From 9ac3e04ae02664323e4c60bb7847196cccc8c118 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Wed, 8 Oct 2025 08:34:05 +0000 Subject: [PATCH 01/15] chore: bump to conan2 --- .env | 4 +- .github/workflows/tests.yml | 12 +++--- VERSION | 2 +- cmake/ExternalProjects.cmake | 78 +++------------------------------- conan-debug.lock | 22 ++++++++++ conan-profile.txt | 15 ------- conan-release.lock | 22 ++++++++++ conanfile.txt | 31 ++++++++++++++ conanprofile.txt | 12 ++++++ docker/faabric-base.dockerfile | 6 ++- docker/faabric.dockerfile | 3 +- src/flat/CMakeLists.txt | 2 +- tasks/dev.py | 32 ++++++++++++++ tasks/util/env.py | 2 + 14 files changed, 142 insertions(+), 101 deletions(-) create mode 100644 conan-debug.lock delete mode 100644 conan-profile.txt create mode 100644 conan-release.lock create mode 100644 conanfile.txt create mode 100644 conanprofile.txt diff --git a/.env b/.env index b030125c9..a8de1017a 100644 --- a/.env +++ b/.env @@ -1,4 +1,4 @@ -FAABRIC_VERSION=0.21.0 -FAABRIC_CLI_IMAGE=ghcr.io/faasm/faabric:0.21.0 +FAABRIC_VERSION=0.22.0 +FAABRIC_CLI_IMAGE=ghcr.io/faasm/faabric:0.22.0 COMPOSE_PROJECT_NAME=faabric-dev CONAN_CACHE_MOUNT_SOURCE=./conan-cache/ diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 957965d0d..ad590f1f4 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -20,7 +20,7 @@ jobs: if: github.event.pull_request.draft == false runs-on: ubuntu-latest container: - image: ghcr.io/faasm/faabric:0.21.0 + image: ghcr.io/faasm/faabric:0.22.0 env: DEPLOYMENT_TYPE: gha-ci steps: @@ -34,7 +34,7 @@ jobs: if: github.event.pull_request.draft == false runs-on: ubuntu-latest container: - image: ghcr.io/faasm/faabric:0.21.0 + image: ghcr.io/faasm/faabric:0.22.0 steps: - name: "Check out code" uses: actions/checkout@v4 @@ -45,7 +45,7 @@ jobs: if: github.event.pull_request.draft == false runs-on: ubuntu-latest container: - image: ghcr.io/faasm/faabric:0.21.0 + image: ghcr.io/faasm/faabric:0.22.0 steps: - name: "Check out code" uses: actions/checkout@v4 @@ -65,7 +65,7 @@ jobs: REDIS_QUEUE_HOST: redis REDIS_STATE_HOST: redis container: - image: ghcr.io/faasm/faabric:0.21.0 + image: ghcr.io/faasm/faabric:0.22.0 options: --privileged services: redis: @@ -104,7 +104,7 @@ jobs: REDIS_QUEUE_HOST: redis REDIS_STATE_HOST: redis container: - image: ghcr.io/faasm/faabric:0.21.0 + image: ghcr.io/faasm/faabric:0.22.0 options: --privileged services: redis: @@ -156,7 +156,7 @@ jobs: REDIS_QUEUE_HOST: redis REDIS_STATE_HOST: redis container: - image: ghcr.io/faasm/faabric:0.21.0 + image: ghcr.io/faasm/faabric:0.22.0 services: redis: image: redis diff --git a/VERSION b/VERSION index 885415662..215740905 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.21.0 +0.22.0 diff --git a/cmake/ExternalProjects.cmake b/cmake/ExternalProjects.cmake index 4f64c643c..3359d532e 100644 --- a/cmake/ExternalProjects.cmake +++ b/cmake/ExternalProjects.cmake @@ -8,81 +8,13 @@ find_package (Threads REQUIRED) list(PREPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_BINARY_DIR}) list(PREPEND CMAKE_PREFIX_PATH ${CMAKE_CURRENT_BINARY_DIR}) -if(NOT EXISTS "${CMAKE_CURRENT_BINARY_DIR}/conan.cmake") - message(STATUS "Downloading conan.cmake from https://github.com/conan-io/cmake-conan") - file(DOWNLOAD "https://raw.githubusercontent.com/conan-io/cmake-conan/0.18.1/conan.cmake" - "${CMAKE_CURRENT_BINARY_DIR}/conan.cmake" - TLS_VERIFY ON) -endif() - -set(CONAN_CMAKE_SILENT_OUTPUT ON CACHE INTERNAL "") -include(${CMAKE_CURRENT_BINARY_DIR}/conan.cmake) - -conan_check(VERSION 1.63.0 REQUIRED) - -# Enable revisions in the conan config -execute_process(COMMAND ${CONAN_CMD} config set general.revisions_enabled=1 - RESULT_VARIABLE RET_CODE) -if(NOT "${RET_CODE}" STREQUAL "0") - message(FATAL_ERROR "Error setting revisions for Conan: '${RET_CODE}'") -endif() - -# -------------------------------- -# Conan dependencies -# -------------------------------- - -conan_cmake_configure( - REQUIRES - "abseil/20220623.0@#732381dc99db29b4cfd293684891da56" - "boost/1.84.0@#7604ce1e7485780469dffb6430f232ea" - "catch2/2.13.9@#8793d3e6287d3684201418de556d98fe" - "flatbuffers/23.5.26@#b153646f6546daab4c7326970b6cd89c" - "hiredis/1.0.2@#370dad964286cadb1f15dc90252e8ef3" - "openssl/3.0.2@#269fa93e5afe8c34bd9a0030d2b8f0fe" - "protobuf/3.20.0@#8e4de7081bea093469c9e6076149b2b4" - "readerwriterqueue/1.0.6@#a95c8da3d68822dec4d4c13fff4b5c96" - "spdlog/1.10.0@#6406c337028e15e56cd6a070cbac54c4" - "zlib/1.2.12@#3b9e037ae1c615d045a06c67d88491ae" - GENERATORS - cmake_find_package - cmake_paths - OPTIONS - boost:error_code_header_only=True - boost:system_no_deprecated=True - boost:zlib=False - boost:bzip2=False - boost:lzma=False - boost:zstd=False - boost:without_locale=True - boost:without_log=True - boost:without_mpi=True - boost:without_python=True - boost:without_test=True - boost:without_wave=True -) - -conan_cmake_autodetect(FAABRIC_CONAN_SETTINGS) - -conan_cmake_install(PATH_OR_REFERENCE . - BUILD outdated - UPDATE - REMOTE conancenter - PROFILE_HOST ${CMAKE_CURRENT_LIST_DIR}/../conan-profile.txt - PROFILE_BUILD ${CMAKE_CURRENT_LIST_DIR}/../conan-profile.txt - SETTINGS ${FAABRIC_CONAN_SETTINGS} -) - -include(${CMAKE_CURRENT_BINARY_DIR}/conan_paths.cmake) - find_package(absl QUIET REQUIRED) -find_package(Boost 1.80.0 QUIET REQUIRED) +find_package(Boost 1.84.0 QUIET REQUIRED COMPONENTS system) find_package(Catch2 QUIET REQUIRED) -find_package(flatbuffers QUIET REQUIRED) +find_package(flatbuffers CONFIG QUIET REQUIRED) find_package(fmt QUIET REQUIRED) find_package(hiredis QUIET REQUIRED) -# 27/01/2023 - Pin OpenSSL to a specific version to avoid incompatibilities -# with the system's (i.e. Ubuntu 22.04) OpenSSL -find_package(OpenSSL 3.0.2 QUIET REQUIRED) +find_package(OpenSSL 3.6.0 QUIET REQUIRED) find_package(Protobuf 3.20.0 QUIET REQUIRED) find_package(readerwriterqueue QUIET REQUIRED) find_package(spdlog QUIET REQUIRED) @@ -121,7 +53,7 @@ FetchContent_Declare(nng_ext ) FetchContent_Declare(zstd_ext GIT_REPOSITORY "https://github.com/facebook/zstd" - GIT_TAG "v1.5.2" + GIT_TAG "v1.5.7" SOURCE_SUBDIR "build/cmake" ) @@ -150,7 +82,7 @@ target_link_libraries(faabric_common_dependencies INTERFACE absl::flat_hash_map absl::strings atomic_queue::atomic_queue - Boost::Boost + Boost::headers Boost::system flatbuffers::flatbuffers hiredis::hiredis diff --git a/conan-debug.lock b/conan-debug.lock new file mode 100644 index 000000000..71d819f76 --- /dev/null +++ b/conan-debug.lock @@ -0,0 +1,22 @@ +{ + "version": "0.5", + "requires": [ + "zlib/1.2.12#2ea72a0bae8b680f8a282e103e0a6880%1743582313.324", + "spdlog/1.10.0#5aed29a37b544d0eb5812b8be74cd7a4%1731353179.805", + "readerwriterqueue/1.0.6#aaa5ff6fac60c2aee591e9e51b063b83%1679587692.891", + "protobuf/3.20.0#9cc01d27c03a1bdd736a031e4b87e1eb%1700600974.257", + "openssl/3.6.0#89e8af1d4a21afcac0557079d23d8890%1759746682.365", + "libbacktrace/cci.20210118#a7691bfccd8caaf66309df196790a5a1%1722218217.276", + "hiredis/1.0.2#6001a683c04da07565e97e76d0d841bd%1744279494.11", + "fmt/8.1.1#d66420384a943dbf1f292d34ef34c11c%1735899179.969", + "flatbuffers/23.5.26#0290575326fe9b2c39a0236ea81d8c30%1743154511.803", + "catch2/2.13.9#17ac9b0b78c63353e304c5744e862a77%1678135818.816", + "boost/1.84.0#40dd9fecacce0ef109851d0e38727fd0%1759416424.265", + "abseil/20220623.0#bbc28aa2a287b46e488cf28bac5bae65%1706008822.296" + ], + "build_requires": [ + "b2/5.3.3#107c15377719889654eb9a162a673975%1750340310.079" + ], + "python_requires": [], + "config_requires": [] +} \ No newline at end of file diff --git a/conan-profile.txt b/conan-profile.txt deleted file mode 100644 index 2edf1e4b0..000000000 --- a/conan-profile.txt +++ /dev/null @@ -1,15 +0,0 @@ -[settings] -os=Linux -os_build=Linux -arch=x86_64 -arch_build=x86_64 -compiler=clang -compiler.version=17 -compiler.libcxx=libstdc++11 -compiler.cppstd=20 -build_type=Release -[options] -[build_requires] -[env] -CC=/usr/bin/clang-17 -CXX=/usr/bin/clang++-17 diff --git a/conan-release.lock b/conan-release.lock new file mode 100644 index 000000000..71d819f76 --- /dev/null +++ b/conan-release.lock @@ -0,0 +1,22 @@ +{ + "version": "0.5", + "requires": [ + "zlib/1.2.12#2ea72a0bae8b680f8a282e103e0a6880%1743582313.324", + "spdlog/1.10.0#5aed29a37b544d0eb5812b8be74cd7a4%1731353179.805", + "readerwriterqueue/1.0.6#aaa5ff6fac60c2aee591e9e51b063b83%1679587692.891", + "protobuf/3.20.0#9cc01d27c03a1bdd736a031e4b87e1eb%1700600974.257", + "openssl/3.6.0#89e8af1d4a21afcac0557079d23d8890%1759746682.365", + "libbacktrace/cci.20210118#a7691bfccd8caaf66309df196790a5a1%1722218217.276", + "hiredis/1.0.2#6001a683c04da07565e97e76d0d841bd%1744279494.11", + "fmt/8.1.1#d66420384a943dbf1f292d34ef34c11c%1735899179.969", + "flatbuffers/23.5.26#0290575326fe9b2c39a0236ea81d8c30%1743154511.803", + "catch2/2.13.9#17ac9b0b78c63353e304c5744e862a77%1678135818.816", + "boost/1.84.0#40dd9fecacce0ef109851d0e38727fd0%1759416424.265", + "abseil/20220623.0#bbc28aa2a287b46e488cf28bac5bae65%1706008822.296" + ], + "build_requires": [ + "b2/5.3.3#107c15377719889654eb9a162a673975%1750340310.079" + ], + "python_requires": [], + "config_requires": [] +} \ No newline at end of file diff --git a/conanfile.txt b/conanfile.txt new file mode 100644 index 000000000..0330dde82 --- /dev/null +++ b/conanfile.txt @@ -0,0 +1,31 @@ +[requires] +abseil/20220623.0 +boost/1.84.0 +catch2/2.13.9 +flatbuffers/23.5.26 +hiredis/1.0.2 +openssl/3.6.0 +protobuf/3.20.0 +readerwriterqueue/1.0.6 +spdlog/1.10.0 +zlib/1.2.12 +# (Optional) Only if CMake can’t find fmt below: +# fmt/9.1.0 + +[generators] +CMakeToolchain +CMakeDeps + +[options] +boost/*:error_code_header_only=True +boost/*:system_no_deprecated=True +boost/*:zlib=False +boost/*:bzip2=False +boost/*:lzma=False +boost/*:zstd=False +boost/*:without_locale=True +boost/*:without_log=True +boost/*:without_mpi=True +boost/*:without_python=True +boost/*:without_test=True +boost/*:without_wave=True diff --git a/conanprofile.txt b/conanprofile.txt new file mode 100644 index 000000000..bfa3e9288 --- /dev/null +++ b/conanprofile.txt @@ -0,0 +1,12 @@ +[settings] +os=Linux +arch=x86_64 +compiler=clang +compiler.version=17 +compiler.libcxx=libstdc++11 +compiler.cppstd=20 +build_type=Release + +[conf] +# Tell Conan which compiler binaries to run +tools.build:compiler_executables={"c": "/usr/bin/clang-17", "cpp": "/usr/bin/clang++-17"} diff --git a/docker/faabric-base.dockerfile b/docker/faabric-base.dockerfile index 2be4e1238..7084fa67c 100644 --- a/docker/faabric-base.dockerfile +++ b/docker/faabric-base.dockerfile @@ -50,18 +50,20 @@ RUN apt update && apt install -y \ unzip # Install up-to-date CMake +ARG CMAKE_VERSION=4.1.2 RUN apt remove --purge --auto-remove cmake \ && mkdir -p /setup \ && cd /setup \ && wget -q -O cmake-linux.sh \ - https://github.com/Kitware/CMake/releases/download/v3.28.0/cmake-3.28.0-linux-x86_64.sh \ + https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-x86_64.sh \ && sh cmake-linux.sh -- --skip-license --prefix=/usr/local \ && apt clean autoclean -y \ && apt autoremove -y # Install Conan +ARG CONAN_VERSION=2.21.0 RUN curl -s -L -o \ - /tmp/conan-latest.deb https://github.com/conan-io/conan/releases/download/1.63.0/conan-ubuntu-64.deb \ + /tmp/conan-latest.deb https://github.com/conan-io/conan/releases/download/${CONAN_VERSION}/conan-${CONAN_VERSION}-amd64.deb \ && sudo dpkg -i /tmp/conan-latest.deb \ && rm -f /tmp/conan-latest.deb diff --git a/docker/faabric.dockerfile b/docker/faabric.dockerfile index 990cf8640..14b0394d6 100644 --- a/docker/faabric.dockerfile +++ b/docker/faabric.dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/faasm/faabric-base:0.21.0 +FROM ghcr.io/faasm/faabric-base:0.22.0 ARG FAABRIC_VERSION # faabic-base image is not re-built often, so tag may be behind @@ -18,6 +18,7 @@ WORKDIR /code/faabric # Python set-up and code builds RUN ./bin/create_venv.sh \ && source venv/bin/activate \ + && inv dev.conan --build=Release \ # Static build && inv dev.cmake --build=Release \ && inv dev.cc faabric \ diff --git a/src/flat/CMakeLists.txt b/src/flat/CMakeLists.txt index 18230c182..5f73f178f 100644 --- a/src/flat/CMakeLists.txt +++ b/src/flat/CMakeLists.txt @@ -10,7 +10,7 @@ add_custom_command( OUTPUT "${FB_HEADER}" DEPENDS faabric.fbs WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} - COMMAND ${CONAN_FLATBUFFERS_ROOT}/bin/flatc + COMMAND flatbuffers::flatc ARGS --cpp --gen-mutable --raw-binary faabric.fbs ) diff --git a/tasks/dev.py b/tasks/dev.py index ad17b6897..fb2e65281 100644 --- a/tasks/dev.py +++ b/tasks/dev.py @@ -3,6 +3,7 @@ from shutil import rmtree from subprocess import run from tasks.util.env import ( + FAABRIC_CONAN_CACHE, FAABRIC_SHARED_BUILD_DIR, FAABRIC_STATIC_BUILD_DIR, FAABRIC_INSTALL_PREFIX, @@ -12,6 +13,30 @@ from invoke import task +@task +def conan(ctx, clean=False, build="Debug"): + """ + Configure dependencies using Conan + """ + conan_lockfile = f"{PROJ_ROOT}/conan-{build.lower()}.lock" + conan_cache = f"{FAABRIC_CONAN_CACHE}/{build.lower()}" + conan_profile = join(PROJ_ROOT, "conanprofile.txt") + + if clean: + run(f"rm -f {conan_lockfile}", shell=True, check=True) + run(f"rm -rf {conan_cache}", shell=True, check=True) + + # Generate a conan lock file if it does not exist. This file should be + # comitted for version control. + if not exists(conan_lockfile): + run("conan remote list", shell=True, check=True) + conan_cmd = f"conan lock create {PROJ_ROOT} -pr:h={conan_profile} -pr:b={conan_profile} -s build_type={build} --lockfile-out={conan_lockfile}" + print(conan_cmd) + run(conan_cmd, shell=True, check=True) + + conan_install_cmd = f"conan install {PROJ_ROOT} -pr:h={conan_profile} -pr:b={conan_profile} -s build_type={build} -of {conan_cache} --build=missing --lockfile={conan_lockfile}" + print(conan_install_cmd) + run(conan_install_cmd, shell=True, check=True) @task def cmake( @@ -42,9 +67,16 @@ def cmake( if build not in build_types: raise RuntimeError("Expected build to be in {}".format(build_types)) + conan_cache = f"{FAABRIC_CONAN_CACHE}/{build.lower()}" + if not exists(conan_cache): + print(f"ERROR: expected conan cache in {conan_cache}") + print("ERROR: make sure to run 'inv dev.conan' first") + raise RuntimeError(f"Expected conan cache in {conan_cache}") + cmd = [ "cmake", "-GNinja", + f"-DCMAKE_TOOLCHAIN_FILE={conan_cache}/conan_toolchain.cmake", "-DCMAKE_INSTALL_PREFIX={}".format(FAABRIC_INSTALL_PREFIX), "-DCMAKE_BUILD_TYPE={}".format(build), "-DBUILD_SHARED_LIBS={}".format("ON" if shared else "OFF"), diff --git a/tasks/util/env.py b/tasks/util/env.py index 42151ea3d..4dfeddd19 100644 --- a/tasks/util/env.py +++ b/tasks/util/env.py @@ -14,6 +14,8 @@ FAABRIC_CONFIG_FILE = join(PROJ_ROOT, "faabric.ini") +FAABRIC_CONAN_CACHE = join(PROJ_ROOT, "conan-cache") + CR_NAME = "ghcr.io/faasm" # This LLVM version is for the LLVM that we use to compile regular C/C++ code From 8a2c4c018d7fc893b849e71a317f63a80db85d27 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Wed, 8 Oct 2025 09:36:57 +0000 Subject: [PATCH 02/15] gha: fix conan-cache-action --- .github/workflows/tests.yml | 9 ++++++--- conanfile.txt | 2 -- docker/planner.dockerfile | 3 ++- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ad590f1f4..332f75a6b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -26,9 +26,12 @@ jobs: steps: - name: "Check-out code" uses: actions/checkout@v4 - - uses: faasm/conan-cache-action@v3 - - name: "Build Conan CMake deps to be shared by all runs" - run: ./bin/inv_wrapper.sh dev.cmake --build Debug --clean + - uses: faasm/conan-cache-action@main + with: + build-type: Debug + from: faabric + - name: "Build Conan deps to be shared by all runs" + run: ./bin/inv_wrapper.sh dev.conan --build Debug --clean docs: if: github.event.pull_request.draft == false diff --git a/conanfile.txt b/conanfile.txt index 0330dde82..6723be707 100644 --- a/conanfile.txt +++ b/conanfile.txt @@ -9,8 +9,6 @@ protobuf/3.20.0 readerwriterqueue/1.0.6 spdlog/1.10.0 zlib/1.2.12 -# (Optional) Only if CMake can’t find fmt below: -# fmt/9.1.0 [generators] CMakeToolchain diff --git a/docker/planner.dockerfile b/docker/planner.dockerfile index 02f3e92fc..c4af2be31 100644 --- a/docker/planner.dockerfile +++ b/docker/planner.dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/faasm/faabric-base:0.21.0 +FROM ghcr.io/faasm/faabric-base:0.22.0 ARG FAABRIC_VERSION # Flag to say we're in a container @@ -15,6 +15,7 @@ RUN rm -rf /code \ && cd /code/faabric \ && ./bin/create_venv.sh \ && source venv/bin/activate \ + && inv dev.conan --build=Release \ && inv dev.cmake --build=Release \ && inv dev.cc planner_server From 4e580b94cfc034c3435dd7da9773f8824be26a78 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Wed, 8 Oct 2025 09:40:15 +0000 Subject: [PATCH 03/15] chore: run python formatting --- .ycm_extra_conf.py | 13 ------------- tasks/dev.py | 2 ++ 2 files changed, 2 insertions(+), 13 deletions(-) delete mode 100644 .ycm_extra_conf.py diff --git a/.ycm_extra_conf.py b/.ycm_extra_conf.py deleted file mode 100644 index 18a613bab..000000000 --- a/.ycm_extra_conf.py +++ /dev/null @@ -1,13 +0,0 @@ -from os.path import dirname, realpath, join, exists - -_PROJ_ROOT = dirname(realpath(__file__)) - - -def Settings(**kwargs): - venv_interpreter = join(_PROJ_ROOT, "venv", "bin", "python") - - if not exists(venv_interpreter): - parent_root = dirname(dirname(_PROJ_ROOT)) - venv_interpreter = join(parent_root, "venv", "bin", "python") - - return {"interpreter_path": venv_interpreter} diff --git a/tasks/dev.py b/tasks/dev.py index fb2e65281..13ea09504 100644 --- a/tasks/dev.py +++ b/tasks/dev.py @@ -13,6 +13,7 @@ from invoke import task + @task def conan(ctx, clean=False, build="Debug"): """ @@ -38,6 +39,7 @@ def conan(ctx, clean=False, build="Debug"): print(conan_install_cmd) run(conan_install_cmd, shell=True, check=True) + @task def cmake( ctx, From ade06ca2fc7d31863ae8acf79803ed06ed553292 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Wed, 8 Oct 2025 10:28:35 +0000 Subject: [PATCH 04/15] chore: more formatting fixes --- .github/workflows/tests.yml | 1 + tasks/dev.py | 12 ++++++++++-- tasks/format_code.py | 1 - 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 332f75a6b..0bfb7fe7e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -30,6 +30,7 @@ jobs: with: build-type: Debug from: faabric + mount-path: conan-cache - name: "Build Conan deps to be shared by all runs" run: ./bin/inv_wrapper.sh dev.conan --build Debug --clean diff --git a/tasks/dev.py b/tasks/dev.py index 13ea09504..ae8626e55 100644 --- a/tasks/dev.py +++ b/tasks/dev.py @@ -31,11 +31,19 @@ def conan(ctx, clean=False, build="Debug"): # comitted for version control. if not exists(conan_lockfile): run("conan remote list", shell=True, check=True) - conan_cmd = f"conan lock create {PROJ_ROOT} -pr:h={conan_profile} -pr:b={conan_profile} -s build_type={build} --lockfile-out={conan_lockfile}" + conan_cmd = ( + f"conan lock create {PROJ_ROOT} -pr:h={conan_profile} " + f"-pr:b={conan_profile} -s build_type={build} " + f"--lockfile-out={conan_lockfile}" + ) print(conan_cmd) run(conan_cmd, shell=True, check=True) - conan_install_cmd = f"conan install {PROJ_ROOT} -pr:h={conan_profile} -pr:b={conan_profile} -s build_type={build} -of {conan_cache} --build=missing --lockfile={conan_lockfile}" + conan_install_cmd = ( + f"conan install {PROJ_ROOT} -pr:h={conan_profile} -pr:b={conan_profile}" + f" -s build_type={build} -of {conan_cache} --build=missing " + f"--lockfile={conan_lockfile}" + ) print(conan_install_cmd) run(conan_install_cmd, shell=True, check=True) diff --git a/tasks/format_code.py b/tasks/format_code.py index 39b97c01b..d30f14ca9 100644 --- a/tasks/format_code.py +++ b/tasks/format_code.py @@ -33,7 +33,6 @@ def format(ctx, check=False): flake8_cmd = [ "python3 -m flake8", - "{}".format("--format" if not check else ""), " ".join(files_to_check), ] flake8_cmd = " ".join(flake8_cmd) From 9a4a2ce18a59d4000d96ccf01c07e90a7bf2f301 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Wed, 8 Oct 2025 11:09:13 +0000 Subject: [PATCH 05/15] chore: fix compilation warnings --- .github/workflows/tests.yml | 25 +++++++++++++++++++------ cmake/ExternalProjects.cmake | 4 +++- include/faabric/util/clock.h | 8 ++++---- src/util/clock.cpp | 8 ++++---- tasks/dev.py | 6 +++--- 5 files changed, 33 insertions(+), 18 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0bfb7fe7e..3db964663 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -77,7 +77,11 @@ jobs: steps: - name: "Check-out code" uses: actions/checkout@v4 - - uses: faasm/conan-cache-action@v3 + - uses: faasm/conan-cache-action@main + with: + build-type: Debug + from: faabric + mount-path: conan-cache - name: "Ping redis" run: redis-cli -h redis ping - name: "Run cmake for tests" @@ -116,7 +120,11 @@ jobs: steps: - name: "Check-out code" uses: actions/checkout@v4 - - uses: faasm/conan-cache-action@v3 + - uses: faasm/conan-cache-action@main + with: + build-type: Debug + from: faabric + mount-path: conan-cache - name: "Ping redis" run: redis-cli -h redis ping - name: "Run cmake for tests" @@ -137,8 +145,11 @@ jobs: # --- Code update --- - name: "Check out code" uses: actions/checkout@v4 - - name: "Conan cache" - uses: faasm/conan-cache-action@v3 + - uses: faasm/conan-cache-action@main + with: + build-type: Debug + from: faabric + mount-path: conan-cache - name: "Build distributed tests" run: ./dist-test/build.sh - name: "Run the distributed tests" @@ -167,9 +178,11 @@ jobs: steps: - name: "Check-out code" uses: actions/checkout@v4 - - uses: faasm/conan-cache-action@v3 + - uses: faasm/conan-cache-action@main with: - build-type: "release" + build-type: Release + from: faabric + mount-path: conan-cache - name: "Run cmake shared" run: ./bin/inv_wrapper.sh dev.cmake --clean --shared --build=Release - name: "Build Faabric shared library" diff --git a/cmake/ExternalProjects.cmake b/cmake/ExternalProjects.cmake index 3359d532e..0f395a35c 100644 --- a/cmake/ExternalProjects.cmake +++ b/cmake/ExternalProjects.cmake @@ -9,7 +9,7 @@ list(PREPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_BINARY_DIR}) list(PREPEND CMAKE_PREFIX_PATH ${CMAKE_CURRENT_BINARY_DIR}) find_package(absl QUIET REQUIRED) -find_package(Boost 1.84.0 QUIET REQUIRED COMPONENTS system) +find_package(Boost 1.84.0 QUIET REQUIRED COMPONENTS filesystem program_options system) find_package(Catch2 QUIET REQUIRED) find_package(flatbuffers CONFIG QUIET REQUIRED) find_package(fmt QUIET REQUIRED) @@ -83,6 +83,8 @@ target_link_libraries(faabric_common_dependencies INTERFACE absl::strings atomic_queue::atomic_queue Boost::headers + Boost::filesystem + Boost::program_options Boost::system flatbuffers::flatbuffers hiredis::hiredis diff --git a/include/faabric/util/clock.h b/include/faabric/util/clock.h index 5f71096c0..0d4e58b9c 100644 --- a/include/faabric/util/clock.h +++ b/include/faabric/util/clock.h @@ -12,13 +12,13 @@ class Clock const TimePoint now(); - const long epochMillis(); + long epochMillis(); - const long timeDiff(const TimePoint& t1, const TimePoint& t2); + long timeDiff(const TimePoint& t1, const TimePoint& t2); - const long timeDiffNano(const TimePoint& t1, const TimePoint& t2); + long timeDiffNano(const TimePoint& t1, const TimePoint& t2); - const long timeDiffMicro(const TimePoint& t1, const TimePoint& t2); + long timeDiffMicro(const TimePoint& t1, const TimePoint& t2); }; Clock& getGlobalClock(); diff --git a/src/util/clock.cpp b/src/util/clock.cpp index 9def1ea19..23bb06822 100644 --- a/src/util/clock.cpp +++ b/src/util/clock.cpp @@ -14,7 +14,7 @@ const TimePoint Clock::now() return std::chrono::steady_clock::now(); } -const long Clock::epochMillis() +long Clock::epochMillis() { long millis = std::chrono::duration_cast( std::chrono::system_clock::now().time_since_epoch()) @@ -23,21 +23,21 @@ const long Clock::epochMillis() return millis; } -const long Clock::timeDiff(const TimePoint& t1, const TimePoint& t2) +long Clock::timeDiff(const TimePoint& t1, const TimePoint& t2) { long age = std::chrono::duration_cast(t1 - t2).count(); return age; } -const long Clock::timeDiffMicro(const TimePoint& t1, const TimePoint& t2) +long Clock::timeDiffMicro(const TimePoint& t1, const TimePoint& t2) { long age = std::chrono::duration_cast(t1 - t2).count(); return age; } -const long Clock::timeDiffNano(const TimePoint& t1, const TimePoint& t2) +long Clock::timeDiffNano(const TimePoint& t1, const TimePoint& t2) { long age = std::chrono::duration_cast(t1 - t2).count(); diff --git a/tasks/dev.py b/tasks/dev.py index ae8626e55..6fde80777 100644 --- a/tasks/dev.py +++ b/tasks/dev.py @@ -40,9 +40,9 @@ def conan(ctx, clean=False, build="Debug"): run(conan_cmd, shell=True, check=True) conan_install_cmd = ( - f"conan install {PROJ_ROOT} -pr:h={conan_profile} -pr:b={conan_profile}" - f" -s build_type={build} -of {conan_cache} --build=missing " - f"--lockfile={conan_lockfile}" + f"conan install {PROJ_ROOT} -pr:h={conan_profile} " + f"-pr:b={conan_profile} -s build_type={build} -of {conan_cache} " + f"--build=missing --lockfile={conan_lockfile}" ) print(conan_install_cmd) run(conan_install_cmd, shell=True, check=True) From 09bc67b2cb475ada099ec991c1b60d1040378dbb Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Wed, 8 Oct 2025 11:38:57 +0000 Subject: [PATCH 06/15] gha: fix conan caching --- .github/workflows/tests.yml | 9 +++------ cmake/ExternalProjects.cmake | 2 +- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3db964663..fe65ed1f5 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -30,9 +30,10 @@ jobs: with: build-type: Debug from: faabric - mount-path: conan-cache - name: "Build Conan deps to be shared by all runs" - run: ./bin/inv_wrapper.sh dev.conan --build Debug --clean + run: | + ./bin/inv_wrapper.sh dev.conan --build Debug --clean + ./bin/inv_wrapper.sh dev.conan --build Release --clean docs: if: github.event.pull_request.draft == false @@ -81,7 +82,6 @@ jobs: with: build-type: Debug from: faabric - mount-path: conan-cache - name: "Ping redis" run: redis-cli -h redis ping - name: "Run cmake for tests" @@ -124,7 +124,6 @@ jobs: with: build-type: Debug from: faabric - mount-path: conan-cache - name: "Ping redis" run: redis-cli -h redis ping - name: "Run cmake for tests" @@ -149,7 +148,6 @@ jobs: with: build-type: Debug from: faabric - mount-path: conan-cache - name: "Build distributed tests" run: ./dist-test/build.sh - name: "Run the distributed tests" @@ -182,7 +180,6 @@ jobs: with: build-type: Release from: faabric - mount-path: conan-cache - name: "Run cmake shared" run: ./bin/inv_wrapper.sh dev.cmake --clean --shared --build=Release - name: "Build Faabric shared library" diff --git a/cmake/ExternalProjects.cmake b/cmake/ExternalProjects.cmake index 0f395a35c..f58c54cc5 100644 --- a/cmake/ExternalProjects.cmake +++ b/cmake/ExternalProjects.cmake @@ -4,7 +4,7 @@ include (ExternalProject) include (FetchContent) find_package (Threads REQUIRED) -# Find conan-generated package descriptions +# Find conan-generated package descriptions. list(PREPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_BINARY_DIR}) list(PREPEND CMAKE_PREFIX_PATH ${CMAKE_CURRENT_BINARY_DIR}) From 8ff1e7b3fdf56a28aeda7027e0b38d0bdb8b71b1 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Wed, 8 Oct 2025 12:11:48 +0000 Subject: [PATCH 07/15] chore: bump protobuf version --- cmake/ExternalProjects.cmake | 2 +- conan-debug.lock | 5 +++-- conanfile.txt | 4 ++-- src/util/json.cpp | 8 +++----- 4 files changed, 9 insertions(+), 10 deletions(-) diff --git a/cmake/ExternalProjects.cmake b/cmake/ExternalProjects.cmake index f58c54cc5..d2e6ae34b 100644 --- a/cmake/ExternalProjects.cmake +++ b/cmake/ExternalProjects.cmake @@ -15,7 +15,7 @@ find_package(flatbuffers CONFIG QUIET REQUIRED) find_package(fmt QUIET REQUIRED) find_package(hiredis QUIET REQUIRED) find_package(OpenSSL 3.6.0 QUIET REQUIRED) -find_package(Protobuf 3.20.0 QUIET REQUIRED) +find_package(Protobuf 6.30.1 QUIET REQUIRED) find_package(readerwriterqueue QUIET REQUIRED) find_package(spdlog QUIET REQUIRED) find_package(ZLIB QUIET REQUIRED) diff --git a/conan-debug.lock b/conan-debug.lock index 71d819f76..ae206fc57 100644 --- a/conan-debug.lock +++ b/conan-debug.lock @@ -4,7 +4,7 @@ "zlib/1.2.12#2ea72a0bae8b680f8a282e103e0a6880%1743582313.324", "spdlog/1.10.0#5aed29a37b544d0eb5812b8be74cd7a4%1731353179.805", "readerwriterqueue/1.0.6#aaa5ff6fac60c2aee591e9e51b063b83%1679587692.891", - "protobuf/3.20.0#9cc01d27c03a1bdd736a031e4b87e1eb%1700600974.257", + "protobuf/6.30.1#97bae23ef6d7f9fcfdb4ded9468ad6de%1747665968.233", "openssl/3.6.0#89e8af1d4a21afcac0557079d23d8890%1759746682.365", "libbacktrace/cci.20210118#a7691bfccd8caaf66309df196790a5a1%1722218217.276", "hiredis/1.0.2#6001a683c04da07565e97e76d0d841bd%1744279494.11", @@ -12,9 +12,10 @@ "flatbuffers/23.5.26#0290575326fe9b2c39a0236ea81d8c30%1743154511.803", "catch2/2.13.9#17ac9b0b78c63353e304c5744e862a77%1678135818.816", "boost/1.84.0#40dd9fecacce0ef109851d0e38727fd0%1759416424.265", - "abseil/20220623.0#bbc28aa2a287b46e488cf28bac5bae65%1706008822.296" + "abseil/20250127.0#faefa3bbf31b5c32933e328d72e42cfe%1754142622.985" ], "build_requires": [ + "cmake/3.31.9#2032c6471fe4f5a3e17f65fed518d545%1758832282.188", "b2/5.3.3#107c15377719889654eb9a162a673975%1750340310.079" ], "python_requires": [], diff --git a/conanfile.txt b/conanfile.txt index 6723be707..d289607c7 100644 --- a/conanfile.txt +++ b/conanfile.txt @@ -1,11 +1,11 @@ [requires] -abseil/20220623.0 +abseil/20250127.0 boost/1.84.0 catch2/2.13.9 flatbuffers/23.5.26 hiredis/1.0.2 openssl/3.6.0 -protobuf/3.20.0 +protobuf/6.30.1 readerwriterqueue/1.0.6 spdlog/1.10.0 zlib/1.2.12 diff --git a/src/util/json.cpp b/src/util/json.cpp index 23d8c460f..ffdd9a2c2 100644 --- a/src/util/json.cpp +++ b/src/util/json.cpp @@ -11,11 +11,10 @@ std::string messageToJson(const google::protobuf::Message& msg) // Set the JSON print options. This is very important to ensure backwards- // compatibility with clients sending HTTP requests to faabric - google::protobuf::util::JsonOptions jsonOptions; + google::protobuf::util::JsonPrintOptions jsonOptions; jsonOptions.always_print_enums_as_ints = true; - google::protobuf::util::Status status = - google::protobuf::util::MessageToJsonString(msg, &jsonStr, jsonOptions); + auto status = google::protobuf::util::MessageToJsonString(msg, &jsonStr, jsonOptions); if (!status.ok()) { SPDLOG_ERROR("Serialising JSON string to protobuf message: {}", status.message().data()); @@ -28,8 +27,7 @@ std::string messageToJson(const google::protobuf::Message& msg) void jsonToMessage(const std::string& jsonStr, google::protobuf::Message* msg) { - google::protobuf::util::Status status = - google::protobuf::util::JsonStringToMessage(jsonStr, msg); + auto status = google::protobuf::util::JsonStringToMessage(jsonStr, msg); if (!status.ok()) { SPDLOG_ERROR("Deserialising JSON string to protobuf message: {}", status.message().data()); From 986bd4892dd0b85c86293cc1def1bcd057ab53af Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Wed, 8 Oct 2025 12:48:59 +0000 Subject: [PATCH 08/15] ci: run dev.conan to generate the generators --- .github/workflows/tests.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index fe65ed1f5..afbe9bea0 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -84,6 +84,8 @@ jobs: from: faabric - name: "Ping redis" run: redis-cli -h redis ping + - name: "Run conan" + run: ./bin/inv_wrapper.sh dev.conan --clean --build=Debug - name: "Run cmake for tests" run: ./bin/inv_wrapper.sh dev.cmake --clean --build=Debug --coverage - name: "Build tests" @@ -126,6 +128,8 @@ jobs: from: faabric - name: "Ping redis" run: redis-cli -h redis ping + - name: "Run conan" + run: ./bin/inv_wrapper.sh dev.conan --clean --build=Debug - name: "Run cmake for tests" run: ./bin/inv_wrapper.sh dev.cmake --clean --build=Debug --sanitiser ${{ matrix.sanitiser }} - name: "Build tests" @@ -180,6 +184,8 @@ jobs: with: build-type: Release from: faabric + - name: "Run conan" + run: ./bin/inv_wrapper.sh dev.conan --clean --build=Release - name: "Run cmake shared" run: ./bin/inv_wrapper.sh dev.cmake --clean --shared --build=Release - name: "Build Faabric shared library" From ccf9e53918e2c426ad2098d6e628c8eca3dd29dc Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Thu, 9 Oct 2025 10:31:15 +0000 Subject: [PATCH 09/15] fix(dist-test): build conan first --- dist-test/build_internal.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/dist-test/build_internal.sh b/dist-test/build_internal.sh index ede7f3a62..f225b625e 100755 --- a/dist-test/build_internal.sh +++ b/dist-test/build_internal.sh @@ -9,6 +9,7 @@ pushd ${PROJ_ROOT} >> /dev/null source ./bin/workon.sh # Run the debug build +inv dev.conan --build=Debug --clean inv dev.cmake --build=Debug --clean inv dev.cc faabric_dist_tests inv dev.cc faabric_dist_test_server From 0f842b2480e04ca6cf30efcd31cb7c9b547d7aaa Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Thu, 9 Oct 2025 10:48:22 +0000 Subject: [PATCH 10/15] fix(gha): temporarily disable TSAN and ASAN --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index afbe9bea0..e293ead1e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -108,7 +108,7 @@ jobs: strategy: fail-fast: false matrix: - sanitiser: [None, Address, Thread, Undefined] + sanitiser: [None, Undefined] env: DEPLOYMENT_TYPE: gha-ci REDIS_QUEUE_HOST: redis From f232fbb5b3c7b0a3929821156bfd79d558c7c84e Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Thu, 9 Oct 2025 13:35:03 +0000 Subject: [PATCH 11/15] fix: use different conan profiles for sanitisers --- .github/workflows/tests.yml | 7 ++++--- conan-profiles/asan.txt | 15 +++++++++++++++ conanprofile.txt => conan-profiles/default.txt | 1 - conan-profiles/tsan.txt | 15 +++++++++++++++ dist-test/build_internal.sh | 2 +- tasks/dev.py | 15 ++++++++++++--- tasks/util/env.py | 1 + thread-sanitizer-ignorelist.txt | 2 ++ 8 files changed, 50 insertions(+), 8 deletions(-) create mode 100644 conan-profiles/asan.txt rename conanprofile.txt => conan-profiles/default.txt (84%) create mode 100644 conan-profiles/tsan.txt diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e293ead1e..d4db420de 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -108,7 +108,7 @@ jobs: strategy: fail-fast: false matrix: - sanitiser: [None, Undefined] + sanitiser: [None, Thread, Address, Undefined] env: DEPLOYMENT_TYPE: gha-ci REDIS_QUEUE_HOST: redis @@ -128,8 +128,9 @@ jobs: from: faabric - name: "Ping redis" run: redis-cli -h redis ping + # Re-run conan with the --clean flag to rebuild the packages with sanitisers - name: "Run conan" - run: ./bin/inv_wrapper.sh dev.conan --clean --build=Debug + run: ./bin/inv_wrapper.sh dev.conan --clean --build=Debug --sanitiser ${{ matrix.sanitiser }} - name: "Run cmake for tests" run: ./bin/inv_wrapper.sh dev.cmake --clean --build=Debug --sanitiser ${{ matrix.sanitiser }} - name: "Build tests" @@ -185,7 +186,7 @@ jobs: build-type: Release from: faabric - name: "Run conan" - run: ./bin/inv_wrapper.sh dev.conan --clean --build=Release + run: ./bin/inv_wrapper.sh dev.conan --build=Release - name: "Run cmake shared" run: ./bin/inv_wrapper.sh dev.cmake --clean --shared --build=Release - name: "Build Faabric shared library" diff --git a/conan-profiles/asan.txt b/conan-profiles/asan.txt new file mode 100644 index 000000000..6822d85a8 --- /dev/null +++ b/conan-profiles/asan.txt @@ -0,0 +1,15 @@ +[settings] +os=Linux +arch=x86_64 +compiler=clang +compiler.version=17 +compiler.libcxx=libstdc++11 +compiler.cppstd=20 +build_type=RelWithDebInfo + +[conf] +tools.build:compiler_executables={"c": "/usr/bin/clang-17", "cpp": "/usr/bin/clang++-17"} +tools.build:cflags=["-fsanitize=address","-fno-omit-frame-pointer","-g","-O1"] +tools.build:cxxflags=["-fsanitize=address","-fno-omit-frame-pointer","-g","-O1"] +tools.build:sharedlinkflags=["-fsanitize=address"] +tools.build:exelinkflags=["-fsanitize=address"] diff --git a/conanprofile.txt b/conan-profiles/default.txt similarity index 84% rename from conanprofile.txt rename to conan-profiles/default.txt index bfa3e9288..94943459e 100644 --- a/conanprofile.txt +++ b/conan-profiles/default.txt @@ -8,5 +8,4 @@ compiler.cppstd=20 build_type=Release [conf] -# Tell Conan which compiler binaries to run tools.build:compiler_executables={"c": "/usr/bin/clang-17", "cpp": "/usr/bin/clang++-17"} diff --git a/conan-profiles/tsan.txt b/conan-profiles/tsan.txt new file mode 100644 index 000000000..364705599 --- /dev/null +++ b/conan-profiles/tsan.txt @@ -0,0 +1,15 @@ +[settings] +os=Linux +arch=x86_64 +compiler=clang +compiler.version=17 +compiler.libcxx=libstdc++11 +compiler.cppstd=20 +build_type=RelWithDebInfo + +[conf] +tools.build:compiler_executables={"c": "/usr/bin/clang-17", "cpp": "/usr/bin/clang++-17"} +tools.build:cxxflags=["-fsanitize=thread","-fno-omit-frame-pointer","-g"] +tools.build:cflags=["-fsanitize=thread","-fno-omit-frame-pointer","-g"] +tools.build:sharedlinkflags=["-fsanitize=thread"] +tools.build:exelinkflags=["-fsanitize=thread"] diff --git a/dist-test/build_internal.sh b/dist-test/build_internal.sh index f225b625e..e5f3dd894 100755 --- a/dist-test/build_internal.sh +++ b/dist-test/build_internal.sh @@ -9,7 +9,7 @@ pushd ${PROJ_ROOT} >> /dev/null source ./bin/workon.sh # Run the debug build -inv dev.conan --build=Debug --clean +inv dev.conan --build=Debug inv dev.cmake --build=Debug --clean inv dev.cc faabric_dist_tests inv dev.cc faabric_dist_test_server diff --git a/tasks/dev.py b/tasks/dev.py index 6fde80777..86145e428 100644 --- a/tasks/dev.py +++ b/tasks/dev.py @@ -4,6 +4,7 @@ from subprocess import run from tasks.util.env import ( FAABRIC_CONAN_CACHE, + FAABRIC_CONAN_PROFILES, FAABRIC_SHARED_BUILD_DIR, FAABRIC_STATIC_BUILD_DIR, FAABRIC_INSTALL_PREFIX, @@ -15,13 +16,18 @@ @task -def conan(ctx, clean=False, build="Debug"): +def conan(ctx, clean=False, build="Debug", sanitiser="None"): """ Configure dependencies using Conan """ conan_lockfile = f"{PROJ_ROOT}/conan-{build.lower()}.lock" conan_cache = f"{FAABRIC_CONAN_CACHE}/{build.lower()}" - conan_profile = join(PROJ_ROOT, "conanprofile.txt") + if sanitiser == "Thread": + conan_profile = join(FAABRIC_CONAN_PROFILES, "tsan.txt") + elif sanitiser == "Address": + conan_profile = join(FAABRIC_CONAN_PROFILES, "asan.txt") + else: + conan_profile = join(FAABRIC_CONAN_PROFILES, "default.txt") if clean: run(f"rm -f {conan_lockfile}", shell=True, check=True) @@ -39,10 +45,13 @@ def conan(ctx, clean=False, build="Debug"): print(conan_cmd) run(conan_cmd, shell=True, check=True) + # Ensure a clean build by re-building all Conan packages + build_type = "*" if clean else "missing" + conan_install_cmd = ( f"conan install {PROJ_ROOT} -pr:h={conan_profile} " f"-pr:b={conan_profile} -s build_type={build} -of {conan_cache} " - f"--build=missing --lockfile={conan_lockfile}" + f"--build={build_type} --lockfile={conan_lockfile}" ) print(conan_install_cmd) run(conan_install_cmd, shell=True, check=True) diff --git a/tasks/util/env.py b/tasks/util/env.py index 4dfeddd19..fbea59b16 100644 --- a/tasks/util/env.py +++ b/tasks/util/env.py @@ -15,6 +15,7 @@ FAABRIC_CONFIG_FILE = join(PROJ_ROOT, "faabric.ini") FAABRIC_CONAN_CACHE = join(PROJ_ROOT, "conan-cache") +FAABRIC_CONAN_PROFILES = join(PROJ_ROOT, "conan-profiles") CR_NAME = "ghcr.io/faasm" diff --git a/thread-sanitizer-ignorelist.txt b/thread-sanitizer-ignorelist.txt index 1013d918f..57ab819ec 100644 --- a/thread-sanitizer-ignorelist.txt +++ b/thread-sanitizer-ignorelist.txt @@ -11,3 +11,5 @@ race:faabric::mpi::MpiWorld::* # Race in ReaderWriterQueue race:moodycamel::* + +race:google::protobuf::* From 80224f47c978b2f5221067c7042ae47a0e75edb6 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Thu, 9 Oct 2025 13:40:46 +0000 Subject: [PATCH 12/15] fix: run cpp formatting --- src/util/json.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/util/json.cpp b/src/util/json.cpp index ffdd9a2c2..60fd7ebd4 100644 --- a/src/util/json.cpp +++ b/src/util/json.cpp @@ -14,7 +14,8 @@ std::string messageToJson(const google::protobuf::Message& msg) google::protobuf::util::JsonPrintOptions jsonOptions; jsonOptions.always_print_enums_as_ints = true; - auto status = google::protobuf::util::MessageToJsonString(msg, &jsonStr, jsonOptions); + auto status = + google::protobuf::util::MessageToJsonString(msg, &jsonStr, jsonOptions); if (!status.ok()) { SPDLOG_ERROR("Serialising JSON string to protobuf message: {}", status.message().data()); From 1bb7b3bf7fee0feef8c730b5612409ca89925664 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Thu, 9 Oct 2025 15:13:25 +0000 Subject: [PATCH 13/15] fix(conan): update conan-release.lock --- conan-release.lock | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/conan-release.lock b/conan-release.lock index 71d819f76..ae206fc57 100644 --- a/conan-release.lock +++ b/conan-release.lock @@ -4,7 +4,7 @@ "zlib/1.2.12#2ea72a0bae8b680f8a282e103e0a6880%1743582313.324", "spdlog/1.10.0#5aed29a37b544d0eb5812b8be74cd7a4%1731353179.805", "readerwriterqueue/1.0.6#aaa5ff6fac60c2aee591e9e51b063b83%1679587692.891", - "protobuf/3.20.0#9cc01d27c03a1bdd736a031e4b87e1eb%1700600974.257", + "protobuf/6.30.1#97bae23ef6d7f9fcfdb4ded9468ad6de%1747665968.233", "openssl/3.6.0#89e8af1d4a21afcac0557079d23d8890%1759746682.365", "libbacktrace/cci.20210118#a7691bfccd8caaf66309df196790a5a1%1722218217.276", "hiredis/1.0.2#6001a683c04da07565e97e76d0d841bd%1744279494.11", @@ -12,9 +12,10 @@ "flatbuffers/23.5.26#0290575326fe9b2c39a0236ea81d8c30%1743154511.803", "catch2/2.13.9#17ac9b0b78c63353e304c5744e862a77%1678135818.816", "boost/1.84.0#40dd9fecacce0ef109851d0e38727fd0%1759416424.265", - "abseil/20220623.0#bbc28aa2a287b46e488cf28bac5bae65%1706008822.296" + "abseil/20250127.0#faefa3bbf31b5c32933e328d72e42cfe%1754142622.985" ], "build_requires": [ + "cmake/3.31.9#2032c6471fe4f5a3e17f65fed518d545%1758832282.188", "b2/5.3.3#107c15377719889654eb9a162a673975%1750340310.079" ], "python_requires": [], From e12e9cc7782e3487cbfc90ac7bbc47d9a747f3c1 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Thu, 9 Oct 2025 17:54:10 +0000 Subject: [PATCH 14/15] fix(ci): add conan toolchain to example's build --- .github/workflows/tests.yml | 2 ++ tasks/examples.py | 8 ++++++++ 2 files changed, 10 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d4db420de..342875d26 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -193,6 +193,8 @@ jobs: run: ./bin/inv_wrapper.sh dev.cc faabric --shared - name: "Install Faabric shared library" run: ./bin/inv_wrapper.sh dev.install faabric --shared + - name: "Re-run conan (needed for examples)" + run: ./bin/inv_wrapper.sh dev.conan --build=Release - name: "Build examples" run: ./bin/inv_wrapper.sh examples - name: "Run example to check" diff --git a/tasks/examples.py b/tasks/examples.py index 81f7bfd4d..c7bfac9bd 100644 --- a/tasks/examples.py +++ b/tasks/examples.py @@ -5,6 +5,7 @@ from shutil import rmtree from subprocess import run from tasks.util.env import ( + FAABRIC_CONAN_CACHE, FAABRIC_INSTALL_PREFIX, LLVM_VERSION_MAJOR, PROJ_ROOT, @@ -28,12 +29,19 @@ def build(ctx, clean=False): if not exists(BUILD_DIR): makedirs(BUILD_DIR) + conan_cache = f"{FAABRIC_CONAN_CACHE}/release" + if not exists(conan_cache): + print(f"ERROR: expected conan cache in {conan_cache}") + print("ERROR: make sure to run 'inv dev.conan' first") + raise RuntimeError(f"Expected conan cache in {conan_cache}") + # Cmake cmake_cmd = " ".join( [ "cmake", "-GNinja", "-DCMAKE_BUILD_TYPE=Release", + f"-DCMAKE_TOOLCHAIN_FILE={conan_cache}/conan_toolchain.cmake", "-DCMAKE_CXX_FLAGS=-I{}".format(INCLUDE_DIR), "-DCMAKE_EXE_LINKER_FLAGS=-L{}".format(LIB_DIR), "-DCMAKE_CXX_COMPILER=/usr/bin/clang++-{}".format( From 2193aefb48c12bc6f74dcd53f7bad898b0476527 Mon Sep 17 00:00:00 2001 From: Carlos Segarra Date: Thu, 9 Oct 2025 18:16:47 +0000 Subject: [PATCH 15/15] fix(gha): don't force --clean on the conan-cache step --- .github/workflows/tests.yml | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 342875d26..494a3e3bd 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -32,8 +32,8 @@ jobs: from: faabric - name: "Build Conan deps to be shared by all runs" run: | - ./bin/inv_wrapper.sh dev.conan --build Debug --clean - ./bin/inv_wrapper.sh dev.conan --build Release --clean + ./bin/inv_wrapper.sh dev.conan --build Debug + ./bin/inv_wrapper.sh dev.conan --build Release docs: if: github.event.pull_request.draft == false @@ -85,7 +85,7 @@ jobs: - name: "Ping redis" run: redis-cli -h redis ping - name: "Run conan" - run: ./bin/inv_wrapper.sh dev.conan --clean --build=Debug + run: ./bin/inv_wrapper.sh dev.conan --build=Debug - name: "Run cmake for tests" run: ./bin/inv_wrapper.sh dev.cmake --clean --build=Debug --coverage - name: "Build tests" @@ -128,9 +128,15 @@ jobs: from: faabric - name: "Ping redis" run: redis-cli -h redis ping - # Re-run conan with the --clean flag to rebuild the packages with sanitisers - - name: "Run conan" + # For sanitised builds, we need to also re-build all Conan dependencies + # so that they are also sanitized. Otherwise this causes unexpected + # crashes. + - name: "Re-build conan deps" + if: ${{ matrix.sanitiser != 'None' }} run: ./bin/inv_wrapper.sh dev.conan --clean --build=Debug --sanitiser ${{ matrix.sanitiser }} + - name: "Run conan" + if: ${{ matrix.sanitiser == 'None' }} + run: ./bin/inv_wrapper.sh dev.conan --build=Debug --sanitiser ${{ matrix.sanitiser }} - name: "Run cmake for tests" run: ./bin/inv_wrapper.sh dev.cmake --clean --build=Debug --sanitiser ${{ matrix.sanitiser }} - name: "Build tests"