diff --git a/.env b/.env index b030125c9..a8de1017a 100644 --- a/.env +++ b/.env @@ -1,4 +1,4 @@ -FAABRIC_VERSION=0.21.0 -FAABRIC_CLI_IMAGE=ghcr.io/faasm/faabric:0.21.0 +FAABRIC_VERSION=0.22.0 +FAABRIC_CLI_IMAGE=ghcr.io/faasm/faabric:0.22.0 COMPOSE_PROJECT_NAME=faabric-dev CONAN_CACHE_MOUNT_SOURCE=./conan-cache/ diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 957965d0d..494a3e3bd 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -20,21 +20,26 @@ jobs: if: github.event.pull_request.draft == false runs-on: ubuntu-latest container: - image: ghcr.io/faasm/faabric:0.21.0 + image: ghcr.io/faasm/faabric:0.22.0 env: DEPLOYMENT_TYPE: gha-ci steps: - name: "Check-out code" uses: actions/checkout@v4 - - uses: faasm/conan-cache-action@v3 - - name: "Build Conan CMake deps to be shared by all runs" - run: ./bin/inv_wrapper.sh dev.cmake --build Debug --clean + - uses: faasm/conan-cache-action@main + with: + build-type: Debug + from: faabric + - name: "Build Conan deps to be shared by all runs" + run: | + ./bin/inv_wrapper.sh dev.conan --build Debug + ./bin/inv_wrapper.sh dev.conan --build Release docs: if: github.event.pull_request.draft == false runs-on: ubuntu-latest container: - image: ghcr.io/faasm/faabric:0.21.0 + image: ghcr.io/faasm/faabric:0.22.0 steps: - name: "Check out code" uses: actions/checkout@v4 @@ -45,7 +50,7 @@ jobs: if: github.event.pull_request.draft == false runs-on: ubuntu-latest container: - image: ghcr.io/faasm/faabric:0.21.0 + image: ghcr.io/faasm/faabric:0.22.0 steps: - name: "Check out code" uses: actions/checkout@v4 @@ -65,7 +70,7 @@ jobs: REDIS_QUEUE_HOST: redis REDIS_STATE_HOST: redis container: - image: ghcr.io/faasm/faabric:0.21.0 + image: ghcr.io/faasm/faabric:0.22.0 options: --privileged services: redis: @@ -73,9 +78,14 @@ jobs: steps: - name: "Check-out code" uses: actions/checkout@v4 - - uses: faasm/conan-cache-action@v3 + - uses: faasm/conan-cache-action@main + with: + build-type: Debug + from: faabric - name: "Ping redis" run: redis-cli -h redis ping + - name: "Run conan" + run: ./bin/inv_wrapper.sh dev.conan --build=Debug - name: "Run cmake for tests" run: ./bin/inv_wrapper.sh dev.cmake --clean --build=Debug --coverage - name: "Build tests" @@ -98,13 +108,13 @@ jobs: strategy: fail-fast: false matrix: - sanitiser: [None, Address, Thread, Undefined] + sanitiser: [None, Thread, Address, Undefined] env: DEPLOYMENT_TYPE: gha-ci REDIS_QUEUE_HOST: redis REDIS_STATE_HOST: redis container: - image: ghcr.io/faasm/faabric:0.21.0 + image: ghcr.io/faasm/faabric:0.22.0 options: --privileged services: redis: @@ -112,9 +122,21 @@ jobs: steps: - name: "Check-out code" uses: actions/checkout@v4 - - uses: faasm/conan-cache-action@v3 + - uses: faasm/conan-cache-action@main + with: + build-type: Debug + from: faabric - name: "Ping redis" run: redis-cli -h redis ping + # For sanitised builds, we need to also re-build all Conan dependencies + # so that they are also sanitized. Otherwise this causes unexpected + # crashes. + - name: "Re-build conan deps" + if: ${{ matrix.sanitiser != 'None' }} + run: ./bin/inv_wrapper.sh dev.conan --clean --build=Debug --sanitiser ${{ matrix.sanitiser }} + - name: "Run conan" + if: ${{ matrix.sanitiser == 'None' }} + run: ./bin/inv_wrapper.sh dev.conan --build=Debug --sanitiser ${{ matrix.sanitiser }} - name: "Run cmake for tests" run: ./bin/inv_wrapper.sh dev.cmake --clean --build=Debug --sanitiser ${{ matrix.sanitiser }} - name: "Build tests" @@ -133,8 +155,10 @@ jobs: # --- Code update --- - name: "Check out code" uses: actions/checkout@v4 - - name: "Conan cache" - uses: faasm/conan-cache-action@v3 + - uses: faasm/conan-cache-action@main + with: + build-type: Debug + from: faabric - name: "Build distributed tests" run: ./dist-test/build.sh - name: "Run the distributed tests" @@ -156,22 +180,27 @@ jobs: REDIS_QUEUE_HOST: redis REDIS_STATE_HOST: redis container: - image: ghcr.io/faasm/faabric:0.21.0 + image: ghcr.io/faasm/faabric:0.22.0 services: redis: image: redis steps: - name: "Check-out code" uses: actions/checkout@v4 - - uses: faasm/conan-cache-action@v3 + - uses: faasm/conan-cache-action@main with: - build-type: "release" + build-type: Release + from: faabric + - name: "Run conan" + run: ./bin/inv_wrapper.sh dev.conan --build=Release - name: "Run cmake shared" run: ./bin/inv_wrapper.sh dev.cmake --clean --shared --build=Release - name: "Build Faabric shared library" run: ./bin/inv_wrapper.sh dev.cc faabric --shared - name: "Install Faabric shared library" run: ./bin/inv_wrapper.sh dev.install faabric --shared + - name: "Re-run conan (needed for examples)" + run: ./bin/inv_wrapper.sh dev.conan --build=Release - name: "Build examples" run: ./bin/inv_wrapper.sh examples - name: "Run example to check" diff --git a/.ycm_extra_conf.py b/.ycm_extra_conf.py deleted file mode 100644 index 18a613bab..000000000 --- a/.ycm_extra_conf.py +++ /dev/null @@ -1,13 +0,0 @@ -from os.path import dirname, realpath, join, exists - -_PROJ_ROOT = dirname(realpath(__file__)) - - -def Settings(**kwargs): - venv_interpreter = join(_PROJ_ROOT, "venv", "bin", "python") - - if not exists(venv_interpreter): - parent_root = dirname(dirname(_PROJ_ROOT)) - venv_interpreter = join(parent_root, "venv", "bin", "python") - - return {"interpreter_path": venv_interpreter} diff --git a/VERSION b/VERSION index 885415662..215740905 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.21.0 +0.22.0 diff --git a/cmake/ExternalProjects.cmake b/cmake/ExternalProjects.cmake index 4f64c643c..d2e6ae34b 100644 --- a/cmake/ExternalProjects.cmake +++ b/cmake/ExternalProjects.cmake @@ -4,86 +4,18 @@ include (ExternalProject) include (FetchContent) find_package (Threads REQUIRED) -# Find conan-generated package descriptions +# Find conan-generated package descriptions. list(PREPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_BINARY_DIR}) list(PREPEND CMAKE_PREFIX_PATH ${CMAKE_CURRENT_BINARY_DIR}) -if(NOT EXISTS "${CMAKE_CURRENT_BINARY_DIR}/conan.cmake") - message(STATUS "Downloading conan.cmake from https://github.com/conan-io/cmake-conan") - file(DOWNLOAD "https://raw.githubusercontent.com/conan-io/cmake-conan/0.18.1/conan.cmake" - "${CMAKE_CURRENT_BINARY_DIR}/conan.cmake" - TLS_VERIFY ON) -endif() - -set(CONAN_CMAKE_SILENT_OUTPUT ON CACHE INTERNAL "") -include(${CMAKE_CURRENT_BINARY_DIR}/conan.cmake) - -conan_check(VERSION 1.63.0 REQUIRED) - -# Enable revisions in the conan config -execute_process(COMMAND ${CONAN_CMD} config set general.revisions_enabled=1 - RESULT_VARIABLE RET_CODE) -if(NOT "${RET_CODE}" STREQUAL "0") - message(FATAL_ERROR "Error setting revisions for Conan: '${RET_CODE}'") -endif() - -# -------------------------------- -# Conan dependencies -# -------------------------------- - -conan_cmake_configure( - REQUIRES - "abseil/20220623.0@#732381dc99db29b4cfd293684891da56" - "boost/1.84.0@#7604ce1e7485780469dffb6430f232ea" - "catch2/2.13.9@#8793d3e6287d3684201418de556d98fe" - "flatbuffers/23.5.26@#b153646f6546daab4c7326970b6cd89c" - "hiredis/1.0.2@#370dad964286cadb1f15dc90252e8ef3" - "openssl/3.0.2@#269fa93e5afe8c34bd9a0030d2b8f0fe" - "protobuf/3.20.0@#8e4de7081bea093469c9e6076149b2b4" - "readerwriterqueue/1.0.6@#a95c8da3d68822dec4d4c13fff4b5c96" - "spdlog/1.10.0@#6406c337028e15e56cd6a070cbac54c4" - "zlib/1.2.12@#3b9e037ae1c615d045a06c67d88491ae" - GENERATORS - cmake_find_package - cmake_paths - OPTIONS - boost:error_code_header_only=True - boost:system_no_deprecated=True - boost:zlib=False - boost:bzip2=False - boost:lzma=False - boost:zstd=False - boost:without_locale=True - boost:without_log=True - boost:without_mpi=True - boost:without_python=True - boost:without_test=True - boost:without_wave=True -) - -conan_cmake_autodetect(FAABRIC_CONAN_SETTINGS) - -conan_cmake_install(PATH_OR_REFERENCE . - BUILD outdated - UPDATE - REMOTE conancenter - PROFILE_HOST ${CMAKE_CURRENT_LIST_DIR}/../conan-profile.txt - PROFILE_BUILD ${CMAKE_CURRENT_LIST_DIR}/../conan-profile.txt - SETTINGS ${FAABRIC_CONAN_SETTINGS} -) - -include(${CMAKE_CURRENT_BINARY_DIR}/conan_paths.cmake) - find_package(absl QUIET REQUIRED) -find_package(Boost 1.80.0 QUIET REQUIRED) +find_package(Boost 1.84.0 QUIET REQUIRED COMPONENTS filesystem program_options system) find_package(Catch2 QUIET REQUIRED) -find_package(flatbuffers QUIET REQUIRED) +find_package(flatbuffers CONFIG QUIET REQUIRED) find_package(fmt QUIET REQUIRED) find_package(hiredis QUIET REQUIRED) -# 27/01/2023 - Pin OpenSSL to a specific version to avoid incompatibilities -# with the system's (i.e. Ubuntu 22.04) OpenSSL -find_package(OpenSSL 3.0.2 QUIET REQUIRED) -find_package(Protobuf 3.20.0 QUIET REQUIRED) +find_package(OpenSSL 3.6.0 QUIET REQUIRED) +find_package(Protobuf 6.30.1 QUIET REQUIRED) find_package(readerwriterqueue QUIET REQUIRED) find_package(spdlog QUIET REQUIRED) find_package(ZLIB QUIET REQUIRED) @@ -121,7 +53,7 @@ FetchContent_Declare(nng_ext ) FetchContent_Declare(zstd_ext GIT_REPOSITORY "https://github.com/facebook/zstd" - GIT_TAG "v1.5.2" + GIT_TAG "v1.5.7" SOURCE_SUBDIR "build/cmake" ) @@ -150,7 +82,9 @@ target_link_libraries(faabric_common_dependencies INTERFACE absl::flat_hash_map absl::strings atomic_queue::atomic_queue - Boost::Boost + Boost::headers + Boost::filesystem + Boost::program_options Boost::system flatbuffers::flatbuffers hiredis::hiredis diff --git a/conan-debug.lock b/conan-debug.lock new file mode 100644 index 000000000..ae206fc57 --- /dev/null +++ b/conan-debug.lock @@ -0,0 +1,23 @@ +{ + "version": "0.5", + "requires": [ + "zlib/1.2.12#2ea72a0bae8b680f8a282e103e0a6880%1743582313.324", + "spdlog/1.10.0#5aed29a37b544d0eb5812b8be74cd7a4%1731353179.805", + "readerwriterqueue/1.0.6#aaa5ff6fac60c2aee591e9e51b063b83%1679587692.891", + "protobuf/6.30.1#97bae23ef6d7f9fcfdb4ded9468ad6de%1747665968.233", + "openssl/3.6.0#89e8af1d4a21afcac0557079d23d8890%1759746682.365", + "libbacktrace/cci.20210118#a7691bfccd8caaf66309df196790a5a1%1722218217.276", + "hiredis/1.0.2#6001a683c04da07565e97e76d0d841bd%1744279494.11", + "fmt/8.1.1#d66420384a943dbf1f292d34ef34c11c%1735899179.969", + "flatbuffers/23.5.26#0290575326fe9b2c39a0236ea81d8c30%1743154511.803", + "catch2/2.13.9#17ac9b0b78c63353e304c5744e862a77%1678135818.816", + "boost/1.84.0#40dd9fecacce0ef109851d0e38727fd0%1759416424.265", + "abseil/20250127.0#faefa3bbf31b5c32933e328d72e42cfe%1754142622.985" + ], + "build_requires": [ + "cmake/3.31.9#2032c6471fe4f5a3e17f65fed518d545%1758832282.188", + "b2/5.3.3#107c15377719889654eb9a162a673975%1750340310.079" + ], + "python_requires": [], + "config_requires": [] +} \ No newline at end of file diff --git a/conan-profiles/asan.txt b/conan-profiles/asan.txt new file mode 100644 index 000000000..6822d85a8 --- /dev/null +++ b/conan-profiles/asan.txt @@ -0,0 +1,15 @@ +[settings] +os=Linux +arch=x86_64 +compiler=clang +compiler.version=17 +compiler.libcxx=libstdc++11 +compiler.cppstd=20 +build_type=RelWithDebInfo + +[conf] +tools.build:compiler_executables={"c": "/usr/bin/clang-17", "cpp": "/usr/bin/clang++-17"} +tools.build:cflags=["-fsanitize=address","-fno-omit-frame-pointer","-g","-O1"] +tools.build:cxxflags=["-fsanitize=address","-fno-omit-frame-pointer","-g","-O1"] +tools.build:sharedlinkflags=["-fsanitize=address"] +tools.build:exelinkflags=["-fsanitize=address"] diff --git a/conan-profile.txt b/conan-profiles/default.txt similarity index 54% rename from conan-profile.txt rename to conan-profiles/default.txt index 2edf1e4b0..94943459e 100644 --- a/conan-profile.txt +++ b/conan-profiles/default.txt @@ -1,15 +1,11 @@ [settings] os=Linux -os_build=Linux arch=x86_64 -arch_build=x86_64 compiler=clang compiler.version=17 compiler.libcxx=libstdc++11 compiler.cppstd=20 build_type=Release -[options] -[build_requires] -[env] -CC=/usr/bin/clang-17 -CXX=/usr/bin/clang++-17 + +[conf] +tools.build:compiler_executables={"c": "/usr/bin/clang-17", "cpp": "/usr/bin/clang++-17"} diff --git a/conan-profiles/tsan.txt b/conan-profiles/tsan.txt new file mode 100644 index 000000000..364705599 --- /dev/null +++ b/conan-profiles/tsan.txt @@ -0,0 +1,15 @@ +[settings] +os=Linux +arch=x86_64 +compiler=clang +compiler.version=17 +compiler.libcxx=libstdc++11 +compiler.cppstd=20 +build_type=RelWithDebInfo + +[conf] +tools.build:compiler_executables={"c": "/usr/bin/clang-17", "cpp": "/usr/bin/clang++-17"} +tools.build:cxxflags=["-fsanitize=thread","-fno-omit-frame-pointer","-g"] +tools.build:cflags=["-fsanitize=thread","-fno-omit-frame-pointer","-g"] +tools.build:sharedlinkflags=["-fsanitize=thread"] +tools.build:exelinkflags=["-fsanitize=thread"] diff --git a/conan-release.lock b/conan-release.lock new file mode 100644 index 000000000..ae206fc57 --- /dev/null +++ b/conan-release.lock @@ -0,0 +1,23 @@ +{ + "version": "0.5", + "requires": [ + "zlib/1.2.12#2ea72a0bae8b680f8a282e103e0a6880%1743582313.324", + "spdlog/1.10.0#5aed29a37b544d0eb5812b8be74cd7a4%1731353179.805", + "readerwriterqueue/1.0.6#aaa5ff6fac60c2aee591e9e51b063b83%1679587692.891", + "protobuf/6.30.1#97bae23ef6d7f9fcfdb4ded9468ad6de%1747665968.233", + "openssl/3.6.0#89e8af1d4a21afcac0557079d23d8890%1759746682.365", + "libbacktrace/cci.20210118#a7691bfccd8caaf66309df196790a5a1%1722218217.276", + "hiredis/1.0.2#6001a683c04da07565e97e76d0d841bd%1744279494.11", + "fmt/8.1.1#d66420384a943dbf1f292d34ef34c11c%1735899179.969", + "flatbuffers/23.5.26#0290575326fe9b2c39a0236ea81d8c30%1743154511.803", + "catch2/2.13.9#17ac9b0b78c63353e304c5744e862a77%1678135818.816", + "boost/1.84.0#40dd9fecacce0ef109851d0e38727fd0%1759416424.265", + "abseil/20250127.0#faefa3bbf31b5c32933e328d72e42cfe%1754142622.985" + ], + "build_requires": [ + "cmake/3.31.9#2032c6471fe4f5a3e17f65fed518d545%1758832282.188", + "b2/5.3.3#107c15377719889654eb9a162a673975%1750340310.079" + ], + "python_requires": [], + "config_requires": [] +} \ No newline at end of file diff --git a/conanfile.txt b/conanfile.txt new file mode 100644 index 000000000..d289607c7 --- /dev/null +++ b/conanfile.txt @@ -0,0 +1,29 @@ +[requires] +abseil/20250127.0 +boost/1.84.0 +catch2/2.13.9 +flatbuffers/23.5.26 +hiredis/1.0.2 +openssl/3.6.0 +protobuf/6.30.1 +readerwriterqueue/1.0.6 +spdlog/1.10.0 +zlib/1.2.12 + +[generators] +CMakeToolchain +CMakeDeps + +[options] +boost/*:error_code_header_only=True +boost/*:system_no_deprecated=True +boost/*:zlib=False +boost/*:bzip2=False +boost/*:lzma=False +boost/*:zstd=False +boost/*:without_locale=True +boost/*:without_log=True +boost/*:without_mpi=True +boost/*:without_python=True +boost/*:without_test=True +boost/*:without_wave=True diff --git a/dist-test/build_internal.sh b/dist-test/build_internal.sh index ede7f3a62..e5f3dd894 100755 --- a/dist-test/build_internal.sh +++ b/dist-test/build_internal.sh @@ -9,6 +9,7 @@ pushd ${PROJ_ROOT} >> /dev/null source ./bin/workon.sh # Run the debug build +inv dev.conan --build=Debug inv dev.cmake --build=Debug --clean inv dev.cc faabric_dist_tests inv dev.cc faabric_dist_test_server diff --git a/docker/faabric-base.dockerfile b/docker/faabric-base.dockerfile index 2be4e1238..7084fa67c 100644 --- a/docker/faabric-base.dockerfile +++ b/docker/faabric-base.dockerfile @@ -50,18 +50,20 @@ RUN apt update && apt install -y \ unzip # Install up-to-date CMake +ARG CMAKE_VERSION=4.1.2 RUN apt remove --purge --auto-remove cmake \ && mkdir -p /setup \ && cd /setup \ && wget -q -O cmake-linux.sh \ - https://github.com/Kitware/CMake/releases/download/v3.28.0/cmake-3.28.0-linux-x86_64.sh \ + https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-x86_64.sh \ && sh cmake-linux.sh -- --skip-license --prefix=/usr/local \ && apt clean autoclean -y \ && apt autoremove -y # Install Conan +ARG CONAN_VERSION=2.21.0 RUN curl -s -L -o \ - /tmp/conan-latest.deb https://github.com/conan-io/conan/releases/download/1.63.0/conan-ubuntu-64.deb \ + /tmp/conan-latest.deb https://github.com/conan-io/conan/releases/download/${CONAN_VERSION}/conan-${CONAN_VERSION}-amd64.deb \ && sudo dpkg -i /tmp/conan-latest.deb \ && rm -f /tmp/conan-latest.deb diff --git a/docker/faabric.dockerfile b/docker/faabric.dockerfile index 990cf8640..14b0394d6 100644 --- a/docker/faabric.dockerfile +++ b/docker/faabric.dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/faasm/faabric-base:0.21.0 +FROM ghcr.io/faasm/faabric-base:0.22.0 ARG FAABRIC_VERSION # faabic-base image is not re-built often, so tag may be behind @@ -18,6 +18,7 @@ WORKDIR /code/faabric # Python set-up and code builds RUN ./bin/create_venv.sh \ && source venv/bin/activate \ + && inv dev.conan --build=Release \ # Static build && inv dev.cmake --build=Release \ && inv dev.cc faabric \ diff --git a/docker/planner.dockerfile b/docker/planner.dockerfile index 02f3e92fc..c4af2be31 100644 --- a/docker/planner.dockerfile +++ b/docker/planner.dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/faasm/faabric-base:0.21.0 +FROM ghcr.io/faasm/faabric-base:0.22.0 ARG FAABRIC_VERSION # Flag to say we're in a container @@ -15,6 +15,7 @@ RUN rm -rf /code \ && cd /code/faabric \ && ./bin/create_venv.sh \ && source venv/bin/activate \ + && inv dev.conan --build=Release \ && inv dev.cmake --build=Release \ && inv dev.cc planner_server diff --git a/include/faabric/util/clock.h b/include/faabric/util/clock.h index 5f71096c0..0d4e58b9c 100644 --- a/include/faabric/util/clock.h +++ b/include/faabric/util/clock.h @@ -12,13 +12,13 @@ class Clock const TimePoint now(); - const long epochMillis(); + long epochMillis(); - const long timeDiff(const TimePoint& t1, const TimePoint& t2); + long timeDiff(const TimePoint& t1, const TimePoint& t2); - const long timeDiffNano(const TimePoint& t1, const TimePoint& t2); + long timeDiffNano(const TimePoint& t1, const TimePoint& t2); - const long timeDiffMicro(const TimePoint& t1, const TimePoint& t2); + long timeDiffMicro(const TimePoint& t1, const TimePoint& t2); }; Clock& getGlobalClock(); diff --git a/src/flat/CMakeLists.txt b/src/flat/CMakeLists.txt index 18230c182..5f73f178f 100644 --- a/src/flat/CMakeLists.txt +++ b/src/flat/CMakeLists.txt @@ -10,7 +10,7 @@ add_custom_command( OUTPUT "${FB_HEADER}" DEPENDS faabric.fbs WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} - COMMAND ${CONAN_FLATBUFFERS_ROOT}/bin/flatc + COMMAND flatbuffers::flatc ARGS --cpp --gen-mutable --raw-binary faabric.fbs ) diff --git a/src/util/clock.cpp b/src/util/clock.cpp index 9def1ea19..23bb06822 100644 --- a/src/util/clock.cpp +++ b/src/util/clock.cpp @@ -14,7 +14,7 @@ const TimePoint Clock::now() return std::chrono::steady_clock::now(); } -const long Clock::epochMillis() +long Clock::epochMillis() { long millis = std::chrono::duration_cast( std::chrono::system_clock::now().time_since_epoch()) @@ -23,21 +23,21 @@ const long Clock::epochMillis() return millis; } -const long Clock::timeDiff(const TimePoint& t1, const TimePoint& t2) +long Clock::timeDiff(const TimePoint& t1, const TimePoint& t2) { long age = std::chrono::duration_cast(t1 - t2).count(); return age; } -const long Clock::timeDiffMicro(const TimePoint& t1, const TimePoint& t2) +long Clock::timeDiffMicro(const TimePoint& t1, const TimePoint& t2) { long age = std::chrono::duration_cast(t1 - t2).count(); return age; } -const long Clock::timeDiffNano(const TimePoint& t1, const TimePoint& t2) +long Clock::timeDiffNano(const TimePoint& t1, const TimePoint& t2) { long age = std::chrono::duration_cast(t1 - t2).count(); diff --git a/src/util/json.cpp b/src/util/json.cpp index 23d8c460f..60fd7ebd4 100644 --- a/src/util/json.cpp +++ b/src/util/json.cpp @@ -11,10 +11,10 @@ std::string messageToJson(const google::protobuf::Message& msg) // Set the JSON print options. This is very important to ensure backwards- // compatibility with clients sending HTTP requests to faabric - google::protobuf::util::JsonOptions jsonOptions; + google::protobuf::util::JsonPrintOptions jsonOptions; jsonOptions.always_print_enums_as_ints = true; - google::protobuf::util::Status status = + auto status = google::protobuf::util::MessageToJsonString(msg, &jsonStr, jsonOptions); if (!status.ok()) { SPDLOG_ERROR("Serialising JSON string to protobuf message: {}", @@ -28,8 +28,7 @@ std::string messageToJson(const google::protobuf::Message& msg) void jsonToMessage(const std::string& jsonStr, google::protobuf::Message* msg) { - google::protobuf::util::Status status = - google::protobuf::util::JsonStringToMessage(jsonStr, msg); + auto status = google::protobuf::util::JsonStringToMessage(jsonStr, msg); if (!status.ok()) { SPDLOG_ERROR("Deserialising JSON string to protobuf message: {}", status.message().data()); diff --git a/tasks/dev.py b/tasks/dev.py index ad17b6897..86145e428 100644 --- a/tasks/dev.py +++ b/tasks/dev.py @@ -3,6 +3,8 @@ from shutil import rmtree from subprocess import run from tasks.util.env import ( + FAABRIC_CONAN_CACHE, + FAABRIC_CONAN_PROFILES, FAABRIC_SHARED_BUILD_DIR, FAABRIC_STATIC_BUILD_DIR, FAABRIC_INSTALL_PREFIX, @@ -13,6 +15,48 @@ from invoke import task +@task +def conan(ctx, clean=False, build="Debug", sanitiser="None"): + """ + Configure dependencies using Conan + """ + conan_lockfile = f"{PROJ_ROOT}/conan-{build.lower()}.lock" + conan_cache = f"{FAABRIC_CONAN_CACHE}/{build.lower()}" + if sanitiser == "Thread": + conan_profile = join(FAABRIC_CONAN_PROFILES, "tsan.txt") + elif sanitiser == "Address": + conan_profile = join(FAABRIC_CONAN_PROFILES, "asan.txt") + else: + conan_profile = join(FAABRIC_CONAN_PROFILES, "default.txt") + + if clean: + run(f"rm -f {conan_lockfile}", shell=True, check=True) + run(f"rm -rf {conan_cache}", shell=True, check=True) + + # Generate a conan lock file if it does not exist. This file should be + # comitted for version control. + if not exists(conan_lockfile): + run("conan remote list", shell=True, check=True) + conan_cmd = ( + f"conan lock create {PROJ_ROOT} -pr:h={conan_profile} " + f"-pr:b={conan_profile} -s build_type={build} " + f"--lockfile-out={conan_lockfile}" + ) + print(conan_cmd) + run(conan_cmd, shell=True, check=True) + + # Ensure a clean build by re-building all Conan packages + build_type = "*" if clean else "missing" + + conan_install_cmd = ( + f"conan install {PROJ_ROOT} -pr:h={conan_profile} " + f"-pr:b={conan_profile} -s build_type={build} -of {conan_cache} " + f"--build={build_type} --lockfile={conan_lockfile}" + ) + print(conan_install_cmd) + run(conan_install_cmd, shell=True, check=True) + + @task def cmake( ctx, @@ -42,9 +86,16 @@ def cmake( if build not in build_types: raise RuntimeError("Expected build to be in {}".format(build_types)) + conan_cache = f"{FAABRIC_CONAN_CACHE}/{build.lower()}" + if not exists(conan_cache): + print(f"ERROR: expected conan cache in {conan_cache}") + print("ERROR: make sure to run 'inv dev.conan' first") + raise RuntimeError(f"Expected conan cache in {conan_cache}") + cmd = [ "cmake", "-GNinja", + f"-DCMAKE_TOOLCHAIN_FILE={conan_cache}/conan_toolchain.cmake", "-DCMAKE_INSTALL_PREFIX={}".format(FAABRIC_INSTALL_PREFIX), "-DCMAKE_BUILD_TYPE={}".format(build), "-DBUILD_SHARED_LIBS={}".format("ON" if shared else "OFF"), diff --git a/tasks/examples.py b/tasks/examples.py index 81f7bfd4d..c7bfac9bd 100644 --- a/tasks/examples.py +++ b/tasks/examples.py @@ -5,6 +5,7 @@ from shutil import rmtree from subprocess import run from tasks.util.env import ( + FAABRIC_CONAN_CACHE, FAABRIC_INSTALL_PREFIX, LLVM_VERSION_MAJOR, PROJ_ROOT, @@ -28,12 +29,19 @@ def build(ctx, clean=False): if not exists(BUILD_DIR): makedirs(BUILD_DIR) + conan_cache = f"{FAABRIC_CONAN_CACHE}/release" + if not exists(conan_cache): + print(f"ERROR: expected conan cache in {conan_cache}") + print("ERROR: make sure to run 'inv dev.conan' first") + raise RuntimeError(f"Expected conan cache in {conan_cache}") + # Cmake cmake_cmd = " ".join( [ "cmake", "-GNinja", "-DCMAKE_BUILD_TYPE=Release", + f"-DCMAKE_TOOLCHAIN_FILE={conan_cache}/conan_toolchain.cmake", "-DCMAKE_CXX_FLAGS=-I{}".format(INCLUDE_DIR), "-DCMAKE_EXE_LINKER_FLAGS=-L{}".format(LIB_DIR), "-DCMAKE_CXX_COMPILER=/usr/bin/clang++-{}".format( diff --git a/tasks/format_code.py b/tasks/format_code.py index 39b97c01b..d30f14ca9 100644 --- a/tasks/format_code.py +++ b/tasks/format_code.py @@ -33,7 +33,6 @@ def format(ctx, check=False): flake8_cmd = [ "python3 -m flake8", - "{}".format("--format" if not check else ""), " ".join(files_to_check), ] flake8_cmd = " ".join(flake8_cmd) diff --git a/tasks/util/env.py b/tasks/util/env.py index 42151ea3d..fbea59b16 100644 --- a/tasks/util/env.py +++ b/tasks/util/env.py @@ -14,6 +14,9 @@ FAABRIC_CONFIG_FILE = join(PROJ_ROOT, "faabric.ini") +FAABRIC_CONAN_CACHE = join(PROJ_ROOT, "conan-cache") +FAABRIC_CONAN_PROFILES = join(PROJ_ROOT, "conan-profiles") + CR_NAME = "ghcr.io/faasm" # This LLVM version is for the LLVM that we use to compile regular C/C++ code diff --git a/thread-sanitizer-ignorelist.txt b/thread-sanitizer-ignorelist.txt index 1013d918f..57ab819ec 100644 --- a/thread-sanitizer-ignorelist.txt +++ b/thread-sanitizer-ignorelist.txt @@ -11,3 +11,5 @@ race:faabric::mpi::MpiWorld::* # Race in ReaderWriterQueue race:moodycamel::* + +race:google::protobuf::*