diff --git a/README.md b/README.md index c2b99df..38b1ddb 100644 --- a/README.md +++ b/README.md @@ -19,13 +19,13 @@ layer of protection against unintentionally linking to any system-provided libraries. The versions of LLVM and Musl as well as a few other tunables are set in `config`: -by default, mainline `llvmorg-21.1.0-rc1` tag is used together with a patched +by default, a reasonably fresh mainline LLVM version is used together with a patched version of Musl that can be obtained at https://github.com/access-softek/musl. The choice of Linux kernel version is mostly arbitrary: it is only used to provide kernel headers to Musl, thus any recent version should work. (As this version does not have to be adjusted by the user, it is defined in the -`scripts/global-vars` file instead.) +`scripts/global-vars.inc.sh` file instead.) Please note that while basic sanity check is performed to make sure the expected SHA1 hashes are checked out under `./src/llvm` and `./src/musl`, it is diff --git a/build.sh b/build.sh index bd60348..a19f2b1 100755 --- a/build.sh +++ b/build.sh @@ -4,6 +4,16 @@ ROOT="$(dirname "$0")" ROOT="$(realpath "$ROOT")" cd "$ROOT" +set -x +. ./scripts/common.inc.sh +. ./scripts/global-vars.inc.sh +set_global_variables host_build "$ROOT" +set_global_variables docker_host "$ROOT" +set_global_variables docker +# Local configuration file may reference the variables defined above. +. ./config +set +x + check_repo_sha() { local repo_path="$1" local expected_sha="$2" @@ -50,8 +60,8 @@ fetch_git_commit() { } fetch_sources() { - . ./config - . ./scripts/global-vars + # Note: the *_host_build variables referenced by this function are the + # same as their *_docker_host counterparts. if [ "$#" != 2 ]; then echo "Usage: docker.sh sources " @@ -62,18 +72,18 @@ fetch_sources() { local llvm_repo="$1" local musl_repo="$2" - mkdir -p "$ROOT/src" - fetch_git_commit "$ROOT/src/llvm" "$llvm_repo" "$LLVM_BRANCH" "$LLVM_SHA" - fetch_git_commit "$ROOT/src/musl" "$musl_repo" "$MUSL_BRANCH" "$MUSL_SHA" + mkdir -p "$SRC_DIR_host_build" + fetch_git_commit "$LLVM_SOURCE_DIR_host_build" "$llvm_repo" "$LLVM_BRANCH" "$LLVM_SHA" + fetch_git_commit "$MUSL_SOURCE_DIR_host_build" "$musl_repo" "$MUSL_BRANCH" "$MUSL_SHA" - local LOCAL_TARBALL_PATH="$ROOT/src/$LINUX_KERNEL_TARBALL_BASENAME" + local LOCAL_TARBALL_PATH="$SRC_DIR_host_build/$LINUX_KERNEL_TARBALL_BASENAME" if [ ! -f "$LOCAL_TARBALL_PATH" ]; then echo "Missing $LOCAL_TARBALL_PATH, downloading from $LINUX_KERNEL_TARBALL_URL..." curl -sSL "$LINUX_KERNEL_TARBALL_URL" -o "$LOCAL_TARBALL_PATH" fi - check_repo_sha "$ROOT/src/llvm" "$LLVM_SHA" - check_repo_sha "$ROOT/src/musl" "$MUSL_SHA" + check_repo_sha "$LLVM_SOURCE_DIR_host_build" "$LLVM_SHA" + check_repo_sha "$MUSL_SOURCE_DIR_host_build" "$MUSL_SHA" local computed_sha256="$(sha256sum "$LOCAL_TARBALL_PATH" | sed 's/[ \t].*$//')" if [ "$computed_sha256" = "$LINUX_KERNEL_SHA256" ]; then @@ -87,37 +97,32 @@ fetch_sources() { } build_in_docker() { - # Path inside the container. - REPO_ROOT=/repo - . ./config - . ./scripts/global-vars - - check_repo_sha "$ROOT/src/llvm" "$LLVM_SHA" - check_repo_sha "$ROOT/src/musl" "$MUSL_SHA" + check_repo_sha "$LLVM_SOURCE_DIR_docker_host" "$LLVM_SHA" + check_repo_sha "$MUSL_SOURCE_DIR_docker_host" "$MUSL_SHA" $DOCKER_CMD build \ -t "$DOCKER_IMAGE_NAME" \ -f Dockerfile.builder \ - --build-arg REPO_ROOT="$REPO_ROOT" \ - "$ROOT" + --build-arg REPO_ROOT="$REPO_ROOT_docker" \ + "$REPO_ROOT_docker_host" $DOCKER_CMD run -ti --rm \ - --volume "$ROOT/output:$OUTPUT_DIR:rw" \ - --volume "$ROOT/ccache:$CCACHE_DIR:rw" \ - --volume "$ROOT/src:$SRC_DIR:ro" \ - --volume "$ROOT/tmp:/tmp:rw" \ + --volume "$OUTPUT_DIR_docker_host:$OUTPUT_DIR_docker:rw" \ + --volume "$CCACHE_DIR_docker_host:$CCACHE_DIR_docker:rw" \ + --volume "$SRC_DIR_docker_host:$SRC_DIR_docker:ro" \ + --volume "$REPO_ROOT_docker_host/tmp:/tmp:rw" \ --tmpfs "$DOCKER_BUILD_INSTALL_DIR:rw,exec,size=2G" \ - --tmpfs "$BUILD_TMP:rw,exec,size=8G" \ - "$DOCKER_IMAGE_NAME" "$REPO_ROOT/scripts/build-in-docker.sh" + --tmpfs "$BUILD_TMP_docker:rw,exec,size=8G" \ + "$DOCKER_IMAGE_NAME" "$REPO_ROOT_docker/scripts/build-in-docker.sh" } build_on_host() { - REPO_ROOT="$ROOT" - . ./config - . ./scripts/global-vars + set -x + reexport_variables host_build export INSTALL_DIR="$HOST_BUILD_INSTALL_DIR" + set +x - check_repo_sha "$ROOT/src/llvm" "$LLVM_SHA" - check_repo_sha "$ROOT/src/musl" "$MUSL_SHA" + check_repo_sha "$LLVM_SOURCE_DIR_host_build" "$LLVM_SHA" + check_repo_sha "$MUSL_SOURCE_DIR_host_build" "$MUSL_SHA" ./scripts/build-on-host.sh } diff --git a/config b/config index 33fa1d0..a9913c3 100644 --- a/config +++ b/config @@ -27,7 +27,7 @@ DOCKER_BUILD_INSTALL_DIR=/opt/llvm-pauth # This path is used as a final installation prefix when building the toolchain # on the host. -HOST_BUILD_INSTALL_DIR="$REPO_ROOT/inst" +HOST_BUILD_INSTALL_DIR="$REPO_ROOT_host_build/inst" # Command to build and run containers on the host: docker, podman, etc. DOCKER_CMD=docker diff --git a/scripts/build-all.sh b/scripts/build-all.sh index 59b1f8c..0f94981 100755 --- a/scripts/build-all.sh +++ b/scripts/build-all.sh @@ -4,14 +4,14 @@ set -e # This script invokes all other build-*.sh scripts. # It is called either by build-in-docker.sh or by build-on-host.sh, depending # on whether the build is performed inside a container. -# The calling script is responsible for `export`ing REPO_ROOT environment -# variable (as explained in ./global-vars). +# The caller of this script is responsible for calling `reexport_variables` beforehand. -set -x cd "$(dirname "$0")" -. "$REPO_ROOT/config" -. ./global-vars -set +x +. ./common.inc.sh +. ../config + +# Export for use by child processes. +export BUILD_OPTIMIZED_RUNTIMES write_clang_config_files() { cat > "$INSTALL_DIR/bin/aarch64-unknown-linux-pauthtest.cfg" <&2 - exit 1 + report_fatal_error "Incomplete build directory is found at $build_dir." fi # Try performing the build step. @@ -63,9 +62,9 @@ try_build() { if "$@"; then touch "$stamp_file_name" else - echo "Execution of '$stamp_prefix' step for '$CROSS_TARGET' failed." 1>&2 - echo "Please remove incomplete build at '$BUILD_DIR' before restarting the build." 1>&2 - exit 1 + report_fatal_error \ + "Execution of '$stamp_prefix' step for '$CROSS_TARGET' failed." \ + "Please remove incomplete build at '$BUILD_DIR' before restarting the build." fi } diff --git a/scripts/build-compiler-rt.sh b/scripts/build-compiler-rt.sh index 97e8dfa..70a1f18 100755 --- a/scripts/build-compiler-rt.sh +++ b/scripts/build-compiler-rt.sh @@ -1,8 +1,7 @@ #!/usr/bin/env sh set -e cd "$(dirname "$0")" -. "$REPO_ROOT/config" -. ./global-vars +. ./common.inc.sh COMPILER_RT_INSTALL_PREFIX="$("$INSTALL_DIR/bin/clang" --print-resource-dir)" normalized_triple="$("$INSTALL_DIR/bin/clang" -target $CROSS_TARGET --print-target-triple)" @@ -11,9 +10,9 @@ normalized_triple="$("$INSTALL_DIR/bin/clang" -target $CROSS_TARGET --print-targ rel_install_prefix="$(realpath --relative-to="$INSTALL_DIR" "$COMPILER_RT_INSTALL_PREFIX")" if [ "${rel_install_prefix#..}" != "${rel_install_prefix}" ]; then # Removing an optional '..' prefix yields a different string - path is relative. - echo "Expected compiler-rt to be installed under $INSTALL_DIR" 1>&2 - echo "The path returned by Clang is $COMPILER_RT_INSTALL_PREFIX" 1>&2 - exit 1 + report_fatal_error \ + "Expected compiler-rt to be installed under $INSTALL_DIR" \ + "The path returned by Clang is $COMPILER_RT_INSTALL_PREFIX" fi cmake \ diff --git a/scripts/build-in-docker.sh b/scripts/build-in-docker.sh index fb76200..9a7a9eb 100755 --- a/scripts/build-in-docker.sh +++ b/scripts/build-in-docker.sh @@ -5,11 +5,15 @@ cd "$(dirname "$0")" # This script is an entry point inside the Docker container. # Its location is expected to be $REPO_ROOT/scripts/build-in-docker.sh. -# Export the REPO_ROOT variable, so it can be used by the 'global-vars' script -# sourced by this script, as well as its subprocesses. -export REPO_ROOT="$(pwd)/.." +# Inside the Docker container, we have a separate process tree, and this script +# is the top-most process among our build scripts. Thus, perform a subset of +# global variable initialization, similar to that performed by build.sh. +. ./common.inc.sh +. ./global-vars.inc.sh +set_global_variables docker . ../config -. ./global-vars + +reexport_variables docker export INSTALL_DIR="$DOCKER_BUILD_INSTALL_DIR" on_exit() { diff --git a/scripts/build-linux-header.sh b/scripts/build-linux-header.sh index d61e5c6..bd0f254 100755 --- a/scripts/build-linux-header.sh +++ b/scripts/build-linux-header.sh @@ -1,8 +1,7 @@ #!/usr/bin/env sh set -e cd "$(dirname "$0")" -. "$REPO_ROOT/config" -. ./global-vars +. ./global-vars.inc.sh KERNEL_ARCH=arm64 TARBALL_PATH="$SRC_DIR/$LINUX_KERNEL_TARBALL_BASENAME" diff --git a/scripts/build-llvm.sh b/scripts/build-llvm.sh index d239f08..3d589da 100755 --- a/scripts/build-llvm.sh +++ b/scripts/build-llvm.sh @@ -1,8 +1,6 @@ #!/usr/bin/env sh set -e cd "$(dirname "$0")" -. "$REPO_ROOT/config" -. ./global-vars cmake \ -S "$LLVM_SOURCE_DIR/llvm" \ diff --git a/scripts/build-musl.sh b/scripts/build-musl.sh index b353741..0ca2954 100755 --- a/scripts/build-musl.sh +++ b/scripts/build-musl.sh @@ -1,8 +1,7 @@ #!/usr/bin/env sh set -e cd "$(dirname "$0")" -. "$REPO_ROOT/config" -. ./global-vars +. ./common.inc.sh mkdir "$BUILD_DIR" cd "$BUILD_DIR" diff --git a/scripts/build-on-host.sh b/scripts/build-on-host.sh index 50cad0a..5172fd2 100755 --- a/scripts/build-on-host.sh +++ b/scripts/build-on-host.sh @@ -6,11 +6,8 @@ cd "$(dirname "$0")" # containeraized build. # Its location is expected to be $REPO_ROOT/scripts/build-on-host.sh. -# Export the REPO_ROOT variable, so it can be used by the 'global-vars' script -# sourced by this script, as well as its subprocesses. -export REPO_ROOT="$(pwd)/.." -. ../config -. ./global-vars +# All global configuration variables are expected to be already exported +# by the calling ./build.sh script. ./build-all.sh diff --git a/scripts/build-runtimes.sh b/scripts/build-runtimes.sh index 2bf2a24..bd7839c 100755 --- a/scripts/build-runtimes.sh +++ b/scripts/build-runtimes.sh @@ -1,12 +1,10 @@ #!/usr/bin/env sh set -e cd "$(dirname "$0")" -. "$REPO_ROOT/config" -. ./global-vars +. ./common.inc.sh if [ -d "$TARGET_PREFIX/include/c++" ]; then - echo "ERROR: The destination directory already exists: $TARGET_PREFIX/include/c++" 1>&2 - exit 1 + report_fatal_error "ERROR: The destination directory already exists: $TARGET_PREFIX/include/c++" fi cmake \ diff --git a/scripts/common.inc.sh b/scripts/common.inc.sh new file mode 100644 index 0000000..19b1ead --- /dev/null +++ b/scripts/common.inc.sh @@ -0,0 +1,26 @@ +# Common shell utilities. + +report_fatal_error() { + local line + for line in "$@"; do + echo "$line" 1>&2 + done + exit 1 +} + +# Prints its second or third argument depending on its first argument: +# +# if_then_else "boolean" "if true" "if false" +# +if_then_else() { + case "$1" in + 1|true|yes) + echo "$2" + ;; + 0|false|no) + echo "$3" + ;; + *) + report_fatal_error "Unexpected boolean value: $1" + esac +} diff --git a/scripts/create-symlinks.sh b/scripts/create-symlinks.sh index 230ccaa..e44a701 100755 --- a/scripts/create-symlinks.sh +++ b/scripts/create-symlinks.sh @@ -1,8 +1,6 @@ #!/usr/bin/env sh set -e cd "$(dirname "$0")" -. "$REPO_ROOT/config" -. ./global-vars cd "$INSTALL_DIR/bin" diff --git a/scripts/global-vars b/scripts/global-vars deleted file mode 100644 index a1285a6..0000000 --- a/scripts/global-vars +++ /dev/null @@ -1,57 +0,0 @@ -# This file defines global variables that are used by most of the build scripts. -# -# The variables defined directly in this file are global, but are not expected -# to be adjusted by the users. -# -# On the other hand, the versions of LLVM and Musl to build, or the variables -# that may have to be adjusted according to the host system configuration are -# defined in the `config` file in the root of this repository instead. - -# REPO_ROOT should contain the path as seen by ./scripts/*.sh (i.e. it should -# be a path inside the container when building using Docker). -# -# NB: Make sure the path is absolute, as relative paths may be interpreted in -# surprising ways in some contexts, such as in the argument of `--toolchain` -# option of CMake. -if [ "x$REPO_ROOT" != "x" ]; then - # Directories mounted from the host to the container in Docker-based build. - # When Docker is used, these path should be as seen from inside the container, - # otherwise they are just absolute paths on the host pointing inside this repo. - OUTPUT_DIR="$REPO_ROOT/output" - CCACHE_DIR="$REPO_ROOT/ccache" - BUILD_TMP="$REPO_ROOT/build" - - SRC_DIR="$REPO_ROOT/src" - LLVM_SOURCE_DIR="$SRC_DIR/llvm" - MUSL_SOURCE_DIR="$SRC_DIR/musl" - - CMAKE_DIR="$REPO_ROOT/cmake" -fi - -# Linux kernel version to be used to provide user-space headers to libc. -# Any recent version should work, so this variable is defined here instead -# of $REPO_ROOT/config. -LINUX_KERNEL_VERSION=6.19.12 -LINUX_KERNEL_TARBALL_BASENAME="linux-$LINUX_KERNEL_VERSION.tar.xz" -LINUX_KERNEL_TARBALL_URL="https://cdn.kernel.org/pub/linux/kernel/v${LINUX_KERNEL_VERSION%%.*}.x/$LINUX_KERNEL_TARBALL_BASENAME" -LINUX_KERNEL_SHA256=ce5c4f1205f9729286b569b037649591555f31ca1e03cc504bd3b70b8e58a8d5 - -CPU_COUNT="$(nproc)" - -# Prints its second or third argument depending on its first argument: -# -# if_then_else "boolean" "if true" "if false" -# -if_then_else() { - case "$1" in - 1|true|yes) - echo "$2" - ;; - 0|false|no) - echo "$3" - ;; - *) - echo "Unexpected boolean value: $1" 1>&2 - exit 1 - esac -} diff --git a/scripts/global-vars.inc.sh b/scripts/global-vars.inc.sh new file mode 100644 index 0000000..b69362e --- /dev/null +++ b/scripts/global-vars.inc.sh @@ -0,0 +1,85 @@ +# This file defines global variables that are used by most of the build scripts. +# +# The variables defined by this file are global, but are not expected +# to be adjusted by the users. +# +# On the other hand, the versions of LLVM and Musl to build, or the variables +# that may have to be adjusted according to the host system configuration are +# defined in the `config` file in the root of this repository instead. + + +# Sets global configuration variables suffixed with "_host_build", "_docker_host", +# or "_docker". +# +# Usage: set_global_variables host_build +# set_global_variables docker_host +# set_global_variables docker +# +# NB: Make sure host_repo_root is absolute, as relative paths may be interpreted +# in surprising ways in some contexts, such as in the argument of `--toolchain` +# option of CMake. +set_global_variables() { + local purpose="$1" + local host_repo_root="$2" # Must be empty if purpose is "docker" + + local repo_root + case "$purpose" in + host_build) + repo_root="$host_repo_root" + ;; + docker_host) + repo_root="$host_repo_root" + ;; + docker) + [ "x$host_repo_root" != "x" ] && \ + report_fatal_error "Do not specify host_repo_root with 'docker'." + + repo_root="/repo" + ;; + *) + report_fatal_error "Expected one of host_build, docker_host, docker." + esac + + # The expression string passed to the 'eval' built-in is something like this: + # OUTPUT_DIR_host_build="$repo_root/output" + + eval REPO_ROOT_$purpose='"$repo_root"' + eval OUTPUT_DIR_$purpose='"$repo_root/output"' + eval CCACHE_DIR_$purpose='"$repo_root/ccache"' + eval BUILD_TMP_$purpose='"$repo_root/build"' + + eval SRC_DIR_$purpose='"$repo_root/src"' + eval LLVM_SOURCE_DIR_$purpose='"$repo_root/src/llvm"' + eval MUSL_SOURCE_DIR_$purpose='"$repo_root/src/musl"' + + eval CMAKE_DIR_$purpose='"$repo_root/cmake"' +} + +# Re-defines previously set global variables without suffix and exports them. +reexport_variables() { + local suffix="$1" + + case "$suffix" in + host_build|docker_host|docker) true ;; + *) report_fatal_error "Expected one of host_build, docker_host, docker." + esac + + local var_name + for var_name in REPO_ROOT OUTPUT_DIR CCACHE_DIR BUILD_TMP \ + SRC_DIR LLVM_SOURCE_DIR MUSL_SOURCE_DIR \ + CMAKE_DIR; do + # Eval-ed expression looks like this: + # export REPO_ROOT="$REPO_ROOT_host_build" + eval "export $var_name=\"\$${var_name}_${suffix}\"" + done +} + +export CPU_COUNT="$(nproc)" + +# Linux kernel version to be used to provide user-space headers to libc. +# Any recent version should work, so this variable is defined here instead +# of $REPO_ROOT/config. +LINUX_KERNEL_VERSION=6.19.12 +LINUX_KERNEL_TARBALL_BASENAME="linux-$LINUX_KERNEL_VERSION.tar.xz" +LINUX_KERNEL_TARBALL_URL="https://cdn.kernel.org/pub/linux/kernel/v${LINUX_KERNEL_VERSION%%.*}.x/$LINUX_KERNEL_TARBALL_BASENAME" +LINUX_KERNEL_SHA256=ce5c4f1205f9729286b569b037649591555f31ca1e03cc504bd3b70b8e58a8d5