From b9601d7cf668516605923e24200825ab96baed24 Mon Sep 17 00:00:00 2001 From: Dmytro Smirnov Date: Mon, 15 Jun 2026 13:56:08 +0300 Subject: [PATCH 1/3] fix: stabilize worker image build --- Dockerfile | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index e9c4266b..6129290f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -54,7 +54,7 @@ RUN apt-get update && \ zip=3.0-15ubuntu2 \ unzip=6.0-28ubuntu7 \ nano=8.4-1 \ - vim=2:9.1.0967-1ubuntu6.5 \ + vim=2:9.1.0967-1ubuntu6.6 \ python3.13=3.13.7-1ubuntu0.4 \ python3.13-venv=3.13.7-1ubuntu0.4 \ supervisor=4.2.5-3 && \ @@ -65,7 +65,7 @@ RUN apt-get update && \ ln -s /opt/az/bin/az /usr/local/bin/az && \ # Clean up pip cache and temp files rm -rf /root/.cache/pip && \ - find /opt/az -type d -name __pycache__ -exec rm -rf {} + 2>/dev/null || true && \ + (find /opt/az -type d -name __pycache__ -exec rm -rf {} + 2>/dev/null || true) && \ apt-get clean && \ rm -rf /tmp/* /var/tmp/* && \ # Set up sources.list.d for child images @@ -74,15 +74,15 @@ RUN apt-get update && \ # Configure the timezone RUN echo $TZ > /etc/timezone && \ - rm /etc/localtime && \ - ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && \ - dpkg-reconfigure -f noninteractive tzdata + rm -f /etc/localtime && \ + ln -snf /usr/share/zoneinfo/$TZ /etc/localtime # Install yq (architecture-aware) RUN ARCH=$(uname -m) && \ if [ "$ARCH" = "x86_64" ]; then ARCH="amd64"; elif [ "$ARCH" = "aarch64" ]; then ARCH="arm64"; fi && \ - curl -sL https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_linux_${ARCH}.tar.gz | tar xz && \ - mv yq_linux_${ARCH} /usr/bin/yq && \ + curl -fsSL "https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_linux_${ARCH}.tar.gz" -o /tmp/yq.tar.gz && \ + tar -xzf /tmp/yq.tar.gz -C /tmp && \ + mv /tmp/yq_linux_${ARCH} /usr/bin/yq && \ rm -rf /tmp/* # Install Google Cloud SDK (architecture-aware) From 7b471fdf46f0f0a6e7c6c78851851792e49355f3 Mon Sep 17 00:00:00 2001 From: Dmytro Smirnov Date: Mon, 15 Jun 2026 13:56:17 +0300 Subject: [PATCH 2/3] feat: emit worker runtime output --- docs/config.md | 12 +++- lib/runtime_output.sh | 101 ++++++++++++++++++++++++++---- src/configs/worker.yaml | 1 + test/modules/25_runtime_output.sh | 63 +++++++++++++++++++ 4 files changed, 163 insertions(+), 14 deletions(-) create mode 100755 test/modules/25_runtime_output.sh diff --git a/docs/config.md b/docs/config.md index 472d8205..d5e3b8dc 100644 --- a/docs/config.md +++ b/docs/config.md @@ -83,9 +83,17 @@ docker run --rm \ By default the worker does not print runtime config details or write output files. The entrypoint logs a short hint that output can be enabled. -Set `WORKER_OUTPUT_FILE` when a deployment or workflow needs runtime config evidence. The worker writes redacted JSON runtime metadata to that path. +Set `WORKER_OUTPUT_FILE` when a deployment or workflow needs runtime config evidence. The worker writes JSON runtime metadata to that path after `worker.yaml`, deployment environment overrides, and secret references have been applied. -Workflow-specific outputs such as `$GITHUB_OUTPUT`, `$GITHUB_STEP_SUMMARY`, or platform annotations should be generated by the workflow from the JSON file. +Set `WORKER_OUTPUT_LOG=true` to also emit the same JSON to container logs as a single minified line prefixed with `WORKER_RUNTIME_OUTPUT_JSON=`. This is useful for Kubernetes or workflow systems where the next step reads container logs instead of a mounted file. + +The output contains: + +- `env`: resolved non-secret environment variables. +- `redacted`: environment variable names that were intentionally omitted because they are configured as secrets or secret references. +- `paths`: the config and environment files used to build the output. + +Workflow-specific outputs such as `$GITHUB_OUTPUT`, `$GITHUB_STEP_SUMMARY`, uploaded artifacts, or platform annotations should be generated by the workflow from the JSON file or the structured log line. ## Related Docs diff --git a/lib/runtime_output.sh b/lib/runtime_output.sh index 736dc2d4..c2a90c23 100644 --- a/lib/runtime_output.sh +++ b/lib/runtime_output.sh @@ -7,7 +7,7 @@ source "${WORKER_LIB_DIR}/worker_config.sh" build_runtime_output_json() { local config_json="$1" - local worker_config_path services_config_path env_json secrets_json + local worker_config_path services_config_path env_json redacted_json worker_config_path=$(get_worker_config_path) services_config_path="${HOME}/.config/worker/services.yaml" @@ -15,8 +15,8 @@ build_runtime_output_json() { services_config_path="${WORKER_CONFIG_DIR}/services.yaml" fi - env_json=$(echo "$config_json" | jq '.config.env // {} | with_entries(.value = "redacted")' 2>/dev/null) || return 1 - secrets_json=$(echo "$config_json" | jq '.config.secrets // {} | with_entries(.value = "redacted")' 2>/dev/null) || return 1 + env_json=$(build_runtime_env_json "$config_json") || return 1 + redacted_json=$(build_runtime_redacted_json "$config_json") || return 1 jq -n \ --arg worker_config_path "$worker_config_path" \ @@ -24,7 +24,7 @@ build_runtime_output_json() { --arg worker_env_file "$WORKER_ENV_FILE" \ --arg generated_at "$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \ --argjson env "$env_json" \ - --argjson secrets "$secrets_json" \ + --argjson redacted "$redacted_json" \ '{ generated_at: $generated_at, paths: { @@ -33,16 +33,87 @@ build_runtime_output_json() { environment: $worker_env_file }, env: $env, - secrets: $secrets, - secret_values: "redacted" + redacted: $redacted }' } +is_runtime_output_redacted_name() { + local config_json="$1" + local name="$2" + + echo "$config_json" | jq -e --arg name "$name" --arg pattern "^(${SUPPORTED_SECRET_PROVIDERS})/.+/.+" ' + (.config.secrets // {} | has($name)) or + ((.config.env // {} | .[$name] // "" | tostring) | test($pattern)) + ' >/dev/null +} + +build_runtime_env_json() { + local config_json="$1" + local names name value json + + if [[ ! -f "$WORKER_ENV_FILE" ]]; then + log_error "Runtime output" "Environment file not found: $WORKER_ENV_FILE" + return 1 + fi + + names=$(grep "^export " "$WORKER_ENV_FILE" | cut -d'=' -f1 | cut -d' ' -f2) + json="{}" + while IFS= read -r name; do + if [[ -z "$name" ]] || is_runtime_output_redacted_name "$config_json" "$name"; then + continue + fi + + value=$(get_env_value "$name") || return 1 + json=$(echo "$json" | jq --arg key "$name" --arg value "$value" '. + {($key): $value}') || return 1 + done <<< "$names" + + echo "$json" | jq -S . +} + +build_runtime_redacted_json() { + local config_json="$1" + local names name json + + if [[ ! -f "$WORKER_ENV_FILE" ]]; then + log_error "Runtime output" "Environment file not found: $WORKER_ENV_FILE" + return 1 + fi + + names=$(grep "^export " "$WORKER_ENV_FILE" | cut -d'=' -f1 | cut -d' ' -f2) + json="[]" + while IFS= read -r name; do + if [[ -n "$name" ]] && is_runtime_output_redacted_name "$config_json" "$name"; then + json=$(echo "$json" | jq --arg name "$name" '. + [$name]') || return 1 + fi + done <<< "$names" + + echo "$json" | jq -S 'unique' +} + +runtime_output_log_enabled() { + case "${WORKER_OUTPUT_LOG:-false}" in + true|TRUE|1|yes|YES|on|ON) + return 0 + ;; + *) + return 1 + ;; + esac +} + +emit_runtime_output_log() { + local runtime_json="$1" + local compact_json + + compact_json=$(echo "$runtime_json" | jq -c .) || return 1 + printf 'WORKER_RUNTIME_OUTPUT_JSON=%s\n' "$compact_json" +} + emit_runtime_output() { local config_json runtime_json - if [[ -z "${WORKER_OUTPUT_FILE:-}" ]]; then - log_info "Runtime output disabled. Set WORKER_OUTPUT_FILE to write redacted JSON runtime config for workflow/deployment integrations." + if [[ -z "${WORKER_OUTPUT_FILE:-}" ]] && ! runtime_output_log_enabled; then + log_info "Runtime output disabled. Set WORKER_OUTPUT_FILE or WORKER_OUTPUT_LOG=true to emit redacted JSON runtime config for workflow/deployment integrations." return 0 fi @@ -52,8 +123,14 @@ emit_runtime_output() { return 1 fi - mkdir -p "$(dirname "$WORKER_OUTPUT_FILE")" || return 1 - install -m 600 /dev/null "$WORKER_OUTPUT_FILE" || return 1 - printf '%s\n' "$runtime_json" > "$WORKER_OUTPUT_FILE" - log_info "Runtime output written to $WORKER_OUTPUT_FILE" + if [[ -n "${WORKER_OUTPUT_FILE:-}" ]]; then + mkdir -p "$(dirname "$WORKER_OUTPUT_FILE")" || return 1 + install -m 600 /dev/null "$WORKER_OUTPUT_FILE" || return 1 + printf '%s\n' "$runtime_json" > "$WORKER_OUTPUT_FILE" + log_info "Runtime output written to $WORKER_OUTPUT_FILE" + fi + + if runtime_output_log_enabled; then + emit_runtime_output_log "$runtime_json" || return 1 + fi } diff --git a/src/configs/worker.yaml b/src/configs/worker.yaml index 9358ed77..aabef6ed 100644 --- a/src/configs/worker.yaml +++ b/src/configs/worker.yaml @@ -4,4 +4,5 @@ version: udx.io/worker-v1/config config: env: WORKER_OUTPUT_FILE: "" + WORKER_OUTPUT_LOG: "false" secrets: {} diff --git a/test/modules/25_runtime_output.sh b/test/modules/25_runtime_output.sh new file mode 100755 index 00000000..a92a79f3 --- /dev/null +++ b/test/modules/25_runtime_output.sh @@ -0,0 +1,63 @@ +#!/bin/bash + +# Source test helpers +# shellcheck source=../test_helpers.sh disable=SC1091 +source "/home/udx/test/test_helpers.sh" + +print_header "Runtime Output Tests" + +# shellcheck source=/home/udx/lib/runtime_output.sh disable=SC1091 +source "${WORKER_LIB_DIR}/runtime_output.sh" + +print_info "Testing: runtime output redacts configured secrets" +RUNTIME_ENV_FILE=$(mktemp) +ORIGINAL_WORKER_ENV_FILE="$WORKER_ENV_FILE" +export WORKER_ENV_FILE="$RUNTIME_ENV_FILE" + +printf 'export PUBLIC_VALUE=%q\n' "visible value" > "$WORKER_ENV_FILE" +printf 'export CONFIG_SECRET=%q\n' "resolved secret" >> "$WORKER_ENV_FILE" +printf 'export CONFIG_REF=%q\n' "resolved reference" >> "$WORKER_ENV_FILE" + +CONFIG_JSON='{ + "config": { + "env": { + "PUBLIC_VALUE": "visible value", + "CONFIG_REF": "gcp/project-id/secret-name" + }, + "secrets": { + "CONFIG_SECRET": "aws/secret-name/us-west-2" + } + } +}' + +RUNTIME_OUTPUT=$(build_runtime_output_json "$CONFIG_JSON") +export WORKER_ENV_FILE="$ORIGINAL_WORKER_ENV_FILE" +rm -f "$RUNTIME_ENV_FILE" + +if ! echo "$RUNTIME_OUTPUT" | jq -e '.env.PUBLIC_VALUE == "visible value"' >/dev/null; then + print_error "runtime output missing non-secret env value" + exit 1 +fi + +if echo "$RUNTIME_OUTPUT" | jq -e '.env.CONFIG_SECRET or .env.CONFIG_REF' >/dev/null; then + print_error "runtime output leaked a redacted env value" + exit 1 +fi + +if ! echo "$RUNTIME_OUTPUT" | jq -e '.redacted == ["CONFIG_REF", "CONFIG_SECRET"]' >/dev/null; then + print_error "runtime output redacted list is incorrect" + exit 1 +fi + +LOG_LINE=$(WORKER_OUTPUT_LOG=true emit_runtime_output_log "$RUNTIME_OUTPUT") +if [[ "$LOG_LINE" != WORKER_RUNTIME_OUTPUT_JSON=* ]]; then + print_error "runtime output log marker is missing" + exit 1 +fi + +if ! echo "${LOG_LINE#WORKER_RUNTIME_OUTPUT_JSON=}" | jq -e '.env.PUBLIC_VALUE == "visible value"' >/dev/null; then + print_error "runtime output log JSON is invalid" + exit 1 +fi + +print_success "All runtime output tests passed" From 33d9bf2b4d96e74b5eb1009a74fb4c7e942c932d Mon Sep 17 00:00:00 2001 From: Dmytro Smirnov Date: Mon, 15 Jun 2026 14:23:42 +0300 Subject: [PATCH 3/3] ci: publish worker runtime output artifact --- .github/workflows/docker-ops.yml | 44 ++++++++++++++++++++++++++++++++ .rabbit/context.yaml | 2 +- 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker-ops.yml b/.github/workflows/docker-ops.yml index 7fa23b49..1f9497b7 100644 --- a/.github/workflows/docker-ops.yml +++ b/.github/workflows/docker-ops.yml @@ -35,3 +35,47 @@ jobs: version_config_path: ci/git-version.yml secrets: docker_token: ${{ secrets.DOCKER_TOKEN }} + + runtime-output: + name: Runtime Output + needs: docker-ops + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Checkout code + uses: actions/checkout@v6 + + - name: Build worker image + run: docker build --pull --tag "worker-runtime-output:${GITHUB_SHA}" . + + - name: Capture runtime output + run: | + set -euo pipefail + mkdir -p runtime-output + sudo chown 500:500 runtime-output + docker run --rm \ + -e WORKER_OUTPUT_FILE=/tmp/worker-runtime-output/runtime.json \ + -e WORKER_OUTPUT_LOG=true \ + -v "${PWD}/runtime-output:/tmp/worker-runtime-output" \ + "worker-runtime-output:${GITHUB_SHA}" \ + /bin/bash -lc 'test -s "$WORKER_OUTPUT_FILE"' + sudo chown "$(id -u):$(id -g)" runtime-output/runtime.json + jq -e '.env | type == "object"' runtime-output/runtime.json + jq -c . runtime-output/runtime.json + + - name: Write runtime output summary + run: | + { + echo "### Worker runtime output" + echo + echo '```json' + jq . runtime-output/runtime.json + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + + - name: Upload runtime output artifact + uses: actions/upload-artifact@v4 + with: + name: worker-runtime-output + path: runtime-output/runtime.json diff --git a/.rabbit/context.yaml b/.rabbit/context.yaml index 80914ee1..6c7fe7e7 100644 --- a/.rabbit/context.yaml +++ b/.rabbit/context.yaml @@ -6,7 +6,7 @@ generator: tool: dev.kit repo: https://github.com/udx/dev.kit version: 0.13.0 - generated_at: 2026-05-31T09:26:48Z + generated_at: 2026-06-15T11:33:01Z sources: homepage: https://udx.dev/kit repository: https://github.com/udx/dev.kit