From 18f14b8cab03316151e9c8962b7ebae08aebdbf6 Mon Sep 17 00:00:00 2001 From: Dmytro Smirnov Date: Sun, 31 May 2026 12:03:05 +0300 Subject: [PATCH 01/32] refactor: simplify worker runtime config --- .github/workflows/docker-ops.yml | 2 +- Dockerfile | 3 +- Makefile | 4 +- bin/entrypoint.sh | 11 +- etc/configs/worker/default.yaml | 11 - lib/auth.sh | 191 --------------- lib/auth/aws.sh | 42 ---- lib/auth/azure.sh | 49 ---- lib/auth/gcp.sh | 84 ------- lib/cleanup.sh | 158 ------------- lib/cli.sh | 4 +- lib/cli/auth.sh | 269 ---------------------- lib/cli/config.sh | 30 +-- lib/cli/env.sh | 40 +--- lib/cli/service.sh | 45 +++- lib/env_handler.sh | 142 ++++++++---- lib/environment.sh | 88 ------- lib/process_manager.sh | 54 ++++- lib/runtime_output.sh | 72 ++++++ lib/secrets.sh | 1 - lib/worker_config.sh | 66 ++---- src/configs/services.yaml | 4 + src/configs/worker.yaml | 7 + src/tests/modules/30_auth.sh | 37 --- {src/tests => test}/main.sh | 4 +- {src/tests => test}/modules/10_config.sh | 9 +- {src/tests => test}/modules/20_env.sh | 9 +- {src/tests => test}/modules/40_service.sh | 2 +- {src/tests => test}/modules/50_sbom.sh | 2 +- {src/tests => test}/modules/60_health.sh | 2 +- {src/tests => test}/test_helpers.sh | 0 31 files changed, 338 insertions(+), 1104 deletions(-) delete mode 100644 etc/configs/worker/default.yaml delete mode 100644 lib/auth.sh delete mode 100644 lib/auth/aws.sh delete mode 100644 lib/auth/azure.sh delete mode 100644 lib/auth/gcp.sh delete mode 100644 lib/cleanup.sh delete mode 100644 lib/cli/auth.sh delete mode 100644 lib/environment.sh create mode 100644 lib/runtime_output.sh create mode 100644 src/configs/services.yaml create mode 100644 src/configs/worker.yaml delete mode 100755 src/tests/modules/30_auth.sh rename {src/tests => test}/main.sh (96%) rename {src/tests => test}/modules/10_config.sh (81%) rename {src/tests => test}/modules/20_env.sh (85%) rename {src/tests => test}/modules/40_service.sh (97%) rename {src/tests => test}/modules/50_sbom.sh (96%) rename {src/tests => test}/modules/60_health.sh (96%) rename {src/tests => test}/test_helpers.sh (100%) diff --git a/.github/workflows/docker-ops.yml b/.github/workflows/docker-ops.yml index c099c85f..4d06c3cc 100644 --- a/.github/workflows/docker-ops.yml +++ b/.github/workflows/docker-ops.yml @@ -12,7 +12,7 @@ name: Docker Ops - "lib/**" - "src/**" - "etc/**" - - "tests/**" + - "test/**" - "Makefile" - "Makefile.variables" - "ci/**" diff --git a/Dockerfile b/Dockerfile index 40cf4459..6f473658 100644 --- a/Dockerfile +++ b/Dockerfile @@ -144,7 +144,8 @@ RUN mkdir -p \ # Copy worker files COPY bin/entrypoint.sh ${WORKER_BIN_DIR}/ COPY lib ${WORKER_LIB_DIR}/ -COPY etc/configs/worker/default.yaml ${WORKER_CONFIG_DIR}/worker.yaml +COPY src/configs/worker.yaml ${WORKER_CONFIG_DIR}/worker.yaml +COPY src/configs/services.yaml ${WORKER_CONFIG_DIR}/services.yaml COPY etc/configs/supervisor ${WORKER_CONFIG_DIR}/supervisor/ # Make scripts executable and initialize environment diff --git a/Makefile b/Makefile index 6387c2d2..9dbb97c7 100644 --- a/Makefile +++ b/Makefile @@ -102,8 +102,8 @@ clean: test: clean @printf "$(COLOR_BLUE)$(SYM_ARROW) Running tests...$(COLOR_RESET)\n" @$(MAKE) run \ - VOLUMES="$(PWD)/src/tests:/home/udx/tests $(PWD)/src/examples/simple-config/.config/worker/worker.yaml:/home/udx/.config/worker/worker.yaml $(PWD)/src/examples/simple-service/.config/worker/services.yaml:/home/udx/.config/worker/services.yaml" \ - COMMAND="/home/udx/tests/main.sh" + VOLUMES="$(PWD)/test:/home/udx/test $(PWD)/src/examples/simple-config/.config/worker/worker.yaml:/home/udx/.config/worker/worker.yaml $(PWD)/src/examples/simple-service/.config/worker/services.yaml:/home/udx/.config/worker/services.yaml" \ + COMMAND="/home/udx/test/main.sh" @printf "$(COLOR_BLUE)$(SYM_ARROW) Following test output...$(COLOR_RESET)\n" @docker logs -f $(CONTAINER_NAME) & LOGS_PID=$$!; \ docker wait $(CONTAINER_NAME) > /dev/null; EXIT_CODE=$$?; \ diff --git a/bin/entrypoint.sh b/bin/entrypoint.sh index 64a94e64..1c4cb7e8 100644 --- a/bin/entrypoint.sh +++ b/bin/entrypoint.sh @@ -6,7 +6,14 @@ source "${WORKER_LIB_DIR}/utils.sh" log_info "Welcome to UDX Worker Container. Initializing environment..." # shellcheck disable=SC1091 -source "${WORKER_LIB_DIR}/environment.sh" +source "${WORKER_LIB_DIR}/worker_config.sh" +# shellcheck disable=SC1091 +source "${WORKER_LIB_DIR}/secrets.sh" +configure_environment || exit 1 + +# shellcheck disable=SC1091 +source "${WORKER_LIB_DIR}/runtime_output.sh" +emit_runtime_output || exit 1 # Start the process manager log_info "Starting process manager..." @@ -32,4 +39,4 @@ if [ "$#" -gt 0 ]; then fi # Keep the container running -wait \ No newline at end of file +wait diff --git a/etc/configs/worker/default.yaml b/etc/configs/worker/default.yaml deleted file mode 100644 index 5ec380b4..00000000 --- a/etc/configs/worker/default.yaml +++ /dev/null @@ -1,11 +0,0 @@ ---- -kind: workerConfig -version: udx.io/worker-v1/config -config: - actors: - - type: gcp - creds: "${GCP_CREDS}" - - type: azure - creds: "${AZURE_CREDS}" - - type: aws - creds: "${AWS_CREDS}" diff --git a/lib/auth.sh b/lib/auth.sh deleted file mode 100644 index 33ab5db0..00000000 --- a/lib/auth.sh +++ /dev/null @@ -1,191 +0,0 @@ -#!/bin/bash - -# shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 -source "${WORKER_LIB_DIR}/utils.sh" - -# Array to track configured providers -declare -a configured_providers=() - -# Function to get env var names for a provider from actors JSON -get_provider_env_vars() { - local provider=$1 - local actors_json=$2 - - # Get all env var names from actor creds that match ${VAR} pattern - # shellcheck disable=SC2016 - echo "$actors_json" | jq -r ".[].creds" 2>/dev/null | \ - grep -o '\${[^}]*}' | sed 's/[\${}]//g' || true -} - -# Function to check if a provider is configured -is_provider_configured() { - local provider=$1 - local actors_json=$2 - - # Get provider's actors - local provider_actors - provider_actors=$(echo "$actors_json" | jq -r "[.[] | select(.type | startswith(\"$provider\"))]" 2>/dev/null) - - if [ -z "$provider_actors" ] || [ "$provider_actors" = "[]" ]; then - return 1 - fi - - # Get all possible env var names from actors - local env_vars - mapfile -t env_vars < <(get_provider_env_vars "$provider" "$provider_actors") - - # Check all possible env vars - for env_var in "${env_vars[@]}"; do - if [ -n "${!env_var}" ]; then - return 0 - fi - done - - # Check each actor's credentials - while IFS= read -r actor; do - [ -z "$actor" ] && continue - - local creds - creds=$(echo "$actor" | jq -r '.creds' 2>/dev/null) - [ "$creds" = "null" ] && continue - - # Evaluate creds as a reference to an environment variable - if [[ "$creds" =~ ^\$\{(.+)\}$ ]]; then - local env_var_name="${BASH_REMATCH[1]}" - creds="${!env_var_name}" - fi - - # If we find any valid credentials, return success - if [ -n "$creds" ]; then - return 0 - fi - done <<< "$(echo "$provider_actors" | jq -r '.[]')" - - return 1 -} - -# Function to authenticate actors -authenticate_actors() { - local actors_json="$1" - - if [[ -z "$actors_json" || "$actors_json" == "null" ]]; then - log_info "No worker actors found in the configuration." - return 0 - fi - - local actors_file - actors_file=$(mktemp) - echo "$actors_json" | jq -c '.[]' > "$actors_file" - - mapfile -t actors_array < "$actors_file" - rm -f "$actors_file" - - # Create local creds dir - log_info "Pre-creating local creds dir" - mkdir -p "$LOCAL_CREDS_DIR" - - for actor in "${actors_array[@]}"; do - local type provider creds auth_script auth_function - - type=$(resolve_env_vars "$(echo "$actor" | jq -r '.type')") - provider=$(echo "$type" | cut -d '-' -f 1) - creds=$(echo "$actor" | jq -r '.creds') - - # Evaluate creds as a reference to an environment variable - if [[ "$creds" =~ ^\$\{(.+)\}$ ]]; then - local env_var_name="${BASH_REMATCH[1]}" - creds="${!env_var_name}" - fi - - # Skip if the credentials are empty or not defined - if [[ -z "$creds" ]]; then - continue - else - log_success "Authentication" "Detected credentials for $type" - fi - - # Explicitly check for JSON format - if echo "$creds" | jq -e . >/dev/null 2>&1; then - log_info "Reading JSON credentials" - # Then, check if it's a file path - elif [[ -f "$creds" ]]; then - log_info "Reading credentials from file: $creds" - creds=$(cat "$creds") - # Finally, check if it's possibly base64 encoded - elif echo "$creds" | base64 --decode &>/dev/null && echo "$creds" | base64 --decode | jq empty &>/dev/null; then - log_info "Reading base64 encoded JSON credentials" - creds=$(echo "$creds" | base64 --decode) - else - log_error "Authentication" "Credentials format not recognized for $provider. Skipping..." - continue - fi - - # Proceed only if creds are valid JSON - if echo "$creds" | jq empty &>/dev/null; then - log_info "Processing credentials for $provider" - auth_script="${WORKER_LIB_DIR}/auth/${provider}.sh" - auth_function="${provider}_authenticate" - - if [[ -f "$auth_script" ]]; then - source "$auth_script" - - if command -v "$auth_function" > /dev/null; then - - if ! authenticate_provider "$provider" "$auth_function" "$creds"; then - log_error "Authentication" "Authentication failed for provider $provider." - return 1 - fi - configured_providers+=("$provider") - else - log_error "Authentication" "Authentication function $auth_function not found for $provider. Skipping..." - continue - fi - else - log_error "Authentication" "Authentication script $auth_script not found for $provider. Skipping..." - continue - fi - else - log_error "Authentication" "Invalid JSON credentials for $provider. Skipping..." - continue - fi - done - - if [[ ${#configured_providers[@]} -eq 0 ]]; then - log_info "No providers creds detected." - fi - - return 0 -} - -# Function to handle provider-specific authentication -authenticate_provider() { - local provider="$1" - local auth_function="$2" - local creds="$3" - local temp_config_file - - # Save the credentials data to a temporary file - temp_config_file=$(mktemp /tmp/actor_creds.XXXXXX) - echo "$creds" > "$temp_config_file" - - # Ensure cleanup with a trap in case of unexpected exit - trap 'rm -f "$temp_config_file"' EXIT - - # Call the authentication function with the temp file - if ! $auth_function "$temp_config_file"; then - log_error "Authentication failed for provider $provider." - return 1 - fi - - # Clean up the temporary file - rm -f "$temp_config_file" - trap - EXIT - - # Set an environment variable to mark successful authorization - export "${provider^^}_AUTHORIZED=true" - - return 0 -} - -# Example usage: -# authenticate_actors "$actors_json" \ No newline at end of file diff --git a/lib/auth/aws.sh b/lib/auth/aws.sh deleted file mode 100644 index e11c3271..00000000 --- a/lib/auth/aws.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash - -# shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 -source "${WORKER_LIB_DIR}/utils.sh" - -# Example usage of the function -# aws_authenticate "/path/to/your/aws_creds.json" - -# Function to authenticate AWS using IAM user credentials -aws_authenticate() { - local creds_json="$1" - - # Read the contents of the file - local creds_content - creds_content=$(cat "$creds_json") - - if [[ -z "$creds_content" ]]; then - log_error "AWS Authentication" "No AWS credentials provided." - return 1 - fi - - # Extract necessary fields from the JSON credentials - local accessKeyId secretAccessKey sessionToken - - accessKeyId=$(echo "$creds_content" | jq -r '.AccessKeyId') - secretAccessKey=$(echo "$creds_content" | jq -r '.SecretAccessKey') - sessionToken=$(echo "$creds_content" | jq -r '.SessionToken') - - if [[ -z "$accessKeyId" || -z "$secretAccessKey" ]]; then - log_error "AWS Authentication" "Missing required AWS credentials." - return 1 - fi - - # Export the credentials as environment variables - export AWS_ACCESS_KEY_ID="$accessKeyId" - export AWS_SECRET_ACCESS_KEY="$secretAccessKey" - if [[ -n "$sessionToken" ]]; then - export AWS_SESSION_TOKEN="$sessionToken" - fi - - log_success "AWS Authentication" "AWS credentials set successfully." -} \ No newline at end of file diff --git a/lib/auth/azure.sh b/lib/auth/azure.sh deleted file mode 100644 index e913180d..00000000 --- a/lib/auth/azure.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash - -# Function to authenticate Azure accounts -# -# Example usage of the function -# azure_authenticate "/path/to/your/azure_creds.json" - -# shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 -source "${WORKER_LIB_DIR}/utils.sh" - -# Function to authenticate Azure accounts -azure_authenticate() { - local creds_json="$1" - - # Read the contents of the file - local creds_content - creds_content=$(cat "$creds_json") - - if [[ -z "$creds_content" ]]; then - log_error "Azure Authentication" "No Azure credentials provided." - return 1 - fi - - # Extract necessary fields from the JSON credentials - local clientId clientSecret subscriptionId tenantId - - clientId=$(echo "$creds_content" | jq -r '.clientId') - clientSecret=$(echo "$creds_content" | jq -r '.clientSecret') - subscriptionId=$(echo "$creds_content" | jq -r '.subscriptionId') - tenantId=$(echo "$creds_content" | jq -r '.tenantId') - - if [[ -z "$clientId" || -z "$clientSecret" || -z "$subscriptionId" || -z "$tenantId" ]]; then - log_error "Azure Authentication" "Missing required Azure credentials." - return 1 - fi - - log_info "Authenticating Azure service principal..." - if ! az login --service-principal -u "$clientId" -p "$clientSecret" --tenant "$tenantId" >/dev/null 2>&1; then - log_error "Azure Authentication" "Azure service principal authentication failed." - return 1 - fi - - if ! az account set --subscription "$subscriptionId" >/dev/null 2>&1; then - log_error "Azure Authentication" "Failed to set Azure subscription." - return 1 - fi - - log_success "Azure Authentication" "Azure service principal authenticated and subscription set." -} diff --git a/lib/auth/gcp.sh b/lib/auth/gcp.sh deleted file mode 100644 index b5791cfa..00000000 --- a/lib/auth/gcp.sh +++ /dev/null @@ -1,84 +0,0 @@ -#!/bin/bash - -# shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 -source "${WORKER_LIB_DIR}/utils.sh" - -# GCP Authentication Module -# Supports: Service Account Keys, Workload Identity Tokens -# All methods use: gcloud auth login --cred-file="$GOOGLE_APPLICATION_CREDENTIALS" -# -# Note: Impersonation is handled externally by setting GOOGLE_APPLICATION_CREDENTIALS -# and CLOUDSDK_AUTH_ACCESS_TOKEN directly, bypassing this module. -# -# Example usage: -# gcp_authenticate "/path/to/gcp_creds.json" -# gcp_authenticate "${GCP_CREDS}" - -# Function to authenticate with GCP -gcp_authenticate() { - local creds_json="$1" - - # Read the contents of the file - local creds_content - creds_content=$(cat "$creds_json") - - if [[ -z "$creds_content" ]]; then - log_error "GCP Authentication" "No GCP credentials provided." - return 1 - fi - - # If GOOGLE_APPLICATION_CREDENTIALS already set, do not override - if [ -n "$GOOGLE_APPLICATION_CREDENTIALS" ]; then - log_info "GCP Authentication" "GOOGLE_APPLICATION_CREDENTIALS already set, skipping authentication." - return 0 - fi - - local creds_file="$LOCAL_CREDS_DIR/gcp_creds.json" - - # Check if this is a service account key that needs private_key normalization - if echo "$creds_content" | jq -e '.private_key' >/dev/null 2>&1; then - # Service account key - normalize private_key field - local clientEmail privateKey projectId - - clientEmail=$(echo "$creds_content" | jq -r '.client_email') - privateKey=$(echo "$creds_content" | jq -r '.private_key') - projectId=$(echo "$creds_content" | jq -r '.project_id') - - # Normalize private_key: handle escaped newlines and spacing issues - privateKey=$(echo "$privateKey" | sed 's/\\n/\n/g' | sed 's/- /\n-/g' | sed 's/ -/-\n/g') - - # Create normalized JSON file - jq -n --arg clientEmail "$clientEmail" --arg privateKey "$privateKey" --arg projectId "$projectId" \ - '{type: "service_account", client_email: $clientEmail, private_key: $privateKey, project_id: $projectId}' > "$creds_file" - else - # Other credential types (workload identity, impersonation) - use as-is - echo "$creds_content" > "$creds_file" - fi - - # Set GOOGLE_APPLICATION_CREDENTIALS for all methods - export GOOGLE_APPLICATION_CREDENTIALS="$creds_file" - - # Set GCP_CREDS for backward compatibility - export GCP_CREDS="$creds_file" - - # Authenticate with gcloud (works for all credential types) - log_info "GCP Authentication" "Authenticating with gcloud..." - if ! gcloud auth login --cred-file="$GOOGLE_APPLICATION_CREDENTIALS" >/dev/null 2>&1; then - log_error "GCP Authentication" "Failed to authenticate with gcloud." - return 1 - fi - - # Extract and set project ID if available - local projectId - projectId=$(echo "$creds_content" | jq -r '.project_id // empty' 2>/dev/null) - - if [[ -n "$projectId" && "$projectId" != "null" ]]; then - if ! gcloud config set project "$projectId" >/dev/null 2>&1; then - log_error "GCP Authentication" "Failed to set GCP project: $projectId" - return 1 - fi - log_success "GCP Authentication" "Authenticated successfully. Project: $projectId" - else - log_success "GCP Authentication" "Authenticated successfully." - fi -} \ No newline at end of file diff --git a/lib/cleanup.sh b/lib/cleanup.sh deleted file mode 100644 index 87de9fdf..00000000 --- a/lib/cleanup.sh +++ /dev/null @@ -1,158 +0,0 @@ -#!/bin/bash - -# Include worker config utilities first -# shellcheck source=/dev/null -source "${WORKER_LIB_DIR}/worker_config.sh" - -# shellcheck source=/dev/null -source "${WORKER_LIB_DIR}/utils.sh" - -# Generic function to clean up authentication for any provider -cleanup_provider() { - local provider=$1 - local logout_cmd=$2 - local list_cmd=$3 - local name=$4 - local cleaned_up=false - - # Check if the provider's CLI is available - if ! command -v "$provider" > /dev/null; then - return 0 # Skip silently if CLI is not available - fi - - # Check if there are active sessions/accounts to clean up - if ! eval "$list_cmd" > /dev/null 2>&1; then - return 0 # Skip silently if no active sessions are found - fi - - # Check if the provider's CLI returns "no credentials" to skip - if eval "$list_cmd" > /dev/null 2>&1 | grep -q "no credentials"; then - return 0 # Skip silently if no active sessions are found - fi - - # Check if the output of the list command is empty - if [ -z "$(eval "$list_cmd" 2> /dev/null)" ]; then - return 0 # Skip silently if no active sessions are found - fi - - log_info "Cleaning up $name authentication" - - # Run the logout command and capture any output or errors - local logout_output - logout_output=$(eval "$logout_cmd" 2>&1) - local logout_status=$? - - # Check if the logout was successful or if an expected error message was returned - if [[ $logout_status -ne 0 ]]; then - if echo "$logout_output" | grep -q -E "No credentials available to revoke|No active sessions|No active accounts"; then - log_info "No active $name credentials to revoke." - else - log_error "Cleanup" "Failed to log out of $name: $logout_output" - return 1 - fi - else - log_success "Cleanup" "$name authentication cleaned up successfully." - cleaned_up=true - fi - - # Return the cleanup status for summary reporting - if [[ "$cleaned_up" == true ]]; then - return 0 - else - return 1 - fi -} - -# Function to clean up credential files -cleanup_cred_files() { - local provider=$1 - local env_var_name="${provider^^}_CREDS" # Convert to uppercase - local creds_value="${!env_var_name}" - - # Skip if no credentials value found - if [[ -z "$creds_value" ]]; then - return 0 - fi - - # If the value is a file path and exists, remove it - if [[ -f "$creds_value" ]]; then - log_info "Removing credential file for $provider: $creds_value" - if rm -f "$creds_value"; then - log_success "Cleanup" "Removed credential file for $provider" - return 0 - else - log_error "Cleanup" "Failed to remove credential file for $provider" - return 1 - fi - fi - - return 0 -} - -# Function to clean up actors based on the providers configured during authentication -cleanup_actors() { - # Check if cleanup is enabled - if [[ "${ACTORS_CLEANUP,,}" != "true" ]]; then - log_info "Actors cleanup is disabled via ACTORS_CLEANUP environment variable" - return 0 - fi - - # Skip cleanup if no providers were configured during authentication - if [[ ${#configured_providers[@]} -eq 0 ]]; then - return 0 - fi - - log_info "Starting cleanup of actors" - - # Track if any actual cleanup was performed - local any_cleanup=false - - # Only clean up providers that were actually configured - for provider in "${configured_providers[@]}"; do - # First cleanup any credential files - if cleanup_cred_files "$provider"; then - any_cleanup=true - fi - - # Then cleanup provider sessions - case "$provider" in - azure) - if cleanup_provider "az" "az logout" "az account show" "Azure"; then - any_cleanup=true - fi - ;; - gcp) - if cleanup_provider "gcloud" "gcloud auth revoke --all && unset GOOGLE_APPLICATION_CREDENTIALS" "gcloud auth list" "GCP"; then - any_cleanup=true - fi - ;; - aws) - if cleanup_provider "aws" "aws sso logout" "aws sso list-accounts" "AWS"; then - any_cleanup=true - fi - ;; - *) - log_warn "Unsupported or unavailable actor type for cleanup: $provider" - ;; - esac - done - - # Log a summary if no cleanup actions were needed - if [[ "$any_cleanup" == false ]]; then - log_info "No active sessions found for any configured providers." - fi - - # Remove local copy creds dir - if [ -d "$LOCAL_CREDS_DIR" ]; then - log_info "Removing local copy creds dir" - rm -rf "$LOCAL_CREDS_DIR" - fi - - # Clear the configured providers array - configured_providers=() - - return 0 -} - -# Example usage -# cleanup_actors diff --git a/lib/cli.sh b/lib/cli.sh index 4ebaf35a..6045c124 100644 --- a/lib/cli.sh +++ b/lib/cli.sh @@ -87,7 +87,7 @@ EOF cat << EOF Run any command without arguments to see its detailed help and usage information. -For example: 'worker auth' will show auth command help. +For example: 'worker service' will show service command help. EOF } @@ -132,4 +132,4 @@ else log_error "CLI" "Unknown command: $command" echo "Run 'worker help' to see available commands." exit 1 -fi \ No newline at end of file +fi diff --git a/lib/cli/auth.sh b/lib/cli/auth.sh deleted file mode 100644 index 23cd77c2..00000000 --- a/lib/cli/auth.sh +++ /dev/null @@ -1,269 +0,0 @@ -#!/bin/bash - -# shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 -source "${WORKER_LIB_DIR}/utils.sh" -source "${WORKER_LIB_DIR}/auth.sh" - -# Show help for auth command -auth_help() { - cat << EOF -Manage authentication and credentials - -Usage: worker auth [command] [provider] [options] - -Available Commands: - status Show authentication status for all or specific provider - login Re-authenticate with provider(s) using available credentials - logout Log out from provider(s) - -Options: - --format Output format for status command (e.g. json) - -Examples: - worker auth status # Show status of all providers - worker auth status azure # Show status of Azure only - worker auth status --format json # Show status in JSON format - worker auth login # Re-auth all providers with available creds - worker auth login azure # Re-auth Azure only - worker auth logout # Log out from all providers -EOF -} - -# Description: Display authentication status for all cloud providers -# Example: worker auth status [--format json] -show_auth_status() { - local target_provider=$1 - local format=$2 - - if [ "$format" != "json" ]; then - log_info "Auth" "Checking authentication status..." - fi - - # Initialize JSON array if json format - local json_output="[" - - # Load config once and extract actors section - local config actors - config=$(load_and_parse_config) - actors=$(get_config_section "$config" "actors") - - # Function to check a specific provider - check_provider_status() { - local provider=$1 - local status="Not configured" - local types="" - - # Get provider's actors - local provider_actors - provider_actors=$(echo "$actors" | jq -r "[.[] | select(.type | startswith(\"$provider\"))]" 2>/dev/null) - - if [ -n "$provider_actors" ] && [ "$provider_actors" != "[]" ]; then - types=$(echo "$provider_actors" | jq -r '.[].type' 2>/dev/null | tr '\n' ' ') - - # First check if provider has credentials - if is_provider_configured "$provider" "$provider_actors"; then - # Then check if it's authenticated - if check_provider_auth "$provider"; then - status="Authenticated" - state="active" - else - status="Needs re-auth" - state="needs_reauth" - fi - else - status="Not configured" - state="missing_creds" - fi - else - status="Not configured" - state="not_configured" - fi - - # Output status based on format - if [ "$format" = "json" ]; then - [ -n "$json_output" ] && [ "$json_output" != "[" ] && json_output+="," - json_output+=$(jq -n \ - --arg provider "$provider" \ - --arg state "$state" \ - --arg status "$status" \ - --arg types "$types" \ - '{provider: $provider, state: $state, status: $status, types: $types}') - else - case "$state" in - "active") log_success "Auth" "$provider: $status" ;; - "needs_reauth") log_info "Auth" "$provider: $status" ;; - "missing_creds") log_warn "Auth" "$provider: $status" ;; - "not_configured") log_info "Auth" "$provider: $status" ;; - esac - fi - } - - # Check status for specific provider or all providers - if [ -n "$target_provider" ]; then - check_provider_status "$target_provider" - else - check_provider_status "aws" - check_provider_status "gcp" - check_provider_status "azure" - - fi - - # Close JSON array if json format - if [ "$format" = "json" ]; then - json_output+="]" - echo "$json_output" - fi -} - -# Login to provider(s) -login_provider() { - local target_provider=$1 - log_info "Auth" "Authenticating providers..." - - # Load config once and extract actors section - local config actors - config=$(load_and_parse_config) - actors=$(get_config_section "$config" "actors") - - if [[ -z "$actors" || "$actors" == "null" ]]; then - log_warn "Auth" "No providers found in configuration" - return 1 - fi - - # Filter actors by provider if specified - if [[ -n "$target_provider" ]]; then - actors=$(echo "$actors" | jq -r "[.[] | select(.type | startswith(\"$target_provider\"))]") - if [[ "$actors" == "[]" ]]; then - log_warn "Auth" "$target_provider: Not configured" - return 1 - fi - fi - - # Use the same authentication flow as entrypoint - if authenticate_actors "$actors"; then - log_success "Auth" "Authentication complete" - return 0 - else - log_warn "Auth" "No providers were authenticated" - return 1 - fi -} - -# Logout from provider(s) -logout_provider() { - local target_provider=$1 - log_info "Auth" "Logging out providers..." - - # Source cleanup utilities - source "${WORKER_LIB_DIR}/cleanup.sh" - - # Function to logout from a specific provider - do_provider_logout() { - local provider=$1 - - case "$provider" in - aws) - cleanup_provider "aws" "aws sso logout" "aws sso list-accounts" "AWS" - ;; - azure) - cleanup_provider "az" "az logout" "az account show" "Azure" - ;; - gcp) - cleanup_provider "gcloud" "gcloud auth revoke --all" "gcloud auth list" "GCP" - ;; - esac - } - - if [ -n "$target_provider" ]; then - do_provider_logout "$target_provider" - else - for provider in aws gcp azure; do - do_provider_logout "$provider" - done - fi -} - - - -# Check if a provider is currently authenticated -check_provider_auth() { - local provider=$1 - - case "$provider" in - aws) - if aws sts get-caller-identity &>/dev/null; then - return 0 - fi - ;; - azure) - # Azure CLI can return non-zero exit code even when it succeeds - # so we check if the output contains valid JSON - if output=$(az account show 2>/dev/null) && echo "$output" | jq empty &>/dev/null; then - return 0 - fi - ;; - gcp) - if gcloud auth list --format="value(account)" 2>/dev/null | grep -q .; then - return 0 - fi - ;; - esac - - return 1 -} - -# Handle auth commands -auth_handler() { - local cmd=$1 - shift - - case $cmd in - status) - local provider="" - local format="" - - # Parse arguments - while [ $# -gt 0 ]; do - case "$1" in - --format) - format="$2" - shift 2 - ;; - --*) - log_error "CLI" "Unknown option: $1" - return 1 - ;; - *) - if [ -z "$provider" ]; then - provider="$1" - else - log_error "CLI" "Unexpected argument: $1" - return 1 - fi - shift - ;; - esac - done - - show_auth_status "$provider" "$format" - return $? - ;; - login) - login_provider "$provider" - return $? - ;; - logout) - logout_provider "$provider" - return $? - ;; - "" | help) - auth_help - return 0 - ;; - *) - log_error "CLI" "Unknown command: auth" - auth_help - return 1 - ;; - esac -} diff --git a/lib/cli/config.sh b/lib/cli/config.sh index 24971cb2..270bd878 100644 --- a/lib/cli/config.sh +++ b/lib/cli/config.sh @@ -92,7 +92,6 @@ edit_config() { # Create file if it doesn't exist if [ ! -f "$config_file" ]; then - # Copy built-in config first to ensure actors section is preserved if [ -f "$BUILT_IN_CONFIG" ]; then cp "$BUILT_IN_CONFIG" "$config_file" || { log_error "Config" "Failed to copy built-in configuration" @@ -128,9 +127,9 @@ EOF show_locations() { cat << EOF Configuration Locations: - Built-in config: $BUILT_IN_CONFIG + Built-in config: $BUILT_IN_CONFIG User config: $USER_CONFIG - Merged config: $MERGED_CONFIG + Active config: $(get_worker_config_path) EOF } @@ -157,7 +156,6 @@ EOF if [ -f "$USER_CONFIG" ]; then log_success "Config" "Configuration initialized at $USER_CONFIG" - merge_worker_configs # Merge with built-in config return 0 else log_error "Config" "Failed to create configuration file" @@ -187,32 +185,12 @@ show_diff() { # Example: worker config apply apply_config() { log_info "Config" "Parsing and applying configuration..." - - # Load and parse the configuration - local config_json - if ! config_json=$(load_and_parse_config); then - log_error "Config" "Failed to load and parse configuration" - return 1 - fi - # Export variables from the configuration - if ! export_variables_from_config "$config_json"; then - log_error "Config" "Failed to export variables from configuration" + if ! configure_environment; then + log_error "Config" "Failed to parse and apply configuration" return 1 fi - # Extract secrets section from config - local secrets_json - secrets_json=$(echo "$config_json" | jq -r '.config.secrets // {}') - - # Fetch and set secrets if any are defined - if [[ "$secrets_json" != "{}" ]]; then - if ! fetch_secrets "$secrets_json"; then - log_error "Config" "Failed to fetch and set secrets" - return 1 - fi - fi - log_success "Config" "Configuration successfully parsed and applied" return 0 } diff --git a/lib/cli/env.sh b/lib/cli/env.sh index 7507f8b2..38346446 100644 --- a/lib/cli/env.sh +++ b/lib/cli/env.sh @@ -255,21 +255,14 @@ show_status() { local format=${1:-text} local env_file_exists=false - local secrets_file_exists=false local env_count=0 - local secrets_count=0 [ -f "$WORKER_ENV_FILE" ] && env_file_exists=true - [ -f "$WORKER_SECRETS_FILE" ] && secrets_file_exists=true if [ "$env_file_exists" = true ]; then env_count=$(grep -c "^export" "$WORKER_ENV_FILE" || echo 0) fi - if [ "$secrets_file_exists" = true ]; then - secrets_count=$(grep -c "^export" "$WORKER_SECRETS_FILE" || echo 0) - fi - case $format in json) { @@ -278,11 +271,6 @@ show_status() { echo " \"file\": \"$WORKER_ENV_FILE\"," echo " \"exists\": $env_file_exists," echo " \"variables\": $env_count" - echo " }," - echo " \"secrets\": {" - echo " \"file\": \"$WORKER_SECRETS_FILE\"," - echo " \"exists\": $secrets_file_exists," - echo " \"variables\": $secrets_count" echo " }" echo "}" } | jq '.' @@ -293,10 +281,6 @@ show_status() { echo "Environment File: $WORKER_ENV_FILE" echo " - Exists: $env_file_exists" echo " - Variables: $env_count" - echo - echo "Secrets File: $WORKER_SECRETS_FILE" - echo " - Exists: $secrets_file_exists" - echo " - Variables: $secrets_count" ;; *) log_error "Env" "Unknown format: $format" @@ -411,29 +395,11 @@ env_handler() { ;; reload) log_info "Env" "Reloading environment from configuration..." - local config_json - if ! config_json=$(load_and_parse_config); then - log_error "Env" "Failed to load and parse configuration" - return 1 - fi - - if ! export_variables_from_config "$config_json"; then - log_error "Env" "Failed to export variables from configuration" + if ! configure_environment; then + log_error "Env" "Failed to reload environment from configuration" return 1 fi - # Extract secrets section from config - local secrets_json - secrets_json=$(echo "$config_json" | jq -r '.config.secrets // {}') - - # Fetch and set secrets if any are defined - if [[ "$secrets_json" != "{}" ]]; then - if ! fetch_secrets "$secrets_json"; then - log_error "Env" "Failed to fetch and set secrets" - return 1 - fi - fi - log_success "Env" "Environment successfully reloaded from configuration" ;; status) @@ -451,4 +417,4 @@ env_handler() { exit 1 ;; esac -} \ No newline at end of file +} diff --git a/lib/cli/service.sh b/lib/cli/service.sh index 98b1af0d..16d55886 100644 --- a/lib/cli/service.sh +++ b/lib/cli/service.sh @@ -5,7 +5,23 @@ source "${WORKER_LIB_DIR}/utils.sh" # Constants SERVICES_CONFIG_DIR="${HOME}/.config/worker" -SERVICES_CONFIG_FILE="${SERVICES_CONFIG_DIR}/services.yaml" +USER_SERVICES_CONFIG_FILE="${SERVICES_CONFIG_DIR}/services.yaml" +BUILT_IN_SERVICES_CONFIG_FILE="${WORKER_CONFIG_DIR}/services.yaml" +SERVICES_CONFIG_FILE="" + +get_services_config_file() { + if [ -f "$USER_SERVICES_CONFIG_FILE" ] && [ -s "$USER_SERVICES_CONFIG_FILE" ]; then + echo "$USER_SERVICES_CONFIG_FILE" + return 0 + fi + + if [ -f "$BUILT_IN_SERVICES_CONFIG_FILE" ] && [ -s "$BUILT_IN_SERVICES_CONFIG_FILE" ]; then + echo "$BUILT_IN_SERVICES_CONFIG_FILE" + return 0 + fi + + return 1 +} # Show help for service command service_help() { @@ -66,9 +82,11 @@ service_handler() { return 0 fi + SERVICES_CONFIG_FILE=$(get_services_config_file) + # Check if services config exists before most commands if [ "$cmd" != "init" ]; then - if [ ! -f "$SERVICES_CONFIG_FILE" ]; then + if [ -z "$SERVICES_CONFIG_FILE" ]; then log_warn "Service" "No services configuration found" log_info "Service" "Run 'worker service' for information about service configuration" return 1 @@ -367,6 +385,29 @@ service_show_config() { fi } +# Description: Initialize a user service configuration +# Example: worker service init +init_service_config() { + mkdir -p "$SERVICES_CONFIG_DIR" || { + log_error "Service" "Failed to create service config directory: $SERVICES_CONFIG_DIR" + return 1 + } + + if [ -f "$USER_SERVICES_CONFIG_FILE" ]; then + log_info "Service" "Service configuration already exists at $USER_SERVICES_CONFIG_FILE" + return 0 + fi + + cat > "$USER_SERVICES_CONFIG_FILE" << 'EOF' +--- +kind: workerService +version: udx.io/worker-v1/service +services: [] +EOF + + log_success "Service" "Service configuration created at $USER_SERVICES_CONFIG_FILE" +} + # Description: Start, stop, or restart a service # Example: worker service restart my-app manage_service() { diff --git a/lib/env_handler.sh b/lib/env_handler.sh index d439c35d..450f6f67 100644 --- a/lib/env_handler.sh +++ b/lib/env_handler.sh @@ -4,7 +4,63 @@ source "${WORKER_LIB_DIR}/utils.sh" # Environment file location -WORKER_ENV_FILE="/etc/worker/environment" +WORKER_ENV_FILE="${WORKER_ENV_FILE:-/etc/worker/environment}" + +ensure_env_file() { + local env_dir + env_dir=$(dirname "$WORKER_ENV_FILE") + + mkdir -p "$env_dir" || { + log_error "Environment" "Failed to create environment directory: $env_dir" + return 1 + } + + touch "$WORKER_ENV_FILE" || { + log_error "Environment" "Failed to create environment file: $WORKER_ENV_FILE" + return 1 + } +} + +escape_env_value() { + local value="$1" + value=${value//\\/\\\\} + value=${value//\"/\\\"} + value=${value//\$/\\\$} + value=${value//\`/\\\`} + printf "%s" "$value" +} + +upsert_env_value() { + local name="$1" + local value="$2" + + if [[ -z "$name" ]]; then + log_error "Environment" "Variable name not provided" + return 1 + fi + + if ! [[ "$name" =~ ^[a-zA-Z_][a-zA-Z0-9_]*$ ]]; then + log_error "Environment" "Invalid variable name: $name" + return 1 + fi + + ensure_env_file || return 1 + + local tmpfile + tmpfile=$(mktemp "${WORKER_ENV_FILE}.tmp.XXXXXX") || { + log_error "Environment" "Failed to create temporary environment file" + return 1 + } + + grep -v "^export $name=" "$WORKER_ENV_FILE" > "$tmpfile" || true + printf 'export %s="%s"\n' "$name" "$(escape_env_value "$value")" >> "$tmpfile" + + mv "$tmpfile" "$WORKER_ENV_FILE" || { + rm -f "$tmpfile" + log_error "Environment" "Failed to update environment file" + return 1 + } +} # Generate environment file with regular variables generate_env_file() { @@ -18,8 +74,7 @@ generate_env_file() { log_info "Environment" "Loading environment variables from configuration" - # Create the environment file - touch "$WORKER_ENV_FILE" + ensure_env_file || return 1 # Process each environment variable while IFS= read -r entry; do @@ -39,12 +94,12 @@ generate_env_file() { # We use printenv to check if it's truly in the environment, not just a shell variable if ! printenv "$key" > /dev/null 2>&1; then # Variable doesn't exist in environment, add it from config - echo "export $key=\"$value\"" >> "$WORKER_ENV_FILE" + upsert_env_value "$key" "$value" || return 1 else # Variable exists in environment, use that value instead local env_value env_value="$(printenv "$key")" - echo "export $key=\"$env_value\"" >> "$WORKER_ENV_FILE" + upsert_env_value "$key" "$env_value" || return 1 log_info "Environment" "Detected [$key] in container environment - using runtime value instead of config value" fi fi @@ -65,18 +120,6 @@ _resolve_and_append_secrets() { return 1 fi - # Create a temporary file for resolved secrets - local temp_file - temp_file=$(mktemp) - - { - echo "" - echo "# Resolved Secrets" - echo "# Generated on $(date)" - echo "# DO NOT EDIT THIS FILE DIRECTLY" - echo - } > "$temp_file" - # Process each secret and resolve it while IFS= read -r secret; do local name value @@ -103,7 +146,7 @@ _resolve_and_append_secrets() { log_error "Environment" "Failed to resolve secret for $name" has_failures=true else - echo "export $name=\"$value\"" >> "$temp_file" + upsert_env_value "$name" "$value" || has_failures=true log_success "Environment" "Resolved secret for $name" fi done < <(echo "$secrets_json" | jq -c 'to_entries[]') @@ -111,14 +154,10 @@ _resolve_and_append_secrets() { # If any secret failed to resolve, error out if [[ "$has_failures" == "true" ]]; then log_error "Environment" "Failed to resolve one or more secrets" - rm -f "$temp_file" return 1 fi - # If we get here, all secrets resolved successfully - cat "$temp_file" >> "$WORKER_ENV_FILE" log_success "Environment" "Added all resolved secrets to environment file" - rm -f "$temp_file" } # Resolve secrets from worker.yaml config.secrets section @@ -133,6 +172,43 @@ resolve_env_var_secrets() { _resolve_and_append_secrets "$1" "false" } +# Configure environment from worker.yaml plus runtime secret references. +configure_environment() { + log_info "Starting environment configuration..." + + local resolved_config + resolved_config=$(load_and_parse_config) + if [[ -z "$resolved_config" ]]; then + log_error "Environment" "Configuration loading failed. Exiting..." + return 1 + fi + + if ! export_variables_from_config "$resolved_config"; then + log_error "Environment" "Failed to export variables." + return 1 + fi + + local secrets + secrets=$(get_config_section "$resolved_config" "secrets") + if [[ $? -eq 0 && -n "$secrets" && "$secrets" != "{}" ]]; then + log_info "Fetching secrets from configuration..." + if ! fetch_secrets "$secrets"; then + log_error "Environment" "Failed to fetch secrets." + return 1 + fi + else + log_info "No secrets defined in the configuration." + fi + + log_info "Checking for secret references in environment variables..." + if ! fetch_secrets_from_env_vars; then + log_error "Environment" "Failed to fetch secrets from environment variables." + return 1 + fi + + log_info "Secure environment setup completed successfully." +} + # Load environment variables and secrets load_environment() { if [ -f "$WORKER_ENV_FILE" ]; then @@ -181,17 +257,13 @@ format_env_vars() { # Initialize environment init_environment() { generate_env_file - generate_secrets_file load_environment - load_secrets } # Update environment when config changes update_environment() { generate_env_file - generate_secrets_file load_environment - load_secrets } # Get environment variable value @@ -205,10 +277,8 @@ get_env_value() { if [ -f "$WORKER_ENV_FILE" ]; then grep "^export $var_name=" "$WORKER_ENV_FILE" | cut -d'=' -f2- | tr -d '"' - elif [ -f "$WORKER_SECRETS_FILE" ]; then - grep "^export $var_name=" "$WORKER_SECRETS_FILE" | cut -d'=' -f2- | tr -d '"' else - log_error "Environment" "Neither environment nor secrets file exists" + log_error "Environment" "Environment file does not exist" return 1 fi } @@ -217,14 +287,13 @@ get_env_value() { list_env_vars() { local show_secrets="$1" local env_vars="" - local secret_vars="" if [ -f "$WORKER_ENV_FILE" ]; then env_vars=$(grep "^export" "$WORKER_ENV_FILE" | cut -d'=' -f1 | cut -d' ' -f2) fi - - if [ "$show_secrets" = "true" ] && [ -f "$WORKER_SECRETS_FILE" ]; then - secret_vars=$(grep "^export" "$WORKER_SECRETS_FILE" | cut -d'=' -f1 | cut -d' ' -f2) + + if [ "$show_secrets" = "true" ]; then + log_warn "Environment" "Secrets are stored in the worker environment file; separate secret listing is no longer used." fi if [ -n "$env_vars" ]; then @@ -232,9 +301,4 @@ list_env_vars() { echo "$env_vars" fi - if [ -n "$secret_vars" ]; then - echo - echo "Secret Variables:" - echo "$secret_vars" - fi -} \ No newline at end of file +} diff --git a/lib/environment.sh b/lib/environment.sh deleted file mode 100644 index 0da68d4a..00000000 --- a/lib/environment.sh +++ /dev/null @@ -1,88 +0,0 @@ -#!/bin/bash - -# Include necessary modules -# shellcheck disable=SC1091 -source "${WORKER_LIB_DIR}/auth.sh" -# shellcheck disable=SC1091 -source "${WORKER_LIB_DIR}/secrets.sh" -# shellcheck disable=SC1091 -source "${WORKER_LIB_DIR}/cleanup.sh" -# shellcheck disable=SC1091 -source "${WORKER_LIB_DIR}/worker_config.sh" - -# shellcheck disable=SC1091 -source "${WORKER_LIB_DIR}/utils.sh" - -# Main function to coordinate environment setup -configure_environment() { - log_info "Starting environment configuration..." - - # Load and resolve the worker configuration - local resolved_config - resolved_config=$(load_and_parse_config) - if [[ -z "$resolved_config" ]]; then - log_error "Environment" "Configuration loading failed. Exiting..." - return 1 - fi - - # Export variables from the configuration - if ! export_variables_from_config "$resolved_config"; then - log_error "Environment" "Failed to export variables." - return 1 - fi - - # Set default envs - if [[ -z "${ACTORS_CLEANUP:-}" ]]; then - export ACTORS_CLEANUP=true - fi - - if [[ -z "${LOCAL_CREDS_DIR:-}" ]]; then - export LOCAL_CREDS_DIR="$HOME/.config/worker/creds" - fi - - # Extract and authenticate actors - local actors - actors=$(get_config_section "$resolved_config" "actors") - if [[ $? -eq 0 && -n "$actors" ]]; then - log_info "Authenticating actors from configuration..." - if ! authenticate_actors "$actors"; then - log_error "Environment" "Failed to authenticate actors." - return 1 - fi - else - log_info "No actors defined in the configuration." - fi - - # Extract and fetch secrets - local secrets - secrets=$(get_config_section "$resolved_config" "secrets") - if [[ $? -eq 0 && -n "$secrets" ]]; then - log_info "Fetching secrets from configuration..." - if ! fetch_secrets "$secrets"; then - log_error "Environment" "Failed to fetch secrets." - return 1 - fi - else - log_info "No secrets defined in the configuration." - fi - - # Fetch secrets from environment variables with provider prefixes - log_info "Checking for secret references in environment variables..." - if ! fetch_secrets_from_env_vars; then - log_error "Environment" "Failed to fetch secrets from environment variables." - return 1 - fi - - # Perform cleanup - log_info "Cleaning up sensitive data..." - if ! cleanup_actors; then - log_error "Environment" "Failed to clean up actors." - return 1 - fi - - # Environment setup complete - log_info "Secure environment setup completed successfully." -} - -# Call the main function -configure_environment diff --git a/lib/process_manager.sh b/lib/process_manager.sh index d6d65ff9..9b9f979f 100644 --- a/lib/process_manager.sh +++ b/lib/process_manager.sh @@ -5,7 +5,8 @@ source "${WORKER_LIB_DIR}/utils.sh" # Define paths USER_CONFIG_PATH="${HOME}/.config/worker/services.yaml" -CONFIG_FILE="${USER_CONFIG_PATH}" +BUILT_IN_CONFIG_PATH="${WORKER_CONFIG_DIR}/services.yaml" +CONFIG_FILE="" # Supervisor configuration paths COMMON_TEMPLATE_FILE="${WORKER_CONFIG_DIR}/supervisor/common.conf" @@ -18,13 +19,19 @@ trap 'handle_supervisor_signals SIGINT' SIGINT # Main execution main() { - # Check if user config exists - if [[ ! -f "${USER_CONFIG_PATH}" ]]; then - log_info "No services configuration found at ${USER_CONFIG_PATH}." + CONFIG_FILE=$(get_service_config_path) + + if [[ -z "$CONFIG_FILE" ]]; then + log_info "No services configuration found." log_info "Run 'worker service' for information about service configuration" exit 0 fi + if ! has_enabled_services; then + log_info "No enabled services found in $CONFIG_FILE." + exit 0 + fi + log_info "Process Manager" "Starting process manager..." if ! configure_and_execute_services; then @@ -46,6 +53,28 @@ main() { wait } +get_service_config_path() { + if [[ -f "$USER_CONFIG_PATH" && -s "$USER_CONFIG_PATH" ]]; then + echo "$USER_CONFIG_PATH" + return 0 + fi + + if [[ -f "$BUILT_IN_CONFIG_PATH" && -s "$BUILT_IN_CONFIG_PATH" ]]; then + echo "$BUILT_IN_CONFIG_PATH" + return 0 + fi + + return 1 +} + +has_enabled_services() { + local enabled_services_count + + enabled_services_count=$(yq e '.services // [] | map(select(.ignore != true)) | length' "$CONFIG_FILE" 2>/dev/null) + + [[ "${enabled_services_count:-0}" -gt 0 ]] +} + # Helper function to parse and process each service configuration parse_service_info() { local service_json="$1" @@ -210,14 +239,11 @@ start_supervisor() { # Function to check for service configurations should_generate_config() { - local enabled_services_count - # Extract services into JSON format - services_yaml=$(yq e -o=json '.services[] | select(.ignore != true)' "$CONFIG_FILE") - # Count the number of items in the JSON array, trimming any newlines or spaces - enabled_services_count=$(echo "$services_yaml" | jq -c '. | length' | tr -d '\n') + if [[ -z "$CONFIG_FILE" ]]; then + CONFIG_FILE=$(get_service_config_path) + fi - # Check if the configuration file exists and there is at least one enabled service - if [ -f "$CONFIG_FILE" ] && [ "${enabled_services_count:-0}" -gt 0 ]; then + if [ -f "$CONFIG_FILE" ] && has_enabled_services; then return 0 else return 1 @@ -226,6 +252,10 @@ should_generate_config() { # Function to configure services configure_services() { + if [[ -z "$CONFIG_FILE" ]]; then + CONFIG_FILE=$(get_service_config_path) + fi + if ! should_generate_config; then log_warn "Process Manager" "No services found in $CONFIG_FILE. No Supervisor configuration generated." return 1 @@ -265,4 +295,4 @@ configure_and_execute_services() { # Only run main if not in service mode if [ -z "$WORKER_SERVICE_MODE" ]; then main -fi \ No newline at end of file +fi diff --git a/lib/runtime_output.sh b/lib/runtime_output.sh new file mode 100644 index 00000000..fb2a4911 --- /dev/null +++ b/lib/runtime_output.sh @@ -0,0 +1,72 @@ +#!/bin/bash + +# shellcheck source=${WORKER_LIB_DIR}/utils.sh disable=SC1091 +source "${WORKER_LIB_DIR}/utils.sh" +# shellcheck source=${WORKER_LIB_DIR}/worker_config.sh disable=SC1091 +source "${WORKER_LIB_DIR}/worker_config.sh" + +get_effective_env_value() { + local name="$1" + printenv "$name" 2>/dev/null || true +} + +build_runtime_output_json() { + local config_json="$1" + local worker_config_path services_config_path env_json secrets_json + + worker_config_path=$(get_worker_config_path) + services_config_path="${HOME}/.config/worker/services.yaml" + if [[ ! -f "$services_config_path" && -f "${WORKER_CONFIG_DIR}/services.yaml" ]]; then + services_config_path="${WORKER_CONFIG_DIR}/services.yaml" + fi + + env_json=$( + echo "$config_json" | jq -r '.config.env // {} | keys[]' 2>/dev/null | while IFS= read -r key; do + [ -n "$key" ] || continue + value=$(get_effective_env_value "$key") + printf '%s\t%s\n' "$key" "$value" + done | jq -Rn ' + reduce inputs as $line ({}; + ($line | split("\t")) as $parts | + . + {($parts[0]): ($parts[1] // "")} + ) + ' + ) + + secrets_json=$(echo "$config_json" | jq '.config.secrets // {}' 2>/dev/null) + + jq -n \ + --arg worker_config_path "$worker_config_path" \ + --arg services_config_path "$services_config_path" \ + --arg worker_env_file "$WORKER_ENV_FILE" \ + --arg generated_at "$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \ + --argjson env "$env_json" \ + --argjson secrets "$secrets_json" \ + '{ + generated_at: $generated_at, + paths: { + worker_config: $worker_config_path, + services_config: $services_config_path, + environment: $worker_env_file + }, + env: $env, + secrets: $secrets, + secret_values: "redacted" + }' +} + +emit_runtime_output() { + local config_json runtime_json + + if [[ -z "${WORKER_OUTPUT_FILE:-}" ]]; then + log_info "Runtime output disabled. Set WORKER_OUTPUT_FILE to write redacted JSON runtime config for workflow/deployment integrations." + return 0 + fi + + config_json=$(load_and_parse_config) || return 1 + runtime_json=$(build_runtime_output_json "$config_json") + + mkdir -p "$(dirname "$WORKER_OUTPUT_FILE")" || return 1 + echo "$runtime_json" > "$WORKER_OUTPUT_FILE" + log_info "Runtime output written to $WORKER_OUTPUT_FILE" +} diff --git a/lib/secrets.sh b/lib/secrets.sh index 6968fede..078b46c8 100644 --- a/lib/secrets.sh +++ b/lib/secrets.sh @@ -20,7 +20,6 @@ if [[ -z "${WORKER_INTERNAL_VARS+x}" ]]; then readonly WORKER_INTERNAL_VARS=( "AZURE_CONFIG_DIR" "AWS_CONFIG_FILE" - "GCP_CREDS" "TZ" ) fi diff --git a/lib/worker_config.sh b/lib/worker_config.sh index 274d96c4..7782418f 100644 --- a/lib/worker_config.sh +++ b/lib/worker_config.sh @@ -5,10 +5,9 @@ source "${WORKER_LIB_DIR}/utils.sh" # shellcheck source=${WORKER_LIB_DIR}/env_handler.sh disable=SC1091 source "${WORKER_LIB_DIR}/env_handler.sh" -# Paths for configurations -BUILT_IN_CONFIG="${WORKER_CONFIG_DIR}/worker.yaml" # Built-in default config -USER_CONFIG="${HOME}/.config/worker/worker.yaml" # Optional user config -MERGED_CONFIG="${WORKER_CONFIG_DIR}/worker.merged.yaml" # Result of merging both configs +# Paths for configuration. +BUILT_IN_CONFIG="${WORKER_CONFIG_DIR}/worker.yaml" +USER_CONFIG="${HOME}/.config/worker/worker.yaml" # Ensure `yq` is available if ! command -v yq >/dev/null 2>&1; then @@ -25,53 +24,34 @@ ensure_config_exists() { fi } -# Merge built-in and user-provided configurations -merge_worker_configs() { - # Ensure the merged configuration file exists - if [ ! -f "$MERGED_CONFIG" ]; then - touch "$MERGED_CONFIG" || { log_error "Worker configuration" "Failed to create merged configuration file at $MERGED_CONFIG"; return 1; } - fi - - # Ensure built-in config exists and has actors section - if ! ensure_config_exists "$BUILT_IN_CONFIG"; then - log_error "Worker configuration" "Built-in configuration not found" - return 1 +# Resolve the runtime configuration path. A mounted user config is preferred; +# otherwise the built-in default keeps the image runnable without mounts. +get_worker_config_path() { + if [[ -f "$USER_CONFIG" && -s "$USER_CONFIG" ]]; then + echo "$USER_CONFIG" + return 0 fi - # First copy built-in config (with actors) to merged config - if ! cp "$BUILT_IN_CONFIG" "$MERGED_CONFIG"; then - log_error "Worker configuration" "Failed to copy built-in configuration" - return 1 + if [[ -f "$BUILT_IN_CONFIG" && -s "$BUILT_IN_CONFIG" ]]; then + echo "$BUILT_IN_CONFIG" + return 0 fi - # If user config exists, merge env and secrets sections - if [[ -f "$USER_CONFIG" && -s "$USER_CONFIG" ]]; then - # Use yq to merge configs, preserving actors from built-in - if ! yq eval-all 'select(fileIndex == 0) * select(fileIndex == 1)' "$MERGED_CONFIG" "$USER_CONFIG" > "${MERGED_CONFIG}.tmp"; then - log_error "Worker configuration" "Failed to merge configurations" - return 1 - fi - - # Check if merge was successful - if [ -s "${MERGED_CONFIG}.tmp" ]; then - mv "${MERGED_CONFIG}.tmp" "$MERGED_CONFIG" - return 0 - else - rm -f "${MERGED_CONFIG}.tmp" - log_error "Worker configuration" "Failed to merge configurations - empty result" - return 1 - fi - fi + echo "$BUILT_IN_CONFIG" } -# Load and parse the merged configuration +# Load and parse the active configuration. load_and_parse_config() { - merge_worker_configs || return 1 + local config_path + config_path=$(get_worker_config_path) + + if ! ensure_config_exists "$config_path"; then + return 1 + fi - # Parse the merged configuration into JSON local json_output - if ! json_output=$(yq eval -o=json "$MERGED_CONFIG" 2>/dev/null); then - log_error "Worker configuration" "Failed to parse merged YAML from $MERGED_CONFIG. yq returned an error." + if ! json_output=$(yq eval -o=json "$config_path" 2>/dev/null); then + log_error "Worker configuration" "Failed to parse YAML from $config_path. yq returned an error." return 1 fi @@ -119,4 +99,4 @@ get_config_section() { fi echo "$extracted_section" -} \ No newline at end of file +} diff --git a/src/configs/services.yaml b/src/configs/services.yaml new file mode 100644 index 00000000..6d999511 --- /dev/null +++ b/src/configs/services.yaml @@ -0,0 +1,4 @@ +--- +kind: workerService +version: udx.io/worker-v1/service +services: [] diff --git a/src/configs/worker.yaml b/src/configs/worker.yaml new file mode 100644 index 00000000..9358ed77 --- /dev/null +++ b/src/configs/worker.yaml @@ -0,0 +1,7 @@ +--- +kind: workerConfig +version: udx.io/worker-v1/config +config: + env: + WORKER_OUTPUT_FILE: "" + secrets: {} diff --git a/src/tests/modules/30_auth.sh b/src/tests/modules/30_auth.sh deleted file mode 100755 index ef011cba..00000000 --- a/src/tests/modules/30_auth.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash - -# Source test helpers -# shellcheck source=../test_helpers.sh disable=SC1091 -source "/home/udx/tests/test_helpers.sh" - -# Test authentication commands -print_header "Authentication Tests" - -# Test auth status -print_info "Testing: auth status" - -# Capture both stdout and stderr -AUTH_OUTPUT=$(worker auth status 2>&1) - -# Test that we got some output -if [ -z "$AUTH_OUTPUT" ]; then - print_error "auth status produced no output" - exit 1 -fi - -# Test that output contains provider status -if ! printf "%s" "$AUTH_OUTPUT" | grep -q "Auth:"; then - print_error "auth status should show provider status" - exit 1 -fi - -# Test auth status json format -print_info "Testing: auth status --format json" -AUTH_JSON=$(worker auth status --format json) -if ! echo "$AUTH_JSON" | jq -e '.[] | select(.provider == "aws") | .status' > /dev/null; then - print_error "auth status json should include provider status" - exit 1 -fi - -# All tests passed -print_success "All authentication tests passed" diff --git a/src/tests/main.sh b/test/main.sh similarity index 96% rename from src/tests/main.sh rename to test/main.sh index 88459e5a..b721bd12 100755 --- a/src/tests/main.sh +++ b/test/main.sh @@ -4,7 +4,7 @@ # Source test helpers # shellcheck source=./test_helpers.sh disable=SC1091 -source "/home/udx/tests/test_helpers.sh" +source "/home/udx/test/test_helpers.sh" # Exit on any error set -e @@ -30,7 +30,7 @@ PASSED=0 FAILED=0 # Directory containing test files -TEST_DIR="/home/udx/tests" +TEST_DIR="/home/udx/test" cd "$TEST_DIR" # Don't exit on test failures diff --git a/src/tests/modules/10_config.sh b/test/modules/10_config.sh similarity index 81% rename from src/tests/modules/10_config.sh rename to test/modules/10_config.sh index cb717822..91590c6c 100755 --- a/src/tests/modules/10_config.sh +++ b/test/modules/10_config.sh @@ -2,7 +2,7 @@ # Source test helpers # shellcheck source=../test_helpers.sh disable=SC1091 -source "/home/udx/tests/test_helpers.sh" +source "/home/udx/test/test_helpers.sh" # Test configuration commands print_header "Configuration Tests" @@ -30,5 +30,12 @@ if ! worker config locations | grep -q "/home/udx/.config/worker"; then exit 1 fi +# Test config apply +print_info "Testing: config apply" +if ! worker config apply; then + print_error "config apply should re-apply current configuration" + exit 1 +fi + # All tests passed print_success "All configuration tests passed" diff --git a/src/tests/modules/20_env.sh b/test/modules/20_env.sh similarity index 85% rename from src/tests/modules/20_env.sh rename to test/modules/20_env.sh index 31ec27f6..f967bfcb 100755 --- a/src/tests/modules/20_env.sh +++ b/test/modules/20_env.sh @@ -2,7 +2,7 @@ # Source test helpers # shellcheck source=../test_helpers.sh disable=SC1091 -source "/home/udx/tests/test_helpers.sh" +source "/home/udx/test/test_helpers.sh" # Test environment commands print_header "Environment Tests" @@ -43,5 +43,12 @@ for var in $CONFIG_ENV; do fi done +# Test environment reload +print_info "Testing: env reload" +if ! worker env reload; then + print_error "env reload should re-apply current configuration" + exit 1 +fi + # All tests passed print_success "All environment tests passed" diff --git a/src/tests/modules/40_service.sh b/test/modules/40_service.sh similarity index 97% rename from src/tests/modules/40_service.sh rename to test/modules/40_service.sh index 63112262..cac2d520 100755 --- a/src/tests/modules/40_service.sh +++ b/test/modules/40_service.sh @@ -2,7 +2,7 @@ # Source test helpers # shellcheck source=../test_helpers.sh disable=SC1091 -source "/home/udx/tests/test_helpers.sh" +source "/home/udx/test/test_helpers.sh" # Test service commands print_header "Service Tests" diff --git a/src/tests/modules/50_sbom.sh b/test/modules/50_sbom.sh similarity index 96% rename from src/tests/modules/50_sbom.sh rename to test/modules/50_sbom.sh index bf4c141b..2ee2683f 100755 --- a/src/tests/modules/50_sbom.sh +++ b/test/modules/50_sbom.sh @@ -2,7 +2,7 @@ # Source test helpers # shellcheck source=../test_helpers.sh disable=SC1091 -source "/home/udx/tests/test_helpers.sh" +source "/home/udx/test/test_helpers.sh" # Test SBOM commands print_header "SBOM Tests" diff --git a/src/tests/modules/60_health.sh b/test/modules/60_health.sh similarity index 96% rename from src/tests/modules/60_health.sh rename to test/modules/60_health.sh index 2ba2e12d..492cff2f 100755 --- a/src/tests/modules/60_health.sh +++ b/test/modules/60_health.sh @@ -2,7 +2,7 @@ # Source test helpers # shellcheck source=../test_helpers.sh disable=SC1091 -source "/home/udx/tests/test_helpers.sh" +source "/home/udx/test/test_helpers.sh" # Test health check commands print_header "Health Check Tests" diff --git a/src/tests/test_helpers.sh b/test/test_helpers.sh similarity index 100% rename from src/tests/test_helpers.sh rename to test/test_helpers.sh From dcb43b219e61a47f376acd79c0472ea07802f563 Mon Sep 17 00:00:00 2001 From: Dmytro Smirnov Date: Sun, 31 May 2026 12:03:17 +0300 Subject: [PATCH 02/32] docs: reorganize worker runtime docs --- README.md | 110 ++++------ deploy-gcp.yml | 18 -- deploy.yml | 33 --- docs/auth/README.md | 31 --- docs/auth/aws.md | 14 -- docs/auth/azure.md | 24 --- docs/auth/gcp.md | 189 ------------------ docs/authorization.md | 94 --------- docs/{development => }/child-images.md | 35 +--- docs/{reference => }/cli.md | 16 +- docs/config.md | 94 +++++++++ docs/{development => }/core-image.md | 6 +- docs/deploy/README.md | 34 ---- docs/deploy/image-override.md | 58 ------ docs/deploy/kubernetes.md | 74 ------- docs/deploy/worker-deployment.md | 58 ------ docs/deployment.md | 54 +++++ .../{development/README.md => development.md} | 10 +- docs/index.md | 34 ---- docs/references/README.md | 6 + docs/references/cloud-providers-auth.md | 27 +++ .../container-structure.md | 4 +- docs/runtime/config.md | 135 ------------- docs/secrets.md | 90 +++++++++ docs/{runtime => }/services.md | 10 +- src/examples/README.md | 7 - src/examples/deploy-image-override/README.md | 29 --- .../deploy-image-override/deploy.template.yml | 8 - src/examples/simple-service/README.md | 2 +- 29 files changed, 346 insertions(+), 958 deletions(-) delete mode 100644 deploy-gcp.yml delete mode 100644 deploy.yml delete mode 100644 docs/auth/README.md delete mode 100644 docs/auth/aws.md delete mode 100644 docs/auth/azure.md delete mode 100644 docs/auth/gcp.md delete mode 100644 docs/authorization.md rename docs/{development => }/child-images.md (62%) rename docs/{reference => }/cli.md (67%) create mode 100644 docs/config.md rename docs/{development => }/core-image.md (84%) delete mode 100644 docs/deploy/README.md delete mode 100644 docs/deploy/image-override.md delete mode 100644 docs/deploy/kubernetes.md delete mode 100644 docs/deploy/worker-deployment.md create mode 100644 docs/deployment.md rename docs/{development/README.md => development.md} (75%) delete mode 100644 docs/index.md create mode 100644 docs/references/README.md create mode 100644 docs/references/cloud-providers-auth.md rename docs/{reference => references}/container-structure.md (96%) delete mode 100644 docs/runtime/config.md create mode 100644 docs/secrets.md rename docs/{runtime => }/services.md (91%) delete mode 100644 src/examples/deploy-image-override/README.md delete mode 100644 src/examples/deploy-image-override/deploy.template.yml diff --git a/README.md b/README.md index 885e73e9..8e12e694 100644 --- a/README.md +++ b/README.md @@ -2,21 +2,23 @@ [![Docker Pulls](https://img.shields.io/docker/pulls/usabilitydynamics/udx-worker.svg)](https://hub.docker.com/r/usabilitydynamics/udx-worker) [![License](https://img.shields.io/github/license/udx/worker.svg)](LICENSE) [![Documentation](https://img.shields.io/badge/docs-udx.dev-blue.svg)](https://udx.dev/worker) -**Secure, containerized environment for DevSecOps automation** +Container runtime foundation for UDX automation images. -[Quick Start](#-quick-start) • [Documentation](#-documentation) • [Development](#️-development) • [Contributing](#-contributing) +[Quick Start](#quick-start) | [Documentation](#documentation) | [Development](#development) -## 🚀 Overview +## Overview -UDX Worker is a containerized solution that simplifies DevSecOps by providing: +UDX Worker is a base container image for automation workloads that need predictable runtime config, secret references, and process supervision. -- 🔒 **Secure Environment**: Built on zero-trust principles -- 🤖 **Automation Support**: Streamlined task execution -- 🔑 **Secret Management**: Automatic detection and resolution from multiple providers -- 📦 **12-Factor Compliance**: Modern application practices -- ♾️ **CI/CD Ready**: Seamless pipeline integration with environment-based overrides +It provides: -## 🏃 Quick Start +- `worker.yaml` for runtime env values, secret references, and opt-in runtime output. +- `services.yaml` for supervised processes inside the container. +- Secret reference resolution from AWS, Azure, and Google Cloud after provider auth exists. +- A shared CLI for inspecting and re-applying container runtime config. +- A stable base for child images that add workload-specific tools. + +## Quick Start ### Prerequisites @@ -63,7 +65,7 @@ docker run -d \ docker logs -f my-service ``` -### Example 2: Secrets Management with Authorization +### Example 2: Secret References ```bash # Define secrets configuration @@ -72,51 +74,34 @@ kind: workerConfig version: udx.io/worker-v1/config config: secrets: - API_KEY: "azure/key-vault/api-key" - DB_PASS: "aws/secrets/database" + API_KEY: "gcp/my-project/api-key" + DB_PASS: "aws/database-password/us-west-2" EOF -# Create base64-encoded Azure credentials -AZURE_CREDS=$(echo '{ - "client_id": "your-client-id", - "client_secret": "your-client-secret", - "tenant_id": "your-tenant-id" -}' | base64) - -# Run with cloud provider credentials +# Run with provider auth injected by the host/platform docker run -d \ --name my-secrets \ -v "$(pwd)/.config/worker:/home/udx/.config/worker" \ - -e AZURE_CREDS="${AZURE_CREDS}" \ usabilitydynamics/udx-worker:latest -# Verify authorization and secrets -docker exec my-secrets worker auth verify -docker exec my-secrets worker env get API_KEY +# Verify resolved environment +docker exec my-secrets worker env show --filter API_KEY ``` -See [Authorization Guide](docs/authorization.md) for supported providers and credential formats (JSON, Base64, File Path). +See [Secrets](docs/secrets.md) for secret references and provider auth boundaries. -### 💡 Simplified Deployment +### Deployment -For easier deployment with automatic credential detection, use the [`@udx/worker-deployment`](https://www.npmjs.com/package/@udx/worker-deployment) CLI: +Deployment uses the host-native tool for the target environment. Mount runtime config into the container and pass provider credentials, workload identity, or secret references through the platform. ```bash -# Install -npm install -g @udx/worker-deployment - -# Generate config -worker config - -# Run with automatic GCP authentication -worker run +docker run --rm \ + -v "$(pwd)/.config/worker:/home/udx/.config/worker:ro" \ + -e API_KEY="gcp/my-project/api-key" \ + usabilitydynamics/udx-worker:latest ``` -Features: -- ✅ Auto-detects GCP credentials (service account keys, impersonation, workload identity) -- ✅ Zero-config for default file names -- ✅ Secure read-only mounts -- ✅ Interactive debugging mode +For Kubernetes, mount `worker.yaml` and `services.yaml` through ConfigMaps or Secrets and deploy the image with normal Kubernetes manifests. ### Development Setup @@ -138,22 +123,18 @@ make test More examples available in [src/examples/README.md](src/examples/README.md). -## 📚 Documentation +## Documentation -### Core Concepts -- [Docs Index](docs/index.md) - Start here -- [Runtime: Services](docs/runtime/services.md) - `services.yaml` -- [Runtime: Config](docs/runtime/config.md) - `worker.yaml` -- [Deployment](docs/deploy/README.md) - `deploy.yml` and `worker-deployment` -- [Authorization](docs/authorization.md) - Credential management -- [CLI Reference](docs/reference/cli.md) - Command line usage - -### Additional Resources -- [Container Structure](docs/reference/container-structure.md) - Directory layout -- [Development](docs/development/README.md) - Build, run, test, child images +- [CLI](docs/cli.md) - runtime inspection and re-apply commands +- [Config](docs/config.md) - `worker.yaml`, env values, runtime output +- [Secrets](docs/secrets.md) - secret references and provider auth boundaries +- [Services](docs/services.md) - `services.yaml` process config +- [Deployment](docs/deployment.md) - Docker, Kubernetes, and CI usage +- [Development](docs/development.md) - Build, test, and child image workflow +- [Reference Docs](docs/references/README.md) - provider auth options and container structure - [Examples](src/examples/README.md) - Runnable samples -## 🛠️ Development +## Development ```bash # Clone repository @@ -174,33 +155,18 @@ make test make help ``` -## 🤝 Contributing - -We welcome contributions! Here's how you can help: - -1. Fork the repository -2. Create a feature branch -3. Commit your changes -4. Push to your branch -5. Open a Pull Request - -Please ensure your PR: -- Follows our coding standards -- Includes appropriate tests -- Updates relevant documentation - -## 🔗 Resources +## Resources - [Docker Hub](https://hub.docker.com/r/usabilitydynamics/udx-worker) - [Documentation](https://udx.dev/worker) - [Product Page](https://udx.io/products/udx-worker) -## 🎯 Custom Development +## Custom Development Need specific features or customizations? [Contact our team](https://udx.io/) for professional development services. -## 📄 License +## License This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details. diff --git a/deploy-gcp.yml b/deploy-gcp.yml deleted file mode 100644 index e5925a66..00000000 --- a/deploy-gcp.yml +++ /dev/null @@ -1,18 +0,0 @@ -# npm install -g @udx/worker-deployment -# worker run --config=deploy-gcp.yml - ---- -kind: workerDeployConfig -version: udx.io/worker-v1/deploy -config: - # Docker image to run - image: "usabilitydynamics/udx-worker:latest" - - env: - ACTORS_CLEANUP: "false" - - # Prefer command + args for special characters - command: "gcloud" - args: - - "auth" - - "list" diff --git a/deploy.yml b/deploy.yml deleted file mode 100644 index 4375754a..00000000 --- a/deploy.yml +++ /dev/null @@ -1,33 +0,0 @@ -# npm install -g @udx/worker-deployment -# worker config -# worker run - ---- -kind: workerDeployConfig -version: udx.io/worker-v1/deploy -config: - # Docker image to run - image: "usabilitydynamics/udx-worker:latest" - - env: - TEST_ENV_SECRET: "gcp/rabbit-ci-dev/worker-secret-test" - TEST_ENV_JSON_KEY: "gcp/rabbit-ci-dev/worker-secret-json-key" - - # Volume mounts (optional) - # Format: "host_path:container_path" or "host_path:container_path:ro" - # volumes: - # - "./worker.yaml:/home/udx/.config/worker/worker.yaml" - - # Ports to expose (optional) - # ports: - # - "80:80" - - # Command to run (optional - if not specified, uses container default) - # Tip: keep command to the executable and put complex values in args - # command: "/usr/local/bin/init.sh" - # args: - # - "--example-flag" - - # Service account impersonation (requires gcloud auth on host) - service_account: - email: "worker-site@rabbit-ci-dev.iam.gserviceaccount.com" diff --git a/docs/auth/README.md b/docs/auth/README.md deleted file mode 100644 index 17927107..00000000 --- a/docs/auth/README.md +++ /dev/null @@ -1,31 +0,0 @@ -# Provider Authentication Guides - -## Overview - -This directory contains detailed authentication documentation for each supported cloud provider. - -## When To Use - -Use these guides when you need provider-specific setup, credential formats, or CLI integration details. - -## Key Concepts - -- Each provider has its own auth flow and credential structure. -- Worker-deployment CLI can simplify authentication setup. - -## Examples - -- `docs/auth/gcp.md` - GCP authentication guide -- `docs/auth/azure.md` - Azure authentication guide (coming soon) -- `docs/auth/aws.md` - AWS authentication guide (coming soon) - -## Common Pitfalls - -- Mixing provider-specific formats. -- Passing plain secrets directly in config files. - -## Related Docs - -- `docs/authorization.md` -- `docs/runtime/config.md` -- `docs/reference/cli.md` diff --git a/docs/auth/aws.md b/docs/auth/aws.md deleted file mode 100644 index d5d7850a..00000000 --- a/docs/auth/aws.md +++ /dev/null @@ -1,14 +0,0 @@ -# AWS Authentication - -> **Coming Soon**: Detailed AWS authentication documentation - -## Quick Reference - -**Environment Variable:** `AWS_CREDS` - -**Supported Formats:** -- JSON -- Base64-encoded JSON -- File path - -For now, see the [general authorization guide](../authorization.md) for credential format examples. diff --git a/docs/auth/azure.md b/docs/auth/azure.md deleted file mode 100644 index 5bd2c822..00000000 --- a/docs/auth/azure.md +++ /dev/null @@ -1,24 +0,0 @@ -# Azure Authentication - -> **Coming Soon**: Detailed Azure authentication documentation - -## Quick Reference - -**Environment Variable:** `AZURE_CREDS` - -**Supported Formats:** -- JSON -- Base64-encoded JSON -- File path - -**Example:** -```json -{ - "client_id": "CLIENT_ID", - "client_secret": "CLIENT_SECRET", - "tenant_id": "TENANT_ID", - "subscription_id": "SUBSCRIPTION_ID" -} -``` - -For now, see the [general authorization guide](../authorization.md) for credential format examples. diff --git a/docs/auth/gcp.md b/docs/auth/gcp.md deleted file mode 100644 index 36b4ee9f..00000000 --- a/docs/auth/gcp.md +++ /dev/null @@ -1,189 +0,0 @@ -# GCP Authentication - -Google Cloud Platform supports multiple authentication methods, each suited for different use cases. - -## Authentication Methods - -### 1. Service Account Key (Most Common) - -Service account keys work for both local development and CI/CD environments. - -**JSON Format:** -```json -{ - "type": "service_account", - "project_id": "my-project-id", - "private_key_id": "key-id", - "private_key": "-----BEGIN PRIVATE KEY-----\n...\n-----END PRIVATE KEY-----\n", - "client_email": "my-sa@my-project.iam.gserviceaccount.com", - "client_id": "123456789", - "auth_uri": "https://accounts.google.com/o/oauth2/auth", - "token_uri": "https://oauth2.googleapis.com/token", - "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", - "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/..." -} -``` - -**Usage:** -```bash -# Via environment variable -export GCP_CREDS='{"type":"service_account",...}' - -# Via file path -export GCP_CREDS="/path/to/service-account-key.json" - -# Via base64 encoding (recommended for CI/CD) -export GCP_CREDS=$(cat service-account-key.json | base64) -``` - -**Features:** -- ✅ Automatic `private_key` normalization (handles escaped newlines) -- ✅ Sets both `GOOGLE_APPLICATION_CREDENTIALS` and `GCP_CREDS` -- ✅ Authenticates with `gcloud` CLI -- ✅ Sets project automatically from `project_id` field - ---- - -### 2. Workload Identity Token (GitHub Actions / CI/CD) - -Keyless authentication using OIDC tokens - no service account keys needed! - -**JSON Format:** -```json -{ - "type": "external_account", - "audience": "//iam.googleapis.com/projects/PROJECT_NUMBER/locations/global/workloadIdentityPools/POOL_ID/providers/PROVIDER_ID", - "subject_token_type": "urn:ietf:params:oauth:token-type:jwt", - "token_url": "https://sts.googleapis.com/v1/token", - "service_account_impersonation_url": "https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/SA_EMAIL:generateAccessToken", - "credential_source": { - "file": "/path/to/token", - "format": { - "type": "text" - } - } -} -``` - -**GitHub Actions Example:** -```yaml -- uses: google-github-actions/auth@v3 - id: auth - with: - workload_identity_provider: ${{ secrets.WIF_PROVIDER }} - service_account: ${{ secrets.WIF_SERVICE_ACCOUNT }} - -- name: Run Worker - env: - GCP_CREDS: ${{ steps.auth.outputs.credentials_file_path }} - run: | - docker run -e GCP_CREDS usabilitydynamics/udx-worker:latest -``` - -**Features:** -- ✅ No long-lived credentials -- ✅ Automatic token refresh -- ✅ Works with Google Cloud client libraries -- ✅ Recommended for CI/CD pipelines - ---- - -### 3. Service Account Impersonation (Local Development) - -Use your personal gcloud credentials to impersonate a service account - no key files needed! - -**Setup:** -```bash -# 1. Authenticate with gcloud -gcloud auth login - -# 2. Set up Application Default Credentials (required for Terraform/SDKs) -gcloud auth application-default login - -# 3. Grant yourself impersonation permission -gcloud iam service-accounts add-iam-policy-binding \ - my-sa@my-project.iam.gserviceaccount.com \ - --member="user:$(gcloud config get-value account)" \ - --role="roles/iam.serviceAccountTokenCreator" \ - --project=MY_PROJECT -``` - -**Usage with worker-deployment CLI:** -```yaml -# deploy.yml -config: - service_account: - email: "my-sa@my-project.iam.gserviceaccount.com" - image: "usabilitydynamics/udx-worker:latest" - command: "worker run my-task" -``` - -```bash -# Run with automatic impersonation -worker run --config=deploy.yml -``` - -**Manual Docker Usage:** -```bash -# Generate impersonation credentials -gcloud auth application-default print-access-token > /tmp/token.txt - -# Run container with impersonation -docker run \ - -e GOOGLE_APPLICATION_CREDENTIALS=/home/udx/adc.json \ - -e CLOUDSDK_AUTH_ACCESS_TOKEN=$(cat /tmp/token.txt) \ - -v ~/.config/gcloud/application_default_credentials.json:/home/udx/adc.json:ro \ - usabilitydynamics/udx-worker:latest -``` - -**Features:** -- ✅ No service account key files -- ✅ Uses your personal credentials -- ✅ Temporary access tokens -- ✅ Easy permission management -- ✅ Works with Terraform, gcloud, and SDKs - -> **Note**: Impersonation bypasses the `gcp_authenticate()` function by setting `GOOGLE_APPLICATION_CREDENTIALS` and `CLOUDSDK_AUTH_ACCESS_TOKEN` directly. - ---- - -## Authentication Priority - -When multiple credential sources are available, the worker uses this priority: - -1. **`GOOGLE_APPLICATION_CREDENTIALS`** - If already set, skip authentication (used for impersonation) -2. **`GCP_CREDS`** - Process through `gcp_authenticate()` function: - - Detect credential type (service account key vs. workload identity token) - - Normalize service account keys (fix escaped newlines in `private_key`) - - Set `GOOGLE_APPLICATION_CREDENTIALS` - - Authenticate with `gcloud auth login --cred-file` - ---- - -## Using worker-deployment CLI - -The [`@udx/worker-deployment`](https://www.npmjs.com/package/@udx/worker-deployment) CLI simplifies GCP authentication: - -**Installation:** -```bash -npm install -g @udx/worker-deployment -``` - -**Quick Start:** -```bash -# Generate config template -worker config - -# Edit deploy.yml with your settings - -# Run with automatic credential detection -worker run -``` - -**Features:** -- ✅ Automatic credential detection (service account keys, impersonation, workload identity) -- ✅ Zero-config for default file names (`gcp-key.json`, `gcp-credentials.json`) -- ✅ Secure read-only mounts -- ✅ Support for custom credential paths - -See the [worker-deployment README](https://github.com/udx/worker-deployment) for detailed examples. diff --git a/docs/authorization.md b/docs/authorization.md deleted file mode 100644 index b6b4e28d..00000000 --- a/docs/authorization.md +++ /dev/null @@ -1,94 +0,0 @@ -# Worker Authorization - -## Overview - -The UDX Worker supports multiple cloud providers and services through environment-based credential management. - -## When To Use - -Use this when you need to: - -- Provide credentials to the worker container. -- Understand supported providers and formats. - -## Key Concepts - -- Credentials can be provided via env vars. -- Secrets can be JSON, Base64, or file paths. -- Secret references are resolved from provider paths in `worker.yaml` and matching runtime env vars. -- Authorization cleanup is controlled by `ACTORS_CLEANUP` (enabled by default). - -## Examples - -### Supported Providers - -| Provider | Environment Variable | Description | -| -------- | -------------------- | --------------------------------- | -| Azure | `AZURE_CREDS` | Azure cloud credentials | -| AWS | `AWS_CREDS` | Amazon Web Services credentials | -| GCP | `GCP_CREDS` | Google Cloud Platform credentials | - -### JSON Format - -```json -{ - "clientId": "CLIENT_ID", - "clientSecret": "CLIENT_SECRET", - "tenantId": "TENANT_ID", - "subscriptionId": "SUBSCRIPTION_ID" -} -``` - -### Base64 Format - -```bash -echo -n '{"clientId":"CLIENT_ID","clientSecret":"CLIENT_SECRET","tenantId":"TENANT_ID","subscriptionId":"SUBSCRIPTION_ID"}' | base64 -``` - -### File Path - -```bash -AZURE_CREDS="/path/to/azure_credentials.json" -``` - -### Provider-Specific Credential Keys - -- Azure (`AZURE_CREDS`): `clientId`, `clientSecret`, `tenantId`, `subscriptionId` -- AWS (`AWS_CREDS`): `AccessKeyId`, `SecretAccessKey`, optional `SessionToken` -- GCP (`GCP_CREDS`): standard GCP credential JSON (service account or other `gcloud --cred-file` compatible JSON) - -### Security Scenario: Long-Lived Credentials - -Example concern: -- A static service principal secret is stored in CI and reused for months. -- If leaked, an attacker can keep resolving secrets from the same vault scope until rotation. - -How worker design can mitigate: -- Fetch only referenced secrets at startup (for example `azure//`). -- Remove local auth artifacts after setup when `ACTORS_CLEANUP=true`. -- Override secret references per environment at deploy time for safer rotation workflows. - -How worker design can exacerbate: -- Resolved secrets are exported to process environment and can live for the container lifetime. -- Services can leak secrets if scripts print env vars or run with verbose shell tracing. -- A single broad credential can unlock multiple vault scopes in one worker instance. - -### Recommended Credential Posture - -- Prefer short-lived credentials or federation-based identity flows over long-lived static secrets. -- Grant least-privilege access to only the required secret scopes. -- Rotate provider credentials and secret values regularly. -- Split high-trust workloads into separate worker deployments when hard isolation is required. - -## Common Pitfalls - -- Using relative credential paths in production. -- Storing secrets in version control. -- Using long-lived provider credentials with broad access scope. -- Reusing one credential principal for unrelated services that require isolation. - -## Related Docs - -- `docs/runtime/config.md` -- `docs/deploy/README.md` -- `docs/auth/README.md` diff --git a/docs/development/child-images.md b/docs/child-images.md similarity index 62% rename from docs/development/child-images.md rename to docs/child-images.md index 7890eb23..287bd539 100644 --- a/docs/development/child-images.md +++ b/docs/child-images.md @@ -20,28 +20,16 @@ Avoid a child image when: ## Key Concepts - Child images extend `usabilitydynamics/udx-worker`. -- `worker gen` creates scaffolding to get started quickly. +- Child image scaffolding can be created manually with a Dockerfile that extends the base worker image. ## Examples -### Generate Scaffolding - -```bash -npm install -g @udx/worker-deployment - -# Generate a child image repo skeleton (dry-run + prompt) -worker gen repo - -# Generate a Dockerfile only (dry-run + prompt) -worker gen dockerfile -``` - ### Minimal Workflow -1. Generate a repo or Dockerfile. +1. Create a Dockerfile. 2. Add dependencies. 3. Build and tag the image. -4. Deploy using `deploy.yml`. +4. Run it with Docker, Kubernetes, or CI/CD. Example Dockerfile: @@ -56,21 +44,20 @@ Build: docker build -t my-org/udx-worker-custom:latest . ``` -Deploy (excerpt): +Run: -```yaml -kind: workerDeployConfig -version: udx.io/worker-v1/deploy -config: - image: "my-org/udx-worker-custom:latest" +```bash +docker run --rm \ + -v "$(pwd)/.config/worker:/home/udx/.config/worker:ro" \ + my-org/udx-worker-custom:latest ``` ## Common Pitfalls - Baking secrets into the image. -- Forgetting to update `deploy.yml` with the child image. +- Forgetting to update the host deployment image reference. ## Related Docs -- `docs/deploy/worker-deployment.md` -- `docs/reference/container-structure.md` +- `docs/deployment.md` +- `docs/references/container-structure.md` diff --git a/docs/reference/cli.md b/docs/cli.md similarity index 67% rename from docs/reference/cli.md rename to docs/cli.md index 58350d86..71b9f1f0 100644 --- a/docs/reference/cli.md +++ b/docs/cli.md @@ -14,17 +14,21 @@ Use the CLI when you need to: ## Key Concepts -- Commands are namespaced (`worker service`, `worker env`, `worker auth`). +- Commands are namespaced (`worker service`, `worker env`, `worker health`, `worker sbom`, `worker config`). - Most commands provide help when run without arguments. +- `worker env reload` and `worker config apply` rerun the same config/env/secret resolution path used by the entrypoint. ## Examples ```bash -# Show auth command help -worker auth - # Show service command help worker service + +# Inspect resolved environment +worker env status + +# Re-apply worker.yaml after provider auth has been established +worker env reload ``` ## Common Pitfalls @@ -34,5 +38,5 @@ worker service ## Related Docs -- `docs/runtime/services.md` -- `docs/runtime/config.md` +- `docs/services.md` +- `docs/config.md` diff --git a/docs/config.md b/docs/config.md new file mode 100644 index 00000000..472d8205 --- /dev/null +++ b/docs/config.md @@ -0,0 +1,94 @@ +# Worker Config (`worker.yaml`) + +## Overview + +`worker.yaml` is the primary runtime configuration file. It defines environment variables, secret references, and opt-in runtime output used inside the worker container. + +## When To Use + +Use this when you need to: + +- Define runtime environment variables. +- Reference secrets that should be resolved at startup. +- Override defaults at runtime without rebuilding images. + +## Key Concepts + +- Runtime-only config: `/home/udx/.config/worker/worker.yaml`. +- Deployment env vars override `worker.yaml` values. +- Secret reference behavior is documented in `docs/secrets.md`. +- Provider authentication is not configured in `worker.yaml`. + +## Examples + +### Basic + +```yaml +kind: workerConfig +version: udx.io/worker-v1/config +config: + env: + APP_MODE: "worker" + AWS_REGION: "us-west-2" + secrets: + DB_PASSWORD: "aws/db-password/us-west-2" + API_KEY: "azure/kv-prod/api-key" +``` + +### Static Env and Secret References + +```yaml +kind: workerConfig +version: udx.io/worker-v1/config +config: + env: + API_KEY: "dev-only-static-key" + secrets: + DB_PASSWORD: "azure/kv-prod/db-password" +``` + +`API_KEY` is injected as-is. `DB_PASSWORD` is resolved from the provider after auth exists and then exported as an environment variable. + +### Runtime Environment and Precedence + +1. **Deployment environment variables** (highest priority) +2. Deployment environment variables containing secret references (resolved at startup) +3. `worker.yaml` `config.secrets` +4. `worker.yaml` `config.env` + +Example override: + +```yaml +# worker.yaml (production defaults) +config: + secrets: + ES_PASSWORD: "gcp/prod-project/es-password" +``` + +Deployment override example: + +```bash +docker run --rm \ + -e ES_PASSWORD="gcp/staging-project/es-password" \ + usabilitydynamics/udx-worker:latest +``` + +## Common Pitfalls + +- Storing plaintext secrets in `worker.yaml`. +- Putting cloud login/session setup in `worker.yaml`. +- Forgetting that deployment env vars override runtime config. + +## Runtime Output + +By default the worker does not print runtime config details or write output files. The entrypoint logs a short hint that output can be enabled. + +Set `WORKER_OUTPUT_FILE` when a deployment or workflow needs runtime config evidence. The worker writes redacted JSON runtime metadata to that path. + +Workflow-specific outputs such as `$GITHUB_OUTPUT`, `$GITHUB_STEP_SUMMARY`, or platform annotations should be generated by the workflow from the JSON file. + +## Related Docs + +- `docs/services.md` +- `docs/secrets.md` +- `docs/deployment.md` diff --git a/docs/development/core-image.md b/docs/core-image.md similarity index 84% rename from docs/development/core-image.md rename to docs/core-image.md index e889ebff..95c3e471 100644 --- a/docs/development/core-image.md +++ b/docs/core-image.md @@ -56,7 +56,7 @@ make log FOLLOW_LOGS=true make test ``` -The test target mounts `src/tests` and example configs into the container and runs `/home/udx/tests/main.sh`. +The test target mounts `test` and example configs into the container and runs `/home/udx/test/main.sh`. ## Common Pitfalls @@ -65,5 +65,5 @@ The test target mounts `src/tests` and example configs into the container and ru ## Related Docs -- `docs/runtime/services.md` -- `docs/runtime/config.md` +- `docs/services.md` +- `docs/config.md` diff --git a/docs/deploy/README.md b/docs/deploy/README.md deleted file mode 100644 index 59e96a94..00000000 --- a/docs/deploy/README.md +++ /dev/null @@ -1,34 +0,0 @@ -# Deployment - -## Overview - -Deployment is external to the worker container. This directory covers how to run the worker image consistently across local machines, CI/CD, and Kubernetes. - -## When To Use - -Use these docs when you need to: - -- Run the worker locally with a consistent config. -- Deploy in CI/CD (GitHub Actions, etc.). -- Mount runtime configs into Kubernetes. - -## Key Concepts - -- `deploy.yml` chooses the image, mounts runtime configs, and defines command/args. -- `worker.yaml` and `services.yaml` are runtime configs inside the container. - -## Examples - -- Local/CI usage: `docs/deploy/worker-deployment.md` -- CI image overrides: `docs/deploy/image-override.md` -- Kubernetes ConfigMaps: `docs/deploy/kubernetes.md` - -## Common Pitfalls - -- Editing runtime configs when you meant to change deployment behavior. -- Hardcoding image tags in CI instead of rendering `deploy.yml`. - -## Related Docs - -- `docs/runtime/config.md` -- `docs/runtime/services.md` diff --git a/docs/deploy/image-override.md b/docs/deploy/image-override.md deleted file mode 100644 index fe1c4e8f..00000000 --- a/docs/deploy/image-override.md +++ /dev/null @@ -1,58 +0,0 @@ -# CI/CD Image Override - -## Overview - -Override the worker image at deploy time by rendering a deploy template with an environment variable. - -## When To Use - -Use this when you: - -- Tag images per build and want to deploy that tag. -- Need to switch images without changing runtime configs. - -## Key Concepts - -- Keep `deploy.template.yml` in repo. -- Render to `deploy.yml` in CI. - -## Examples - -Template (`deploy.template.yml`): - -```yaml -kind: workerDeployConfig -version: udx.io/worker-v1/deploy -config: - image: "${WORKER_IMAGE}" - command: "echo" - args: - - "Hello from CI/CD" -``` - -Render + run: - -```bash -export WORKER_IMAGE="usabilitydynamics/udx-worker:latest" -envsubst < deploy.template.yml > deploy.yml -worker run --config=deploy.yml -``` - -If `envsubst` is unavailable: - -```bash -export WORKER_IMAGE="usabilitydynamics/udx-worker:latest" -sed "s|\${WORKER_IMAGE}|${WORKER_IMAGE}|g" deploy.template.yml > deploy.yml -worker run --config=deploy.yml -``` - -Example directory: `src/examples/deploy-image-override/` - -## Common Pitfalls - -- Committing generated `deploy.yml` with build-specific tags. -- Forgetting to set `WORKER_IMAGE` in CI. - -## Related Docs - -- `docs/deploy/worker-deployment.md` diff --git a/docs/deploy/kubernetes.md b/docs/deploy/kubernetes.md deleted file mode 100644 index 011c54f2..00000000 --- a/docs/deploy/kubernetes.md +++ /dev/null @@ -1,74 +0,0 @@ -# Kubernetes Deployment - -## Overview - -Kubernetes can mount `worker.yaml` and `services.yaml` into the container using ConfigMaps/Secrets. This keeps runtime configuration separate from the image. - -## When To Use - -Use this when you deploy the worker as a Kubernetes Deployment and want environment-specific configuration without rebuilding images. - -## Key Concepts - -- Store runtime configs as ConfigMaps (or Secrets for sensitive data). -- Mount into `/home/udx/.config/worker/`. - -## Examples - -```yaml -apiVersion: v1 -kind: ConfigMap -metadata: - name: udx-worker-config -data: - worker.yaml: | - kind: workerConfig - version: udx.io/worker-v1/config - config: - env: - LOG_LEVEL: "info" - services.yaml: | - kind: workerService - version: udx.io/worker-v1/service - services: - - name: "logger" - command: "bash -c 'echo \"[startup]\"; while true; do echo \"[tick]\"; sleep 5; done'" - autostart: true - ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: udx-worker -spec: - replicas: 1 - selector: - matchLabels: - app: udx-worker - template: - metadata: - labels: - app: udx-worker - spec: - containers: - - name: worker - image: usabilitydynamics/udx-worker:latest - volumeMounts: - - name: worker-config - mountPath: /home/udx/.config/worker - readOnly: true - volumes: - - name: worker-config - configMap: - name: udx-worker-config -``` - -## Common Pitfalls - -- Mounting configs to the wrong path. -- Putting secrets in ConfigMaps instead of Kubernetes Secrets. - -## Related Docs - -- `docs/runtime/config.md` -- `docs/runtime/services.md` diff --git a/docs/deploy/worker-deployment.md b/docs/deploy/worker-deployment.md deleted file mode 100644 index 47240118..00000000 --- a/docs/deploy/worker-deployment.md +++ /dev/null @@ -1,58 +0,0 @@ -# worker-deployment CLI - -## Overview - -`@udx/worker-deployment` standardizes how you run the worker image across different hosts and use cases. It keeps image selection, mounts, and runtime args in a single `deploy.yml` file. - -## When To Use - -Use `worker-deployment` when you want: - -- A consistent local run command (`worker run`). -- The same config to work in CI/CD runners. -- A portable deployment format across laptops and ephemeral hosts. - -## Key Concepts - -- `deploy.yml` is the deployment config. -- `worker.yaml` and `services.yaml` are runtime configs mounted into the container. - -## Examples - -### Quick Start - -```bash -npm install -g @udx/worker-deployment - -# Generate a template -worker config - -# Edit deploy.yml, then run -worker run -``` - -### Minimal Config - -```yaml -kind: workerDeployConfig -version: udx.io/worker-v1/deploy -config: - image: "usabilitydynamics/udx-worker:latest" - volumes: - - "./.config/worker:/home/udx/.config/worker:ro" - command: "echo" - args: - - "Hello from deploy.yml" -``` - -## Common Pitfalls - -- Forgetting to mount `worker.yaml`/`services.yaml` into the container. -- Putting runtime logic in `deploy.yml` instead of `services.yaml`. - -## Related Docs - -- `docs/deploy/README.md` -- `docs/deploy/image-override.md` -- `docs/runtime/config.md` -- `docs/runtime/services.md` diff --git a/docs/deployment.md b/docs/deployment.md new file mode 100644 index 00000000..d39fb1da --- /dev/null +++ b/docs/deployment.md @@ -0,0 +1,54 @@ +# Deployment + +## Overview + +Deployment is external to the worker container. Use the host-native tool for the target environment: `docker run`, Docker Compose, `kubectl apply`, or the CI/CD platform's deployment step. + +## When To Use + +Use this when you need to: + +- Run the worker locally with Docker. +- Deploy in CI/CD (GitHub Actions, etc.). +- Mount runtime configs into a container runtime or orchestrator. + +## Key Concepts + +- Host tooling chooses the image, mounts runtime configs, and defines command/args. +- `worker.yaml` and `services.yaml` are runtime configs inside the container. +- Provider auth should be established by the host/platform or by the command running inside the container. +- Prefer native identity/env/token injection over custom credential volumes where the platform supports it. + +## Examples + +- Docker: + +```bash +docker run --rm \ + -v "$(pwd)/.config/worker:/home/udx/.config/worker:ro" \ + -e API_KEY="gcp/my-project/api-key" \ + usabilitydynamics/udx-worker:latest +``` + +- Docker with a child image: + +```bash +docker run --rm \ + -v "$(pwd)/.config/worker:/home/udx/.config/worker:ro" \ + my-org/udx-worker-custom:latest +``` + +For orchestrators such as Kubernetes, mount `worker.yaml` and `services.yaml` through the platform's normal config/secret primitives and keep deployment manifests outside the worker image. + +## Common Pitfalls + +- Editing runtime configs when you meant to change deployment behavior. +- Baking secrets into images instead of using env vars, mounted files, or Kubernetes Secrets. +- Putting runtime process definitions in host deployment config instead of `services.yaml`. +- Expecting worker deployment logic to create cloud sessions; deployment and auth are external concerns. + +## Related Docs + +- `docs/config.md` +- `docs/services.md` +- `docs/secrets.md` diff --git a/docs/development/README.md b/docs/development.md similarity index 75% rename from docs/development/README.md rename to docs/development.md index 8b09984a..afcc6b4a 100644 --- a/docs/development/README.md +++ b/docs/development.md @@ -17,10 +17,10 @@ Use these docs when you need to: - **Core image**: modify this repo when changing worker behavior. - **Child image**: extend the core image for extra dependencies. -## Examples +## Guides -- Core image workflow: `docs/development/core-image.md` -- Child image workflow: `docs/development/child-images.md` +- Core image workflow: `docs/core-image.md` +- Child image workflow: `docs/child-images.md` ## Common Pitfalls @@ -29,5 +29,5 @@ Use these docs when you need to: ## Related Docs -- `docs/deploy/README.md` -- `docs/reference/container-structure.md` +- `docs/deployment.md` +- `docs/references/container-structure.md` diff --git a/docs/index.md b/docs/index.md deleted file mode 100644 index 76132df4..00000000 --- a/docs/index.md +++ /dev/null @@ -1,34 +0,0 @@ -# UDX Worker Documentation - -## Overview - -This documentation is organized by concern so it is clear what happens **inside** the worker container (runtime) vs what happens **outside** at deployment time. - -## When To Use - -Start here if you need to: - -- Configure runtime behavior (`services.yaml`, `worker.yaml`). -- Deploy the worker image across environments. -- Build the core image or child images. - -## Key Concepts - -- Runtime config lives inside the container. -- Deployment config selects the image and mounts runtime files. - -## Examples - -- Runtime: `docs/runtime/services.md` -- Deployment: `docs/deploy/README.md` -- Development: `docs/development/README.md` - -## Common Pitfalls - -- Mixing runtime config with deployment config. -- Editing the image when you only need a runtime change. - -## Related Docs - -- `docs/runtime/config.md` -- `docs/deploy/worker-deployment.md` diff --git a/docs/references/README.md b/docs/references/README.md new file mode 100644 index 00000000..7ad9476c --- /dev/null +++ b/docs/references/README.md @@ -0,0 +1,6 @@ +# Reference Docs + +Reference docs are stable lookup material for related platform context, best practices, and durable interpretation material. + +- [Cloud Providers Auth](cloud-providers-auth.md) +- [Container Structure](container-structure.md) diff --git a/docs/references/cloud-providers-auth.md b/docs/references/cloud-providers-auth.md new file mode 100644 index 00000000..df578b06 --- /dev/null +++ b/docs/references/cloud-providers-auth.md @@ -0,0 +1,27 @@ +# Cloud Providers Auth + +Cloud auth is intentionally outside the worker runtime. The worker only passes through provider environment variables/files and uses provider CLIs or SDK behavior after auth exists. + +## Options Matrix + +Prefer identity mechanisms that the runtime platform injects without worker-specific credential volumes. + +| Provider | Preferred path | File or volume fallback | Worker role | +|---|---|---|---| +| AWS | ECS task role, EKS Pod Identity, IRSA, or CI federation that exports standard AWS env/token variables | Shared AWS config/credentials files or projected web identity token files | Pass through AWS env/files and resolve secrets only after auth exists. | +| Azure | GitHub OIDC with `azure/login`, managed identity, or AKS workload identity | Azure CLI profile/config directory or projected federated token file when CLI tooling requires it | Pass through Azure env/files and resolve secrets only after auth exists. | +| Google Cloud | Attached service account, GKE/Cloud Run identity, or Workload Identity Federation | ADC or gcloud config files such as `GOOGLE_APPLICATION_CREDENTIALS` / `CLOUDSDK_CONFIG` | Pass through ADC/config and resolve secrets only after auth exists. | + +## Practical Rule + +Use the platform-native identity path first. Use mounted credential files only for local development, legacy tools, or provider CLIs that specifically require file-backed config. + +When auth happens inside the container, run the provider-native auth command first, then run: + +```bash +worker env reload +``` + +## Related Docs + +- `docs/secrets.md` diff --git a/docs/reference/container-structure.md b/docs/references/container-structure.md similarity index 96% rename from docs/reference/container-structure.md rename to docs/references/container-structure.md index 94385ab3..93fc67d2 100644 --- a/docs/reference/container-structure.md +++ b/docs/references/container-structure.md @@ -69,5 +69,5 @@ Python: ## Related Docs -- `docs/development/child-images.md` -- `docs/deploy/README.md` +- `docs/child-images.md` +- `docs/deployment.md` diff --git a/docs/runtime/config.md b/docs/runtime/config.md deleted file mode 100644 index c0c42af2..00000000 --- a/docs/runtime/config.md +++ /dev/null @@ -1,135 +0,0 @@ -# Worker Configuration (`worker.yaml`) - -## Overview - -`worker.yaml` is the primary runtime configuration file. It defines environment variables and secret references used **inside** the worker container. - -## When To Use - -Use this when you need to: - -- Define runtime environment variables. -- Reference secrets from cloud providers. -- Override defaults at runtime without rebuilding images. - -## Key Concepts - -- Runtime-only config: `/home/udx/.config/worker/worker.yaml`. -- Deployment env vars override `worker.yaml` values. -- Secret references use `provider//` format. -- Provider reference formats: - - `azure//` - - `gcp//` - - `aws//` - -## Examples - -### Basic - -```yaml -kind: workerConfig -version: udx.io/worker-v1/config -config: - env: - AZURE_CLIENT_ID: "12345678-1234-1234-1234-1234567890ab" - AWS_REGION: "us-west-2" - secrets: - DB_PASSWORD: "aws/db-password/us-west-2" - API_KEY: "azure/kv-prod/api-key" -``` - -### Static Secret Injection (`API_KEY`) and External Secret Resolution (`DB_PASSWORD`) - -```yaml -kind: workerConfig -version: udx.io/worker-v1/config -config: - env: - API_KEY: "dev-only-static-key" - secrets: - DB_PASSWORD: "azure/kv-prod/db-password" -``` - -`API_KEY` is injected as-is. `DB_PASSWORD` is resolved from the provider and then exported as an environment variable. - -### Secret References in `config.env` - -```yaml -config: - env: - DATABASE_URL: "gcp/my-project/db-connection-string" - API_TOKEN: "azure/kv-prod/api-token" - LOG_LEVEL: "info" -``` - -If an `env` value matches a secret reference format, the worker resolves it at startup. - -### Separate Secret Scopes for Different Services - -Use separate variable names in `worker.yaml`, then consume the right variable in each service: - -```yaml -# worker.yaml -kind: workerConfig -version: udx.io/worker-v1/config -config: - secrets: - SERVICE_A_DB_PASSWORD: "azure/kv-service-a/db-password" - SERVICE_B_DB_PASSWORD: "azure/kv-service-b/db-password" -``` - -```yaml -# services.yaml -kind: workerService -version: udx.io/worker-v1/service -services: - - name: "serviceA" - command: "bash -lc 'exec /home/udx/bin/service_a.sh'" - envs: - - "SERVICE_NAME=serviceA" - - - name: "serviceB" - command: "bash -lc 'exec /home/udx/bin/service_b.sh'" - envs: - - "SERVICE_NAME=serviceB" -``` - -`serviceA` should read `SERVICE_A_DB_PASSWORD`, and `serviceB` should read `SERVICE_B_DB_PASSWORD`. -For strict isolation boundaries, run separate worker instances with separate identities. - -### Runtime Environment and Precedence - -1. **Deployment environment variables** (highest priority) -2. Deployment environment variables containing secret references (resolved at startup) -3. `worker.yaml` `config.secrets` -4. `worker.yaml` `config.env` - -Example override: - -```yaml -# worker.yaml (production defaults) -config: - secrets: - ES_PASSWORD: "gcp/prod-project/es-password" -``` - -```yaml -# Kubernetes deployment (staging override) -spec: - containers: - - name: worker - env: - - name: ES_PASSWORD - value: "gcp/staging-project/es-password" -``` - -## Common Pitfalls - -- Storing plaintext secrets in `worker.yaml`. -- Forgetting that deployment env vars override runtime config. - -## Related Docs - -- `docs/runtime/services.md` -- `docs/deploy/README.md` -- `docs/deploy/kubernetes.md` diff --git a/docs/secrets.md b/docs/secrets.md new file mode 100644 index 00000000..8f9f8844 --- /dev/null +++ b/docs/secrets.md @@ -0,0 +1,90 @@ +# Secrets + +## Overview + +The worker resolves secret references from `worker.yaml` and environment variables after provider auth already exists. It does not log in to cloud providers or manage credential sessions. + +## When To Use + +Use this when you need to: + +- Define secret references in `worker.yaml`. +- Pass secret references through deployment environment variables. +- Re-run secret resolution after a command authenticates inside the container. + +## Key Concepts + +- Secret values are exported into the worker environment file and become available to services. +- `config.secrets` maps environment variable names to provider references. +- `config.env` values that look like provider references are resolved too. +- Deployment env vars override `worker.yaml` and can also contain secret references. +- Provider auth is external: Docker, Kubernetes, CI/CD, workload identity, mounted credentials, or a command inside the container owns login/session setup. + +Supported reference formats: + +- `azure//` +- `gcp//` +- `aws//` + +## Examples + +### `worker.yaml` + +```yaml +kind: workerConfig +version: udx.io/worker-v1/config +config: + secrets: + DB_PASSWORD: "azure/kv-prod/db-password" + API_KEY: "gcp/my-project/api-key" + env: + DATABASE_URL: "aws/database-url/us-west-2" +``` + +### Deployment Env Reference + +```bash +docker run --rm \ + -v "$(pwd)/.config/worker:/home/udx/.config/worker:ro" \ + -e API_KEY="gcp/my-project/api-key" \ + usabilitydynamics/udx-worker:latest +``` + +### Internal Auth Then Re-Resolve + +For development, testing, validation, or runbook workflows, authenticate with provider-native tooling inside the container and then rerun worker resolution: + +```bash +worker env reload +``` + +`worker config apply` is equivalent when the intent is to re-apply `worker.yaml`. + +This is still not a worker login feature. The auth command and credential storage remain owned by the user, child image, workflow, or runbook. + +### Resolve One Reference + +```bash +worker env resolve gcp/my-project/api-key +``` + +## Credential Posture + +- Prefer short-lived credentials, workload identity, or federation over static keys. +- Grant least-privilege access to only the required secret scopes. +- Pass only the provider env vars and mounted files needed by the workload. +- Mount credential directories read-only when files are required. +- Split high-trust workloads into separate worker deployments when isolation matters. + +## Common Pitfalls + +- Expecting built-in provider login commands. +- Storing plaintext secrets in `worker.yaml`. +- Reusing one broad credential principal for unrelated workloads. +- Adding custom credential volumes when platform identity or injected env/token files are available. + +## Related Docs + +- `docs/config.md` +- `docs/services.md` +- `docs/references/cloud-providers-auth.md` diff --git a/docs/runtime/services.md b/docs/services.md similarity index 91% rename from docs/runtime/services.md rename to docs/services.md index 2fae3e05..afab0211 100644 --- a/docs/runtime/services.md +++ b/docs/services.md @@ -1,4 +1,4 @@ -# Service Configuration (`services.yaml`) +# Service Config (`services.yaml`) ## Overview @@ -14,7 +14,7 @@ Use this when you need to: ## Key Concepts - Runtime-only: it lives inside the container at `/home/udx/.config/worker/`. -- Image selection happens at deployment time (see `docs/deploy/README.md`). +- Image selection happens at deployment time (see `docs/deployment.md`). - Each service is configured with a single `command` string (there is no `args` field in `services.yaml`). ## Examples @@ -115,12 +115,12 @@ services: ## Common Pitfalls -- Using `services.yaml` to select the image (use `deploy.yml` instead). +- Using `services.yaml` to select the image; image selection belongs to Docker, Kubernetes, or CI/CD deployment config. - Forgetting to mount `services.yaml` into the container. - Expecting an `args` field in `services.yaml` (put arguments directly in `command`). - Putting provider references (for example `azure/...`) in `services.yaml` `envs`. ## Related Docs -- `docs/runtime/config.md` -- `docs/deploy/README.md` +- `docs/config.md` +- `docs/deployment.md` diff --git a/src/examples/README.md b/src/examples/README.md index 96b3b68e..789cb6cc 100644 --- a/src/examples/README.md +++ b/src/examples/README.md @@ -15,10 +15,3 @@ A set of small service scripts used to demonstrate supervisor behavior: Minimal `worker.yaml` used by tests and quick local runs: - `.config/worker/worker.yaml` - -## deploy-image-override - -Shows how to override the worker image in CI/CD using a deploy template: - -- `deploy.template.yml` -- `README.md` diff --git a/src/examples/deploy-image-override/README.md b/src/examples/deploy-image-override/README.md deleted file mode 100644 index 3f05a663..00000000 --- a/src/examples/deploy-image-override/README.md +++ /dev/null @@ -1,29 +0,0 @@ -# Deploy Image Override (CI/CD) - -This example shows how to override the worker image at deploy time using an environment variable and a deploy template. - -Related docs: `docs/deploy/README.md` - -## Steps - -```bash -cd src/examples/deploy-image-override - -export WORKER_IMAGE="usabilitydynamics/udx-worker:latest" - -# Render deploy.yml from the template -envsubst < deploy.template.yml > deploy.yml - -# Run container using worker-deployment -worker run --config=deploy.yml -``` - -## Notes - -- `services.yaml` controls processes inside the container, not the container image. -- The image is selected in `deploy.yml` (worker-deployment). -- If `envsubst` is unavailable, use: - -```bash -sed "s|\${WORKER_IMAGE}|${WORKER_IMAGE}|g" deploy.template.yml > deploy.yml -``` diff --git a/src/examples/deploy-image-override/deploy.template.yml b/src/examples/deploy-image-override/deploy.template.yml deleted file mode 100644 index 87e7108b..00000000 --- a/src/examples/deploy-image-override/deploy.template.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -kind: workerDeployConfig -version: udx.io/worker-v1/deploy -config: - image: "${WORKER_IMAGE}" - command: "echo" - args: - - "Hello from CI/CD" diff --git a/src/examples/simple-service/README.md b/src/examples/simple-service/README.md index 44044644..1596add6 100644 --- a/src/examples/simple-service/README.md +++ b/src/examples/simple-service/README.md @@ -2,7 +2,7 @@ These scripts demonstrate common service behaviors for `services.yaml`. -Related docs: `docs/runtime/services.md` +Related docs: `docs/services.md` ## Scripts From a501d2d1aaab1272acdf7b101a3e4d73349b6de4 Mon Sep 17 00:00:00 2001 From: Dmytro Smirnov Date: Sun, 31 May 2026 12:28:08 +0300 Subject: [PATCH 03/32] fix: include runtime configs in docker build context --- .dockerignore | 17 +++++--- .rabbit/context.yaml | 101 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 111 insertions(+), 7 deletions(-) create mode 100644 .rabbit/context.yaml diff --git a/.dockerignore b/.dockerignore index c3127dc1..5059a361 100644 --- a/.dockerignore +++ b/.dockerignore @@ -7,8 +7,10 @@ .gitignore .gitattributes -# CI/CD and GitHub-related files +# CI/CD and repo metadata ci/* +.github/* +.rabbit/ # macOS system files .DS_Store @@ -21,12 +23,12 @@ ci/* .vscode/ *.swp *.swo -.github/* -# Documentation files +# Documentation and development-only files docs/* README.md SECURITY.md +test/ # Local development scripts and helper files Makefile @@ -40,11 +42,12 @@ Makefile.variables *.bak *.old -# Ignore test configuration files +# Runtime defaults copied by Dockerfile live under src/configs. +# Keep the rest of src out of the image build context. src/* +!src/configs/ +!src/configs/** -# Ignore IDE specific and Prettier configuration files -.vscode/ +# Formatter/editor local files *.iml .prettierignore -.DS_Store diff --git a/.rabbit/context.yaml b/.rabbit/context.yaml new file mode 100644 index 00000000..80914ee1 --- /dev/null +++ b/.rabbit/context.yaml @@ -0,0 +1,101 @@ +# Generated by dev.kit repo — do not edit manually. +# Run `dev.kit repo` to refresh. +kind: repoContext +version: udx.dev/dev.kit/v1 +generator: + tool: dev.kit + repo: https://github.com/udx/dev.kit + version: 0.13.0 + generated_at: 2026-05-31T09:26:48Z + sources: + homepage: https://udx.dev/kit + repository: https://github.com/udx/dev.kit + package: https://www.npmjs.com/package/@udx/dev-kit + installation: https://github.com/udx/dev.kit/blob/latest/docs/installation.md + +repo: + name: worker + archetype: manifest-repo + +# Refs — Direct-read files and paths that define the repo contract. +# Note: Include only files or directories a repo consumer should read before code exploration. +# Note: Prefer README, focused docs, workflows, manifests, and explicit operational files. +# Note: Exclude broad implementation directories unless they are the contract themselves. + +refs: + - ./README.md + - ./Makefile + - ./docs/child-images.md + - ./docs/core-image.md + - ./src/configs/services.yaml + - ./src/configs/worker.yaml + - ./.github/workflows + - ./Dockerfile + - ./docs + +# Commands — Canonical repo entrypoints detected from strong repo signals. +# Note: Prefer declared make targets and package scripts before regex matches in docs. +# Note: Emit only commands that can be traced to a concrete source. +# Note: Record the source path so the command can be reviewed and corrected. + +commands: + verify: + run: make test + source: Makefile + build: + run: make build + source: Makefile + run: + run: make run + source: Makefile + +# Dependencies — Meaningful dependency-repo contracts such as reusable workflows, images, or versioned manifests this repo relies on. +# Note: Capture execution-shaping behavior defined outside the current checkout. +# Note: Avoid promoting standard package inventory or ordinary GitHub action refs into top-level context. +# Note: Normalize same-org versioned refs into repo slugs when possible. + +dependencies: + - repo: udx/reusable-workflows + kind: reusable workflow + resolved: true + archetype: workflow-repo + description: Reusable GitHub Actions workflow templates for CI/CD + used_by: + - .github/workflows/context7_sync.yml + - .github/workflows/docker-ops.yml + - repo: ubuntu:25.10 + kind: base image + resolved: false + used_by: + - Dockerfile + +# Manifests — YAML files that define repo-specific workflow, deploy, or contract behavior. +# Note: Include custom config/manifests that materially shape repo behavior or contract understanding. +# Note: Do not include workflow YAML only because it lives under .github/workflows. +# Note: Promote workflow files only when they declare reusable workflow refs or other repo-specific execution contracts. +# Note: Prefer structured kind and description metadata from the manifest itself. +# Note: Include hidden or nested contract dirs when they contain repo-owned manifests with meaningful metadata. + +manifests: + - path: .github/workflows/context7_sync.yml + kind: githubWorkflow + - path: .github/workflows/docker-ops.yml + kind: githubWorkflow + - path: src/configs/services.yaml + kind: workerService + declared_as: udx.io/worker-v1/service + source_repo: udx/worker + used_by: + - Dockerfile + evidence: + - version: udx.io/worker-v1/service + - path reference: Dockerfile + - path: src/configs/worker.yaml + kind: workerConfig + declared_as: udx.io/worker-v1/config + source_repo: udx/worker + used_by: + - Dockerfile + evidence: + - version: udx.io/worker-v1/config + - path reference: Dockerfile From f66f38617ad995f32c489e96ca6f466490b47b74 Mon Sep 17 00:00:00 2001 From: Dmytro Smirnov Date: Sun, 31 May 2026 12:29:13 +0300 Subject: [PATCH 04/32] ci: run docker ops for dockerignore changes --- .github/workflows/docker-ops.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docker-ops.yml b/.github/workflows/docker-ops.yml index 4d06c3cc..7fa23b49 100644 --- a/.github/workflows/docker-ops.yml +++ b/.github/workflows/docker-ops.yml @@ -7,6 +7,7 @@ name: Docker Ops - "**" paths: - ".github/workflows/docker-ops.yml" + - ".dockerignore" - "Dockerfile" - "bin/**" - "lib/**" From 257bbe4b110cbbb061e0bbdffb73a5016a559fdd Mon Sep 17 00:00:00 2001 From: Dmytro Smirnov Date: Tue, 2 Jun 2026 11:27:19 +0300 Subject: [PATCH 05/32] ci: add copilot docker dependency updater --- .../workflows/docker-dependency-updater.yml | 191 ++++++++++++++++ ci/docker-dependency-probe.sh | 205 ++++++++++++++++++ 2 files changed, 396 insertions(+) create mode 100644 .github/workflows/docker-dependency-updater.yml create mode 100755 ci/docker-dependency-probe.sh diff --git a/.github/workflows/docker-dependency-updater.yml b/.github/workflows/docker-dependency-updater.yml new file mode 100644 index 00000000..b56a9dca --- /dev/null +++ b/.github/workflows/docker-dependency-updater.yml @@ -0,0 +1,191 @@ +--- +name: Docker Dependency Updater + +"on": + schedule: + - cron: "0 5 * * 1" + workflow_dispatch: + +permissions: + contents: read + +jobs: + build-dependency-report: + runs-on: ubuntu-24.04 + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Build unpinned dependency probe + run: | + chmod +x ci/docker-dependency-probe.sh + ci/docker-dependency-probe.sh \ + --dockerfile Dockerfile \ + --image worker-deps-probe:${{ github.run_id }} \ + --report docker-dependency-report.json + + - name: Upload dependency report + uses: actions/upload-artifact@v5 + with: + name: docker-dependency-report + path: docker-dependency-report.json + + update-with-copilot: + needs: build-dependency-report + runs-on: ubuntu-24.04 + + permissions: + contents: write + pull-requests: write + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + with: + ref: ${{ github.event.repository.default_branch }} + + - name: Stop if an update PR is already open + id: existing-pr + uses: actions/github-script@v8 + with: + script: | + const owner = context.repo.owner; + const repo = context.repo.repo; + const head = `${owner}:automation/docker-dependency-updates`; + + const prs = await github.rest.pulls.list({ + owner, + repo, + state: "open", + head, + per_page: 100 + }); + + core.setOutput("found", prs.data.length > 0 ? "true" : "false"); + if (prs.data.length > 0) { + core.notice(`Update PR already open: ${prs.data[0].html_url}`); + } + + - name: Download dependency report + if: steps.existing-pr.outputs.found != 'true' + uses: actions/download-artifact@v6 + with: + name: docker-dependency-report + + - name: Set up Node.js + if: steps.existing-pr.outputs.found != 'true' + uses: actions/setup-node@v6 + with: + node-version: "22" + + - name: Install Copilot CLI + if: steps.existing-pr.outputs.found != 'true' + run: npm install -g @github/copilot + + - name: Update Dockerfile with Copilot CLI + if: steps.existing-pr.outputs.found != 'true' + env: + COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} + COPILOT_ALLOW_ALL: "true" + COPILOT_AUTO_UPDATE: "false" + run: | + set -euo pipefail + + if [ -z "${COPILOT_GITHUB_TOKEN:-}" ]; then + echo "COPILOT_GITHUB_TOKEN secret is required. Use a fine-grained token with Copilot Requests permission." >&2 + exit 1 + fi + + cat > /tmp/docker-dependency-updater-prompt.md <<'EOF' + You are updating Dockerfile dependency pins for this repository. + + Intent: + - Read Dockerfile and docker-dependency-report.json. + - The report was produced by building a temporary copy of Dockerfile with dependency version pins removed. + - Treat the report as the primary evidence for versions that install successfully for the current base image. + - Update only Dockerfile dependency pins and ARG values when the report shows a newer installed version. + - Preserve the ubuntu base image tag unless explicitly necessary to make the reported versions valid. + - Keep AWS CLI unpinned unless Dockerfile already pins it; mention its latest observed version in the changelog only. + - Do not edit workflow files, docs, tests, or application code. + - Do not commit, push, or create a pull request; this workflow will do that. + + Expected Dockerfile update categories: + - apt package pins from dependencies.apt[].installed + - PIP_VERSION from dependencies.pip[name="pip"].installed + - AZURE_CLI_VERSION from dependencies.pip[name="azure-cli"].installed + - YQ_VERSION from dependencies.github_releases[name="yq"].installed + - GCLOUD_VERSION from dependencies.archives[name="google-cloud-sdk"].installed + + After editing: + - Print a concise changelog of every version change. + - Print any dependency that was observed but intentionally not pinned. + EOF + + copilot \ + --prompt "$(cat /tmp/docker-dependency-updater-prompt.md)" \ + --attachment docker-dependency-report.json \ + --allow-all-tools \ + --allow-all-urls \ + --no-ask-user \ + --no-auto-update \ + --silent \ + --share copilot-docker-dependency-session.md + + - name: Detect changes + if: steps.existing-pr.outputs.found != 'true' + id: changes + shell: bash + run: | + set -euo pipefail + if git diff --quiet -- Dockerfile; then + echo "changed=false" >> "${GITHUB_OUTPUT}" + else + echo "changed=true" >> "${GITHUB_OUTPUT}" + git diff -- Dockerfile > docker-dependency-update.diff + fi + + - name: Validate Docker build + if: steps.existing-pr.outputs.found != 'true' && steps.changes.outputs.changed == 'true' + run: docker build --progress=plain -t dependency-update-validation . + + - name: Prepare pull request body + if: steps.existing-pr.outputs.found != 'true' && steps.changes.outputs.changed == 'true' + run: | + { + echo "## Summary" + echo + echo "Updates Dockerfile dependency pins using an unpinned probe build and Copilot CLI." + echo + echo "## Evidence" + echo + echo "- Probe report artifact: \`docker-dependency-report.json\`" + echo "- Copilot session artifact: \`copilot-docker-dependency-session.md\`" + echo "- Validation: \`docker build --progress=plain -t dependency-update-validation .\`" + echo + echo "## Diff" + echo + echo '```diff' + sed -n '1,220p' docker-dependency-update.diff + echo '```' + } > docker-dependency-pr-body.md + + - name: Upload Copilot session + if: steps.existing-pr.outputs.found != 'true' + uses: actions/upload-artifact@v5 + with: + name: copilot-docker-dependency-session + path: | + copilot-docker-dependency-session.md + docker-dependency-update.diff + if-no-files-found: ignore + + - name: Create pull request + if: steps.existing-pr.outputs.found != 'true' && steps.changes.outputs.changed == 'true' + uses: peter-evans/create-pull-request@v7 + with: + commit-message: "chore(deps): update Docker dependency pins" + title: "chore(deps): update Docker dependency pins" + body-path: docker-dependency-pr-body.md + branch: automation/docker-dependency-updates + delete-branch: true diff --git a/ci/docker-dependency-probe.sh b/ci/docker-dependency-probe.sh new file mode 100755 index 00000000..2cef151f --- /dev/null +++ b/ci/docker-dependency-probe.sh @@ -0,0 +1,205 @@ +#!/usr/bin/env bash +set -euo pipefail + +dockerfile="${DOCKERFILE:-Dockerfile}" +image="${PROBE_IMAGE:-worker-deps-probe:${GITHUB_RUN_ID:-local}}" +report="${PROBE_REPORT:-docker-dependency-report.json}" + +usage() { + cat <<'EOF' +Usage: ci/docker-dependency-probe.sh [--dockerfile Dockerfile] [--image tag] [--report path] + +Build a temporary Dockerfile with dependency pins removed, then report the +versions that the image actually installs. +EOF +} + +while [ "$#" -gt 0 ]; do + case "$1" in + --dockerfile) + dockerfile="$2" + shift 2 + ;; + --image) + image="$2" + shift 2 + ;; + --report) + report="$2" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "Unknown argument: $1" >&2 + usage >&2 + exit 2 + ;; + esac +done + +if [ ! -f "${dockerfile}" ]; then + echo "Dockerfile not found: ${dockerfile}" >&2 + exit 1 +fi + +tmpdir="$(mktemp -d)" +probe_dockerfile="${tmpdir}/Dockerfile.unpinned-probe" +trap 'rm -rf "${tmpdir}"' EXIT + +base_image="$(awk '$1 == "FROM" { print $2; exit }' "${dockerfile}")" + +mapfile -t apt_packages < <( + awk ' + /apt-get install -y --no-install-recommends/ { in_block=1; next } + in_block { + line=$0 + gsub(/\\/, "", line) + gsub(/^[[:space:]]+/, "", line) + if (line ~ /^[[:alnum:].+-]+=/) { + split(line, parts, "=") + print parts[1] + } + if ($0 ~ /&&[[:space:]]*\\?$/) { + in_block=0 + } + } + ' "${dockerfile}" +) + +awk ' + /^ARG (AZURE_CLI_VERSION|PIP_VERSION|YQ_VERSION|GCLOUD_VERSION)=/ { + next + } + + /^# Install yq/ { + block = "yq" + print + next + } + + /^# Install Google Cloud SDK/ { + block = "gcloud" + gcloud_inserted = 0 + print + next + } + + /^# Install AWS CLI/ { + block = "aws" + print + next + } + + /^#/ && $0 !~ /^# Install/ { + block = "" + print + next + } + + { + line = $0 + + if (line ~ /apt-get install -y --no-install-recommends/) { + in_apt = 1 + } else if (in_apt && line ~ /^[[:space:]]+[A-Za-z0-9.+-]+=/) { + sub(/=[^[:space:]]+/, "", line) + } + + if (line ~ /pip install .*--upgrade pip==\$\{PIP_VERSION\}/) { + sub(/==\$\{PIP_VERSION\}/, "", line) + } + + if (line ~ /pip install .*azure-cli==\$\{AZURE_CLI_VERSION\}/) { + sub(/==\$\{AZURE_CLI_VERSION\}/, "", line) + } + + if (block == "yq" && line ~ /github.com\/mikefarah\/yq\/releases\/download/) { + print " YQ_VERSION=\"$(curl -fsSL https://api.github.com/repos/mikefarah/yq/releases/latest | jq -r .tag_name)\" && \\" + sub(/download\/v\$\{YQ_VERSION\}/, "download/${YQ_VERSION}", line) + } + + if (block == "gcloud" && gcloud_inserted == 0 && line ~ /^RUN ARCH=\$\(uname -m\) && \\$/) { + print line + print " GCLOUD_VERSION=\"$(curl -fsSL https://dl.google.com/dl/cloudsdk/channels/rapid/components-2.json | jq -r .version)\" && \\" + gcloud_inserted = 1 + next + } + + print line + + if (in_apt && line ~ /&&[[:space:]]*\\?$/) { + in_apt = 0 + } + } +' "${dockerfile}" > "${probe_dockerfile}" + +if [ "${PROBE_SKIP_BUILD:-false}" = "true" ]; then + echo "Skipping probe image build and using existing image: ${image}" >&2 +else + echo "Building dependency probe image: ${image}" >&2 + docker build --pull --no-cache -f "${probe_dockerfile}" -t "${image}" . +fi + +echo "Collecting dependency inventory from: ${image}" >&2 +docker run -i --rm --entrypoint bash "${image}" -s -- "${base_image}" "${apt_packages[@]}" > "${report}" <<'EOF' +set -euo pipefail + +base_image="$1" +shift + +apt_json="$( + dpkg-query -W -f='${binary:Package}\t${Version}\n' "$@" \ + | jq -R -s ' + split("\n") + | map(select(length > 0)) + | map(split("\t") | {name: .[0], installed: .[1]}) + ' +)" + +os_pretty="$(. /etc/os-release && printf '%s' "${PRETTY_NAME}")" +pip_version="$(/opt/az/bin/pip --version | awk '{ print $2 }')" +azure_cli_version="$(/opt/az/bin/pip show azure-cli | awk -F': ' '$1 == "Version" { print $2 }')" +yq_version="$(yq --version | awk '{ print $NF }' | sed 's/^v//')" +gcloud_version="$(gcloud version --format=json | jq -r '."Google Cloud SDK"')" +aws_cli_version="$(aws --version 2>&1 | awk '{ print $1 }' | sed 's#aws-cli/##')" + +jq -n \ + --arg generated_at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + --arg base_image "${base_image}" \ + --arg os "${os_pretty}" \ + --arg arch "$(uname -m)" \ + --arg pip "${pip_version}" \ + --arg azure_cli "${azure_cli_version}" \ + --arg yq "${yq_version}" \ + --arg gcloud "${gcloud_version}" \ + --arg aws_cli "${aws_cli_version}" \ + --argjson apt "${apt_json}" \ + '{ + generated_at: $generated_at, + method: "unpinned Dockerfile probe build", + base_image: $base_image, + runtime: { + os: $os, + architecture: $arch + }, + dependencies: { + apt: $apt, + pip: [ + {name: "pip", installed: $pip}, + {name: "azure-cli", installed: $azure_cli} + ], + github_releases: [ + {name: "yq", installed: $yq, source: "mikefarah/yq"} + ], + archives: [ + {name: "google-cloud-sdk", installed: $gcloud, source: "Google Cloud SDK rapid channel"}, + {name: "aws-cli", installed: $aws_cli, source: "awscli.amazonaws.com latest zip", pinned_in_dockerfile: false} + ] + } + }' +EOF + +echo "Dependency report written to ${report}" >&2 From 4444067f25832d7ce73553dc6861929f845a80fd Mon Sep 17 00:00:00 2001 From: Dmytro Smirnov Date: Tue, 2 Jun 2026 12:58:33 +0300 Subject: [PATCH 06/32] ci: temporarily test docker dependency updater on pr branch --- .github/workflows/docker-dependency-updater.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker-dependency-updater.yml b/.github/workflows/docker-dependency-updater.yml index b56a9dca..1c391ab3 100644 --- a/.github/workflows/docker-dependency-updater.yml +++ b/.github/workflows/docker-dependency-updater.yml @@ -4,6 +4,13 @@ name: Docker Dependency Updater "on": schedule: - cron: "0 5 * * 1" + push: + branches: + - cleanup-worker-runtime-docs + paths: + - ".github/workflows/docker-dependency-updater.yml" + - "ci/docker-dependency-probe.sh" + - "Dockerfile" workflow_dispatch: permissions: @@ -43,7 +50,7 @@ jobs: - name: Checkout repository uses: actions/checkout@v6 with: - ref: ${{ github.event.repository.default_branch }} + ref: ${{ github.ref }} - name: Stop if an update PR is already open id: existing-pr From 8db710301583d9b596194c7a0431f6c357c8c844 Mon Sep 17 00:00:00 2001 From: Dmytro Smirnov Date: Tue, 2 Jun 2026 13:02:08 +0300 Subject: [PATCH 07/32] ci: inline dependency report for copilot cli --- .github/workflows/docker-dependency-updater.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker-dependency-updater.yml b/.github/workflows/docker-dependency-updater.yml index 1c391ab3..8acc0bf2 100644 --- a/.github/workflows/docker-dependency-updater.yml +++ b/.github/workflows/docker-dependency-updater.yml @@ -127,11 +127,20 @@ jobs: After editing: - Print a concise changelog of every version change. - Print any dependency that was observed but intentionally not pinned. + + Dependency report: EOF + { + echo + echo '```json' + cat docker-dependency-report.json + echo + echo '```' + } >> /tmp/docker-dependency-updater-prompt.md + copilot \ --prompt "$(cat /tmp/docker-dependency-updater-prompt.md)" \ - --attachment docker-dependency-report.json \ --allow-all-tools \ --allow-all-urls \ --no-ask-user \ From 0ac47e80f0a4ef46bbf560adf458755c23e62f8c Mon Sep 17 00:00:00 2001 From: Dmytro Smirnov Date: Tue, 2 Jun 2026 13:10:13 +0300 Subject: [PATCH 08/32] ci: limit dependency pr contents to dockerfile --- .github/workflows/docker-dependency-updater.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docker-dependency-updater.yml b/.github/workflows/docker-dependency-updater.yml index 8acc0bf2..1cdedcef 100644 --- a/.github/workflows/docker-dependency-updater.yml +++ b/.github/workflows/docker-dependency-updater.yml @@ -204,4 +204,5 @@ jobs: title: "chore(deps): update Docker dependency pins" body-path: docker-dependency-pr-body.md branch: automation/docker-dependency-updates + add-paths: Dockerfile delete-branch: true From ea342059accc83cdc0b0b7556c2364d031d02b33 Mon Sep 17 00:00:00 2001 From: Dmytro Smirnov Date: Tue, 2 Jun 2026 13:13:08 +0300 Subject: [PATCH 09/32] ci: remove temporary dependency updater push trigger --- .github/workflows/docker-dependency-updater.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/workflows/docker-dependency-updater.yml b/.github/workflows/docker-dependency-updater.yml index 1cdedcef..d3fe6d52 100644 --- a/.github/workflows/docker-dependency-updater.yml +++ b/.github/workflows/docker-dependency-updater.yml @@ -4,13 +4,6 @@ name: Docker Dependency Updater "on": schedule: - cron: "0 5 * * 1" - push: - branches: - - cleanup-worker-runtime-docs - paths: - - ".github/workflows/docker-dependency-updater.yml" - - "ci/docker-dependency-probe.sh" - - "Dockerfile" workflow_dispatch: permissions: From 17b5db5834283ad753f7a16766d72997a012231b Mon Sep 17 00:00:00 2001 From: Dmytro Smirnov Date: Tue, 2 Jun 2026 14:09:50 +0300 Subject: [PATCH 10/32] ci: inline docker dependency probe workflow --- .../workflows/docker-dependency-updater.yml | 452 +++++++++++++++--- ci/configs/docker-dependency-probe.yaml | 86 ++++ ci/docker-dependency-probe.sh | 205 -------- ci/prompts/docker-dependency-updater.md | 23 + 4 files changed, 492 insertions(+), 274 deletions(-) create mode 100644 ci/configs/docker-dependency-probe.yaml delete mode 100755 ci/docker-dependency-probe.sh create mode 100644 ci/prompts/docker-dependency-updater.md diff --git a/.github/workflows/docker-dependency-updater.yml b/.github/workflows/docker-dependency-updater.yml index d3fe6d52..a10785a2 100644 --- a/.github/workflows/docker-dependency-updater.yml +++ b/.github/workflows/docker-dependency-updater.yml @@ -10,49 +10,77 @@ permissions: contents: read jobs: - build-dependency-report: + configure: runs-on: ubuntu-24.04 + permissions: + contents: read + pull-requests: read + + outputs: + config_path: ${{ steps.config.outputs.config_path }} + dockerfile: ${{ steps.config.outputs.dockerfile }} + existing_pr_found: ${{ steps.existing-pr.outputs.found }} + image: ${{ steps.config.outputs.image }} + prompt_path: ${{ steps.config.outputs.prompt_path }} + probe_dockerfile: ${{ steps.config.outputs.probe_dockerfile }} + report_path: ${{ steps.config.outputs.report_path }} + update_branch: ${{ steps.config.outputs.update_branch }} + pr_title: ${{ steps.config.outputs.pr_title }} + commit_message: ${{ steps.config.outputs.commit_message }} + validation_command: ${{ steps.config.outputs.validation_command }} + should_run: ${{ steps.decision.outputs.should_run }} + skip_reason: ${{ steps.decision.outputs.skip_reason }} + steps: - name: Checkout repository uses: actions/checkout@v6 - - name: Build unpinned dependency probe + - name: Load dependency updater config + id: config run: | - chmod +x ci/docker-dependency-probe.sh - ci/docker-dependency-probe.sh \ - --dockerfile Dockerfile \ - --image worker-deps-probe:${{ github.run_id }} \ - --report docker-dependency-report.json + set -euo pipefail - - name: Upload dependency report - uses: actions/upload-artifact@v5 - with: - name: docker-dependency-report - path: docker-dependency-report.json + config_path="ci/configs/docker-dependency-probe.yaml" - update-with-copilot: - needs: build-dependency-report - runs-on: ubuntu-24.04 + ruby -ryaml -e ' + config = YAML.load_file(ARGV.fetch(0)) || {} + workflow = config["workflow"] || {} + paths = config["paths"] || {} + validation = Array(config["validation"]) - permissions: - contents: write - pull-requests: write + outputs = { + "config_path" => ARGV.fetch(0), + "dockerfile" => config["dockerfile"] || "Dockerfile", + "image" => "worker-deps-probe:#{ENV.fetch("GITHUB_RUN_ID")}", + "prompt_path" => paths["prompt"] || "ci/prompts/docker-dependency-updater.md", + "probe_dockerfile" => paths["probe_dockerfile"] || ".tmp/dependency-probe/Dockerfile", + "report_path" => paths["report"] || "docker-dependency-report.json", + "update_branch" => workflow["update_branch"] || "automation/docker-dependency-updates", + "pr_title" => workflow["pull_request_title"] || "chore(deps): update Docker dependency pins", + "commit_message" => workflow["commit_message"] || "chore(deps): update Docker dependency pins", + "validation_command" => validation.first || "docker build --progress=plain -t dependency-update-validation ." + } - steps: - - name: Checkout repository - uses: actions/checkout@v6 - with: - ref: ${{ github.ref }} + File.open(ENV.fetch("GITHUB_OUTPUT"), "a") do |out| + outputs.each { |key, value| out.puts("#{key}=#{value}") } + end + ' "${config_path}" + + test -f "${config_path}" + test -f "$(ruby -ryaml -e 'config = YAML.load_file(ARGV.fetch(0)) || {}; paths = config["paths"] || {}; puts paths["prompt"] || "ci/prompts/docker-dependency-updater.md"' "${config_path}")" + test -f "$(ruby -ryaml -e 'config = YAML.load_file(ARGV.fetch(0)) || {}; puts config["dockerfile"] || "Dockerfile"' "${config_path}")" - name: Stop if an update PR is already open id: existing-pr uses: actions/github-script@v8 + env: + UPDATE_BRANCH: ${{ steps.config.outputs.update_branch }} with: script: | const owner = context.repo.owner; const repo = context.repo.repo; - const head = `${owner}:automation/docker-dependency-updates`; + const head = `${owner}:${process.env.UPDATE_BRANCH}`; const prs = await github.rest.pulls.list({ owner, @@ -67,28 +95,334 @@ jobs: core.notice(`Update PR already open: ${prs.data[0].html_url}`); } + - name: Decide updater execution + id: decision + run: | + set -euo pipefail + + if [ "${EXISTING_PR_FOUND}" = "true" ]; then + echo "should_run=false" >> "${GITHUB_OUTPUT}" + echo "skip_reason=Update PR already open for ${UPDATE_BRANCH}" >> "${GITHUB_OUTPUT}" + exit 0 + fi + + echo "should_run=true" >> "${GITHUB_OUTPUT}" + echo "skip_reason=" >> "${GITHUB_OUTPUT}" + env: + EXISTING_PR_FOUND: ${{ steps.existing-pr.outputs.found }} + UPDATE_BRANCH: ${{ steps.config.outputs.update_branch }} + + build-dependency-report: + needs: configure + if: needs.configure.outputs.should_run == 'true' + runs-on: ubuntu-24.04 + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Render unpinned probe Dockerfile + run: | + mkdir -p "$(dirname "${PROBE_DOCKERFILE}")" + + ruby -ryaml -e ' + config = YAML.load_file(ENV.fetch("PROBE_CONFIG")) || {} + dockerfile = ENV.fetch("DOCKERFILE") + output = ENV.fetch("PROBE_DOCKERFILE") + + flatten_specs = lambda do |value| + case value + when Array + value + when Hash + value.flat_map do |group, specs| + Array(specs).map { |spec| spec.merge("group" => spec["group"] || group) } + end + else + [] + end + end + + probes = flatten_specs.call(config["probes"]) + flatten_specs.call(config["observed_only"]) + resolvers = Array(config["latest_resolvers"]) + managed_args = probes.map { |probe| probe["maps_to_arg"] }.compact + managed_args += resolvers.map { |resolver| resolver["arg"] }.compact + managed_args = managed_args.uniq + + inserted = {} + active_blocks = {} + in_apt = false + + File.open(output, "w") do |out| + File.foreach(dockerfile) do |original_line| + resolvers.each do |resolver| + arg = resolver["arg"] + block_start = resolver["block_start_pattern"] + next if arg.nil? || block_start.nil? + + active_blocks[arg] = true if original_line.match?(Regexp.new(block_start)) + active_blocks[arg] = false if original_line.match?(/^# /) && !original_line.match?(Regexp.new(block_start)) + end + + arg_match = original_line.match(/^ARG\s+([A-Za-z_][A-Za-z0-9_]*)=/) + next if arg_match && managed_args.include?(arg_match[1]) + + line = original_line.dup + + if line.include?("apt-get install -y --no-install-recommends") + in_apt = true + elsif in_apt + line = line.sub(/^(\s+[A-Za-z0-9.+-]+)=\S+(.+)$/, "\\1\\2") + end + + managed_args.each do |arg| + line = line.gsub("==${#{arg}}", "") + end + + resolvers.each do |resolver| + arg = resolver["arg"] + next if arg.nil? || inserted[arg] + next if resolver["block_start_pattern"] && !active_blocks[arg] + + insert_before = resolver["insert_before_pattern"] + if insert_before && line.match?(Regexp.new(insert_before)) + out.puts " #{resolver.fetch("assignment")} && \\" + line = line.gsub(resolver.dig("rewrite", "from"), resolver.dig("rewrite", "to")) if resolver["rewrite"] + inserted[arg] = true + end + end + + out.write line + + resolvers.each do |resolver| + arg = resolver["arg"] + next if arg.nil? || inserted[arg] + next if resolver["block_start_pattern"] && !active_blocks[arg] + + insert_after = resolver["insert_after_pattern"] + if insert_after && line.match?(Regexp.new(insert_after)) + out.puts " #{resolver.fetch("assignment")} && \\" + inserted[arg] = true + end + end + + in_apt = false if in_apt && line.match?(/&&\s*\\?$/) + end + end + ' + env: + DOCKERFILE: ${{ needs.configure.outputs.dockerfile }} + PROBE_CONFIG: ${{ needs.configure.outputs.config_path }} + PROBE_DOCKERFILE: ${{ needs.configure.outputs.probe_dockerfile }} + + - name: Build unpinned dependency probe + run: docker build --pull --no-cache -f "${PROBE_DOCKERFILE}" -t "${PROBE_IMAGE}" . + env: + PROBE_DOCKERFILE: ${{ needs.configure.outputs.probe_dockerfile }} + PROBE_IMAGE: ${{ needs.configure.outputs.image }} + + - name: Discover pinned apt packages + run: | + mkdir -p .tmp/dependency-probe + + awk ' + /apt-get install -y --no-install-recommends/ { in_block=1; next } + in_block { + line=$0 + gsub(/\\/, "", line) + gsub(/^[[:space:]]+/, "", line) + if (line ~ /^[[:alnum:].+-]+=/) { + split(line, parts, "=") + print parts[1] + } + if ($0 ~ /&&[[:space:]]*\\?$/) { + in_block=0 + } + } + ' "${DOCKERFILE}" > .tmp/dependency-probe/apt-packages.txt + env: + DOCKERFILE: ${{ needs.configure.outputs.dockerfile }} + + - name: Collect apt inventory + run: | + set -euo pipefail + + mapfile -t apt_packages < .tmp/dependency-probe/apt-packages.txt + + docker run -i --rm --entrypoint bash "${PROBE_IMAGE}" -s -- "${apt_packages[@]}" > .tmp/dependency-probe/apt.json <<'EOF' + set -euo pipefail + + dpkg-query -W -f='${binary:Package}\t${Version}\n' "$@" \ + | jq -R -s ' + split("\n") + | map(select(length > 0)) + | map(split("\t") | {name: .[0], installed: .[1]}) + ' + EOF + env: + PROBE_IMAGE: ${{ needs.configure.outputs.image }} + + - name: Collect runtime metadata + run: | + docker run -i --rm --entrypoint bash "${PROBE_IMAGE}" -s > .tmp/dependency-probe/runtime.json <<'EOF' + set -euo pipefail + + jq -n \ + --arg os "$(. /etc/os-release && printf '%s' "${PRETTY_NAME}")" \ + --arg arch "$(uname -m)" \ + '{os: $os, architecture: $arch}' + EOF + env: + PROBE_IMAGE: ${{ needs.configure.outputs.image }} + + - name: Collect configured probe inventory + run: | + set -euo pipefail + + config_json="$(ruby -ryaml -rjson -e 'puts JSON.generate(YAML.load_file(ARGV.fetch(0)) || {})' "${PROBE_CONFIG}")" + + docker run -i --rm --entrypoint bash \ + -e PROBE_CONFIG_JSON="${config_json}" \ + "${PROBE_IMAGE}" -s > .tmp/dependency-probe/probes.json <<'EOF' + set -euo pipefail + + run_probe_commands() { + local section="$1" + + printf '%s' "${PROBE_CONFIG_JSON:-{}}" \ + | jq -c " + def probe_specs: + if type == \"array\" then + .[] + elif type == \"object\" then + to_entries[] | .key as \$group | .value[] | . + {group: (.group // \$group)} + else + empty + end; + + .${section} // {} | probe_specs + " \ + | while IFS= read -r spec; do + name="$(jq -r '.name' <<< "${spec}")" + command="$(jq -r '.command' <<< "${spec}")" + installed="$(bash -o pipefail -c "${command}")" + + jq -n \ + --arg name "${name}" \ + --arg type "$(jq -r '.type // "command"' <<< "${spec}")" \ + --arg command "${command}" \ + --arg installed "${installed}" \ + --arg maps_to_arg "$(jq -r '.maps_to_arg // empty' <<< "${spec}")" \ + --arg group "$(jq -r '.group // empty' <<< "${spec}")" \ + --arg source "$(jq -r '.source // empty' <<< "${spec}")" \ + --arg reason "$(jq -r '.reason // empty' <<< "${spec}")" \ + '{ + name: $name, + type: $type, + command: $command, + installed: $installed + } + + (if $maps_to_arg == "" then {} else {maps_to_arg: $maps_to_arg} end) + + (if $group == "" then {} else {group: $group} end) + + (if $source == "" then {} else {source: $source} end) + + (if $reason == "" then {} else {reason: $reason, pinned_in_dockerfile: false} end)' + done \ + | jq -s '.' + } + + probes_json="$(run_probe_commands probes)" + observed_only_json="$(run_probe_commands observed_only)" + + jq -n \ + --argjson probes "${probes_json}" \ + --argjson observed_only "${observed_only_json}" \ + '{probes: $probes, observed_only: $observed_only}' + EOF + env: + PROBE_CONFIG: ${{ needs.configure.outputs.config_path }} + PROBE_IMAGE: ${{ needs.configure.outputs.image }} + + - name: Assemble dependency report + run: | + set -euo pipefail + + base_image="$(awk '$1 == "FROM" { print $2; exit }' "${DOCKERFILE}")" + + jq -n \ + --arg generated_at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + --arg base_image "${base_image}" \ + --slurpfile runtime .tmp/dependency-probe/runtime.json \ + --slurpfile apt .tmp/dependency-probe/apt.json \ + --slurpfile probes .tmp/dependency-probe/probes.json \ + '{ + generated_at: $generated_at, + method: "unpinned Dockerfile probe build", + base_image: $base_image, + runtime: $runtime[0], + dependencies: { + apt: $apt[0], + probes: $probes[0].probes, + observed_only: $probes[0].observed_only + } + }' > "${PROBE_REPORT}" + + report_tmp="${PROBE_REPORT}.tmp" + jq \ + --arg config_path "${PROBE_CONFIG}" \ + --rawfile config_yaml "${PROBE_CONFIG}" \ + '.probe_config = {path: $config_path, format: "yaml", content: $config_yaml}' \ + "${PROBE_REPORT}" > "${report_tmp}" + mv "${report_tmp}" "${PROBE_REPORT}" + env: + DOCKERFILE: ${{ needs.configure.outputs.dockerfile }} + PROBE_CONFIG: ${{ needs.configure.outputs.config_path }} + PROBE_REPORT: ${{ needs.configure.outputs.report_path }} + + - name: Upload dependency report + uses: actions/upload-artifact@v5 + with: + name: docker-dependency-report + path: | + ${{ needs.configure.outputs.report_path }} + ${{ needs.configure.outputs.probe_dockerfile }} + + update-with-copilot: + needs: + - configure + - build-dependency-report + if: needs.configure.outputs.should_run == 'true' + runs-on: ubuntu-24.04 + + permissions: + contents: write + pull-requests: write + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + with: + ref: ${{ github.ref }} + - name: Download dependency report - if: steps.existing-pr.outputs.found != 'true' uses: actions/download-artifact@v6 with: name: docker-dependency-report - name: Set up Node.js - if: steps.existing-pr.outputs.found != 'true' uses: actions/setup-node@v6 with: node-version: "22" - name: Install Copilot CLI - if: steps.existing-pr.outputs.found != 'true' run: npm install -g @github/copilot - name: Update Dockerfile with Copilot CLI - if: steps.existing-pr.outputs.found != 'true' env: COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} COPILOT_ALLOW_ALL: "true" COPILOT_AUTO_UPDATE: "false" + PROMPT_PATH: ${{ needs.configure.outputs.prompt_path }} + REPORT_PATH: ${{ needs.configure.outputs.report_path }} run: | set -euo pipefail @@ -97,37 +431,14 @@ jobs: exit 1 fi - cat > /tmp/docker-dependency-updater-prompt.md <<'EOF' - You are updating Dockerfile dependency pins for this repository. - - Intent: - - Read Dockerfile and docker-dependency-report.json. - - The report was produced by building a temporary copy of Dockerfile with dependency version pins removed. - - Treat the report as the primary evidence for versions that install successfully for the current base image. - - Update only Dockerfile dependency pins and ARG values when the report shows a newer installed version. - - Preserve the ubuntu base image tag unless explicitly necessary to make the reported versions valid. - - Keep AWS CLI unpinned unless Dockerfile already pins it; mention its latest observed version in the changelog only. - - Do not edit workflow files, docs, tests, or application code. - - Do not commit, push, or create a pull request; this workflow will do that. - - Expected Dockerfile update categories: - - apt package pins from dependencies.apt[].installed - - PIP_VERSION from dependencies.pip[name="pip"].installed - - AZURE_CLI_VERSION from dependencies.pip[name="azure-cli"].installed - - YQ_VERSION from dependencies.github_releases[name="yq"].installed - - GCLOUD_VERSION from dependencies.archives[name="google-cloud-sdk"].installed - - After editing: - - Print a concise changelog of every version change. - - Print any dependency that was observed but intentionally not pinned. - - Dependency report: - EOF + cp "${PROMPT_PATH}" /tmp/docker-dependency-updater-prompt.md { + echo + echo "Dependency report:" echo echo '```json' - cat docker-dependency-report.json + cat "${REPORT_PATH}" echo echo '```' } >> /tmp/docker-dependency-updater-prompt.md @@ -142,24 +453,25 @@ jobs: --share copilot-docker-dependency-session.md - name: Detect changes - if: steps.existing-pr.outputs.found != 'true' id: changes shell: bash run: | set -euo pipefail - if git diff --quiet -- Dockerfile; then + if git diff --quiet -- "${DOCKERFILE}"; then echo "changed=false" >> "${GITHUB_OUTPUT}" else echo "changed=true" >> "${GITHUB_OUTPUT}" - git diff -- Dockerfile > docker-dependency-update.diff + git diff -- "${DOCKERFILE}" > docker-dependency-update.diff fi + env: + DOCKERFILE: ${{ needs.configure.outputs.dockerfile }} - name: Validate Docker build - if: steps.existing-pr.outputs.found != 'true' && steps.changes.outputs.changed == 'true' - run: docker build --progress=plain -t dependency-update-validation . + if: steps.changes.outputs.changed == 'true' + run: ${{ needs.configure.outputs.validation_command }} - name: Prepare pull request body - if: steps.existing-pr.outputs.found != 'true' && steps.changes.outputs.changed == 'true' + if: steps.changes.outputs.changed == 'true' run: | { echo "## Summary" @@ -168,9 +480,9 @@ jobs: echo echo "## Evidence" echo - echo "- Probe report artifact: \`docker-dependency-report.json\`" + echo "- Probe report artifact: \`${REPORT_PATH}\`" echo "- Copilot session artifact: \`copilot-docker-dependency-session.md\`" - echo "- Validation: \`docker build --progress=plain -t dependency-update-validation .\`" + echo "- Validation: \`${VALIDATION_COMMAND}\`" echo echo "## Diff" echo @@ -178,9 +490,11 @@ jobs: sed -n '1,220p' docker-dependency-update.diff echo '```' } > docker-dependency-pr-body.md + env: + REPORT_PATH: ${{ needs.configure.outputs.report_path }} + VALIDATION_COMMAND: ${{ needs.configure.outputs.validation_command }} - name: Upload Copilot session - if: steps.existing-pr.outputs.found != 'true' uses: actions/upload-artifact@v5 with: name: copilot-docker-dependency-session @@ -190,12 +504,12 @@ jobs: if-no-files-found: ignore - name: Create pull request - if: steps.existing-pr.outputs.found != 'true' && steps.changes.outputs.changed == 'true' + if: steps.changes.outputs.changed == 'true' uses: peter-evans/create-pull-request@v7 with: - commit-message: "chore(deps): update Docker dependency pins" - title: "chore(deps): update Docker dependency pins" + commit-message: ${{ needs.configure.outputs.commit_message }} + title: ${{ needs.configure.outputs.pr_title }} body-path: docker-dependency-pr-body.md - branch: automation/docker-dependency-updates - add-paths: Dockerfile + branch: ${{ needs.configure.outputs.update_branch }} + add-paths: ${{ needs.configure.outputs.dockerfile }} delete-branch: true diff --git a/ci/configs/docker-dependency-probe.yaml b/ci/configs/docker-dependency-probe.yaml new file mode 100644 index 00000000..fa4b3ff1 --- /dev/null +++ b/ci/configs/docker-dependency-probe.yaml @@ -0,0 +1,86 @@ +--- +# Lightweight hints for Docker dependency update automation. +# The probe script treats Dockerfile as the source of truth and this file as +# repo-specific guidance for Copilot and future probe extensions. + +dockerfile: Dockerfile + +workflow: + update_branch: automation/docker-dependency-updates + pull_request_title: "chore(deps): update Docker dependency pins" + commit_message: "chore(deps): update Docker dependency pins" + +paths: + prompt: ci/prompts/docker-dependency-updater.md + probe_dockerfile: .tmp/dependency-probe/Dockerfile + report: docker-dependency-report.json + +apt: + discover: true + +args: + discover_version_args: true + +latest_resolvers: + - arg: YQ_VERSION + insert_before_pattern: github.com/mikefarah/yq/releases/download + assignment: >- + YQ_VERSION="$(curl -fsSL https://api.github.com/repos/mikefarah/yq/releases/latest | jq -r .tag_name)" + rewrite: + from: download/v${YQ_VERSION} + to: download/${YQ_VERSION} + + - arg: GCLOUD_VERSION + block_start_pattern: ^# Install Google Cloud SDK + insert_after_pattern: ^RUN ARCH=\$\(uname -m\) && \\$ + assignment: >- + GCLOUD_VERSION="$(curl -fsSL https://dl.google.com/dl/cloudsdk/channels/rapid/components-2.json | jq -r .version)" + +probes: + pip: + - name: pip + type: command + command: >- + /opt/az/bin/pip --version | awk '{ print $2 }' + maps_to_arg: PIP_VERSION + + - name: azure-cli + type: command + command: >- + /opt/az/bin/pip show azure-cli | awk -F': ' '$1 == "Version" { print $2 }' + maps_to_arg: AZURE_CLI_VERSION + + github_releases: + - name: yq + type: command + command: >- + yq --version | awk '{ print $NF }' | sed 's/^v//' + maps_to_arg: YQ_VERSION + source: mikefarah/yq + + archives: + - name: google-cloud-sdk + type: command + command: >- + gcloud version --format=json | jq -r '."Google Cloud SDK"' + maps_to_arg: GCLOUD_VERSION + source: Google Cloud SDK rapid channel + +observed_only: + archives: + - name: aws-cli + type: command + command: >- + aws --version 2>&1 | awk '{ print $1 }' | sed 's#aws-cli/##' + reason: Dockerfile installs the latest AWS CLI zip and does not pin it. + +copilot: + intent: + - Use Dockerfile and the generated dependency report as primary evidence. + - Use this file as hints, not as the source of truth. + - Update only Dockerfile dependency pins and ARG values. + - Preserve the ubuntu base image tag unless explicitly necessary. + - Report versioned Dockerfile dependencies that are missing probe coverage. + +validation: + - docker build --progress=plain -t dependency-update-validation . diff --git a/ci/docker-dependency-probe.sh b/ci/docker-dependency-probe.sh deleted file mode 100755 index 2cef151f..00000000 --- a/ci/docker-dependency-probe.sh +++ /dev/null @@ -1,205 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -dockerfile="${DOCKERFILE:-Dockerfile}" -image="${PROBE_IMAGE:-worker-deps-probe:${GITHUB_RUN_ID:-local}}" -report="${PROBE_REPORT:-docker-dependency-report.json}" - -usage() { - cat <<'EOF' -Usage: ci/docker-dependency-probe.sh [--dockerfile Dockerfile] [--image tag] [--report path] - -Build a temporary Dockerfile with dependency pins removed, then report the -versions that the image actually installs. -EOF -} - -while [ "$#" -gt 0 ]; do - case "$1" in - --dockerfile) - dockerfile="$2" - shift 2 - ;; - --image) - image="$2" - shift 2 - ;; - --report) - report="$2" - shift 2 - ;; - -h|--help) - usage - exit 0 - ;; - *) - echo "Unknown argument: $1" >&2 - usage >&2 - exit 2 - ;; - esac -done - -if [ ! -f "${dockerfile}" ]; then - echo "Dockerfile not found: ${dockerfile}" >&2 - exit 1 -fi - -tmpdir="$(mktemp -d)" -probe_dockerfile="${tmpdir}/Dockerfile.unpinned-probe" -trap 'rm -rf "${tmpdir}"' EXIT - -base_image="$(awk '$1 == "FROM" { print $2; exit }' "${dockerfile}")" - -mapfile -t apt_packages < <( - awk ' - /apt-get install -y --no-install-recommends/ { in_block=1; next } - in_block { - line=$0 - gsub(/\\/, "", line) - gsub(/^[[:space:]]+/, "", line) - if (line ~ /^[[:alnum:].+-]+=/) { - split(line, parts, "=") - print parts[1] - } - if ($0 ~ /&&[[:space:]]*\\?$/) { - in_block=0 - } - } - ' "${dockerfile}" -) - -awk ' - /^ARG (AZURE_CLI_VERSION|PIP_VERSION|YQ_VERSION|GCLOUD_VERSION)=/ { - next - } - - /^# Install yq/ { - block = "yq" - print - next - } - - /^# Install Google Cloud SDK/ { - block = "gcloud" - gcloud_inserted = 0 - print - next - } - - /^# Install AWS CLI/ { - block = "aws" - print - next - } - - /^#/ && $0 !~ /^# Install/ { - block = "" - print - next - } - - { - line = $0 - - if (line ~ /apt-get install -y --no-install-recommends/) { - in_apt = 1 - } else if (in_apt && line ~ /^[[:space:]]+[A-Za-z0-9.+-]+=/) { - sub(/=[^[:space:]]+/, "", line) - } - - if (line ~ /pip install .*--upgrade pip==\$\{PIP_VERSION\}/) { - sub(/==\$\{PIP_VERSION\}/, "", line) - } - - if (line ~ /pip install .*azure-cli==\$\{AZURE_CLI_VERSION\}/) { - sub(/==\$\{AZURE_CLI_VERSION\}/, "", line) - } - - if (block == "yq" && line ~ /github.com\/mikefarah\/yq\/releases\/download/) { - print " YQ_VERSION=\"$(curl -fsSL https://api.github.com/repos/mikefarah/yq/releases/latest | jq -r .tag_name)\" && \\" - sub(/download\/v\$\{YQ_VERSION\}/, "download/${YQ_VERSION}", line) - } - - if (block == "gcloud" && gcloud_inserted == 0 && line ~ /^RUN ARCH=\$\(uname -m\) && \\$/) { - print line - print " GCLOUD_VERSION=\"$(curl -fsSL https://dl.google.com/dl/cloudsdk/channels/rapid/components-2.json | jq -r .version)\" && \\" - gcloud_inserted = 1 - next - } - - print line - - if (in_apt && line ~ /&&[[:space:]]*\\?$/) { - in_apt = 0 - } - } -' "${dockerfile}" > "${probe_dockerfile}" - -if [ "${PROBE_SKIP_BUILD:-false}" = "true" ]; then - echo "Skipping probe image build and using existing image: ${image}" >&2 -else - echo "Building dependency probe image: ${image}" >&2 - docker build --pull --no-cache -f "${probe_dockerfile}" -t "${image}" . -fi - -echo "Collecting dependency inventory from: ${image}" >&2 -docker run -i --rm --entrypoint bash "${image}" -s -- "${base_image}" "${apt_packages[@]}" > "${report}" <<'EOF' -set -euo pipefail - -base_image="$1" -shift - -apt_json="$( - dpkg-query -W -f='${binary:Package}\t${Version}\n' "$@" \ - | jq -R -s ' - split("\n") - | map(select(length > 0)) - | map(split("\t") | {name: .[0], installed: .[1]}) - ' -)" - -os_pretty="$(. /etc/os-release && printf '%s' "${PRETTY_NAME}")" -pip_version="$(/opt/az/bin/pip --version | awk '{ print $2 }')" -azure_cli_version="$(/opt/az/bin/pip show azure-cli | awk -F': ' '$1 == "Version" { print $2 }')" -yq_version="$(yq --version | awk '{ print $NF }' | sed 's/^v//')" -gcloud_version="$(gcloud version --format=json | jq -r '."Google Cloud SDK"')" -aws_cli_version="$(aws --version 2>&1 | awk '{ print $1 }' | sed 's#aws-cli/##')" - -jq -n \ - --arg generated_at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ - --arg base_image "${base_image}" \ - --arg os "${os_pretty}" \ - --arg arch "$(uname -m)" \ - --arg pip "${pip_version}" \ - --arg azure_cli "${azure_cli_version}" \ - --arg yq "${yq_version}" \ - --arg gcloud "${gcloud_version}" \ - --arg aws_cli "${aws_cli_version}" \ - --argjson apt "${apt_json}" \ - '{ - generated_at: $generated_at, - method: "unpinned Dockerfile probe build", - base_image: $base_image, - runtime: { - os: $os, - architecture: $arch - }, - dependencies: { - apt: $apt, - pip: [ - {name: "pip", installed: $pip}, - {name: "azure-cli", installed: $azure_cli} - ], - github_releases: [ - {name: "yq", installed: $yq, source: "mikefarah/yq"} - ], - archives: [ - {name: "google-cloud-sdk", installed: $gcloud, source: "Google Cloud SDK rapid channel"}, - {name: "aws-cli", installed: $aws_cli, source: "awscli.amazonaws.com latest zip", pinned_in_dockerfile: false} - ] - } - }' -EOF - -echo "Dependency report written to ${report}" >&2 diff --git a/ci/prompts/docker-dependency-updater.md b/ci/prompts/docker-dependency-updater.md new file mode 100644 index 00000000..9e3e5e8f --- /dev/null +++ b/ci/prompts/docker-dependency-updater.md @@ -0,0 +1,23 @@ +You are updating Dockerfile dependency pins for this repository. + +Intent: +- Read Dockerfile and docker-dependency-report.json. +- The report was produced by building a temporary copy of Dockerfile with dependency version pins removed. +- The report may include probe_config.content from ci/configs/docker-dependency-probe.yaml. +- Treat the report as the primary evidence for versions that install successfully for the current base image. +- Treat probe_config.content as repo-specific hints, not as the source of truth. +- Update only Dockerfile dependency pins and ARG values when the report shows a newer installed version. +- Preserve the ubuntu base image tag unless explicitly necessary to make the reported versions valid. +- Keep observed-only dependencies unpinned unless Dockerfile already pins them; mention their latest observed versions in the changelog only. +- Do not edit workflow files, docs, tests, or application code. +- Do not commit, push, or create a pull request; this workflow will do that. +- If Dockerfile contains a versioned dependency that is not represented in the report or probe_config, mention it as missing probe coverage. + +Expected Dockerfile update categories: +- apt package pins from dependencies.apt[].installed +- ARG pins from dependencies.probes[] entries with maps_to_arg values +- observed-only tools from dependencies.observed_only[] entries + +After editing: +- Print a concise changelog of every version change. +- Print any dependency that was observed but intentionally not pinned. From c40156e245326ebf00aa4fac1231ac69a696381b Mon Sep 17 00:00:00 2001 From: Dmytro Smirnov Date: Tue, 2 Jun 2026 14:13:42 +0300 Subject: [PATCH 11/32] ci: mount probe config for docker inventory --- .github/workflows/docker-dependency-updater.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/docker-dependency-updater.yml b/.github/workflows/docker-dependency-updater.yml index a10785a2..f4cb13c5 100644 --- a/.github/workflows/docker-dependency-updater.yml +++ b/.github/workflows/docker-dependency-updater.yml @@ -279,18 +279,18 @@ jobs: run: | set -euo pipefail - config_json="$(ruby -ryaml -rjson -e 'puts JSON.generate(YAML.load_file(ARGV.fetch(0)) || {})' "${PROBE_CONFIG}")" + ruby -ryaml -rjson -e 'puts JSON.generate(YAML.load_file(ARGV.fetch(0)) || {})' \ + "${PROBE_CONFIG}" > .tmp/dependency-probe/config.json docker run -i --rm --entrypoint bash \ - -e PROBE_CONFIG_JSON="${config_json}" \ + -v "${PWD}/.tmp/dependency-probe/config.json:/tmp/probe-config.json:ro" \ "${PROBE_IMAGE}" -s > .tmp/dependency-probe/probes.json <<'EOF' set -euo pipefail run_probe_commands() { local section="$1" - printf '%s' "${PROBE_CONFIG_JSON:-{}}" \ - | jq -c " + jq -c " def probe_specs: if type == \"array\" then .[] @@ -301,7 +301,7 @@ jobs: end; .${section} // {} | probe_specs - " \ + " /tmp/probe-config.json \ | while IFS= read -r spec; do name="$(jq -r '.name' <<< "${spec}")" command="$(jq -r '.command' <<< "${spec}")" From 43f6610de4d3f48e4635973ffcd9acf3905ecca3 Mon Sep 17 00:00:00 2001 From: Dmytro Smirnov Date: Tue, 2 Jun 2026 15:49:59 +0300 Subject: [PATCH 12/32] ci: clarify docker dependency updater run output --- .../workflows/docker-dependency-updater.yml | 181 ++++++++++++++---- ci/configs/docker-dependency-probe.yaml | 4 +- 2 files changed, 147 insertions(+), 38 deletions(-) diff --git a/.github/workflows/docker-dependency-updater.yml b/.github/workflows/docker-dependency-updater.yml index f4cb13c5..cfaa224d 100644 --- a/.github/workflows/docker-dependency-updater.yml +++ b/.github/workflows/docker-dependency-updater.yml @@ -9,8 +9,11 @@ name: Docker Dependency Updater permissions: contents: read +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" + jobs: - configure: + config: runs-on: ubuntu-24.04 permissions: @@ -70,6 +73,7 @@ jobs: test -f "${config_path}" test -f "$(ruby -ryaml -e 'config = YAML.load_file(ARGV.fetch(0)) || {}; paths = config["paths"] || {}; puts paths["prompt"] || "ci/prompts/docker-dependency-updater.md"' "${config_path}")" test -f "$(ruby -ryaml -e 'config = YAML.load_file(ARGV.fetch(0)) || {}; puts config["dockerfile"] || "Dockerfile"' "${config_path}")" + echo "Config loaded: ${config_path}" - name: Stop if an update PR is already open id: existing-pr @@ -92,7 +96,9 @@ jobs: core.setOutput("found", prs.data.length > 0 ? "true" : "false"); if (prs.data.length > 0) { - core.notice(`Update PR already open: ${prs.data[0].html_url}`); + console.log(`Update PR already open: ${prs.data[0].html_url}`); + } else { + console.log(`No open update PR found for ${head}`); } - name: Decide updater execution @@ -103,19 +109,37 @@ jobs: if [ "${EXISTING_PR_FOUND}" = "true" ]; then echo "should_run=false" >> "${GITHUB_OUTPUT}" echo "skip_reason=Update PR already open for ${UPDATE_BRANCH}" >> "${GITHUB_OUTPUT}" + echo "Config decision: skip, update PR already open for ${UPDATE_BRANCH}" exit 0 fi echo "should_run=true" >> "${GITHUB_OUTPUT}" echo "skip_reason=" >> "${GITHUB_OUTPUT}" + echo "Config decision: run analyze/apply, no open updater PR found." env: EXISTING_PR_FOUND: ${{ steps.existing-pr.outputs.found }} UPDATE_BRANCH: ${{ steps.config.outputs.update_branch }} - build-dependency-report: - needs: configure - if: needs.configure.outputs.should_run == 'true' + - name: Write config summary + if: always() + run: | + { + echo "## Docker Dependency Updater" + echo + echo "| Job | Result | Details |" + echo "| --- | --- | --- |" + echo "| config | ${{ job.status }} | Dockerfile: \`${{ steps.config.outputs.dockerfile }}\`; branch: \`${{ steps.config.outputs.update_branch }}\`; should run: \`${{ steps.decision.outputs.should_run }}\` |" + } >> "${GITHUB_STEP_SUMMARY}" + + analyze: + needs: config + if: needs.config.outputs.should_run == 'true' runs-on: ubuntu-24.04 + outputs: + apt_count: ${{ steps.report.outputs.apt_count }} + observed_count: ${{ steps.report.outputs.observed_count }} + probe_count: ${{ steps.report.outputs.probe_count }} + report_path: ${{ needs.config.outputs.report_path }} steps: - name: Checkout repository @@ -210,16 +234,19 @@ jobs: end end ' + echo "Analyze render: ${DOCKERFILE} -> ${PROBE_DOCKERFILE}" env: - DOCKERFILE: ${{ needs.configure.outputs.dockerfile }} - PROBE_CONFIG: ${{ needs.configure.outputs.config_path }} - PROBE_DOCKERFILE: ${{ needs.configure.outputs.probe_dockerfile }} + DOCKERFILE: ${{ needs.config.outputs.dockerfile }} + PROBE_CONFIG: ${{ needs.config.outputs.config_path }} + PROBE_DOCKERFILE: ${{ needs.config.outputs.probe_dockerfile }} - name: Build unpinned dependency probe - run: docker build --pull --no-cache -f "${PROBE_DOCKERFILE}" -t "${PROBE_IMAGE}" . + run: | + docker build --pull --no-cache -f "${PROBE_DOCKERFILE}" -t "${PROBE_IMAGE}" . + echo "Analyze build: built ${PROBE_IMAGE} from ${PROBE_DOCKERFILE}" env: - PROBE_DOCKERFILE: ${{ needs.configure.outputs.probe_dockerfile }} - PROBE_IMAGE: ${{ needs.configure.outputs.image }} + PROBE_DOCKERFILE: ${{ needs.config.outputs.probe_dockerfile }} + PROBE_IMAGE: ${{ needs.config.outputs.image }} - name: Discover pinned apt packages run: | @@ -240,8 +267,11 @@ jobs: } } ' "${DOCKERFILE}" > .tmp/dependency-probe/apt-packages.txt + + apt_count="$(wc -l < .tmp/dependency-probe/apt-packages.txt | tr -d ' ')" + echo "Analyze apt discovery: found ${apt_count} pinned apt packages in ${DOCKERFILE}" env: - DOCKERFILE: ${{ needs.configure.outputs.dockerfile }} + DOCKERFILE: ${{ needs.config.outputs.dockerfile }} - name: Collect apt inventory run: | @@ -259,8 +289,11 @@ jobs: | map(split("\t") | {name: .[0], installed: .[1]}) ' EOF + + apt_count="$(jq 'length' .tmp/dependency-probe/apt.json)" + echo "Analyze apt inventory: collected installed versions for ${apt_count} packages" env: - PROBE_IMAGE: ${{ needs.configure.outputs.image }} + PROBE_IMAGE: ${{ needs.config.outputs.image }} - name: Collect runtime metadata run: | @@ -272,8 +305,12 @@ jobs: --arg arch "$(uname -m)" \ '{os: $os, architecture: $arch}' EOF + + os="$(jq -r '.os' .tmp/dependency-probe/runtime.json)" + arch="$(jq -r '.architecture' .tmp/dependency-probe/runtime.json)" + echo "Analyze runtime: ${os} on ${arch}" env: - PROBE_IMAGE: ${{ needs.configure.outputs.image }} + PROBE_IMAGE: ${{ needs.config.outputs.image }} - name: Collect configured probe inventory run: | @@ -338,11 +375,16 @@ jobs: --argjson observed_only "${observed_only_json}" \ '{probes: $probes, observed_only: $observed_only}' EOF + + probe_count="$(jq '.probes | length' .tmp/dependency-probe/probes.json)" + observed_count="$(jq '.observed_only | length' .tmp/dependency-probe/probes.json)" + echo "Analyze probes: collected ${probe_count} managed probes and ${observed_count} observed-only probes" env: - PROBE_CONFIG: ${{ needs.configure.outputs.config_path }} - PROBE_IMAGE: ${{ needs.configure.outputs.image }} + PROBE_CONFIG: ${{ needs.config.outputs.config_path }} + PROBE_IMAGE: ${{ needs.config.outputs.image }} - name: Assemble dependency report + id: report run: | set -euo pipefail @@ -373,25 +415,59 @@ jobs: '.probe_config = {path: $config_path, format: "yaml", content: $config_yaml}' \ "${PROBE_REPORT}" > "${report_tmp}" mv "${report_tmp}" "${PROBE_REPORT}" + + apt_count="$(jq '.dependencies.apt | length' "${PROBE_REPORT}")" + probe_count="$(jq '.dependencies.probes | length' "${PROBE_REPORT}")" + observed_count="$(jq '.dependencies.observed_only | length' "${PROBE_REPORT}")" + printf 'apt_count=%s\n' "${apt_count}" >> "${GITHUB_OUTPUT}" + printf 'probe_count=%s\n' "${probe_count}" >> "${GITHUB_OUTPUT}" + printf 'observed_count=%s\n' "${observed_count}" >> "${GITHUB_OUTPUT}" + echo "Analyze report: wrote ${PROBE_REPORT} with apt=${apt_count}, probes=${probe_count}, observed-only=${observed_count}" env: - DOCKERFILE: ${{ needs.configure.outputs.dockerfile }} - PROBE_CONFIG: ${{ needs.configure.outputs.config_path }} - PROBE_REPORT: ${{ needs.configure.outputs.report_path }} + DOCKERFILE: ${{ needs.config.outputs.dockerfile }} + PROBE_CONFIG: ${{ needs.config.outputs.config_path }} + PROBE_REPORT: ${{ needs.config.outputs.report_path }} - name: Upload dependency report uses: actions/upload-artifact@v5 with: name: docker-dependency-report path: | - ${{ needs.configure.outputs.report_path }} - ${{ needs.configure.outputs.probe_dockerfile }} + ${{ needs.config.outputs.report_path }} + ${{ needs.config.outputs.probe_dockerfile }} + + - name: Write analyze summary + if: always() + run: | + apt_count="n/a" + probe_count="n/a" + observed_count="n/a" + + if [ -f "${PROBE_REPORT}" ]; then + apt_count="$(jq '.dependencies.apt | length' "${PROBE_REPORT}")" + probe_count="$(jq '.dependencies.probes | length' "${PROBE_REPORT}")" + observed_count="$(jq '.dependencies.observed_only | length' "${PROBE_REPORT}")" + fi + + { + echo "## Docker Dependency Updater" + echo + echo "| Job | Result | Details |" + echo "| --- | --- | --- |" + echo "| analyze | ${{ job.status }} | Report: \`${PROBE_REPORT}\`; apt: \`${apt_count}\`; probes: \`${probe_count}\`; observed-only: \`${observed_count}\` |" + } >> "${GITHUB_STEP_SUMMARY}" + env: + PROBE_REPORT: ${{ needs.config.outputs.report_path }} - update-with-copilot: + apply: needs: - - configure - - build-dependency-report - if: needs.configure.outputs.should_run == 'true' + - config + - analyze + if: needs.config.outputs.should_run == 'true' runs-on: ubuntu-24.04 + outputs: + changed: ${{ steps.changes.outputs.changed }} + pr_url: ${{ steps.create-pr.outputs.pull-request-url }} permissions: contents: write @@ -414,15 +490,17 @@ jobs: node-version: "22" - name: Install Copilot CLI - run: npm install -g @github/copilot + run: | + npm install -g @github/copilot + echo "Apply setup: installed Copilot CLI" - name: Update Dockerfile with Copilot CLI env: COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} COPILOT_ALLOW_ALL: "true" COPILOT_AUTO_UPDATE: "false" - PROMPT_PATH: ${{ needs.configure.outputs.prompt_path }} - REPORT_PATH: ${{ needs.configure.outputs.report_path }} + PROMPT_PATH: ${{ needs.config.outputs.prompt_path }} + REPORT_PATH: ${{ needs.config.outputs.report_path }} run: | set -euo pipefail @@ -452,6 +530,8 @@ jobs: --silent \ --share copilot-docker-dependency-session.md + echo "Apply Copilot: completed; session saved to copilot-docker-dependency-session.md" + - name: Detect changes id: changes shell: bash @@ -459,16 +539,23 @@ jobs: set -euo pipefail if git diff --quiet -- "${DOCKERFILE}"; then echo "changed=false" >> "${GITHUB_OUTPUT}" + echo "Apply changes: no ${DOCKERFILE} changes" else echo "changed=true" >> "${GITHUB_OUTPUT}" git diff -- "${DOCKERFILE}" > docker-dependency-update.diff + changed_lines="$(wc -l < docker-dependency-update.diff | tr -d ' ')" + echo "Apply changes: ${DOCKERFILE} changed; diff has ${changed_lines} lines" fi env: - DOCKERFILE: ${{ needs.configure.outputs.dockerfile }} + DOCKERFILE: ${{ needs.config.outputs.dockerfile }} - name: Validate Docker build if: steps.changes.outputs.changed == 'true' - run: ${{ needs.configure.outputs.validation_command }} + run: | + bash -lc "${VALIDATION_COMMAND}" + echo "Apply validation: succeeded: ${VALIDATION_COMMAND}" + env: + VALIDATION_COMMAND: ${{ needs.config.outputs.validation_command }} - name: Prepare pull request body if: steps.changes.outputs.changed == 'true' @@ -490,9 +577,10 @@ jobs: sed -n '1,220p' docker-dependency-update.diff echo '```' } > docker-dependency-pr-body.md + echo "Apply PR body: prepared docker-dependency-pr-body.md" env: - REPORT_PATH: ${{ needs.configure.outputs.report_path }} - VALIDATION_COMMAND: ${{ needs.configure.outputs.validation_command }} + REPORT_PATH: ${{ needs.config.outputs.report_path }} + VALIDATION_COMMAND: ${{ needs.config.outputs.validation_command }} - name: Upload Copilot session uses: actions/upload-artifact@v5 @@ -504,12 +592,33 @@ jobs: if-no-files-found: ignore - name: Create pull request + id: create-pr if: steps.changes.outputs.changed == 'true' uses: peter-evans/create-pull-request@v7 with: - commit-message: ${{ needs.configure.outputs.commit_message }} - title: ${{ needs.configure.outputs.pr_title }} + commit-message: ${{ needs.config.outputs.commit_message }} + title: ${{ needs.config.outputs.pr_title }} body-path: docker-dependency-pr-body.md - branch: ${{ needs.configure.outputs.update_branch }} - add-paths: ${{ needs.configure.outputs.dockerfile }} + branch: ${{ needs.config.outputs.update_branch }} + add-paths: ${{ needs.config.outputs.dockerfile }} delete-branch: true + + - name: Write apply summary + if: always() + run: | + pr_url="${PR_URL:-}" + if [ -z "${pr_url}" ]; then + pr_url="n/a" + fi + + { + echo "## Docker Dependency Updater" + echo + echo "| Job | Result | Details |" + echo "| --- | --- | --- |" + echo "| config | ${{ needs.config.result }} | Dockerfile: \`${{ needs.config.outputs.dockerfile }}\`; branch: \`${{ needs.config.outputs.update_branch }}\`; should run: \`${{ needs.config.outputs.should_run }}\` |" + echo "| analyze | ${{ needs.analyze.result }} | Report: \`${{ needs.analyze.outputs.report_path }}\`; apt: \`${{ needs.analyze.outputs.apt_count }}\`; probes: \`${{ needs.analyze.outputs.probe_count }}\`; observed-only: \`${{ needs.analyze.outputs.observed_count }}\` |" + echo "| apply | ${{ job.status }} | Dockerfile changed: \`${{ steps.changes.outputs.changed }}\`; PR: ${pr_url} |" + } >> "${GITHUB_STEP_SUMMARY}" + env: + PR_URL: ${{ steps.create-pr.outputs.pull-request-url }} diff --git a/ci/configs/docker-dependency-probe.yaml b/ci/configs/docker-dependency-probe.yaml index fa4b3ff1..df3534e8 100644 --- a/ci/configs/docker-dependency-probe.yaml +++ b/ci/configs/docker-dependency-probe.yaml @@ -1,7 +1,7 @@ --- # Lightweight hints for Docker dependency update automation. -# The probe script treats Dockerfile as the source of truth and this file as -# repo-specific guidance for Copilot and future probe extensions. +# The workflow treats Dockerfile as the source of truth and this file as +# repo-specific guidance for the analyzer, Copilot, and future probe extensions. dockerfile: Dockerfile From 35dc0aacdd8004ad45b8a31503268dc8a60b9325 Mon Sep 17 00:00:00 2001 From: Dmytro Smirnov Date: Tue, 2 Jun 2026 15:53:39 +0300 Subject: [PATCH 13/32] ci: rename docker automation workflow --- .github/workflows/docker-dependency-updater.yml | 10 +++++----- ci/configs/docker-dependency-probe.yaml | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/docker-dependency-updater.yml b/.github/workflows/docker-dependency-updater.yml index cfaa224d..12d6e091 100644 --- a/.github/workflows/docker-dependency-updater.yml +++ b/.github/workflows/docker-dependency-updater.yml @@ -1,5 +1,5 @@ --- -name: Docker Dependency Updater +name: "UDX Automation: Docker Dependency Updater" "on": schedule: @@ -60,7 +60,7 @@ jobs: "probe_dockerfile" => paths["probe_dockerfile"] || ".tmp/dependency-probe/Dockerfile", "report_path" => paths["report"] || "docker-dependency-report.json", "update_branch" => workflow["update_branch"] || "automation/docker-dependency-updates", - "pr_title" => workflow["pull_request_title"] || "chore(deps): update Docker dependency pins", + "pr_title" => workflow["pull_request_title"] || "UDX Automation: Docker dependency updates", "commit_message" => workflow["commit_message"] || "chore(deps): update Docker dependency pins", "validation_command" => validation.first || "docker build --progress=plain -t dependency-update-validation ." } @@ -124,7 +124,7 @@ jobs: if: always() run: | { - echo "## Docker Dependency Updater" + echo "## UDX Automation: Docker Dependency Updater" echo echo "| Job | Result | Details |" echo "| --- | --- | --- |" @@ -450,7 +450,7 @@ jobs: fi { - echo "## Docker Dependency Updater" + echo "## UDX Automation: Docker Dependency Updater" echo echo "| Job | Result | Details |" echo "| --- | --- | --- |" @@ -612,7 +612,7 @@ jobs: fi { - echo "## Docker Dependency Updater" + echo "## UDX Automation: Docker Dependency Updater" echo echo "| Job | Result | Details |" echo "| --- | --- | --- |" diff --git a/ci/configs/docker-dependency-probe.yaml b/ci/configs/docker-dependency-probe.yaml index df3534e8..e920738a 100644 --- a/ci/configs/docker-dependency-probe.yaml +++ b/ci/configs/docker-dependency-probe.yaml @@ -7,7 +7,7 @@ dockerfile: Dockerfile workflow: update_branch: automation/docker-dependency-updates - pull_request_title: "chore(deps): update Docker dependency pins" + pull_request_title: "UDX Automation: Docker dependency updates" commit_message: "chore(deps): update Docker dependency pins" paths: From f153e2a007ac74728f7ae4b7fa7ff89e6fabe4f9 Mon Sep 17 00:00:00 2001 From: Dmytro Smirnov Date: Tue, 2 Jun 2026 15:54:49 +0300 Subject: [PATCH 14/32] ci: align dependency upgrade workflow title --- .github/workflows/docker-dependency-updater.yml | 10 +++++----- ci/configs/docker-dependency-probe.yaml | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/docker-dependency-updater.yml b/.github/workflows/docker-dependency-updater.yml index 12d6e091..5b267bc6 100644 --- a/.github/workflows/docker-dependency-updater.yml +++ b/.github/workflows/docker-dependency-updater.yml @@ -1,5 +1,5 @@ --- -name: "UDX Automation: Docker Dependency Updater" +name: udx-automation / dependency upgrade "on": schedule: @@ -60,7 +60,7 @@ jobs: "probe_dockerfile" => paths["probe_dockerfile"] || ".tmp/dependency-probe/Dockerfile", "report_path" => paths["report"] || "docker-dependency-report.json", "update_branch" => workflow["update_branch"] || "automation/docker-dependency-updates", - "pr_title" => workflow["pull_request_title"] || "UDX Automation: Docker dependency updates", + "pr_title" => workflow["pull_request_title"] || "udx-automation / dependency upgrade", "commit_message" => workflow["commit_message"] || "chore(deps): update Docker dependency pins", "validation_command" => validation.first || "docker build --progress=plain -t dependency-update-validation ." } @@ -124,7 +124,7 @@ jobs: if: always() run: | { - echo "## UDX Automation: Docker Dependency Updater" + echo "## udx-automation / dependency upgrade" echo echo "| Job | Result | Details |" echo "| --- | --- | --- |" @@ -450,7 +450,7 @@ jobs: fi { - echo "## UDX Automation: Docker Dependency Updater" + echo "## udx-automation / dependency upgrade" echo echo "| Job | Result | Details |" echo "| --- | --- | --- |" @@ -612,7 +612,7 @@ jobs: fi { - echo "## UDX Automation: Docker Dependency Updater" + echo "## udx-automation / dependency upgrade" echo echo "| Job | Result | Details |" echo "| --- | --- | --- |" diff --git a/ci/configs/docker-dependency-probe.yaml b/ci/configs/docker-dependency-probe.yaml index e920738a..70e5f717 100644 --- a/ci/configs/docker-dependency-probe.yaml +++ b/ci/configs/docker-dependency-probe.yaml @@ -7,7 +7,7 @@ dockerfile: Dockerfile workflow: update_branch: automation/docker-dependency-updates - pull_request_title: "UDX Automation: Docker dependency updates" + pull_request_title: "udx-automation / dependency upgrade" commit_message: "chore(deps): update Docker dependency pins" paths: From 35207ef217f69fad1a5ef9f1880eec450413cc7d Mon Sep 17 00:00:00 2001 From: Dmytro Smirnov Date: Tue, 2 Jun 2026 15:57:39 +0300 Subject: [PATCH 15/32] ci: request worker review for dependency upgrades --- .github/workflows/docker-dependency-updater.yml | 8 ++++++-- ci/configs/docker-dependency-probe.yaml | 2 ++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-dependency-updater.yml b/.github/workflows/docker-dependency-updater.yml index 5b267bc6..2372570a 100644 --- a/.github/workflows/docker-dependency-updater.yml +++ b/.github/workflows/docker-dependency-updater.yml @@ -30,6 +30,7 @@ jobs: report_path: ${{ steps.config.outputs.report_path }} update_branch: ${{ steps.config.outputs.update_branch }} pr_title: ${{ steps.config.outputs.pr_title }} + pr_team_reviewers: ${{ steps.config.outputs.pr_team_reviewers }} commit_message: ${{ steps.config.outputs.commit_message }} validation_command: ${{ steps.config.outputs.validation_command }} should_run: ${{ steps.decision.outputs.should_run }} @@ -61,6 +62,7 @@ jobs: "report_path" => paths["report"] || "docker-dependency-report.json", "update_branch" => workflow["update_branch"] || "automation/docker-dependency-updates", "pr_title" => workflow["pull_request_title"] || "udx-automation / dependency upgrade", + "pr_team_reviewers" => Array(workflow["pull_request_team_reviewers"] || []).join(","), "commit_message" => workflow["commit_message"] || "chore(deps): update Docker dependency pins", "validation_command" => validation.first || "docker build --progress=plain -t dependency-update-validation ." } @@ -128,7 +130,7 @@ jobs: echo echo "| Job | Result | Details |" echo "| --- | --- | --- |" - echo "| config | ${{ job.status }} | Dockerfile: \`${{ steps.config.outputs.dockerfile }}\`; branch: \`${{ steps.config.outputs.update_branch }}\`; should run: \`${{ steps.decision.outputs.should_run }}\` |" + echo "| config | ${{ job.status }} | Dockerfile: \`${{ steps.config.outputs.dockerfile }}\`; branch: \`${{ steps.config.outputs.update_branch }}\`; reviewers: \`${{ steps.config.outputs.pr_team_reviewers }}\`; should run: \`${{ steps.decision.outputs.should_run }}\` |" } >> "${GITHUB_STEP_SUMMARY}" analyze: @@ -601,6 +603,8 @@ jobs: body-path: docker-dependency-pr-body.md branch: ${{ needs.config.outputs.update_branch }} add-paths: ${{ needs.config.outputs.dockerfile }} + team-reviewers: ${{ needs.config.outputs.pr_team_reviewers }} + draft: false delete-branch: true - name: Write apply summary @@ -616,7 +620,7 @@ jobs: echo echo "| Job | Result | Details |" echo "| --- | --- | --- |" - echo "| config | ${{ needs.config.result }} | Dockerfile: \`${{ needs.config.outputs.dockerfile }}\`; branch: \`${{ needs.config.outputs.update_branch }}\`; should run: \`${{ needs.config.outputs.should_run }}\` |" + echo "| config | ${{ needs.config.result }} | Dockerfile: \`${{ needs.config.outputs.dockerfile }}\`; branch: \`${{ needs.config.outputs.update_branch }}\`; reviewers: \`${{ needs.config.outputs.pr_team_reviewers }}\`; should run: \`${{ needs.config.outputs.should_run }}\` |" echo "| analyze | ${{ needs.analyze.result }} | Report: \`${{ needs.analyze.outputs.report_path }}\`; apt: \`${{ needs.analyze.outputs.apt_count }}\`; probes: \`${{ needs.analyze.outputs.probe_count }}\`; observed-only: \`${{ needs.analyze.outputs.observed_count }}\` |" echo "| apply | ${{ job.status }} | Dockerfile changed: \`${{ steps.changes.outputs.changed }}\`; PR: ${pr_url} |" } >> "${GITHUB_STEP_SUMMARY}" diff --git a/ci/configs/docker-dependency-probe.yaml b/ci/configs/docker-dependency-probe.yaml index 70e5f717..59eba8f6 100644 --- a/ci/configs/docker-dependency-probe.yaml +++ b/ci/configs/docker-dependency-probe.yaml @@ -8,6 +8,8 @@ dockerfile: Dockerfile workflow: update_branch: automation/docker-dependency-updates pull_request_title: "udx-automation / dependency upgrade" + pull_request_team_reviewers: + - worker commit_message: "chore(deps): update Docker dependency pins" paths: From 24761843ca6b53cd4c5fa027febcaf07fb16ef5e Mon Sep 17 00:00:00 2001 From: Dmytro Smirnov Date: Tue, 2 Jun 2026 16:05:27 +0300 Subject: [PATCH 16/32] ci: use conventional dependency PR title --- .github/workflows/docker-dependency-updater.yml | 2 +- ci/configs/docker-dependency-probe.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-dependency-updater.yml b/.github/workflows/docker-dependency-updater.yml index 2372570a..5acd4a2e 100644 --- a/.github/workflows/docker-dependency-updater.yml +++ b/.github/workflows/docker-dependency-updater.yml @@ -61,7 +61,7 @@ jobs: "probe_dockerfile" => paths["probe_dockerfile"] || ".tmp/dependency-probe/Dockerfile", "report_path" => paths["report"] || "docker-dependency-report.json", "update_branch" => workflow["update_branch"] || "automation/docker-dependency-updates", - "pr_title" => workflow["pull_request_title"] || "udx-automation / dependency upgrade", + "pr_title" => workflow["pull_request_title"] || "chore(deps): docker dependency upgrade", "pr_team_reviewers" => Array(workflow["pull_request_team_reviewers"] || []).join(","), "commit_message" => workflow["commit_message"] || "chore(deps): update Docker dependency pins", "validation_command" => validation.first || "docker build --progress=plain -t dependency-update-validation ." diff --git a/ci/configs/docker-dependency-probe.yaml b/ci/configs/docker-dependency-probe.yaml index 59eba8f6..cec9d215 100644 --- a/ci/configs/docker-dependency-probe.yaml +++ b/ci/configs/docker-dependency-probe.yaml @@ -7,7 +7,7 @@ dockerfile: Dockerfile workflow: update_branch: automation/docker-dependency-updates - pull_request_title: "udx-automation / dependency upgrade" + pull_request_title: "chore(deps): docker dependency upgrade" pull_request_team_reviewers: - worker commit_message: "chore(deps): update Docker dependency pins" From 73447c0a645700e42c45cbc12a2dff0d0b8be5d0 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 2 Jun 2026 16:17:35 +0300 Subject: [PATCH 17/32] chore(deps): update Docker dependency pins (#129) Co-authored-by: fqjony <12067297+fqjony@users.noreply.github.com> --- Dockerfile | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6f473658..aed3e0bb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,10 +4,10 @@ FROM ubuntu:25.10 # Set the maintainer of the image LABEL maintainer="UDX CAG Team" -ARG AZURE_CLI_VERSION=2.85.0 -ARG PIP_VERSION=26.0.1 +ARG AZURE_CLI_VERSION=2.87.0 +ARG PIP_VERSION=26.1.2 ARG YQ_VERSION=4.53.2 -ARG GCLOUD_VERSION=565.0.0 +ARG GCLOUD_VERSION=570.0.0 # Set base environment variables ENV DEBIAN_FRONTEND=noninteractive \ @@ -43,18 +43,18 @@ USER root RUN apt-get update && \ apt-get install -y --no-install-recommends \ tzdata=2026a-0ubuntu0.25.10.1 \ - curl=8.14.1-2ubuntu1.2 \ + curl=8.14.1-2ubuntu1.3 \ bash=5.2.37-2ubuntu5 \ apt-utils=3.1.6ubuntu2 \ gettext=0.23.1-2build2 \ gnupg2=2.4.8-2ubuntu2.1 \ ca-certificates=20250419 \ lsb-release=12.1-1 \ - jq=1.8.1-3ubuntu1 \ + jq=1.8.1-3ubuntu1.1 \ zip=3.0-15ubuntu2 \ unzip=6.0-28ubuntu7 \ nano=8.4-1 \ - vim=2:9.1.0967-1ubuntu6.2 \ + vim=2:9.1.0967-1ubuntu6.5 \ python3.13=3.13.7-1ubuntu0.4 \ python3.13-venv=3.13.7-1ubuntu0.4 \ supervisor=4.2.5-3 && \ From e215ab3d5c2302378b44641dcfb6250b2809ce05 Mon Sep 17 00:00:00 2001 From: Dmytro Smirnov Date: Tue, 2 Jun 2026 16:18:45 +0300 Subject: [PATCH 18/32] ci: enable auto merge for dependency PRs --- .../workflows/docker-dependency-updater.yml | 32 +++++++++++++++++-- ci/configs/docker-dependency-probe.yaml | 2 ++ 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-dependency-updater.yml b/.github/workflows/docker-dependency-updater.yml index 5acd4a2e..13be3359 100644 --- a/.github/workflows/docker-dependency-updater.yml +++ b/.github/workflows/docker-dependency-updater.yml @@ -31,6 +31,8 @@ jobs: update_branch: ${{ steps.config.outputs.update_branch }} pr_title: ${{ steps.config.outputs.pr_title }} pr_team_reviewers: ${{ steps.config.outputs.pr_team_reviewers }} + pr_auto_merge: ${{ steps.config.outputs.pr_auto_merge }} + pr_merge_method: ${{ steps.config.outputs.pr_merge_method }} commit_message: ${{ steps.config.outputs.commit_message }} validation_command: ${{ steps.config.outputs.validation_command }} should_run: ${{ steps.decision.outputs.should_run }} @@ -63,6 +65,8 @@ jobs: "update_branch" => workflow["update_branch"] || "automation/docker-dependency-updates", "pr_title" => workflow["pull_request_title"] || "chore(deps): docker dependency upgrade", "pr_team_reviewers" => Array(workflow["pull_request_team_reviewers"] || []).join(","), + "pr_auto_merge" => workflow.fetch("pull_request_auto_merge", true).to_s, + "pr_merge_method" => workflow["pull_request_merge_method"] || "squash", "commit_message" => workflow["commit_message"] || "chore(deps): update Docker dependency pins", "validation_command" => validation.first || "docker build --progress=plain -t dependency-update-validation ." } @@ -130,7 +134,7 @@ jobs: echo echo "| Job | Result | Details |" echo "| --- | --- | --- |" - echo "| config | ${{ job.status }} | Dockerfile: \`${{ steps.config.outputs.dockerfile }}\`; branch: \`${{ steps.config.outputs.update_branch }}\`; reviewers: \`${{ steps.config.outputs.pr_team_reviewers }}\`; should run: \`${{ steps.decision.outputs.should_run }}\` |" + echo "| config | ${{ job.status }} | Dockerfile: \`${{ steps.config.outputs.dockerfile }}\`; branch: \`${{ steps.config.outputs.update_branch }}\`; reviewers: \`${{ steps.config.outputs.pr_team_reviewers }}\`; auto-merge: \`${{ steps.config.outputs.pr_auto_merge }}\`; should run: \`${{ steps.decision.outputs.should_run }}\` |" } >> "${GITHUB_STEP_SUMMARY}" analyze: @@ -607,6 +611,30 @@ jobs: draft: false delete-branch: true + - name: Enable pull request auto-merge + if: >- + steps.changes.outputs.changed == 'true' && + steps.create-pr.outputs.pull-request-url != '' && + needs.config.outputs.pr_auto_merge == 'true' + run: | + set -euo pipefail + + case "${MERGE_METHOD}" in + merge|rebase|squash) + gh pr merge "${PR_URL}" --auto "--${MERGE_METHOD}" + ;; + *) + echo "Unsupported merge method: ${MERGE_METHOD}" >&2 + exit 1 + ;; + esac + + echo "Apply auto-merge: enabled ${MERGE_METHOD} auto-merge for ${PR_URL}" + env: + GH_TOKEN: ${{ github.token }} + MERGE_METHOD: ${{ needs.config.outputs.pr_merge_method }} + PR_URL: ${{ steps.create-pr.outputs.pull-request-url }} + - name: Write apply summary if: always() run: | @@ -620,7 +648,7 @@ jobs: echo echo "| Job | Result | Details |" echo "| --- | --- | --- |" - echo "| config | ${{ needs.config.result }} | Dockerfile: \`${{ needs.config.outputs.dockerfile }}\`; branch: \`${{ needs.config.outputs.update_branch }}\`; reviewers: \`${{ needs.config.outputs.pr_team_reviewers }}\`; should run: \`${{ needs.config.outputs.should_run }}\` |" + echo "| config | ${{ needs.config.result }} | Dockerfile: \`${{ needs.config.outputs.dockerfile }}\`; branch: \`${{ needs.config.outputs.update_branch }}\`; reviewers: \`${{ needs.config.outputs.pr_team_reviewers }}\`; auto-merge: \`${{ needs.config.outputs.pr_auto_merge }}\`; should run: \`${{ needs.config.outputs.should_run }}\` |" echo "| analyze | ${{ needs.analyze.result }} | Report: \`${{ needs.analyze.outputs.report_path }}\`; apt: \`${{ needs.analyze.outputs.apt_count }}\`; probes: \`${{ needs.analyze.outputs.probe_count }}\`; observed-only: \`${{ needs.analyze.outputs.observed_count }}\` |" echo "| apply | ${{ job.status }} | Dockerfile changed: \`${{ steps.changes.outputs.changed }}\`; PR: ${pr_url} |" } >> "${GITHUB_STEP_SUMMARY}" diff --git a/ci/configs/docker-dependency-probe.yaml b/ci/configs/docker-dependency-probe.yaml index cec9d215..8ab31ab8 100644 --- a/ci/configs/docker-dependency-probe.yaml +++ b/ci/configs/docker-dependency-probe.yaml @@ -10,6 +10,8 @@ workflow: pull_request_title: "chore(deps): docker dependency upgrade" pull_request_team_reviewers: - worker + pull_request_auto_merge: true + pull_request_merge_method: squash commit_message: "chore(deps): update Docker dependency pins" paths: From 21fdfe014df2b54477ed0320be78b8d311d59c87 Mon Sep 17 00:00:00 2001 From: Dmytro Smirnov Date: Tue, 2 Jun 2026 23:23:11 +0300 Subject: [PATCH 19/32] ci: simplify docker dependency probe config --- .../workflows/docker-dependency-updater.yml | 70 ++++---- ci/configs/docker-dependency-probe.yaml | 161 ++++++++++-------- 2 files changed, 124 insertions(+), 107 deletions(-) diff --git a/.github/workflows/docker-dependency-updater.yml b/.github/workflows/docker-dependency-updater.yml index 13be3359..f75706f4 100644 --- a/.github/workflows/docker-dependency-updater.yml +++ b/.github/workflows/docker-dependency-updater.yml @@ -49,36 +49,32 @@ jobs: config_path="ci/configs/docker-dependency-probe.yaml" - ruby -ryaml -e ' - config = YAML.load_file(ARGV.fetch(0)) || {} - workflow = config["workflow"] || {} - paths = config["paths"] || {} - validation = Array(config["validation"]) - - outputs = { - "config_path" => ARGV.fetch(0), - "dockerfile" => config["dockerfile"] || "Dockerfile", - "image" => "worker-deps-probe:#{ENV.fetch("GITHUB_RUN_ID")}", - "prompt_path" => paths["prompt"] || "ci/prompts/docker-dependency-updater.md", - "probe_dockerfile" => paths["probe_dockerfile"] || ".tmp/dependency-probe/Dockerfile", - "report_path" => paths["report"] || "docker-dependency-report.json", - "update_branch" => workflow["update_branch"] || "automation/docker-dependency-updates", - "pr_title" => workflow["pull_request_title"] || "chore(deps): docker dependency upgrade", - "pr_team_reviewers" => Array(workflow["pull_request_team_reviewers"] || []).join(","), - "pr_auto_merge" => workflow.fetch("pull_request_auto_merge", true).to_s, - "pr_merge_method" => workflow["pull_request_merge_method"] || "squash", - "commit_message" => workflow["commit_message"] || "chore(deps): update Docker dependency pins", - "validation_command" => validation.first || "docker build --progress=plain -t dependency-update-validation ." - } + kind="$(yq -r '.kind // ""' "${config_path}")" + version="$(yq -r '.version // ""' "${config_path}")" + if [ "${kind}" != "DockerDependencyProbe" ] || [ "${version}" != "1" ]; then + echo "Expected ${config_path} to declare kind=DockerDependencyProbe and version=1" >&2 + exit 1 + fi - File.open(ENV.fetch("GITHUB_OUTPUT"), "a") do |out| - outputs.each { |key, value| out.puts("#{key}=#{value}") } - end - ' "${config_path}" + { + echo "config_path=${config_path}" + echo "dockerfile=$(yq -r '.dockerfile.path // "Dockerfile"' "${config_path}")" + echo "image=worker-deps-probe:${GITHUB_RUN_ID}" + echo "prompt_path=$(yq -r '.copilot.prompt // "ci/prompts/docker-dependency-updater.md"' "${config_path}")" + echo "probe_dockerfile=$(yq -r '.dockerfile.probe.dockerfile // ".tmp/dependency-probe/Dockerfile"' "${config_path}")" + echo "report_path=$(yq -r '.analysis.report // "docker-dependency-report.json"' "${config_path}")" + echo "update_branch=$(yq -r '.workflow.update_branch // "automation/docker-dependency-updates"' "${config_path}")" + echo "pr_title=$(yq -r '.workflow.pull_request.title // "chore(deps): docker dependency upgrade"' "${config_path}")" + echo "pr_team_reviewers=$(yq -r '.workflow.pull_request.team_reviewers // [] | join(",")' "${config_path}")" + echo "pr_auto_merge=$(yq -r '.workflow.pull_request.auto_merge // true' "${config_path}")" + echo "pr_merge_method=$(yq -r '.workflow.pull_request.merge_method // "squash"' "${config_path}")" + echo "commit_message=$(yq -r '.workflow.commit_message // "chore(deps): update Docker dependency pins"' "${config_path}")" + echo "validation_command=$(yq -r '.workflow.validation.command // "docker build --progress=plain -t dependency-update-validation ."' "${config_path}")" + } >> "${GITHUB_OUTPUT}" test -f "${config_path}" - test -f "$(ruby -ryaml -e 'config = YAML.load_file(ARGV.fetch(0)) || {}; paths = config["paths"] || {}; puts paths["prompt"] || "ci/prompts/docker-dependency-updater.md"' "${config_path}")" - test -f "$(ruby -ryaml -e 'config = YAML.load_file(ARGV.fetch(0)) || {}; puts config["dockerfile"] || "Dockerfile"' "${config_path}")" + test -f "$(yq -r '.copilot.prompt // "ci/prompts/docker-dependency-updater.md"' "${config_path}")" + test -f "$(yq -r '.dockerfile.path // "Dockerfile"' "${config_path}")" echo "Config loaded: ${config_path}" - name: Stop if an update PR is already open @@ -154,9 +150,10 @@ jobs: - name: Render unpinned probe Dockerfile run: | mkdir -p "$(dirname "${PROBE_DOCKERFILE}")" + yq -o=json '.' "${PROBE_CONFIG}" > .tmp/dependency-probe/config.json - ruby -ryaml -e ' - config = YAML.load_file(ENV.fetch("PROBE_CONFIG")) || {} + ruby -rjson -e ' + config = JSON.parse(File.read(ENV.fetch("PROBE_CONFIG_JSON"))) dockerfile = ENV.fetch("DOCKERFILE") output = ENV.fetch("PROBE_DOCKERFILE") @@ -173,8 +170,13 @@ jobs: end end - probes = flatten_specs.call(config["probes"]) + flatten_specs.call(config["observed_only"]) - resolvers = Array(config["latest_resolvers"]) + dependencies = config["dependencies"] || {} + dockerfile_config = config["dockerfile"].is_a?(Hash) ? config["dockerfile"] : {} + probe_config = dockerfile_config["probe"] || {} + + probes = flatten_specs.call(dependencies["probes"]) + probes += flatten_specs.call(dependencies["observed_only"]) + resolvers = Array(probe_config["version_resolvers"]) managed_args = probes.map { |probe| probe["maps_to_arg"] }.compact managed_args += resolvers.map { |resolver| resolver["arg"] }.compact managed_args = managed_args.uniq @@ -244,6 +246,7 @@ jobs: env: DOCKERFILE: ${{ needs.config.outputs.dockerfile }} PROBE_CONFIG: ${{ needs.config.outputs.config_path }} + PROBE_CONFIG_JSON: .tmp/dependency-probe/config.json PROBE_DOCKERFILE: ${{ needs.config.outputs.probe_dockerfile }} - name: Build unpinned dependency probe @@ -322,8 +325,7 @@ jobs: run: | set -euo pipefail - ruby -ryaml -rjson -e 'puts JSON.generate(YAML.load_file(ARGV.fetch(0)) || {})' \ - "${PROBE_CONFIG}" > .tmp/dependency-probe/config.json + yq -o=json '.' "${PROBE_CONFIG}" > .tmp/dependency-probe/config.json docker run -i --rm --entrypoint bash \ -v "${PWD}/.tmp/dependency-probe/config.json:/tmp/probe-config.json:ro" \ @@ -343,7 +345,7 @@ jobs: empty end; - .${section} // {} | probe_specs + (.dependencies.${section} // {}) | probe_specs " /tmp/probe-config.json \ | while IFS= read -r spec; do name="$(jq -r '.name' <<< "${spec}")" diff --git a/ci/configs/docker-dependency-probe.yaml b/ci/configs/docker-dependency-probe.yaml index 8ab31ab8..baa62d8b 100644 --- a/ci/configs/docker-dependency-probe.yaml +++ b/ci/configs/docker-dependency-probe.yaml @@ -1,90 +1,105 @@ --- -# Lightweight hints for Docker dependency update automation. -# The workflow treats Dockerfile as the source of truth and this file as -# repo-specific guidance for the analyzer, Copilot, and future probe extensions. +# Lightweight hints for Docker dependency update automation. Dockerfile remains +# the source of truth; this file only describes repo-specific automation knobs. +kind: DockerDependencyProbe +version: 1 -dockerfile: Dockerfile +# Dockerfile inputs and probe-time rewrites. +dockerfile: + path: Dockerfile + # The probe Dockerfile is a temporary unpinned copy used to build a container + # that reports the versions Docker installs when pins are removed. + probe: + dockerfile: .tmp/dependency-probe/Dockerfile + + # ARGs that must be resolved inside the probe build after their pinned ARG + # declarations are removed. These are probe rewrites, not final update rules. + version_resolvers: + - arg: YQ_VERSION + insert_before_pattern: github.com/mikefarah/yq/releases/download + assignment: >- + YQ_VERSION="$(curl -fsSL https://api.github.com/repos/mikefarah/yq/releases/latest | jq -r .tag_name)" + rewrite: + from: download/v${YQ_VERSION} + to: download/${YQ_VERSION} + + - arg: GCLOUD_VERSION + block_start_pattern: ^# Install Google Cloud SDK + insert_after_pattern: ^RUN ARCH=\$\(uname -m\) && \\$ + assignment: >- + GCLOUD_VERSION="$(curl -fsSL https://dl.google.com/dl/cloudsdk/channels/rapid/components-2.json | jq -r .version)" + +# GitHub Actions behavior for generated dependency upgrade PRs. workflow: update_branch: automation/docker-dependency-updates - pull_request_title: "chore(deps): docker dependency upgrade" - pull_request_team_reviewers: - - worker - pull_request_auto_merge: true - pull_request_merge_method: squash commit_message: "chore(deps): update Docker dependency pins" -paths: - prompt: ci/prompts/docker-dependency-updater.md - probe_dockerfile: .tmp/dependency-probe/Dockerfile - report: docker-dependency-report.json + pull_request: + title: "chore(deps): docker dependency upgrade" + team_reviewers: + - worker + auto_merge: true + merge_method: squash -apt: - discover: true + validation: + command: docker build --progress=plain -t dependency-update-validation . -args: - discover_version_args: true - -latest_resolvers: - - arg: YQ_VERSION - insert_before_pattern: github.com/mikefarah/yq/releases/download - assignment: >- - YQ_VERSION="$(curl -fsSL https://api.github.com/repos/mikefarah/yq/releases/latest | jq -r .tag_name)" - rewrite: - from: download/v${YQ_VERSION} - to: download/${YQ_VERSION} +# Analysis output. +analysis: + report: docker-dependency-report.json - - arg: GCLOUD_VERSION - block_start_pattern: ^# Install Google Cloud SDK - insert_after_pattern: ^RUN ARCH=\$\(uname -m\) && \\$ - assignment: >- - GCLOUD_VERSION="$(curl -fsSL https://dl.google.com/dl/cloudsdk/channels/rapid/components-2.json | jq -r .version)" +# Dependency groups. Apt is discovered from Dockerfile pins, while the other +# registries use explicit commands against the unpinned probe image. +dependencies: + apt: + discover: true + source: ubuntu apt repositories -probes: - pip: - - name: pip - type: command - command: >- - /opt/az/bin/pip --version | awk '{ print $2 }' - maps_to_arg: PIP_VERSION + # Managed probes are pinned dependencies that should map back to Dockerfile + # ARGs. Groups are provider/source families and can be extended without + # workflow edits. + probes: + pip: + - name: pip + type: command + command: >- + /opt/az/bin/pip --version | awk '{ print $2 }' + maps_to_arg: PIP_VERSION - - name: azure-cli - type: command - command: >- - /opt/az/bin/pip show azure-cli | awk -F': ' '$1 == "Version" { print $2 }' - maps_to_arg: AZURE_CLI_VERSION + - name: azure-cli + type: command + command: >- + /opt/az/bin/pip show azure-cli | awk -F': ' '$1 == "Version" { print $2 }' + maps_to_arg: AZURE_CLI_VERSION - github_releases: - - name: yq - type: command - command: >- - yq --version | awk '{ print $NF }' | sed 's/^v//' - maps_to_arg: YQ_VERSION - source: mikefarah/yq + github_releases: + - name: yq + type: command + command: >- + yq --version | awk '{ print $NF }' | sed 's/^v//' + maps_to_arg: YQ_VERSION + source: mikefarah/yq - archives: - - name: google-cloud-sdk - type: command - command: >- - gcloud version --format=json | jq -r '."Google Cloud SDK"' - maps_to_arg: GCLOUD_VERSION - source: Google Cloud SDK rapid channel + archives: + - name: google-cloud-sdk + type: command + command: >- + gcloud version --format=json | jq -r '."Google Cloud SDK"' + maps_to_arg: GCLOUD_VERSION + source: Google Cloud SDK rapid channel -observed_only: - archives: - - name: aws-cli - type: command - command: >- - aws --version 2>&1 | awk '{ print $1 }' | sed 's#aws-cli/##' - reason: Dockerfile installs the latest AWS CLI zip and does not pin it. + # Observed-only probes report tools that are installed dynamically but are not + # pinned in the Dockerfile today. + observed_only: + archives: + - name: aws-cli + type: command + command: >- + aws --version 2>&1 | awk '{ print $1 }' | sed 's#aws-cli/##' + reason: Dockerfile installs the latest AWS CLI zip and does not pin it. +# Copilot prompt input. Behavior lives in the prompt file so instructions are +# not duplicated across two places. copilot: - intent: - - Use Dockerfile and the generated dependency report as primary evidence. - - Use this file as hints, not as the source of truth. - - Update only Dockerfile dependency pins and ARG values. - - Preserve the ubuntu base image tag unless explicitly necessary. - - Report versioned Dockerfile dependencies that are missing probe coverage. - -validation: - - docker build --progress=plain -t dependency-update-validation . + prompt: ci/prompts/docker-dependency-updater.md From 148c2c9a15e4cdad613a6b9739d196a87dfec88a Mon Sep 17 00:00:00 2001 From: Dmytro Smirnov Date: Tue, 2 Jun 2026 23:26:01 +0300 Subject: [PATCH 20/32] ci: delegate external dependency lookup to copilot --- .../workflows/docker-dependency-updater.yml | 53 +++---------------- ci/configs/docker-dependency-probe.yaml | 23 ++------ ci/prompts/docker-dependency-updater.md | 10 ++-- 3 files changed, 19 insertions(+), 67 deletions(-) diff --git a/.github/workflows/docker-dependency-updater.yml b/.github/workflows/docker-dependency-updater.yml index f75706f4..8f32a01e 100644 --- a/.github/workflows/docker-dependency-updater.yml +++ b/.github/workflows/docker-dependency-updater.yml @@ -171,31 +171,17 @@ jobs: end dependencies = config["dependencies"] || {} - dockerfile_config = config["dockerfile"].is_a?(Hash) ? config["dockerfile"] : {} - probe_config = dockerfile_config["probe"] || {} - probes = flatten_specs.call(dependencies["probes"]) - probes += flatten_specs.call(dependencies["observed_only"]) - resolvers = Array(probe_config["version_resolvers"]) - managed_args = probes.map { |probe| probe["maps_to_arg"] }.compact - managed_args += resolvers.map { |resolver| resolver["arg"] }.compact - managed_args = managed_args.uniq - - inserted = {} - active_blocks = {} + managed_args = probes + .select { |probe| (probe["update_strategy"] || "unpin_probe") == "unpin_probe" } + .map { |probe| probe["maps_to_arg"] } + .compact + .uniq + in_apt = false File.open(output, "w") do |out| File.foreach(dockerfile) do |original_line| - resolvers.each do |resolver| - arg = resolver["arg"] - block_start = resolver["block_start_pattern"] - next if arg.nil? || block_start.nil? - - active_blocks[arg] = true if original_line.match?(Regexp.new(block_start)) - active_blocks[arg] = false if original_line.match?(/^# /) && !original_line.match?(Regexp.new(block_start)) - end - arg_match = original_line.match(/^ARG\s+([A-Za-z_][A-Za-z0-9_]*)=/) next if arg_match && managed_args.include?(arg_match[1]) @@ -211,33 +197,8 @@ jobs: line = line.gsub("==${#{arg}}", "") end - resolvers.each do |resolver| - arg = resolver["arg"] - next if arg.nil? || inserted[arg] - next if resolver["block_start_pattern"] && !active_blocks[arg] - - insert_before = resolver["insert_before_pattern"] - if insert_before && line.match?(Regexp.new(insert_before)) - out.puts " #{resolver.fetch("assignment")} && \\" - line = line.gsub(resolver.dig("rewrite", "from"), resolver.dig("rewrite", "to")) if resolver["rewrite"] - inserted[arg] = true - end - end - out.write line - resolvers.each do |resolver| - arg = resolver["arg"] - next if arg.nil? || inserted[arg] - next if resolver["block_start_pattern"] && !active_blocks[arg] - - insert_after = resolver["insert_after_pattern"] - if insert_after && line.match?(Regexp.new(insert_after)) - out.puts " #{resolver.fetch("assignment")} && \\" - inserted[arg] = true - end - end - in_apt = false if in_apt && line.match?(/&&\s*\\?$/) end end @@ -358,6 +319,7 @@ jobs: --arg command "${command}" \ --arg installed "${installed}" \ --arg maps_to_arg "$(jq -r '.maps_to_arg // empty' <<< "${spec}")" \ + --arg update_strategy "$(jq -r '.update_strategy // empty' <<< "${spec}")" \ --arg group "$(jq -r '.group // empty' <<< "${spec}")" \ --arg source "$(jq -r '.source // empty' <<< "${spec}")" \ --arg reason "$(jq -r '.reason // empty' <<< "${spec}")" \ @@ -368,6 +330,7 @@ jobs: installed: $installed } + (if $maps_to_arg == "" then {} else {maps_to_arg: $maps_to_arg} end) + + (if $update_strategy == "" then {} else {update_strategy: $update_strategy} end) + (if $group == "" then {} else {group: $group} end) + (if $source == "" then {} else {source: $source} end) + (if $reason == "" then {} else {reason: $reason, pinned_in_dockerfile: false} end)' diff --git a/ci/configs/docker-dependency-probe.yaml b/ci/configs/docker-dependency-probe.yaml index baa62d8b..27dc593a 100644 --- a/ci/configs/docker-dependency-probe.yaml +++ b/ci/configs/docker-dependency-probe.yaml @@ -4,7 +4,7 @@ kind: DockerDependencyProbe version: 1 -# Dockerfile inputs and probe-time rewrites. +# Dockerfile input and probe build output. dockerfile: path: Dockerfile @@ -13,23 +13,6 @@ dockerfile: probe: dockerfile: .tmp/dependency-probe/Dockerfile - # ARGs that must be resolved inside the probe build after their pinned ARG - # declarations are removed. These are probe rewrites, not final update rules. - version_resolvers: - - arg: YQ_VERSION - insert_before_pattern: github.com/mikefarah/yq/releases/download - assignment: >- - YQ_VERSION="$(curl -fsSL https://api.github.com/repos/mikefarah/yq/releases/latest | jq -r .tag_name)" - rewrite: - from: download/v${YQ_VERSION} - to: download/${YQ_VERSION} - - - arg: GCLOUD_VERSION - block_start_pattern: ^# Install Google Cloud SDK - insert_after_pattern: ^RUN ARCH=\$\(uname -m\) && \\$ - assignment: >- - GCLOUD_VERSION="$(curl -fsSL https://dl.google.com/dl/cloudsdk/channels/rapid/components-2.json | jq -r .version)" - # GitHub Actions behavior for generated dependency upgrade PRs. workflow: update_branch: automation/docker-dependency-updates @@ -63,12 +46,14 @@ dependencies: pip: - name: pip type: command + update_strategy: unpin_probe command: >- /opt/az/bin/pip --version | awk '{ print $2 }' maps_to_arg: PIP_VERSION - name: azure-cli type: command + update_strategy: unpin_probe command: >- /opt/az/bin/pip show azure-cli | awk -F': ' '$1 == "Version" { print $2 }' maps_to_arg: AZURE_CLI_VERSION @@ -76,6 +61,7 @@ dependencies: github_releases: - name: yq type: command + update_strategy: copilot_latest command: >- yq --version | awk '{ print $NF }' | sed 's/^v//' maps_to_arg: YQ_VERSION @@ -84,6 +70,7 @@ dependencies: archives: - name: google-cloud-sdk type: command + update_strategy: copilot_latest command: >- gcloud version --format=json | jq -r '."Google Cloud SDK"' maps_to_arg: GCLOUD_VERSION diff --git a/ci/prompts/docker-dependency-updater.md b/ci/prompts/docker-dependency-updater.md index 9e3e5e8f..a990a362 100644 --- a/ci/prompts/docker-dependency-updater.md +++ b/ci/prompts/docker-dependency-updater.md @@ -2,11 +2,12 @@ You are updating Dockerfile dependency pins for this repository. Intent: - Read Dockerfile and docker-dependency-report.json. -- The report was produced by building a temporary copy of Dockerfile with dependency version pins removed. +- The report was produced by building a temporary copy of Dockerfile with apt pins and selected ARG pins removed. - The report may include probe_config.content from ci/configs/docker-dependency-probe.yaml. -- Treat the report as the primary evidence for versions that install successfully for the current base image. +- Treat the report as the primary evidence for versions that install successfully for the current base image when a dependency uses `update_strategy: unpin_probe`. - Treat probe_config.content as repo-specific hints, not as the source of truth. -- Update only Dockerfile dependency pins and ARG values when the report shows a newer installed version. +- For dependencies with `update_strategy: copilot_latest`, use the report and Dockerfile to identify the current installed version, then check the dependency source for the latest stable version. +- Update only Dockerfile dependency pins and ARG values when the report or checked source shows a newer version. - Preserve the ubuntu base image tag unless explicitly necessary to make the reported versions valid. - Keep observed-only dependencies unpinned unless Dockerfile already pins them; mention their latest observed versions in the changelog only. - Do not edit workflow files, docs, tests, or application code. @@ -15,7 +16,8 @@ Intent: Expected Dockerfile update categories: - apt package pins from dependencies.apt[].installed -- ARG pins from dependencies.probes[] entries with maps_to_arg values +- ARG pins from dependencies.probes[] entries with maps_to_arg values and `update_strategy: unpin_probe` +- ARG pins from dependencies.probes[] entries with maps_to_arg values and `update_strategy: copilot_latest`, after checking their source - observed-only tools from dependencies.observed_only[] entries After editing: From 01020a175e443a163a187337213309cc90770f9c Mon Sep 17 00:00:00 2001 From: Dmytro Smirnov Date: Tue, 2 Jun 2026 23:28:51 +0300 Subject: [PATCH 21/32] ci: render dependency probe with jq --- .../workflows/docker-dependency-updater.yml | 108 +++++++++--------- 1 file changed, 57 insertions(+), 51 deletions(-) diff --git a/.github/workflows/docker-dependency-updater.yml b/.github/workflows/docker-dependency-updater.yml index 8f32a01e..823c8613 100644 --- a/.github/workflows/docker-dependency-updater.yml +++ b/.github/workflows/docker-dependency-updater.yml @@ -149,65 +149,71 @@ jobs: - name: Render unpinned probe Dockerfile run: | + mkdir -p .tmp/dependency-probe mkdir -p "$(dirname "${PROBE_DOCKERFILE}")" yq -o=json '.' "${PROBE_CONFIG}" > .tmp/dependency-probe/config.json - ruby -rjson -e ' - config = JSON.parse(File.read(ENV.fetch("PROBE_CONFIG_JSON"))) - dockerfile = ENV.fetch("DOCKERFILE") - output = ENV.fetch("PROBE_DOCKERFILE") - - flatten_specs = lambda do |value| - case value - when Array - value - when Hash - value.flat_map do |group, specs| - Array(specs).map { |spec| spec.merge("group" => spec["group"] || group) } - end + jq -r ' + def probe_specs: + if type == "array" then + .[] + elif type == "object" then + to_entries[] | .value[] else - [] - end - end - - dependencies = config["dependencies"] || {} - probes = flatten_specs.call(dependencies["probes"]) - managed_args = probes - .select { |probe| (probe["update_strategy"] || "unpin_probe") == "unpin_probe" } - .map { |probe| probe["maps_to_arg"] } - .compact - .uniq - - in_apt = false - - File.open(output, "w") do |out| - File.foreach(dockerfile) do |original_line| - arg_match = original_line.match(/^ARG\s+([A-Za-z_][A-Za-z0-9_]*)=/) - next if arg_match && managed_args.include?(arg_match[1]) - - line = original_line.dup - - if line.include?("apt-get install -y --no-install-recommends") - in_apt = true - elsif in_apt - line = line.sub(/^(\s+[A-Za-z0-9.+-]+)=\S+(.+)$/, "\\1\\2") - end - - managed_args.each do |arg| - line = line.gsub("==${#{arg}}", "") - end - - out.write line - - in_apt = false if in_apt && line.match?(/&&\s*\\?$/) - end - end - ' + empty + end; + + [ + .dependencies.probes // {} + | probe_specs + | select((.update_strategy // "unpin_probe") == "unpin_probe") + | .maps_to_arg // empty + ] + | unique + | .[] + ' .tmp/dependency-probe/config.json > .tmp/dependency-probe/managed-args.txt + + awk -v args_file=.tmp/dependency-probe/managed-args.txt ' + BEGIN { + while ((getline arg < args_file) > 0) { + managed[arg] = 1 + } + } + + /^ARG[[:space:]]+[A-Za-z_][A-Za-z0-9_]*=/ { + arg_name = $0 + sub(/^ARG[[:space:]]+/, "", arg_name) + sub(/=.*/, "", arg_name) + if (arg_name in managed) { + next + } + } + + { + line = $0 + + if (line ~ /apt-get install -y --no-install-recommends/) { + in_apt = 1 + } else if (in_apt) { + sub(/^([[:space:]]+[A-Za-z0-9.+-]+)=\S+(.*)$/, "\\1\\2", line) + } + + for (arg in managed) { + gsub("==\\$\\{" arg "\\}", "", line) + } + + print line + + if (in_apt && line ~ /&&[[:space:]]*\\?$/) { + in_apt = 0 + } + } + ' "${DOCKERFILE}" > "${PROBE_DOCKERFILE}" + echo "Analyze render: ${DOCKERFILE} -> ${PROBE_DOCKERFILE}" env: DOCKERFILE: ${{ needs.config.outputs.dockerfile }} PROBE_CONFIG: ${{ needs.config.outputs.config_path }} - PROBE_CONFIG_JSON: .tmp/dependency-probe/config.json PROBE_DOCKERFILE: ${{ needs.config.outputs.probe_dockerfile }} - name: Build unpinned dependency probe From 6c8328b82fe6b8caa53326b7ecb5810d5a9f86ab Mon Sep 17 00:00:00 2001 From: Dmytro Smirnov Date: Tue, 2 Jun 2026 23:41:20 +0300 Subject: [PATCH 22/32] ci: simplify docker dependency probe --- .../workflows/docker-dependency-updater.yml | 206 ++---------------- ci/configs/docker-dependency-probe.yaml | 56 +---- ci/prompts/docker-dependency-updater.md | 24 +- 3 files changed, 35 insertions(+), 251 deletions(-) diff --git a/.github/workflows/docker-dependency-updater.yml b/.github/workflows/docker-dependency-updater.yml index 823c8613..01ba20ff 100644 --- a/.github/workflows/docker-dependency-updater.yml +++ b/.github/workflows/docker-dependency-updater.yml @@ -139,67 +139,25 @@ jobs: runs-on: ubuntu-24.04 outputs: apt_count: ${{ steps.report.outputs.apt_count }} - observed_count: ${{ steps.report.outputs.observed_count }} - probe_count: ${{ steps.report.outputs.probe_count }} report_path: ${{ needs.config.outputs.report_path }} steps: - name: Checkout repository uses: actions/checkout@v6 - - name: Render unpinned probe Dockerfile + - name: Render apt probe Dockerfile run: | mkdir -p .tmp/dependency-probe mkdir -p "$(dirname "${PROBE_DOCKERFILE}")" - yq -o=json '.' "${PROBE_CONFIG}" > .tmp/dependency-probe/config.json - - jq -r ' - def probe_specs: - if type == "array" then - .[] - elif type == "object" then - to_entries[] | .value[] - else - empty - end; - - [ - .dependencies.probes // {} - | probe_specs - | select((.update_strategy // "unpin_probe") == "unpin_probe") - | .maps_to_arg // empty - ] - | unique - | .[] - ' .tmp/dependency-probe/config.json > .tmp/dependency-probe/managed-args.txt - - awk -v args_file=.tmp/dependency-probe/managed-args.txt ' - BEGIN { - while ((getline arg < args_file) > 0) { - managed[arg] = 1 - } - } - - /^ARG[[:space:]]+[A-Za-z_][A-Za-z0-9_]*=/ { - arg_name = $0 - sub(/^ARG[[:space:]]+/, "", arg_name) - sub(/=.*/, "", arg_name) - if (arg_name in managed) { - next - } - } + awk ' { line = $0 if (line ~ /apt-get install -y --no-install-recommends/) { in_apt = 1 } else if (in_apt) { - sub(/^([[:space:]]+[A-Za-z0-9.+-]+)=\S+(.*)$/, "\\1\\2", line) - } - - for (arg in managed) { - gsub("==\\$\\{" arg "\\}", "", line) + sub(/^([[:space:]]+[A-Za-z0-9.+-]+)=[^[:space:]]+(.*)$/, "\\1\\2", line) } print line @@ -210,13 +168,12 @@ jobs: } ' "${DOCKERFILE}" > "${PROBE_DOCKERFILE}" - echo "Analyze render: ${DOCKERFILE} -> ${PROBE_DOCKERFILE}" + echo "Analyze render: created apt-only probe ${PROBE_DOCKERFILE} from ${DOCKERFILE}" env: DOCKERFILE: ${{ needs.config.outputs.dockerfile }} - PROBE_CONFIG: ${{ needs.config.outputs.config_path }} PROBE_DOCKERFILE: ${{ needs.config.outputs.probe_dockerfile }} - - name: Build unpinned dependency probe + - name: Build apt dependency probe run: | docker build --pull --no-cache -f "${PROBE_DOCKERFILE}" -t "${PROBE_IMAGE}" . echo "Analyze build: built ${PROBE_IMAGE} from ${PROBE_DOCKERFILE}" @@ -224,9 +181,10 @@ jobs: PROBE_DOCKERFILE: ${{ needs.config.outputs.probe_dockerfile }} PROBE_IMAGE: ${{ needs.config.outputs.image }} - - name: Discover pinned apt packages + - name: Write apt dependency report + id: report run: | - mkdir -p .tmp/dependency-probe + set -euo pipefail awk ' /apt-get install -y --no-install-recommends/ { in_block=1; next } @@ -244,15 +202,6 @@ jobs: } ' "${DOCKERFILE}" > .tmp/dependency-probe/apt-packages.txt - apt_count="$(wc -l < .tmp/dependency-probe/apt-packages.txt | tr -d ' ')" - echo "Analyze apt discovery: found ${apt_count} pinned apt packages in ${DOCKERFILE}" - env: - DOCKERFILE: ${{ needs.config.outputs.dockerfile }} - - - name: Collect apt inventory - run: | - set -euo pipefail - mapfile -t apt_packages < .tmp/dependency-probe/apt-packages.txt docker run -i --rm --entrypoint bash "${PROBE_IMAGE}" -s -- "${apt_packages[@]}" > .tmp/dependency-probe/apt.json <<'EOF' @@ -266,122 +215,18 @@ jobs: ' EOF - apt_count="$(jq 'length' .tmp/dependency-probe/apt.json)" - echo "Analyze apt inventory: collected installed versions for ${apt_count} packages" - env: - PROBE_IMAGE: ${{ needs.config.outputs.image }} - - - name: Collect runtime metadata - run: | - docker run -i --rm --entrypoint bash "${PROBE_IMAGE}" -s > .tmp/dependency-probe/runtime.json <<'EOF' - set -euo pipefail - - jq -n \ - --arg os "$(. /etc/os-release && printf '%s' "${PRETTY_NAME}")" \ - --arg arch "$(uname -m)" \ - '{os: $os, architecture: $arch}' - EOF - - os="$(jq -r '.os' .tmp/dependency-probe/runtime.json)" - arch="$(jq -r '.architecture' .tmp/dependency-probe/runtime.json)" - echo "Analyze runtime: ${os} on ${arch}" - env: - PROBE_IMAGE: ${{ needs.config.outputs.image }} - - - name: Collect configured probe inventory - run: | - set -euo pipefail - - yq -o=json '.' "${PROBE_CONFIG}" > .tmp/dependency-probe/config.json - - docker run -i --rm --entrypoint bash \ - -v "${PWD}/.tmp/dependency-probe/config.json:/tmp/probe-config.json:ro" \ - "${PROBE_IMAGE}" -s > .tmp/dependency-probe/probes.json <<'EOF' - set -euo pipefail - - run_probe_commands() { - local section="$1" - - jq -c " - def probe_specs: - if type == \"array\" then - .[] - elif type == \"object\" then - to_entries[] | .key as \$group | .value[] | . + {group: (.group // \$group)} - else - empty - end; - - (.dependencies.${section} // {}) | probe_specs - " /tmp/probe-config.json \ - | while IFS= read -r spec; do - name="$(jq -r '.name' <<< "${spec}")" - command="$(jq -r '.command' <<< "${spec}")" - installed="$(bash -o pipefail -c "${command}")" - - jq -n \ - --arg name "${name}" \ - --arg type "$(jq -r '.type // "command"' <<< "${spec}")" \ - --arg command "${command}" \ - --arg installed "${installed}" \ - --arg maps_to_arg "$(jq -r '.maps_to_arg // empty' <<< "${spec}")" \ - --arg update_strategy "$(jq -r '.update_strategy // empty' <<< "${spec}")" \ - --arg group "$(jq -r '.group // empty' <<< "${spec}")" \ - --arg source "$(jq -r '.source // empty' <<< "${spec}")" \ - --arg reason "$(jq -r '.reason // empty' <<< "${spec}")" \ - '{ - name: $name, - type: $type, - command: $command, - installed: $installed - } - + (if $maps_to_arg == "" then {} else {maps_to_arg: $maps_to_arg} end) - + (if $update_strategy == "" then {} else {update_strategy: $update_strategy} end) - + (if $group == "" then {} else {group: $group} end) - + (if $source == "" then {} else {source: $source} end) - + (if $reason == "" then {} else {reason: $reason, pinned_in_dockerfile: false} end)' - done \ - | jq -s '.' - } - - probes_json="$(run_probe_commands probes)" - observed_only_json="$(run_probe_commands observed_only)" - - jq -n \ - --argjson probes "${probes_json}" \ - --argjson observed_only "${observed_only_json}" \ - '{probes: $probes, observed_only: $observed_only}' - EOF - - probe_count="$(jq '.probes | length' .tmp/dependency-probe/probes.json)" - observed_count="$(jq '.observed_only | length' .tmp/dependency-probe/probes.json)" - echo "Analyze probes: collected ${probe_count} managed probes and ${observed_count} observed-only probes" - env: - PROBE_CONFIG: ${{ needs.config.outputs.config_path }} - PROBE_IMAGE: ${{ needs.config.outputs.image }} - - - name: Assemble dependency report - id: report - run: | - set -euo pipefail - base_image="$(awk '$1 == "FROM" { print $2; exit }' "${DOCKERFILE}")" jq -n \ --arg generated_at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ --arg base_image "${base_image}" \ - --slurpfile runtime .tmp/dependency-probe/runtime.json \ --slurpfile apt .tmp/dependency-probe/apt.json \ - --slurpfile probes .tmp/dependency-probe/probes.json \ '{ generated_at: $generated_at, - method: "unpinned Dockerfile probe build", + method: "apt-only Dockerfile probe build", base_image: $base_image, - runtime: $runtime[0], dependencies: { - apt: $apt[0], - probes: $probes[0].probes, - observed_only: $probes[0].observed_only + apt: $apt[0] } }' > "${PROBE_REPORT}" @@ -394,15 +239,12 @@ jobs: mv "${report_tmp}" "${PROBE_REPORT}" apt_count="$(jq '.dependencies.apt | length' "${PROBE_REPORT}")" - probe_count="$(jq '.dependencies.probes | length' "${PROBE_REPORT}")" - observed_count="$(jq '.dependencies.observed_only | length' "${PROBE_REPORT}")" printf 'apt_count=%s\n' "${apt_count}" >> "${GITHUB_OUTPUT}" - printf 'probe_count=%s\n' "${probe_count}" >> "${GITHUB_OUTPUT}" - printf 'observed_count=%s\n' "${observed_count}" >> "${GITHUB_OUTPUT}" - echo "Analyze report: wrote ${PROBE_REPORT} with apt=${apt_count}, probes=${probe_count}, observed-only=${observed_count}" + echo "Analyze report: wrote ${PROBE_REPORT} with ${apt_count} apt packages" env: DOCKERFILE: ${{ needs.config.outputs.dockerfile }} PROBE_CONFIG: ${{ needs.config.outputs.config_path }} + PROBE_IMAGE: ${{ needs.config.outputs.image }} PROBE_REPORT: ${{ needs.config.outputs.report_path }} - name: Upload dependency report @@ -417,13 +259,9 @@ jobs: if: always() run: | apt_count="n/a" - probe_count="n/a" - observed_count="n/a" if [ -f "${PROBE_REPORT}" ]; then apt_count="$(jq '.dependencies.apt | length' "${PROBE_REPORT}")" - probe_count="$(jq '.dependencies.probes | length' "${PROBE_REPORT}")" - observed_count="$(jq '.dependencies.observed_only | length' "${PROBE_REPORT}")" fi { @@ -431,7 +269,7 @@ jobs: echo echo "| Job | Result | Details |" echo "| --- | --- | --- |" - echo "| analyze | ${{ job.status }} | Report: \`${PROBE_REPORT}\`; apt: \`${apt_count}\`; probes: \`${probe_count}\`; observed-only: \`${observed_count}\` |" + echo "| analyze | ${{ job.status }} | Report: \`${PROBE_REPORT}\`; apt: \`${apt_count}\` |" } >> "${GITHUB_STEP_SUMMARY}" env: PROBE_REPORT: ${{ needs.config.outputs.report_path }} @@ -486,20 +324,8 @@ jobs: exit 1 fi - cp "${PROMPT_PATH}" /tmp/docker-dependency-updater-prompt.md - - { - echo - echo "Dependency report:" - echo - echo '```json' - cat "${REPORT_PATH}" - echo - echo '```' - } >> /tmp/docker-dependency-updater-prompt.md - copilot \ - --prompt "$(cat /tmp/docker-dependency-updater-prompt.md)" \ + --prompt "$(cat "${PROMPT_PATH}")" \ --allow-all-tools \ --allow-all-urls \ --no-ask-user \ @@ -540,7 +366,7 @@ jobs: { echo "## Summary" echo - echo "Updates Dockerfile dependency pins using an unpinned probe build and Copilot CLI." + echo "Updates Dockerfile dependency pins using an apt-only probe build and Copilot CLI." echo echo "## Evidence" echo @@ -620,7 +446,7 @@ jobs: echo "| Job | Result | Details |" echo "| --- | --- | --- |" echo "| config | ${{ needs.config.result }} | Dockerfile: \`${{ needs.config.outputs.dockerfile }}\`; branch: \`${{ needs.config.outputs.update_branch }}\`; reviewers: \`${{ needs.config.outputs.pr_team_reviewers }}\`; auto-merge: \`${{ needs.config.outputs.pr_auto_merge }}\`; should run: \`${{ needs.config.outputs.should_run }}\` |" - echo "| analyze | ${{ needs.analyze.result }} | Report: \`${{ needs.analyze.outputs.report_path }}\`; apt: \`${{ needs.analyze.outputs.apt_count }}\`; probes: \`${{ needs.analyze.outputs.probe_count }}\`; observed-only: \`${{ needs.analyze.outputs.observed_count }}\` |" + echo "| analyze | ${{ needs.analyze.result }} | Report: \`${{ needs.analyze.outputs.report_path }}\`; apt: \`${{ needs.analyze.outputs.apt_count }}\` |" echo "| apply | ${{ job.status }} | Dockerfile changed: \`${{ steps.changes.outputs.changed }}\`; PR: ${pr_url} |" } >> "${GITHUB_STEP_SUMMARY}" env: diff --git a/ci/configs/docker-dependency-probe.yaml b/ci/configs/docker-dependency-probe.yaml index 27dc593a..5afae533 100644 --- a/ci/configs/docker-dependency-probe.yaml +++ b/ci/configs/docker-dependency-probe.yaml @@ -8,8 +8,8 @@ version: 1 dockerfile: path: Dockerfile - # The probe Dockerfile is a temporary unpinned copy used to build a container - # that reports the versions Docker installs when pins are removed. + # The probe Dockerfile is a temporary copy used to build a container that + # reports the apt versions Docker installs when apt pins are removed. probe: dockerfile: .tmp/dependency-probe/Dockerfile @@ -32,60 +32,14 @@ workflow: analysis: report: docker-dependency-report.json -# Dependency groups. Apt is discovered from Dockerfile pins, while the other -# registries use explicit commands against the unpinned probe image. +# Dependency groups. Apt is discovered from Dockerfile pins and probed +# deterministically; non-apt dependency discovery is delegated to Copilot from +# the Dockerfile and prompt. dependencies: apt: discover: true source: ubuntu apt repositories - # Managed probes are pinned dependencies that should map back to Dockerfile - # ARGs. Groups are provider/source families and can be extended without - # workflow edits. - probes: - pip: - - name: pip - type: command - update_strategy: unpin_probe - command: >- - /opt/az/bin/pip --version | awk '{ print $2 }' - maps_to_arg: PIP_VERSION - - - name: azure-cli - type: command - update_strategy: unpin_probe - command: >- - /opt/az/bin/pip show azure-cli | awk -F': ' '$1 == "Version" { print $2 }' - maps_to_arg: AZURE_CLI_VERSION - - github_releases: - - name: yq - type: command - update_strategy: copilot_latest - command: >- - yq --version | awk '{ print $NF }' | sed 's/^v//' - maps_to_arg: YQ_VERSION - source: mikefarah/yq - - archives: - - name: google-cloud-sdk - type: command - update_strategy: copilot_latest - command: >- - gcloud version --format=json | jq -r '."Google Cloud SDK"' - maps_to_arg: GCLOUD_VERSION - source: Google Cloud SDK rapid channel - - # Observed-only probes report tools that are installed dynamically but are not - # pinned in the Dockerfile today. - observed_only: - archives: - - name: aws-cli - type: command - command: >- - aws --version 2>&1 | awk '{ print $1 }' | sed 's#aws-cli/##' - reason: Dockerfile installs the latest AWS CLI zip and does not pin it. - # Copilot prompt input. Behavior lives in the prompt file so instructions are # not duplicated across two places. copilot: diff --git a/ci/prompts/docker-dependency-updater.md b/ci/prompts/docker-dependency-updater.md index a990a362..5ff6f4f6 100644 --- a/ci/prompts/docker-dependency-updater.md +++ b/ci/prompts/docker-dependency-updater.md @@ -1,24 +1,28 @@ You are updating Dockerfile dependency pins for this repository. +Inputs: +- Dockerfile: `Dockerfile` +- Dependency report: `docker-dependency-report.json` + Intent: -- Read Dockerfile and docker-dependency-report.json. -- The report was produced by building a temporary copy of Dockerfile with apt pins and selected ARG pins removed. +- Read both input files before editing. +- The report was produced by building a temporary copy of Dockerfile with apt pins removed. - The report may include probe_config.content from ci/configs/docker-dependency-probe.yaml. -- Treat the report as the primary evidence for versions that install successfully for the current base image when a dependency uses `update_strategy: unpin_probe`. -- Treat probe_config.content as repo-specific hints, not as the source of truth. -- For dependencies with `update_strategy: copilot_latest`, use the report and Dockerfile to identify the current installed version, then check the dependency source for the latest stable version. +- Treat the report as the primary evidence for apt package versions that install successfully for the current base image. +- Treat Dockerfile as the source of truth for every non-apt dependency. Detect ARG-pinned versions, URL-pinned versions, package-manager pins, and dynamically installed tools directly from Dockerfile. +- For non-apt dependencies, identify the upstream source from Dockerfile context, then check that source for the latest stable version. - Update only Dockerfile dependency pins and ARG values when the report or checked source shows a newer version. - Preserve the ubuntu base image tag unless explicitly necessary to make the reported versions valid. -- Keep observed-only dependencies unpinned unless Dockerfile already pins them; mention their latest observed versions in the changelog only. +- Keep dynamically installed dependencies unpinned unless Dockerfile already pins them; mention them in the changelog only. - Do not edit workflow files, docs, tests, or application code. - Do not commit, push, or create a pull request; this workflow will do that. -- If Dockerfile contains a versioned dependency that is not represented in the report or probe_config, mention it as missing probe coverage. +- If a Dockerfile dependency cannot be checked confidently, leave it unchanged and mention the reason in the changelog. Expected Dockerfile update categories: - apt package pins from dependencies.apt[].installed -- ARG pins from dependencies.probes[] entries with maps_to_arg values and `update_strategy: unpin_probe` -- ARG pins from dependencies.probes[] entries with maps_to_arg values and `update_strategy: copilot_latest`, after checking their source -- observed-only tools from dependencies.observed_only[] entries +- Non-apt ARG pins discovered in Dockerfile after checking their upstream source. +- Non-apt URL/package-manager pins discovered in Dockerfile after checking their upstream source. +- Dynamically installed tools discovered in Dockerfile that intentionally remain unpinned. After editing: - Print a concise changelog of every version change. From 3bcf58da3fbdacb1aa44c39b909292b19c0c3984 Mon Sep 17 00:00:00 2001 From: Dmytro Smirnov Date: Tue, 2 Jun 2026 23:52:56 +0300 Subject: [PATCH 23/32] ci: clarify docker dependency prompt output --- ci/prompts/docker-dependency-updater.md | 48 ++++++++++++++++--------- 1 file changed, 32 insertions(+), 16 deletions(-) diff --git a/ci/prompts/docker-dependency-updater.md b/ci/prompts/docker-dependency-updater.md index 5ff6f4f6..ca43af0e 100644 --- a/ci/prompts/docker-dependency-updater.md +++ b/ci/prompts/docker-dependency-updater.md @@ -4,26 +4,42 @@ Inputs: - Dockerfile: `Dockerfile` - Dependency report: `docker-dependency-report.json` -Intent: +Automation contract: - Read both input files before editing. -- The report was produced by building a temporary copy of Dockerfile with apt pins removed. -- The report may include probe_config.content from ci/configs/docker-dependency-probe.yaml. -- Treat the report as the primary evidence for apt package versions that install successfully for the current base image. -- Treat Dockerfile as the source of truth for every non-apt dependency. Detect ARG-pinned versions, URL-pinned versions, package-manager pins, and dynamically installed tools directly from Dockerfile. -- For non-apt dependencies, identify the upstream source from Dockerfile context, then check that source for the latest stable version. -- Update only Dockerfile dependency pins and ARG values when the report or checked source shows a newer version. +- The dependency report already resolves current apt package versions for the configured Ubuntu base image. +- You are responsible for detecting and checking every non-apt dependency directly from Dockerfile. + +Dependency handling: +- For apt packages, update Dockerfile pins only from `dependencies.apt[].installed` in the dependency report. +- For non-apt dependencies, detect ARG-pinned versions, URL-pinned versions, package-manager pins, and dynamically installed tools from Dockerfile. +- For each non-apt pinned dependency, identify its upstream source from Dockerfile context and check the latest stable version. +- Update only Dockerfile dependency pins and ARG values when the report or upstream source shows a newer version. - Preserve the ubuntu base image tag unless explicitly necessary to make the reported versions valid. - Keep dynamically installed dependencies unpinned unless Dockerfile already pins them; mention them in the changelog only. +- If a Dockerfile dependency cannot be checked confidently, leave it unchanged and mention the reason in the changelog. + +Editing rules: +- Edit only Dockerfile dependency pins and ARG values. - Do not edit workflow files, docs, tests, or application code. - Do not commit, push, or create a pull request; this workflow will do that. -- If a Dockerfile dependency cannot be checked confidently, leave it unchanged and mention the reason in the changelog. -Expected Dockerfile update categories: -- apt package pins from dependencies.apt[].installed -- Non-apt ARG pins discovered in Dockerfile after checking their upstream source. -- Non-apt URL/package-manager pins discovered in Dockerfile after checking their upstream source. -- Dynamically installed tools discovered in Dockerfile that intentionally remain unpinned. +Output: +- Print a changelog using this template: + +```text +Docker dependency changelog + +Updated: +- : -> () + +Unchanged: +- : () + +Observed but not pinned: +- : () + +Notes: +- +``` -After editing: -- Print a concise changelog of every version change. -- Print any dependency that was observed but intentionally not pinned. +- Omit empty sections except `Notes`. From 7165f5f8a16e44730b072e7f3f9d98a49f19a463 Mon Sep 17 00:00:00 2001 From: Dmytro Smirnov Date: Wed, 3 Jun 2026 12:05:00 +0300 Subject: [PATCH 24/32] ci: simplify docker dependency updater --- .../workflows/docker-dependency-updater.yml | 259 ++++++++---------- ci/configs/docker-dependency-probe.yaml | 46 ---- ci/prompts/docker-dependency-updater.md | 6 +- 3 files changed, 118 insertions(+), 193 deletions(-) delete mode 100644 ci/configs/docker-dependency-probe.yaml diff --git a/.github/workflows/docker-dependency-updater.yml b/.github/workflows/docker-dependency-updater.yml index 01ba20ff..4d6fa9c7 100644 --- a/.github/workflows/docker-dependency-updater.yml +++ b/.github/workflows/docker-dependency-updater.yml @@ -11,6 +11,17 @@ permissions: env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" + DOCKERFILE: Dockerfile + PROBE_DOCKERFILE: .tmp/dependency-probe/Dockerfile + REPORT_PATH: docker-dependency-report.json + UPDATE_BRANCH: docker-dependency-updates + PR_TITLE: "chore(deps): docker dependency upgrade" + PR_TEAM_REVIEWERS: worker + PR_AUTO_MERGE: "true" + PR_MERGE_METHOD: squash + COMMIT_MESSAGE: "chore(deps): update Docker dependency pins" + VALIDATION_COMMAND: docker build --progress=plain -t dependency-update-validation . + COPILOT_PROMPT_PATH: ci/prompts/docker-dependency-updater.md jobs: config: @@ -21,7 +32,6 @@ jobs: pull-requests: read outputs: - config_path: ${{ steps.config.outputs.config_path }} dockerfile: ${{ steps.config.outputs.dockerfile }} existing_pr_found: ${{ steps.existing-pr.outputs.found }} image: ${{ steps.config.outputs.image }} @@ -42,40 +52,32 @@ jobs: - name: Checkout repository uses: actions/checkout@v6 - - name: Load dependency updater config + - name: Load dependency updater defaults id: config run: | set -euo pipefail - config_path="ci/configs/docker-dependency-probe.yaml" - - kind="$(yq -r '.kind // ""' "${config_path}")" - version="$(yq -r '.version // ""' "${config_path}")" - if [ "${kind}" != "DockerDependencyProbe" ] || [ "${version}" != "1" ]; then - echo "Expected ${config_path} to declare kind=DockerDependencyProbe and version=1" >&2 - exit 1 - fi - { - echo "config_path=${config_path}" - echo "dockerfile=$(yq -r '.dockerfile.path // "Dockerfile"' "${config_path}")" echo "image=worker-deps-probe:${GITHUB_RUN_ID}" - echo "prompt_path=$(yq -r '.copilot.prompt // "ci/prompts/docker-dependency-updater.md"' "${config_path}")" - echo "probe_dockerfile=$(yq -r '.dockerfile.probe.dockerfile // ".tmp/dependency-probe/Dockerfile"' "${config_path}")" - echo "report_path=$(yq -r '.analysis.report // "docker-dependency-report.json"' "${config_path}")" - echo "update_branch=$(yq -r '.workflow.update_branch // "automation/docker-dependency-updates"' "${config_path}")" - echo "pr_title=$(yq -r '.workflow.pull_request.title // "chore(deps): docker dependency upgrade"' "${config_path}")" - echo "pr_team_reviewers=$(yq -r '.workflow.pull_request.team_reviewers // [] | join(",")' "${config_path}")" - echo "pr_auto_merge=$(yq -r '.workflow.pull_request.auto_merge // true' "${config_path}")" - echo "pr_merge_method=$(yq -r '.workflow.pull_request.merge_method // "squash"' "${config_path}")" - echo "commit_message=$(yq -r '.workflow.commit_message // "chore(deps): update Docker dependency pins"' "${config_path}")" - echo "validation_command=$(yq -r '.workflow.validation.command // "docker build --progress=plain -t dependency-update-validation ."' "${config_path}")" + echo "dockerfile=${DOCKERFILE}" + echo "prompt_path=${COPILOT_PROMPT_PATH}" + echo "probe_dockerfile=${PROBE_DOCKERFILE}" + echo "report_path=${REPORT_PATH}" + echo "update_branch=${UPDATE_BRANCH}" + echo "pr_title=${PR_TITLE}" + echo "pr_team_reviewers=${PR_TEAM_REVIEWERS}" + echo "pr_auto_merge=${PR_AUTO_MERGE}" + echo "pr_merge_method=${PR_MERGE_METHOD}" + echo "commit_message=${COMMIT_MESSAGE}" + echo "validation_command=${VALIDATION_COMMAND}" } >> "${GITHUB_OUTPUT}" - test -f "${config_path}" - test -f "$(yq -r '.copilot.prompt // "ci/prompts/docker-dependency-updater.md"' "${config_path}")" - test -f "$(yq -r '.dockerfile.path // "Dockerfile"' "${config_path}")" - echo "Config loaded: ${config_path}" + test -n "${DOCKERFILE}" + test -n "${COPILOT_PROMPT_PATH}" + test -n "${PROBE_DOCKERFILE}" + test -f "${DOCKERFILE}" + test -f "${COPILOT_PROMPT_PATH}" + echo "Dependency updater defaults loaded for ${DOCKERFILE}" - name: Stop if an update PR is already open id: existing-pr @@ -117,38 +119,40 @@ jobs: echo "should_run=true" >> "${GITHUB_OUTPUT}" echo "skip_reason=" >> "${GITHUB_OUTPUT}" - echo "Config decision: run analyze/apply, no open updater PR found." + echo "Config decision: run upgrade, no open updater PR found." env: EXISTING_PR_FOUND: ${{ steps.existing-pr.outputs.found }} UPDATE_BRANCH: ${{ steps.config.outputs.update_branch }} - - name: Write config summary - if: always() - run: | - { - echo "## udx-automation / dependency upgrade" - echo - echo "| Job | Result | Details |" - echo "| --- | --- | --- |" - echo "| config | ${{ job.status }} | Dockerfile: \`${{ steps.config.outputs.dockerfile }}\`; branch: \`${{ steps.config.outputs.update_branch }}\`; reviewers: \`${{ steps.config.outputs.pr_team_reviewers }}\`; auto-merge: \`${{ steps.config.outputs.pr_auto_merge }}\`; should run: \`${{ steps.decision.outputs.should_run }}\` |" - } >> "${GITHUB_STEP_SUMMARY}" - - analyze: + upgrade: needs: config if: needs.config.outputs.should_run == 'true' runs-on: ubuntu-24.04 outputs: - apt_count: ${{ steps.report.outputs.apt_count }} - report_path: ${{ needs.config.outputs.report_path }} + apt_count: ${{ steps.probe.outputs.apt_count }} + changed: ${{ steps.changes.outputs.changed }} + pr_url: ${{ steps.create-pr.outputs.pull-request-url }} + + permissions: + contents: write + pull-requests: write steps: - name: Checkout repository uses: actions/checkout@v6 + with: + ref: ${{ github.ref }} - - name: Render apt probe Dockerfile + - name: Generate no-pin apt probe report + id: probe run: | - mkdir -p .tmp/dependency-probe - mkdir -p "$(dirname "${PROBE_DOCKERFILE}")" + set -euo pipefail + + probe_dir="$(dirname "${PROBE_DOCKERFILE}")" + apt_packages_path="${probe_dir}/apt-packages.txt" + apt_versions_path="${probe_dir}/apt.tsv" + + mkdir -p "${probe_dir}" awk ' { @@ -168,51 +172,30 @@ jobs: } ' "${DOCKERFILE}" > "${PROBE_DOCKERFILE}" - echo "Analyze render: created apt-only probe ${PROBE_DOCKERFILE} from ${DOCKERFILE}" - env: - DOCKERFILE: ${{ needs.config.outputs.dockerfile }} - PROBE_DOCKERFILE: ${{ needs.config.outputs.probe_dockerfile }} - - - name: Build apt dependency probe - run: | - docker build --pull --no-cache -f "${PROBE_DOCKERFILE}" -t "${PROBE_IMAGE}" . - echo "Analyze build: built ${PROBE_IMAGE} from ${PROBE_DOCKERFILE}" - env: - PROBE_DOCKERFILE: ${{ needs.config.outputs.probe_dockerfile }} - PROBE_IMAGE: ${{ needs.config.outputs.image }} - - - name: Write apt dependency report - id: report - run: | - set -euo pipefail - awk ' /apt-get install -y --no-install-recommends/ { in_block=1; next } in_block { line=$0 + sub(/#.*/, "", line) gsub(/\\/, "", line) - gsub(/^[[:space:]]+/, "", line) + gsub(/^[[:space:]]+|[[:space:]]+$/, "", line) if (line ~ /^[[:alnum:].+-]+=/) { - split(line, parts, "=") - print parts[1] + sub(/=.*/, "", line) + print line } if ($0 ~ /&&[[:space:]]*\\?$/) { in_block=0 } } - ' "${DOCKERFILE}" > .tmp/dependency-probe/apt-packages.txt + ' "${DOCKERFILE}" > "${apt_packages_path}" - mapfile -t apt_packages < .tmp/dependency-probe/apt-packages.txt + mapfile -t apt_packages < "${apt_packages_path}" - docker run -i --rm --entrypoint bash "${PROBE_IMAGE}" -s -- "${apt_packages[@]}" > .tmp/dependency-probe/apt.json <<'EOF' + docker build --pull --no-cache -f "${PROBE_DOCKERFILE}" -t "${PROBE_IMAGE}" . + docker run -i --rm --entrypoint bash "${PROBE_IMAGE}" -s -- "${apt_packages[@]}" > "${apt_versions_path}" <<'EOF' set -euo pipefail - dpkg-query -W -f='${binary:Package}\t${Version}\n' "$@" \ - | jq -R -s ' - split("\n") - | map(select(length > 0)) - | map(split("\t") | {name: .[0], installed: .[1]}) - ' + dpkg-query -W -f='${binary:Package}\t${Version}\n' "$@" EOF base_image="$(awk '$1 == "FROM" { print $2; exit }' "${DOCKERFILE}")" @@ -220,30 +203,34 @@ jobs: jq -n \ --arg generated_at "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ --arg base_image "${base_image}" \ - --slurpfile apt .tmp/dependency-probe/apt.json \ + --arg dockerfile "${DOCKERFILE}" \ + --arg probe_dockerfile "${PROBE_DOCKERFILE}" \ + --rawfile apt_versions "${apt_versions_path}" \ '{ generated_at: $generated_at, - method: "apt-only Dockerfile probe build", + method: "no-pin apt probe", base_image: $base_image, + strategy: { + summary: "Render a temporary Dockerfile with apt package pins removed, build it against the configured base image, then read installed versions with dpkg-query.", + dockerfile: $dockerfile, + probe_dockerfile: $probe_dockerfile + }, dependencies: { - apt: $apt[0] + apt: ( + $apt_versions + | split("\n") + | map(select(length > 0)) + | map(split("\t") | {name: .[0], installed: .[1]}) + ) } }' > "${PROBE_REPORT}" - report_tmp="${PROBE_REPORT}.tmp" - jq \ - --arg config_path "${PROBE_CONFIG}" \ - --rawfile config_yaml "${PROBE_CONFIG}" \ - '.probe_config = {path: $config_path, format: "yaml", content: $config_yaml}' \ - "${PROBE_REPORT}" > "${report_tmp}" - mv "${report_tmp}" "${PROBE_REPORT}" - apt_count="$(jq '.dependencies.apt | length' "${PROBE_REPORT}")" - printf 'apt_count=%s\n' "${apt_count}" >> "${GITHUB_OUTPUT}" - echo "Analyze report: wrote ${PROBE_REPORT} with ${apt_count} apt packages" + echo "apt_count=${apt_count}" >> "${GITHUB_OUTPUT}" + echo "Upgrade probe: wrote ${PROBE_REPORT} with ${apt_count} apt packages" env: DOCKERFILE: ${{ needs.config.outputs.dockerfile }} - PROBE_CONFIG: ${{ needs.config.outputs.config_path }} + PROBE_DOCKERFILE: ${{ needs.config.outputs.probe_dockerfile }} PROBE_IMAGE: ${{ needs.config.outputs.image }} PROBE_REPORT: ${{ needs.config.outputs.report_path }} @@ -255,50 +242,6 @@ jobs: ${{ needs.config.outputs.report_path }} ${{ needs.config.outputs.probe_dockerfile }} - - name: Write analyze summary - if: always() - run: | - apt_count="n/a" - - if [ -f "${PROBE_REPORT}" ]; then - apt_count="$(jq '.dependencies.apt | length' "${PROBE_REPORT}")" - fi - - { - echo "## udx-automation / dependency upgrade" - echo - echo "| Job | Result | Details |" - echo "| --- | --- | --- |" - echo "| analyze | ${{ job.status }} | Report: \`${PROBE_REPORT}\`; apt: \`${apt_count}\` |" - } >> "${GITHUB_STEP_SUMMARY}" - env: - PROBE_REPORT: ${{ needs.config.outputs.report_path }} - - apply: - needs: - - config - - analyze - if: needs.config.outputs.should_run == 'true' - runs-on: ubuntu-24.04 - outputs: - changed: ${{ steps.changes.outputs.changed }} - pr_url: ${{ steps.create-pr.outputs.pull-request-url }} - - permissions: - contents: write - pull-requests: write - - steps: - - name: Checkout repository - uses: actions/checkout@v6 - with: - ref: ${{ github.ref }} - - - name: Download dependency report - uses: actions/download-artifact@v6 - with: - name: docker-dependency-report - - name: Set up Node.js uses: actions/setup-node@v6 with: @@ -307,7 +250,7 @@ jobs: - name: Install Copilot CLI run: | npm install -g @github/copilot - echo "Apply setup: installed Copilot CLI" + echo "Upgrade setup: installed Copilot CLI" - name: Update Dockerfile with Copilot CLI env: @@ -333,7 +276,7 @@ jobs: --silent \ --share copilot-docker-dependency-session.md - echo "Apply Copilot: completed; session saved to copilot-docker-dependency-session.md" + echo "Upgrade Copilot: completed; session saved to copilot-docker-dependency-session.md" - name: Detect changes id: changes @@ -342,12 +285,12 @@ jobs: set -euo pipefail if git diff --quiet -- "${DOCKERFILE}"; then echo "changed=false" >> "${GITHUB_OUTPUT}" - echo "Apply changes: no ${DOCKERFILE} changes" + echo "Upgrade changes: no ${DOCKERFILE} changes" else echo "changed=true" >> "${GITHUB_OUTPUT}" git diff -- "${DOCKERFILE}" > docker-dependency-update.diff changed_lines="$(wc -l < docker-dependency-update.diff | tr -d ' ')" - echo "Apply changes: ${DOCKERFILE} changed; diff has ${changed_lines} lines" + echo "Upgrade changes: ${DOCKERFILE} changed; diff has ${changed_lines} lines" fi env: DOCKERFILE: ${{ needs.config.outputs.dockerfile }} @@ -356,7 +299,7 @@ jobs: if: steps.changes.outputs.changed == 'true' run: | bash -lc "${VALIDATION_COMMAND}" - echo "Apply validation: succeeded: ${VALIDATION_COMMAND}" + echo "Upgrade validation: succeeded: ${VALIDATION_COMMAND}" env: VALIDATION_COMMAND: ${{ needs.config.outputs.validation_command }} @@ -366,7 +309,7 @@ jobs: { echo "## Summary" echo - echo "Updates Dockerfile dependency pins using an apt-only probe build and Copilot CLI." + echo "Updates Dockerfile dependency pins using a no-pin apt probe and Copilot CLI." echo echo "## Evidence" echo @@ -380,7 +323,7 @@ jobs: sed -n '1,220p' docker-dependency-update.diff echo '```' } > docker-dependency-pr-body.md - echo "Apply PR body: prepared docker-dependency-pr-body.md" + echo "Upgrade PR body: prepared docker-dependency-pr-body.md" env: REPORT_PATH: ${{ needs.config.outputs.report_path }} VALIDATION_COMMAND: ${{ needs.config.outputs.validation_command }} @@ -426,13 +369,21 @@ jobs: ;; esac - echo "Apply auto-merge: enabled ${MERGE_METHOD} auto-merge for ${PR_URL}" + echo "Upgrade auto-merge: enabled ${MERGE_METHOD} auto-merge for ${PR_URL}" env: GH_TOKEN: ${{ github.token }} MERGE_METHOD: ${{ needs.config.outputs.pr_merge_method }} PR_URL: ${{ steps.create-pr.outputs.pull-request-url }} - - name: Write apply summary + report: + needs: + - config + - upgrade + if: always() + runs-on: ubuntu-24.04 + + steps: + - name: Write workflow summary if: always() run: | pr_url="${PR_URL:-}" @@ -440,14 +391,32 @@ jobs: pr_url="n/a" fi + skip_reason="${SKIP_REASON:-}" + if [ -z "${skip_reason}" ]; then + skip_reason="n/a" + fi + + apt_count="${APT_COUNT:-}" + if [ -z "${apt_count}" ]; then + apt_count="n/a" + fi + + changed="${CHANGED:-}" + if [ -z "${changed}" ]; then + changed="n/a" + fi + { echo "## udx-automation / dependency upgrade" echo echo "| Job | Result | Details |" echo "| --- | --- | --- |" echo "| config | ${{ needs.config.result }} | Dockerfile: \`${{ needs.config.outputs.dockerfile }}\`; branch: \`${{ needs.config.outputs.update_branch }}\`; reviewers: \`${{ needs.config.outputs.pr_team_reviewers }}\`; auto-merge: \`${{ needs.config.outputs.pr_auto_merge }}\`; should run: \`${{ needs.config.outputs.should_run }}\` |" - echo "| analyze | ${{ needs.analyze.result }} | Report: \`${{ needs.analyze.outputs.report_path }}\`; apt: \`${{ needs.analyze.outputs.apt_count }}\` |" - echo "| apply | ${{ job.status }} | Dockerfile changed: \`${{ steps.changes.outputs.changed }}\`; PR: ${pr_url} |" + echo "| upgrade | ${{ needs.upgrade.result }} | Report: \`${{ needs.config.outputs.report_path }}\`; apt: \`${apt_count}\`; Dockerfile changed: \`${changed}\`; PR: ${pr_url}; skip: \`${skip_reason}\` |" + echo "| report | ${{ job.status }} | Summary written |" } >> "${GITHUB_STEP_SUMMARY}" env: - PR_URL: ${{ steps.create-pr.outputs.pull-request-url }} + APT_COUNT: ${{ needs.upgrade.outputs.apt_count }} + CHANGED: ${{ needs.upgrade.outputs.changed }} + PR_URL: ${{ needs.upgrade.outputs.pr_url }} + SKIP_REASON: ${{ needs.config.outputs.skip_reason }} diff --git a/ci/configs/docker-dependency-probe.yaml b/ci/configs/docker-dependency-probe.yaml deleted file mode 100644 index 5afae533..00000000 --- a/ci/configs/docker-dependency-probe.yaml +++ /dev/null @@ -1,46 +0,0 @@ ---- -# Lightweight hints for Docker dependency update automation. Dockerfile remains -# the source of truth; this file only describes repo-specific automation knobs. -kind: DockerDependencyProbe -version: 1 - -# Dockerfile input and probe build output. -dockerfile: - path: Dockerfile - - # The probe Dockerfile is a temporary copy used to build a container that - # reports the apt versions Docker installs when apt pins are removed. - probe: - dockerfile: .tmp/dependency-probe/Dockerfile - -# GitHub Actions behavior for generated dependency upgrade PRs. -workflow: - update_branch: automation/docker-dependency-updates - commit_message: "chore(deps): update Docker dependency pins" - - pull_request: - title: "chore(deps): docker dependency upgrade" - team_reviewers: - - worker - auto_merge: true - merge_method: squash - - validation: - command: docker build --progress=plain -t dependency-update-validation . - -# Analysis output. -analysis: - report: docker-dependency-report.json - -# Dependency groups. Apt is discovered from Dockerfile pins and probed -# deterministically; non-apt dependency discovery is delegated to Copilot from -# the Dockerfile and prompt. -dependencies: - apt: - discover: true - source: ubuntu apt repositories - -# Copilot prompt input. Behavior lives in the prompt file so instructions are -# not duplicated across two places. -copilot: - prompt: ci/prompts/docker-dependency-updater.md diff --git a/ci/prompts/docker-dependency-updater.md b/ci/prompts/docker-dependency-updater.md index ca43af0e..a84580b1 100644 --- a/ci/prompts/docker-dependency-updater.md +++ b/ci/prompts/docker-dependency-updater.md @@ -6,11 +6,13 @@ Inputs: Automation contract: - Read both input files before editing. -- The dependency report already resolves current apt package versions for the configured Ubuntu base image. +- The dependency report already resolves current apt package versions for the configured Ubuntu base image using a no-pin apt probe. +- The no-pin apt probe is authoritative for apt package updates: it was built from a temporary Dockerfile where apt pins were removed, then queried with `dpkg-query`. - You are responsible for detecting and checking every non-apt dependency directly from Dockerfile. Dependency handling: -- For apt packages, update Dockerfile pins only from `dependencies.apt[].installed` in the dependency report. +- For apt packages, update Dockerfile pins only from `dependencies.apt[].installed` in the dependency report. Do not use apt websites, package search pages, or guessed versions for apt pins. +- If an apt package from Dockerfile is missing from the report, leave that package unchanged and explain it in the changelog. - For non-apt dependencies, detect ARG-pinned versions, URL-pinned versions, package-manager pins, and dynamically installed tools from Dockerfile. - For each non-apt pinned dependency, identify its upstream source from Dockerfile context and check the latest stable version. - Update only Dockerfile dependency pins and ARG values when the report or upstream source shows a newer version. From e2877047b85967f27155df9af7a9eb048fc1184e Mon Sep 17 00:00:00 2001 From: Dmytro Smirnov Date: Wed, 3 Jun 2026 12:07:04 +0300 Subject: [PATCH 25/32] ci: make docker apt probe awk portable --- .github/workflows/docker-dependency-updater.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker-dependency-updater.yml b/.github/workflows/docker-dependency-updater.yml index 4d6fa9c7..da89651b 100644 --- a/.github/workflows/docker-dependency-updater.yml +++ b/.github/workflows/docker-dependency-updater.yml @@ -161,7 +161,11 @@ jobs: if (line ~ /apt-get install -y --no-install-recommends/) { in_apt = 1 } else if (in_apt) { - sub(/^([[:space:]]+[A-Za-z0-9.+-]+)=[^[:space:]]+(.*)$/, "\\1\\2", line) + prefix = line + suffix = line + sub(/=.*/, "", prefix) + sub(/^[[:space:]]+[A-Za-z0-9.+-]+=[^[:space:]]+/, "", suffix) + line = prefix suffix } print line From ec49024e3dcdbd90dd38a51d59b1f3d37776935f Mon Sep 17 00:00:00 2001 From: Dmytro Smirnov Date: Wed, 3 Jun 2026 12:14:04 +0300 Subject: [PATCH 26/32] ci: avoid dependency updater reviewer permission failure --- .github/workflows/docker-dependency-updater.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/docker-dependency-updater.yml b/.github/workflows/docker-dependency-updater.yml index da89651b..37984e79 100644 --- a/.github/workflows/docker-dependency-updater.yml +++ b/.github/workflows/docker-dependency-updater.yml @@ -16,7 +16,6 @@ env: REPORT_PATH: docker-dependency-report.json UPDATE_BRANCH: docker-dependency-updates PR_TITLE: "chore(deps): docker dependency upgrade" - PR_TEAM_REVIEWERS: worker PR_AUTO_MERGE: "true" PR_MERGE_METHOD: squash COMMIT_MESSAGE: "chore(deps): update Docker dependency pins" @@ -40,7 +39,6 @@ jobs: report_path: ${{ steps.config.outputs.report_path }} update_branch: ${{ steps.config.outputs.update_branch }} pr_title: ${{ steps.config.outputs.pr_title }} - pr_team_reviewers: ${{ steps.config.outputs.pr_team_reviewers }} pr_auto_merge: ${{ steps.config.outputs.pr_auto_merge }} pr_merge_method: ${{ steps.config.outputs.pr_merge_method }} commit_message: ${{ steps.config.outputs.commit_message }} @@ -65,7 +63,6 @@ jobs: echo "report_path=${REPORT_PATH}" echo "update_branch=${UPDATE_BRANCH}" echo "pr_title=${PR_TITLE}" - echo "pr_team_reviewers=${PR_TEAM_REVIEWERS}" echo "pr_auto_merge=${PR_AUTO_MERGE}" echo "pr_merge_method=${PR_MERGE_METHOD}" echo "commit_message=${COMMIT_MESSAGE}" @@ -351,7 +348,6 @@ jobs: body-path: docker-dependency-pr-body.md branch: ${{ needs.config.outputs.update_branch }} add-paths: ${{ needs.config.outputs.dockerfile }} - team-reviewers: ${{ needs.config.outputs.pr_team_reviewers }} draft: false delete-branch: true @@ -415,7 +411,7 @@ jobs: echo echo "| Job | Result | Details |" echo "| --- | --- | --- |" - echo "| config | ${{ needs.config.result }} | Dockerfile: \`${{ needs.config.outputs.dockerfile }}\`; branch: \`${{ needs.config.outputs.update_branch }}\`; reviewers: \`${{ needs.config.outputs.pr_team_reviewers }}\`; auto-merge: \`${{ needs.config.outputs.pr_auto_merge }}\`; should run: \`${{ needs.config.outputs.should_run }}\` |" + echo "| config | ${{ needs.config.result }} | Dockerfile: \`${{ needs.config.outputs.dockerfile }}\`; branch: \`${{ needs.config.outputs.update_branch }}\`; auto-merge: \`${{ needs.config.outputs.pr_auto_merge }}\`; should run: \`${{ needs.config.outputs.should_run }}\` |" echo "| upgrade | ${{ needs.upgrade.result }} | Report: \`${{ needs.config.outputs.report_path }}\`; apt: \`${apt_count}\`; Dockerfile changed: \`${changed}\`; PR: ${pr_url}; skip: \`${skip_reason}\` |" echo "| report | ${{ job.status }} | Summary written |" } >> "${GITHUB_STEP_SUMMARY}" From 1ae748fbc0fe31c7d3a62336bee36f655a7e51da Mon Sep 17 00:00:00 2001 From: Dmytro Smirnov Date: Wed, 3 Jun 2026 12:16:40 +0300 Subject: [PATCH 27/32] ci: leave docker build validation to CI --- .github/workflows/docker-dependency-updater.yml | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/.github/workflows/docker-dependency-updater.yml b/.github/workflows/docker-dependency-updater.yml index 37984e79..fb848765 100644 --- a/.github/workflows/docker-dependency-updater.yml +++ b/.github/workflows/docker-dependency-updater.yml @@ -19,7 +19,6 @@ env: PR_AUTO_MERGE: "true" PR_MERGE_METHOD: squash COMMIT_MESSAGE: "chore(deps): update Docker dependency pins" - VALIDATION_COMMAND: docker build --progress=plain -t dependency-update-validation . COPILOT_PROMPT_PATH: ci/prompts/docker-dependency-updater.md jobs: @@ -42,7 +41,6 @@ jobs: pr_auto_merge: ${{ steps.config.outputs.pr_auto_merge }} pr_merge_method: ${{ steps.config.outputs.pr_merge_method }} commit_message: ${{ steps.config.outputs.commit_message }} - validation_command: ${{ steps.config.outputs.validation_command }} should_run: ${{ steps.decision.outputs.should_run }} skip_reason: ${{ steps.decision.outputs.skip_reason }} @@ -66,7 +64,6 @@ jobs: echo "pr_auto_merge=${PR_AUTO_MERGE}" echo "pr_merge_method=${PR_MERGE_METHOD}" echo "commit_message=${COMMIT_MESSAGE}" - echo "validation_command=${VALIDATION_COMMAND}" } >> "${GITHUB_OUTPUT}" test -n "${DOCKERFILE}" @@ -296,14 +293,6 @@ jobs: env: DOCKERFILE: ${{ needs.config.outputs.dockerfile }} - - name: Validate Docker build - if: steps.changes.outputs.changed == 'true' - run: | - bash -lc "${VALIDATION_COMMAND}" - echo "Upgrade validation: succeeded: ${VALIDATION_COMMAND}" - env: - VALIDATION_COMMAND: ${{ needs.config.outputs.validation_command }} - - name: Prepare pull request body if: steps.changes.outputs.changed == 'true' run: | @@ -316,7 +305,7 @@ jobs: echo echo "- Probe report artifact: \`${REPORT_PATH}\`" echo "- Copilot session artifact: \`copilot-docker-dependency-session.md\`" - echo "- Validation: \`${VALIDATION_COMMAND}\`" + echo "- Build validation: handled by the repository Docker/CI workflows" echo echo "## Diff" echo @@ -327,7 +316,6 @@ jobs: echo "Upgrade PR body: prepared docker-dependency-pr-body.md" env: REPORT_PATH: ${{ needs.config.outputs.report_path }} - VALIDATION_COMMAND: ${{ needs.config.outputs.validation_command }} - name: Upload Copilot session uses: actions/upload-artifact@v5 From 7062d553c03559f3afd7a10fff59efb4a48e76f1 Mon Sep 17 00:00:00 2001 From: Dmytro Smirnov Date: Wed, 3 Jun 2026 12:25:54 +0300 Subject: [PATCH 28/32] ci: constrain dependency updater execution --- .../workflows/docker-dependency-updater.yml | 50 +++++++++++++++---- ci/prompts/docker-dependency-apt.md | 6 +++ ci/prompts/docker-dependency-guardrails.md | 14 ++++++ ci/prompts/docker-dependency-nonapt.md | 7 +++ ci/prompts/docker-dependency-output.md | 20 ++++++++ ci/prompts/docker-dependency-updater.md | 47 ----------------- 6 files changed, 88 insertions(+), 56 deletions(-) create mode 100644 ci/prompts/docker-dependency-apt.md create mode 100644 ci/prompts/docker-dependency-guardrails.md create mode 100644 ci/prompts/docker-dependency-nonapt.md create mode 100644 ci/prompts/docker-dependency-output.md delete mode 100644 ci/prompts/docker-dependency-updater.md diff --git a/.github/workflows/docker-dependency-updater.yml b/.github/workflows/docker-dependency-updater.yml index fb848765..976ed529 100644 --- a/.github/workflows/docker-dependency-updater.yml +++ b/.github/workflows/docker-dependency-updater.yml @@ -6,6 +6,10 @@ name: udx-automation / dependency upgrade - cron: "0 5 * * 1" workflow_dispatch: +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + permissions: contents: read @@ -19,11 +23,16 @@ env: PR_AUTO_MERGE: "true" PR_MERGE_METHOD: squash COMMIT_MESSAGE: "chore(deps): update Docker dependency pins" - COPILOT_PROMPT_PATH: ci/prompts/docker-dependency-updater.md + COPILOT_PROMPT_PARTS: >- + ci/prompts/docker-dependency-guardrails.md + ci/prompts/docker-dependency-apt.md + ci/prompts/docker-dependency-nonapt.md + ci/prompts/docker-dependency-output.md jobs: config: runs-on: ubuntu-24.04 + timeout-minutes: 5 permissions: contents: read @@ -33,7 +42,7 @@ jobs: dockerfile: ${{ steps.config.outputs.dockerfile }} existing_pr_found: ${{ steps.existing-pr.outputs.found }} image: ${{ steps.config.outputs.image }} - prompt_path: ${{ steps.config.outputs.prompt_path }} + prompt_parts: ${{ steps.config.outputs.prompt_parts }} probe_dockerfile: ${{ steps.config.outputs.probe_dockerfile }} report_path: ${{ steps.config.outputs.report_path }} update_branch: ${{ steps.config.outputs.update_branch }} @@ -56,7 +65,7 @@ jobs: { echo "image=worker-deps-probe:${GITHUB_RUN_ID}" echo "dockerfile=${DOCKERFILE}" - echo "prompt_path=${COPILOT_PROMPT_PATH}" + echo "prompt_parts=${COPILOT_PROMPT_PARTS}" echo "probe_dockerfile=${PROBE_DOCKERFILE}" echo "report_path=${REPORT_PATH}" echo "update_branch=${UPDATE_BRANCH}" @@ -67,10 +76,12 @@ jobs: } >> "${GITHUB_OUTPUT}" test -n "${DOCKERFILE}" - test -n "${COPILOT_PROMPT_PATH}" + test -n "${COPILOT_PROMPT_PARTS}" test -n "${PROBE_DOCKERFILE}" test -f "${DOCKERFILE}" - test -f "${COPILOT_PROMPT_PATH}" + for prompt_part in ${COPILOT_PROMPT_PARTS}; do + test -f "${prompt_part}" + done echo "Dependency updater defaults loaded for ${DOCKERFILE}" - name: Stop if an update PR is already open @@ -122,6 +133,7 @@ jobs: needs: config if: needs.config.outputs.should_run == 'true' runs-on: ubuntu-24.04 + timeout-minutes: 20 outputs: apt_count: ${{ steps.probe.outputs.apt_count }} changed: ${{ steps.changes.outputs.changed }} @@ -139,6 +151,7 @@ jobs: - name: Generate no-pin apt probe report id: probe + timeout-minutes: 8 run: | set -euo pipefail @@ -234,6 +247,7 @@ jobs: - name: Upload dependency report uses: actions/upload-artifact@v5 + timeout-minutes: 3 with: name: docker-dependency-report path: | @@ -242,20 +256,23 @@ jobs: - name: Set up Node.js uses: actions/setup-node@v6 + timeout-minutes: 3 with: node-version: "22" - name: Install Copilot CLI + timeout-minutes: 3 run: | npm install -g @github/copilot echo "Upgrade setup: installed Copilot CLI" - name: Update Dockerfile with Copilot CLI + timeout-minutes: 8 env: COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} COPILOT_ALLOW_ALL: "true" COPILOT_AUTO_UPDATE: "false" - PROMPT_PATH: ${{ needs.config.outputs.prompt_path }} + PROMPT_PARTS: ${{ needs.config.outputs.prompt_parts }} REPORT_PATH: ${{ needs.config.outputs.report_path }} run: | set -euo pipefail @@ -265,8 +282,19 @@ jobs: exit 1 fi + prompt_path=".tmp/dependency-upgrade/copilot-prompt.md" + mkdir -p "$(dirname "${prompt_path}")" + : > "${prompt_path}" + for prompt_part in ${PROMPT_PARTS}; do + test -f "${prompt_part}" + { + cat "${prompt_part}" + echo + } >> "${prompt_path}" + done + copilot \ - --prompt "$(cat "${PROMPT_PATH}")" \ + --prompt "$(cat "${prompt_path}")" \ --allow-all-tools \ --allow-all-urls \ --no-ask-user \ @@ -319,6 +347,7 @@ jobs: - name: Upload Copilot session uses: actions/upload-artifact@v5 + timeout-minutes: 3 with: name: copilot-docker-dependency-session path: | @@ -330,6 +359,7 @@ jobs: id: create-pr if: steps.changes.outputs.changed == 'true' uses: peter-evans/create-pull-request@v7 + timeout-minutes: 5 with: commit-message: ${{ needs.config.outputs.commit_message }} title: ${{ needs.config.outputs.pr_title }} @@ -342,8 +372,9 @@ jobs: - name: Enable pull request auto-merge if: >- steps.changes.outputs.changed == 'true' && - steps.create-pr.outputs.pull-request-url != '' && - needs.config.outputs.pr_auto_merge == 'true' + steps.create-pr.outputs.pull-request-url != '' && + needs.config.outputs.pr_auto_merge == 'true' + timeout-minutes: 3 run: | set -euo pipefail @@ -369,6 +400,7 @@ jobs: - upgrade if: always() runs-on: ubuntu-24.04 + timeout-minutes: 5 steps: - name: Write workflow summary diff --git a/ci/prompts/docker-dependency-apt.md b/ci/prompts/docker-dependency-apt.md new file mode 100644 index 00000000..92fa0508 --- /dev/null +++ b/ci/prompts/docker-dependency-apt.md @@ -0,0 +1,6 @@ +Apt dependency rules: +- The dependency report already resolves current apt package versions for the configured Ubuntu base image using a no-pin apt probe. +- The no-pin apt probe is authoritative for apt package updates: it was built from a temporary Dockerfile where apt pins were removed, then queried with `dpkg-query`. +- For apt packages, update Dockerfile pins only from `dependencies.apt[].installed` in the dependency report. +- Do not use apt websites, package search pages, or guessed versions for apt pins. +- If an apt package from Dockerfile is missing from the report, leave that package unchanged and explain it in the changelog. diff --git a/ci/prompts/docker-dependency-guardrails.md b/ci/prompts/docker-dependency-guardrails.md new file mode 100644 index 00000000..538b3f33 --- /dev/null +++ b/ci/prompts/docker-dependency-guardrails.md @@ -0,0 +1,14 @@ +You are updating Dockerfile dependency pins for this repository. + +Inputs: +- Dockerfile: `Dockerfile` +- Dependency report: `docker-dependency-report.json` + +Hard boundaries: +- This is an edit-only dependency update task. +- Read both input files before editing. +- Edit only Dockerfile dependency pins and ARG values. +- Do not edit workflow files, docs, tests, or application code. +- Do not validate, build, test, run the container pipeline, inspect GitHub Actions runs, wait for workflows, create pull requests, commit, push, or request reviews. +- Do not run `docker`, `make`, test commands, CI commands, `gh run`, `gh workflow`, `gh pr`, `git commit`, `git push`, or any command that waits on external workflow state. +- If an update cannot be verified from the dependency report or upstream release metadata without validation, leave it unchanged and mention why. diff --git a/ci/prompts/docker-dependency-nonapt.md b/ci/prompts/docker-dependency-nonapt.md new file mode 100644 index 00000000..e38d90d0 --- /dev/null +++ b/ci/prompts/docker-dependency-nonapt.md @@ -0,0 +1,7 @@ +Non-apt dependency rules: +- Detect ARG-pinned versions, URL-pinned versions, package-manager pins, and dynamically installed tools from Dockerfile. +- For each non-apt pinned dependency, identify its upstream source from Dockerfile context and check the latest stable version. +- Update only Dockerfile dependency pins and ARG values when the dependency report or upstream source shows a newer version. +- Preserve the ubuntu base image tag unless explicitly necessary to make reported apt versions valid. +- Keep dynamically installed dependencies unpinned unless Dockerfile already pins them; mention them in the changelog only. +- If a Dockerfile dependency cannot be checked confidently, leave it unchanged and mention the reason in the changelog. diff --git a/ci/prompts/docker-dependency-output.md b/ci/prompts/docker-dependency-output.md new file mode 100644 index 00000000..7c38fa2b --- /dev/null +++ b/ci/prompts/docker-dependency-output.md @@ -0,0 +1,20 @@ +Output: +- Print a changelog using this template: + +```text +Docker dependency changelog + +Updated: +- : -> () + +Unchanged: +- : () + +Observed but not pinned: +- : () + +Notes: +- +``` + +- Omit empty sections except `Notes`. diff --git a/ci/prompts/docker-dependency-updater.md b/ci/prompts/docker-dependency-updater.md deleted file mode 100644 index a84580b1..00000000 --- a/ci/prompts/docker-dependency-updater.md +++ /dev/null @@ -1,47 +0,0 @@ -You are updating Dockerfile dependency pins for this repository. - -Inputs: -- Dockerfile: `Dockerfile` -- Dependency report: `docker-dependency-report.json` - -Automation contract: -- Read both input files before editing. -- The dependency report already resolves current apt package versions for the configured Ubuntu base image using a no-pin apt probe. -- The no-pin apt probe is authoritative for apt package updates: it was built from a temporary Dockerfile where apt pins were removed, then queried with `dpkg-query`. -- You are responsible for detecting and checking every non-apt dependency directly from Dockerfile. - -Dependency handling: -- For apt packages, update Dockerfile pins only from `dependencies.apt[].installed` in the dependency report. Do not use apt websites, package search pages, or guessed versions for apt pins. -- If an apt package from Dockerfile is missing from the report, leave that package unchanged and explain it in the changelog. -- For non-apt dependencies, detect ARG-pinned versions, URL-pinned versions, package-manager pins, and dynamically installed tools from Dockerfile. -- For each non-apt pinned dependency, identify its upstream source from Dockerfile context and check the latest stable version. -- Update only Dockerfile dependency pins and ARG values when the report or upstream source shows a newer version. -- Preserve the ubuntu base image tag unless explicitly necessary to make the reported versions valid. -- Keep dynamically installed dependencies unpinned unless Dockerfile already pins them; mention them in the changelog only. -- If a Dockerfile dependency cannot be checked confidently, leave it unchanged and mention the reason in the changelog. - -Editing rules: -- Edit only Dockerfile dependency pins and ARG values. -- Do not edit workflow files, docs, tests, or application code. -- Do not commit, push, or create a pull request; this workflow will do that. - -Output: -- Print a changelog using this template: - -```text -Docker dependency changelog - -Updated: -- : -> () - -Unchanged: -- : () - -Observed but not pinned: -- : () - -Notes: -- -``` - -- Omit empty sections except `Notes`. From e5cc8a3c8a873ad236277009e8c7b8f91a358a78 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 3 Jun 2026 09:30:55 +0000 Subject: [PATCH 29/32] chore(deps): update Docker dependency pins (#131) Co-authored-by: fqjony <12067297+fqjony@users.noreply.github.com> --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index aed3e0bb..a8f2a3e4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,7 +7,7 @@ LABEL maintainer="UDX CAG Team" ARG AZURE_CLI_VERSION=2.87.0 ARG PIP_VERSION=26.1.2 ARG YQ_VERSION=4.53.2 -ARG GCLOUD_VERSION=570.0.0 +ARG GCLOUD_VERSION=571.0.0 # Set base environment variables ENV DEBIAN_FRONTEND=noninteractive \ From ac33570e9c56a3b980613596506af83b85d9d331 Mon Sep 17 00:00:00 2001 From: Dmytro Smirnov Date: Wed, 3 Jun 2026 12:59:54 +0300 Subject: [PATCH 30/32] ci: make dependency updater auto-merge best effort --- .github/workflows/docker-dependency-updater.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-dependency-updater.yml b/.github/workflows/docker-dependency-updater.yml index 976ed529..176bff26 100644 --- a/.github/workflows/docker-dependency-updater.yml +++ b/.github/workflows/docker-dependency-updater.yml @@ -380,7 +380,11 @@ jobs: case "${MERGE_METHOD}" in merge|rebase|squash) - gh pr merge "${PR_URL}" --auto "--${MERGE_METHOD}" + if gh pr merge "${PR_URL}" --auto "--${MERGE_METHOD}"; then + echo "Upgrade auto-merge: enabled ${MERGE_METHOD} auto-merge for ${PR_URL}" + else + echo "Upgrade auto-merge: could not enable ${MERGE_METHOD} auto-merge for ${PR_URL}; leaving PR open or already mergeable." >&2 + fi ;; *) echo "Unsupported merge method: ${MERGE_METHOD}" >&2 @@ -388,7 +392,6 @@ jobs: ;; esac - echo "Upgrade auto-merge: enabled ${MERGE_METHOD} auto-merge for ${PR_URL}" env: GH_TOKEN: ${{ github.token }} MERGE_METHOD: ${{ needs.config.outputs.pr_merge_method }} From 23a6fae7f020ee70a07740bf9de1afdb77c93e44 Mon Sep 17 00:00:00 2001 From: Dmytro Smirnov Date: Wed, 3 Jun 2026 13:13:55 +0300 Subject: [PATCH 31/32] fix: address bot runtime feedback --- .../workflows/docker-dependency-updater.yml | 18 ++++- Dockerfile | 3 +- lib/cli/env.sh | 61 ++++++----------- lib/env_handler.sh | 65 +++++++++---------- lib/process_manager.sh | 38 ++++++++++- lib/runtime_output.sh | 29 +++------ 6 files changed, 112 insertions(+), 102 deletions(-) diff --git a/.github/workflows/docker-dependency-updater.yml b/.github/workflows/docker-dependency-updater.yml index 176bff26..006465ff 100644 --- a/.github/workflows/docker-dependency-updater.yml +++ b/.github/workflows/docker-dependency-updater.yml @@ -304,6 +304,22 @@ jobs: echo "Upgrade Copilot: completed; session saved to copilot-docker-dependency-session.md" + - name: Guard Dockerfile-only changes + timeout-minutes: 1 + run: | + set -euo pipefail + + unexpected_changes="$(git diff --name-only | awk -v dockerfile="${DOCKERFILE}" '$0 != dockerfile')" + if [ -n "${unexpected_changes}" ]; then + echo "Upgrade guard: Copilot modified tracked files outside ${DOCKERFILE}:" >&2 + printf '%s\n' "${unexpected_changes}" >&2 + exit 1 + fi + + echo "Upgrade guard: tracked changes are limited to ${DOCKERFILE}" + env: + DOCKERFILE: ${{ needs.config.outputs.dockerfile }} + - name: Detect changes id: changes shell: bash @@ -358,7 +374,7 @@ jobs: - name: Create pull request id: create-pr if: steps.changes.outputs.changed == 'true' - uses: peter-evans/create-pull-request@v7 + uses: peter-evans/create-pull-request@22a9089034f40e5a961c8808d113e2c98fb63676 timeout-minutes: 5 with: commit-message: ${{ needs.config.outputs.commit_message }} diff --git a/Dockerfile b/Dockerfile index a8f2a3e4..e9c4266b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -139,7 +139,7 @@ RUN mkdir -p \ # Create and set permissions for environment files touch ${WORKER_CONFIG_DIR}/environment && \ chown ${USER}:${USER} ${WORKER_CONFIG_DIR}/environment && \ - chmod 644 ${WORKER_CONFIG_DIR}/environment + chmod 600 ${WORKER_CONFIG_DIR}/environment # Copy worker files COPY bin/entrypoint.sh ${WORKER_BIN_DIR}/ @@ -182,6 +182,7 @@ RUN \ find ${WORKER_BASE_DIR} ${WORKER_CONFIG_DIR} ${WORKER_LIB_DIR} ${WORKER_BIN_DIR} -type d -exec chmod 755 {} + && \ # Set base file permissions find ${WORKER_CONFIG_DIR} -type f -exec chmod 644 {} + && \ + chmod 600 ${WORKER_CONFIG_DIR}/environment && \ find ${WORKER_LIB_DIR} -type f ! -name process_manager.sh -exec chmod 644 {} + && \ # Make specific files executable chmod 755 \ diff --git a/lib/cli/env.sh b/lib/cli/env.sh index 38346446..64072ff5 100644 --- a/lib/cli/env.sh +++ b/lib/cli/env.sh @@ -41,54 +41,38 @@ EOF # Example: worker env show --format json --filter AWS_* --include-secrets show_environment() { local format=${1:-text} - local filter=$2 + local filter=${2:-} local include_secrets=${3:-false} - + local names # Check if environment file exists if [ ! -f "$WORKER_ENV_FILE" ]; then log_error "Env" "Environment file not found" return 1 fi + + names=$(grep "^export " "$WORKER_ENV_FILE" | cut -d'=' -f1 | cut -d' ' -f2) case $format in json) - # Get variables and convert to JSON - local vars - if [ -n "$filter" ]; then - vars=$(grep "^export $filter" "$WORKER_ENV_FILE") - else - vars=$(grep "^export" "$WORKER_ENV_FILE") - fi - - # Convert to JSON - local json="{" - local first=true - while IFS= read -r line; do - if [[ $line =~ ^export[[:space:]]+([^=]+)=\"([^\"]*)\" ]]; then - if [ "$first" = true ]; then - first=false - else - json="$json," - fi - key=${BASH_REMATCH[1]} - value=${BASH_REMATCH[2]} - json="$json\"$key\":\"$value\"" + local json="{}" + while IFS= read -r name; do + # shellcheck disable=SC2053 # Env filters intentionally support globs like AWS_*. + if [[ -n "$name" && ( -z "$filter" || "$name" == $filter ) ]]; then + local value + value=$(get_env_value "$name") + json=$(echo "$json" | jq --arg key "$name" --arg value "$value" '. + {($key): $value}') fi - done <<< "$vars" - json="$json}" - if command -v jq >/dev/null 2>&1; then - echo "$json" | jq . - else - echo "$json" - fi + done <<< "$names" + echo "$json" | jq . ;; text) - if [ -n "$filter" ]; then - grep "^export $filter" "$WORKER_ENV_FILE" | sed 's/export \([^=]*\)="\([^"]*\)"/\1=\2/' - else - grep "^export" "$WORKER_ENV_FILE" | sed 's/export \([^=]*\)="\([^"]*\)"/\1=\2/' - fi + while IFS= read -r name; do + # shellcheck disable=SC2053 # Env filters intentionally support globs like AWS_*. + if [[ -n "$name" && ( -z "$filter" || "$name" == $filter ) ]]; then + printf '%s=%s\n' "$name" "$(get_env_value "$name")" + fi + done <<< "$names" ;; *) log_error "Env" "Unknown format: $format" @@ -119,13 +103,8 @@ set_environment() { return 1 fi - # Add to environment file if [ -f "$WORKER_ENV_FILE" ]; then - # Remove existing declaration if any - sed -i "/^export $name=/d" "$WORKER_ENV_FILE" - # Add new declaration - echo "export $name=\"$value\"" >> "$WORKER_ENV_FILE" - # Export in current session + upsert_env_value "$name" "$value" || return 1 export "$name=$value" log_success "Env" "Set $name to '$value'" else diff --git a/lib/env_handler.sh b/lib/env_handler.sh index 450f6f67..1ad6d663 100644 --- a/lib/env_handler.sh +++ b/lib/env_handler.sh @@ -19,15 +19,11 @@ ensure_env_file() { log_error "Environment" "Failed to create environment file: $WORKER_ENV_FILE" return 1 } -} -escape_env_value() { - local value="$1" - value=${value//\\/\\\\} - value=${value//\"/\\\"} - value=${value//\$/\\\$} - value=${value//\`/\\\`} - printf "%s" "$value" + chmod 600 "$WORKER_ENV_FILE" || { + log_error "Environment" "Failed to restrict environment file permissions: $WORKER_ENV_FILE" + return 1 + } } upsert_env_value() { @@ -53,13 +49,18 @@ upsert_env_value() { } grep -v "^export $name=" "$WORKER_ENV_FILE" > "$tmpfile" || true - printf 'export %s="%s"\n' "$name" "$(escape_env_value "$value")" >> "$tmpfile" + printf 'export %s=%q\n' "$name" "$value" >> "$tmpfile" mv "$tmpfile" "$WORKER_ENV_FILE" || { rm -f "$tmpfile" log_error "Environment" "Failed to update environment file" return 1 } + + chmod 600 "$WORKER_ENV_FILE" || { + log_error "Environment" "Failed to restrict environment file permissions: $WORKER_ENV_FILE" + return 1 + } } # Generate environment file with regular variables @@ -76,34 +77,20 @@ generate_env_file() { ensure_env_file || return 1 - # Process each environment variable while IFS= read -r entry; do - # Extract key and value using string manipulation instead of regex - if [[ $entry == export* ]]; then - # Remove 'export ' prefix - local kv_pair=${entry#export } - # Extract key (everything before =) - local key=${kv_pair%%=*} - # Extract value (everything after = and remove quotes) - local value=${kv_pair#*=} - value=${value//\"/} - value=${value#\"} - value=${value%\"} - - # Check if the environment variable is exported (available in the environment) - # We use printenv to check if it's truly in the environment, not just a shell variable - if ! printenv "$key" > /dev/null 2>&1; then - # Variable doesn't exist in environment, add it from config - upsert_env_value "$key" "$value" || return 1 - else - # Variable exists in environment, use that value instead - local env_value - env_value="$(printenv "$key")" - upsert_env_value "$key" "$env_value" || return 1 - log_info "Environment" "Detected [$key] in container environment - using runtime value instead of config value" - fi + local key value + key=$(echo "$entry" | jq -r '.key') + value=$(echo "$entry" | jq -r '.value | tostring') + + if ! printenv "$key" > /dev/null 2>&1; then + upsert_env_value "$key" "$value" || return 1 + else + local env_value + env_value="$(printenv "$key")" + upsert_env_value "$key" "$env_value" || return 1 + log_info "Environment" "Detected [$key] in container environment - using runtime value instead of config value" fi - done < <(echo "$config" | yq eval '.config.env | to_entries | .[] | "export " + .key + "=\"" + .value + "\""' -) + done < <(echo "$config" | jq -c '.config.env // {} | to_entries[]') } # Internal function to resolve and append secrets @@ -206,6 +193,8 @@ configure_environment() { return 1 fi + load_environment + log_info "Secure environment setup completed successfully." } @@ -276,7 +265,11 @@ get_env_value() { fi if [ -f "$WORKER_ENV_FILE" ]; then - grep "^export $var_name=" "$WORKER_ENV_FILE" | cut -d'=' -f2- | tr -d '"' + ( + # shellcheck source=/dev/null + source "$WORKER_ENV_FILE" + printenv "$var_name" + ) else log_error "Environment" "Environment file does not exist" return 1 diff --git a/lib/process_manager.sh b/lib/process_manager.sh index 9b9f979f..a7e9ef93 100644 --- a/lib/process_manager.sh +++ b/lib/process_manager.sh @@ -17,8 +17,27 @@ FINAL_CONFIG="${WORKER_CONFIG_DIR}/supervisor/supervisord.conf" trap 'handle_supervisor_signals SIGTERM' SIGTERM trap 'handle_supervisor_signals SIGINT' SIGINT +ensure_process_manager_dependencies() { + local missing=false + + for command in yq jq; do + if ! command -v "$command" >/dev/null 2>&1; then + log_error "Process Manager" "$command is not installed. Please ensure it is available in the PATH." + missing=true + fi + done + + [[ "$missing" == "false" ]] +} + # Main execution main() { + local enabled_services_count + + if ! ensure_process_manager_dependencies; then + exit 1 + fi + CONFIG_FILE=$(get_service_config_path) if [[ -z "$CONFIG_FILE" ]]; then @@ -27,7 +46,11 @@ main() { exit 0 fi - if ! has_enabled_services; then + if ! enabled_services_count=$(count_enabled_services); then + exit 1 + fi + + if [[ "${enabled_services_count:-0}" -eq 0 ]]; then log_info "No enabled services found in $CONFIG_FILE." exit 0 fi @@ -67,10 +90,21 @@ get_service_config_path() { return 1 } +count_enabled_services() { + local enabled_services_count + + if ! enabled_services_count=$(yq e '.services // [] | map(select(.ignore != true)) | length' "$CONFIG_FILE" 2>/dev/null); then + log_error "Process Manager" "Failed to parse services configuration: $CONFIG_FILE" + return 1 + fi + + echo "${enabled_services_count:-0}" +} + has_enabled_services() { local enabled_services_count - enabled_services_count=$(yq e '.services // [] | map(select(.ignore != true)) | length' "$CONFIG_FILE" 2>/dev/null) + enabled_services_count=$(count_enabled_services) || return 1 [[ "${enabled_services_count:-0}" -gt 0 ]] } diff --git a/lib/runtime_output.sh b/lib/runtime_output.sh index fb2a4911..736dc2d4 100644 --- a/lib/runtime_output.sh +++ b/lib/runtime_output.sh @@ -5,11 +5,6 @@ source "${WORKER_LIB_DIR}/utils.sh" # shellcheck source=${WORKER_LIB_DIR}/worker_config.sh disable=SC1091 source "${WORKER_LIB_DIR}/worker_config.sh" -get_effective_env_value() { - local name="$1" - printenv "$name" 2>/dev/null || true -} - build_runtime_output_json() { local config_json="$1" local worker_config_path services_config_path env_json secrets_json @@ -20,20 +15,8 @@ build_runtime_output_json() { services_config_path="${WORKER_CONFIG_DIR}/services.yaml" fi - env_json=$( - echo "$config_json" | jq -r '.config.env // {} | keys[]' 2>/dev/null | while IFS= read -r key; do - [ -n "$key" ] || continue - value=$(get_effective_env_value "$key") - printf '%s\t%s\n' "$key" "$value" - done | jq -Rn ' - reduce inputs as $line ({}; - ($line | split("\t")) as $parts | - . + {($parts[0]): ($parts[1] // "")} - ) - ' - ) - - secrets_json=$(echo "$config_json" | jq '.config.secrets // {}' 2>/dev/null) + env_json=$(echo "$config_json" | jq '.config.env // {} | with_entries(.value = "redacted")' 2>/dev/null) || return 1 + secrets_json=$(echo "$config_json" | jq '.config.secrets // {} | with_entries(.value = "redacted")' 2>/dev/null) || return 1 jq -n \ --arg worker_config_path "$worker_config_path" \ @@ -64,9 +47,13 @@ emit_runtime_output() { fi config_json=$(load_and_parse_config) || return 1 - runtime_json=$(build_runtime_output_json "$config_json") + if ! runtime_json=$(build_runtime_output_json "$config_json"); then + log_error "Runtime output" "Failed to build runtime output JSON" + return 1 + fi mkdir -p "$(dirname "$WORKER_OUTPUT_FILE")" || return 1 - echo "$runtime_json" > "$WORKER_OUTPUT_FILE" + install -m 600 /dev/null "$WORKER_OUTPUT_FILE" || return 1 + printf '%s\n' "$runtime_json" > "$WORKER_OUTPUT_FILE" log_info "Runtime output written to $WORKER_OUTPUT_FILE" } From b73f401a4fcc2de1ea2497aad8a540648b3b6c8e Mon Sep 17 00:00:00 2001 From: Dmytro Smirnov Date: Wed, 3 Jun 2026 13:35:41 +0300 Subject: [PATCH 32/32] fix: mask env secrets in CLI output --- README.md | 4 ++-- lib/cli/env.sh | 31 ++++++++++++++++++++++++++++--- 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 8e12e694..0ec732a6 100644 --- a/README.md +++ b/README.md @@ -84,8 +84,8 @@ docker run -d \ -v "$(pwd)/.config/worker:/home/udx/.config/worker" \ usabilitydynamics/udx-worker:latest -# Verify resolved environment -docker exec my-secrets worker env show --filter API_KEY +# Verify the resolved environment without printing the secret value +docker exec my-secrets sh -lc 'worker env show --filter API_KEY --format json | jq -e '\''has("API_KEY") and .API_KEY != ""'\'' >/dev/null' ``` See [Secrets](docs/secrets.md) for secret references and provider auth boundaries. diff --git a/lib/cli/env.sh b/lib/cli/env.sh index 64072ff5..dfd37e3d 100644 --- a/lib/cli/env.sh +++ b/lib/cli/env.sh @@ -23,7 +23,7 @@ Available Commands: Options: --format Output format (text/json) --filter Filter variables by prefix - --include-secrets Include secrets in output (masked) + --include-secrets Include unmasked secrets in output Examples: worker env show # Show all environment variables @@ -39,6 +39,31 @@ EOF # Description: Display environment variables with optional filtering # Options: --format text|json, --filter PATTERN, --include-secrets # Example: worker env show --format json --filter AWS_* --include-secrets +is_secret_env_name() { + local name="$1" + local config + + config=$(load_and_parse_config) || return 1 + echo "$config" | jq -e --arg name "$name" --arg pattern "^(${SUPPORTED_SECRET_PROVIDERS})/.+/.+" ' + (.config.secrets // {} | has($name)) or + ((.config.env // {} | .[$name] // "" | tostring) | test($pattern)) + ' >/dev/null +} + +format_env_value_for_output() { + local name="$1" + local include_secrets="$2" + local value + + if [[ "$include_secrets" != "true" ]] && is_secret_env_name "$name"; then + printf '%s' '********' + return 0 + fi + + value=$(get_env_value "$name") || return 1 + printf '%s' "$value" +} + show_environment() { local format=${1:-text} local filter=${2:-} @@ -60,7 +85,7 @@ show_environment() { # shellcheck disable=SC2053 # Env filters intentionally support globs like AWS_*. if [[ -n "$name" && ( -z "$filter" || "$name" == $filter ) ]]; then local value - value=$(get_env_value "$name") + value=$(format_env_value_for_output "$name" "$include_secrets") json=$(echo "$json" | jq --arg key "$name" --arg value "$value" '. + {($key): $value}') fi done <<< "$names" @@ -70,7 +95,7 @@ show_environment() { while IFS= read -r name; do # shellcheck disable=SC2053 # Env filters intentionally support globs like AWS_*. if [[ -n "$name" && ( -z "$filter" || "$name" == $filter ) ]]; then - printf '%s=%s\n' "$name" "$(get_env_value "$name")" + printf '%s=%s\n' "$name" "$(format_env_value_for_output "$name" "$include_secrets")" fi done <<< "$names" ;;