diff --git a/.github/workflows/ansible-lint.yml b/.github/workflows/ansible-lint.yml index a08ac59..373d2ae 100644 --- a/.github/workflows/ansible-lint.yml +++ b/.github/workflows/ansible-lint.yml @@ -16,8 +16,11 @@ on: push: branches: - main - - release-1.0 + - 'release-[0-9]+\.[0-9]+' pull_request: + branches: + - main + - 'release-[0-9]+\.[0-9]+' types: [opened, synchronize, reopened, ready_for_review] permissions: diff --git a/.github/workflows/bandit.yml b/.github/workflows/bandit.yml index 0a976fb..50c2c81 100644 --- a/.github/workflows/bandit.yml +++ b/.github/workflows/bandit.yml @@ -16,8 +16,11 @@ on: push: branches: - main - - release-1.0 + - 'release-[0-9]+\.[0-9]+' pull_request: + branches: + - main + - 'release-[0-9]+\.[0-9]+' types: [opened, synchronize, reopened, ready_for_review] schedule: # Weekly re-scan so new findings surface without a code change. diff --git a/.github/workflows/checkov.yml b/.github/workflows/checkov.yml index 49d884f..b61a886 100644 --- a/.github/workflows/checkov.yml +++ b/.github/workflows/checkov.yml @@ -16,8 +16,11 @@ on: push: branches: - main - - release-1.0 + - 'release-[0-9]+\.[0-9]+' pull_request: + branches: + - main + - 'release-[0-9]+\.[0-9]+' types: [opened, synchronize, reopened, ready_for_review] schedule: # Weekly re-scan so new findings surface without a code change. diff --git a/.github/workflows/shellcheck.yml b/.github/workflows/shellcheck.yml index 1e868fe..5266420 100644 --- a/.github/workflows/shellcheck.yml +++ b/.github/workflows/shellcheck.yml @@ -16,8 +16,11 @@ on: push: branches: - main - - release-1.0 + - 'release-[0-9]+\.[0-9]+' pull_request: + branches: + - main + - 'release-[0-9]+\.[0-9]+' types: [opened, synchronize, reopened, ready_for_review] permissions: diff --git a/.github/workflows/trivy.yml b/.github/workflows/trivy.yml index d695e54..f82ef50 100644 --- a/.github/workflows/trivy.yml +++ b/.github/workflows/trivy.yml @@ -16,8 +16,11 @@ on: push: branches: - main - - release-1.0 + - 'release-[0-9]+\.[0-9]+' pull_request: + branches: + - main + - 'release-[0-9]+\.[0-9]+' types: [opened, synchronize, reopened, ready_for_review] schedule: # Weekly re-scan so new findings surface without a code change. diff --git a/model_manager/lib/helpers.sh b/model_manager/lib/helpers.sh index b8ac43e..1a6a0ab 100644 --- a/model_manager/lib/helpers.sh +++ b/model_manager/lib/helpers.sh @@ -235,6 +235,37 @@ pod_progress() { esac } +# wait_pods_gone — block until no pods for a model remain in the namespace. +# +# Deleting the serving CR (LLMInferenceService/InferenceService) returns as soon +# as the CR object itself is gone, but the KServe-owned Deployment → ReplicaSet → +# Pods are torn down asynchronously by the garbage collector and honour their +# graceful-termination period. Those pods stay in phase Running (Terminating) +# and keep their pinned CPUs / NRI balloon until they are truly removed — which +# is why an immediate cpu-collisions-check still lists them. This waits for that +# tail so `undeploy --wait` only returns once the CPUs are actually reclaimed. +# +# Bounded by the timeout so a pod wedged by a finalizer or stuck termination +# never blocks the CLI forever. Returns 0 once the pods are gone, 1 on timeout. +wait_pods_gone() { + local namespace="$1" name="$2" timeout="${3:-900}" + local selector="app.kubernetes.io/name=$name" + local start=$SECONDS end=$((SECONDS + timeout)) + local last_beat=0 remaining + while (( SECONDS < end )); do + remaining=$(kubectl get pods -n "$namespace" -l "$selector" \ + -o name 2>/dev/null | wc -l) + remaining=${remaining//[[:space:]]/} + (( remaining == 0 )) && return 0 + if (( SECONDS - last_beat >= 15 )); then + info " ${DIM}[$(fmt_duration $((SECONDS - start)))] waiting for ${remaining} pod(s) to terminate...${RESET}" + last_beat=$SECONDS + fi + sleep 3 + done + return 1 +} + patch_pool_status() { local name="$1" namespace="$2" diff --git a/model_manager/model-manager b/model_manager/model-manager index fb8668f..83cdb6d 100755 --- a/model_manager/model-manager +++ b/model_manager/model-manager @@ -34,11 +34,17 @@ source "$MM_LIB/cpu_policy.sh" # ── Global option state ────────────────────────────────────────────────────── DRY_RUN=false WAIT=false -WAIT_TIMEOUT=900 +WAIT_TIMEOUT=900 # deploy default; large because model load + warmup is slow +WAIT_TIMEOUT_SET=false # true once --wait-timeout is given explicitly +UNDEPLOY_WAIT_TIMEOUT=180 # undeploy default: pod graceful shutdown is quick (3 min ceiling) DOWNLOAD_TIMEOUT=7200 EXTRA_ARGS=() EXTRA_ENV=() +# Extra metadata labels for the serving CR. Lets a caller stamp its own +# ownership marker: the built-in managed-by=model-manager is shared by every +# caller, so it cannot distinguish who deployed a model. +EXTRA_LABELS=() # Per-model / per-version override JSON, set during resolution. Safe defaults # here so build_env_block / build_args_block never see an unset var (a brace @@ -65,6 +71,19 @@ require_int() { [[ "$2" =~ ^[0-9]+$ ]] || abort "--$1 requires a positive integer, got: $2" } +# Validate a --label argument up front: a malformed key or value would otherwise +# only surface as a rejected apply, long after the weights download. +require_label() { + [[ "$1" == *=* ]] || abort "--label requires KEY=VALUE, got: $1" + local key="${1%%=*}" value="${1#*=}" + [[ -n "$key" ]] || abort "--label key must not be empty: $1" + # Optional DNS-subdomain prefix, then a name; value may be empty (k8s allows it). + [[ "$key" =~ ^([a-z0-9]([-a-z0-9.]*[a-z0-9])?/)?[a-zA-Z0-9]([-_.a-zA-Z0-9]*[a-zA-Z0-9])?$ ]] \ + || abort "--label key is not a valid Kubernetes label key: $key" + [[ -z "$value" || "$value" =~ ^[a-zA-Z0-9]([-_.a-zA-Z0-9]*[a-zA-Z0-9])?$ ]] \ + || abort "--label value is not a valid Kubernetes label value: $value" +} + # ── Usage ──────────────────────────────────────────────────────────────────── usage() { cat < Pin deployment to a specific node ${GREEN}--env${RESET} KEY=VAL Add/override env var (repeatable) ${GREEN}--arg${RESET} "--flag=val" Append a runtime arg (repeatable) - ${GREEN}--wait${RESET} Block until model is Ready - ${GREEN}--wait-timeout${RESET} N Seconds to wait (default: 900) + ${GREEN}--label${RESET} KEY=VAL Add a metadata label to the serving CR (repeatable); + use it to mark ownership — managed-by=model-manager + is set for every caller and cannot distinguish them + ${GREEN}--wait${RESET} deploy: block until model is Ready; undeploy: block until pods terminate + ${GREEN}--wait-timeout${RESET} N Seconds to wait (default: deploy 900, undeploy 180) ${GREEN}--dry-run${RESET} Print manifest without applying ${BOLD}${CYAN}Examples:${RESET} @@ -101,6 +123,7 @@ ${BOLD}${CYAN}Examples:${RESET} ./model-manager deploy llama3-8b-awq --server-version 0.19.1 --wait ./model-manager deploy --id meta-llama/Llama-3.2-1B-Instruct --cpu 8 --memory 16Gi ./model-manager undeploy llama3-8b-awq + ./model-manager undeploy llama3-8b-awq --wait # block until pods terminate + CPUs are freed ${BOLD}${CYAN}For status / scale / logs, use kubectl:${RESET} kubectl get llminferenceservices,inferenceservices -n llm-inference @@ -135,7 +158,7 @@ parse_args() { case "$1" in --dry-run) DRY_RUN=true; shift ;; --wait) WAIT=true; shift ;; - --wait-timeout) require_int wait-timeout "$2"; WAIT_TIMEOUT="$2"; shift 2 ;; + --wait-timeout) require_int wait-timeout "$2"; WAIT_TIMEOUT="$2"; WAIT_TIMEOUT_SET=true; shift 2 ;; --id) OPT_ID="$2"; shift 2 ;; --name) OPT_NAME="$2"; shift 2 ;; --server) OPT_SERVER="$2"; shift 2 ;; @@ -152,6 +175,7 @@ parse_args() { --node) OPT_NODE="$2"; shift 2 ;; --env) EXTRA_ENV+=("$2"); shift 2 ;; --arg) EXTRA_ARGS+=("$2"); shift 2 ;; + --label) require_label "$2"; EXTRA_LABELS+=("$2"); shift 2 ;; -h|--help) usage; exit 0 ;; -v|--version) echo "model-manager $MM_VERSION"; exit 0 ;; --) shift; POSITIONAL+=("$@"); break ;; @@ -470,40 +494,29 @@ build_chat_template_blocks() { " } -# Args block. vLLM applies repeated flags last-wins, so we simply append in -# ascending precedence and let the engine resolve duplicates: +# Args block. Precedence (lowest → highest), deduped by flag name so a higher +# layer REPLACES rather than appends — the emitted list matches vLLM's +# last-wins runtime semantics but the CRD carries only one occurrence: # category defaults → version args delta → models.yaml per-model args # → --served-model-name / --chat-template → --arg CLI build_args_block() { local rj rj=$(runtime_json "$MODEL_RUNTIME") - local out="" arg # NB: OVMS args come entirely from the ClusterServingRuntime (--config_path); # the InferenceService adds none. The model is pre-exported to IR by the prep # job, so no --source_model here. - while IFS= read -r arg; do - [[ -n "$arg" ]] && out+=" - \"$arg\" -" - done < <(jq -r --arg c "$MODEL_CATEGORY" '.categories[$c].args // [] | .[]' <<< "$rj" 2>/dev/null) - - # Per-version args delta (from the selected version). Always initialized upstream. - while IFS= read -r arg; do - [[ -n "$arg" ]] && out+=" - \"$arg\" -" - done < <(jq -r '.[]' <<< "$RUNTIME_VERSION_ARGS_JSON" 2>/dev/null) - - # Per-model args from models.yaml — central, reusable override. - while IFS= read -r arg; do - [[ -n "$arg" ]] && out+=" - \"$arg\" -" - done < <(jq -r '.[]' <<< "$MODEL_ARGS_JSON" 2>/dev/null) + # Concatenate every source into one JSON array, low-precedence first. + local all_json + all_json=$(jq -cn \ + --argjson cat "$(jq -c --arg c "$MODEL_CATEGORY" '.categories[$c].args // []' <<< "$rj")" \ + --argjson ver "$RUNTIME_VERSION_ARGS_JSON" \ + --argjson model "$MODEL_ARGS_JSON" \ + '$cat + $ver + $model') if [[ "$MODEL_KIND" == "LLMInferenceService" ]]; then - out+=" - \"--served-model-name\" - - \"$MODEL_NAME\" -" + all_json=$(jq -c --arg n "$MODEL_NAME" '. + ["--served-model-name", $n]' <<< "$all_json") # Optional per-model chat template (models.yaml: chat_template). # Either form resolves to a PATH here, never to inline Jinja: an absolute # path is passed through untouched, while inline Jinja is written to a @@ -513,22 +526,75 @@ build_args_block() { # apply_chat_template_cm). Placed before EXTRA_ARGS so a user # --arg "--chat-template=..." still wins. if [[ -n "${MODEL_CHAT_TEMPLATE:-}" ]]; then - out+=" - \"--chat-template\" - - $(yaml_quote "$(chat_template_path)") -" + all_json=$(jq -c --arg t "$(chat_template_path)" \ + '. + ["--chat-template", $t]' <<< "$all_json") fi fi - for arg in "${EXTRA_ARGS[@]}"; do - out+=" - \"$arg\" + # EXTRA_ARGS from --arg CLI — highest precedence. + if (( ${#EXTRA_ARGS[@]} > 0 )); then + local extra_json + extra_json=$(printf '%s\n' "${EXTRA_ARGS[@]}" | jq -Rc '[inputs]') + all_json=$(jq -c --argjson e "$extra_json" '. + $e' <<< "$all_json") + fi + + # Dedupe by flag name, keeping the LAST occurrence. Recognizes three forms + # per entry: "--flag=value" (inline), "--flag" "value" (paired, two slots), + # "--flag" (boolean). Non-flag tokens (no leading --) are kept in place and + # deduped by their literal value. + local deduped_json + deduped_json=$(jq -c ' + . as $a + | [range(0; $a | length)] + | reduce .[] as $i ({out: [], skip: false}; + if .skip then .skip = false + elif ($a[$i] | test("^--[^=]+=")) then + .out += [{key: ($a[$i] | capture("^(?--[^=]+)=") | .k), tokens: [$a[$i]]}] + elif ($a[$i] | test("^--")) then + if ($i + 1 < ($a | length)) and (($a[$i+1] | test("^--")) | not) then + (.out += [{key: $a[$i], tokens: [$a[$i], $a[$i+1]]}]) | .skip = true + else + .out += [{key: $a[$i], tokens: [$a[$i]]}] + end + else + .out += [{key: $a[$i], tokens: [$a[$i]]}] + end + ) + | .out + | reverse + | reduce .[] as $e ({seen: {}, out: []}; + if .seen[$e.key] then . else (.seen[$e.key] = true | .out += [$e]) end + ) + | .out + | reverse + | map(.tokens[]) + ' <<< "$all_json") + + # Emit YAML list items. yaml_quote uses jq @json so values with special + # characters (spaces, quotes, backslashes) render as valid YAML scalars. + local out="" arg + while IFS= read -r arg; do + out+=" - $(yaml_quote "$arg") " - done + done < <(jq -r '.[]' <<< "$deduped_json") # Leading newline before list items, or inline `[]` when empty — mirrors # build_env_block so `args:` is never rendered as null. if [[ -z "$out" ]]; then printf '[]'; else printf '\n%s' "${out%$'\n'}"; fi } +# Extra metadata.labels entries from --label, appended under the built-in labels. +# Leading newline per entry and none trailing — same idiom as build_args_block, +# because `$(...)` strips trailing newlines but not leading ones. The templates +# therefore put {{EXTRA_LABELS}} at the end of the last built-in label line. +build_labels_block() { + local out="" pair + for pair in "${EXTRA_LABELS[@]}"; do + out+=$'\n'" ${pair%%=*}: $(yaml_quote "${pair#*=}")" + done + printf '%s' "$out" +} + # Optional nodeAffinity block. # Includes soft preference for --node and hard NRI max-tp requirement when applicable. # When require_amx is enabled, adds a hard requirement for AMX-capable nodes (NFD label). @@ -667,10 +733,11 @@ nri_preflight_gate() { } render_manifest() { - local env_block args_block affinity_block + local env_block args_block affinity_block labels_block resolve_tp_size # mutates EXTRA_ARGS; sets TP_SIZE env_block=$(build_env_block) args_block=$(build_args_block) + labels_block=$(build_labels_block) # NB: ordering is deliberately unchanged from the NRI-only version — # build_affinity_block runs BEFORE the policy block, so the hard max-tp # nodeAffinity that build_cpu_policy_annotations would set via @@ -694,7 +761,8 @@ render_manifest() { "GATEWAY_NAMESPACE=${MM_INFERENCE_GATEWAY_NS:-llm-inference}" \ "CHAT_TEMPLATE_MOUNT=$CHAT_TEMPLATE_MOUNT" \ "CHAT_TEMPLATE_VOLUME=$CHAT_TEMPLATE_VOLUME" \ - "AFFINITY=$affinity_block" + "AFFINITY=$affinity_block" \ + "EXTRA_LABELS=$labels_block" ;; InferenceService) local template="$MM_TEMPLATES/inference-service.yaml" @@ -707,7 +775,8 @@ render_manifest() { "RUNTIME_NAME=${RUNTIME_NAME:-$MODEL_RUNTIME-runtime}" \ "CPU=$MODEL_CPU" "MEMORY=$MODEL_MEMORY" "REPLICAS=$MODEL_REPLICAS" \ "ARGS=$args_block" "ENV=$env_block" "AFFINITY=$affinity_block" \ - "NRI_ANNOTATIONS=$CPU_POLICY_ANNOTATIONS" + "NRI_ANNOTATIONS=$CPU_POLICY_ANNOTATIONS" \ + "EXTRA_LABELS=$labels_block" ;; *) abort "Unknown model kind: $MODEL_KIND" ;; esac @@ -1315,7 +1384,28 @@ cmd_undeploy() { --ignore-not-found >/dev/null 2>&1 || true aigateway_deregister "$MODEL_NAME" "$MODEL_NAMESPACE" "$MODEL_KIND" "$MODEL_ROUTING" litellm_deregister "$MODEL_NAME" - ok "$MODEL_NAME undeployed" + + # Deleting the CR above is asynchronous: the serving pods enter Terminating + # and keep their pinned CPUs / NRI balloon until the graceful shutdown + # completes. With --wait, block until they are actually gone so callers (and + # a follow-up cpu-collisions-check) see the CPUs reclaimed. Bounded by + # --wait-timeout so a stuck termination can't hang the CLI indefinitely. + if [[ "$WAIT" == true ]]; then + # Undeploy uses a short default (pod shutdown is quick); an explicit + # --wait-timeout always wins. The wait returns the instant the pods are + # gone, so this value is only a ceiling, not a fixed delay. + local timeout="$UNDEPLOY_WAIT_TIMEOUT" + [[ "$WAIT_TIMEOUT_SET" == true ]] && timeout="$WAIT_TIMEOUT" + info "Waiting for $MODEL_NAME pods to terminate (timeout: ${timeout}s)..." + if wait_pods_gone "$MODEL_NAMESPACE" "$MODEL_NAME" "$timeout"; then + ok "$MODEL_NAME undeployed (all pods terminated, CPUs released)" + else + warn "$MODEL_NAME undeployed, but pods are still terminating after ${timeout}s — CPUs/balloons may not be reclaimed yet" + warn " Check: kubectl get pods -n $MODEL_NAMESPACE -l app.kubernetes.io/name=$MODEL_NAME" + fi + else + ok "$MODEL_NAME undeployed" + fi } # Discover a live deployment by name when it has no catalog entry (e.g. an diff --git a/model_manager/templates/inference-service-ovms.yaml b/model_manager/templates/inference-service-ovms.yaml index 62f6791..3fc11aa 100644 --- a/model_manager/templates/inference-service-ovms.yaml +++ b/model_manager/templates/inference-service-ovms.yaml @@ -5,7 +5,7 @@ metadata: namespace: {{NAMESPACE}} labels: app.kubernetes.io/name: {{MODEL_NAME}} - app.kubernetes.io/managed-by: model-manager + app.kubernetes.io/managed-by: model-manager{{EXTRA_LABELS}} spec: predictor: {{NRI_ANNOTATIONS}} minReplicas: {{REPLICAS}} diff --git a/model_manager/templates/inference-service.yaml b/model_manager/templates/inference-service.yaml index dd66074..cc89f00 100644 --- a/model_manager/templates/inference-service.yaml +++ b/model_manager/templates/inference-service.yaml @@ -5,7 +5,7 @@ metadata: namespace: {{NAMESPACE}} labels: app.kubernetes.io/name: {{MODEL_NAME}} - app.kubernetes.io/managed-by: model-manager + app.kubernetes.io/managed-by: model-manager{{EXTRA_LABELS}} spec: predictor: {{NRI_ANNOTATIONS}} minReplicas: {{REPLICAS}} diff --git a/model_manager/templates/llm-inference-service.yaml b/model_manager/templates/llm-inference-service.yaml index 8395814..dec2215 100644 --- a/model_manager/templates/llm-inference-service.yaml +++ b/model_manager/templates/llm-inference-service.yaml @@ -5,7 +5,7 @@ metadata: namespace: {{NAMESPACE}} labels: app.kubernetes.io/name: {{MODEL_NAME}} - app.kubernetes.io/managed-by: model-manager + app.kubernetes.io/managed-by: model-manager{{EXTRA_LABELS}} spec: {{NRI_ANNOTATIONS}} model: uri: {{MODEL_URI}} diff --git a/roles/llm_services/defaults/main.yaml b/roles/llm_services/defaults/main.yaml index 3af4cff..d86b989 100644 --- a/roles/llm_services/defaults/main.yaml +++ b/roles/llm_services/defaults/main.yaml @@ -38,14 +38,36 @@ llm_services_namespace: "llm-inference" # ============================================================================= llm_services_model_store_pvc: "model-store" llm_services_model_store_size: "100Gi" + +# Set storage_backend in env//global_config.yaml, not this. The core storage +# role publishes _resolved_storage_backend in preflight, already trimmed and +# lowercased; the fallback keeps this role working without core's preflight. +_llm_storage_backend: >- + {{ _resolved_storage_backend | default(storage_backend | default('local-path')) + | string | trim | lower }} + +# model-store is one PVC mounted by every predictor pod, so it needs +# ReadWriteMany as soon as pods can land on more than one node. Only local-path +# cannot provide it (node-local directories); nfs, CephFS and ontap-nas all can. llm_services_model_store_access_mode: >- - {{ 'ReadWriteOnce' if storage_backend | default('local-path') == 'local-path' - else 'ReadWriteMany' }} + {{ 'ReadWriteOnce' if _llm_storage_backend == 'local-path' else 'ReadWriteMany' }} + +# StorageClass for model-store. Named only for the backends whose *default* +# StorageClass is not the right one for a shared model cache (ceph defaults to +# ceph-block, which is RWO). Empty means "omit storageClassName" so the PVC binds +# to the cluster default, which core asserts is exactly one - that is the +# netapp-trident path. Never fall back to 'local-path': it only exists when the +# backend IS local-path, so that fallback pointed model-store at a StorageClass +# that does not exist and the PVC stayed Pending forever. llm_services_model_store_storage_class: >- - {{ 'local-path' if storage_backend | default('local-path') == 'local-path' - else 'nfs-client' if storage_backend == 'nfs' - else 'ceph-fs' if storage_backend == 'ceph' - else 'local-path' }} + {{ 'local-path' if _llm_storage_backend == 'local-path' + else 'nfs-client' if _llm_storage_backend == 'nfs' + else 'ceph-fs' if _llm_storage_backend == 'ceph' + else '' }} + +# Tolerates null, which `| length` would crash on. +_llm_sc_requested: >- + {{ llm_services_model_store_storage_class | default('', true) | string | trim }} # Deploy files/runtimes/*.yaml as namespace-scoped ServingRuntimes. # Needed for InferenceService CRs; LLMInferenceService CRs ignore these. diff --git a/roles/llm_services/tasks/install.yaml b/roles/llm_services/tasks/install.yaml index 155f4b3..cb4c989 100644 --- a/roles/llm_services/tasks/install.yaml +++ b/roles/llm_services/tasks/install.yaml @@ -52,13 +52,24 @@ labels: app.kubernetes.io/name: "{{ llm_services_model_store_pvc }}" app.kubernetes.io/managed-by: ansible - spec: - accessModes: - - "{{ llm_services_model_store_access_mode }}" - resources: - requests: - storage: "{{ llm_services_model_store_size }}" - storageClassName: "{{ llm_services_model_store_storage_class }}" + spec: "{{ _llm_model_store_spec }}" + vars: + _llm_model_store_spec_base: + accessModes: + - "{{ llm_services_model_store_access_mode }}" + resources: + requests: + storage: "{{ llm_services_model_store_size }}" + # An empty storage class must render as *no* storageClassName key, so the PVC + # binds to the cluster default StorageClass. storageClassName: "" means the + # opposite - dynamic provisioning disabled - and the PVC stays Pending. + # _llm_sc_requested is the null/whitespace-tolerant form of + # llm_services_model_store_storage_class (defaults/main.yaml). + _llm_model_store_spec: >- + {{ _llm_model_store_spec_base + | combine({'storageClassName': _llm_sc_requested}) + if _llm_sc_requested | length > 0 + else _llm_model_store_spec_base }} # -- CRD readiness gates (fail fast if KServe isn't installed) -- diff --git a/roles/nfd/tasks/install.yaml b/roles/nfd/tasks/install.yaml index bcdfa90..4324344 100644 --- a/roles/nfd/tasks/install.yaml +++ b/roles/nfd/tasks/install.yaml @@ -69,6 +69,24 @@ cpu: "200m" memory: "128Mi" + # NFD worker mounts hostPath (/sys, /etc/kubernetes/node-feature-discovery, + # /usr/src) and typically uses hostNetwork; namespace-level PSS applies to + # every pod in the namespace, so master/gc must sit at the same floor the + # worker requires — "privileged". + - name: "nfd | Label namespace — PSS" + kubernetes.core.k8s: + state: present + definition: + apiVersion: v1 + kind: Namespace + metadata: + name: "{{ nfd_namespace }}" + labels: + pod-security.kubernetes.io/enforce: "privileged" + pod-security.kubernetes.io/audit: "privileged" + pod-security.kubernetes.io/warn: "privileged" + when: enforce_pss | default(false) | bool + # Wait for NFD worker pods to be running on all nodes so labels are applied - name: "nfd | Wait for worker DaemonSet ready" kubernetes.core.k8s_info: