From c57e5331652b9843e1f9b30e126b43beed473f40 Mon Sep 17 00:00:00 2001 From: Dalibor Kovacevic <56942801+RobiladK@users.noreply.github.com> Date: Fri, 7 Aug 2026 20:19:33 -0700 Subject: [PATCH] Harden the Zava SRE Agent deployment and correlation demo Align the Zava AKS/PostgreSQL sample with current SRE Agent deployment contracts, improve repeatable break/fix behavior, document private networking and response-plan routing, and teach investigations to correlate nearby alerts by mechanism rather than timing.`n`nCo-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .gitignore | 3 + .../.github/skills/deploying-demo/SKILL.md | 19 +- .../skills/managing-sre-agent/SKILL.md | 32 +- .../.github/skills/running-demo/SKILL.md | 51 ++- .../running-demo/scripts/break-bad-deploy.ps1 | 5 + .../running-demo/scripts/break-compound.ps1 | 124 +++++++ .../running-demo/scripts/break-db-perf.ps1 | 5 + .../running-demo/scripts/fix-compound.ps1 | 56 +++ labs/zava-aks-postgres/AGENTS.md | 59 ++- labs/zava-aks-postgres/README.md | 120 +++--- .../docs/aks-access-and-auth.md | 202 ++++++++++ labs/zava-aks-postgres/infra/main.bicep | 40 ++ labs/zava-aks-postgres/infra/main.bicepparam | 6 +- labs/zava-aks-postgres/infra/main.json | 344 +++++++++++++++--- .../zava-aks-postgres/infra/modules/aks.bicep | 5 +- .../modules/firewall-agent-dataplane.bicep | 75 ++++ .../infra/modules/monitoring.bicep | 55 +++ .../infra/modules/sre-agent.bicep | 229 +++++++++--- .../infra/modules/subscription-reader.bicep | 15 + .../infra/modules/vnet.bicep | 45 ++- .../scripts/_aks-helpers.ps1 | 57 +++ .../scripts/post-provision.ps1 | 22 +- labs/zava-aks-postgres/scripts/pre-down.ps1 | 76 +++- .../scripts/setup-sre-agent.ps1 | 174 ++++++++- .../sre-config/custom-instructions.md | 21 ++ .../knowledge-base/zava-architecture.md | 9 +- .../vnet-integrated-keyvault/README.md | 2 +- .../vnet-integrated-keyvault/variables.tf | 4 +- 28 files changed, 1601 insertions(+), 254 deletions(-) create mode 100644 labs/zava-aks-postgres/.github/skills/running-demo/scripts/break-compound.ps1 create mode 100644 labs/zava-aks-postgres/.github/skills/running-demo/scripts/fix-compound.ps1 create mode 100644 labs/zava-aks-postgres/docs/aks-access-and-auth.md create mode 100644 labs/zava-aks-postgres/infra/modules/firewall-agent-dataplane.bicep create mode 100644 labs/zava-aks-postgres/infra/modules/subscription-reader.bicep create mode 100644 labs/zava-aks-postgres/sre-config/custom-instructions.md diff --git a/.gitignore b/.gitignore index 300f7ca36..65f87743c 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,6 @@ Thumbs.db Desktop.ini samples/deployment-compliance/skills/.DS_Store + +# Browser automation artifacts +.playwright-mcp/ diff --git a/labs/zava-aks-postgres/.github/skills/deploying-demo/SKILL.md b/labs/zava-aks-postgres/.github/skills/deploying-demo/SKILL.md index d2abe2005..d8c9b9abe 100644 --- a/labs/zava-aks-postgres/.github/skills/deploying-demo/SKILL.md +++ b/labs/zava-aks-postgres/.github/skills/deploying-demo/SKILL.md @@ -10,10 +10,14 @@ Run these and install anything missing: - `az version` — need 2.60+ - `azd version` — need 1.9+ - `pwsh -v` — need 7.4+ +- Azure permission: Owner, User Access Administrator, or equivalent + `Microsoft.Authorization/roleAssignments/write` at subscription scope. The + template grants the agent runtime identity subscription Reader for correlation + context and removes it during `azd down`. > Note: `kubectl` is **not** required on your local workstation. The cluster is private. Operator > scripts in this repo go through `az aks command invoke` (wrapped by `scripts/_aks-helpers.ps1`). -> The SRE Agent reaches the cluster the same way through its `az` CLI tools — no kubeconfig either side. +> The SRE Agent uses its built-in `RunKubectlReadCommand` and `RunKubectlWriteCommand` tools instead. ## Phase 1: Azure Deployment 1. Check if user has a subscription: `az account show` @@ -25,7 +29,7 @@ Run these and install anything missing: ## Phase 2: Verify Deployment The AKS API server is private (`enablePrivateCluster: true`) — local kubectl -will not work. Use the same path the SRE Agent uses: +will not work. Human operators use the Azure-proxied command-invoke path: ```powershell . .\scripts\_aks-helpers.ps1 @@ -46,12 +50,19 @@ $ip = ($r.logs -replace '[^\d\.]','').Trim() -Command "kubectl exec -n zava-demo deploy/zava-api -- wget -qO- http://localhost:3001/api/health" ``` +> If you are manually testing from an SRE Agent chat and absolutely need terminal-native kubectl, +> first run a read-only command against this cluster with `RunKubectlReadCommand`. That warms the +> process-local AKS CA path; terminal kubectl can then use an already-valid kubeconfig in the same +> runtime. A runtime restart clears the warm-up. Since `RunKubectl*` accepts the same kubectl +> commands, prefer it directly. See `docs/aks-access-and-auth.md` for the kubeconfig, TLS, +> authentication, authorization, and private-network mechanics behind this behavior. + ## Phase 3: Sync knowledge + verify SRE Agent The agent itself — connectors, custom skills, response plans, autonomous mode, Azure Monitor binding — is already provisioned by Bicep during `azd up`. This -script just uploads knowledge files (the one data-plane piece with no ARM API) -and prints a verification readout of the Bicep-deployed assets. +script uploads knowledge files, syncs the agent-global custom instructions, +enables the Microsoft Learn MCP tools, and verifies the complete configuration. 1. Get azd values: `$env:SRE_AGENT_ENDPOINT = azd env get-value SRE_AGENT_ENDPOINT` (and RESOURCE_GROUP, SRE_AGENT_NAME) 2. Run: `.\scripts\setup-sre-agent.ps1` (auto-detects ResourceGroup and AgentName from `azd env`) diff --git a/labs/zava-aks-postgres/.github/skills/managing-sre-agent/SKILL.md b/labs/zava-aks-postgres/.github/skills/managing-sre-agent/SKILL.md index 2c313e186..f5c68254d 100644 --- a/labs/zava-aks-postgres/.github/skills/managing-sre-agent/SKILL.md +++ b/labs/zava-aks-postgres/.github/skills/managing-sre-agent/SKILL.md @@ -10,17 +10,23 @@ For **this demo**, agent configuration is declared in Bicep `Microsoft.App/agents/*` ARM resources: - **Agent settings** — autonomous mode, High access level, Azure Monitor incident binding -- **Connectors** — `app-insights`, `log-analytics`, `azure-monitor` (MonitorClient), `microsoft-learn` (MCP) -- **Custom skills** — `database-incidents`, `performance-incidents`, `application-incidents`, `general-triage` (the unknown bucket), `proactive-health-check` (auto-selected by description; max 5 concurrent) -- **Response plans / incident filters** — `zava-database`, `zava-performance`, `zava-application` (known-good, autonomous) + `zava-unknown` (catch-all, Review mode) - (routed by `titleContains` / `titleNotContains`) +- **Connectors** — `app-insights`, `log-analytics`, `azure-monitor` (MonitorClient), `learn-docs` (Microsoft Learn no-auth MCP) +- **Custom skills** — `database-incidents`, `performance-incidents`, `application-incidents`, `general-triage` (the unknown bucket), `proactive-health-check`, `incident-correlation` (auto-selected by description; max 5 concurrent) +- **Response plans / incident filters** — `zava-database`, `zava-performance`, `zava-application` (purpose-built, autonomous) + `zava-unknown` (bounded fallback, Review mode), routed by `titleContains` / `titleNotContains` - **RBAC** — system-assigned managed identity granted Reader, Monitoring Reader, - Contributor, and AKS RBAC Cluster Admin on the resource group + Contributor, and AKS RBAC Cluster Admin on the resource group; the runtime + user-assigned identity also has subscription-level Reader so the + correlation skill can read Alerts Management and Resource Health event feeds To change any of these, **edit the Bicep and run `azd provision`**. There is no data-plane CLI tool for them in this repo. -## Knowledge base (the one data-plane piece) +Do not design overlapping response plans around an assumed priority or +specificity rule. Treat multiple matches as undefined, keep purpose-built +filters mutually exclusive where routing matters, and make any fallback both +positively scoped and explicitly exclude every known route. + +## Data-plane configuration ARM does not yet surface SRE Agent knowledge files, so they're uploaded by `scripts/setup-sre-agent.ps1`: @@ -42,6 +48,11 @@ To remove a knowledge file: delete the local `.md`, then delete the correspondin `.md` from the agent's Builder UI > Knowledge sources view (the script does not delete remote files that are no longer present locally). +The same script also syncs the singleton agent-global custom instructions from +`sre-config/custom-instructions.md` and enables the Microsoft Learn MCP tools. +Keep global instructions short; detailed procedures belong in a skill so they +load only when relevant. + ## When helping users 1. **"Add a skill / response plan / connector"** — edit `infra/modules/sre-agent.bicep` @@ -52,10 +63,15 @@ script does not delete remote files that are no longer present locally). output reports `[OK]` or `[MISSING]` for every Bicep-deployed asset. 4. **Activity-log alerts gotcha** — they fire as Sev4 regardless of the configured severity, so response plan filters must match all severities (Bicep already does). -5. **Runbook philosophy** — the five skills (`database-incidents`, `performance-incidents`, - `application-incidents`, `general-triage`, `proactive-health-check`) in `sre-agent.bicep` +5. **Runbook philosophy** — the six skills (`database-incidents`, `performance-incidents`, + `application-incidents`, `general-triage`, `proactive-health-check`, `incident-correlation`) in `sre-agent.bicep` state the facts the agent can't infer (the RBAC it holds, what each alert means, which table to look at) — e.g. the `database-incidents` runbook's `postgres-unreachable` triage table maps alert → ARM-state check → action TYPE — while keeping the actual remediation at the action-type level, NOT copy-paste SQL/kubectl recipes. Preserve both halves when adding/modifying skills. See AGENTS.md "Non-Obvious Things" for the full rationale. +6. **Kubernetes tool guidance** — wire `RunKubectlReadCommand` / `RunKubectlWriteCommand` + into runtime skills and use them directly. If an ad-hoc chat must use terminal-native + kubectl, first issue a built-in read against the same cluster to warm the process-local + AKS CA path; the terminal command still needs an already-valid kubeconfig, and the warm-up + is lost on runtime restart. This is a side note, not a reason to add `RunInTerminal` to skills. diff --git a/labs/zava-aks-postgres/.github/skills/running-demo/SKILL.md b/labs/zava-aks-postgres/.github/skills/running-demo/SKILL.md index b468e1fa4..54c9c86b9 100644 --- a/labs/zava-aks-postgres/.github/skills/running-demo/SKILL.md +++ b/labs/zava-aks-postgres/.github/skills/running-demo/SKILL.md @@ -12,7 +12,7 @@ This skill drives the full demo using Playwright MCP for browser control. Execut ```powershell # AKS is a private cluster — kubectl from your local workstation won't work without VPN/jumpbox. # Use `Invoke-AksCommand` (wraps `az aks command invoke` for human-operator polling/diagnostics). -# The SRE Agent uses native kubectl; this helper is for human operators without the agent's VNet/DNS/proxy setup. +# The SRE Agent uses the built-in RunKubectl* system tools; this helper is for human operators. . .\scripts\_aks-helpers.ps1 $rg = (azd env get-value RESOURCE_GROUP) $aks = (azd env get-value AKS_CLUSTER_NAME) @@ -24,6 +24,12 @@ $storeUrl = "http://$ip" $agentUrl = (azd env get-value AGENT_PORTAL_URL) # deep-links to this agent's blade — sign in if prompted ``` +When observing or prompting the SRE Agent, prefer its built-in `RunKubectlReadCommand` and +`RunKubectlWriteCommand`; they accept the same kubectl commands used in a terminal. If a manual +test absolutely requires terminal-native kubectl, run a built-in read against the cluster first +to warm the process-local AKS CA path, then use the existing valid terminal kubeconfig. The warm-up +does not survive a runtime restart. + ## Scenario 1: Database Outage ### Step 1: Show healthy state @@ -81,7 +87,7 @@ Wait 30 seconds. ### Step 4: Watch the agent 1. Check SRE Agent portal for investigation -2. Agent needs to find the K8s NetworkPolicy via native `kubectl get networkpolicy -n zava-demo -o yaml` and remove it via `kubectl delete networkpolicy database-tier-isolation -n zava-demo` (run in its sandbox terminal) — this is harder than Scenario 1 and may take longer +2. Agent needs to find the K8s NetworkPolicy with `RunKubectlReadCommand` using `kubectl get networkpolicy -n zava-demo -o yaml`, then remove it with `RunKubectlWriteCommand` using `kubectl delete networkpolicy database-tier-isolation -n zava-demo` - this is harder than Scenario 1 and may take longer 3. Poll for NetworkPolicy removal (the AKS API server is private — go through ARM): ```powershell Invoke-AksCommand -ResourceGroup $rg -ClusterName $aks -Command "kubectl get networkpolicy -n zava-demo" @@ -117,7 +123,7 @@ If the script aborts with "Telemetry pipeline is dead", the api pods stopped sen 4. (`break-db-perf.ps1` already launched a 15-min in-cluster Kubernetes Job (`zava-cat-load` in the `zava-demo` namespace) that hammers `/api/products/category/` over the cluster-internal Service DNS. This pushes real traffic past the alert's 30ms threshold — the 1Hz `__probe` is excluded by the alert KQL. The Job auto-cleans 60s after completion via `ttlSecondsAfterFinished`; `fix-db-perf.ps1` also deletes it explicitly. Run with `-NoLoad` to skip.) ### Step 4: Watch agent -1. Monitor SRE Agent portal — it should detect slow response times via App Insights, identify the missing index, and run `CREATE INDEX CONCURRENTLY` in-cluster via `bin/run-sql.js` (the agent runs native `kubectl exec -n zava-demo deploy/zava-api -- node bin/run-sql.js ""` from its sandbox terminal — the helper reuses the pod's workload identity) +1. Monitor SRE Agent portal - it should detect slow response times via App Insights, identify the missing index, and run `CREATE INDEX CONCURRENTLY` in-cluster via `bin/run-sql.js` (`RunKubectlWriteCommand` executes `kubectl exec -n zava-demo deploy/zava-api -- node bin/run-sql.js ""`; the helper reuses the pod's workload identity) 2. Do not run `fix-db-perf.ps1` as part of the demo — same rule as the other scenarios: the script is post-demo cleanup, not an agent-failure fallback. ### Step 5: Show recovery @@ -170,6 +176,45 @@ If the script aborts with "Telemetry pipeline is dead", the api pods stopped sen 1. Navigate to `$storeUrl/api/products` — returns 200 again 2. Navigate to `$storeUrl` — products load; take screenshot +## Scenario 5: Compound Independent Faults + +This scenario overlaps Scenario 3 and Scenario 4 by 90 seconds. It should +produce two separate alerts and two independent causes, not one causal story. + +### Step 1: Confirm healthy state +1. Navigate to `$storeUrl` and `$storeUrl/api/health` +2. Confirm products load and the database is connected + +### Step 2: Break both paths +```powershell +.\.github\skills\running-demo\scripts\break-compound.ps1 +``` +The script drops both category indexes, starts the sustained category load, +waits 90 seconds, then deploys `FAULT_INJECT=500`. + +### Step 3: Verify the overlap +1. `$storeUrl/api/products` returns HTTP 500 while `/api/health` remains healthy. +2. Query `/api/diagnostics`; both category indexes are absent and product scans are sequential. +3. Confirm the `zava-cat-load` Job is active through `Invoke-AksCommand`. +4. Expect both `Zava-products-query-slow` and `Zava-http-5xx-errors` within 5-10 minutes. + +### Step 4: Grade the investigation +A correct investigation enumerates the nearby alerts and disabled rule inventory, +then proves the mechanisms are independent: +- 5xx failures are app-local (`localhost:3001`) and correlate with the rollout. +- PostgreSQL CPU/latency rises, but its slow queries succeed and create no failed PG dependencies. +- `Zava-db-cpu-saturation` is present but disabled. + +Do not accept alert timestamps alone as causality; every dispatching rule uses +PT5M evaluation and the 90-second injection order can be reversed at alert time. + +### Step 5: Cleanup +Let the SRE Agent remediate during a demo. For post-demo cleanup or test teardown: +```powershell +.\.github\skills\running-demo\scripts\fix-compound.ps1 +``` +Verify `/api/products` returns 200, both indexes exist, and the load Job is gone. + ## Chat demo: interrogate the hub firewall (network device) No break needed — this shows the agent treating the **hub Azure Firewall** as a queryable "network device" in the hub-and-spoke topology. diff --git a/labs/zava-aks-postgres/.github/skills/running-demo/scripts/break-bad-deploy.ps1 b/labs/zava-aks-postgres/.github/skills/running-demo/scripts/break-bad-deploy.ps1 index 8d6da0c76..22e032ee8 100644 --- a/labs/zava-aks-postgres/.github/skills/running-demo/scripts/break-bad-deploy.ps1 +++ b/labs/zava-aks-postgres/.github/skills/running-demo/scripts/break-bad-deploy.ps1 @@ -35,6 +35,11 @@ $ErrorActionPreference = "Stop" . "$PSScriptRoot\..\..\..\..\scripts\_aks-helpers.ps1" $ctx = Resolve-AksContext -ResourceGroup $ResourceGroup -ClusterName $ClusterName +# Azure Monitor's stateful per-rule instance is separate from agent-side merge. +# Refuse to inject a new fault while the prior condition is still Fired, and +# close a resolved prior instance so this run dispatches as a fresh alert. +Reset-DemoAlertRule -ResourceGroup $ctx.ResourceGroup -AlertRuleName 'Zava-http-5xx-errors' + # Telemetry precheck. Zava-http-5xx-errors evaluates the requests/failed metric, # which is derived from AppRequests telemetry. If the api isn't currently sending # telemetry to the workspace, the alert can never fire no matter how many 500s the diff --git a/labs/zava-aks-postgres/.github/skills/running-demo/scripts/break-compound.ps1 b/labs/zava-aks-postgres/.github/skills/running-demo/scripts/break-compound.ps1 new file mode 100644 index 000000000..b535ce5fa --- /dev/null +++ b/labs/zava-aks-postgres/.github/skills/running-demo/scripts/break-compound.ps1 @@ -0,0 +1,124 @@ +#Requires -Version 7.4 +# Break COMPOUND (Scenario 5): two INDEPENDENT faults, overlapping in time. +# +# Why this scenario exists +# ------------------------ +# Scenarios 1-4 each inject exactly one fault, so every alert the agent sees has +# exactly one cause and "diagnose the alert you were handed" always works. Real +# incidents are not that tidy, and the lab was teaching a habit that breaks in +# production: treating the dispatched alert as the whole story. +# +# This scenario runs Scenario 3 (drop the category indexes + load generator) and +# Scenario 4 (FAULT_INJECT=500 bad deploy) so their impact windows overlap. Two +# alerts then fire within seconds of each other: +# +# Zava-products-query-slow <- category endpoints breach the 30 ms threshold +# Zava-http-5xx-errors <- GET /api/products returns 500 +# +# The tempting-but-WRONG read is "the database got slow, so the API started +# failing" — a clean causal story that fits the timestamps perfectly and is +# false. The mechanisms are disjoint and the telemetry says so: +# +# 5xx path : HTTP 500, failed dependencies ONLY on localhost:3001, +# ZERO failed dependencies against the PG target. +# slow path : PG cpu_percent pegs ~90%, queries are SLOW BUT SUCCEED, so +# they produce no dependency FAILURES at all. +# +# If DB saturation were causing the 5xx you would see PG dependency failures or +# 503 timeouts. Neither appears. Two faults, one window, no causal link. +# +# What to watch for +# ----------------- +# A good investigation notices there are two alerts, checks whether one explains +# the other, finds it does not, and reports two independent incidents. A weak one +# merges them into a single tidy narrative and "fixes" the DB while the bad deploy +# keeps serving 500s. Note also that `Zava-db-cpu-saturation` ships DISABLED +# (infra/modules/monitoring.bicep), so the causal DB signal never alerts at all — +# the agent has to enumerate the alert RULE inventory to discover it was muted. +# +# Ordering note: the two breaks are deliberately started close together but NOT +# atomically. Alert fire order will NOT match fault-injection order — every +# dispatching rule is PT5M/PT5M, so detection latency swamps the 90-second offset. +# That is the point: alert timestamps cannot establish causality here. +param( + [string]$ResourceGroup = "", + [string]$ClusterName = "", + [string]$Namespace = "zava-demo", + # Must outlast the 5-min alert window plus agent dispatch and investigation. + # Default matches break-db-perf.ps1 so the perf fault is still live while the + # agent works the 5xx thread (and vice versa) — if the load run ends early the + # overlap disappears and the scenario degrades to two sequential incidents. + [int]$LoadMinutes = 20, + [switch]$SkipTelemetryCheck +) + +$ErrorActionPreference = "Stop" +. "$PSScriptRoot\..\..\..\..\scripts\_aks-helpers.ps1" +$ctx = Resolve-AksContext -ResourceGroup $ResourceGroup -ClusterName $ClusterName + +Write-Host "=== Scenario 5: COMPOUND break (two independent faults) ===" -ForegroundColor Magenta +Write-Host "This injects a DB-performance fault and an APP fault so their alerts co-fire." -ForegroundColor Magenta +Write-Host "They are NOT causally related. See the header of this script for why." -ForegroundColor Magenta +Write-Host "" + +# Delegate to the existing single-fault scripts rather than duplicating their +# logic. They already carry the load-bearing details (telemetry precheck, drop +# BOTH indexes, roll the deployment to clear PG plan cache and rewake the OTel +# exporter, Job-based load generator) and those must not drift out of sync here. +$dbPerf = Join-Path $PSScriptRoot 'break-db-perf.ps1' +$badDeploy = Join-Path $PSScriptRoot 'break-bad-deploy.ps1' + +# --- Fault A: DB performance (index drop + sustained category load) --------- +# Run this FIRST: it rolls the api deployment as part of its normal flow. Doing +# it after the FAULT_INJECT change would create another rollout revision and +# muddy the deployment-correlation signal the agent uses for the 5xx fault. +Write-Host "[1/2] Injecting DB-performance fault (drop category indexes + load)..." -ForegroundColor Yellow +$dbArgs = @{ + ResourceGroup = $ctx.ResourceGroup + ClusterName = $ctx.ClusterName + Namespace = $Namespace + LoadMinutes = $LoadMinutes +} +if ($SkipTelemetryCheck) { $dbArgs['SkipTelemetryCheck'] = $true } +$dbCliArgs = @( + '-ResourceGroup', $dbArgs.ResourceGroup, + '-ClusterName', $dbArgs.ClusterName, + '-Namespace', $dbArgs.Namespace, + '-LoadMinutes', $dbArgs.LoadMinutes +) +if ($SkipTelemetryCheck) { $dbCliArgs += '-SkipTelemetryCheck' } +& pwsh -NoProfile -File $dbPerf @dbCliArgs +if ($LASTEXITCODE -ne 0) { + Write-Error "break-db-perf.ps1 failed (exit $LASTEXITCODE). Aborting before injecting the second fault so you don't end up with a half-broken environment that's hard to reason about. Fix the first fault (fix-db-perf.ps1) and retry." + exit 1 +} + +# Let the perf fault establish a measurable signal before layering the app fault +# on top. Without this the two onsets land in the same telemetry bucket and even +# a careful investigation cannot separate them — which would make the scenario +# unfair rather than instructive. +Write-Host "`nWaiting 90s for the DB-perf signal to establish before the second fault..." -ForegroundColor DarkGray +Start-Sleep -Seconds 90 + +# --- Fault B: bad deploy (FAULT_INJECT=500 on GET /api/products) ------------ +Write-Host "`n[2/2] Injecting APP fault (FAULT_INJECT=500 bad deploy)..." -ForegroundColor Yellow +& pwsh -NoProfile -File $badDeploy -ResourceGroup $ctx.ResourceGroup -ClusterName $ctx.ClusterName -Namespace $Namespace +if ($LASTEXITCODE -ne 0) { + Write-Error "break-bad-deploy.ps1 failed (exit $LASTEXITCODE). The DB-performance fault IS still active — run fix-db-perf.ps1 to clean up, or re-run just break-bad-deploy.ps1 to complete the compound scenario." + exit 1 +} + +Write-Host "" +Write-Host "=== Compound break complete ===" -ForegroundColor Magenta +Write-Host "Expect TWO alerts within ~5-10 min: Zava-products-query-slow and Zava-http-5xx-errors." -ForegroundColor Cyan +Write-Host "They open SEPARATE investigation threads (merge is disabled on every response plan)." -ForegroundColor Cyan +Write-Host "" +Write-Host "Grading the agent:" -ForegroundColor Cyan +Write-Host " GOOD - notices both alerts, tests whether one explains the other, finds disjoint" -ForegroundColor Green +Write-Host " mechanisms (500 + localhost:3001 only vs PG CPU with no dep failures)," -ForegroundColor Green +Write-Host " reports TWO independent incidents, and fixes both." -ForegroundColor Green +Write-Host " WEAK - merges them into one story ('slow DB caused the 5xx'), fixes only the DB," -ForegroundColor Yellow +Write-Host " and leaves FAULT_INJECT serving 500s." -ForegroundColor Yellow +Write-Host " Also worth watching: does it discover that Zava-db-cpu-saturation is DISABLED?" -ForegroundColor Yellow +Write-Host "" +Write-Host "Fix both with: .\.github\skills\running-demo\scripts\fix-compound.ps1" -ForegroundColor Cyan diff --git a/labs/zava-aks-postgres/.github/skills/running-demo/scripts/break-db-perf.ps1 b/labs/zava-aks-postgres/.github/skills/running-demo/scripts/break-db-perf.ps1 index 9fe15be9e..934ca9fbf 100644 --- a/labs/zava-aks-postgres/.github/skills/running-demo/scripts/break-db-perf.ps1 +++ b/labs/zava-aks-postgres/.github/skills/running-demo/scripts/break-db-perf.ps1 @@ -28,6 +28,11 @@ $ErrorActionPreference = "Stop" . "$PSScriptRoot\..\..\..\..\scripts\_aks-helpers.ps1" $ctx = Resolve-AksContext -ResourceGroup $ResourceGroup -ClusterName $ClusterName +# Azure Monitor's stateful per-rule instance is separate from agent-side merge. +# Refuse to inject a new fault while the prior condition is still Fired, and +# close a resolved prior instance so this run dispatches as a fresh alert. +Reset-DemoAlertRule -ResourceGroup $ctx.ResourceGroup -AlertRuleName 'Zava-products-query-slow' + # Telemetry precheck. The Zava-products-query-slow alert is a scheduled KQL # query against AppRequests. If the api isn't currently sending telemetry to # the workspace, the alert can never fire no matter how slow the queries are diff --git a/labs/zava-aks-postgres/.github/skills/running-demo/scripts/fix-compound.ps1 b/labs/zava-aks-postgres/.github/skills/running-demo/scripts/fix-compound.ps1 new file mode 100644 index 000000000..c1bd96b30 --- /dev/null +++ b/labs/zava-aks-postgres/.github/skills/running-demo/scripts/fix-compound.ps1 @@ -0,0 +1,56 @@ +#Requires -Version 7.4 +# Fix COMPOUND (Scenario 5): undo both independent faults. +# +# Post-demo cleanup / fallback, exactly like the other fix scripts — during a +# live demo you want the SRE Agent to remediate, not this script. +# +# Order matters for a clean readout: undo the APP fault first so GET /api/products +# stops returning 500, THEN recreate the indexes and stop the load generator. Doing +# it the other way round leaves the app serving 500s while latency has already +# recovered, which looks like the DB fix caused an app regression. +# +# Both halves run even if the first reports a failure — a partial cleanup that +# leaves one fault live is worse than a noisy one, and each underlying fix script +# is independently idempotent. +param( + [string]$ResourceGroup = "", + [string]$ClusterName = "", + [string]$Namespace = "zava-demo" +) + +$ErrorActionPreference = "Stop" +. "$PSScriptRoot\..\..\..\..\scripts\_aks-helpers.ps1" +$ctx = Resolve-AksContext -ResourceGroup $ResourceGroup -ClusterName $ClusterName + +Write-Host "=== Scenario 5: COMPOUND fix (undo both faults) ===" -ForegroundColor Magenta + +$failures = @() + +# --- Undo Fault B: bad deploy ---------------------------------------------- +Write-Host "`n[1/2] Reverting APP fault (rollout undo + strip FAULT_INJECT)..." -ForegroundColor Yellow +try { + & pwsh -NoProfile -File (Join-Path $PSScriptRoot 'fix-bad-deploy.ps1') -ResourceGroup $ctx.ResourceGroup -ClusterName $ctx.ClusterName -Namespace $Namespace + if ($LASTEXITCODE -ne 0) { $failures += "fix-bad-deploy.ps1 (exit $LASTEXITCODE)" } +} catch { + $failures += "fix-bad-deploy.ps1 ($($_.Exception.Message))" +} + +# --- Undo Fault A: DB performance ------------------------------------------ +# Continue regardless of the result above so we never strand the index drop. +Write-Host "`n[2/2] Reverting DB-performance fault (recreate indexes + stop load Job)..." -ForegroundColor Yellow +try { + & pwsh -NoProfile -File (Join-Path $PSScriptRoot 'fix-db-perf.ps1') -ResourceGroup $ctx.ResourceGroup -ClusterName $ctx.ClusterName -Namespace $Namespace + if ($LASTEXITCODE -ne 0) { $failures += "fix-db-perf.ps1 (exit $LASTEXITCODE)" } +} catch { + $failures += "fix-db-perf.ps1 ($($_.Exception.Message))" +} + +Write-Host "" +if ($failures.Count -gt 0) { + Write-Error "Compound fix completed with failures: $($failures -join '; '). Re-run the individual fix script(s) above — both are idempotent. Verify manually: GET /api/products returns 200, and both idx_products_category and idx_products_category_name exist." + exit 1 +} + +Write-Host "=== Compound fix complete ===" -ForegroundColor Magenta +Write-Host "Verify: GET /api/products returns 200, category endpoints back to ~3ms, PG cpu_percent back to baseline." -ForegroundColor Cyan +Write-Host "Both alerts should auto-mitigate; the agent may also have closed them itself." -ForegroundColor Cyan diff --git a/labs/zava-aks-postgres/AGENTS.md b/labs/zava-aks-postgres/AGENTS.md index b904520b7..9fe54723e 100644 --- a/labs/zava-aks-postgres/AGENTS.md +++ b/labs/zava-aks-postgres/AGENTS.md @@ -7,7 +7,7 @@ Azure SRE Agent demo — AKS + PostgreSQL e-commerce app with break/fix scenario ## Getting Started When a user clones this repo, guide them through setup: -1. Check prerequisites: `az`, `azd`, `pwsh` — install any missing. (`kubectl` is **not** required on the operator workstation: the AKS cluster is private, and operator in-cluster operations run through `az aks command invoke`; the deployed SRE Agent uses native `kubectl` — authenticated by its managed identity via `kubelogin` — for in-cluster work.) +1. Check prerequisites: `az`, `azd`, `pwsh`, plus Owner/User Access Administrator (or equivalent role-assignment write permission) at subscription scope — install anything missing. (`kubectl` is **not** required on the operator workstation: the AKS cluster is private, operator operations use `az aks command invoke`, and the SRE Agent uses its built-in Kubernetes tools.) 2. Run `azd up` — pick their default subscription, use `swedencentral` region 3. After deploy completes, run `scripts/setup-sre-agent.ps1` to upload the knowledge file and verify the Bicep-deployed agent (the agent itself, connectors, skills, response plans, mode, incident binding are all already provisioned by Bicep — this script only handles the data-plane KB upload and a verification readout) 4. Open the storefront in browser to verify it works @@ -22,7 +22,7 @@ These are gotchas for someone editing this repo's IaC or Bicep — they're *not* - **K8s manifests** use `${VAR}` placeholders — substituted by `post-provision.ps1`, not Helm/Kustomize. - **No `azd deploy api`** — there's no `services:` block in `azure.yaml`. Container images are built by `post-provision.ps1` via `az acr build`. To iterate: `az acr build --registry $acr --image zava-api:latest ./src/api` then `az aks command invoke … kubectl rollout restart deployment …`. - **Activity log alerts default to Sev4** because `Microsoft.Insights/activityLogAlerts` rules don't expose a severity field for the categories we use (Administrative). Response plans must include `Sev4` in `priorities[]` (this repo includes all severities) or activity-log-driven incidents won't match. -- **Response plans are granular: 3 known-good filters + 1 unknown bucket.** `infra/modules/sre-agent.bicep` defines four `incidentFilters`, routed by `titleContains` (handlingAgent `default`, so the agent picks a skill by description): `zava-database` (`postgres`), `zava-performance` (`query-slow`), `zava-application` (`http-5xx`) — all autonomous — plus `zava-unknown`, a catch-all (`titleContainsAny ['Zava','postgres']` + `titleNotContains ['postgres','query-slow','http-5xx']`) that runs in **Review** mode with deep investigation for novel incidents. There is NO documented precedence when multiple filters match, so the buckets are kept non-overlapping via `titleNotContains`; the unknown bucket is bounded to demo-named alerts so it can't sweep in subscription noise. Exercise it with the disabled `Zava-unknown-test` alert. +- **Response plans are granular: 3 purpose-built filters + 1 unknown bucket.** `infra/modules/sre-agent.bicep` defines four `incidentFilters` (handlingAgent `meta_agent`, the current built-in dispatcher that picks skills by description): `zava-database` (`postgres`), `zava-performance` (`query-slow`), and `zava-application` (`http-5xx`) are autonomous; `zava-unknown` matches other `Zava` alerts in **Review** mode. Treat overlapping response-plan matches as nondeterministic/undefined: there is no customer-defined priority or "most specific wins" rule, and the runtime uses the first matching plan in mutable filter order. Build purpose-specific filters, make them mutually exclusive where routing matters, and give the fallback explicit `titleNotContains` exclusions for every known route. This sample's fallback is also positively bounded by `titleContains: 'Zava'`, so it cannot sweep in unrelated subscription alerts. Exercise it with the disabled `Zava-unknown-test` alert. - **One primary dispatching alert per failure domain (de-noised).** A single root cause used to fire up to 5 alerts → 5 threads. Now there are **three enabled** dispatching alerts: `postgres-unreachable` (DB stop OR network partition — see the next bullet), `Zava-products-query-slow` (perf), `Zava-http-5xx-errors` (app). The 5xx alert fires purely on the 5xx count — **no DB self-suppression**, and agent **merge is off on all plans**, so we don't dedupe: a DB outage that also returns 5xx opens its own app thread alongside the `postgres-unreachable` thread (each real symptom surfaces its own investigation). The `performance-incidents` skill corroborates from the underlying metrics, which flow independently of any alert — `cpu_percent` (PG `AllMetrics` diagnostic → `AzureMetrics`) and the `zava.products.category.query.duration_ms` custom metric (OTel → `AppMetrics`); we deploy only the dispatching alerts the demo actually fires, not disabled metric-alert examples. The NSG-change activity alerts (noise from platform-managed NSG changes) and the redundant `postgres-server-stopped` activity alert were removed. - **Dispatching scheduled-query alerts use `evaluationFrequency: PT5M`, not `PT1M`.** Verified live across all three scenarios: at 1-minute evaluation the alert fires and is *acknowledged* but the SRE agent does **not** open an autonomous investigation thread; at 5-minute evaluation it dispatches and remediates end-to-end. `postgres-unreachable`, `Zava-products-query-slow`, and `Zava-http-5xx-errors` are therefore all `PT5M`/`PT5M` (eval/window). The DB/perf signals are unambiguous, so firing ~3-5 min after the break (vs ~2 min) is well within demo tolerance. Keep any new dispatching alert at `PT5M`. - **One `postgres-unreachable` alert for BOTH DB scenarios — diagnose cause from ARM state, not error text.** A stopped PG Flexible Server and a NetworkPolicy/NSG block both present at the app as connection **timeouts** (measured ~1650 timeout traces vs ~42 ECONNREFUSED on a stop), so the old `postgres-server-down`(ECONNREFUSED)/`postgres-network-blocked`(ETIMEDOUT) split could not actually tell them apart — a stop fired the *network-blocked* alert. The fix: one symptom alert (`postgres-unreachable`), and the `database-incidents` skill reads PG ARM `state` — `Stopped` → start it; `Ready` but unreachable → find the NetworkPolicy/NSG. Agent **merge is off on every plan** (no agent-side dedup). Because both DB scenarios share the one `postgres-unreachable` rule, Azure Monitor won't emit a fresh alert instance while the prior one is still `Fired`, so the `database-incidents` runbook has the agent **resolve (close) the alert as its final step** once recovery is verified (`az rest … /changestate?…&newState=Closed`, using its existing Contributor `Microsoft.AlertsManagement/alerts/changestate/action` right) — that clears it immediately so a back-to-back stop→partition run dispatches fresh, with `autoMitigate` (~15-30 min) as the fallback. Don't rely on splitting the rule — the two signals are indistinguishable (above). @@ -33,28 +33,69 @@ These are gotchas for someone editing this repo's IaC or Bicep — they're *not* - **Scenario 3 break rolls the api deployment after dropping the indexes.** Two reasons: (1) the api's `pg` pool retains cached query plans on each connection that referenced the now-missing indexes; rolling forces fresh plans + a clean seq_scan signal. (2) The OTel/AppInsights exporter has been observed to silently wedge on long-uptime pods (4h+ in our runs), causing telemetry to stop without a pod restart. The roll guarantees a fresh exporter for the load run that follows. - **Scenario 3 break has a telemetry precheck.** `break-db-perf.ps1` queries `AppRequests | where AppRoleName == 'zava-api' | last 10m` against the Log Analytics workspace and aborts loudly with `exit 1` if zero rows. The slow-query alert is a scheduled KQL against `AppRequests` — without telemetry it can never fire and the SRE Agent never gets dispatched, but the failure mode looks like "agent ignored the alert" 30 minutes later. Pass `-SkipTelemetryCheck` to bypass (e.g. fresh deploy where the api hasn't had time to send any telemetry yet). - **Scenario 3 load generator runs as a Kubernetes Job** (`k8s/jobs/load-categories.yaml`), not a local `Start-ThreadJob`. The Copilot CLI / azd hook environment recycles the PowerShell process between calls, which killed the previous in-process ThreadJob immediately and produced ~zero alert-eligible traffic. The Job survives script exit, hits cluster-internal Service DNS (the operator's workstation is not in the network path), and auto-cleans via `ttlSecondsAfterFinished: 60`. `break-db-perf.ps1` substitutes `${ACR_NAME}`/`${DURATION_MIN}` and applies via `Invoke-AksCommand -Files`; `fix-db-perf.ps1` deletes the Job after recreating the indexes. -- **Skills are split by incident domain.** The five skills are `database-incidents`, `performance-incidents`, `application-incidents`, `general-triage`, and `proactive-health-check`; there is no unified `dbIncidentSkill`. The known-facts triage table for `postgres-unreachable` lives in the `database-incidents` runbook and maps `alert → ARM-state check → action TYPE` (Stopped → start it; Ready but unreachable → investigate NetworkPolicy/NSG). Preserve that definitive triage guidance while keeping remediation at the action-type level, not copy-paste SQL/kubectl recipes. +- **Scenario 5 (`break-compound.ps1`) is the only multi-fault scenario — and the lab needs one.** Scenarios 1-4 inject exactly one fault each, so "diagnose the alert you were handed" always works and the lab quietly teaches a habit that breaks in production. Scenario 5 runs Scenario 3 and Scenario 4 with a 90-second offset so `Zava-products-query-slow` and `Zava-http-5xx-errors` co-fire from **genuinely independent** causes. The tempting-and-wrong read is "the DB got slow, so the API started failing" — it fits the timestamps perfectly and is false. The 90s offset is load-bearing: without it both onsets land in the same telemetry bucket and even a careful investigation can't separate them, which makes the scenario unfair rather than instructive. Fault A runs first because `break-db-perf.ps1` rolls the api deployment; injecting `FAULT_INJECT` first would put another rollout revision in the way of the deployment-correlation signal the 5xx path depends on. +- **Skills are split by incident domain.** The six skills are `database-incidents`, `performance-incidents`, `application-incidents`, `general-triage`, `proactive-health-check`, and `incident-correlation`; there is no unified `dbIncidentSkill`. The "max 5 concurrent" cap is on skills ACTIVE in a thread, not skills defined — a domain skill plus `incident-correlation` is 2 of 5, the intended pairing. The known-facts triage table for `postgres-unreachable` lives in the `database-incidents` runbook and maps `alert → ARM-state check → action TYPE` (Stopped → start it; Ready but unreachable → investigate NetworkPolicy/NSG). Preserve that definitive triage guidance while keeping remediation at the action-type level, not copy-paste SQL/kubectl recipes. - **Alert `description` strings in `monitoring.bicep` are agent-readable payload, not cosmetic Bicep strings.** Azure Monitor includes the alert description in the incident context the SRE Agent reads. They MUST stay symptom-only — never re-add "Likely cause: …", "Remediation: …", "(Scenario N)", or any specific resource name (table, index, NetworkPolicy) the agent could pattern-match instead of diagnosing. The descriptions describe what was observed; the runbook + KB explain how to investigate. +- **Cross-alert correlation is split across TWO surfaces on purpose — don't consolidate them.** Every response plan runs `mergeEnabled: false` and Azure Monitor merging is same-alert-rule-only, so each fired alert opens an ISOLATED thread with no visibility into what else fired. Structural isolation is the default; the only way an investigation sees the wider picture is if it pulls it. That guidance lives in two places: (1) `sre-config/custom-instructions.md` — the always-on, agent-scoped **custom instructions** blob (~125 words, appended to every thread) carrying only the *trigger* ("the alert is a signal, not the story"); (2) the `incident-correlation` **skill** in `sre-agent.bicep` carrying the *method* (exact REST calls, the alert-rule inventory sweep, Service Health, and the two correctness rules), loaded on demand. The split is the token-cost design: always-on nudge is cheap, the method is only paid for when the nudge fires. **Do NOT** move this into alert `description`s (forbidden above, and a per-alert string can't express a cross-alert idea) or duplicate it into the four `incidentFilters` as per-plan instructions (four copies to drift, and each fires only *after* routing already narrowed the incident to one bucket — the exact tunnel vision it's meant to break). +- **The two correctness rules in the correlation skill are load-bearing — a correlation sweep without them makes the agent WORSE.** (1) *Alert fire order is not causal order*: every dispatching rule is `PT5M`/`PT5M`, so detection latency is up to 5 min + ingestion lag and a symptom alert routinely fires BEFORE its cause. Any gap under ~7 min proves nothing; establish onset from raw telemetry in 1-2 min buckets. (2) *Co-firing is not causation*: verified against real lab telemetry where two alerts fired 5 seconds apart from unrelated faults. The discriminator is one `dependencies` query split by `target` alongside `resultCode` — HTTP 500 + failures only on `localhost:3001` = app; 503 + failures on the PG target = DB unreachable; **no dependency failures but PG `cpu_percent` high = DB saturation (queries are slow but SUCCEED, so failure-based signals stay clean)**. Without these rules the agent confidently merges independent incidents into one tidy false narrative. +- **`Zava-db-cpu-saturation` is deployed DISABLED on purpose (`enableDbCpuSaturationAlert=false`) — it is not dead code.** It previously existed in live demo RGs but in no template, so `azd up` and reality disagreed; it is now declared in `monitoring.bicep` to fix that drift while preserving the teaching value. When Scenario 3 runs, PG `cpu_percent` pegs ~90% and the ONLY alerts that fire are downstream symptom alerts — the *causal* signal is silent, exactly like a real org that muted a noisy rule months ago. An agent reasoning only from its dispatched alert can't recover the cause; one that enumerates the alert **rule inventory** (not just fired alerts) finds a disabled rule on the very resource it's investigating. Set the param `true` to convert the scenario into a genuine causal+symptom co-firing case instead. Enumerating rules is `az monitor metrics alert list` **plus** a `scheduledQueryRules` ARM GET — metric and log alerts are different resource types and `az monitor metrics alert list` does not return log-search rules. +- **Azure Service Health has a working REST API and the agent should use it before ever concluding "platform health event".** `GET /subscriptions//providers/Microsoft.ResourceHealth/events?api-version=2022-10-01&queryStartTime=` returns `ServiceIssue` / `PlannedMaintenance` / `HealthAdvisory` records subscription-wide; per-resource state is `.../resourceGroups//providers/Microsoft.ResourceHealth/availabilityStatuses?api-version=2023-07-01-preview`. Historically this lab's PG-blip investigations concluded "platform health event" from *absence* of contrary evidence while live `PlannedMaintenance` notices for Azure Database for PostgreSQL were sitting unqueried in this API. Note `Microsoft.ResourceHealth/emergingIssues` returns 404 on this subscription and a single-event GET by tracking ID returns `TrackingIdNotFound` — use the **list** form. - **Knowledge base is intentionally minimal.** It documents only what the agent can't infer from world knowledge: environment-specific names (RG, namespace, `bin/run-sql.js` helper), the private-network reachability constraint, and counterintuitive Azure behaviors (e.g. delegated-subnet NSG semantics, shared App Insights workspace requiring `AppRoleName` filter). Don't re-add KQL recipes, per-failure-mode "where to look" lists, control-plane-vs-data-plane primers, or layered diagnostic surface tables — that's all SRE common sense or public Azure docs and the agent already has it. Spoon-feeding makes the agent worse, not better: it pattern-matches the recipe instead of reasoning from telemetry. - **The Log Analytics workspace has no `dailyQuotaGb` cap.** Demo alerts must always be able to fire, so workspace ingestion is uncapped. Volume is bounded at the SDK layer instead: `logger.js` only ships warn/error to OTel (HTTP auto-instrumentation already records access logs as `AppRequests`), and high-volume App Insights tables are pinned to the 4-day per-table minimum. - **The api workload identity needs `Monitoring Metrics Publisher` on the AI resource.** The `@azure/monitor-opentelemetry` SDK uses AAD auth (via the workload-identity-injected federated token) for the breeze ingestion endpoint when `AZURE_CLIENT_ID` is set in the pod env, even if a connection string is also provided. Without the role, exports are silently rejected and no telemetry flows. The role is granted in `identity.bicep`'s `appMetricsPublisher` resource (scoped to the AI component, not the RG) — don't drop it. - **Scenario 2 NetworkPolicy is named `database-tier-isolation`, not `block-postgres`.** This is deliberate de-spoon-feeding: the old name let the agent reach the diagnosis purely from string-matching. The new name reads like a security-architect zero-trust attempt that accidentally over-blocks; the agent has to read the egress rules and reason about the destination subnet to find it. - **NSG deny rule installed by `break-network.ps1` is a red herring, not a "breadcrumb".** PG Flexible Server private access uses a delegated subnet whose routing/policy is platform-managed (see [subnet-delegation-overview](https://learn.microsoft.com/azure/virtual-network/subnet-delegation-overview) and [PG private networking](https://learn.microsoft.com/azure/postgresql/network/concepts-networking-private)), so a user-added NSG rule on 5432 looks like the smoking gun in config but isn't the active enforcement point. The agent is expected to cross-reference the KB and discount it. Don't reintroduce framing that calls it a "cosmetic clue" — that telegraphs the answer. - **No `SendOutlookEmail` tool wired into the skills.** Email-out requires an OAuth consent flow that has to be completed by an interactive user in the agent's portal — there is no Bicep/ARM verb to provision it on the agent's behalf. Adding it to `skill.tools[]` without that consent makes the skill *fail to load*. If you want post-remediation email for your own deployment, follow the public docs ([Microsoft 365 connector for SRE Agent](https://learn.microsoft.com/azure/sre-agent/)) to grant consent in the portal, then re-add `'SendOutlookEmail'` to the relevant skill's `tools` array and a "send a summary email" line to the runbook. -- **The network is hub-and-spoke (three VNets), not one flat VNet.** `vnet.bicep` deploys a **hub** (`vnet-Zava-hub-*`, 10.10.0.0/22 — `AzureFirewallSubnet` + the Azure Firewall, a reserved `GatewaySubnet` for a future ExpressRoute/VPN gateway, and `pe-subnet` for the AMPLS private endpoint), a **platform spoke** (`vnet-Zava-platform-*`, 10.20.0.0/16 — `aks-subnet` + delegated `db-subnet`), and an **agent spoke** (`vnet-Zava-agent-*`, 10.30.0.0/24 — the delegated `agent-subnet`). The firewall lives in the **hub**; the agent subnet force-tunnels to it via a UDR (`0.0.0.0/0` → firewall private IP `10.10.0.4`) over VNet peering, and the firewall policy's `sourceAddresses` is the agent subnet (`10.30.0.0/28`) — update both if you renumber. This is safe because the agent reaches AKS via native `kubectl` over the private API-server path and PG through the in-cluster helper, never raw DB sockets, so it never needed to share a VNet with them. Consequence for scripts: anything that picks "the VNet" must select the one containing its target subnet — `break-network.ps1` now queries `[?subnets[?name=='aks-subnet']]`, not `[0]`. +- **The network is hub-and-spoke (three VNets), not one flat VNet.** `vnet.bicep` deploys a **hub** (`vnet-Zava-hub-*`, 10.10.0.0/22 — `AzureFirewallSubnet` + the Azure Firewall, a reserved `GatewaySubnet` for a future ExpressRoute/VPN gateway, and `pe-subnet` for the AMPLS private endpoint), a **platform spoke** (`vnet-Zava-platform-*`, 10.20.0.0/16 — `aks-subnet` + delegated `db-subnet`), and an **agent spoke** (`vnet-Zava-agent-*`, 10.30.0.0/24 — the delegated `agent-subnet` at 10.30.0.0/27). `/27` is the minimum accepted size: 32 total addresses minus Azure's 5 reserved addresses leaves the required 27 usable addresses. The firewall lives in the **hub**; the agent subnet force-tunnels to it via a UDR (`0.0.0.0/0` → firewall private IP `10.10.0.4`) over VNet peering, and every firewall rule takes its source from the single `agentSubnetPrefix` variable — keep that variable at `/27` unless the product requirement changes. Consequence for scripts: anything that picks "the VNet" must select the one containing its target subnet — `break-network.ps1` now queries `[?subnets[?name=='aks-subnet']]`, not `[0]`. - **Agent VNet injection is REGIONAL; cross-region reach is via PEERING.** The `agent-subnet` (delegated to `Microsoft.App/environments`) **must be in the same region as the `Microsoft.App/agents` resource** — VNet injection is regional, not a tuning knob ([SRE Agent subnet requirements](https://learn.microsoft.com/azure/sre-agent/network-integration#configure-azure-vnet-mode): *"The subnet must be in the same region as your SRE Agent resource"*). A single-region `azd up` satisfies this automatically — `vnet.bicep` deploys all three VNets and `sre-agent.bicep` deploys the agent with the same `location: location`; don't move the agent subnet to another region expecting injection to work. The agent's **reach is NOT regional**, though: peered to the hub, it routes to anything the hub peers to — **other Azure regions over global VNet peering**, **on-prem over the reserved `GatewaySubnet` gateway** (*"as long as your network routes and rules allow it"*). The on-prem example is just one instance of this. `vnet.bicep` ships a **commented `remote-region` global-peering example** (after the local peerings) and the README's *"Reaching other regions and on-premises"* section is the narrative. Because the agent is force-tunneled to the hub firewall, reaching a new peered range also needs a firewall network rule (`agent-subnet → that range`), not just the peering. -- **Native kubectl requires the AKS private-DNS link, and that link is now codified.** `vnet.bicep` already adds the agent-subnet → AKS API :443 firewall rule and SNAT. The remaining requirement is linking the AKS-managed private-DNS zone (`.privatelink..azmk8s.io`, created in the node resource group after cluster creation) to the agent VNet. That zone name is dynamic, so it cannot be a static Bicep resource; `scripts/post-provision.ps1` **Step 4b** now discovers the node RG, azmk8s.io zone, and agent VNet, then idempotently creates the `agent-link` virtual-network link on every deploy. Before this was codified, fresh deploys could spend minutes floundering on private AKS DNS before falling back/proceeding. -- **The Microsoft Learn MCP connector routes entirely through the hub Azure Firewall (no platform bypass) — `raw.githubusercontent.com` must be allow-listed or it silently surfaces zero tools.** The `microsoft-learn` connector is a *Streamable-HTTP* MCP server (`endpoint: https://learn.microsoft.com/api/mcp`). We keep **`allowHttpMcpServerNetworkAccess: false`** (the default) on purpose: when `true`, the platform routes the MCP runtime endpoint as `Rewrite{RoutingMode=Platform}` — a broker that egresses *outside* the customer VNet, bypassing our firewall (an egress escape hatch that contradicts the lab's "every connection gated by our firewall" thesis; empirically, with it on, the MCP runtime to `learn.microsoft.com` never appears in our `AZFWApplicationRule` logs). With it `false`, the MCP host falls under AzureVNet's **default-Allow** and egresses via the VNet → forced-tunnel → hub Azure Firewall. So both the in-sandbox `mcp-broker`'s **server-bits fetch from `raw.githubusercontent.com`** (the `microsoftdocs/mcp` repo, during the `tools/list` handshake) AND the **runtime stream to `learn.microsoft.com`** are gated by our firewall. `vnet.bicep`'s `allow-microsoft-learn` collection allows `learn.microsoft.com` + an `allow-github-raw-mcp-bits` rule scoped to **`raw.githubusercontent.com`** only (the single host the agent hits — verified in `AZFWApplicationRule` denials; no `*.githubusercontent.com` wildcard). Without the GitHub-raw allow the connector provisions `Succeeded` yet shows **"no active connection"** with **zero** tools (a raw GET to `/api/mcp` returns `405` "use a streamable HTTP transport", so the endpoint is reachable — it's the bits fetch that's blocked). This is a **Standard** firewall, so L7 matching is FQDN/SNI only — pinning the exact repo path would need Azure Firewall **Premium** + TLS inspection (`targetUrls`). (The only true pod-side bypass is the platform `ExperimentalSettings.HttpMcpInSandbox` flag, which defaults to the locked-down in-sandbox broker and isn't exposed in our bicep.) **Separately**, MCP connector tools ship `defaultMode: disabled` (skill-gated — they only surface once a skill like `database-incidents` is active). To make the 3 Learn tools part of the **global** tool roster, `setup-sre-agent.ps1` Step 2b enables them via `POST /api/v2/agent/tools/configure` (`{overrides:[{name,enabled}]}`, merge semantics) — there is **no ARM/Bicep property** for per-tool state (the agent's `permissions` stays `null`; Microsoft's `srectl tool config set` CLI exists for exactly this). +- **Agent skills use the built-in Kubernetes system tools; terminal-native kubectl is only an ad-hoc fallback.** `RunKubectlReadCommand` and `RunKubectlWriteCommand` own the private-network, identity, and temporary connection setup required for each call. They do not rewrite `~/.kube/config`. If an exceptional chat must use an already-valid terminal kubeconfig, first issue a harmless built-in read against the same cluster; never make an incident runbook depend on that fallback. Keep the learner explanation in [`docs/aks-access-and-auth.md`](docs/aks-access-and-auth.md), not in the runtime knowledge file. +- **The Microsoft Learn MCP connector routes entirely through the hub Azure Firewall (no platform bypass) — `raw.githubusercontent.com` must be allow-listed or it silently surfaces zero tools.** Its ARM name is **`learn-docs`**, deliberately avoiding `azd`'s generic reserved-word warning for resource names containing `microsoft`; the endpoint remains Microsoft Learn. It is a no-auth *Streamable-HTTP* MCP server (`endpoint: https://learn.microsoft.com/api/mcp`): do not add `authType: CustomHeaders`. The proven healthy shape uses `dataSource: placeholder` and selects the three `learn-docs_microsoft_*` tools explicitly. We keep **`allowHttpMcpServerNetworkAccess: false`** (the default) on purpose: when `true`, the platform routes the MCP runtime endpoint as `Rewrite{RoutingMode=Platform}` — a broker that egresses *outside* the customer VNet, bypassing our firewall. With it `false`, the MCP host egresses via the VNet → forced-tunnel → hub Azure Firewall. Both the in-sandbox `mcp-broker` server-bits fetch from `raw.githubusercontent.com` and the runtime stream to `learn.microsoft.com` are therefore gated by our firewall. Without the GitHub-raw allow the connector can provision `Succeeded` yet expose zero tools. The setup script enables the three discovered tools globally through `/api/v2/agent/tools/configure`. +- **The agent could not reach its OWN data plane until `allowAgentSelfManagement` was added — and the failure looks like a TLS bug, not a firewall denial.** The agent's sandbox egress is force-tunneled to the hub firewall, whose allow-list covers ARM / Entra / Graph / Learn / the AKS API. The agent data plane is a *different host* — `https://--...azuresre.ai` — and was never allow-listed, so every call to `/api/v2/agent/*` (custom instructions, `tools/configure`) and `/api/v2/extendedAgent/*` (hooks, knowledge files) died with `SSL_ERROR_SYSCALL` while DNS resolved and TCP 443 connected. That reads like a certificate problem and sends you looking at the TLS-inspecting proxy CA; it is not. `infra/modules/firewall-agent-dataplane.bicep` deploys a separate rule collection group after the agent exists, pinned to that agent's exact FQDN. **Two traps:** (1) the token *audience* is `https://azuresre.dev` but the network *host* is `*.azuresre.ai` — allow-listing the audience domain does nothing; (2) a conditional Bicep module would leave the old rule behind under incremental ARM deployment. The module is therefore always deployed and writes an empty rule collection when `allowAgentSelfManagement=false`, which actually revokes existing access. Standard firewall handles exact FQDN/SNI matching; no Premium or broad wildcard is needed. **Security note:** this is a self-modification path — the same API that lets the agent read its config lets it rewrite its own skills and always-on prompts. That's intentional here; set the param false to keep data-plane config operator/CI-only. +- **The custom-instructions text is deliberately ~125 words, and should get SHORTER as models improve.** `sre-config/custom-instructions.md` encodes only what the model cannot infer from inside a session: (1) the structural fact that each alert arrives in an isolated thread, so silence about other alerts is an artifact rather than evidence; (2) explicit permission to spend tokens on breadth before concluding. Everything else — how to correlate, that correlation isn't causation — is reasoning a competent model already has, and restating it makes the agent mechanical while burning always-on budget. It is also the ONLY global budget there is (the API is a singleton), so every word competes with every future global instruction. The text is scoped ("before you commit to a root cause") so it stays inert in ordinary chat; an unconditional "always check X" fires on "what's my subscription id?" and trains ritual behavior. Method stays in the skill, trigger stays here. - **The hub Azure Firewall is the demo's "network device".** `firewall-diagnostics.bicep` ships its logs to Log Analytics as resource-specific `AZFW*` tables (`logAnalyticsDestinationType: 'Dedicated'`) so the agent can interrogate it *indirectly* (KQL on `AZFWNetworkRule` / `AZFWApplicationRule` / …) as well as *directly* (ARM reads of its policy/rules). Don't drop the diagnostic setting or the `Dedicated` flag — the KB points the agent at those tables, which only exist in Dedicated mode. The agent already holds Reader/Monitoring Reader on the RG, so no new role is needed for the direct path. -- **Agent AMPLS lockdown is ON by default (`lockAgentToPrivateMonitor = true`).** `monitor-private-link.bicep` always creates the Azure Monitor Private Link Scope, scoped resources (LA + App Insights), the private endpoint, and the five `privatelink.*` DNS zones (linked to the hub). By default it ALSO links those zones to the **agent** spoke, and `vnet.bicep` drops the public `AzureMonitor` service tag from the firewall L4 rule, so the agent reaches Log Analytics / App Insights only over the AMPLS private endpoint (maximum restraint). The agent remains fully functional under it: it queries Log Analytics / App Insights and remediates incidents end-to-end (dispatch → investigate via Monitor + native kubectl → `kubectl rollout undo` → verify) over the private path. The Monitor query connector is platform-brokered, so dropping the public `AzureMonitor` tag from the agent-VNet firewall doesn't gate it. Set `lockAgentToPrivateMonitor = false` to keep the public Monitor path. The **platform/workload** spoke is a separate concern: `linkWorkloadVnetsToPrivateMonitor` stays **false** by default because linking it forces the app's App Insights traffic onto the private endpoint — and the regional ingestion host (`-N.in.applicationinsights.azure.com`, from the component's connection string) can resolve into the private zone without a matching record → NXDOMAIN → the app silently stops shipping telemetry (a documented private-link DNS pitfall; this lab doesn't validate the workload's private path). The agent's lockdown is independent (it only queries Monitor, over its own spoke). Don't switch the AMPLS access mode to `PrivateOnly` (resource-level) without testing — that can block operator public queries region-wide. -- **Each deployment needs its own resource group — and a *fresh* RG name when redeploying.** `uniqueSuffix` is derived from `subscription().subscriptionId + resourceGroupName`, so a second instance must use a new env with a distinct `ZAVA_RG_NAME` (different RG name) to avoid resource-name collisions. **Critically, after an `azd down` do NOT redeploy to the same RG name:** that regenerates the same workspace name/resource ID, and the Azure Log Search Alerts backend retains stale state for the deleted workspace — every `scheduledQueryRules` create then fails with `BadRequest: The provided credentials have insufficient access to perform the requested operation` (even as Owner, even with a direct workspace role; metric alerts are unaffected). A new `ZAVA_RG_NAME` → new suffix → new workspace ID → alerts deploy cleanly. For a locked-down corp landing zone (this targets a permissive dev sub), expect Azure Policy to require hardening the demo doesn't apply: `disableLocalAccounts` on AKS, a Premium ACR + private endpoint, `publicNetworkAccess: 'Disabled'` on the workspace/App Insights, and a policy exemption for the firewall public IP (PostgreSQL is already VNet-integrated). Not validated here. +- **Agent AMPLS lockdown is ON by default (`lockAgentToPrivateMonitor = true`).** `monitor-private-link.bicep` always creates the Azure Monitor Private Link Scope, scoped resources (LA + App Insights), the private endpoint, and the five `privatelink.*` DNS zones (linked to the hub). By default it ALSO links those zones to the **agent** spoke, and `vnet.bicep` drops the public `AzureMonitor` service tag from the firewall L4 rule, so the agent reaches Log Analytics / App Insights only over the AMPLS private endpoint (maximum restraint). The agent remains fully functional under it: it queries Log Analytics / App Insights and remediates incidents end-to-end through Monitor and the built-in Kubernetes tools. The Monitor query connector is platform-brokered, so dropping the public `AzureMonitor` tag from the agent-VNet firewall doesn't gate it. Set `lockAgentToPrivateMonitor = false` to keep the public Monitor path. The **platform/workload** spoke is a separate concern: `linkWorkloadVnetsToPrivateMonitor` stays **false** by default because linking it forces the app's App Insights traffic onto the private endpoint — and the regional ingestion host (`-N.in.applicationinsights.azure.com`, from the component's connection string) can resolve into the private zone without a matching record → NXDOMAIN → the app silently stops shipping telemetry (a documented private-link DNS pitfall; this lab doesn't validate the workload's private path). The agent's lockdown is independent (it only queries Monitor, over its own spoke). Don't switch the AMPLS access mode to `PrivateOnly` (resource-level) without testing — that can block operator public queries region-wide. +- **Each deployment needs its own resource group — and a *fresh* RG name when redeploying.** `infra/main.bicepparam` now derives the default as `rg-$AZURE_ENV_NAME`, so `azd env new ` automatically isolates the deployment; `ZAVA_RG_NAME` remains an optional override. `uniqueSuffix` is derived from `subscription().subscriptionId + resourceGroupName`, so distinct environments avoid resource-name collisions. **Critically, after an `azd down` do NOT redeploy to the same environment/RG name:** that regenerates the same workspace name/resource ID, and the Azure Log Search Alerts backend retains stale state for the deleted workspace — every `scheduledQueryRules` create then fails with `BadRequest: The provided credentials have insufficient access to perform the requested operation` (even as Owner, even with a direct workspace role; metric alerts are unaffected). A fresh environment name → new RG → new suffix → new workspace ID → alerts deploy cleanly. For a locked-down corp landing zone (this targets a permissive dev sub), expect Azure Policy to require hardening the demo doesn't apply: `disableLocalAccounts` on AKS, a Premium ACR + private endpoint, `publicNetworkAccess: 'Disabled'` on the workspace/App Insights, and a policy exemption for the firewall public IP (PostgreSQL is already VNet-integrated). Not validated here. ## Project-local skills (Copilot CLI) For agents that support Copilot CLI's project-local skills under `.github/skills/`: - `deploying-demo` — Full deployment workflow (prerequisites, azd up, SRE Agent config, verification) -- `running-demo` — Break/fix scenarios with browser verification +- `running-demo` — Break/fix scenarios with browser verification (Scenarios 1-4 single-fault, Scenario 5 compound) - `managing-sre-agent` — Create/manage SRE Agent skills, response plans, knowledge files +## Data-plane config (not ARM-exposed) + +`scripts/setup-sre-agent.ps1` handles everything Bicep can't express at `2025-05-01-preview`: +knowledge-file sync (Step 2), Learn MCP tool enablement (2b), and **custom instructions** (2c). +**Custom instructions — the always-on global prompt (undocumented API).** This is the surface the portal's +"Custom instructions" box writes, and the thing appended to EVERY thread (chat, incident, scheduled task) +regardless of which response plan or skill matched. Captured from the portal's own network trace and +verified `200 OK`: + +``` +GET/PUT {agentEndpoint}/api/v2/agent/customInstructions +body: { "instructions": "" } +``` + +Three things differ from every other data-plane object, and each one bites: +1. **It is a SINGLETON**, not a named collection — there is no `/{name}` segment and exactly one blob per + agent. Everything you want globally must be concatenated into one document, which is itself an argument + for keeping it short. +2. **The body is FLAT** — `{ instructions }` — not the `{ name, type, tags, properties }` envelope used by + `/api/v2/extendedAgent/*` objects. +3. **It lives under `/api/v2/agent/`** — same family as the `tools/configure` call in Step 2b — NOT + under `/api/v2/extendedAgent/`. + +**Do not confuse this with `commonPrompts`** (`PUT /api/v2/extendedAgent/commonprompts/{name}`, as used by +`sreagent-templates`). That is a NAMED collection which *subagents* opt into via their own +`commonPrompts: [...]` field — a different feature. It is not what the portal's global box writes and it is +not appended to every thread. Building against it is a silent no-op for the global-instructions use case. + +Source of truth is [`sre-config/custom-instructions.md`](sre-config/custom-instructions.md). **The file +content IS the payload, verbatim** — there is no metadata wrapper and no comment syntax to strip, so keep +rationale in this file, never inside it. Step 2c GETs the current value and skips when unchanged; it strips +`\r` on both sides first because the service normalises stored line endings to CRLF, so an LF file would +otherwise look "changed" on every run. + Other agents can ignore this section. diff --git a/labs/zava-aks-postgres/README.md b/labs/zava-aks-postgres/README.md index 36c10eabb..570c6b790 100644 --- a/labs/zava-aks-postgres/README.md +++ b/labs/zava-aks-postgres/README.md @@ -16,6 +16,12 @@ azd up # Deploy everything (~25 min) azd down --force --purge # Tear down when done ``` +Deployment requires **Owner**, **User Access Administrator**, or equivalent +`Microsoft.Authorization/roleAssignments/write` permission at subscription scope. +The correlation skill reads subscription-wide alert and Service Health context, +so the template grants its runtime identity the built-in Reader role there. The +`predown` hook removes that assignment before deleting the resource group. + ## What You Get | Component | Details | @@ -25,7 +31,7 @@ azd down --force --purge # Tear down when done | **Monitoring** | App Insights + Log Analytics (4-day retention on noisy tables, no daily ingestion cap, 100% sampling; probes filtered at the alert KQL so alerts fire fast) + **3 enabled dispatching Azure Monitor alerts**: `postgres-unreachable` (covers both DB-stop and network-partition scenarios), `Zava-products-query-slow`, and `Zava-http-5xx-errors`, so one root cause = one incident thread. The app emits a **custom OpenTelemetry metric** (`zava.products.category.query.duration_ms`) and PG emits `cpu_percent`; the slow-query alert is **paired** with both as corroboration the agent queries during investigation (kept as *disabled* metric-alert examples rather than separate dispatching alerts, to avoid duplicate threads). | | **SRE Agent** | Anthropic-backed agent (Preview channel). Connectors, skills, response plans, and Azure Monitor incident binding declared in `infra/modules/sre-agent.bicep`. Knowledge-file upload via `scripts/setup-sre-agent.ps1` (ARM doesn't surface that yet). Default agent + rich skills, no subagent handoff. | | **Telemetry access** | App Insights, Log Analytics, and Azure Monitor exposed via **connectors** | -| **Demo Scenarios** | 4 break/fix scenarios with scripts | +| **Demo Scenarios** | 5 break/fix scenarios with scripts | ## Architecture @@ -41,9 +47,9 @@ HUB VNet 10.10.0.0/22 (shared edge / security) │ peering │ peering + UDR (forced tunnel) │ │ PLATFORM spoke 10.20.0.0/16 AGENT spoke 10.30.0.0/24 - ├─ AKS (private API server) └─ SRE Agent (VNet-injected) - │ ├─ zava-storefront • reaches AKS via native kubectl + ARM - │ └─ zava-api ──► PostgreSQL 16 (native kubectl exec, run-sql.js) + ├─ AKS (private API server) └─ SRE Agent (VNet-injected, /27 subnet) + │ ├─ zava-storefront • built-in Kubernetes tools + ARM + │ └─ zava-api ──► PostgreSQL 16 (in-pod run-sql.js helper) └─ db-subnet (delegated) (Entra auth) • all egress → hub firewall only App Insights + Log Analytics (AppRequests, AppMetrics, AZFW* firewall logs, KubeEvents …) @@ -98,6 +104,41 @@ While the UI shows `agent investigating`, the SRE Agent is actually working the Liveness and readiness (`/livez`) stay green, and `/api/health` can stay healthy for app-only regressions, so the platform looks healthy while only the app route regresses — deployment-signal correlation is what ties the symptom to its cause. +### Scenario 5: Compound — two independent faults, one window +```powershell +.\.github\skills\running-demo\scripts\break-compound.ps1 # Scenario 3 + Scenario 4, offset by 90s +# TWO alerts co-fire (Zava-products-query-slow + Zava-http-5xx-errors) into SEPARATE threads +# (merge is disabled on every response plan). They are NOT causally related. +.\.github\skills\running-demo\scripts\fix-compound.ps1 # Fallback: undoes both +``` +Scenarios 1–4 inject exactly one fault each, so "diagnose the alert you were handed" always works — a habit that +breaks in production. This scenario is the counterexample. The tempting read is *"the database got slow, so the API +started failing"*: it fits the timestamps perfectly and it is **false**. The mechanisms are disjoint — + +| Signal | 5xx fault | Slow-query fault | +|---|---|---| +| Status code | HTTP **500** | n/a (requests succeed) | +| Failed dependencies | `localhost:3001` **only** | **none** — queries are slow but *succeed* | +| PG dependency failures | **zero** | **zero** | +| PG `cpu_percent` | baseline | pegged ~90% | + +If DB saturation were causing the 5xx you would see PG dependency failures or 503 timeouts. Neither appears. +Two faults, one window, no causal link. + +Two further traps are built in. **Alert fire order is not causal order** — every dispatching rule is `PT5M`/`PT5M`, +so detection latency swamps the 90-second injection offset. And the *causal* DB signal never alerts at all: +`Zava-db-cpu-saturation` ships **disabled** (see [`AGENTS.md`](AGENTS.md)), mimicking an org that muted a noisy +rule months ago, so the agent has to enumerate the alert **rule inventory** — not just fired alerts — to discover it. + +Handling this well is what the `incident-correlation` skill and the always-on +[`sre-config/custom-instructions.md`](sre-config/custom-instructions.md) nudge +exist for: an alert is a signal, not the story. + +The correlation skill reads subscription-scoped Alerts Management and Resource +Health event feeds. The agent's runtime user-assigned identity therefore receives +the built-in **Reader** role at subscription scope. Monitor query and remediation +rights remain limited to the demo resource group. + ## SRE Agent Management Agent configuration is fully declarative in **`infra/modules/sre-agent.bicep`** — @@ -105,21 +146,28 @@ connectors, custom skills, response plans / incident filters, autonomous mode, a Monitor incident binding all flow through `Microsoft.App/agents/*` ARM resources. To change them, edit the Bicep and run `azd provision`. -The only data-plane state ARM doesn't yet surface is **knowledge file upload** — handled -by `scripts/setup-sre-agent.ps1`, which also verifies the Bicep-deployed assets are live. -Drop new `*.md` files into `sre-config/knowledge-base/` and re-run the script to sync. +Residual data-plane state is handled by `scripts/setup-sre-agent.ps1`: knowledge-file +upload, the singleton agent-global custom instructions, and Microsoft Learn MCP tool +enablement. The script also verifies the Bicep-deployed assets are live. Drop new +`*.md` files into `sre-config/knowledge-base/` and re-run the script to sync. ## How the Agent Operates Against a Private Backend ### Network posture: VNet-injected, egress locked down behind an Azure Firewall -The agent is **injected into a dedicated agent spoke VNet** (a delegated `agent-subnet`) and its sandbox egress is **locked down behind an Azure Firewall** — default-deny, with a tight allow-list (ARM, Entra, Microsoft Graph, Azure Monitor, Microsoft Learn, GitHub raw for the Learn MCP server bits) **plus a narrow rule to the AKS API server** that enables native kubectl. The agent sits **inside** the VNet and operates the cluster with **native `kubectl`** (authenticated by its own managed identity) and runs PostgreSQL SQL through an in-cluster pod. ARM and Azure Monitor go over the control plane; nothing else gets out. The point is a fully locked-down, in-VNet agent: it sits inside the customer network boundary yet its blast radius is constrained to the allow-list and its least-privilege identity grants. See **[Native kubectl (enabled)](#native-kubectl-enabled)** for exactly how the API-server path + auth are wired (and how to close it for a command-invoke-only posture). +The agent is **injected into a dedicated `/28` agent spoke subnet** and its sandbox egress is **locked down behind an Azure Firewall** with a tight allow-list. Agent skills operate Kubernetes through the built-in `RunKubectlReadCommand` and `RunKubectlWriteCommand` system tools. PostgreSQL SQL runs through the in-cluster helper invoked by the write tool. + +> **Kubernetes tool choice:** incident runbooks use the built-in `RunKubectlReadCommand` and `RunKubectlWriteCommand` tools because they own the private-network, identity, and temporary connection setup. For an exceptional ad-hoc terminal command with an already-valid kubeconfig, issue a harmless built-in read against the same cluster first. Do not build runbooks around that fallback. + +> **Learn the full access path:** [`docs/aks-access-and-auth.md`](docs/aks-access-and-auth.md) explains kubeconfig anatomy, managed-identity token acquisition, Azure RBAC, the two TLS trust hops behind the warm-up behavior, private API-server DNS and routing, operator/CI access choices, and why a public FQDN on a private cluster is not a public API endpoint. -> **What "VNet-injected" means here:** the agent's egress mode is **AzureVNet** (real VNet egress) routed through the Azure Firewall **and** a TLS-inspecting forward proxy that re-signs certificates. The firewall allow-list is ARM/Entra/Graph, Azure Monitor, Microsoft Learn (+ GitHub raw for the Learn MCP server bits), **and a rule (with SNAT) to the AKS API server** — so the agent reaches the private API server and uses **native `kubectl`** (managed-identity auth via `kubelogin`, trusting the egress-proxy CA). PostgreSQL SQL runs from an in-cluster pod (a real VNet NIC) via `kubectl exec`. Egress allow/deny decisions are visible in the SRE Agent UI under **Workspace Configuration → Inspect → Network audit** (Preview). To revert to a command-invoke-only posture (no API-server line of sight), remove the API-server firewall rule + SNAT — see [Native kubectl (enabled)](#native-kubectl-enabled). +> **What "VNet-injected" means here:** the agent's egress mode is **AzureVNet** (real VNet egress) routed through the Azure Firewall. Egress allow/deny decisions are visible in the SRE Agent UI under **Workspace Configuration → Inspect → Network audit** (Preview). Kubernetes access in the skills remains through the built-in system tools. > **Scope:** the firewall + forced-tunnel route govern the **agent sandbox's internet egress** (the `agent-subnet` only). They do not restrict private intra-VNet traffic, the AKS subnet's own egress, or what the agent can make AKS do via its Cluster Admin RBAC — those are governed by Kubernetes RBAC and the agent's action boundary, not this firewall. -One consequence is worth calling out, because it shapes Scenario 3's remediation: **DDL like `CREATE INDEX` is data-plane only.** No managed PG service (Azure PG Flex, RDS, Cloud SQL) exposes catalog mutation through its cloud control plane. The agent reads `pg_stat_*` to diagnose the missing index and applies the DDL the same way — by running the in-cluster helper from a workload that's already in the VNet (the api pod), reached via native `kubectl exec`: +> **The agent's own URL is allowed.** `allowAgentSelfManagement=true` (the default) adds an HTTPS rule for the exact platform-assigned agent data-plane FQDN (`--...azuresre.ai`). The rule does not allow the broad `*.azuresre.ai` wildcard. This permits custom-instruction, knowledge, and tool-configuration calls to the agent's own API through the hub firewall, including agent-initiated configuration changes. Set the parameter to `false` when configuration must remain operator/CI-owned. + +One consequence is worth calling out, because it shapes Scenario 3's remediation: **DDL like `CREATE INDEX` is data-plane only.** No managed PG service (Azure PG Flex, RDS, Cloud SQL) exposes catalog mutation through its cloud control plane. The agent reads `pg_stat_*` to diagnose the missing index and applies the DDL through the in-cluster helper using `RunKubectlWriteCommand`: ``` kubectl exec deploy/zava-api -n zava-demo -- node bin/run-sql.js '' @@ -130,46 +178,31 @@ kubectl exec deploy/zava-api -n zava-demo -- node bin/run-sql.js '' | Component | Endpoint | How the agent works on it | |---|---|---| | Storefront / nginx ingress | Public LoadBalancer IP | HTTP from anywhere | -| AKS API server | **Private** (AKS private-DNS zone linked to the agent VNet; firewall rule + SNAT to the API server) | Native `kubectl`, authenticated by the agent's Entra identity (*Cluster Admin* RBAC) | -| Pods, services, node IPs | Private (VNet only) | Native `kubectl ` (`get`, `logs`, `describe`, `delete`, `apply`, `exec`, `rollout`) | -| PostgreSQL Flex (port 5432) | **Private only** — `publicNetworkAccess: Disabled`, VNet-delegated | State/config: `az postgres flexible-server`. SQL (reads + DDL): native `kubectl exec deploy/zava-api -- node bin/run-sql.js ''` — the in-cluster pod (a real VNet NIC) reuses the pod's PG Entra identity | +| AKS API server | **Private** | Built-in `RunKubectlReadCommand` / `RunKubectlWriteCommand` tools | +| Pods, services, node IPs | Private (VNet only) | Built-in Kubernetes system tools | +| PostgreSQL Flex (port 5432) | **Private only** — `publicNetworkAccess: Disabled`, VNet-delegated | State/config: `az postgres flexible-server`. SQL (reads + DDL): `RunKubectlWriteCommand` invokes `kubectl exec deploy/zava-api -- node bin/run-sql.js ''` | ### What the agent can do (from inside the locked-down VNet) | Plane | Read | Write / remediate | |---|---|---| | **AKS control plane** | `az aks show / nodepool list / get-upgrades` | `az aks start / stop / update / nodepool scale / rotate-certs` | -| **Kubernetes (via native `kubectl`)** | `kubectl get … / logs / describe` | `kubectl delete networkpolicy …` (Scenario 2), `kubectl rollout undo …`, `kubectl exec deploy/zava-api -- node bin/run-sql.js 'CREATE INDEX …'` (Scenario 3) | -| **PostgreSQL** | Control: `az postgres flexible-server show / parameter list / backup list / server-logs list / replica list`. Data (reads + DDL): native `kubectl exec … bin/run-sql.js` | `az postgres flexible-server start` (**Scenario 1**), `restart`, `update`, `parameter set`, `replica create`, `restore`, `ad-admin create` | +| **Kubernetes** | `RunKubectlReadCommand` | `RunKubectlWriteCommand` for NetworkPolicy deletion, rollout undo, and in-pod SQL helper execution | +| **PostgreSQL** | Control: `az postgres flexible-server show / parameter list / backup list / server-logs list / replica list`. Data (reads + DDL): in-cluster helper through `RunKubectlWriteCommand` | `az postgres flexible-server start` (**Scenario 1**), `restart`, `update`, `parameter set`, `replica create`, `restore`, `ad-admin create` | | **Networking** | `az network nsg / vnet / private-dns show`, plus the hub firewall as a device: `az network firewall [policy] show` (Reader-covered) and its `AZFW*` logs (KQL) | `az network nsg rule create / delete` (Scenario 2 cleanup) | | **Telemetry** | App Insights, Log Analytics, and Azure Monitor connectors (KQL + metrics) — API-based, no network reachability needed | Alert / action group create / update | ### Running PostgreSQL SQL -SQL — reads (`pg_stat_*`) and read-mostly DDL like `CREATE INDEX CONCURRENTLY` and `ANALYZE` — runs through the in-cluster `bin/run-sql.js` helper in the application pod (which reuses the pod's PostgreSQL Entra identity), invoked through native `kubectl exec`: +SQL — reads (`pg_stat_*`) and read-mostly DDL like `CREATE INDEX CONCURRENTLY` and `ANALYZE` — runs through the in-cluster `bin/run-sql.js` helper in the application pod, invoked with `RunKubectlWriteCommand`: ``` kubectl exec deploy/zava-api -n zava-demo -- node bin/run-sql.js '' ``` -### Native kubectl (enabled) - -This lab is configured so the agent uses **native `kubectl`** against the private cluster — the agent runs `kubectl get nodes` / `get pods` / `rollout undo` / `exec … run-sql.js` directly. The deploy now completes the private-cluster path automatically; two infra enablers plus a three-step in-session setup make it work: +### Kubernetes system tools -**Infra — in `vnet.bicep` + `scripts/post-provision.ps1`:** -1. **DNS** — link the AKS-managed private-DNS zone `.privatelink..azmk8s.io` (in the cluster's `MC_…` resource group) to the **agent** VNet so the sandbox resolves the API-server FQDN. The zone name is dynamic and unknown until AKS creates it, so this cannot be a static Bicep resource. `scripts/post-provision.ps1` **Step 4b** discovers the node resource group, azmk8s.io private-DNS zone, and agent VNet, then idempotently creates the `agent-link` virtual-network link on every deploy: - ``` - ZONE=$(az network private-dns zone list -g --query "[?contains(name,'azmk8s')].name|[0]" -o tsv) - az network private-dns link vnet create -g -z $ZONE -n agent-link -v -e false - ``` -2. **Firewall** — `vnet.bicep` adds an allow rule (`agent-subnet 10.30.0.0/28 → aks-subnet 10.20.0.0/20 :443`) **and SNATs** all traffic (`snat.privateRanges = 255.255.255.255/32`). SNAT is essential: the API server's NSG only admits the `VirtualNetwork` tag and the agent spoke isn't *directly* peered to the platform spoke, so the agent's source IP is rewritten to the firewall's hub IP (which *is* in the tag) — that also makes the return path symmetric without touching the AKS subnet's routing. - -**Agent in-session setup — encoded in the skill runbook:** -1. `az aks get-credentials -g -n --overwrite-existing` -2. `kubelogin convert-kubeconfig -l azurecli` — non-interactive managed-identity auth (the default device-code flow hangs in a sandbox). -3. The sandbox egress is a **TLS-inspecting forward proxy** that re-signs certs, so merge its CA (`/etc/ssl/certs/adc-egress-proxy-ca.crt`) into the kubeconfig's cluster `certificate-authority-data` so kubectl trusts the connection. - -**`az aks command invoke`:** this lab does **not** use it for the agent — the agent is native-`kubectl`-only. It remains an option for a deployment that deliberately closes the firewall path to the API server (remove the `allow-agent-to-aks-api` rule + SNAT from `vnet.bicep`); see the [private-cluster docs](https://learn.microsoft.com/en-us/azure/aks/access-private-cluster). +Skills that need Kubernetes list `RunKubectlReadCommand` and, when remediation or `exec` is required, `RunKubectlWriteCommand`. These are the canonical runtime path. Do not add `RunInTerminal`, Python wrappers, kubeconfig setup, `kubelogin`, or proxy certificate manipulation to skill instructions. ## Hub-and-Spoke & Talking to Network Devices @@ -177,11 +210,11 @@ The network is modeled as **hub-and-spoke**, the shape most enterprises actually - **Hub VNet** (`vnet-Zava-hub-*`, 10.10.0.0/22) holds the shared **Azure Firewall** (the agent's single egress point), a reserved **`GatewaySubnet`** where an **ExpressRoute/VPN gateway** to on-prem would attach, and the **Azure Monitor Private Link Scope (AMPLS)** private endpoint. - **Platform spoke** (`vnet-Zava-platform-*`, 10.20.0.0/16) holds the workload — AKS + PostgreSQL. -- **Agent spoke** (`vnet-Zava-agent-*`, 10.30.0.0/24) holds the VNet-injected SRE Agent; its egress is force-tunneled to the hub firewall over VNet peering (UDR `0.0.0.0/0` → firewall). +- **Agent spoke** (`vnet-Zava-agent-*`, 10.30.0.0/24) holds the VNet-injected SRE Agent in `agent-subnet` (`10.30.0.0/27`). `/27` is the minimum: after Azure reserves five addresses, 27 usable addresses remain. Its egress is force-tunneled to the hub firewall over VNet peering (UDR `0.0.0.0/0` → firewall). > **The agent's VNet is regional — its *reach* is not.** VNet injection is a **regional binding**: the `agent-subnet` you inject the agent into **must be in the same Azure region as the SRE Agent resource** — Microsoft's docs are explicit, *"The subnet must be in the same region as your SRE Agent resource"* ([SRE Agent subnet requirements](https://learn.microsoft.com/azure/sre-agent/network-integration#configure-azure-vnet-mode)). You **cannot** inject an agent that lives in *region A* into a subnet in *region B*. But that co-regional subnet only fixes **where the agent runs** — it does **not** limit **what the agent can reach**. Once injected, the agent reaches whatever its VNet can route to, including resources in **other Azure regions** (over [global VNet peering](https://learn.microsoft.com/azure/virtual-network/virtual-network-peering-overview)) and **on-premises** networks (over ExpressRoute/VPN) — *"as long as your network routes and rules allow it"* ([SRE Agent traffic routing](https://learn.microsoft.com/azure/sre-agent/network-integration#how-azure-vnet-mode-works)). In this lab all three VNets are co-regional, but the cross-region path is the **same mechanism** as the on-prem path — see [Reaching other regions and on-premises](#reaching-other-regions-and-on-premises). -This proves the agent operates identically when it's isolated in its own management spoke and reaches everything through a *shared* firewall — the real customer pattern. It's behavior-preserving because the agent reaches AKS via native `kubectl` over the private API-server path and PostgreSQL through an in-cluster pod — not raw DB sockets; moving it into a separate spoke changes only *which* firewall inspects its egress. +This proves the agent operates identically when isolated in its own management spoke and reaches everything through a *shared* firewall — the real customer pattern. Kubernetes operations use the built-in system tools, and PostgreSQL access stays inside the application pod rather than opening raw DB sockets. ### The hub firewall doubles as a "network device" the agent can interrogate @@ -223,7 +256,7 @@ The on-prem example is therefore just **one instance** of the general rule, not The Log Analytics workspace and Application Insights are scoped to an **Azure Monitor Private Link Scope** with a private endpoint in the hub (`infra/modules/monitor-private-link.bicep`). By default (`lockAgentToPrivateMonitor = true`) the **agent is locked to the private path**: its Monitor private-DNS zones are linked to the agent VNet and the public `AzureMonitor` service tag is dropped from the firewall L4 allow-list, so the agent reaches Log Analytics / Application Insights only over the AMPLS private endpoint (maximum restraint). Set `lockAgentToPrivateMonitor = false` to keep the public allow-listed Monitor path instead. -> **The agent stays fully functional under the lockdown.** With the lockdown on, the agent still queries Log Analytics / Application Insights and remediates incidents end-to-end (dispatch → investigate via Monitor + native kubectl → `kubectl rollout undo` → verify) over the private path. The agent's Monitor query connector is platform-brokered, so dropping the public `AzureMonitor` tag from the agent-VNet firewall doesn't gate it. +> **The agent stays fully functional under the lockdown.** With the lockdown on, the agent still queries Log Analytics / Application Insights and remediates incidents end-to-end through Monitor and the built-in Kubernetes tools. The agent's Monitor query connector is platform-brokered, so dropping the public `AzureMonitor` tag from the agent-VNet firewall doesn't gate it. > **Workload (app) telemetry stays public by default.** `linkWorkloadVnetsToPrivateMonitor = false` on purpose: linking the *platform* spoke to the Monitor private-DNS zones forces the app's App Insights traffic onto the private endpoint, which only works if every endpoint in its connection string is served by the AMPLS zones. The regional App Insights **ingestion** host (`-N.in.applicationinsights.azure.com`, from the component's connection string) is the classic gap: if it resolves into the private zone without a matching record it returns NXDOMAIN and the app silently stops shipping telemetry — a [documented private-link DNS pitfall](https://learn.microsoft.com/azure/azure-monitor/logs/private-link-security). This lab doesn't validate the workload's private path, so it's left public; the agent's lockdown is independent (it only *queries* Monitor, over its own spoke). Enable the toggle only after validating the workload's ingestion endpoints. For resource-level lockdown, switch the AMPLS access mode to `PrivateOnly` (riskier — can block operator public queries region-wide). @@ -231,7 +264,7 @@ The Log Analytics workspace and Application Insights are scoped to an **Azure Mo This demo targets a permissive dev/sandbox subscription and works there as-is: it ships a Standard Azure Firewall with a public IP, a Basic ACR, AKS with local accounts enabled, and default public network access on the Log Analytics workspace / Application Insights (PostgreSQL is already VNet-integrated, with no public endpoint). A locked-down corporate landing zone with strict Azure Policy would likely require hardening those: `disableLocalAccounts` on AKS, a Premium ACR with a private endpoint, `publicNetworkAccess: 'Disabled'` on the workspace/App Insights, and a policy exemption for the firewall public IP. That hardened path isn't validated here. -> **`disableLocalAccounts` and the SRE Agent.** On a hardened cluster the agent can use **native `kubectl`** authenticated by its own managed identity (`kubelogin convert-kubeconfig -l azurecli`) over a private network path to the API server (private-DNS link + firewall/SNAT). This sample does not override the built-in `RunKubectl*` tool state. +> **`disableLocalAccounts` and the SRE Agent.** Kubernetes operations use the built-in `RunKubectl*` system tools and the agent's existing AKS RBAC grant. ## Platform Behaviors @@ -241,17 +274,21 @@ For repo/IaC author gotchas (Sev4 quirk, NSG-vs-NetworkPolicy, container-image b Azure Monitor itself does NOT link or merge incidents across different alert rules. Each alert rule fires independently, and the same rule re-firing just updates the existing alert's count (with `autoMitigate: true`, it flips to `Resolved` when the condition clears). +The break scripts make repeat runs fail-safe: before injecting a fault they inspect the relevant stateful alert instance. A still-`Fired` prior condition aborts the script because Azure Monitor cannot produce a fresh dispatch; a resolved prior instance is closed so the next activation arrives as `New`. This is intentionally different from making the rules stateless, which would emit another alert every evaluation and create duplicate investigation threads during one sustained fault. + **Consequence for the two DB scenarios:** they share the single `postgres-unreachable` rule, so while the first alert is still `Fired`, a back-to-back second break just updates that instance instead of opening a new one — and the SRE Agent only dispatches on a *new* alert. To handle this, the `database-incidents` runbook has the agent **close the alert as its final step** once it verifies recovery (it holds the Contributor right for `Microsoft.AlertsManagement/alerts/changestate/action`), so the next DB break dispatches fresh; `autoMitigate` (~15-30 min) is the fallback if it doesn't. The other scenarios use distinct rules, so this only ever affected DB stop ↔ network partition. -**This sample disables agent-side merge on all four response plans** (`mergeEnabled: false`, `mergeWindowHours: 0`) — every incident opens its OWN investigation thread, with no deduplication. (For reference: when merge is *on*, the agent folds any matching alert arriving within `mergeWindowHours` into the most recent open thread for that plan instead of dispatching a new one. That deduplication can quietly hide real, distinct incidents, so this demo keeps it off.) +**This sample disables agent-side merge on all four response plans** (`mergeEnabled: false`; `mergeWindowHours: 3` remains a schema-valid but inactive value) — every incident opens its OWN investigation thread, with no deduplication. (For reference: when merge is *on*, the agent folds any matching alert arriving within `mergeWindowHours` into the most recent open thread for that plan instead of dispatching a new one. That deduplication can quietly hide real, distinct incidents, so this demo keeps it off.) + +The four response plans / incident filters are `zava-database` (`postgres`), `zava-performance` (`query-slow`), `zava-application` (`http-5xx`), and `zava-unknown` (other `Zava` alerts, Review mode). Response plans do not have a customer-defined priority or "most specific wins" rule, so overlapping matches should be treated as undefined. Prefer purpose-built filters that do not overlap; if you keep a fallback, positively bound its scope and explicitly exclude every known route. This sample does both with `titleContains: 'Zava'` plus `titleNotContains` for the three known tokens. -The four response plans / incident filters are `zava-database` (`postgres`), `zava-performance` (`query-slow`), `zava-application` (`http-5xx`), and `zava-unknown` (catch-all, Review mode). The `Zava-http-5xx-errors` alert also **no longer self-suppresses** on DB errors — a DB outage that returns 5xx will open both a `postgres-unreachable` thread and an app thread, so every real symptom surfaces its own investigation. The only deduplication left is the Azure Monitor shared-rule stateful behavior described in the box above, which affects only back-to-back DB stop ↔ partition. +The `Zava-http-5xx-errors` alert also **no longer self-suppresses** on DB errors — a DB outage that returns 5xx will open both a `postgres-unreachable` thread and an app thread, so every real symptom surfaces its own investigation. The only deduplication left is the Azure Monitor shared-rule stateful behavior described in the box above, which affects only back-to-back DB stop ↔ partition. A brand-new `azd env new` always gets fresh dispatch because the SRE Agent name includes a per-env suffix (e.g. `sre-agent-zava-awpo`), so a new env gets a new agent with an empty thread store. ### Microsoft Learn MCP (Streamable-HTTP) connector -The `microsoft-learn` connector is a remote **Streamable-HTTP** MCP server (`https://learn.microsoft.com/api/mcp`). It is configured to route **entirely through the hub Azure Firewall** — no platform bypass. Three non-obvious things: +The `learn-docs` connector is a no-auth remote **Streamable-HTTP** MCP server for Microsoft Learn (`https://learn.microsoft.com/api/mcp`). The neutral ARM name avoids `azd`'s generic reserved-word warning for names containing `microsoft`; it does not change the service or endpoint. Its three tools are selected in Bicep and it routes **entirely through the hub Azure Firewall** — no platform bypass. Three non-obvious things: 0. **No platform escape hatch (`allowHttpMcpServerNetworkAccess: false`).** Left at its default-off on purpose. When `true`, the platform routes the MCP runtime endpoint as `Rewrite{RoutingMode=Platform}` — a broker that egresses *outside* the VNet, bypassing this firewall (it never even appears in the `AZFW*` logs). With it off, the MCP host falls under AzureVNet's default-Allow and egresses via the VNet → forced-tunnel → the firewall, so the runtime stream to `learn.microsoft.com` is gated by **our** allow-list like everything else — consistent with the lockdown thesis. (The only true pod-side bypass is the platform `ExperimentalSettings.HttpMcpInSandbox` flag, which defaults to the locked-down in-sandbox broker and isn't exposed here.) 1. **Its server bits come from GitHub raw.** The in-sandbox `mcp-broker` fetches the connector's server bits from `raw.githubusercontent.com` (the `microsoftdocs/mcp` repo) during the `tools/list` handshake. The firewall therefore allow-lists `raw.githubusercontent.com` (`allow-github-raw-mcp-bits` in `vnet.bicep`). Without it the connector provisions but shows *"no active connection"* with **zero tools**, even though `learn.microsoft.com` itself is reachable (a raw GET to `/api/mcp` returns `405` "use a streamable HTTP transport"). The connection idle-disconnects and re-handshakes, so the rule is needed durably, not just on first use. It's scoped to that single host — this is a **Standard** firewall, which matches FQDN/SNI only; pinning the exact repo path (`raw.githubusercontent.com/microsoftdocs/mcp/*`) would require Azure Firewall **Premium** + TLS inspection (`targetUrls`). @@ -264,7 +301,7 @@ The `microsoft-learn` connector is a remote **Streamable-HTTP** MCP server (`htt - [Azure Developer CLI (azd)](https://learn.microsoft.com/azure/developer/azure-developer-cli/install-azd) (1.9+) - [PowerShell 7.4+](https://learn.microsoft.com/powershell/scripting/install/installing-powershell) — **required on Windows, WSL, Linux, or macOS**; `azd up` runs a pre-provision check and fast-fails if `pwsh` is missing -> **Note:** `kubectl` is **not** required on your local workstation. The AKS cluster is private; operator in-cluster operations in this repo go through `az aks command invoke` (wrapped by `Invoke-AksCommand` in `scripts/_aks-helpers.ps1`) because the operator workstation lacks the agent's VNet/DNS/proxy setup. The SRE Agent uses native `kubectl` (private-DNS link + firewall rule + SNAT to the API server) — see "How the Agent Operates Against a Private Backend" above. +> **Note:** `kubectl` is **not** required on your local workstation. The AKS cluster is private; operator in-cluster operations use `az aks command invoke` (wrapped by `Invoke-AksCommand`), while the SRE Agent uses its built-in Kubernetes tools. > **Region default:** `azd up` will prompt for a location. The Bicep default is `swedencentral` (validated end-to-end there). To deploy elsewhere, pick another region at the prompt or run `azd env set AZURE_LOCATION ` before `azd up`. Any region with availability for AKS, PostgreSQL Flexible Server, and the SRE Agent resource provider works. @@ -286,6 +323,7 @@ zava-aks-postgres/ ├── infra/ # Bicep (AKS, PostgreSQL, SRE Agent, monitoring) ├── src/api/ # Express.js API ├── src/storefront/ # Zava Athletic storefront UI +├── docs/ # Architecture and access explainers + images ├── k8s/ # Kubernetes manifests (${VAR} substitution) ├── scripts/ # azd lifecycle hooks + shared helper │ ├── _aks-helpers.ps1 # Invoke-AksCommand wrapper (REST fallback) diff --git a/labs/zava-aks-postgres/docs/aks-access-and-auth.md b/labs/zava-aks-postgres/docs/aks-access-and-auth.md new file mode 100644 index 000000000..335f50cd9 --- /dev/null +++ b/labs/zava-aks-postgres/docs/aks-access-and-auth.md @@ -0,0 +1,202 @@ +# AKS access, identity, and kubeconfig in this lab + +This lab deliberately uses a private AKS API server and a VNet-injected SRE Agent. That makes Kubernetes access more realistic than a public demo cluster, but it also means several independent controls must all succeed before a `kubectl` command works. + +Use this guide to understand the access path. Incident runbooks should still use the agent's built-in `RunKubectlReadCommand` and `RunKubectlWriteCommand` tools. + +## The four gates behind a successful kubectl call + +`kubectl get pods` is one command, but it crosses four separate gates: + +| Gate | Question | Typical failure | +|---|---|---| +| Name resolution and routing | Can the caller resolve and reach the API server's private endpoint? | DNS failure, timeout, or no route | +| TLS trust | Does each side of the connection trust the certificate presented by the next hop? | `x509: certificate signed by unknown authority`, reset, or proxy handshake failure | +| Authentication | Can the caller obtain a valid Microsoft Entra token for AKS? | `kubelogin` or token acquisition error | +| Authorization | Is that Entra identity allowed to perform the requested Kubernetes action? | HTTP 403 `Forbidden` | + +These gates are independent. A fresh token does not create a network route. A valid kubeconfig does not grant RBAC. Repairing the client certificate chain does not necessarily repair an intermediary's trust of the upstream AKS server. + +## The Zava cluster posture + +The lab declares the following AKS settings in [`infra/modules/aks.bicep`](../infra/modules/aks.bicep): + +```bicep +aadProfile: { + managed: true + enableAzureRBAC: true +} +apiServerAccessProfile: { + enablePrivateCluster: true + privateDNSZone: 'system' + enablePrivateClusterPublicFQDN: false +} +``` + +The practical result is: + +- The Kubernetes API is reached through a private endpoint. +- AKS creates and manages a private DNS zone for the API server. +- No public API-server FQDN is published for the cluster. +- Microsoft Entra ID authenticates callers. +- Azure RBAC authorizes Kubernetes API operations. +- Both SRE Agent identities receive the **Azure Kubernetes Service RBAC Cluster Admin** role on this demo cluster. + +The broad cluster-admin grant is intentional for an autonomous break/fix lab. A production deployment should normally replace it with namespace-scoped Reader, Writer, or Admin assignments that match the agent's required actions. + +## What a kubeconfig contains + +A kubeconfig is connection configuration, not a universal access credential. Its important sections are: + +```yaml +clusters: +- name: zava + cluster: + server: https://:443 + certificate-authority-data: + +users: +- name: zava-identity + user: + exec: + command: kubelogin + args: + - get-token + - --login + - msi + +contexts: +- name: zava + context: + cluster: zava + user: zava-identity + namespace: zava-demo + +current-context: zava +``` + +- **`clusters`** says where the API server is and which certificate authority signs its TLS certificate. +- **`users`** says how the client obtains credentials. Modern Entra-integrated AKS configurations commonly use the `kubelogin` exec plugin rather than embedding a long-lived token. +- **`contexts`** pair one cluster with one user and, optionally, a default namespace. +- **`current-context`** selects the active pairing. + +`az aks get-credentials` retrieves connection configuration through the Azure Resource Manager control plane and normally merges it into `~/.kube/config`. Permission to retrieve that configuration and permission to use the Kubernetes API are separate checks. On an Entra-integrated cluster, the eventual data-plane access still depends on the signed-in identity's AKS RBAC assignment. + +## The agent's built-in Kubernetes path + +For this lab, a built-in call follows this logical flow: + +```text +RunKubectlReadCommand / RunKubectlWriteCommand + | + | 1. Resolve AKS connection information through Azure + | 2. Select the agent managed identity + | 3. Build a temporary per-call kubeconfig + | 4. Add the runtime proxy CA needed by the client-side TLS hop + | 5. Obtain an Entra token through kubelogin managed-identity login + v +VNet-injected runtime and egress proxy + | + | 6. Reach the AKS private endpoint over the connected VNets + | 7. Validate the AKS server using the cluster CA + v +AKS API server + | + | 8. Authorize the identity with Azure RBAC for Kubernetes + v +Requested Kubernetes operation +``` + +The temporary kubeconfig is deleted after the tool call. A built-in call does **not** rewrite or leave a repaired kubeconfig in `~/.kube/config`. + +This is why the built-in tools are more than command wrappers. They accept normal kubectl arguments, but the runtime also supplies the managed identity, temporary kubeconfig, certificate handling, private network path, and tool policy. + +## Why a built-in call can "warm up" terminal kubectl + +The current preview runtime has two TLS trust relationships when traffic passes through its egress intermediary: + +1. **Client to intermediary:** kubectl must trust the certificate presented by the runtime intermediary. The built-in path adds the runtime proxy CA to its temporary kubeconfig. +2. **Intermediary to AKS:** the intermediary must trust the real AKS API-server certificate. The runtime keeps the AKS cluster CA in process-local state. + +An existing terminal kubeconfig can already have valid client-side configuration while the second trust relationship is cold. In that state: + +- signing in again changes identity state, not upstream TLS trust; +- `az aks get-credentials` refreshes the terminal kubeconfig, not the intermediary's process-local CA state; +- `kubelogin convert-kubeconfig` changes token acquisition, not the intermediary's upstream trust; +- adding a client-side CA can repair the first TLS hop, but not the second. + +A successful built-in `RunKubectlReadCommand` or `RunKubectlWriteCommand` against the same cluster registers the AKS CA in the current runtime process. In the live Zava test, terminal `kubectl get nodes` and `kubectl get pods -n zava-demo` then succeeded immediately with the existing kubeconfig and no additional login, credential refresh, `kubelogin`, or CA changes. + +That warm-up is: + +- process-local; +- temporary; +- lost when the runtime restarts; +- not a replacement for a valid terminal kubeconfig; +- not something incident runbooks should depend on. + +If an exceptional ad-hoc chat truly needs terminal-native kubectl, first complete a harmless built-in read against the same cluster, then use the already-valid terminal kubeconfig. For normal reads, writes, `exec`, rollout, and NetworkPolicy operations, send the kubectl command directly to `RunKubectlReadCommand` or `RunKubectlWriteCommand`. + +## Why not preserve the temporary kubeconfig? + +The runtime-generated file is temporary and contains no bearer token, password, client private key, or client certificate. Authentication happens when `kubelogin` obtains a short-lived token for the selected managed identity. + +Do not treat that as permission to persist arbitrary kubeconfigs. Other kubeconfigs can contain credentials, every copy can reveal cluster and tenant metadata, and certificate or endpoint changes can make a saved copy stale. More importantly, preserving the client file does not repair the separate upstream trust state described above. + +For incident automation, let the built-in Kubernetes tools generate current connection material for each call. If terminal-native kubectl is required for an exceptional ad-hoc task, treat its kubeconfig as short-lived configuration and rebuild it rather than storing it as durable agent knowledge. + +## Human and CI access paths + +The agent's built-in tools are not the only valid way to operate a private cluster. The right path depends on who is calling and whether the access is interactive or automated. + +| Caller | Recommended path | Why | +|---|---|---| +| SRE Agent incident skill | `RunKubectlReadCommand` / `RunKubectlWriteCommand` | Uses the runtime identity, private path, and temporary connection setup | +| Lab operator without private network connectivity | `az aks command invoke` through `Invoke-AksCommand` | Runs kubectl through the Azure API without exposing the private endpoint to the workstation | +| Developer or administrator | Workstation or jump host connected through VNet peering, VPN, ExpressRoute, or Bastion | Supports full interactive Kubernetes tooling over the private endpoint | +| CI/CD automation | Self-hosted runner on a connected VNet | Stable private DNS and routing for programmatic access | + +This repo uses `az aks command invoke` for setup and demo scripts because it keeps the workstation prerequisites small. Microsoft documents that path as an operator convenience, not a general programmatic-access transport: it depends on an in-cluster command pod, has a 60-second ARM scheduling timeout, and limits output to 512 KB. + +## API-server exposure options + +The following choices are easy to conflate: + +| Model | Network behavior | When it fits | +|---|---|---| +| **Private cluster, public FQDN disabled** - this lab | The API server has a private endpoint and private DNS only. Callers need connected private networking or an Azure-brokered command path. | Enterprise isolation, private automation, and demonstrations of VNet-injected operations | +| **Private cluster, public FQDN enabled** | AKS publishes a publicly resolvable DNS name, but API communication still goes to the private endpoint. It does **not** create a public API endpoint. | Connected clients that need public DNS resolution while retaining private network reachability | +| **Public API server with authorized IP ranges** | The API server has a public endpoint and rejects source addresses outside configured CIDRs. | Simpler external administration when a public control-plane endpoint is acceptable | + +API-server authorized IP ranges are not a way to make a private cluster selectively public. Microsoft documents that authorized IP ranges cannot be used with private clusters. + +The lab chose the first model because it demonstrates the customer pattern the sample is intended to teach: + +- the agent operates from a dedicated management spoke; +- the Kubernetes control plane is not exposed to the internet; +- private DNS, peering, identity, and authorization remain visible architectural concerns; +- human setup can still use `az aks command invoke`; +- production-style private runners, VPN, or ExpressRoute can replace command invoke without changing the cluster posture. + +## Troubleshooting by layer + +| Symptom | Most likely layer | Check | +|---|---|---| +| API FQDN does not resolve | Private DNS | VNet link, DNS forwarding, and whether the caller uses a connected network | +| TCP timeout or no route | Network | Peering, route tables, NSGs, firewall rules, VPN/ExpressRoute | +| `x509: certificate signed by unknown authority` | Client-side TLS trust | Kubeconfig CA data and any runtime intermediary CA | +| Reset or HTTP/2 error after client trust is correct | Intermediary upstream TLS trust | In the current preview runtime, run a built-in read against the same cluster | +| `kubelogin` or token error | Authentication | Login mode, managed-identity client ID, Entra tenant, token audience | +| HTTP 403 `Forbidden` | Authorization | AKS Azure RBAC assignment and its scope | +| `az aks get-credentials` denied | ARM credential retrieval | Cluster User/Admin credential role, which is separate from Kubernetes data-plane RBAC | + +Avoid changing several layers at once. First establish private DNS and routing, then TLS, then token acquisition, then RBAC. + +## References + +- [Create a private AKS cluster](https://learn.microsoft.com/azure/aks/private-clusters) +- [Access a private AKS cluster with command invoke](https://learn.microsoft.com/azure/aks/access-private-cluster) +- [Use Microsoft Entra ID authorization for the Kubernetes API](https://learn.microsoft.com/azure/aks/manage-azure-rbac) +- [Control access to AKS kubeconfig](https://learn.microsoft.com/azure/aks/control-kubeconfig-access) +- [API server authorized IP ranges](https://learn.microsoft.com/azure/aks/api-server-authorized-ip-ranges) +- [kubelogin documentation](https://azure.github.io/kubelogin/) diff --git a/labs/zava-aks-postgres/infra/main.bicep b/labs/zava-aks-postgres/infra/main.bicep index abac9faa1..fc533de14 100644 --- a/labs/zava-aks-postgres/infra/main.bicep +++ b/labs/zava-aks-postgres/infra/main.bicep @@ -23,6 +23,14 @@ connector is platform-brokered, so dropping the public `AzureMonitor` tag from t firewall does not gate it.) Set false to keep the public allow-listed Monitor path instead.''') param lockAgentToPrivateMonitor bool = true +@description('''Allow the agent to reach its OWN data-plane endpoint (`*.azuresre.ai`) through the hub +firewall, default true. Required for the agent to read or write the configuration surfaces ARM does not +expose — custom instructions, hooks, knowledge files, and global tool enablement. This is also a +self-modification path (the agent can rewrite its own skills and always-on prompts); set false to keep +data-plane config strictly operator/CI-applied. The firewall module remains deployed with an empty +rule collection when false so incremental deployments revoke previously granted access.''') +param allowAgentSelfManagement bool = true + // 4-char hash of the env name appended to the SRE Agent name. Empty when // environmentName is blank (e.g. raw `az deployment sub create`), preserving // the legacy `sre-agent-${uniqueSuffix}` shape for that path. @@ -97,6 +105,22 @@ module identity 'modules/identity.bicep' = { } } +// Cross-alert correlation queries the subscription-scoped Alerts Management and +// Resource Health event feeds so it can distinguish same-RG incidents from wider +// platform events. RG-scoped roles cannot read upward, so the runtime UMI needs +// Reader at subscription scope. Monitor query permissions remain RG-scoped. +// +// The principal ID is runtime-known. Pass it into a nested deployment so the +// assignment name can be keyed to the actual principal rather than the reusable +// identity name; that avoids RoleAssignmentUpdateNotPermitted after recreation. +module correlationSubscriptionReader 'modules/subscription-reader.bicep' = { + scope: subscription() + name: 'correlation-subscription-reader' + params: { + principalId: identity.outputs.sreAgentIdentityPrincipalId + } +} + module sreAgent 'modules/sre-agent.bicep' = { scope: rg name: 'sre-agent' @@ -117,6 +141,21 @@ module sreAgent 'modules/sre-agent.bicep' = { } } +// Agent data-plane firewall rule — pinned to the agent's exact hostname (not the +// broad *.azuresre.ai wildcard). Deployed AFTER the agent resource because the +// hostname is platform-assigned at agent creation time and cannot be computed in +// Bicep before it exists. See firewall-agent-dataplane.bicep header. +module firewallAgentDataPlane 'modules/firewall-agent-dataplane.bicep' = { + scope: rg + name: 'firewall-agent-dataplane' + params: { + firewallPolicyName: vnet.outputs.firewallPolicyName + agentEndpoint: sreAgent.outputs.agentEndpoint + agentSubnetPrefix: vnet.outputs.agentSubnetPrefix + enabled: allowAgentSelfManagement + } +} + // Azure Monitor Private Link Scope (AMPLS) — the private ingress/egress path for // Azure Monitor. The agent is locked to the private path by default // (lockAgentToPrivateMonitor); the platform/workload spoke stays on the public @@ -207,6 +246,7 @@ output PG_SERVER_NAME string = postgresql.outputs.serverName output APP_IDENTITY_NAME string = identity.outputs.appIdentityName output APP_IDENTITY_CLIENT_ID string = identity.outputs.appIdentityClientId output APP_IDENTITY_PRINCIPAL_ID string = identity.outputs.appIdentityPrincipalId +output SRE_AGENT_PRINCIPAL_ID string = identity.outputs.sreAgentIdentityPrincipalId output LOG_ANALYTICS_WORKSPACE_ID string = monitoring.outputs.logAnalyticsWorkspaceId // NOTE: do NOT mark this @secure(). `azd env get-value` (used by post-provision.ps1) // silently omits secure outputs and returns the literal "ERROR: key not found" text diff --git a/labs/zava-aks-postgres/infra/main.bicepparam b/labs/zava-aks-postgres/infra/main.bicepparam index afca45eb6..9411b14ac 100644 --- a/labs/zava-aks-postgres/infra/main.bicepparam +++ b/labs/zava-aks-postgres/infra/main.bicepparam @@ -1,10 +1,12 @@ using './main.bicep' +var azdEnvironmentName = readEnvironmentVariable('AZURE_ENV_NAME', 'zava-aks-postgres') + param location = 'swedencentral' -param resourceGroupName = readEnvironmentVariable('ZAVA_RG_NAME', 'rg-zava-aks-postgres') +param resourceGroupName = readEnvironmentVariable('ZAVA_RG_NAME', 'rg-${azdEnvironmentName}') // AZD sets AZURE_ENV_NAME automatically (e.g. 'zava-oneshot-1514'). Read it // here so the per-env SRE Agent suffix is derivable at deployment-plan time. // When deploying without azd (raw `az deployment sub create`), this falls back // to '' and the agent name keeps its legacy `sre-agent-${uniqueSuffix}` shape. -param environmentName = readEnvironmentVariable('AZURE_ENV_NAME', '') +param environmentName = azdEnvironmentName diff --git a/labs/zava-aks-postgres/infra/main.json b/labs/zava-aks-postgres/infra/main.json index 9ec08e531..17aa22325 100644 --- a/labs/zava-aks-postgres/infra/main.json +++ b/labs/zava-aks-postgres/infra/main.json @@ -5,7 +5,7 @@ "_generator": { "name": "bicep", "version": "0.43.1.21952", - "templateHash": "17454459993975253304" + "templateHash": "4794228390861712736" } }, "parameters": { @@ -43,6 +43,13 @@ "metadata": { "description": "Lock the SRE Agent down to PRIVATE-ONLY Azure Monitor (maximum restraint).\nWhen true (default): the agent\\'s Monitor private-DNS zones are linked to the agent VNet AND the\npublic `AzureMonitor` service tag is removed from the firewall L4 allow-list, so the agent reaches\nLog Analytics / Application Insights only via the AMPLS private endpoint over the hub/spoke.\n\nThe agent remains fully functional under this lockdown: it queries Log Analytics / Application\nInsights and remediates incidents end-to-end over the private path. (The agent\\'s Monitor query\nconnector is platform-brokered, so dropping the public `AzureMonitor` tag from the agent-VNet\nfirewall does not gate it.) Set false to keep the public allow-listed Monitor path instead." } + }, + "allowAgentSelfManagement": { + "type": "bool", + "defaultValue": true, + "metadata": { + "description": "Allow the agent to reach its OWN data-plane endpoint (`*.azuresre.ai`) through the hub\nfirewall, default true. Required for the agent to read or write the configuration surfaces ARM does not\nexpose — custom instructions, hooks, knowledge files, and global tool enablement. This is also a\nself-modification path (the agent can rewrite its own skills and always-on prompts); set false to keep\ndata-plane config strictly operator/CI-applied. The firewall module remains deployed with an empty\nrule collection when false so incremental deployments revoke previously granted access." + } } }, "variables": { @@ -83,7 +90,7 @@ "_generator": { "name": "bicep", "version": "0.43.1.21952", - "templateHash": "7676869149456753884" + "templateHash": "12420303598575479335" } }, "parameters": { @@ -103,7 +110,7 @@ "type": "bool", "defaultValue": true, "metadata": { - "description": "Lock the agent to PRIVATE-ONLY Azure Monitor (default true). When true, the public\n`AzureMonitor` service tag is dropped from the firewall L4 allow-list and the agent reaches Monitor\nvia the AMPLS private endpoint (10.10.2.0/27, rule allow-agent-to-ampls) + the linked private-DNS\nzones. The agent remains fully functional under this lockdown (Monitor queries, native kubectl, and\nincident remediation all work). See the main.bicep param doc." + "description": "Lock the agent to PRIVATE-ONLY Azure Monitor (default true). When true, the public\n`AzureMonitor` service tag is dropped from the firewall L4 allow-list and the agent reaches Monitor\nvia the AMPLS private endpoint (10.10.2.0/27, rule allow-agent-to-ampls) + the linked private-DNS\nzones. The agent remains fully functional under this lockdown (Monitor queries, Kubernetes tools, and\nincident remediation all work). See the main.bicep param doc." } } }, @@ -112,6 +119,7 @@ "platformVnetName": "[format('vnet-Zava-platform-{0}', parameters('uniqueSuffix'))]", "agentVnetName": "[format('vnet-Zava-agent-{0}', parameters('uniqueSuffix'))]", "nsgName": "[format('nsg-aks-{0}', parameters('uniqueSuffix'))]", + "agentSubnetPrefix": "10.30.0.0/27", "firewallPrivateIp": "10.10.0.4" }, "resources": [ @@ -252,7 +260,7 @@ { "name": "agent-subnet", "properties": { - "addressPrefix": "10.30.0.0/28", + "addressPrefix": "[variables('agentSubnetPrefix')]", "routeTable": { "id": "[resourceId('Microsoft.Network/routeTables', format('rt-agent-{0}', parameters('uniqueSuffix')))]" }, @@ -401,7 +409,7 @@ "TCP" ], "sourceAddresses": [ - "10.30.0.0/28" + "[variables('agentSubnetPrefix')]" ], "destinationAddresses": [ "168.63.129.16" @@ -423,12 +431,12 @@ { "ruleType": "NetworkRule", "name": "agent-to-apiserver", - "description": "Agent subnet -> AKS API server (enables native kubectl)", + "description": "Agent subnet -> AKS private API server", "ipProtocols": [ "TCP" ], "sourceAddresses": [ - "10.30.0.0/28" + "[variables('agentSubnetPrefix')]" ], "destinationAddresses": [ "10.20.0.0/20" @@ -455,7 +463,7 @@ "TCP" ], "sourceAddresses": [ - "10.30.0.0/28" + "[variables('agentSubnetPrefix')]" ], "destinationAddresses": [ "10.10.2.0/27" @@ -482,7 +490,7 @@ "TCP" ], "sourceAddresses": [ - "10.30.0.0/28" + "[variables('agentSubnetPrefix')]" ], "destinationAddresses": "[if(parameters('lockAgentToPrivateMonitor'), createArray('AzureResourceManager', 'AzureActiveDirectory'), createArray('AzureResourceManager', 'AzureActiveDirectory', 'AzureMonitor'))]", "destinationPorts": [ @@ -504,7 +512,7 @@ "name": "allow-arm-aad-graph", "description": "FQDN access to ARM, Entra ID, and Microsoft Graph", "sourceAddresses": [ - "10.30.0.0/28" + "[variables('agentSubnetPrefix')]" ], "protocols": [ { @@ -533,7 +541,7 @@ "name": "allow-learn-microsoft-com", "description": "Microsoft Learn docs + MCP runtime endpoint (the agent looks up Azure/AKS/PostgreSQL guidance here)", "sourceAddresses": [ - "10.30.0.0/28" + "[variables('agentSubnetPrefix')]" ], "protocols": [ { @@ -551,7 +559,7 @@ "name": "allow-github-raw-mcp-bits", "description": "GitHub raw content — the Microsoft Learn MCP connector fetches its server bits here to complete the tool-discovery handshake", "sourceAddresses": [ - "10.30.0.0/28" + "[variables('agentSubnetPrefix')]" ], "protocols": [ { @@ -649,6 +657,10 @@ "type": "string", "value": "[format('{0}/subnets/agent-subnet', resourceId('Microsoft.Network/virtualNetworks', variables('agentVnetName')))]" }, + "agentSubnetPrefix": { + "type": "string", + "value": "[variables('agentSubnetPrefix')]" + }, "privateDnsZoneId": { "type": "string", "value": "[resourceId('Microsoft.Network/privateDnsZones', format('{0}.private.postgres.database.azure.com', parameters('uniqueSuffix')))]" @@ -677,6 +689,10 @@ "type": "string", "value": "[format('afw-Zava-{0}', parameters('uniqueSuffix'))]" }, + "firewallPolicyName": { + "type": "string", + "value": "[format('afw-policy-Zava-{0}', parameters('uniqueSuffix'))]" + }, "firewallId": { "type": "string", "value": "[resourceId('Microsoft.Network/azureFirewalls', format('afw-Zava-{0}', parameters('uniqueSuffix')))]" @@ -802,7 +818,7 @@ "_generator": { "name": "bicep", "version": "0.43.1.21952", - "templateHash": "4090068017172206758" + "templateHash": "318792492186991763" } }, "parameters": { @@ -829,6 +845,13 @@ "metadata": { "description": "Resource group ID — scope for activity-log-based alerts." } + }, + "enableDbCpuSaturationAlert": { + "type": "bool", + "defaultValue": false, + "metadata": { + "description": "Enable the PostgreSQL CPU-saturation metric alert. Deployed DISABLED by default on purpose — see the resource comment: the disabled causal alert is what makes the correlation scenario teach anything." + } } }, "variables": { @@ -1050,6 +1073,45 @@ "[resourceId('Microsoft.OperationalInsights/workspaces', variables('lawName'))]" ] }, + { + "type": "Microsoft.Insights/metricAlerts", + "apiVersion": "2018-03-01", + "name": "Zava-db-cpu-saturation", + "location": "global", + "properties": { + "severity": 3, + "enabled": "[parameters('enableDbCpuSaturationAlert')]", + "evaluationFrequency": "PT1M", + "windowSize": "PT5M", + "scopes": [ + "[resourceId('Microsoft.DBforPostgreSQL/flexibleServers', parameters('postgresServerName'))]" + ], + "criteria": { + "odata.type": "Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria", + "allOf": [ + { + "criterionType": "StaticThresholdCriterion", + "name": "pgCpu", + "metricName": "cpu_percent", + "metricNamespace": "Microsoft.DBforPostgreSQL/flexibleServers", + "operator": "GreaterThan", + "threshold": 80, + "timeAggregation": "Average" + } + ] + }, + "actions": [ + { + "actionGroupId": "[resourceId('Microsoft.Insights/actionGroups', format('ag-Zava-sre-{0}', parameters('uniqueSuffix')))]" + } + ], + "autoMitigate": true, + "description": "Zava Demo: PostgreSQL server CPU averaged above 80% over 5 minutes." + }, + "dependsOn": [ + "[resourceId('Microsoft.Insights/actionGroups', format('ag-Zava-sre-{0}', parameters('uniqueSuffix')))]" + ] + }, { "type": "Microsoft.Insights/activityLogAlerts", "apiVersion": "2023-01-01-preview", @@ -1684,6 +1746,60 @@ "[subscriptionResourceId('Microsoft.Resources/resourceGroups', parameters('resourceGroupName'))]" ] }, + { + "type": "Microsoft.Resources/deployments", + "apiVersion": "2025-04-01", + "name": "correlation-subscription-reader", + "location": "[deployment().location]", + "properties": { + "expressionEvaluationOptions": { + "scope": "inner" + }, + "mode": "Incremental", + "parameters": { + "principalId": { + "value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, parameters('resourceGroupName')), 'Microsoft.Resources/deployments', 'identity'), '2025-04-01').outputs.sreAgentIdentityPrincipalId.value]" + } + }, + "template": { + "$schema": "https://schema.management.azure.com/schemas/2018-05-01/subscriptionDeploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "metadata": { + "_generator": { + "name": "bicep", + "version": "0.43.1.21952", + "templateHash": "17563515106843481408" + } + }, + "parameters": { + "principalId": { + "type": "string", + "metadata": { + "description": "Principal object ID that receives subscription Reader." + } + } + }, + "variables": { + "readerRoleId": "acdd72a7-3385-48ef-bd42-f606fba81ae7" + }, + "resources": [ + { + "type": "Microsoft.Authorization/roleAssignments", + "apiVersion": "2022-04-01", + "name": "[guid(subscription().id, parameters('principalId'), variables('readerRoleId'))]", + "properties": { + "roleDefinitionId": "[subscriptionResourceId('Microsoft.Authorization/roleDefinitions', variables('readerRoleId'))]", + "principalId": "[parameters('principalId')]", + "principalType": "ServicePrincipal" + } + } + ] + } + }, + "dependsOn": [ + "[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, parameters('resourceGroupName')), 'Microsoft.Resources/deployments', 'identity')]" + ] + }, { "type": "Microsoft.Resources/deployments", "apiVersion": "2025-04-01", @@ -1733,7 +1849,7 @@ "_generator": { "name": "bicep", "version": "0.43.1.21952", - "templateHash": "5106647395530293630" + "templateHash": "9718037598915593578" } }, "parameters": { @@ -1832,46 +1948,49 @@ "aiResourceName": "[last(split(parameters('appInsightsId'), '/'))]", "lawResourceName": "[last(split(parameters('logAnalyticsId'), '/'))]", "rgName": "[resourceGroup().name]", - "sharedContext": "Resource Group `@@RG@@`. App namespace `zava-demo`. Deployments `zava-api` / `zava-storefront`. App Insights cloud_RoleName `zava-api`.\r\n\r\nYou operate with your own managed identity (Entra) — AKS RBAC Cluster Admin, Reader + Monitoring Reader + Contributor on the resource group, and PostgreSQL Entra admin. These are sufficient: do NOT attempt `az role assignment create` (it is denied — if you think you need a role you lack, your diagnosis is wrong, back up). Your sandbox egress is forced through an Azure Firewall (allow-list: ARM, Entra, Microsoft Graph, Microsoft Learn over public service tags, plus the AKS API server over the hub/spoke; Azure Monitor is reached privately via the AMPLS private endpoint by default) AND a TLS-inspecting forward proxy that re-signs certificates. This cluster is wired for native `kubectl` (the agent VNet has the AKS private-DNS zone linked and a firewall rule + SNAT to the API server): you run `kubectl` yourself as a bash command in your sandbox terminal (`RunInTerminal`). One-time setup per session: (1) `az aks get-credentials -g @@RG@@ -n --overwrite-existing` (find the cluster via `az aks list -g @@RG@@ --query \"[0].name\" -o tsv`); (2) `kubelogin convert-kubeconfig -l azurecli` — non-interactive managed-identity auth (the DEFAULT device-code flow hangs; do not use it); (3) trust the egress proxy by merging its CA `/etc/ssl/certs/adc-egress-proxy-ca.crt` into the kubeconfig cluster's `certificate-authority-data`. Then `kubectl get nodes` works; run kubectl in your terminal for pods, logs, events, NetworkPolicies, rollouts, and the in-cluster SQL helper `kubectl exec -n zava-demo deploy/zava-api -- node bin/run-sql.js ''`. Never install DB clients (`psql`, `psycopg2`) or open a raw socket to PostgreSQL. Reach ARM over the control plane; reach Azure Monitor (Log Analytics / Application Insights) with your Monitor query tools — they work normally (this deployment locks the agent's Monitor access to the AMPLS private endpoint by default, and your tools operate fine over it). Filter every App Insights / Log Analytics query by `AppRoleName == 'zava-api'` — the workspace is shared with your own ARM-poll telemetry.", + "sharedContext": "Resource Group `@@RG@@`. App namespace `zava-demo`. Deployments `zava-api` / `zava-storefront`. App Insights cloud_RoleName `zava-api`.\r\n\r\nYou operate with your own managed identity (Entra) — AKS RBAC Cluster Admin, Reader + Monitoring Reader + Contributor on the resource group, Reader at subscription scope for cross-alert and Service Health context, and PostgreSQL Entra admin. These are sufficient: do NOT attempt `az role assignment create` (it is denied — if you think you need a role you lack, your diagnosis is wrong, back up). Use the built-in `RunKubectlReadCommand` and `RunKubectlWriteCommand` system tools for Kubernetes; they accept the same kubectl commands as a terminal. Use the read tool for inspection and the write tool for `delete`, `rollout`, and `exec` operations. Do not replace them with terminal-native kubectl, login repair, kubeconfig setup, or Python wrappers. Run PostgreSQL SQL through the in-cluster helper with the write tool: `kubectl exec -n zava-demo deploy/zava-api -- node bin/run-sql.js ''`. Never install DB clients (`psql`, `psycopg2`) or open a raw socket to PostgreSQL. Reach ARM over the control plane; reach Azure Monitor (Log Analytics / Application Insights) with your Monitor query tools — they work normally (this deployment locks the agent's Monitor access to the AMPLS private endpoint by default, and your tools operate fine over it). Filter every App Insights / Log Analytics query by `AppRoleName == 'zava-api'` — the workspace is shared with your own ARM-poll telemetry.", "databaseSkill": { "description": "Use for Zava PostgreSQL AVAILABILITY incidents — alert `postgres-unreachable` (zava-api cannot reach PostgreSQL; connection refused or, more often, timeout). Diagnose the cause from ARM state — stopped server vs network partition — and remediate: restart the server, or remove the in-cluster Kubernetes NetworkPolicy / matching NSG deny rule that blocks PG egress.", "tools": [ "RunAzCliReadCommands", "RunAzCliWriteCommands", - "RunInTerminal", + "RunKubectlReadCommand", + "RunKubectlWriteCommand", "SearchMemory", - "microsoft-learn_microsoft_docs_search", - "microsoft-learn_microsoft_docs_fetch" + "learn-docs_microsoft_docs_search", + "learn-docs_microsoft_docs_fetch" ], - "skillContent": "## Database availability runbook (Zava)\r\n\r\n@@SHARED@@\r\n\r\nYou diagnose from telemetry, then remediate within the permitted-action boundary; outside it, summarize and stop.\r\n\r\nThe alert `postgres-unreachable` means zava-api cannot reach PostgreSQL — it logged connection failures (refused or, far more often, **timeouts**). A stopped server and a network block BOTH look like timeouts at the app, so **diagnose the cause from ARM state, not the error text**:\r\n\r\n| PG ARM `state` | Cause | Action |\r\n|---|---|---|\r\n| `Stopped` | The server was stopped. | **Start it**: `az postgres flexible-server start`. |\r\n| `Ready` (app still can't connect) | A network block. | Two enforcement surfaces sit between the app and PG: an NSG deny rule on the AKS subnet (often a RED HERRING — PG's private access uses a platform-managed delegated subnet) and a Kubernetes **NetworkPolicy** in `zava-demo` (usually the real cause). Inspect both — `az network nsg rule list` and `kubectl get networkpolicy -A -o yaml` (run in your terminal) — then delete the offending NetworkPolicy with `kubectl delete networkpolicy -n zava-demo` (and any matching NSG deny rule on the AKS subnet). |\r\n\r\n## Permitted autonomous actions\r\n- Start / restart / parameter-set on PostgreSQL Flexible Server.\r\n- Delete a NetworkPolicy in `zava-demo` whose egress blocks PG, and delete a matching NSG deny rule on the AKS subnet.\r\n\r\n## Out of scope (summarize + stop)\r\n- `DROP`, DML, schema migrations, role/grant changes; cluster scale / node deletion / VNet changes; any IAM modification.\r\n\r\n## Verify\r\nPG `state == Ready`; zava-api connection-error traces stop.\r\n\r\n## Close the loop (resolve the alert)\r\nAfter confirming recovery, **resolve the `postgres-unreachable` alert you were handling** instead of waiting for Azure Monitor's auto-mitigate. Auto-mitigate lags ~15-30 min, and while the alert lingers in a fired state Azure Monitor dedupes the NEXT distinct database incident into this same alert instance — so no new investigation dispatches until it clears. Closing it yourself keeps the loop tight. Take the alert's ARM id from your incident context (form `/subscriptions/.../providers/Microsoft.AlertsManagement/alerts/`); if you don't have it, list open ones with `az rest --method GET --url \"https://management.azure.com/subscriptions//providers/Microsoft.AlertsManagement/alerts?api-version=2018-05-05&alertRule=postgres-unreachable\"`. Then close it:\r\n`az rest --method POST --url \"https://management.azure.com/changestate?api-version=2018-05-05&newState=Closed\"`\r\n(your Contributor role grants `Microsoft.AlertsManagement/alerts/changestate/action`).\r\n", + "skillContent": "## Database availability runbook (Zava)\r\n\r\n@@SHARED@@\r\n\r\nYou diagnose from telemetry, then remediate within the permitted-action boundary; outside it, summarize and stop.\r\n\r\nThe alert `postgres-unreachable` means zava-api cannot reach PostgreSQL — it logged connection failures (refused or, far more often, **timeouts**). A stopped server and a network block BOTH look like timeouts at the app, so **diagnose the cause from ARM state, not the error text**:\r\n\r\n| PG ARM `state` | Cause | Action |\r\n|---|---|---|\r\n| `Stopped` | The server was stopped. | **Start it**: `az postgres flexible-server start`. |\r\n| `Ready` (app still can't connect) | A network block. | Two enforcement surfaces sit between the app and PG: an NSG deny rule on the AKS subnet (often a RED HERRING — PG's private access uses a platform-managed delegated subnet) and a Kubernetes **NetworkPolicy** in `zava-demo` (usually the real cause). Inspect both with `az network nsg rule list` and `RunKubectlReadCommand`, then delete the offending NetworkPolicy with `RunKubectlWriteCommand` (and any matching NSG deny rule on the AKS subnet). |\r\n\r\n## Permitted autonomous actions\r\n- Start / restart / parameter-set on PostgreSQL Flexible Server.\r\n- Delete a NetworkPolicy in `zava-demo` whose egress blocks PG, and delete a matching NSG deny rule on the AKS subnet.\r\n\r\n## Out of scope (summarize + stop)\r\n- `DROP`, DML, schema migrations, role/grant changes; cluster scale / node deletion / VNet changes; any IAM modification.\r\n\r\n## Verify\r\nPG `state == Ready`; zava-api connection-error traces stop.\r\n\r\n## Close the loop (resolve the alert)\r\nAfter confirming recovery, **resolve the `postgres-unreachable` alert you were handling** instead of waiting for Azure Monitor's auto-mitigate. Auto-mitigate lags ~15-30 min, and while the alert lingers in a fired state Azure Monitor dedupes the NEXT distinct database incident into this same alert instance — so no new investigation dispatches until it clears. Closing it yourself keeps the loop tight. Take the alert's ARM id from your incident context (form `/subscriptions/.../providers/Microsoft.AlertsManagement/alerts/`); if you don't have it, list open ones with `az rest --method GET --url \"https://management.azure.com/subscriptions//providers/Microsoft.AlertsManagement/alerts?api-version=2018-05-05&alertRule=postgres-unreachable\"`. Then close it:\r\n`az rest --method POST --url \"https://management.azure.com/changestate?api-version=2018-05-05&newState=Closed\"`\r\n(your Contributor role grants `Microsoft.AlertsManagement/alerts/changestate/action`).\r\n", "additionalFiles": [], "sourcePluginInstallation": null }, "performanceSkill": { - "description": "Use for Zava query-LATENCY / slow-endpoint incidents — alert `Zava-products-query-slow` (a /api/products/category endpoint breached its latency threshold). The bottleneck is at PostgreSQL (missing/disabled index, plan regression), not pods/CPU. Corroborate with the custom latency metric + PG CPU, then apply read-mostly DDL (CREATE INDEX) via the in-cluster SQL helper.", + "description": "Use for Zava query-LATENCY / slow-endpoint incidents — alert `Zava-products-query-slow` (a /api/products/category endpoint breached its latency threshold). Diagnose the PostgreSQL query path, check nearby alerts with `incident-correlation`, and do not treat a co-firing 5xx as the same cause without a direct dependency-failure mechanism. Apply read-mostly DDL (CREATE INDEX) via the in-cluster SQL helper when the query plan proves it is needed.", "tools": [ "RunAzCliReadCommands", "RunAzCliWriteCommands", - "RunInTerminal", + "RunKubectlReadCommand", + "RunKubectlWriteCommand", "SearchMemory", - "microsoft-learn_microsoft_docs_search", - "microsoft-learn_microsoft_docs_fetch" + "learn-docs_microsoft_docs_search", + "learn-docs_microsoft_docs_fetch" ], - "skillContent": "## Query-performance runbook (Zava)\r\n\r\n@@SHARED@@\r\n\r\n`Zava-products-query-slow` fires when a `/api/products/category/` endpoint averages above its latency threshold (healthy baseline ~3 ms). The bottleneck is almost always at the DATABASE (missing/disabled index, plan regression, statistics drift), NOT pods/CPU/memory — never restart pods or scale the cluster for this alert.\r\n\r\n## Corroborate across logs + metrics + traces (REQUIRED — these are paired with the alert, not separate alerts)\r\n1. **Log** (the alert): `AppRequests | where AppRoleName == 'zava-api' | where Name startswith 'GET /api/products/category/' and Name !contains '__probe' | summarize avg(DurationMs) by Name`.\r\n2. **Custom metric**: `AppMetrics | where Name == 'zava.products.category.query.duration_ms' | extend Category = tostring(Properties['category']) | where Category != '__probe' | summarize sum(Sum)/sum(ItemCount) by Category`.\r\n3. **PG saturation metric**: `AzureMetrics` for `cpu_percent` on the PG server (heavy seq scans drive CPU up).\r\n4. **Trace**: `AppDependencies` PostgreSQL-call latency.\r\nAgreement across all four points at the database query, not the app tier.\r\n\r\n## Diagnose at PostgreSQL (in-cluster SQL helper)\r\n`kubectl exec -n zava-demo deploy/zava-api -- node bin/run-sql.js ''` — run kubectl in your terminal, set up per shared context. Inspect `pg_stat_user_indexes` (low/zero `idx_scan` on a hot table is a strong signal), `pg_stat_user_tables` (high `seq_scan`), `pg_stat_statements` (top mean-time), and `EXPLAIN`.\r\n\r\n## Permitted autonomous actions\r\n- Read-mostly DDL on PostgreSQL via the in-cluster helper: `CREATE INDEX CONCURRENTLY IF NOT EXISTS`, `ANALYZE`, `REINDEX CONCURRENTLY`.\r\n\r\n## Out of scope (summarize + stop)\r\n- `DROP`, DML, schema migrations; pod restarts / cluster scale for this alert; any IAM modification.\r\n\r\n## Verify\r\nThe category endpoint's avg latency returns to baseline; `idx_scan` climbs on the new index; the alert auto-mitigates.\r\n", + "skillContent": "## Query-performance runbook (Zava)\r\n\r\n@@SHARED@@\r\n\r\n`Zava-products-query-slow` fires when a `/api/products/category/` endpoint averages above its latency threshold (healthy baseline ~3 ms). The bottleneck is almost always at the DATABASE (missing/disabled index, plan regression, statistics drift), NOT pods/CPU/memory — never restart pods or scale the cluster for this alert.\r\n\r\n## Corroborate across logs + metrics + traces (REQUIRED — these are paired with the alert, not separate alerts)\r\n1. **Log** (the alert): `AppRequests | where AppRoleName == 'zava-api' | where Name startswith 'GET /api/products/category/' and Name !contains '__probe' | summarize avg(DurationMs) by Name`.\r\n2. **Custom metric**: `AppMetrics | where Name == 'zava.products.category.query.duration_ms' | extend Category = tostring(Properties['category']) | where Category != '__probe' | summarize sum(Sum)/sum(ItemCount) by Category`.\r\n3. **PG saturation metric**: `AzureMetrics` for `cpu_percent` on the PG server (heavy seq scans drive CPU up).\r\n4. **Trace**: `AppDependencies` PostgreSQL-call latency.\r\nAgreement across all four points at the database query, not the app tier.\r\n\r\n## Cross-alert guard\r\nLoad `incident-correlation` to check fired-alert history before assigning a shared root cause; alert-rule inventory cannot tell you what fired. A co-firing 5xx is NOT corroboration for a slow-query diagnosis. Split `AppDependencies` by target and result code: slow but successful PostgreSQL calls establish this latency fault, but they cannot explain HTTP 500s whose failed dependencies are only app-local. Different mechanisms mean independent incidents, even when their alert times and resource group match. If the other alert is already acknowledged, report the relationship but leave its remediation to that thread.\r\n\r\n## Diagnose at PostgreSQL (in-cluster SQL helper)\r\nUse `RunKubectlWriteCommand` to execute `kubectl exec -n zava-demo deploy/zava-api -- node bin/run-sql.js ''`. Inspect `pg_stat_user_indexes` (low/zero `idx_scan` on a hot table is a strong signal), `pg_stat_user_tables` (high `seq_scan`), `pg_stat_statements` (top mean-time), and `EXPLAIN`.\r\n\r\n## Permitted autonomous actions\r\n- Read-mostly DDL on PostgreSQL via the in-cluster helper: `CREATE INDEX CONCURRENTLY IF NOT EXISTS`, `ANALYZE`, `REINDEX CONCURRENTLY`.\r\n\r\n## Out of scope (summarize + stop)\r\n- `DROP`, DML, schema migrations; pod restarts / cluster scale for this alert; any IAM modification.\r\n\r\n## Verify\r\nThe category endpoint's avg latency returns to baseline; `idx_scan` climbs on the new index; the alert auto-mitigates.\r\n", "additionalFiles": [], "sourcePluginInstallation": null }, "applicationSkill": { - "description": "Use for Zava APPLICATION-layer HTTP 5xx incidents — alert `Zava-http-5xx-errors` (zava-api returning HTTP 5xx). This is typically an app/route regression such as a bad deploy, but a DB outage also produces 5xx, so FIRST rule out DB/perf; if PG is healthy, correlate the 5xx onset with a recent rollout and roll back to the previous good revision.", + "description": "Use for Zava APPLICATION-layer HTTP 5xx incidents — alert `Zava-http-5xx-errors` (zava-api returning HTTP 5xx). Rule out a direct DB failure path, check nearby alerts with `incident-correlation`, and correlate the 5xx onset with a recent rollout. Do not attribute it to a co-firing latency alert unless dependency failures prove that mechanism.", "tools": [ "RunAzCliReadCommands", "RunAzCliWriteCommands", - "RunInTerminal", + "RunKubectlReadCommand", + "RunKubectlWriteCommand", "SearchMemory", - "microsoft-learn_microsoft_docs_search", - "microsoft-learn_microsoft_docs_fetch" + "learn-docs_microsoft_docs_search", + "learn-docs_microsoft_docs_fetch" ], - "skillContent": "## Application 5xx runbook (Zava)\r\n\r\n@@SHARED@@\r\n\r\n`Zava-http-5xx-errors` fires when zava-api returns >5 HTTP 5xx in 5 min. It does NOT self-suppress on DB errors, so a DB outage (which also returns 5xx) can fire this alert too — therefore your FIRST step is to rule out a DB/perf root cause. If PostgreSQL is healthy and there is no slow-query symptom, this is an APP-layer regression.\r\n\r\n## Investigate\r\n1. Briefly confirm it is not DB/perf after all: PG `state == Ready`, no ECONNREFUSED/ETIMEDOUT traces, `/api/products` latency normal. If a DB or slow-query symptom is actually present, defer to the database / performance runbook.\r\n2. App regressions are usually shipped by a deploy. Every change to the `zava-api` Deployment pod template creates a new ReplicaSet **revision**. Check whether the 5xx onset lines up with a recent rollout: native `kubectl rollout history deployment/zava-api -n zava-demo` and `KubeEvents` (Azure Monitor) (`ScalingReplicaSet` timestamps). Note the liveness AND readiness probes both hit `/livez` (shallow, no DB call), so pods stay Ready through an app-route regression and the platform looks healthy while the app is broken; `/api/health` is a separate app health endpoint (it pings the DB) and can also stay green for a route-only regression — deployment correlation is the tie.\r\n\r\n## Permitted autonomous actions\r\n- Roll back a `zava-demo` deployment to its previous revision (native `kubectl rollout undo deployment/zava-api -n zava-demo`) when a 5xx regression correlates with a recent rollout.\r\n- Restart deployments in `zava-demo`.\r\n\r\n## Out of scope (summarize + stop)\r\n- Schema/role/IAM changes; cluster scale / node deletion / VNet changes.\r\n\r\n## Verify\r\n`GET /api/products` returns 200; 5xx rate returns to baseline; the alert auto-mitigates.\r\n", + "skillContent": "## Application 5xx runbook (Zava)\r\n\r\n@@SHARED@@\r\n\r\n`Zava-http-5xx-errors` fires when zava-api returns >5 HTTP 5xx in 5 min. It does NOT self-suppress on DB errors, so a DB outage (which also returns 5xx) can fire this alert too — therefore your FIRST step is to rule out a DB/perf root cause. If PostgreSQL is healthy and there is no slow-query symptom, this is an APP-layer regression.\r\n\r\n## Investigate\r\n1. Briefly confirm it is not DB/perf after all: PG `state == Ready`, no ECONNREFUSED/ETIMEDOUT traces, `/api/products` latency normal. If a DB or slow-query symptom is actually present, defer to the database / performance runbook.\r\n2. App regressions are usually shipped by a deploy. Every change to the `zava-api` Deployment pod template creates a new ReplicaSet **revision**. Check whether the 5xx onset lines up with a recent rollout using `RunKubectlReadCommand` for `kubectl rollout history deployment/zava-api -n zava-demo` and `KubeEvents` (Azure Monitor) (`ScalingReplicaSet` timestamps). Note the liveness AND readiness probes both hit `/livez` (shallow, no DB call), so pods stay Ready through an app-route regression and the platform looks healthy while the app is broken; `/api/health` is a separate app health endpoint (it pings the DB) and can also stay green for a route-only regression — deployment correlation is the tie.\r\n\r\nLoad `incident-correlation` to check fired-alert history; alert-rule inventory cannot tell you what fired. Shared timing is not a mechanism: split `AppDependencies` by target and result code before claiming the other alert caused this one. A slow-query alert with successful PostgreSQL dependencies does not explain HTTP 500s whose failed dependency is app-local. If the other alert is already acknowledged, report the relationship but leave its remediation to that thread.\r\n\r\n## Permitted autonomous actions\r\n- Roll back a `zava-demo` deployment to its previous revision with `RunKubectlWriteCommand` (`kubectl rollout undo deployment/zava-api -n zava-demo`) when a 5xx regression correlates with a recent rollout.\r\n- Restart deployments in `zava-demo`.\r\n\r\n## Out of scope (summarize + stop)\r\n- Schema/role/IAM changes; cluster scale / node deletion / VNet changes.\r\n\r\n## Verify\r\n`GET /api/products` returns 200; 5xx rate returns to baseline; the alert auto-mitigates.\r\n", "additionalFiles": [], "sourcePluginInstallation": null }, @@ -1880,8 +1999,8 @@ "tools": [ "RunAzCliReadCommands", "SearchMemory", - "microsoft-learn_microsoft_docs_search", - "microsoft-learn_microsoft_docs_fetch" + "learn-docs_microsoft_docs_search", + "learn-docs_microsoft_docs_fetch" ], "skillContent": "## General triage runbook (Zava — unknown incidents)\r\n\r\n@@SHARED@@\r\n\r\nThis is the catch-all for incidents that do NOT match a known scenario (PostgreSQL availability, query performance, or application 5xx). You run in REVIEW mode: investigate thoroughly and PROPOSE actions for human approval — do not autonomously change resources beyond read-only/safe inspection.\r\n\r\n## Approach (first principles)\r\n1. Parse the alert: which rule fired, severity, the impacted Azure resource (`alertTargetIDs` / scope) and the symptom in the description.\r\n2. Establish blast radius and a baseline: is the app serving traffic (`AppRequests` success rate for `AppRoleName == 'zava-api'`), is PostgreSQL `Ready`, are pods healthy (via `KubeEvents` in Azure Monitor — this skill is read-only, so use telemetry rather than `kubectl`)?\r\n3. Gather the relevant telemetry for the impacted resource (Azure Monitor metrics/logs, `KubeEvents`, recent `az monitor activity-log` changes, the hub firewall `AZFW*` logs if egress-related).\r\n4. Form 1–3 ranked hypotheses with the evidence for each.\r\n5. Propose a concrete, least-privilege remediation and the verification step — then stop for approval. If it maps to a known scenario after all, recommend the matching skill.\r\n\r\n## Boundaries\r\nRead-only investigation is always allowed. Any mutating action requires approval (Review mode). Never `az role assignment create`. Never `DROP` / DML / schema / IAM changes.\r\n", "additionalFiles": [], @@ -1893,14 +2012,24 @@ "RunAzCliReadCommands", "SearchMemory", "ExecutePythonCode", - "microsoft-learn_microsoft_code_sample_search", - "microsoft-learn_microsoft_docs_fetch", - "microsoft-learn_microsoft_docs_search" + "learn-docs_microsoft_code_sample_search", + "learn-docs_microsoft_docs_fetch", + "learn-docs_microsoft_docs_search" ], "skillContent": "## Proactive Health Check\r\n\r\nPull current signals; complete silently if everything is in baseline.\r\n\r\nAlways filter App Insights queries by `AppRoleName == 'zava-api'` — the workspace is shared with SRE Agent's own ARM polling, which dominates unfiltered queries.\r\n\r\nWhat \"baseline\" means for Zava:\r\n\r\n1. Request success rate >99% on `/api/*` over the last 15 minutes; single-digit ms avg/p95 on `/api/products*`.\r\n2. Zero `ECONNREFUSED` / `ETIMEDOUT` / \"timeout exceeded when trying to connect\" exceptions or traces from `zava-api` in the last 15 minutes.\r\n3. PostgreSQL Flexible Server `state == Ready`.\r\n\r\nIf any of those is missed, hand off to the matching domain skill: `database-incidents` (connectivity), `performance-incidents` (latency), or `application-incidents` (5xx). If everything is in baseline, complete silently.\r\n", "additionalFiles": [], "sourcePluginInstallation": null }, + "correlationSkill": { + "description": "Use during a Zava incident when the dispatched alert may be only part of the story: another alert fired nearby, the evidence does not add up, remediation did not hold, or a symptom appears to precede its cause. Enumerates other Azure Monitor alerts, disabled alert rules that may hide the causal signal, and Azure Service Health, then distinguishes one causal chain from independent faults that merely overlapped.", + "tools": [ + "RunAzCliReadCommands", + "SearchMemory" + ], + "skillContent": "## Cross-alert correlation runbook (Zava)\r\n\r\nResource Group `@@RG@@`.\r\n\r\nYou were dispatched on ONE alert. That alert is a filter someone wrote in advance, on one signal, with one threshold — it is evidence, not a conclusion, and it cannot tell you whether it is the cause, a symptom, or a coincidence. Every response plan in this deployment has merge DISABLED, so a single root cause opens several INDEPENDENT threads that cannot see each other. Nobody assembles the forest for you. Pull it.\r\n\r\n## 1. What else fired? (the forest)\r\n\r\n`az graph query` is usually unavailable (resource-graph extension absent). Use the Alerts Management REST API:\r\n\r\n`az rest --method get --url \"https://management.azure.com/subscriptions//providers/Microsoft.AlertsManagement/alerts?api-version=2019-05-05-preview&timeRange=1d&pageCount=250\" --query \"value[].{ruleId:properties.essentials.alertRule, rg:properties.essentials.targetResourceGroup, sev:properties.essentials.severity, cond:properties.essentials.monitorCondition, start:properties.essentials.startDateTime, target:properties.essentials.targetResource}\" -o json`\r\n\r\n- `pageCount` MUST be 1..250 (larger returns BadRequest). `timeRange` accepts 1h/1d/7d/30d only.\r\n- `RunAzCliReadCommands` rejects shell pipes and `&&` — issue one command per call.\r\n- `alertRule` is already the full rule resource ID in this API; the projection names it `ruleId`.\r\n- For LOG alerts `targetResource` is the Log Analytics WORKSPACE, not the app or DB. Use `rg` / `ruleId` to identify the environment; do not group by `targetResource`.\r\n- This fired-alert feed and the rule inventory in step 2 answer different questions. Never conclude \"nothing else fired\" from `scheduledQueryRules` or `az monitor metrics alert list`.\r\n\r\n## 2. Which alerts SHOULD have fired but did not? (the silent cause)\r\n\r\nA missing alert is a finding. Enumerate the rule INVENTORY, not just fired alerts:\r\n\r\n`az monitor metrics alert list -g @@RG@@ --query \"[].{name:name, enabled:enabled, scopes:scopes}\" -o json`\r\n`az rest --method get --url \"https://management.azure.com/subscriptions//resourceGroups/@@RG@@/providers/microsoft.insights/scheduledQueryRules?api-version=2023-03-15-preview\" --query \"value[].{name:name, enabled:properties.enabled, window:properties.windowSize, freq:properties.evaluationFrequency}\" -o json`\r\n\r\nIf a rule sits on the resource you are investigating and is `enabled: false`, the causal signal is MUTED — query that metric directly rather than concluding the resource was healthy. This deployment ships `Zava-db-cpu-saturation` disabled on purpose; PG CPU can be pegged at 90% with no DB alert anywhere.\r\n\r\n## 3. Is the platform itself the cause?\r\n\r\nAzure Service Health, subscription-scoped — covers regional outages and planned maintenance:\r\n\r\n`az rest --method get --url \"https://management.azure.com/subscriptions//providers/Microsoft.ResourceHealth/events?api-version=2022-10-01&queryStartTime=\" --query \"value[].{type:properties.eventType, level:properties.eventLevel, status:properties.status, title:properties.title, start:properties.impactStartTime}\" -o json`\r\n\r\n`eventType` is `ServiceIssue` (outage), `PlannedMaintenance`, or `HealthAdvisory`. Per-resource view:\r\n`az rest --method get --url \"https://management.azure.com/subscriptions//resourceGroups/@@RG@@/providers/Microsoft.ResourceHealth/availabilityStatuses?api-version=2023-07-01-preview\" --query \"value[].{res:id, avail:properties.availabilityState, summary:properties.summary}\" -o json`\r\n\r\nCheck this BEFORE concluding \"platform health event\" from absence of evidence. A `PlannedMaintenance` event naming PostgreSQL turns a guess into a fact, and it changes the remediation: you wait and document instead of chasing config.\r\n\r\n## 4. TWO RULES — violate these and correlation makes you WORSE, not better\r\n\r\nFilter every App Insights query by `AppRoleName == 'zava-api'`; the workspace also contains the agent's own ARM-poll telemetry.\r\n\r\n**Alert fire order is NOT causal order.** Every dispatching rule here is `PT5M` window / `PT5M` evaluation, so detection latency is up to 5 min plus ingestion lag. A symptom alert can fire BEFORE the cause alert. Any gap under ~7 minutes proves nothing about ordering. Establish onset from raw telemetry in 1-2 minute buckets, never from alert timestamps.\r\n\r\n**Co-firing is NOT causation.** Two alerts seconds apart can be two unrelated faults. Before claiming a causal chain, confirm the mechanism in telemetry:\r\n\r\n| Observation | Reading |\r\n|---|---|\r\n| HTTP 500, failed dependencies ONLY on `localhost:3001`, zero PG dependency failures | app-layer regression |\r\n| HTTP 503, failed dependencies against the PG target | DB unreachable |\r\n| No dependency FAILURES but PG `cpu_percent` high and latency up | DB saturation — slow but SUCCESSFUL queries, so failure-based signals stay clean |\r\n\r\nThe single fastest discriminator is one `dependencies` query split by `target` alongside `resultCode`. If two co-firing alerts have different mechanisms, they are independent — report them as separate incidents and do not merge the narrative.\r\n\r\n**Environment containment:** alerts from a DIFFERENT resource group are a different Zava stack. Same-RG co-firing only identifies the candidate environment; it does NOT suggest a shared cause. Cross-RG simultaneity may justify checking for a platform event (step 3), but still proves nothing by itself. Never merge findings across resource groups without that check.\r\n\r\n## 5. Report\r\n\r\nState plainly which it was: (a) one cause, several alerts; (b) several independent causes that overlapped; or (c) this alert is the whole story. If (c), say so in one line and move on — a correlation sweep that finds nothing is a successful sweep, not wasted work.\r\n\r\nIf an independent alert is already `Acknowledged`, its own thread is active. Attach the correlation finding to your report, but do not execute that other domain's remediation from this thread; concurrent duplicate writes can conflict.\r\n\r\n## Boundaries\r\nRead-only. This skill never remediates — hand off to `database-incidents`, `performance-incidents`, or `application-incidents` with the correlation context attached.\r\n", + "additionalFiles": [], + "sourcePluginInstallation": null + }, "defaultPriorities": [ "Sev0", "Sev1", @@ -1919,14 +2048,13 @@ "titleContainsAny": [], "titleNotContains": [], "agentMode": "autonomous", - "handlingAgent": "default", + "handlingAgent": "meta_agent", "handlingAgents": null, "owningTeamId": "", "owningTeamIds": [], "maxAutomatedInvestigationAttempts": 3, - "deepInvestigationEnabled": false, "mergeEnabled": false, - "mergeWindowHours": 0, + "mergeWindowHours": 3, "isEnabled": true, "icmFilterSettings": null, "azMonitorFilterSettings": { @@ -1945,14 +2073,13 @@ "titleContainsAny": [], "titleNotContains": [], "agentMode": "autonomous", - "handlingAgent": "default", + "handlingAgent": "meta_agent", "handlingAgents": null, "owningTeamId": "", "owningTeamIds": [], "maxAutomatedInvestigationAttempts": 3, - "deepInvestigationEnabled": false, "mergeEnabled": false, - "mergeWindowHours": 0, + "mergeWindowHours": 3, "isEnabled": true, "icmFilterSettings": null, "azMonitorFilterSettings": { @@ -1971,14 +2098,13 @@ "titleContainsAny": [], "titleNotContains": [], "agentMode": "autonomous", - "handlingAgent": "default", + "handlingAgent": "meta_agent", "handlingAgents": null, "owningTeamId": "", "owningTeamIds": [], "maxAutomatedInvestigationAttempts": 3, - "deepInvestigationEnabled": false, "mergeEnabled": false, - "mergeWindowHours": 0, + "mergeWindowHours": 3, "isEnabled": true, "icmFilterSettings": null, "azMonitorFilterSettings": { @@ -1992,26 +2118,22 @@ "priorities": "[variables('defaultPriorities')]", "incidentType": "", "alertId": "", - "titleContains": "", + "titleContains": "Zava", "titleContainsAll": [], - "titleContainsAny": [ - "Zava", - "postgres" - ], + "titleContainsAny": [], "titleNotContains": [ "postgres", "query-slow", "http-5xx" ], "agentMode": "review", - "handlingAgent": "default", + "handlingAgent": "meta_agent", "handlingAgents": null, "owningTeamId": "", "owningTeamIds": [], "maxAutomatedInvestigationAttempts": 2, - "deepInvestigationEnabled": true, "mergeEnabled": false, - "mergeWindowHours": 0, + "mergeWindowHours": 3, "isEnabled": true, "icmFilterSettings": null, "azMonitorFilterSettings": { @@ -2193,14 +2315,23 @@ { "type": "Microsoft.App/agents/connectors", "apiVersion": "2025-05-01-preview", - "name": "[format('{0}/{1}', parameters('agentName'), 'microsoft-learn')]", + "name": "[format('{0}/{1}', parameters('agentName'), 'learn-docs')]", "properties": { "dataConnectorType": "Mcp", - "dataSource": "zava-aks-postgres-microsoft-learn-mcp", + "dataSource": "placeholder", "extendedProperties": { "type": "http", "endpoint": "https://learn.microsoft.com/api/mcp", - "authType": "CustomHeaders" + "selectedTools": [ + "learn-docs_microsoft_docs_search", + "learn-docs_microsoft_code_sample_search", + "learn-docs_microsoft_docs_fetch" + ], + "toolsVisibleToMetaAgent": [ + "learn-docs_microsoft_docs_search", + "learn-docs_microsoft_code_sample_search", + "learn-docs_microsoft_docs_fetch" + ] }, "identity": "" }, @@ -2276,6 +2407,17 @@ "[resourceId('Microsoft.App/agents', parameters('agentName'))]" ] }, + { + "type": "Microsoft.App/agents/skills", + "apiVersion": "2025-05-01-preview", + "name": "[format('{0}/{1}', parameters('agentName'), 'incident-correlation')]", + "properties": { + "value": "[base64(string(union(variables('correlationSkill'), createObject('skillContent', replace(variables('correlationSkill').skillContent, '@@RG@@', variables('rgName'))))))]" + }, + "dependsOn": [ + "[resourceId('Microsoft.App/agents', parameters('agentName'))]" + ] + }, { "type": "Microsoft.App/agents/incidentFilters", "apiVersion": "2025-05-01-preview", @@ -2353,6 +2495,96 @@ "[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, parameters('resourceGroupName')), 'Microsoft.Resources/deployments', 'vnet')]" ] }, + { + "type": "Microsoft.Resources/deployments", + "apiVersion": "2025-04-01", + "name": "firewall-agent-dataplane", + "resourceGroup": "[parameters('resourceGroupName')]", + "properties": { + "expressionEvaluationOptions": { + "scope": "inner" + }, + "mode": "Incremental", + "parameters": { + "firewallPolicyName": { + "value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, parameters('resourceGroupName')), 'Microsoft.Resources/deployments', 'vnet'), '2025-04-01').outputs.firewallPolicyName.value]" + }, + "agentEndpoint": { + "value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, parameters('resourceGroupName')), 'Microsoft.Resources/deployments', 'sre-agent'), '2025-04-01').outputs.agentEndpoint.value]" + }, + "agentSubnetPrefix": { + "value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, parameters('resourceGroupName')), 'Microsoft.Resources/deployments', 'vnet'), '2025-04-01').outputs.agentSubnetPrefix.value]" + }, + "enabled": { + "value": "[parameters('allowAgentSelfManagement')]" + } + }, + "template": { + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "metadata": { + "_generator": { + "name": "bicep", + "version": "0.43.1.21952", + "templateHash": "1430610225671267128" + } + }, + "parameters": { + "firewallPolicyName": { + "type": "string", + "metadata": { + "description": "Name of the firewall policy to add the rule collection group to." + } + }, + "agentEndpoint": { + "type": "string", + "metadata": { + "description": "The agent data-plane endpoint URL (e.g. https://--...azuresre.ai). The host is extracted automatically." + } + }, + "agentSubnetPrefix": { + "type": "string", + "defaultValue": "10.30.0.0/27", + "metadata": { + "description": "Source address prefix for the agent subnet (must match vnet.bicep)." + } + }, + "enabled": { + "type": "bool", + "defaultValue": true, + "metadata": { + "description": "Whether the exact-host data-plane allow rule is active. False removes the rule on incremental deployments." + } + } + }, + "variables": { + "agentFqdn": "[split(replace(parameters('agentEndpoint'), 'https://', ''), '/')[0]]" + }, + "resources": [ + { + "type": "Microsoft.Network/firewallPolicies/ruleCollectionGroups", + "apiVersion": "2023-11-01", + "name": "[format('{0}/{1}', parameters('firewallPolicyName'), 'AgentDataPlaneRuleCollectionGroup')]", + "properties": { + "priority": 250, + "ruleCollections": "[if(parameters('enabled'), createArray(createObject('ruleCollectionType', 'FirewallPolicyFilterRuleCollection', 'name', 'allow-agent-data-plane', 'priority', 100, 'action', createObject('type', 'Allow'), 'rules', createArray(createObject('ruleType', 'ApplicationRule', 'name', 'allow-agent-data-plane', 'description', format('SRE Agent data-plane: {0}', variables('agentFqdn')), 'sourceAddresses', createArray(parameters('agentSubnetPrefix')), 'protocols', createArray(createObject('protocolType', 'Https', 'port', 443)), 'targetFqdns', createArray(variables('agentFqdn')))))), createArray())]" + } + } + ], + "outputs": { + "agentDataPlaneFqdn": { + "type": "string", + "value": "[variables('agentFqdn')]" + } + } + } + }, + "dependsOn": [ + "[subscriptionResourceId('Microsoft.Resources/resourceGroups', parameters('resourceGroupName'))]", + "[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, parameters('resourceGroupName')), 'Microsoft.Resources/deployments', 'sre-agent')]", + "[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, parameters('resourceGroupName')), 'Microsoft.Resources/deployments', 'vnet')]" + ] + }, { "type": "Microsoft.Resources/deployments", "apiVersion": "2025-04-01", @@ -3048,6 +3280,10 @@ "type": "string", "value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, parameters('resourceGroupName')), 'Microsoft.Resources/deployments', 'identity'), '2025-04-01').outputs.appIdentityPrincipalId.value]" }, + "SRE_AGENT_PRINCIPAL_ID": { + "type": "string", + "value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, parameters('resourceGroupName')), 'Microsoft.Resources/deployments', 'identity'), '2025-04-01').outputs.sreAgentIdentityPrincipalId.value]" + }, "LOG_ANALYTICS_WORKSPACE_ID": { "type": "string", "value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, parameters('resourceGroupName')), 'Microsoft.Resources/deployments', 'monitoring'), '2025-04-01').outputs.logAnalyticsWorkspaceId.value]" diff --git a/labs/zava-aks-postgres/infra/modules/aks.bicep b/labs/zava-aks-postgres/infra/modules/aks.bicep index b22f1941d..fdf289109 100644 --- a/labs/zava-aks-postgres/infra/modules/aks.bicep +++ b/labs/zava-aks-postgres/infra/modules/aks.bicep @@ -35,9 +35,8 @@ resource aks 'Microsoft.ContainerService/managedClusters@2024-09-01' = { } // Enterprise hardening: private API server. The control plane is // unreachable from the public internet. Human operators reach it through - // `az aks command invoke` (Azure-proxied kubectl); the SRE Agent uses - // native `kubectl` over the private API-server path (managed-identity - // `kubelogin`). Cluster Admin RBAC for the agent + // `az aks command invoke` (Azure-proxied kubectl); the SRE Agent uses its + // built-in Kubernetes system tools. Cluster Admin RBAC for the agent // identities is granted in `sre-agent.bicep`. apiServerAccessProfile: { enablePrivateCluster: true diff --git a/labs/zava-aks-postgres/infra/modules/firewall-agent-dataplane.bicep b/labs/zava-aks-postgres/infra/modules/firewall-agent-dataplane.bicep new file mode 100644 index 000000000..a1d9be5d7 --- /dev/null +++ b/labs/zava-aks-postgres/infra/modules/firewall-agent-dataplane.bicep @@ -0,0 +1,75 @@ +// Firewall rule: allow the SRE Agent to reach its OWN data-plane endpoint. +// +// Deployed as a SEPARATE rule collection group (not inside vnet.bicep's +// DefaultNetworkRuleCollectionGroup) so it can depend on the agent resource +// and use the agent's exact hostname — which is platform-assigned AFTER the +// agent is created and cannot be computed at vnet-deploy time. +// +// The agent's sandbox egress is forced through the hub firewall. Without this +// rule the agent cannot read or write the configuration surfaces ARM does not +// expose (custom instructions, hooks, knowledge files, tool enablement). The +// failure is confusing: DNS resolves, TCP 443 connects, then TLS is RESET +// (SSL_ERROR_SYSCALL) — it reads like a certificate problem, not a firewall +// denial. +// +// SECURITY NOTE: this is a self-modification path. The same data-plane API +// that lets the agent READ its config also lets it WRITE it (skills, always-on +// prompts). For this lab that is the point; set allowAgentSelfManagement=false +// in main.bicep to deploy this collection group with no allow rules. + +@description('Name of the firewall policy to add the rule collection group to.') +param firewallPolicyName string + +@description('The agent data-plane endpoint URL (e.g. https://--...azuresre.ai). The host is extracted automatically.') +param agentEndpoint string + +@description('Source address prefix for the agent subnet (must match vnet.bicep).') +param agentSubnetPrefix string = '10.30.0.0/27' + +@description('Whether the exact-host data-plane allow rule is active. False removes the rule on incremental deployments.') +param enabled bool = true + +// Extract the FQDN from the full endpoint URL. +// agentEndpoint is e.g. 'https://sre-agent-zava-g5kb--c618a660.6d6a35f1.swedencentral.azuresre.ai' +var agentFqdn = split(replace(agentEndpoint, 'https://', ''), '/')[0] + +resource firewallPolicy 'Microsoft.Network/firewallPolicies@2023-11-01' existing = { + name: firewallPolicyName +} + +// Separate rule collection group — priority 250, after the existing +// DefaultNetworkRuleCollectionGroup (priority 200). Using a distinct group +// avoids having to duplicate the full rule set from vnet.bicep. +resource agentDataPlaneRcg 'Microsoft.Network/firewallPolicies/ruleCollectionGroups@2023-11-01' = { + parent: firewallPolicy + name: 'AgentDataPlaneRuleCollectionGroup' + properties: { + priority: 250 + ruleCollections: enabled ? [ + { + ruleCollectionType: 'FirewallPolicyFilterRuleCollection' + name: 'allow-agent-data-plane' + priority: 100 + action: { type: 'Allow' } + rules: [ + { + ruleType: 'ApplicationRule' + name: 'allow-agent-data-plane' + // Pinned to this agent's exact hostname — not the broad *.azuresre.ai + // wildcard. Standard firewall handles exact FQDNs fine (FQDN/SNI match). + // + // NOTE: the token AUDIENCE is `https://azuresre.dev` but the network + // HOST is `*.azuresre.ai` — two different domains. Allow-listing the + // audience domain does nothing. + description: 'SRE Agent data-plane: ${agentFqdn}' + sourceAddresses: [agentSubnetPrefix] + protocols: [{ protocolType: 'Https', port: 443 }] + targetFqdns: [agentFqdn] + } + ] + } + ] : [] + } +} + +output agentDataPlaneFqdn string = agentFqdn diff --git a/labs/zava-aks-postgres/infra/modules/monitoring.bicep b/labs/zava-aks-postgres/infra/modules/monitoring.bicep index d9453e28d..51d9ced38 100644 --- a/labs/zava-aks-postgres/infra/modules/monitoring.bicep +++ b/labs/zava-aks-postgres/infra/modules/monitoring.bicep @@ -10,6 +10,9 @@ param postgresServerName string @description('Resource group ID — scope for activity-log-based alerts.') param resourceGroupId string +@description('Enable the PostgreSQL CPU-saturation metric alert. Deployed DISABLED by default on purpose — see the resource comment: the disabled causal alert is what makes the correlation scenario teach anything.') +param enableDbCpuSaturationAlert bool = false + var lawName = 'law-Zava-${uniqueSuffix}' var aiName = 'ai-Zava-${uniqueSuffix}' @@ -299,6 +302,58 @@ resource alertProductsSlow 'Microsoft.Insights/scheduledQueryRules@2023-03-15-pr // az monitor activity-log alert update -g -n Zava-unknown-test --enabled true // az tag update --operation merge --tags zava-drill=on --resource-id // Disable again afterwards. +// Alert: PostgreSQL CPU saturation (metric alert on cpu_percent). +// +// DEPLOYED DISABLED ON PURPOSE. This is not an oversight and not dead code — +// it is the load-bearing piece of the cross-alert correlation scenario, and it +// was previously drift (it existed in several live demo RGs but in no template, +// so `azd up` and reality disagreed). Declaring it here fixes the drift while +// preserving the teaching value. +// +// Why disabled: when Scenario 3 (missing index) runs, PG `cpu_percent` pegs at +// ~90% for the length of the load run, but the ONLY alerts that fire are the +// downstream symptom alerts (`Zava-products-query-slow`, and `Zava-http-5xx-errors` +// if the app is also degraded). The *causal* signal is silent. That is exactly the +// real-world failure mode this lab should teach: an operator muted a noisy rule +// months ago, so the alert that would have named the cause never fires and the +// responder only ever sees symptoms. An agent that reasons only from the alert it +// was dispatched on cannot recover the cause; an agent that enumerates the alert +// RULE inventory (not just fired alerts) finds a disabled rule sitting on the very +// resource it is investigating, and knows to go query that metric directly. +// +// Set enableDbCpuSaturationAlert=true to turn it on — the scenario then becomes a +// genuine multi-alert co-firing case (causal + symptom) instead of a silent-cause +// case. Both are useful; the default teaches the harder lesson. +resource alertDbCpuSaturation 'Microsoft.Insights/metricAlerts@2018-03-01' = { + name: 'Zava-db-cpu-saturation' + location: 'global' + properties: { + severity: 3 + enabled: enableDbCpuSaturationAlert + evaluationFrequency: 'PT1M' + windowSize: 'PT5M' + scopes: [pgServer.id] + criteria: { + 'odata.type': 'Microsoft.Azure.Monitor.SingleResourceMultipleMetricCriteria' + allOf: [ + { + criterionType: 'StaticThresholdCriterion' + name: 'pgCpu' + metricName: 'cpu_percent' + metricNamespace: 'Microsoft.DBforPostgreSQL/flexibleServers' + operator: 'GreaterThan' + threshold: 80 + timeAggregation: 'Average' + } + ] + } + actions: [{ actionGroupId: actionGroup.id }] + autoMitigate: true + // Symptom-only by design — do NOT name a suspected index, query, or scenario here. + description: 'Zava Demo: PostgreSQL server CPU averaged above 80% over 5 minutes.' + } +} + resource alertUnknownTest 'Microsoft.Insights/activityLogAlerts@2023-01-01-preview' = { name: 'Zava-unknown-test' location: 'global' diff --git a/labs/zava-aks-postgres/infra/modules/sre-agent.bicep b/labs/zava-aks-postgres/infra/modules/sre-agent.bicep index 6a685d8ae..4e47d90fe 100644 --- a/labs/zava-aks-postgres/infra/modules/sre-agent.bicep +++ b/labs/zava-aks-postgres/infra/modules/sre-agent.bicep @@ -67,8 +67,8 @@ resource sreAgent 'Microsoft.App/agents@2025-05-01-preview' = { // the delegated agent subnet, with ALL egress forced (UDR) through the Azure // Firewall. The firewall allow-list is deliberately minimal — the control plane // (ARM, Entra, Microsoft Graph) and Microsoft Learn over public service tags, - // plus the AKS API server over the hub/spoke (so the agent uses native - // `kubectl`). Azure Monitor is private-only by default (lockAgentToPrivateMonitor): + // plus the AKS API server over the hub/spoke for the built-in RunKubectl* + // system tools. Azure Monitor is private-only by default (lockAgentToPrivateMonitor): // public AzureMonitor is dropped and the agent reaches Log Analytics / App // Insights over the AMPLS private endpoint; the agent remains fully functional // over it. It does NOT permit a raw socket to PostgreSQL:5432, so SQL runs @@ -104,9 +104,9 @@ resource sreAgent 'Microsoft.App/agents@2025-05-01-preview' = { // Registries/CodeRepositories empty). Egress mode is AzureVNet, so the agent // gets REAL VNet egress (not an HTTP-proxy) — but every connection is gated by // the Azure Firewall above. Its rules permit ARM/Entra/Graph + Microsoft Learn - // (public service tags) and the AKS API server over the hub/spoke (TCP 443 — - // native kubectl is enabled; the agent VNet has the AKS private-DNS zone - // linked + a firewall rule + SNAT). Azure Monitor is private-only by default + // (public service tags) and the AKS API server over the hub/spoke (TCP 443; + // the agent VNet has the AKS private-DNS zone linked + a firewall rule + + // SNAT). Azure Monitor is private-only by default // (public AzureMonitor dropped; agent linked to the AMPLS private DNS) — the // agent remains fully functional over it. Everything else is denied by // design — the agent still cannot open a raw socket to PostgreSQL:5432. @@ -251,14 +251,23 @@ resource logAnalyticsConnector 'Microsoft.App/agents/connectors@2025-05-01-previ #disable-next-line BCP081 resource microsoftLearnConnector 'Microsoft.App/agents/connectors@2025-05-01-preview' = { parent: sreAgent - name: 'microsoft-learn' + name: 'learn-docs' properties: { dataConnectorType: 'Mcp' - dataSource: 'zava-aks-postgres-microsoft-learn-mcp' + dataSource: 'placeholder' extendedProperties: { type: 'http' endpoint: 'https://learn.microsoft.com/api/mcp' - authType: 'CustomHeaders' + selectedTools: [ + 'learn-docs_microsoft_docs_search' + 'learn-docs_microsoft_code_sample_search' + 'learn-docs_microsoft_docs_fetch' + ] + toolsVisibleToMetaAgent: [ + 'learn-docs_microsoft_docs_search' + 'learn-docs_microsoft_code_sample_search' + 'learn-docs_microsoft_docs_fetch' + ] } identity: '' } @@ -290,17 +299,18 @@ resource azureMonitorConnector 'Microsoft.App/agents/connectors@2025-05-01-previ var sharedContext = '''Resource Group `@@RG@@`. App namespace `zava-demo`. Deployments `zava-api` / `zava-storefront`. App Insights cloud_RoleName `zava-api`. -You operate with your own managed identity (Entra) — AKS RBAC Cluster Admin, Reader + Monitoring Reader + Contributor on the resource group, and PostgreSQL Entra admin. These are sufficient: do NOT attempt `az role assignment create` (it is denied — if you think you need a role you lack, your diagnosis is wrong, back up). Your sandbox egress is forced through an Azure Firewall (allow-list: ARM, Entra, Microsoft Graph, Microsoft Learn over public service tags, plus the AKS API server over the hub/spoke; Azure Monitor is reached privately via the AMPLS private endpoint by default) AND a TLS-inspecting forward proxy that re-signs certificates. This cluster is wired for native `kubectl` (the agent VNet has the AKS private-DNS zone linked and a firewall rule + SNAT to the API server): you run `kubectl` yourself as a bash command in your sandbox terminal (`RunInTerminal`). One-time setup per session: (1) `az aks get-credentials -g @@RG@@ -n --overwrite-existing` (find the cluster via `az aks list -g @@RG@@ --query "[0].name" -o tsv`); (2) `kubelogin convert-kubeconfig -l azurecli` — non-interactive managed-identity auth (the DEFAULT device-code flow hangs; do not use it); (3) trust the egress proxy by merging its CA `/etc/ssl/certs/adc-egress-proxy-ca.crt` into the kubeconfig cluster's `certificate-authority-data`. Then `kubectl get nodes` works; run kubectl in your terminal for pods, logs, events, NetworkPolicies, rollouts, and the in-cluster SQL helper `kubectl exec -n zava-demo deploy/zava-api -- node bin/run-sql.js ''`. Never install DB clients (`psql`, `psycopg2`) or open a raw socket to PostgreSQL. Reach ARM over the control plane; reach Azure Monitor (Log Analytics / Application Insights) with your Monitor query tools — they work normally (this deployment locks the agent's Monitor access to the AMPLS private endpoint by default, and your tools operate fine over it). Filter every App Insights / Log Analytics query by `AppRoleName == 'zava-api'` — the workspace is shared with your own ARM-poll telemetry.''' +You operate with your own managed identity (Entra) — AKS RBAC Cluster Admin, Reader + Monitoring Reader + Contributor on the resource group, Reader at subscription scope for cross-alert and Service Health context, and PostgreSQL Entra admin. These are sufficient: do NOT attempt `az role assignment create` (it is denied — if you think you need a role you lack, your diagnosis is wrong, back up). Use the built-in `RunKubectlReadCommand` and `RunKubectlWriteCommand` system tools for Kubernetes; they accept the same kubectl commands as a terminal. Use the read tool for inspection and the write tool for `delete`, `rollout`, and `exec` operations. Do not replace them with terminal-native kubectl, login repair, kubeconfig setup, or Python wrappers. Run PostgreSQL SQL through the in-cluster helper with the write tool: `kubectl exec -n zava-demo deploy/zava-api -- node bin/run-sql.js ''`. Never install DB clients (`psql`, `psycopg2`) or open a raw socket to PostgreSQL. Reach ARM over the control plane; reach Azure Monitor (Log Analytics / Application Insights) with your Monitor query tools — they work normally (this deployment locks the agent's Monitor access to the AMPLS private endpoint by default, and your tools operate fine over it). Filter every App Insights / Log Analytics query by `AppRoleName == 'zava-api'` — the workspace is shared with your own ARM-poll telemetry.''' var databaseSkill = { description: 'Use for Zava PostgreSQL AVAILABILITY incidents — alert `postgres-unreachable` (zava-api cannot reach PostgreSQL; connection refused or, more often, timeout). Diagnose the cause from ARM state — stopped server vs network partition — and remediate: restart the server, or remove the in-cluster Kubernetes NetworkPolicy / matching NSG deny rule that blocks PG egress.' tools: [ 'RunAzCliReadCommands' 'RunAzCliWriteCommands' - 'RunInTerminal' + 'RunKubectlReadCommand' + 'RunKubectlWriteCommand' 'SearchMemory' - 'microsoft-learn_microsoft_docs_search' - 'microsoft-learn_microsoft_docs_fetch' + 'learn-docs_microsoft_docs_search' + 'learn-docs_microsoft_docs_fetch' ] skillContent: '''## Database availability runbook (Zava) @@ -313,7 +323,7 @@ The alert `postgres-unreachable` means zava-api cannot reach PostgreSQL — it l | PG ARM `state` | Cause | Action | |---|---|---| | `Stopped` | The server was stopped. | **Start it**: `az postgres flexible-server start`. | -| `Ready` (app still can't connect) | A network block. | Two enforcement surfaces sit between the app and PG: an NSG deny rule on the AKS subnet (often a RED HERRING — PG's private access uses a platform-managed delegated subnet) and a Kubernetes **NetworkPolicy** in `zava-demo` (usually the real cause). Inspect both — `az network nsg rule list` and `kubectl get networkpolicy -A -o yaml` (run in your terminal) — then delete the offending NetworkPolicy with `kubectl delete networkpolicy -n zava-demo` (and any matching NSG deny rule on the AKS subnet). | +| `Ready` (app still can't connect) | A network block. | Two enforcement surfaces sit between the app and PG: an NSG deny rule on the AKS subnet (often a RED HERRING — PG's private access uses a platform-managed delegated subnet) and a Kubernetes **NetworkPolicy** in `zava-demo` (usually the real cause). Inspect both with `az network nsg rule list` and `RunKubectlReadCommand`, then delete the offending NetworkPolicy with `RunKubectlWriteCommand` (and any matching NSG deny rule on the AKS subnet). | ## Permitted autonomous actions - Start / restart / parameter-set on PostgreSQL Flexible Server. @@ -335,14 +345,15 @@ After confirming recovery, **resolve the `postgres-unreachable` alert you were h } var performanceSkill = { - description: 'Use for Zava query-LATENCY / slow-endpoint incidents — alert `Zava-products-query-slow` (a /api/products/category endpoint breached its latency threshold). The bottleneck is at PostgreSQL (missing/disabled index, plan regression), not pods/CPU. Corroborate with the custom latency metric + PG CPU, then apply read-mostly DDL (CREATE INDEX) via the in-cluster SQL helper.' + description: 'Use for Zava query-LATENCY / slow-endpoint incidents — alert `Zava-products-query-slow` (a /api/products/category endpoint breached its latency threshold). Diagnose the PostgreSQL query path, check nearby alerts with `incident-correlation`, and do not treat a co-firing 5xx as the same cause without a direct dependency-failure mechanism. Apply read-mostly DDL (CREATE INDEX) via the in-cluster SQL helper when the query plan proves it is needed.' tools: [ 'RunAzCliReadCommands' 'RunAzCliWriteCommands' - 'RunInTerminal' + 'RunKubectlReadCommand' + 'RunKubectlWriteCommand' 'SearchMemory' - 'microsoft-learn_microsoft_docs_search' - 'microsoft-learn_microsoft_docs_fetch' + 'learn-docs_microsoft_docs_search' + 'learn-docs_microsoft_docs_fetch' ] skillContent: '''## Query-performance runbook (Zava) @@ -357,8 +368,11 @@ var performanceSkill = { 4. **Trace**: `AppDependencies` PostgreSQL-call latency. Agreement across all four points at the database query, not the app tier. +## Cross-alert guard +Load `incident-correlation` to check fired-alert history before assigning a shared root cause; alert-rule inventory cannot tell you what fired. A co-firing 5xx is NOT corroboration for a slow-query diagnosis. Split `AppDependencies` by target and result code: slow but successful PostgreSQL calls establish this latency fault, but they cannot explain HTTP 500s whose failed dependencies are only app-local. Different mechanisms mean independent incidents, even when their alert times and resource group match. If the other alert is already acknowledged, report the relationship but leave its remediation to that thread. + ## Diagnose at PostgreSQL (in-cluster SQL helper) -`kubectl exec -n zava-demo deploy/zava-api -- node bin/run-sql.js ''` — run kubectl in your terminal, set up per shared context. Inspect `pg_stat_user_indexes` (low/zero `idx_scan` on a hot table is a strong signal), `pg_stat_user_tables` (high `seq_scan`), `pg_stat_statements` (top mean-time), and `EXPLAIN`. +Use `RunKubectlWriteCommand` to execute `kubectl exec -n zava-demo deploy/zava-api -- node bin/run-sql.js ''`. Inspect `pg_stat_user_indexes` (low/zero `idx_scan` on a hot table is a strong signal), `pg_stat_user_tables` (high `seq_scan`), `pg_stat_statements` (top mean-time), and `EXPLAIN`. ## Permitted autonomous actions - Read-mostly DDL on PostgreSQL via the in-cluster helper: `CREATE INDEX CONCURRENTLY IF NOT EXISTS`, `ANALYZE`, `REINDEX CONCURRENTLY`. @@ -374,14 +388,15 @@ The category endpoint's avg latency returns to baseline; `idx_scan` climbs on th } var applicationSkill = { - description: 'Use for Zava APPLICATION-layer HTTP 5xx incidents — alert `Zava-http-5xx-errors` (zava-api returning HTTP 5xx). This is typically an app/route regression such as a bad deploy, but a DB outage also produces 5xx, so FIRST rule out DB/perf; if PG is healthy, correlate the 5xx onset with a recent rollout and roll back to the previous good revision.' + description: 'Use for Zava APPLICATION-layer HTTP 5xx incidents — alert `Zava-http-5xx-errors` (zava-api returning HTTP 5xx). Rule out a direct DB failure path, check nearby alerts with `incident-correlation`, and correlate the 5xx onset with a recent rollout. Do not attribute it to a co-firing latency alert unless dependency failures prove that mechanism.' tools: [ 'RunAzCliReadCommands' 'RunAzCliWriteCommands' - 'RunInTerminal' + 'RunKubectlReadCommand' + 'RunKubectlWriteCommand' 'SearchMemory' - 'microsoft-learn_microsoft_docs_search' - 'microsoft-learn_microsoft_docs_fetch' + 'learn-docs_microsoft_docs_search' + 'learn-docs_microsoft_docs_fetch' ] skillContent: '''## Application 5xx runbook (Zava) @@ -391,10 +406,12 @@ var applicationSkill = { ## Investigate 1. Briefly confirm it is not DB/perf after all: PG `state == Ready`, no ECONNREFUSED/ETIMEDOUT traces, `/api/products` latency normal. If a DB or slow-query symptom is actually present, defer to the database / performance runbook. -2. App regressions are usually shipped by a deploy. Every change to the `zava-api` Deployment pod template creates a new ReplicaSet **revision**. Check whether the 5xx onset lines up with a recent rollout: native `kubectl rollout history deployment/zava-api -n zava-demo` and `KubeEvents` (Azure Monitor) (`ScalingReplicaSet` timestamps). Note the liveness AND readiness probes both hit `/livez` (shallow, no DB call), so pods stay Ready through an app-route regression and the platform looks healthy while the app is broken; `/api/health` is a separate app health endpoint (it pings the DB) and can also stay green for a route-only regression — deployment correlation is the tie. +2. App regressions are usually shipped by a deploy. Every change to the `zava-api` Deployment pod template creates a new ReplicaSet **revision**. Check whether the 5xx onset lines up with a recent rollout using `RunKubectlReadCommand` for `kubectl rollout history deployment/zava-api -n zava-demo` and `KubeEvents` (Azure Monitor) (`ScalingReplicaSet` timestamps). Note the liveness AND readiness probes both hit `/livez` (shallow, no DB call), so pods stay Ready through an app-route regression and the platform looks healthy while the app is broken; `/api/health` is a separate app health endpoint (it pings the DB) and can also stay green for a route-only regression — deployment correlation is the tie. + +Load `incident-correlation` to check fired-alert history; alert-rule inventory cannot tell you what fired. Shared timing is not a mechanism: split `AppDependencies` by target and result code before claiming the other alert caused this one. A slow-query alert with successful PostgreSQL dependencies does not explain HTTP 500s whose failed dependency is app-local. If the other alert is already acknowledged, report the relationship but leave its remediation to that thread. ## Permitted autonomous actions -- Roll back a `zava-demo` deployment to its previous revision (native `kubectl rollout undo deployment/zava-api -n zava-demo`) when a 5xx regression correlates with a recent rollout. +- Roll back a `zava-demo` deployment to its previous revision with `RunKubectlWriteCommand` (`kubectl rollout undo deployment/zava-api -n zava-demo`) when a 5xx regression correlates with a recent rollout. - Restart deployments in `zava-demo`. ## Out of scope (summarize + stop) @@ -412,8 +429,8 @@ var generalTriageSkill = { tools: [ 'RunAzCliReadCommands' 'SearchMemory' - 'microsoft-learn_microsoft_docs_search' - 'microsoft-learn_microsoft_docs_fetch' + 'learn-docs_microsoft_docs_search' + 'learn-docs_microsoft_docs_fetch' ] skillContent: '''## General triage runbook (Zava — unknown incidents) @@ -441,9 +458,9 @@ var proactiveHealthSkill = { 'RunAzCliReadCommands' 'SearchMemory' 'ExecutePythonCode' - 'microsoft-learn_microsoft_code_sample_search' - 'microsoft-learn_microsoft_docs_fetch' - 'microsoft-learn_microsoft_docs_search' + 'learn-docs_microsoft_code_sample_search' + 'learn-docs_microsoft_docs_fetch' + 'learn-docs_microsoft_docs_search' ] skillContent: '''## Proactive Health Check @@ -463,6 +480,96 @@ If any of those is missed, hand off to the matching domain skill: `database-inci sourcePluginInstallation: null } +// Cross-alert correlation ("is this the tree or the forest?"). +// +// This skill exists because the platform CANNOT hand the agent a forest view. +// Every response plan here runs `mergeEnabled: false`, and Azure Monitor merging +// is same-alert-rule-only, so each fired alert opens its OWN isolated thread with +// no visibility into what else fired. Structural isolation is the default. The +// only way an investigation sees the wider picture is if it PULLS it. +// +// Deliberately NOT put in the alert `description` fields (AGENTS.md forbids +// semantics there, and a per-alert string can't express a cross-alert idea), and +// NOT duplicated into all four incidentFilters (four copies = drift). The cheap +// always-on trigger lives in `sre-config/custom-instructions.md`, applied to the +// agent-global customInstructions surface by scripts/setup-sre-agent.ps1 (Step 2c); +// this skill carries the expensive procedure and loads only when that trigger fires. +// That split is the token-cost design: ~200 always-on tokens, full method on demand. +var correlationSkill = { + description: 'Use during a Zava incident when the dispatched alert may be only part of the story: another alert fired nearby, the evidence does not add up, remediation did not hold, or a symptom appears to precede its cause. Enumerates other Azure Monitor alerts, disabled alert rules that may hide the causal signal, and Azure Service Health, then distinguishes one causal chain from independent faults that merely overlapped.' + tools: [ + 'RunAzCliReadCommands' + 'SearchMemory' + ] + skillContent: '''## Cross-alert correlation runbook (Zava) + +Resource Group `@@RG@@`. + +You were dispatched on ONE alert. That alert is a filter someone wrote in advance, on one signal, with one threshold — it is evidence, not a conclusion, and it cannot tell you whether it is the cause, a symptom, or a coincidence. Every response plan in this deployment has merge DISABLED, so a single root cause opens several INDEPENDENT threads that cannot see each other. Nobody assembles the forest for you. Pull it. + +## 1. What else fired? (the forest) + +`az graph query` is usually unavailable (resource-graph extension absent). Use the Alerts Management REST API: + +`az rest --method get --url "https://management.azure.com/subscriptions//providers/Microsoft.AlertsManagement/alerts?api-version=2019-05-05-preview&timeRange=1d&pageCount=250" --query "value[].{ruleId:properties.essentials.alertRule, rg:properties.essentials.targetResourceGroup, sev:properties.essentials.severity, cond:properties.essentials.monitorCondition, start:properties.essentials.startDateTime, target:properties.essentials.targetResource}" -o json` + +- `pageCount` MUST be 1..250 (larger returns BadRequest). `timeRange` accepts 1h/1d/7d/30d only. +- `RunAzCliReadCommands` rejects shell pipes and `&&` — issue one command per call. +- `alertRule` is already the full rule resource ID in this API; the projection names it `ruleId`. +- For LOG alerts `targetResource` is the Log Analytics WORKSPACE, not the app or DB. Use `rg` / `ruleId` to identify the environment; do not group by `targetResource`. +- This fired-alert feed and the rule inventory in step 2 answer different questions. Never conclude "nothing else fired" from `scheduledQueryRules` or `az monitor metrics alert list`. + +## 2. Which alerts SHOULD have fired but did not? (the silent cause) + +A missing alert is a finding. Enumerate the rule INVENTORY, not just fired alerts: + +`az monitor metrics alert list -g @@RG@@ --query "[].{name:name, enabled:enabled, scopes:scopes}" -o json` +`az rest --method get --url "https://management.azure.com/subscriptions//resourceGroups/@@RG@@/providers/microsoft.insights/scheduledQueryRules?api-version=2023-03-15-preview" --query "value[].{name:name, enabled:properties.enabled, window:properties.windowSize, freq:properties.evaluationFrequency}" -o json` + +If a rule sits on the resource you are investigating and is `enabled: false`, the causal signal is MUTED — query that metric directly rather than concluding the resource was healthy. This deployment ships `Zava-db-cpu-saturation` disabled on purpose; PG CPU can be pegged at 90% with no DB alert anywhere. + +## 3. Is the platform itself the cause? + +Azure Service Health, subscription-scoped — covers regional outages and planned maintenance: + +`az rest --method get --url "https://management.azure.com/subscriptions//providers/Microsoft.ResourceHealth/events?api-version=2022-10-01&queryStartTime=" --query "value[].{type:properties.eventType, level:properties.eventLevel, status:properties.status, title:properties.title, start:properties.impactStartTime}" -o json` + +`eventType` is `ServiceIssue` (outage), `PlannedMaintenance`, or `HealthAdvisory`. Per-resource view: +`az rest --method get --url "https://management.azure.com/subscriptions//resourceGroups/@@RG@@/providers/Microsoft.ResourceHealth/availabilityStatuses?api-version=2023-07-01-preview" --query "value[].{res:id, avail:properties.availabilityState, summary:properties.summary}" -o json` + +Check this BEFORE concluding "platform health event" from absence of evidence. A `PlannedMaintenance` event naming PostgreSQL turns a guess into a fact, and it changes the remediation: you wait and document instead of chasing config. + +## 4. TWO RULES — violate these and correlation makes you WORSE, not better + +Filter every App Insights query by `AppRoleName == 'zava-api'`; the workspace also contains the agent's own ARM-poll telemetry. + +**Alert fire order is NOT causal order.** Every dispatching rule here is `PT5M` window / `PT5M` evaluation, so detection latency is up to 5 min plus ingestion lag. A symptom alert can fire BEFORE the cause alert. Any gap under ~7 minutes proves nothing about ordering. Establish onset from raw telemetry in 1-2 minute buckets, never from alert timestamps. + +**Co-firing is NOT causation.** Two alerts seconds apart can be two unrelated faults. Before claiming a causal chain, confirm the mechanism in telemetry: + +| Observation | Reading | +|---|---| +| HTTP 500, failed dependencies ONLY on `localhost:3001`, zero PG dependency failures | app-layer regression | +| HTTP 503, failed dependencies against the PG target | DB unreachable | +| No dependency FAILURES but PG `cpu_percent` high and latency up | DB saturation — slow but SUCCESSFUL queries, so failure-based signals stay clean | + +The single fastest discriminator is one `dependencies` query split by `target` alongside `resultCode`. If two co-firing alerts have different mechanisms, they are independent — report them as separate incidents and do not merge the narrative. + +**Environment containment:** alerts from a DIFFERENT resource group are a different Zava stack. Same-RG co-firing only identifies the candidate environment; it does NOT suggest a shared cause. Cross-RG simultaneity may justify checking for a platform event (step 3), but still proves nothing by itself. Never merge findings across resource groups without that check. + +## 5. Report + +State plainly which it was: (a) one cause, several alerts; (b) several independent causes that overlapped; or (c) this alert is the whole story. If (c), say so in one line and move on — a correlation sweep that finds nothing is a successful sweep, not wasted work. + +If an independent alert is already `Acknowledged`, its own thread is active. Attach the correlation finding to your report, but do not execute that other domain's remediation from this thread; concurrent duplicate writes can conflict. + +## Boundaries +Read-only. This skill never remediates — hand off to `database-incidents`, `performance-incidents`, or `application-incidents` with the correlation context attached. +''' + additionalFiles: [] + sourcePluginInstallation: null +} + // NOTE: Browser-based site diagnosis is intentionally NOT defined as an SRE // Agent skill. The `BrowseWebPage` / Browser Operator tool is not generally // available to deployed SRE Agents, so a skill that references it would never @@ -526,18 +633,33 @@ resource skillProactiveHealth 'Microsoft.App/agents/skills@2025-05-01-preview' = } } +// Sixth skill. The "max 5 concurrent" limit is on skills ACTIVE in a thread, not +// on skills defined — a domain skill plus this correlation skill is 2 of 5, which +// is the intended pairing. +#disable-next-line BCP081 +resource skillCorrelation 'Microsoft.App/agents/skills@2025-05-01-preview' = { + parent: sreAgent + name: 'incident-correlation' + properties: { + value: base64(string(union(correlationSkill, { + skillContent: replace(correlationSkill.skillContent, '@@RG@@', rgName) + }))) + } +} + // --- Incident filters (a.k.a. response plans) ------------------------------ // Granular routing: one filter per known DOMAIN (database / performance / -// application) plus an UNKNOWN catch-all. handlingAgent 'default' -> the agent +// application) plus an UNKNOWN catch-all. handlingAgent 'meta_agent' -> the agent // picks a skill by description (skills are NOT linked to filters), so the filter // names/tokens and the skill descriptions are kept aligned. // -// There is NO documented precedence when multiple filters match, so the buckets -// are made NON-OVERLAPPING: each known filter matches one alert token, and the -// unknown filter excludes all known tokens via titleNotContains. The unknown -// bucket is bounded to the demo's own alerts (titleContainsAny 'Zava' / 'postgres') -// so it can't sweep in unrelated subscription noise, and runs in REVIEW mode with -// deep investigation — investigate + propose, don't auto-act on a novel incident. +// Overlapping matches have no customer-controlled priority or specificity rule: +// the runtime uses the first matching plan in mutable filter order. Treat that +// choice as undefined and keep routes NON-OVERLAPPING. Each known filter matches +// one alert token; the fallback positively matches this demo's `Zava` prefix and +// explicitly excludes every known token. It therefore cannot sweep in unrelated +// subscription alerts and runs in REVIEW mode — investigate + propose, don't +// auto-act on a novel incident. var defaultPriorities = [ 'Sev0' @@ -560,12 +682,11 @@ var databaseFilter = { titleContainsAny: [] titleNotContains: [] agentMode: 'autonomous' - handlingAgent: 'default' + handlingAgent: 'meta_agent' handlingAgents: null owningTeamId: '' owningTeamIds: [] maxAutomatedInvestigationAttempts: 3 - deepInvestigationEnabled: false // Merge OFF on every plan — no agent-side deduplication. We want each scenario to // open its OWN investigation thread, not fold into a prior one (dedup hid real // incidents in testing). NOTE: the two DB scenarios still share the one @@ -574,7 +695,7 @@ var databaseFilter = { // — see monitoring.bicep alertDbUnreachable. That is an Azure Monitor stateful-alert // behavior, independent of this (already-off) agent merge setting. mergeEnabled: false - mergeWindowHours: 0 + mergeWindowHours: 3 isEnabled: true icmFilterSettings: null azMonitorFilterSettings: { @@ -594,15 +715,14 @@ var performanceFilter = { titleContainsAny: [] titleNotContains: [] agentMode: 'autonomous' - handlingAgent: 'default' + handlingAgent: 'meta_agent' handlingAgents: null owningTeamId: '' owningTeamIds: [] maxAutomatedInvestigationAttempts: 3 - deepInvestigationEnabled: false // Merge OFF — no dedup; every perf incident opens its own thread. mergeEnabled: false - mergeWindowHours: 0 + mergeWindowHours: 3 isEnabled: true icmFilterSettings: null azMonitorFilterSettings: { @@ -622,15 +742,14 @@ var applicationFilter = { titleContainsAny: [] titleNotContains: [] agentMode: 'autonomous' - handlingAgent: 'default' + handlingAgent: 'meta_agent' handlingAgents: null owningTeamId: '' owningTeamIds: [] maxAutomatedInvestigationAttempts: 3 - deepInvestigationEnabled: false // Merge OFF — no dedup; every 5xx incident opens its own thread. mergeEnabled: false - mergeWindowHours: 0 + mergeWindowHours: 3 isEnabled: true icmFilterSettings: null azMonitorFilterSettings: { @@ -639,9 +758,9 @@ var applicationFilter = { } } -// Unknown / catch-all bucket. Bounded to demo-named alerts (Zava* / postgres*), -// excludes every known routing token, runs in Review mode with deep investigation -// and fewer auto-attempts. Exercise it with the (disabled-by-default) +// Unknown / catch-all bucket. Positively bounded to Zava-named alerts, excludes +// every known routing token, and runs in Review mode with fewer auto-attempts. +// Exercise it with the (disabled-by-default) // `Zava-unknown-test` alert in monitoring.bicep. var unknownFilter = { incidentPlatform: 'AzMonitor' @@ -649,27 +768,23 @@ var unknownFilter = { priorities: defaultPriorities incidentType: '' alertId: '' - titleContains: '' + titleContains: 'Zava' titleContainsAll: [] - titleContainsAny: [ - 'Zava' - 'postgres' - ] + titleContainsAny: [] titleNotContains: [ 'postgres' 'query-slow' 'http-5xx' ] agentMode: 'review' - handlingAgent: 'default' + handlingAgent: 'meta_agent' handlingAgents: null owningTeamId: '' owningTeamIds: [] maxAutomatedInvestigationAttempts: 2 - deepInvestigationEnabled: true // Merge OFF — no dedup; every novel incident opens its own (Review-mode) thread. mergeEnabled: false - mergeWindowHours: 0 + mergeWindowHours: 3 isEnabled: true icmFilterSettings: null azMonitorFilterSettings: { @@ -721,5 +836,3 @@ output agentSystemPrincipalId string = sreAgent.identity.principalId // Deep-link straight to this agent's blade so the operator lands on the // Threads tab without having to pick the agent from a list. output agentPortalUrl string = 'https://sre.azure.com/agents${sreAgent.id}' - - diff --git a/labs/zava-aks-postgres/infra/modules/subscription-reader.bicep b/labs/zava-aks-postgres/infra/modules/subscription-reader.bicep new file mode 100644 index 000000000..a336674c0 --- /dev/null +++ b/labs/zava-aks-postgres/infra/modules/subscription-reader.bicep @@ -0,0 +1,15 @@ +targetScope = 'subscription' + +@description('Principal object ID that receives subscription Reader.') +param principalId string + +var readerRoleId = 'acdd72a7-3385-48ef-bd42-f606fba81ae7' + +resource reader 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(subscription().id, principalId, readerRoleId) + properties: { + roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', readerRoleId) + principalId: principalId + principalType: 'ServicePrincipal' + } +} diff --git a/labs/zava-aks-postgres/infra/modules/vnet.bicep b/labs/zava-aks-postgres/infra/modules/vnet.bicep index 6459c1537..4f494419e 100644 --- a/labs/zava-aks-postgres/infra/modules/vnet.bicep +++ b/labs/zava-aks-postgres/infra/modules/vnet.bicep @@ -7,7 +7,7 @@ param uniqueSuffix string @description('''Lock the agent to PRIVATE-ONLY Azure Monitor (default true). When true, the public `AzureMonitor` service tag is dropped from the firewall L4 allow-list and the agent reaches Monitor via the AMPLS private endpoint (10.10.2.0/27, rule allow-agent-to-ampls) + the linked private-DNS -zones. The agent remains fully functional under this lockdown (Monitor queries, native kubectl, and +zones. The agent remains fully functional under this lockdown (Monitor queries, Kubernetes tools, and incident remediation all work). See the main.bicep param doc.''') param lockAgentToPrivateMonitor bool = true @@ -41,7 +41,7 @@ param lockAgentToPrivateMonitor bool = true // └─ db-subnet 10.20.16.0/24 PostgreSQL Flexible Server delegation. // // SPOKE 2 — agent vnet-Zava-agent-* 10.30.0.0/24 (the SRE Agent) -// └─ agent-subnet 10.30.0.0/28 Microsoft.App/environments delegation; +// └─ agent-subnet 10.30.0.0/27 Microsoft.App/environments delegation; // ALL egress forced to the hub firewall // via a UDR (0.0.0.0/0 → firewall private // IP) over peering. @@ -79,6 +79,7 @@ var hubVnetName = 'vnet-Zava-hub-${uniqueSuffix}' var platformVnetName = 'vnet-Zava-platform-${uniqueSuffix}' var agentVnetName = 'vnet-Zava-agent-${uniqueSuffix}' var nsgName = 'nsg-aks-${uniqueSuffix}' +var agentSubnetPrefix = '10.30.0.0/27' // First usable address in AzureFirewallSubnet (Azure reserves .0-.3 of the // subnet). Kept in sync with the AzureFirewallSubnet prefix (10.10.0.0/26). @@ -226,10 +227,11 @@ resource agentVnet 'Microsoft.Network/virtualNetworks@2024-01-01' = { { // SRE Agent workload subnet — delegated to Microsoft.App/environments so // the agent's sandbox is injected here, with all egress forced through - // the hub Azure Firewall (route table above). Minimum size is /28. + // the hub Azure Firewall (route table above). /27 is the minimum: + // 32 total addresses minus 5 Azure-reserved addresses = 27 usable. name: 'agent-subnet' properties: { - addressPrefix: '10.30.0.0/28' + addressPrefix: agentSubnetPrefix routeTable: { id: agentRouteTable.id } delegations: [ { @@ -372,7 +374,7 @@ resource peerAgentToHub 'Microsoft.Network/virtualNetworks/virtualNetworkPeering // // range to the ruleCollectionGroup below (the module already SNATs all egress, // // so the return path stays symmetric): // // { name: 'allow-agent-to-remote-region', ruleType: 'NetworkRule', -// // sourceAddresses: ['10.30.0.0/28'], destinationAddresses: ['10.40.0.0/24'], +// // sourceAddresses: [agentSubnetPrefix], destinationAddresses: ['10.40.0.0/24'], // // destinationPorts: ['*'], ipProtocols: ['Any'] } // // Cross-region peering alone is enough for VNet-to-VNet traffic that ISN'T // // force-tunneled; this lab force-tunnels the agent, hence the extra firewall rule. @@ -402,8 +404,8 @@ resource firewallPolicy 'Microsoft.Network/firewallPolicies@2024-05-01' = { enableProxy: true } threatIntelMode: 'Deny' - // SNAT all traffic, including private destinations. This is what lets the - // agent reach the PRIVATE AKS API server with native kubectl: the API + // SNAT all traffic, including private destinations. This supports the + // private AKS API-server network path: the API // server's NSG only admits the `VirtualNetwork` service tag, and the agent // spoke is NOT directly peered to the platform spoke — so without SNAT the // agent's 10.30.x.x source would be denied and the return path asymmetric. @@ -426,7 +428,7 @@ resource firewallPolicy 'Microsoft.Network/firewallPolicies@2024-05-01' = { // exposing the cluster API server publicly. // AzureCloud is deliberately NOT used (it covers ~65k prefixes including // third-party SaaS); precise service tags are used instead. The source is the -// agent spoke's subnet (10.30.0.0/28). +// agent spoke's subnet (agentSubnetPrefix). // // To let the agent reach a NETWORK DEVICE or other private service DIRECTLY, its // management endpoint must be HTTPS and its FQDN added BOTH here (an application @@ -451,7 +453,7 @@ resource ruleCollectionGroup 'Microsoft.Network/firewallPolicies/ruleCollectionG name: 'allow-azure-dns' description: 'DNS resolution via Azure DNS (required for the firewall DNS proxy)' ipProtocols: ['UDP', 'TCP'] - sourceAddresses: ['10.30.0.0/28'] + sourceAddresses: [agentSubnetPrefix] destinationAddresses: ['168.63.129.16'] destinationPorts: ['53'] } @@ -462,9 +464,8 @@ resource ruleCollectionGroup 'Microsoft.Network/firewallPolicies/ruleCollectionG // PRIVATE AKS API server (10.20.0.4, in the platform spoke's aks-subnet) // on 443. Combined with (a) linking the AKS private-DNS zone to the agent // VNet (a post-deploy step — the zone is AKS-managed in the MC_* RG) and - // (b) the SNAT on the policy above, this is what makes native `kubectl` - // work from the agent. Omit this collection (and the SNAT) for a - // command-invoke-only lab (no API-server line of sight). + // (b) the SNAT on the policy above, this completes direct API-server + // network reachability from the agent subnet. ruleCollectionType: 'FirewallPolicyFilterRuleCollection' name: 'allow-agent-to-aks-api' priority: 210 @@ -473,9 +474,9 @@ resource ruleCollectionGroup 'Microsoft.Network/firewallPolicies/ruleCollectionG { ruleType: 'NetworkRule' name: 'agent-to-apiserver' - description: 'Agent subnet -> AKS API server (enables native kubectl)' + description: 'Agent subnet -> AKS private API server' ipProtocols: ['TCP'] - sourceAddresses: ['10.30.0.0/28'] + sourceAddresses: [agentSubnetPrefix] destinationAddresses: ['10.20.0.0/20'] destinationPorts: ['443'] } @@ -500,7 +501,7 @@ resource ruleCollectionGroup 'Microsoft.Network/firewallPolicies/ruleCollectionG name: 'agent-to-ampls-pe' description: 'Agent subnet -> AMPLS private endpoint (private Azure Monitor)' ipProtocols: ['TCP'] - sourceAddresses: ['10.30.0.0/28'] + sourceAddresses: [agentSubnetPrefix] destinationAddresses: ['10.10.2.0/27'] destinationPorts: ['443'] } @@ -517,7 +518,7 @@ resource ruleCollectionGroup 'Microsoft.Network/firewallPolicies/ruleCollectionG name: 'allow-azure-services-l4' description: 'L4 access to Azure services via precise service tags (NOT AzureCloud)' ipProtocols: ['TCP'] - sourceAddresses: ['10.30.0.0/28'] + sourceAddresses: [agentSubnetPrefix] // AzureMonitor is dropped by DEFAULT (lockAgentToPrivateMonitor=true) // so the agent reaches Monitor only over the AMPLS private endpoint — // private-only / maximum restraint; the agent remains fully functional @@ -544,7 +545,7 @@ resource ruleCollectionGroup 'Microsoft.Network/firewallPolicies/ruleCollectionG ruleType: 'ApplicationRule' name: 'allow-arm-aad-graph' description: 'FQDN access to ARM, Entra ID, and Microsoft Graph' - sourceAddresses: ['10.30.0.0/28'] + sourceAddresses: [agentSubnetPrefix] protocols: [{ protocolType: 'Https', port: 443 }] targetFqdns: [ 'management.azure.com' @@ -552,6 +553,10 @@ resource ruleCollectionGroup 'Microsoft.Network/firewallPolicies/ruleCollectionG 'graph.microsoft.com' ] } + // The agent's OWN data-plane rule (allow-agent-data-plane) is NOT here — + // it lives in firewall-agent-dataplane.bicep, deployed AFTER the agent + // resource so it can pin to the agent's exact hostname (platform-assigned + // at creation time, not computable in Bicep beforehand). See main.bicep. ] } { @@ -564,7 +569,7 @@ resource ruleCollectionGroup 'Microsoft.Network/firewallPolicies/ruleCollectionG ruleType: 'ApplicationRule' name: 'allow-learn-microsoft-com' description: 'Microsoft Learn docs + MCP runtime endpoint (the agent looks up Azure/AKS/PostgreSQL guidance here)' - sourceAddresses: ['10.30.0.0/28'] + sourceAddresses: [agentSubnetPrefix] protocols: [{ protocolType: 'Https', port: 443 }] targetFqdns: [ 'learn.microsoft.com' @@ -585,7 +590,7 @@ resource ruleCollectionGroup 'Microsoft.Network/firewallPolicies/ruleCollectionG // (raw.githubusercontent.com/microsoftdocs/mcp/*) you'd need Azure // Firewall Premium + TLS inspection (targetUrls). See README caveats. description: 'GitHub raw content — the Microsoft Learn MCP connector fetches its server bits here to complete the tool-discovery handshake' - sourceAddresses: ['10.30.0.0/28'] + sourceAddresses: [agentSubnetPrefix] protocols: [{ protocolType: 'Https', port: 443 }] targetFqdns: [ 'raw.githubusercontent.com' @@ -644,6 +649,7 @@ output nsgName string = nsg.name output aksSubnetId string = '${platformVnet.id}/subnets/aks-subnet' output dbSubnetId string = '${platformVnet.id}/subnets/db-subnet' output agentSubnetId string = '${agentVnet.id}/subnets/agent-subnet' +output agentSubnetPrefix string = agentSubnetPrefix output privateDnsZoneId string = privateDnsZone.id // Hub-and-spoke outputs (consumed by the AMPLS + firewall-diagnostics modules). @@ -653,5 +659,6 @@ output platformVnetId string = platformVnet.id output agentVnetId string = agentVnet.id output peSubnetId string = '${hubVnet.id}/subnets/pe-subnet' output firewallName string = firewall.name +output firewallPolicyName string = firewallPolicy.name output firewallId string = firewall.id output firewallPrivateIp string = firewallPrivateIp diff --git a/labs/zava-aks-postgres/scripts/_aks-helpers.ps1 b/labs/zava-aks-postgres/scripts/_aks-helpers.ps1 index 9da24d3ab..29c4ae9f0 100644 --- a/labs/zava-aks-postgres/scripts/_aks-helpers.ps1 +++ b/labs/zava-aks-postgres/scripts/_aks-helpers.ps1 @@ -203,3 +203,60 @@ function Resolve-AksContext { } return [pscustomobject]@{ ResourceGroup = $ResourceGroup; ClusterName = $ClusterName } } + +function Reset-DemoAlertRule { + <# + .SYNOPSIS + Makes a stateful Azure Monitor alert rule ready for another demo run. + + .DESCRIPTION + Agent-side response-plan merging is disabled, but Azure Monitor still keeps + one stateful alert instance per rule. A prior instance that remains Fired + prevents a new activation and therefore prevents a fresh agent dispatch. + This helper fails before fault injection if the old condition is still + active, and closes a resolved instance so the next activation is New. + #> + param( + [Parameter(Mandatory)] [string]$ResourceGroup, + [Parameter(Mandatory)] [string]$AlertRuleName + ) + + $sub = (az account show --query id -o tsv 2>$null).Trim() + if (-not $sub) { throw "Not logged in to az. Run 'az login'." } + $token = (az account get-access-token --resource 'https://management.azure.com/' --query accessToken -o tsv 2>$null).Trim() + if (-not $token) { throw "Could not acquire an Azure Resource Manager token. Run 'az login'." } + $headers = @{ Authorization = "Bearer $token" } + + $url = "https://management.azure.com/subscriptions/$sub/providers/Microsoft.AlertsManagement/alerts?api-version=2019-05-05-preview&timeRange=30d&pageCount=250" + $response = Invoke-RestMethod -Method Get -Uri $url -Headers $headers + $alerts = @($response.value | Where-Object { + $essentials = $_.properties.essentials + $rule = [string]$essentials.alertRule + $ruleName = if ($rule.Contains('/')) { $rule.Split('/')[-1] } else { $rule } + $essentials.targetResourceGroup -eq $ResourceGroup -and $ruleName -eq $AlertRuleName + } | Sort-Object { [datetime]$_.properties.essentials.startDateTime } -Descending) + + if ($alerts.Count -eq 0) { + Write-Host "Alert preflight: no prior $AlertRuleName instance." -ForegroundColor DarkGray + return + } + + $latest = $alerts[0] + $essentials = $latest.properties.essentials + if ($essentials.monitorCondition -eq 'Fired') { + throw "Alert preflight: prior '$AlertRuleName' condition is still Fired. Restore the previous fault and wait for Azure Monitor to report Resolved before starting another run; otherwise no fresh agent dispatch is possible." + } + + if ($essentials.alertState -ne 'Closed') { + $alertId = [string]$latest.id + $changeStateUrl = "https://management.azure.com${alertId}/changestate?api-version=2018-05-05&newState=Closed" + try { + Invoke-RestMethod -Method Post -Uri $changeStateUrl -Headers $headers | Out-Null + } catch { + throw "Could not close prior '$AlertRuleName' alert instance." + } + Write-Host "Alert preflight: closed prior resolved $AlertRuleName instance." -ForegroundColor DarkGray + } else { + Write-Host "Alert preflight: prior $AlertRuleName instance is resolved and closed." -ForegroundColor DarkGray + } +} diff --git a/labs/zava-aks-postgres/scripts/post-provision.ps1 b/labs/zava-aks-postgres/scripts/post-provision.ps1 index 19e82dc3d..21e8c9c17 100644 --- a/labs/zava-aks-postgres/scripts/post-provision.ps1 +++ b/labs/zava-aks-postgres/scripts/post-provision.ps1 @@ -37,8 +37,8 @@ Write-Host "Loading azd environment values..." -ForegroundColor Yellow # Resolve the target resource group early — the Azure-discovery fallback below # needs it. RESOURCE_GROUP is an azd OUTPUT (only present after a successful -# provision); ZAVA_RG_NAME is the INPUT the env always carries, so it is the -# reliable bootstrap. +# provision); ZAVA_RG_NAME is an optional override. Fresh environments default +# to rg-$AZURE_ENV_NAME, matching infra/main.bicepparam. function Get-TargetRg { foreach ($k in 'RESOURCE_GROUP', 'ZAVA_RG_NAME') { $v = [Environment]::GetEnvironmentVariable($k) @@ -48,6 +48,8 @@ function Get-TargetRg { } if ($v -and "$v".Trim() -and "$v" -notmatch '^ERROR') { return "$v".Trim() } } + $environmentName = [Environment]::GetEnvironmentVariable('AZURE_ENV_NAME') + if ($environmentName) { return "rg-$environmentName" } return $null } $script:TargetRg = Get-TargetRg @@ -230,16 +232,12 @@ if (-not $proxyReady) { } Write-Host "" -# ── Step 4b: Link AKS private DNS zone to the agent VNet (native kubectl) ───── -# The SRE Agent runs VNet-injected in its own spoke and is wired for NATIVE -# kubectl (the firewall rule agent-subnet -> API:443 and SNAT are in vnet.bicep). -# The one piece that can ONLY be done post-deploy: AKS creates its private DNS +# ── Step 4b: Link AKS private DNS zone to the agent VNet ───────────────────── +# AKS creates its private DNS # zone (.privatelink..azmk8s.io) in the node resource group with a # name not known until the cluster exists, so the virtual-network link to the -# agent spoke can't be a static Bicep resource. Without this link the agent can't -# resolve the private API server and flounders on kubectl before falling back to -# `az aks command invoke`. Idempotent. -Write-Host "=== Step 4b: Linking AKS private DNS zone to agent VNet (native kubectl) ===" -ForegroundColor Green +# agent spoke can't be a static Bicep resource. Idempotent. +Write-Host "=== Step 4b: Linking AKS private DNS zone to agent VNet ===" -ForegroundColor Green $mcRg = az aks show -g $RG -n $AKS_NAME --query nodeResourceGroup -o tsv 2>$null $aksDnsZone = az network private-dns zone list -g $mcRg --query "[?contains(name,'azmk8s.io')].name | [0]" -o tsv 2>$null $agentVnetId = az network vnet list -g $RG --query "[?contains(name,'agent')].id | [0]" -o tsv 2>$null @@ -251,9 +249,9 @@ if ($mcRg -and $aksDnsZone -and $agentVnetId) { az network private-dns link vnet create -g $mcRg -z $aksDnsZone -n agent-link ` --virtual-network $agentVnetId --registration-enabled false -o none 2>$null if ($LASTEXITCODE -eq 0) { - Write-Host " Linked AKS private DNS zone -> agent VNet (native kubectl enabled)" -ForegroundColor Green + Write-Host " Linked AKS private DNS zone -> agent VNet" -ForegroundColor Green } else { - Write-Host " (warning: could not link AKS DNS zone; agent falls back to 'az aks command invoke')" -ForegroundColor Yellow + Write-Host " (warning: could not link AKS DNS zone to agent VNet)" -ForegroundColor Yellow } } } else { diff --git a/labs/zava-aks-postgres/scripts/pre-down.ps1 b/labs/zava-aks-postgres/scripts/pre-down.ps1 index 9ce326d02..69482cd2e 100644 --- a/labs/zava-aks-postgres/scripts/pre-down.ps1 +++ b/labs/zava-aks-postgres/scripts/pre-down.ps1 @@ -1,9 +1,8 @@ #Requires -Version 7.4 <# .SYNOPSIS - azd `predown` hook — unlinks the Azure Monitor Private Link Scope (AMPLS) - before teardown so `azd down --purge` can delete/purge the Log Analytics - workspace and Application Insights component. + azd `predown` hook — removes subscription-level demo access and unlinks the + Azure Monitor Private Link Scope (AMPLS) before resource-group teardown. .DESCRIPTION AMPLS pins its scoped resources: a Log Analytics workspace (or App Insights @@ -12,9 +11,9 @@ `CannotDeleteWorkspaceWhenLinkedToPrivateLinkScopes`, which aborts the whole `azd down` and orphans the resource group. - This hook removes every scopedResource from every AMPLS in the resource group - (cheap, instant, and reversible — a re-provision recreates them), so the - subsequent workspace/App-Insights deletion + soft-delete purge succeeds. + This hook removes the runtime identity's subscription Reader assignment and + every scopedResource from every AMPLS in the resource group. A later + re-provision recreates both. Idempotent: a no-op when there is no AMPLS (e.g. the lab was deployed with the private-link module disabled) or when the resource group is already gone. @@ -31,10 +30,10 @@ Set-StrictMode -Version Latest if (-not $ResourceGroup) { # RESOURCE_GROUP is an azd OUTPUT (only persisted after a fully successful - # provision); ZAVA_RG_NAME is the always-present INPUT. Fall back to it so - # teardown of a partially-provisioned env (the exact case this hook hardens) - # still finds the RG. Guard against `azd env get-value`'s "ERROR: ..." string - # for a missing key (non-zero exit, printed to stdout). + # provision); ZAVA_RG_NAME is an optional override. Fall back to the standard + # rg-$AZURE_ENV_NAME default so teardown of a partially-provisioned env still + # finds the RG. Guard against `azd env get-value`'s "ERROR: ..." string for a + # missing key (non-zero exit, printed to stdout). foreach ($k in 'RESOURCE_GROUP', 'ZAVA_RG_NAME') { $v = [Environment]::GetEnvironmentVariable($k) if (-not $v) { @@ -43,15 +42,62 @@ if (-not $ResourceGroup) { } if ($v -and "$v".Trim() -and "$v" -notmatch '^ERROR') { $ResourceGroup = "$v".Trim(); break } } + if (-not $ResourceGroup) { + $environmentName = [Environment]::GetEnvironmentVariable('AZURE_ENV_NAME') + if ($environmentName) { $ResourceGroup = "rg-$environmentName" } + } } -if (-not $ResourceGroup) { - Write-Host "pre-down: RESOURCE_GROUP not set — nothing to unlink." -ForegroundColor DarkGray +$savedPrincipalId = [Environment]::GetEnvironmentVariable('SRE_AGENT_PRINCIPAL_ID') +if (-not $savedPrincipalId) { + $savedPrincipalId = azd env get-value SRE_AGENT_PRINCIPAL_ID 2>$null + if ($LASTEXITCODE -ne 0 -or "$savedPrincipalId" -match '^ERROR') { + $savedPrincipalId = $null + } +} + +if (-not $ResourceGroup -and -not $savedPrincipalId) { + Write-Host "pre-down: neither resource group nor saved runtime principal is available — nothing to clean." -ForegroundColor DarkGray return } -if ((az group exists -n $ResourceGroup 2>$null) -ne 'true') { - Write-Host "pre-down: resource group '$ResourceGroup' does not exist — nothing to unlink." -ForegroundColor DarkGray +# The correlation role assignment lives outside the resource group and would +# otherwise survive `azd down`. +$resourceGroupExists = $ResourceGroup -and ((az group exists -n $ResourceGroup 2>$null) -eq 'true') +$principalIds = @() +if ($savedPrincipalId) { + $principalIds += "$savedPrincipalId".Trim() +} +if ($resourceGroupExists) { + $livePrincipalIds = az identity list -g $ResourceGroup ` + --query "[?starts_with(name, 'id-sre-agent-')].principalId" -o tsv 2>$null + $principalIds += @($livePrincipalIds -split "`n" | Where-Object { $_ } | ForEach-Object { $_.Trim() }) +} +$principalIds = @($principalIds | Where-Object { $_ } | Sort-Object -Unique) + +$subscriptionId = az account show --query id -o tsv 2>$null +if ($LASTEXITCODE -eq 0 -and $subscriptionId) { + $subscriptionScope = "/subscriptions/$subscriptionId" + foreach ($principalId in $principalIds) { + $assignmentIds = az role assignment list ` + --assignee-object-id $principalId ` + --scope $subscriptionScope ` + --query "[?roleDefinitionName=='Reader' && scope=='$subscriptionScope'].id" ` + -o tsv 2>$null + + foreach ($assignmentId in ($assignmentIds -split "`n" | Where-Object { $_ })) { + $assignmentId = $assignmentId.Trim() + Write-Host "pre-down: removing subscription Reader assignment for runtime identity..." + az role assignment delete --ids $assignmentId 2>$null + if ($LASTEXITCODE -ne 0) { + Write-Host " (warning: failed to remove '$assignmentId'; remove it manually to avoid an orphaned assignment)" -ForegroundColor Yellow + } + } + } +} + +if (-not $resourceGroupExists) { + Write-Host "pre-down: resource group is absent; subscription assignment cleanup is complete." -ForegroundColor DarkGray return } @@ -90,4 +136,4 @@ foreach ($ampls in ($amplsNames -split "`n" | Where-Object { $_ })) { } } -Write-Host "pre-down: AMPLS unlink complete — workspace/App Insights are now deletable." -ForegroundColor Green +Write-Host "pre-down: external access cleanup and AMPLS unlink complete." -ForegroundColor Green diff --git a/labs/zava-aks-postgres/scripts/setup-sre-agent.ps1 b/labs/zava-aks-postgres/scripts/setup-sre-agent.ps1 index 379c44824..010332660 100644 --- a/labs/zava-aks-postgres/scripts/setup-sre-agent.ps1 +++ b/labs/zava-aks-postgres/scripts/setup-sre-agent.ps1 @@ -5,7 +5,7 @@ .DESCRIPTION Most agent configuration is now declarative in Bicep (infra/modules/sre-agent.bicep): autonomous mode, AzMonitor incident - platform, connectors (app-insights, log-analytics, azure-monitor, microsoft-learn), + platform, connectors (app-insights, log-analytics, azure-monitor, learn-docs), custom skills, and incident filters / response plans all flow through Microsoft.App/agents/* ARM resources. @@ -17,6 +17,7 @@ and there is NO ARM/Bicep property for per-tool state (the agent's `permissions` stays null) — Microsoft's own `srectl tool config set` CLI exists for exactly this (POST /api/v2/agent/tools/configure). + - Agent-global custom instructions sync (the cross-alert correlation nudge) - Verification of Bicep-deployed assets .EXAMPLE .\scripts\setup-sre-agent.ps1 @@ -195,15 +196,32 @@ Write-Host (" Summary: {0} uploaded, {1} replaced, {2} skipped, {3} failed (of # exactly this. The underlying call is POST /api/v2/agent/tools/configure with # merge semantics: { overrides: [{ name, enabled }] }. # -# The tools only appear in the catalog AFTER the microsoft-learn MCP connector +# The tools only appear in the catalog AFTER the learn-docs connector # completes its first tools/list handshake (which needs the GitHub-raw firewall # allow in vnet.bicep + a warm connection), so we poll for them before enabling. Write-Host "`nStep 2b: Enabling Microsoft Learn MCP tools globally..." -ForegroundColor Yellow -$learnTools = @( - 'microsoft-learn_microsoft_docs_search', - 'microsoft-learn_microsoft_code_sample_search', - 'microsoft-learn_microsoft_docs_fetch' +$learnToolSets = @( + [pscustomobject]@{ + Connector = 'learn-docs' + Tools = @( + 'learn-docs_microsoft_docs_search', + 'learn-docs_microsoft_code_sample_search', + 'learn-docs_microsoft_docs_fetch' + ) + }, + # Migration compatibility for an azd run that compiled the old template + # before this repository was updated to the azd-safe connector name. + [pscustomobject]@{ + Connector = 'microsoft-learn' + Tools = @( + 'microsoft-learn_microsoft_docs_search', + 'microsoft-learn_microsoft_code_sample_search', + 'microsoft-learn_microsoft_docs_fetch' + ) + } ) +$learnConnectorName = $learnToolSets[0].Connector +$learnTools = $learnToolSets[0].Tools $catalog = @(); $present = @() $toolDeadline = (Get-Date).AddMinutes(3) do { @@ -211,14 +229,22 @@ do { $tr = $client.GetAsync("$agentEndpoint/api/v2/agent/tools").Result if ($tr.IsSuccessStatusCode) { $catalog = @(($tr.Content.ReadAsStringAsync().Result | ConvertFrom-Json).data) } } catch {} - $present = @($learnTools | Where-Object { $_ -in $catalog.name }) + + foreach ($toolSet in $learnToolSets) { + $candidatePresent = @($toolSet.Tools | Where-Object { $_ -in $catalog.name }) + if ($candidatePresent.Count -gt $present.Count) { + $learnConnectorName = $toolSet.Connector + $learnTools = $toolSet.Tools + $present = $candidatePresent + } + } if ($present.Count -eq $learnTools.Count) { break } Start-Sleep -Seconds 15 } while ((Get-Date) -lt $toolDeadline) if ($present.Count -lt $learnTools.Count) { Write-Host " [WARN] Only $($present.Count)/$($learnTools.Count) Learn MCP tools visible in the catalog yet — the" -ForegroundColor Yellow - Write-Host " microsoft-learn MCP connection is still warming up (it fetches its server bits from" -ForegroundColor Yellow + Write-Host " $learnConnectorName connection is still warming up (it fetches its server bits from" -ForegroundColor Yellow Write-Host " raw.githubusercontent.com; confirm the allow-github-raw-mcp-bits firewall rule exists)." -ForegroundColor Yellow Write-Host " Re-run this script shortly to finish enabling them." -ForegroundColor Yellow } @@ -239,23 +265,101 @@ if ($present.Count -gt 0) { } } +# --- Step 2c: Sync custom instructions (data-plane only) -------------------- +# Custom instructions are the agent-scoped, ALWAYS-ON prompt appended to EVERY +# thread — chat, incident, scheduled task — regardless of which response plan or +# skill matched. This is the surface the portal's "Custom instructions" box writes. +# +# Data-plane contract: +# GET/PUT {agentEndpoint}/api/v2/agent/customInstructions +# body: { "instructions": "" } +# +# We ship the correlation nudge: the cheap always-on trigger telling the agent an +# alert is a signal rather than the whole story, pointing at the +# `incident-correlation` SKILL (Bicep) for the actual queries. +Write-Host "`nStep 2c: Syncing custom instructions..." -ForegroundColor Yellow +$ciPath = Join-Path $PSScriptRoot "..\sre-config\custom-instructions.md" +$ciText = $null +$ciCurrent = $null +$normalizeInstructions = { param($s) if ($null -eq $s) { '' } else { $s.Replace("`r", '').Trim() } } +if (-not (Test-Path $ciPath)) { + Write-Host " WARNING: sre-config/custom-instructions.md is missing; correlation guidance cannot be synced." -ForegroundColor Yellow +} else { + # The file content IS the payload verbatim — there is no metadata wrapper and + # no comment syntax to strip, so keep rationale in AGENTS.md, never in here. + $ciText = ([System.IO.File]::ReadAllText($ciPath)).Replace('@@RG@@', $ResourceGroup).Trim() + + # Compare against what's live so a re-run is a no-op. The service normalises + # line endings to CRLF on write, so strip \r on BOTH sides before comparing — + # otherwise a file saved with LF looks "changed" on every single run. + try { + $getResp = $client.GetAsync("$agentEndpoint/api/v2/agent/customInstructions").Result + if ($getResp.IsSuccessStatusCode) { + $ciCurrent = ($getResp.Content.ReadAsStringAsync().Result | ConvertFrom-Json).instructions + } + } catch {} + + if ((& $normalizeInstructions $ciCurrent) -eq (& $normalizeInstructions $ciText)) { + Write-Host " [skip] custom instructions unchanged ($($ciText.Length) chars)" -ForegroundColor DarkGray + } else { + $ciBody = @{ instructions = $ciText } | ConvertTo-Json -Depth 4 -Compress + $ciContent = [System.Net.Http.StringContent]::new($ciBody, [System.Text.Encoding]::UTF8, "application/json") + $ciResp = $client.PutAsync("$agentEndpoint/api/v2/agent/customInstructions", $ciContent).Result + if ($ciResp.IsSuccessStatusCode) { + $verb = if ([string]::IsNullOrWhiteSpace($ciCurrent)) { "set" } else { "replaced" } + Write-Host " [ok] custom instructions $verb ($($ciText.Length) chars, appended to every thread)" -ForegroundColor Green + } else { + # A 403/timeout from inside the agent sandbox usually means the exact-host + # allow-agent-data-plane firewall rule is missing. + Write-Host " WARNING: custom instructions returned $($ciResp.StatusCode): $($ciResp.Content.ReadAsStringAsync().Result)" -ForegroundColor Yellow + } + $ciContent.Dispose() + } +} + # --- Step 3: Verify Bicep-deployed assets ---------------------------------- Write-Host "`nStep 3: Verifying Bicep-deployed configuration..." -ForegroundColor Yellow $allGood = $true +$armToken = (az account get-access-token --resource "https://management.azure.com/" --query accessToken -o tsv 2>$null).Trim() +if (-not $armToken) { + throw "Could not acquire an Azure Resource Manager token for post-provision verification." +} +$armHeaders = @{ Authorization = "Bearer $armToken"; Accept = "application/json" } function Get-AgentChildren { param([string]$Kind) - (az rest --method GET --url "${agentArmId}/${Kind}?api-version=$apiVersion" 2>$null | ConvertFrom-Json).value + + $url = "https://management.azure.com${agentArmId}/${Kind}?api-version=$apiVersion" + for ($attempt = 1; $attempt -le 6; $attempt++) { + try { + $response = Invoke-RestMethod -Method Get -Uri $url -Headers $armHeaders + $valueProperty = $response.PSObject.Properties['value'] + if ($valueProperty) { + return @($valueProperty.Value) + } + } catch { + if ($attempt -eq 6) { + throw "Could not list agent $Kind after $attempt attempts: $($_.Exception.Message)" + } + } + + if ($attempt -lt 6) { Start-Sleep -Seconds 5 } + } + + throw "Agent $Kind list response did not contain a value collection after 6 attempts." } $connectors = @(Get-AgentChildren -Kind "connectors") -$expectedConnectors = @("app-insights","log-analytics","azure-monitor","microsoft-learn") +$expectedConnectors = @("app-insights","log-analytics","azure-monitor") $missingConnectors = $expectedConnectors | Where-Object { $_ -notin $connectors.name } -if (-not $missingConnectors) { Write-Host " [OK] Connectors: $($connectors.Count) (app-insights, log-analytics, azure-monitor, microsoft-learn)" -ForegroundColor Green } +$learnConnector = $connectors | Where-Object { $_.name -in @("learn-docs", "microsoft-learn") } | Select-Object -First 1 +if (-not $learnConnector) { $missingConnectors += "learn-docs" } +else { $learnConnectorName = $learnConnector.name } +if (-not $missingConnectors) { Write-Host " [OK] Connectors: $($connectors.Count) (app-insights, log-analytics, azure-monitor, $($learnConnector.name))" -ForegroundColor Green } else { Write-Host " [MISSING] Connectors: $($missingConnectors -join ', ') — re-run azd provision" -ForegroundColor Red; $allGood = $false } $skills = @(Get-AgentChildren -Kind "skills") -$expectedSkills = @("database-incidents","performance-incidents","application-incidents","general-triage","proactive-health-check") +$expectedSkills = @("database-incidents","performance-incidents","application-incidents","general-triage","proactive-health-check","incident-correlation") $missingSkills = $expectedSkills | Where-Object { $_ -notin $skills.name } if (-not $missingSkills) { Write-Host " [OK] Custom skills: $($skills.Count)" -ForegroundColor Green } else { Write-Host " [MISSING] Skills: $($missingSkills -join ', ') — re-run azd provision" -ForegroundColor Red; $allGood = $false } @@ -287,6 +391,27 @@ if ($expectedKb.Count -eq 0) { Write-Host " [MISSING] Knowledge files: $($missingKb -join ', ') — re-run Step 2 (upload) above" -ForegroundColor Red; $allGood = $false } +$verifiedInstructions = $null +$customInstructionsVerified = $false +if (-not $ciText) { + Write-Host " [MISSING] Custom instructions source file — restore sre-config/custom-instructions.md" -ForegroundColor Red + $allGood = $false +} else { + try { + $verifyCiResp = $client.GetAsync("$agentEndpoint/api/v2/agent/customInstructions").Result + if ($verifyCiResp.IsSuccessStatusCode) { + $verifiedInstructions = ($verifyCiResp.Content.ReadAsStringAsync().Result | ConvertFrom-Json).instructions + } + } catch {} + if ((& $normalizeInstructions $verifiedInstructions) -eq (& $normalizeInstructions $ciText)) { + Write-Host " [OK] Custom instructions match local source ($($ciText.Length) chars)" -ForegroundColor Green + $customInstructionsVerified = $true + } else { + Write-Host " [MISSING] Custom instructions do not match local source — re-run Step 2c" -ForegroundColor Red + $allGood = $false + } +} + if ($agent.properties.actionConfiguration.mode -ne "autonomous") { Write-Host " [WARN] Agent mode: $($agent.properties.actionConfiguration.mode) (expected autonomous)" -ForegroundColor Yellow; $allGood = $false } else { Write-Host " [OK] Mode: autonomous + access $($agent.properties.actionConfiguration.accessLevel)" -ForegroundColor Green } @@ -307,11 +432,11 @@ try { if ($learnEnabled.Count -eq $learnTools.Count) { Write-Host " [OK] Microsoft Learn MCP tools enabled globally: $($learnEnabled.Count)/$($learnTools.Count)" -ForegroundColor Green } else { - Write-Host " [WARN] Learn MCP tools enabled globally: $($learnEnabled.Count)/$($learnTools.Count) (MCP connection may still be warming up)" -ForegroundColor Yellow; $allGood = $false + Write-Host " [WARN] Learn MCP tools enabled globally: $($learnEnabled.Count)/$($learnTools.Count) (MCP connection may still be warming up)" -ForegroundColor Yellow } -if ($allGood) { Write-Host " All Bicep + data-plane assets verified." -ForegroundColor Green } -else { Write-Host " Some assets missing — see above." -ForegroundColor Yellow } +if ($allGood) { Write-Host " All required Bicep + data-plane assets verified." -ForegroundColor Green } +else { Write-Host " Required assets are missing — see above." -ForegroundColor Red } $client.Dispose() @@ -323,15 +448,28 @@ Write-Host "========================================`n" -ForegroundColor Cyan Write-Host " DEPLOYED BY BICEP (verified above, not done by this script):" -ForegroundColor DarkGray Write-Host " [x] Agent: autonomous mode + High access" Write-Host " [x] Incident platform: Azure Monitor" -Write-Host " [x] Connectors: app-insights, log-analytics, azure-monitor, microsoft-learn" -Write-Host " [x] Custom skills: database-incidents, performance-incidents, application-incidents, general-triage, proactive-health-check" +Write-Host " [x] Connectors: app-insights, log-analytics, azure-monitor, $learnConnectorName" +Write-Host " [x] Custom skills: database-incidents, performance-incidents, application-incidents, general-triage, proactive-health-check, incident-correlation" Write-Host " [x] Response plans (incident filters): zava-database, zava-performance, zava-application, zava-unknown" Write-Host "`n DONE BY THIS SCRIPT (data plane — no ARM API yet):" -ForegroundColor Cyan Write-Host (" [x] Knowledge files synced: {0} local file(s) ({1} uploaded, {2} replaced, {3} skipped, {4} failed)" -f $kbLocalFiles.Count, $uploaded, $replaced, $skipped, $failed) -Write-Host (" [x] Microsoft Learn MCP tools enabled globally: {0}/{1} (docs_search, code_sample_search, docs_fetch)" -f $learnEnabled.Count, $learnTools.Count) +if ($learnEnabled.Count -eq $learnTools.Count) { + Write-Host (" [x] Microsoft Learn MCP tools enabled globally: {0}/{1} (docs_search, code_sample_search, docs_fetch)" -f $learnEnabled.Count, $learnTools.Count) +} else { + Write-Host (" [!] Microsoft Learn MCP tools enabled globally: {0}/{1} (connector warm-up/runtime issue; nonfatal)" -f $learnEnabled.Count, $learnTools.Count) -ForegroundColor Yellow +} +if ($customInstructionsVerified) { + Write-Host (" [x] Custom instructions synced and verified: {0} chars" -f $ciText.Length) +} else { + Write-Host " [ ] Custom instructions not verified" -ForegroundColor Red +} Write-Host "`n NEXT STEPS:" -ForegroundColor Cyan Write-Host " Run a break scenario:" Write-Host " .\.github\skills\running-demo\scripts\break-sql.ps1 # Stop PostgreSQL" Write-Host " .\.github\skills\running-demo\scripts\break-network.ps1 # Block DB traffic" Write-Host " .\.github\skills\running-demo\scripts\break-db-perf.ps1 # Drop index" +Write-Host " .\.github\skills\running-demo\scripts\break-bad-deploy.ps1 # Ship a bad rollout" +Write-Host " .\.github\skills\running-demo\scripts\break-compound.ps1 # Two independent faults" Write-Host " Watch the agent: https://sre.azure.com/agents$agentArmId`n" + +if (-not $allGood) { exit 1 } diff --git a/labs/zava-aks-postgres/sre-config/custom-instructions.md b/labs/zava-aks-postgres/sre-config/custom-instructions.md new file mode 100644 index 000000000..46b92ce7e --- /dev/null +++ b/labs/zava-aks-postgres/sre-config/custom-instructions.md @@ -0,0 +1,21 @@ +## Incident context is partial by construction + +When you are working an alert, remember that it arrives in a thread that cannot +see other alerts or investigations. That isolation is a platform artifact — it +is not evidence that nothing else is happening. + +Before you commit to a root cause, widen the frame: what else fired nearby, +which alert rules are muted on the resource, and whether Azure Service Health +already explains it. + +Use the `incident-correlation` skill for the nearby-alert check; do not substitute +alert-rule inventory for fired-alert history. If another alert fired in the same +resource group within 10 minutes, shared timing and resource group prove only +overlap. Require a direct mechanism: for HTTP failures, split dependencies by +target and result code. Slow successful PostgreSQL calls cannot explain HTTP +500s whose only failed dependency is an app-local target. Report independent +causes when mechanisms differ. If the other alert is already acknowledged, +leave its remediation to its own thread. + +When no nearby alert exists and the evidence is already clear, do not force a +correlation sweep. "I checked; this alert is the whole story" is complete. diff --git a/labs/zava-aks-postgres/sre-config/knowledge-base/zava-architecture.md b/labs/zava-aks-postgres/sre-config/knowledge-base/zava-architecture.md index a035fc400..40cce9627 100644 --- a/labs/zava-aks-postgres/sre-config/knowledge-base/zava-architecture.md +++ b/labs/zava-aks-postgres/sre-config/knowledge-base/zava-architecture.md @@ -5,12 +5,11 @@ A Node.js e-commerce app (`zava-storefront` + `zava-api`) on AKS with Azure Post ## Environment specifics ### How your sandbox reaches things (and what it can't) -You operate with your **own managed identity** (no app credentials, no passwords). Your sandbox egress is forced through an Azure Firewall (allow-list: ARM, Entra, Graph, Microsoft Learn) **and a TLS-inspecting forward proxy** that re-signs certificates. The firewall ALSO permits the agent subnet to reach the AKS API server (so you **can use native `kubectl`**). Azure Monitor is **private-only by default**: the public `AzureMonitor` tag is dropped and your Monitor DNS resolves to the AMPLS private endpoint, but your Log Analytics / Application Insights query tools work normally over it — just query as usual. Surfaces: +You operate with your **own managed identity** (no app credentials, no passwords). Your sandbox egress is forced through an Azure Firewall (allow-list: ARM, Entra, Graph, Microsoft Learn). Azure Monitor is **private-only by default**: the public `AzureMonitor` tag is dropped and your Monitor DNS resolves to the AMPLS private endpoint, but your Log Analytics / Application Insights query tools work normally over it — just query as usual. Surfaces: 1. **ARM control plane** via `az` (`RunAzCliReadCommands` / `RunAzCliWriteCommands`) — PG state/start/stop/parameters, NSG rules, role lookups, identity, AKS metadata, Azure Monitor. -2. **Kubernetes** via native `kubectl`, which you run yourself as a bash command in your sandbox terminal (`RunInTerminal`). It is authenticated by your Entra identity (AKS RBAC Cluster Admin). One-time setup per session: (a) `az aks get-credentials -g @@RG@@ -n --overwrite-existing`; (b) `kubelogin convert-kubeconfig -l azurecli` (non-interactive managed-identity auth — the default device-code flow hangs); (c) merge the egress-proxy CA `/etc/ssl/certs/adc-egress-proxy-ca.crt` into the kubeconfig cluster's `certificate-authority-data` so kubectl trusts the re-signed TLS. Then use kubectl directly for pods, logs, events, deployments, NetworkPolicies, rollouts. -3. **Terminal** via `RunInTerminal` — `python3`/`node`/scripts inside your sandbox; egress is the same firewalled allow-list. Use it for compute, not for opening DB sockets directly. -4. **PostgreSQL SQL** — run SQL (reads `pg_stat_*`, and read-mostly DDL like `CREATE INDEX CONCURRENTLY` / `ANALYZE`) through the in-cluster helper from an app pod (a real VNet NIC): `kubectl exec -n zava-demo deploy/zava-api -- node bin/run-sql.js ''` (reuses the app pod's PG Entra identity). +2. **Kubernetes** via the built-in `RunKubectlReadCommand` and `RunKubectlWriteCommand` system tools; they accept the same kubectl commands as a terminal. Incident runbooks use these tools directly rather than setting up terminal-native kubectl. +3. **PostgreSQL SQL** — use `RunKubectlWriteCommand` to run SQL (reads `pg_stat_*`, and read-mostly DDL like `CREATE INDEX CONCURRENTLY` / `ANALYZE`) through the in-cluster helper from an app pod: `kubectl exec -n zava-demo deploy/zava-api -- node bin/run-sql.js ''` (reuses the app pod's PG Entra identity). Note: do **not** rely on database tools that open their PostgreSQL connection from outside the platform-spoke VNet (where PostgreSQL lives) -- they can't reach this private server, and the resulting timeout can be misread as "stopped/network-blocked". Run SQL through the in-cluster helper instead. @@ -55,7 +54,7 @@ App regressions are often shipped by a deployment, not caused by infra. Every ch ## Hub-and-spoke network and the hub firewall -You run VNet-injected in your **own spoke** (`vnet-Zava-agent-*`, `agent-subnet` 10.30.0.0/28), with all egress forced through a **shared Azure Firewall in the hub** (`vnet-Zava-hub-*`) over VNet peering. The workload — AKS and PostgreSQL — sits in a separate **platform spoke** (`vnet-Zava-platform-*`). Your agent subnet is pinned to **your own region** (VNet injection is regional — the subnet must be in the same region as you), but that only fixes *where you run*, not *what you can reach*: peering lets you operate on resources in **other Azure regions** (global VNet peering) or **on-prem** (ExpressRoute/VPN) too — here everything you act on is co-regional, so no cross-region hop is needed. Nothing about how you operate changes: you reach the private AKS API server through native `kubectl` run from your sandbox terminal, PostgreSQL through the in-cluster `bin/run-sql.js` helper, ARM / Entra / Microsoft Learn over allow-listed HTTPS, and Azure Monitor (Log Analytics / App Insights) over the AMPLS private endpoint by default — all through the hub firewall, and your Monitor query tools work normally over the private path. You never need raw L3 reachability to the other spokes. +You run VNet-injected in your **own spoke** (`vnet-Zava-agent-*`, `agent-subnet` 10.30.0.0/27), with all egress forced through a **shared Azure Firewall in the hub** (`vnet-Zava-hub-*`) over VNet peering. The workload — AKS and PostgreSQL — sits in a separate **platform spoke** (`vnet-Zava-platform-*`). Your agent subnet is pinned to **your own region** (VNet injection is regional — the subnet must be in the same region as you), but that only fixes *where you run*, not *what you can reach*: peering lets you operate on resources in **other Azure regions** (global VNet peering) or **on-prem** (ExpressRoute/VPN) too — here everything you act on is co-regional, so no cross-region hop is needed. Use the built-in Kubernetes system tools for cluster operations; use ARM / Entra / Microsoft Learn over allow-listed HTTPS and Azure Monitor over the AMPLS private endpoint by default. When an incident has a network/egress dimension, the **hub Azure Firewall is itself an inspectable resource**: read its policy and rule collections over ARM (your Reader role covers `az network firewall [policy] show`), and see what it actually allowed or denied in the resource-specific **`AZFW*`** Log Analytics tables (`AZFWNetworkRule`, `AZFWApplicationRule`, `AZFWNatRule`, `AZFWDnsQuery`) — those tables exist because the firewall's diagnostic setting uses the `Dedicated` destination. There is no third-party network device in this environment, and your sandbox egress is allow-listed HTTPS only, so you cannot open a raw TCP/SSH socket to a device IP; a device's own telemetry (if one shipped syslog/CEF to this workspace) would be the path, never a direct connection. diff --git a/sreagent-templates/examples/vnet-integrated-keyvault/README.md b/sreagent-templates/examples/vnet-integrated-keyvault/README.md index 746ff5168..d51cf587b 100644 --- a/sreagent-templates/examples/vnet-integrated-keyvault/README.md +++ b/sreagent-templates/examples/vnet-integrated-keyvault/README.md @@ -27,7 +27,7 @@ refuses: | Resource | Setting that matters | |---|---| | VNet `sreagent-vnet` (`10.30.0.0/24`) | One VNet, two subnets | -| ‣ `agent-subnet` (`10.30.0.0/28`) | Delegated to `Microsoft.App/environments` - agent injected here | +| ‣ `agent-subnet` (`10.30.0.0/27`) | Delegated to `Microsoft.App/environments` - minimum 27 usable addresses after Azure reservations | | ‣ `pe-subnet` (`10.30.0.32/27`) | Holds the Key Vault private endpoint | | Key Vault | `public_network_access_enabled = false`, `default_action = Deny`, `bypass = None`, RBAC | | Private endpoint `pe-kv` | `subresource = vault`, in `pe-subnet` | diff --git a/sreagent-templates/examples/vnet-integrated-keyvault/variables.tf b/sreagent-templates/examples/vnet-integrated-keyvault/variables.tf index 187da8797..d748c1194 100644 --- a/sreagent-templates/examples/vnet-integrated-keyvault/variables.tf +++ b/sreagent-templates/examples/vnet-integrated-keyvault/variables.tf @@ -47,9 +47,9 @@ variable "vnet_address_space" { } variable "agent_subnet_prefix" { - description = "Delegated subnet the SRE Agent is injected into (Microsoft.App/environments)." + description = "Delegated /27-or-larger subnet the SRE Agent is injected into (Microsoft.App/environments)." type = string - default = "10.30.0.0/28" + default = "10.30.0.0/27" } variable "pe_subnet_prefix" {