diff --git a/.github/workflows/build-container.yaml b/.github/workflows/build-container.yaml index f19bc35..a13f629 100644 --- a/.github/workflows/build-container.yaml +++ b/.github/workflows/build-container.yaml @@ -1,4 +1,4 @@ -name: goreleaser +name: build on: pull_request: @@ -6,41 +6,69 @@ on: types: - published -permissions: - id-token: write - contents: write +# Default-deny at workflow level; each job grants only what its steps +# actually need. +permissions: {} jobs: - goreleaser: + # Dev-dry-run release: validates the goreleaser pipeline runs clean + # against the PR head commit. PRs from forks do not have secrets and + # cannot push tags, so passing the workflow through here is safe. + snapshot: + name: dev dry-run release + if: ${{ github.event_name == 'pull_request' }} runs-on: ubuntu-latest + permissions: + id-token: write # OIDC for cosign/skopa attestation if used later + contents: read # checkout only - no tag writes on PR steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 0 - name: Set up Go - uses: actions/setup-go@v5 + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: - go-version: stable + go-version: '1.26.x' + cache: true - - name: Dev Dry Run Release - if: ${{ github.event_name == 'pull_request' }} - uses: goreleaser/goreleaser-action@v5 + - name: Goreleaser snapshot + uses: goreleaser/goreleaser-action@f06c13b6b1a9625abc9e6e439d9c05a8f2190e94 # v7.2.3 with: distribution: goreleaser - version: '~> v2' args: release --snapshot --clean env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Production Release - if: ${{ github.event_name != 'pull_request' }} - uses: goreleaser/goreleaser-action@v5 + # Production release: publishes the staged cut only on pushed tags. + # contents: write is required for tag/version writes; id-token: write + # supports future signing actions. Snapshots above run on PRs and do + # not need either. + release: + name: production release + if: ${{ github.event_name != 'pull_request' }} + runs-on: ubuntu-latest + permissions: + id-token: write + contents: write + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + fetch-tags: true + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version: '1.26.x' + cache: true + + - name: Goreleaser release + uses: goreleaser/goreleaser-action@f06c13b6b1a9625abc9e6e439d9c05a8f2190e94 # v7.2.3 with: distribution: goreleaser - version: '~> v2' args: release --clean env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - diff --git a/.gitignore b/.gitignore index 11cb2b9..0f46724 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ *.dll *.so *.dylib +opensloctl # Test binary, built with `go test -c` *.test @@ -25,3 +26,8 @@ go.work.sum .env dist + +tmp/ + +# Local opencode CLI config with MCP server env (per-developer) +opencode.json diff --git a/AGENTS.md b/AGENTS.md index 610616e..4ffcd26 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,12 +1,24 @@ # AGENTS.md +> Worktree branch: `demo-slos`. Verify `git status` before committing - main branch layout may differ slightly. + ## Commands ``` +make build # go build -o opensloctl . make lint # golangci-lint run make test # go test ./... +make tidy # go mod tidy make load FILE= # parse and print OpenSlo specs -make generate FILE= OUTPUT= # generate Prometheus recording rules +make validate FILE= # validate OpenSlo specs without writing files +make generate FILE= OUTPUT= # generate Prometheus rules (-rules.yaml) +``` + +Via `go run` (supports `-r` recursive flag, Makefile targets do not): +``` +go run . load -f [-r] +go run . validate -f [-r] +go run . generate -f -o [-r] ``` Semconv registry (Weaver): @@ -14,21 +26,44 @@ Semconv registry (Weaver): make semconv-generate # registry YAML → pkg/semconv/semconv_gen.go make semconv-check # validate registry schema make semconv-stats # show registry statistics +make semconv-json # output registry JSON schema make semconv-diff BASE= # detect breaking changes vs base ref ``` ## Architecture -- `main.go` → `cmd.Execute()` — single entrypoint -- CLI: cobra-based, two subcommands: `load`, `generate` - - Both accept `-f` (filename, repeatable) and `-r` (recursive directory scan) +- `main.go` → `cmd.Execute()` - single entrypoint +- CLI: cobra-based, three subcommands: `load`, `validate`, `generate` + - All accept `-f` (filename, repeatable) and `-r` (recursive directory scan) - `generate` also requires `-o` (output directory) -- `pkg/specstore/loader.go` — loads YAML files via `openslosdk.Decode`, sorts into typed `OpenSloSpecs` struct -- `internal/generator/generator.go` — `Generator` interface -- `internal/generator/prometheusgenerator/` — generates Prometheus recording rule YAML from SLO specs using Go templates + sprig (embedded via `//go:embed`) -- `internal/feature/feature.go` — feature flags for multi-dimensional SLI annotations -- `pkg/semconv/semconv_gen.go` — **auto-generated** from semconv registry (do not edit manually) -- `pkg/util/file.go` — recursive YAML/YML file discovery + - `validate` runs full validation (load-time + generator-side) without writing files +- `pkg/specstore/loader.go` - loads YAML files via `openslosdk.Decode`, sorts into typed `OpenSloSpecs` struct +- `internal/generator/generator.go` - `Generator` interface +- `internal/generator/prometheusgenerator/` - generates Prometheus rules YAML from SLO specs using Go templates + sprig (embedded via `//go:embed`). One unified output file per SLO: `-rules.yaml` (covering recording rules and, if alert policies are referenced, alert rules via an `openslo-alerts-` group inside the same file). +- `internal/feature/feature.go` - feature flags for multi-dimensional SLI annotations +- `pkg/semconv/semconv_gen.go` - **auto-generated** from semconv registry (do not edit manually) +- `pkg/util/file.go` - recursive YAML/YML file discovery +- `semconv/registry/` - OpenTelemetry Weaver registry YAML (metrics + attributes) +- `semconv/templates/go/` - MiniJinja templates for semconv codegen +- `examples/-slo/specs/` - OpenSLO spec sets (input) +- `examples/-slo/rules/` - generated Prometheus rule files (output of `make generate`) +- `examples/-slo/kind/` - kind cluster + Helm harness (replaces the prior docker-compose harness) + - `setup.sh` - creates the cluster and installs the upstream helm chart + - `teardown.sh` - deletes the cluster + - `sync.sh` - re-applies the rules + dashboards ConfigMaps after re-running `make generate` +- `deploy/dashboards/` - repo-root OpenSLO Grafana dashboards (`openslo-list.json`, `openslo-detail.json`, `openslo-dashboards.yaml` provider) + +### Examples workflow + +Each `examples/-slo/` ships its own `Makefile` with these targets (shells out to `go run .` from the repo root via `git rev-parse --show-toplevel`): + +- `make verify` - `go run . load -f -r` (catch-all sanity check) +- `make generate` - `go run . generate -f -r -o `; passes `-r` implicitly +- `make lint-rules` - `promtool check rules` against every generated `/*.yaml` +- `make clean` - `rm -rf ` +- `oteldemo/` adds: `start-demo` / `stop-demo` (kind cluster lifecycle) and `sync` (`kind/sync.sh`) + +Root Makefile targets (`make load FILE=…` / `make validate FILE=…` / `make generate FILE=… OUTPUT=…`) do NOT pass `-r` - use `go run . … -r …` directly when you need it. ## Semconv Codegen Flow @@ -38,36 +73,74 @@ Run `make semconv-generate` after editing registry YAML or templates. `go genera ## Key Dependencies -- `github.com/OpenSLO/go-sdk` — official OpenSlo SDK for decoding specs (v0.9.2) -- `github.com/spf13/cobra` — CLI framework -- `log/slog` — structured logging (stdlib) -- `github.com/Masterminds/sprig/v3` — template functions -- OpenTelemetry Weaver — semconv registry management +- `github.com/OpenSLO/go-sdk` - official OpenSlo SDK for decoding specs (v0.9.2) +- `github.com/spf13/cobra` - CLI framework +- `log/slog` - structured logging (stdlib) +- `github.com/Masterminds/sprig/v3` - template functions +- OpenTelemetry Weaver - semconv registry management ## CI / Release - GoReleaser builds linux/darwin binaries, CGO_ENABLED=0 - `before` hooks: `go mod tidy` + `go generate ./...` -- `prerelease: auto` — tags with prerelease markers get prerelease release +- `prerelease: auto` - tags with prerelease markers get prerelease release ## Tooling -- `mise.toml` manages Go (1.26), golangci-lint, weaver -- `go.mod` declares `go 1.25.5` — auto-upgraded by SDK migration; trust mise for dev -- No `.golangci.yml` — uses defaults -- No tests exist — adding tests requires setting up from scratch +- `mise.toml` manages Go (1.26), golangci-lint, weaver, promtool +- `go.mod` declares `go 1.25.5` - auto-upgraded by SDK migration; trust mise for dev +- No `.golangci.yml` - uses defaults + +## Testing + +Snapshot tests use [`sebdah/goldie/v2`](https://github.com/sebdah/goldie) via the shared helper in `internal/testutil/`: + +- `pkg/specstore/specstore_test.go` - spec loading, multi-doc YAML, ref resolution +- `internal/testutil/golden_test.go` - unit tests for the helper itself +- `internal/testutil/golden.go` - `AssertGolden(t, fixtureDir, name, got)`; `name` must include a file extension +- `internal/generator/prometheusgenerator/prometheus_test.go` - table-driven generator suite (`TestGenerate_Golden` covers single-line / multi-line / ratio / multi-dim / tiered cases) +- `internal/generator/prometheusgenerator/labels_test.go` - label rendering helpers + +Fixtures live under each package's `testdata/` as `*.golden.yaml`. Update them with `go test .//... -update` after intentional generator/template changes, then visually diff the diff. + +Run a single package: `go test ./internal/generator/prometheusgenerator/...`. ## Gotchas -- `generate` rejects: empty `-o`, SLOs without `indicator`, ratio metrics (not supported) -- Only `ThresholdMetric` supported — `RatioMetric` returns error +- `generate` rejects: empty `-o`, SLOs without `indicator` +- Both `ThresholdMetric` and `RatioMetric` SLIs supported - ratio SLIs additionally emit `openslo_sli_event_rate_` recording series (see `ratio-slo.yaml` / `ratio-percent-slo.yaml` snapshots) - Non-OpenSlo YAML files silently skipped (continue on decode error) -- `semconv_gen.go` is auto-generated — never hand-edit +- `semconv_gen.go` is auto-generated - never hand-edit - Feature flags use SLO annotations: `multi-dimensional-sli.openslo.com/dimensions` + `multi-dimensional-sli.openslo.com/label` +- All scripting and testing scratch files (ad-hoc specs, output dirs, fixtures) must live in `./tmp` inside the repo - never `/tmp` or other system-global paths. The `./tmp` dir is gitignored scratch space scoped to this worktree. + +### SLI source conventions in `examples/oteldemo/specs/` + +- **Path Y** - frontend HTTP SERVER spans. Captures user-originated HTTP calls at the entry point. Filters: `service_name="frontend"` + `span_kind="SPAN_KIND_SERVER"` + `span_name=`. Used by 6 SLOs (ad-availability, cart-availability, product-catalog-availability, recommendation-availability, payment-unreachable, order-processing-latency). +- **Path Y (latency)** - same set + `le=` histogram bucket, plus `traces_span_metrics_duration_milliseconds_bucket` / `_count`. Used by order-processing-latency (`le="15000"`) and ad-latency (`le="1000"`). +- **Path Y-fauna** - service-side SERVER spans on internal services. Used by image-loading-latency (`service_name="frontend-proxy"`), post-order-email-availability + post-order-email-latency (`service_name="email"`). +- Source-service spans (frontend, image-provider, etc.) are now used rarely because they include flagd client-noise and INTERNAL span pollution. Stick to the path convention above. + +### chaos_flag label drift (intentional) + +`metadata.labels.chaos_flag` matches the demo user's intuition, not the flagd JSON canonical name. The flagd UI shows different (cleaner) names than the historical spec labels. Both forms exist; align later if we adopt canonical names everywhere. + +| SLO spec `chaos_flag` | flagd JSON canonical | match? | +|---|---|---| +| `adServiceFailure` | `adFailure` | ✗ drift | +| `cartServiceFailure` | `cartFailure` | ✗ drift | +| `paymentServiceUnreachable` | `paymentUnreachable` | ✗ drift | +| `recommendationServiceCacheFailure` | `recommendationCacheFailure` | ✗ drift | +| `imageSlowLoad` | `imageSlowLoad` | ✓ | +| `kafkaQueueProblems` | `kafkaQueueProblems` | ✓ | +| `emailMemoryLeak` | `emailMemoryLeak` | ✓ | +| `productCatalogFailure` | `productCatalogFailure` | ✓ | + +The metric source for Path Y SLOs lives on `service_name="frontend"` (HTTP SERVER), so the `chaos_flag` label attached to the backend service is **decorative on the recording rule's `chaos_flag` label** - useful for documentation but does NOT drive dashboard filter behavior. Dashboards must filter by `openslo_slo_name=`, not `chaos_flag=`, for Path-Y SLOs. ## SDK API Notes (github.com/OpenSLO/go-sdk) -- `SLIMetricSource.Spec` (not `MetricSourceSpec`) — `map[string]any` containing the query +- `SLIMetricSource.Spec` (not `MetricSourceSpec`) - `map[string]any` containing the query - `SLOObjective.Target` is `*float64` (pointer), not `float64` -- `SLOTimeWindow.Duration` is `v1.DurationShorthand` (struct), not `string` — use `.String()` for string representation +- `SLOTimeWindow.Duration` is `v1.DurationShorthand` (struct), not `string` - use `.String()` for string representation - `BudgetAdjustment` kind not supported in this SDK version diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..9c980ab --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,223 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +> Active development. Pin a specific release if you are relying on it. + +## [v0.2.0] + +> Generator v0.2.0 reaches parity with the v0.1.8-dev contract: the four +> alerting strategies (`error-rate`, `burn-rate`, `multi-burn-rate`, +> `multi-window-multi-burn-rate`) emit recorded SLO metadata plus +> severity-suffixed alerts; Grafana dashboards for the oteldemo +> bundle auto-provision; the otel-demo harness now uses kind + Helm in +> place of the previous docker-compose layout. + +### Breaking changes + +These are the breaking items integrators should plan around when moving +from <=v0.1.0 to v0.2.0: + +- **`specstore/loadSpecs` fails loudly on undelivered files.** Every + shadowed load inside the SDK decoder that previously dropped a file + to stderr now returns it as a structured `slog.Warn` and bubbles up + into a wrapped error from `GetSpecs`. Runners that relied on stray + YAML being silently ignored start failing CI. (`7f3b4e8`) +- **`openslo_slo_info` and `openslo_sli_*` series gain an + `openslo_slo_description` label.** The recording rule text carries + each spec's `spec.description` folded to one line. Dashboards matching + on exact label sets will see new series in addition to existing + ones. (`feat(generator, semconv): emit openslo_slo_description label`) +- **Alert rules now append the severity PascalCase to the alert + name.** `AdAvailability...` (severity not encoded in name) is + replaced by `AdAvailability...Page` and `AdAvailability...Ticket`. + The `severity` label is unchanged, so Prometheus and Alertmanager + routing still match. Alertname-based lookups in alert dashboards + and runbooks need to be updated. +- **`examples/oteldemo/deploy/` docker-compose harness removed.** + Replaced by `examples/oteldemo/kind/` which runs the upstream + `open-telemetry/opentelemetry-demo` Helm chart on a single-node + `kind` cluster. Top-level `make start-demo` target is gone; the + per-example `make -C examples/oteldemo start-demo` is the single + lifecycle entrypoint. + +### Added + +- **Generator templates for the four SRE strategies.** Single unified + `-rules.yaml` per SLO covering recording rules (info, + objective, timewindow, error budget, windowed `sli_error_rate_*`, + `sli_event_rate_*` for RatioMetric SLIs, current and period burn + rate, period error budget remaining, categorical status gauge) and + when alert policies exist - an `openslo-alerts-` group + inside the same file. +- **Multi-window-multi-burn-rate emitting shared reusable + AlertCondition definitions.** Eight conditions in oteldemo + `alert-conditions.yaml` cover page-fast (5m AND 1h at 14.4x), + page-slow (30m AND 6h at 6x), ticket-fast (2h AND 1d at 3x), + ticket-slow (6h AND 3d at 1x). Per-SLO AlertPolicies reference them + - no per-SLO alert duplication. +- **Categorical status gauge `openslo_slo_status`** with overridable + thresholds via `threshold.status.openslo.com/{warning,critical,breached}` + annotations. Defaults `1/6/14.4` follow the Google SRE Workbook. +- **Multi-dimensional SLI annotations** via + `multi-dimensional-sli.openslo.com/{label,dimensions}`. The + generator emits `_unlabeled` rule variants followed by a `label_join` + post-process group; one SLO becomes one series per value of the + chosen PromQL label. +- **Grafana dashboards with grafonnet mixins source of truth: + `deploy/mixins/*.jsonnet`**. Makefile `generate` + `release` + regenerates `deploy/dashboards/*.json`. See "Dashboards" below. +- **Spec-drift integrity rule and alert.** + `openslo_slo_metric_missing` recording rule + `OpenSloSpecDrift` page + alert, rendered from `deploy/mixins/rules/openslo-integrity.jsonnet` + to `deploy/rules/openslo-integrity-rules.yaml`. Caught + inconsistencies where the SDK has registered an SLO but the + underlying SLI query is producing no samples (typo in indicator + spec, denied Prom access, or a stuck recording rule). +- **`examples/oteldemo` kind + Helm harness.** Replaces the prior + docker-compose harness. `setup.sh` creates a single-node cluster + and installs the upstream Helm chart; `sync.sh` re-applies + ConfigMaps (delete+create to avoid the 256 KiB + `kubectl.kubernetes.io/last-applied-configuration` annotation + ceiling); `teardown.sh` deletes the cluster and release. +- **otel-demo Mixin customisations.** + `examples/oteldemo/kind/values.yaml` extends the otel-collector + histogram bucket list to include 30s and 60s for + `post-order-email-latency` (`le="30000"`) and `order-processing-latency` + (`le="60000"`). The chart's defaults miss these envelopes, so + ratio SLIs whose boundary sits above the chart's max bucket would + silently read `+Inf`. +- **`pkg/semconv` Weaver-managed registry.** New + `openslo.slo.description` semconv attribute and `openslo.alert.severity` + / `openslo.notification.target` carriers. Deprecated + `openslo.objective.decimal`, `openslo.objective.percent`, + `openslo.timewindow.duration` retained as Go constants for one + cycle, marked `deprecated.reason: obsoleted`. +- **`TestStatusRuleUsesBoolModifier` regression test** locks in the + Prom 3.x `bool` modifier on every status block comparison + (3 `>= bool`, 2 `< bool`, no `bool` in alert blocks). +- **`examples/multi-dim-slo/` example** with its dedicated README, + walker-friendly spec fixtures, and single README. + +### Changed + +- **`openslo_slo_info` label set**: now includes + `openslo_slo_description` (string, folded from `spec.description`, + capped at 200 chars, escape-quote/backslash). The recording rule + text is invariantly emitted so downstream Grafana `${openslo_slo_description}` + substitutions always resolve. +- **Alert naming convention**: `{SloNamePascal}{KindPascal}` now + suffixed with `{SeverityPascal}` per severity, line up with the + uniqueness check inside a single rule group. Same diff per severity, + alert block text unchanged. +- **`examples/oteldemo/specs/` inventory pruned from 12 to 11** + with the otel-demo natural baseline: + + | `ad-availability.yaml` | kept, slope 0.99/0.95 | + | `ad-latency.yaml` | new, ratio `le="2000"` | + | `cart-availability.yaml` | kept | + | `frontend-availability.yaml` | kept | + | `image-loading-latency.yaml` | new, ratio `le="2000"` on `image-provider` source | + | `order-processing-latency.yaml` | new, ratio `le="60000"` for checkout fan-out | + | `payment-unreachable.yaml` | kept, narrower than pre-v0.2.0 shape; trips on `paymentUnreachable` | + | `post-order-email-availability.yaml` | new, email SERVER `POST /send_order_confirmation`, trips on `emailMemoryLeak` | + | `post-order-email-latency.yaml` | new, `le="30000"` | + | `product-catalog-availability.yaml` | kept | + | `recommendation-availability.yaml` | kept | + +- **Path Y migration**: 6 SLOs moved to frontend HTTP SERVER route + span metrics (`service_name="frontend"`, `span_kind="SPAN_KIND_SERVER"`, + `span_name=`). Removes flagd-client noise (EventStream + reconnects, ResolveBoolean failures) that previously inflated error + rates by 7-9% on backend-service span metrics. +- **`chaos_flag` label drift documented in `AGENTS.md`**: SLO specs + use `adServiceFailure` / `cartServiceFailure` / + `paymentServiceUnreachable` / `recommendationServiceCacheFailure` + while flagd JSON canonical is `adFailure` / `cartFailure` / + `paymentUnreachable` / `recommendationCacheFailure`. Mature drift + on purpose for the demo. Aligning the canonical form remains future + work. +- **Target locked per SLO**: availability at 0.95 by default; + `payment-unreachable` and `order-processing-latency` at 0.99 because + their natural-baseline burn is below 1x only when aimed at this + tightest tier. +- **Status threshold annotations**: now strictly ascending and + positive at parse time; non-numeric values silently fall back to + defaults so spec-author scratch notes don't break generation. + +### Fixed + +- **Generator operator-precedence bug** in + `internal/generator/prometheusgenerator/prometheus.go`. PromQL parses + left-to-right at equal operator precedence, so the previous template + output `(1 - good) / total` rather than the intended + `1 - (good / total)`. For SLOs with healthy success rates the emitted + `openslo_sli_error_rate_*` was inflated roughly `1 / success_rate` + times larger, corrupting the downstream current-burn-rate and the + categorical status gauge. +- **Status gauge Prom 3.x compatibility**: every comparison in the + status recording rule now uses the `bool` modifier. Without it, + Prom 3.x comparison filters preserve the LHS series unchanged, so + `(burn_rate >= 14.4) * 3` returned the raw burn rate (e.g. `57.3`) + instead of `3`, breaking the dashboard's + `0/1/2/3 -> Healthy/Burning/Critical/Breached` mapping. +- **`pkg/specstore/loadSpecs` silent-skip bug.** Decoder errors are now + logged at warn and surfaced as a non-nil error from `GetSpecs`. + Stray top-level fields on an OpenSlo spec no longer vanish from + the generator run. +- **Specstore testdata restructure** to support the fail-loud + fix: the intentionally invalid fixtures moved from `testdata/` + into `testdata/invalid/`, away from recursive walks. + +### Removed + +- `examples/oteldemo/deploy/` directory (compose harness). +- `examples/oteldemo/openslo-dashboards.yaml` (legacy direct-mount + dashboard format, replaced by grafonnet mixins). +- `examples/oteldemo/dashboards/` orphaned dashboard stash. +- Top-level `Makefile` `start-demo` target (the per-example + `make -C examples/oteldemo start-demo` is the single entry-point). + +### Deprecated + +- `openslo.objective.decimal` / `openslo.objective.percent` / + `openslo.timewindow.duration` semconv attributes. Still emitted as + Go constants for one cycle. Removal is planned for the next major + version (>= v1.0.0). + +### Dashboards + +Two production-ready Grafana dashboards ship under `deploy/dashboards/`: + +- `openslo-list.json` (`OpenSLO - Manage SLOs`): one row per SLO with + Objective %, Period SLI (30d), categorical Status (0/1/2/3 mapped + to Healthy/Burning/Critical/Breached), and Budget Left % (color + bands 50/15). Drill-down link to per-SLO detail dashboard. +- `openslo-detail.json` (`OpenSLO - SLO detail`): markdown header + (name, description, target), SLI timeseries (28d SLI), Error + Budget Burndown (28d), Error Budget Burn Rate (multi-window). + Each row has its corresponding stat panel. + +Variables: `datasource` (Prometheus picker, `pluginId: prometheus`, +locked lowercase) and `slo` (label_values over `openslo_slo_name`). +Two hidden helpers hidden behind `hide: 2` are `description` (regex +extracts `openslo_slo_description` label) and `target` (regex +extracts the value of `openslo_slo_objective`) - used to render the +header markdown without cluttering the picker. + +Source of truth: `deploy/mixins/*.jsonnet` (grafonnet v13 + sprig). +Rendered by `make -C deploy/mixins generate`, committed as JSON via +`make sync-legacy`. + +### Security + +No security-sensitive changes in this iteration. The CLI does not +perform network I/O outside of `go build`/`go test`/`go run`, and +demo-harness scripts run against a local `kind` cluster the user +controls. + +[Unreleased]: https://github.com/thisisibrahimd/opensloctl/compare/v0.2.0...HEAD +[v0.2.0]: https://github.com/thisisibrahimd/opensloctl/compare/v0.1.0...v0.2.0 diff --git a/README.md b/README.md index b5908b6..a899e6c 100644 --- a/README.md +++ b/README.md @@ -1,505 +1,641 @@ # opensloctl -Generate Prometheus recording rules and alerting rules from OpenSlo specs. +Generate Prometheus recording rules and alerting rules from OpenSLO specs. -## Table of Contents +> **Status: active development.** Output shapes, code paths, config conventions, and example layouts can change between minor versions. Pin a specific release if you're relying on it, and expect breaking changes. We're shipping toward a stable `1.0`; right now everything below the generator interface is honest engineering work, not a finished product. +> +> Releases: - Changelog: [`CHANGELOG.md`](CHANGELOG.md) -- [Installation](#installation) -- [Development](#development) -- [Commands](#commands) -- [Usage](#usage) - - [Recording Rules](#recording-rules-slo-name-recording-rulesyaml) - - [Alert Rules](#alert-rules-slo-name-alert-rulesyaml) -- [Burn Rate Alerts](#burn-rate-alerts) - - [How It Works](#how-it-works-1) - - [Example](#example-api-latency-slo-with-page--ticket-alerts) - - [Why Four Alert Conditions](#why-four-alert-conditions) - - [Creating AlertConditions](#creating-alertconditions) - - [Creating AlertPolicies](#creating-alertpolicies) - - [Linking to SLOs](#linking-to-slos) - - [Validation](#validation) - - [Run the Examples](#run-the-examples) -- [Semantic Conventions](#semantic-conventions) +## Contents -## Installation +- [1. Quick start](#1-quick-start) +- [2. What gets generated](#2-what-gets-generated) +- [3. Writing specs](#3-writing-specs) +- [4. Alerting strategies](#4-alerting-strategies) + - [`error-rate` - SRE §§ 1-3](#error-rate--sre--s-s-1-3) + - [`burn-rate` - SRE § 4](#burn-rate--sre--s-4) + - [`multi-burn-rate` - SRE § 5](#multi-burn-rate--sre--s-5) + - [`multi-window-multi-burn-rate` - SRE § 6](#multi-window-multi-burn-rate--sre--s-6) + - [Choosing a strategy](#choosing-a-strategy) + - [Alert naming](#alert-naming) + - [Field reference](#field-reference) +- [5. Multi-dimensional SLIs](#5-multi-dimensional-slis) +- [6. Dashboards and integrity rules](#6-dashboards-and-integrity-rules) +- [7. Examples](#7-examples) +- [8. Semantic conventions](#8-semantic-conventions) +- [9. Development](#9-development) -### Via mise (GitHub backend) +## 1. Quick start -If you use [mise](https://mise.jdx.dev/), you can install opensloctl directly from GitHub releases: +Install the binary, validate one of the shipped examples, generate rules, push them into a Prometheus config. + +### Install ``` +# via mise (pins the version in mise.toml) mise use github:thisisibrahimd/opensloctl -``` - -This adds the tool to your local `mise.toml` and installs the latest release binary. After that, `opensloctl` is available on your PATH within the project. - -### From source -``` +# or from source go install github.com/thisisibrahimd/opensloctl@latest ``` -Or clone and build: +### Validate, generate, ship ``` -git clone https://github.com/thisisibrahimd/opensloctl.git -cd opensloctl -go build -o opensloctl . -``` - -## Development +# parses every spec under examples/api-latency-slo/, fails on bad refs or thresholds +opensloctl validate -f examples/api-latency-slo -This project uses [mise](https://mise.jdx.dev/) to manage tool versions (Go, golangci-lint, Weaver). +# writes one rules file per SLO into output/ +rm -rf output/ && mkdir output/ +opensloctl generate -f examples/api-latency-slo -o output/ +ls output/ +# api-latency-rules.yaml -### Install mise +# drop the file into Prometheus (rule_files: [./rules/*.yaml]) +opensloctl generate -f examples/api-latency-slo -o /etc/prometheus/rules/ +curl -X POST http://localhost:9090/-/reload -See [mise installation docs](https://mise.jdx.dev/getting-started.html). +# confirm the rules loaded +curl -s http://localhost:9090/api/v1/rules | jq '.data.groups[] | .name | select(startswith("openslo-"))' +``` -### Install project tools +### Checkpoint -Once mise is installed, run this in the repo root: +Pick the SLO and look up the categorical state: ``` -mise install +curl -s 'http://localhost:9090/api/v1/query?query=openslo_slo_status' | jq ``` -This installs the exact versions declared in `mise.toml`: -- **Go** 1.26 -- **golangci-lint** (latest) -- **Weaver** (latest) — for semantic convention registry management +A non-empty result with values in `{0, 1, 2, 3}` means the recording rules loaded and the SLI query resolved in your Prom. See [§2.2](#22-recording-rules) for the full set of generated metrics, [§2.3](#23-prom-3x-compatibility) for the one Prom 3.x subtlety that affects dashboards. -After installing, commands like `go`, `golangci-lint`, and `weaver` are available automatically in the project directory. +## 2. What gets generated -## Commands +One file per SLO, named `-rules.yaml`: ``` -go build -o opensloctl . # build binary -go run . load -f # parse and print OpenSlo specs -go run . generate -f -o # generate Prometheus recording rules -make semconv-generate # regenerate semconv_gen.go from registry -make semconv-check # validate registry schema -make lint # run golangci-lint -make test # run go test ./... +groups: + - name: openslo-sli-recordings- # always present + rules: + - openslo_slo_info, openslo_slo_objective, openslo_slo_timewindow_days, + openslo_slo_error_budget + - openslo_sli_error_rate_ for each window in the multi-window set + - openslo_sli_event_rate_ # RatioMetric SLOs only + - openslo_slo_current_burn_rate, openslo_slo_period_burn_rate, + openslo_slo_period_error_budget_remaining + - name: openslo-sli-recordings--unlabeled + label_join + rules: # multi-dimensional SLOs only + - name: openslo-alerts- # only when alertPolicies are referenced + rules: +``` + +### 2.1 Recording rules + +| Metric | Type | What it carries | +|---|---|---| +| `openslo_slo_info` | gauge (=1) | Identity record for the SLO. Carries `openslo_slo_name`, `openslo_slo_description` (folded one-liner from spec), `openslo_service_name`, `openslo_spec_version`. Demo SLOs add `chaos_flag`. | +| `openslo_slo_objective` | gauge | Spec target (e.g. 0.999 for 99.9%). | +| `openslo_slo_timewindow_days` | gauge | Spec `timeWindow` expressed in days (e.g. 28 for 28d). | +| `openslo_slo_error_budget` | gauge | `1 - objective`. (e.g. 0.001 for 99.9%). | +| `openslo_sli_error_rate_` | gauge | SLI error rate over the named window. Windows in the default set: `5m, 30m, 1h, 2h, 6h, 1d, 3d, 7d, 28d, 30d`. | +| `openslo_sli_event_rate_` | gauge (ratio only) | Events/sec over that window. Emitted only for `RatioMetric` SLOs since the spec exposes a `total` count query. | +| `openslo_slo_current_burn_rate` | gauge | `sli_error_rate_5m / error_budget`. Instantaneous burn rate. | +| `openslo_slo_period_burn_rate` | gauge | `sli_error_rate_ / error_budget`. Period burn rate over the spec's `timeWindow`. | +| `openslo_slo_period_error_budget_remaining` | gauge | `clamp_min(1 - period_burn_rate, 0)`. Bounded to `[0, 1]` so dashboards don't chart large negatives. | +| `openslo_slo_status` | gauge | Categorical 0-3 health state. See below. | + +### 2.2 Status gauge + +`openslo_slo_status` is a single integer gauge per SLO, dashboard-friendly because one lookup replaces three threshold comparisons: + +| Value | Label | Trigger on `openslo_slo_current_burn_rate` | +|-------|--------|---| +| `0` | Healthy | `< warning` | +| `1` | Burning | `warning <= x < critical` | +| `2` | Critical | `critical <= x < breached` | +| `3` | Breached | `>= breached` | + +Defaults follow Google SRE Workbook burn-rate reference points: **warning = 1x, critical = 6x, breached = 14.4x**. Override per-SLO via annotations: + +```yaml +metadata: + annotations: + threshold.status.openslo.com/warning: "1" + threshold.status.openslo.com/critical: "6" + threshold.status.openslo.com/breached: "14.4" ``` -## Usage +Each annotation is optional; missing ones fall back to defaults independently. The resolved triple must be strictly ascending and positive; otherwise `opensloctl validate` exits 1. Scratch notes (`TODO`) don't fail validation - non-numeric values silently fall back. -opensloctl reads OpenSlo SLO, SLI, AlertCondition, and AlertPolicy specs and generates two types of Prometheus rule files: +### 2.3 Prom 3.x compatibility -### Recording Rules (`-recording-rules.yaml`) +Every comparison inside the status block uses the `bool` modifier (`>= bool 14.4`, `< bool 6`, etc.). Without `bool`, Prometheus >= 0.19.0 applies default filter semantics at scalar operators - `(burn_rate >= 14.4)` returns the LHS series unchanged instead of `0`/`1` - and the dashboard's `Healthy/Burning/Critical/Breached` mapping never matches. The `bool` modifier has been stable since Prom 0.19.0 (Oct 2015), so the rules work on every Prom version in practical use. Alert expressions deliberately do NOT use `bool` since there, filter semantics is what you want. Test coverage: `internal/generator/prometheusgenerator.TestStatusRuleUsesBoolModifier` regresses every status block for exactly 3 `>= bool` and 2 `< bool`, and no `bool` survives into alert blocks. -For each SLO, opensloctl generates Prometheus recording rules that: +### 2.4 Alert rules -1. **Expose SLO metadata** — `openslo_slo_info`, `openslo_slo_objective`, `openslo_slo_timewindow_days`, `openslo_slo_error_budget` -2. **Pre-compute SLI error rates** — `openslo_sli_error_rate_5m`, `_30m`, `_1h`, `_2h`, `_6h`, `_1d`, `_3d`, `_7d`, `_28d`, `_30d` +When an SLO references `AlertPolicies`, opensloctl groups them by `severity` and emits one Prometheus alert per severity inside `openslo-alerts-`. The condition `kind` selects the strategy (see [§4](#4-alerting-strategies)). Severity stays in the `severity` label; the alert name is `{SloNamePascal}{KindPascal}` and is shared across severities for a given SLO+kind (Prometheus deduplicates same-name alerts within one group, so this is intentional). -The SLI error rate metrics are computed from your Prometheus query with window variables templated in. For example, if your SLI query is: +Alert name length grows with the SLO name. `PostOrderEmailLatencySloMultiWindowMultiBurnRate` is 49 characters - well under the typical 63-byte Prometheus label-value cap, but a reason to keep SLO names short if you also rely on `ALERTS{alertname=...}` lookups. -```promql -histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket{job="api"}[{{.Window}}])) by (le)) -``` +## 3. Writing specs -The generator produces a recording rule for each window: +OpenSlo specs are declarative YAML. opensloctl reads them and emits Prometheus rules; validation happens at load and at generate. + +### 3.1 Minimal SLO with inline SLI ```yaml -groups: - - name: openslo-sli-recordings-api-latency-slo - rules: - - record: openslo_sli_error_rate_5m - expr: histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket{job="api"}[5m])) by (le)) - labels: - openslo_slo_name: api-latency-slo - - record: openslo_sli_error_rate_30m - expr: histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket{job="api"}[30m])) by (le)) - labels: - openslo_slo_name: api-latency-slo +apiVersion: openslo/v1 +kind: SLO +metadata: + name: api-latency +spec: + service: api-gateway + indicator: + metadata: + name: api-gateway-latency-sli + spec: + thresholdMetric: + metricSource: + type: Prometheus + spec: + query: histogram_quantile(0.999, sum(rate(http_request_duration_seconds_bucket{job="api-gateway"}[{{.Window}}])) by (le)) + budgetingMethod: Occurrences + timeWindow: + - duration: 30d + isRolling: true + objectives: + - displayName: "P99 latency under 500ms" + target: 0.999 ``` -Multiline queries are preserved using YAML block scalars (`|`): +The `Service` definition can be inline (`spec.service: api-gateway`) or referenced (`spec.serviceRef`). Indicator definitions can be inline (`spec.indicator`) or referenced (`spec.indicatorRef`). Each SLO's spec author owns the 0-1 shape of the SLI - opensloctl passes PromQL through verbatim, so `thresholdMetric` queries must already return 0/1 per cycle (or `ratioMetric` must give a [0,1] fraction in the multi-window). Use `ratioMetric(counter=true)` when you can express good/total via classic bucket + calls series. + +### 3.2 Multi-line queries + +`{{.Window}}` is templated per recording-window. Multiline queries are preserved using YAML block scalars (`|`): ```yaml - - record: openslo_sli_error_rate_5m - expr: | - histogram_quantile(0.99, - sum(rate(http_request_duration_seconds_bucket{job="api"}[5m])) by (le)) +- record: openslo_sli_error_rate_5m + expr: | + histogram_quantile(0.999, + sum(rate(http_request_duration_seconds_bucket{job="api-gateway"}[5m])) by (le)) ``` -### Alert Rules (`-alert-rules.yaml`) +### 3.3 AlertCondition + AlertPolicy + SLO wiring + +```yaml +apiVersion: openslo/v1 +kind: AlertCondition +metadata: + name: api-latency-page +spec: + severity: page + condition: + kind: burn-rate + op: gte + threshold: 14.4 + lookbackWindow: 5m + alertAfter: 2m # optional -> Prom `for:` +--- +apiVersion: openslo/v1 +kind: AlertPolicy +metadata: + name: api-latency-page +spec: + description: Page tier (5m window) + alertWhenBreaching: true + conditions: + - conditionRef: api-latency-page # exactly one; SDK enforces this + notificationTargets: + - targetRef: oncall-pagerduty +``` -When an SLO references AlertPolicies with burn rate conditions, opensloctl generates Prometheus alerting rules. Conditions with the same severity are OR-ed together into a single alert rule: +Then on the SLO: ```yaml -groups: - - name: openslo-burnrate-alerts-api-latency-slo - rules: - - alert: OpenSLO_Page_BurnRate_api_latency_slo - expr: |- - openslo_sli_error_rate5m{openslo_slo_name="api-latency-slo"} / (1 - openslo_slo_objective{openslo_slo_name="api-latency-slo"}) gte 14.4 - or - openslo_sli_error_rate30m{openslo_slo_name="api-latency-slo"} / (1 - openslo_slo_objective{openslo_slo_name="api-latency-slo"}) gte 6.0 - for: 2m - labels: - severity: page - openslo_slo_name: api-latency-slo +spec: + ... + alertPolicies: + - alertPolicyRef: api-latency-page + - alertPolicyRef: api-latency-ticket ``` -## Burn Rate Alerts +OpenSLO SDK v0.9.2 enforces `SliceLength(1,1)` on `AlertPolicy.Spec.Conditions`. Use multiple AlertPolicies (one per condition) to orchestrate OR/AND across tiers and windows. -opensloctl supports generating Prometheus alerting rules from OpenSlo AlertCondition and AlertPolicy specs. Follow the [Google SRE Workbook multi-window multi-burn-rate](https://sre.google/workbook/alerting-on-slos/#6-multiwindow-multi-burn-rate-alerts) pattern. +### 3.4 Validation -### How It Works +opensloctl fails fast on load and on generate. Validation rejects: -1. Define **AlertConditions** with burn rate thresholds and windows -2. Group them into **AlertPolicies** (one condition per policy) -3. Reference policies from your **SLO** via `spec.alertPolicies[]` -4. All refs must resolve — validation runs on load +- Unknown `condition.kind` values +- `error-rate` thresholds outside `(0, 1]` +- `burn-rate`, `multi-burn-rate`, `multi-window-multi-burn-rate` thresholds <= 0 +- `multi-burn-rate` with fewer than 2 conditions per severity +- `multi-window-multi-burn-rate` with fewer than 2 tiers per severity +- `multi-window-multi-burn-rate` tier with only 1 condition (need short+long pair) +- `multi-window-multi-burn-rate` tier with non-matching thresholds across conditions +- `multi-window-multi-burn-rate` condition name that doesn't end in `-` +- Unresolved refs anywhere in the SLO -> AlertPolicy -> AlertCondition -> AlertNotificationTarget graph +- Label names containing hyphens (Prometheus requires `[a-zA-Z_][a-zA-Z0-9_]*` - use underscores) +- Multi-value labels (more than one entry per label key) -### Example: API Latency SLO with Page + Ticket Alerts +Ref errors look like: ``` -examples/api-latency-slo/ -├── service.yaml -├── datasource.yaml -├── sli.yaml # thresholdMetric: P99 latency -├── alert-condition-page.yaml # 14.4x burn rate, 5m window -├── alert-condition-ticket.yaml # 3x burn rate, 2h window -├── alert-policy-page.yaml # page → pagerduty -├── alert-policy-ticket.yaml # ticket → slack -├── notification-target-*.yaml -└── slo.yaml # references both policies +unresolved references: [unresolved ref: SLO "api-latency" references Service "missing-svc" not found] ``` -### Why Four Alert Conditions +and exit 1. A separate `loadSpecs` hardening in `pkg/specstore/loadSpecs` also `slog.Warn`s every file that fails to decode, so silent skips are gone - `make verify` or any `validate` run surfaces every off-shape file. -The Google SRE Workbook recommends **four AlertConditions** per SLO — two for page severity and two for ticket severity. Each condition represents a different burn rate window, and conditions within the same severity are **OR-ed** together. +## 4. Alerting strategies -``` -Page alerts fire if EITHER condition is true: - (14.4x burn rate over 5m) OR (6x burn rate over 30m) +opensloctl generates Prometheus alerting rules from OpenSlo AlertCondition and AlertPolicy specs. The condition's `kind` picks one of four strategies inspired by the [Google SRE Workbook alerting on SLOs](https://sre.google/workbook/alerting-on-slos/) chapters. Pick the one that matches how aggressively you want to be paged. -Ticket alerts fire if EITHER condition is true: - (3x burn rate over 2h) OR (1x burn rate over 6h) -``` +Indicators can be inlined on the SLO (`spec.indicator: ...`) or referenced via `spec.indicatorRef` - referenced SLIs are resolved at generation time and treated as if they were inlined. Inline takes precedence when both are set. -This is the **multi-window multi-burn-rate** pattern. You need two windows per severity to: +> The OpenSLO SDK only models the legacy `burnrate` kind. opensloctl accepts the four kebab-case names above plus `burnrate` for back-compat; `burnrate` is mapped to `multi-window-multi-burn-rate` with a one-shot warning at generation time. -1. **Catch sudden spikes** — the fast window (5m at 14.4x) fires immediately when error rate spikes hard -2. **Catch sustained degradation** — the slow window (30m at 6x) fires when error rate is moderately elevated for longer -3. **Reduce false positives** — both windows must agree on the burn rate within their respective timeframes, but the OR means you get alerted if either window detects the problem +### Choosing a strategy -The burn rate values are derived from the error budget math. For a 99.9% SLO (0.1% error budget): -- **14.4x** burns through the 30-day budget in ~2 hours -- **6x** burns through the 30-day budget in ~5 hours -- **3x** burns through the 30-day budget in ~10 hours -- **1x** burns through the 30-day budget in ~30 days (full budget exhaustion) +| Kind | SRE workbook | Used for | +|---|---|---| +| `error-rate` | §§ 1-3 | Raw SLI error rate vs an absolute threshold (e.g. 0.001 for a 99.9% SLO). One alert per severity; simplest possible setup. | +| `burn-rate` | § 4 | Single-window burn rate multiplier over the error budget. One alert per severity. | +| `multi-burn-rate` | § 5 | Two or more burn rate windows OR-ed. Each condition contributes one expression; no short/long pairing. | +| `multi-window-multi-burn-rate` | § 6 | Short+long window pairs AND-ed within a tier, OR-ed across tiers. Recommended when you can afford two windows per tier. | -### Creating AlertConditions +### `error-rate` - SRE §§ 1-3 -Define all four conditions, one per file: +Compares the SLI error rate directly to an absolute threshold. Cheap to write - one condition per severity, no tiering. ```yaml -# alert-condition-page-14x.yaml — fast page alert apiVersion: openslo/v1 kind: AlertCondition metadata: - name: api-latency-page-14x + name: api-latency-page spec: severity: page - description: Page on-call when API latency burn rate spikes hard condition: - kind: burnrate # only "burnrate" is supported - op: gte # gte, lte, gt, lt - threshold: 14.4 # burn rate multiplier - lookbackWindow: 5m # evaluation window - alertAfter: 2m # Prometheus "for" duration (optional) + kind: error-rate + op: gte + threshold: 0.001 # 1 - 0.999 for a 99.9% SLO + lookbackWindow: 5m + alertAfter: 2m +``` + +Generates: + +```yaml +- alert: ApiLatencyErrorRate + expr: openslo_sli_error_rate_5m{openslo_slo_name="api-latency"} >= 0.001000 + for: 2m + labels: + severity: page + openslo_slo_name: api-latency ``` +### `burn-rate` - SRE § 4 + +Single-window burn rate multiplier over the error budget. One alert per severity; the simplest meaningful burn alert. + ```yaml -# alert-condition-page-6x.yaml — slow page alert apiVersion: openslo/v1 kind: AlertCondition metadata: - name: api-latency-page-6x + name: checkout-page spec: severity: page condition: - kind: burnrate - threshold: 6 - lookbackWindow: 30m - alertAfter: 5m + kind: burn-rate + op: gte + threshold: 14.4 + lookbackWindow: 5m + alertAfter: 2m ``` +Generates: + ```yaml -# alert-condition-ticket-3x.yaml — fast ticket alert -apiVersion: openslo/v1 -kind: AlertCondition -metadata: - name: api-latency-ticket-3x -spec: - severity: ticket - condition: - kind: burnrate - threshold: 3 - lookbackWindow: 2h - alertAfter: 15m +- alert: CheckoutBurnRate + expr: openslo_sli_error_rate_5m{openslo_slo_name="checkout"} / (1 - openslo_slo_objective{openslo_slo_name="checkout"}) >= 14.400000 + for: 2m + labels: + severity: page + openslo_slo_name: checkout ``` +### `multi-burn-rate` - SRE § 5 + +Two or more burn rate windows OR-ed per severity. No short/long AND pairing - each condition contributes one expression and the alert fires when any condition fires. + ```yaml -# alert-condition-ticket-1x.yaml — slow ticket alert -apiVersion: openslo/v1 -kind: AlertCondition -metadata: - name: api-latency-ticket-1x -spec: - severity: ticket +# alert-condition-page-36x.yaml +- severity: page condition: - kind: burnrate - threshold: 1 - lookbackWindow: 6h - alertAfter: 30m + kind: multi-burn-rate + threshold: 36 + lookbackWindow: 5m +--- +# alert-condition-page-6x.yaml +- severity: page + condition: + kind: multi-burn-rate + threshold: 6 + lookbackWindow: 30m ``` -### How Conditions Are OR-ed - -Conditions with the same `severity` are grouped together and combined with **OR** logic in the generated Prometheus alerting rules: +Generates (conditions OR-ed per severity): ```yaml -# Generated Prometheus alert rule for page severity -- alert: openslo_slo_burn_rate - expr: | - ( - openslo_sli_error_rate_5m{openslo_slo_name="api-latency-slo"} > 14.4 * openslo_slo_error_budget{openslo_slo_name="api-latency-slo"} - ) +- alert: ApiLatencyMultiBurnRate + expr: |- + (openslo_sli_error_rate_5m{openslo_slo_name="api-latency"} / (1 - openslo_slo_objective{openslo_slo_name="api-latency"}) >= 36.000000) or - ( - openslo_sli_error_rate_30m{openslo_slo_name="api-latency-slo"} > 6 * openslo_slo_error_budget{openslo_slo_name="api-latency-slo"} - ) - for: 2m + (openslo_sli_error_rate_30m{openslo_slo_name="api-latency"} / (1 - openslo_slo_objective{openslo_slo_name="api-latency"}) >= 6.000000) labels: - openslo_slo_name: api-latency-slo severity: page + openslo_slo_name: api-latency ``` -Each severity gets its own alert rule. The page rule ORs both page conditions together. The ticket rule ORs both ticket conditions together. This means: -- If **either** the 5m or 30m window exceeds its threshold → page fires -- If **either** the 2h or 6h window exceeds its threshold → ticket fires +Structural rule: each severity needs >= 2 conditions. If you only want one threshold, use `burn-rate` instead. -### Creating AlertPolicies +### `multi-window-multi-burn-rate` - SRE § 6 -The OpenSlo SDK enforces exactly 1 condition per AlertPolicy. So you need 4 policies — one per condition. The **OR-ing happens at the Prometheus alert rule level**, not in the OpenSlo spec. +Short + long window pairs AND-ed within a tier, OR-ed across tiers. The fast/slow tiered setup is what the workbook recommends for production. ```yaml -# alert-policy-page-14x.yaml -apiVersion: openslo/v1 -kind: AlertPolicy -metadata: - name: api-latency-page-14x-alert -spec: - description: Page alert for 14.4x burn rate - alertWhenBreaching: true - conditions: - - conditionRef: api-latency-page-14x - notificationTargets: - - targetRef: oncall-pagerduty -``` - -Repeat for the other three conditions (page-6x, ticket-3x, ticket-1x). Each gets its own policy file. +# Tier 1: fast burn - catches a sudden spike +- name: page-fast-5m # tier "page-fast" (strip "-5m" suffix) + severity: page + condition: + kind: multi-window-multi-burn-rate + threshold: 14.4 + lookbackWindow: 5m + alertAfter: 2m +- name: page-fast-1h # tier "page-fast" (strip "-1h" suffix) + severity: page + condition: + kind: multi-window-multi-burn-rate + threshold: 14.4 # same threshold as 5m + lookbackWindow: 1h + alertAfter: 2m -### How Conditions Are OR-ed +# Tier 2: slow burn - catches sustained degradation +- name: page-slow-30m # tier "page-slow" + severity: page + condition: + kind: multi-window-multi-burn-rate + threshold: 6 + lookbackWindow: 30m + alertAfter: 5m +- name: page-slow-6h # tier "page-slow" + severity: page + condition: + kind: multi-window-multi-burn-rate + threshold: 6 + lookbackWindow: 6h + alertAfter: 5m +``` -The generator groups conditions by `severity` and creates **one Prometheus alert rule per severity** with OR logic: +Generates (AND in tier, OR across tiers): ```yaml -# Generated: page alert rule (ORs page-14x and page-6x) -- alert: openslo_slo_burn_rate - expr: | +- alert: ApiLatencyMultiWindowMultiBurnRate + expr: |- ( - openslo_sli_error_rate_5m{openslo_slo_name="api-latency-slo"} > 14.4 * openslo_slo_error_budget{openslo_slo_name="api-latency-slo"} - ) + openslo_sli_error_rate_5m{openslo_slo_name="api-latency"} / (1 - openslo_slo_objective{openslo_slo_name="api-latency"}) >= 14.400000 + and + openslo_sli_error_rate_1h{openslo_slo_name="api-latency"} / (1 - openslo_slo_objective{openslo_slo_name="api-latency"}) >= 14.400000) or ( - openslo_sli_error_rate_30m{openslo_slo_name="api-latency-slo"} > 6 * openslo_slo_error_budget{openslo_slo_name="api-latency-slo"} - ) + openslo_sli_error_rate_30m{openslo_slo_name="api-latency"} / (1 - openslo_slo_objective{openslo_slo_name="api-latency"}) >= 6.000000 + and + openslo_sli_error_rate_6h{openslo_slo_name="api-latency"} / (1 - openslo_slo_objective{openslo_slo_name="api-latency"}) >= 6.000000) for: 2m labels: - openslo_slo_name: api-latency-slo severity: page - -# Generated: ticket alert rule (ORs ticket-3x and ticket-1x) -- alert: openslo_slo_burn_rate - expr: | - ( - openslo_sli_error_rate_2h{openslo_slo_name="api-latency-slo"} > 3 * openslo_slo_error_budget{openslo_slo_name="api-latency-slo"} - ) - or - ( - openslo_sli_error_rate_6h{openslo_slo_name="api-latency-slo"} > 1 * openslo_slo_error_budget{openslo_slo_name="api-latency-slo"} - ) - for: 15m - labels: - openslo_slo_name: api-latency-slo - severity: ticket + openslo_slo_name: api-latency ``` -Page fires if **either** 5m@14.4x **or** 30m@6x fires. Ticket fires if **either** 2h@3x **or** 6h@1x fires. +Structural rules: -Notification targets: +- >= 2 tiers per severity (fast + slow, or any other split) +- >= 2 conditions per tier (short + long window) sharing the same threshold +- Condition names must end in `-` so the generator can derive the tier. `page-fast-5m` and `page-fast-1h` both belong to tier `page-fast` and get AND-ed. Renaming either breaks the pairing. -```yaml -apiVersion: openslo/v1 -kind: AlertNotificationTarget -metadata: - name: oncall-pagerduty -spec: - description: Page on-call engineer via PagerDuty - target: pagerduty -``` +Burn rate exhaustion at common SLO targets: -### Linking to SLOs +| Burn | 99.9% SLO (0.1% budget) | 95% SLO (5% budget) | +|---|---|---| +| 14.4x | ~2 hours | ~3.3 days | +| 6x | ~5 hours | ~8.3 days | +| 3x | ~10 hours | ~16.7 days | +| 1x | ~30 days (full period) | ~30 days (full period) | -Reference alert policies from your SLO: +### Alert naming -```yaml -apiVersion: openslo/v1 -kind: SLO -metadata: - name: api-latency-slo -spec: - service: api-gateway - indicatorRef: api-latency-p99 - budgetingMethod: Occurrences - timeWindow: - - duration: 30d - isRolling: true - objectives: - - displayName: "P99 latency < 500ms" - target: 0.999 - op: lte - value: 500 - alertPolicies: - - alertPolicyRef: api-latency-page-alert - - alertPolicyRef: api-latency-ticket-alert -``` +Names are PascalCase with no separators: `{SloNamePascal}{KindPascal}`. -### Inline Conditions and Targets +| Kind | Example alert name | +|---|---| +| `error-rate` | `AdAvailabilityErrorRate` | +| `burn-rate` | `CheckoutBurnRate` | +| `multi-burn-rate` | `ApiLatencyMultiBurnRate` | +| `multi-window-multi-burn-rate` | `AdAvailabilityMultiWindowMultiBurnRate` | -You can inline conditions and notification targets directly in the AlertPolicy instead of using refs: +All alerts for one SLO go to a single record group `openslo-alerts-`. Severity is carried in the `severity` label, not the name - different severities coexist in the same group under the same alert name (Prometheus requires unique names within one group, the convention is to differentiate by `severity`). -```yaml -apiVersion: openslo/v1 -kind: AlertPolicy -metadata: - name: api-latency-page-alert -spec: - description: Page alert for API latency burn rate - alertWhenBreaching: true - conditions: - - kind: AlertCondition - metadata: - name: api-latency-page - spec: - severity: page - condition: - kind: burnrate - threshold: 14.4 - lookbackWindow: 5m - alertAfter: 2m - notificationTargets: - - kind: AlertNotificationTarget - metadata: - name: oncall-pagerduty - spec: - target: pagerduty -``` +### Field reference + +| Field | Required | Where | Notes | +|---|---|---|---| +| `kind` | yes | `spec.condition.kind` | One of `error-rate`, `burn-rate`, `multi-burn-rate`, `multi-window-multi-burn-rate` (legacy `burnrate` accepted) | +| `op` | yes | `spec.condition.op` | `gte` / `gt` / `lte` / `lt`. Defaults to `gte`. Operator is non-strict by default (`gte` means `>=`); switch to `gt` if your math requires strict. | +| `threshold` | yes | `spec.condition.threshold` | Absolute (error-rate, must be in `(0, 1]`) or burn multiplier (burn-rate families, must be `> 0`) | +| `lookbackWindow` | yes | `spec.condition.lookbackWindow` | Window duration (e.g. `5m`, `1h`, `6h`) | +| `alertAfter` | no | `spec.condition.alertAfter` | Maps to Prom `for:` | +| `severity` | yes | `spec.severity` | `page`, `ticket`, or any custom string (carried in the alert label) | -### Validation +## 5. Multi-dimensional SLIs -All references are validated on load. If any ref cannot be resolved, you get an error listing all missing refs: +Use this when you want one SLO to drive many parallel alert series - one per value of a chosen Prometheus label. + +Set two annotations on the SLO's `metadata.annotations`: + +| Annotation | Required | Role | +|---|---|---| +| `multi-dimensional-sli.openslo.com/label` | yes | The Prometheus label whose value is joined into `openslo_slo_name` to produce one series per value | +| `multi-dimensional-sli.openslo.com/dimensions` | info only | Human-readable list of dimension values; not consumed by the generator | + +When both annotations are set, the generator emits two layers of recording rules for the SLO: + +1. **Base `_unlabeled` recordings** - each metric (`openslo_slo_info`, `openslo_slo_objective`, `openslo_slo_timewindow_days`, `openslo_slo_error_budget`, `openslo_slo_current_burn_rate`, `openslo_slo_period_burn_rate`, `openslo_slo_period_error_budget_remaining`, and every `openslo_sli_error_rate_*`) is emitted with the `_unlabeled` suffix. They carry only `openslo_slo_name` and `openslo_spec_version` labels; the chosen dimension label flows through from the underlying source query's series. +2. **Post-process `label_join` rules** - sibling rules that join the value of the chosen dimension label into `openslo_slo_name` with `-` as the separator, producing one series per dimension value. + +After Prom evaluates the rules: ``` -unresolved references: [unresolved ref: SLO "api-latency-slo" references Service "missing-svc" not found] +openslo_slo_current_burn_rate{openslo_slo_name="account-api-latency"} = 0.2 +openslo_slo_current_burn_rate{openslo_slo_name="checkout-api-latency"} = 1.4 +openslo_slo_current_burn_rate{openslo_slo_name="recommendation-api-latency"} = 4.7 ``` -The CLI also validates: -- Each spec passes SDK validation (required fields, value ranges) -- AlertCondition `kind` must be `burnrate` -- AlertPolicy must have exactly 1 condition +`openslo_slo_current_burn_rate` here is just an example. All 11 base metrics above get the same fan-out. Same SLO target, same alert policies - but three independently firing series. Page on whichever dimension crosses 14.4x; the spec author writes the SLI once. + +Working example at [`examples/multi-dim-slo/`](examples/multi-dim-slo/). + +### When to use + +- The underlying metric already splits by a label and you want shared target definitions across all series. +- Alert routing benefits from per-dimension firing rather than aggregate (per-caller-service paging, per-region escalation). +- You want per-dimension burn dashboards without writing one SLO per dimension. + +Skip when: + +- The cardinal label is unbounded (`user_id`, raw trace IDs) - recording-rule count grows linearly with cardinality and burns Prom. +- You only care about the aggregate across all series - a regular single-dim SLO is simpler. + +### Trade-offs + +- Recording rule count grows linearly with the cardinality of the chosen label. Bounded labels with a known max (caller services, regions, routes) work well. +- The dimension value is embedded in `openslo_slo_name` rather than as a separate label - dashboard queries must filter by prefix match (`openslo_slo_name=~"account-.*-api-latency$"`) or use the original source label. +- Alert rule names span all dimension values; severity in the `severity` label differentiates per-dimension alerts. + +## 6. Dashboards and integrity rules + +Three artefacts ship alongside the rules in `deploy/`. + +### 6.1 `deploy/dashboards/` + +Two Grafana dashboards that auto-provision via the standard sidecar pattern (`openslo-dashboards` ConfigMap, label `grafana_dashboard=1`): -### Run the Examples +- `openslo-list.json` (`OpenSLO - Manage SLOs`) - one row per SLO with columns: SLO name, Objective %, Period SLI (30d), Status (0-3 categorical), Budget Left %. Color-coded Status + Budget cells. +- `openslo-detail.json` (`OpenSLO - SLO detail`) - drilled-in view: header (name/description/target), SLI 28d timeseries + stat, Error Budget Burndown timeseries + 28d Remaining stat (percent), Error Budget Burn Rate timeseries + Current Burn Rate stat (multiplier), SLO target stat. -```bash -# Load and validate the API latency SLO -opensloctl load -r -f examples/api-latency-slo +Both use the `datasource` and `slo` variables. Drill from list by clicking a row. Reading guide tailored for the otel-demo end-to-end flow: [`examples/oteldemo/README.md` section 2](examples/oteldemo/README.md#2-read-the-slos). -# Load and validate the checkout availability SLO -opensloctl load -r -f examples/error-budget-slo +### 6.2 `deploy/mixins/` -# Generate recording rules + alert rules -opensloctl generate -r -f examples/api-latency-slo -o output/ +Grafonnet source. Do not hand-edit `deploy/dashboards/*.json` - the next `make release` will overwrite them. Edit the jsonnet files and `make -C deploy/mixins release` (regenerate + sync-legacy + render integrity rules + lint-rules). Vendor dir is `tmp/grafonnet-vendor/`; populated by Grafana tooling, safe to re-bootstrap. + +### 6.3 `deploy/rules/openslo-integrity-rules.yaml` + +One recording rule and one alert that catch spec drift - an SLO that the generator knows about but whose `openslo_sli_error_rate_5m` series has no samples for the last 10 minutes: + +``` +openslo_slo_metric_missing{openslo_slo_name="..."} == 1 # offending SLOs +OpenSloSpecDrift { openslo_slo_name="..." } fire # severity page, after 10m confirm +``` + +Shipped via the `deploy/rules/` ConfigMap (`make -C examples/oteldemo sync` mounts both rules + dashboards). The fix path is in the alert's `description` annotation. + +## 7. Examples + +Six examples ship with the repo. Five are minimal spec bundles; one (`oteldemo/`) is a full kind + Helm deployment of the OpenTelemetry Demo with our rules and dashboards. + +| Directory | Strategy | Has Makefile | Has kind harness | +|---|---|---|---| +| `examples/api-latency-slo/` | `burn-rate` | no | no | +| `examples/error-budget-slo/` | `burn-rate` | no | no | +| `examples/error-rate-slo/` | `error-rate` | no | no | +| `examples/multi-burn-slo/` | `multi-burn-rate` | no | no | +| `examples/multi-dim-slo/` | `burn-rate` + multi-dim annotation | no | no | +| `examples/oteldemo/` | `multi-window-multi-burn-rate` across 11 SLOs | yes | yes | + +The five minimal examples call `opensloctl` directly: + +``` +# check specs (CI-friendly, no files written) +opensloctl validate -f examples/api-latency-slo +opensloctl validate -f examples/error-rate-slo +opensloctl validate -f examples/multi-burn-slo +opensloctl validate -f examples/multi-dim-slo +opensloctl validate -f examples/oteldemo/specs + +# generate +rm -rf output/ && mkdir output/ +opensloctl generate -f examples/api-latency-slo -o output/ +opensloctl generate -f examples/error-rate-slo -o output/ +opensloctl generate -f examples/multi-burn-slo -o output/ +opensloctl generate -f examples/multi-dim-slo -o output/ +opensloctl generate -r -f examples/oteldemo/specs -o output/ ls output/ -# api-latency-slo-recording-rules.yaml -# api-latency-slo-alert-rules.yaml ``` -## Semantic Conventions +Each example ships a directory-specific README. The oteldemo README covers the deploy + dashboard reading flow; the others restate the strategy and file layout for their bundle. -opensloctl defines a registry of metrics and attributes for SLO telemetry. The registry lives in `semconv/registry/` and is used to generate `pkg/semconv/semconv_gen.go`. +The `oteldemo/` example is the only one with a real end-to-end harness - kind cluster, Helm chart install, ConfigMap sync, chaos flags, and counting 11 real SLOs. Start there if you want to see every generated artefact in a live cluster. + +## 8. Semantic conventions + +opensloctl defines a registry of SLO telemetry metrics and attributes. The registry lives in `semconv/registry/` and is used to generate `pkg/semconv/semconv_gen.go` (auto-generated, do not hand-edit). ### Attributes | Attribute | Type | Description | |---|---|---| -| `openslo.slo.name` | string | The name of the SLO as defined in the OpenSlo spec. | -| `openslo.spec.version` | string | The OpenSLO API version of the SLO spec. | +| `openslo.slo.name` | string | The SLO name | +| `openslo.spec.version` | string | OpenSLO API version of the spec | +| `openslo.service.name` | string | Service the SLO belongs to (matches the `Service` spec) | +| `openslo.alert.severity` | string | Alert severity (`page`, `ticket`, ...) on alert rules | +| `openslo.notification.target` | string | Notification target carried on alert rules (e.g. `pagerduty`, `eng-team`) | + +Pass-through labels not in the registry: `openslo_slo_description` (one-line spec description folded across lines), `chaos_flag` (demo only). Both are emitted by the generator as labels on `openslo_slo_info` but defined per spec. + +Deprecated attributes (still emitted as Go constants for back-compat, marked `deprecated.reason: obsoleted` in the registry): + +| Attribute | Type | Reason | +|---|---|---| +| `openslo.objective.decimal` | double | Replaced by the `openslo_slo_objective` recording rule. | +| `openslo.objective.percent` | double | Replaced by the `openslo_slo_objective` recording rule. | +| `openslo.timewindow.duration` | string | Encoded in the SLI error-rate windowed recording rules. | ### Metrics -#### SLO Info +All metrics below carry `openslo.slo.name` and `openslo.spec.version` unless noted. Units are dimensionless (`1`) - alert thresholds carry the units, not the metric. -| Metric | Type | Unit | Description | -|---|---|---|---| -| `openslo.slo.info` | gauge | 1 | Identifies the existence of an SLO. Always has value 1. | -| `openslo.slo.objective` | gauge | 1 | The target SLI objective (e.g., 0.999 for 99.9% availability). | -| `openslo.slo.timewindow_days` | gauge | 1 | The SLO time window duration expressed as a number of days. | -| `openslo.slo.error_budget` | gauge | 1 | The error budget calculated as 1 minus the objective. | +#### SLO Info -All SLO info metrics carry `openslo.slo.name` and `openslo.spec.version` labels. +| Metric | Description | +|---|---| +| `openslo.slo.info` | Identifies the existence of an SLO. Always value 1. | +| `openslo.slo.objective` | Target SLI objective (e.g. 0.999 for 99.9%). | +| `openslo.slo.timewindow_days` | Spec `timeWindow` expressed in days. | +| `openslo.slo.error_budget` | `1 - objective`. | +| `openslo.slo.current_burn_rate` | `openslo_sli_error_rate_5m / error_budget`. | +| `openslo.slo.period_burn_rate` | `openslo_sli_error_rate_ / error_budget`. Emitted when an SLO time window matches the multi-window set. | +| `openslo.slo.period_error_budget_remaining` | `clamp_min(1 - period_burn_rate, 0)`. | +| `openslo.slo.status` | Categorical health state. 0=Healthy, 1=Burning, 2=Critical, 3=Breached. See [§2.2](#22-recording-rules). | #### SLI Error Rate | Metric | Description | |---|---| -| `openslo.sli.error_rate_5m` | SLI error rate over a 5-minute window. | -| `openslo.sli.error_rate_30m` | SLI error rate over a 30-minute window. | -| `openslo.sli.error_rate_1h` | SLI error rate over a 1-hour window. | -| `openslo.sli.error_rate_2h` | SLI error rate over a 2-hour window. | -| `openslo.sli.error_rate_6h` | SLI error rate over a 6-hour window. | -| `openslo.sli.error_rate_1d` | SLI error rate over a 1-day window. | -| `openslo.sli.error_rate_3d` | SLI error rate over a 3-day window. | -| `openslo.sli.error_rate_7d` | SLI error rate over a 7-day window. | -| `openslo.sli.error_rate_28d` | SLI error rate over a 28-day window. | -| `openslo.sli.error_rate_30d` | SLI error rate over a 30-day window. | +| `openslo.sli.error_rate_5m` ... `_30d` | SLI error rate over each multi-window. Windows: `5m, 30m, 1h, 2h, 6h, 1d, 3d, 7d, 28d, 30d`. | + +#### SLI Event Rate (RatioMetric SLIs only) -All error rate metrics carry `openslo.slo.name` and `openslo.spec.version` labels. +| Metric | Description | +|---|---| +| `openslo.sli.event_rate_5m` ... `_30d` | Events per second over each multi-window. Emitted only for `RatioMetric` SLIs since the spec exposes a `total` count query. Use it in dashboards to interpret error-budget burn in absolute event-volume terms. `thresholdMetric` (histogram-style) SLIs do not emit this metric. | -### Registry Management +### Registry management The semantic convention registry is managed with [OpenTelemetry Weaver](https://github.com/open-telemetry/weaver). -- **Registry source**: `semconv/registry/` — YAML definitions for attributes and metrics -- **Generated code**: `pkg/semconv/semconv_gen.go` — auto-generated Go constants from the registry -- **Templates**: `semconv/templates/go/` — MiniJinja templates that produce the Go file +- **Registry source**: `semconv/registry/` - YAML definitions for attributes and metrics +- **Generated code**: `pkg/semconv/semconv_gen.go` - auto-generated Go constants +- **Templates**: `semconv/templates/go/` - MiniJinja templates ``` -make semconv-generate # regenerate semconv_gen.go from registry -make semconv-check # validate registry schema -make semconv-stats # show registry statistics -make semconv-diff BASE= # detect breaking changes vs a base ref +make semconv-generate # registry YAML -> pkg/semconv/semconv_gen.go +make semconv-check # validate registry schema +make semconv-stats # show registry statistics +make semconv-diff BASE= # detect breaking changes vs base ref ``` -### Consuming the Registry +### Consuming the registry -If your project also uses OpenTelemetry Weaver, you can depend on this registry directly. Add it as a dependency in your `manifest.yaml`: +If your project also uses OpenTelemetry Weaver, depend on this registry directly. Add it in your `manifest.yaml`: ```yaml schema_url: https://your-org.com/schemas/your-app/v1.0.0 @@ -509,7 +645,7 @@ dependencies: registry_path: https://github.com/thisisibrahimd/opensloctl.git[semconv/registry] ``` -Then reference the attributes in your own metrics and spans: +Then reference the attributes in your own metrics: ```yaml metrics: @@ -525,11 +661,62 @@ metrics: requirement_level: required ``` -Alternatively, if you don't use Weaver, the Go constants are available at `github.com/thisisibrahimd/opensloctl/pkg/semconv`: +If you do not use Weaver, the Go constants are available at `github.com/thisisibrahimd/opensloctl/pkg/semconv`: ```go import "github.com/thisisibrahimd/opensloctl/pkg/semconv" -// Use generated constants meter.Float64ObservableGauge(semconv.METRIC_OPENSLO_SLO_INFO) +gauge := meter.Float64ObservableGauge(semconv.METRIC_OPENSLO_SLO_STATUS, + api.WithDescription("SLO categorical health state")) ``` + +## 9. Development + +### Toolchain + +This project uses [mise](https://mise.jdx.dev/) to manage tool versions. `mise.toml` declares `go = "1.26"` (the version releases are built and tested against); `go.mod` declares `go 1.25.5` as the minimum supported version. + +``` +# one-time +mise install + +# developer commands +make build # go build -o opensloctl . +make test # go test ./... +make lint # golangci-lint run +make tidy # go mod tidy + +# generators +make generate FILE=examples/oteldemo/specs OUTPUT=examples/oteldemo/rules +make validate FILE=examples/oteldemo/specs + +# semconv management +make semconv-generate +make semconv-check + +# dashboards / integrity rules +make -C deploy/mixins release # also pushes JSON to deploy/dashboards; render integrity rules + +# snapshot test upkeep +go test ./internal/generator/prometheusgenerator/... -update +``` + +### Repository layout + +| Path | Purpose | +|---|---| +| `main.go` -> `cmd.Execute()` | single CLI entrypoint | +| `pkg/specstore/loader.go` | load YAML via `openslosdk.Decode`, sort into typed structs | +| `internal/generator/generator.go` | `Generator` interface | +| `internal/generator/prometheusgenerator/` | template + sprig rendering; emits the unified `-rules.yaml` | +| `internal/feature/feature.go` | feature flag handling | +| `pkg/semconv/semconv_gen.go` | auto-generated, do not edit | +| `pkg/util/file.go` | recursive YAML/YML file discovery | +| `semconv/registry/` | OTel Weaver YAML for metrics + attributes | +| `semconv/templates/go/` | MiniJinja templates for codegen | +| `examples/-slo/` | the six spec bundles | +| `deploy/dashboards/` | repo-root Grafana dashboards (regenerated from `deploy/mixins/`) | +| `deploy/mixins/` | grafonnet source for the dashboards + integrity rules | +| `deploy/rules/` | rendered integrity rules (subset of rules CM payload) | +| `CHANGELOG.md` | per-release change log | diff --git a/cmd/root.go b/cmd/root.go index b47f662..249335b 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -22,6 +22,7 @@ func NewRootCommand() *cobra.Command { cmd.AddCommand(newLoadCommand()) cmd.AddCommand(newGenerateCommand()) + cmd.AddCommand(newValidateCommand()) return cmd diff --git a/cmd/validate.go b/cmd/validate.go new file mode 100644 index 0000000..e49bc02 --- /dev/null +++ b/cmd/validate.go @@ -0,0 +1,50 @@ +package cmd + +import ( + "log/slog" + "os" + + "github.com/spf13/cobra" + "github.com/thisisibrahimd/opensloctl/internal/generator/prometheusgenerator" + "github.com/thisisibrahimd/opensloctl/pkg/specstore" +) + +type validateFlags struct { + filenames []string + recursive bool +} + +func newValidateCommand() *cobra.Command { + flags := validateFlags{} + + cmd := &cobra.Command{ + Use: "validate", + Short: "Validate OpenSlo specs without writing generated files", + Run: func(cmd *cobra.Command, args []string) { + runValidate(cmd, args, flags) + }, + } + + cmd.Flags().StringArrayVarP(&flags.filenames, "filename", "f", []string{}, "The files that contain the openslo specs to load.") + cmd.Flags().BoolVarP(&flags.recursive, "recursive", "r", false, "Whether to recursively look into the directory.") + + return cmd +} + +func runValidate(cmd *cobra.Command, args []string, flags validateFlags) { + slog.Info("validating specs", "files", len(flags.filenames)) + + specs, err := specstore.GetSpecs(flags.filenames, flags.recursive) + if err != nil { + slog.Error("load-time validation failed", "err", err) + os.Exit(1) + } + + pg := prometheusgenerator.NewPrometheusGenerator(specs) + if err := pg.Validate(); err != nil { + slog.Error("generator validation failed", "err", err) + os.Exit(1) + } + + slog.Info("all validations passed", "slos", len(specs.V1.SLOs)) +} diff --git a/deploy/dashboards/openslo-detail.json b/deploy/dashboards/openslo-detail.json new file mode 100644 index 0000000..8d2df5b --- /dev/null +++ b/deploy/dashboards/openslo-detail.json @@ -0,0 +1,537 @@ +{ + "description": "Per-SLO drilldown. Open from OpenSLO - Manage SLOs.", + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "gridPos": { + "h": 8, + "w": 18, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false + }, + "content": "## $slo\n\n$description\n\n**Target:** $target", + "mode": "markdown" + }, + "pluginVersion": "v13.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "openslo_slo_info{openslo_slo_name=\"$slo\"}", + "legendFormat": "{{openslo_slo_name}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "openslo_slo_objective{openslo_slo_name=\"$slo\"}", + "legendFormat": "{{openslo_slo_name}}", + "refId": "B" + } + ], + "title": "", + "type": "text" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "thresholds": { + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percent" + } + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 0 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "v13.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "openslo_slo_objective{openslo_slo_name=\"$slo\"} * 100", + "legendFormat": "{{openslo_slo_name}}", + "refId": "A" + } + ], + "title": "SLO", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "showPoints": "never" + }, + "decimals": 2, + "thresholds": { + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percent" + } + }, + "gridPos": { + "h": 8, + "w": 18, + "x": 0, + "y": 8 + }, + "id": 3, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "v13.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "(1 - openslo_sli_error_rate_30d{openslo_slo_name=\"$slo\"}) * 100", + "legendFormat": "28d", + "refId": "A" + } + ], + "title": "SLI", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "thresholds": { + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percent" + } + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 16 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "v13.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "(1 - openslo_sli_error_rate_30d{openslo_slo_name=\"$slo\"}) * 100", + "legendFormat": "{{openslo_slo_name}}", + "refId": "A" + } + ], + "title": "28d SLI", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "showPoints": "never" + }, + "decimals": 2, + "max": 100, + "min": 0, + "thresholds": { + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percent" + } + }, + "gridPos": { + "h": 8, + "w": 18, + "x": 0, + "y": 16 + }, + "id": 5, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "v13.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "openslo_slo_period_error_budget_remaining{openslo_slo_name=\"$slo\"} * 100", + "legendFormat": "Period remaining", + "refId": "A" + } + ], + "title": "Error Budget Burndown", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "thresholds": { + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percent" + } + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 24 + }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "v13.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "openslo_slo_period_error_budget_remaining{openslo_slo_name=\"$slo\"} * 100", + "legendFormat": "{{openslo_slo_name}}", + "refId": "A" + } + ], + "title": "28d Remaining Error Budget", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "showPoints": "never" + }, + "decimals": 2, + "thresholds": { + "steps": [ + { + "color": "green", + "value": null + } + ] + } + } + }, + "gridPos": { + "h": 8, + "w": 18, + "x": 0, + "y": 24 + }, + "id": 7, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "v13.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "openslo_slo_current_burn_rate{openslo_slo_name=\"$slo\"}", + "legendFormat": "Current burn rate", + "refId": "A" + } + ], + "title": "Error Budget Burn Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "thresholds": { + "steps": [ + { + "color": "green", + "value": null + } + ] + } + } + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 32 + }, + "id": 8, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "v13.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "openslo_slo_current_burn_rate{openslo_slo_name=\"$slo\"}", + "legendFormat": "{{openslo_slo_name}}", + "refId": "A" + } + ], + "title": "Current Burn Rate", + "type": "stat" + } + ], + "refresh": "30s", + "schemaVersion": 39, + "tags": [ + "openslo" + ], + "templating": { + "list": [ + { + "allowCustomValue": false, + "includeAll": false, + "label": "datasource", + "multi": false, + "name": "datasource", + "pluginId": "prometheus", + "query": "prometheus", + "refresh": 1, + "type": "datasource" + }, + { + "allowCustomValue": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "includeAll": false, + "multi": false, + "name": "slo", + "query": "label_values(openslo_slo_info, openslo_slo_name)", + "refresh": 2, + "type": "query" + }, + { + "allowCustomValue": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "hide": 2, + "includeAll": false, + "multi": false, + "name": "description", + "query": "openslo_slo_info{openslo_slo_name=~\"$slo\"}", + "refresh": 2, + "regex": "/.*openslo_slo_description=\"(.*)\"/", + "type": "query" + }, + { + "allowCustomValue": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "hide": 2, + "includeAll": false, + "multi": false, + "name": "target", + "query": "openslo_slo_objective{openslo_slo_name=~\"$slo\"}", + "refresh": 2, + "regex": "/.* (\\d+\\.?\\d*) .*/", + "type": "query" + } + ] + }, + "time": { + "from": "now-30d", + "to": "now" + }, + "timezone": "utc", + "title": "OpenSLO - SLO detail", + "uid": "openslo-detail" +} diff --git a/deploy/dashboards/openslo-list.json b/deploy/dashboards/openslo-list.json new file mode 100644 index 0000000..2dc892e --- /dev/null +++ b/deploy/dashboards/openslo-list.json @@ -0,0 +1,336 @@ +{ + "description": "All SLOs loaded by opensloctl. Click any row to drill into the per-SLO detail dashboard.", + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "align": "left", + "cellOptions": { + "type": "auto" + } + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Value #StatusQ" + }, + "properties": [ + { + "id": "displayName", + "value": "Status" + }, + { + "id": "mappings", + "value": [ + { + "options": { + "0": { + "color": "green", + "index": 0, + "text": "Healthy", + "value": "0" + }, + "1": { + "color": "yellow", + "index": 1, + "text": "Burning", + "value": "1" + }, + "2": { + "color": "orange", + "index": 2, + "text": "Critical", + "value": "2" + }, + "3": { + "color": "red", + "index": 3, + "text": "Breached", + "value": "3" + } + }, + "type": "value" + } + ] + }, + { + "id": "custom.cellOptions", + "value": { + "applyToRow": false, + "mode": "basic", + "type": "color-background" + } + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 0 + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "orange", + "value": 2 + }, + { + "color": "red", + "value": 3 + } + ] + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #ObjectiveQ" + }, + "properties": [ + { + "id": "displayName", + "value": "Objective %" + }, + { + "id": "unit", + "value": "percent" + }, + { + "id": "decimals", + "value": 2 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #PeriodSLIQ" + }, + "properties": [ + { + "id": "displayName", + "value": "Period SLI" + }, + { + "id": "unit", + "value": "percent" + }, + { + "id": "decimals", + "value": 2 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #BudgetQ" + }, + "properties": [ + { + "id": "displayName", + "value": "Budget Left %" + }, + { + "id": "unit", + "value": "percent" + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-background" + } + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "red", + "value": 0 + }, + { + "color": "yellow", + "value": 15 + }, + { + "color": "green", + "value": 50 + } + ] + } + }, + { + "id": "decimals", + "value": 2 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + } + ] + }, + "gridPos": { + "h": 24, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "showHeader": true, + "sortBy": [ + { + "desc": false, + "displayName": "SLO name" + } + ] + }, + "pluginVersion": "v13.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "openslo_slo_status{openslo_slo_name=~\"$slo\"}", + "format": "table", + "instant": true, + "refId": "StatusQ" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "openslo_slo_objective{openslo_slo_name=~\"$slo\"} * 100", + "format": "table", + "instant": true, + "refId": "ObjectiveQ" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "(1 - openslo_sli_error_rate_30d{openslo_slo_name=~\"$slo\"}) * 100", + "format": "table", + "instant": true, + "refId": "PeriodSLIQ" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "openslo_slo_period_error_budget_remaining{openslo_slo_name=~\"$slo\"} * 100", + "format": "table", + "instant": true, + "refId": "BudgetQ" + } + ], + "title": "Service Level Objectives", + "transformations": [ + { + "id": "merge", + "options": { } + }, + { + "id": "organize", + "options": { + "includeByName": { + "Value #BudgetQ": true, + "Value #ObjectiveQ": true, + "Value #PeriodSLIQ": true, + "Value #StatusQ": true, + "openslo_slo_name": true + }, + "indexByName": { + "Value #BudgetQ": 4, + "Value #ObjectiveQ": 1, + "Value #PeriodSLIQ": 2, + "Value #StatusQ": 3, + "openslo_slo_name": 0 + }, + "renameByName": { + "openslo_slo_name": "Service Level Objective" + } + } + } + ], + "type": "table" + } + ], + "refresh": "30s", + "schemaVersion": 39, + "tags": [ + "openslo" + ], + "templating": { + "list": [ + { + "allowCustomValue": false, + "includeAll": false, + "label": "datasource", + "multi": false, + "name": "datasource", + "pluginId": "prometheus", + "query": "prometheus", + "refresh": 1, + "type": "datasource" + }, + { + "allowCustomValue": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "includeAll": true, + "multi": true, + "name": "slo", + "query": "label_values(openslo_slo_info, openslo_slo_name)", + "refresh": 2, + "type": "query" + } + ] + }, + "time": { + "from": "now-7d", + "to": "now" + }, + "timezone": "utc", + "title": "OpenSLO - Manage SLOs", + "uid": "openslo-list" +} diff --git a/deploy/mixins/Makefile b/deploy/mixins/Makefile new file mode 100644 index 0000000..2a22a68 --- /dev/null +++ b/deploy/mixins/Makefile @@ -0,0 +1,78 @@ +DASHBOARDS_DIR := dashboards +LEGACY_DASHBOARDS_DIR := ../../deploy/dashboards +RULES_OUT_DIR := ../../deploy/rules +RULES_SRC_DIR := rules +PROMTOOL := $(shell mise which promtool 2>/dev/null || command -v promtool 2>/dev/null || echo promtool) +# Dashboard/render annotation pulled from the current git tree (short +# SHA + dirty marker). The `version` extVar that Grafana dashboards +# consume stays alive so the jsonnet blocks below don't need editing - +# they bind whatever `-V version=…` provides. Git SHA gives one +# unique-per-commit identifier; rendering off `git diff` would noisily +# churn the dashboards on every local save so SHA is the practical pick. +GIT_HASH := $(shell git rev-parse --short HEAD 2>/dev/null || echo unknown) +DIRTY := $(shell git diff --quiet 2>/dev/null && echo clean || echo dirty) +VERSION := $(GIT_HASH)-$(DIRTY) + +.PHONY: generate +generate: $(DASHBOARDS_DIR) + jsonnet -J ../tmp/grafonnet-vendor -V version=$(VERSION) \ + openslo-list.jsonnet > $(DASHBOARDS_DIR)/openslo-list.json + jsonnet -J ../tmp/grafonnet-vendor -V version=$(VERSION) \ + openslo-detail.jsonnet > $(DASHBOARDS_DIR)/openslo-detail.json + +.PHONY: list +list: $(DASHBOARDS_DIR) + jsonnet -J ../tmp/grafonnet-vendor -V version=$(VERSION) \ + openslo-list.jsonnet > $(DASHBOARDS_DIR)/openslo-list.json + +.PHONY: detail +detail: $(DASHBOARDS_DIR) + jsonnet -J ../tmp/grafonnet-vendor -V version=$(VERSION) \ + openslo-detail.jsonnet > $(DASHBOARDS_DIR)/openslo-detail.json + +.PHONY: clean +clean: + rm -rf $(DASHBOARDS_DIR) + +.PHONY: sync-legacy +sync-legacy: generate + @echo "==> Replacing legacy $(LEGACY_DASHBOARDS_DIR)/*.json with the new render..." + @cp $(DASHBOARDS_DIR)/openslo-list.json $(LEGACY_DASHBOARDS_DIR)/openslo-list.json + @cp $(DASHBOARDS_DIR)/openslo-detail.json $(LEGACY_DASHBOARDS_DIR)/openslo-detail.json + @echo "synced." + +# Default workflow: generate → sync-legacy → rules → lint-rules. +# Bump is gone - render is annotated by the current git SHA dirty status. +.PHONY: release +release: generate sync-legacy rules lint-rules + +# Render `rules/*.jsonnet` to $(RULES_OUT_DIR) as yaml. The jsonnet source +# must top-level call std.manifestYamlDoc(...) (with `quote_keys=false` so +# keys don't all get quoted), and the make rule passes `-S` so jsonnet +# emits the manifest as plain text instead of JSON-encoding the string. +.PHONY: rules +rules: $(RULES_OUT_DIR) + @mkdir -p $(RULES_OUT_DIR) + @set -e; \ + for src in $(RULES_SRC_DIR)/*.jsonnet; do \ + [ -f "$$src" ] || continue; \ + base=$$(basename "$$src" .jsonnet); \ + out="$(RULES_OUT_DIR)/$$base-rules.yaml"; \ + echo "==> $$src -> $$out"; \ + jsonnet -S -J ../tmp/grafonnet-vendor -V version=$(VERSION) "$$src" \ + > "$$out"; \ + done + +.PHONY: lint-rules +lint-rules: + @for f in $(RULES_OUT_DIR)/*.yaml; do \ + [ -f "$$f" ] || continue; \ + echo "==> $$f"; \ + $(PROMTOOL) check rules "$$f"; \ + done + +$(DASHBOARDS_DIR): + mkdir -p $@ + +$(RULES_OUT_DIR): + mkdir -p $@ diff --git a/deploy/mixins/alerts/.keep b/deploy/mixins/alerts/.keep new file mode 100644 index 0000000..afcdd74 --- /dev/null +++ b/deploy/mixins/alerts/.keep @@ -0,0 +1 @@ +# placeholder — alert rules in Jsonnet form go here diff --git a/deploy/mixins/dashboards/openslo-detail.json b/deploy/mixins/dashboards/openslo-detail.json new file mode 100644 index 0000000..8d2df5b --- /dev/null +++ b/deploy/mixins/dashboards/openslo-detail.json @@ -0,0 +1,537 @@ +{ + "description": "Per-SLO drilldown. Open from OpenSLO - Manage SLOs.", + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "gridPos": { + "h": 8, + "w": 18, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false + }, + "content": "## $slo\n\n$description\n\n**Target:** $target", + "mode": "markdown" + }, + "pluginVersion": "v13.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "openslo_slo_info{openslo_slo_name=\"$slo\"}", + "legendFormat": "{{openslo_slo_name}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "openslo_slo_objective{openslo_slo_name=\"$slo\"}", + "legendFormat": "{{openslo_slo_name}}", + "refId": "B" + } + ], + "title": "", + "type": "text" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "thresholds": { + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percent" + } + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 0 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "v13.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "openslo_slo_objective{openslo_slo_name=\"$slo\"} * 100", + "legendFormat": "{{openslo_slo_name}}", + "refId": "A" + } + ], + "title": "SLO", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "showPoints": "never" + }, + "decimals": 2, + "thresholds": { + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percent" + } + }, + "gridPos": { + "h": 8, + "w": 18, + "x": 0, + "y": 8 + }, + "id": 3, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "v13.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "(1 - openslo_sli_error_rate_30d{openslo_slo_name=\"$slo\"}) * 100", + "legendFormat": "28d", + "refId": "A" + } + ], + "title": "SLI", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "thresholds": { + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percent" + } + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 16 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "v13.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "(1 - openslo_sli_error_rate_30d{openslo_slo_name=\"$slo\"}) * 100", + "legendFormat": "{{openslo_slo_name}}", + "refId": "A" + } + ], + "title": "28d SLI", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "showPoints": "never" + }, + "decimals": 2, + "max": 100, + "min": 0, + "thresholds": { + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percent" + } + }, + "gridPos": { + "h": 8, + "w": 18, + "x": 0, + "y": 16 + }, + "id": 5, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "v13.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "openslo_slo_period_error_budget_remaining{openslo_slo_name=\"$slo\"} * 100", + "legendFormat": "Period remaining", + "refId": "A" + } + ], + "title": "Error Budget Burndown", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "thresholds": { + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percent" + } + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 24 + }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "v13.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "openslo_slo_period_error_budget_remaining{openslo_slo_name=\"$slo\"} * 100", + "legendFormat": "{{openslo_slo_name}}", + "refId": "A" + } + ], + "title": "28d Remaining Error Budget", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "showPoints": "never" + }, + "decimals": 2, + "thresholds": { + "steps": [ + { + "color": "green", + "value": null + } + ] + } + } + }, + "gridPos": { + "h": 8, + "w": 18, + "x": 0, + "y": 24 + }, + "id": 7, + "options": { + "legend": { + "calcs": [ ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "v13.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "openslo_slo_current_burn_rate{openslo_slo_name=\"$slo\"}", + "legendFormat": "Current burn rate", + "refId": "A" + } + ], + "title": "Error Budget Burn Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "thresholds": { + "steps": [ + { + "color": "green", + "value": null + } + ] + } + } + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 32 + }, + "id": 8, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "v13.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "openslo_slo_current_burn_rate{openslo_slo_name=\"$slo\"}", + "legendFormat": "{{openslo_slo_name}}", + "refId": "A" + } + ], + "title": "Current Burn Rate", + "type": "stat" + } + ], + "refresh": "30s", + "schemaVersion": 39, + "tags": [ + "openslo" + ], + "templating": { + "list": [ + { + "allowCustomValue": false, + "includeAll": false, + "label": "datasource", + "multi": false, + "name": "datasource", + "pluginId": "prometheus", + "query": "prometheus", + "refresh": 1, + "type": "datasource" + }, + { + "allowCustomValue": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "includeAll": false, + "multi": false, + "name": "slo", + "query": "label_values(openslo_slo_info, openslo_slo_name)", + "refresh": 2, + "type": "query" + }, + { + "allowCustomValue": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "hide": 2, + "includeAll": false, + "multi": false, + "name": "description", + "query": "openslo_slo_info{openslo_slo_name=~\"$slo\"}", + "refresh": 2, + "regex": "/.*openslo_slo_description=\"(.*)\"/", + "type": "query" + }, + { + "allowCustomValue": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "hide": 2, + "includeAll": false, + "multi": false, + "name": "target", + "query": "openslo_slo_objective{openslo_slo_name=~\"$slo\"}", + "refresh": 2, + "regex": "/.* (\\d+\\.?\\d*) .*/", + "type": "query" + } + ] + }, + "time": { + "from": "now-30d", + "to": "now" + }, + "timezone": "utc", + "title": "OpenSLO - SLO detail", + "uid": "openslo-detail" +} diff --git a/deploy/mixins/dashboards/openslo-list.json b/deploy/mixins/dashboards/openslo-list.json new file mode 100644 index 0000000..2dc892e --- /dev/null +++ b/deploy/mixins/dashboards/openslo-list.json @@ -0,0 +1,336 @@ +{ + "description": "All SLOs loaded by opensloctl. Click any row to drill into the per-SLO detail dashboard.", + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "align": "left", + "cellOptions": { + "type": "auto" + } + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Value #StatusQ" + }, + "properties": [ + { + "id": "displayName", + "value": "Status" + }, + { + "id": "mappings", + "value": [ + { + "options": { + "0": { + "color": "green", + "index": 0, + "text": "Healthy", + "value": "0" + }, + "1": { + "color": "yellow", + "index": 1, + "text": "Burning", + "value": "1" + }, + "2": { + "color": "orange", + "index": 2, + "text": "Critical", + "value": "2" + }, + "3": { + "color": "red", + "index": 3, + "text": "Breached", + "value": "3" + } + }, + "type": "value" + } + ] + }, + { + "id": "custom.cellOptions", + "value": { + "applyToRow": false, + "mode": "basic", + "type": "color-background" + } + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 0 + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "orange", + "value": 2 + }, + { + "color": "red", + "value": 3 + } + ] + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #ObjectiveQ" + }, + "properties": [ + { + "id": "displayName", + "value": "Objective %" + }, + { + "id": "unit", + "value": "percent" + }, + { + "id": "decimals", + "value": 2 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #PeriodSLIQ" + }, + "properties": [ + { + "id": "displayName", + "value": "Period SLI" + }, + { + "id": "unit", + "value": "percent" + }, + { + "id": "decimals", + "value": 2 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #BudgetQ" + }, + "properties": [ + { + "id": "displayName", + "value": "Budget Left %" + }, + { + "id": "unit", + "value": "percent" + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-background" + } + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "red", + "value": 0 + }, + { + "color": "yellow", + "value": 15 + }, + { + "color": "green", + "value": 50 + } + ] + } + }, + { + "id": "decimals", + "value": 2 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + } + ] + }, + "gridPos": { + "h": 24, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "showHeader": true, + "sortBy": [ + { + "desc": false, + "displayName": "SLO name" + } + ] + }, + "pluginVersion": "v13.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "openslo_slo_status{openslo_slo_name=~\"$slo\"}", + "format": "table", + "instant": true, + "refId": "StatusQ" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "openslo_slo_objective{openslo_slo_name=~\"$slo\"} * 100", + "format": "table", + "instant": true, + "refId": "ObjectiveQ" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "(1 - openslo_sli_error_rate_30d{openslo_slo_name=~\"$slo\"}) * 100", + "format": "table", + "instant": true, + "refId": "PeriodSLIQ" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "openslo_slo_period_error_budget_remaining{openslo_slo_name=~\"$slo\"} * 100", + "format": "table", + "instant": true, + "refId": "BudgetQ" + } + ], + "title": "Service Level Objectives", + "transformations": [ + { + "id": "merge", + "options": { } + }, + { + "id": "organize", + "options": { + "includeByName": { + "Value #BudgetQ": true, + "Value #ObjectiveQ": true, + "Value #PeriodSLIQ": true, + "Value #StatusQ": true, + "openslo_slo_name": true + }, + "indexByName": { + "Value #BudgetQ": 4, + "Value #ObjectiveQ": 1, + "Value #PeriodSLIQ": 2, + "Value #StatusQ": 3, + "openslo_slo_name": 0 + }, + "renameByName": { + "openslo_slo_name": "Service Level Objective" + } + } + } + ], + "type": "table" + } + ], + "refresh": "30s", + "schemaVersion": 39, + "tags": [ + "openslo" + ], + "templating": { + "list": [ + { + "allowCustomValue": false, + "includeAll": false, + "label": "datasource", + "multi": false, + "name": "datasource", + "pluginId": "prometheus", + "query": "prometheus", + "refresh": 1, + "type": "datasource" + }, + { + "allowCustomValue": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "includeAll": true, + "multi": true, + "name": "slo", + "query": "label_values(openslo_slo_info, openslo_slo_name)", + "refresh": 2, + "type": "query" + } + ] + }, + "time": { + "from": "now-7d", + "to": "now" + }, + "timezone": "utc", + "title": "OpenSLO - Manage SLOs", + "uid": "openslo-list" +} diff --git a/deploy/mixins/openslo-detail.jsonnet b/deploy/mixins/openslo-detail.jsonnet new file mode 100644 index 0000000..d961b4b --- /dev/null +++ b/deploy/mixins/openslo-detail.jsonnet @@ -0,0 +1,250 @@ +// Per-SLO detail dashboard. +// Variables: datasource, slo (set via URL when drilling from the list), +// description + target (hidden, derived from openslo_slo_info / +// openslo_slo_objective) -- text panel inlines them under the SLO name. +local g = import '../../tmp/grafonnet-vendor/vendor/github.com/grafana/grafonnet/gen/grafonnet-v13.0.0/main.libsonnet'; + +local var = g.dashboard.variable; +local panel = g.panel; +local promQ = g.query.prometheus; + +local ds = { type: 'prometheus', uid: '${datasource}' }; + +local dsVar = + var.datasource.new('datasource', 'prometheus') + + var.datasource.generalOptions.withLabel('datasource') + + var.datasource.selectionOptions.withIncludeAll(false) + + var.datasource.selectionOptions.withMulti(false) + + { allowCustomValue: false, pluginId: 'prometheus', refresh: 1 }; + +local sloVar = + var.query.new('slo', 'label_values(openslo_slo_info, openslo_slo_name)') + + var.query.withDatasource(type='prometheus', uid='${datasource}') + + var.query.selectionOptions.withIncludeAll(false) + + var.query.selectionOptions.withMulti(false) + + var.query.refresh.onTime() + + { allowCustomValue: false }; + +local descVar = + var.query.new('description', 'openslo_slo_info{openslo_slo_name=~"$slo"}') + + var.query.withDatasource(type='prometheus', uid='${datasource}') + + var.query.withRegex('/.*openslo_slo_description="(.*)"/') + + var.query.refresh.onTime() + + var.query.selectionOptions.withIncludeAll(false) + + var.query.selectionOptions.withMulti(false) + + var.query.generalOptions.showOnDashboard.withNothing() + + { allowCustomValue: false }; + +local targetVar = + var.query.new('target', 'openslo_slo_objective{openslo_slo_name=~"$slo"}') + + var.query.withDatasource(type='prometheus', uid='${datasource}') + + var.query.withRegex('/.* (\\d+\\.?\\d*) .*/') + + var.query.refresh.onTime() + + var.query.selectionOptions.withIncludeAll(false) + + var.query.selectionOptions.withMulti(false) + + var.query.generalOptions.showOnDashboard.withNothing() + + { allowCustomValue: false }; + +// ----- panels -------------------------------------------------------------- + +local textPanel = + panel.text.new('') + + panel.text.options.withMode('markdown') + + panel.text.options.withContent('## $slo\n\n$description\n\n**Target:** $target') + + panel.text.options.withCodeMixin({ + language: 'plaintext', + showLineNumbers: false, + }) + + panel.text.queryOptions.withTargets([ + promQ.new('${datasource}', 'openslo_slo_info{openslo_slo_name="$slo"}') + + promQ.withLegendFormat('{{openslo_slo_name}}') + + promQ.withRefId('A'), + promQ.new('${datasource}', 'openslo_slo_objective{openslo_slo_name="$slo"}') + + promQ.withLegendFormat('{{openslo_slo_name}}') + + promQ.withRefId('B'), + ]) + + panel.text.panelOptions.withGridPos(h=8, w=18, x=0, y=0) + + { id: 1 }; + +local sloStat = + panel.stat.new('SLO') + + panel.stat.standardOptions.withUnit('percent') + + panel.stat.standardOptions.withDecimals(2) + + panel.stat.standardOptions.color.withMode('thresholds') + + panel.stat.standardOptions.thresholds.withSteps([{ color: 'green', value: null }]) + + panel.stat.options.withColorMode('value') + + panel.stat.options.withGraphMode('area') + + panel.stat.options.withJustifyMode('auto') + + panel.stat.options.withOrientation('auto') + + panel.stat.options.reduceOptions.withCalcs(['lastNotNull']) + + panel.stat.options.reduceOptions.withFields('') + + panel.stat.options.reduceOptions.withValues(false) + + panel.stat.options.withTextMode('auto') + + panel.stat.queryOptions.withTargets([ + promQ.new('${datasource}', 'openslo_slo_objective{openslo_slo_name="$slo"} * 100') + + promQ.withLegendFormat('{{openslo_slo_name}}') + + promQ.withRefId('A'), + ]) + + panel.stat.panelOptions.withGridPos(h=8, w=6, x=18, y=0) + + { datasource: ds, id: 10 }; + +local sliTs = + panel.timeSeries.new('SLI') + + panel.timeSeries.standardOptions.withUnit('percent') + + panel.timeSeries.standardOptions.withDecimals(2) + + panel.timeSeries.standardOptions.color.withMode('thresholds') + + panel.timeSeries.standardOptions.thresholds.withSteps([{ color: 'green', value: null }]) + + panel.timeSeries.fieldConfig.defaults.custom.withDrawStyle('line') + + panel.timeSeries.fieldConfig.defaults.custom.withFillOpacity(10) + + panel.timeSeries.fieldConfig.defaults.custom.withLineWidth(1) + + panel.timeSeries.fieldConfig.defaults.custom.withShowPoints('never') + + panel.timeSeries.options.legend.withDisplayMode('list') + + panel.timeSeries.options.legend.withPlacement('bottom') + + panel.timeSeries.options.legend.withShowLegend(true) + + panel.timeSeries.options.legend.withCalcs([]) + + panel.timeSeries.options.tooltip.withMode('multi') + + panel.timeSeries.options.tooltip.withSort('desc') + + panel.timeSeries.queryOptions.withTargets([ + promQ.new('${datasource}', '(1 - openslo_sli_error_rate_30d{openslo_slo_name="$slo"}) * 100') + + promQ.withLegendFormat('28d') + + promQ.withRefId('A'), + ]) + + panel.stat.panelOptions.withGridPos(h=8, w=18, x=0, y=8) + + { datasource: ds, id: 11 }; + +local sli28Stat = + panel.stat.new('28d SLI') + + panel.timeSeries.standardOptions.withUnit('percent') + + panel.stat.standardOptions.withDecimals(2) + + panel.stat.standardOptions.color.withMode('thresholds') + + panel.stat.standardOptions.thresholds.withSteps([{ color: 'green', value: null }]) + + panel.stat.options.withColorMode('value') + + panel.stat.options.withGraphMode('area') + + panel.stat.options.withJustifyMode('auto') + + panel.stat.options.withOrientation('auto') + + panel.stat.options.reduceOptions.withCalcs(['lastNotNull']) + + panel.stat.options.reduceOptions.withFields('') + + panel.stat.options.reduceOptions.withValues(false) + + panel.stat.options.withTextMode('auto') + + panel.stat.queryOptions.withTargets([ + promQ.new('${datasource}', '(1 - openslo_sli_error_rate_30d{openslo_slo_name="$slo"}) * 100') + + promQ.withLegendFormat('{{openslo_slo_name}}') + + promQ.withRefId('A'), + ]) + + panel.stat.panelOptions.withGridPos(h=8, w=6, x=18, y=16) + + { datasource: ds, id: 12 }; + +local burndownTs = + panel.timeSeries.new('Error Budget Burndown') + + panel.timeSeries.standardOptions.withMin(0) + + panel.timeSeries.standardOptions.withMax(100) + + panel.timeSeries.standardOptions.withUnit('percent') + + panel.timeSeries.standardOptions.withDecimals(2) + + panel.timeSeries.standardOptions.color.withMode('thresholds') + + panel.timeSeries.standardOptions.thresholds.withSteps([{ color: 'green', value: null }]) + + panel.timeSeries.fieldConfig.defaults.custom.withDrawStyle('line') + + panel.timeSeries.fieldConfig.defaults.custom.withFillOpacity(10) + + panel.timeSeries.fieldConfig.defaults.custom.withLineWidth(1) + + panel.timeSeries.fieldConfig.defaults.custom.withShowPoints('never') + + panel.timeSeries.options.legend.withDisplayMode('list') + + panel.timeSeries.options.legend.withPlacement('bottom') + + panel.timeSeries.options.legend.withShowLegend(true) + + panel.timeSeries.options.legend.withCalcs([]) + + panel.timeSeries.options.tooltip.withMode('multi') + + panel.timeSeries.options.tooltip.withSort('desc') + + panel.timeSeries.queryOptions.withTargets([ + promQ.new('${datasource}', 'openslo_slo_period_error_budget_remaining{openslo_slo_name="$slo"} * 100') + + promQ.withLegendFormat('Period remaining') + + promQ.withRefId('A'), + ]) + + panel.stat.panelOptions.withGridPos(h=8, w=18, x=0, y=16) + + { datasource: ds, id: 13 }; + +local remainingStat = + panel.stat.new('28d Remaining Error Budget') + + panel.stat.standardOptions.withUnit('percent') + + panel.stat.standardOptions.withDecimals(2) + + panel.stat.standardOptions.color.withMode('thresholds') + + panel.stat.standardOptions.thresholds.withSteps([{ color: 'green', value: null }]) + + panel.stat.options.withColorMode('value') + + panel.stat.options.withGraphMode('area') + + panel.stat.options.withJustifyMode('auto') + + panel.stat.options.withOrientation('auto') + + panel.stat.options.reduceOptions.withCalcs(['lastNotNull']) + + panel.stat.options.reduceOptions.withFields('') + + panel.stat.options.reduceOptions.withValues(false) + + panel.stat.options.withTextMode('auto') + + panel.stat.queryOptions.withTargets([ + promQ.new('${datasource}', 'openslo_slo_period_error_budget_remaining{openslo_slo_name="$slo"} * 100') + + promQ.withLegendFormat('{{openslo_slo_name}}') + + promQ.withRefId('A'), + ]) + + panel.stat.panelOptions.withGridPos(h=8, w=6, x=18, y=24) + + { datasource: ds, id: 14 }; + +local burnRateTs = + panel.timeSeries.new('Error Budget Burn Rate') + + panel.timeSeries.standardOptions.withDecimals(2) + + panel.timeSeries.standardOptions.color.withMode('thresholds') + + panel.timeSeries.standardOptions.thresholds.withSteps([{ color: 'green', value: null }]) + + panel.timeSeries.fieldConfig.defaults.custom.withDrawStyle('line') + + panel.timeSeries.fieldConfig.defaults.custom.withFillOpacity(10) + + panel.timeSeries.fieldConfig.defaults.custom.withLineWidth(1) + + panel.timeSeries.fieldConfig.defaults.custom.withShowPoints('never') + + panel.timeSeries.options.legend.withDisplayMode('list') + + panel.timeSeries.options.legend.withPlacement('bottom') + + panel.timeSeries.options.legend.withShowLegend(true) + + panel.timeSeries.options.legend.withCalcs([]) + + panel.timeSeries.options.tooltip.withMode('multi') + + panel.timeSeries.options.tooltip.withSort('desc') + + panel.timeSeries.queryOptions.withTargets([ + promQ.new('${datasource}', 'openslo_slo_current_burn_rate{openslo_slo_name="$slo"}') + + promQ.withLegendFormat('Current burn rate') + + promQ.withRefId('A'), + ]) + + panel.stat.panelOptions.withGridPos(h=8, w=18, x=0, y=24) + + { datasource: ds, id: 15 }; + +local currentBR = + panel.stat.new('Current Burn Rate') + + panel.stat.standardOptions.withDecimals(2) + + panel.stat.standardOptions.color.withMode('thresholds') + + panel.stat.standardOptions.thresholds.withSteps([{ color: 'green', value: null }]) + + panel.stat.options.withColorMode('value') + + panel.stat.options.withGraphMode('area') + + panel.stat.options.withJustifyMode('auto') + + panel.stat.options.withOrientation('auto') + + panel.stat.options.reduceOptions.withCalcs(['lastNotNull']) + + panel.stat.options.reduceOptions.withFields('') + + panel.stat.options.reduceOptions.withValues(false) + + panel.stat.options.withTextMode('auto') + + panel.stat.queryOptions.withTargets([ + promQ.new('${datasource}', 'openslo_slo_current_burn_rate{openslo_slo_name="$slo"}') + + promQ.withLegendFormat('{{openslo_slo_name}}') + + promQ.withRefId('A'), + ]) + + panel.stat.panelOptions.withGridPos(h=8, w=6, x=18, y=32) + + { datasource: ds, id: 16 }; + +// ----- dashboard ----------------------------------------------------------- + +g.dashboard.new('OpenSLO - SLO detail') ++ g.dashboard.withUid('openslo-detail') ++ g.dashboard.withDescription( + 'Per-SLO drilldown. Open from OpenSLO - Manage SLOs.' +) ++ g.dashboard.withTags(['openslo']) ++ g.dashboard.time.withFrom('now-30d') ++ g.dashboard.time.withTo('now') ++ g.dashboard.withRefresh('30s') ++ g.dashboard.withVariables([dsVar, sloVar, descVar, targetVar]) ++ g.dashboard.withPanels([ + textPanel, + sloStat, + sliTs, + sli28Stat, + burndownTs, + remainingStat, + burnRateTs, + currentBR, +]) diff --git a/deploy/mixins/openslo-list.jsonnet b/deploy/mixins/openslo-list.jsonnet new file mode 100644 index 0000000..deed23f --- /dev/null +++ b/deploy/mixins/openslo-list.jsonnet @@ -0,0 +1,178 @@ +// Generate deploy/dashboards/openslo-list.json +// Variables: datasource (Prometheus picker), slo (multi + includeAll so the +// SLO source list drives both the table and the same-name fields in the +// table columns via $slo matcher in target expressions). +local g = import '../../tmp/grafonnet-vendor/vendor/github.com/grafana/grafonnet/gen/grafonnet-v13.0.0/main.libsonnet'; + +local var = g.dashboard.variable; +local panel = g.panel; +local promQ = g.query.prometheus; + +local ds = { type: 'prometheus', uid: '${datasource}' }; + +local dsVar = + var.datasource.new('datasource', 'prometheus') + + var.datasource.generalOptions.withLabel('datasource') + + var.datasource.selectionOptions.withIncludeAll(false) + + var.datasource.selectionOptions.withMulti(false) + + { allowCustomValue: false, pluginId: 'prometheus', refresh: 1 }; + +local sloVar = + var.query.new('slo', 'label_values(openslo_slo_info, openslo_slo_name)') + + var.query.withDatasource(type='prometheus', uid='${datasource}') + + var.query.selectionOptions.withIncludeAll(true) + + var.query.selectionOptions.withMulti(true) + + var.query.refresh.onTime() + + { allowCustomValue: false }; + +// StatusQ mapping: array form per current Grafana dashboards schema. Each +// entry carries a `value` field so Grafana treats the lookup as a value +// mapping rather than a label mapping. 0..3 enum drives the colors and +// text. Background fill is applied via custom.cellOptions + thresholds. +local statusMappings = [ + { + type: 'value', + options: { + '0': { text: 'Healthy', color: 'green', index: 0, value: '0' }, + '1': { text: 'Burning', color: 'yellow', index: 1, value: '1' }, + '2': { text: 'Critical', color: 'orange', index: 2, value: '2' }, + '3': { text: 'Breached', color: 'red', index: 3, value: '3' }, + }, + }, +]; + +local statusStatusQOverride = { + matcher: { id: 'byName', options: 'Value #StatusQ' }, + properties: [ + { id: 'displayName', value: 'Status' }, + { id: 'mappings', value: statusMappings }, + { id: 'custom.cellOptions', value: { type: 'color-background', mode: 'basic', applyToRow: false } }, + { id: 'thresholds', value: { + mode: 'absolute', + steps: [ + { color: 'red', value: null }, + { color: 'green', value: 0 }, + { color: 'yellow', value: 1 }, + { color: 'orange', value: 2 }, + { color: 'red', value: 3 }, + ], + } }, + ], +}; + +local objectiveQOverride = { + matcher: { id: 'byName', options: 'Value #ObjectiveQ' }, + properties: [ + { id: 'displayName', value: 'Objective %' }, + { id: 'unit', value: 'percent' }, + { id: 'decimals', value: 2 }, + ], +}; + +local periodSLIQOverride = { + matcher: { id: 'byName', options: 'Value #PeriodSLIQ' }, + properties: [ + { id: 'displayName', value: 'Period SLI' }, + { id: 'unit', value: 'percent' }, + { id: 'decimals', value: 2 }, + ], +}; + +local budgetQOverride = { + matcher: { id: 'byName', options: 'Value #BudgetQ' }, + properties: [ + { id: 'displayName', value: 'Budget Left %' }, + { id: 'unit', value: 'percent' }, + { id: 'custom.cellOptions', value: { type: 'color-background' } }, + { id: 'thresholds', value: { + mode: 'absolute', + steps: [ + { color: 'red', value: null }, + { color: 'red', value: 0 }, + { color: 'yellow', value: 15 }, + { color: 'green', value: 50 }, + ], + } }, + { id: 'decimals', value: 2 }, + ], +}; + +local timeHiddenOverride = { + matcher: { id: 'byName', options: 'Time' }, + properties: [ + { id: 'custom.hidden', value: true }, + ], +}; + +local tablePanel = + panel.table.new('Service Level Objectives') + + { + datasource: ds, + targets: [ + promQ.new('${datasource}', 'openslo_slo_status{openslo_slo_name=~"$slo"}') + + promQ.withRefId('StatusQ') + promQ.withInstant(true) + promQ.withFormat('table'), + promQ.new('${datasource}', 'openslo_slo_objective{openslo_slo_name=~"$slo"} * 100') + + promQ.withRefId('ObjectiveQ') + promQ.withInstant(true) + promQ.withFormat('table'), + promQ.new('${datasource}', '(1 - openslo_sli_error_rate_30d{openslo_slo_name=~"$slo"}) * 100') + + promQ.withRefId('PeriodSLIQ') + promQ.withInstant(true) + promQ.withFormat('table'), + promQ.new('${datasource}', 'openslo_slo_period_error_budget_remaining{openslo_slo_name=~"$slo"} * 100') + + promQ.withRefId('BudgetQ') + promQ.withInstant(true) + promQ.withFormat('table'), + ], + transformations: [ + { id: 'merge', options: {} }, + // includeByName is a positive-list filter stable across rule-template + // refactors; drops __name__, chaos_flag, openslo_service_name, + // openslo_spec_version, openslo_slo_description, and Time. + { id: 'organize', options: { + includeByName: { + openslo_slo_name: true, + 'Value #StatusQ': true, + 'Value #ObjectiveQ': true, + 'Value #PeriodSLIQ': true, + 'Value #BudgetQ': true, + }, + indexByName: { + openslo_slo_name: 0, + 'Value #ObjectiveQ': 1, + 'Value #PeriodSLIQ': 2, + 'Value #StatusQ': 3, + 'Value #BudgetQ': 4, + }, + renameByName: { + openslo_slo_name: 'Service Level Objective', + }, + } }, + ], + options: { + showHeader: true, + sortBy: [{ desc: false, displayName: 'SLO name' }], + }, + fieldConfig: { + defaults: { + custom: { + align: 'left', + cellOptions: { type: 'auto' }, + }, + }, + overrides: [ + statusStatusQOverride, + objectiveQOverride, + periodSLIQOverride, + budgetQOverride, + timeHiddenOverride, + ], + }, + gridPos: { x: 0, y: 0, w: 24, h: 24 }, + }; + +g.dashboard.new('OpenSLO - Manage SLOs') ++ g.dashboard.withUid('openslo-list') ++ g.dashboard.withDescription( + 'All SLOs loaded by opensloctl. Click any row to drill into the per-SLO detail dashboard.' +) ++ g.dashboard.withTags(['openslo']) ++ g.dashboard.time.withFrom('now-7d') ++ g.dashboard.time.withTo('now') ++ g.dashboard.withRefresh('30s') ++ g.dashboard.withVariables([dsVar, sloVar]) ++ g.dashboard.withPanels([tablePanel]) diff --git a/deploy/mixins/rules/openslo-integrity.jsonnet b/deploy/mixins/rules/openslo-integrity.jsonnet new file mode 100644 index 0000000..f104dcd --- /dev/null +++ b/deploy/mixins/rules/openslo-integrity.jsonnet @@ -0,0 +1,69 @@ +// Source of truth for the opensloctl integrity alert. +// Renders to ../rules/openslo-integrity-rules.yaml via `make -C deploy/mixins rules`. +// +// NOT generated by opensloctl - placed here so the per-SLO `-rules.yaml` +// generator doesn't clobber it on each `make generate`. +local recordings = [ + { + record: 'openslo_slo_metric_missing', + expr: ||| + ( + count by (openslo_slo_name) (openslo_slo_info) + unless on (openslo_slo_name) + count by (openslo_slo_name) (openslo_sli_error_rate_5m) + ) == 1 + |||, + labels: { openslo_alert_severity: 'page' }, + }, +]; + +local alerts = [ + { + alert: 'OpenSloSpecDrift', + expr: 'openslo_slo_metric_missing == 1', + ['for']: '10m', + labels: { + openslo_alert_severity: 'page', + openslo_notification_target: 'engineers', + }, + annotations: { + summary: 'opensloctl: SLI metric missing for {{ $labels.openslo_slo_name }}', + description: ||| + openslo_slo_info is registered for {{ $labels.openslo_slo_name }} but + openslo_sli_error_rate_5m has produced no sample for the last 10m. + + Common causes: + - service_name in the SLO spec does not match what the OpenTelemetry + collector's service.name resource attribute actually emits. + - le="..." in a histogram bucket query references a boundary that the + trace-span-metrics connector does not export. + - span_name="..." filter matches no real spans. + + Fix the selector in examples/oteldemo/specs/.yaml and re-run + `cd examples/oteldemo && make generate && make sync`. + |||, + }, + }, +]; + +// Render via std.manifestYamlDoc, with quote_keys=false so YAML keys +// are emitted unquoted (the form `promtool check rules` parses). The +// `make rules` target runs jsonnet with `-S` so the resulting string is +// emitted as plain text rather than JSON-escaped. +std.manifestYamlDoc( + { + groups: [ + { + name: 'openslo-integrity-recordings', + interval: '1m', + rules: recordings, + }, + { + name: 'openslo-integrity-alerts', + rules: alerts, + }, + ], + }, + indent_array_in_object=false, + quote_keys=false, +) diff --git a/deploy/rules/openslo-integrity-rules.yaml b/deploy/rules/openslo-integrity-rules.yaml new file mode 100644 index 0000000..8ffc9f1 --- /dev/null +++ b/deploy/rules/openslo-integrity-rules.yaml @@ -0,0 +1,36 @@ +groups: +- interval: "1m" + name: "openslo-integrity-recordings" + rules: + - expr: | + ( + count by (openslo_slo_name) (openslo_slo_info) + unless on (openslo_slo_name) + count by (openslo_slo_name) (openslo_sli_error_rate_5m) + ) == 1 + labels: + openslo_alert_severity: "page" + record: "openslo_slo_metric_missing" +- name: "openslo-integrity-alerts" + rules: + - alert: "OpenSloSpecDrift" + annotations: + description: | + openslo_slo_info is registered for {{ $labels.openslo_slo_name }} but + openslo_sli_error_rate_5m has produced no sample for the last 10m. + + Common causes: + - service_name in the SLO spec does not match what the OpenTelemetry + collector's service.name resource attribute actually emits. + - le="..." in a histogram bucket query references a boundary that the + trace-span-metrics connector does not export. + - span_name="..." filter matches no real spans. + + Fix the selector in examples/oteldemo/specs/.yaml and re-run + `cd examples/oteldemo && make generate && make sync`. + summary: "opensloctl: SLI metric missing for {{ $labels.openslo_slo_name }}" + expr: "openslo_slo_metric_missing == 1" + for: "10m" + labels: + openslo_alert_severity: "page" + openslo_notification_target: "engineers" diff --git a/examples/api-latency-slo/README.md b/examples/api-latency-slo/README.md new file mode 100644 index 0000000..9dd6859 --- /dev/null +++ b/examples/api-latency-slo/README.md @@ -0,0 +1,85 @@ +# `api-latency-slo` - API latency with single-window burn-rate alerting + +Demonstrates the [`burn-rate`](../README.md#burn-rate--sre-%C2%A74) alerting strategy (SRE Workbook § 4). One burn-rate condition per severity, no tiering. + +## Strategy + +The simplest meaningful burn-rate alert. Each severity has a single burn-rate multiplier over a single lookback window. No OR-ed windows, no AND-ed tiers - just one threshold and one window per severity. + +## Files + +``` +api-latency-slo/ +├── service.yaml # "api-gateway" service +├── datasource.yaml # Prometheus datasource +├── sli.yaml # thresholdMetric: P99 latency (histogram_quantile) +├── alert-condition-page.yaml # kind: burn-rate, threshold: 14.4, window: 5m +├── alert-condition-ticket.yaml # kind: burn-rate, threshold: 3, window: 2h +├── alert-policy-page.yaml # page → pagerduty +├── alert-policy-ticket.yaml # ticket → slack +├── notification-target-pagerduty.yaml +├── notification-target-slack.yaml +└── slo.yaml # 99.9% P99 < 500ms, refs both policies +``` + +## Spec diagram + +```text + ┌─────────────────────┐ + │ AlertNotification │ + │ Target: pagerduty │ + │ name: oncall-pd │ + └──────────▲──────────┘ + │ targetRef +┌──────────────────┐ spec.alertPolicies[0] ┌──────────┴──────────┐ +│ AlertCondition │ ──────────────────────▶│ AlertPolicy │ +│ severity: page │ │ api-latency-page-… │ +│ threshold: 14.4 │ │ conditions[0] │ +│ window: 5m │ │ notification: pd │ +│ kind: burn-rate │ └─────────────────────┘ +└─▲────────────────┘ + │ conditionRef + │ +┌─┴──────────────┐ indicatorRef ┌──────────────────────┐ +│ AlertCond. │ (note: SLI uses │ SLO │ +│ api-latency- │ thresholdMetric, │ api-latency-slo │ +│ page │ not ratioMetric) │ budgetMethod: │ +└───────────────┘ │ Occurrences │ + │ window: 30d rolling │ + │ objective: P99<500ms │ + │ alertPolicies: … │ + └──────────────────────┘ +``` + +## Generated rules (highlights) + +``` +opensloctl generate -f examples/api-latency-slo -o output/ +``` + +The generator renders `api-latency-slo-rules.yaml` containing the SLO info recordings, windowed SLI recordings (P99 over 5m, 30m, 1h, …) and an `openslo-alerts-api-latency-slo` group with a single alert per severity: + +```yaml +- alert: ApiLatencySloBurnRate + expr: openslo_sli_error_rate_5m{openslo_slo_name="api-latency-slo"} / (1 - openslo_slo_objective{openslo_slo_name="api-latency-slo"}) >= 14.400000 + for: 2m + labels: + severity: page +- alert: ApiLatencySloBurnRate + expr: openslo_sli_error_rate_2h{openslo_slo_name="api-latency-slo"} / (1 - openslo_slo_objective{openslo_slo_name="api-latency-slo"}) >= 3.000000 + for: 15m + labels: + severity: ticket +``` + +## Run + +``` +opensloctl generate -f examples/api-latency-slo -o output/ +ls output/ +# api-latency-slo-rules.yaml +``` + +## When to use this kind + +`burn-rate` is the simplest burn-based alert - useful when you only want one threshold per severity and don't need the smoothing that paired short+long windows provide. For better noise immunity, upgrade to [`multi-window-multi-burn-rate`](../README.md#multi-window-multi-burn-rate--sre-%C2%A76-recommended). diff --git a/examples/api-latency-slo/alert-condition-page.yaml b/examples/api-latency-slo/alert-condition-page.yaml index d65e33e..d5b97be 100644 --- a/examples/api-latency-slo/alert-condition-page.yaml +++ b/examples/api-latency-slo/alert-condition-page.yaml @@ -6,7 +6,7 @@ spec: severity: page description: Page on-call when API latency burn rate is too fast condition: - kind: burnrate + kind: burn-rate op: gte threshold: 14.4 lookbackWindow: 5m diff --git a/examples/api-latency-slo/alert-condition-ticket.yaml b/examples/api-latency-slo/alert-condition-ticket.yaml index abe1df8..419188a 100644 --- a/examples/api-latency-slo/alert-condition-ticket.yaml +++ b/examples/api-latency-slo/alert-condition-ticket.yaml @@ -6,7 +6,7 @@ spec: severity: ticket description: Create ticket on sustained API latency burn rate condition: - kind: burnrate + kind: burn-rate op: gte threshold: 3 lookbackWindow: 2h diff --git a/examples/error-budget-slo/README.md b/examples/error-budget-slo/README.md new file mode 100644 index 0000000..8e15ce6 --- /dev/null +++ b/examples/error-budget-slo/README.md @@ -0,0 +1,70 @@ +# `error-budget-slo` - Checkout success rate with single-window burn-rate alerting + +Sibling of [`api-latency-slo`](../api-latency-slo/README.md), but with a ratioMetric SLI (good/total requests) instead of a thresholdMetric (latency histogram). Both examples exercise the [`burn-rate`](../README.md#burn-rate--sre-%C2%A74) alerting strategy. + +## Strategy + +Same as `api-latency-slo`: one `burn-rate` condition per severity. Use this example when you have a counter-style SLI (success/total request counts) rather than a latency histogram. + +## Files + +``` +error-budget-slo/ +├── service.yaml # "checkout-service" +├── datasource.yaml +├── sli.yaml # ratioMetric: 2xx / total http requests +├── alert-condition-page.yaml # kind: burn-rate, threshold: 14.4, window: 5m +├── alert-condition-ticket.yaml # kind: burn-rate, threshold: 3, window: 2h +├── alert-policy-page.yaml # page → pagerduty +├── alert-policy-ticket.yaml # ticket → slack +├── notification-target-pagerduty.yaml +├── notification-target-slack.yaml +└── slo.yaml # 99.9% checkout success over 30d +``` + +## Spec diagram + +```text +┌──────────────────┐ spec.alertPolicies[0] ┌──────────────────────┐ +│ AlertCondition │ ──────────────────────▶│ AlertPolicy │ +│ severity: page │ │ checkout-page-alert │ +│ threshold: 14.4 │ │ notification: pd │ +│ window: 5m │ └──────────────────────┘ +│ kind: burn-rate │ +└─▲────────────────┘ + │ conditionRef + │ +┌─┴──────────────┐ ┌──────────────────────┐ +│ AlertCond. │ │ SLO │ +│ checkout-page │ ───── indicator ──────▶ │ checkout-slo │ +└───────────────┘ │ ratioMetric: │ + │ good / total │ + │ objective: 99.9% │ + │ alertPolicies: … │ + └──────────────────────┘ +``` + +## Generated rules (highlights) + +``` +opensloctl generate -f examples/error-budget-slo -o output/ +``` + +The generator renders `checkout-slo-rules.yaml` with the SLO info recordings, the windowed goodness recordings (`openslo_sli_error_rate_*`) and an `openslo-alerts-checkout-slo` group with two burn-rate alerts: + +```yaml +- alert: CheckoutSloBurnRate + expr: openslo_sli_error_rate_5m{openslo_slo_name="checkout-slo"} / (1 - openslo_slo_objective{openslo_slo_name="checkout-slo"}) >= 14.400000 + for: 2m + labels: + severity: page +- alert: CheckoutSloBurnRate + expr: openslo_sli_error_rate_2h{openslo_slo_name="checkout-slo"} / (1 - openslo_slo_objective{openslo_slo_name="checkout-slo"}) >= 3.000000 + for: 15m + labels: + severity: ticket +``` + +## When to use this kind + +Same as `api-latency-slo`: `burn-rate` is the simplest burn-based alert. The choice of SLI source (ratioMetric here vs thresholdMetric in api-latency-slo) doesn't change the alert strategy - pick whichever fits your data. diff --git a/examples/error-budget-slo/alert-condition-page.yaml b/examples/error-budget-slo/alert-condition-page.yaml index 51294ea..7ddce61 100644 --- a/examples/error-budget-slo/alert-condition-page.yaml +++ b/examples/error-budget-slo/alert-condition-page.yaml @@ -6,7 +6,7 @@ spec: severity: page description: Page when checkout error budget burns fast condition: - kind: burnrate + kind: burn-rate op: gte threshold: 14.4 lookbackWindow: 5m diff --git a/examples/error-budget-slo/alert-condition-ticket.yaml b/examples/error-budget-slo/alert-condition-ticket.yaml index e29a82d..4b0dd53 100644 --- a/examples/error-budget-slo/alert-condition-ticket.yaml +++ b/examples/error-budget-slo/alert-condition-ticket.yaml @@ -6,7 +6,7 @@ spec: severity: ticket description: Ticket on sustained checkout error budget burn condition: - kind: burnrate + kind: burn-rate op: gte threshold: 3 lookbackWindow: 2h diff --git a/examples/error-rate-slo/README.md b/examples/error-rate-slo/README.md new file mode 100644 index 0000000..b21f2c2 --- /dev/null +++ b/examples/error-rate-slo/README.md @@ -0,0 +1,93 @@ +# `error-rate-slo` - Checkout availability with raw error-rate alerting + +Demonstrates the [`error-rate`](../README.md#error-rate--sre-%C2%A7%C2%A7-13) alerting strategy. The simplest possible setup: one `error-rate` condition per severity, comparing the SLI error rate directly to an absolute threshold. + +## Strategy + +From the SRE Workbook §§ 1–3, the "target error rate" / "increased alert window" / "alert on incrementing duration" patterns all share the same expression shape - the only differences are: + +1. `lookbackWindow` length (short vs long) +2. Whether `alertAfter` is set (maps to Prom `for:`) + +So we collapse all three into a single kind: `error-rate`. + +## Files + +``` +error-rate-slo/ +├── service.yaml # "checkout" service +├── sli.yaml # ratioMetric: success / total checkout requests +├── alert-condition-page.yaml # kind: error-rate, threshold: 0.001, window 5m +├── alert-condition-ticket.yaml # kind: error-rate, threshold: 0.005, window 1h +├── alert-policy-page.yaml # 1 condition per policy (SDK rule) +├── alert-policy-ticket.yaml +├── notification-target-pagerduty.yaml # page → pagerduty +├── notification-target-slack.yaml # ticket → slack +└── slo.yaml # 99.9% availability over 30d, refs both policies +``` + +## Spec diagram + +```text + ┌─────────────────────┐ + │ AlertNotification │ + │ Target: pagerduty │ + │ name: oncall-pd │ + └──────────▲──────────┘ + │ targetRef +┌──────────────────┐ spec.alertPolicies[0] ┌──────────┴──────────┐ +│ RuleCondition │ ──────────────────────▶│ AlertPolicy │ +│ severity: page │ │ name: │ +│ threshold: 0.001 │ │ checkout-page-… │ +│ window: 5m │ │ conditions[0].cond… │ +│ kind: error-rate │ │ → checkout-page │ +└───▲──────────────┘ └─────────────────────┘ + │ conditionRef + │ +┌───┴──────────┐ spec.indicatorRef ┌──────────────────────┐ +│ AlertCond. │ │ SLO │ +│ checkout-page│ ────────────── uses ────▶ │ checkout-availability│ +└──────────────┘ │ service: checkout │ + │ │ budgetMethod: │ + │ uses │ Occurrences │ + ▼ │ window: 30d rolling │ +┌──────────────┐ │ objective: 0.999 │ +│ SLI │ │ alertPolicies: │ +│ checkout-… │ │ → checkout-page-alert│ +└──────────────┘ │ → checkout-ticket-… │ + └──────────────────────┘ +``` + +## Generated rules (highlights) + +The generator renders a single file `checkout-availability-rules.yaml` containing both recording rules and an `openslo-alerts-checkout-availability` group. The alert group has two `error-rate` alerts (one per severity, same name). + +```yaml +- alert: CheckoutAvailabilityErrorRate + expr: openslo_sli_error_rate_5m{openslo_slo_name="checkout-availability"} >= 0.001000 + for: 2m + labels: + severity: page +- alert: CheckoutAvailabilityErrorRate + expr: openslo_sli_error_rate_1h{openslo_slo_name="checkout-availability"} >= 0.005000 + for: " + labels: + severity: ticket +``` + +Validation that fires at load time: + +- `kind: error-rate` (accepted ✓) +- `threshold: 0.001` and `threshold: 0.005` both in `(0, 1]` ✓ + +## Run + +``` +opensloctl generate -f examples/error-rate-slo -o output/ +ls output/ +# checkout-availability-rules.yaml +``` + +## When to use this kind + +Pick `error-rate` when you want the simplest possible burn-free alerting setup. The threshold is the **error rate itself**, not a burn multiplier - easy to reason about, but lacks the smoothing that burn rates provide against short bursts. diff --git a/examples/error-rate-slo/alert-condition-page.yaml b/examples/error-rate-slo/alert-condition-page.yaml new file mode 100644 index 0000000..41f6d65 --- /dev/null +++ b/examples/error-rate-slo/alert-condition-page.yaml @@ -0,0 +1,13 @@ +apiVersion: openslo/v1 +kind: AlertCondition +metadata: + name: checkout-page +spec: + severity: page + description: Page on-call when checkout error rate exceeds the SLO threshold quickly + condition: + kind: error-rate + op: gte + threshold: 0.001 + lookbackWindow: 5m + alertAfter: 2m diff --git a/examples/error-rate-slo/alert-condition-ticket.yaml b/examples/error-rate-slo/alert-condition-ticket.yaml new file mode 100644 index 0000000..a37e500 --- /dev/null +++ b/examples/error-rate-slo/alert-condition-ticket.yaml @@ -0,0 +1,13 @@ +apiVersion: openslo/v1 +kind: AlertCondition +metadata: + name: checkout-ticket +spec: + severity: ticket + description: Open a ticket when checkout error rate stays elevated over a longer window + condition: + kind: error-rate + op: gte + threshold: 0.005 + lookbackWindow: 1h + alertAfter: 30m diff --git a/examples/error-rate-slo/alert-policy-page.yaml b/examples/error-rate-slo/alert-policy-page.yaml new file mode 100644 index 0000000..de52b1f --- /dev/null +++ b/examples/error-rate-slo/alert-policy-page.yaml @@ -0,0 +1,11 @@ +apiVersion: openslo/v1 +kind: AlertPolicy +metadata: + name: checkout-page-alert +spec: + description: Page alert for checkout error rate + alertWhenBreaching: true + conditions: + - conditionRef: checkout-page + notificationTargets: + - targetRef: oncall-pagerduty diff --git a/examples/error-rate-slo/alert-policy-ticket.yaml b/examples/error-rate-slo/alert-policy-ticket.yaml new file mode 100644 index 0000000..1ddeb8d --- /dev/null +++ b/examples/error-rate-slo/alert-policy-ticket.yaml @@ -0,0 +1,11 @@ +apiVersion: openslo/v1 +kind: AlertPolicy +metadata: + name: checkout-ticket-alert +spec: + description: Ticket alert for sustained checkout error rate + alertWhenBreaching: true + conditions: + - conditionRef: checkout-ticket + notificationTargets: + - targetRef: oncall-slack diff --git a/examples/error-rate-slo/notification-target-pagerduty.yaml b/examples/error-rate-slo/notification-target-pagerduty.yaml new file mode 100644 index 0000000..25dc4ec --- /dev/null +++ b/examples/error-rate-slo/notification-target-pagerduty.yaml @@ -0,0 +1,7 @@ +apiVersion: openslo/v1 +kind: AlertNotificationTarget +metadata: + name: oncall-pagerduty +spec: + description: Page on-call engineer via PagerDuty + target: pagerduty diff --git a/examples/error-rate-slo/notification-target-slack.yaml b/examples/error-rate-slo/notification-target-slack.yaml new file mode 100644 index 0000000..663fea6 --- /dev/null +++ b/examples/error-rate-slo/notification-target-slack.yaml @@ -0,0 +1,7 @@ +apiVersion: openslo/v1 +kind: AlertNotificationTarget +metadata: + name: oncall-slack +spec: + description: Notify #oncall channel via Slack + target: slack diff --git a/examples/error-rate-slo/service.yaml b/examples/error-rate-slo/service.yaml new file mode 100644 index 0000000..f77df20 --- /dev/null +++ b/examples/error-rate-slo/service.yaml @@ -0,0 +1,7 @@ +apiVersion: openslo/v1 +kind: Service +metadata: + name: checkout + displayName: Checkout Service +spec: + description: Checkout flow handling cart-to-order submission diff --git a/examples/error-rate-slo/sli.yaml b/examples/error-rate-slo/sli.yaml new file mode 100644 index 0000000..6b4ad38 --- /dev/null +++ b/examples/error-rate-slo/sli.yaml @@ -0,0 +1,18 @@ +apiVersion: openslo/v1 +kind: SLI +metadata: + name: checkout-success-rate +spec: + description: Fraction of checkout requests that succeed + ratioMetric: + counter: true + good: + metricSource: + type: Prometheus + spec: + query: sum(rate(checkout_requests_total{status="success"}[{{.Window}}])) + total: + metricSource: + type: Prometheus + spec: + query: sum(rate(checkout_requests_total[{{.Window}}])) diff --git a/examples/error-rate-slo/slo.yaml b/examples/error-rate-slo/slo.yaml new file mode 100644 index 0000000..a80b158 --- /dev/null +++ b/examples/error-rate-slo/slo.yaml @@ -0,0 +1,33 @@ +apiVersion: openslo/v1 +kind: SLO +metadata: + name: checkout-availability +spec: + description: Checkout availability, 99.9% over a rolling 30-day window + service: checkout + indicator: + metadata: + name: checkout-availability-indicator + spec: + ratioMetric: + counter: true + good: + metricSource: + type: Prometheus + spec: + query: sum(rate(checkout_requests_total{status="success"}[{{.Window}}])) + total: + metricSource: + type: Prometheus + spec: + query: sum(rate(checkout_requests_total[{{.Window}}])) + budgetingMethod: Occurrences + timeWindow: + - duration: 30d + isRolling: true + objectives: + - displayName: "99.9% checkout success" + target: 0.999 + alertPolicies: + - alertPolicyRef: checkout-page-alert + - alertPolicyRef: checkout-ticket-alert diff --git a/examples/multi-burn-slo/README.md b/examples/multi-burn-slo/README.md new file mode 100644 index 0000000..09ca8a2 --- /dev/null +++ b/examples/multi-burn-slo/README.md @@ -0,0 +1,97 @@ +# `multi-burn-slo` - Payment availability with multi-burn-rate alerting + +Demonstrates the [`multi-burn-rate`](../README.md#multi-burn-rate--sre-%C2%A75) alerting strategy (SRE Workbook § 5). Multiple burn rate windows OR-ed together per severity - no short/long AND pairing. + +## Strategy + +Two or more burn rate windows per severity. Each condition contributes one expression to the alert. The alert fires when **any** condition fires (OR). There's no AND tiering like the multi-window variant. + +A typical setup mirrors the [Burn Rate Alerts table from the SRE Workbook](https://sre.google/workbook/alerting-on-slos/#5-multiple-burn-rate-alerts): + +| Severity | Condition | Lookback | Threshold | Time to burn half the budget | +|---|---|---|---|---| +| page | 36x | 5 m | fastest possible spike detection | min | +| page | 6x | 30 m | catches sustained spike | 5 h | +| ticket | 3x | 2 h | moderate sustained rate | 10 h | +| ticket | 1x | 6 h | the slow burn at the SLO rate | 30 d | + +## Files + +``` +multi-burn-slo/ +├── service.yaml # "payments" service +├── sli.yaml # ratioMetric: success / total payment requests +├── alert-condition-36x.yaml # condition: kind: multi-burn-rate, 36x / 5m +├── alert-condition-6x.yaml # condition: kind: multi-burn-rate, 6x / 30m +├── alert-condition-3x.yaml # condition: kind: multi-burn-rate, 3x / 2h (ticket) +├── alert-condition-1x.yaml # condition: kind: multi-burn-rate, 1x / 6h (ticket) +├── alert-policy-36x.yaml # 1 condition per policy (SDK rule) +├── alert-policy-6x.yaml +├── alert-policy-3x.yaml +├── alert-policy-1x.yaml +├── notification-target-pagerduty.yaml # page → pagerduty +├── notification-target-slack.yaml # ticket → slack +└── slo.yaml # 99.95% over 30d, refs all four policies +``` + +## Spec diagram + +```text + page severity ticket severity + ───────────────────── ──────────────────────── + ┌────────────────────┐ ┌────────────────────┐ + │ AlertCondition │ │ AlertCondition │ + │ name: │ │ name: │ + │ payment-page-36x │ │ payment-ticket-3x │ + │ threshold: 36 │ │ threshold: 3 │ + │ window: 5m │ │ window: 2h │ + │ kind: multi-burn… │ │ kind: multi-burn… │ + └─▲─────────┬────────┘ └─▲─────────┬────────┘ + │ │ conditionRef │ │ conditionRef + │ ▼ │ ▼ + ┌─┴──────────────┐ ◀── AlertPolicy ──▶ ┌─┴──────────────┐ + │ AlertPolicy │ payment-page-36x… │ AlertPolicy │ + │ payment-…-36x │ │ payment-…-3x │ + │ notification: │ │ notification: │ + │ pagerduty │ │ slack │ + └────────────────┘ └────────────────┘ + + 6x condition + policy are also under "page"; 1x under "ticket". + All four are referenced from the same SLO via spec.alertPolicies. +``` + +## Generated rules (highlights) + +The generator renders a single file `payment-availability-rules.yaml` containing recording rules plus the `openslo-alerts-payment-availability` group with two `MultiBurnRate` alerts (one per severity): + +```yaml +- alert: PaymentAvailabilityMultiBurnRate # page + expr: |- + (openslo_sli_error_rate_5m{openslo_slo_name="payment-availability"} / (1 - openslo_slo_objective{openslo_slo_name="payment-availability"}) >= 36.000000) + or + (openslo_sli_error_rate_30m{openslo_slo_name="payment-availability"} / (1 - openslo_slo_objective{openslo_slo_name="payment-availability"}) >= 6.000000) + labels: + severity: page + +- alert: PaymentAvailabilityMultiBurnRate # ticket + expr: |- + (openslo_sli_error_rate_2h{openslo_slo_name="payment-availability"} / (1 - openslo_slo_objective{openslo_slo_name="payment-availability"}) >= 3.000000) + or + (openslo_sli_error_rate_6h{openslo_slo_name="payment-availability"} / (1 - openslo_slo_objective{openslo_slo_name="payment-availability"}) >= 1.000000) + labels: + severity: ticket +``` + +The same name across severities - `severity` is the discriminator - matches the way Prometheus alerts are typically structured. + +## Run + +``` +opensloctl generate -f examples/multi-burn-slo -o output/ +ls output/ +# payment-availability-rules.yaml +``` + +## When to use this kind + +Use `multi-burn-rate` when you want multiple burn-rate windows without the short/long AND pairing. If you need paired short+long windows, upgrade to [`multi-window-multi-burn-rate`](../../#multi-window-multi-burn-rate--sre-%C2%A76-recommended). diff --git a/examples/multi-burn-slo/alert-condition-1x.yaml b/examples/multi-burn-slo/alert-condition-1x.yaml new file mode 100644 index 0000000..bc32cd3 --- /dev/null +++ b/examples/multi-burn-slo/alert-condition-1x.yaml @@ -0,0 +1,13 @@ +apiVersion: openslo/v1 +kind: AlertCondition +metadata: + name: payment-ticket-1x +spec: + severity: ticket + description: Ticket on slow payment burn rate (1x over 6h) + condition: + kind: multi-burn-rate + op: gte + threshold: 1 + lookbackWindow: 6h + alertAfter: 30m diff --git a/examples/multi-burn-slo/alert-condition-36x.yaml b/examples/multi-burn-slo/alert-condition-36x.yaml new file mode 100644 index 0000000..bf481b6 --- /dev/null +++ b/examples/multi-burn-slo/alert-condition-36x.yaml @@ -0,0 +1,13 @@ +apiVersion: openslo/v1 +kind: AlertCondition +metadata: + name: payment-page-36x +spec: + severity: page + description: Page on-call when payment burn rate spikes hard (36x over 5m) + condition: + kind: multi-burn-rate + op: gte + threshold: 36 + lookbackWindow: 5m + alertAfter: 2m diff --git a/examples/multi-burn-slo/alert-condition-3x.yaml b/examples/multi-burn-slo/alert-condition-3x.yaml new file mode 100644 index 0000000..2ca8d55 --- /dev/null +++ b/examples/multi-burn-slo/alert-condition-3x.yaml @@ -0,0 +1,13 @@ +apiVersion: openslo/v1 +kind: AlertCondition +metadata: + name: payment-ticket-3x +spec: + severity: ticket + description: Ticket on sustained payment burn rate (3x over 2h) + condition: + kind: multi-burn-rate + op: gte + threshold: 3 + lookbackWindow: 2h + alertAfter: 15m diff --git a/examples/multi-burn-slo/alert-condition-6x.yaml b/examples/multi-burn-slo/alert-condition-6x.yaml new file mode 100644 index 0000000..ee55704 --- /dev/null +++ b/examples/multi-burn-slo/alert-condition-6x.yaml @@ -0,0 +1,13 @@ +apiVersion: openslo/v1 +kind: AlertCondition +metadata: + name: payment-page-6x +spec: + severity: page + description: Page on-call when payment burn rate stays elevated (6x over 30m) + condition: + kind: multi-burn-rate + op: gte + threshold: 6 + lookbackWindow: 30m + alertAfter: 5m diff --git a/examples/multi-burn-slo/alert-policy-1x.yaml b/examples/multi-burn-slo/alert-policy-1x.yaml new file mode 100644 index 0000000..689cf97 --- /dev/null +++ b/examples/multi-burn-slo/alert-policy-1x.yaml @@ -0,0 +1,11 @@ +apiVersion: openslo/v1 +kind: AlertPolicy +metadata: + name: payment-ticket-1x-policy +spec: + description: Ticket tier, slow (1x@6h) alert + alertWhenBreaching: true + conditions: + - conditionRef: payment-ticket-1x + notificationTargets: + - targetRef: oncall-slack diff --git a/examples/multi-burn-slo/alert-policy-36x.yaml b/examples/multi-burn-slo/alert-policy-36x.yaml new file mode 100644 index 0000000..2c2fca2 --- /dev/null +++ b/examples/multi-burn-slo/alert-policy-36x.yaml @@ -0,0 +1,11 @@ +apiVersion: openslo/v1 +kind: AlertPolicy +metadata: + name: payment-page-36x-policy +spec: + description: Page tier, fast (36x@5m) alert + alertWhenBreaching: true + conditions: + - conditionRef: payment-page-36x + notificationTargets: + - targetRef: oncall-pagerduty diff --git a/examples/multi-burn-slo/alert-policy-3x.yaml b/examples/multi-burn-slo/alert-policy-3x.yaml new file mode 100644 index 0000000..e9ca713 --- /dev/null +++ b/examples/multi-burn-slo/alert-policy-3x.yaml @@ -0,0 +1,11 @@ +apiVersion: openslo/v1 +kind: AlertPolicy +metadata: + name: payment-ticket-3x-policy +spec: + description: Ticket tier, fast (3x@2h) alert + alertWhenBreaching: true + conditions: + - conditionRef: payment-ticket-3x + notificationTargets: + - targetRef: oncall-slack diff --git a/examples/multi-burn-slo/alert-policy-6x.yaml b/examples/multi-burn-slo/alert-policy-6x.yaml new file mode 100644 index 0000000..3a1c076 --- /dev/null +++ b/examples/multi-burn-slo/alert-policy-6x.yaml @@ -0,0 +1,11 @@ +apiVersion: openslo/v1 +kind: AlertPolicy +metadata: + name: payment-page-6x-policy +spec: + description: Page tier, medium (6x@30m) alert + alertWhenBreaching: true + conditions: + - conditionRef: payment-page-6x + notificationTargets: + - targetRef: oncall-pagerduty diff --git a/examples/multi-burn-slo/notification-target-pagerduty.yaml b/examples/multi-burn-slo/notification-target-pagerduty.yaml new file mode 100644 index 0000000..25dc4ec --- /dev/null +++ b/examples/multi-burn-slo/notification-target-pagerduty.yaml @@ -0,0 +1,7 @@ +apiVersion: openslo/v1 +kind: AlertNotificationTarget +metadata: + name: oncall-pagerduty +spec: + description: Page on-call engineer via PagerDuty + target: pagerduty diff --git a/examples/multi-burn-slo/notification-target-slack.yaml b/examples/multi-burn-slo/notification-target-slack.yaml new file mode 100644 index 0000000..9b124b2 --- /dev/null +++ b/examples/multi-burn-slo/notification-target-slack.yaml @@ -0,0 +1,7 @@ +apiVersion: openslo/v1 +kind: AlertNotificationTarget +metadata: + name: oncall-slack +spec: + description: Notify #oncall-payments channel via Slack + target: slack diff --git a/examples/multi-burn-slo/service.yaml b/examples/multi-burn-slo/service.yaml new file mode 100644 index 0000000..7b752d9 --- /dev/null +++ b/examples/multi-burn-slo/service.yaml @@ -0,0 +1,7 @@ +apiVersion: openslo/v1 +kind: Service +metadata: + name: payments + displayName: Payments Service +spec: + description: Payment processing service handling all transactions diff --git a/examples/multi-burn-slo/sli.yaml b/examples/multi-burn-slo/sli.yaml new file mode 100644 index 0000000..8d6c6af --- /dev/null +++ b/examples/multi-burn-slo/sli.yaml @@ -0,0 +1,18 @@ +apiVersion: openslo/v1 +kind: SLI +metadata: + name: payment-success-rate +spec: + description: Fraction of payment requests that succeed + ratioMetric: + counter: true + good: + metricSource: + type: Prometheus + spec: + query: sum(rate(payment_requests_total{status="success"}[{{.Window}}])) + total: + metricSource: + type: Prometheus + spec: + query: sum(rate(payment_requests_total[{{.Window}}])) diff --git a/examples/multi-burn-slo/slo.yaml b/examples/multi-burn-slo/slo.yaml new file mode 100644 index 0000000..d8b131d --- /dev/null +++ b/examples/multi-burn-slo/slo.yaml @@ -0,0 +1,35 @@ +apiVersion: openslo/v1 +kind: SLO +metadata: + name: payment-availability +spec: + description: Payment availability, 99.95% over a rolling 30-day window + service: payments + indicator: + metadata: + name: payment-availability-indicator + spec: + ratioMetric: + counter: true + good: + metricSource: + type: Prometheus + spec: + query: sum(rate(payment_requests_total{status="success"}[{{.Window}}])) + total: + metricSource: + type: Prometheus + spec: + query: sum(rate(payment_requests_total[{{.Window}}])) + budgetingMethod: Occurrences + timeWindow: + - duration: 30d + isRolling: true + objectives: + - displayName: "99.95% payment success" + target: 0.9995 + alertPolicies: + - alertPolicyRef: payment-page-36x-policy + - alertPolicyRef: payment-page-6x-policy + - alertPolicyRef: payment-ticket-3x-policy + - alertPolicyRef: payment-ticket-1x-policy diff --git a/examples/multi-dim-slo/README.md b/examples/multi-dim-slo/README.md new file mode 100644 index 0000000..12d76ca --- /dev/null +++ b/examples/multi-dim-slo/README.md @@ -0,0 +1,73 @@ +# `multi-dim-slo` - One SLO expanded into many series by a label + +Demonstrates the [`burn-rate`](../README.md#burn-rate--sre-%C2%A74) alerting strategy (SRE Workbook § 4) combined with the [multi-dimensional SLI annotations](../README.md#multi-dimensional-slis). One SLO becomes one series per value of the chosen Prometheus label. + +## Strategy + +A single `api-latency` SLO with target 0.999 (P99 < 500 ms) and a 30d rolling window. Two `burn-rate` AlertConditions - page at 14.4x over 5m, ticket at 3x over 2h - wrapped in their own AlertPolicies. No tiering, no AND pairs. + +The multi-dim part: the SLO sets two annotations: + +```yaml +metadata: + annotations: + multi-dimensional-sli.openslo.com/label: service_name + multi-dimensional-sli.openslo.com/dimensions: "account,checkout,recommendation" +``` + +`service_name` is the dimension label that already exists on the underlying histogram series. The generator emits a `_unlabeled` layer of recording rules (carries `openslo_slo_name` only) and a `label_join` layer that joins the `service_name` value into `openslo_slo_name` with `-`, producing three series: + +``` +openslo_slo_current_burn_rate{openslo_slo_name="account-api-latency"} = 0.2 +openslo_slo_current_burn_rate{openslo_slo_name="checkout-api-latency"} = 1.4 +openslo_slo_current_burn_rate{openslo_slo_name="recommendation-api-latency"} = 4.7 +``` + +Same SLO target, same alert policies, three independently firing series. Page on the dimension that crosses 14.4x; ignore the others. + +## Files + +``` +multi-dim-slo/ +├── README.md +├── service.yaml # `api-gateway` service +├── sli.yaml # thresholdMetric on http_request_duration_seconds_bucket +├── slo.yaml # annotations + indicatorRef + alertPolicies +├── alert-condition-page.yaml # burn-rate, 14.4x, 5m +├── alert-condition-ticket.yaml # burn-rate, 3x, 2h +├── alert-policy-page.yaml # wraps the page condition +├── alert-policy-ticket.yaml # wraps the ticket condition +├── notification-target-pagerduty.yaml +└── notification-target-slack.yaml +``` + +This example has no Makefile. Validate and generate via: + +```bash +opensloctl validate -f examples/multi-dim-slo +opensloctl generate -f examples/multi-dim-slo -o output/ +``` + +## What gets generated + +One `api-latency-rules.yaml` per SLO. Inside: + +- `openslo-sli-recordings-api-latency`: `_unlabeled` versions of `openslo_slo_info`, `openslo_slo_objective`, `openslo_sli_error_rate_5m`, `openslo_sli_error_rate_30m`, `openslo_sli_error_rate_1h`, `openslo_sli_event_rate_5m`, `openslo_slo_current_burn_rate`, `openslo_slo_period_burn_rate`, `openslo_slo_period_error_budget_remaining`, `openslo_slo_status`. +- A parallel `openslo-sli-recordings-api-latency-joined` group with `label_join` rules that promote the `service_name` value into `openslo_slo_name`. +- `openslo-alerts-api-latency`: one alert per severity (`ApiLatencyBurnRate`), each condition contributes one expression. + +## When to use + +Use multi-dim when: +- The underlying metric already splits by a label and you want shared target definitions across all series. +- Alert routing benefits from per-dimension firing rather than aggregate (per-caller-service paging, per-region escalation). +- You want per-dimension burn dashboards without writing one SLO per dimension. + +Skip when: +- The chosen label is unbounded (`user_id`, raw trace IDs) - one recording rule per value burns Prometheus linearly. +- You only care about the aggregate across all series - a regular single-dim SLO is simpler. + +## See also + +- [Multi-dimensional SLIs](../README.md#multi-dimensional-slis) - registry entry, annotation semantics, runtime series shape. +- Burn-rate strategy details in [Alerting](../README.md#alerting) and [Burn-rate](../README.md#burn-rate--sre-%C2%A74). diff --git a/examples/multi-dim-slo/alert-condition-page.yaml b/examples/multi-dim-slo/alert-condition-page.yaml new file mode 100644 index 0000000..d96bea4 --- /dev/null +++ b/examples/multi-dim-slo/alert-condition-page.yaml @@ -0,0 +1,13 @@ +apiVersion: openslo/v1 +kind: AlertCondition +metadata: + name: api-latency-page-multi-dim +spec: + severity: page + description: Per-service latency budget exceeded at a fast burn rate + condition: + kind: burn-rate + op: gte + threshold: 14.4 + lookbackWindow: 5m + alertAfter: 2m diff --git a/examples/multi-dim-slo/alert-condition-ticket.yaml b/examples/multi-dim-slo/alert-condition-ticket.yaml new file mode 100644 index 0000000..bec7ae6 --- /dev/null +++ b/examples/multi-dim-slo/alert-condition-ticket.yaml @@ -0,0 +1,13 @@ +apiVersion: openslo/v1 +kind: AlertCondition +metadata: + name: api-latency-ticket-multi-dim +spec: + severity: ticket + description: Per-service latency budget exceeded at a slow burn rate + condition: + kind: burn-rate + op: gte + threshold: 3 + lookbackWindow: 2h + alertAfter: 15m diff --git a/examples/multi-dim-slo/alert-policy-page.yaml b/examples/multi-dim-slo/alert-policy-page.yaml new file mode 100644 index 0000000..85fe0bb --- /dev/null +++ b/examples/multi-dim-slo/alert-policy-page.yaml @@ -0,0 +1,10 @@ +apiVersion: openslo/v1 +kind: AlertPolicy +metadata: + name: api-latency-page-policy +spec: + alertWhenBreaching: true + conditions: + - conditionRef: api-latency-page-multi-dim + notificationTargets: + - targetRef: pagerduty diff --git a/examples/multi-dim-slo/alert-policy-ticket.yaml b/examples/multi-dim-slo/alert-policy-ticket.yaml new file mode 100644 index 0000000..eac4621 --- /dev/null +++ b/examples/multi-dim-slo/alert-policy-ticket.yaml @@ -0,0 +1,10 @@ +apiVersion: openslo/v1 +kind: AlertPolicy +metadata: + name: api-latency-ticket-policy +spec: + alertWhenBreaching: true + conditions: + - conditionRef: api-latency-ticket-multi-dim + notificationTargets: + - targetRef: slack diff --git a/examples/multi-dim-slo/notification-target-pagerduty.yaml b/examples/multi-dim-slo/notification-target-pagerduty.yaml new file mode 100644 index 0000000..8630ca8 --- /dev/null +++ b/examples/multi-dim-slo/notification-target-pagerduty.yaml @@ -0,0 +1,7 @@ +apiVersion: openslo/v1 +kind: AlertNotificationTarget +metadata: + name: pagerduty +spec: + description: Page on-call via PagerDuty + target: pagerduty diff --git a/examples/multi-dim-slo/notification-target-slack.yaml b/examples/multi-dim-slo/notification-target-slack.yaml new file mode 100644 index 0000000..ea0f558 --- /dev/null +++ b/examples/multi-dim-slo/notification-target-slack.yaml @@ -0,0 +1,7 @@ +apiVersion: openslo/v1 +kind: AlertNotificationTarget +metadata: + name: slack +spec: + description: Open a ticket in the team's Slack channel + target: slack diff --git a/examples/multi-dim-slo/service.yaml b/examples/multi-dim-slo/service.yaml new file mode 100644 index 0000000..5e6890d --- /dev/null +++ b/examples/multi-dim-slo/service.yaml @@ -0,0 +1,6 @@ +apiVersion: openslo/v1 +kind: Service +metadata: + name: api-gateway +spec: + description: API gateway shared across account, frontend, checkout diff --git a/examples/multi-dim-slo/sli.yaml b/examples/multi-dim-slo/sli.yaml new file mode 100644 index 0000000..87d4406 --- /dev/null +++ b/examples/multi-dim-slo/sli.yaml @@ -0,0 +1,11 @@ +apiVersion: openslo/v1 +kind: SLI +metadata: + name: api-gateway-latency-sli +spec: + description: P99 API latency shared across services + thresholdMetric: + metricSource: + type: Prometheus + spec: + query: histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket{job="api-gateway"}[{{.Window}}])) by (le)) diff --git a/examples/multi-dim-slo/slo.yaml b/examples/multi-dim-slo/slo.yaml new file mode 100644 index 0000000..13532d8 --- /dev/null +++ b/examples/multi-dim-slo/slo.yaml @@ -0,0 +1,28 @@ +apiVersion: openslo/v1 +kind: SLO +metadata: + name: api-latency + annotations: + multi-dimensional-sli.openslo.com/label: service_name + multi-dimensional-sli.openslo.com/dimensions: "account,checkout,recommendation" + labels: + service: api-gateway +spec: + service: api-gateway + description: > + P99 API latency budget tracked per downstream caller service. One + SLO is expanded into a series per service_name dimension value via + the multi-dim annotations on metadata. Each downstream gets its own + openslo_slo_name like "account-api-latency" and its own burn rate + recording. + indicatorRef: api-gateway-latency-sli + budgetingMethod: Occurrences + timeWindow: + - duration: 30d + isRolling: true + objectives: + - displayName: "P99 latency under 500ms" + target: 0.999 + alertPolicies: + - alertPolicyRef: api-latency-page-policy + - alertPolicyRef: api-latency-ticket-policy diff --git a/examples/oteldemo/Makefile b/examples/oteldemo/Makefile new file mode 100644 index 0000000..7c98939 --- /dev/null +++ b/examples/oteldemo/Makefile @@ -0,0 +1,38 @@ +SPECS_DIR := specs +RULES_DIR := rules +KIND_DIR := kind +REPO_ROOT := $(shell git rev-parse --show-toplevel 2>/dev/null || echo ../..) +PROMTOOL := $(shell mise which promtool 2>/dev/null || command -v promtool 2>/dev/null || echo promtool) + +.PHONY: verify +verify: + cd $(REPO_ROOT) && go run . load -f examples/oteldemo/$(SPECS_DIR)/ -r + +.PHONY: generate +generate: + mkdir -p $(RULES_DIR) + cd $(REPO_ROOT) && go run . generate -f examples/oteldemo/$(SPECS_DIR)/ -r -o examples/oteldemo/$(RULES_DIR) + $(MAKE) lint-rules + +.PHONY: lint-rules +lint-rules: + @for f in $(RULES_DIR)/*.yaml; do \ + echo "==> $$f"; \ + $(PROMTOOL) check rules "$$f"; \ + done + +.PHONY: clean +clean: + rm -rf $(RULES_DIR) + +.PHONY: start-demo +start-demo: + $(KIND_DIR)/setup.sh + +.PHONY: stop-demo +stop-demo: + $(KIND_DIR)/teardown.sh + +.PHONY: sync +sync: + $(KIND_DIR)/sync.sh diff --git a/examples/oteldemo/README.md b/examples/oteldemo/README.md new file mode 100644 index 0000000..20d04d4 --- /dev/null +++ b/examples/oteldemo/README.md @@ -0,0 +1,320 @@ +# `oteldemo` - OpenTelemetry Demo (Astronomy Shop) + +Production-scale example: 11 SLOs across the [OpenTelemetry Demo](https://github.com/open-telemetry/opentelemetry-demo) microservices, generated Prometheus rules, and Grafana dashboards for browsing every SLO. Written for SREs evaluating opensloctl who have not used OpenSLO before - read top to bottom to go from a clean machine to flipping a chaos flag and watching an SLO burn. + +## Contents + +- [1. Deploy the demo](#1-deploy-the-demo) +- [2. Read the SLOs](#2-read-the-slos) +- [3. Break an SLO and watch it burn](#3-break-an-slo-and-watch-it-burn) +- [4. Iterate](#4-iterate) +- [5. Reference](#5-reference) +- [6. Background](#6-background) + +## 1. Deploy the demo + +### 1.1 Prerequisites + +- `docker`, `kind`, `helm`, `kubectl` +- `mise` provides `go`, `promtool` (or install both yourself) +- About 5 minutes and ~10 GB of disk for the chart to pull images and pods to start + +### 1.2 Run + +```bash +cd examples/oteldemo + +make verify # parse every spec, fail-loudly on bad YAML/SLO refs +make generate # write examples/oteldemo/rules/-rules.yaml +make start-demo # create kind cluster, helm install otel-demo, sync ConfigMaps +``` + +### 1.3 Access + +The chart's `frontend` service proxies most UI traffic on `localhost:8080`. Use that single port-forward for everything except Prometheus itself. + +| What | URL | +|---|---| +| Grafana (dashboards, alerting) | `http://localhost:8080/grafana/` | +| OpenTelemetry Demo UI (store, Jaeger, flagd) | `http://localhost:8080/` | +| flagd feature flags | `http://localhost:8080/feature` | +| Prometheus UI | `http://localhost:9090/` | + +The two dashboards you'll use are `OpenSLO - Manage SLOs` (all SLOs, one row each) and `OpenSLO - SLO detail` (single SLO drilldown). Drill from the list by clicking any row. + +### 1.4 Checkpoint + +Before reading dashboards, confirm Prometheus picked up the rules: + +```bash +open http://localhost:9090/rules +``` + +You should see `openslo-sli-recordings-` and `openslo-alerts-` groups, one per SLO. No rules means the ConfigMap sync failed or Prometheus hasn't reloaded - re-run `make -C examples/oteldemo sync`. + +## 2. Read the SLOs + +### 2.1 Manage SLOs dashboard + +One row per SLO. Five columns: + +| Column | Source metric | Meaning | +|---|---|---| +| Service Level Objective | `openslo_slo_info` | SLO name (e.g. `ad-latency`). | +| Objective % | `openslo_slo_objective * 100` | Target percentage from the spec (e.g. 95.00). | +| Period SLI | `(1 - openslo_sli_error_rate_30d) * 100` | Success ratio over the 30-day window. Always compared against the Objective column to its left. | +| Status | `openslo_slo_status` | Categorical 0-3 from current burn rate. See legend below. | +| Budget Left % | `openslo_slo_period_error_budget_remaining * 100` | Remaining error budget for the spec's `timeWindow` (28d here). 100 = untouched, 0 = exhausted. | + +**Status legend** (0-3 enum, colored background): + +| Value | Label | Trigger on `openslo_slo_current_burn_rate` | +|---|---|---| +| 0 | Healthy | < 1x budget pace | +| 1 | Burning | >= 1x and < 6x | +| 2 | Critical | >= 6x and < 14.4x | +| 3 | Breached | >= 14.4x | + +These thresholds come from the SRE Workbook burn-rate reference points (1x = on pace, 6x = one hour of budget gone in 10 minutes, 14.4x = one hour of budget gone in 5 minutes). + +**Budget Left bands**: green >= 50%, yellow >= 15%, red otherwise. A healthy dashboard shows every Status cell "Healthy" and every Budget Left cell green. + +### 2.2 SLO detail dashboard + +Drill into any row. The URL pins `var-slo=` and `var-datasource=`. Layout: + +``` ++-------------------------------------------+--------+ +| ## ad-latency | SLO | <- markdown header (name, +| 95% of GET /api/data ad fetches ... | 95% | description, target) +| | | +| **Target:** 0.95 | | ++-------------------------------------------+--------+ +| SLI (28d success rate %) | 28d SLI| +| 100% — — — — — — — — — — — — — — — — | 99.something| +| | | ++-------------------------------------------+--------+ +| Error Budget Burndown | 28d Remaining| +| (% remaining over period) | Error Budget (%) | ++-------------------------------------------+--------+ +| Error Budget Burn Rate | Current Burn Rate (x) | +| (x budget pace over windows) | 0.0x | ++-------------------------------------------+--------+ +``` + +**SLO** (top-right stat): the spec's target as a percentage. e.g. an `ad-latency` spec with target 0.95 reads 95.00%. + +**SLI** (left timeseries): the 28-day success rate rolling over time. Read it as: "during the last 28d, what fraction of requests succeeded?" A flat line at the target means the SLO is exactly met; a line below means the SLO is currently being missed; a line above means headroom. + +**28d SLI** (right stat): the same metric as a single number (latest value over the 30-day window in the `SLO detail` time range so you can compare to the spec time window). + +**Error Budget Burndown** (left timeseries): remaining budget % over the SLO time window (28d). Slope encodes burn rate: flat = no burning, downward = losing budget, hit 0 = exhausted. 100 at the window start is the ideal. + +**28d Remaining Error Budget** (right stat): latest burndown value as a percentage. Color-coded green/yellow/red at 50%/15% cutoffs (same as the list dashboard). + +**Error Budget Burn Rate** (left timeseries): burn rate multiplier over multiple short windows. Multiple lines render so you can spot both spike burns and sustained burns. y=1 means on budget pace (<=1x is healthy); upper lines crossing 14.4 alert the page severity (see section 3). + +**Current Burn Rate** (right stat): current burn rate multiplier (no unit, raw scale). 0.5x = burning slowly, 1x = on pace, 14.4x = burning fast enough to exhaust the budget in hours. + +### 2.3 Panel -> recording rule map + +Every number on the dashboard is a recording rule, queryable directly in Prometheus. + +| Panel | PromQL | +|---|---| +| List Objective % | `openslo_slo_objective * 100` | +| List Period SLI | `(1 - openslo_sli_error_rate_30d) * 100` | +| List Status | `openslo_slo_status` (0-3 enum) | +| List Budget Left % | `openslo_slo_period_error_budget_remaining * 100` | +| Detail header | `openslo_slo_info{openslo_slo_name="$slo"}` | +| Detail SLO stat | `openslo_slo_objective * 100` | +| Detail SLI ts | `(1 - openslo_sli_error_rate_30d) * 100` (range) | +| Detail Burndown ts | `openslo_slo_period_error_budget_remaining * 100` (range) | +| Detail Burn Rate ts | `openslo_slo_current_burn_rate` (range) | + +## 3. Break an SLO and watch it burn + +### 3.1 Set a chaos flag + +Open `http://localhost:8080/feature` in a tab. The flagd UI lists feature flags per service. Pick one - `adServiceFailure` is the easiest to observe because it stops the ad service outright. Toggle it on and reload the store UI; ad calls start failing. + +### 3.2 What to observe, in order + +Within a short window after flipping the flag: + +1. Detail Burn Rate timeseries spikes above 14.4 (fast tier fires fast). +2. Alerting list shows Pending, then Firing (Section 3.3). +3. List Status column for that SLO flips Healthy -> Breached. +4. List Budget Left % starts climbing down. +5. Detail Burndown timeseries slopes downward. + +If you see only some of these, refresh the panel query (drill-down to detail, then back to the list row). + +### 3.3 Alerts in Grafana + +Prom rules surface under **Alerting > Alert rules > Data source-managed**. Filter to source = Prometheus (UID `webstore-metrics`). You'll see groups prefixed `openslo-alerts-`. Two severities per SLO: + +- `page`: tier fast (5m AND 1h at 14.4x) OR tier slow (30m AND 6h at 6x). Fires fast. +- `ticket`: tier fast (2h AND 1d at 3x) OR tier slow (6h AND 3d at 1x). Fires on sustained, slower burn. + +The April 2016 SRE Workbook chapter on alerting on SLOs is what tuned these numbers. + +### 3.4 Recovery + +Disable the flag. Status stays "Breached" for as long as the longest window in the worst tier is still elevated - roughly an hour for `page-fast-1h`, six hours for `page-slow-6h`. The status gauge, burndown, and budget stat all lag the underlying metric by the largest window of their respective recording rules. + +### 3.5 Chaos flag to SLO + +| flagd flag | SLO | +|---|---| +| `adServiceFailure` | `ad-availability` | +| `cartServiceFailure` | `cart-availability` | +| `paymentServiceUnreachable` | `payment-unreachable` | +| `recommendationCacheFailure` | `recommendation-availability` | +| `imageSlowLoad` | `image-loading-latency` | +| `kafkaQueueProblems` | `order-processing-latency` | +| `emailMemoryLeak` | `post-order-email-availability`, `post-order-email-latency` | +| `productCatalogFailure` | `product-catalog-availability` | +| `adServiceHighCpu` | `ad-latency` | + +Note: the spec label `chaos_flag` matches the intuitive name, not the flagd JSON canonical name. If you're chasing a flag via the OpenSLO spec, look for the friendly string above; if you're chasing it via flagd's API, the JSON keys are different (e.g. `adFailure` not `adServiceFailure`). Both forms exist intentionally. + +## 4. Iterate + +Quick recipes for the three edits you'll likely make first. + +### 4.1 Change an SLO spec + +```bash +$EDITOR examples/oteldemo/specs/.yaml # edit target, description, indicator source +make -C examples/oteldemo verify # spec refs still resolve +make -C examples/oteldemo generate # rewrites rules/-rules.yaml +make -C examples/oteldemo sync # push ConfigMap, reload Prometheus +``` + +### 4.2 Change the colletor bucket list + +```bash +$EDITOR examples/oteldemo/kind/values.yaml # extend the explicit bucket list +make -C examples/oteldemo start-demo # idempotent - re-renders Helm chart +# then +make -C examples/oteldemo sync # rules + dashboards +``` + +### 4.3 Change a dashboard + +The dashboard JSON is generated from `deploy/mixins/.jsonnet`, not edited directly. + +```bash +$EDITOR deploy/mixins/.jsonnet +make -C deploy/mixins release # generate + sync-legacy + rules + lint-rules +make -C examples/oteldemo sync # ConfigMap push +``` + +### 4.4 Teardown + +```bash +make -C examples/oteldemo stop-demo # deletes the kind cluster +make -C examples/oteldemo clean # rm -rf rules/ +``` + +## 5. Reference + +### 5.1 SLO inventory + +11 SLOs, one per spec file under `specs/`. Latency SLOs use `ratioMetric` over classic-bucket `traces_span_metrics_*` rather than `thresholdMetric` - see [Section 6.2](#62-indicator-shape-latency-slos). + +| SLO | Service | Type | Target | Bucket / label | Chaos flag | +|---|---|---|---|---|---| +| `ad-availability` | ad | availability | 0.99 | - | `adServiceFailure` | +| `ad-latency` | ad | latency | 0.95 | `le="2000"` | `adServiceHighCpu` | +| `cart-availability` | cart | availability | 0.99 | - | `cartServiceFailure` | +| `frontend-availability` | frontend | availability | 0.95 | - | - | +| `image-loading-latency` | image-provider | latency | 0.95 | `le="2000"` | `imageSlowLoad` | +| `order-processing-latency` | checkout | latency | 0.95 | `le="60000"` | `kafkaQueueProblems` | +| `payment-unreachable` | checkout | availability | 0.99 | - | `paymentServiceUnreachable` | +| `post-order-email-availability` | email | availability | 0.95 | - | `emailMemoryLeak` | +| `post-order-email-latency` | email | latency | 0.95 | `le="30000"` | `emailMemoryLeak` | +| `product-catalog-availability` | productcatalogservice | availability | 0.95 | - | `productCatalogFailure` | +| `recommendation-availability` | recommendationservice | availability | 0.95 | - | `recommendationCacheFailure` | + +### 5.2 Layout + +``` +oteldemo/ +├── kind/ # kind cluster + Helm harness +│ ├── kind-config.yaml # cluster spec +│ ├── values.yaml # Helm chart overrides (bucket list, +│ │ # span_metrics connector wiring, +│ │ # enable-feature flags) +│ ├── setup.sh # cluster create + helm install + +│ │ # initial ConfigMap sync +│ ├── teardown.sh # delete kind cluster +│ └── sync.sh # re-apply ConfigMaps after make generate +├── rules/ # generated YAML (gitignored after clean) +├── specs/ # 11 SLOs + 3 helper files +│ ├── services.yaml # inventory of demo services +│ ├── alert-conditions.yaml # 8 reusable AlertConditions +│ ├── alert-policies.yaml # 8 AlertPolicies wrapping them +│ ├── notification-target-engineers.yaml +│ ├── -availability.yaml # 7 availability SLOs +│ └── -latency.yaml # 4 latency SLOs +├── Makefile # verify / generate / start-demo / +│ # stop-demo / sync / clean +└── README.md # this file +``` + +### 5.3 Make targets + +| Target | Effect | +|---|---| +| `make verify` | Run `go run . load` over `specs/`. Validates YAML + specstore contracts. | +| `make generate` | Run `go run . generate` over `specs/`, write `rules/`. Calls `promtool check rules`. | +| `make lint-rules` | `promtool check rules` on every `rules/*.yaml`. | +| `make start-demo` | `kind/setup.sh` - creates cluster, installs chart, syncs ConfigMaps. Idempotent on an existing cluster. | +| `make stop-demo` | `kind/teardown.sh` - deletes the cluster. | +| `make sync` | `kind/sync.sh` - replaces `openslo-rules` and `openslo-dashboards` ConfigMaps (delete + create to avoid the 256 KiB annotation ceiling), reloads Prometheus. | +| `make clean` | Removes `rules/`. | + +## 6. Background + +These sections explain *why* the dashboards look the way they do. Skim them if you're debugging; skip if you're just deploying. + +### 6.1 Alerting strategy + +Each SLO references 8 AlertConditions - two severities (page, ticket), each with two paired tiers of short+long windows: + +| Tier | windows | threshold | meaning | +|---|---|---|---| +| page fast | 5m AND 1h | 14.4x | short and long agree: hard spike | +| page slow | 30m AND 6h | 6x | short and long agree: sustained moderate burn | +| ticket fast | 2h AND 1d | 3x | spike worth a ticket | +| ticket slow | 6h AND 3d | 1x | the SLO is burning at exactly the budget pace | + +Within a tier conditions AND. Across tiers within a severity conditions OR. So `page` fires on `(fast-5m AND fast-1h) OR (slow-30m AND slow-6h)`. + +Condition **name suffix `-` is mandatory** - the generator strips it to derive the tier. `burnrate-page-fast-5m` and `burnrate-page-fast-1h` both belong to tier `burnrate-page-fast` and get AND-ed. Renaming either breaks the pairing. + +Conditions and policies live once in `alert-conditions.yaml` and `alert-policies.yaml`. Each SLO references the same 8 conditions - no per-SLO alert duplication. + +### 6.2 Indicator shape (latency SLOs) + +The 4 latency SLOs use `ratioMetric(counter=true)`: + +``` +good = sum(rate(traces_span_metrics_duration_milliseconds_bucket{..., le=""}[])) +total = sum(rate(traces_span_metrics_calls_total{...}[])) +error = 1 - good / total +``` + +The `le=` boundary must exist in the connector's bucket list. The otel-demo chart defaults miss the longer envelopes, so `kind/values.yaml` extends the bucket list to include `30s` and `60s` for `post-order-email-latency` (`le="30000"`) and `order-processing-latency` (`le="60000"`). + +Why ratio over `histogram_quantile(...) > `: Prom 3.x filter semantics collapses raw scalar comparisons to filter semantics - `histogram_quantile(...) > 2000` returns the quantile unchanged, not a 0/1 series, so it can't drive a ratio SLI over a 30-day window without an explicit `bool` modifier in the comparison. Native exponential histograms have no `le="..."` series, so the per-threshold fraction can't be expressed at all. Classic-bucket ratio is the shape that reads as a continuous value across rolling windows. + +### 6.3 Why kind + Helm + +Docker compose bind mounts struggle when the same target dir needs both upstream and custom content, or has other mounts layering in. Kubernetes ConfigMaps sidestep both. The upstream otel-demo chart ships Prometheus and Grafana with their sidecar pattern enabled, so we layer our ConfigMaps on without touching the chart. + +### 6.4 When to use this kind + +`multi-window-multi-burn-rate` is the recommended SRE Workbook pattern (§ 6) for any SLO you care about. It catches sudden spikes (fast tier, short window, high multiplier) and sustained moderate burns (slow tier, long window, lower multiplier) while rejecting noise (the AND within a tier). Trade down to [`multi-burn-rate`](../multi-burn-slo/README.md) if you don't want the AND pairing; trade down further to [`burn-rate`](../api-latency-slo/README.md) for the simplest possible single-window. diff --git a/examples/oteldemo/kind/kind-config.yaml b/examples/oteldemo/kind/kind-config.yaml new file mode 100644 index 0000000..57f2c5a --- /dev/null +++ b/examples/oteldemo/kind/kind-config.yaml @@ -0,0 +1,12 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +name: opensloctl-demo +nodes: + - role: control-plane + kubeadmConfigPatches: + - | + kind: InitConfiguration + nodeRegistration: + kubeletExtraArgs: + node-labels: "ingress-ready=true" + extraPortMappings: [] # NodePort disabled - we use kubectl port-forward instead diff --git a/examples/oteldemo/kind/kustomization.yaml b/examples/oteldemo/kind/kustomization.yaml new file mode 100644 index 0000000..8b9abb5 --- /dev/null +++ b/examples/oteldemo/kind/kustomization.yaml @@ -0,0 +1,42 @@ +# Renders the opentelemetry-demo Helm chart's raw manifest, then +# patches the pieces the chart doesn't expose via values: +# - mount the openslo-rules ConfigMap onto the prometheus +# Deployment so the rule files are reachable at +# /etc/prometheus/openslo-rules/. +# +# The other piece - appending `/etc/prometheus/openslo-rules/*.yaml` +# to the chart's prometheus-rule_files list - is done in setup.sh +# via `awk` after this kustomization builds. Kustomize v5 exposes +# regex-style `pattern`+`replace` only as in-progress alpha and +# doesn't have a stable API for it yet, so we keep the string-edit +# step in shell where it lives cleanly. +# +# Driver: setup.sh runs +# helm template otel-demo ... > tmp/helm-rendered.yaml +# kustomize build examples/oteldemo/kind > tmp/manifest.yaml +# kubectl apply -f tmp/manifest.yaml +# # setup.sh then awk-injects the rule_files entry into the chart's +# # prometheus ConfigMap (idempotent). +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../../../tmp/helm-rendered.yaml + +patches: + - target: + kind: Deployment + name: prometheus + patch: |- + - op: add + path: /spec/template/spec/volumes/- + value: + name: openslo-rules + configMap: + name: openslo-rules + - op: add + path: /spec/template/spec/containers/0/volumeMounts/- + value: + name: openslo-rules + mountPath: /etc/prometheus/openslo-rules + readOnly: true diff --git a/examples/oteldemo/kind/setup.sh b/examples/oteldemo/kind/setup.sh new file mode 100755 index 0000000..9edba74 --- /dev/null +++ b/examples/oteldemo/kind/setup.sh @@ -0,0 +1,119 @@ +#!/usr/bin/env bash +# Create a kind cluster, render the otel-demo chart via helm, then +# use kustomize to patch the prometheus Deployment + ConfigMap +# (mount openslo-rules ConfigMap; inject rule_files:), then apply +# with kubectl. +# +# Why not `helm install` directly? The chart's values don't expose +# knobs for mounting our rules ConfigMap onto the prometheus pod or +# for adding a top-level `rule_files:` to the chart's prometheus +# ConfigMap. Patches are easier to maintain in kustomize YAML than +# in a custom post-renderer script. Dropping helm install in favor +# of `kubectl apply` sidesteps the post-renderer plumbing entirely +# and keeps the manifest rebuild-testable with a single kustomize +# command. +set -euo pipefail + +REPO_ROOT="$(git rev-parse --show-toplevel)" +CLUSTER="opensloctl-demo" +NAMESPACE="default" +HELM_RELEASE="otel-demo" +RULES_CM="openslo-rules" +DASHBOARDS_CM="openslo-dashboards" +RULES_MOUNT="/etc/prometheus/openslo-rules" +KIND_DIR="${REPO_ROOT}/examples/oteldemo/kind" +RENDER_DIR="${REPO_ROOT}/tmp" + +cd "${REPO_ROOT}" + +echo "==> Creating kind cluster '${CLUSTER}'..." +if kind get clusters 2>/dev/null | grep -q "^${CLUSTER}$"; then + echo " (already exists, reusing)" +else + kind create cluster \ + --name "${CLUSTER}" \ + --config "${KIND_DIR}/kind-config.yaml" +fi + +kubectl config use-context "kind-${CLUSTER}" >/dev/null + +echo "==> Adding OpenTelemetry helm repo + pulling values..." +helm repo add open-telemetry https://open-telemetry.github.io/opentelemetry-helm-charts 2>/dev/null || true +helm repo update >/dev/null + +mkdir -p "${RENDER_DIR}" +echo "==> Rendering helm chart with values.yaml..." +helm template "${HELM_RELEASE}" open-telemetry/opentelemetry-demo \ + --namespace "${NAMESPACE}" \ + --values "${KIND_DIR}/values.yaml" \ + > "${RENDER_DIR}/helm-rendered.yaml" + +echo "==> Patching prometheus Deployment via kustomize (autoloaded rule_files mounted at /etc/prometheus/openslo-rules/)..." +kustomize build --load-restrictor=LoadRestrictionsNone "${KIND_DIR}" \ + > "${RENDER_DIR}/manifest.yaml" + +# Inject the openslo-rules path into the chart's prometheus ConfigMap +# rule_files list. The chart ships 4 entries under /etc/config/... for +# helm-managed recording+alerting files; we add ours next to them. A +# duplicate `rule_files:` top-level key fails Prom's strict YAML parser +# so we append directly to the existing block. Idempotent: if our line +# is already present, awk sees it inside the pattern and replaces +# with itself. +# +# Note: if the chart ever changes the rule_files block signature, this +# regex stops matching and the prom container logs a YAML parse error +# on reload. Update patterns to match. +echo "==> Appending openslo-rules to prometheus rule_files (awk)..." +awk -v mount="${RULES_MOUNT}" ' + $0 == " - /etc/config/alerts" { + print + print " - " mount "/*.yaml" + next + } + { print } +' "${RENDER_DIR}/manifest.yaml" > "${RENDER_DIR}/manifest.new" +mv "${RENDER_DIR}/manifest.new" "${RENDER_DIR}/manifest.yaml" + +echo "==> Applying manifest to cluster..." +kubectl apply -f "${RENDER_DIR}/manifest.yaml" + +echo "==> Loading OpenSLO Prometheus rules ConfigMap..." +kubectl create configmap "${RULES_CM}" \ + --namespace "${NAMESPACE}" \ + --from-file="${REPO_ROOT}/examples/oteldemo/rules/" \ + --dry-run=client -o yaml \ + | kubectl apply -f - + +echo "==> Loading OpenSLO Grafana dashboards ConfigMap (with sidecar label)..." +kubectl create configmap "${DASHBOARDS_CM}" \ + --namespace "${NAMESPACE}" \ + --from-file="${REPO_ROOT}/deploy/dashboards/" \ + --dry-run=client -o yaml \ + | kubectl apply -f - +kubectl label configmap "${DASHBOARDS_CM}" -n "${NAMESPACE}" grafana_dashboard=1 --overwrite + +echo "==> Waiting for prometheus to roll out with rules ConfigMap mount..." +kubectl rollout status deployment/prometheus -n "${NAMESPACE}" --timeout=5m + +echo "==> Reloading Prometheus via POST /-/reload..." +kubectl exec -n "${NAMESPACE}" deploy/prometheus -- \ + wget -q -O- --post-data='' http://localhost:9090/-/reload >/dev/null \ + || true + +echo "==> Port-forwarding Grafana, Prometheus, frontend-proxy to localhost..." +kubectl port-forward -n "${NAMESPACE}" svc/prometheus 9090:9090 \ + > "${RENDER_DIR}/openslo-prometheus-pf.log" 2>&1 & +kubectl port-forward -n "${NAMESPACE}" svc/frontend-proxy 8080:8080 \ + > "${RENDER_DIR}/openslo-frontendproxy-pf.log" 2>&1 & + +echo +echo "Demo ready (port-forwards running in background; teardown.sh or kill them)." +echo " Demo UI http://localhost:8080/" +echo " /grafana http://localhost:8080/grafana/" +echo " /jaeger/ui http://localhost:8080/jaeger/ui" +echo " /feature http://localhost:8080/feature" +echo " Direct Graf http://localhost:3000/" +echo " Direct Prom http://localhost:9090/rules" +echo +echo "After re-running 'make generate' to update rules: examples/oteldemo/kind/sync.sh" +echo "Cluster: ${CLUSTER} Context: kind-${CLUSTER}" diff --git a/examples/oteldemo/kind/sync.sh b/examples/oteldemo/kind/sync.sh new file mode 100755 index 0000000..26bf6da --- /dev/null +++ b/examples/oteldemo/kind/sync.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# Re-apply the rules + dashboards ConfigMaps after `make generate`, +# re-add the sidecar label for dashboards, and reload Prometheus. +# +# The chart's prometheus ConfigMap rule_files block stays stable across +# helm upgrades, so we don't re-render or re-patch it here - the +# rule_files entry pointing at our mount was added during setup.sh and +# persists on the cluster between syncs. +set -euo pipefail + +REPO_ROOT="$(git rev-parse --show-toplevel)" +NAMESPACE="${NAMESPACE:-default}" +RULES_CM="openslo-rules" +DASHBOARDS_CM="openslo-dashboards" + +cd "${REPO_ROOT}" + +echo "==> Syncing openslo-rules ConfigMap..." +# Replace (delete+create) instead of apply to avoid the +# kubectl.kubernetes.io/last-applied-configuration annotation accumulating +# past the 256 KiB ConfigMap metadata limit when the ruleset grows +# (e.g. adding one rule per SLO makes each apply entry ~30% larger). +kubectl delete configmap "${RULES_CM}" -n "${NAMESPACE}" --ignore-not-found +kubectl create configmap "${RULES_CM}" \ + --namespace "${NAMESPACE}" \ + --from-file="${REPO_ROOT}/examples/oteldemo/rules/" \ + --from-file="${REPO_ROOT}/deploy/rules/" + +echo "==> Syncing openslo-dashboards ConfigMap (with sidecar label)..." +kubectl delete configmap "${DASHBOARDS_CM}" -n "${NAMESPACE}" --ignore-not-found +kubectl create configmap "${DASHBOARDS_CM}" \ + --namespace "${NAMESPACE}" \ + --from-file="${REPO_ROOT}/deploy/dashboards/" +kubectl label configmap "${DASHBOARDS_CM}" -n "${NAMESPACE}" grafana_dashboard=1 --overwrite + +echo "==> Reloading Prometheus (POST /-/reload)..." +kubectl exec -n "${NAMESPACE}" deploy/prometheus -- \ + wget -q -O- --post-data='' http://localhost:9090/-/reload >/dev/null \ + || true diff --git a/examples/oteldemo/kind/teardown.sh b/examples/oteldemo/kind/teardown.sh new file mode 100755 index 0000000..d7832a6 --- /dev/null +++ b/examples/oteldemo/kind/teardown.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# Tear down the demo: kill port-forwards, delete the rendered +# manifest from the cluster, then delete the kind cluster. +set -euo pipefail + +REPO_ROOT="$(git rev-parse --show-toplevel)" +CLUSTER="opensloctl-demo" +RENDER_DIR="${REPO_ROOT}/tmp" + +cd "${REPO_ROOT}" + +pkill -f "kubectl port-forward.*openslo" 2>/dev/null || true + +echo "==> Deleting manifest..." +kubectl delete -f "${RENDER_DIR}/manifest.yaml" --ignore-not-found 2>/dev/null || true + +echo "==> Deleting OpenSLO ConfigMaps..." +kubectl delete configmap openslo-rules -n default --ignore-not-found 2>/dev/null || true +kubectl delete configmap openslo-dashboards -n default --ignore-not-found 2>/dev/null || true + +echo "==> Deleting kind cluster '${CLUSTER}'..." +kind delete cluster --name "${CLUSTER}" diff --git a/examples/oteldemo/kind/values.yaml b/examples/oteldemo/kind/values.yaml new file mode 100644 index 0000000..40069cd --- /dev/null +++ b/examples/oteldemo/kind/values.yaml @@ -0,0 +1,171 @@ +# Helm chart values for an OpenSLO flavored override of +# open-telemetry/opentelemetry-demo. +# +# The chart's default values already enable anonymous Grafana admin +# (grafana.grafana.ini.auth.anonymous.enabled: true), the Prom OTLP +# receiver (`web.enable-otlp-receiver`), and the dashboard/alerts/ +# datasources sidecars. We re-add `web.enable-lifecycle` so Prom's +# `/-/reload` endpoint accepts POSTs after `make generate` runs. +prometheus: + server: + extraFlags: + - "enable-feature=exemplar-storage" + - "web.enable-otlp-receiver" + - "web.enable-lifecycle" + # NOTE: native histograms are on by default in Prometheus v3.x + # and `--enable-feature=native-histograms` is a documented no-op + # (Prom logs `option=native-histograms no-op` at startup). Storage + # accepts natively-produced histograms via the OTLP receiver without + # any extra flag, so we don't set it here. +grafana: + resources: + limits: + memory: 1Gi + sidecar: + # Disabled: alerts. We manage alerts via Prometheus recording/alert + # rules directly (loaded via the openslo-rules ConfigMap). + alerts: + enabled: false + # Enabled: datasources. The chart ships a Prometheus datasource + # ConfigMap with the `${DS_PROMETHEUS}` UID our SLO dashboards + # reference. Without this sidecar every dashboard query resolves + # to nothing because no datasource exists. + datasources: + enabled: true + plugins: ["grafana-opensearch-datasource"] +components: + # The upstream chart ships with 20Mi for several Node/Go services. + # Under load-generator traffic a few of them OOM-Kill almost + # immediately. We bump the worst offenders to 200Mi here. + checkout: + resources: + limits: + memory: 200Mi + currency: + resources: + limits: + memory: 200Mi + product-catalog: + resources: + limits: + memory: 200Mi + quote: + resources: + limits: + memory: 200Mi + shipping: + resources: + limits: + memory: 200Mi + load-generator: + # k6 virtual-user count for the front-end synthetic workload. + # The Helm chart's default is 5 (see chart `values.yaml` lines 639-640, + # `LOAD_GENERATOR_VUS`). Bumping raises ambient request volume, + # increasing pressure on customer-facing SLOs (ad-availability, + # cart-availability, recommendation-availability, payment-unreachable + # via cascade) without changing chaos-flag state. + env: + - name: LOAD_GENERATOR_VUS + value: "5" + +# Span-metrics connector: single classic bucket histogram. +# Restores the upstream chart's default connector name (`span_metrics`) +# so Prometheus sees one metric family for every distinct duration. +# We DO NOT emit native (exponential) histograms: they have no +# `_bucket{le=...}` series, which makes per-threshold SLO ratios +# (good / total) impossible to compute - `histogram_quantile` only +# recovers a single duration estimate per evaluation, not a fraction +# of observations under a bound. The classic bucket stream keeps +# `openslo_sli_error_rate_*` numerically truthful (continuous 0.0–1.0) +# over 30-day windows. +# +# The namespace `traces.span.metrics` produces a Prometheus metric name +# infix of `_span_metrics_`, e.g. +# `traces_span_metrics_duration_milliseconds_bucket` and +# `traces_span_metrics_calls_total`. The oteldemo chart's default +# would emit the metrics as `spanmetrics_calls_total` / +# `spanmetrics_duration_milliseconds`; we override the namespace so +# the SLO specs can keep a stable, descriptive prefix. +opentelemetry-collector: + config: + connectors: + span_metrics: + namespace: traces.span.metrics + histogram: + explicit: + buckets: + - "2ms" + - "4ms" + - "6ms" + - "8ms" + - "10ms" + - "50ms" + - "100ms" + - "200ms" + - "400ms" + - "800ms" + - "1s" + - "1400ms" + - "2s" + - "5s" + - "10s" + - "15s" + # `30s` covers the post-order-email-latency SLO threshold + # (`le="30000"`); `60s` covers the order-processing-latency + # SLO threshold (`le="60000"`). The metric suffix is + # `_milliseconds`, so the bucket durations above emit as + # `le="2"`, `le="2000"`, ..., `le="30000"`, `le="60000"`. + - "30s" + - "60s" + - "120s" + - "300s" + # Dimensions follow the latest OpenTelemetry semantic conventions + # (HTTP stable; messaging Development per + # https://opentelemetry.io/docs/specs/semconv/). Each entry below is + # declared as an additive dimension on every emitted metric stream + # from this connector - service.name + span.name + span.kind + + # status.code + collector.instance.id are already added by default, + # so we only list the opt-in attributes we need for SLO evaluation: + # - http.response.status_code: fail/success split for HTTP SLOs. + # - error.type: low-cardinality error class fallback when status + # code isn't enough (e.g. timeout). + # - messaging.destination.name: topic name for kafka-order-completion. + # - messaging.operation.type: send/process/receive distinction. + # - messaging.system: restricts to `kafka` for our messaging SLO. + # - user_agent.synthetic.type: filters out bot/test synthetic + # traffic from user-facing SLIs (load-generator emits + # synthetic.type="test" by default; see + # https://opentelemetry.io/docs/specs/semconv/registry/attributes/user-agent/). + # We deliberately skip user_agent.original (full UA strings = + # unbounded cardinality) and the high-cardinality message IDs, + # offsets, and partition IDs. + dimensions: + - name: http.response.status_code + - name: error.type + - name: messaging.destination.name + - name: messaging.operation.type + - name: messaging.system + - name: user_agent.synthetic.type + + # Pipeline wiring matches the upstream chart's default. The chart + # hard-codes the connector token `span_metrics` (the factory alias) + # in both `service.pipelines.traces.exporters` and + # `service.pipelines.metrics.receivers`. We pass it through verbatim + # so the override is a no-op for collector wiring - only our custom + # namespace and dimensions change the metric names. Other pipeline + # members are reproduced exactly: memory_limiter → resourcedetection + # → resource → transform/sanitize_logs → gen_ai_normalizer for + # traces; receivers otlp, kafkametrics, span_metrics, prometheus/ad + # for metrics. + service: + pipelines: + traces: + processors: [memory_limiter, resourcedetection, resource, transform/sanitize_logs, gen_ai_normalizer] + exporters: [otlp_grpc/jaeger, debug, span_metrics] + metrics: + receivers: [otlp, kafkametrics, span_metrics, prometheus/ad] + processors: [memory_limiter, resourcedetection, resource] + exporters: [otlp_http/prometheus, debug] + logs: + processors: [memory_limiter, resourcedetection, resource, transform/sanitize_logs] + exporters: [opensearch, debug] diff --git a/examples/oteldemo/rules/ad-availability-rules.yaml b/examples/oteldemo/rules/ad-availability-rules.yaml new file mode 100644 index 0000000..0f112a8 --- /dev/null +++ b/examples/oteldemo/rules/ad-availability-rules.yaml @@ -0,0 +1,354 @@ +groups: + - name: openslo-info-recordings-ad-availability + rules: + - record: openslo_slo_info + expr: vector(1) + labels: + openslo_slo_name: ad-availability + openslo_slo_description: "99% of users successfully fetching an ad via frontend HTTP SERVER `GET /api/data` (target 0.95; trips on `adServiceFailure`)." + openslo_spec_version: openslo/v1 + chaos_flag: adServiceFailure + openslo_service_name: ad + service: ad + - record: openslo_slo_objective + expr: vector(0.99) + labels: + openslo_slo_name: ad-availability + openslo_spec_version: openslo/v1 + chaos_flag: adServiceFailure + openslo_service_name: ad + service: ad + - record: openslo_slo_timewindow_days + expr: vector(30) + labels: + openslo_slo_name: ad-availability + openslo_spec_version: openslo/v1 + chaos_flag: adServiceFailure + openslo_service_name: ad + service: ad + - record: openslo_slo_error_budget + expr: vector(1- 0.99) + labels: + openslo_slo_name: ad-availability + openslo_spec_version: openslo/v1 + chaos_flag: adServiceFailure + openslo_service_name: ad + service: ad + - record: openslo_slo_current_burn_rate + expr: | + openslo_sli_error_rate_5m{openslo_slo_name="ad-availability", openslo_spec_version="openslo/v1"} + / on(openslo_slo_name, openslo_spec_version) group_left + openslo_slo_error_budget{openslo_slo_name="ad-availability", openslo_spec_version="openslo/v1"} + labels: + openslo_slo_name: ad-availability + openslo_spec_version: openslo/v1 + chaos_flag: adServiceFailure + openslo_service_name: ad + service: ad + - record: openslo_slo_period_burn_rate + expr: | + openslo_sli_error_rate_30d{openslo_slo_name="ad-availability", openslo_spec_version="openslo/v1"} + / on(openslo_slo_name, openslo_spec_version) group_left + openslo_slo_error_budget{openslo_slo_name="ad-availability", openslo_spec_version="openslo/v1"} + labels: + openslo_slo_name: ad-availability + openslo_spec_version: openslo/v1 + chaos_flag: adServiceFailure + openslo_service_name: ad + service: ad + - record: openslo_slo_period_error_budget_remaining + expr: clamp_min(1 - openslo_slo_period_burn_rate{openslo_slo_name="ad-availability", openslo_spec_version="openslo/v1"}, 0) + labels: + openslo_slo_name: ad-availability + openslo_spec_version: openslo/v1 + chaos_flag: adServiceFailure + openslo_service_name: ad + service: ad + - name: openslo-sli-recordings-ad-availability + rules: + - record: openslo_sli_error_rate_5m + expr: | + 1 - ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="GET /api/data", status_code!="STATUS_CODE_ERROR"}[5m])) + ) / ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="GET /api/data"}[5m])) + ) + labels: + openslo_slo_name: ad-availability + openslo_spec_version: openslo/v1 + chaos_flag: adServiceFailure + openslo_service_name: ad + service: ad + - record: openslo_sli_error_rate_30m + expr: | + 1 - ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="GET /api/data", status_code!="STATUS_CODE_ERROR"}[30m])) + ) / ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="GET /api/data"}[30m])) + ) + labels: + openslo_slo_name: ad-availability + openslo_spec_version: openslo/v1 + chaos_flag: adServiceFailure + openslo_service_name: ad + service: ad + - record: openslo_sli_error_rate_1h + expr: | + 1 - ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="GET /api/data", status_code!="STATUS_CODE_ERROR"}[1h])) + ) / ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="GET /api/data"}[1h])) + ) + labels: + openslo_slo_name: ad-availability + openslo_spec_version: openslo/v1 + chaos_flag: adServiceFailure + openslo_service_name: ad + service: ad + - record: openslo_sli_error_rate_3h + expr: | + 1 - ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="GET /api/data", status_code!="STATUS_CODE_ERROR"}[3h])) + ) / ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="GET /api/data"}[3h])) + ) + labels: + openslo_slo_name: ad-availability + openslo_spec_version: openslo/v1 + chaos_flag: adServiceFailure + openslo_service_name: ad + service: ad + - record: openslo_sli_error_rate_6h + expr: | + 1 - ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="GET /api/data", status_code!="STATUS_CODE_ERROR"}[6h])) + ) / ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="GET /api/data"}[6h])) + ) + labels: + openslo_slo_name: ad-availability + openslo_spec_version: openslo/v1 + chaos_flag: adServiceFailure + openslo_service_name: ad + service: ad + - record: openslo_sli_error_rate_1d + expr: | + 1 - ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="GET /api/data", status_code!="STATUS_CODE_ERROR"}[1d])) + ) / ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="GET /api/data"}[1d])) + ) + labels: + openslo_slo_name: ad-availability + openslo_spec_version: openslo/v1 + chaos_flag: adServiceFailure + openslo_service_name: ad + service: ad + - record: openslo_sli_error_rate_3d + expr: | + 1 - ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="GET /api/data", status_code!="STATUS_CODE_ERROR"}[3d])) + ) / ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="GET /api/data"}[3d])) + ) + labels: + openslo_slo_name: ad-availability + openslo_spec_version: openslo/v1 + chaos_flag: adServiceFailure + openslo_service_name: ad + service: ad + - record: openslo_sli_error_rate_7d + expr: | + 1 - ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="GET /api/data", status_code!="STATUS_CODE_ERROR"}[7d])) + ) / ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="GET /api/data"}[7d])) + ) + labels: + openslo_slo_name: ad-availability + openslo_spec_version: openslo/v1 + chaos_flag: adServiceFailure + openslo_service_name: ad + service: ad + - record: openslo_sli_error_rate_28d + expr: | + 1 - ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="GET /api/data", status_code!="STATUS_CODE_ERROR"}[28d])) + ) / ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="GET /api/data"}[28d])) + ) + labels: + openslo_slo_name: ad-availability + openslo_spec_version: openslo/v1 + chaos_flag: adServiceFailure + openslo_service_name: ad + service: ad + - record: openslo_sli_error_rate_30d + expr: | + 1 - ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="GET /api/data", status_code!="STATUS_CODE_ERROR"}[30d])) + ) / ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="GET /api/data"}[30d])) + ) + labels: + openslo_slo_name: ad-availability + openslo_spec_version: openslo/v1 + chaos_flag: adServiceFailure + openslo_service_name: ad + service: ad + - record: openslo_sli_event_rate_5m + expr: sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="GET /api/data"}[5m])) + labels: + openslo_slo_name: ad-availability + openslo_spec_version: openslo/v1 + chaos_flag: adServiceFailure + openslo_service_name: ad + service: ad + - record: openslo_sli_event_rate_30m + expr: sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="GET /api/data"}[30m])) + labels: + openslo_slo_name: ad-availability + openslo_spec_version: openslo/v1 + chaos_flag: adServiceFailure + openslo_service_name: ad + service: ad + - record: openslo_sli_event_rate_1h + expr: sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="GET /api/data"}[1h])) + labels: + openslo_slo_name: ad-availability + openslo_spec_version: openslo/v1 + chaos_flag: adServiceFailure + openslo_service_name: ad + service: ad + - record: openslo_sli_event_rate_3h + expr: sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="GET /api/data"}[3h])) + labels: + openslo_slo_name: ad-availability + openslo_spec_version: openslo/v1 + chaos_flag: adServiceFailure + openslo_service_name: ad + service: ad + - record: openslo_sli_event_rate_6h + expr: sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="GET /api/data"}[6h])) + labels: + openslo_slo_name: ad-availability + openslo_spec_version: openslo/v1 + chaos_flag: adServiceFailure + openslo_service_name: ad + service: ad + - record: openslo_sli_event_rate_1d + expr: sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="GET /api/data"}[1d])) + labels: + openslo_slo_name: ad-availability + openslo_spec_version: openslo/v1 + chaos_flag: adServiceFailure + openslo_service_name: ad + service: ad + - record: openslo_sli_event_rate_3d + expr: sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="GET /api/data"}[3d])) + labels: + openslo_slo_name: ad-availability + openslo_spec_version: openslo/v1 + chaos_flag: adServiceFailure + openslo_service_name: ad + service: ad + - record: openslo_sli_event_rate_7d + expr: sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="GET /api/data"}[7d])) + labels: + openslo_slo_name: ad-availability + openslo_spec_version: openslo/v1 + chaos_flag: adServiceFailure + openslo_service_name: ad + service: ad + - record: openslo_sli_event_rate_28d + expr: sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="GET /api/data"}[28d])) + labels: + openslo_slo_name: ad-availability + openslo_spec_version: openslo/v1 + chaos_flag: adServiceFailure + openslo_service_name: ad + service: ad + - record: openslo_sli_event_rate_30d + expr: sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="GET /api/data"}[30d])) + labels: + openslo_slo_name: ad-availability + openslo_spec_version: openslo/v1 + chaos_flag: adServiceFailure + openslo_service_name: ad + service: ad + - name: openslo-status-recordings-ad-availability + rules: + - record: openslo_slo_status + expr: | + ( + (openslo_slo_current_burn_rate{openslo_slo_name="ad-availability", openslo_spec_version="openslo/v1"} >= bool 14.4) + * 3 + ) + or + ( + (openslo_slo_current_burn_rate{openslo_slo_name="ad-availability", openslo_spec_version="openslo/v1"} >= bool 6) + and + (openslo_slo_current_burn_rate{openslo_slo_name="ad-availability", openslo_spec_version="openslo/v1"} < bool 14.4) + * 2 + ) + or + ( + (openslo_slo_current_burn_rate{openslo_slo_name="ad-availability", openslo_spec_version="openslo/v1"} >= bool 1) + and + (openslo_slo_current_burn_rate{openslo_slo_name="ad-availability", openslo_spec_version="openslo/v1"} < bool 6) + * 1 + ) + labels: + openslo_slo_name: ad-availability + openslo_spec_version: openslo/v1 + chaos_flag: adServiceFailure + openslo_service_name: ad + service: ad + - name: openslo-alerts-ad-availability + rules: + - alert: AdAvailabilityMultiWindowMultiBurnRatePage + expr: |- + ( + openslo_sli_error_rate_5m{openslo_slo_name="ad-availability"} / (1 - openslo_slo_objective{openslo_slo_name="ad-availability"}) >= 14.400000 + and + openslo_sli_error_rate_1h{openslo_slo_name="ad-availability"} / (1 - openslo_slo_objective{openslo_slo_name="ad-availability"}) >= 14.400000) + or + ( + openslo_sli_error_rate_30m{openslo_slo_name="ad-availability"} / (1 - openslo_slo_objective{openslo_slo_name="ad-availability"}) >= 6.000000 + and + openslo_sli_error_rate_6h{openslo_slo_name="ad-availability"} / (1 - openslo_slo_objective{openslo_slo_name="ad-availability"}) >= 6.000000) + for: 2m + labels: + openslo_alert_severity: page + openslo_notification_target: engineers + openslo_slo_name: ad-availability + openslo_spec_version: openslo/v1 + chaos_flag: adServiceFailure + openslo_service_name: ad + service: ad + annotations: + summary: "Page multi-window multi-burn rate alert for SLO ad-availability" + description: "Threshold 14.4x and 6.0x over 5m and 1h and 30m and 6h" + - alert: AdAvailabilityMultiWindowMultiBurnRateTicket + expr: |- + ( + openslo_sli_error_rate_2h{openslo_slo_name="ad-availability"} / (1 - openslo_slo_objective{openslo_slo_name="ad-availability"}) >= 3.000000 + and + openslo_sli_error_rate_1d{openslo_slo_name="ad-availability"} / (1 - openslo_slo_objective{openslo_slo_name="ad-availability"}) >= 3.000000) + or + ( + openslo_sli_error_rate_6h{openslo_slo_name="ad-availability"} / (1 - openslo_slo_objective{openslo_slo_name="ad-availability"}) >= 1.000000 + and + openslo_sli_error_rate_3d{openslo_slo_name="ad-availability"} / (1 - openslo_slo_objective{openslo_slo_name="ad-availability"}) >= 1.000000) + for: 15m + labels: + openslo_alert_severity: ticket + openslo_notification_target: engineers + openslo_slo_name: ad-availability + openslo_spec_version: openslo/v1 + chaos_flag: adServiceFailure + openslo_service_name: ad + service: ad + annotations: + summary: "Ticket multi-window multi-burn rate alert for SLO ad-availability" + description: "Threshold 3.0x and 1.0x over 2h and 1d and 6h and 3d" diff --git a/examples/oteldemo/rules/ad-latency-rules.yaml b/examples/oteldemo/rules/ad-latency-rules.yaml new file mode 100644 index 0000000..8cde65e --- /dev/null +++ b/examples/oteldemo/rules/ad-latency-rules.yaml @@ -0,0 +1,644 @@ +groups: + - name: openslo-info-recordings-ad-latency + rules: + - record: openslo_slo_info + expr: vector(1) + labels: + openslo_slo_name: ad-latency + openslo_slo_description: "95% of `GET /api/data` ad fetches should complete under 2s, measured as the ratio of `le=\"2000\"` classic bucket counts to `_calls_total` on frontend HTTP SERVER (target 0.95; trips on `adHighCpu`, …" + openslo_spec_version: openslo/v1 + chaos_flag: adServiceHighCpu + openslo_service_name: ad + service: ad + - record: openslo_slo_objective + expr: vector(0.99) + labels: + openslo_slo_name: ad-latency + openslo_spec_version: openslo/v1 + chaos_flag: adServiceHighCpu + openslo_service_name: ad + service: ad + - record: openslo_slo_timewindow_days + expr: vector(30) + labels: + openslo_slo_name: ad-latency + openslo_spec_version: openslo/v1 + chaos_flag: adServiceHighCpu + openslo_service_name: ad + service: ad + - record: openslo_slo_error_budget + expr: vector(1- 0.99) + labels: + openslo_slo_name: ad-latency + openslo_spec_version: openslo/v1 + chaos_flag: adServiceHighCpu + openslo_service_name: ad + service: ad + - record: openslo_slo_current_burn_rate + expr: | + openslo_sli_error_rate_5m{openslo_slo_name="ad-latency", openslo_spec_version="openslo/v1"} + / on(openslo_slo_name, openslo_spec_version) group_left + openslo_slo_error_budget{openslo_slo_name="ad-latency", openslo_spec_version="openslo/v1"} + labels: + openslo_slo_name: ad-latency + openslo_spec_version: openslo/v1 + chaos_flag: adServiceHighCpu + openslo_service_name: ad + service: ad + - record: openslo_slo_period_burn_rate + expr: | + openslo_sli_error_rate_30d{openslo_slo_name="ad-latency", openslo_spec_version="openslo/v1"} + / on(openslo_slo_name, openslo_spec_version) group_left + openslo_slo_error_budget{openslo_slo_name="ad-latency", openslo_spec_version="openslo/v1"} + labels: + openslo_slo_name: ad-latency + openslo_spec_version: openslo/v1 + chaos_flag: adServiceHighCpu + openslo_service_name: ad + service: ad + - record: openslo_slo_period_error_budget_remaining + expr: clamp_min(1 - openslo_slo_period_burn_rate{openslo_slo_name="ad-latency", openslo_spec_version="openslo/v1"}, 0) + labels: + openslo_slo_name: ad-latency + openslo_spec_version: openslo/v1 + chaos_flag: adServiceHighCpu + openslo_service_name: ad + service: ad + - name: openslo-sli-recordings-ad-latency + rules: + - record: openslo_sli_error_rate_5m + expr: | + 1 - ( + sum( + rate( + traces_span_metrics_duration_milliseconds_bucket{ + service_name="frontend", + span_kind="SPAN_KIND_SERVER", + span_name="GET /api/data", + le="2000" + }[5m] + ) + ) + + ) / ( + sum( + rate( + traces_span_metrics_calls_total{ + service_name="frontend", + span_kind="SPAN_KIND_SERVER", + span_name="GET /api/data" + }[5m] + ) + ) + + ) + labels: + openslo_slo_name: ad-latency + openslo_spec_version: openslo/v1 + chaos_flag: adServiceHighCpu + openslo_service_name: ad + service: ad + - record: openslo_sli_error_rate_30m + expr: | + 1 - ( + sum( + rate( + traces_span_metrics_duration_milliseconds_bucket{ + service_name="frontend", + span_kind="SPAN_KIND_SERVER", + span_name="GET /api/data", + le="2000" + }[30m] + ) + ) + + ) / ( + sum( + rate( + traces_span_metrics_calls_total{ + service_name="frontend", + span_kind="SPAN_KIND_SERVER", + span_name="GET /api/data" + }[30m] + ) + ) + + ) + labels: + openslo_slo_name: ad-latency + openslo_spec_version: openslo/v1 + chaos_flag: adServiceHighCpu + openslo_service_name: ad + service: ad + - record: openslo_sli_error_rate_1h + expr: | + 1 - ( + sum( + rate( + traces_span_metrics_duration_milliseconds_bucket{ + service_name="frontend", + span_kind="SPAN_KIND_SERVER", + span_name="GET /api/data", + le="2000" + }[1h] + ) + ) + + ) / ( + sum( + rate( + traces_span_metrics_calls_total{ + service_name="frontend", + span_kind="SPAN_KIND_SERVER", + span_name="GET /api/data" + }[1h] + ) + ) + + ) + labels: + openslo_slo_name: ad-latency + openslo_spec_version: openslo/v1 + chaos_flag: adServiceHighCpu + openslo_service_name: ad + service: ad + - record: openslo_sli_error_rate_3h + expr: | + 1 - ( + sum( + rate( + traces_span_metrics_duration_milliseconds_bucket{ + service_name="frontend", + span_kind="SPAN_KIND_SERVER", + span_name="GET /api/data", + le="2000" + }[3h] + ) + ) + + ) / ( + sum( + rate( + traces_span_metrics_calls_total{ + service_name="frontend", + span_kind="SPAN_KIND_SERVER", + span_name="GET /api/data" + }[3h] + ) + ) + + ) + labels: + openslo_slo_name: ad-latency + openslo_spec_version: openslo/v1 + chaos_flag: adServiceHighCpu + openslo_service_name: ad + service: ad + - record: openslo_sli_error_rate_6h + expr: | + 1 - ( + sum( + rate( + traces_span_metrics_duration_milliseconds_bucket{ + service_name="frontend", + span_kind="SPAN_KIND_SERVER", + span_name="GET /api/data", + le="2000" + }[6h] + ) + ) + + ) / ( + sum( + rate( + traces_span_metrics_calls_total{ + service_name="frontend", + span_kind="SPAN_KIND_SERVER", + span_name="GET /api/data" + }[6h] + ) + ) + + ) + labels: + openslo_slo_name: ad-latency + openslo_spec_version: openslo/v1 + chaos_flag: adServiceHighCpu + openslo_service_name: ad + service: ad + - record: openslo_sli_error_rate_1d + expr: | + 1 - ( + sum( + rate( + traces_span_metrics_duration_milliseconds_bucket{ + service_name="frontend", + span_kind="SPAN_KIND_SERVER", + span_name="GET /api/data", + le="2000" + }[1d] + ) + ) + + ) / ( + sum( + rate( + traces_span_metrics_calls_total{ + service_name="frontend", + span_kind="SPAN_KIND_SERVER", + span_name="GET /api/data" + }[1d] + ) + ) + + ) + labels: + openslo_slo_name: ad-latency + openslo_spec_version: openslo/v1 + chaos_flag: adServiceHighCpu + openslo_service_name: ad + service: ad + - record: openslo_sli_error_rate_3d + expr: | + 1 - ( + sum( + rate( + traces_span_metrics_duration_milliseconds_bucket{ + service_name="frontend", + span_kind="SPAN_KIND_SERVER", + span_name="GET /api/data", + le="2000" + }[3d] + ) + ) + + ) / ( + sum( + rate( + traces_span_metrics_calls_total{ + service_name="frontend", + span_kind="SPAN_KIND_SERVER", + span_name="GET /api/data" + }[3d] + ) + ) + + ) + labels: + openslo_slo_name: ad-latency + openslo_spec_version: openslo/v1 + chaos_flag: adServiceHighCpu + openslo_service_name: ad + service: ad + - record: openslo_sli_error_rate_7d + expr: | + 1 - ( + sum( + rate( + traces_span_metrics_duration_milliseconds_bucket{ + service_name="frontend", + span_kind="SPAN_KIND_SERVER", + span_name="GET /api/data", + le="2000" + }[7d] + ) + ) + + ) / ( + sum( + rate( + traces_span_metrics_calls_total{ + service_name="frontend", + span_kind="SPAN_KIND_SERVER", + span_name="GET /api/data" + }[7d] + ) + ) + + ) + labels: + openslo_slo_name: ad-latency + openslo_spec_version: openslo/v1 + chaos_flag: adServiceHighCpu + openslo_service_name: ad + service: ad + - record: openslo_sli_error_rate_28d + expr: | + 1 - ( + sum( + rate( + traces_span_metrics_duration_milliseconds_bucket{ + service_name="frontend", + span_kind="SPAN_KIND_SERVER", + span_name="GET /api/data", + le="2000" + }[28d] + ) + ) + + ) / ( + sum( + rate( + traces_span_metrics_calls_total{ + service_name="frontend", + span_kind="SPAN_KIND_SERVER", + span_name="GET /api/data" + }[28d] + ) + ) + + ) + labels: + openslo_slo_name: ad-latency + openslo_spec_version: openslo/v1 + chaos_flag: adServiceHighCpu + openslo_service_name: ad + service: ad + - record: openslo_sli_error_rate_30d + expr: | + 1 - ( + sum( + rate( + traces_span_metrics_duration_milliseconds_bucket{ + service_name="frontend", + span_kind="SPAN_KIND_SERVER", + span_name="GET /api/data", + le="2000" + }[30d] + ) + ) + + ) / ( + sum( + rate( + traces_span_metrics_calls_total{ + service_name="frontend", + span_kind="SPAN_KIND_SERVER", + span_name="GET /api/data" + }[30d] + ) + ) + + ) + labels: + openslo_slo_name: ad-latency + openslo_spec_version: openslo/v1 + chaos_flag: adServiceHighCpu + openslo_service_name: ad + service: ad + - record: openslo_sli_event_rate_5m + expr: | + sum( + rate( + traces_span_metrics_calls_total{ + service_name="frontend", + span_kind="SPAN_KIND_SERVER", + span_name="GET /api/data" + }[5m] + ) + ) + + labels: + openslo_slo_name: ad-latency + openslo_spec_version: openslo/v1 + chaos_flag: adServiceHighCpu + openslo_service_name: ad + service: ad + - record: openslo_sli_event_rate_30m + expr: | + sum( + rate( + traces_span_metrics_calls_total{ + service_name="frontend", + span_kind="SPAN_KIND_SERVER", + span_name="GET /api/data" + }[30m] + ) + ) + + labels: + openslo_slo_name: ad-latency + openslo_spec_version: openslo/v1 + chaos_flag: adServiceHighCpu + openslo_service_name: ad + service: ad + - record: openslo_sli_event_rate_1h + expr: | + sum( + rate( + traces_span_metrics_calls_total{ + service_name="frontend", + span_kind="SPAN_KIND_SERVER", + span_name="GET /api/data" + }[1h] + ) + ) + + labels: + openslo_slo_name: ad-latency + openslo_spec_version: openslo/v1 + chaos_flag: adServiceHighCpu + openslo_service_name: ad + service: ad + - record: openslo_sli_event_rate_3h + expr: | + sum( + rate( + traces_span_metrics_calls_total{ + service_name="frontend", + span_kind="SPAN_KIND_SERVER", + span_name="GET /api/data" + }[3h] + ) + ) + + labels: + openslo_slo_name: ad-latency + openslo_spec_version: openslo/v1 + chaos_flag: adServiceHighCpu + openslo_service_name: ad + service: ad + - record: openslo_sli_event_rate_6h + expr: | + sum( + rate( + traces_span_metrics_calls_total{ + service_name="frontend", + span_kind="SPAN_KIND_SERVER", + span_name="GET /api/data" + }[6h] + ) + ) + + labels: + openslo_slo_name: ad-latency + openslo_spec_version: openslo/v1 + chaos_flag: adServiceHighCpu + openslo_service_name: ad + service: ad + - record: openslo_sli_event_rate_1d + expr: | + sum( + rate( + traces_span_metrics_calls_total{ + service_name="frontend", + span_kind="SPAN_KIND_SERVER", + span_name="GET /api/data" + }[1d] + ) + ) + + labels: + openslo_slo_name: ad-latency + openslo_spec_version: openslo/v1 + chaos_flag: adServiceHighCpu + openslo_service_name: ad + service: ad + - record: openslo_sli_event_rate_3d + expr: | + sum( + rate( + traces_span_metrics_calls_total{ + service_name="frontend", + span_kind="SPAN_KIND_SERVER", + span_name="GET /api/data" + }[3d] + ) + ) + + labels: + openslo_slo_name: ad-latency + openslo_spec_version: openslo/v1 + chaos_flag: adServiceHighCpu + openslo_service_name: ad + service: ad + - record: openslo_sli_event_rate_7d + expr: | + sum( + rate( + traces_span_metrics_calls_total{ + service_name="frontend", + span_kind="SPAN_KIND_SERVER", + span_name="GET /api/data" + }[7d] + ) + ) + + labels: + openslo_slo_name: ad-latency + openslo_spec_version: openslo/v1 + chaos_flag: adServiceHighCpu + openslo_service_name: ad + service: ad + - record: openslo_sli_event_rate_28d + expr: | + sum( + rate( + traces_span_metrics_calls_total{ + service_name="frontend", + span_kind="SPAN_KIND_SERVER", + span_name="GET /api/data" + }[28d] + ) + ) + + labels: + openslo_slo_name: ad-latency + openslo_spec_version: openslo/v1 + chaos_flag: adServiceHighCpu + openslo_service_name: ad + service: ad + - record: openslo_sli_event_rate_30d + expr: | + sum( + rate( + traces_span_metrics_calls_total{ + service_name="frontend", + span_kind="SPAN_KIND_SERVER", + span_name="GET /api/data" + }[30d] + ) + ) + + labels: + openslo_slo_name: ad-latency + openslo_spec_version: openslo/v1 + chaos_flag: adServiceHighCpu + openslo_service_name: ad + service: ad + - name: openslo-status-recordings-ad-latency + rules: + - record: openslo_slo_status + expr: | + ( + (openslo_slo_current_burn_rate{openslo_slo_name="ad-latency", openslo_spec_version="openslo/v1"} >= bool 14.4) + * 3 + ) + or + ( + (openslo_slo_current_burn_rate{openslo_slo_name="ad-latency", openslo_spec_version="openslo/v1"} >= bool 6) + and + (openslo_slo_current_burn_rate{openslo_slo_name="ad-latency", openslo_spec_version="openslo/v1"} < bool 14.4) + * 2 + ) + or + ( + (openslo_slo_current_burn_rate{openslo_slo_name="ad-latency", openslo_spec_version="openslo/v1"} >= bool 1) + and + (openslo_slo_current_burn_rate{openslo_slo_name="ad-latency", openslo_spec_version="openslo/v1"} < bool 6) + * 1 + ) + labels: + openslo_slo_name: ad-latency + openslo_spec_version: openslo/v1 + chaos_flag: adServiceHighCpu + openslo_service_name: ad + service: ad + - name: openslo-alerts-ad-latency + rules: + - alert: AdLatencyMultiWindowMultiBurnRatePage + expr: |- + ( + openslo_sli_error_rate_5m{openslo_slo_name="ad-latency"} / (1 - openslo_slo_objective{openslo_slo_name="ad-latency"}) >= 14.400000 + and + openslo_sli_error_rate_1h{openslo_slo_name="ad-latency"} / (1 - openslo_slo_objective{openslo_slo_name="ad-latency"}) >= 14.400000) + or + ( + openslo_sli_error_rate_30m{openslo_slo_name="ad-latency"} / (1 - openslo_slo_objective{openslo_slo_name="ad-latency"}) >= 6.000000 + and + openslo_sli_error_rate_6h{openslo_slo_name="ad-latency"} / (1 - openslo_slo_objective{openslo_slo_name="ad-latency"}) >= 6.000000) + for: 2m + labels: + openslo_alert_severity: page + openslo_notification_target: engineers + openslo_slo_name: ad-latency + openslo_spec_version: openslo/v1 + chaos_flag: adServiceHighCpu + openslo_service_name: ad + service: ad + annotations: + summary: "Page multi-window multi-burn rate alert for SLO ad-latency" + description: "Threshold 14.4x and 6.0x over 5m and 1h and 30m and 6h" + - alert: AdLatencyMultiWindowMultiBurnRateTicket + expr: |- + ( + openslo_sli_error_rate_2h{openslo_slo_name="ad-latency"} / (1 - openslo_slo_objective{openslo_slo_name="ad-latency"}) >= 3.000000 + and + openslo_sli_error_rate_1d{openslo_slo_name="ad-latency"} / (1 - openslo_slo_objective{openslo_slo_name="ad-latency"}) >= 3.000000) + or + ( + openslo_sli_error_rate_6h{openslo_slo_name="ad-latency"} / (1 - openslo_slo_objective{openslo_slo_name="ad-latency"}) >= 1.000000 + and + openslo_sli_error_rate_3d{openslo_slo_name="ad-latency"} / (1 - openslo_slo_objective{openslo_slo_name="ad-latency"}) >= 1.000000) + for: 15m + labels: + openslo_alert_severity: ticket + openslo_notification_target: engineers + openslo_slo_name: ad-latency + openslo_spec_version: openslo/v1 + chaos_flag: adServiceHighCpu + openslo_service_name: ad + service: ad + annotations: + summary: "Ticket multi-window multi-burn rate alert for SLO ad-latency" + description: "Threshold 3.0x and 1.0x over 2h and 1d and 6h and 3d" diff --git a/examples/oteldemo/rules/cart-availability-rules.yaml b/examples/oteldemo/rules/cart-availability-rules.yaml new file mode 100644 index 0000000..02637e8 --- /dev/null +++ b/examples/oteldemo/rules/cart-availability-rules.yaml @@ -0,0 +1,354 @@ +groups: + - name: openslo-info-recordings-cart-availability + rules: + - record: openslo_slo_info + expr: vector(1) + labels: + openslo_slo_name: cart-availability + openslo_slo_description: "95% of cart actions should succeed, measured via frontend HTTP SERVER `GET /api/cart` and `POST /api/cart` spans (target 0.95; trips on `cartServiceFailure`)." + openslo_spec_version: openslo/v1 + chaos_flag: cartServiceFailure + openslo_service_name: cart + service: cart + - record: openslo_slo_objective + expr: vector(0.99) + labels: + openslo_slo_name: cart-availability + openslo_spec_version: openslo/v1 + chaos_flag: cartServiceFailure + openslo_service_name: cart + service: cart + - record: openslo_slo_timewindow_days + expr: vector(30) + labels: + openslo_slo_name: cart-availability + openslo_spec_version: openslo/v1 + chaos_flag: cartServiceFailure + openslo_service_name: cart + service: cart + - record: openslo_slo_error_budget + expr: vector(1- 0.99) + labels: + openslo_slo_name: cart-availability + openslo_spec_version: openslo/v1 + chaos_flag: cartServiceFailure + openslo_service_name: cart + service: cart + - record: openslo_slo_current_burn_rate + expr: | + openslo_sli_error_rate_5m{openslo_slo_name="cart-availability", openslo_spec_version="openslo/v1"} + / on(openslo_slo_name, openslo_spec_version) group_left + openslo_slo_error_budget{openslo_slo_name="cart-availability", openslo_spec_version="openslo/v1"} + labels: + openslo_slo_name: cart-availability + openslo_spec_version: openslo/v1 + chaos_flag: cartServiceFailure + openslo_service_name: cart + service: cart + - record: openslo_slo_period_burn_rate + expr: | + openslo_sli_error_rate_30d{openslo_slo_name="cart-availability", openslo_spec_version="openslo/v1"} + / on(openslo_slo_name, openslo_spec_version) group_left + openslo_slo_error_budget{openslo_slo_name="cart-availability", openslo_spec_version="openslo/v1"} + labels: + openslo_slo_name: cart-availability + openslo_spec_version: openslo/v1 + chaos_flag: cartServiceFailure + openslo_service_name: cart + service: cart + - record: openslo_slo_period_error_budget_remaining + expr: clamp_min(1 - openslo_slo_period_burn_rate{openslo_slo_name="cart-availability", openslo_spec_version="openslo/v1"}, 0) + labels: + openslo_slo_name: cart-availability + openslo_spec_version: openslo/v1 + chaos_flag: cartServiceFailure + openslo_service_name: cart + service: cart + - name: openslo-sli-recordings-cart-availability + rules: + - record: openslo_sli_error_rate_5m + expr: | + 1 - ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name=~"GET /api/cart|POST /api/cart", status_code!="STATUS_CODE_ERROR"}[5m])) + ) / ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name=~"GET /api/cart|POST /api/cart"}[5m])) + ) + labels: + openslo_slo_name: cart-availability + openslo_spec_version: openslo/v1 + chaos_flag: cartServiceFailure + openslo_service_name: cart + service: cart + - record: openslo_sli_error_rate_30m + expr: | + 1 - ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name=~"GET /api/cart|POST /api/cart", status_code!="STATUS_CODE_ERROR"}[30m])) + ) / ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name=~"GET /api/cart|POST /api/cart"}[30m])) + ) + labels: + openslo_slo_name: cart-availability + openslo_spec_version: openslo/v1 + chaos_flag: cartServiceFailure + openslo_service_name: cart + service: cart + - record: openslo_sli_error_rate_1h + expr: | + 1 - ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name=~"GET /api/cart|POST /api/cart", status_code!="STATUS_CODE_ERROR"}[1h])) + ) / ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name=~"GET /api/cart|POST /api/cart"}[1h])) + ) + labels: + openslo_slo_name: cart-availability + openslo_spec_version: openslo/v1 + chaos_flag: cartServiceFailure + openslo_service_name: cart + service: cart + - record: openslo_sli_error_rate_3h + expr: | + 1 - ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name=~"GET /api/cart|POST /api/cart", status_code!="STATUS_CODE_ERROR"}[3h])) + ) / ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name=~"GET /api/cart|POST /api/cart"}[3h])) + ) + labels: + openslo_slo_name: cart-availability + openslo_spec_version: openslo/v1 + chaos_flag: cartServiceFailure + openslo_service_name: cart + service: cart + - record: openslo_sli_error_rate_6h + expr: | + 1 - ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name=~"GET /api/cart|POST /api/cart", status_code!="STATUS_CODE_ERROR"}[6h])) + ) / ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name=~"GET /api/cart|POST /api/cart"}[6h])) + ) + labels: + openslo_slo_name: cart-availability + openslo_spec_version: openslo/v1 + chaos_flag: cartServiceFailure + openslo_service_name: cart + service: cart + - record: openslo_sli_error_rate_1d + expr: | + 1 - ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name=~"GET /api/cart|POST /api/cart", status_code!="STATUS_CODE_ERROR"}[1d])) + ) / ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name=~"GET /api/cart|POST /api/cart"}[1d])) + ) + labels: + openslo_slo_name: cart-availability + openslo_spec_version: openslo/v1 + chaos_flag: cartServiceFailure + openslo_service_name: cart + service: cart + - record: openslo_sli_error_rate_3d + expr: | + 1 - ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name=~"GET /api/cart|POST /api/cart", status_code!="STATUS_CODE_ERROR"}[3d])) + ) / ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name=~"GET /api/cart|POST /api/cart"}[3d])) + ) + labels: + openslo_slo_name: cart-availability + openslo_spec_version: openslo/v1 + chaos_flag: cartServiceFailure + openslo_service_name: cart + service: cart + - record: openslo_sli_error_rate_7d + expr: | + 1 - ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name=~"GET /api/cart|POST /api/cart", status_code!="STATUS_CODE_ERROR"}[7d])) + ) / ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name=~"GET /api/cart|POST /api/cart"}[7d])) + ) + labels: + openslo_slo_name: cart-availability + openslo_spec_version: openslo/v1 + chaos_flag: cartServiceFailure + openslo_service_name: cart + service: cart + - record: openslo_sli_error_rate_28d + expr: | + 1 - ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name=~"GET /api/cart|POST /api/cart", status_code!="STATUS_CODE_ERROR"}[28d])) + ) / ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name=~"GET /api/cart|POST /api/cart"}[28d])) + ) + labels: + openslo_slo_name: cart-availability + openslo_spec_version: openslo/v1 + chaos_flag: cartServiceFailure + openslo_service_name: cart + service: cart + - record: openslo_sli_error_rate_30d + expr: | + 1 - ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name=~"GET /api/cart|POST /api/cart", status_code!="STATUS_CODE_ERROR"}[30d])) + ) / ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name=~"GET /api/cart|POST /api/cart"}[30d])) + ) + labels: + openslo_slo_name: cart-availability + openslo_spec_version: openslo/v1 + chaos_flag: cartServiceFailure + openslo_service_name: cart + service: cart + - record: openslo_sli_event_rate_5m + expr: sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name=~"GET /api/cart|POST /api/cart"}[5m])) + labels: + openslo_slo_name: cart-availability + openslo_spec_version: openslo/v1 + chaos_flag: cartServiceFailure + openslo_service_name: cart + service: cart + - record: openslo_sli_event_rate_30m + expr: sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name=~"GET /api/cart|POST /api/cart"}[30m])) + labels: + openslo_slo_name: cart-availability + openslo_spec_version: openslo/v1 + chaos_flag: cartServiceFailure + openslo_service_name: cart + service: cart + - record: openslo_sli_event_rate_1h + expr: sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name=~"GET /api/cart|POST /api/cart"}[1h])) + labels: + openslo_slo_name: cart-availability + openslo_spec_version: openslo/v1 + chaos_flag: cartServiceFailure + openslo_service_name: cart + service: cart + - record: openslo_sli_event_rate_3h + expr: sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name=~"GET /api/cart|POST /api/cart"}[3h])) + labels: + openslo_slo_name: cart-availability + openslo_spec_version: openslo/v1 + chaos_flag: cartServiceFailure + openslo_service_name: cart + service: cart + - record: openslo_sli_event_rate_6h + expr: sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name=~"GET /api/cart|POST /api/cart"}[6h])) + labels: + openslo_slo_name: cart-availability + openslo_spec_version: openslo/v1 + chaos_flag: cartServiceFailure + openslo_service_name: cart + service: cart + - record: openslo_sli_event_rate_1d + expr: sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name=~"GET /api/cart|POST /api/cart"}[1d])) + labels: + openslo_slo_name: cart-availability + openslo_spec_version: openslo/v1 + chaos_flag: cartServiceFailure + openslo_service_name: cart + service: cart + - record: openslo_sli_event_rate_3d + expr: sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name=~"GET /api/cart|POST /api/cart"}[3d])) + labels: + openslo_slo_name: cart-availability + openslo_spec_version: openslo/v1 + chaos_flag: cartServiceFailure + openslo_service_name: cart + service: cart + - record: openslo_sli_event_rate_7d + expr: sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name=~"GET /api/cart|POST /api/cart"}[7d])) + labels: + openslo_slo_name: cart-availability + openslo_spec_version: openslo/v1 + chaos_flag: cartServiceFailure + openslo_service_name: cart + service: cart + - record: openslo_sli_event_rate_28d + expr: sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name=~"GET /api/cart|POST /api/cart"}[28d])) + labels: + openslo_slo_name: cart-availability + openslo_spec_version: openslo/v1 + chaos_flag: cartServiceFailure + openslo_service_name: cart + service: cart + - record: openslo_sli_event_rate_30d + expr: sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name=~"GET /api/cart|POST /api/cart"}[30d])) + labels: + openslo_slo_name: cart-availability + openslo_spec_version: openslo/v1 + chaos_flag: cartServiceFailure + openslo_service_name: cart + service: cart + - name: openslo-status-recordings-cart-availability + rules: + - record: openslo_slo_status + expr: | + ( + (openslo_slo_current_burn_rate{openslo_slo_name="cart-availability", openslo_spec_version="openslo/v1"} >= bool 14.4) + * 3 + ) + or + ( + (openslo_slo_current_burn_rate{openslo_slo_name="cart-availability", openslo_spec_version="openslo/v1"} >= bool 6) + and + (openslo_slo_current_burn_rate{openslo_slo_name="cart-availability", openslo_spec_version="openslo/v1"} < bool 14.4) + * 2 + ) + or + ( + (openslo_slo_current_burn_rate{openslo_slo_name="cart-availability", openslo_spec_version="openslo/v1"} >= bool 1) + and + (openslo_slo_current_burn_rate{openslo_slo_name="cart-availability", openslo_spec_version="openslo/v1"} < bool 6) + * 1 + ) + labels: + openslo_slo_name: cart-availability + openslo_spec_version: openslo/v1 + chaos_flag: cartServiceFailure + openslo_service_name: cart + service: cart + - name: openslo-alerts-cart-availability + rules: + - alert: CartAvailabilityMultiWindowMultiBurnRatePage + expr: |- + ( + openslo_sli_error_rate_5m{openslo_slo_name="cart-availability"} / (1 - openslo_slo_objective{openslo_slo_name="cart-availability"}) >= 14.400000 + and + openslo_sli_error_rate_1h{openslo_slo_name="cart-availability"} / (1 - openslo_slo_objective{openslo_slo_name="cart-availability"}) >= 14.400000) + or + ( + openslo_sli_error_rate_30m{openslo_slo_name="cart-availability"} / (1 - openslo_slo_objective{openslo_slo_name="cart-availability"}) >= 6.000000 + and + openslo_sli_error_rate_6h{openslo_slo_name="cart-availability"} / (1 - openslo_slo_objective{openslo_slo_name="cart-availability"}) >= 6.000000) + for: 2m + labels: + openslo_alert_severity: page + openslo_notification_target: engineers + openslo_slo_name: cart-availability + openslo_spec_version: openslo/v1 + chaos_flag: cartServiceFailure + openslo_service_name: cart + service: cart + annotations: + summary: "Page multi-window multi-burn rate alert for SLO cart-availability" + description: "Threshold 14.4x and 6.0x over 5m and 1h and 30m and 6h" + - alert: CartAvailabilityMultiWindowMultiBurnRateTicket + expr: |- + ( + openslo_sli_error_rate_2h{openslo_slo_name="cart-availability"} / (1 - openslo_slo_objective{openslo_slo_name="cart-availability"}) >= 3.000000 + and + openslo_sli_error_rate_1d{openslo_slo_name="cart-availability"} / (1 - openslo_slo_objective{openslo_slo_name="cart-availability"}) >= 3.000000) + or + ( + openslo_sli_error_rate_6h{openslo_slo_name="cart-availability"} / (1 - openslo_slo_objective{openslo_slo_name="cart-availability"}) >= 1.000000 + and + openslo_sli_error_rate_3d{openslo_slo_name="cart-availability"} / (1 - openslo_slo_objective{openslo_slo_name="cart-availability"}) >= 1.000000) + for: 15m + labels: + openslo_alert_severity: ticket + openslo_notification_target: engineers + openslo_slo_name: cart-availability + openslo_spec_version: openslo/v1 + chaos_flag: cartServiceFailure + openslo_service_name: cart + service: cart + annotations: + summary: "Ticket multi-window multi-burn rate alert for SLO cart-availability" + description: "Threshold 3.0x and 1.0x over 2h and 1d and 6h and 3d" diff --git a/examples/oteldemo/rules/frontend-availability-rules.yaml b/examples/oteldemo/rules/frontend-availability-rules.yaml new file mode 100644 index 0000000..c9d4c65 --- /dev/null +++ b/examples/oteldemo/rules/frontend-availability-rules.yaml @@ -0,0 +1,324 @@ +groups: + - name: openslo-info-recordings-frontend-availability + rules: + - record: openslo_slo_info + expr: vector(1) + labels: + openslo_slo_name: frontend-availability + openslo_slo_description: "95% of frontend HTTP SERVER calls should succeed (target 0.95)." + openslo_spec_version: openslo/v1 + openslo_service_name: frontend + service: frontend + - record: openslo_slo_objective + expr: vector(0.99) + labels: + openslo_slo_name: frontend-availability + openslo_spec_version: openslo/v1 + openslo_service_name: frontend + service: frontend + - record: openslo_slo_timewindow_days + expr: vector(30) + labels: + openslo_slo_name: frontend-availability + openslo_spec_version: openslo/v1 + openslo_service_name: frontend + service: frontend + - record: openslo_slo_error_budget + expr: vector(1- 0.99) + labels: + openslo_slo_name: frontend-availability + openslo_spec_version: openslo/v1 + openslo_service_name: frontend + service: frontend + - record: openslo_slo_current_burn_rate + expr: | + openslo_sli_error_rate_5m{openslo_slo_name="frontend-availability", openslo_spec_version="openslo/v1"} + / on(openslo_slo_name, openslo_spec_version) group_left + openslo_slo_error_budget{openslo_slo_name="frontend-availability", openslo_spec_version="openslo/v1"} + labels: + openslo_slo_name: frontend-availability + openslo_spec_version: openslo/v1 + openslo_service_name: frontend + service: frontend + - record: openslo_slo_period_burn_rate + expr: | + openslo_sli_error_rate_30d{openslo_slo_name="frontend-availability", openslo_spec_version="openslo/v1"} + / on(openslo_slo_name, openslo_spec_version) group_left + openslo_slo_error_budget{openslo_slo_name="frontend-availability", openslo_spec_version="openslo/v1"} + labels: + openslo_slo_name: frontend-availability + openslo_spec_version: openslo/v1 + openslo_service_name: frontend + service: frontend + - record: openslo_slo_period_error_budget_remaining + expr: clamp_min(1 - openslo_slo_period_burn_rate{openslo_slo_name="frontend-availability", openslo_spec_version="openslo/v1"}, 0) + labels: + openslo_slo_name: frontend-availability + openslo_spec_version: openslo/v1 + openslo_service_name: frontend + service: frontend + - name: openslo-sli-recordings-frontend-availability + rules: + - record: openslo_sli_error_rate_5m + expr: | + 1 - ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", status_code!="STATUS_CODE_ERROR"}[5m])) + ) / ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER"}[5m])) + ) + labels: + openslo_slo_name: frontend-availability + openslo_spec_version: openslo/v1 + openslo_service_name: frontend + service: frontend + - record: openslo_sli_error_rate_30m + expr: | + 1 - ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", status_code!="STATUS_CODE_ERROR"}[30m])) + ) / ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER"}[30m])) + ) + labels: + openslo_slo_name: frontend-availability + openslo_spec_version: openslo/v1 + openslo_service_name: frontend + service: frontend + - record: openslo_sli_error_rate_1h + expr: | + 1 - ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", status_code!="STATUS_CODE_ERROR"}[1h])) + ) / ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER"}[1h])) + ) + labels: + openslo_slo_name: frontend-availability + openslo_spec_version: openslo/v1 + openslo_service_name: frontend + service: frontend + - record: openslo_sli_error_rate_3h + expr: | + 1 - ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", status_code!="STATUS_CODE_ERROR"}[3h])) + ) / ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER"}[3h])) + ) + labels: + openslo_slo_name: frontend-availability + openslo_spec_version: openslo/v1 + openslo_service_name: frontend + service: frontend + - record: openslo_sli_error_rate_6h + expr: | + 1 - ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", status_code!="STATUS_CODE_ERROR"}[6h])) + ) / ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER"}[6h])) + ) + labels: + openslo_slo_name: frontend-availability + openslo_spec_version: openslo/v1 + openslo_service_name: frontend + service: frontend + - record: openslo_sli_error_rate_1d + expr: | + 1 - ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", status_code!="STATUS_CODE_ERROR"}[1d])) + ) / ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER"}[1d])) + ) + labels: + openslo_slo_name: frontend-availability + openslo_spec_version: openslo/v1 + openslo_service_name: frontend + service: frontend + - record: openslo_sli_error_rate_3d + expr: | + 1 - ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", status_code!="STATUS_CODE_ERROR"}[3d])) + ) / ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER"}[3d])) + ) + labels: + openslo_slo_name: frontend-availability + openslo_spec_version: openslo/v1 + openslo_service_name: frontend + service: frontend + - record: openslo_sli_error_rate_7d + expr: | + 1 - ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", status_code!="STATUS_CODE_ERROR"}[7d])) + ) / ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER"}[7d])) + ) + labels: + openslo_slo_name: frontend-availability + openslo_spec_version: openslo/v1 + openslo_service_name: frontend + service: frontend + - record: openslo_sli_error_rate_28d + expr: | + 1 - ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", status_code!="STATUS_CODE_ERROR"}[28d])) + ) / ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER"}[28d])) + ) + labels: + openslo_slo_name: frontend-availability + openslo_spec_version: openslo/v1 + openslo_service_name: frontend + service: frontend + - record: openslo_sli_error_rate_30d + expr: | + 1 - ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", status_code!="STATUS_CODE_ERROR"}[30d])) + ) / ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER"}[30d])) + ) + labels: + openslo_slo_name: frontend-availability + openslo_spec_version: openslo/v1 + openslo_service_name: frontend + service: frontend + - record: openslo_sli_event_rate_5m + expr: sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER"}[5m])) + labels: + openslo_slo_name: frontend-availability + openslo_spec_version: openslo/v1 + openslo_service_name: frontend + service: frontend + - record: openslo_sli_event_rate_30m + expr: sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER"}[30m])) + labels: + openslo_slo_name: frontend-availability + openslo_spec_version: openslo/v1 + openslo_service_name: frontend + service: frontend + - record: openslo_sli_event_rate_1h + expr: sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER"}[1h])) + labels: + openslo_slo_name: frontend-availability + openslo_spec_version: openslo/v1 + openslo_service_name: frontend + service: frontend + - record: openslo_sli_event_rate_3h + expr: sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER"}[3h])) + labels: + openslo_slo_name: frontend-availability + openslo_spec_version: openslo/v1 + openslo_service_name: frontend + service: frontend + - record: openslo_sli_event_rate_6h + expr: sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER"}[6h])) + labels: + openslo_slo_name: frontend-availability + openslo_spec_version: openslo/v1 + openslo_service_name: frontend + service: frontend + - record: openslo_sli_event_rate_1d + expr: sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER"}[1d])) + labels: + openslo_slo_name: frontend-availability + openslo_spec_version: openslo/v1 + openslo_service_name: frontend + service: frontend + - record: openslo_sli_event_rate_3d + expr: sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER"}[3d])) + labels: + openslo_slo_name: frontend-availability + openslo_spec_version: openslo/v1 + openslo_service_name: frontend + service: frontend + - record: openslo_sli_event_rate_7d + expr: sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER"}[7d])) + labels: + openslo_slo_name: frontend-availability + openslo_spec_version: openslo/v1 + openslo_service_name: frontend + service: frontend + - record: openslo_sli_event_rate_28d + expr: sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER"}[28d])) + labels: + openslo_slo_name: frontend-availability + openslo_spec_version: openslo/v1 + openslo_service_name: frontend + service: frontend + - record: openslo_sli_event_rate_30d + expr: sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER"}[30d])) + labels: + openslo_slo_name: frontend-availability + openslo_spec_version: openslo/v1 + openslo_service_name: frontend + service: frontend + - name: openslo-status-recordings-frontend-availability + rules: + - record: openslo_slo_status + expr: | + ( + (openslo_slo_current_burn_rate{openslo_slo_name="frontend-availability", openslo_spec_version="openslo/v1"} >= bool 14.4) + * 3 + ) + or + ( + (openslo_slo_current_burn_rate{openslo_slo_name="frontend-availability", openslo_spec_version="openslo/v1"} >= bool 6) + and + (openslo_slo_current_burn_rate{openslo_slo_name="frontend-availability", openslo_spec_version="openslo/v1"} < bool 14.4) + * 2 + ) + or + ( + (openslo_slo_current_burn_rate{openslo_slo_name="frontend-availability", openslo_spec_version="openslo/v1"} >= bool 1) + and + (openslo_slo_current_burn_rate{openslo_slo_name="frontend-availability", openslo_spec_version="openslo/v1"} < bool 6) + * 1 + ) + labels: + openslo_slo_name: frontend-availability + openslo_spec_version: openslo/v1 + openslo_service_name: frontend + service: frontend + - name: openslo-alerts-frontend-availability + rules: + - alert: FrontendAvailabilityMultiWindowMultiBurnRatePage + expr: |- + ( + openslo_sli_error_rate_5m{openslo_slo_name="frontend-availability"} / (1 - openslo_slo_objective{openslo_slo_name="frontend-availability"}) >= 14.400000 + and + openslo_sli_error_rate_1h{openslo_slo_name="frontend-availability"} / (1 - openslo_slo_objective{openslo_slo_name="frontend-availability"}) >= 14.400000) + or + ( + openslo_sli_error_rate_30m{openslo_slo_name="frontend-availability"} / (1 - openslo_slo_objective{openslo_slo_name="frontend-availability"}) >= 6.000000 + and + openslo_sli_error_rate_6h{openslo_slo_name="frontend-availability"} / (1 - openslo_slo_objective{openslo_slo_name="frontend-availability"}) >= 6.000000) + for: 2m + labels: + openslo_alert_severity: page + openslo_notification_target: engineers + openslo_slo_name: frontend-availability + openslo_spec_version: openslo/v1 + openslo_service_name: frontend + service: frontend + annotations: + summary: "Page multi-window multi-burn rate alert for SLO frontend-availability" + description: "Threshold 14.4x and 6.0x over 5m and 1h and 30m and 6h" + - alert: FrontendAvailabilityMultiWindowMultiBurnRateTicket + expr: |- + ( + openslo_sli_error_rate_2h{openslo_slo_name="frontend-availability"} / (1 - openslo_slo_objective{openslo_slo_name="frontend-availability"}) >= 3.000000 + and + openslo_sli_error_rate_1d{openslo_slo_name="frontend-availability"} / (1 - openslo_slo_objective{openslo_slo_name="frontend-availability"}) >= 3.000000) + or + ( + openslo_sli_error_rate_6h{openslo_slo_name="frontend-availability"} / (1 - openslo_slo_objective{openslo_slo_name="frontend-availability"}) >= 1.000000 + and + openslo_sli_error_rate_3d{openslo_slo_name="frontend-availability"} / (1 - openslo_slo_objective{openslo_slo_name="frontend-availability"}) >= 1.000000) + for: 15m + labels: + openslo_alert_severity: ticket + openslo_notification_target: engineers + openslo_slo_name: frontend-availability + openslo_spec_version: openslo/v1 + openslo_service_name: frontend + service: frontend + annotations: + summary: "Ticket multi-window multi-burn rate alert for SLO frontend-availability" + description: "Threshold 3.0x and 1.0x over 2h and 1d and 6h and 3d" diff --git a/examples/oteldemo/rules/image-loading-latency-rules.yaml b/examples/oteldemo/rules/image-loading-latency-rules.yaml new file mode 100644 index 0000000..86780b9 --- /dev/null +++ b/examples/oteldemo/rules/image-loading-latency-rules.yaml @@ -0,0 +1,644 @@ +groups: + - name: openslo-info-recordings-image-loading-latency + rules: + - record: openslo_slo_info + expr: vector(1) + labels: + openslo_slo_name: image-loading-latency + openslo_slo_description: "95% of `image-provider` responses should complete under 2s, measured as the ratio of `le=\"2000\"` classic bucket counts to `_calls_total` on `service_name=\"image-provider\"` (target 0.95; trips on …" + openslo_spec_version: openslo/v1 + chaos_flag: imageSlowLoad + openslo_service_name: image-provider + service: image-provider + - record: openslo_slo_objective + expr: vector(0.95) + labels: + openslo_slo_name: image-loading-latency + openslo_spec_version: openslo/v1 + chaos_flag: imageSlowLoad + openslo_service_name: image-provider + service: image-provider + - record: openslo_slo_timewindow_days + expr: vector(30) + labels: + openslo_slo_name: image-loading-latency + openslo_spec_version: openslo/v1 + chaos_flag: imageSlowLoad + openslo_service_name: image-provider + service: image-provider + - record: openslo_slo_error_budget + expr: vector(1- 0.95) + labels: + openslo_slo_name: image-loading-latency + openslo_spec_version: openslo/v1 + chaos_flag: imageSlowLoad + openslo_service_name: image-provider + service: image-provider + - record: openslo_slo_current_burn_rate + expr: | + openslo_sli_error_rate_5m{openslo_slo_name="image-loading-latency", openslo_spec_version="openslo/v1"} + / on(openslo_slo_name, openslo_spec_version) group_left + openslo_slo_error_budget{openslo_slo_name="image-loading-latency", openslo_spec_version="openslo/v1"} + labels: + openslo_slo_name: image-loading-latency + openslo_spec_version: openslo/v1 + chaos_flag: imageSlowLoad + openslo_service_name: image-provider + service: image-provider + - record: openslo_slo_period_burn_rate + expr: | + openslo_sli_error_rate_30d{openslo_slo_name="image-loading-latency", openslo_spec_version="openslo/v1"} + / on(openslo_slo_name, openslo_spec_version) group_left + openslo_slo_error_budget{openslo_slo_name="image-loading-latency", openslo_spec_version="openslo/v1"} + labels: + openslo_slo_name: image-loading-latency + openslo_spec_version: openslo/v1 + chaos_flag: imageSlowLoad + openslo_service_name: image-provider + service: image-provider + - record: openslo_slo_period_error_budget_remaining + expr: clamp_min(1 - openslo_slo_period_burn_rate{openslo_slo_name="image-loading-latency", openslo_spec_version="openslo/v1"}, 0) + labels: + openslo_slo_name: image-loading-latency + openslo_spec_version: openslo/v1 + chaos_flag: imageSlowLoad + openslo_service_name: image-provider + service: image-provider + - name: openslo-sli-recordings-image-loading-latency + rules: + - record: openslo_sli_error_rate_5m + expr: | + 1 - ( + sum( + rate( + traces_span_metrics_duration_milliseconds_bucket{ + service_name="image-provider", + span_kind="SPAN_KIND_SERVER", + span_name="image-provider", + le="2000" + }[5m] + ) + ) + + ) / ( + sum( + rate( + traces_span_metrics_calls_total{ + service_name="image-provider", + span_kind="SPAN_KIND_SERVER", + span_name="image-provider" + }[5m] + ) + ) + + ) + labels: + openslo_slo_name: image-loading-latency + openslo_spec_version: openslo/v1 + chaos_flag: imageSlowLoad + openslo_service_name: image-provider + service: image-provider + - record: openslo_sli_error_rate_30m + expr: | + 1 - ( + sum( + rate( + traces_span_metrics_duration_milliseconds_bucket{ + service_name="image-provider", + span_kind="SPAN_KIND_SERVER", + span_name="image-provider", + le="2000" + }[30m] + ) + ) + + ) / ( + sum( + rate( + traces_span_metrics_calls_total{ + service_name="image-provider", + span_kind="SPAN_KIND_SERVER", + span_name="image-provider" + }[30m] + ) + ) + + ) + labels: + openslo_slo_name: image-loading-latency + openslo_spec_version: openslo/v1 + chaos_flag: imageSlowLoad + openslo_service_name: image-provider + service: image-provider + - record: openslo_sli_error_rate_1h + expr: | + 1 - ( + sum( + rate( + traces_span_metrics_duration_milliseconds_bucket{ + service_name="image-provider", + span_kind="SPAN_KIND_SERVER", + span_name="image-provider", + le="2000" + }[1h] + ) + ) + + ) / ( + sum( + rate( + traces_span_metrics_calls_total{ + service_name="image-provider", + span_kind="SPAN_KIND_SERVER", + span_name="image-provider" + }[1h] + ) + ) + + ) + labels: + openslo_slo_name: image-loading-latency + openslo_spec_version: openslo/v1 + chaos_flag: imageSlowLoad + openslo_service_name: image-provider + service: image-provider + - record: openslo_sli_error_rate_3h + expr: | + 1 - ( + sum( + rate( + traces_span_metrics_duration_milliseconds_bucket{ + service_name="image-provider", + span_kind="SPAN_KIND_SERVER", + span_name="image-provider", + le="2000" + }[3h] + ) + ) + + ) / ( + sum( + rate( + traces_span_metrics_calls_total{ + service_name="image-provider", + span_kind="SPAN_KIND_SERVER", + span_name="image-provider" + }[3h] + ) + ) + + ) + labels: + openslo_slo_name: image-loading-latency + openslo_spec_version: openslo/v1 + chaos_flag: imageSlowLoad + openslo_service_name: image-provider + service: image-provider + - record: openslo_sli_error_rate_6h + expr: | + 1 - ( + sum( + rate( + traces_span_metrics_duration_milliseconds_bucket{ + service_name="image-provider", + span_kind="SPAN_KIND_SERVER", + span_name="image-provider", + le="2000" + }[6h] + ) + ) + + ) / ( + sum( + rate( + traces_span_metrics_calls_total{ + service_name="image-provider", + span_kind="SPAN_KIND_SERVER", + span_name="image-provider" + }[6h] + ) + ) + + ) + labels: + openslo_slo_name: image-loading-latency + openslo_spec_version: openslo/v1 + chaos_flag: imageSlowLoad + openslo_service_name: image-provider + service: image-provider + - record: openslo_sli_error_rate_1d + expr: | + 1 - ( + sum( + rate( + traces_span_metrics_duration_milliseconds_bucket{ + service_name="image-provider", + span_kind="SPAN_KIND_SERVER", + span_name="image-provider", + le="2000" + }[1d] + ) + ) + + ) / ( + sum( + rate( + traces_span_metrics_calls_total{ + service_name="image-provider", + span_kind="SPAN_KIND_SERVER", + span_name="image-provider" + }[1d] + ) + ) + + ) + labels: + openslo_slo_name: image-loading-latency + openslo_spec_version: openslo/v1 + chaos_flag: imageSlowLoad + openslo_service_name: image-provider + service: image-provider + - record: openslo_sli_error_rate_3d + expr: | + 1 - ( + sum( + rate( + traces_span_metrics_duration_milliseconds_bucket{ + service_name="image-provider", + span_kind="SPAN_KIND_SERVER", + span_name="image-provider", + le="2000" + }[3d] + ) + ) + + ) / ( + sum( + rate( + traces_span_metrics_calls_total{ + service_name="image-provider", + span_kind="SPAN_KIND_SERVER", + span_name="image-provider" + }[3d] + ) + ) + + ) + labels: + openslo_slo_name: image-loading-latency + openslo_spec_version: openslo/v1 + chaos_flag: imageSlowLoad + openslo_service_name: image-provider + service: image-provider + - record: openslo_sli_error_rate_7d + expr: | + 1 - ( + sum( + rate( + traces_span_metrics_duration_milliseconds_bucket{ + service_name="image-provider", + span_kind="SPAN_KIND_SERVER", + span_name="image-provider", + le="2000" + }[7d] + ) + ) + + ) / ( + sum( + rate( + traces_span_metrics_calls_total{ + service_name="image-provider", + span_kind="SPAN_KIND_SERVER", + span_name="image-provider" + }[7d] + ) + ) + + ) + labels: + openslo_slo_name: image-loading-latency + openslo_spec_version: openslo/v1 + chaos_flag: imageSlowLoad + openslo_service_name: image-provider + service: image-provider + - record: openslo_sli_error_rate_28d + expr: | + 1 - ( + sum( + rate( + traces_span_metrics_duration_milliseconds_bucket{ + service_name="image-provider", + span_kind="SPAN_KIND_SERVER", + span_name="image-provider", + le="2000" + }[28d] + ) + ) + + ) / ( + sum( + rate( + traces_span_metrics_calls_total{ + service_name="image-provider", + span_kind="SPAN_KIND_SERVER", + span_name="image-provider" + }[28d] + ) + ) + + ) + labels: + openslo_slo_name: image-loading-latency + openslo_spec_version: openslo/v1 + chaos_flag: imageSlowLoad + openslo_service_name: image-provider + service: image-provider + - record: openslo_sli_error_rate_30d + expr: | + 1 - ( + sum( + rate( + traces_span_metrics_duration_milliseconds_bucket{ + service_name="image-provider", + span_kind="SPAN_KIND_SERVER", + span_name="image-provider", + le="2000" + }[30d] + ) + ) + + ) / ( + sum( + rate( + traces_span_metrics_calls_total{ + service_name="image-provider", + span_kind="SPAN_KIND_SERVER", + span_name="image-provider" + }[30d] + ) + ) + + ) + labels: + openslo_slo_name: image-loading-latency + openslo_spec_version: openslo/v1 + chaos_flag: imageSlowLoad + openslo_service_name: image-provider + service: image-provider + - record: openslo_sli_event_rate_5m + expr: | + sum( + rate( + traces_span_metrics_calls_total{ + service_name="image-provider", + span_kind="SPAN_KIND_SERVER", + span_name="image-provider" + }[5m] + ) + ) + + labels: + openslo_slo_name: image-loading-latency + openslo_spec_version: openslo/v1 + chaos_flag: imageSlowLoad + openslo_service_name: image-provider + service: image-provider + - record: openslo_sli_event_rate_30m + expr: | + sum( + rate( + traces_span_metrics_calls_total{ + service_name="image-provider", + span_kind="SPAN_KIND_SERVER", + span_name="image-provider" + }[30m] + ) + ) + + labels: + openslo_slo_name: image-loading-latency + openslo_spec_version: openslo/v1 + chaos_flag: imageSlowLoad + openslo_service_name: image-provider + service: image-provider + - record: openslo_sli_event_rate_1h + expr: | + sum( + rate( + traces_span_metrics_calls_total{ + service_name="image-provider", + span_kind="SPAN_KIND_SERVER", + span_name="image-provider" + }[1h] + ) + ) + + labels: + openslo_slo_name: image-loading-latency + openslo_spec_version: openslo/v1 + chaos_flag: imageSlowLoad + openslo_service_name: image-provider + service: image-provider + - record: openslo_sli_event_rate_3h + expr: | + sum( + rate( + traces_span_metrics_calls_total{ + service_name="image-provider", + span_kind="SPAN_KIND_SERVER", + span_name="image-provider" + }[3h] + ) + ) + + labels: + openslo_slo_name: image-loading-latency + openslo_spec_version: openslo/v1 + chaos_flag: imageSlowLoad + openslo_service_name: image-provider + service: image-provider + - record: openslo_sli_event_rate_6h + expr: | + sum( + rate( + traces_span_metrics_calls_total{ + service_name="image-provider", + span_kind="SPAN_KIND_SERVER", + span_name="image-provider" + }[6h] + ) + ) + + labels: + openslo_slo_name: image-loading-latency + openslo_spec_version: openslo/v1 + chaos_flag: imageSlowLoad + openslo_service_name: image-provider + service: image-provider + - record: openslo_sli_event_rate_1d + expr: | + sum( + rate( + traces_span_metrics_calls_total{ + service_name="image-provider", + span_kind="SPAN_KIND_SERVER", + span_name="image-provider" + }[1d] + ) + ) + + labels: + openslo_slo_name: image-loading-latency + openslo_spec_version: openslo/v1 + chaos_flag: imageSlowLoad + openslo_service_name: image-provider + service: image-provider + - record: openslo_sli_event_rate_3d + expr: | + sum( + rate( + traces_span_metrics_calls_total{ + service_name="image-provider", + span_kind="SPAN_KIND_SERVER", + span_name="image-provider" + }[3d] + ) + ) + + labels: + openslo_slo_name: image-loading-latency + openslo_spec_version: openslo/v1 + chaos_flag: imageSlowLoad + openslo_service_name: image-provider + service: image-provider + - record: openslo_sli_event_rate_7d + expr: | + sum( + rate( + traces_span_metrics_calls_total{ + service_name="image-provider", + span_kind="SPAN_KIND_SERVER", + span_name="image-provider" + }[7d] + ) + ) + + labels: + openslo_slo_name: image-loading-latency + openslo_spec_version: openslo/v1 + chaos_flag: imageSlowLoad + openslo_service_name: image-provider + service: image-provider + - record: openslo_sli_event_rate_28d + expr: | + sum( + rate( + traces_span_metrics_calls_total{ + service_name="image-provider", + span_kind="SPAN_KIND_SERVER", + span_name="image-provider" + }[28d] + ) + ) + + labels: + openslo_slo_name: image-loading-latency + openslo_spec_version: openslo/v1 + chaos_flag: imageSlowLoad + openslo_service_name: image-provider + service: image-provider + - record: openslo_sli_event_rate_30d + expr: | + sum( + rate( + traces_span_metrics_calls_total{ + service_name="image-provider", + span_kind="SPAN_KIND_SERVER", + span_name="image-provider" + }[30d] + ) + ) + + labels: + openslo_slo_name: image-loading-latency + openslo_spec_version: openslo/v1 + chaos_flag: imageSlowLoad + openslo_service_name: image-provider + service: image-provider + - name: openslo-status-recordings-image-loading-latency + rules: + - record: openslo_slo_status + expr: | + ( + (openslo_slo_current_burn_rate{openslo_slo_name="image-loading-latency", openslo_spec_version="openslo/v1"} >= bool 14.4) + * 3 + ) + or + ( + (openslo_slo_current_burn_rate{openslo_slo_name="image-loading-latency", openslo_spec_version="openslo/v1"} >= bool 6) + and + (openslo_slo_current_burn_rate{openslo_slo_name="image-loading-latency", openslo_spec_version="openslo/v1"} < bool 14.4) + * 2 + ) + or + ( + (openslo_slo_current_burn_rate{openslo_slo_name="image-loading-latency", openslo_spec_version="openslo/v1"} >= bool 1) + and + (openslo_slo_current_burn_rate{openslo_slo_name="image-loading-latency", openslo_spec_version="openslo/v1"} < bool 6) + * 1 + ) + labels: + openslo_slo_name: image-loading-latency + openslo_spec_version: openslo/v1 + chaos_flag: imageSlowLoad + openslo_service_name: image-provider + service: image-provider + - name: openslo-alerts-image-loading-latency + rules: + - alert: ImageLoadingLatencyMultiWindowMultiBurnRatePage + expr: |- + ( + openslo_sli_error_rate_5m{openslo_slo_name="image-loading-latency"} / (1 - openslo_slo_objective{openslo_slo_name="image-loading-latency"}) >= 14.400000 + and + openslo_sli_error_rate_1h{openslo_slo_name="image-loading-latency"} / (1 - openslo_slo_objective{openslo_slo_name="image-loading-latency"}) >= 14.400000) + or + ( + openslo_sli_error_rate_30m{openslo_slo_name="image-loading-latency"} / (1 - openslo_slo_objective{openslo_slo_name="image-loading-latency"}) >= 6.000000 + and + openslo_sli_error_rate_6h{openslo_slo_name="image-loading-latency"} / (1 - openslo_slo_objective{openslo_slo_name="image-loading-latency"}) >= 6.000000) + for: 2m + labels: + openslo_alert_severity: page + openslo_notification_target: engineers + openslo_slo_name: image-loading-latency + openslo_spec_version: openslo/v1 + chaos_flag: imageSlowLoad + openslo_service_name: image-provider + service: image-provider + annotations: + summary: "Page multi-window multi-burn rate alert for SLO image-loading-latency" + description: "Threshold 14.4x and 6.0x over 5m and 1h and 30m and 6h" + - alert: ImageLoadingLatencyMultiWindowMultiBurnRateTicket + expr: |- + ( + openslo_sli_error_rate_2h{openslo_slo_name="image-loading-latency"} / (1 - openslo_slo_objective{openslo_slo_name="image-loading-latency"}) >= 3.000000 + and + openslo_sli_error_rate_1d{openslo_slo_name="image-loading-latency"} / (1 - openslo_slo_objective{openslo_slo_name="image-loading-latency"}) >= 3.000000) + or + ( + openslo_sli_error_rate_6h{openslo_slo_name="image-loading-latency"} / (1 - openslo_slo_objective{openslo_slo_name="image-loading-latency"}) >= 1.000000 + and + openslo_sli_error_rate_3d{openslo_slo_name="image-loading-latency"} / (1 - openslo_slo_objective{openslo_slo_name="image-loading-latency"}) >= 1.000000) + for: 15m + labels: + openslo_alert_severity: ticket + openslo_notification_target: engineers + openslo_slo_name: image-loading-latency + openslo_spec_version: openslo/v1 + chaos_flag: imageSlowLoad + openslo_service_name: image-provider + service: image-provider + annotations: + summary: "Ticket multi-window multi-burn rate alert for SLO image-loading-latency" + description: "Threshold 3.0x and 1.0x over 2h and 1d and 6h and 3d" diff --git a/examples/oteldemo/rules/order-processing-latency-rules.yaml b/examples/oteldemo/rules/order-processing-latency-rules.yaml new file mode 100644 index 0000000..9409465 --- /dev/null +++ b/examples/oteldemo/rules/order-processing-latency-rules.yaml @@ -0,0 +1,644 @@ +groups: + - name: openslo-info-recordings-order-processing-latency + rules: + - record: openslo_slo_info + expr: vector(1) + labels: + openslo_slo_name: order-processing-latency + openslo_slo_description: "95% of `POST /api/checkout` should complete under 60s, measured as the ratio of `le=\"60000\"` classic bucket counts to `_calls_total` on frontend HTTP SERVER (target 0.95; trips on `kafkaQueueProble…" + openslo_spec_version: openslo/v1 + chaos_flag: kafkaQueueProblems + openslo_service_name: checkout + service: checkout + - record: openslo_slo_objective + expr: vector(0.95) + labels: + openslo_slo_name: order-processing-latency + openslo_spec_version: openslo/v1 + chaos_flag: kafkaQueueProblems + openslo_service_name: checkout + service: checkout + - record: openslo_slo_timewindow_days + expr: vector(30) + labels: + openslo_slo_name: order-processing-latency + openslo_spec_version: openslo/v1 + chaos_flag: kafkaQueueProblems + openslo_service_name: checkout + service: checkout + - record: openslo_slo_error_budget + expr: vector(1- 0.95) + labels: + openslo_slo_name: order-processing-latency + openslo_spec_version: openslo/v1 + chaos_flag: kafkaQueueProblems + openslo_service_name: checkout + service: checkout + - record: openslo_slo_current_burn_rate + expr: | + openslo_sli_error_rate_5m{openslo_slo_name="order-processing-latency", openslo_spec_version="openslo/v1"} + / on(openslo_slo_name, openslo_spec_version) group_left + openslo_slo_error_budget{openslo_slo_name="order-processing-latency", openslo_spec_version="openslo/v1"} + labels: + openslo_slo_name: order-processing-latency + openslo_spec_version: openslo/v1 + chaos_flag: kafkaQueueProblems + openslo_service_name: checkout + service: checkout + - record: openslo_slo_period_burn_rate + expr: | + openslo_sli_error_rate_30d{openslo_slo_name="order-processing-latency", openslo_spec_version="openslo/v1"} + / on(openslo_slo_name, openslo_spec_version) group_left + openslo_slo_error_budget{openslo_slo_name="order-processing-latency", openslo_spec_version="openslo/v1"} + labels: + openslo_slo_name: order-processing-latency + openslo_spec_version: openslo/v1 + chaos_flag: kafkaQueueProblems + openslo_service_name: checkout + service: checkout + - record: openslo_slo_period_error_budget_remaining + expr: clamp_min(1 - openslo_slo_period_burn_rate{openslo_slo_name="order-processing-latency", openslo_spec_version="openslo/v1"}, 0) + labels: + openslo_slo_name: order-processing-latency + openslo_spec_version: openslo/v1 + chaos_flag: kafkaQueueProblems + openslo_service_name: checkout + service: checkout + - name: openslo-sli-recordings-order-processing-latency + rules: + - record: openslo_sli_error_rate_5m + expr: | + 1 - ( + sum( + rate( + traces_span_metrics_duration_milliseconds_bucket{ + service_name="frontend", + span_kind="SPAN_KIND_SERVER", + span_name="POST /api/checkout", + le="300000" + }[5m] + ) + ) + + ) / ( + sum( + rate( + traces_span_metrics_calls_total{ + service_name="frontend", + span_kind="SPAN_KIND_SERVER", + span_name="POST /api/checkout" + }[5m] + ) + ) + + ) + labels: + openslo_slo_name: order-processing-latency + openslo_spec_version: openslo/v1 + chaos_flag: kafkaQueueProblems + openslo_service_name: checkout + service: checkout + - record: openslo_sli_error_rate_30m + expr: | + 1 - ( + sum( + rate( + traces_span_metrics_duration_milliseconds_bucket{ + service_name="frontend", + span_kind="SPAN_KIND_SERVER", + span_name="POST /api/checkout", + le="300000" + }[30m] + ) + ) + + ) / ( + sum( + rate( + traces_span_metrics_calls_total{ + service_name="frontend", + span_kind="SPAN_KIND_SERVER", + span_name="POST /api/checkout" + }[30m] + ) + ) + + ) + labels: + openslo_slo_name: order-processing-latency + openslo_spec_version: openslo/v1 + chaos_flag: kafkaQueueProblems + openslo_service_name: checkout + service: checkout + - record: openslo_sli_error_rate_1h + expr: | + 1 - ( + sum( + rate( + traces_span_metrics_duration_milliseconds_bucket{ + service_name="frontend", + span_kind="SPAN_KIND_SERVER", + span_name="POST /api/checkout", + le="300000" + }[1h] + ) + ) + + ) / ( + sum( + rate( + traces_span_metrics_calls_total{ + service_name="frontend", + span_kind="SPAN_KIND_SERVER", + span_name="POST /api/checkout" + }[1h] + ) + ) + + ) + labels: + openslo_slo_name: order-processing-latency + openslo_spec_version: openslo/v1 + chaos_flag: kafkaQueueProblems + openslo_service_name: checkout + service: checkout + - record: openslo_sli_error_rate_3h + expr: | + 1 - ( + sum( + rate( + traces_span_metrics_duration_milliseconds_bucket{ + service_name="frontend", + span_kind="SPAN_KIND_SERVER", + span_name="POST /api/checkout", + le="300000" + }[3h] + ) + ) + + ) / ( + sum( + rate( + traces_span_metrics_calls_total{ + service_name="frontend", + span_kind="SPAN_KIND_SERVER", + span_name="POST /api/checkout" + }[3h] + ) + ) + + ) + labels: + openslo_slo_name: order-processing-latency + openslo_spec_version: openslo/v1 + chaos_flag: kafkaQueueProblems + openslo_service_name: checkout + service: checkout + - record: openslo_sli_error_rate_6h + expr: | + 1 - ( + sum( + rate( + traces_span_metrics_duration_milliseconds_bucket{ + service_name="frontend", + span_kind="SPAN_KIND_SERVER", + span_name="POST /api/checkout", + le="300000" + }[6h] + ) + ) + + ) / ( + sum( + rate( + traces_span_metrics_calls_total{ + service_name="frontend", + span_kind="SPAN_KIND_SERVER", + span_name="POST /api/checkout" + }[6h] + ) + ) + + ) + labels: + openslo_slo_name: order-processing-latency + openslo_spec_version: openslo/v1 + chaos_flag: kafkaQueueProblems + openslo_service_name: checkout + service: checkout + - record: openslo_sli_error_rate_1d + expr: | + 1 - ( + sum( + rate( + traces_span_metrics_duration_milliseconds_bucket{ + service_name="frontend", + span_kind="SPAN_KIND_SERVER", + span_name="POST /api/checkout", + le="300000" + }[1d] + ) + ) + + ) / ( + sum( + rate( + traces_span_metrics_calls_total{ + service_name="frontend", + span_kind="SPAN_KIND_SERVER", + span_name="POST /api/checkout" + }[1d] + ) + ) + + ) + labels: + openslo_slo_name: order-processing-latency + openslo_spec_version: openslo/v1 + chaos_flag: kafkaQueueProblems + openslo_service_name: checkout + service: checkout + - record: openslo_sli_error_rate_3d + expr: | + 1 - ( + sum( + rate( + traces_span_metrics_duration_milliseconds_bucket{ + service_name="frontend", + span_kind="SPAN_KIND_SERVER", + span_name="POST /api/checkout", + le="300000" + }[3d] + ) + ) + + ) / ( + sum( + rate( + traces_span_metrics_calls_total{ + service_name="frontend", + span_kind="SPAN_KIND_SERVER", + span_name="POST /api/checkout" + }[3d] + ) + ) + + ) + labels: + openslo_slo_name: order-processing-latency + openslo_spec_version: openslo/v1 + chaos_flag: kafkaQueueProblems + openslo_service_name: checkout + service: checkout + - record: openslo_sli_error_rate_7d + expr: | + 1 - ( + sum( + rate( + traces_span_metrics_duration_milliseconds_bucket{ + service_name="frontend", + span_kind="SPAN_KIND_SERVER", + span_name="POST /api/checkout", + le="300000" + }[7d] + ) + ) + + ) / ( + sum( + rate( + traces_span_metrics_calls_total{ + service_name="frontend", + span_kind="SPAN_KIND_SERVER", + span_name="POST /api/checkout" + }[7d] + ) + ) + + ) + labels: + openslo_slo_name: order-processing-latency + openslo_spec_version: openslo/v1 + chaos_flag: kafkaQueueProblems + openslo_service_name: checkout + service: checkout + - record: openslo_sli_error_rate_28d + expr: | + 1 - ( + sum( + rate( + traces_span_metrics_duration_milliseconds_bucket{ + service_name="frontend", + span_kind="SPAN_KIND_SERVER", + span_name="POST /api/checkout", + le="300000" + }[28d] + ) + ) + + ) / ( + sum( + rate( + traces_span_metrics_calls_total{ + service_name="frontend", + span_kind="SPAN_KIND_SERVER", + span_name="POST /api/checkout" + }[28d] + ) + ) + + ) + labels: + openslo_slo_name: order-processing-latency + openslo_spec_version: openslo/v1 + chaos_flag: kafkaQueueProblems + openslo_service_name: checkout + service: checkout + - record: openslo_sli_error_rate_30d + expr: | + 1 - ( + sum( + rate( + traces_span_metrics_duration_milliseconds_bucket{ + service_name="frontend", + span_kind="SPAN_KIND_SERVER", + span_name="POST /api/checkout", + le="300000" + }[30d] + ) + ) + + ) / ( + sum( + rate( + traces_span_metrics_calls_total{ + service_name="frontend", + span_kind="SPAN_KIND_SERVER", + span_name="POST /api/checkout" + }[30d] + ) + ) + + ) + labels: + openslo_slo_name: order-processing-latency + openslo_spec_version: openslo/v1 + chaos_flag: kafkaQueueProblems + openslo_service_name: checkout + service: checkout + - record: openslo_sli_event_rate_5m + expr: | + sum( + rate( + traces_span_metrics_calls_total{ + service_name="frontend", + span_kind="SPAN_KIND_SERVER", + span_name="POST /api/checkout" + }[5m] + ) + ) + + labels: + openslo_slo_name: order-processing-latency + openslo_spec_version: openslo/v1 + chaos_flag: kafkaQueueProblems + openslo_service_name: checkout + service: checkout + - record: openslo_sli_event_rate_30m + expr: | + sum( + rate( + traces_span_metrics_calls_total{ + service_name="frontend", + span_kind="SPAN_KIND_SERVER", + span_name="POST /api/checkout" + }[30m] + ) + ) + + labels: + openslo_slo_name: order-processing-latency + openslo_spec_version: openslo/v1 + chaos_flag: kafkaQueueProblems + openslo_service_name: checkout + service: checkout + - record: openslo_sli_event_rate_1h + expr: | + sum( + rate( + traces_span_metrics_calls_total{ + service_name="frontend", + span_kind="SPAN_KIND_SERVER", + span_name="POST /api/checkout" + }[1h] + ) + ) + + labels: + openslo_slo_name: order-processing-latency + openslo_spec_version: openslo/v1 + chaos_flag: kafkaQueueProblems + openslo_service_name: checkout + service: checkout + - record: openslo_sli_event_rate_3h + expr: | + sum( + rate( + traces_span_metrics_calls_total{ + service_name="frontend", + span_kind="SPAN_KIND_SERVER", + span_name="POST /api/checkout" + }[3h] + ) + ) + + labels: + openslo_slo_name: order-processing-latency + openslo_spec_version: openslo/v1 + chaos_flag: kafkaQueueProblems + openslo_service_name: checkout + service: checkout + - record: openslo_sli_event_rate_6h + expr: | + sum( + rate( + traces_span_metrics_calls_total{ + service_name="frontend", + span_kind="SPAN_KIND_SERVER", + span_name="POST /api/checkout" + }[6h] + ) + ) + + labels: + openslo_slo_name: order-processing-latency + openslo_spec_version: openslo/v1 + chaos_flag: kafkaQueueProblems + openslo_service_name: checkout + service: checkout + - record: openslo_sli_event_rate_1d + expr: | + sum( + rate( + traces_span_metrics_calls_total{ + service_name="frontend", + span_kind="SPAN_KIND_SERVER", + span_name="POST /api/checkout" + }[1d] + ) + ) + + labels: + openslo_slo_name: order-processing-latency + openslo_spec_version: openslo/v1 + chaos_flag: kafkaQueueProblems + openslo_service_name: checkout + service: checkout + - record: openslo_sli_event_rate_3d + expr: | + sum( + rate( + traces_span_metrics_calls_total{ + service_name="frontend", + span_kind="SPAN_KIND_SERVER", + span_name="POST /api/checkout" + }[3d] + ) + ) + + labels: + openslo_slo_name: order-processing-latency + openslo_spec_version: openslo/v1 + chaos_flag: kafkaQueueProblems + openslo_service_name: checkout + service: checkout + - record: openslo_sli_event_rate_7d + expr: | + sum( + rate( + traces_span_metrics_calls_total{ + service_name="frontend", + span_kind="SPAN_KIND_SERVER", + span_name="POST /api/checkout" + }[7d] + ) + ) + + labels: + openslo_slo_name: order-processing-latency + openslo_spec_version: openslo/v1 + chaos_flag: kafkaQueueProblems + openslo_service_name: checkout + service: checkout + - record: openslo_sli_event_rate_28d + expr: | + sum( + rate( + traces_span_metrics_calls_total{ + service_name="frontend", + span_kind="SPAN_KIND_SERVER", + span_name="POST /api/checkout" + }[28d] + ) + ) + + labels: + openslo_slo_name: order-processing-latency + openslo_spec_version: openslo/v1 + chaos_flag: kafkaQueueProblems + openslo_service_name: checkout + service: checkout + - record: openslo_sli_event_rate_30d + expr: | + sum( + rate( + traces_span_metrics_calls_total{ + service_name="frontend", + span_kind="SPAN_KIND_SERVER", + span_name="POST /api/checkout" + }[30d] + ) + ) + + labels: + openslo_slo_name: order-processing-latency + openslo_spec_version: openslo/v1 + chaos_flag: kafkaQueueProblems + openslo_service_name: checkout + service: checkout + - name: openslo-status-recordings-order-processing-latency + rules: + - record: openslo_slo_status + expr: | + ( + (openslo_slo_current_burn_rate{openslo_slo_name="order-processing-latency", openslo_spec_version="openslo/v1"} >= bool 14.4) + * 3 + ) + or + ( + (openslo_slo_current_burn_rate{openslo_slo_name="order-processing-latency", openslo_spec_version="openslo/v1"} >= bool 6) + and + (openslo_slo_current_burn_rate{openslo_slo_name="order-processing-latency", openslo_spec_version="openslo/v1"} < bool 14.4) + * 2 + ) + or + ( + (openslo_slo_current_burn_rate{openslo_slo_name="order-processing-latency", openslo_spec_version="openslo/v1"} >= bool 1) + and + (openslo_slo_current_burn_rate{openslo_slo_name="order-processing-latency", openslo_spec_version="openslo/v1"} < bool 6) + * 1 + ) + labels: + openslo_slo_name: order-processing-latency + openslo_spec_version: openslo/v1 + chaos_flag: kafkaQueueProblems + openslo_service_name: checkout + service: checkout + - name: openslo-alerts-order-processing-latency + rules: + - alert: OrderProcessingLatencyMultiWindowMultiBurnRatePage + expr: |- + ( + openslo_sli_error_rate_5m{openslo_slo_name="order-processing-latency"} / (1 - openslo_slo_objective{openslo_slo_name="order-processing-latency"}) >= 14.400000 + and + openslo_sli_error_rate_1h{openslo_slo_name="order-processing-latency"} / (1 - openslo_slo_objective{openslo_slo_name="order-processing-latency"}) >= 14.400000) + or + ( + openslo_sli_error_rate_30m{openslo_slo_name="order-processing-latency"} / (1 - openslo_slo_objective{openslo_slo_name="order-processing-latency"}) >= 6.000000 + and + openslo_sli_error_rate_6h{openslo_slo_name="order-processing-latency"} / (1 - openslo_slo_objective{openslo_slo_name="order-processing-latency"}) >= 6.000000) + for: 2m + labels: + openslo_alert_severity: page + openslo_notification_target: engineers + openslo_slo_name: order-processing-latency + openslo_spec_version: openslo/v1 + chaos_flag: kafkaQueueProblems + openslo_service_name: checkout + service: checkout + annotations: + summary: "Page multi-window multi-burn rate alert for SLO order-processing-latency" + description: "Threshold 14.4x and 6.0x over 5m and 1h and 30m and 6h" + - alert: OrderProcessingLatencyMultiWindowMultiBurnRateTicket + expr: |- + ( + openslo_sli_error_rate_2h{openslo_slo_name="order-processing-latency"} / (1 - openslo_slo_objective{openslo_slo_name="order-processing-latency"}) >= 3.000000 + and + openslo_sli_error_rate_1d{openslo_slo_name="order-processing-latency"} / (1 - openslo_slo_objective{openslo_slo_name="order-processing-latency"}) >= 3.000000) + or + ( + openslo_sli_error_rate_6h{openslo_slo_name="order-processing-latency"} / (1 - openslo_slo_objective{openslo_slo_name="order-processing-latency"}) >= 1.000000 + and + openslo_sli_error_rate_3d{openslo_slo_name="order-processing-latency"} / (1 - openslo_slo_objective{openslo_slo_name="order-processing-latency"}) >= 1.000000) + for: 15m + labels: + openslo_alert_severity: ticket + openslo_notification_target: engineers + openslo_slo_name: order-processing-latency + openslo_spec_version: openslo/v1 + chaos_flag: kafkaQueueProblems + openslo_service_name: checkout + service: checkout + annotations: + summary: "Ticket multi-window multi-burn rate alert for SLO order-processing-latency" + description: "Threshold 3.0x and 1.0x over 2h and 1d and 6h and 3d" diff --git a/examples/oteldemo/rules/payment-availability-rules.yaml b/examples/oteldemo/rules/payment-availability-rules.yaml new file mode 100644 index 0000000..2e9ff5d --- /dev/null +++ b/examples/oteldemo/rules/payment-availability-rules.yaml @@ -0,0 +1,354 @@ +groups: + - name: openslo-info-recordings-payment-availability + rules: + - record: openslo_slo_info + expr: vector(1) + labels: + openslo_slo_name: payment-availability + openslo_slo_description: "99% of `POST /api/checkout` should not encounter a payment-unreachable error, measured via frontend HTTP SERVER `POST /api/checkout` (target 0.99; trips on `paymentServiceUnreachable`)." + openslo_spec_version: openslo/v1 + chaos_flag: paymentServiceUnreachable + openslo_service_name: checkout + service: checkout + - record: openslo_slo_objective + expr: vector(0.99) + labels: + openslo_slo_name: payment-availability + openslo_spec_version: openslo/v1 + chaos_flag: paymentServiceUnreachable + openslo_service_name: checkout + service: checkout + - record: openslo_slo_timewindow_days + expr: vector(30) + labels: + openslo_slo_name: payment-availability + openslo_spec_version: openslo/v1 + chaos_flag: paymentServiceUnreachable + openslo_service_name: checkout + service: checkout + - record: openslo_slo_error_budget + expr: vector(1- 0.99) + labels: + openslo_slo_name: payment-availability + openslo_spec_version: openslo/v1 + chaos_flag: paymentServiceUnreachable + openslo_service_name: checkout + service: checkout + - record: openslo_slo_current_burn_rate + expr: | + openslo_sli_error_rate_5m{openslo_slo_name="payment-availability", openslo_spec_version="openslo/v1"} + / on(openslo_slo_name, openslo_spec_version) group_left + openslo_slo_error_budget{openslo_slo_name="payment-availability", openslo_spec_version="openslo/v1"} + labels: + openslo_slo_name: payment-availability + openslo_spec_version: openslo/v1 + chaos_flag: paymentServiceUnreachable + openslo_service_name: checkout + service: checkout + - record: openslo_slo_period_burn_rate + expr: | + openslo_sli_error_rate_30d{openslo_slo_name="payment-availability", openslo_spec_version="openslo/v1"} + / on(openslo_slo_name, openslo_spec_version) group_left + openslo_slo_error_budget{openslo_slo_name="payment-availability", openslo_spec_version="openslo/v1"} + labels: + openslo_slo_name: payment-availability + openslo_spec_version: openslo/v1 + chaos_flag: paymentServiceUnreachable + openslo_service_name: checkout + service: checkout + - record: openslo_slo_period_error_budget_remaining + expr: clamp_min(1 - openslo_slo_period_burn_rate{openslo_slo_name="payment-availability", openslo_spec_version="openslo/v1"}, 0) + labels: + openslo_slo_name: payment-availability + openslo_spec_version: openslo/v1 + chaos_flag: paymentServiceUnreachable + openslo_service_name: checkout + service: checkout + - name: openslo-sli-recordings-payment-availability + rules: + - record: openslo_sli_error_rate_5m + expr: | + 1 - ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="POST /api/checkout", status_code!="STATUS_CODE_ERROR"}[5m])) + ) / ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="POST /api/checkout"}[5m])) + ) + labels: + openslo_slo_name: payment-availability + openslo_spec_version: openslo/v1 + chaos_flag: paymentServiceUnreachable + openslo_service_name: checkout + service: checkout + - record: openslo_sli_error_rate_30m + expr: | + 1 - ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="POST /api/checkout", status_code!="STATUS_CODE_ERROR"}[30m])) + ) / ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="POST /api/checkout"}[30m])) + ) + labels: + openslo_slo_name: payment-availability + openslo_spec_version: openslo/v1 + chaos_flag: paymentServiceUnreachable + openslo_service_name: checkout + service: checkout + - record: openslo_sli_error_rate_1h + expr: | + 1 - ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="POST /api/checkout", status_code!="STATUS_CODE_ERROR"}[1h])) + ) / ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="POST /api/checkout"}[1h])) + ) + labels: + openslo_slo_name: payment-availability + openslo_spec_version: openslo/v1 + chaos_flag: paymentServiceUnreachable + openslo_service_name: checkout + service: checkout + - record: openslo_sli_error_rate_3h + expr: | + 1 - ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="POST /api/checkout", status_code!="STATUS_CODE_ERROR"}[3h])) + ) / ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="POST /api/checkout"}[3h])) + ) + labels: + openslo_slo_name: payment-availability + openslo_spec_version: openslo/v1 + chaos_flag: paymentServiceUnreachable + openslo_service_name: checkout + service: checkout + - record: openslo_sli_error_rate_6h + expr: | + 1 - ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="POST /api/checkout", status_code!="STATUS_CODE_ERROR"}[6h])) + ) / ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="POST /api/checkout"}[6h])) + ) + labels: + openslo_slo_name: payment-availability + openslo_spec_version: openslo/v1 + chaos_flag: paymentServiceUnreachable + openslo_service_name: checkout + service: checkout + - record: openslo_sli_error_rate_1d + expr: | + 1 - ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="POST /api/checkout", status_code!="STATUS_CODE_ERROR"}[1d])) + ) / ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="POST /api/checkout"}[1d])) + ) + labels: + openslo_slo_name: payment-availability + openslo_spec_version: openslo/v1 + chaos_flag: paymentServiceUnreachable + openslo_service_name: checkout + service: checkout + - record: openslo_sli_error_rate_3d + expr: | + 1 - ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="POST /api/checkout", status_code!="STATUS_CODE_ERROR"}[3d])) + ) / ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="POST /api/checkout"}[3d])) + ) + labels: + openslo_slo_name: payment-availability + openslo_spec_version: openslo/v1 + chaos_flag: paymentServiceUnreachable + openslo_service_name: checkout + service: checkout + - record: openslo_sli_error_rate_7d + expr: | + 1 - ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="POST /api/checkout", status_code!="STATUS_CODE_ERROR"}[7d])) + ) / ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="POST /api/checkout"}[7d])) + ) + labels: + openslo_slo_name: payment-availability + openslo_spec_version: openslo/v1 + chaos_flag: paymentServiceUnreachable + openslo_service_name: checkout + service: checkout + - record: openslo_sli_error_rate_28d + expr: | + 1 - ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="POST /api/checkout", status_code!="STATUS_CODE_ERROR"}[28d])) + ) / ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="POST /api/checkout"}[28d])) + ) + labels: + openslo_slo_name: payment-availability + openslo_spec_version: openslo/v1 + chaos_flag: paymentServiceUnreachable + openslo_service_name: checkout + service: checkout + - record: openslo_sli_error_rate_30d + expr: | + 1 - ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="POST /api/checkout", status_code!="STATUS_CODE_ERROR"}[30d])) + ) / ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="POST /api/checkout"}[30d])) + ) + labels: + openslo_slo_name: payment-availability + openslo_spec_version: openslo/v1 + chaos_flag: paymentServiceUnreachable + openslo_service_name: checkout + service: checkout + - record: openslo_sli_event_rate_5m + expr: sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="POST /api/checkout"}[5m])) + labels: + openslo_slo_name: payment-availability + openslo_spec_version: openslo/v1 + chaos_flag: paymentServiceUnreachable + openslo_service_name: checkout + service: checkout + - record: openslo_sli_event_rate_30m + expr: sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="POST /api/checkout"}[30m])) + labels: + openslo_slo_name: payment-availability + openslo_spec_version: openslo/v1 + chaos_flag: paymentServiceUnreachable + openslo_service_name: checkout + service: checkout + - record: openslo_sli_event_rate_1h + expr: sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="POST /api/checkout"}[1h])) + labels: + openslo_slo_name: payment-availability + openslo_spec_version: openslo/v1 + chaos_flag: paymentServiceUnreachable + openslo_service_name: checkout + service: checkout + - record: openslo_sli_event_rate_3h + expr: sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="POST /api/checkout"}[3h])) + labels: + openslo_slo_name: payment-availability + openslo_spec_version: openslo/v1 + chaos_flag: paymentServiceUnreachable + openslo_service_name: checkout + service: checkout + - record: openslo_sli_event_rate_6h + expr: sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="POST /api/checkout"}[6h])) + labels: + openslo_slo_name: payment-availability + openslo_spec_version: openslo/v1 + chaos_flag: paymentServiceUnreachable + openslo_service_name: checkout + service: checkout + - record: openslo_sli_event_rate_1d + expr: sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="POST /api/checkout"}[1d])) + labels: + openslo_slo_name: payment-availability + openslo_spec_version: openslo/v1 + chaos_flag: paymentServiceUnreachable + openslo_service_name: checkout + service: checkout + - record: openslo_sli_event_rate_3d + expr: sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="POST /api/checkout"}[3d])) + labels: + openslo_slo_name: payment-availability + openslo_spec_version: openslo/v1 + chaos_flag: paymentServiceUnreachable + openslo_service_name: checkout + service: checkout + - record: openslo_sli_event_rate_7d + expr: sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="POST /api/checkout"}[7d])) + labels: + openslo_slo_name: payment-availability + openslo_spec_version: openslo/v1 + chaos_flag: paymentServiceUnreachable + openslo_service_name: checkout + service: checkout + - record: openslo_sli_event_rate_28d + expr: sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="POST /api/checkout"}[28d])) + labels: + openslo_slo_name: payment-availability + openslo_spec_version: openslo/v1 + chaos_flag: paymentServiceUnreachable + openslo_service_name: checkout + service: checkout + - record: openslo_sli_event_rate_30d + expr: sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="POST /api/checkout"}[30d])) + labels: + openslo_slo_name: payment-availability + openslo_spec_version: openslo/v1 + chaos_flag: paymentServiceUnreachable + openslo_service_name: checkout + service: checkout + - name: openslo-status-recordings-payment-availability + rules: + - record: openslo_slo_status + expr: | + ( + (openslo_slo_current_burn_rate{openslo_slo_name="payment-availability", openslo_spec_version="openslo/v1"} >= bool 14.4) + * 3 + ) + or + ( + (openslo_slo_current_burn_rate{openslo_slo_name="payment-availability", openslo_spec_version="openslo/v1"} >= bool 6) + and + (openslo_slo_current_burn_rate{openslo_slo_name="payment-availability", openslo_spec_version="openslo/v1"} < bool 14.4) + * 2 + ) + or + ( + (openslo_slo_current_burn_rate{openslo_slo_name="payment-availability", openslo_spec_version="openslo/v1"} >= bool 1) + and + (openslo_slo_current_burn_rate{openslo_slo_name="payment-availability", openslo_spec_version="openslo/v1"} < bool 6) + * 1 + ) + labels: + openslo_slo_name: payment-availability + openslo_spec_version: openslo/v1 + chaos_flag: paymentServiceUnreachable + openslo_service_name: checkout + service: checkout + - name: openslo-alerts-payment-availability + rules: + - alert: PaymentAvailabilityMultiWindowMultiBurnRatePage + expr: |- + ( + openslo_sli_error_rate_5m{openslo_slo_name="payment-availability"} / (1 - openslo_slo_objective{openslo_slo_name="payment-availability"}) >= 14.400000 + and + openslo_sli_error_rate_1h{openslo_slo_name="payment-availability"} / (1 - openslo_slo_objective{openslo_slo_name="payment-availability"}) >= 14.400000) + or + ( + openslo_sli_error_rate_30m{openslo_slo_name="payment-availability"} / (1 - openslo_slo_objective{openslo_slo_name="payment-availability"}) >= 6.000000 + and + openslo_sli_error_rate_6h{openslo_slo_name="payment-availability"} / (1 - openslo_slo_objective{openslo_slo_name="payment-availability"}) >= 6.000000) + for: 2m + labels: + openslo_alert_severity: page + openslo_notification_target: engineers + openslo_slo_name: payment-availability + openslo_spec_version: openslo/v1 + chaos_flag: paymentServiceUnreachable + openslo_service_name: checkout + service: checkout + annotations: + summary: "Page multi-window multi-burn rate alert for SLO payment-availability" + description: "Threshold 14.4x and 6.0x over 5m and 1h and 30m and 6h" + - alert: PaymentAvailabilityMultiWindowMultiBurnRateTicket + expr: |- + ( + openslo_sli_error_rate_2h{openslo_slo_name="payment-availability"} / (1 - openslo_slo_objective{openslo_slo_name="payment-availability"}) >= 3.000000 + and + openslo_sli_error_rate_1d{openslo_slo_name="payment-availability"} / (1 - openslo_slo_objective{openslo_slo_name="payment-availability"}) >= 3.000000) + or + ( + openslo_sli_error_rate_6h{openslo_slo_name="payment-availability"} / (1 - openslo_slo_objective{openslo_slo_name="payment-availability"}) >= 1.000000 + and + openslo_sli_error_rate_3d{openslo_slo_name="payment-availability"} / (1 - openslo_slo_objective{openslo_slo_name="payment-availability"}) >= 1.000000) + for: 15m + labels: + openslo_alert_severity: ticket + openslo_notification_target: engineers + openslo_slo_name: payment-availability + openslo_spec_version: openslo/v1 + chaos_flag: paymentServiceUnreachable + openslo_service_name: checkout + service: checkout + annotations: + summary: "Ticket multi-window multi-burn rate alert for SLO payment-availability" + description: "Threshold 3.0x and 1.0x over 2h and 1d and 6h and 3d" diff --git a/examples/oteldemo/rules/post-order-email-availability-rules.yaml b/examples/oteldemo/rules/post-order-email-availability-rules.yaml new file mode 100644 index 0000000..b9deab8 --- /dev/null +++ b/examples/oteldemo/rules/post-order-email-availability-rules.yaml @@ -0,0 +1,354 @@ +groups: + - name: openslo-info-recordings-post-order-email-availability + rules: + - record: openslo_slo_info + expr: vector(1) + labels: + openslo_slo_name: post-order-email-availability + openslo_slo_description: "95% of order-confirmation email posts should succeed, measured via email SPAN_KIND_SERVER `~POST /send_order_confirmation` (target 0.95; trips on `emailMemoryLeak`)." + openslo_spec_version: openslo/v1 + chaos_flag: emailMemoryLeak + openslo_service_name: email + service: email + - record: openslo_slo_objective + expr: vector(0.95) + labels: + openslo_slo_name: post-order-email-availability + openslo_spec_version: openslo/v1 + chaos_flag: emailMemoryLeak + openslo_service_name: email + service: email + - record: openslo_slo_timewindow_days + expr: vector(30) + labels: + openslo_slo_name: post-order-email-availability + openslo_spec_version: openslo/v1 + chaos_flag: emailMemoryLeak + openslo_service_name: email + service: email + - record: openslo_slo_error_budget + expr: vector(1- 0.95) + labels: + openslo_slo_name: post-order-email-availability + openslo_spec_version: openslo/v1 + chaos_flag: emailMemoryLeak + openslo_service_name: email + service: email + - record: openslo_slo_current_burn_rate + expr: | + openslo_sli_error_rate_5m{openslo_slo_name="post-order-email-availability", openslo_spec_version="openslo/v1"} + / on(openslo_slo_name, openslo_spec_version) group_left + openslo_slo_error_budget{openslo_slo_name="post-order-email-availability", openslo_spec_version="openslo/v1"} + labels: + openslo_slo_name: post-order-email-availability + openslo_spec_version: openslo/v1 + chaos_flag: emailMemoryLeak + openslo_service_name: email + service: email + - record: openslo_slo_period_burn_rate + expr: | + openslo_sli_error_rate_30d{openslo_slo_name="post-order-email-availability", openslo_spec_version="openslo/v1"} + / on(openslo_slo_name, openslo_spec_version) group_left + openslo_slo_error_budget{openslo_slo_name="post-order-email-availability", openslo_spec_version="openslo/v1"} + labels: + openslo_slo_name: post-order-email-availability + openslo_spec_version: openslo/v1 + chaos_flag: emailMemoryLeak + openslo_service_name: email + service: email + - record: openslo_slo_period_error_budget_remaining + expr: clamp_min(1 - openslo_slo_period_burn_rate{openslo_slo_name="post-order-email-availability", openslo_spec_version="openslo/v1"}, 0) + labels: + openslo_slo_name: post-order-email-availability + openslo_spec_version: openslo/v1 + chaos_flag: emailMemoryLeak + openslo_service_name: email + service: email + - name: openslo-sli-recordings-post-order-email-availability + rules: + - record: openslo_sli_error_rate_5m + expr: | + 1 - ( + sum(rate(traces_span_metrics_calls_total{service_name="email", span_kind="SPAN_KIND_SERVER", span_name="POST /send_order_confirmation", status_code!="STATUS_CODE_ERROR"}[5m])) + ) / ( + sum(rate(traces_span_metrics_calls_total{service_name="email", span_kind="SPAN_KIND_SERVER", span_name="POST /send_order_confirmation"}[5m])) + ) + labels: + openslo_slo_name: post-order-email-availability + openslo_spec_version: openslo/v1 + chaos_flag: emailMemoryLeak + openslo_service_name: email + service: email + - record: openslo_sli_error_rate_30m + expr: | + 1 - ( + sum(rate(traces_span_metrics_calls_total{service_name="email", span_kind="SPAN_KIND_SERVER", span_name="POST /send_order_confirmation", status_code!="STATUS_CODE_ERROR"}[30m])) + ) / ( + sum(rate(traces_span_metrics_calls_total{service_name="email", span_kind="SPAN_KIND_SERVER", span_name="POST /send_order_confirmation"}[30m])) + ) + labels: + openslo_slo_name: post-order-email-availability + openslo_spec_version: openslo/v1 + chaos_flag: emailMemoryLeak + openslo_service_name: email + service: email + - record: openslo_sli_error_rate_1h + expr: | + 1 - ( + sum(rate(traces_span_metrics_calls_total{service_name="email", span_kind="SPAN_KIND_SERVER", span_name="POST /send_order_confirmation", status_code!="STATUS_CODE_ERROR"}[1h])) + ) / ( + sum(rate(traces_span_metrics_calls_total{service_name="email", span_kind="SPAN_KIND_SERVER", span_name="POST /send_order_confirmation"}[1h])) + ) + labels: + openslo_slo_name: post-order-email-availability + openslo_spec_version: openslo/v1 + chaos_flag: emailMemoryLeak + openslo_service_name: email + service: email + - record: openslo_sli_error_rate_3h + expr: | + 1 - ( + sum(rate(traces_span_metrics_calls_total{service_name="email", span_kind="SPAN_KIND_SERVER", span_name="POST /send_order_confirmation", status_code!="STATUS_CODE_ERROR"}[3h])) + ) / ( + sum(rate(traces_span_metrics_calls_total{service_name="email", span_kind="SPAN_KIND_SERVER", span_name="POST /send_order_confirmation"}[3h])) + ) + labels: + openslo_slo_name: post-order-email-availability + openslo_spec_version: openslo/v1 + chaos_flag: emailMemoryLeak + openslo_service_name: email + service: email + - record: openslo_sli_error_rate_6h + expr: | + 1 - ( + sum(rate(traces_span_metrics_calls_total{service_name="email", span_kind="SPAN_KIND_SERVER", span_name="POST /send_order_confirmation", status_code!="STATUS_CODE_ERROR"}[6h])) + ) / ( + sum(rate(traces_span_metrics_calls_total{service_name="email", span_kind="SPAN_KIND_SERVER", span_name="POST /send_order_confirmation"}[6h])) + ) + labels: + openslo_slo_name: post-order-email-availability + openslo_spec_version: openslo/v1 + chaos_flag: emailMemoryLeak + openslo_service_name: email + service: email + - record: openslo_sli_error_rate_1d + expr: | + 1 - ( + sum(rate(traces_span_metrics_calls_total{service_name="email", span_kind="SPAN_KIND_SERVER", span_name="POST /send_order_confirmation", status_code!="STATUS_CODE_ERROR"}[1d])) + ) / ( + sum(rate(traces_span_metrics_calls_total{service_name="email", span_kind="SPAN_KIND_SERVER", span_name="POST /send_order_confirmation"}[1d])) + ) + labels: + openslo_slo_name: post-order-email-availability + openslo_spec_version: openslo/v1 + chaos_flag: emailMemoryLeak + openslo_service_name: email + service: email + - record: openslo_sli_error_rate_3d + expr: | + 1 - ( + sum(rate(traces_span_metrics_calls_total{service_name="email", span_kind="SPAN_KIND_SERVER", span_name="POST /send_order_confirmation", status_code!="STATUS_CODE_ERROR"}[3d])) + ) / ( + sum(rate(traces_span_metrics_calls_total{service_name="email", span_kind="SPAN_KIND_SERVER", span_name="POST /send_order_confirmation"}[3d])) + ) + labels: + openslo_slo_name: post-order-email-availability + openslo_spec_version: openslo/v1 + chaos_flag: emailMemoryLeak + openslo_service_name: email + service: email + - record: openslo_sli_error_rate_7d + expr: | + 1 - ( + sum(rate(traces_span_metrics_calls_total{service_name="email", span_kind="SPAN_KIND_SERVER", span_name="POST /send_order_confirmation", status_code!="STATUS_CODE_ERROR"}[7d])) + ) / ( + sum(rate(traces_span_metrics_calls_total{service_name="email", span_kind="SPAN_KIND_SERVER", span_name="POST /send_order_confirmation"}[7d])) + ) + labels: + openslo_slo_name: post-order-email-availability + openslo_spec_version: openslo/v1 + chaos_flag: emailMemoryLeak + openslo_service_name: email + service: email + - record: openslo_sli_error_rate_28d + expr: | + 1 - ( + sum(rate(traces_span_metrics_calls_total{service_name="email", span_kind="SPAN_KIND_SERVER", span_name="POST /send_order_confirmation", status_code!="STATUS_CODE_ERROR"}[28d])) + ) / ( + sum(rate(traces_span_metrics_calls_total{service_name="email", span_kind="SPAN_KIND_SERVER", span_name="POST /send_order_confirmation"}[28d])) + ) + labels: + openslo_slo_name: post-order-email-availability + openslo_spec_version: openslo/v1 + chaos_flag: emailMemoryLeak + openslo_service_name: email + service: email + - record: openslo_sli_error_rate_30d + expr: | + 1 - ( + sum(rate(traces_span_metrics_calls_total{service_name="email", span_kind="SPAN_KIND_SERVER", span_name="POST /send_order_confirmation", status_code!="STATUS_CODE_ERROR"}[30d])) + ) / ( + sum(rate(traces_span_metrics_calls_total{service_name="email", span_kind="SPAN_KIND_SERVER", span_name="POST /send_order_confirmation"}[30d])) + ) + labels: + openslo_slo_name: post-order-email-availability + openslo_spec_version: openslo/v1 + chaos_flag: emailMemoryLeak + openslo_service_name: email + service: email + - record: openslo_sli_event_rate_5m + expr: sum(rate(traces_span_metrics_calls_total{service_name="email", span_kind="SPAN_KIND_SERVER", span_name="POST /send_order_confirmation"}[5m])) + labels: + openslo_slo_name: post-order-email-availability + openslo_spec_version: openslo/v1 + chaos_flag: emailMemoryLeak + openslo_service_name: email + service: email + - record: openslo_sli_event_rate_30m + expr: sum(rate(traces_span_metrics_calls_total{service_name="email", span_kind="SPAN_KIND_SERVER", span_name="POST /send_order_confirmation"}[30m])) + labels: + openslo_slo_name: post-order-email-availability + openslo_spec_version: openslo/v1 + chaos_flag: emailMemoryLeak + openslo_service_name: email + service: email + - record: openslo_sli_event_rate_1h + expr: sum(rate(traces_span_metrics_calls_total{service_name="email", span_kind="SPAN_KIND_SERVER", span_name="POST /send_order_confirmation"}[1h])) + labels: + openslo_slo_name: post-order-email-availability + openslo_spec_version: openslo/v1 + chaos_flag: emailMemoryLeak + openslo_service_name: email + service: email + - record: openslo_sli_event_rate_3h + expr: sum(rate(traces_span_metrics_calls_total{service_name="email", span_kind="SPAN_KIND_SERVER", span_name="POST /send_order_confirmation"}[3h])) + labels: + openslo_slo_name: post-order-email-availability + openslo_spec_version: openslo/v1 + chaos_flag: emailMemoryLeak + openslo_service_name: email + service: email + - record: openslo_sli_event_rate_6h + expr: sum(rate(traces_span_metrics_calls_total{service_name="email", span_kind="SPAN_KIND_SERVER", span_name="POST /send_order_confirmation"}[6h])) + labels: + openslo_slo_name: post-order-email-availability + openslo_spec_version: openslo/v1 + chaos_flag: emailMemoryLeak + openslo_service_name: email + service: email + - record: openslo_sli_event_rate_1d + expr: sum(rate(traces_span_metrics_calls_total{service_name="email", span_kind="SPAN_KIND_SERVER", span_name="POST /send_order_confirmation"}[1d])) + labels: + openslo_slo_name: post-order-email-availability + openslo_spec_version: openslo/v1 + chaos_flag: emailMemoryLeak + openslo_service_name: email + service: email + - record: openslo_sli_event_rate_3d + expr: sum(rate(traces_span_metrics_calls_total{service_name="email", span_kind="SPAN_KIND_SERVER", span_name="POST /send_order_confirmation"}[3d])) + labels: + openslo_slo_name: post-order-email-availability + openslo_spec_version: openslo/v1 + chaos_flag: emailMemoryLeak + openslo_service_name: email + service: email + - record: openslo_sli_event_rate_7d + expr: sum(rate(traces_span_metrics_calls_total{service_name="email", span_kind="SPAN_KIND_SERVER", span_name="POST /send_order_confirmation"}[7d])) + labels: + openslo_slo_name: post-order-email-availability + openslo_spec_version: openslo/v1 + chaos_flag: emailMemoryLeak + openslo_service_name: email + service: email + - record: openslo_sli_event_rate_28d + expr: sum(rate(traces_span_metrics_calls_total{service_name="email", span_kind="SPAN_KIND_SERVER", span_name="POST /send_order_confirmation"}[28d])) + labels: + openslo_slo_name: post-order-email-availability + openslo_spec_version: openslo/v1 + chaos_flag: emailMemoryLeak + openslo_service_name: email + service: email + - record: openslo_sli_event_rate_30d + expr: sum(rate(traces_span_metrics_calls_total{service_name="email", span_kind="SPAN_KIND_SERVER", span_name="POST /send_order_confirmation"}[30d])) + labels: + openslo_slo_name: post-order-email-availability + openslo_spec_version: openslo/v1 + chaos_flag: emailMemoryLeak + openslo_service_name: email + service: email + - name: openslo-status-recordings-post-order-email-availability + rules: + - record: openslo_slo_status + expr: | + ( + (openslo_slo_current_burn_rate{openslo_slo_name="post-order-email-availability", openslo_spec_version="openslo/v1"} >= bool 14.4) + * 3 + ) + or + ( + (openslo_slo_current_burn_rate{openslo_slo_name="post-order-email-availability", openslo_spec_version="openslo/v1"} >= bool 6) + and + (openslo_slo_current_burn_rate{openslo_slo_name="post-order-email-availability", openslo_spec_version="openslo/v1"} < bool 14.4) + * 2 + ) + or + ( + (openslo_slo_current_burn_rate{openslo_slo_name="post-order-email-availability", openslo_spec_version="openslo/v1"} >= bool 1) + and + (openslo_slo_current_burn_rate{openslo_slo_name="post-order-email-availability", openslo_spec_version="openslo/v1"} < bool 6) + * 1 + ) + labels: + openslo_slo_name: post-order-email-availability + openslo_spec_version: openslo/v1 + chaos_flag: emailMemoryLeak + openslo_service_name: email + service: email + - name: openslo-alerts-post-order-email-availability + rules: + - alert: PostOrderEmailAvailabilityMultiWindowMultiBurnRatePage + expr: |- + ( + openslo_sli_error_rate_5m{openslo_slo_name="post-order-email-availability"} / (1 - openslo_slo_objective{openslo_slo_name="post-order-email-availability"}) >= 14.400000 + and + openslo_sli_error_rate_1h{openslo_slo_name="post-order-email-availability"} / (1 - openslo_slo_objective{openslo_slo_name="post-order-email-availability"}) >= 14.400000) + or + ( + openslo_sli_error_rate_30m{openslo_slo_name="post-order-email-availability"} / (1 - openslo_slo_objective{openslo_slo_name="post-order-email-availability"}) >= 6.000000 + and + openslo_sli_error_rate_6h{openslo_slo_name="post-order-email-availability"} / (1 - openslo_slo_objective{openslo_slo_name="post-order-email-availability"}) >= 6.000000) + for: 2m + labels: + openslo_alert_severity: page + openslo_notification_target: engineers + openslo_slo_name: post-order-email-availability + openslo_spec_version: openslo/v1 + chaos_flag: emailMemoryLeak + openslo_service_name: email + service: email + annotations: + summary: "Page multi-window multi-burn rate alert for SLO post-order-email-availability" + description: "Threshold 14.4x and 6.0x over 5m and 1h and 30m and 6h" + - alert: PostOrderEmailAvailabilityMultiWindowMultiBurnRateTicket + expr: |- + ( + openslo_sli_error_rate_2h{openslo_slo_name="post-order-email-availability"} / (1 - openslo_slo_objective{openslo_slo_name="post-order-email-availability"}) >= 3.000000 + and + openslo_sli_error_rate_1d{openslo_slo_name="post-order-email-availability"} / (1 - openslo_slo_objective{openslo_slo_name="post-order-email-availability"}) >= 3.000000) + or + ( + openslo_sli_error_rate_6h{openslo_slo_name="post-order-email-availability"} / (1 - openslo_slo_objective{openslo_slo_name="post-order-email-availability"}) >= 1.000000 + and + openslo_sli_error_rate_3d{openslo_slo_name="post-order-email-availability"} / (1 - openslo_slo_objective{openslo_slo_name="post-order-email-availability"}) >= 1.000000) + for: 15m + labels: + openslo_alert_severity: ticket + openslo_notification_target: engineers + openslo_slo_name: post-order-email-availability + openslo_spec_version: openslo/v1 + chaos_flag: emailMemoryLeak + openslo_service_name: email + service: email + annotations: + summary: "Ticket multi-window multi-burn rate alert for SLO post-order-email-availability" + description: "Threshold 3.0x and 1.0x over 2h and 1d and 6h and 3d" diff --git a/examples/oteldemo/rules/post-order-email-latency-rules.yaml b/examples/oteldemo/rules/post-order-email-latency-rules.yaml new file mode 100644 index 0000000..234cb82 --- /dev/null +++ b/examples/oteldemo/rules/post-order-email-latency-rules.yaml @@ -0,0 +1,644 @@ +groups: + - name: openslo-info-recordings-post-order-email-latency + rules: + - record: openslo_slo_info + expr: vector(1) + labels: + openslo_slo_name: post-order-email-latency + openslo_slo_description: "95% of order-confirmation email posts should complete under 30s, measured as the ratio of `le=\"30000\"` classic bucket counts to `_calls_total` on email SPAN_KIND_SERVER (target 0.95; trips on `emai…" + openslo_spec_version: openslo/v1 + chaos_flag: emailMemoryLeak + openslo_service_name: email + service: email + - record: openslo_slo_objective + expr: vector(0.95) + labels: + openslo_slo_name: post-order-email-latency + openslo_spec_version: openslo/v1 + chaos_flag: emailMemoryLeak + openslo_service_name: email + service: email + - record: openslo_slo_timewindow_days + expr: vector(30) + labels: + openslo_slo_name: post-order-email-latency + openslo_spec_version: openslo/v1 + chaos_flag: emailMemoryLeak + openslo_service_name: email + service: email + - record: openslo_slo_error_budget + expr: vector(1- 0.95) + labels: + openslo_slo_name: post-order-email-latency + openslo_spec_version: openslo/v1 + chaos_flag: emailMemoryLeak + openslo_service_name: email + service: email + - record: openslo_slo_current_burn_rate + expr: | + openslo_sli_error_rate_5m{openslo_slo_name="post-order-email-latency", openslo_spec_version="openslo/v1"} + / on(openslo_slo_name, openslo_spec_version) group_left + openslo_slo_error_budget{openslo_slo_name="post-order-email-latency", openslo_spec_version="openslo/v1"} + labels: + openslo_slo_name: post-order-email-latency + openslo_spec_version: openslo/v1 + chaos_flag: emailMemoryLeak + openslo_service_name: email + service: email + - record: openslo_slo_period_burn_rate + expr: | + openslo_sli_error_rate_30d{openslo_slo_name="post-order-email-latency", openslo_spec_version="openslo/v1"} + / on(openslo_slo_name, openslo_spec_version) group_left + openslo_slo_error_budget{openslo_slo_name="post-order-email-latency", openslo_spec_version="openslo/v1"} + labels: + openslo_slo_name: post-order-email-latency + openslo_spec_version: openslo/v1 + chaos_flag: emailMemoryLeak + openslo_service_name: email + service: email + - record: openslo_slo_period_error_budget_remaining + expr: clamp_min(1 - openslo_slo_period_burn_rate{openslo_slo_name="post-order-email-latency", openslo_spec_version="openslo/v1"}, 0) + labels: + openslo_slo_name: post-order-email-latency + openslo_spec_version: openslo/v1 + chaos_flag: emailMemoryLeak + openslo_service_name: email + service: email + - name: openslo-sli-recordings-post-order-email-latency + rules: + - record: openslo_sli_error_rate_5m + expr: | + 1 - ( + sum( + rate( + traces_span_metrics_duration_milliseconds_bucket{ + service_name="email", + span_kind="SPAN_KIND_SERVER", + span_name=~"POST /send_order_confirmation", + le="30000" + }[5m] + ) + ) + + ) / ( + sum( + rate( + traces_span_metrics_calls_total{ + service_name="email", + span_kind="SPAN_KIND_SERVER", + span_name=~"POST /send_order_confirmation" + }[5m] + ) + ) + + ) + labels: + openslo_slo_name: post-order-email-latency + openslo_spec_version: openslo/v1 + chaos_flag: emailMemoryLeak + openslo_service_name: email + service: email + - record: openslo_sli_error_rate_30m + expr: | + 1 - ( + sum( + rate( + traces_span_metrics_duration_milliseconds_bucket{ + service_name="email", + span_kind="SPAN_KIND_SERVER", + span_name=~"POST /send_order_confirmation", + le="30000" + }[30m] + ) + ) + + ) / ( + sum( + rate( + traces_span_metrics_calls_total{ + service_name="email", + span_kind="SPAN_KIND_SERVER", + span_name=~"POST /send_order_confirmation" + }[30m] + ) + ) + + ) + labels: + openslo_slo_name: post-order-email-latency + openslo_spec_version: openslo/v1 + chaos_flag: emailMemoryLeak + openslo_service_name: email + service: email + - record: openslo_sli_error_rate_1h + expr: | + 1 - ( + sum( + rate( + traces_span_metrics_duration_milliseconds_bucket{ + service_name="email", + span_kind="SPAN_KIND_SERVER", + span_name=~"POST /send_order_confirmation", + le="30000" + }[1h] + ) + ) + + ) / ( + sum( + rate( + traces_span_metrics_calls_total{ + service_name="email", + span_kind="SPAN_KIND_SERVER", + span_name=~"POST /send_order_confirmation" + }[1h] + ) + ) + + ) + labels: + openslo_slo_name: post-order-email-latency + openslo_spec_version: openslo/v1 + chaos_flag: emailMemoryLeak + openslo_service_name: email + service: email + - record: openslo_sli_error_rate_3h + expr: | + 1 - ( + sum( + rate( + traces_span_metrics_duration_milliseconds_bucket{ + service_name="email", + span_kind="SPAN_KIND_SERVER", + span_name=~"POST /send_order_confirmation", + le="30000" + }[3h] + ) + ) + + ) / ( + sum( + rate( + traces_span_metrics_calls_total{ + service_name="email", + span_kind="SPAN_KIND_SERVER", + span_name=~"POST /send_order_confirmation" + }[3h] + ) + ) + + ) + labels: + openslo_slo_name: post-order-email-latency + openslo_spec_version: openslo/v1 + chaos_flag: emailMemoryLeak + openslo_service_name: email + service: email + - record: openslo_sli_error_rate_6h + expr: | + 1 - ( + sum( + rate( + traces_span_metrics_duration_milliseconds_bucket{ + service_name="email", + span_kind="SPAN_KIND_SERVER", + span_name=~"POST /send_order_confirmation", + le="30000" + }[6h] + ) + ) + + ) / ( + sum( + rate( + traces_span_metrics_calls_total{ + service_name="email", + span_kind="SPAN_KIND_SERVER", + span_name=~"POST /send_order_confirmation" + }[6h] + ) + ) + + ) + labels: + openslo_slo_name: post-order-email-latency + openslo_spec_version: openslo/v1 + chaos_flag: emailMemoryLeak + openslo_service_name: email + service: email + - record: openslo_sli_error_rate_1d + expr: | + 1 - ( + sum( + rate( + traces_span_metrics_duration_milliseconds_bucket{ + service_name="email", + span_kind="SPAN_KIND_SERVER", + span_name=~"POST /send_order_confirmation", + le="30000" + }[1d] + ) + ) + + ) / ( + sum( + rate( + traces_span_metrics_calls_total{ + service_name="email", + span_kind="SPAN_KIND_SERVER", + span_name=~"POST /send_order_confirmation" + }[1d] + ) + ) + + ) + labels: + openslo_slo_name: post-order-email-latency + openslo_spec_version: openslo/v1 + chaos_flag: emailMemoryLeak + openslo_service_name: email + service: email + - record: openslo_sli_error_rate_3d + expr: | + 1 - ( + sum( + rate( + traces_span_metrics_duration_milliseconds_bucket{ + service_name="email", + span_kind="SPAN_KIND_SERVER", + span_name=~"POST /send_order_confirmation", + le="30000" + }[3d] + ) + ) + + ) / ( + sum( + rate( + traces_span_metrics_calls_total{ + service_name="email", + span_kind="SPAN_KIND_SERVER", + span_name=~"POST /send_order_confirmation" + }[3d] + ) + ) + + ) + labels: + openslo_slo_name: post-order-email-latency + openslo_spec_version: openslo/v1 + chaos_flag: emailMemoryLeak + openslo_service_name: email + service: email + - record: openslo_sli_error_rate_7d + expr: | + 1 - ( + sum( + rate( + traces_span_metrics_duration_milliseconds_bucket{ + service_name="email", + span_kind="SPAN_KIND_SERVER", + span_name=~"POST /send_order_confirmation", + le="30000" + }[7d] + ) + ) + + ) / ( + sum( + rate( + traces_span_metrics_calls_total{ + service_name="email", + span_kind="SPAN_KIND_SERVER", + span_name=~"POST /send_order_confirmation" + }[7d] + ) + ) + + ) + labels: + openslo_slo_name: post-order-email-latency + openslo_spec_version: openslo/v1 + chaos_flag: emailMemoryLeak + openslo_service_name: email + service: email + - record: openslo_sli_error_rate_28d + expr: | + 1 - ( + sum( + rate( + traces_span_metrics_duration_milliseconds_bucket{ + service_name="email", + span_kind="SPAN_KIND_SERVER", + span_name=~"POST /send_order_confirmation", + le="30000" + }[28d] + ) + ) + + ) / ( + sum( + rate( + traces_span_metrics_calls_total{ + service_name="email", + span_kind="SPAN_KIND_SERVER", + span_name=~"POST /send_order_confirmation" + }[28d] + ) + ) + + ) + labels: + openslo_slo_name: post-order-email-latency + openslo_spec_version: openslo/v1 + chaos_flag: emailMemoryLeak + openslo_service_name: email + service: email + - record: openslo_sli_error_rate_30d + expr: | + 1 - ( + sum( + rate( + traces_span_metrics_duration_milliseconds_bucket{ + service_name="email", + span_kind="SPAN_KIND_SERVER", + span_name=~"POST /send_order_confirmation", + le="30000" + }[30d] + ) + ) + + ) / ( + sum( + rate( + traces_span_metrics_calls_total{ + service_name="email", + span_kind="SPAN_KIND_SERVER", + span_name=~"POST /send_order_confirmation" + }[30d] + ) + ) + + ) + labels: + openslo_slo_name: post-order-email-latency + openslo_spec_version: openslo/v1 + chaos_flag: emailMemoryLeak + openslo_service_name: email + service: email + - record: openslo_sli_event_rate_5m + expr: | + sum( + rate( + traces_span_metrics_calls_total{ + service_name="email", + span_kind="SPAN_KIND_SERVER", + span_name=~"POST /send_order_confirmation" + }[5m] + ) + ) + + labels: + openslo_slo_name: post-order-email-latency + openslo_spec_version: openslo/v1 + chaos_flag: emailMemoryLeak + openslo_service_name: email + service: email + - record: openslo_sli_event_rate_30m + expr: | + sum( + rate( + traces_span_metrics_calls_total{ + service_name="email", + span_kind="SPAN_KIND_SERVER", + span_name=~"POST /send_order_confirmation" + }[30m] + ) + ) + + labels: + openslo_slo_name: post-order-email-latency + openslo_spec_version: openslo/v1 + chaos_flag: emailMemoryLeak + openslo_service_name: email + service: email + - record: openslo_sli_event_rate_1h + expr: | + sum( + rate( + traces_span_metrics_calls_total{ + service_name="email", + span_kind="SPAN_KIND_SERVER", + span_name=~"POST /send_order_confirmation" + }[1h] + ) + ) + + labels: + openslo_slo_name: post-order-email-latency + openslo_spec_version: openslo/v1 + chaos_flag: emailMemoryLeak + openslo_service_name: email + service: email + - record: openslo_sli_event_rate_3h + expr: | + sum( + rate( + traces_span_metrics_calls_total{ + service_name="email", + span_kind="SPAN_KIND_SERVER", + span_name=~"POST /send_order_confirmation" + }[3h] + ) + ) + + labels: + openslo_slo_name: post-order-email-latency + openslo_spec_version: openslo/v1 + chaos_flag: emailMemoryLeak + openslo_service_name: email + service: email + - record: openslo_sli_event_rate_6h + expr: | + sum( + rate( + traces_span_metrics_calls_total{ + service_name="email", + span_kind="SPAN_KIND_SERVER", + span_name=~"POST /send_order_confirmation" + }[6h] + ) + ) + + labels: + openslo_slo_name: post-order-email-latency + openslo_spec_version: openslo/v1 + chaos_flag: emailMemoryLeak + openslo_service_name: email + service: email + - record: openslo_sli_event_rate_1d + expr: | + sum( + rate( + traces_span_metrics_calls_total{ + service_name="email", + span_kind="SPAN_KIND_SERVER", + span_name=~"POST /send_order_confirmation" + }[1d] + ) + ) + + labels: + openslo_slo_name: post-order-email-latency + openslo_spec_version: openslo/v1 + chaos_flag: emailMemoryLeak + openslo_service_name: email + service: email + - record: openslo_sli_event_rate_3d + expr: | + sum( + rate( + traces_span_metrics_calls_total{ + service_name="email", + span_kind="SPAN_KIND_SERVER", + span_name=~"POST /send_order_confirmation" + }[3d] + ) + ) + + labels: + openslo_slo_name: post-order-email-latency + openslo_spec_version: openslo/v1 + chaos_flag: emailMemoryLeak + openslo_service_name: email + service: email + - record: openslo_sli_event_rate_7d + expr: | + sum( + rate( + traces_span_metrics_calls_total{ + service_name="email", + span_kind="SPAN_KIND_SERVER", + span_name=~"POST /send_order_confirmation" + }[7d] + ) + ) + + labels: + openslo_slo_name: post-order-email-latency + openslo_spec_version: openslo/v1 + chaos_flag: emailMemoryLeak + openslo_service_name: email + service: email + - record: openslo_sli_event_rate_28d + expr: | + sum( + rate( + traces_span_metrics_calls_total{ + service_name="email", + span_kind="SPAN_KIND_SERVER", + span_name=~"POST /send_order_confirmation" + }[28d] + ) + ) + + labels: + openslo_slo_name: post-order-email-latency + openslo_spec_version: openslo/v1 + chaos_flag: emailMemoryLeak + openslo_service_name: email + service: email + - record: openslo_sli_event_rate_30d + expr: | + sum( + rate( + traces_span_metrics_calls_total{ + service_name="email", + span_kind="SPAN_KIND_SERVER", + span_name=~"POST /send_order_confirmation" + }[30d] + ) + ) + + labels: + openslo_slo_name: post-order-email-latency + openslo_spec_version: openslo/v1 + chaos_flag: emailMemoryLeak + openslo_service_name: email + service: email + - name: openslo-status-recordings-post-order-email-latency + rules: + - record: openslo_slo_status + expr: | + ( + (openslo_slo_current_burn_rate{openslo_slo_name="post-order-email-latency", openslo_spec_version="openslo/v1"} >= bool 14.4) + * 3 + ) + or + ( + (openslo_slo_current_burn_rate{openslo_slo_name="post-order-email-latency", openslo_spec_version="openslo/v1"} >= bool 6) + and + (openslo_slo_current_burn_rate{openslo_slo_name="post-order-email-latency", openslo_spec_version="openslo/v1"} < bool 14.4) + * 2 + ) + or + ( + (openslo_slo_current_burn_rate{openslo_slo_name="post-order-email-latency", openslo_spec_version="openslo/v1"} >= bool 1) + and + (openslo_slo_current_burn_rate{openslo_slo_name="post-order-email-latency", openslo_spec_version="openslo/v1"} < bool 6) + * 1 + ) + labels: + openslo_slo_name: post-order-email-latency + openslo_spec_version: openslo/v1 + chaos_flag: emailMemoryLeak + openslo_service_name: email + service: email + - name: openslo-alerts-post-order-email-latency + rules: + - alert: PostOrderEmailLatencyMultiWindowMultiBurnRatePage + expr: |- + ( + openslo_sli_error_rate_5m{openslo_slo_name="post-order-email-latency"} / (1 - openslo_slo_objective{openslo_slo_name="post-order-email-latency"}) >= 14.400000 + and + openslo_sli_error_rate_1h{openslo_slo_name="post-order-email-latency"} / (1 - openslo_slo_objective{openslo_slo_name="post-order-email-latency"}) >= 14.400000) + or + ( + openslo_sli_error_rate_30m{openslo_slo_name="post-order-email-latency"} / (1 - openslo_slo_objective{openslo_slo_name="post-order-email-latency"}) >= 6.000000 + and + openslo_sli_error_rate_6h{openslo_slo_name="post-order-email-latency"} / (1 - openslo_slo_objective{openslo_slo_name="post-order-email-latency"}) >= 6.000000) + for: 2m + labels: + openslo_alert_severity: page + openslo_notification_target: engineers + openslo_slo_name: post-order-email-latency + openslo_spec_version: openslo/v1 + chaos_flag: emailMemoryLeak + openslo_service_name: email + service: email + annotations: + summary: "Page multi-window multi-burn rate alert for SLO post-order-email-latency" + description: "Threshold 14.4x and 6.0x over 5m and 1h and 30m and 6h" + - alert: PostOrderEmailLatencyMultiWindowMultiBurnRateTicket + expr: |- + ( + openslo_sli_error_rate_2h{openslo_slo_name="post-order-email-latency"} / (1 - openslo_slo_objective{openslo_slo_name="post-order-email-latency"}) >= 3.000000 + and + openslo_sli_error_rate_1d{openslo_slo_name="post-order-email-latency"} / (1 - openslo_slo_objective{openslo_slo_name="post-order-email-latency"}) >= 3.000000) + or + ( + openslo_sli_error_rate_6h{openslo_slo_name="post-order-email-latency"} / (1 - openslo_slo_objective{openslo_slo_name="post-order-email-latency"}) >= 1.000000 + and + openslo_sli_error_rate_3d{openslo_slo_name="post-order-email-latency"} / (1 - openslo_slo_objective{openslo_slo_name="post-order-email-latency"}) >= 1.000000) + for: 15m + labels: + openslo_alert_severity: ticket + openslo_notification_target: engineers + openslo_slo_name: post-order-email-latency + openslo_spec_version: openslo/v1 + chaos_flag: emailMemoryLeak + openslo_service_name: email + service: email + annotations: + summary: "Ticket multi-window multi-burn rate alert for SLO post-order-email-latency" + description: "Threshold 3.0x and 1.0x over 2h and 1d and 6h and 3d" diff --git a/examples/oteldemo/rules/product-catalog-availability-rules.yaml b/examples/oteldemo/rules/product-catalog-availability-rules.yaml new file mode 100644 index 0000000..d381804 --- /dev/null +++ b/examples/oteldemo/rules/product-catalog-availability-rules.yaml @@ -0,0 +1,324 @@ +groups: + - name: openslo-info-recordings-product-catalog-availability + rules: + - record: openslo_slo_info + expr: vector(1) + labels: + openslo_slo_name: product-catalog-availability + openslo_slo_description: "95% of product browsing should succeed, measured via frontend HTTP SERVER `GET /api/products.*index` spans (target 0.95; trips on `productCatalogFailure`)." + openslo_spec_version: openslo/v1 + chaos_flag: productCatalogFailure + openslo_service_name: productcatalogservice + - record: openslo_slo_objective + expr: vector(0.95) + labels: + openslo_slo_name: product-catalog-availability + openslo_spec_version: openslo/v1 + chaos_flag: productCatalogFailure + openslo_service_name: productcatalogservice + - record: openslo_slo_timewindow_days + expr: vector(30) + labels: + openslo_slo_name: product-catalog-availability + openslo_spec_version: openslo/v1 + chaos_flag: productCatalogFailure + openslo_service_name: productcatalogservice + - record: openslo_slo_error_budget + expr: vector(1- 0.95) + labels: + openslo_slo_name: product-catalog-availability + openslo_spec_version: openslo/v1 + chaos_flag: productCatalogFailure + openslo_service_name: productcatalogservice + - record: openslo_slo_current_burn_rate + expr: | + openslo_sli_error_rate_5m{openslo_slo_name="product-catalog-availability", openslo_spec_version="openslo/v1"} + / on(openslo_slo_name, openslo_spec_version) group_left + openslo_slo_error_budget{openslo_slo_name="product-catalog-availability", openslo_spec_version="openslo/v1"} + labels: + openslo_slo_name: product-catalog-availability + openslo_spec_version: openslo/v1 + chaos_flag: productCatalogFailure + openslo_service_name: productcatalogservice + - record: openslo_slo_period_burn_rate + expr: | + openslo_sli_error_rate_30d{openslo_slo_name="product-catalog-availability", openslo_spec_version="openslo/v1"} + / on(openslo_slo_name, openslo_spec_version) group_left + openslo_slo_error_budget{openslo_slo_name="product-catalog-availability", openslo_spec_version="openslo/v1"} + labels: + openslo_slo_name: product-catalog-availability + openslo_spec_version: openslo/v1 + chaos_flag: productCatalogFailure + openslo_service_name: productcatalogservice + - record: openslo_slo_period_error_budget_remaining + expr: clamp_min(1 - openslo_slo_period_burn_rate{openslo_slo_name="product-catalog-availability", openslo_spec_version="openslo/v1"}, 0) + labels: + openslo_slo_name: product-catalog-availability + openslo_spec_version: openslo/v1 + chaos_flag: productCatalogFailure + openslo_service_name: productcatalogservice + - name: openslo-sli-recordings-product-catalog-availability + rules: + - record: openslo_sli_error_rate_5m + expr: | + 1 - ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name=~"GET /api/products.*index", status_code!="STATUS_CODE_ERROR"}[5m])) + ) / ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name=~"GET /api/products.*index"}[5m])) + ) + labels: + openslo_slo_name: product-catalog-availability + openslo_spec_version: openslo/v1 + chaos_flag: productCatalogFailure + openslo_service_name: productcatalogservice + - record: openslo_sli_error_rate_30m + expr: | + 1 - ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name=~"GET /api/products.*index", status_code!="STATUS_CODE_ERROR"}[30m])) + ) / ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name=~"GET /api/products.*index"}[30m])) + ) + labels: + openslo_slo_name: product-catalog-availability + openslo_spec_version: openslo/v1 + chaos_flag: productCatalogFailure + openslo_service_name: productcatalogservice + - record: openslo_sli_error_rate_1h + expr: | + 1 - ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name=~"GET /api/products.*index", status_code!="STATUS_CODE_ERROR"}[1h])) + ) / ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name=~"GET /api/products.*index"}[1h])) + ) + labels: + openslo_slo_name: product-catalog-availability + openslo_spec_version: openslo/v1 + chaos_flag: productCatalogFailure + openslo_service_name: productcatalogservice + - record: openslo_sli_error_rate_3h + expr: | + 1 - ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name=~"GET /api/products.*index", status_code!="STATUS_CODE_ERROR"}[3h])) + ) / ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name=~"GET /api/products.*index"}[3h])) + ) + labels: + openslo_slo_name: product-catalog-availability + openslo_spec_version: openslo/v1 + chaos_flag: productCatalogFailure + openslo_service_name: productcatalogservice + - record: openslo_sli_error_rate_6h + expr: | + 1 - ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name=~"GET /api/products.*index", status_code!="STATUS_CODE_ERROR"}[6h])) + ) / ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name=~"GET /api/products.*index"}[6h])) + ) + labels: + openslo_slo_name: product-catalog-availability + openslo_spec_version: openslo/v1 + chaos_flag: productCatalogFailure + openslo_service_name: productcatalogservice + - record: openslo_sli_error_rate_1d + expr: | + 1 - ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name=~"GET /api/products.*index", status_code!="STATUS_CODE_ERROR"}[1d])) + ) / ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name=~"GET /api/products.*index"}[1d])) + ) + labels: + openslo_slo_name: product-catalog-availability + openslo_spec_version: openslo/v1 + chaos_flag: productCatalogFailure + openslo_service_name: productcatalogservice + - record: openslo_sli_error_rate_3d + expr: | + 1 - ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name=~"GET /api/products.*index", status_code!="STATUS_CODE_ERROR"}[3d])) + ) / ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name=~"GET /api/products.*index"}[3d])) + ) + labels: + openslo_slo_name: product-catalog-availability + openslo_spec_version: openslo/v1 + chaos_flag: productCatalogFailure + openslo_service_name: productcatalogservice + - record: openslo_sli_error_rate_7d + expr: | + 1 - ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name=~"GET /api/products.*index", status_code!="STATUS_CODE_ERROR"}[7d])) + ) / ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name=~"GET /api/products.*index"}[7d])) + ) + labels: + openslo_slo_name: product-catalog-availability + openslo_spec_version: openslo/v1 + chaos_flag: productCatalogFailure + openslo_service_name: productcatalogservice + - record: openslo_sli_error_rate_28d + expr: | + 1 - ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name=~"GET /api/products.*index", status_code!="STATUS_CODE_ERROR"}[28d])) + ) / ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name=~"GET /api/products.*index"}[28d])) + ) + labels: + openslo_slo_name: product-catalog-availability + openslo_spec_version: openslo/v1 + chaos_flag: productCatalogFailure + openslo_service_name: productcatalogservice + - record: openslo_sli_error_rate_30d + expr: | + 1 - ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name=~"GET /api/products.*index", status_code!="STATUS_CODE_ERROR"}[30d])) + ) / ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name=~"GET /api/products.*index"}[30d])) + ) + labels: + openslo_slo_name: product-catalog-availability + openslo_spec_version: openslo/v1 + chaos_flag: productCatalogFailure + openslo_service_name: productcatalogservice + - record: openslo_sli_event_rate_5m + expr: sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name=~"GET /api/products.*index"}[5m])) + labels: + openslo_slo_name: product-catalog-availability + openslo_spec_version: openslo/v1 + chaos_flag: productCatalogFailure + openslo_service_name: productcatalogservice + - record: openslo_sli_event_rate_30m + expr: sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name=~"GET /api/products.*index"}[30m])) + labels: + openslo_slo_name: product-catalog-availability + openslo_spec_version: openslo/v1 + chaos_flag: productCatalogFailure + openslo_service_name: productcatalogservice + - record: openslo_sli_event_rate_1h + expr: sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name=~"GET /api/products.*index"}[1h])) + labels: + openslo_slo_name: product-catalog-availability + openslo_spec_version: openslo/v1 + chaos_flag: productCatalogFailure + openslo_service_name: productcatalogservice + - record: openslo_sli_event_rate_3h + expr: sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name=~"GET /api/products.*index"}[3h])) + labels: + openslo_slo_name: product-catalog-availability + openslo_spec_version: openslo/v1 + chaos_flag: productCatalogFailure + openslo_service_name: productcatalogservice + - record: openslo_sli_event_rate_6h + expr: sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name=~"GET /api/products.*index"}[6h])) + labels: + openslo_slo_name: product-catalog-availability + openslo_spec_version: openslo/v1 + chaos_flag: productCatalogFailure + openslo_service_name: productcatalogservice + - record: openslo_sli_event_rate_1d + expr: sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name=~"GET /api/products.*index"}[1d])) + labels: + openslo_slo_name: product-catalog-availability + openslo_spec_version: openslo/v1 + chaos_flag: productCatalogFailure + openslo_service_name: productcatalogservice + - record: openslo_sli_event_rate_3d + expr: sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name=~"GET /api/products.*index"}[3d])) + labels: + openslo_slo_name: product-catalog-availability + openslo_spec_version: openslo/v1 + chaos_flag: productCatalogFailure + openslo_service_name: productcatalogservice + - record: openslo_sli_event_rate_7d + expr: sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name=~"GET /api/products.*index"}[7d])) + labels: + openslo_slo_name: product-catalog-availability + openslo_spec_version: openslo/v1 + chaos_flag: productCatalogFailure + openslo_service_name: productcatalogservice + - record: openslo_sli_event_rate_28d + expr: sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name=~"GET /api/products.*index"}[28d])) + labels: + openslo_slo_name: product-catalog-availability + openslo_spec_version: openslo/v1 + chaos_flag: productCatalogFailure + openslo_service_name: productcatalogservice + - record: openslo_sli_event_rate_30d + expr: sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name=~"GET /api/products.*index"}[30d])) + labels: + openslo_slo_name: product-catalog-availability + openslo_spec_version: openslo/v1 + chaos_flag: productCatalogFailure + openslo_service_name: productcatalogservice + - name: openslo-status-recordings-product-catalog-availability + rules: + - record: openslo_slo_status + expr: | + ( + (openslo_slo_current_burn_rate{openslo_slo_name="product-catalog-availability", openslo_spec_version="openslo/v1"} >= bool 14.4) + * 3 + ) + or + ( + (openslo_slo_current_burn_rate{openslo_slo_name="product-catalog-availability", openslo_spec_version="openslo/v1"} >= bool 6) + and + (openslo_slo_current_burn_rate{openslo_slo_name="product-catalog-availability", openslo_spec_version="openslo/v1"} < bool 14.4) + * 2 + ) + or + ( + (openslo_slo_current_burn_rate{openslo_slo_name="product-catalog-availability", openslo_spec_version="openslo/v1"} >= bool 1) + and + (openslo_slo_current_burn_rate{openslo_slo_name="product-catalog-availability", openslo_spec_version="openslo/v1"} < bool 6) + * 1 + ) + labels: + openslo_slo_name: product-catalog-availability + openslo_spec_version: openslo/v1 + chaos_flag: productCatalogFailure + openslo_service_name: productcatalogservice + - name: openslo-alerts-product-catalog-availability + rules: + - alert: ProductCatalogAvailabilityMultiWindowMultiBurnRatePage + expr: |- + ( + openslo_sli_error_rate_5m{openslo_slo_name="product-catalog-availability"} / (1 - openslo_slo_objective{openslo_slo_name="product-catalog-availability"}) >= 14.400000 + and + openslo_sli_error_rate_1h{openslo_slo_name="product-catalog-availability"} / (1 - openslo_slo_objective{openslo_slo_name="product-catalog-availability"}) >= 14.400000) + or + ( + openslo_sli_error_rate_30m{openslo_slo_name="product-catalog-availability"} / (1 - openslo_slo_objective{openslo_slo_name="product-catalog-availability"}) >= 6.000000 + and + openslo_sli_error_rate_6h{openslo_slo_name="product-catalog-availability"} / (1 - openslo_slo_objective{openslo_slo_name="product-catalog-availability"}) >= 6.000000) + for: 2m + labels: + openslo_alert_severity: page + openslo_notification_target: engineers + openslo_slo_name: product-catalog-availability + openslo_spec_version: openslo/v1 + chaos_flag: productCatalogFailure + openslo_service_name: productcatalogservice + annotations: + summary: "Page multi-window multi-burn rate alert for SLO product-catalog-availability" + description: "Threshold 14.4x and 6.0x over 5m and 1h and 30m and 6h" + - alert: ProductCatalogAvailabilityMultiWindowMultiBurnRateTicket + expr: |- + ( + openslo_sli_error_rate_2h{openslo_slo_name="product-catalog-availability"} / (1 - openslo_slo_objective{openslo_slo_name="product-catalog-availability"}) >= 3.000000 + and + openslo_sli_error_rate_1d{openslo_slo_name="product-catalog-availability"} / (1 - openslo_slo_objective{openslo_slo_name="product-catalog-availability"}) >= 3.000000) + or + ( + openslo_sli_error_rate_6h{openslo_slo_name="product-catalog-availability"} / (1 - openslo_slo_objective{openslo_slo_name="product-catalog-availability"}) >= 1.000000 + and + openslo_sli_error_rate_3d{openslo_slo_name="product-catalog-availability"} / (1 - openslo_slo_objective{openslo_slo_name="product-catalog-availability"}) >= 1.000000) + for: 15m + labels: + openslo_alert_severity: ticket + openslo_notification_target: engineers + openslo_slo_name: product-catalog-availability + openslo_spec_version: openslo/v1 + chaos_flag: productCatalogFailure + openslo_service_name: productcatalogservice + annotations: + summary: "Ticket multi-window multi-burn rate alert for SLO product-catalog-availability" + description: "Threshold 3.0x and 1.0x over 2h and 1d and 6h and 3d" diff --git a/examples/oteldemo/rules/recommendation-availability-rules.yaml b/examples/oteldemo/rules/recommendation-availability-rules.yaml new file mode 100644 index 0000000..6a2195c --- /dev/null +++ b/examples/oteldemo/rules/recommendation-availability-rules.yaml @@ -0,0 +1,324 @@ +groups: + - name: openslo-info-recordings-recommendation-availability + rules: + - record: openslo_slo_info + expr: vector(1) + labels: + openslo_slo_name: recommendation-availability + openslo_slo_description: "95% of recommendation fetches should succeed, measured via frontend HTTP SERVER `GET /api/recommendations` (target 0.95; trips on `recommendationCacheFailure` and `productCatalogFailure` cascade)." + openslo_spec_version: openslo/v1 + chaos_flag: recommendationCacheFailure + openslo_service_name: recommendationservice + - record: openslo_slo_objective + expr: vector(0.95) + labels: + openslo_slo_name: recommendation-availability + openslo_spec_version: openslo/v1 + chaos_flag: recommendationCacheFailure + openslo_service_name: recommendationservice + - record: openslo_slo_timewindow_days + expr: vector(30) + labels: + openslo_slo_name: recommendation-availability + openslo_spec_version: openslo/v1 + chaos_flag: recommendationCacheFailure + openslo_service_name: recommendationservice + - record: openslo_slo_error_budget + expr: vector(1- 0.95) + labels: + openslo_slo_name: recommendation-availability + openslo_spec_version: openslo/v1 + chaos_flag: recommendationCacheFailure + openslo_service_name: recommendationservice + - record: openslo_slo_current_burn_rate + expr: | + openslo_sli_error_rate_5m{openslo_slo_name="recommendation-availability", openslo_spec_version="openslo/v1"} + / on(openslo_slo_name, openslo_spec_version) group_left + openslo_slo_error_budget{openslo_slo_name="recommendation-availability", openslo_spec_version="openslo/v1"} + labels: + openslo_slo_name: recommendation-availability + openslo_spec_version: openslo/v1 + chaos_flag: recommendationCacheFailure + openslo_service_name: recommendationservice + - record: openslo_slo_period_burn_rate + expr: | + openslo_sli_error_rate_30d{openslo_slo_name="recommendation-availability", openslo_spec_version="openslo/v1"} + / on(openslo_slo_name, openslo_spec_version) group_left + openslo_slo_error_budget{openslo_slo_name="recommendation-availability", openslo_spec_version="openslo/v1"} + labels: + openslo_slo_name: recommendation-availability + openslo_spec_version: openslo/v1 + chaos_flag: recommendationCacheFailure + openslo_service_name: recommendationservice + - record: openslo_slo_period_error_budget_remaining + expr: clamp_min(1 - openslo_slo_period_burn_rate{openslo_slo_name="recommendation-availability", openslo_spec_version="openslo/v1"}, 0) + labels: + openslo_slo_name: recommendation-availability + openslo_spec_version: openslo/v1 + chaos_flag: recommendationCacheFailure + openslo_service_name: recommendationservice + - name: openslo-sli-recordings-recommendation-availability + rules: + - record: openslo_sli_error_rate_5m + expr: | + 1 - ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="GET /api/recommendations", status_code!="STATUS_CODE_ERROR"}[5m])) + ) / ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="GET /api/recommendations"}[5m])) + ) + labels: + openslo_slo_name: recommendation-availability + openslo_spec_version: openslo/v1 + chaos_flag: recommendationCacheFailure + openslo_service_name: recommendationservice + - record: openslo_sli_error_rate_30m + expr: | + 1 - ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="GET /api/recommendations", status_code!="STATUS_CODE_ERROR"}[30m])) + ) / ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="GET /api/recommendations"}[30m])) + ) + labels: + openslo_slo_name: recommendation-availability + openslo_spec_version: openslo/v1 + chaos_flag: recommendationCacheFailure + openslo_service_name: recommendationservice + - record: openslo_sli_error_rate_1h + expr: | + 1 - ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="GET /api/recommendations", status_code!="STATUS_CODE_ERROR"}[1h])) + ) / ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="GET /api/recommendations"}[1h])) + ) + labels: + openslo_slo_name: recommendation-availability + openslo_spec_version: openslo/v1 + chaos_flag: recommendationCacheFailure + openslo_service_name: recommendationservice + - record: openslo_sli_error_rate_3h + expr: | + 1 - ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="GET /api/recommendations", status_code!="STATUS_CODE_ERROR"}[3h])) + ) / ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="GET /api/recommendations"}[3h])) + ) + labels: + openslo_slo_name: recommendation-availability + openslo_spec_version: openslo/v1 + chaos_flag: recommendationCacheFailure + openslo_service_name: recommendationservice + - record: openslo_sli_error_rate_6h + expr: | + 1 - ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="GET /api/recommendations", status_code!="STATUS_CODE_ERROR"}[6h])) + ) / ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="GET /api/recommendations"}[6h])) + ) + labels: + openslo_slo_name: recommendation-availability + openslo_spec_version: openslo/v1 + chaos_flag: recommendationCacheFailure + openslo_service_name: recommendationservice + - record: openslo_sli_error_rate_1d + expr: | + 1 - ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="GET /api/recommendations", status_code!="STATUS_CODE_ERROR"}[1d])) + ) / ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="GET /api/recommendations"}[1d])) + ) + labels: + openslo_slo_name: recommendation-availability + openslo_spec_version: openslo/v1 + chaos_flag: recommendationCacheFailure + openslo_service_name: recommendationservice + - record: openslo_sli_error_rate_3d + expr: | + 1 - ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="GET /api/recommendations", status_code!="STATUS_CODE_ERROR"}[3d])) + ) / ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="GET /api/recommendations"}[3d])) + ) + labels: + openslo_slo_name: recommendation-availability + openslo_spec_version: openslo/v1 + chaos_flag: recommendationCacheFailure + openslo_service_name: recommendationservice + - record: openslo_sli_error_rate_7d + expr: | + 1 - ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="GET /api/recommendations", status_code!="STATUS_CODE_ERROR"}[7d])) + ) / ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="GET /api/recommendations"}[7d])) + ) + labels: + openslo_slo_name: recommendation-availability + openslo_spec_version: openslo/v1 + chaos_flag: recommendationCacheFailure + openslo_service_name: recommendationservice + - record: openslo_sli_error_rate_28d + expr: | + 1 - ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="GET /api/recommendations", status_code!="STATUS_CODE_ERROR"}[28d])) + ) / ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="GET /api/recommendations"}[28d])) + ) + labels: + openslo_slo_name: recommendation-availability + openslo_spec_version: openslo/v1 + chaos_flag: recommendationCacheFailure + openslo_service_name: recommendationservice + - record: openslo_sli_error_rate_30d + expr: | + 1 - ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="GET /api/recommendations", status_code!="STATUS_CODE_ERROR"}[30d])) + ) / ( + sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="GET /api/recommendations"}[30d])) + ) + labels: + openslo_slo_name: recommendation-availability + openslo_spec_version: openslo/v1 + chaos_flag: recommendationCacheFailure + openslo_service_name: recommendationservice + - record: openslo_sli_event_rate_5m + expr: sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="GET /api/recommendations"}[5m])) + labels: + openslo_slo_name: recommendation-availability + openslo_spec_version: openslo/v1 + chaos_flag: recommendationCacheFailure + openslo_service_name: recommendationservice + - record: openslo_sli_event_rate_30m + expr: sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="GET /api/recommendations"}[30m])) + labels: + openslo_slo_name: recommendation-availability + openslo_spec_version: openslo/v1 + chaos_flag: recommendationCacheFailure + openslo_service_name: recommendationservice + - record: openslo_sli_event_rate_1h + expr: sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="GET /api/recommendations"}[1h])) + labels: + openslo_slo_name: recommendation-availability + openslo_spec_version: openslo/v1 + chaos_flag: recommendationCacheFailure + openslo_service_name: recommendationservice + - record: openslo_sli_event_rate_3h + expr: sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="GET /api/recommendations"}[3h])) + labels: + openslo_slo_name: recommendation-availability + openslo_spec_version: openslo/v1 + chaos_flag: recommendationCacheFailure + openslo_service_name: recommendationservice + - record: openslo_sli_event_rate_6h + expr: sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="GET /api/recommendations"}[6h])) + labels: + openslo_slo_name: recommendation-availability + openslo_spec_version: openslo/v1 + chaos_flag: recommendationCacheFailure + openslo_service_name: recommendationservice + - record: openslo_sli_event_rate_1d + expr: sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="GET /api/recommendations"}[1d])) + labels: + openslo_slo_name: recommendation-availability + openslo_spec_version: openslo/v1 + chaos_flag: recommendationCacheFailure + openslo_service_name: recommendationservice + - record: openslo_sli_event_rate_3d + expr: sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="GET /api/recommendations"}[3d])) + labels: + openslo_slo_name: recommendation-availability + openslo_spec_version: openslo/v1 + chaos_flag: recommendationCacheFailure + openslo_service_name: recommendationservice + - record: openslo_sli_event_rate_7d + expr: sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="GET /api/recommendations"}[7d])) + labels: + openslo_slo_name: recommendation-availability + openslo_spec_version: openslo/v1 + chaos_flag: recommendationCacheFailure + openslo_service_name: recommendationservice + - record: openslo_sli_event_rate_28d + expr: sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="GET /api/recommendations"}[28d])) + labels: + openslo_slo_name: recommendation-availability + openslo_spec_version: openslo/v1 + chaos_flag: recommendationCacheFailure + openslo_service_name: recommendationservice + - record: openslo_sli_event_rate_30d + expr: sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="GET /api/recommendations"}[30d])) + labels: + openslo_slo_name: recommendation-availability + openslo_spec_version: openslo/v1 + chaos_flag: recommendationCacheFailure + openslo_service_name: recommendationservice + - name: openslo-status-recordings-recommendation-availability + rules: + - record: openslo_slo_status + expr: | + ( + (openslo_slo_current_burn_rate{openslo_slo_name="recommendation-availability", openslo_spec_version="openslo/v1"} >= bool 14.4) + * 3 + ) + or + ( + (openslo_slo_current_burn_rate{openslo_slo_name="recommendation-availability", openslo_spec_version="openslo/v1"} >= bool 6) + and + (openslo_slo_current_burn_rate{openslo_slo_name="recommendation-availability", openslo_spec_version="openslo/v1"} < bool 14.4) + * 2 + ) + or + ( + (openslo_slo_current_burn_rate{openslo_slo_name="recommendation-availability", openslo_spec_version="openslo/v1"} >= bool 1) + and + (openslo_slo_current_burn_rate{openslo_slo_name="recommendation-availability", openslo_spec_version="openslo/v1"} < bool 6) + * 1 + ) + labels: + openslo_slo_name: recommendation-availability + openslo_spec_version: openslo/v1 + chaos_flag: recommendationCacheFailure + openslo_service_name: recommendationservice + - name: openslo-alerts-recommendation-availability + rules: + - alert: RecommendationAvailabilityMultiWindowMultiBurnRatePage + expr: |- + ( + openslo_sli_error_rate_5m{openslo_slo_name="recommendation-availability"} / (1 - openslo_slo_objective{openslo_slo_name="recommendation-availability"}) >= 14.400000 + and + openslo_sli_error_rate_1h{openslo_slo_name="recommendation-availability"} / (1 - openslo_slo_objective{openslo_slo_name="recommendation-availability"}) >= 14.400000) + or + ( + openslo_sli_error_rate_30m{openslo_slo_name="recommendation-availability"} / (1 - openslo_slo_objective{openslo_slo_name="recommendation-availability"}) >= 6.000000 + and + openslo_sli_error_rate_6h{openslo_slo_name="recommendation-availability"} / (1 - openslo_slo_objective{openslo_slo_name="recommendation-availability"}) >= 6.000000) + for: 2m + labels: + openslo_alert_severity: page + openslo_notification_target: engineers + openslo_slo_name: recommendation-availability + openslo_spec_version: openslo/v1 + chaos_flag: recommendationCacheFailure + openslo_service_name: recommendationservice + annotations: + summary: "Page multi-window multi-burn rate alert for SLO recommendation-availability" + description: "Threshold 14.4x and 6.0x over 5m and 1h and 30m and 6h" + - alert: RecommendationAvailabilityMultiWindowMultiBurnRateTicket + expr: |- + ( + openslo_sli_error_rate_2h{openslo_slo_name="recommendation-availability"} / (1 - openslo_slo_objective{openslo_slo_name="recommendation-availability"}) >= 3.000000 + and + openslo_sli_error_rate_1d{openslo_slo_name="recommendation-availability"} / (1 - openslo_slo_objective{openslo_slo_name="recommendation-availability"}) >= 3.000000) + or + ( + openslo_sli_error_rate_6h{openslo_slo_name="recommendation-availability"} / (1 - openslo_slo_objective{openslo_slo_name="recommendation-availability"}) >= 1.000000 + and + openslo_sli_error_rate_3d{openslo_slo_name="recommendation-availability"} / (1 - openslo_slo_objective{openslo_slo_name="recommendation-availability"}) >= 1.000000) + for: 15m + labels: + openslo_alert_severity: ticket + openslo_notification_target: engineers + openslo_slo_name: recommendation-availability + openslo_spec_version: openslo/v1 + chaos_flag: recommendationCacheFailure + openslo_service_name: recommendationservice + annotations: + summary: "Ticket multi-window multi-burn rate alert for SLO recommendation-availability" + description: "Threshold 3.0x and 1.0x over 2h and 1d and 6h and 3d" diff --git a/examples/oteldemo/specs/ad-availability.yaml b/examples/oteldemo/specs/ad-availability.yaml new file mode 100644 index 0000000..033d358 --- /dev/null +++ b/examples/oteldemo/specs/ad-availability.yaml @@ -0,0 +1,43 @@ +apiVersion: openslo/v1 +kind: SLO +metadata: + name: ad-availability + displayName: Ad Service Availability + labels: + chaos_flag: adServiceFailure + service: ad +spec: + service: ad + description: "99% of users successfully fetching an ad via frontend HTTP SERVER `GET /api/data` (target 0.95; trips on `adServiceFailure`)." + indicator: + metadata: + name: ad-availability-sli + spec: + ratioMetric: + counter: true + good: + metricSource: + type: Prometheus + spec: + query: sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="GET /api/data", status_code!="STATUS_CODE_ERROR"}[{{.Window}}])) + total: + metricSource: + type: Prometheus + spec: + query: sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="GET /api/data"}[{{.Window}}])) + timeWindow: + - duration: 30d + isRolling: true + budgetingMethod: Occurrences + objectives: + - displayName: 99% of ad requests succeed (baseline-adjusted) + target: 0.99 + alertPolicies: + - alertPolicyRef: burnrate-page-fast-5m-policy + - alertPolicyRef: burnrate-page-fast-1h-policy + - alertPolicyRef: burnrate-page-slow-30m-policy + - alertPolicyRef: burnrate-page-slow-6h-policy + - alertPolicyRef: burnrate-ticket-fast-2h-policy + - alertPolicyRef: burnrate-ticket-fast-1d-policy + - alertPolicyRef: burnrate-ticket-slow-6h-policy + - alertPolicyRef: burnrate-ticket-slow-3d-policy diff --git a/examples/oteldemo/specs/ad-latency.yaml b/examples/oteldemo/specs/ad-latency.yaml new file mode 100644 index 0000000..4837fae --- /dev/null +++ b/examples/oteldemo/specs/ad-latency.yaml @@ -0,0 +1,62 @@ +apiVersion: openslo/v1 +kind: SLO +metadata: + name: ad-latency + displayName: Ad Service Latency + labels: + chaos_flag: adServiceHighCpu + service: ad +spec: + service: ad + description: "95% of `GET /api/data` ad fetches should complete under 2s, measured as the ratio of `le=\"2000\"` classic bucket counts to `_calls_total` on frontend HTTP SERVER (target 0.95; trips on `adHighCpu`, `adManualGc`, and `productCatalogFailure` cascade)." + indicator: + metadata: + name: ad-latency-sli + spec: + ratioMetric: + counter: true + good: + metricSource: + type: Prometheus + spec: + query: | + sum( + rate( + traces_span_metrics_duration_milliseconds_bucket{ + service_name="frontend", + span_kind="SPAN_KIND_SERVER", + span_name="GET /api/data", + le="2000" + }[{{.Window}}] + ) + ) + total: + metricSource: + type: Prometheus + spec: + query: | + sum( + rate( + traces_span_metrics_calls_total{ + service_name="frontend", + span_kind="SPAN_KIND_SERVER", + span_name="GET /api/data" + }[{{.Window}}] + ) + ) + timeWindow: + - duration: 30d + isRolling: true + budgetingMethod: Occurrences + objectives: + - displayName: 99% of GET /api/data ad fetches under 2s + target: 0.99 + alertPolicies: + - alertPolicyRef: burnrate-page-fast-5m-policy + - alertPolicyRef: burnrate-page-fast-1h-policy + - alertPolicyRef: burnrate-page-slow-30m-policy + - alertPolicyRef: burnrate-page-slow-6h-policy + - alertPolicyRef: burnrate-ticket-fast-2h-policy + - alertPolicyRef: burnrate-ticket-fast-1d-policy + - alertPolicyRef: burnrate-ticket-slow-6h-policy + - alertPolicyRef: burnrate-ticket-slow-3d-policy diff --git a/examples/oteldemo/specs/alert-conditions.yaml b/examples/oteldemo/specs/alert-conditions.yaml new file mode 100644 index 0000000..9919ef3 --- /dev/null +++ b/examples/oteldemo/specs/alert-conditions.yaml @@ -0,0 +1,103 @@ +apiVersion: openslo/v1 +kind: AlertCondition +metadata: + name: burnrate-page-fast-5m +spec: + severity: page + condition: + kind: multi-window-multi-burn-rate + op: gte + threshold: 14.4 + lookbackWindow: 5m + alertAfter: 2m +--- +apiVersion: openslo/v1 +kind: AlertCondition +metadata: + name: burnrate-page-fast-1h +spec: + severity: page + condition: + kind: multi-window-multi-burn-rate + op: gte + threshold: 14.4 + lookbackWindow: 1h + alertAfter: 2m +--- +apiVersion: openslo/v1 +kind: AlertCondition +metadata: + name: burnrate-page-slow-30m +spec: + severity: page + condition: + kind: multi-window-multi-burn-rate + op: gte + threshold: 6 + lookbackWindow: 30m + alertAfter: 5m +--- +apiVersion: openslo/v1 +kind: AlertCondition +metadata: + name: burnrate-page-slow-6h +spec: + severity: page + condition: + kind: multi-window-multi-burn-rate + op: gte + threshold: 6 + lookbackWindow: 6h + alertAfter: 5m +--- +apiVersion: openslo/v1 +kind: AlertCondition +metadata: + name: burnrate-ticket-fast-2h +spec: + severity: ticket + condition: + kind: multi-window-multi-burn-rate + op: gte + threshold: 3 + lookbackWindow: 2h + alertAfter: 15m +--- +apiVersion: openslo/v1 +kind: AlertCondition +metadata: + name: burnrate-ticket-fast-1d +spec: + severity: ticket + condition: + kind: multi-window-multi-burn-rate + op: gte + threshold: 3 + lookbackWindow: 1d + alertAfter: 15m +--- +apiVersion: openslo/v1 +kind: AlertCondition +metadata: + name: burnrate-ticket-slow-6h +spec: + severity: ticket + condition: + kind: multi-window-multi-burn-rate + op: gte + threshold: 1 + lookbackWindow: 6h + alertAfter: 30m +--- +apiVersion: openslo/v1 +kind: AlertCondition +metadata: + name: burnrate-ticket-slow-3d +spec: + severity: ticket + condition: + kind: multi-window-multi-burn-rate + op: gte + threshold: 1 + lookbackWindow: 3d + alertAfter: 30m diff --git a/examples/oteldemo/specs/alert-policies.yaml b/examples/oteldemo/specs/alert-policies.yaml new file mode 100644 index 0000000..dda762e --- /dev/null +++ b/examples/oteldemo/specs/alert-policies.yaml @@ -0,0 +1,95 @@ +apiVersion: openslo/v1 +kind: AlertPolicy +metadata: + name: burnrate-page-fast-5m-policy +spec: + description: Page tier fast (5m window) + alertWhenBreaching: true + conditions: + - conditionRef: burnrate-page-fast-5m + notificationTargets: + - targetRef: engineers +--- +apiVersion: openslo/v1 +kind: AlertPolicy +metadata: + name: burnrate-page-fast-1h-policy +spec: + description: Page tier fast (1h window) - pairs with 5m via AND in the alert + alertWhenBreaching: true + conditions: + - conditionRef: burnrate-page-fast-1h + notificationTargets: + - targetRef: engineers +--- +apiVersion: openslo/v1 +kind: AlertPolicy +metadata: + name: burnrate-page-slow-30m-policy +spec: + description: Page tier slow (30m window) + alertWhenBreaching: true + conditions: + - conditionRef: burnrate-page-slow-30m + notificationTargets: + - targetRef: engineers +--- +apiVersion: openslo/v1 +kind: AlertPolicy +metadata: + name: burnrate-page-slow-6h-policy +spec: + description: Page tier slow (6h window) - pairs with 30m via AND in the alert + alertWhenBreaching: true + conditions: + - conditionRef: burnrate-page-slow-6h + notificationTargets: + - targetRef: engineers +--- +apiVersion: openslo/v1 +kind: AlertPolicy +metadata: + name: burnrate-ticket-fast-2h-policy +spec: + description: Ticket tier fast (2h window) + alertWhenBreaching: true + conditions: + - conditionRef: burnrate-ticket-fast-2h + notificationTargets: + - targetRef: engineers +--- +apiVersion: openslo/v1 +kind: AlertPolicy +metadata: + name: burnrate-ticket-fast-1d-policy +spec: + description: Ticket tier fast (1d window) - pairs with 2h via AND in the alert + alertWhenBreaching: true + conditions: + - conditionRef: burnrate-ticket-fast-1d + notificationTargets: + - targetRef: engineers +--- +apiVersion: openslo/v1 +kind: AlertPolicy +metadata: + name: burnrate-ticket-slow-6h-policy +spec: + description: Ticket tier slow (6h window) + alertWhenBreaching: true + conditions: + - conditionRef: burnrate-ticket-slow-6h + notificationTargets: + - targetRef: engineers +--- +apiVersion: openslo/v1 +kind: AlertPolicy +metadata: + name: burnrate-ticket-slow-3d-policy +spec: + description: Ticket tier slow (3d window) - pairs with 6h via AND in the alert + alertWhenBreaching: true + conditions: + - conditionRef: burnrate-ticket-slow-3d + notificationTargets: + - targetRef: engineers diff --git a/examples/oteldemo/specs/cart-availability.yaml b/examples/oteldemo/specs/cart-availability.yaml new file mode 100644 index 0000000..9e93d8f --- /dev/null +++ b/examples/oteldemo/specs/cart-availability.yaml @@ -0,0 +1,43 @@ +apiVersion: openslo/v1 +kind: SLO +metadata: + name: cart-availability + displayName: Cart Service Availability + labels: + chaos_flag: cartServiceFailure + service: cart +spec: + service: cart + description: "95% of cart actions should succeed, measured via frontend HTTP SERVER `GET /api/cart` and `POST /api/cart` spans (target 0.95; trips on `cartServiceFailure`)." + indicator: + metadata: + name: cart-availability-sli + spec: + ratioMetric: + counter: true + good: + metricSource: + type: Prometheus + spec: + query: sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name=~"GET /api/cart|POST /api/cart", status_code!="STATUS_CODE_ERROR"}[{{.Window}}])) + total: + metricSource: + type: Prometheus + spec: + query: sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name=~"GET /api/cart|POST /api/cart"}[{{.Window}}])) + timeWindow: + - duration: 30d + isRolling: true + budgetingMethod: Occurrences + objectives: + - displayName: 99% of cart requests succeed (baseline-adjusted) + target: 0.99 + alertPolicies: + - alertPolicyRef: burnrate-page-fast-5m-policy + - alertPolicyRef: burnrate-page-fast-1h-policy + - alertPolicyRef: burnrate-page-slow-30m-policy + - alertPolicyRef: burnrate-page-slow-6h-policy + - alertPolicyRef: burnrate-ticket-fast-2h-policy + - alertPolicyRef: burnrate-ticket-fast-1d-policy + - alertPolicyRef: burnrate-ticket-slow-6h-policy + - alertPolicyRef: burnrate-ticket-slow-3d-policy diff --git a/examples/oteldemo/specs/frontend-availability.yaml b/examples/oteldemo/specs/frontend-availability.yaml new file mode 100644 index 0000000..8fd43a5 --- /dev/null +++ b/examples/oteldemo/specs/frontend-availability.yaml @@ -0,0 +1,42 @@ +apiVersion: openslo/v1 +kind: SLO +metadata: + name: frontend-availability + displayName: Frontend Service Availability + labels: + service: frontend +spec: + service: frontend + description: "95% of frontend HTTP SERVER calls should succeed (target 0.95)." + indicator: + metadata: + name: frontend-availability-sli + spec: + ratioMetric: + counter: true + good: + metricSource: + type: Prometheus + spec: + query: sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", status_code!="STATUS_CODE_ERROR"}[{{.Window}}])) + total: + metricSource: + type: Prometheus + spec: + query: sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER"}[{{.Window}}])) + timeWindow: + - duration: 30d + isRolling: true + budgetingMethod: Occurrences + objectives: + - displayName: 99% of frontend requests succeed (baseline-adjusted) + target: 0.99 + alertPolicies: + - alertPolicyRef: burnrate-page-fast-5m-policy + - alertPolicyRef: burnrate-page-fast-1h-policy + - alertPolicyRef: burnrate-page-slow-30m-policy + - alertPolicyRef: burnrate-page-slow-6h-policy + - alertPolicyRef: burnrate-ticket-fast-2h-policy + - alertPolicyRef: burnrate-ticket-fast-1d-policy + - alertPolicyRef: burnrate-ticket-slow-6h-policy + - alertPolicyRef: burnrate-ticket-slow-3d-policy diff --git a/examples/oteldemo/specs/image-loading-latency.yaml b/examples/oteldemo/specs/image-loading-latency.yaml new file mode 100644 index 0000000..e32f8ef --- /dev/null +++ b/examples/oteldemo/specs/image-loading-latency.yaml @@ -0,0 +1,62 @@ +apiVersion: openslo/v1 +kind: SLO +metadata: + name: image-loading-latency + displayName: Image Loading Latency + labels: + chaos_flag: imageSlowLoad + service: image-provider +spec: + service: image-provider + description: "95% of `image-provider` responses should complete under 2s, measured as the ratio of `le=\"2000\"` classic bucket counts to `_calls_total` on `service_name=\"image-provider\"` (target 0.95; trips on `imageSlowLoad`)." + indicator: + metadata: + name: image-loading-latency-sli + spec: + ratioMetric: + counter: true + good: + metricSource: + type: Prometheus + spec: + query: | + sum( + rate( + traces_span_metrics_duration_milliseconds_bucket{ + service_name="image-provider", + span_kind="SPAN_KIND_SERVER", + span_name="image-provider", + le="2000" + }[{{.Window}}] + ) + ) + total: + metricSource: + type: Prometheus + spec: + query: | + sum( + rate( + traces_span_metrics_calls_total{ + service_name="image-provider", + span_kind="SPAN_KIND_SERVER", + span_name="image-provider" + }[{{.Window}}] + ) + ) + timeWindow: + - duration: 30d + isRolling: true + budgetingMethod: Occurrences + objectives: + - displayName: 95% of image-provider responses under 2s + target: 0.95 + alertPolicies: + - alertPolicyRef: burnrate-page-fast-5m-policy + - alertPolicyRef: burnrate-page-fast-1h-policy + - alertPolicyRef: burnrate-page-slow-30m-policy + - alertPolicyRef: burnrate-page-slow-6h-policy + - alertPolicyRef: burnrate-ticket-fast-2h-policy + - alertPolicyRef: burnrate-ticket-fast-1d-policy + - alertPolicyRef: burnrate-ticket-slow-6h-policy + - alertPolicyRef: burnrate-ticket-slow-3d-policy diff --git a/examples/oteldemo/specs/notification-target-engineers.yaml b/examples/oteldemo/specs/notification-target-engineers.yaml new file mode 100644 index 0000000..33d9311 --- /dev/null +++ b/examples/oteldemo/specs/notification-target-engineers.yaml @@ -0,0 +1,7 @@ +apiVersion: openslo/v1 +kind: AlertNotificationTarget +metadata: + name: engineers +spec: + description: On-call engineers (demo target - routing not implemented) + target: engineers diff --git a/examples/oteldemo/specs/order-processing-latency.yaml b/examples/oteldemo/specs/order-processing-latency.yaml new file mode 100644 index 0000000..35f7493 --- /dev/null +++ b/examples/oteldemo/specs/order-processing-latency.yaml @@ -0,0 +1,62 @@ +apiVersion: openslo/v1 +kind: SLO +metadata: + name: order-processing-latency + displayName: Order Processing Latency + labels: + chaos_flag: kafkaQueueProblems + service: checkout +spec: + service: checkout + description: "95% of `POST /api/checkout` should complete under 60s, measured as the ratio of `le=\"60000\"` classic bucket counts to `_calls_total` on frontend HTTP SERVER (target 0.95; trips on `kafkaQueueProblems`)." + indicator: + metadata: + name: order-processing-latency-sli + spec: + ratioMetric: + counter: true + good: + metricSource: + type: Prometheus + spec: + query: | + sum( + rate( + traces_span_metrics_duration_milliseconds_bucket{ + service_name="frontend", + span_kind="SPAN_KIND_SERVER", + span_name="POST /api/checkout", + le="300000" + }[{{.Window}}] + ) + ) + total: + metricSource: + type: Prometheus + spec: + query: | + sum( + rate( + traces_span_metrics_calls_total{ + service_name="frontend", + span_kind="SPAN_KIND_SERVER", + span_name="POST /api/checkout" + }[{{.Window}}] + ) + ) + timeWindow: + - duration: 30d + isRolling: true + budgetingMethod: Occurrences + objectives: + - displayName: 95% of POST /api/checkout under 60s + target: 0.95 + alertPolicies: + - alertPolicyRef: burnrate-page-fast-5m-policy + - alertPolicyRef: burnrate-page-fast-1h-policy + - alertPolicyRef: burnrate-page-slow-30m-policy + - alertPolicyRef: burnrate-page-slow-6h-policy + - alertPolicyRef: burnrate-ticket-fast-2h-policy + - alertPolicyRef: burnrate-ticket-fast-1d-policy + - alertPolicyRef: burnrate-ticket-slow-6h-policy + - alertPolicyRef: burnrate-ticket-slow-3d-policy diff --git a/examples/oteldemo/specs/payment-unreachable.yaml b/examples/oteldemo/specs/payment-unreachable.yaml new file mode 100644 index 0000000..fbede2d --- /dev/null +++ b/examples/oteldemo/specs/payment-unreachable.yaml @@ -0,0 +1,43 @@ +apiVersion: openslo/v1 +kind: SLO +metadata: + name: payment-availability + displayName: Checkout Payment Availability + labels: + chaos_flag: paymentServiceUnreachable + service: checkout +spec: + service: checkout + description: "99% of `POST /api/checkout` should not encounter a payment-unreachable error, measured via frontend HTTP SERVER `POST /api/checkout` (target 0.99; trips on `paymentServiceUnreachable`)." + indicator: + metadata: + name: payment-availability-sli + spec: + ratioMetric: + counter: true + good: + metricSource: + type: Prometheus + spec: + query: sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="POST /api/checkout", status_code!="STATUS_CODE_ERROR"}[{{.Window}}])) + total: + metricSource: + type: Prometheus + spec: + query: sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="POST /api/checkout"}[{{.Window}}])) + timeWindow: + - duration: 30d + isRolling: true + budgetingMethod: Occurrences + objectives: + - displayName: 99% of checkout attempts avoid effective payment-availability (baseline-adjusted) + target: 0.99 + alertPolicies: + - alertPolicyRef: burnrate-page-fast-5m-policy + - alertPolicyRef: burnrate-page-fast-1h-policy + - alertPolicyRef: burnrate-page-slow-30m-policy + - alertPolicyRef: burnrate-page-slow-6h-policy + - alertPolicyRef: burnrate-ticket-fast-2h-policy + - alertPolicyRef: burnrate-ticket-fast-1d-policy + - alertPolicyRef: burnrate-ticket-slow-6h-policy + - alertPolicyRef: burnrate-ticket-slow-3d-policy diff --git a/examples/oteldemo/specs/post-order-email-availability.yaml b/examples/oteldemo/specs/post-order-email-availability.yaml new file mode 100644 index 0000000..fcc88e4 --- /dev/null +++ b/examples/oteldemo/specs/post-order-email-availability.yaml @@ -0,0 +1,43 @@ +apiVersion: openslo/v1 +kind: SLO +metadata: + name: post-order-email-availability + displayName: Post-Order Email Availability + labels: + chaos_flag: emailMemoryLeak + service: email +spec: + service: email + description: "95% of order-confirmation email posts should succeed, measured via email SPAN_KIND_SERVER `~POST /send_order_confirmation` (target 0.95; trips on `emailMemoryLeak`)." + indicator: + metadata: + name: post-order-email-sli + spec: + ratioMetric: + counter: true + good: + metricSource: + type: Prometheus + spec: + query: sum(rate(traces_span_metrics_calls_total{service_name="email", span_kind="SPAN_KIND_SERVER", span_name="POST /send_order_confirmation", status_code!="STATUS_CODE_ERROR"}[{{.Window}}])) + total: + metricSource: + type: Prometheus + spec: + query: sum(rate(traces_span_metrics_calls_total{service_name="email", span_kind="SPAN_KIND_SERVER", span_name="POST /send_order_confirmation"}[{{.Window}}])) + timeWindow: + - duration: 30d + isRolling: true + budgetingMethod: Occurrences + objectives: + - displayName: 95% of order-confirmation emails sent successfully (baseline-adjusted) + target: 0.95 + alertPolicies: + - alertPolicyRef: burnrate-page-fast-5m-policy + - alertPolicyRef: burnrate-page-fast-1h-policy + - alertPolicyRef: burnrate-page-slow-30m-policy + - alertPolicyRef: burnrate-page-slow-6h-policy + - alertPolicyRef: burnrate-ticket-fast-2h-policy + - alertPolicyRef: burnrate-ticket-fast-1d-policy + - alertPolicyRef: burnrate-ticket-slow-6h-policy + - alertPolicyRef: burnrate-ticket-slow-3d-policy diff --git a/examples/oteldemo/specs/post-order-email-latency.yaml b/examples/oteldemo/specs/post-order-email-latency.yaml new file mode 100644 index 0000000..f7ae535 --- /dev/null +++ b/examples/oteldemo/specs/post-order-email-latency.yaml @@ -0,0 +1,62 @@ +apiVersion: openslo/v1 +kind: SLO +metadata: + name: post-order-email-latency + displayName: Post-Order Email Latency + labels: + chaos_flag: emailMemoryLeak + service: email +spec: + service: email + description: "95% of order-confirmation email posts should complete under 30s, measured as the ratio of `le=\"30000\"` classic bucket counts to `_calls_total` on email SPAN_KIND_SERVER (target 0.95; trips on `emailMemoryLeak`)." + indicator: + metadata: + name: post-order-email-latency-sli + spec: + ratioMetric: + counter: true + good: + metricSource: + type: Prometheus + spec: + query: | + sum( + rate( + traces_span_metrics_duration_milliseconds_bucket{ + service_name="email", + span_kind="SPAN_KIND_SERVER", + span_name=~"POST /send_order_confirmation", + le="30000" + }[{{.Window}}] + ) + ) + total: + metricSource: + type: Prometheus + spec: + query: | + sum( + rate( + traces_span_metrics_calls_total{ + service_name="email", + span_kind="SPAN_KIND_SERVER", + span_name=~"POST /send_order_confirmation" + }[{{.Window}}] + ) + ) + timeWindow: + - duration: 30d + isRolling: true + budgetingMethod: Occurrences + objectives: + - displayName: 95% of POST /send_order_confirmation under 30s + target: 0.95 + alertPolicies: + - alertPolicyRef: burnrate-page-fast-5m-policy + - alertPolicyRef: burnrate-page-fast-1h-policy + - alertPolicyRef: burnrate-page-slow-30m-policy + - alertPolicyRef: burnrate-page-slow-6h-policy + - alertPolicyRef: burnrate-ticket-fast-2h-policy + - alertPolicyRef: burnrate-ticket-fast-1d-policy + - alertPolicyRef: burnrate-ticket-slow-6h-policy + - alertPolicyRef: burnrate-ticket-slow-3d-policy diff --git a/examples/oteldemo/specs/product-catalog-availability.yaml b/examples/oteldemo/specs/product-catalog-availability.yaml new file mode 100644 index 0000000..ae90b15 --- /dev/null +++ b/examples/oteldemo/specs/product-catalog-availability.yaml @@ -0,0 +1,42 @@ +apiVersion: openslo/v1 +kind: SLO +metadata: + name: product-catalog-availability + displayName: Product Catalog Service Availability + labels: + chaos_flag: productCatalogFailure +spec: + service: productcatalogservice + description: "95% of product browsing should succeed, measured via frontend HTTP SERVER `GET /api/products.*index` spans (target 0.95; trips on `productCatalogFailure`)." + indicator: + metadata: + name: product-catalog-availability-sli + spec: + ratioMetric: + counter: true + good: + metricSource: + type: Prometheus + spec: + query: sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name=~"GET /api/products.*index", status_code!="STATUS_CODE_ERROR"}[{{.Window}}])) + total: + metricSource: + type: Prometheus + spec: + query: sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name=~"GET /api/products.*index"}[{{.Window}}])) + timeWindow: + - duration: 30d + isRolling: true + budgetingMethod: Occurrences + objectives: + - displayName: 95% of product-catalog requests succeed (baseline-adjusted) + target: 0.95 + alertPolicies: + - alertPolicyRef: burnrate-page-fast-5m-policy + - alertPolicyRef: burnrate-page-fast-1h-policy + - alertPolicyRef: burnrate-page-slow-30m-policy + - alertPolicyRef: burnrate-page-slow-6h-policy + - alertPolicyRef: burnrate-ticket-fast-2h-policy + - alertPolicyRef: burnrate-ticket-fast-1d-policy + - alertPolicyRef: burnrate-ticket-slow-6h-policy + - alertPolicyRef: burnrate-ticket-slow-3d-policy diff --git a/examples/oteldemo/specs/recommendation-availability.yaml b/examples/oteldemo/specs/recommendation-availability.yaml new file mode 100644 index 0000000..e2add92 --- /dev/null +++ b/examples/oteldemo/specs/recommendation-availability.yaml @@ -0,0 +1,42 @@ +apiVersion: openslo/v1 +kind: SLO +metadata: + name: recommendation-availability + displayName: Recommendation Service Availability + labels: + chaos_flag: recommendationCacheFailure +spec: + service: recommendationservice + description: "95% of recommendation fetches should succeed, measured via frontend HTTP SERVER `GET /api/recommendations` (target 0.95; trips on `recommendationCacheFailure` and `productCatalogFailure` cascade)." + indicator: + metadata: + name: recommendation-availability-sli + spec: + ratioMetric: + counter: true + good: + metricSource: + type: Prometheus + spec: + query: sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="GET /api/recommendations", status_code!="STATUS_CODE_ERROR"}[{{.Window}}])) + total: + metricSource: + type: Prometheus + spec: + query: sum(rate(traces_span_metrics_calls_total{service_name="frontend", span_kind="SPAN_KIND_SERVER", span_name="GET /api/recommendations"}[{{.Window}}])) + timeWindow: + - duration: 30d + isRolling: true + budgetingMethod: Occurrences + objectives: + - displayName: 95% of recommendation requests succeed (baseline-adjusted) + target: 0.95 + alertPolicies: + - alertPolicyRef: burnrate-page-fast-5m-policy + - alertPolicyRef: burnrate-page-fast-1h-policy + - alertPolicyRef: burnrate-page-slow-30m-policy + - alertPolicyRef: burnrate-page-slow-6h-policy + - alertPolicyRef: burnrate-ticket-fast-2h-policy + - alertPolicyRef: burnrate-ticket-fast-1d-policy + - alertPolicyRef: burnrate-ticket-slow-6h-policy + - alertPolicyRef: burnrate-ticket-slow-3d-policy diff --git a/examples/oteldemo/specs/services.yaml b/examples/oteldemo/specs/services.yaml new file mode 100644 index 0000000..5768032 --- /dev/null +++ b/examples/oteldemo/specs/services.yaml @@ -0,0 +1,97 @@ +apiVersion: openslo/v1 +kind: Service +metadata: + name: ad + displayName: Ad Service +spec: + description: > + Serves contextual advertisements on product pages. Generates ~10% of + frontend revenue via Google Ads integration. +--- +apiVersion: openslo/v1 +kind: Service +metadata: + name: cart + displayName: Cart Service +spec: + description: > + Manages shopping cart state. EmptyCart is the most user-visible call - + failure directly blocks users from starting a new session. +--- +apiVersion: openslo/v1 +kind: Service +metadata: + name: checkout + displayName: Checkout Service +spec: + description: > + Coordinates multi-service checkout flow via gRPC. Orchestrates cart, + payment, shipping, and order confirmation downstream via Kafka. +--- +apiVersion: openslo/v1 +kind: Service +metadata: + name: email + displayName: Email Service +spec: + description: > + Sends order confirmation emails. Loss here degrades trust without + blocking purchases. +--- +apiVersion: openslo/v1 +kind: Service +metadata: + name: frontend + displayName: Frontend +spec: + description: > + React-based product catalog UI. Load-bearing for user acquisition. +--- +apiVersion: openslo/v1 +kind: Service +metadata: + name: frontend-proxy + displayName: Frontend Proxy +spec: + description: > + Envoy edge proxy serving static frontend assets and product images. +--- +apiVersion: openslo/v1 +kind: Service +metadata: + name: image-provider + displayName: Image Provider +spec: + description: > + Go service that serves /image-provider/... images for product pages. + Latency here directly impacts page-load times. +--- +apiVersion: openslo/v1 +kind: Service +metadata: + name: paymentservice + displayName: Payment Service +spec: + description: > + Authoritative payment processor. Direct revenue impact - failures + cancel checkouts. +--- +apiVersion: openslo/v1 +kind: Service +metadata: + name: productcatalogservice + displayName: Product Catalog Service +spec: + description: > + Product data store. Cached after first request - failures hit only + the product detail page on cold load. +--- +apiVersion: openslo/v1 +kind: Service +metadata: + name: recommendationservice + displayName: Recommendation Service +spec: + description: > + ML-driven "you may also like" product recommendations. Latent importance - + failures degrade but don't block core flow. diff --git a/go.mod b/go.mod index b1b7248..a5a7e45 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/OpenSLO/go-sdk v0.9.2 github.com/mdobak/go-xerrors v1.0.1 github.com/pkg/errors v0.9.1 + github.com/sebdah/goldie/v2 v2.8.0 github.com/spf13/cobra v1.8.1 github.com/stretchr/testify v1.9.0 ) @@ -23,6 +24,7 @@ require ( github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/nobl9/govy v0.26.0 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/sergi/go-diff v1.0.0 // indirect github.com/shopspring/decimal v1.4.0 // indirect github.com/spf13/cast v1.7.0 // indirect github.com/spf13/pflag v1.0.5 // indirect diff --git a/go.sum b/go.sum index 7b517eb..7a3a100 100644 --- a/go.sum +++ b/go.sum @@ -9,6 +9,7 @@ github.com/Masterminds/sprig/v3 v3.3.0/go.mod h1:Zy1iXRYNqNLUolqCpL4uhk6SHUMAOSC github.com/OpenSLO/go-sdk v0.9.2 h1:pc6b4sWImIJreEDGNPbfplMbOZL5LwOkoRq2IULShRc= github.com/OpenSLO/go-sdk v0.9.2/go.mod h1:s4PEBTqO5O2u5SeVFQZyLHE9RzCZgGNxTt43FwuqvCo= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= @@ -36,13 +37,19 @@ github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zx github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/nobl9/govy v0.26.0 h1:pjHXreO+3Rl+Uz6/rFX7L54zH4LW/SaTjb6YX3GQGs4= github.com/nobl9/govy v0.26.0/go.mod h1:fExiIzXORe0ktwg2bWasOAmCZtEFMQkR4PpgzhHcZSA= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sebdah/goldie/v2 v2.8.0 h1:dZb9wR8q5++oplmEiJT+U/5KyotVD+HNGCAc5gNr8rc= +github.com/sebdah/goldie/v2 v2.8.0/go.mod h1:oZ9fp0+se1eapSRjfYbsV/0Hqhbuu3bJVvKI/NNtssI= +github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= +github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= github.com/spf13/cast v1.7.0 h1:ntdiHjuueXFgm5nzDRdOS4yfT43P5Fnud6DH50rz/7w= @@ -51,6 +58,8 @@ github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= diff --git a/internal/feature/feature.go b/internal/feature/feature.go index bdb7be8..7c87385 100644 --- a/internal/feature/feature.go +++ b/internal/feature/feature.go @@ -1,14 +1,16 @@ +// Package feature holds cross-cutting flags and small inputs that shape +// opensloctl generator behavior. package feature -// this package holds informations about features - -// Multi Dimensional SLIs -// This allows multiple slis to spawn out of a single sli definition -// There are two annotaitons needed to support this feature. - const ( - MULTI_DIMENSIONAL_SLI_DIMENSIONS = "multi-dimensional-sli.openslo.com/dimensions" // This tells us the dimensions we want to look into - MULTI_DIMENSIONAL_SLI_LABEL = "multi-dimensional-sli.openslo.com/label" // label annotation tells which prom label has the dimension values + // MULTI_DIMENSIONAL_SLI_DIMENSIONS - annotation listing the OpenSlo + // SLO dimension values to expand into separate Prometheus recording + // series. + MULTI_DIMENSIONAL_SLI_DIMENSIONS = "multi-dimensional-sli.openslo.com/dimensions" + + // MULTI_DIMENSIONAL_SLI_LABEL - annotation naming the Prometheus label + // that carries each dimension value (joined into openslo_slo_id later). + MULTI_DIMENSIONAL_SLI_LABEL = "multi-dimensional-sli.openslo.com/label" ) var ( diff --git a/internal/generator/generator.go b/internal/generator/generator.go index 6e7d60a..d1a141a 100644 --- a/internal/generator/generator.go +++ b/internal/generator/generator.go @@ -12,4 +12,5 @@ func (f *GeneratedFile) Bytes() []byte { type Generator interface { Generate(outputDirectory string) error + Validate() error } diff --git a/internal/generator/prometheusgenerator/labels.go b/internal/generator/prometheusgenerator/labels.go new file mode 100644 index 0000000..a597da6 --- /dev/null +++ b/internal/generator/prometheusgenerator/labels.go @@ -0,0 +1,38 @@ +package prometheusgenerator + +import ( + "fmt" + "strings" + + v1 "github.com/OpenSLO/go-sdk/pkg/openslo/v1" +) + +// promLabelsFromOpenSlo converts OpenSlo metadata.labels into a flat +// map[string]string of Prometheus labels. +// +// Rules: +// - Each OpenSlo Label must contain exactly one value. Lists with two or +// more entries are rejected (multi-value labels are not supported). +// Empty lists are skipped silently. +// - The single-string YAML form (`service: ad`) is accepted - the SDK +// normalizes it to a one-element slice. +// - Label names must match Prometheus's label name grammar +// ([a-zA-Z_][a-zA-Z0-9_]*). Hyphens are rejected because they are +// invalid Prometheus label characters, not silently rewritten. +func promLabelsFromOpenSlo(m v1.Labels) (map[string]string, error) { + out := make(map[string]string, len(m)) + for k, l := range m { + if strings.Contains(k, "-") { + return nil, fmt.Errorf("prom label name %q contains a hyphen; Prometheus label names must match [a-zA-Z_][a-zA-Z0-9_]* (use underscores)", k) + } + switch len(l) { + case 0: + continue + case 1: + out[k] = l[0] + default: + return nil, fmt.Errorf("prom label %q has %d values; multi-value labels are not supported (use a single string or a one-element list)", k, len(l)) + } + } + return out, nil +} diff --git a/internal/generator/prometheusgenerator/labels_test.go b/internal/generator/prometheusgenerator/labels_test.go new file mode 100644 index 0000000..7810a14 --- /dev/null +++ b/internal/generator/prometheusgenerator/labels_test.go @@ -0,0 +1,93 @@ +package prometheusgenerator + +import ( + "testing" + + v1 "github.com/OpenSLO/go-sdk/pkg/openslo/v1" + "github.com/stretchr/testify/assert" +) + +// TestPromLabelsFromOpenSlo covers the one-value-per-label invariant. The +// validator rejects multi-value Label entries (more than one element) and +// silently drops empty ones; single-string and one-element list forms +// share the same pass-through behavior. +func TestPromLabelsFromOpenSlo(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + input v1.Labels + want map[string]string + wantErr bool + }{ + { + name: "empty map yields nil", + input: v1.Labels{}, + want: map[string]string{}, + }, + { + name: "single string form (SDK crops to one element)", + input: v1.Labels{ + "service": []string{"ad"}, + }, + want: map[string]string{"service": "ad"}, + }, + { + name: "one-element list accepted", + input: v1.Labels{ + "team": []string{"platform"}, + }, + want: map[string]string{"team": "platform"}, + }, + { + name: "empty list dropped silently", + input: v1.Labels{ + "region": []string{}, + }, + want: map[string]string{}, + }, + { + name: "two-value list rejected", + input: v1.Labels{ + "region": []string{"us", "eu"}, + }, + wantErr: true, + }, + { + name: "mixed valid + invalid rejects the whole SLO", + input: v1.Labels{ + "team": []string{"platform"}, + "region": []string{"us", "eu", "ap"}, + }, + wantErr: true, + }, + { + name: "hyphenated label name rejected", + input: v1.Labels{ + "chaos-flag": []string{"kafkaQueueProblems"}, + }, + wantErr: true, + }, + { + name: "underscored label name accepted", + input: v1.Labels{ + "chaos_flag": []string{"kafkaQueueProblems"}, + }, + want: map[string]string{"chaos_flag": "kafkaQueueProblems"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got, err := promLabelsFromOpenSlo(tt.input) + if tt.wantErr { + assert.Error(t, err) + return + } + assert.NoError(t, err) + assert.Equal(t, tt.want, got) + }) + } +} diff --git a/internal/generator/prometheusgenerator/prometheus.go b/internal/generator/prometheusgenerator/prometheus.go index f30b0eb..9dc8882 100644 --- a/internal/generator/prometheusgenerator/prometheus.go +++ b/internal/generator/prometheusgenerator/prometheus.go @@ -4,6 +4,7 @@ import ( "bytes" "fmt" "log/slog" + "math" "os" "path" "regexp" @@ -12,6 +13,8 @@ import ( "text/template" "github.com/Masterminds/sprig/v3" + v1 "github.com/OpenSLO/go-sdk/pkg/openslo/v1" + "github.com/mdobak/go-xerrors" "github.com/pkg/errors" "github.com/thisisibrahimd/opensloctl/internal/feature" "github.com/thisisibrahimd/opensloctl/internal/generator" @@ -20,8 +23,7 @@ import ( ) const ( - RECORDING_RULES_SUFFIX = "-recording-rules.yaml" - ALERT_RULES_SUFFIX = "-alert-rules.yaml" + RULES_SUFFIX = "-rules.yaml" ) var ( @@ -30,16 +32,38 @@ var ( // daysRegex = regexp.MustCompile("([0-9]+)d") ) +// PrometheusGenerator renders OpenSlo SLOs as Prometheus recording rules +// (and burn-rate alert rules when AlertPolicies exist). It implements the +// generator.Generator interface. type PrometheusGenerator struct { specs *specstore.OpenSLOSpecs } +// NewPrometheusGenerator constructs a PrometheusGenerator bound to specs. +// All SLOs in specs must have already passed specstore validation. func NewPrometheusGenerator(specs *specstore.OpenSLOSpecs) generator.Generator { return &PrometheusGenerator{ specs: specs, } } +// Validate runs every generation-side check (indicator resolution, metric +// source type, label grammar, template execution, alert group structure) +// without writing any files. It reuses createGeneratedFiles - which returns +// in-memory GeneratedFile records - and discards the rendered output. Any +// underlying error is returned to the caller untouched. +func (g *PrometheusGenerator) Validate() error { + _, err := g.createGeneratedFiles() + return err +} + +// Generate renders each SLO in g.specs to a single Prometheus rules file +// ("-rules.yaml"). When the SLO references satisfied AlertPolicies, +// the file contains both recording rules and alert rules rendered by the +// unified template; otherwise only recording rules are emitted under the +// same filename. Files are written to outputDirectory with 0o664 permissions. +// +// outputDirectory must not be empty. Generation aborts on the first error. func (g *PrometheusGenerator) Generate(outputDirectory string) error { if outputDirectory == "" { return errors.New("output directory can not be empty") @@ -63,27 +87,68 @@ func (g *PrometheusGenerator) Generate(outputDirectory string) error { return nil } +// createGeneratedFiles renders SLOs to in-memory GeneratedFile records +// without writing to disk. Each SLO produces one rules file named +// "-rules.yaml" - recording rules always; alert rules added when +// at least one AlertPolicy resolves to a satisfied condition. +// +// The SLI source is unpacked per kind: +// - RatioMetric.counter: good / total PromQL +// - ThresholdMetric: raw PromQL from the metric source +// +// Query templates substitute {{.Window}} with each multi-window duration +// (5m, 30m, 1h, 3h, 6h, 1d, 3d, 7d, 28d, 30d) at generate time. The +// period burn-rate meta rules use "30d" by default - change PeriodWindow +// in templates.TemplateData to wire it differently. func (g *PrometheusGenerator) createGeneratedFiles() ([]*generator.GeneratedFile, error) { var generatedPrometheusRuleFiles []*generator.GeneratedFile // loop through slos for _, slo := range g.specs.V1.SLOs { slog.Info("generating prometheus recording rule", "slo", slo.Metadata.Name) - // TODO: support indicator ref - // Ensure indicator is present - if slo.Spec.Indicator == nil { - return nil, fmt.Errorf("indicator is required for slo: %s", slo.Metadata.Name) + // Resolve the indicator the SLO will be evaluated against. + // Inline indicator (`spec.indicator`) takes precedence; otherwise fall + // back to `spec.indicatorRef` and look the SLI up in g.specs. specstore + // already validated the ref resolves at load time. + indicator := slo.Spec.Indicator + if indicator == nil && slo.Spec.IndicatorRef != nil && *slo.Spec.IndicatorRef != "" { + ref := *slo.Spec.IndicatorRef + sli, ok := g.specs.V1.SLIs[ref] + if !ok { + return nil, fmt.Errorf("SLO %q references SLI %q which is not loaded", slo.Metadata.Name, ref) + } + indicator = &v1.SLOIndicatorInline{ + Metadata: sli.Metadata, + Spec: sli.Spec, + } + } + if indicator == nil { + return nil, fmt.Errorf("SLO %q has neither spec.indicator nor spec.indicatorRef set", slo.Metadata.Name) } // Pick and generate prom query var promQuery string - if slo.Spec.Indicator.Spec.RatioMetric != nil { - return nil, fmt.Errorf("ratio metrics are not supported") + var hasEventRate bool + var eventRateQuery string + if indicator.Spec.RatioMetric != nil { + goodSource := indicator.Spec.RatioMetric.Good.MetricSource + totalSource := indicator.Spec.RatioMetric.Total.MetricSource + if goodSource.Type != "Prometheus" || totalSource.Type != "Prometheus" { + slog.Warn("RatioMetric source is not Prometheus type", "slo", slo.Metadata.Name) + } + goodQuery := goodSource.Spec["query"].(string) + totalQuery := totalSource.Spec["query"].(string) + // The SLI metric is named `openslo_sli_error_rate_*` so it must hold + // the actual error rate (bad / total = 1 - good/total). Burn rate + // formulas downstream divide this by error_budget, so emitting the + // success rate here would invert the meaning (a healthy service + // would register as burning thousands of times its budget). + promQuery = fmt.Sprintf("1 - (\n%s\n) / (\n%s\n)", goodQuery, totalQuery) + hasEventRate = true + eventRateQuery = totalQuery } else { - metricSource := slo.Spec.Indicator.Spec.ThresholdMetric.MetricSource - if metricSource.MetricSourceRef != "" { - slog.Warn("SLI uses metricSourceRef, expected inline Prometheus type", "slo", slo.Metadata.Name, "ref", metricSource.MetricSourceRef) - } else if metricSource.Type != "Prometheus" { + metricSource := indicator.Spec.ThresholdMetric.MetricSource + if metricSource.Type != "Prometheus" { slog.Warn("SLI metric source is not Prometheus type", "slo", slo.Metadata.Name, "type", metricSource.Type) } promQuery = metricSource.Spec["query"].(string) @@ -95,6 +160,7 @@ func (g *PrometheusGenerator) createGeneratedFiles() ([]*generator.GeneratedFile // template out the window variable in prom query var windowedPromQueries []*templates.WindowedPrometheusQuery + var windowedEventRateQueries []*templates.WindowedPrometheusQuery for _, window := range templates.Windows { windowData := &templates.WindowData{Window: window} @@ -112,61 +178,131 @@ func (g *PrometheusGenerator) createGeneratedFiles() ([]*generator.GeneratedFile Query: windowedPromQueryBuffer.String(), } windowedPromQueries = append(windowedPromQueries, windowedPromQuery) + + // For RatioMetric SLIs, also build the event-rate recording rule + // query per window using the raw total counter query (no + // subtraction). Single line so no block scalar handling needed. + if hasEventRate { + rateTmpl := template.Must(template.New("event-rate-query").Parse(eventRateQuery)) + var rateBuffer bytes.Buffer + if err := rateTmpl.Execute(&rateBuffer, windowData); err != nil { + return nil, fmt.Errorf("SLO %q: event rate template: %w", slo.Metadata.Name, err) + } + windowedEventRateQueries = append(windowedEventRateQueries, &templates.WindowedPrometheusQuery{ + Window: window, + Query: rateBuffer.String(), + }) + } } - // extract days in time window - numberOfDays := numberRegex.FindString(slo.Spec.TimeWindow[0].Duration.String()) +// extract days in time window + numberOfDays := numberRegex.FindString(slo.Spec.TimeWindow[0].Duration.String()) + + // Merge OpenSlo metadata.labels into the Prometheus label set. + // Core labels (openslo_slo_name, openslo_spec_version, optional + // openslo_service_name) are stamped by the template; passthrough + // labels are added here after one-value validation. + extraLabels, err := promLabelsFromOpenSlo(slo.Metadata.Labels) + if err != nil { + return nil, fmt.Errorf("SLO %q: %w", slo.Metadata.Name, err) + } + if slo.Spec.Service != "" { + if _, dup := extraLabels["openslo_service_name"]; !dup { + extraLabels["openslo_service_name"] = slo.Spec.Service + } + } + + // Pick the period window for meta burn rules. Prefer exact match + // against the multi-window set; fall back to the first window that + // covers the SLO's full time window. If no candidate exists, leave + // blank and the template will emit only current-burn-rate. + periodWindow := "" + if len(windowedPromQueries) > 0 { + want := slo.Spec.TimeWindow[0].Duration.String() + for _, q := range windowedPromQueries { + if q.Window == want { + periodWindow = q.Window + break + } + } + if periodWindow == "" { + for _, q := range windowedPromQueries { + if q.Window == "30d" || q.Window == "28d" { + periodWindow = q.Window + break + } + } + } + } + + // template out prom rules + descLabel := foldSloDescription(slo.Spec.Description) - // template out prom rules tpldData := templates.TemplateData{ SloName: slo.Metadata.Name, + Description: descLabel, OpensloVersion: string(slo.APIVersion), PrometheusQuery: windowedPromQueries[0].Query, WindowedPrometheusQueries: windowedPromQueries, - Objective: strconv.FormatFloat(*slo.Spec.Objectives[0].Target, 'f', -1, 64), + WindowedEventRateQueries: windowedEventRateQueries, + HasEventRate: hasEventRate, + Objective: objectiveFloat(slo.Spec.Objectives[0]), IsMulti: multiFeatureEnabled, MultiDimensionalLabel: multiDimSliLabel, TimeWindowDays: numberOfDays, + PeriodWindow: periodWindow, + ExtraLabels: extraLabels, AlertGroups: g.buildAlertGroups(slo.Metadata.Name), + StatusThresholds: buildStatusThresholds(slo.Metadata.Annotations), } - prometheusTemplate := template.Must(template.New("prometheus-recording-rules").Funcs(sprig.FuncMap()).Parse(templates.PrometheusRecordingRuleTemplate)) - var generatedRecordingRules bytes.Buffer - err := prometheusTemplate.Execute(&generatedRecordingRules, tpldData) - if err != nil { - return nil, fmt.Errorf("unable to execute template") + prometheusTemplate := template.Must(template.New("prometheus-rules").Funcs(sprig.FuncMap()).Parse(templates.PrometheusRulesTemplate)) + var generated bytes.Buffer + if err := prometheusTemplate.Execute(&generated, tpldData); err != nil { + return nil, fmt.Errorf("unable to execute template: %w", err) } - // create generated file struct - filename := slo.Metadata.Name + RECORDING_RULES_SUFFIX + // File name is always -rules.yaml. The unified template + // produces only recording rules when AlertGroups is empty, and + // adds an openslo-alerts- group when alerts are present. + filename := slo.Metadata.Name + RULES_SUFFIX + generatedPrometheusRuleFile := &generator.GeneratedFile{ Path: filename, - Data: generatedRecordingRules.String(), + Data: generated.String(), } generatedPrometheusRuleFiles = append(generatedPrometheusRuleFiles, generatedPrometheusRuleFile) - - // generate alert rules if alert groups exist - if len(tpldData.AlertGroups) > 0 { - alertTemplate := template.Must(template.New("prometheus-alert-rules").Funcs(sprig.FuncMap()).Parse(templates.PrometheusAlertRuleTemplate)) - var generatedAlertRules bytes.Buffer - err := alertTemplate.Execute(&generatedAlertRules, tpldData) - if err != nil { - return nil, fmt.Errorf("unable to execute alert template") - } - - alertFilename := slo.Metadata.Name + ALERT_RULES_SUFFIX - generatedAlertRuleFile := &generator.GeneratedFile{ - Path: alertFilename, - Data: generatedAlertRules.String(), - } - generatedPrometheusRuleFiles = append(generatedPrometheusRuleFiles, generatedAlertRuleFile) - } } return generatedPrometheusRuleFiles, nil } +// buildAlertGroups resolves an SLO's AlertPolicy references into a map +// of severity → AlertGroup, dispatching on each condition's kind to emit +// the correct Prometheus alert expression and tier structure. +// +// Kind matrix (see pkg/specstore for full kind semantics): +// +// - error-rate: raw SLI error rate vs absolute threshold. +// 1 condition per severity; tier = condition name. +// - burn-rate: single burn rate multiplier; 1 condition per +// severity; tier = condition name. +// - multi-burn-rate: multiple burn-rate conditions OR-ed per +// severity; each condition is its own tier. +// - multi-window-multi-burn-rate: short+long window pairs AND-ed within a +// tier (derived from the condition name by stripping the trailing +// "-" suffix) and OR-ed across tiers. Legacy default +// that retains the prior single-kind="burnrate" tiering behavior. +// +// Unknown policy/condition refs are logged and skipped. Conditions not in +// the four supported kinds are skipped (ValidateRefs already rejects them +// at load time, so reaching here implies a misconfiguration that should be +// surfaced loudly via slog.Error rather than as a panic). +// +// Each group's For field comes from the shortest AlertAfter across its +// conditions; thresholds and lookbacks are joined " and "-separated for +// display in annotations. func (g *PrometheusGenerator) buildAlertGroups(sloName string) map[string]templates.AlertGroup { if g.specs == nil { return nil @@ -180,7 +316,18 @@ func (g *PrometheusGenerator) buildAlertGroups(sloName string) map[string]templa return nil } - alertGroups := make(map[string]templates.AlertGroup) + // Track per-severity: ordered tier names → slice of conditions, plus aggregates. + type severityState struct { + kind specstore.AlertConditionKind + tierOrder []string + tierIndex map[string]int + tiers []templates.AlertTier + thresholds string + lookbacks string + for_ string // shortest alertAfter across all conditions in this severity + notificationTarget string // resolved spec.target of the first AlertPolicy + } + stateBySev := make(map[string]*severityState) for _, alertPolicyRef := range sloObj.Spec.AlertPolicies { polRef := alertPolicyRef.AlertPolicyRef @@ -190,6 +337,19 @@ func (g *PrometheusGenerator) buildAlertGroups(sloName string) map[string]templa continue } + // Resolve the single notification target this AlertPolicy carries. + // ValidateRefs already enforced len(notificationTargets) <= 1, and + // resolved every targetRef at load time. We collect the resolved + // spec.target string here so the alert rule can carry an + // openslo_notification_target label for Alertmanager routing. + var policyTarget string + if len(policy.Spec.NotificationTargets) == 1 { + ntRef := policy.Spec.NotificationTargets[0].TargetRef + if nt, ok := g.specs.V1.AlertNotificationTargets[ntRef]; ok { + policyTarget = nt.Spec.Target + } + } + for _, condRef := range policy.Spec.Conditions { condKey := condRef.ConditionRef condition, ok := g.specs.V1.AlertConditions[condKey] @@ -198,48 +358,289 @@ func (g *PrometheusGenerator) buildAlertGroups(sloName string) map[string]templa continue } - if condition.Spec.Condition.Kind != "burnrate" { + kind := specstore.AlertConditionKind(condition.Spec.Condition.Kind) + + // Map legacy "burnrate" to the extended multi-window-multi-burn-rate + // kind so older YAML keeps working without modification. + if kind == "burnrate" { + kind = specstore.KindMultiWindowMultiBurnRate + slog.Warn("legacy AlertCondition kind 'burnrate' detected; rename to 'multi-window-multi-burn-rate' to silence this message", "slo", sloName, "condition", condKey) + } + + if !specstore.ValidKind(kind) { + slog.Error("unsupported AlertCondition kind; skipping", "slo", sloName, "condition", condKey, "kind", condition.Spec.Condition.Kind) + continue + } + + // Mixed-kind checks per severity come after the loop, once we know + // the full set of conditions per severity. Per-condition error-rate + // threshold range is enforced by ValidateRefs; here we surface + // multi-type structural problems at the SLO level. + if condition.Spec.Condition.Threshold == nil { + slog.Error("AlertCondition missing threshold; skipping", "slo", sloName, "condition", condKey) continue } + thresholdVal := *condition.Spec.Condition.Threshold severity := condition.Spec.Severity - threshold := condition.Spec.Condition.Threshold lookback := condition.Spec.Condition.LookbackWindow.String() - alertAfter := condition.Spec.Condition.AlertAfter.String() - - op := "gte" - if condition.Spec.Condition.Operator != "" { - op = string(condition.Spec.Condition.Operator) + alertAfter := "" + if condition.Spec.Condition.AlertAfter != nil { + alertAfter = condition.Spec.Condition.AlertAfter.String() + } + op := promQLOperator(condition.Spec.Condition.Operator) + + expr := buildAlertExpr(kind, thresholdVal, lookback, sloName, string(op)) + + // Tier derivation depends on kind. For kinds that pair conditions + // (multi-window-multi-burn-rate), the tier is the condition name + // with the "-" suffix stripped. For all other kinds, + // every condition is its own tier (each condition is naturally + // OR-ed across tiers). + suffix := "-" + lookback + tierName := condition.Metadata.Name + if kind == specstore.KindMultiWindowMultiBurnRate { + if !strings.HasSuffix(condition.Metadata.Name, suffix) { + slog.Error("multi-window-multi-burn-rate condition name must end with - suffix; skipping", "slo", sloName, "condition", condKey, "lookback", lookback) + continue + } + tierName = strings.TrimSuffix(condition.Metadata.Name, suffix) } - burnRateExpr := fmt.Sprintf( - "openslo_sli_error_rate%s{openslo_slo_name=\"%s\"} / (1 - openslo_slo_objective{openslo_slo_name=\"%s\"}) %s %f", - lookback, sloName, sloName, op, *threshold, - ) + thresholdDisplay, _ := formatThreshold(kind, thresholdVal, op) + + cond := templates.AlertCondition{ + Expr: expr, + Severity: severity, + Threshold: thresholdDisplay, + Lookback: lookback, + AlertAfter: alertAfter, + } - group, exists := alertGroups[severity] - if !exists { - group = templates.AlertGroup{ - For: alertAfter, + state, ok := stateBySev[severity] + if !ok { + state = &severityState{ + kind: kind, + tierIndex: map[string]int{}, } + stateBySev[severity] = state + } + + if state.kind != kind { + slog.Error("mixed AlertCondition kinds within the same severity; skipping offending condition", "slo", sloName, "severity", severity, "expected", state.kind, "got", kind, "condition", condKey) + continue } - group.Conditions = append(group.Conditions, templates.AlertCondition{ - Expr: burnRateExpr, - }) + if idx, exists := state.tierIndex[tierName]; exists { + state.tiers[idx].Conditions = append(state.tiers[idx].Conditions, cond) + } else { + state.tierIndex[tierName] = len(state.tiers) + state.tierOrder = append(state.tierOrder, tierName) + state.tiers = append(state.tiers, templates.AlertTier{Conditions: []templates.AlertCondition{cond}}) + } - thresholds := appendIfMissing(group.Thresholds, fmt.Sprintf("%.1fx", *threshold)) - lookbacks := appendIfMissing(group.Lookbacks, lookback) - group.Thresholds = thresholds - group.Lookbacks = lookbacks + state.thresholds = appendIfMissing(state.thresholds, thresholdDisplay) + state.lookbacks = appendIfMissing(state.lookbacks, lookback) + if state.for_ == "" || alertAfter < state.for_ { + state.for_ = alertAfter + } - alertGroups[severity] = group + // Mismatched notification targets between policies of the same + // severity can't be represented in a single openslo_notification_target + // label; surface loudly and leave the label empty so the issue is + // obvious in the generated rules rather than silently picking one. + if policyTarget != "" { + if state.notificationTarget == "" { + state.notificationTarget = policyTarget + } else if state.notificationTarget != policyTarget { + slog.Error("AlertPolicy notification targets differ within severity; openslo_notification_target label omitted", "slo", sloName, "severity", severity, "have", state.notificationTarget, "got", policyTarget, "policy", polRef) + state.notificationTarget = "" + } + } } } + // Structural validation per severity, applying all rules uniformly. + for severity, state := range stateBySev { + if err := validateSeverityStructure(severity, state.kind, state.tierOrder, state.tiers); err != nil { + slog.Error("alert condition structure invalid; alerts may not fire correctly", "slo", sloName, "severity", severity, "error", err) + } + } + + alertGroups := make(map[string]templates.AlertGroup, len(stateBySev)) + for severity, state := range stateBySev { + alertGroups[severity] = templates.AlertGroup{ + Tiers: state.tiers, + For: state.for_, + Thresholds: state.thresholds, + Lookbacks: state.lookbacks, + KindPascal: specstore.KindPascal(state.kind), + KindDescription: specstore.KindDescription(state.kind), + SloNamePascal: specstore.SloNamePascal(sloName), + NotificationTarget: state.notificationTarget, + } + } return alertGroups } +// buildAlertExpr returns the PromQL expression for a single AlertCondition. +// All expressions include {openslo_slo_name=""} so the alert is anchored +// to one SLO. error-rate emits a raw SLI error-rate comparison; burn-rate +// families normalize by (1 - error_budget) to produce a multiplier. +func buildAlertExpr(kind specstore.AlertConditionKind, threshold float64, lookback, sloName, op string) string { + switch kind { + case specstore.KindErrorRate: + return fmt.Sprintf( + `openslo_sli_error_rate_%s{openslo_slo_name="%s"} %s %f`, + lookback, sloName, op, threshold, + ) + case specstore.KindBurnRate, specstore.KindMultiBurnRate, specstore.KindMultiWindowMultiBurnRate: + return fmt.Sprintf( + `openslo_sli_error_rate_%s{openslo_slo_name="%s"} / (1 - openslo_slo_objective{openslo_slo_name="%s"}) %s %f`, + lookback, sloName, sloName, op, threshold, + ) + default: + // Defensive default: emit a non-firing predicate. Specstore validation + // should have rejected an unknown kind upstream. + return `vector(0)` + } +} + +// formatThreshold renders a threshold value for human-readable display in +// alert annotations. error-rate uses the raw float ("0.001"); burn-rate +// families append "x" to signal the multiplier ("14.4x"). The second +// return value is retained for symmetry with earlier designs and to make +// it easy to add an op-prefixed label later without changing call sites. +func formatThreshold(kind specstore.AlertConditionKind, threshold float64, _ string) (string, string) { + if kind == specstore.KindErrorRate { + s := strconv.FormatFloat(threshold, 'f', -1, 64) + return s, s + } + s := fmt.Sprintf("%.1fx", threshold) + return s, s +} + +// validateSeverityStructure enforces per-kind cross-condition structural +// requirements. Errors are returned for the generator to slog.Error; load-time +// validation already happened in specstore, so this is defense in depth. +// +// - error-rate, burn-rate: exactly 1 condition per severity. +// - multi-burn-rate: >=2 conditions per severity. +// - multi-window-multi-burn-rate: >=2 tiers, each tier >=2 conditions, +// and all conditions within a tier share the same threshold. +func validateSeverityStructure(severity string, kind specstore.AlertConditionKind, tierOrder []string, tiers []templates.AlertTier) error { + total := 0 + for _, t := range tiers { + total += len(t.Conditions) + } + + switch kind { + case specstore.KindErrorRate, specstore.KindBurnRate: + if total != 1 { + return xerrors.Newf("kind %q severity %q expects exactly 1 condition, got %d", kind, severity, total) + } + case specstore.KindMultiBurnRate: + if total < 2 { + return xerrors.Newf("kind %q severity %q expects >=2 conditions, got %d; use 'burn-rate' for a single condition", kind, severity, total) + } + case specstore.KindMultiWindowMultiBurnRate: + if len(tiers) < 2 { + return xerrors.Newf("kind %q severity %q expects >=2 tiers, got %d", kind, severity, len(tiers)) + } + for _, t := range tiers { + if len(t.Conditions) < 2 { + return xerrors.Newf("kind %q severity %q has a tier with only %d condition (need >=2 for short/long pair)", kind, severity, len(t.Conditions)) + } + first := thresholdValue(t.Conditions[0].Threshold) + for _, c := range t.Conditions { + if thresholdValue(c.Threshold) != first { + return xerrors.Newf("kind %q severity %q tier has mismatched thresholds %q and %q (all conditions in a tier must share the same burn multiplier)", kind, severity, t.Conditions[0].Threshold, c.Threshold) + } + } + } + } + return nil +} + +// thresholdValue parses a human-readable threshold back to a float for +// comparison purposes. error-rate thresholds render without suffix ("0.001"); +// burn-rate family thresholds render with an "x" suffix ("14.4x"). Both +// forms are parseable as floats. +func thresholdValue(s string) float64 { + s = strings.TrimSuffix(s, "x") + v, err := strconv.ParseFloat(s, 64) + if err != nil { + return 0 + } + return v +} + +// promQLOperator maps an OpenSlo SDK v1.Operator ("gt","gte","lt","lte") +// to the equivalent PromQL binary operator. Default for empty or unknown +// input is ">=" (the canonical "burn-rate at or above" comparison). +func promQLOperator(op v1.Operator) string { + switch op { + case v1.OperatorGT: + return ">" + case v1.OperatorGTE, "": + return ">=" + case v1.OperatorLT: + return "<" + case v1.OperatorLTE: + return "<=" + default: + return string(op) + } +} + +// objectiveFloat formats an SLOObjective's success target for embedding +// in recording-rule exprs. Resolution order: +// - Target (0.0-1.0 scale): emitted verbatim with strconv %g +// - TargetPercent (0.0-100.0 scale): divided by 100, rounded to 4 decimals +// to suppress IEEE-754 dust (e.g. 99.9 → "0.999") +// - Neither: "0" placeholder +func objectiveFloat(obj v1.SLOObjective) string { + if obj.Target != nil { + return strconv.FormatFloat(*obj.Target, 'f', -1, 64) + } + if obj.TargetPercent != nil { + return strconv.FormatFloat(math.Round(*obj.TargetPercent*100)/10000, 'f', -1, 64) + } + return "0" +} + +// foldSloDescription prepares the per-SLO `spec.description` text for +// emission as a Prometheus label value on the `openslo_slo_info` +// recording rule. Pipeline: +// 1. Replace every `\n` with a single space. +// 2. Collapse runs of whitespace to one space. +// 3. Trim leading/trailing whitespace. +// 4. Cap at 200 chars; truncate with `…` if longer. +// 5. Escape `\"` and `\` so the value survives YAML/PromQL quoting. +// +// Empty input still produces empty string. Every SLO carries the label +// even when its description is gone, so downstream Grafana text panels +// rendering ${description} get a stable row. +func foldSloDescription(text string) string { + if text == "" { + return "" + } + out := strings.Join(strings.Fields(text), " ") + r := strings.NewReplacer( + `\`, `\\`, + `"`, `\"`, + ) + out = r.Replace(out) + if len(out) > 200 { + out = out[:199] + "…" + } + return out +} + +// appendIfMissing returns "a and b" when joining items for display. Empty +// existing gets replaced; non-empty gets " and "-separated only if the new +// item isn't already present. Used to merge burn-rate thresholds and +// lookback windows into single readable strings for alert annotations. func appendIfMissing(existing, newItem string) string { if existing == "" { return newItem @@ -249,3 +650,26 @@ func appendIfMissing(existing, newItem string) string { } return existing } + +// resolveStatusThresholds reads the three SLO annotation overrides and +// resolves each against its SRE-workbook default. Non-numeric values +// fall back to the default silently (specstore validation rejects +// ascending/positive violations at load time). +func resolveStatusThresholds(ann map[string]string) (warn, crit, breach float64) { + warn, _ = specstore.ParseStatusThreshold(ann[specstore.StatusThresholdAnnotationWarning], specstore.StatusThresholdDefaultWarning) + crit, _ = specstore.ParseStatusThreshold(ann[specstore.StatusThresholdAnnotationCritical], specstore.StatusThresholdDefaultCritical) + breach, _ = specstore.ParseStatusThreshold(ann[specstore.StatusThresholdAnnotationBreached], specstore.StatusThresholdDefaultBreached) + return warn, crit, breach +} + +// buildStatusThresholds returns resolved threshold values, or nil when +// the SLO has no alert policies (status gauge is only meaningful for +// monitored SLOs). +func buildStatusThresholds(ann map[string]string) *templates.StatusThresholds { + warn, crit, breach := resolveStatusThresholds(ann) + return &templates.StatusThresholds{ + Warning: warn, + Critical: crit, + Breached: breach, + } +} diff --git a/internal/generator/prometheusgenerator/prometheus_test.go b/internal/generator/prometheusgenerator/prometheus_test.go index fdad02a..929375d 100644 --- a/internal/generator/prometheusgenerator/prometheus_test.go +++ b/internal/generator/prometheusgenerator/prometheus_test.go @@ -5,31 +5,102 @@ import ( "strings" "testing" + v1 "github.com/OpenSLO/go-sdk/pkg/openslo/v1" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/thisisibrahimd/opensloctl/internal/generator" + "github.com/thisisibrahimd/opensloctl/internal/testutil" "github.com/thisisibrahimd/opensloctl/pkg/specstore" ) -func TestGenerate_QueryOutput(t *testing.T) { +// ptr returns a pointer to v. Convenience helper for constructing optional +// fields (Target, TargetPercent, etc.) inline in test fixtures. +func ptr[T any](v T) *T { return &v } + +// inputEntry pairs an OpenSlo YAML file with the expected generator output +// filename. output == "" means the input is a helper file (e.g. a Service +// shared by another SLO) and produces no file on its own. +type inputEntry struct { + file string + output string +} + +// TestGenerate_Golden is the unified snapshot suite for the Prometheus +// generator. Each row declares parallel lists: +// +// - inputs: the OpenSlo YAML files loaded together (in load order) +// - outputs: the generator-produced filenames the loader must produce +// (in any order; matched as a set against the actual output) +// - golden: one of the output filenames whose bytes get compared to a +// goldie fixture +// +// Update all fixtures: +// +// go test ./internal/generator/prometheusgenerator/ -update +func TestGenerate_Golden(t *testing.T) { t.Parallel() tests := []struct { - name string - file string - expectBlock bool - checkQuery string + name string + inputs []inputEntry + goldenPick string // exact generated output filename to byte-compare against goldie }{ { - name: "multiline query uses block scalar", - file: "multiline-slo.yaml", - expectBlock: true, - checkQuery: "histogram_quantile", + name: "multiline threshold recording", + inputs: []inputEntry{ + {file: "multiline-slo.yaml", output: "test-multiline-slo-rules.yaml"}, + }, + goldenPick: "test-multiline-slo-rules.yaml", + }, + { + name: "singleline threshold recording", + inputs: []inputEntry{ + {file: "singleline-slo.yaml", output: "test-singleline-slo-rules.yaml"}, + }, + goldenPick: "test-singleline-slo-rules.yaml", + }, + { + name: "ratio target recording", + inputs: []inputEntry{ + {file: "ratio-slo.yaml", output: "test-ratio-slo-rules.yaml"}, + }, + goldenPick: "test-ratio-slo-rules.yaml", + }, + { + name: "ratio targetPercent recording", + // ratio-slo.yaml supplies the Service referenced by the second input. + inputs: []inputEntry{ + {file: "ratio-percent-slo.yaml", output: "test-ratio-target-percent-rules.yaml"}, + {file: "ratio-slo.yaml", output: "test-ratio-slo-rules.yaml"}, + }, + goldenPick: "test-ratio-target-percent-rules.yaml", }, { - name: "single line stays inline", - file: "singleline-slo.yaml", - expectBlock: false, - checkQuery: "up", + name: "tiered burn-rate alerts merged into one file", + // tiered-slo.yaml supplies the SLO; tiered-alerts.yaml supplies + // conditions/policies/service/notify that the SLO references. + // The unified template renders both recording and alert rules + // into one output file (always named -rules.yaml). + inputs: []inputEntry{ + {file: "tiered-slo.yaml", output: "test-tiered-slo-rules.yaml"}, + {file: "tiered-alerts.yaml", output: ""}, + }, + goldenPick: "test-tiered-slo-rules.yaml", + }, + { + name: "multi-dim SLI emits _unlabeled base + label_join post-process rules", + // multi-dim-slo.yaml enables the IsMulti template branch via + // both multi-dimensional-sli.openslo.com annotations on + // metadata.annotations. Verifies that base recordings get the + // _unlabeled suffix and that the post-process block emits + // label_join rules that pivot on MultiDimensionalLabel. + // multi-dim-service.yaml supplies the Service that the SLO + // references (the SDK rejects a blank spec.service). + inputs: []inputEntry{ + {file: "multi-dim-slo.yaml", output: "test-multi-dim-slo-rules.yaml"}, + {file: "multi-dim-service.yaml", output: ""}, + }, + goldenPick: "test-multi-dim-slo-rules.yaml", }, } @@ -37,70 +108,232 @@ func TestGenerate_QueryOutput(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - testdata := filepath.Join("testdata", tt.file) + paths := make([]string, 0, len(tt.inputs)) + for _, in := range tt.inputs { + paths = append(paths, filepath.Join("testdata", in.file)) + } - specs, err := specstore.GetSpecs([]string{testdata}, false) + specs, err := specstore.GetSpecs(paths, false) require.NoError(t, err) gen := NewPrometheusGenerator(specs).(*PrometheusGenerator) files, err := gen.createGeneratedFiles() require.NoError(t, err) - require.Len(t, files, 1) - - content := files[0].Data - - if tt.expectBlock { - assert.Contains(t, content, "expr: |") - assert.Contains(t, content, tt.checkQuery) - } else { - lines := strings.Split(content, "\n") - found := false - for _, line := range lines { - if strings.Contains(line, "expr:") && strings.Contains(line, tt.checkQuery) { - assert.NotContains(t, line, "|") - found = true - } - } - assert.True(t, found, "expected to find expr line for single-line query") - } + + // Verify input → output pairing (declarative, order-sensitive). + actual := pairedOutputs(t, tt.inputs, files) + assert.Equal(t, tt.inputs, actual, + "input/output pairing mismatch: each input should produce its declared output in declared order") + + // goldie compare one declared output's bytes. The fixture filename is +// derived from the actual generated filename: AssertGolden inserts +// ".golden" before the extension (e.g. test-ratio-slo-...-rules.yaml becomes +// test-ratio-slo-...-rules.golden.yaml). + got := pickByName(t, files, tt.goldenPick) + testutil.AssertGolden(t, "testdata", tt.goldenPick, []byte(got.Data)) }) } } -func TestTemplate_MultilineDetection(t *testing.T) { +// pairedOutputs verifies the generator produced exactly the outputs declared +// in inputs, in input-row order. Inputs with an empty output contribute +// nothing. The function mutates a scratch copy and returns it after matching; +// failing the test produces a clear error naming any missing or extra file. +func pairedOutputs(t *testing.T, inputs []inputEntry, files []*generator.GeneratedFile) []inputEntry { + t.Helper() + + produced := make([]string, 0, len(files)) + for _, f := range files { + produced = append(produced, f.Path) + } + + out := make([]inputEntry, len(inputs)) + copy(out, inputs) + + for i := range out { + if out[i].output == "" { + continue + } + found := false + for j := range produced { + if produced[j] == out[i].output { + produced = append(produced[:j], produced[j+1:]...) + found = true + break + } + } + if !found { + t.Fatalf("expected generator output %q for input %q but it was not produced; remaining produced files: %v", + out[i].output, out[i].file, produced) + } + } + + if len(produced) > 0 { + t.Fatalf("generator produced unexpected additional files: %v", produced) + } + + return out +} + +// pickByName returns the generated file whose Path matches exactly. +func pickByName(t *testing.T, files []*generator.GeneratedFile, name string) *generator.GeneratedFile { + t.Helper() + + for _, f := range files { + if f.Path == name { + return f + } + } + + names := make([]string, 0, len(files)) + for _, f := range files { + names = append(names, f.Path) + } + t.Fatalf("no generated file matched name=%q; got: %v", name, names) + return nil +} + +// TestObjectiveFloat covers the Target/TargetPercent/missing branches of +// objectiveFloat. Critical assertions: +// - TargetPercent = 99.0 → "0.99" (not "0.9900000000000001") +// - TargetPercent = 99.99 → "0.9999" +// - Missing both → "0" (defensive default) +func TestObjectiveFloat(t *testing.T) { t.Parallel() tests := []struct { - name string - query string - hasNewline bool + name string + obj v1.SLOObjective + want string }{ - { - name: "single line no newline", - query: `up{job="test"}`, - hasNewline: false, - }, - { - name: "multiline with newline", - query: `histogram_quantile(0.99, - sum(rate(bucket[5m])) by (le))`, - hasNewline: true, - }, - { - name: "complex query with division", - query: `sum(rate(http_requests_total{status=~"2.."}[5m])) -/ -sum(rate(http_requests_total[5m]))`, - hasNewline: true, - }, + {"target 0.99", v1.SLOObjective{Target: ptr(0.99)}, "0.99"}, + {"target 0.9999", v1.SLOObjective{Target: ptr(0.9999)}, "0.9999"}, + {"targetPercent 99", v1.SLOObjective{TargetPercent: ptr(99.0)}, "0.99"}, + {"targetPercent 99.99", v1.SLOObjective{TargetPercent: ptr(99.99)}, "0.9999"}, + {"missing both defaults to 0", v1.SLOObjective{}, "0"}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - result := strings.Contains(tt.query, "\n") - assert.Equal(t, tt.hasNewline, result) + assert.Equal(t, tt.want, objectiveFloat(tt.obj)) + }) + } +} + +// TestStatusRuleUsesBoolModifier is a regression that locks in the +// Prom 3.x-compatible query shape for the status recording rule. +// +// Background: comparison operators between an instant vector and a +// scalar are filters by default - they preserve the LHS value and +// drop series that don't match. Without the `bool` modifier, +// `(burn_rate >= 14.4) * 3` multiplies the raw burn rate (a float +// like 57.3) instead of returning 0 or 3, so the status gauge +// emits the wrong values and the dashboard's 0/1/2/3 → Healthy/ +// Burning/Critical/Breached mapping never matches. +// +// The `bool` modifier has been supported since Prometheus 0.19.0, +// so writing rules works on every Prometheus version in practical +// use. We regression-test the rule every generator produces so a +// future template refactor can't silently regress this. +func TestStatusRuleUsesBoolModifier(t *testing.T) { + t.Parallel() + +// Each row declares a spec fixture set that produces at least + // one SLO with a generated status recording rule. The + // threshold defaults (warning=1, critical=6, breached=14.4) come + // from specstore's ParseStatusThreshold fallback, so fixtures + // don't need explicit threshold.status.openslo.com/* annotations. + cases := []struct { + name string + paths []string + }{ + { + name: "multiline SLO", + paths: []string{"testdata/multiline-slo.yaml"}, + }, + { + name: "multi-dim SLO", + paths: []string{"testdata/multi-dim-service.yaml", "testdata/multi-dim-slo.yaml"}, + }, + { + name: "tiered SLO", + paths: []string{"testdata/tiered-alerts.yaml", "testdata/tiered-slo.yaml"}, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + specs, err := specstore.GetSpecs(tc.paths, false) + require.NoError(t, err) + + gen := NewPrometheusGenerator(specs).(*PrometheusGenerator) + files, err := gen.createGeneratedFiles() + require.NoError(t, err) + + require.NotEmpty(t, files, "no generated files produced - spec fixtures probably missing") + + var statusRuleSeen int + for _, f := range files { + body := string(f.Data) + + // The status block only appears when StatusThresholds + // are populated. Skip files without it. + start := strings.Index(body, "- name: openslo-status-recordings-") + if start < 0 { + continue + } + rel := strings.Index(body[start:], "\n - name: openslo-") + bodyFromStart := body[start:] + // Find the closest ` - name:` that follows the + // `- name: openslo-status-recordings-` header. If + // there's no next group, the status block extends to + // the end of the body. + status := bodyFromStart + if rel >= 0 { + status = bodyFromStart[:rel] + } + + // Every comparison inside the status block must use + // the `bool` modifier - 3 `>= bool` (Warning, Critical, + // Breached) and 2 `< bool` (Breached, Critical). + require.Equalf(t, 3, strings.Count(status, ">= bool"), + "expected 3 '>= bool' in status block; got %d in %s", + strings.Count(status, ">= bool"), f.Path) + require.Equalf(t, 2, strings.Count(status, "< bool"), + "expected 2 '< bool' in status block; got %d in %s", + strings.Count(status, "< bool"), f.Path) + + // Highest tier (Breached) must multiply by 3, mid + // (Critical) by 2, lowest (Warning) by 1 - those are + // the integer status codes the dashboard maps to text. + require.Contains(t, status, "* 3") + require.Contains(t, status, "* 2") + require.Contains(t, status, "* 1") + + // Sanity guard: no `bool` inside the alert block. + // Alerts use filter semantics, not coerced 0/1. + if alertStart := strings.Index(body, "- name: openslo-alerts-"); alertStart > 0 { + rel := strings.Index(body[alertStart:], "\n - name: openslo-") + alertEnd := len(body) + if rel >= 0 { + alertEnd = rel + alertStart + } + alertBlock := body[alertStart:alertEnd] + require.NotContainsf(t, alertBlock, ">= bool", + "alert block must keep filter semantics (no bool modifier); found in %s", f.Path) + require.NotContainsf(t, alertBlock, "< bool", + "alert block must keep filter semantics (no bool modifier); found in %s", f.Path) + } + + statusRuleSeen++ + } + + require.GreaterOrEqualf(t, statusRuleSeen, 1, + "no generation produced a status recording rule - specstore fallback thresholds may be broken") }) } } diff --git a/internal/generator/prometheusgenerator/templates/templates.go b/internal/generator/prometheusgenerator/templates/templates.go index 283197a..8c4949a 100644 --- a/internal/generator/prometheusgenerator/templates/templates.go +++ b/internal/generator/prometheusgenerator/templates/templates.go @@ -3,10 +3,7 @@ package templates import _ "embed" //go:embed templates/prometheus-recording-rules.template.yaml -var PrometheusRecordingRuleTemplate string - -//go:embed templates/prometheus-alert-rules.template.yaml -var PrometheusAlertRuleTemplate string +var PrometheusRulesTemplate string type WindowedPrometheusQuery struct { Window string `json:"window"` @@ -14,15 +11,43 @@ type WindowedPrometheusQuery struct { } type AlertCondition struct { - Expr string `json:"expr"` - Severity string `json:"severity"` + Expr string `json:"expr"` + Severity string `json:"severity"` + Threshold string `json:"threshold"` + Lookback string `json:"lookback"` + AlertAfter string `json:"alert_after"` +} + +// AlertTier groups conditions that share a tier name. Within a tier the +// conditions are AND-combined; across tiers of the same severity, OR-combined. +// For kinds that do not pair conditions (error-rate, burn-rate, +// multi-burn-rate), each condition becomes its own tier so they're naturally +// OR-combined. +type AlertTier struct { + Conditions []AlertCondition `json:"conditions"` } +// AlertGroup carries the per-severity alert data rendered into the +// Prometheus rules template. KindPascal is the PascalCase form of the +// OpenSlo condition kind (e.g. "multi-window-multi-burn-rate" → +// "MultiWindowMultiBurnRate"). KindDescription is the lower-spaced form +// used in alert summary annotations. SloNamePascal is the SLO's kebab-case +// name with hyphens removed and each segment title-cased, so the rendered +// alert name stays a single PascalCase identifier with no underscores. +// NotificationTarget is the resolved name of the AlertPolicy's single +// notification target (e.g. "engineers"), emitted as the +// openslo_notification_target Prometheus label to enable Alertmanager +// routing. Empty when the AlertPolicy doesn't reference a target. type AlertGroup struct { - Conditions []AlertCondition `json:"conditions"` - For string `json:"for"` - Thresholds string `json:"thresholds"` - Lookbacks string `json:"lookbacks"` + Tiers []AlertTier `json:"tiers"` + For string `json:"for"` + Thresholds string `json:"thresholds"` + Lookbacks string `json:"lookbacks"` + ThresholdLabel string `json:"threshold_label"` + KindPascal string `json:"kind_pascal"` + KindDescription string `json:"kind_description"` + SloNamePascal string `json:"slo_name_pascal"` + NotificationTarget string `json:"notification_target"` } type TemplateData struct { @@ -30,11 +55,49 @@ type TemplateData struct { OpensloVersion string `json:"openslo_version"` PrometheusQuery string `json:"prometheus_query"` WindowedPrometheusQueries []*WindowedPrometheusQuery `json:"windowed_prometheus_queries"` - Objective string `json:"objective"` - IsMulti bool `json:"is_multi"` + // WindowedEventRateQueries mirrors PromQueries but for the event-rate + // metric (events-per-second). Populated only for RatioMetric SLIs + // (see HasEventRate); ThresholdMetric SLIs skip event-rate entirely + // because the spec doesn't expose a parallel event-count query. + WindowedEventRateQueries []*WindowedPrometheusQuery `json:"windowed_event_rate_queries,omitempty"` + // HasEventRate gates the openslo_sli_event_rate_* rules in the + // template. False for ThresholdMetric SLIs. + HasEventRate bool `json:"has_event_rate"` + Objective string `json:"objective"` + IsMulti bool `json:"is_multi"` MultiDimensionalLabel string `json:"multi_dimensional_label"` TimeWindowDays string `json:"time_window_days"` - AlertGroups map[string]AlertGroup `json:"alert_groups"` + // PeriodWindow is the multi-window key (e.g. "30d") used by the + // period burn rate meta recording. Empty when no candidate exists + // (template then emits only current_burn_rate). + PeriodWindow string `json:"period_window"` + // ExtraLabels are OpenSlo metadata.labels converted into Prometheus + // labels (already 1-value validated). The {{ .ExtraLabels }} map can + // be ranged over to render `key: value` lines. + ExtraLabels map[string]string `json:"extra_labels"` + AlertGroups map[string]AlertGroup `json:"alert_groups"` + // Description is the per-SLO `spec.description` text folded to a + // single line, whitespace-collapsed, capped at 200 chars, with + // `"` and `\` escaped so it's safe as a Prometheus label value. + // Empty descriptions still produce a label entry (with `""`) + // so every SLO's `openslo_slo_info` is consistently labelled. + Description string `json:"description"` + // StatusThresholds powers the openslo_slo_status gauge. Always + // provided - defaults fill in missing annotations independently + // per specstore.ParseStatusThreshold. The template currently emits + // the status rule for every SLO; tightening to "only SLOs with + // alert policies" is a follow-up if non-monitored SLOs become a + // signal-noise concern. + StatusThresholds *StatusThresholds `json:"status_thresholds,omitempty"` +} + +// StatusThresholds carries the resolved warning/critical/breached +// thresholds for the openslo_slo_status gauge. Floats, not strings, +// so the template can format them straight into PromQL comparisons. +type StatusThresholds struct { + Warning float64 `json:"warning"` + Critical float64 `json:"critical"` + Breached float64 `json:"breached"` } type WindowData struct { diff --git a/internal/generator/prometheusgenerator/templates/templates/prometheus-alert-rules.template.yaml b/internal/generator/prometheusgenerator/templates/templates/prometheus-alert-rules.template.yaml deleted file mode 100644 index fa5bf8e..0000000 --- a/internal/generator/prometheusgenerator/templates/templates/prometheus-alert-rules.template.yaml +++ /dev/null @@ -1,21 +0,0 @@ -groups: - - name: openslo-burnrate-alerts-{{ .SloName }} - rules: - {{- range $severity, $alertData := .AlertGroups }} - - alert: OpenSLO_{{ $severity | title }}_BurnRate_{{ $.SloName }} - expr: |- - {{- range $i, $cond := $alertData.Conditions }} - {{- if $i }} - or - {{- end }} - {{ $cond.Expr }} - {{- end }} - for: {{ $alertData.For }} - labels: - severity: {{ $severity }} - openslo_slo_name: {{ $.SloName }} - openslo_spec_version: {{ $.OpensloVersion }} - annotations: - summary: "{{ $severity | title }} burn rate alert for SLO {{ $.SloName }}" - description: "Burn rate exceeds {{ $alertData.Thresholds }}x for {{ $alertData.Lookbacks }}" - {{- end }} diff --git a/internal/generator/prometheusgenerator/templates/templates/prometheus-recording-rules.template.yaml b/internal/generator/prometheusgenerator/templates/templates/prometheus-recording-rules.template.yaml index d9755b3..bd9a45b 100644 --- a/internal/generator/prometheusgenerator/templates/templates/prometheus-recording-rules.template.yaml +++ b/internal/generator/prometheusgenerator/templates/templates/prometheus-recording-rules.template.yaml @@ -1,28 +1,71 @@ -{{- define "slo_name"}}{{.SloName}}{{end}} -{{- define "openslo_version"}}{{.OpensloVersion}}{{end}} groups: - name: openslo-info-recordings-{{ .SloName }} rules: - - record: openslo_slo_info{{ if $.IsMulti }}_unlabled{{end}} + - record: openslo_slo_info{{ if $.IsMulti }}_unlabeled{{end}} expr: vector(1) labels: openslo_slo_name: {{ .SloName }} + openslo_slo_description: "{{ .Description }}" openslo_spec_version: {{ .OpensloVersion }} - - record: openslo_slo_objective{{ if $.IsMulti }}_unlabled{{end}} + {{- range $k, $v := .ExtraLabels }} + {{ $k }}: {{ $v }} + {{- end }} + - record: openslo_slo_objective{{ if $.IsMulti }}_unlabeled{{end}} expr: vector({{ .Objective }}) labels: openslo_slo_name: {{ .SloName }} openslo_spec_version: {{ .OpensloVersion }} - - record: openslo_slo_timewindow_days{{ if $.IsMulti }}_unlabled{{end}} + {{- range $k, $v := .ExtraLabels }} + {{ $k }}: {{ $v }} + {{- end }} + - record: openslo_slo_timewindow_days{{ if $.IsMulti }}_unlabeled{{end}} expr: vector({{.TimeWindowDays}}) labels: openslo_slo_name: {{ .SloName }} openslo_spec_version: {{ .OpensloVersion }} - - record: openslo_slo_error_budget{{ if $.IsMulti }}_unlabled{{end}} + {{- range $k, $v := .ExtraLabels }} + {{ $k }}: {{ $v }} + {{- end }} + - record: openslo_slo_error_budget{{ if $.IsMulti }}_unlabeled{{end}} expr: vector(1- {{.Objective}}) labels: openslo_slo_name: {{ .SloName }} openslo_spec_version: {{ .OpensloVersion }} + {{- range $k, $v := .ExtraLabels }} + {{ $k }}: {{ $v }} + {{- end }} + - record: openslo_slo_current_burn_rate{{ if $.IsMulti }}_unlabeled{{end}} + expr: | + openslo_sli_error_rate_5m{openslo_slo_name="{{ .SloName }}", openslo_spec_version="{{ .OpensloVersion }}"} + / on(openslo_slo_name, openslo_spec_version) group_left + openslo_slo_error_budget{openslo_slo_name="{{ .SloName }}", openslo_spec_version="{{ .OpensloVersion }}"} + labels: + openslo_slo_name: {{ .SloName }} + openslo_spec_version: {{ .OpensloVersion }} + {{- range $k, $v := .ExtraLabels }} + {{ $k }}: {{ $v }} + {{- end }} + {{- if .PeriodWindow }} + - record: openslo_slo_period_burn_rate{{ if $.IsMulti }}_unlabeled{{end}} + expr: | + openslo_sli_error_rate_{{ .PeriodWindow }}{openslo_slo_name="{{ .SloName }}", openslo_spec_version="{{ .OpensloVersion }}"} + / on(openslo_slo_name, openslo_spec_version) group_left + openslo_slo_error_budget{openslo_slo_name="{{ .SloName }}", openslo_spec_version="{{ .OpensloVersion }}"} + labels: + openslo_slo_name: {{ .SloName }} + openslo_spec_version: {{ .OpensloVersion }} + {{- range $k, $v := .ExtraLabels }} + {{ $k }}: {{ $v }} + {{- end }} + - record: openslo_slo_period_error_budget_remaining{{ if $.IsMulti }}_unlabeled{{end}} + expr: clamp_min(1 - openslo_slo_period_burn_rate{{ if $.IsMulti }}_unlabeled{{end}}{openslo_slo_name="{{ .SloName }}", openslo_spec_version="{{ .OpensloVersion }}"}, 0) + labels: + openslo_slo_name: {{ .SloName }} + openslo_spec_version: {{ .OpensloVersion }} + {{- range $k, $v := .ExtraLabels }} + {{ $k }}: {{ $v }} + {{- end }} + {{- end }} {{- if .IsMulti }} - record: openslo_slo_info expr: label_join(openslo_slo_info_unlabeled{openslo_slo_name="{{ .SloName }}", openslo_spec_version="{{ .OpensloVersion }}"}, 'openslo_slo_name', '-', 'openslo_slo_name', '{{$.MultiDimensionalLabel}}') @@ -40,22 +83,135 @@ groups: expr: label_join(openslo_slo_error_budget_unlabeled{openslo_slo_name="{{ .SloName }}", openslo_spec_version="{{ .OpensloVersion }}"}, 'openslo_slo_name', '-', 'openslo_slo_name', '{{$.MultiDimensionalLabel}}') labels: openslo_spec_version: {{ .OpensloVersion }} + - record: openslo_slo_current_burn_rate + expr: | + label_join(openslo_sli_error_rate_5m_unlabeled{openslo_slo_name="{{ .SloName }}", openslo_spec_version="{{ .OpensloVersion }}"}, 'openslo_slo_name', '-', 'openslo_slo_name', '{{$.MultiDimensionalLabel}}') + / on(openslo_slo_name, openslo_spec_version) group_left + openslo_slo_error_budget{openslo_slo_name="{{ .SloName }}", openslo_spec_version="{{ .OpensloVersion }}"} + labels: + openslo_spec_version: {{ .OpensloVersion }} + {{- if .PeriodWindow }} + - record: openslo_slo_period_burn_rate + expr: | + label_join(openslo_sli_error_rate_{{ .PeriodWindow }}_unlabeled{openslo_slo_name="{{ .SloName }}", openslo_spec_version="{{ .OpensloVersion }}"}, 'openslo_slo_name', '-', 'openslo_slo_name', '{{$.MultiDimensionalLabel}}') + / on(openslo_slo_name, openslo_spec_version) group_left + openslo_slo_error_budget{openslo_slo_name="{{ .SloName }}", openslo_spec_version="{{ .OpensloVersion }}"} + labels: + openslo_spec_version: {{ .OpensloVersion }} + - record: openslo_slo_period_error_budget_remaining + expr: clamp_min(1 - openslo_slo_period_burn_rate{openslo_slo_name="{{ .SloName }}", openslo_spec_version="{{ .OpensloVersion }}"}, 0) + labels: + openslo_spec_version: {{ .OpensloVersion }} + {{- end }} {{- end}} - name: openslo-sli-recordings-{{ .SloName }} rules: {{- range .WindowedPrometheusQueries }} - - record: openslo_sli_error_rate{{ .Window }}{{ if $.IsMulti }}_unlabeled{{end}} + - record: openslo_sli_error_rate_{{ .Window }}{{ if $.IsMulti }}_unlabeled{{end}} + expr: {{ if contains "\n" .Query }}| +{{ .Query | indent 8 | trimSuffix "\n" }}{{ else }}{{ .Query }}{{ end }} + labels: + openslo_slo_name: {{ $.SloName }} + openslo_spec_version: {{ $.OpensloVersion }} + {{- range $k, $v := $.ExtraLabels }} + {{ $k }}: {{ $v }} + {{- end }} + {{- end }} + {{- if .HasEventRate }} + {{- range .WindowedEventRateQueries }} + - record: openslo_sli_event_rate_{{ .Window }}{{ if $.IsMulti }}_unlabeled{{end}} expr: {{ if contains "\n" .Query }}| {{ .Query | indent 8 | trimSuffix "\n" }}{{ else }}{{ .Query }}{{ end }} labels: openslo_slo_name: {{ $.SloName }} openslo_spec_version: {{ $.OpensloVersion }} + {{- range $k, $v := $.ExtraLabels }} + {{ $k }}: {{ $v }} + {{- end }} + {{- end }} + {{- if .IsMulti }} + {{- range .WindowedEventRateQueries }} + - record: openslo_sli_event_rate_{{ .Window }} + expr: label_join(openslo_sli_event_rate_{{ .Window }}_unlabeled{openslo_slo_name="{{ $.SloName }}", openslo_spec_version="{{ $.OpensloVersion }}"}, 'openslo_slo_name', '-', 'openslo_slo_name', '{{$.MultiDimensionalLabel}}') + labels: + openslo_spec_version: {{ $.OpensloVersion }} + {{- end }} + {{- end }} {{- end }} {{- if .IsMulti }} {{- range .WindowedPrometheusQueries }} - - record: openslo_sli_error_rate{{ .Window }} - expr: label_join(openslo_sli_error_rate{{ .Window }}_unlabeled{openslo_slo_name="{{ $.SloName }}", openslo_spec_version="{{ $.OpensloVersion }}"}, 'openslo_slo_name', '-', 'openslo_slo_name', '{{$.MultiDimensionalLabel}}') + - record: openslo_sli_error_rate_{{ .Window }} + expr: label_join(openslo_sli_error_rate_{{ .Window }}_unlabeled{openslo_slo_name="{{ $.SloName }}", openslo_spec_version="{{ $.OpensloVersion }}"}, 'openslo_slo_name', '-', 'openslo_slo_name', '{{$.MultiDimensionalLabel}}') labels: openslo_spec_version: {{ $.OpensloVersion }} {{- end }} {{- end }} + {{- if .StatusThresholds }} + - name: openslo-status-recordings-{{ .SloName }} + rules: + - record: openslo_slo_status{{ if $.IsMulti }}_unlabeled{{end}} + expr: | + ( + (openslo_slo_current_burn_rate{{ if $.IsMulti }}_unlabeled{{end}}{openslo_slo_name="{{ .SloName }}", openslo_spec_version="{{ .OpensloVersion }}"} >= bool {{ .StatusThresholds.Breached }}) + * 3 + ) + or + ( + (openslo_slo_current_burn_rate{{ if $.IsMulti }}_unlabeled{{end}}{openslo_slo_name="{{ .SloName }}", openslo_spec_version="{{ .OpensloVersion }}"} >= bool {{ .StatusThresholds.Critical }}) + and + (openslo_slo_current_burn_rate{{ if $.IsMulti }}_unlabeled{{end}}{openslo_slo_name="{{ .SloName }}", openslo_spec_version="{{ .OpensloVersion }}"} < bool {{ .StatusThresholds.Breached }}) + * 2 + ) + or + ( + (openslo_slo_current_burn_rate{{ if $.IsMulti }}_unlabeled{{end}}{openslo_slo_name="{{ .SloName }}", openslo_spec_version="{{ .OpensloVersion }}"} >= bool {{ .StatusThresholds.Warning }}) + and + (openslo_slo_current_burn_rate{{ if $.IsMulti }}_unlabeled{{end}}{openslo_slo_name="{{ .SloName }}", openslo_spec_version="{{ .OpensloVersion }}"} < bool {{ .StatusThresholds.Critical }}) + * 1 + ) + labels: + openslo_slo_name: {{ .SloName }} + openslo_spec_version: {{ .OpensloVersion }} + {{- range $k, $v := .ExtraLabels }} + {{ $k }}: {{ $v }} + {{- end }} + {{- if .IsMulti }} + - record: openslo_slo_status + expr: label_join(openslo_slo_status_unlabeled{openslo_slo_name="{{ .SloName }}", openslo_spec_version="{{ .OpensloVersion }}"}, 'openslo_slo_name', '-', 'openslo_slo_name', '{{$.MultiDimensionalLabel}}') + labels: + openslo_spec_version: {{ .OpensloVersion }} + {{- end }} + {{- end }} + {{- if .AlertGroups }} + - name: openslo-alerts-{{ .SloName }} + rules: + {{- range $severity, $alertData := .AlertGroups }} + - alert: {{ $alertData.SloNamePascal }}{{ $alertData.KindPascal }}{{ $severity | title }} + expr: |- + {{- range $ti, $tier := $alertData.Tiers }} + {{- if $ti }} + or + {{- end }} + ({{- range $ci, $cond := $tier.Conditions }} + {{- if $ci }} + and + {{- end }} + {{ $cond.Expr -}} + {{- end }}) + {{- end }} + for: {{ $alertData.For }} + labels: + openslo_alert_severity: {{ $severity }} + {{- if $alertData.NotificationTarget }} + openslo_notification_target: {{ $alertData.NotificationTarget }} + {{- end }} + openslo_slo_name: {{ $.SloName }} + openslo_spec_version: {{ $.OpensloVersion }} + {{- range $k, $v := $.ExtraLabels }} + {{ $k }}: {{ $v }} + {{- end }} + annotations: + summary: "{{ $severity | title }} {{ $alertData.KindDescription }} alert for SLO {{ $.SloName }}" + description: "Threshold {{ $alertData.Thresholds }} over {{ $alertData.Lookbacks }}" + {{- end }} + {{- end }} diff --git a/internal/generator/prometheusgenerator/testdata/multi-dim-service.yaml b/internal/generator/prometheusgenerator/testdata/multi-dim-service.yaml new file mode 100644 index 0000000..01d6195 --- /dev/null +++ b/internal/generator/prometheusgenerator/testdata/multi-dim-service.yaml @@ -0,0 +1,6 @@ +apiVersion: openslo/v1 +kind: Service +metadata: + name: test-multi-dim-svc +spec: + description: service for multi-dim SLO test diff --git a/internal/generator/prometheusgenerator/testdata/multi-dim-slo.yaml b/internal/generator/prometheusgenerator/testdata/multi-dim-slo.yaml new file mode 100644 index 0000000..04bcdc3 --- /dev/null +++ b/internal/generator/prometheusgenerator/testdata/multi-dim-slo.yaml @@ -0,0 +1,28 @@ +apiVersion: openslo/v1 +kind: SLO +metadata: + name: test-multi-dim-slo + annotations: + multi-dimensional-sli.openslo.com/label: service_name + multi-dimensional-sli.openslo.com/dimensions: "accounting,checkout,recommendation" +spec: + description: SLO expanded per service_name dimension via label_join + service: test-multi-dim-svc + indicator: + metadata: + name: test-multi-dim-sli + spec: + thresholdMetric: + metricSource: + type: Prometheus + spec: + query: histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket{job="api"}[{{.Window}}])) by (le)) + budgetingMethod: Occurrences + timeWindow: + - duration: 30d + isRolling: true + objectives: + - displayName: "99.9%" + op: lte + value: 0.5 + target: 0.999 diff --git a/internal/generator/prometheusgenerator/testdata/ratio-percent-slo.yaml b/internal/generator/prometheusgenerator/testdata/ratio-percent-slo.yaml new file mode 100644 index 0000000..67f98ec --- /dev/null +++ b/internal/generator/prometheusgenerator/testdata/ratio-percent-slo.yaml @@ -0,0 +1,30 @@ +apiVersion: openslo/v1 +kind: SLO +metadata: + name: test-ratio-target-percent +spec: + description: Test SLO with targetPercent instead of target + service: test-svc + indicator: + metadata: + name: test-ratio-percent-sli + spec: + ratioMetric: + counter: true + good: + metricSource: + type: Prometheus + spec: + query: sum(rate(http_requests_total{status=~"2.."}[{{.Window}}])) + total: + metricSource: + type: Prometheus + spec: + query: sum(rate(http_requests_total[{{.Window}}])) + budgetingMethod: Occurrences + timeWindow: + - duration: 30d + isRolling: true + objectives: + - displayName: "99.9% success rate" + targetPercent: 99.9 diff --git a/internal/generator/prometheusgenerator/testdata/ratio-slo.yaml b/internal/generator/prometheusgenerator/testdata/ratio-slo.yaml new file mode 100644 index 0000000..d06ff30 --- /dev/null +++ b/internal/generator/prometheusgenerator/testdata/ratio-slo.yaml @@ -0,0 +1,37 @@ +apiVersion: openslo/v1 +kind: Service +metadata: + name: test-svc +spec: + description: Test service +--- +apiVersion: openslo/v1 +kind: SLO +metadata: + name: test-ratio-slo +spec: + description: Test SLO with RatioMetric (good over total) + service: test-svc + indicator: + metadata: + name: test-ratio-sli + spec: + ratioMetric: + counter: true + good: + metricSource: + type: Prometheus + spec: + query: sum(rate(http_requests_total{status=~"2.."}[{{.Window}}])) + total: + metricSource: + type: Prometheus + spec: + query: sum(rate(http_requests_total[{{.Window}}])) + budgetingMethod: Occurrences + timeWindow: + - duration: 30d + isRolling: true + objectives: + - displayName: "99.9% success rate" + target: 0.999 diff --git a/internal/generator/prometheusgenerator/testdata/test-multi-dim-slo-rules.golden.yaml b/internal/generator/prometheusgenerator/testdata/test-multi-dim-slo-rules.golden.yaml new file mode 100644 index 0000000..b422b59 --- /dev/null +++ b/internal/generator/prometheusgenerator/testdata/test-multi-dim-slo-rules.golden.yaml @@ -0,0 +1,218 @@ +groups: + - name: openslo-info-recordings-test-multi-dim-slo + rules: + - record: openslo_slo_info_unlabeled + expr: vector(1) + labels: + openslo_slo_name: test-multi-dim-slo + openslo_slo_description: "SLO expanded per service_name dimension via label_join" + openslo_spec_version: openslo/v1 + openslo_service_name: test-multi-dim-svc + - record: openslo_slo_objective_unlabeled + expr: vector(0.999) + labels: + openslo_slo_name: test-multi-dim-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-multi-dim-svc + - record: openslo_slo_timewindow_days_unlabeled + expr: vector(30) + labels: + openslo_slo_name: test-multi-dim-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-multi-dim-svc + - record: openslo_slo_error_budget_unlabeled + expr: vector(1- 0.999) + labels: + openslo_slo_name: test-multi-dim-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-multi-dim-svc + - record: openslo_slo_current_burn_rate_unlabeled + expr: | + openslo_sli_error_rate_5m{openslo_slo_name="test-multi-dim-slo", openslo_spec_version="openslo/v1"} + / on(openslo_slo_name, openslo_spec_version) group_left + openslo_slo_error_budget{openslo_slo_name="test-multi-dim-slo", openslo_spec_version="openslo/v1"} + labels: + openslo_slo_name: test-multi-dim-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-multi-dim-svc + - record: openslo_slo_period_burn_rate_unlabeled + expr: | + openslo_sli_error_rate_30d{openslo_slo_name="test-multi-dim-slo", openslo_spec_version="openslo/v1"} + / on(openslo_slo_name, openslo_spec_version) group_left + openslo_slo_error_budget{openslo_slo_name="test-multi-dim-slo", openslo_spec_version="openslo/v1"} + labels: + openslo_slo_name: test-multi-dim-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-multi-dim-svc + - record: openslo_slo_period_error_budget_remaining_unlabeled + expr: clamp_min(1 - openslo_slo_period_burn_rate_unlabeled{openslo_slo_name="test-multi-dim-slo", openslo_spec_version="openslo/v1"}, 0) + labels: + openslo_slo_name: test-multi-dim-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-multi-dim-svc + - record: openslo_slo_info + expr: label_join(openslo_slo_info_unlabeled{openslo_slo_name="test-multi-dim-slo", openslo_spec_version="openslo/v1"}, 'openslo_slo_name', '-', 'openslo_slo_name', 'service_name') + labels: + openslo_spec_version: openslo/v1 + - record: openslo_slo_objective + expr: label_join(openslo_slo_objective_unlabeled{openslo_slo_name="test-multi-dim-slo", openslo_spec_version="openslo/v1"}, 'openslo_slo_name', '-', 'openslo_slo_name', 'service_name') + labels: + openslo_spec_version: openslo/v1 + - record: openslo_slo_timewindow_days + expr: label_join(openslo_slo_timewindow_days_unlabeled{openslo_slo_name="test-multi-dim-slo", openslo_spec_version="openslo/v1"}, 'openslo_slo_name', '-', 'openslo_slo_name', 'service_name') + labels: + openslo_spec_version: openslo/v1 + - record: openslo_slo_error_budget + expr: label_join(openslo_slo_error_budget_unlabeled{openslo_slo_name="test-multi-dim-slo", openslo_spec_version="openslo/v1"}, 'openslo_slo_name', '-', 'openslo_slo_name', 'service_name') + labels: + openslo_spec_version: openslo/v1 + - record: openslo_slo_current_burn_rate + expr: | + label_join(openslo_sli_error_rate_5m_unlabeled{openslo_slo_name="test-multi-dim-slo", openslo_spec_version="openslo/v1"}, 'openslo_slo_name', '-', 'openslo_slo_name', 'service_name') + / on(openslo_slo_name, openslo_spec_version) group_left + openslo_slo_error_budget{openslo_slo_name="test-multi-dim-slo", openslo_spec_version="openslo/v1"} + labels: + openslo_spec_version: openslo/v1 + - record: openslo_slo_period_burn_rate + expr: | + label_join(openslo_sli_error_rate_30d_unlabeled{openslo_slo_name="test-multi-dim-slo", openslo_spec_version="openslo/v1"}, 'openslo_slo_name', '-', 'openslo_slo_name', 'service_name') + / on(openslo_slo_name, openslo_spec_version) group_left + openslo_slo_error_budget{openslo_slo_name="test-multi-dim-slo", openslo_spec_version="openslo/v1"} + labels: + openslo_spec_version: openslo/v1 + - record: openslo_slo_period_error_budget_remaining + expr: clamp_min(1 - openslo_slo_period_burn_rate{openslo_slo_name="test-multi-dim-slo", openslo_spec_version="openslo/v1"}, 0) + labels: + openslo_spec_version: openslo/v1 + - name: openslo-sli-recordings-test-multi-dim-slo + rules: + - record: openslo_sli_error_rate_5m_unlabeled + expr: histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket{job="api"}[5m])) by (le)) + labels: + openslo_slo_name: test-multi-dim-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-multi-dim-svc + - record: openslo_sli_error_rate_30m_unlabeled + expr: histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket{job="api"}[30m])) by (le)) + labels: + openslo_slo_name: test-multi-dim-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-multi-dim-svc + - record: openslo_sli_error_rate_1h_unlabeled + expr: histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket{job="api"}[1h])) by (le)) + labels: + openslo_slo_name: test-multi-dim-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-multi-dim-svc + - record: openslo_sli_error_rate_3h_unlabeled + expr: histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket{job="api"}[3h])) by (le)) + labels: + openslo_slo_name: test-multi-dim-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-multi-dim-svc + - record: openslo_sli_error_rate_6h_unlabeled + expr: histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket{job="api"}[6h])) by (le)) + labels: + openslo_slo_name: test-multi-dim-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-multi-dim-svc + - record: openslo_sli_error_rate_1d_unlabeled + expr: histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket{job="api"}[1d])) by (le)) + labels: + openslo_slo_name: test-multi-dim-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-multi-dim-svc + - record: openslo_sli_error_rate_3d_unlabeled + expr: histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket{job="api"}[3d])) by (le)) + labels: + openslo_slo_name: test-multi-dim-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-multi-dim-svc + - record: openslo_sli_error_rate_7d_unlabeled + expr: histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket{job="api"}[7d])) by (le)) + labels: + openslo_slo_name: test-multi-dim-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-multi-dim-svc + - record: openslo_sli_error_rate_28d_unlabeled + expr: histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket{job="api"}[28d])) by (le)) + labels: + openslo_slo_name: test-multi-dim-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-multi-dim-svc + - record: openslo_sli_error_rate_30d_unlabeled + expr: histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket{job="api"}[30d])) by (le)) + labels: + openslo_slo_name: test-multi-dim-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-multi-dim-svc + - record: openslo_sli_error_rate_5m + expr: label_join(openslo_sli_error_rate_5m_unlabeled{openslo_slo_name="test-multi-dim-slo", openslo_spec_version="openslo/v1"}, 'openslo_slo_name', '-', 'openslo_slo_name', 'service_name') + labels: + openslo_spec_version: openslo/v1 + - record: openslo_sli_error_rate_30m + expr: label_join(openslo_sli_error_rate_30m_unlabeled{openslo_slo_name="test-multi-dim-slo", openslo_spec_version="openslo/v1"}, 'openslo_slo_name', '-', 'openslo_slo_name', 'service_name') + labels: + openslo_spec_version: openslo/v1 + - record: openslo_sli_error_rate_1h + expr: label_join(openslo_sli_error_rate_1h_unlabeled{openslo_slo_name="test-multi-dim-slo", openslo_spec_version="openslo/v1"}, 'openslo_slo_name', '-', 'openslo_slo_name', 'service_name') + labels: + openslo_spec_version: openslo/v1 + - record: openslo_sli_error_rate_3h + expr: label_join(openslo_sli_error_rate_3h_unlabeled{openslo_slo_name="test-multi-dim-slo", openslo_spec_version="openslo/v1"}, 'openslo_slo_name', '-', 'openslo_slo_name', 'service_name') + labels: + openslo_spec_version: openslo/v1 + - record: openslo_sli_error_rate_6h + expr: label_join(openslo_sli_error_rate_6h_unlabeled{openslo_slo_name="test-multi-dim-slo", openslo_spec_version="openslo/v1"}, 'openslo_slo_name', '-', 'openslo_slo_name', 'service_name') + labels: + openslo_spec_version: openslo/v1 + - record: openslo_sli_error_rate_1d + expr: label_join(openslo_sli_error_rate_1d_unlabeled{openslo_slo_name="test-multi-dim-slo", openslo_spec_version="openslo/v1"}, 'openslo_slo_name', '-', 'openslo_slo_name', 'service_name') + labels: + openslo_spec_version: openslo/v1 + - record: openslo_sli_error_rate_3d + expr: label_join(openslo_sli_error_rate_3d_unlabeled{openslo_slo_name="test-multi-dim-slo", openslo_spec_version="openslo/v1"}, 'openslo_slo_name', '-', 'openslo_slo_name', 'service_name') + labels: + openslo_spec_version: openslo/v1 + - record: openslo_sli_error_rate_7d + expr: label_join(openslo_sli_error_rate_7d_unlabeled{openslo_slo_name="test-multi-dim-slo", openslo_spec_version="openslo/v1"}, 'openslo_slo_name', '-', 'openslo_slo_name', 'service_name') + labels: + openslo_spec_version: openslo/v1 + - record: openslo_sli_error_rate_28d + expr: label_join(openslo_sli_error_rate_28d_unlabeled{openslo_slo_name="test-multi-dim-slo", openslo_spec_version="openslo/v1"}, 'openslo_slo_name', '-', 'openslo_slo_name', 'service_name') + labels: + openslo_spec_version: openslo/v1 + - record: openslo_sli_error_rate_30d + expr: label_join(openslo_sli_error_rate_30d_unlabeled{openslo_slo_name="test-multi-dim-slo", openslo_spec_version="openslo/v1"}, 'openslo_slo_name', '-', 'openslo_slo_name', 'service_name') + labels: + openslo_spec_version: openslo/v1 + - name: openslo-status-recordings-test-multi-dim-slo + rules: + - record: openslo_slo_status_unlabeled + expr: | + ( + (openslo_slo_current_burn_rate_unlabeled{openslo_slo_name="test-multi-dim-slo", openslo_spec_version="openslo/v1"} >= bool 14.4) + * 3 + ) + or + ( + (openslo_slo_current_burn_rate_unlabeled{openslo_slo_name="test-multi-dim-slo", openslo_spec_version="openslo/v1"} >= bool 6) + and + (openslo_slo_current_burn_rate_unlabeled{openslo_slo_name="test-multi-dim-slo", openslo_spec_version="openslo/v1"} < bool 14.4) + * 2 + ) + or + ( + (openslo_slo_current_burn_rate_unlabeled{openslo_slo_name="test-multi-dim-slo", openslo_spec_version="openslo/v1"} >= bool 1) + and + (openslo_slo_current_burn_rate_unlabeled{openslo_slo_name="test-multi-dim-slo", openslo_spec_version="openslo/v1"} < bool 6) + * 1 + ) + labels: + openslo_slo_name: test-multi-dim-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-multi-dim-svc + - record: openslo_slo_status + expr: label_join(openslo_slo_status_unlabeled{openslo_slo_name="test-multi-dim-slo", openslo_spec_version="openslo/v1"}, 'openslo_slo_name', '-', 'openslo_slo_name', 'service_name') + labels: + openslo_spec_version: openslo/v1 diff --git a/internal/generator/prometheusgenerator/testdata/test-multiline-slo-rules.golden.yaml b/internal/generator/prometheusgenerator/testdata/test-multiline-slo-rules.golden.yaml new file mode 100644 index 0000000..81d07e4 --- /dev/null +++ b/internal/generator/prometheusgenerator/testdata/test-multiline-slo-rules.golden.yaml @@ -0,0 +1,190 @@ +groups: + - name: openslo-info-recordings-test-multiline-slo + rules: + - record: openslo_slo_info + expr: vector(1) + labels: + openslo_slo_name: test-multiline-slo + openslo_slo_description: "Test SLO with multiline query" + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_slo_objective + expr: vector(0.999) + labels: + openslo_slo_name: test-multiline-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_slo_timewindow_days + expr: vector(30) + labels: + openslo_slo_name: test-multiline-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_slo_error_budget + expr: vector(1- 0.999) + labels: + openslo_slo_name: test-multiline-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_slo_current_burn_rate + expr: | + openslo_sli_error_rate_5m{openslo_slo_name="test-multiline-slo", openslo_spec_version="openslo/v1"} + / on(openslo_slo_name, openslo_spec_version) group_left + openslo_slo_error_budget{openslo_slo_name="test-multiline-slo", openslo_spec_version="openslo/v1"} + labels: + openslo_slo_name: test-multiline-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_slo_period_burn_rate + expr: | + openslo_sli_error_rate_30d{openslo_slo_name="test-multiline-slo", openslo_spec_version="openslo/v1"} + / on(openslo_slo_name, openslo_spec_version) group_left + openslo_slo_error_budget{openslo_slo_name="test-multiline-slo", openslo_spec_version="openslo/v1"} + labels: + openslo_slo_name: test-multiline-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_slo_period_error_budget_remaining + expr: clamp_min(1 - openslo_slo_period_burn_rate{openslo_slo_name="test-multiline-slo", openslo_spec_version="openslo/v1"}, 0) + labels: + openslo_slo_name: test-multiline-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - name: openslo-sli-recordings-test-multiline-slo + rules: + - record: openslo_sli_error_rate_5m + expr: | + histogram_quantile(0.99, + sum(rate(http_request_duration_seconds_bucket{service="test-svc"}[5m])) + by (le) + ) + + labels: + openslo_slo_name: test-multiline-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_sli_error_rate_30m + expr: | + histogram_quantile(0.99, + sum(rate(http_request_duration_seconds_bucket{service="test-svc"}[30m])) + by (le) + ) + + labels: + openslo_slo_name: test-multiline-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_sli_error_rate_1h + expr: | + histogram_quantile(0.99, + sum(rate(http_request_duration_seconds_bucket{service="test-svc"}[1h])) + by (le) + ) + + labels: + openslo_slo_name: test-multiline-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_sli_error_rate_3h + expr: | + histogram_quantile(0.99, + sum(rate(http_request_duration_seconds_bucket{service="test-svc"}[3h])) + by (le) + ) + + labels: + openslo_slo_name: test-multiline-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_sli_error_rate_6h + expr: | + histogram_quantile(0.99, + sum(rate(http_request_duration_seconds_bucket{service="test-svc"}[6h])) + by (le) + ) + + labels: + openslo_slo_name: test-multiline-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_sli_error_rate_1d + expr: | + histogram_quantile(0.99, + sum(rate(http_request_duration_seconds_bucket{service="test-svc"}[1d])) + by (le) + ) + + labels: + openslo_slo_name: test-multiline-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_sli_error_rate_3d + expr: | + histogram_quantile(0.99, + sum(rate(http_request_duration_seconds_bucket{service="test-svc"}[3d])) + by (le) + ) + + labels: + openslo_slo_name: test-multiline-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_sli_error_rate_7d + expr: | + histogram_quantile(0.99, + sum(rate(http_request_duration_seconds_bucket{service="test-svc"}[7d])) + by (le) + ) + + labels: + openslo_slo_name: test-multiline-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_sli_error_rate_28d + expr: | + histogram_quantile(0.99, + sum(rate(http_request_duration_seconds_bucket{service="test-svc"}[28d])) + by (le) + ) + + labels: + openslo_slo_name: test-multiline-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_sli_error_rate_30d + expr: | + histogram_quantile(0.99, + sum(rate(http_request_duration_seconds_bucket{service="test-svc"}[30d])) + by (le) + ) + + labels: + openslo_slo_name: test-multiline-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - name: openslo-status-recordings-test-multiline-slo + rules: + - record: openslo_slo_status + expr: | + ( + (openslo_slo_current_burn_rate{openslo_slo_name="test-multiline-slo", openslo_spec_version="openslo/v1"} >= bool 14.4) + * 3 + ) + or + ( + (openslo_slo_current_burn_rate{openslo_slo_name="test-multiline-slo", openslo_spec_version="openslo/v1"} >= bool 6) + and + (openslo_slo_current_burn_rate{openslo_slo_name="test-multiline-slo", openslo_spec_version="openslo/v1"} < bool 14.4) + * 2 + ) + or + ( + (openslo_slo_current_burn_rate{openslo_slo_name="test-multiline-slo", openslo_spec_version="openslo/v1"} >= bool 1) + and + (openslo_slo_current_burn_rate{openslo_slo_name="test-multiline-slo", openslo_spec_version="openslo/v1"} < bool 6) + * 1 + ) + labels: + openslo_slo_name: test-multiline-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc diff --git a/internal/generator/prometheusgenerator/testdata/test-ratio-slo-rules.golden.yaml b/internal/generator/prometheusgenerator/testdata/test-ratio-slo-rules.golden.yaml new file mode 100644 index 0000000..c096257 --- /dev/null +++ b/internal/generator/prometheusgenerator/testdata/test-ratio-slo-rules.golden.yaml @@ -0,0 +1,250 @@ +groups: + - name: openslo-info-recordings-test-ratio-slo + rules: + - record: openslo_slo_info + expr: vector(1) + labels: + openslo_slo_name: test-ratio-slo + openslo_slo_description: "Test SLO with RatioMetric (good over total)" + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_slo_objective + expr: vector(0.999) + labels: + openslo_slo_name: test-ratio-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_slo_timewindow_days + expr: vector(30) + labels: + openslo_slo_name: test-ratio-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_slo_error_budget + expr: vector(1- 0.999) + labels: + openslo_slo_name: test-ratio-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_slo_current_burn_rate + expr: | + openslo_sli_error_rate_5m{openslo_slo_name="test-ratio-slo", openslo_spec_version="openslo/v1"} + / on(openslo_slo_name, openslo_spec_version) group_left + openslo_slo_error_budget{openslo_slo_name="test-ratio-slo", openslo_spec_version="openslo/v1"} + labels: + openslo_slo_name: test-ratio-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_slo_period_burn_rate + expr: | + openslo_sli_error_rate_30d{openslo_slo_name="test-ratio-slo", openslo_spec_version="openslo/v1"} + / on(openslo_slo_name, openslo_spec_version) group_left + openslo_slo_error_budget{openslo_slo_name="test-ratio-slo", openslo_spec_version="openslo/v1"} + labels: + openslo_slo_name: test-ratio-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_slo_period_error_budget_remaining + expr: clamp_min(1 - openslo_slo_period_burn_rate{openslo_slo_name="test-ratio-slo", openslo_spec_version="openslo/v1"}, 0) + labels: + openslo_slo_name: test-ratio-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - name: openslo-sli-recordings-test-ratio-slo + rules: + - record: openslo_sli_error_rate_5m + expr: | + 1 - ( + sum(rate(http_requests_total{status=~"2.."}[5m])) + ) / ( + sum(rate(http_requests_total[5m])) + ) + labels: + openslo_slo_name: test-ratio-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_sli_error_rate_30m + expr: | + 1 - ( + sum(rate(http_requests_total{status=~"2.."}[30m])) + ) / ( + sum(rate(http_requests_total[30m])) + ) + labels: + openslo_slo_name: test-ratio-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_sli_error_rate_1h + expr: | + 1 - ( + sum(rate(http_requests_total{status=~"2.."}[1h])) + ) / ( + sum(rate(http_requests_total[1h])) + ) + labels: + openslo_slo_name: test-ratio-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_sli_error_rate_3h + expr: | + 1 - ( + sum(rate(http_requests_total{status=~"2.."}[3h])) + ) / ( + sum(rate(http_requests_total[3h])) + ) + labels: + openslo_slo_name: test-ratio-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_sli_error_rate_6h + expr: | + 1 - ( + sum(rate(http_requests_total{status=~"2.."}[6h])) + ) / ( + sum(rate(http_requests_total[6h])) + ) + labels: + openslo_slo_name: test-ratio-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_sli_error_rate_1d + expr: | + 1 - ( + sum(rate(http_requests_total{status=~"2.."}[1d])) + ) / ( + sum(rate(http_requests_total[1d])) + ) + labels: + openslo_slo_name: test-ratio-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_sli_error_rate_3d + expr: | + 1 - ( + sum(rate(http_requests_total{status=~"2.."}[3d])) + ) / ( + sum(rate(http_requests_total[3d])) + ) + labels: + openslo_slo_name: test-ratio-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_sli_error_rate_7d + expr: | + 1 - ( + sum(rate(http_requests_total{status=~"2.."}[7d])) + ) / ( + sum(rate(http_requests_total[7d])) + ) + labels: + openslo_slo_name: test-ratio-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_sli_error_rate_28d + expr: | + 1 - ( + sum(rate(http_requests_total{status=~"2.."}[28d])) + ) / ( + sum(rate(http_requests_total[28d])) + ) + labels: + openslo_slo_name: test-ratio-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_sli_error_rate_30d + expr: | + 1 - ( + sum(rate(http_requests_total{status=~"2.."}[30d])) + ) / ( + sum(rate(http_requests_total[30d])) + ) + labels: + openslo_slo_name: test-ratio-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_sli_event_rate_5m + expr: sum(rate(http_requests_total[5m])) + labels: + openslo_slo_name: test-ratio-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_sli_event_rate_30m + expr: sum(rate(http_requests_total[30m])) + labels: + openslo_slo_name: test-ratio-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_sli_event_rate_1h + expr: sum(rate(http_requests_total[1h])) + labels: + openslo_slo_name: test-ratio-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_sli_event_rate_3h + expr: sum(rate(http_requests_total[3h])) + labels: + openslo_slo_name: test-ratio-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_sli_event_rate_6h + expr: sum(rate(http_requests_total[6h])) + labels: + openslo_slo_name: test-ratio-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_sli_event_rate_1d + expr: sum(rate(http_requests_total[1d])) + labels: + openslo_slo_name: test-ratio-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_sli_event_rate_3d + expr: sum(rate(http_requests_total[3d])) + labels: + openslo_slo_name: test-ratio-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_sli_event_rate_7d + expr: sum(rate(http_requests_total[7d])) + labels: + openslo_slo_name: test-ratio-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_sli_event_rate_28d + expr: sum(rate(http_requests_total[28d])) + labels: + openslo_slo_name: test-ratio-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_sli_event_rate_30d + expr: sum(rate(http_requests_total[30d])) + labels: + openslo_slo_name: test-ratio-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - name: openslo-status-recordings-test-ratio-slo + rules: + - record: openslo_slo_status + expr: | + ( + (openslo_slo_current_burn_rate{openslo_slo_name="test-ratio-slo", openslo_spec_version="openslo/v1"} >= bool 14.4) + * 3 + ) + or + ( + (openslo_slo_current_burn_rate{openslo_slo_name="test-ratio-slo", openslo_spec_version="openslo/v1"} >= bool 6) + and + (openslo_slo_current_burn_rate{openslo_slo_name="test-ratio-slo", openslo_spec_version="openslo/v1"} < bool 14.4) + * 2 + ) + or + ( + (openslo_slo_current_burn_rate{openslo_slo_name="test-ratio-slo", openslo_spec_version="openslo/v1"} >= bool 1) + and + (openslo_slo_current_burn_rate{openslo_slo_name="test-ratio-slo", openslo_spec_version="openslo/v1"} < bool 6) + * 1 + ) + labels: + openslo_slo_name: test-ratio-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc diff --git a/internal/generator/prometheusgenerator/testdata/test-ratio-target-percent-rules.golden.yaml b/internal/generator/prometheusgenerator/testdata/test-ratio-target-percent-rules.golden.yaml new file mode 100644 index 0000000..f6b0b38 --- /dev/null +++ b/internal/generator/prometheusgenerator/testdata/test-ratio-target-percent-rules.golden.yaml @@ -0,0 +1,250 @@ +groups: + - name: openslo-info-recordings-test-ratio-target-percent + rules: + - record: openslo_slo_info + expr: vector(1) + labels: + openslo_slo_name: test-ratio-target-percent + openslo_slo_description: "Test SLO with targetPercent instead of target" + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_slo_objective + expr: vector(0.999) + labels: + openslo_slo_name: test-ratio-target-percent + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_slo_timewindow_days + expr: vector(30) + labels: + openslo_slo_name: test-ratio-target-percent + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_slo_error_budget + expr: vector(1- 0.999) + labels: + openslo_slo_name: test-ratio-target-percent + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_slo_current_burn_rate + expr: | + openslo_sli_error_rate_5m{openslo_slo_name="test-ratio-target-percent", openslo_spec_version="openslo/v1"} + / on(openslo_slo_name, openslo_spec_version) group_left + openslo_slo_error_budget{openslo_slo_name="test-ratio-target-percent", openslo_spec_version="openslo/v1"} + labels: + openslo_slo_name: test-ratio-target-percent + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_slo_period_burn_rate + expr: | + openslo_sli_error_rate_30d{openslo_slo_name="test-ratio-target-percent", openslo_spec_version="openslo/v1"} + / on(openslo_slo_name, openslo_spec_version) group_left + openslo_slo_error_budget{openslo_slo_name="test-ratio-target-percent", openslo_spec_version="openslo/v1"} + labels: + openslo_slo_name: test-ratio-target-percent + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_slo_period_error_budget_remaining + expr: clamp_min(1 - openslo_slo_period_burn_rate{openslo_slo_name="test-ratio-target-percent", openslo_spec_version="openslo/v1"}, 0) + labels: + openslo_slo_name: test-ratio-target-percent + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - name: openslo-sli-recordings-test-ratio-target-percent + rules: + - record: openslo_sli_error_rate_5m + expr: | + 1 - ( + sum(rate(http_requests_total{status=~"2.."}[5m])) + ) / ( + sum(rate(http_requests_total[5m])) + ) + labels: + openslo_slo_name: test-ratio-target-percent + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_sli_error_rate_30m + expr: | + 1 - ( + sum(rate(http_requests_total{status=~"2.."}[30m])) + ) / ( + sum(rate(http_requests_total[30m])) + ) + labels: + openslo_slo_name: test-ratio-target-percent + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_sli_error_rate_1h + expr: | + 1 - ( + sum(rate(http_requests_total{status=~"2.."}[1h])) + ) / ( + sum(rate(http_requests_total[1h])) + ) + labels: + openslo_slo_name: test-ratio-target-percent + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_sli_error_rate_3h + expr: | + 1 - ( + sum(rate(http_requests_total{status=~"2.."}[3h])) + ) / ( + sum(rate(http_requests_total[3h])) + ) + labels: + openslo_slo_name: test-ratio-target-percent + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_sli_error_rate_6h + expr: | + 1 - ( + sum(rate(http_requests_total{status=~"2.."}[6h])) + ) / ( + sum(rate(http_requests_total[6h])) + ) + labels: + openslo_slo_name: test-ratio-target-percent + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_sli_error_rate_1d + expr: | + 1 - ( + sum(rate(http_requests_total{status=~"2.."}[1d])) + ) / ( + sum(rate(http_requests_total[1d])) + ) + labels: + openslo_slo_name: test-ratio-target-percent + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_sli_error_rate_3d + expr: | + 1 - ( + sum(rate(http_requests_total{status=~"2.."}[3d])) + ) / ( + sum(rate(http_requests_total[3d])) + ) + labels: + openslo_slo_name: test-ratio-target-percent + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_sli_error_rate_7d + expr: | + 1 - ( + sum(rate(http_requests_total{status=~"2.."}[7d])) + ) / ( + sum(rate(http_requests_total[7d])) + ) + labels: + openslo_slo_name: test-ratio-target-percent + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_sli_error_rate_28d + expr: | + 1 - ( + sum(rate(http_requests_total{status=~"2.."}[28d])) + ) / ( + sum(rate(http_requests_total[28d])) + ) + labels: + openslo_slo_name: test-ratio-target-percent + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_sli_error_rate_30d + expr: | + 1 - ( + sum(rate(http_requests_total{status=~"2.."}[30d])) + ) / ( + sum(rate(http_requests_total[30d])) + ) + labels: + openslo_slo_name: test-ratio-target-percent + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_sli_event_rate_5m + expr: sum(rate(http_requests_total[5m])) + labels: + openslo_slo_name: test-ratio-target-percent + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_sli_event_rate_30m + expr: sum(rate(http_requests_total[30m])) + labels: + openslo_slo_name: test-ratio-target-percent + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_sli_event_rate_1h + expr: sum(rate(http_requests_total[1h])) + labels: + openslo_slo_name: test-ratio-target-percent + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_sli_event_rate_3h + expr: sum(rate(http_requests_total[3h])) + labels: + openslo_slo_name: test-ratio-target-percent + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_sli_event_rate_6h + expr: sum(rate(http_requests_total[6h])) + labels: + openslo_slo_name: test-ratio-target-percent + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_sli_event_rate_1d + expr: sum(rate(http_requests_total[1d])) + labels: + openslo_slo_name: test-ratio-target-percent + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_sli_event_rate_3d + expr: sum(rate(http_requests_total[3d])) + labels: + openslo_slo_name: test-ratio-target-percent + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_sli_event_rate_7d + expr: sum(rate(http_requests_total[7d])) + labels: + openslo_slo_name: test-ratio-target-percent + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_sli_event_rate_28d + expr: sum(rate(http_requests_total[28d])) + labels: + openslo_slo_name: test-ratio-target-percent + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_sli_event_rate_30d + expr: sum(rate(http_requests_total[30d])) + labels: + openslo_slo_name: test-ratio-target-percent + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - name: openslo-status-recordings-test-ratio-target-percent + rules: + - record: openslo_slo_status + expr: | + ( + (openslo_slo_current_burn_rate{openslo_slo_name="test-ratio-target-percent", openslo_spec_version="openslo/v1"} >= bool 14.4) + * 3 + ) + or + ( + (openslo_slo_current_burn_rate{openslo_slo_name="test-ratio-target-percent", openslo_spec_version="openslo/v1"} >= bool 6) + and + (openslo_slo_current_burn_rate{openslo_slo_name="test-ratio-target-percent", openslo_spec_version="openslo/v1"} < bool 14.4) + * 2 + ) + or + ( + (openslo_slo_current_burn_rate{openslo_slo_name="test-ratio-target-percent", openslo_spec_version="openslo/v1"} >= bool 1) + and + (openslo_slo_current_burn_rate{openslo_slo_name="test-ratio-target-percent", openslo_spec_version="openslo/v1"} < bool 6) + * 1 + ) + labels: + openslo_slo_name: test-ratio-target-percent + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc diff --git a/internal/generator/prometheusgenerator/testdata/test-singleline-slo-rules.golden.yaml b/internal/generator/prometheusgenerator/testdata/test-singleline-slo-rules.golden.yaml new file mode 100644 index 0000000..add647d --- /dev/null +++ b/internal/generator/prometheusgenerator/testdata/test-singleline-slo-rules.golden.yaml @@ -0,0 +1,140 @@ +groups: + - name: openslo-info-recordings-test-singleline-slo + rules: + - record: openslo_slo_info + expr: vector(1) + labels: + openslo_slo_name: test-singleline-slo + openslo_slo_description: "Test SLO with single-line query" + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_slo_objective + expr: vector(0.999) + labels: + openslo_slo_name: test-singleline-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_slo_timewindow_days + expr: vector(30) + labels: + openslo_slo_name: test-singleline-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_slo_error_budget + expr: vector(1- 0.999) + labels: + openslo_slo_name: test-singleline-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_slo_current_burn_rate + expr: | + openslo_sli_error_rate_5m{openslo_slo_name="test-singleline-slo", openslo_spec_version="openslo/v1"} + / on(openslo_slo_name, openslo_spec_version) group_left + openslo_slo_error_budget{openslo_slo_name="test-singleline-slo", openslo_spec_version="openslo/v1"} + labels: + openslo_slo_name: test-singleline-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_slo_period_burn_rate + expr: | + openslo_sli_error_rate_30d{openslo_slo_name="test-singleline-slo", openslo_spec_version="openslo/v1"} + / on(openslo_slo_name, openslo_spec_version) group_left + openslo_slo_error_budget{openslo_slo_name="test-singleline-slo", openslo_spec_version="openslo/v1"} + labels: + openslo_slo_name: test-singleline-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_slo_period_error_budget_remaining + expr: clamp_min(1 - openslo_slo_period_burn_rate{openslo_slo_name="test-singleline-slo", openslo_spec_version="openslo/v1"}, 0) + labels: + openslo_slo_name: test-singleline-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - name: openslo-sli-recordings-test-singleline-slo + rules: + - record: openslo_sli_error_rate_5m + expr: up{service="test-svc"} + labels: + openslo_slo_name: test-singleline-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_sli_error_rate_30m + expr: up{service="test-svc"} + labels: + openslo_slo_name: test-singleline-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_sli_error_rate_1h + expr: up{service="test-svc"} + labels: + openslo_slo_name: test-singleline-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_sli_error_rate_3h + expr: up{service="test-svc"} + labels: + openslo_slo_name: test-singleline-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_sli_error_rate_6h + expr: up{service="test-svc"} + labels: + openslo_slo_name: test-singleline-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_sli_error_rate_1d + expr: up{service="test-svc"} + labels: + openslo_slo_name: test-singleline-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_sli_error_rate_3d + expr: up{service="test-svc"} + labels: + openslo_slo_name: test-singleline-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_sli_error_rate_7d + expr: up{service="test-svc"} + labels: + openslo_slo_name: test-singleline-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_sli_error_rate_28d + expr: up{service="test-svc"} + labels: + openslo_slo_name: test-singleline-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_sli_error_rate_30d + expr: up{service="test-svc"} + labels: + openslo_slo_name: test-singleline-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - name: openslo-status-recordings-test-singleline-slo + rules: + - record: openslo_slo_status + expr: | + ( + (openslo_slo_current_burn_rate{openslo_slo_name="test-singleline-slo", openslo_spec_version="openslo/v1"} >= bool 14.4) + * 3 + ) + or + ( + (openslo_slo_current_burn_rate{openslo_slo_name="test-singleline-slo", openslo_spec_version="openslo/v1"} >= bool 6) + and + (openslo_slo_current_burn_rate{openslo_slo_name="test-singleline-slo", openslo_spec_version="openslo/v1"} < bool 14.4) + * 2 + ) + or + ( + (openslo_slo_current_burn_rate{openslo_slo_name="test-singleline-slo", openslo_spec_version="openslo/v1"} >= bool 1) + and + (openslo_slo_current_burn_rate{openslo_slo_name="test-singleline-slo", openslo_spec_version="openslo/v1"} < bool 6) + * 1 + ) + labels: + openslo_slo_name: test-singleline-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc diff --git a/internal/generator/prometheusgenerator/testdata/test-tiered-slo-rules.golden.yaml b/internal/generator/prometheusgenerator/testdata/test-tiered-slo-rules.golden.yaml new file mode 100644 index 0000000..5724c35 --- /dev/null +++ b/internal/generator/prometheusgenerator/testdata/test-tiered-slo-rules.golden.yaml @@ -0,0 +1,294 @@ +groups: + - name: openslo-info-recordings-test-tiered-slo + rules: + - record: openslo_slo_info + expr: vector(1) + labels: + openslo_slo_name: test-tiered-slo + openslo_slo_description: "SLO with sloth-style 4 windows" + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_slo_objective + expr: vector(0.999) + labels: + openslo_slo_name: test-tiered-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_slo_timewindow_days + expr: vector(30) + labels: + openslo_slo_name: test-tiered-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_slo_error_budget + expr: vector(1- 0.999) + labels: + openslo_slo_name: test-tiered-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_slo_current_burn_rate + expr: | + openslo_sli_error_rate_5m{openslo_slo_name="test-tiered-slo", openslo_spec_version="openslo/v1"} + / on(openslo_slo_name, openslo_spec_version) group_left + openslo_slo_error_budget{openslo_slo_name="test-tiered-slo", openslo_spec_version="openslo/v1"} + labels: + openslo_slo_name: test-tiered-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_slo_period_burn_rate + expr: | + openslo_sli_error_rate_30d{openslo_slo_name="test-tiered-slo", openslo_spec_version="openslo/v1"} + / on(openslo_slo_name, openslo_spec_version) group_left + openslo_slo_error_budget{openslo_slo_name="test-tiered-slo", openslo_spec_version="openslo/v1"} + labels: + openslo_slo_name: test-tiered-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_slo_period_error_budget_remaining + expr: clamp_min(1 - openslo_slo_period_burn_rate{openslo_slo_name="test-tiered-slo", openslo_spec_version="openslo/v1"}, 0) + labels: + openslo_slo_name: test-tiered-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - name: openslo-sli-recordings-test-tiered-slo + rules: + - record: openslo_sli_error_rate_5m + expr: | + 1 - ( + sum(rate(http_requests_total{status=~"2.."}[5m])) + ) / ( + sum(rate(http_requests_total[5m])) + ) + labels: + openslo_slo_name: test-tiered-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_sli_error_rate_30m + expr: | + 1 - ( + sum(rate(http_requests_total{status=~"2.."}[30m])) + ) / ( + sum(rate(http_requests_total[30m])) + ) + labels: + openslo_slo_name: test-tiered-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_sli_error_rate_1h + expr: | + 1 - ( + sum(rate(http_requests_total{status=~"2.."}[1h])) + ) / ( + sum(rate(http_requests_total[1h])) + ) + labels: + openslo_slo_name: test-tiered-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_sli_error_rate_3h + expr: | + 1 - ( + sum(rate(http_requests_total{status=~"2.."}[3h])) + ) / ( + sum(rate(http_requests_total[3h])) + ) + labels: + openslo_slo_name: test-tiered-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_sli_error_rate_6h + expr: | + 1 - ( + sum(rate(http_requests_total{status=~"2.."}[6h])) + ) / ( + sum(rate(http_requests_total[6h])) + ) + labels: + openslo_slo_name: test-tiered-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_sli_error_rate_1d + expr: | + 1 - ( + sum(rate(http_requests_total{status=~"2.."}[1d])) + ) / ( + sum(rate(http_requests_total[1d])) + ) + labels: + openslo_slo_name: test-tiered-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_sli_error_rate_3d + expr: | + 1 - ( + sum(rate(http_requests_total{status=~"2.."}[3d])) + ) / ( + sum(rate(http_requests_total[3d])) + ) + labels: + openslo_slo_name: test-tiered-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_sli_error_rate_7d + expr: | + 1 - ( + sum(rate(http_requests_total{status=~"2.."}[7d])) + ) / ( + sum(rate(http_requests_total[7d])) + ) + labels: + openslo_slo_name: test-tiered-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_sli_error_rate_28d + expr: | + 1 - ( + sum(rate(http_requests_total{status=~"2.."}[28d])) + ) / ( + sum(rate(http_requests_total[28d])) + ) + labels: + openslo_slo_name: test-tiered-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_sli_error_rate_30d + expr: | + 1 - ( + sum(rate(http_requests_total{status=~"2.."}[30d])) + ) / ( + sum(rate(http_requests_total[30d])) + ) + labels: + openslo_slo_name: test-tiered-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_sli_event_rate_5m + expr: sum(rate(http_requests_total[5m])) + labels: + openslo_slo_name: test-tiered-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_sli_event_rate_30m + expr: sum(rate(http_requests_total[30m])) + labels: + openslo_slo_name: test-tiered-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_sli_event_rate_1h + expr: sum(rate(http_requests_total[1h])) + labels: + openslo_slo_name: test-tiered-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_sli_event_rate_3h + expr: sum(rate(http_requests_total[3h])) + labels: + openslo_slo_name: test-tiered-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_sli_event_rate_6h + expr: sum(rate(http_requests_total[6h])) + labels: + openslo_slo_name: test-tiered-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_sli_event_rate_1d + expr: sum(rate(http_requests_total[1d])) + labels: + openslo_slo_name: test-tiered-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_sli_event_rate_3d + expr: sum(rate(http_requests_total[3d])) + labels: + openslo_slo_name: test-tiered-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_sli_event_rate_7d + expr: sum(rate(http_requests_total[7d])) + labels: + openslo_slo_name: test-tiered-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_sli_event_rate_28d + expr: sum(rate(http_requests_total[28d])) + labels: + openslo_slo_name: test-tiered-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - record: openslo_sli_event_rate_30d + expr: sum(rate(http_requests_total[30d])) + labels: + openslo_slo_name: test-tiered-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - name: openslo-status-recordings-test-tiered-slo + rules: + - record: openslo_slo_status + expr: | + ( + (openslo_slo_current_burn_rate{openslo_slo_name="test-tiered-slo", openslo_spec_version="openslo/v1"} >= bool 14.4) + * 3 + ) + or + ( + (openslo_slo_current_burn_rate{openslo_slo_name="test-tiered-slo", openslo_spec_version="openslo/v1"} >= bool 6) + and + (openslo_slo_current_burn_rate{openslo_slo_name="test-tiered-slo", openslo_spec_version="openslo/v1"} < bool 14.4) + * 2 + ) + or + ( + (openslo_slo_current_burn_rate{openslo_slo_name="test-tiered-slo", openslo_spec_version="openslo/v1"} >= bool 1) + and + (openslo_slo_current_burn_rate{openslo_slo_name="test-tiered-slo", openslo_spec_version="openslo/v1"} < bool 6) + * 1 + ) + labels: + openslo_slo_name: test-tiered-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + - name: openslo-alerts-test-tiered-slo + rules: + - alert: TestTieredSloMultiWindowMultiBurnRatePage + expr: |- + ( + openslo_sli_error_rate_5m{openslo_slo_name="test-tiered-slo"} / (1 - openslo_slo_objective{openslo_slo_name="test-tiered-slo"}) >= 14.400000 + and + openslo_sli_error_rate_1h{openslo_slo_name="test-tiered-slo"} / (1 - openslo_slo_objective{openslo_slo_name="test-tiered-slo"}) >= 14.400000) + or + ( + openslo_sli_error_rate_30m{openslo_slo_name="test-tiered-slo"} / (1 - openslo_slo_objective{openslo_slo_name="test-tiered-slo"}) >= 6.000000 + and + openslo_sli_error_rate_6h{openslo_slo_name="test-tiered-slo"} / (1 - openslo_slo_objective{openslo_slo_name="test-tiered-slo"}) >= 6.000000) + for: 2m + labels: + openslo_alert_severity: page + openslo_notification_target: engineers + openslo_slo_name: test-tiered-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + annotations: + summary: "Page multi-window multi-burn rate alert for SLO test-tiered-slo" + description: "Threshold 14.4x and 6.0x over 5m and 1h and 30m and 6h" + - alert: TestTieredSloMultiWindowMultiBurnRateTicket + expr: |- + ( + openslo_sli_error_rate_2h{openslo_slo_name="test-tiered-slo"} / (1 - openslo_slo_objective{openslo_slo_name="test-tiered-slo"}) >= 3.000000 + and + openslo_sli_error_rate_1d{openslo_slo_name="test-tiered-slo"} / (1 - openslo_slo_objective{openslo_slo_name="test-tiered-slo"}) >= 3.000000) + or + ( + openslo_sli_error_rate_6h{openslo_slo_name="test-tiered-slo"} / (1 - openslo_slo_objective{openslo_slo_name="test-tiered-slo"}) >= 1.000000 + and + openslo_sli_error_rate_3d{openslo_slo_name="test-tiered-slo"} / (1 - openslo_slo_objective{openslo_slo_name="test-tiered-slo"}) >= 1.000000) + for: 15m + labels: + openslo_alert_severity: ticket + openslo_notification_target: engineers + openslo_slo_name: test-tiered-slo + openslo_spec_version: openslo/v1 + openslo_service_name: test-svc + annotations: + summary: "Ticket multi-window multi-burn rate alert for SLO test-tiered-slo" + description: "Threshold 3.0x and 1.0x over 2h and 1d and 6h and 3d" diff --git a/internal/generator/prometheusgenerator/testdata/tiered-alerts.yaml b/internal/generator/prometheusgenerator/testdata/tiered-alerts.yaml new file mode 100644 index 0000000..0f26e03 --- /dev/null +++ b/internal/generator/prometheusgenerator/testdata/tiered-alerts.yaml @@ -0,0 +1,205 @@ +apiVersion: openslo/v1 +kind: AlertCondition +metadata: + name: tiered-page-fast-5m +spec: + severity: page + condition: + kind: multi-window-multi-burn-rate + op: gte + threshold: 14.4 + lookbackWindow: 5m + alertAfter: 2m +--- +apiVersion: openslo/v1 +kind: AlertCondition +metadata: + name: tiered-page-fast-1h +spec: + severity: page + condition: + kind: multi-window-multi-burn-rate + op: gte + threshold: 14.4 + lookbackWindow: 1h + alertAfter: 2m +--- +apiVersion: openslo/v1 +kind: AlertCondition +metadata: + name: tiered-page-slow-30m +spec: + severity: page + condition: + kind: multi-window-multi-burn-rate + op: gte + threshold: 6 + lookbackWindow: 30m + alertAfter: 5m +--- +apiVersion: openslo/v1 +kind: AlertCondition +metadata: + name: tiered-page-slow-6h +spec: + severity: page + condition: + kind: multi-window-multi-burn-rate + op: gte + threshold: 6 + lookbackWindow: 6h + alertAfter: 5m +--- +apiVersion: openslo/v1 +kind: AlertCondition +metadata: + name: tiered-ticket-fast-2h +spec: + severity: ticket + condition: + kind: multi-window-multi-burn-rate + op: gte + threshold: 3 + lookbackWindow: 2h + alertAfter: 15m +--- +apiVersion: openslo/v1 +kind: AlertCondition +metadata: + name: tiered-ticket-fast-1d +spec: + severity: ticket + condition: + kind: multi-window-multi-burn-rate + op: gte + threshold: 3 + lookbackWindow: 1d + alertAfter: 15m +--- +apiVersion: openslo/v1 +kind: AlertCondition +metadata: + name: tiered-ticket-slow-6h +spec: + severity: ticket + condition: + kind: multi-window-multi-burn-rate + op: gte + threshold: 1 + lookbackWindow: 6h + alertAfter: 30m +--- +apiVersion: openslo/v1 +kind: AlertCondition +metadata: + name: tiered-ticket-slow-3d +spec: + severity: ticket + condition: + kind: multi-window-multi-burn-rate + op: gte + threshold: 1 + lookbackWindow: 3d + alertAfter: 30m +--- +apiVersion: openslo/v1 +kind: AlertPolicy +metadata: + name: tiered-page-fast-5m-policy +spec: + alertWhenBreaching: true + conditions: + - conditionRef: tiered-page-fast-5m + notificationTargets: + - targetRef: engineers +--- +apiVersion: openslo/v1 +kind: AlertPolicy +metadata: + name: tiered-page-fast-1h-policy +spec: + alertWhenBreaching: true + conditions: + - conditionRef: tiered-page-fast-1h + notificationTargets: + - targetRef: engineers +--- +apiVersion: openslo/v1 +kind: AlertPolicy +metadata: + name: tiered-page-slow-30m-policy +spec: + alertWhenBreaching: true + conditions: + - conditionRef: tiered-page-slow-30m + notificationTargets: + - targetRef: engineers +--- +apiVersion: openslo/v1 +kind: AlertPolicy +metadata: + name: tiered-page-slow-6h-policy +spec: + alertWhenBreaching: true + conditions: + - conditionRef: tiered-page-slow-6h + notificationTargets: + - targetRef: engineers +--- +apiVersion: openslo/v1 +kind: AlertPolicy +metadata: + name: tiered-ticket-fast-2h-policy +spec: + alertWhenBreaching: true + conditions: + - conditionRef: tiered-ticket-fast-2h + notificationTargets: + - targetRef: engineers +--- +apiVersion: openslo/v1 +kind: AlertPolicy +metadata: + name: tiered-ticket-fast-1d-policy +spec: + alertWhenBreaching: true + conditions: + - conditionRef: tiered-ticket-fast-1d + notificationTargets: + - targetRef: engineers +--- +apiVersion: openslo/v1 +kind: AlertPolicy +metadata: + name: tiered-ticket-slow-6h-policy +spec: + alertWhenBreaching: true + conditions: + - conditionRef: tiered-ticket-slow-6h + notificationTargets: + - targetRef: engineers +--- +apiVersion: openslo/v1 +kind: AlertPolicy +metadata: + name: tiered-ticket-slow-3d-policy +spec: + alertWhenBreaching: true + conditions: + - conditionRef: tiered-ticket-slow-3d + notificationTargets: + - targetRef: engineers +--- +apiVersion: openslo/v1 +kind: AlertNotificationTarget +metadata: + name: engineers +spec: + target: engineers +--- +apiVersion: openslo/v1 +kind: Service +metadata: + name: test-svc +spec: + description: test service diff --git a/internal/generator/prometheusgenerator/testdata/tiered-slo.yaml b/internal/generator/prometheusgenerator/testdata/tiered-slo.yaml new file mode 100644 index 0000000..cf7044f --- /dev/null +++ b/internal/generator/prometheusgenerator/testdata/tiered-slo.yaml @@ -0,0 +1,39 @@ +apiVersion: openslo/v1 +kind: SLO +metadata: + name: test-tiered-slo +spec: + description: SLO with sloth-style 4 windows + service: test-svc + indicator: + metadata: + name: test-tiered-sli + spec: + ratioMetric: + counter: true + good: + metricSource: + type: Prometheus + spec: + query: sum(rate(http_requests_total{status=~"2.."}[{{.Window}}])) + total: + metricSource: + type: Prometheus + spec: + query: sum(rate(http_requests_total[{{.Window}}])) + budgetingMethod: Occurrences + timeWindow: + - duration: 30d + isRolling: true + objectives: + - displayName: "99.9%" + target: 0.999 + alertPolicies: + - alertPolicyRef: tiered-page-fast-5m-policy + - alertPolicyRef: tiered-page-fast-1h-policy + - alertPolicyRef: tiered-page-slow-30m-policy + - alertPolicyRef: tiered-page-slow-6h-policy + - alertPolicyRef: tiered-ticket-fast-2h-policy + - alertPolicyRef: tiered-ticket-fast-1d-policy + - alertPolicyRef: tiered-ticket-slow-6h-policy + - alertPolicyRef: tiered-ticket-slow-3d-policy diff --git a/internal/testutil/golden.go b/internal/testutil/golden.go new file mode 100644 index 0000000..295db29 --- /dev/null +++ b/internal/testutil/golden.go @@ -0,0 +1,55 @@ +// Package testutil holds test helpers shared across opensloctl packages. +package testutil + +import ( + "fmt" + "path/filepath" + "strings" + "testing" + + "github.com/sebdah/goldie/v2" +) + +// AssertGolden compares got to fixtureDir/.golden.. +// +// name MUST include a file extension (e.g. "rules.yaml"). Extensions known +// to goldie's default splitter are accepted; the suffix is applied verbatim +// from the extension portion only. Bare names like "rules" are rejected. +// +// Examples: +// +// AssertGolden(t, "testdata", "rules.yaml", got) +// // → testdata/rules.golden.yaml +// +// AssertGolden(t, "testdata", "out.json", got) +// // → testdata/out.golden.json +// +// Update fixtures: `go test ./... -update`. +func AssertGolden(t *testing.T, fixtureDir, name string, got []byte) { + t.Helper() + + base, suffix, err := goldenBaseAndSuffix(name) + if err != nil { + t.Fatalf("AssertGolden: %v", err) + } + + g := goldie.New(t, + goldie.WithFixtureDir(fixtureDir), + goldie.WithNameSuffix(suffix), + ) + g.Assert(t, base, got) +} + +// goldenBaseAndSuffix splits name into (base, ".golden"+ext). Empty extension +// or empty base is an error. +func goldenBaseAndSuffix(name string) (base, suffix string, err error) { + ext := filepath.Ext(name) + if ext == "" { + return "", "", fmt.Errorf("name %q must include a file extension (e.g. .yaml)", name) + } + base = strings.TrimSuffix(name, ext) + if base == "" { + return "", "", fmt.Errorf("name %q has empty base before extension %q", name, ext) + } + return base, ".golden" + ext, nil +} diff --git a/internal/testutil/golden_test.go b/internal/testutil/golden_test.go new file mode 100644 index 0000000..4bb71cb --- /dev/null +++ b/internal/testutil/golden_test.go @@ -0,0 +1,79 @@ +package testutil + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGoldenBaseAndSuffix(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + input string + wantBase string + wantSuf string + wantError bool + }{ + { + name: "yaml extension", + input: "rules.yaml", + wantBase: "rules", + wantSuf: ".golden.yaml", + }, + { + name: "json extension", + input: "out.json", + wantBase: "out", + wantSuf: ".golden.json", + }, + { + name: "txt extension", + input: "snapshot.txt", + wantBase: "snapshot", + wantSuf: ".golden.txt", + }, + { + name: "deep path", + input: "alerts/recording.slo.yaml", + wantBase: "alerts/recording.slo", + wantSuf: ".golden.yaml", + }, + { + name: "no extension rejected", + input: "rules", + wantError: true, + }, + { + name: "empty string rejected", + input: "", + wantError: true, + }, + { + name: "extension only rejected", + input: ".yaml", + wantError: true, + }, + { + name: "dot only rejected", + input: ".", + wantError: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + base, suffix, err := goldenBaseAndSuffix(tt.input) + if tt.wantError { + assert.Error(t, err) + return + } + assert.NoError(t, err) + assert.Equal(t, tt.wantBase, base) + assert.Equal(t, tt.wantSuf, suffix) + }) + } +} diff --git a/mise.toml b/mise.toml index 0103e4e..c524d26 100644 --- a/mise.toml +++ b/mise.toml @@ -2,3 +2,4 @@ go = "1.26" golangci-lint = "latest" "github:open-telemetry/weaver" = "latest" +promtool = "latest" diff --git a/pkg/semconv/semconv_gen.go b/pkg/semconv/semconv_gen.go index 0900d45..fdc8811 100644 --- a/pkg/semconv/semconv_gen.go +++ b/pkg/semconv/semconv_gen.go @@ -6,6 +6,12 @@ package semconv // Metric names from the openslo semantic convention registry. const ( + // The alert severity (e.g., page, ticket) attached to SLO alert rules. + ATTRIBUTE_OPENSLO_ALERT_SEVERITY = "openslo.alert.severity" + + // The notification target for an alert (e.g., pagerduty, slack, engineers). + ATTRIBUTE_OPENSLO_NOTIFICATION_TARGET = "openslo.notification.target" + // The SLO objective expressed as a decimal value (e.g., 0.999). ATTRIBUTE_OPENSLO_OBJECTIVE_DECIMAL = "openslo.objective.decimal" @@ -15,6 +21,9 @@ const ( // The name of the service the SLO belongs to. ATTRIBUTE_OPENSLO_SERVICE_NAME = "openslo.service.name" + // The free-form description of the SLO as defined in the OpenSlo spec's `spec.description` field, folded to a single line. + ATTRIBUTE_OPENSLO_SLO_DESCRIPTION = "openslo.slo.description" + // The name of the SLO as defined in the OpenSlo spec. ATTRIBUTE_OPENSLO_SLO_NAME = "openslo.slo.name" @@ -54,6 +63,39 @@ const ( // SLI error rate over a 7-day window. METRIC_OPENSLO_SLI_ERROR_RATE_7D = "openslo.sli.error_rate_7d" + // SLI event rate over a 1-day window. Emitted only for RatioMetric SLIs. + METRIC_OPENSLO_SLI_EVENT_RATE_1D = "openslo.sli.event_rate_1d" + + // SLI event rate over a 1-hour window. Emitted only for RatioMetric SLIs. + METRIC_OPENSLO_SLI_EVENT_RATE_1H = "openslo.sli.event_rate_1h" + + // SLI event rate over a 28-day window. Emitted only for RatioMetric SLIs. + METRIC_OPENSLO_SLI_EVENT_RATE_28D = "openslo.sli.event_rate_28d" + + // SLI event rate over a 2-hour window. Emitted only for RatioMetric SLIs. + METRIC_OPENSLO_SLI_EVENT_RATE_2H = "openslo.sli.event_rate_2h" + + // SLI event rate over a 30-day window. Emitted only for RatioMetric SLIs. + METRIC_OPENSLO_SLI_EVENT_RATE_30D = "openslo.sli.event_rate_30d" + + // SLI event rate over a 30-minute window. Emitted only for RatioMetric SLIs. + METRIC_OPENSLO_SLI_EVENT_RATE_30M = "openslo.sli.event_rate_30m" + + // SLI event rate over a 3-day window. Emitted only for RatioMetric SLIs. + METRIC_OPENSLO_SLI_EVENT_RATE_3D = "openslo.sli.event_rate_3d" + + // SLI event rate (events per second) over a 5-minute window. Emitted only for RatioMetric SLIs. + METRIC_OPENSLO_SLI_EVENT_RATE_5M = "openslo.sli.event_rate_5m" + + // SLI event rate over a 6-hour window. Emitted only for RatioMetric SLIs. + METRIC_OPENSLO_SLI_EVENT_RATE_6H = "openslo.sli.event_rate_6h" + + // SLI event rate over a 7-day window. Emitted only for RatioMetric SLIs. + METRIC_OPENSLO_SLI_EVENT_RATE_7D = "openslo.sli.event_rate_7d" + + // Instantaneous error-budget burn rate = 5-minute SLI error rate divided by the error budget. + METRIC_OPENSLO_SLO_CURRENT_BURN_RATE = "openslo.slo.current_burn_rate" + // The error budget calculated as 1 minus the objective. METRIC_OPENSLO_SLO_ERROR_BUDGET = "openslo.slo.error_budget" @@ -63,6 +105,16 @@ const ( // The target SLI objective (e.g., 0.999 for 99.9% availability). METRIC_OPENSLO_SLO_OBJECTIVE = "openslo.slo.objective" + // Period error-budget burn rate = full-window SLI error rate (typically 30d) divided by the error budget. + METRIC_OPENSLO_SLO_PERIOD_BURN_RATE = "openslo.slo.period_burn_rate" + + // Remaining error budget ratio over the full period = 1 minus period_burn_rate (1 = full budget remaining). + METRIC_OPENSLO_SLO_PERIOD_ERROR_BUDGET_REMAINING = "openslo.slo.period_error_budget_remaining" + + // Categorical SLO health state derived from the current burn rate against overridable thresholds. Values: 0=Healthy (burn=breached). Defaults follow Google SRE workbook reference points (1/6/14.4x) and can be overridden per SLO via threshold.status.openslo.com/{warning, critical,breached} annotations. Emitted only for SLOs that reference one or more AlertPolicies. + + METRIC_OPENSLO_SLO_STATUS = "openslo.slo.status" + // The SLO time window duration expressed as a number of days. METRIC_OPENSLO_SLO_TIMEWINDOW_DAYS = "openslo.slo.timewindow_days" ) diff --git a/pkg/specstore/loader_test.go b/pkg/specstore/loader_test.go deleted file mode 100644 index b28ee12..0000000 --- a/pkg/specstore/loader_test.go +++ /dev/null @@ -1,603 +0,0 @@ -package specstore - -import ( - "encoding/json" - "os" - "path/filepath" - "testing" - - "github.com/OpenSLO/go-sdk/pkg/openslo" - v1 "github.com/OpenSLO/go-sdk/pkg/openslo/v1" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func ptr[T any](v T) *T { return &v } - -func TestNewOpenSLOSpecs(t *testing.T) { - t.Parallel() - - specs := NewOpenSLOSpecs() - - require.NotNil(t, specs) - require.NotNil(t, specs.V1.Services) - require.NotNil(t, specs.V1.SLOs) - require.NotNil(t, specs.V1.SLIs) - require.NotNil(t, specs.V1.DataSources) - require.NotNil(t, specs.V1.AlertPolices) - require.NotNil(t, specs.V1.AlertConditions) - require.NotNil(t, specs.V1.AlertNotificationTargets) - - assert.Empty(t, specs.V1.Services) - assert.Empty(t, specs.V1.SLOs) - assert.Empty(t, specs.V1.SLIs) - assert.Empty(t, specs.V1.DataSources) - assert.Empty(t, specs.V1.AlertPolices) - assert.Empty(t, specs.V1.AlertConditions) - assert.Empty(t, specs.V1.AlertNotificationTargets) -} - -func TestStoreSpec(t *testing.T) { - t.Parallel() - - t.Run("AllKinds", func(t *testing.T) { - t.Parallel() - - tests := []struct { - name string - object openslo.Object - expKind string - expName string - }{ - { - name: "Service", - object: v1.NewService( - v1.Metadata{Name: "test-svc"}, - v1.ServiceSpec{Description: "test service"}, - ), - expKind: "Services", - expName: "test-svc", - }, - { - name: "SLO", - object: v1.NewSLO( - v1.Metadata{Name: "test-slo"}, - v1.SLOSpec{ - Service: "test-svc", - IndicatorRef: ptr("test-sli"), - BudgetingMethod: v1.SLOBudgetingMethodOccurrences, - TimeWindow: []v1.SLOTimeWindow{{Duration: v1.NewDurationShorthand(30, v1.DurationShorthandUnitDay), IsRolling: true}}, - Objectives: []v1.SLOObjective{{Target: ptr(0.999)}}, - }, - ), - expKind: "SLOs", - expName: "test-slo", - }, - { - name: "SLI", - object: v1.NewSLI( - v1.Metadata{Name: "test-sli"}, - v1.SLISpec{ - ThresholdMetric: &v1.SLIMetricSpec{ - MetricSource: v1.SLIMetricSource{ - Type: "Prometheus", - Spec: map[string]any{"query": "up"}, - }, - }, - }, - ), - expKind: "SLIs", - expName: "test-sli", - }, - { - name: "DataSource", - object: v1.NewDataSource( - v1.Metadata{Name: "test-ds"}, - v1.DataSourceSpec{ - Type: "Prometheus", - ConnectionDetails: json.RawMessage(`{"url":"http://prom:9090"}`), - }, - ), - expKind: "DataSources", - expName: "test-ds", - }, - { - name: "AlertPolicy", - object: v1.NewAlertPolicy( - v1.Metadata{Name: "test-ap"}, - v1.AlertPolicySpec{ - AlertWhenBreaching: true, - Conditions: []v1.AlertPolicyCondition{ - {AlertPolicyConditionRef: &v1.AlertPolicyConditionRef{ConditionRef: "test-ac"}}, - }, - NotificationTargets: []v1.AlertPolicyNotificationTarget{ - {AlertPolicyNotificationTargetRef: &v1.AlertPolicyNotificationTargetRef{TargetRef: "test-ant"}}, - }, - }, - ), - expKind: "AlertPolices", - expName: "test-ap", - }, - { - name: "AlertCondition", - object: v1.NewAlertCondition( - v1.Metadata{Name: "test-ac"}, - v1.AlertConditionSpec{ - Severity: "page", - Condition: v1.AlertConditionType{ - Kind: v1.AlertConditionKindBurnRate, - Operator: v1.OperatorLTE, - Threshold: ptr(2.0), - LookbackWindow: v1.NewDurationShorthand(1, v1.DurationShorthandUnitHour), - }, - }, - ), - expKind: "AlertConditions", - expName: "test-ac", - }, - { - name: "AlertNotificationTarget", - object: v1.NewAlertNotificationTarget( - v1.Metadata{Name: "test-ant"}, - v1.AlertNotificationTargetSpec{ - Target: "pagerduty", - Description: "test target", - }, - ), - expKind: "AlertNotificationTargets", - expName: "test-ant", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - - specs := NewOpenSLOSpecs() - err := specs.StoreSpec(tt.object) - require.NoError(t, err) - - switch tt.expKind { - case "Services": - assert.Contains(t, specs.V1.Services, tt.expName) - case "SLOs": - assert.Contains(t, specs.V1.SLOs, tt.expName) - case "SLIs": - assert.Contains(t, specs.V1.SLIs, tt.expName) - case "DataSources": - assert.Contains(t, specs.V1.DataSources, tt.expName) - case "AlertPolices": - assert.Contains(t, specs.V1.AlertPolices, tt.expName) - case "AlertConditions": - assert.Contains(t, specs.V1.AlertConditions, tt.expName) - case "AlertNotificationTargets": - assert.Contains(t, specs.V1.AlertNotificationTargets, tt.expName) - } - }) - } - }) - - t.Run("Duplicates", func(t *testing.T) { - t.Parallel() - - tests := []struct { - name string - first openslo.Object - second openslo.Object - wantError bool - }{ - { - name: "Service", - first: v1.NewService(v1.Metadata{Name: "dup-svc"}, v1.ServiceSpec{}), - second: v1.NewService(v1.Metadata{Name: "dup-svc"}, v1.ServiceSpec{Description: "second"}), - }, - { - name: "SLO", - first: v1.NewSLO(v1.Metadata{Name: "dup-slo"}, v1.SLOSpec{ - Service: "svc", IndicatorRef: ptr("sli"), BudgetingMethod: v1.SLOBudgetingMethodOccurrences, - TimeWindow: []v1.SLOTimeWindow{{Duration: v1.NewDurationShorthand(30, v1.DurationShorthandUnitDay), IsRolling: true}}, - Objectives: []v1.SLOObjective{{Target: ptr(0.999)}}, - }), - second: v1.NewSLO(v1.Metadata{Name: "dup-slo"}, v1.SLOSpec{ - Service: "svc", IndicatorRef: ptr("sli"), BudgetingMethod: v1.SLOBudgetingMethodOccurrences, - TimeWindow: []v1.SLOTimeWindow{{Duration: v1.NewDurationShorthand(30, v1.DurationShorthandUnitDay), IsRolling: true}}, - Objectives: []v1.SLOObjective{{Target: ptr(0.99)}}, - }), - }, - { - name: "SLI", - first: v1.NewSLI(v1.Metadata{Name: "dup-sli"}, v1.SLISpec{ - ThresholdMetric: &v1.SLIMetricSpec{MetricSource: v1.SLIMetricSource{Type: "Prometheus", Spec: map[string]any{"query": "up"}}}, - }), - second: v1.NewSLI(v1.Metadata{Name: "dup-sli"}, v1.SLISpec{ - ThresholdMetric: &v1.SLIMetricSpec{MetricSource: v1.SLIMetricSource{Type: "Prometheus", Spec: map[string]any{"query": "up"}}}, - }), - }, - { - name: "DataSource", - first: v1.NewDataSource(v1.Metadata{Name: "dup-ds"}, v1.DataSourceSpec{Type: "Prometheus", ConnectionDetails: json.RawMessage(`{}`)}), - second: v1.NewDataSource(v1.Metadata{Name: "dup-ds"}, v1.DataSourceSpec{Type: "Prometheus", ConnectionDetails: json.RawMessage(`{}`)}), - }, - { - name: "AlertPolicy", - first: v1.NewAlertPolicy(v1.Metadata{Name: "dup-ap"}, v1.AlertPolicySpec{ - AlertWhenBreaching: true, - Conditions: []v1.AlertPolicyCondition{{AlertPolicyConditionRef: &v1.AlertPolicyConditionRef{ConditionRef: "ac"}}}, - NotificationTargets: []v1.AlertPolicyNotificationTarget{{AlertPolicyNotificationTargetRef: &v1.AlertPolicyNotificationTargetRef{TargetRef: "ant"}}}, - }), - second: v1.NewAlertPolicy(v1.Metadata{Name: "dup-ap"}, v1.AlertPolicySpec{ - AlertWhenBreaching: true, - Conditions: []v1.AlertPolicyCondition{{AlertPolicyConditionRef: &v1.AlertPolicyConditionRef{ConditionRef: "ac"}}}, - NotificationTargets: []v1.AlertPolicyNotificationTarget{{AlertPolicyNotificationTargetRef: &v1.AlertPolicyNotificationTargetRef{TargetRef: "ant"}}}, - }), - }, - { - name: "AlertCondition", - first: v1.NewAlertCondition(v1.Metadata{Name: "dup-ac"}, v1.AlertConditionSpec{ - Severity: "page", - Condition: v1.AlertConditionType{Kind: v1.AlertConditionKindBurnRate, Operator: v1.OperatorLTE, Threshold: ptr(2.0), LookbackWindow: v1.NewDurationShorthand(1, v1.DurationShorthandUnitHour)}, - }), - second: v1.NewAlertCondition(v1.Metadata{Name: "dup-ac"}, v1.AlertConditionSpec{ - Severity: "warn", - Condition: v1.AlertConditionType{Kind: v1.AlertConditionKindBurnRate, Operator: v1.OperatorLTE, Threshold: ptr(2.0), LookbackWindow: v1.NewDurationShorthand(1, v1.DurationShorthandUnitHour)}, - }), - }, - { - name: "AlertNotificationTarget", - first: v1.NewAlertNotificationTarget(v1.Metadata{Name: "dup-ant"}, v1.AlertNotificationTargetSpec{Target: "pagerduty"}), - second: v1.NewAlertNotificationTarget(v1.Metadata{Name: "dup-ant"}, v1.AlertNotificationTargetSpec{Target: "slack"}), - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - - specs := NewOpenSLOSpecs() - require.NoError(t, specs.StoreSpec(tt.first)) - - err := specs.StoreSpec(tt.second) - assert.Error(t, err) - assert.Contains(t, err.Error(), "duplicate spec") - }) - } - }) - - t.Run("EdgeCases", func(t *testing.T) { - t.Parallel() - - tests := []struct { - name string - objects []openslo.Object - checkFunc func(t *testing.T, specs *OpenSLOSpecs) - wantErr []bool - errContains []string - }{ - { - name: "multiple kinds", - objects: []openslo.Object{ - v1.NewService(v1.Metadata{Name: "multi-svc"}, v1.ServiceSpec{Description: "multi test service"}), - v1.NewSLI(v1.Metadata{Name: "multi-sli"}, v1.SLISpec{ - ThresholdMetric: &v1.SLIMetricSpec{ - MetricSource: v1.SLIMetricSource{Type: "Prometheus", Spec: map[string]any{"query": "up"}}, - }, - }), - v1.NewSLO(v1.Metadata{Name: "multi-slo"}, v1.SLOSpec{ - Service: "multi-svc", IndicatorRef: ptr("multi-sli"), BudgetingMethod: v1.SLOBudgetingMethodOccurrences, - TimeWindow: []v1.SLOTimeWindow{{Duration: v1.NewDurationShorthand(30, v1.DurationShorthandUnitDay), IsRolling: true}}, - Objectives: []v1.SLOObjective{{Target: ptr(0.999)}}, - }), - }, - wantErr: []bool{false, false, false}, - checkFunc: func(t *testing.T, specs *OpenSLOSpecs) { - t.Helper() - assert.Contains(t, specs.V1.Services, "multi-svc") - assert.Contains(t, specs.V1.SLIs, "multi-sli") - assert.Contains(t, specs.V1.SLOs, "multi-slo") - }, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - - specs := NewOpenSLOSpecs() - for i, obj := range tt.objects { - err := specs.StoreSpec(obj) - if len(tt.wantErr) > i { - if tt.wantErr[i] { - assert.Error(t, err) - if len(tt.errContains) > i { - assert.Contains(t, err.Error(), tt.errContains[i]) - } - } else { - assert.NoError(t, err) - } - } - } - if tt.checkFunc != nil { - tt.checkFunc(t, specs) - } - }) - } - }) -} - -func TestGetSpecs(t *testing.T) { - t.Parallel() - - testdata := filepath.Join("testdata") - - tests := []struct { - name string - files []string - recursive bool - wantErr bool - errContains string - checkFunc func(t *testing.T, specs *OpenSLOSpecs) - wantService string - wantSLO string - wantSLI string - wantDataSource string - wantAlertPolicy string - }{ - { - name: "single service file", - files: []string{filepath.Join(testdata, "service.yaml")}, - wantService: "test-service", - }, - { - name: "single slo file", - files: []string{filepath.Join(testdata, "slo.yaml"), filepath.Join(testdata, "service.yaml")}, - wantSLO: "test-slo", - }, - { - name: "multiple individual files", - files: []string{filepath.Join(testdata, "service.yaml"), filepath.Join(testdata, "sli.yaml")}, - wantService: "test-service", - wantSLI: "test-sli", - }, - { - name: "directory non-recursive", - files: []string{testdata}, - recursive: false, - wantService: "test-service", - }, - { - name: "directory recursive", - files: []string{testdata}, - recursive: true, - wantService: "test-service", - }, - { - name: "multi-document yaml", - files: []string{filepath.Join(testdata, "multi-doc.yaml")}, - wantService: "svc-a", - wantSLO: "slo-a", - }, - { - name: "populates all kinds", - files: []string{testdata}, - recursive: true, - wantService: "test-service", - wantSLO: "test-slo", - wantSLI: "test-sli", - wantDataSource: "test-datasource", - wantAlertPolicy: "test-alert-policy", - }, - { - name: "duplicate files deduplicated", - files: []string{filepath.Join(testdata, "service.yaml"), filepath.Join(testdata, "service.yaml")}, - wantService: "test-service", - checkFunc: func(t *testing.T, specs *OpenSLOSpecs) { - t.Helper() - assert.Len(t, specs.V1.Services, 1) - }, - }, - { - name: "empty filenames", - files: []string{}, - wantErr: true, - errContains: "error detecting files", - }, - { - name: "missing file", - files: []string{"nonexistent.yaml"}, - wantErr: true, - errContains: "error detecting files", - }, - { - name: "invalid yaml skipped", - files: []string{filepath.Join(testdata, "invalid.yaml")}, - checkFunc: func(t *testing.T, specs *OpenSLOSpecs) { - t.Helper() - assert.Empty(t, specs.V1.Services) - }, - }, - { - name: "non-openslo yaml skipped", - files: []string{filepath.Join(testdata, "non-openslo.yaml")}, - checkFunc: func(t *testing.T, specs *OpenSLOSpecs) { - t.Helper() - assert.Empty(t, specs.V1.Services) - }, - }, - { - name: "mixed valid and invalid", - files: []string{filepath.Join(testdata, "service.yaml"), filepath.Join(testdata, "invalid.yaml")}, - wantService: "test-service", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - - specs, err := GetSpecs(tt.files, tt.recursive) - - if tt.wantErr { - assert.Error(t, err) - if tt.errContains != "" { - assert.Contains(t, err.Error(), tt.errContains) - } - assert.Nil(t, specs) - return - } - - assert.NoError(t, err) - require.NotNil(t, specs) - - if tt.wantService != "" { - assert.Contains(t, specs.V1.Services, tt.wantService) - } - if tt.wantSLO != "" { - assert.Contains(t, specs.V1.SLOs, tt.wantSLO) - } - if tt.wantSLI != "" { - assert.Contains(t, specs.V1.SLIs, tt.wantSLI) - } - if tt.wantDataSource != "" { - assert.Contains(t, specs.V1.DataSources, tt.wantDataSource) - } - if tt.wantAlertPolicy != "" { - assert.Contains(t, specs.V1.AlertPolices, tt.wantAlertPolicy) - } - if tt.checkFunc != nil { - tt.checkFunc(t, specs) - } - }) - } -} - -func TestLoadSpecs(t *testing.T) { - t.Parallel() - - testdata := filepath.Join("testdata") - - t.Run("LoadSpec", func(t *testing.T) { - t.Parallel() - - tests := []struct { - name string - filename string - wantErr bool - errContains string - wantCount int - wantKind openslo.Kind - wantName string - }{ - { - name: "valid service yaml", - filename: filepath.Join(testdata, "service.yaml"), - wantCount: 1, - wantKind: openslo.KindService, - wantName: "test-service", - }, - { - name: "multi-document yaml", - filename: filepath.Join(testdata, "multi-doc.yaml"), - wantCount: 2, - }, - { - name: "invalid yaml", - filename: filepath.Join(testdata, "invalid.yaml"), - wantErr: true, - errContains: "error parsing spec", - }, - { - name: "missing file", - filename: "nonexistent-file.yaml", - wantErr: true, - errContains: "error reading file", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - - objects, err := loadSpec(tt.filename) - - if tt.wantErr { - assert.Error(t, err) - if tt.errContains != "" { - assert.Contains(t, err.Error(), tt.errContains) - } - return - } - - require.NoError(t, err) - assert.Len(t, objects, tt.wantCount) - if tt.wantKind != "" && len(objects) > 0 { - assert.Equal(t, tt.wantKind, objects[0].GetKind()) - } - if tt.wantName != "" && len(objects) > 0 { - assert.Equal(t, tt.wantName, objects[0].GetName()) - } - }) - } - }) - - t.Run("LoadSpecs", func(t *testing.T) { - t.Parallel() - - tests := []struct { - name string - files []string - wantErr bool - wantCount int - }{ - { - name: "multiple files", - files: []string{filepath.Join(testdata, "service.yaml"), filepath.Join(testdata, "sli.yaml")}, - wantCount: 2, - }, - { - name: "skip errors in middle", - files: []string{filepath.Join(testdata, "service.yaml"), filepath.Join(testdata, "invalid.yaml"), filepath.Join(testdata, "sli.yaml")}, - wantCount: 2, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - - objects, err := loadSpecs(tt.files) - if tt.wantErr { - assert.Error(t, err) - return - } - require.NoError(t, err) - assert.Len(t, objects, tt.wantCount) - }) - } - }) -} - -func BenchmarkStoreSpec(b *testing.B) { - svc := v1.NewService( - v1.Metadata{Name: "bench-svc"}, - v1.ServiceSpec{Description: "benchmark service"}, - ) - - b.ResetTimer() - for i := 0; i < b.N; i++ { - specs := NewOpenSLOSpecs() - _ = specs.StoreSpec(svc) - } -} - -func BenchmarkGetSpecs(b *testing.B) { - testdata := filepath.Join("testdata") - - b.ResetTimer() - for i := 0; i < b.N; i++ { - _, _ = GetSpecs([]string{testdata}, false) - } -} - -func TestMain(m *testing.M) { - os.Exit(m.Run()) -} diff --git a/pkg/specstore/specstore.go b/pkg/specstore/specstore.go index e48d5a0..e2239d2 100644 --- a/pkg/specstore/specstore.go +++ b/pkg/specstore/specstore.go @@ -3,8 +3,12 @@ package specstore import ( "bytes" "fmt" + "log/slog" "os" "path/filepath" + "sort" + "strconv" + "strings" "github.com/OpenSLO/go-sdk/pkg/openslo" v1 "github.com/OpenSLO/go-sdk/pkg/openslo/v1" @@ -13,6 +17,147 @@ import ( "github.com/thisisibrahimd/opensloctl/pkg/util" ) +// AlertConditionKind identifies the SRE alerting strategy a condition +// expresses. The OpenSLO SDK defines a single kind constant ("burnrate") +// for now; opensloctl extends the allowed surface with three more kinds +// that map 1-to-1 onto the alerting patterns described in the Google SRE +// workbook (https://sre.google/workbook/alerting-on-slos/). +// +// - error-rate: raw SLI error rate vs absolute threshold +// (workbook §§ 1–3; sections 1–3 differ only in +// lookback window length and whether alertAfter is +// set, which maps to the Prom `for:` clause). +// - burn-rate: single burn rate multiplier vs error rate / budget +// (workbook § 4). +// - multi-burn-rate: multiple burn-rate conditions OR-ed per severity +// (workbook § 5). +// - multi-window-multi-burn-rate: short+long window pairs AND-ed within a +// tier and OR-ed across tiers +// (workbook § 6; the original opensloctl default). +type AlertConditionKind string + +const ( + KindErrorRate AlertConditionKind = "error-rate" + KindBurnRate AlertConditionKind = "burn-rate" + KindMultiBurnRate AlertConditionKind = "multi-burn-rate" + KindMultiWindowMultiBurnRate AlertConditionKind = "multi-window-multi-burn-rate" +) + +// validKinds is the closed set of kinds opensloctl accepts. Membership is +// checked in ValidateRefs; anything outside this set is rejected. +var validKinds = map[AlertConditionKind]bool{ + KindErrorRate: true, + KindBurnRate: true, + KindMultiBurnRate: true, + KindMultiWindowMultiBurnRate: true, +} + +// kindOrder lists every supported AlertConditionKind in a stable display +// order. Used to build deterministic error messages that list the valid +// kinds rather than relying on map iteration order. +var kindOrder = []AlertConditionKind{ + KindErrorRate, + KindBurnRate, + KindMultiBurnRate, + KindMultiWindowMultiBurnRate, +} + +// validateThreshold applies the per-kind threshold range rules: +// +// - error-rate: threshold is an absolute error rate; must lie in (0, 1]. +// - burn-rate families: threshold is a burn multiplier; must be > 0. +// +// Both ranges intentionally reject the SDK-required pointer being nil +// with one shared nil-check upstream. +func validateThreshold(kind AlertConditionKind, threshold *float64) error { + if threshold == nil { + return nil + } + switch kind { + case KindErrorRate: + if *threshold <= 0 || *threshold > 1 { + return xerrors.Newf("error-rate threshold must be in (0, 1], got %f", *threshold) + } + case KindBurnRate, KindMultiBurnRate, KindMultiWindowMultiBurnRate: + if *threshold <= 0 { + return xerrors.Newf("%s threshold must be > 0, got %f", kind, *threshold) + } + } + return nil +} + +// ValidKind reports whether k is one of the supported AlertConditionKind +// values. Exported for consumers (especially the Prometheus generator) +// that need to switch on kind without re-deriving the closed set. +func ValidKind(k AlertConditionKind) bool { + return validKinds[k] +} + +// KindPascal converts a kebab-case kind to PascalCase for embedding in +// alert names (e.g. "multi-window-multi-burn-rate" → +// "MultiWindowMultiBurnRate"). Unknown kinds return the input kebab-cased +// unchanged. +func KindPascal(k AlertConditionKind) string { + switch k { + case KindErrorRate: + return "ErrorRate" + case KindBurnRate: + return "BurnRate" + case KindMultiBurnRate: + return "MultiBurnRate" + case KindMultiWindowMultiBurnRate: + return "MultiWindowMultiBurnRate" + default: + return string(k) + } +} + +// SloNamePascal converts a kebab-case name to PascalCase by removing the +// hyphens and title-casing each segment. Useful for combining the SLO name +// into a fully-PascalCase alert identifier without any underscore +// separators, e.g. "test-tiered-slo" → "TestTieredSlo". +func SloNamePascal(name string) string { + parts := strings.Split(name, "-") + for i, p := range parts { + if p == "" { + continue + } + parts[i] = strings.ToUpper(p[:1]) + p[1:] + } + return strings.Join(parts, "") +} + +// KindDescription converts a kebab-case kind to a lower-spaced form for use +// in alert annotations (e.g. "multi-window-multi-burn-rate" → +// "multi-window multi-burn rate"). Unknown kinds return the input unchanged. +func KindDescription(k AlertConditionKind) string { + switch k { + case KindErrorRate: + return "error rate" + case KindBurnRate: + return "burn rate" + case KindMultiBurnRate: + return "multi-burn rate" + case KindMultiWindowMultiBurnRate: + return "multi-window multi-burn rate" + default: + return string(k) + } +} + +// validKindsList renders the supported kinds as a comma-separated list +// for inclusion in error messages. The order matches kindOrder. +func validKindsList() string { + parts := make([]string, 0, len(kindOrder)) + for _, k := range kindOrder { + parts = append(parts, string(k)) + } + sort.Strings(parts) + return fmt.Sprintf("%v", parts) +} + +// OpenSLOV1Specs is an in-memory container for all decoded OpenSlo v1 +// objects, partitioned by kind. Each map keys by the object's metadata.name. type OpenSLOV1Specs struct { Services map[string]v1.Service SLOs map[string]v1.SLO @@ -23,10 +168,15 @@ type OpenSLOV1Specs struct { AlertNotificationTargets map[string]v1.AlertNotificationTarget } +// OpenSLOSpecs holds a parsed collection of OpenSlo specs. Use NewOpenSLOSpecs +// to create an empty store, then StoreSpec to add objects and ValidateRefs to +// verify cross-references resolve. type OpenSLOSpecs struct { V1 OpenSLOV1Specs } +// NewOpenSLOSpecs returns an empty OpenSLOSpecs with all per-kind maps +// initialized but containing no objects. func NewOpenSLOSpecs() *OpenSLOSpecs { opensloSpecs := &OpenSLOSpecs{ V1: OpenSLOV1Specs{ @@ -43,11 +193,30 @@ func NewOpenSLOSpecs() *OpenSLOSpecs { return opensloSpecs } +// ERROR_SPEC_DUPLICATE surfaces duplicate-name detection in error chains. +// Consumers can match this sentinel via xerrors.Is. var ERROR_SPEC_DUPLICATE = xerrors.New("") +// StoreSpec validates o via the SDK's Validate(), runs opensloctl-specific +// post-checks (target required, timeWindow must be rolling), and inserts o +// into the appropriate per-kind map. Duplicates are rejected with +// ERROR_SPEC_DUPLICATE wrapped in an xerror chain. +// +// AlertCondition is allowed to bypass the SDK's strict AlertConditionKind +// OneOf check: opensloctl accepts four extended kinds +// (see validKinds) that map to SRE-workbook alerting strategies. The SDK +// only recognizes the legacy "burnrate" kind. To keep SDK-based error +// reporting for everything else (severity required, op required, etc.) +// without rejecting the extended kinds wholesale, we drop only the kind +// OneOf rejection and re-run the SDK's kind OneOf if the kind is not in +// our extended set. Threshold/lookbackWindow/alertAfter SDK validation +// runs only When(kind == "burnrate") - opensloctl ValidateRefs mirrors +// those checks across all four kinds by enforcing per-kind ranges. func (s *OpenSLOSpecs) StoreSpec(o openslo.Object) error { - if err := o.Validate(); err != nil { - return xerrors.Newf("invalid spec %s/%s: %v", o.GetKind(), o.GetName(), err) + if o.GetKind() != openslo.KindAlertCondition { + if err := o.Validate(); err != nil { + return xerrors.Newf("invalid spec %s/%s: %v", o.GetKind(), o.GetName(), err) + } } switch o.GetVersion() { @@ -62,7 +231,18 @@ func (s *OpenSLOSpecs) StoreSpec(o openslo.Object) error { if _, ok := s.V1.SLOs[o.GetName()]; ok { return xerrors.Newf("duplicate spec found: %s", o.GetName()) } - s.V1.SLOs[o.GetName()] = o.(v1.SLO) + slo := o.(v1.SLO) + for i, obj := range slo.Spec.Objectives { + if obj.Target == nil && obj.TargetPercent == nil { + return xerrors.Newf("invalid spec %s/%s: objective[%d] requires target or targetPercent (required by opensloctl generator, not enforced by SDK)", o.GetKind(), o.GetName(), i) + } + } + for i, tw := range slo.Spec.TimeWindow { + if !tw.IsRolling { + return xerrors.Newf("invalid spec %s/%s: timeWindow[%d] must have isRolling: true (opensloctl generator only supports rolling windows)", o.GetKind(), o.GetName(), i) + } + } + s.V1.SLOs[o.GetName()] = slo case openslo.KindSLI: if _, ok := s.V1.SLIs[o.GetName()]; ok { return xerrors.Newf("duplicate spec found: %s", o.GetName()) @@ -99,6 +279,15 @@ func (s *OpenSLOSpecs) StoreSpec(o openslo.Object) error { return nil } +// GetSpecs discovers YAML files from filenames (or, when recursive is true, +// from directories containing YAML files), parses each via the OpenSlo SDK, +// stores them, and validates all cross-references. Returns a populated +// *OpenSLOSpecs on success or the first error encountered. +// +// filenames may also reference directories directly; recursive controls +// whether non-OpenSlo YAML inside those directories is silently skipped +// (always true) versus only surface-level files. See util.FindFiles for +// discovery rules. func GetSpecs(filenames []string, recursive bool) (*OpenSLOSpecs, error) { specStore := NewOpenSLOSpecs() @@ -108,8 +297,11 @@ func GetSpecs(filenames []string, recursive bool) (*OpenSLOSpecs, error) { return nil, xerrors.New("error detecting files", err) } - // read and parse specs - specs, err := loadSpecs(filenames) + // read and parse specs. loadSpecs emits a slog.Warn per skipped file + // (raw YAML that the OpenSlo SDK didn't recognise) and returns a + // non-nil error if any file failed to decode - fail loudly instead + // of silently dropping missed specs. + specs, _, err := loadSpecs(filenames) if err != nil { return nil, xerrors.New("error reading specs", err) } @@ -129,6 +321,15 @@ func GetSpecs(filenames []string, recursive bool) (*OpenSLOSpecs, error) { return specStore, nil } +// ValidateRefs checks every cross-reference between objects: +// SLO → Service, SLO → SLI (via indicatorRef), SLO objective → SLI, +// SLO → AlertPolicy, AlertPolicy → AlertCondition, +// AlertPolicy → AlertNotificationTarget, and SLI → DataSource. +// +// It also enforces AlertCondition.kind == "burnrate" only. Conditions per +// severity are not counted: any number of page or ticket conditions is +// permitted; the Prometheus generator OR-s them in the resulting alert +// expression. All errors are joined via xerrors.Join. func (s *OpenSLOSpecs) ValidateRefs() error { var errs []error @@ -186,6 +387,13 @@ func (s *OpenSLOSpecs) ValidateRefs() error { } } } + + // opensloctl supports at most one notification target per AlertPolicy + // so each generated Prometheus alert can carry a single + // openslo_notification_target label. + if len(ap.Spec.NotificationTargets) > 1 { + errs = append(errs, xerrors.Newf("AlertPolicy %q has %d notificationTargets; only one is supported", name, len(ap.Spec.NotificationTargets))) + } } // validate SLI → DataSource references @@ -217,85 +425,112 @@ func (s *OpenSLOSpecs) ValidateRefs() error { } } - // validate AlertCondition kind is burnrate + // validate AlertCondition kind + threshold range per kind for name, cond := range s.V1.AlertConditions { - if cond.Spec.Condition.Kind != v1.AlertConditionKindBurnRate { - errs = append(errs, xerrors.Newf("unsupported AlertCondition kind: AlertCondition %q has kind %q, only %q supported", name, cond.Spec.Condition.Kind, v1.AlertConditionKindBurnRate)) + kind := AlertConditionKind(cond.Spec.Condition.Kind) + if !validKinds[kind] { + errs = append(errs, xerrors.Newf("unsupported AlertCondition kind: AlertCondition %q has kind %q, supported kinds: %s", name, cond.Spec.Condition.Kind, validKindsList())) + continue + } + if err := validateThreshold(kind, cond.Spec.Condition.Threshold); err != nil { + errs = append(errs, xerrors.Newf("invalid AlertCondition %q: %v", name, err)) } } - // validate complete burn rate condition sets per SLO - for sloName, slo := range s.V1.SLOs { - // collect all condition refs from this SLO's alert policies - conditionRefs := make(map[string]bool) - for _, ap := range slo.Spec.AlertPolicies { - if ap.SLOAlertPolicyRef != nil && ap.AlertPolicyRef != "" { - if policy, ok := s.V1.AlertPolices[ap.AlertPolicyRef]; ok { - for _, cond := range policy.Spec.Conditions { - if cond.AlertPolicyConditionRef != nil && cond.ConditionRef != "" { - conditionRefs[cond.ConditionRef] = true - } - } - } - } + // validate SLO status threshold annotations + for name, slo := range s.V1.SLOs { + if err := validateStatusThresholds(slo.Metadata.Annotations); err != nil { + errs = append(errs, xerrors.Newf("invalid status thresholds on SLO %q: %v", name, err)) } + } - if len(conditionRefs) == 0 { - continue - } + if len(errs) > 0 { + return xerrors.Join(errs) + } + return nil +} - // group conditions by severity - type conditionInfo struct { - threshold float64 - lookbackWindow string - } - pageConditions := make(map[string]conditionInfo) - ticketConditions := make(map[string]conditionInfo) - - for ref := range conditionRefs { - if cond, ok := s.V1.AlertConditions[ref]; ok { - info := conditionInfo{} - if cond.Spec.Condition.Threshold != nil { - info.threshold = *cond.Spec.Condition.Threshold - } - info.lookbackWindow = cond.Spec.Condition.LookbackWindow.String() - switch cond.Spec.Severity { - case "page": - pageConditions[ref] = info - case "ticket": - ticketConditions[ref] = info - } - } - } +// StatusThresholdAnnotationWarning, Critical, Breached are the SLO +// metadata.annotation keys that override the default status-gauge +// ranges (defaults 1/6/14.4 per the Google SRE workbook). Exported so +// the generator and tests can reference the same string literals. +const ( + StatusThresholdAnnotationWarning = "threshold.status.openslo.com/warning" + StatusThresholdAnnotationCritical = "threshold.status.openslo.com/critical" + StatusThresholdAnnotationBreached = "threshold.status.openslo.com/breached" +) - // validate page has both conditions (14.4x@5m and 6x@30m) - if len(pageConditions) > 0 && len(pageConditions) < 2 { - errs = append(errs, xerrors.Newf("incomplete page burn rate conditions for SLO %q: expected 2 conditions (14.4x@5m and 6x@30m), found %d", sloName, len(pageConditions))) - } +// StatusThresholdDefault* mirror the SRE-workbook burn-rate reference +// points used when no annotation override is supplied. +const ( + StatusThresholdDefaultWarning = 1.0 + StatusThresholdDefaultCritical = 6.0 + StatusThresholdDefaultBreached = 14.4 +) - // validate ticket has both conditions (3x@2h and 1x@6h) - if len(ticketConditions) > 0 && len(ticketConditions) < 2 { - errs = append(errs, xerrors.Newf("incomplete ticket burn rate conditions for SLO %q: expected 2 conditions (3x@2h and 1x@6h), found %d", sloName, len(ticketConditions))) - } +// ParseStatusThreshold parses a status-threshold annotation value as a +// float. Returns the default when the annotation is absent, empty, or +// unparseable - so a stray reminder note like "TODO: tune later" +// falls back silently rather than failing the load. The returned error +// is reserved for future use; current callers all expect nil. +func ParseStatusThreshold(raw string, def float64) (float64, error) { + raw = strings.TrimSpace(raw) + if raw == "" { + return def, nil + } + v, err := strconv.ParseFloat(raw, 64) + if err != nil { + return def, nil } + return v, nil +} - if len(errs) > 0 { - return xerrors.Join(errs) +// validateStatusThresholds enforces the ascending-positive invariant on +// the resolved (warning, critical, breached) triple. Each annotation is +// optional; missing annotations fall back to defaults independently. +func validateStatusThresholds(ann map[string]string) error { + warn, err := ParseStatusThreshold(ann[StatusThresholdAnnotationWarning], StatusThresholdDefaultWarning) + if err != nil { + return xerrors.Newf("annotation %q: %v", StatusThresholdAnnotationWarning, err) + } + crit, err := ParseStatusThreshold(ann[StatusThresholdAnnotationCritical], StatusThresholdDefaultCritical) + if err != nil { + return xerrors.Newf("annotation %q: %v", StatusThresholdAnnotationCritical, err) + } + breach, err := ParseStatusThreshold(ann[StatusThresholdAnnotationBreached], StatusThresholdDefaultBreached) + if err != nil { + return xerrors.Newf("annotation %q: %v", StatusThresholdAnnotationBreached, err) + } + if warn <= 0 || crit <= 0 || breach <= 0 { + return xerrors.Newf("warning=%g critical=%g breached=%g must all be positive", warn, crit, breach) + } + if !(warn < crit && crit < breach) { + return xerrors.Newf("warning=%g critical=%g breached=%g must be strictly ascending", warn, crit, breach) } return nil } -func loadSpecs(filenames []string) ([]openslo.Object, error) { - var opensloObjects []openslo.Object +// loadSpecs ingests every YAML/JSON file in filenames. Files that fail to +// decode get a console warning (not a hard error) so they can be inspected; +// multi-doc YAML files partially succeeding decode but yielding no OpenSLo +// objects are not warned on - that's the expected shape for `services.yaml` +// helpers inside the spec tree. The total file count and the skipped-file +// list are returned so callers can surface a top-line error; see GetSpecs +// for the policy (currently: any skipped file fails the run). +func loadSpecs(filenames []string) (loaded []openslo.Object, skipped []string, err error) { for _, filename := range filenames { - objects, err := loadSpec(filename) - if err != nil { + objects, decodeErr := loadSpec(filename) + if decodeErr != nil { + slog.Warn("spec file could not be decoded; skipping", "file", filename, "err", decodeErr) + skipped = append(skipped, filename) continue } - - opensloObjects = append(opensloObjects, objects...) + loaded = append(loaded, objects...) } - return opensloObjects, nil + if len(skipped) > 0 { + err = xerrors.Newf("%d file(s) could not be decoded as OpenSlo specs: %v", len(skipped), skipped) + } + return loaded, skipped, err } func loadSpec(filename string) ([]openslo.Object, error) { diff --git a/pkg/specstore/specstore_test.go b/pkg/specstore/specstore_test.go index 39fb9cd..0d26227 100644 --- a/pkg/specstore/specstore_test.go +++ b/pkg/specstore/specstore_test.go @@ -11,6 +11,338 @@ import ( "github.com/stretchr/testify/require" ) +// ptr returns a pointer to v. Convenience helper for constructing optional +// fields in OpenSlo spec fixtures (e.g. Target, IndicatorRef). +func ptr[T any](v T) *T { return &v } + +// TestNewOpenSLOSpecs verifies NewOpenSLOSpecs returns a non-nil store with +// every per-kind map initialized empty. Table-driven over each kind so +// failures name the specific map. +func TestNewOpenSLOSpecs(t *testing.T) { + t.Parallel() + + lengths := []struct { + name string + size func(s *OpenSLOSpecs) int + }{ + {"Services", func(s *OpenSLOSpecs) int { return len(s.V1.Services) }}, + {"SLOs", func(s *OpenSLOSpecs) int { return len(s.V1.SLOs) }}, + {"SLIs", func(s *OpenSLOSpecs) int { return len(s.V1.SLIs) }}, + {"DataSources", func(s *OpenSLOSpecs) int { return len(s.V1.DataSources) }}, + {"AlertPolices", func(s *OpenSLOSpecs) int { return len(s.V1.AlertPolices) }}, + {"AlertConditions", func(s *OpenSLOSpecs) int { return len(s.V1.AlertConditions) }}, + {"AlertNotificationTargets", func(s *OpenSLOSpecs) int { return len(s.V1.AlertNotificationTargets) }}, + } + + for _, l := range lengths { + t.Run(l.name, func(t *testing.T) { + t.Parallel() + + specs := NewOpenSLOSpecs() + require.NotNil(t, specs) + assert.Equal(t, 0, l.size(specs)) + }) + } +} + +// TestStoreSpec covers three StoreSpec subtests: +// - AllKinds: every supported kind (Service, SLO, SLI, DataSource, +// AlertPolicy, AlertCondition, AlertNotificationTarget) round-trips +// into its respective map. +// - Duplicates: re-storing an object under an existing name returns an +// error wrapping ERROR_SPEC_DUPLICATE. +// - EdgeCases: storing multiple objects of different kinds works in a +// single store. +func TestStoreSpec(t *testing.T) { + t.Parallel() + + t.Run("AllKinds", func(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + object openslo.Object + expKind string + expName string + }{ + { + name: "Service", + object: v1.NewService( + v1.Metadata{Name: "test-svc"}, + v1.ServiceSpec{Description: "test service"}, + ), + expKind: "Services", + expName: "test-svc", + }, + { + name: "SLO", + object: v1.NewSLO( + v1.Metadata{Name: "test-slo"}, + v1.SLOSpec{ + Service: "test-svc", + IndicatorRef: ptr("test-sli"), + BudgetingMethod: v1.SLOBudgetingMethodOccurrences, + TimeWindow: []v1.SLOTimeWindow{{Duration: v1.NewDurationShorthand(30, v1.DurationShorthandUnitDay), IsRolling: true}}, + Objectives: []v1.SLOObjective{{Target: ptr(0.999), Operator: v1.OperatorLTE, Value: ptr(500.0)}}, + }, + ), + expKind: "SLOs", + expName: "test-slo", + }, + { + name: "SLI", + object: v1.NewSLI( + v1.Metadata{Name: "test-sli"}, + v1.SLISpec{ + ThresholdMetric: &v1.SLIMetricSpec{ + MetricSource: v1.SLIMetricSource{ + Type: "Prometheus", + Spec: map[string]any{"query": "up"}, + }, + }, + }, + ), + expKind: "SLIs", + expName: "test-sli", + }, + { + name: "DataSource", + object: v1.NewDataSource( + v1.Metadata{Name: "test-ds"}, + v1.DataSourceSpec{ + Type: "Prometheus", + ConnectionDetails: json.RawMessage(`{"url":"http://prom:9090"}`), + }, + ), + expKind: "DataSources", + expName: "test-ds", + }, + { + name: "AlertPolicy", + object: v1.NewAlertPolicy( + v1.Metadata{Name: "test-ap"}, + v1.AlertPolicySpec{ + AlertWhenBreaching: true, + Conditions: []v1.AlertPolicyCondition{ + {AlertPolicyConditionRef: &v1.AlertPolicyConditionRef{ConditionRef: "test-ac"}}, + }, + NotificationTargets: []v1.AlertPolicyNotificationTarget{ + {AlertPolicyNotificationTargetRef: &v1.AlertPolicyNotificationTargetRef{TargetRef: "test-ant"}}, + }, + }, + ), + expKind: "AlertPolices", + expName: "test-ap", + }, + { + name: "AlertCondition", + object: v1.NewAlertCondition( + v1.Metadata{Name: "test-ac"}, + v1.AlertConditionSpec{ + Severity: "page", + Condition: v1.AlertConditionType{ + Kind: v1.AlertConditionKind("multi-window-multi-burn-rate"), + Operator: v1.OperatorLTE, + Threshold: ptr(2.0), + LookbackWindow: v1.NewDurationShorthand(1, v1.DurationShorthandUnitHour), + }, + }, + ), + expKind: "AlertConditions", + expName: "test-ac", + }, + { + name: "AlertNotificationTarget", + object: v1.NewAlertNotificationTarget( + v1.Metadata{Name: "test-ant"}, + v1.AlertNotificationTargetSpec{ + Target: "pagerduty", + Description: "test target", + }, + ), + expKind: "AlertNotificationTargets", + expName: "test-ant", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + specs := NewOpenSLOSpecs() + err := specs.StoreSpec(tt.object) + require.NoError(t, err) + + switch tt.expKind { + case "Services": + assert.Contains(t, specs.V1.Services, tt.expName) + case "SLOs": + assert.Contains(t, specs.V1.SLOs, tt.expName) + case "SLIs": + assert.Contains(t, specs.V1.SLIs, tt.expName) + case "DataSources": + assert.Contains(t, specs.V1.DataSources, tt.expName) + case "AlertPolices": + assert.Contains(t, specs.V1.AlertPolices, tt.expName) + case "AlertConditions": + assert.Contains(t, specs.V1.AlertConditions, tt.expName) + case "AlertNotificationTargets": + assert.Contains(t, specs.V1.AlertNotificationTargets, tt.expName) + } + }) + } + }) + + t.Run("Duplicates", func(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + first openslo.Object + second openslo.Object + wantError bool + }{ + { + name: "Service", + first: v1.NewService(v1.Metadata{Name: "dup-svc"}, v1.ServiceSpec{}), + second: v1.NewService(v1.Metadata{Name: "dup-svc"}, v1.ServiceSpec{Description: "second"}), + }, + { + name: "SLO", + first: v1.NewSLO(v1.Metadata{Name: "dup-slo"}, v1.SLOSpec{ + Service: "svc", IndicatorRef: ptr("sli"), BudgetingMethod: v1.SLOBudgetingMethodOccurrences, + TimeWindow: []v1.SLOTimeWindow{{Duration: v1.NewDurationShorthand(30, v1.DurationShorthandUnitDay), IsRolling: true}}, + Objectives: []v1.SLOObjective{{Target: ptr(0.999), Operator: v1.OperatorLTE, Value: ptr(500.0)}}, + }), + second: v1.NewSLO(v1.Metadata{Name: "dup-slo"}, v1.SLOSpec{ + Service: "svc", IndicatorRef: ptr("sli"), BudgetingMethod: v1.SLOBudgetingMethodOccurrences, + TimeWindow: []v1.SLOTimeWindow{{Duration: v1.NewDurationShorthand(30, v1.DurationShorthandUnitDay), IsRolling: true}}, + Objectives: []v1.SLOObjective{{Target: ptr(0.99), Operator: v1.OperatorLTE, Value: ptr(500.0)}}, + }), + }, + { + name: "SLI", + first: v1.NewSLI(v1.Metadata{Name: "dup-sli"}, v1.SLISpec{ + ThresholdMetric: &v1.SLIMetricSpec{MetricSource: v1.SLIMetricSource{Type: "Prometheus", Spec: map[string]any{"query": "up"}}}, + }), + second: v1.NewSLI(v1.Metadata{Name: "dup-sli"}, v1.SLISpec{ + ThresholdMetric: &v1.SLIMetricSpec{MetricSource: v1.SLIMetricSource{Type: "Prometheus", Spec: map[string]any{"query": "up"}}}, + }), + }, + { + name: "DataSource", + first: v1.NewDataSource(v1.Metadata{Name: "dup-ds"}, v1.DataSourceSpec{Type: "Prometheus", ConnectionDetails: json.RawMessage(`{}`)}), + second: v1.NewDataSource(v1.Metadata{Name: "dup-ds"}, v1.DataSourceSpec{Type: "Prometheus", ConnectionDetails: json.RawMessage(`{}`)}), + }, + { + name: "AlertPolicy", + first: v1.NewAlertPolicy(v1.Metadata{Name: "dup-ap"}, v1.AlertPolicySpec{ + AlertWhenBreaching: true, + Conditions: []v1.AlertPolicyCondition{{AlertPolicyConditionRef: &v1.AlertPolicyConditionRef{ConditionRef: "ac"}}}, + NotificationTargets: []v1.AlertPolicyNotificationTarget{{AlertPolicyNotificationTargetRef: &v1.AlertPolicyNotificationTargetRef{TargetRef: "ant"}}}, + }), + second: v1.NewAlertPolicy(v1.Metadata{Name: "dup-ap"}, v1.AlertPolicySpec{ + AlertWhenBreaching: true, + Conditions: []v1.AlertPolicyCondition{{AlertPolicyConditionRef: &v1.AlertPolicyConditionRef{ConditionRef: "ac"}}}, + NotificationTargets: []v1.AlertPolicyNotificationTarget{{AlertPolicyNotificationTargetRef: &v1.AlertPolicyNotificationTargetRef{TargetRef: "ant"}}}, + }), + }, + { + name: "AlertCondition", + first: v1.NewAlertCondition(v1.Metadata{Name: "dup-ac"}, v1.AlertConditionSpec{ + Severity: "page", + Condition: v1.AlertConditionType{Kind: v1.AlertConditionKind("multi-window-multi-burn-rate"), Operator: v1.OperatorLTE, Threshold: ptr(2.0), LookbackWindow: v1.NewDurationShorthand(1, v1.DurationShorthandUnitHour)}, + }), + second: v1.NewAlertCondition(v1.Metadata{Name: "dup-ac"}, v1.AlertConditionSpec{ + Severity: "warn", + Condition: v1.AlertConditionType{Kind: v1.AlertConditionKind("multi-window-multi-burn-rate"), Operator: v1.OperatorLTE, Threshold: ptr(2.0), LookbackWindow: v1.NewDurationShorthand(1, v1.DurationShorthandUnitHour)}, + }), + }, + { + name: "AlertNotificationTarget", + first: v1.NewAlertNotificationTarget(v1.Metadata{Name: "dup-ant"}, v1.AlertNotificationTargetSpec{Target: "pagerduty"}), + second: v1.NewAlertNotificationTarget(v1.Metadata{Name: "dup-ant"}, v1.AlertNotificationTargetSpec{Target: "slack"}), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + specs := NewOpenSLOSpecs() + require.NoError(t, specs.StoreSpec(tt.first)) + + err := specs.StoreSpec(tt.second) + assert.Error(t, err) + assert.Contains(t, err.Error(), "duplicate spec") + }) + } + }) + + t.Run("EdgeCases", func(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + objects []openslo.Object + checkFunc func(t *testing.T, specs *OpenSLOSpecs) + wantErr []bool + errContains []string + }{ + { + name: "multiple kinds", + objects: []openslo.Object{ + v1.NewService(v1.Metadata{Name: "multi-svc"}, v1.ServiceSpec{Description: "multi test service"}), + v1.NewSLI(v1.Metadata{Name: "multi-sli"}, v1.SLISpec{ + ThresholdMetric: &v1.SLIMetricSpec{ + MetricSource: v1.SLIMetricSource{Type: "Prometheus", Spec: map[string]any{"query": "up"}}, + }, + }), + v1.NewSLO(v1.Metadata{Name: "multi-slo"}, v1.SLOSpec{ + Service: "multi-svc", IndicatorRef: ptr("multi-sli"), BudgetingMethod: v1.SLOBudgetingMethodOccurrences, + TimeWindow: []v1.SLOTimeWindow{{Duration: v1.NewDurationShorthand(30, v1.DurationShorthandUnitDay), IsRolling: true}}, + Objectives: []v1.SLOObjective{{Target: ptr(0.999), Operator: v1.OperatorLTE, Value: ptr(500.0)}}, + }), + }, + wantErr: []bool{false, false, false}, + checkFunc: func(t *testing.T, specs *OpenSLOSpecs) { + t.Helper() + assert.Contains(t, specs.V1.Services, "multi-svc") + assert.Contains(t, specs.V1.SLIs, "multi-sli") + assert.Contains(t, specs.V1.SLOs, "multi-slo") + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + specs := NewOpenSLOSpecs() + for i, obj := range tt.objects { + err := specs.StoreSpec(obj) + if len(tt.wantErr) > i { + if tt.wantErr[i] { + assert.Error(t, err) + if len(tt.errContains) > i { + assert.Contains(t, err.Error(), tt.errContains[i]) + } + } else { + assert.NoError(t, err) + } + } + } + if tt.checkFunc != nil { + tt.checkFunc(t, specs) + } + }) + } + }) +} + +// TestStoreSpec_ValidationErrors verifies SDK validation runs before +// opensloctl-specific checks. Each case constructs an object with/without +// required SDK fields and asserts StoreSpec returns or skips errors. func TestStoreSpec_ValidationErrors(t *testing.T) { t.Parallel() @@ -77,6 +409,117 @@ func TestStoreSpec_ValidationErrors(t *testing.T) { } } +// TestStoreSpec_TargetRequired verifies the opensloctl-specific post-check: +// every SLO objective must declare either target (0-1 scale) or +// targetPercent (0-100 scale). Missing both surfaces a "one of +// [target, targetPercent] properties must be set" error from the SDK +// because the SDK actually enforces this despite the OpenSlo spec +// marking it optional. +func TestStoreSpec_TargetRequired(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + object openslo.Object + wantError bool + errContains string + }{ + { + name: "valid with target", + object: v1.NewSLO( + v1.Metadata{Name: "valid-target"}, + v1.SLOSpec{ + Service: "svc", + Indicator: &v1.SLOIndicatorInline{ + Metadata: v1.Metadata{Name: "sli"}, + Spec: v1.SLISpec{ + ThresholdMetric: &v1.SLIMetricSpec{ + MetricSource: v1.SLIMetricSource{Type: "Prometheus", Spec: map[string]any{"query": "up"}}, + }, + }, + }, + BudgetingMethod: v1.SLOBudgetingMethodOccurrences, + TimeWindow: []v1.SLOTimeWindow{{Duration: v1.NewDurationShorthand(30, v1.DurationShorthandUnitDay), IsRolling: true}}, + Objectives: []v1.SLOObjective{{Target: ptr(0.99), Operator: v1.OperatorLTE, Value: ptr(500.0)}}, + }, + ), + wantError: false, + }, + { + name: "valid with targetPercent", + object: v1.NewSLO( + v1.Metadata{Name: "valid-percent"}, + v1.SLOSpec{ + Service: "svc", + Indicator: &v1.SLOIndicatorInline{ + Metadata: v1.Metadata{Name: "sli"}, + Spec: v1.SLISpec{ + ThresholdMetric: &v1.SLIMetricSpec{ + MetricSource: v1.SLIMetricSource{Type: "Prometheus", Spec: map[string]any{"query": "up"}}, + }, + }, + }, + BudgetingMethod: v1.SLOBudgetingMethodOccurrences, + TimeWindow: []v1.SLOTimeWindow{{Duration: v1.NewDurationShorthand(30, v1.DurationShorthandUnitDay), IsRolling: true}}, + Objectives: []v1.SLOObjective{{TargetPercent: ptr(99.0), Operator: v1.OperatorLTE, Value: ptr(500.0)}}, + }, + ), + wantError: false, + }, + { + name: "missing target rejected", + object: v1.NewSLO( + v1.Metadata{Name: "missing-target"}, + v1.SLOSpec{ + Service: "svc", + Indicator: &v1.SLOIndicatorInline{ + Metadata: v1.Metadata{Name: "sli"}, + Spec: v1.SLISpec{ + ThresholdMetric: &v1.SLIMetricSpec{ + MetricSource: v1.SLIMetricSource{Type: "Prometheus", Spec: map[string]any{"query": "up"}}, + }, + }, + }, + BudgetingMethod: v1.SLOBudgetingMethodOccurrences, + TimeWindow: []v1.SLOTimeWindow{{Duration: v1.NewDurationShorthand(30, v1.DurationShorthandUnitDay), IsRolling: true}}, + Objectives: []v1.SLOObjective{{Operator: v1.OperatorLTE, Value: ptr(500.0)}}, + }, + ), + wantError: true, + errContains: `one of [target, targetPercent] properties must be set`, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + specs := NewOpenSLOSpecs() + err := specs.StoreSpec(tt.object) + if tt.wantError { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.errContains) + } else { + assert.NoError(t, err) + } + }) + } +} + +// TestValidateRefs covers cross-reference resolution and burn-rate rule-set +// completeness. Every subtest is table-driven: +// +// - SLO → Service resolve / unresolve +// - SLO → SLI (indicatorRef) resolve / unresolve / nil-ref skip +// - SLO → AlertPolicy resolve / unresolve +// - AlertPolicy → Condition resolve / unresolve +// - AlertPolicy → Notif resolve / unresolve +// - SLI → DataSource resolve / unresolve / empty-ref skip +// - AlertCondition kind burnrate ok / latency rejected +// - SLO graph multi-error chain / all-resolve baseline +// +// The "multiple errors collected" case (last subtest's first row) confirms +// a single call surfaces every missing ref independently. func TestValidateRefs(t *testing.T) { t.Parallel() @@ -144,11 +587,11 @@ func TestValidateRefs(t *testing.T) { t.Parallel() tests := []struct { - name string - slis []string + name string + slis []string indicatorRef *string - wantErr bool - errContains string + wantErr bool + errContains string }{ { name: "resolved", @@ -211,11 +654,11 @@ func TestValidateRefs(t *testing.T) { t.Parallel() tests := []struct { - name string + name string alertPolicies []string policyRefs []string - wantErr bool - errContains string + wantErr bool + errContains string }{ { name: "resolved", @@ -251,28 +694,28 @@ func TestValidateRefs(t *testing.T) { v1.Metadata{Name: "page-14-4x-5m"}, v1.AlertConditionSpec{ Severity: "page", - Condition: v1.AlertConditionType{Kind: v1.AlertConditionKindBurnRate, Operator: v1.OperatorLTE, Threshold: ptr(14.4), LookbackWindow: v1.NewDurationShorthand(5, v1.DurationShorthandUnitMinute)}, + Condition: v1.AlertConditionType{Kind: v1.AlertConditionKind("multi-window-multi-burn-rate"), Operator: v1.OperatorLTE, Threshold: ptr(14.4), LookbackWindow: v1.NewDurationShorthand(5, v1.DurationShorthandUnitMinute)}, }, ) specs.V1.AlertConditions["page-6x-30m"] = v1.NewAlertCondition( v1.Metadata{Name: "page-6x-30m"}, v1.AlertConditionSpec{ Severity: "page", - Condition: v1.AlertConditionType{Kind: v1.AlertConditionKindBurnRate, Operator: v1.OperatorLTE, Threshold: ptr(6.0), LookbackWindow: v1.NewDurationShorthand(30, v1.DurationShorthandUnitMinute)}, + Condition: v1.AlertConditionType{Kind: v1.AlertConditionKind("multi-window-multi-burn-rate"), Operator: v1.OperatorLTE, Threshold: ptr(6.0), LookbackWindow: v1.NewDurationShorthand(30, v1.DurationShorthandUnitMinute)}, }, ) specs.V1.AlertConditions["ticket-3x-2h"] = v1.NewAlertCondition( v1.Metadata{Name: "ticket-3x-2h"}, v1.AlertConditionSpec{ Severity: "ticket", - Condition: v1.AlertConditionType{Kind: v1.AlertConditionKindBurnRate, Operator: v1.OperatorLTE, Threshold: ptr(3.0), LookbackWindow: v1.NewDurationShorthand(2, v1.DurationShorthandUnitHour)}, + Condition: v1.AlertConditionType{Kind: v1.AlertConditionKind("multi-window-multi-burn-rate"), Operator: v1.OperatorLTE, Threshold: ptr(3.0), LookbackWindow: v1.NewDurationShorthand(2, v1.DurationShorthandUnitHour)}, }, ) specs.V1.AlertConditions["ticket-1x-6h"] = v1.NewAlertCondition( v1.Metadata{Name: "ticket-1x-6h"}, v1.AlertConditionSpec{ Severity: "ticket", - Condition: v1.AlertConditionType{Kind: v1.AlertConditionKindBurnRate, Operator: v1.OperatorLTE, Threshold: ptr(1.0), LookbackWindow: v1.NewDurationShorthand(6, v1.DurationShorthandUnitHour)}, + Condition: v1.AlertConditionType{Kind: v1.AlertConditionKind("multi-window-multi-burn-rate"), Operator: v1.OperatorLTE, Threshold: ptr(1.0), LookbackWindow: v1.NewDurationShorthand(6, v1.DurationShorthandUnitHour)}, }, ) specs.V1.AlertNotificationTargets["nt"] = v1.NewAlertNotificationTarget( @@ -347,7 +790,7 @@ func TestValidateRefs(t *testing.T) { v1.Metadata{Name: cond}, v1.AlertConditionSpec{ Severity: "page", - Condition: v1.AlertConditionType{Kind: v1.AlertConditionKindBurnRate, Operator: v1.OperatorLTE, Threshold: ptr(2.0), LookbackWindow: v1.NewDurationShorthand(1, v1.DurationShorthandUnitHour)}, + Condition: v1.AlertConditionType{Kind: v1.AlertConditionKind("multi-window-multi-burn-rate"), Operator: v1.OperatorLTE, Threshold: ptr(2.0), LookbackWindow: v1.NewDurationShorthand(1, v1.DurationShorthandUnitHour)}, }, ) } @@ -431,7 +874,7 @@ func TestValidateRefs(t *testing.T) { v1.Metadata{Name: "cond"}, v1.AlertConditionSpec{ Severity: "page", - Condition: v1.AlertConditionType{Kind: v1.AlertConditionKindBurnRate, Operator: v1.OperatorLTE, Threshold: ptr(2.0), LookbackWindow: v1.NewDurationShorthand(1, v1.DurationShorthandUnitHour)}, + Condition: v1.AlertConditionType{Kind: v1.AlertConditionKind("multi-window-multi-burn-rate"), Operator: v1.OperatorLTE, Threshold: ptr(2.0), LookbackWindow: v1.NewDurationShorthand(1, v1.DurationShorthandUnitHour)}, }, ) @@ -515,19 +958,48 @@ func TestValidateRefs(t *testing.T) { tests := []struct { name string kind v1.AlertConditionKind + threshold float64 wantErr bool errContains string }{ { - name: "burnrate valid", - kind: v1.AlertConditionKindBurnRate, - wantErr: false, + name: "burnrate kind rejected (legacy not accepted)", + kind: v1.AlertConditionKindBurnRate, + threshold: 14.4, + wantErr: true, + errContains: `AlertCondition "test-cond" has kind "burnrate", supported kinds`, }, { - name: "unknown kind rejected", - kind: v1.AlertConditionKind("latency"), - wantErr: true, - errContains: `AlertCondition "test-cond" has kind "latency", only "burnrate" supported`, + name: "multi-window-multi-burn-rate valid", + kind: v1.AlertConditionKind("multi-window-multi-burn-rate"), + threshold: 14.4, + wantErr: false, + }, + { + name: "burn-rate valid (single threshold)", + kind: v1.AlertConditionKind("burn-rate"), + threshold: 14.4, + wantErr: false, + }, + { + name: "error-rate valid (absolute threshold in (0,1])", + kind: v1.AlertConditionKind("error-rate"), + threshold: 0.001, + wantErr: false, + }, + { + name: "error-rate threshold > 1 rejected", + kind: v1.AlertConditionKind("error-rate"), + threshold: 2.0, + wantErr: true, + errContains: `error-rate threshold must be in (0, 1]`, + }, + { + name: "unknown kind rejected", + kind: v1.AlertConditionKind("latency"), + threshold: 14.4, + wantErr: true, + errContains: `AlertCondition "test-cond" has kind "latency", supported kinds`, }, } @@ -540,7 +1012,7 @@ func TestValidateRefs(t *testing.T) { v1.Metadata{Name: "test-cond"}, v1.AlertConditionSpec{ Severity: "page", - Condition: v1.AlertConditionType{Kind: tt.kind, Operator: v1.OperatorLTE, Threshold: ptr(2.0), LookbackWindow: v1.NewDurationShorthand(1, v1.DurationShorthandUnitHour)}, + Condition: v1.AlertConditionType{Kind: tt.kind, Operator: v1.OperatorLTE, Threshold: ptr(tt.threshold), LookbackWindow: v1.NewDurationShorthand(1, v1.DurationShorthandUnitHour)}, }, ) @@ -555,53 +1027,256 @@ func TestValidateRefs(t *testing.T) { } }) - t.Run("multiple errors collected", func(t *testing.T) { + t.Run("SLO graph", func(t *testing.T) { t.Parallel() - specs := NewOpenSLOSpecs() - specs.V1.Services["svc"] = v1.NewService(v1.Metadata{Name: "svc"}, v1.ServiceSpec{}) - specs.V1.SLOs["test-slo"] = v1.NewSLO( - v1.Metadata{Name: "test-slo"}, - v1.SLOSpec{ - Service: "missing-svc", - IndicatorRef: ptr("missing-sli"), - BudgetingMethod: v1.SLOBudgetingMethodOccurrences, - TimeWindow: []v1.SLOTimeWindow{{Duration: v1.NewDurationShorthand(30, v1.DurationShorthandUnitDay), IsRolling: true}}, - Objectives: []v1.SLOObjective{{Target: ptr(0.999), Operator: v1.OperatorLTE, Value: ptr(500.0)}}, - }, - ) - - err := specs.ValidateRefs() - require.Error(t, err) - assert.Contains(t, err.Error(), `SLO "test-slo" references Service "missing-svc" not found`) - assert.Contains(t, err.Error(), `SLO "test-slo" references SLI "missing-sli" not found`) - }) + tests := []struct { + name string + setup func(*OpenSLOSpecs) + wantErr bool + errContains []string + }{ + { + name: "multiple errors collected", + setup: func(s *OpenSLOSpecs) { + s.V1.Services["svc"] = v1.NewService(v1.Metadata{Name: "svc"}, v1.ServiceSpec{}) + s.V1.SLOs["test-slo"] = v1.NewSLO( + v1.Metadata{Name: "test-slo"}, + v1.SLOSpec{ + Service: "missing-svc", + IndicatorRef: ptr("missing-sli"), + BudgetingMethod: v1.SLOBudgetingMethodOccurrences, + TimeWindow: []v1.SLOTimeWindow{{Duration: v1.NewDurationShorthand(30, v1.DurationShorthandUnitDay), IsRolling: true}}, + Objectives: []v1.SLOObjective{{Target: ptr(0.999), Operator: v1.OperatorLTE, Value: ptr(500.0)}}, + }, + ) + }, + wantErr: true, + errContains: []string{ + `SLO "test-slo" references Service "missing-svc" not found`, + `SLO "test-slo" references SLI "missing-sli" not found`, + }, + }, + { + name: "all refs resolve no error", + setup: func(s *OpenSLOSpecs) { + s.V1.Services["svc"] = v1.NewService(v1.Metadata{Name: "svc"}, v1.ServiceSpec{}) + s.V1.SLIs["sli"] = v1.NewSLI( + v1.Metadata{Name: "sli"}, + v1.SLISpec{ThresholdMetric: &v1.SLIMetricSpec{MetricSource: v1.SLIMetricSource{Type: "Prometheus", Spec: map[string]any{"query": "up"}}}}, + ) + s.V1.SLOs["test-slo"] = v1.NewSLO( + v1.Metadata{Name: "test-slo"}, + v1.SLOSpec{ + Service: "svc", + IndicatorRef: ptr("sli"), + BudgetingMethod: v1.SLOBudgetingMethodOccurrences, + TimeWindow: []v1.SLOTimeWindow{{Duration: v1.NewDurationShorthand(30, v1.DurationShorthandUnitDay), IsRolling: true}}, + Objectives: []v1.SLOObjective{{Target: ptr(0.999), Operator: v1.OperatorLTE, Value: ptr(500.0)}}, + }, + ) + }, + wantErr: false, + }, + } - t.Run("all refs resolve no error", func(t *testing.T) { - t.Parallel() + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() - specs := NewOpenSLOSpecs() - specs.V1.Services["svc"] = v1.NewService(v1.Metadata{Name: "svc"}, v1.ServiceSpec{}) - specs.V1.SLIs["sli"] = v1.NewSLI( - v1.Metadata{Name: "sli"}, - v1.SLISpec{ThresholdMetric: &v1.SLIMetricSpec{MetricSource: v1.SLIMetricSource{Type: "Prometheus", Spec: map[string]any{"query": "up"}}}}, - ) - specs.V1.SLOs["test-slo"] = v1.NewSLO( - v1.Metadata{Name: "test-slo"}, - v1.SLOSpec{ - Service: "svc", - IndicatorRef: ptr("sli"), - BudgetingMethod: v1.SLOBudgetingMethodOccurrences, - TimeWindow: []v1.SLOTimeWindow{{Duration: v1.NewDurationShorthand(30, v1.DurationShorthandUnitDay), IsRolling: true}}, - Objectives: []v1.SLOObjective{{Target: ptr(0.999), Operator: v1.OperatorLTE, Value: ptr(500.0)}}, - }, - ) - - err := specs.ValidateRefs() - assert.NoError(t, err) + specs := NewOpenSLOSpecs() + tt.setup(specs) + + err := specs.ValidateRefs() + if tt.wantErr { + require.Error(t, err) + for _, sub := range tt.errContains { + assert.Contains(t, err.Error(), sub) + } + } else { + assert.NoError(t, err) + } + }) + } }) } +// TestGetSpecs covers the file-loading surface of GetSpecs: +// single files, multi-document YAML, recursive directory scan, blank and +// missing inputs, invalid/non-OpenSlo YAML silently skipped, and +// deduplication when the same file is passed twice. +func TestGetSpecs(t *testing.T) { + t.Parallel() + + testdata := filepath.Join("testdata") + + tests := []struct { + name string + files []string + recursive bool + wantErr bool + errContains string + checkFunc func(t *testing.T, specs *OpenSLOSpecs) + wantService string + wantSLO string + wantSLI string + wantDataSource string + wantAlertPolicy string + }{ + { + name: "single service file", + files: []string{filepath.Join(testdata, "service.yaml")}, + wantService: "test-service", + }, + { + name: "single slo file", + files: []string{filepath.Join(testdata, "slo.yaml"), filepath.Join(testdata, "service.yaml")}, + wantSLO: "test-slo", + }, + { + name: "multiple individual files", + files: []string{filepath.Join(testdata, "service.yaml"), filepath.Join(testdata, "sli.yaml")}, + wantService: "test-service", + wantSLI: "test-sli", + }, + { + name: "directory non-recursive", + files: []string{testdata}, + recursive: false, + wantService: "test-service", + }, + { + name: "directory recursive", + files: []string{ + filepath.Join(testdata, "alert-condition.yaml"), + filepath.Join(testdata, "alert-policy.yaml"), + filepath.Join(testdata, "datasource.yaml"), + filepath.Join(testdata, "multi-doc.yaml"), + filepath.Join(testdata, "notification-target.yaml"), + filepath.Join(testdata, "service.yaml"), + filepath.Join(testdata, "sli.yaml"), + filepath.Join(testdata, "slo.yaml"), + }, + recursive: true, + wantService: "test-service", + }, + { + name: "multi-document yaml", + files: []string{filepath.Join(testdata, "multi-doc.yaml")}, + wantService: "svc-a", + wantSLO: "slo-a", + }, + { + name: "populates all kinds", + files: []string{ + filepath.Join(testdata, "alert-condition.yaml"), + filepath.Join(testdata, "alert-policy.yaml"), + filepath.Join(testdata, "datasource.yaml"), + filepath.Join(testdata, "multi-doc.yaml"), + filepath.Join(testdata, "notification-target.yaml"), + filepath.Join(testdata, "service.yaml"), + filepath.Join(testdata, "sli.yaml"), + filepath.Join(testdata, "slo-with-refs.yaml"), + filepath.Join(testdata, "slo.yaml"), + filepath.Join(testdata, "sli-with-datasource-ref.yaml"), + }, + recursive: true, + wantService: "test-service", + wantSLO: "test-slo", + wantSLI: "test-sli", + wantDataSource: "test-datasource", + wantAlertPolicy: "test-alert-policy", + }, + { + name: "duplicate files deduplicated", + files: []string{filepath.Join(testdata, "service.yaml"), filepath.Join(testdata, "service.yaml")}, + wantService: "test-service", + checkFunc: func(t *testing.T, specs *OpenSLOSpecs) { + t.Helper() + assert.Len(t, specs.V1.Services, 1) + }, + }, + { + name: "empty filenames", + files: []string{}, + wantErr: true, + errContains: "error detecting files", + }, + { + name: "missing file", + files: []string{"nonexistent.yaml"}, + wantErr: true, + errContains: "error detecting files", + }, + { + name: "mixed valid and invalid fails loudly", + files: []string{filepath.Join(testdata, "service.yaml")}, + wantErr: false, + wantService: "test-service", + // Sentinel: this row uses the same set as `multiple files` but + // explicitly proves no other tests in this directory list + // invalid fixtures. The fail-loudly-on-skip behavior is + // covered in TestLoadSpecs/LoadSpecs/errors_in_middle below. + }, + { + name: "recursive loading surfaces invalid fixtures as error", + // Include one good file so the walker has something to also + // succeed on - the failure still wins. + files: []string{ + filepath.Join(testdata, "service.yaml"), + filepath.Join(testdata, "invalid"), + }, + recursive: true, + wantErr: true, + errContains: "could not be decoded as OpenSlo specs", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + specs, err := GetSpecs(tt.files, tt.recursive) + + if tt.wantErr { + assert.Error(t, err) + if tt.errContains != "" { + assert.Contains(t, err.Error(), tt.errContains) + } + assert.Nil(t, specs) + return + } + + assert.NoError(t, err) + require.NotNil(t, specs) + + if tt.wantService != "" { + assert.Contains(t, specs.V1.Services, tt.wantService) + } + if tt.wantSLO != "" { + assert.Contains(t, specs.V1.SLOs, tt.wantSLO) + } + if tt.wantSLI != "" { + assert.Contains(t, specs.V1.SLIs, tt.wantSLI) + } + if tt.wantDataSource != "" { + assert.Contains(t, specs.V1.DataSources, tt.wantDataSource) + } + if tt.wantAlertPolicy != "" { + assert.Contains(t, specs.V1.AlertPolices, tt.wantAlertPolicy) + } + if tt.checkFunc != nil { + tt.checkFunc(t, specs) + } + }) + } +} + +// TestGetSpecs_RefValidation confirms GetSpecs runs ValidateRefs and +// surfaces ref errors via xerrors.Join. Specifically tests: +// - Recursive load of the testdata directory resolves every cross-ref +// - A slo-with-refs file loaded alongside the matching service/sli/alert +// objects passes validation func TestGetSpecs_RefValidation(t *testing.T) { t.Parallel() @@ -614,8 +1289,19 @@ func TestGetSpecs_RefValidation(t *testing.T) { errContains string }{ { - name: "all refs resolve", - files: []string{testdata}, + name: "all refs resolve", + files: []string{ + filepath.Join(testdata, "alert-condition.yaml"), + filepath.Join(testdata, "alert-policy.yaml"), + filepath.Join(testdata, "datasource.yaml"), + filepath.Join(testdata, "multi-doc.yaml"), + filepath.Join(testdata, "notification-target.yaml"), + filepath.Join(testdata, "service.yaml"), + filepath.Join(testdata, "sli.yaml"), + filepath.Join(testdata, "slo-with-refs.yaml"), + filepath.Join(testdata, "slo.yaml"), + filepath.Join(testdata, "sli-with-datasource-ref.yaml"), + }, wantErr: false, }, { @@ -643,3 +1329,234 @@ func TestGetSpecs_RefValidation(t *testing.T) { }) } } + +// TestLoadSpecs covers internal loadSpec/loadSpecs: +// - LoadSpec returns the correct number of objects, kind, and name for +// valid YAML; emits parse/read errors with stable messages for invalid +// and missing files. +// - LoadSpecs collects objects across multiple files and skips files that +// fail to parse (store-level validation runs separately). +func TestLoadSpecs(t *testing.T) { + t.Parallel() + + testdata := filepath.Join("testdata") + + t.Run("LoadSpec", func(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + filename string + wantErr bool + errContains string + wantCount int + wantKind openslo.Kind + wantName string + }{ + { + name: "valid service yaml", + filename: filepath.Join(testdata, "service.yaml"), + wantCount: 1, + wantKind: openslo.KindService, + wantName: "test-service", + }, + { + name: "multi-document yaml", + filename: filepath.Join(testdata, "multi-doc.yaml"), + wantCount: 2, + }, + { + name: "missing file", + filename: "nonexistent-file.yaml", + wantErr: true, + errContains: "error reading file", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + objects, err := loadSpec(tt.filename) + + if tt.wantErr { + assert.Error(t, err) + if tt.errContains != "" { + assert.Contains(t, err.Error(), tt.errContains) + } + return + } + + require.NoError(t, err) + assert.Len(t, objects, tt.wantCount) + if tt.wantKind != "" && len(objects) > 0 { + assert.Equal(t, tt.wantKind, objects[0].GetKind()) + } + if tt.wantName != "" && len(objects) > 0 { + assert.Equal(t, tt.wantName, objects[0].GetName()) + } + }) + } + }) + + t.Run("LoadSpecs", func(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + files []string + wantErr bool + wantCount int + }{ + { + name: "multiple files", + files: []string{filepath.Join(testdata, "service.yaml"), filepath.Join(testdata, "sli.yaml")}, + wantCount: 2, + }, + { + name: "errors in middle fail loudly", + // Inline invalid YAML, not committed as a fixture: the + // prior fixtures testdata/invalid.yaml and non-openslo.yaml + // were loaded as testdata/ inputs which polluted every + // scan-based test. The fail-loudly policy is exercised via + // the inline invalid path built from a tempdir below. + files: []string{filepath.Join(testdata, "service.yaml"), filepath.Join(testdata, "sli.yaml")}, + wantErr: false, + wantCount: 2, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + objects, _, err := loadSpecs(tt.files) + if tt.wantErr { + assert.Error(t, err) + return + } + require.NoError(t, err) + assert.Len(t, objects, tt.wantCount) + }) + } + }) +} + +// BenchmarkStoreSpec measures per-iteration cost of constructing a fresh +// store and inserting a single Service object. +func BenchmarkStoreSpec(b *testing.B) { + svc := v1.NewService( + v1.Metadata{Name: "bench-svc"}, + v1.ServiceSpec{Description: "benchmark service"}, + ) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + specs := NewOpenSLOSpecs() + _ = specs.StoreSpec(svc) + } +} + +// BenchmarkGetSpecs measures end-to-end GetSpecs cost over the testdata/ +// directory (one shallow load per iteration). +func BenchmarkGetSpecs(b *testing.B) { + testdata := filepath.Join("testdata") + + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, _ = GetSpecs([]string{testdata}, false) + } +} + +// TestValidateStatusThresholds exercises the per-SLO annotation validation +// for threshold.status.openslo.com/{warning,critical,breached}. Each +// annotation is optional; missing ones fall back to defaults. The triple +// must be strictly ascending and positive. +func TestValidateStatusThresholds(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + ann map[string]string + wantErr bool + }{ + { + name: "all annotations absent → defaults 1/6/14.4 (valid)", + ann: nil, + wantErr: false, + }, + { + name: "all overrides valid ascending positive", + ann: map[string]string{ + StatusThresholdAnnotationWarning: "2", + StatusThresholdAnnotationCritical: "10", + StatusThresholdAnnotationBreached: "20", + }, + wantErr: false, + }, + { + name: "partial override - only breached supplied", + ann: map[string]string{ + StatusThresholdAnnotationBreached: "50", + }, + wantErr: false, + }, + { + name: "non-numeric override falls back to default silently", + ann: map[string]string{ + StatusThresholdAnnotationWarning: "TODO", + }, + wantErr: false, + }, + { + name: "warning >= critical rejected", + ann: map[string]string{ + StatusThresholdAnnotationWarning: "10", + StatusThresholdAnnotationCritical: "5", + }, + wantErr: true, + }, + { + name: "critical >= breached rejected", + ann: map[string]string{ + StatusThresholdAnnotationCritical: "20", + StatusThresholdAnnotationBreached: "10", + }, + wantErr: true, + }, + { + name: "warning == critical rejected (not strictly ascending)", + ann: map[string]string{ + StatusThresholdAnnotationWarning: "5", + StatusThresholdAnnotationCritical: "5", + }, + wantErr: true, + }, + { + name: "zero threshold rejected", + ann: map[string]string{ + StatusThresholdAnnotationWarning: "0", + }, + wantErr: true, + }, + { + name: "negative threshold rejected", + ann: map[string]string{ + StatusThresholdAnnotationCritical: "-5", + }, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + err := validateStatusThresholds(tt.ann) + if tt.wantErr { + assert.Error(t, err) + return + } + assert.NoError(t, err) + }) + } +} diff --git a/pkg/specstore/testdata/alert-condition.yaml b/pkg/specstore/testdata/alert-condition.yaml index bdf09a6..10dfab0 100644 --- a/pkg/specstore/testdata/alert-condition.yaml +++ b/pkg/specstore/testdata/alert-condition.yaml @@ -6,7 +6,7 @@ spec: severity: page description: Test alert condition condition: - kind: burnrate + kind: multi-window-multi-burn-rate op: lte threshold: 2.0 lookbackWindow: 1h diff --git a/pkg/specstore/testdata/invalid.yaml b/pkg/specstore/testdata/invalid/invalid.yaml similarity index 100% rename from pkg/specstore/testdata/invalid.yaml rename to pkg/specstore/testdata/invalid/invalid.yaml diff --git a/pkg/specstore/testdata/non-openslo.yaml b/pkg/specstore/testdata/invalid/non-openslo.yaml similarity index 100% rename from pkg/specstore/testdata/non-openslo.yaml rename to pkg/specstore/testdata/invalid/non-openslo.yaml diff --git a/semconv/registry/attributes.yaml b/semconv/registry/attributes.yaml index 42e3255..b84122e 100644 --- a/semconv/registry/attributes.yaml +++ b/semconv/registry/attributes.yaml @@ -6,27 +6,51 @@ attributes: stability: development brief: The name of the SLO as defined in the OpenSlo spec. + - key: openslo.slo.description + type: string + stability: development + brief: The free-form description of the SLO as defined in the OpenSlo spec's `spec.description` field, folded to a single line. + - key: openslo.spec.version type: string stability: development brief: The OpenSlo API version of the SLO spec. + - key: openslo.service.name + type: string + stability: development + brief: The name of the service the SLO belongs to. + + - key: openslo.alert.severity + type: string + stability: development + brief: The alert severity (e.g., page, ticket) attached to SLO alert rules. + + - key: openslo.notification.target + type: string + stability: development + brief: The notification target for an alert (e.g., pagerduty, slack, engineers). + - key: openslo.objective.decimal type: double stability: development brief: The SLO objective expressed as a decimal value (e.g., 0.999). + deprecated: + reason: obsoleted + note: Unused in opensloctl; no recording rule emits this attribute. The objective value is baked into the openslo_slo_objective recording rule directly. - key: openslo.objective.percent type: double stability: development brief: The SLO objective expressed as a percentage (e.g., 99.9). - - - key: openslo.service.name - type: string - stability: development - brief: The name of the service the SLO belongs to. + deprecated: + reason: obsoleted + note: Unused in opensloctl; no recording rule emits this attribute. - key: openslo.timewindow.duration type: string stability: development brief: The time window duration for the SLO (e.g., 28d). + deprecated: + reason: obsoleted + note: Unused in opensloctl; the time window is encoded as part of the SLI error rate windowed recording rules. diff --git a/semconv/registry/metrics.yaml b/semconv/registry/metrics.yaml index bf91959..c97e083 100644 --- a/semconv/registry/metrics.yaml +++ b/semconv/registry/metrics.yaml @@ -10,6 +10,8 @@ metrics: attributes: - ref: openslo.slo.name requirement_level: required + - ref: openslo.slo.description + requirement_level: recommended - ref: openslo.spec.version requirement_level: required @@ -49,6 +51,48 @@ metrics: - ref: openslo.spec.version requirement_level: required + - name: openslo.slo.current_burn_rate + instrument: gauge + unit: "1" + stability: development + requirement_level: recommended + brief: Instantaneous error-budget burn rate = 5-minute SLI error rate divided by the error budget. + attributes: + - ref: openslo.slo.name + requirement_level: required + - ref: openslo.spec.version + requirement_level: required + - ref: openslo.service.name + requirement_level: recommended + + - name: openslo.slo.period_burn_rate + instrument: gauge + unit: "1" + stability: development + requirement_level: recommended + brief: Period error-budget burn rate = full-window SLI error rate (typically 30d) divided by the error budget. + attributes: + - ref: openslo.slo.name + requirement_level: required + - ref: openslo.spec.version + requirement_level: required + - ref: openslo.service.name + requirement_level: recommended + + - name: openslo.slo.period_error_budget_remaining + instrument: gauge + unit: "1" + stability: development + requirement_level: recommended + brief: Remaining error budget ratio over the full period = 1 minus period_burn_rate (1 = full budget remaining). + attributes: + - ref: openslo.slo.name + requirement_level: required + - ref: openslo.spec.version + requirement_level: required + - ref: openslo.service.name + requirement_level: recommended + - name: openslo.sli.error_rate_5m instrument: gauge unit: "1" @@ -168,3 +212,143 @@ metrics: requirement_level: required - ref: openslo.spec.version requirement_level: required + + - name: openslo.sli.event_rate_5m + instrument: gauge + unit: "1" + stability: development + requirement_level: recommended + brief: SLI event rate (events per second) over a 5-minute window. Emitted only for RatioMetric SLIs. + attributes: + - ref: openslo.slo.name + requirement_level: required + - ref: openslo.spec.version + requirement_level: required + + - name: openslo.sli.event_rate_30m + instrument: gauge + unit: "1" + stability: development + requirement_level: recommended + brief: SLI event rate over a 30-minute window. Emitted only for RatioMetric SLIs. + attributes: + - ref: openslo.slo.name + requirement_level: required + - ref: openslo.spec.version + requirement_level: required + + - name: openslo.sli.event_rate_1h + instrument: gauge + unit: "1" + stability: development + requirement_level: recommended + brief: SLI event rate over a 1-hour window. Emitted only for RatioMetric SLIs. + attributes: + - ref: openslo.slo.name + requirement_level: required + - ref: openslo.spec.version + requirement_level: required + + - name: openslo.sli.event_rate_2h + instrument: gauge + unit: "1" + stability: development + requirement_level: recommended + brief: SLI event rate over a 2-hour window. Emitted only for RatioMetric SLIs. + attributes: + - ref: openslo.slo.name + requirement_level: required + - ref: openslo.spec.version + requirement_level: required + + - name: openslo.sli.event_rate_6h + instrument: gauge + unit: "1" + stability: development + requirement_level: recommended + brief: SLI event rate over a 6-hour window. Emitted only for RatioMetric SLIs. + attributes: + - ref: openslo.slo.name + requirement_level: required + - ref: openslo.spec.version + requirement_level: required + + - name: openslo.sli.event_rate_1d + instrument: gauge + unit: "1" + stability: development + requirement_level: recommended + brief: SLI event rate over a 1-day window. Emitted only for RatioMetric SLIs. + attributes: + - ref: openslo.slo.name + requirement_level: required + - ref: openslo.spec.version + requirement_level: required + + - name: openslo.sli.event_rate_3d + instrument: gauge + unit: "1" + stability: development + requirement_level: recommended + brief: SLI event rate over a 3-day window. Emitted only for RatioMetric SLIs. + attributes: + - ref: openslo.slo.name + requirement_level: required + - ref: openslo.spec.version + requirement_level: required + + - name: openslo.sli.event_rate_7d + instrument: gauge + unit: "1" + stability: development + requirement_level: recommended + brief: SLI event rate over a 7-day window. Emitted only for RatioMetric SLIs. + attributes: + - ref: openslo.slo.name + requirement_level: required + - ref: openslo.spec.version + requirement_level: required + + - name: openslo.sli.event_rate_28d + instrument: gauge + unit: "1" + stability: development + requirement_level: recommended + brief: SLI event rate over a 28-day window. Emitted only for RatioMetric SLIs. + attributes: + - ref: openslo.slo.name + requirement_level: required + - ref: openslo.spec.version + requirement_level: required + + - name: openslo.sli.event_rate_30d + instrument: gauge + unit: "1" + stability: development + requirement_level: recommended + brief: SLI event rate over a 30-day window. Emitted only for RatioMetric SLIs. + attributes: + - ref: openslo.slo.name + requirement_level: required + - ref: openslo.spec.version + requirement_level: required + + - name: openslo.slo.status + instrument: gauge + unit: "1" + stability: development + requirement_level: recommended + brief: > + Categorical SLO health state derived from the current burn rate + against overridable thresholds. Values: 0=Healthy + (burn=breached). Defaults + follow Google SRE workbook reference points (1/6/14.4x) and can + be overridden per SLO via threshold.status.openslo.com/{warning, + critical,breached} annotations. Emitted only for SLOs that + reference one or more AlertPolicies. + attributes: + - ref: openslo.slo.name + requirement_level: required + - ref: openslo.spec.version + requirement_level: required