From 68dd6bdefc909266e18eb2d76ffe41659d160f61 Mon Sep 17 00:00:00 2001 From: Pavlov Alexandr Date: Sun, 14 Jun 2026 23:41:54 +0700 Subject: [PATCH] feat(observability): real ordered log-level filter via detected_level + pino mapping MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the cross-format substring-regex Min-level stopgap in the Logs (Loki) dashboard (uid solid-stats-logs) with a real ordered level filter. The dropdown now emits LogQL label filters over the level / detected_level fields instead of matching message substrings, so an info line containing the word "error" no longer matches Warning+. Loki 3.6.x already attaches a query-time detected_level for the text-format apps (rabbitmq/postgres/glitchtip), but maps server-2's pino NUMERIC "level":30/40/50 to detected_level: unknown, hiding our main app under any level filter. Add an Alloy pipeline stage (80-alloy.yaml) scoped to app="server-2" that parses the pino JSON level, maps the numeric severity to a normalized text level (info/warn/error/fatal), and exposes it as STRUCTURED METADATA named level — not a new stream label, so the Phase 15 5-key label allowlist and stream cardinality are unchanged. The dashboard filter ORs both fields (| level=~... or detected_level=~...) so a single Min-level dropdown works across all apps. Closes the logs-level-filtering-todo brief (option 1 detected_level + option 2 Alloy mapping fallback for pino). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../61-grafana-logs-dashboard.yaml | 15 ++++----- k8s/observability/80-alloy.yaml | 31 ++++++++++++++++++- 2 files changed, 38 insertions(+), 8 deletions(-) diff --git a/k8s/observability/61-grafana-logs-dashboard.yaml b/k8s/observability/61-grafana-logs-dashboard.yaml index bc45f48..c57a035 100644 --- a/k8s/observability/61-grafana-logs-dashboard.yaml +++ b/k8s/observability/61-grafana-logs-dashboard.yaml @@ -79,27 +79,28 @@ data: "label": "Min level", "type": "custom", "hide": 0, - "query": "All : . , Warning+ : (?i)(\"level\":[456]0|\"level\":\"(warn|warning|error|fatal|critical)\"|\\[(warn|warning|error|fatal|crit|critical)\\]) , Error+ : (?i)(\"level\":[56]0|\"level\":\"(error|fatal|critical)\"|\\[(error|fatal|crit|critical)\\])", + "description": "Ordered level filter using real level fields, not message substrings. server-2 (pino) gets a normalized `level` via structured metadata from Alloy; the other apps use Loki's query-time `detected_level`. The OR covers both so a single dropdown works across all apps.", + "query": "All : , Warning+ : | level=~\"warn|error|fatal|critical\" or detected_level=~\"warn|error|fatal|critical\" , Error+ : | level=~\"error|fatal|critical\" or detected_level=~\"error|fatal|critical\"", "current": { "selected": true, "text": "All", - "value": "." + "value": "" }, "options": [ { "selected": true, "text": "All", - "value": "." + "value": "" }, { "selected": false, "text": "Warning+", - "value": "(?i)(\"level\":[456]0|\"level\":\"(warn|warning|error|fatal|critical)\"|\\[(warn|warning|error|fatal|crit|critical)\\])" + "value": " | level=~\"warn|error|fatal|critical\" or detected_level=~\"warn|error|fatal|critical\"" }, { "selected": false, "text": "Error+", - "value": "(?i)(\"level\":[56]0|\"level\":\"(error|fatal|critical)\"|\\[(error|fatal|crit|critical)\\])" + "value": " | level=~\"error|fatal|critical\" or detected_level=~\"error|fatal|critical\"" } ], "includeAll": false, @@ -161,7 +162,7 @@ data: { "refId": "A", "datasource": "Loki", - "expr": "sum by (app) (count_over_time({namespace=~\"$namespace\", app=~\"$app\"} |~ \"(?i)$search\" |~ `$level` [$__interval]))", + "expr": "sum by (app) (count_over_time({namespace=~\"$namespace\", app=~\"$app\"} |~ \"(?i)$search\" $level [$__interval]))", "legendFormat": "{{app}}", "queryType": "range" } @@ -204,7 +205,7 @@ data: { "refId": "A", "datasource": "Loki", - "expr": "{namespace=~\"$namespace\", app=~\"$app\"} |~ \"(?i)$search\" |~ `$level`", + "expr": "{namespace=~\"$namespace\", app=~\"$app\"} |~ \"(?i)$search\" $level", "queryType": "range", "maxLines": 500 } diff --git a/k8s/observability/80-alloy.yaml b/k8s/observability/80-alloy.yaml index c3dc38c..70cd6bd 100644 --- a/k8s/observability/80-alloy.yaml +++ b/k8s/observability/80-alloy.yaml @@ -89,8 +89,37 @@ data: forward_to = [loki.process.drop_extra_labels.receiver] } - // Drop any remaining labels not in the 5-key allowlist (LOG-02, T-15-05) + // Normalize server-2's pino NUMERIC level into a real ordered level, exposed + // as STRUCTURED METADATA (not a label — keep the 5-key label allowlist below to + // avoid stream cardinality blow-up, LOG-02). Loki's query-time detected_level + // already gives a real level to the text-format apps (rabbitmq/postgres/glitchtip) + // but maps pino's numeric "level":30/40/50 to `unknown`, hiding server-2 under a + // level filter. This stage fixes that. See plans logs-level-filtering-todo brief. loki.process "drop_extra_labels" { + // Scope to server-2 only; other apps keep their query-time detected_level. + stage.match { + selector = "{app=\"server-2\"}" + + // Pino logs JSON; pull the numeric level out of the line into `level`. + stage.json { + expressions = { "level" = "level" } + } + + // Map pino numeric severity -> normalized text level, overwriting `level`. + // Pino: trace=10 debug=20 info=30 warn=40 error=50 fatal=60. `.Value` is the + // current `level` value; the result is written back to the `level` key. + stage.template { + source = "level" + template = "{{ if eq .Value \"60\" }}fatal{{ else if eq .Value \"50\" }}error{{ else if eq .Value \"40\" }}warn{{ else if eq .Value \"30\" }}info{{ else if eq .Value \"20\" }}debug{{ else if eq .Value \"10\" }}trace{{ else }}unknown{{ end }}" + } + + // Expose as structured metadata named `level` (queryable, not a stream label). + stage.structured_metadata { + values = { "level" = "" } + } + } + + // Drop any remaining labels not in the 5-key allowlist (LOG-02, T-15-05) stage.label_keep { values = ["namespace", "pod", "container", "app", "job"] }