From 927b674493bc8b9428b2197b3d942a94de5ecad5 Mon Sep 17 00:00:00 2001 From: Leo Parente <23251360+leoparente@users.noreply.github.com> Date: Thu, 25 Jun 2026 13:34:11 -0300 Subject: [PATCH 1/6] docs(k8s): add pktvisor sidecar deployment example (#743) --- k8s/pktvisor-sidecar.yaml | 75 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 k8s/pktvisor-sidecar.yaml diff --git a/k8s/pktvisor-sidecar.yaml b/k8s/pktvisor-sidecar.yaml new file mode 100644 index 000000000..20afa6cdb --- /dev/null +++ b/k8s/pktvisor-sidecar.yaml @@ -0,0 +1,75 @@ +# pktvisor as a sidecar: observes the pod's eth0 NIC traffic and exposes +# Prometheus metrics on :10853. See README.md in this directory for usage, +# verification, the Prometheus scrape config, and how to adapt to your workload. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: pktvisor-demo + labels: + app: pktvisor-demo +spec: + replicas: 1 + selector: + matchLabels: + app: pktvisor-demo + template: + metadata: + labels: + app: pktvisor-demo + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "10853" + prometheus.io/path: "/metrics" + spec: + containers: + # ---- your workload: replace this container with your own ---- + - name: app + image: nginx:stable-alpine + ports: + - containerPort: 80 + + # ---- OPTIONAL traffic generator (delete for real workloads) ---- + # wget's an EXTERNAL URL so the request leaves the pod over eth0, which + # pktvisord observes (loopback traffic would NOT be seen). Requires + # cluster egress; repoint TARGET_URL or remove this container as needed. + - name: traffic-gen + image: busybox:stable + env: + - name: TARGET_URL + value: "http://example.com" + - name: INTERVAL + value: "5" + command: ["/bin/sh", "-c"] + args: + - 'while true; do wget -q -O /dev/null "$TARGET_URL" || true; sleep "$INTERVAL"; done' + + # ---- pktvisord sidecar: capture on eth0, serve /metrics on :10853 ---- + - name: pktvisord + image: netboxlabs/pktvisor + # The image ENTRYPOINT (/entry-cp.sh) selects the binary from the first + # arg and forwards the rest to pktvisord (adding crashpad/geo defaults). + # Keep the leading "pktvisord". + # -l 0.0.0.0 : bind the metrics server on all interfaces; the default + # is localhost, which Prometheus could not reach at POD_IP. + # -H .../32 : single-host CIDR so pktvisord attributes in/out direction + # (a bare IP throws "invalid CIDR"; use /128 on IPv6 clusters). + # eth0 : run the default net+dhcp+dns+pcap policy on the pod NIC. + args: ["pktvisord", "-l", "0.0.0.0", "-H", "$(POD_IP)/32", "eth0"] + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + ports: + - name: metrics + containerPort: 10853 + securityContext: + capabilities: + add: ["NET_RAW"] # minimum capability for packet capture; NOT privileged + resources: + requests: + cpu: "50m" + memory: "128Mi" + limits: + cpu: "500m" + memory: "512Mi" From 912dd42ee337c6167a91e8869fcfad0f337e1651 Mon Sep 17 00:00:00 2001 From: Leo Parente <23251360+leoparente@users.noreply.github.com> Date: Thu, 25 Jun 2026 13:34:11 -0300 Subject: [PATCH 2/6] docs(k8s): document the pktvisor sidecar example (#743) --- k8s/README.md | 138 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 138 insertions(+) create mode 100644 k8s/README.md diff --git a/k8s/README.md b/k8s/README.md new file mode 100644 index 000000000..28242e261 --- /dev/null +++ b/k8s/README.md @@ -0,0 +1,138 @@ +# pktvisor on Kubernetes (sidecar) + +Run `pktvisord` as a **sidecar** in a pod to observe that pod's network traffic +(`eth0`) and expose Prometheus metrics on `:10853`. + +``` ++------------------- Pod (shared network namespace) -------------------+ +| | +| [ app ] [ traffic-gen (optional) ] [ pktvisord ] | +| nginx wget loop -> external URL sniffs eth0 | +| :10853/metrics | ++----------------------------------------------------------------------+ + ^ + | Prometheus scrapes via pod annotations +``` + +Containers in a pod share one network namespace, so the `pktvisord` sidecar sees +the same `eth0` as your application container. + +## Prerequisites + +- A Kubernetes cluster and `kubectl`. +- The target namespace must allow the `NET_RAW` capability — PodSecurity + `baseline` or `privileged`, **not** `restricted`. pktvisord needs `NET_RAW` + for packet capture; it does **not** need `privileged`. +- Prometheus (or Grafana Agent) configured to scrape pods by annotation — see + [Prometheus scrape configuration](#prometheus-scrape-configuration). +- The example captures `eth0`, the pod's primary interface on most CNIs. If your + pods use a different name, change `eth0` in the manifest args (or use `auto` to + let pktvisord pick the busiest interface). Find the name with + `kubectl exec -c app -- ip -o link`. + +## Deploy + +```shell +kubectl apply -f pktvisor-sidecar.yaml +``` + +Creates a `pktvisor-demo` Deployment with three containers: your app (`nginx` +placeholder), an optional traffic generator, and the `pktvisord` sidecar. + +## Verify + +```shell +kubectl rollout status deploy/pktvisor-demo +kubectl port-forward deploy/pktvisor-demo 10853:10853 +# in another terminal: +curl -s localhost:10853/metrics | grep -E '^(dns|packets)_' | head +``` + +You should see non-zero `packets_*` and `dns_*` series (the traffic generator +drives them) — a populated `/metrics` is the "it's working" signal. pktvisord +aggregates over ~60s windows, so if the first scrape looks sparse, wait a few +seconds and retry. Note: the +published image redirects pktvisord's own stdout to a file, so +`kubectl logs deploy/pktvisor-demo -c pktvisord` is usually **empty** — rely on +`/metrics`, not logs. (If the sidecar `CrashLoopBackOff`s instead, re-check the +`NET_RAW` capability and the `-H .../32` CIDR.) + +## Generating observable traffic + +pktvisord watches **`eth0`**, so it sees the pod's **ingress/egress** — not +loopback. The optional `traffic-gen` container produces egress by `wget`-ing an +external URL (`TARGET_URL`, default `http://example.com`). To see ingress, send +requests to the app from outside the pod (e.g. the `port-forward` above, or a +Service). Repoint with `TARGET_URL`/`INTERVAL`, or delete the `traffic-gen` +container for real workloads. + +No internet egress (air-gapped, or a default-deny `NetworkPolicy`)? The +per-iteration DNS lookups still go to cluster DNS over `eth0`, so `dns_*` +populates regardless; for richer `packets_*`, point `TARGET_URL` at an in-cluster +Service, or drive the app via the `port-forward` above. + +## Prometheus scrape configuration + +The pod carries the standard annotations: + +```yaml +prometheus.io/scrape: "true" +prometheus.io/port: "10853" +prometheus.io/path: "/metrics" +``` + +A vanilla Prometheus discovers it with a pod job (no Prometheus Operator +required): + +```yaml +scrape_configs: + - job_name: kubernetes-pods + kubernetes_sd_configs: + - role: pod + relabel_configs: + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape] + action: keep + regex: "true" + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path] + action: replace + target_label: __metrics_path__ + regex: (.+) + - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port] + action: replace + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:$2 + target_label: __address__ + - source_labels: [__meta_kubernetes_namespace] + target_label: namespace + - source_labels: [__meta_kubernetes_pod_name] + target_label: pod +``` + +> Using the Prometheus Operator (kube-prometheus-stack)? Expose the sidecar with +> a `Service` on port `10853` and create a `ServiceMonitor` selecting it, instead +> of using annotations. + +## Grafana dashboard + +Import the community dashboard **ID 14221**, or the JSON at +`../centralized_collection/prometheus/grafana-dashboard-prometheus.json`. + +## Use with your own workload + +Add the sidecar to any existing pod spec: + +1. Add the three `prometheus.io/*` pod annotations. For a Deployment/StatefulSet + they go on the **pod template** (`spec.template.metadata.annotations`), not the + top-level object metadata; for a bare Pod they go on `metadata.annotations`. +2. Add the `pktvisord` container — including the `POD_IP` downward-API env, the + `NET_RAW` capability, and `args: ["pktvisord", "-l", "0.0.0.0", "-H", "$(POD_IP)/32", "eth0"]`. +3. Remove the demo `app` and `traffic-gen` containers. + +## Security note + +The sidecar adds only `NET_RAW` (not `privileged`) — the minimum capability +libpcap/AF_PACKET needs to capture packets. Promiscuous mode (which would need +`NET_ADMIN`) is not required to see the pod's own traffic. The PodSecurity +`restricted` profile disallows `NET_RAW`; use `baseline`/`privileged` or a +tailored policy. Note `-l 0.0.0.0` exposes `/metrics` on the pod IP (no auth) so +Prometheus can scrape it — restrict with a `NetworkPolicy` if needed. From 9a859e3c4654a21e6588662177ecf7f1326a3227 Mon Sep 17 00:00:00 2001 From: Leo Parente <23251360+leoparente@users.noreply.github.com> Date: Thu, 25 Jun 2026 13:51:10 -0300 Subject: [PATCH 3/6] docs(k8s): correct PodSecurity NET_RAW prerequisite; clarify apply path and pod SD dedup (#743) --- k8s/README.md | 36 ++++++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/k8s/README.md b/k8s/README.md index 28242e261..23b8a49bc 100644 --- a/k8s/README.md +++ b/k8s/README.md @@ -20,9 +20,13 @@ the same `eth0` as your application container. ## Prerequisites - A Kubernetes cluster and `kubectl`. -- The target namespace must allow the `NET_RAW` capability — PodSecurity - `baseline` or `privileged`, **not** `restricted`. pktvisord needs `NET_RAW` - for packet capture; it does **not** need `privileged`. +- The target namespace must allow pods to **add the `NET_RAW` capability**. Under + the Pod Security Standards only the `privileged` level permits this (or an + exemption / custom admission policy that allows `NET_RAW`); both `baseline` and + `restricted` reject it — `baseline` only allows adding `NET_BIND_SERVICE`. This + is about the namespace PSS *level*; the container itself is **not** run as + `privileged: true` — it only adds the single `NET_RAW` capability pktvisord + needs to capture packets. - Prometheus (or Grafana Agent) configured to scrape pods by annotation — see [Prometheus scrape configuration](#prometheus-scrape-configuration). - The example captures `eth0`, the pod's primary interface on most CNIs. If your @@ -33,7 +37,8 @@ the same `eth0` as your application container. ## Deploy ```shell -kubectl apply -f pktvisor-sidecar.yaml +# from the repo root (or use the bare filename from inside k8s/) +kubectl apply -f k8s/pktvisor-sidecar.yaml ``` Creates a `pktvisor-demo` Deployment with three containers: your app (`nginx` @@ -108,6 +113,14 @@ scrape_configs: target_label: pod ``` +`role: pod` generates one target per declared container port, so a +multi-container pod (here `app` + `pktvisord`) is discovered as several targets +that these rules all rewrite to the same `POD_IP:10853`. Prometheus deduplicates +targets whose final label sets are identical, so the endpoint is still scraped +only **once**. Don't add a container-name/port `keep` to this shared job to +"fix" the duplication — it would drop every other annotated pod; if you want +explicit selection, do it inside a pktvisor-only scrape job. + > Using the Prometheus Operator (kube-prometheus-stack)? Expose the sidecar with > a `Service` on port `10853` and create a `ServiceMonitor` selecting it, instead > of using annotations. @@ -130,9 +143,12 @@ Add the sidecar to any existing pod spec: ## Security note -The sidecar adds only `NET_RAW` (not `privileged`) — the minimum capability -libpcap/AF_PACKET needs to capture packets. Promiscuous mode (which would need -`NET_ADMIN`) is not required to see the pod's own traffic. The PodSecurity -`restricted` profile disallows `NET_RAW`; use `baseline`/`privileged` or a -tailored policy. Note `-l 0.0.0.0` exposes `/metrics` on the pod IP (no auth) so -Prometheus can scrape it — restrict with a `NetworkPolicy` if needed. +The sidecar adds only the `NET_RAW` capability (it is **not** run as a +`privileged` container) — the minimum libpcap/AF_PACKET needs to capture packets. +Promiscuous mode (which would need `NET_ADMIN`) is not required to see the pod's +own traffic. Under the Pod Security Standards, both the `baseline` and +`restricted` levels reject adding `NET_RAW` (baseline only allows adding +`NET_BIND_SERVICE`), so run this in a namespace at the `privileged` level or with +an exemption / custom policy that permits it. Note `-l 0.0.0.0` exposes +`/metrics` on the pod IP (no auth) so Prometheus can scrape it — restrict with a +`NetworkPolicy` if needed. From 7ac5c4df3c68e2a05b2c1de28f02b7f9cf86c467 Mon Sep 17 00:00:00 2001 From: Leo Parente <23251360+leoparente@users.noreply.github.com> Date: Thu, 25 Jun 2026 15:34:18 -0300 Subject: [PATCH 4/6] docs(k8s): add NET_ADMIN for promiscuous capture; IPv6-safe scrape regex; portable iface lookup (#743) --- k8s/README.md | 40 +++++++++++++++++++++------------------ k8s/pktvisor-sidecar.yaml | 5 ++++- 2 files changed, 26 insertions(+), 19 deletions(-) diff --git a/k8s/README.md b/k8s/README.md index 23b8a49bc..c10d913ef 100644 --- a/k8s/README.md +++ b/k8s/README.md @@ -20,19 +20,21 @@ the same `eth0` as your application container. ## Prerequisites - A Kubernetes cluster and `kubectl`. -- The target namespace must allow pods to **add the `NET_RAW` capability**. Under - the Pod Security Standards only the `privileged` level permits this (or an - exemption / custom admission policy that allows `NET_RAW`); both `baseline` and - `restricted` reject it — `baseline` only allows adding `NET_BIND_SERVICE`. This - is about the namespace PSS *level*; the container itself is **not** run as - `privileged: true` — it only adds the single `NET_RAW` capability pktvisord - needs to capture packets. +- The target namespace must allow pods to **add the `NET_RAW` and `NET_ADMIN` + capabilities** (pktvisord captures in promiscuous mode — see the security note). + Under the Pod Security Standards only the `privileged` level permits adding + these (or an exemption / custom admission policy that allows them); both + `baseline` and `restricted` reject them — `baseline` only allows adding + `NET_BIND_SERVICE`. This is about the namespace PSS *level*; the container + itself is **not** run as `privileged: true` — it only adds those two + capabilities pktvisord needs to capture packets. - Prometheus (or Grafana Agent) configured to scrape pods by annotation — see [Prometheus scrape configuration](#prometheus-scrape-configuration). - The example captures `eth0`, the pod's primary interface on most CNIs. If your pods use a different name, change `eth0` in the manifest args (or use `auto` to let pktvisord pick the busiest interface). Find the name with - `kubectl exec -c app -- ip -o link`. + `kubectl exec -c app -- ls /sys/class/net` (works without `iproute2`, + which the minimal demo images don't ship). ## Deploy @@ -60,7 +62,7 @@ seconds and retry. Note: the published image redirects pktvisord's own stdout to a file, so `kubectl logs deploy/pktvisor-demo -c pktvisord` is usually **empty** — rely on `/metrics`, not logs. (If the sidecar `CrashLoopBackOff`s instead, re-check the -`NET_RAW` capability and the `-H .../32` CIDR.) +`NET_RAW`/`NET_ADMIN` capabilities and the `-H .../32` CIDR.) ## Generating observable traffic @@ -104,7 +106,8 @@ scrape_configs: regex: (.+) - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port] action: replace - regex: ([^:]+)(?::\d+)?;(\d+) + # first group matches a bracketed IPv6 literal or an IPv4/host (no colons) + regex: (\[.+\]|[^:]+)(?::\d+)?;(\d+) replacement: $1:$2 target_label: __address__ - source_labels: [__meta_kubernetes_namespace] @@ -138,17 +141,18 @@ Add the sidecar to any existing pod spec: they go on the **pod template** (`spec.template.metadata.annotations`), not the top-level object metadata; for a bare Pod they go on `metadata.annotations`. 2. Add the `pktvisord` container — including the `POD_IP` downward-API env, the - `NET_RAW` capability, and `args: ["pktvisord", "-l", "0.0.0.0", "-H", "$(POD_IP)/32", "eth0"]`. + `NET_RAW` and `NET_ADMIN` capabilities, and `args: ["pktvisord", "-l", "0.0.0.0", "-H", "$(POD_IP)/32", "eth0"]`. 3. Remove the demo `app` and `traffic-gen` containers. ## Security note -The sidecar adds only the `NET_RAW` capability (it is **not** run as a -`privileged` container) — the minimum libpcap/AF_PACKET needs to capture packets. -Promiscuous mode (which would need `NET_ADMIN`) is not required to see the pod's -own traffic. Under the Pod Security Standards, both the `baseline` and -`restricted` levels reject adding `NET_RAW` (baseline only allows adding -`NET_BIND_SERVICE`), so run this in a namespace at the `privileged` level or with -an exemption / custom policy that permits it. Note `-l 0.0.0.0` exposes +The sidecar adds the `NET_RAW` and `NET_ADMIN` capabilities (it is **not** run as +a `privileged` container): `NET_RAW` opens the AF_PACKET/raw capture socket, and +`NET_ADMIN` is required because pktvisord opens the interface in **promiscuous +mode** — matching the project's documented bare-metal `setcap +cap_net_raw,cap_net_admin`. Under the Pod Security Standards, both the `baseline` +and `restricted` levels reject adding these capabilities (baseline only allows +adding `NET_BIND_SERVICE`), so run this in a namespace at the `privileged` level +or with an exemption / custom policy that permits them. Note `-l 0.0.0.0` exposes `/metrics` on the pod IP (no auth) so Prometheus can scrape it — restrict with a `NetworkPolicy` if needed. diff --git a/k8s/pktvisor-sidecar.yaml b/k8s/pktvisor-sidecar.yaml index 20afa6cdb..a7ff194d6 100644 --- a/k8s/pktvisor-sidecar.yaml +++ b/k8s/pktvisor-sidecar.yaml @@ -65,7 +65,10 @@ spec: containerPort: 10853 securityContext: capabilities: - add: ["NET_RAW"] # minimum capability for packet capture; NOT privileged + # NET_RAW opens the AF_PACKET/raw capture socket; NET_ADMIN is needed + # because pktvisord captures in promiscuous mode (matches the project's + # documented `setcap cap_net_raw,cap_net_admin`). Still NOT privileged. + add: ["NET_RAW", "NET_ADMIN"] resources: requests: cpu: "50m" From 5d6ec1dfa2edf2cbb7755a42701055aef1055835 Mon Sep 17 00:00:00 2001 From: Leo Parente <23251360+leoparente@users.noreply.github.com> Date: Thu, 25 Jun 2026 15:45:31 -0300 Subject: [PATCH 5/6] docs(k8s): correct default-deny DNS egress + ingress-via-loopback caveats (#743) --- k8s/README.md | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/k8s/README.md b/k8s/README.md index c10d913ef..00ba734b4 100644 --- a/k8s/README.md +++ b/k8s/README.md @@ -68,15 +68,20 @@ published image redirects pktvisord's own stdout to a file, so pktvisord watches **`eth0`**, so it sees the pod's **ingress/egress** — not loopback. The optional `traffic-gen` container produces egress by `wget`-ing an -external URL (`TARGET_URL`, default `http://example.com`). To see ingress, send -requests to the app from outside the pod (e.g. the `port-forward` above, or a -Service). Repoint with `TARGET_URL`/`INTERVAL`, or delete the `traffic-gen` -container for real workloads. - -No internet egress (air-gapped, or a default-deny `NetworkPolicy`)? The -per-iteration DNS lookups still go to cluster DNS over `eth0`, so `dns_*` -populates regardless; for richer `packets_*`, point `TARGET_URL` at an in-cluster -Service, or drive the app via the `port-forward` above. +external URL (`TARGET_URL`, default `http://example.com`). To generate observable +**ingress**, drive the app from *another* pod or through a Service so the request +enters over `eth0` — e.g. +`kubectl run client --rm -it --image=busybox --restart=Never -- wget -qO- :80`. +(`kubectl port-forward` reaches the app over the pod's *loopback*, so that +traffic does **not** appear on `eth0`.) Repoint with `TARGET_URL`/`INTERVAL`, or +delete the `traffic-gen` container for real workloads. + +No internet egress? If only *external* egress is blocked but in-cluster traffic +is allowed, the generator's DNS lookups still reach cluster DNS over `eth0`, so +`dns_*` populates; for richer `packets_*`, point `TARGET_URL` at an in-cluster +Service. Under a **default-deny `NetworkPolicy`**, egress to kube-dns is blocked +too — add an egress rule allowing DNS (UDP/TCP `53` to the kube-dns/CoreDNS pods), +otherwise name resolution itself fails and the metrics stay empty. ## Prometheus scrape configuration From e873985de33c14b1c65a3eb6442dd1f0b3ed0779 Mon Sep 17 00:00:00 2001 From: Leo Parente <23251360+leoparente@users.noreply.github.com> Date: Thu, 25 Jun 2026 17:47:42 -0300 Subject: [PATCH 6/6] docs(k8s): document IPv6-only/primary cluster args (-l ::, /128) (#743) --- k8s/README.md | 4 ++++ k8s/pktvisor-sidecar.yaml | 8 +++++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/k8s/README.md b/k8s/README.md index 00ba734b4..4305689a6 100644 --- a/k8s/README.md +++ b/k8s/README.md @@ -35,6 +35,10 @@ the same `eth0` as your application container. let pktvisord pick the busiest interface). Find the name with `kubectl exec -c app -- ls /sys/class/net` (works without `iproute2`, which the minimal demo images don't ship). +- The manifest's args are **IPv4** — the default `status.podIP` family, including + on dual-stack clusters. On an **IPv6-only or IPv6-primary** cluster, change the + sidecar args to bind IPv6 and use a `/128` host spec: `-l ::` (in place of + `-l 0.0.0.0`) and `-H $(POD_IP)/128` (in place of `/32`). ## Deploy diff --git a/k8s/pktvisor-sidecar.yaml b/k8s/pktvisor-sidecar.yaml index a7ff194d6..e9838c9a0 100644 --- a/k8s/pktvisor-sidecar.yaml +++ b/k8s/pktvisor-sidecar.yaml @@ -49,11 +49,13 @@ spec: # The image ENTRYPOINT (/entry-cp.sh) selects the binary from the first # arg and forwards the rest to pktvisord (adding crashpad/geo defaults). # Keep the leading "pktvisord". - # -l 0.0.0.0 : bind the metrics server on all interfaces; the default - # is localhost, which Prometheus could not reach at POD_IP. + # -l 0.0.0.0 : bind the metrics server on all IPv4 interfaces; the + # default is localhost, unreachable by Prometheus at POD_IP. # -H .../32 : single-host CIDR so pktvisord attributes in/out direction - # (a bare IP throws "invalid CIDR"; use /128 on IPv6 clusters). + # (a bare IP throws "invalid CIDR"). # eth0 : run the default net+dhcp+dns+pcap policy on the pod NIC. + # These values are IPv4 (the default status.podIP family). On an IPv6-only + # or IPv6-primary cluster, use -l :: and -H $(POD_IP)/128 instead. args: ["pktvisord", "-l", "0.0.0.0", "-H", "$(POD_IP)/32", "eth0"] env: - name: POD_IP