feat(machine-a-tron): k8s controller for mock BMC services

.github/workflows/ci.yaml

@@ -171,6 +171,7 @@ jobs:
               - 'Cargo.lock'
               - 'Makefile.toml'
               - 'rust-toolchain.toml'
+              - 'dev/k8s/machine-a-tron-controller/**'
       - name: Calculate version
         id: version
         run: |
@@ -816,6 +817,32 @@ jobs:
       timeout_minutes: 60
     secrets: inherit
 
+  build-mat-k8s-controller:
+    if: >-
+      ${{
+        always()
+        && github.event_name != 'schedule'
+        && needs.prepare.result == 'success'
+        && needs.prepare.outputs.source_files_changed == 'true'
+      }}
+    needs:
+      - prepare
+    uses: ./.github/workflows/docker-build.yml
+    with:
+      dockerfile_path: dev/k8s/machine-a-tron-controller/Dockerfile
+      context_path: dev/k8s/machine-a-tron-controller
+      image_name: ${{ needs.prepare.outputs.image_registry }}/mat-k8s-controller
+      image_tag: ${{ needs.prepare.outputs.version }}
+      additional_tags: ${{ needs.prepare.outputs.image_registry }}/mat-k8s-controller:${{ needs.prepare.outputs.major_minor_version }}-latest
+      platforms: linux/amd64,linux/arm64
+      runner: linux-amd64-cpu4
+      push: ${{ needs.prepare.outputs.publish_images == 'true' }}
+      load: false
+      scan: true
+      tag_latest: false
+      timeout_minutes: 30
+    secrets: inherit
+
   test-release-container-services:
     if: >-
       ${{
@@ -1834,6 +1861,7 @@ jobs:
       - build-release-container-x86_64
       - build-release-container-aarch64
       - build-machine-a-tron
+      - build-mat-k8s-controller
       - test-release-container-services
       - build-boot-artifacts-x86
       - build-boot-artifacts-bfb
@@ -1938,6 +1966,7 @@ jobs:
       - build-release-container-x86_64
       - build-release-container-aarch64
       - build-machine-a-tron
+      - build-mat-k8s-controller
       - test-release-container-services
       - build-boot-artifacts-x86
       - build-boot-artifacts-bfb
@@ -2017,6 +2046,8 @@ jobs:
       - security-secret-scan
       - lint-police
       - check-rest-core-proto-sync
+      - build-machine-a-tron
+      - build-mat-k8s-controller
     steps:
       - name: Decide pass/fail
         env:

crates/machine-a-tron/README.md

@@ -9,7 +9,7 @@ periodically report health network observations.
 
 ## Usage
 
-```
+```text
 target/debug/machine-a-tron -h
 Usage: machine-a-tron [OPTIONS] --relay-address <RELAY_ADDRESS> <NUM_HOSTS> [NICO_API]
 
@@ -120,4 +120,4 @@ setup with the appropriate overrides for redfish so that it will send all reques
 > for all libredfish calls. If you want to go back to running the TUI locally, you'll want to manually edit the
 > generated nico-api-site-config.toml and drop the `override_target_host` line. You may also want to edit the
 > `mat.toml` in the same directory and set the host_count to 0 so that the in-cluster machine-a-tron doesn't run any
-> mock machines.
+> mock machines.

dev/k8s/machine-a-tron-controller/Dockerfile

@@ -0,0 +1,22 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# Build stage
+FROM golang:1.26.4-alpine@sha256:3ad57304ad93bbec8548a0437ad9e06a455660655d9af011d58b993f6f615648 AS builder
+
+WORKDIR /app
+
+COPY go.mod go.sum ./
+RUN go mod download
+
+COPY . .
+RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w" -o mat-k8s-controller ./cmd/mat-k8s-controller
+
+# Runtime stage
+FROM gcr.io/distroless/static:nonroot@sha256:f7f8f729987ad0fdf6b05eeeae94b26e6a0f613bdf46feea7fc40f7bd72953e6
+
+COPY --from=builder /app/mat-k8s-controller /mat-k8s-controller
+
+USER nonroot:nonroot
+
+ENTRYPOINT ["/mat-k8s-controller"]

dev/k8s/machine-a-tron-controller/Makefile

@@ -0,0 +1,39 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+.PHONY: all build test test-coverage lint fmt docker-build docker-build-latest clean run verify
+
+TAG ?= dev
+
+all: build
+
+build:
+	@mkdir -p bin
+	go build -o bin/mat-k8s-controller ./cmd/mat-k8s-controller
+
+test:
+	go test -v ./...
+
+test-coverage:
+	go test -coverprofile=coverage.out ./...
+	go tool cover -html=coverage.out -o coverage.html
+
+lint:
+	golangci-lint run ./...
+
+fmt:
+	go fmt ./...
+
+docker-build:
+	docker build -t mat-k8s-controller:$(TAG) .
+
+docker-build-latest:
+	docker build -t mat-k8s-controller:latest .
+
+clean:
+	rm -rf bin/ coverage.out coverage.html
+
+run:
+	go run ./cmd/mat-k8s-controller --kubeconfig=$(KUBECONFIG) --log-level=debug
+
+verify: lint test

dev/k8s/machine-a-tron-controller/README.md

@@ -0,0 +1,114 @@
+# Machine-a-tron Kubernetes Controller
+
+Kubernetes controller that auto-discovers machine-a-tron pods and creates
+Services for mock BMC endpoints.
+
+## Features
+
+- Auto-discovers machine-a-tron pods via `nvidia-infra-controller/mat-service=true`
+  label
+- Creates ClusterIP Services with BMC IP for each mock BMC
+- Supports Redfish (TCP 443) ports
+- Multi-pod deployments with pod-specific routing
+- Automatic cleanup of stale Services
+
+## Build
+
+```bash
+docker build -t mat-k8s-controller:latest .
+kind load docker-image mat-k8s-controller:latest --name <cluster>
+```
+
+## Configuration
+
+| Flag | Env Var | Default | Description |
+|------|---------|---------|-------------|
+| `--namespace` | `NAMESPACE` | `nico-system` | Kubernetes namespace |
+| `--discovery-selector` | `DISCOVERY_SELECTOR` | `nvidia-infra-controller/mat-service=true` | Label selector for discovery |
+| `--sync-interval` | `SYNC_INTERVAL` | `30s` | Reconciliation interval |
+| `--target-selector` | `TARGET_SELECTOR` | `app.kubernetes.io/name=nico-machine-a-tron` | Pod selector for Services |
+| `--insecure-skip-verify` | `INSECURE_SKIP_VERIFY` | `false` | Skip TLS verification (dev only) |
+| `--log-level` | `LOG_LEVEL` | `info` | Log level |
+| `--bmc-mock-port` | `BMC_MOCK_PORT` | `1266` | BMC mock service port |
+| `--kubeconfig` | `KUBECONFIG` | (empty) | Path to kubeconfig (dev only, uses in-cluster config if empty) |
+
+## Helm Deployment
+
+Enable in parent chart:
+
+```yaml
+mat-k8s-controller:
+  enabled: true
+  image:
+    pullPolicy: Never  # For local images
+  config:
+    insecureSkipVerify: true  # Only for dev with self-signed certs
+```
+
+## Service Structure
+
+Created Services have:
+
+**Labels:**
+
+- `app.kubernetes.io/managed-by: mat-k8s-controller`
+- `nvidia-infra-controller/mat-id: <uuid>`
+- `nvidia-infra-controller/mat-machine-type: host|dpu`
+- `nvidia-infra-controller/pod-name: <pod>` (multi-pod)
+
+**Annotations:**
+
+- `nvidia-infra-controller/mat-bmc-ip`
+- `nvidia-infra-controller/mat-api-state`
+- `nvidia-infra-controller/mat-power-state`
+- `nvidia-infra-controller/mat-hardware-type`
+
+## Development
+
+```bash
+make build
+make test
+make run KUBECONFIG="$HOME/.kube/config"
+```
+
+## Troubleshooting
+
+### ClusterIP already allocated
+
+BMC IP is outside ServiceCIDR or already in use.
+
+**Solutions:**
+
+1. Reserve a ServiceCIDR for machine-a-tron (K8s 1.29+)
+2. Use a CIDR within the cluster's ServiceCIDR
+3. Delete conflicting Services
+
+### ClusterIP change detected
+
+BMC IP changed but ClusterIP is immutable. Controller will delete and recreate
+the Service.
+
+## Architecture
+
+```mermaid
+flowchart LR
+    subgraph MAT[machine-a-tron pods]
+        MAT0[mat-0/bmc-mock]
+        MAT1[mat-1/bmc-mock]
+    end
+
+    subgraph Controller
+        Discovery[Service Discovery]
+        Reconciler[Reconciler]
+    end
+
+    subgraph K8s[Kubernetes Services]
+        SVC1[mat-bmc-host-xxx]
+        SVC2[mat-bmc-dpu-yyy]
+    end
+
+    Discovery -->|discovers| MAT0
+    Discovery -->|discovers| MAT1
+    Reconciler -->|polls /machines/status| MAT
+    Reconciler -->|creates/updates/deletes| K8s
+```