docker-compose.dev.yml

services:
  natkit-v0-bridge:
    # ⚠️ The three core services had NO restart policy at all, while the frontend,
    # control plane, worker, mosquitto and ntp all had one -- so a crashed bridge,
    # backend or broker stayed down while the rest of the stack looked healthy.
    # TEC-NATKIT-71's premise ("every service declares a restart policy") was
    # checked against docker-compose.yml and was never true of this file.
    #
    # `unless-stopped` rather than `always`: a container you stopped BY HAND while
    # debugging must stay stopped. `always` brings it back on the next podman
    # restart regardless, which is the wrong answer on a dev stack.
    restart: unless-stopped
    image: natkit-bridge:dev
    build:
      context: ./libnatkit
      dockerfile: Dockerfile_libnatkit_bridge
      args:
        NATKIT_CMAKE_BUILD_PARALLEL: ${NATKIT_CMAKE_BUILD_PARALLEL:-}
    environment:
      # ⚠️ WHERE THE BROKERS ARE, and NOT optional (TEC-NATKIT-76). Both binaries
      # fall back to localhost when these are unset, and podman-compose gives each
      # service its own netns and its own IP on natkit-v0-kafka-network -- so
      # `localhost` is THIS CONTAINER, not the broker. The failure is not a startup
      # error: the service comes up, logs "Using Kafka broker at localhost:29093",
      # retries a refused connection forever, and serves an EMPTY STREAM LIST. The
      # UI shows no sensors and looks like the hardware is off.
      #
      # They were lost when this file stopped being an override of
      # docker-compose.yml and became self-contained: the values were never in this
      # file, they were inherited. Same root as TEC-NATKIT-70 (the published port),
      # and the same lesson -- a self-contained file has to carry everything the
      # merge used to supply.
      #
      # 9092 is the DOCKER_NETWORK_LISTENER, the one advertised as
      # natkit-v0-kafka:9092 and therefore the only one whose advertised address
      # resolves from inside this network.
      LIBNATKIT_KAFKA_BROKER_ADDRESS: natkit-v0-kafka
      LIBNATKIT_KAFKA_BROKER_PORT: "9092"
      LIBNATKIT_MQTT_BROKER_ADDRESS: mosquitto
      LIBNATKIT_MQTT_BROKER_PORT: "1883"
      # Per-message wire tracing, OFF by default: it printed every frame's fully
      # decoded payload in both directions (~100 lines per bulk IMU frame) and
      # drowned every other service in `compose logs`. Set to 1 to debug the wire.
      NATKIT_BRIDGE_LOG_MESSAGES: ${NATKIT_BRIDGE_LOG_MESSAGES:-}

  natkit-v0-backend:
    # Published so HOST tooling can reach the backend directly -- raw WebSocket
    # clients, curl against /api, and a frontend run outside the container with
    # VITE_DEV_BACKEND_URL. The in-container frontend does not need it (it proxies
    # to natkit-v0-backend:7409 over the compose network), but the prod file
    # publishes it and this file is now used INSTEAD of that one, so omitting it
    # silently removed host access that development depends on (TEC-NATKIT-70).
    ports:
      - "7409:7409"
    restart: unless-stopped
    image: natkit-backend:dev
    build:
      context: ./libnatkit
      dockerfile: Dockerfile_natkit_backend
      args:
        NATKIT_CMAKE_BUILD_PARALLEL: ${NATKIT_CMAKE_BUILD_PARALLEL:-}
    environment:
      # ⚠️ WHERE THE BROKERS ARE, and NOT optional (TEC-NATKIT-76). Both binaries
      # fall back to localhost when these are unset, and podman-compose gives each
      # service its own netns and its own IP on natkit-v0-kafka-network -- so
      # `localhost` is THIS CONTAINER, not the broker. The failure is not a startup
      # error: the service comes up, logs "Using Kafka broker at localhost:29093",
      # retries a refused connection forever, and serves an EMPTY STREAM LIST. The
      # UI shows no sensors and looks like the hardware is off.
      #
      # They were lost when this file stopped being an override of
      # docker-compose.yml and became self-contained: the values were never in this
      # file, they were inherited. Same root as TEC-NATKIT-70 (the published port),
      # and the same lesson -- a self-contained file has to carry everything the
      # merge used to supply.
      #
      # 9092 is the DOCKER_NETWORK_LISTENER, the one advertised as
      # natkit-v0-kafka:9092 and therefore the only one whose advertised address
      # resolves from inside this network.
      LIBNATKIT_KAFKA_BROKER_ADDRESS: natkit-v0-kafka
      LIBNATKIT_KAFKA_BROKER_PORT: "9092"
      LIBNATKIT_MQTT_BROKER_ADDRESS: mosquitto
      LIBNATKIT_MQTT_BROKER_PORT: "1883"
      # Dev-only: bypass authentication so the panel/WebSocket need no login in
      # the local dev stack. NEVER set this in prod (see docker-compose.portainer.yml).
      NATKIT_AUTH_DISABLED: "true"
      # Share one auth session DB with the ML control-plane so a natkit_session
      # cookie issued at login validates on the control-plane's WebSocket.
      NATKIT_AUTH_DB_PATH: /auth/auth.sqlite3
      # Phase 5: the backend is the sole client of the ML control plane and
      # proxies it over /ws/stream_viewer (browser no longer hits :8786 directly).
      NATKIT_ML_CONTROL_PLANE_URL: ws://natkit-v0-ml-control-plane:8786
      # Persist saved stream graphs on a named volume so they survive container
      # recreate (the default ./data/stream_graphs.json lives in the ephemeral
      # container layer and is lost on `down`/rebuild).
      NATKIT_STREAM_GRAPH_STORE: /graphs/stream_graphs.json
      # Experiments live on the same volume: an experiment owns a board and (from
      # Phase 2) its recorded instances, so losing this store orphans history.
      NATKIT_EXPERIMENT_STORE: /graphs/experiments.json
      # Workspaces and profiles belong on the same volume for the same reason.
      # ⚠️ Both defaulted to ./data/*.json — the ephemeral layer — so a container
      # recreate lost them WHILE THEIR CONTENTS SURVIVED: membership lives on the
      # member (`workspace_id` on experiments, boards and profiles), so every
      # experiment would point at a workspace that no longer exists, and the picker
      # shows that as an empty list. It reads as "my experiments are gone".
      NATKIT_WORKSPACE_STORE: /graphs/workspaces.json
      NATKIT_PROFILE_STORE: /graphs/profiles.json
      # Materialized instance artifacts (Parquet + the markers sidecar). These ARE
      # the permanence guarantee — the whole point is that the data has left Kafka —
      # so they live on their own named volume, never the container layer.
      NATKIT_INSTANCE_STORE: /instances
    volumes:
      - natkit-v0-auth:/auth
      - natkit-v0-graphs:/graphs
      - natkit-v0-instances:/instances
      # Read trained models the control plane persists here (Phase 5, slice C)
      # so a classify node can load model_path directly.
      - natkit-v0-models:/models:ro

  natkit-v0-ml-control-plane:
    image: natkit-ml-control-plane:dev
    build:
      context: .
      dockerfile: Dockerfile_natkit_ml_control_plane
      args:
        NATKIT_CMAKE_BUILD_PARALLEL: ${NATKIT_CMAKE_BUILD_PARALLEL:-}
    restart: always
    environment:
      NATKIT_ML_CONTROL_PLANE_BROKER: ${NATKIT_ML_CONTROL_PLANE_BROKER:-natkit-v0-kafka:9092}
      NATKIT_ML_CONTROL_PLANE_WORKER_THREADS: ${NATKIT_ML_CONTROL_PLANE_WORKER_THREADS:-}
      NATKIT_ML_CONTROL_PLANE_MAX_WORKER_RESTART_ATTEMPTS: ${NATKIT_ML_CONTROL_PLANE_MAX_WORKER_RESTART_ATTEMPTS:-}
      # Same auth DB as natkit-v0-backend (see above) so login sessions resolve here.
      NATKIT_AUTH_DB_PATH: /auth/auth.sqlite3
      # Persist trained models to the shared volume the backend reads (slice C).
      NATKIT_ML_ARTIFACTS_DIR: /models
    ports:
      - "8786:8786"
    volumes:
      - ./uploads/ml-control-plane:/state:Z
      - natkit-v0-auth:/auth
      - natkit-v0-models:/models
      # Instance artifacts, READ-ONLY: training reads a recording's Parquet straight
      # off disk (Phase 6), and the paths come from the backend, so they have to
      # resolve identically here. Read-only because a sealed instance is a
      # historical fact — the trainer is a consumer of it, never a writer.
      - natkit-v0-instances:/instances:ro
    networks:
      - natkit-v0-kafka-network
    depends_on:
      - natkit-v0-kafka

  natkit-v0-ml-worker-a:
    image: natkit-ml-control-plane:dev
    # The control plane already supplies embedded worker slots; this is the
    # external worker, and it comes up with the rest of the stack (Zach,
    # 2026-08-20). It was previously behind a `profiles: ["ml-worker"]` gate.
    #
    # ⚠️ It needs a real shared-auth account, read from .env as
    # NATKIT_ML_WORKER_AUTH_USERNAME / _PASSWORD. Those default to EMPTY, so on a
    # machine without them the worker will crash-loop rather than silently not
    # exist -- which is the trade the profile was making. If you hit that, put the
    # credentials in .env; do not re-add the profile without saying so here.
    command:
      - python
      - /workspace/libnatkit/scripts/natkit_ml_worker.py
      - --control-plane-url
      - ${NATKIT_ML_WORKER_CONTROL_PLANE_URL:-ws://natkit-v0-ml-control-plane:8786}
      - --worker-id
      - ${NATKIT_ML_WORKER_ID_A:-worker-remote-a}
      - --worker-threads
      - "${NATKIT_ML_WORKER_THREADS_A:-4}"
      - --scratch-root
      - /state/scratch
    restart: always
    environment:
      # The worker authenticates to the shared auth store by logging in against
      # the backend's /api/auth/login. derive_auth_base_url() would point this at
      # the control-plane host (which doesn't serve auth) in this topology, so set
      # it explicitly to the backend. Credentials come from .env and must match a
      # real account (see NATKIT_ML_WORKER_AUTH_* there).
      NATKIT_AUTH_BASE_URL: http://natkit-v0-backend:7409
      NATKIT_AUTH_USERNAME: ${NATKIT_ML_WORKER_AUTH_USERNAME:-}
      NATKIT_AUTH_PASSWORD: ${NATKIT_ML_WORKER_AUTH_PASSWORD:-}
    volumes:
      - ./uploads/ml-worker-a:/state:Z
      # Same instance artifacts, same path, read-only: a remote worker runs the very
      # same pipeline, so an instance-backed job must resolve its Parquet here too.
      - natkit-v0-instances:/instances:ro
      # Trained artifacts land on the shared models volume (as the in-process path
      # already does) so a classify node can load what a remote worker produced.
      - natkit-v0-models:/models
    networks:
      - natkit-v0-kafka-network
    depends_on:
      - natkit-v0-ml-control-plane

  natkit-v0-frontend:
    image: natkit-frontend:dev
    build:
      context: ./frontend
      dockerfile: Dockerfile.dev
    restart: always
    environment:
      VITE_DEV_BACKEND_URL: http://natkit-v0-backend:7409
    ports:
      - "8080:5173"
    volumes:
      - ./frontend:/app:Z
      - natkit-v0-frontend-node-modules:/app/node_modules
    networks:
      - natkit-v0-kafka-network
    depends_on:
      - natkit-v0-backend

  # --- Standalone-stack services (TEC-NATKIT-70) ---------------------------
  #
  # ⚠️ These are COPIED VERBATIM from docker-compose.yml, which is the definition
  # actually in use, NOT from the docker-compose.dev.yml.new draft. That draft's
  # kafka would have been a data-loss trap: a different image
  # (apache/kafka:3.7.0 vs confluentinc/cp-kafka:7.8.0), no published ports, no
  # natkit-v0-kafka-data volume, and only one listener instead of the four the rig
  # uses -- so host tooling on 29092 would have broken and topics would not have
  # survived a recreate.
  #
  # Why they are here at all: this file is documented (ENVIRONMENT.html) as
  # `podman-compose -f docker-compose.dev.yml up -d`, i.e. standalone, but it had
  # neither these three services nor a `networks:` block, so running it alone
  # failed with "missing networks: natkit-v0-kafka-network". Combining it with
  # docker-compose.yml instead made the FRONTEND unstartable, because `ports`
  # lists CONCATENATE on merge: 8080:80 from the base plus 8080:5173 from here
  # bound host 8080 twice and podman reported "rootlessport conflict with ID 1".
  #
  # Self-contained is the fix that matches the documentation. The two files are
  # now alternatives, not layers -- do not pass both.

  natkit-v0-kafka:
    # ⚠️ The broker had no restart policy either, and it is the one every other
    # service depends on. See the note on natkit-v0-bridge.
    restart: unless-stopped
    image: confluentinc/cp-kafka:7.8.0
    hostname: natkit-v0-kafka
    ports:
      - 9092:9092
      - 9093:9093
      - 29092:29092
      - 29093:29093
    environment:
      KAFKA_KRAFT_MODE: "true"
      KAFKA_PROCESS_ROLES: controller,broker
      KAFKA_NODE_ID: 1
      KAFKA_CONTROLLER_ID: 1
      CLUSTER_ID: ReR6t0tUSZGaK6IO8K4lwA
      KAFKA_CONTROLLER_QUORUM_VOTERS: "1@natkit-v0-kafka:9093"
      KAFKA_LISTENERS: "DOCKER_NETWORK_LISTENER://0.0.0.0:9092,CONTROLLER_LISTENER://0.0.0.0:9093,LOCALHOST_LISTENER://0.0.0.0:29092,REMOTE_LISTENER://0.0.0.0:29093"
      # For remote access, set NATKIT_HOST_IP environment variable before running docker-compose
      # Example: NATKIT_HOST_IP=192.168.1.100 docker-compose up -d
      KAFKA_ADVERTISED_LISTENERS: "DOCKER_NETWORK_LISTENER://natkit-v0-kafka:9092,LOCALHOST_LISTENER://127.0.0.1:29092,REMOTE_LISTENER://${NATKIT_HOST_IP:-127.0.0.1}:29093"
      KAFKA_INTER_BROKER_LISTENER_NAME: "DOCKER_NETWORK_LISTENER"
      KAFKA_CONTROLLER_LISTENER_NAMES: "CONTROLLER_LISTENER"
      # Persist KRaft data on a named volume. This used to be /tmp/kraft-storage,
      # i.e. the container's writable layer: every `up --build` or container
      # recreate silently destroyed every topic, which cost us recorded sessions
      # twice. Recorded experiment instances are meant to be permanent, so the
      # broker's storage cannot be ephemeral. (docker-compose.portainer.yml has
      # always done this; the dev/base stack was the outlier.)
      KAFKA_LOG_DIRS: "/var/lib/kafka/data"
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: DOCKER_NETWORK_LISTENER:PLAINTEXT,CONTROLLER_LISTENER:PLAINTEXT,LOCALHOST_LISTENER:PLAINTEXT,REMOTE_LISTENER:PLAINTEXT
    volumes:
      - natkit-v0-kafka-data:/var/lib/kafka/data
    networks:
      - natkit-v0-kafka-network

  # natkit-v0-schema-registry:
  #   image: confluentinc/cp-schema-registry:7.0.7
  #   hostname: natkit-v0-schema-registry
  #   ports:
  #   - 38081:38081
  #   depends_on:
  #   - natkit-v0-kafka
  #   environment:
  #     SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: natkit-v0-zookeeper:32181
  #     SCHEMA_REGISTRY_HOST_NAME: natkit-v0-schema-registry
  #     SCHEMA_REGISTRY_LISTENERS: http://natkit-v0-schema-registry:38081
  #     SCHEMA_REGISTRY_DEBUG: "true"
  #     SCHEMA_REGISTRY_KAFKASTORE_SECURITY_PROTOCOL: PLAINTEXT
  #     SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: PLAINTEXT://natkit-v0-kafka:29092
  #   networks:
  #     - natkit-v0-kafka-network

  # natkit-v0-kafka-rest:
  #   image: confluentinc/cp-kafka-rest:7.0.7
  #   hostname: natkit-v0-kafka-rest
  #   ports:
  #   - 38082:38082
  #   depends_on:
  #     - natkit-v0-schema-registry
  #   environment:
  #     KAFKA_REST_ZOOKEEPER_CONNECT: natkit-v0-zookeeper:32181
  #     KAFKA_REST_SCHEMA_REGISTRY_URL: natkit-v0-schema-registry:38081
  #     KAFKA_REST_HOST_NAME: natkit-v0-kafka-rest
  #     KAFKA_REST_LISTENERS: http://natkit-v0-kafka-rest:38082
  #     KAFKA_REST_BOOTSTRAP_SERVERS: PLAINTEXT://natkit-v0-kafka:29092
  #   networks:
  #     - natkit-v0-kafka-network
  mosquitto:
    image: eclipse-mosquitto:2.0.22
    hostname: mosquitto
    container_name: mosquitto
    restart: unless-stopped
    ports:
      - "1883:1883"
      - "9001:9001"
    volumes:
      - ./mosquitto/etc:/etc/mosquitto:Z
      - ./mosquitto/log:/mosquitto/log:Z
      - ./mosquitto/mosquitto.conf:/mosquitto/config/mosquitto.conf:Z
    networks:
      - natkit-v0-kafka-network
  ntp:
    image: cturra/ntp:latest
    container_name: ntp
    restart: always
    ports:
      - 123:123/udp
    environment:
      - NTP_SERVERS=subitaneous.cpsc.ucalgary.ca
      - LOG_LEVEL=0
      - TZ=America/Edmonton
  #      - NOCLIENTLOG=true
  #      - ENABLE_NTS=true

volumes:
  natkit-v0-kafka-data:
  natkit-v0-instances:
  natkit-v0-auth:
  natkit-v0-graphs:
  natkit-v0-frontend-node-modules:
  natkit-v0-models:

networks:
  # Declared here so the file stands alone; same name and settings as the prod
  # file, so a container created by either lands on the same network.
  natkit-v0-kafka-network:
    external: false