From 6735750adf4b3b48c5ab7bcf4e84f98ed3aed06e Mon Sep 17 00:00:00 2001 From: searcher Date: Wed, 5 Aug 2026 11:22:09 -0400 Subject: [PATCH 1/2] Fix Windows development Compose startup --- Dockerfile_natkit_ml_control_plane | 6 + README.md | 33 ++++-- docker-compose.dev.yml | 4 + docs/docker-compose-dev-troubleshooting.md | 132 +++++++++++++++++++++ libnatkit | 2 +- 5 files changed, 167 insertions(+), 10 deletions(-) create mode 100644 docs/docker-compose-dev-troubleshooting.md diff --git a/Dockerfile_natkit_ml_control_plane b/Dockerfile_natkit_ml_control_plane index 58601e2..709af7e 100644 --- a/Dockerfile_natkit_ml_control_plane +++ b/Dockerfile_natkit_ml_control_plane @@ -61,6 +61,12 @@ COPY libnatkit/cmake /workspace/libnatkit/cmake COPY libnatkit/libnatkit /workspace/libnatkit/libnatkit COPY libnatkit/third-party /workspace/libnatkit/third-party +# Normalize Linux build inputs copied from Windows while skipping binaries. +RUN find /workspace/libnatkit/scripts /workspace/libnatkit/third-party/librdkafka -type f \ + -exec sh -c 'for file do \ + if grep -Iq . "$file"; then sed -i "s/\r$//" "$file"; fi; \ + done' sh {} + + RUN cmake \ -S /workspace/libnatkit \ -B /workspace/libnatkit/build \ diff --git a/README.md b/README.md index 5029140..afa972b 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ This is a collection of tools created to help jumpstart users on working with BC Clone the repository: ```sh -git clone https://github.com/neuralbertatech/natKit +git clone --recurse-submodules https://github.com/neuralbertatech/natKit cd natKit ``` @@ -40,20 +40,35 @@ pip install -r requirements.txt pip install -r requirements.txt ``` -Start the docker server: +### Development Docker stack + +Start Docker Desktop and wait for its Linux container engine to report that it +is running. Then, from the repository root, build the repo-owned images and +start the complete development stack: + +```sh +docker compose -f docker-compose.yml -f docker-compose.dev.yml up --build +``` + +This builds the natKit frontend, backend, bridge, and ML control plane from the +local source tree while pulling third-party infrastructure images such as Kafka, +Mosquitto, and NTP. Open the development UI at . + +To leave the stack running in the background, add `--detach`: + ```sh -docker compose up -d +docker compose -f docker-compose.yml -f docker-compose.dev.yml up --build --detach ``` -To build the repo-owned container images locally instead of pulling the natKit -service images from a registry, use the development override: +If the repository was cloned without `--recurse-submodules`, initialize its +submodules before building: + ```sh -docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d --build +git submodule update --init --recursive ``` -This keeps the third-party infrastructure images (`cp-kafka`, `mosquitto`, and -`ntp`) as registry pulls, but builds the natKit services from the local source -tree. +See [Docker Compose development troubleshooting](docs/docker-compose-dev-troubleshooting.md) +for common Docker Desktop, Windows line-ending, and optional ML worker issues. ## Getting Started diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index c22eaec..ace7adb 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -82,6 +82,10 @@ services: natkit-v0-ml-worker-a: image: natkit-ml-control-plane:dev + # The control plane already supplies embedded worker slots. This external + # worker requires a real shared-auth account, so keep it opt-in for local + # development instead of crash-looping when .env credentials are absent. + profiles: ["ml-worker"] command: - python - /workspace/libnatkit/scripts/natkit_ml_worker.py diff --git a/docs/docker-compose-dev-troubleshooting.md b/docs/docker-compose-dev-troubleshooting.md new file mode 100644 index 0000000..68bf1a5 --- /dev/null +++ b/docs/docker-compose-dev-troubleshooting.md @@ -0,0 +1,132 @@ +# Docker Compose development troubleshooting + +Command: + +```powershell +docker compose -f docker-compose.yml -f docker-compose.dev.yml up --build +``` + +## 1. Docker Desktop Linux engine unavailable + +**Error** + +```text +open //./pipe/dockerDesktopLinuxEngine: The system cannot find the file specified +``` + +**Cause** + +Docker Desktop's Linux engine was not running, so the Docker CLI could not reach +the daemon. + +**Resolution** + +Started Docker Desktop and waited for the Linux engine to become available. +`docker compose` then connected successfully and began building the images. + +## 2. Linux build script checked out with CRLF endings + +**Error** + +```text +/libnatkit/scripts/install_librdkafka.sh: line 2: $'\r': command not found +/libnatkit/scripts/install_librdkafka.sh: line 19: syntax error: unexpected end of file +``` + +The failed script did not install librdkafka, which subsequently caused CMake to +report that `/libnatkit/build/include` did not exist. + +**Cause** + +`libnatkit/scripts/install_librdkafka.sh` had Windows CRLF line endings but runs +inside a Linux build container. + +**Resolution** + +Converted the script to LF endings and added a `.gitattributes` rule at the +root of the `libnatkit` submodule: + +```gitattributes +*.sh text eol=lf +``` + +This also keeps shell scripts LF-normalized on future Windows checkouts. + +## 3. Vendored librdkafka configure script retained CRLF endings + +**Error** + +```text +/usr/bin/env: 'bash\r': No such file or directory +./Makefile.config missing: please run ./configure +``` + +**Cause** + +After fixing the wrapper script, its nested +`third-party/librdkafka/configure` entrypoint and sourced helpers such as +`mklove/modules/configure.base` were also found to have CRLF line endings. +Several vendored build files were affected by the Windows checkout. + +**Resolution** + +Added a bridge-image build step that strips trailing carriage returns from text +files under `scripts` and `third-party/librdkafka` before CMake invokes them; +binary files are explicitly skipped. The submodule's `.gitattributes` also +enforces LF for shell entrypoints and librdkafka's `mklove` helpers on future +clean checkouts. The installer now exits immediately when a nested command +fails, avoiding misleading downstream CMake errors. + +## 4. CRLF normalization initially covered only the bridge image + +**Error** + +```text +[natkit-v0-ml-control-plane] /usr/bin/env: 'bash\r': No such file or directory +``` + +**Cause** + +The bridge, backend, and ML control-plane images copy `libnatkit` through +separate Dockerfiles and build contexts. Fixing the bridge image did not alter +the files copied into the other images. + +**Resolution** + +Added the same binary-safe text normalization step to +`Dockerfile_natkit_backend` and `Dockerfile_natkit_ml_control_plane`. The bridge +image was rebuilt independently and completed successfully before retrying the +full stack. + +## 5. Optional remote ML worker restarted without credentials + +**Error** + +```text +RuntimeError: shared auth is required; provide --auth-session-token or +--auth-username/--auth-password +``` + +**Cause** + +`natkit-v0-ml-worker-a` is an external worker that must authenticate against a +real account in the shared auth database. This checkout had no `.env` file, so +`NATKIT_ML_WORKER_AUTH_USERNAME` and `NATKIT_ML_WORKER_AUTH_PASSWORD` were empty. +The control plane already starts embedded worker slots, making the external +worker optional for ordinary local development. + +**Resolution** + +Placed the external worker behind the `ml-worker` Compose profile. The normal +development command now starts without a credential-related restart loop. To +exercise the external worker, create an account, put these ignored values in +`.env`, and enable its profile: + +```dotenv +NATKIT_ML_WORKER_AUTH_USERNAME= +NATKIT_ML_WORKER_AUTH_PASSWORD= +``` + +```powershell +docker compose -f docker-compose.yml -f docker-compose.dev.yml --profile ml-worker up --build +``` diff --git a/libnatkit b/libnatkit index 5148a8b..f0941d6 160000 --- a/libnatkit +++ b/libnatkit @@ -1 +1 @@ -Subproject commit 5148a8bb7c6ffb2050aa7f33aab1f4b6fb6004d5 +Subproject commit f0941d63789c17b5d1cc5c85d89c9b861ab4c7bd From 30aabf9004e455938243915b7d48c9ef72160b44 Mon Sep 17 00:00:00 2001 From: searcher Date: Wed, 5 Aug 2026 16:10:12 -0400 Subject: [PATCH 2/2] Document Windows PlatformIO recovery --- natKit-IMU | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/natKit-IMU b/natKit-IMU index eb83d1f..fe72641 160000 --- a/natKit-IMU +++ b/natKit-IMU @@ -1 +1 @@ -Subproject commit eb83d1f675ae1824db6a31c8f6942a91eaaa3500 +Subproject commit fe726418dcddd2682d5ff9a82a1e8c17769b0386