From 33643e728c8754a7349e519002e0088c50afea73 Mon Sep 17 00:00:00 2001 From: Julien Bouquiaux Date: Thu, 16 Jul 2026 17:08:59 +0200 Subject: [PATCH 1/3] add dev profile --- .docker/app/Dockerfile | 27 ++++++++++++-- docker-compose.yml | 76 +++++++++++++++++++++++++++++++++++++++ pydatalab/docs/INSTALL.md | 23 ++++++++++++ 3 files changed, 124 insertions(+), 2 deletions(-) diff --git a/.docker/app/Dockerfile b/.docker/app/Dockerfile index 2fcd9c522..559009d3d 100644 --- a/.docker/app/Dockerfile +++ b/.docker/app/Dockerfile @@ -1,5 +1,5 @@ # syntax=docker/dockerfile:1 -FROM node:22.22-bullseye-slim AS build +FROM node:22.22-bullseye-slim AS base SHELL ["/bin/bash", "--login", "-c"] WORKDIR /app @@ -13,6 +13,30 @@ COPY webapp/package.json webapp/yarn.lock ./ # Using a custom node_modules location to avoid mounting it outside of docker RUN --mount=type=cache,target=/root/.cache/yarn yarn install --frozen-lockfile --modules-folder /node_modules +# dev target: deps installed, source mounted at runtime — no production build. +# Also installs the system libraries the Cypress test runner needs (GTK/X11 +# runtime libs plus xvfb for a virtual display), so the webapp test suites can +# run inside the container. +FROM base AS dev +RUN apt update \ + && apt install -y --no-install-recommends \ + libgtk2.0-0 \ + libgtk-3-0 \ + libgbm-dev \ + libnotify-dev \ + libnss3 \ + libxss1 \ + libasound2 \ + libxtst6 \ + xauth \ + xvfb \ + && rm -rf /var/lib/apt/lists/* +COPY webapp ./ + +# build target: runs the production webpack bundle. Derives from `base` rather +# than `dev` so production builds don't pull in the Cypress system libraries. +FROM base AS build +COPY webapp ./ # These get replaced by the entrypoint script for production builds. # Set the real values in `.env` files or an external docker-compose. ENV NODE_ENV=production @@ -28,7 +52,6 @@ ARG VUE_APP_AUTOMATICALLY_GENERATE_ID_DEFAULT=magic-generate-id-setting ARG VUE_APP_GIT_VERSION=0.0.0+ci ENV VUE_APP_GIT_VERSION=${VUE_APP_GIT_VERSION} -COPY webapp ./ RUN /node_modules/.bin/vue-cli-service build FROM node:22.22-bullseye-slim AS production diff --git a/docker-compose.yml b/docker-compose.yml index 20de8af64..58853bdd5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -57,6 +57,82 @@ services: networks: - backend + # --------------------------------------------------------------------------- + # Development services (`docker compose --profile dev up`). + # + # These give a hot-reloading, source-mounted environment for local + # development. + # --------------------------------------------------------------------------- + database-dev: + profiles: ["dev"] + build: + context: . + dockerfile: .docker/mongo/Dockerfile + volumes: + - datalab-dev-dbdata:/data/db + ports: + # Published on host port 27018 to avoid clashing with a + # developer's native MongoDB. + - "27018:27017" + networks: + - backend + + api-dev: + profiles: ["dev"] + build: + context: . + dockerfile: .docker/server/Dockerfile + target: api + command: ["/opt/.venv/bin/invoke", "dev.serve", "--host", "0.0.0.0", "--port", "5001"] + depends_on: + - database-dev + volumes: + - ./pydatalab:/app + - datalab-dev-files:/app/files + ports: + - "5001:5001" + networks: + - backend + environment: + - PYDATALAB_MONGO_URI=mongodb://database-dev:27017/${DATALAB_DB_NAME:-datalabvue} + - PYDATALAB_FILE_DIRECTORY=/app/files + + app-dev: + profiles: ["dev"] + build: + context: . + dockerfile: .docker/app/Dockerfile + target: dev + command: + ["/node_modules/.bin/vue-cli-service", "serve", "--host", "0.0.0.0", "--port", "8081"] + volumes: + - ./webapp:/app + ports: + - "8081:8081" + environment: + - NODE_ENV=development + - VUE_APP_API_URL=${VUE_APP_API_URL:-http://localhost:5001} + - CHOKIDAR_USEPOLLING=${CHOKIDAR_USEPOLLING:-true} + healthcheck: + # The dev image has no HEALTHCHECK (that lives in the production stage); + # this lets `docker compose up --wait` (used in CI) block until the + # webpack dev server has compiled and is serving. + test: + [ + "CMD", + "node", + "-e", + "fetch('http://localhost:8081').then(r => process.exit(r.ok ? 0 : 1)).catch(() => process.exit(1))", + ] + interval: 15s + timeout: 5s + retries: 3 + start_period: 180s + networks: backend: driver: bridge + +volumes: + datalab-dev-dbdata: + datalab-dev-files: diff --git a/pydatalab/docs/INSTALL.md b/pydatalab/docs/INSTALL.md index 2586954b8..0c3a965f6 100644 --- a/pydatalab/docs/INSTALL.md +++ b/pydatalab/docs/INSTALL.md @@ -30,6 +30,29 @@ If you are not familiar with `git` or GitHub, you can do worse than reading thro Your local development *datalab* can be configured with all the options as a real *datalab*; these are expected in the same places as described in [Server configuration](https://docs.datalab-org.io/en/latest/config/), and can be set with environment variables or a config file, with additional config and secrets provided in `.env` files in the `pydatalab/` and `webapp/` directories for development purposes. +### Docker development environment + +The complete development stack can be run with Docker Compose: + +```shell +docker compose --profile dev up --build +``` + +This starts the web app at [http://localhost:8081](http://localhost:8081), the API at +[http://localhost:5001](http://localhost:5001), and MongoDB at +`mongodb://localhost:27018`. Changes under `pydatalab/` and `webapp/` are +bind-mounted into the containers and trigger the respective development servers to reload. + +The development database and uploaded files are stored in separate Docker volumes. To use +a different database name, set `DATALAB_DB_NAME` when starting the stack, for example: + +```shell +DATALAB_DB_NAME=my-project docker compose --profile dev up +``` + +Stop the stack with `docker compose --profile dev down`. Add `--volumes` to also remove its +development database and uploaded files. + ### `pydatalab` server installation The instructions in this section will leave you with a running *datalab* server on your host machine, as implemented in the `pydatalab` Python package. From 7f63e78211e311cca907ca3cc1da68bed3e9a9bf Mon Sep 17 00:00:00 2001 From: Julien Bouquiaux Date: Thu, 16 Jul 2026 17:09:39 +0200 Subject: [PATCH 2/3] test docker dev profile --- .github/workflows/ci.yml | 57 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 303d67c3c..eaadac314 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -196,6 +196,63 @@ jobs: # Test that plugin block is listed in info endpoint curl -s http://localhost:5000/info/blocks | jq '.data | any(.id == "example")' + docker-dev: + name: Test dev Docker builds + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + fetch-tags: true + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + + - name: Validate Docker Compose configuration + run: docker compose config --quiet + + - name: Build dev Docker images + uses: docker/bake-action@v7 + with: + files: docker-compose.yml + load: true + source: . + targets: "app-dev,api-dev,database-dev" + set: | + app-dev.cache-from=type=gha,scope=${{ github.ref_name }}-build-app-dev + app-dev.cache-from=type=gha,scope=main-build-app-dev + app-dev.cache-to=type=gha,scope=${{ github.ref_name }}-build-app-dev,mode=max + api-dev.cache-from=type=gha,scope=${{ github.ref_name }}-build-api + api-dev.cache-from=type=gha,scope=main-build-api + database-dev.cache-from=type=gha,scope=${{ github.ref_name }}-build-database + database-dev.cache-from=type=gha,scope=main-build-database + api-dev.args.SETUPTOOLS_SCM_PRETEND_VERSION=0.0.0+ci + app-dev.tags=datalab-app-dev:latest + api-dev.tags=datalab-api-dev:latest + database-dev.tags=datalab-database-dev:latest + + - name: Start dev services + run: | + # Boot the hot-reloading dev profile and block on healthchecks + # (webpack dev server compiled, API answering /healthcheck/is_ready). + docker compose --profile dev up --no-build --force-recreate -d --wait + + - name: Check the dev API and app respond + run: | + curl --fail http://localhost:5001/healthcheck/is_ready + curl --fail --silent --output /dev/null http://localhost:8081 + + - name: Show dev service diagnostics + if: failure() + run: | + docker compose --profile dev logs + docker compose --profile dev ps + + - name: Stop dev services + if: always() + run: docker compose --profile dev down --volumes + e2e: name: e2e tests runs-on: ubuntu-latest From 6c666f80d16dbb2835c297751f9bd506b28cb2a9 Mon Sep 17 00:00:00 2001 From: Julien Bouquiaux Date: Fri, 17 Jul 2026 10:30:59 +0200 Subject: [PATCH 3/3] Complete INSTALL.md --- pydatalab/docs/INSTALL.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pydatalab/docs/INSTALL.md b/pydatalab/docs/INSTALL.md index 0c3a965f6..c177464f1 100644 --- a/pydatalab/docs/INSTALL.md +++ b/pydatalab/docs/INSTALL.md @@ -43,6 +43,12 @@ This starts the web app at [http://localhost:8081](http://localhost:8081), the A `mongodb://localhost:27018`. Changes under `pydatalab/` and `webapp/` are bind-mounted into the containers and trigger the respective development servers to reload. +After connection, you can make yourself admin with the command +```shell +docker compose exec api-dev /opt/.venv/bin/invoke admin.change-user-role \ + --display-name "Your Name" --role admin +``` + The development database and uploaded files are stored in separate Docker volumes. To use a different database name, set `DATALAB_DB_NAME` when starting the stack, for example: