Improve QC normalization robustness, enforce JSON-safe API responses, and align local/devcontainer defaults

.devcontainer/devcontainer.json

@@ -1,25 +1,37 @@
 {
-  "name": "scanomatic-ci-like",
+  "name": "scanomatic-dev",
   "build": {
     "dockerfile": "Dockerfile",
     "context": ".."
   },
+  "workspaceFolder": "/workspaces/${localWorkspaceFolderBasename}",
+  "workspaceMount": "source=${localWorkspaceFolder},target=/workspaces/${localWorkspaceFolderBasename},type=bind",
+  "containerUser": "vscode",
   "remoteUser": "vscode",
   "updateRemoteUserUID": true,
+  "runArgs": [
+    "--init"
+  ],
   "mounts": [
-    "source=/var/run/docker.sock,target=/var/run/docker.sock,type=bind"
+    "source=/home/joakim/work/scanomatic/.git,target=/home/joakim/work/scanomatic/.git,type=bind",
+    "source=/var/run/docker.sock,target=/var/run/docker.sock,type=bind",
+    "source=scanomatic-uv-cache,target=/home/vscode/.cache/uv,type=volume",
+    "source=scanomatic-pip-cache,target=/home/vscode/.cache/pip,type=volume",
+    "source=scanomatic-npm-cache,target=/home/vscode/.npm,type=volume"
   ],
   "containerEnv": {
     "DISPLAY": ":99",
-    "CHROME_BIN": "/usr/bin/chromium"
+    "CHROME_BIN": "/usr/bin/chromium",
+    "SCANOMATIC_DATA": "/workspaces/${localWorkspaceFolderBasename}/.scan-o-matic"
   },
   "forwardPorts": [5000],
   "onCreateCommand": "bash .devcontainer/fix-docker-socket-perms.sh",
+  "postStartCommand": "bash .devcontainer/fix-docker-socket-perms.sh",
   "postCreateCommand": "bash .devcontainer/post-create.sh",
   "customizations": {
     "vscode": {
       "settings": {
-        "python.defaultInterpreterPath": "/usr/local/bin/python",
+        "python.defaultInterpreterPath": "${workspaceFolder}/.venv/bin/python",
         "python.testing.pytestEnabled": true
       },
       "extensions": [

.devcontainer/fix-docker-socket-perms.sh

@@ -1,6 +1,8 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
+CURRENT_USER="$(id -un)"
+
 # Align the container user with the mounted docker socket GID.
 if [[ ! -S /var/run/docker.sock ]]; then
   echo "No docker socket mount found; skipping docker group alignment."
@@ -15,10 +17,10 @@ if [[ -z "${EXISTING_GROUP}" ]]; then
   sudo groupadd --gid "${SOCK_GID}" "${TARGET_GROUP}"
 fi
 
-if id -nG "${USER}" | grep -qw "${TARGET_GROUP}"; then
-  echo "User ${USER} already belongs to ${TARGET_GROUP}."
+if id -nG "${CURRENT_USER}" | grep -qw "${TARGET_GROUP}"; then
+  echo "User ${CURRENT_USER} already belongs to ${TARGET_GROUP}."
 else
-  sudo usermod -aG "${TARGET_GROUP}" "${USER}"
-  echo "Added ${USER} to ${TARGET_GROUP}."
+  sudo usermod -aG "${TARGET_GROUP}" "${CURRENT_USER}"
+  echo "Added ${CURRENT_USER} to ${TARGET_GROUP}."
   echo "Rebuild/Reopen once more if docker commands still fail due to cached session groups."
 fi

.devcontainer/post-create.sh

@@ -21,12 +21,78 @@ fix_node_modules_permissions() {
   fi
 }
 
+# npm cache may be persisted from previous runs as root-owned files.
+# Ensure npm can create cache and log directories before npm ci.
+fix_npm_cache_permissions() {
+  local npm_cache_dir="${NPM_CONFIG_CACHE:-$HOME/.npm}"
+
+  if [[ ! -e "${npm_cache_dir}" ]]; then
+    return
+  fi
+
+  if [[ -w "${npm_cache_dir}" ]]; then
+    return
+  fi
+
+  echo "Fixing ownership for npm cache at ${npm_cache_dir}..."
+  if command -v sudo >/dev/null 2>&1; then
+    sudo chown -R "$(id -u):$(id -g)" "${npm_cache_dir}"
+  else
+    echo "sudo is unavailable; using a local npm cache for this run"
+    export NPM_CONFIG_CACHE="$PWD/.npm-cache"
+    mkdir -p "${NPM_CONFIG_CACHE}"
+  fi
+}
+
+fix_uv_cache_permissions() {
+  local uv_cache_dir="${UV_CACHE_DIR:-$HOME/.cache/uv}"
+
+  if [[ ! -e "${uv_cache_dir}" ]]; then
+    return
+  fi
+
+  if [[ -w "${uv_cache_dir}" ]]; then
+    return
+  fi
+
+  echo "Fixing ownership for uv cache at ${uv_cache_dir}..."
+  if command -v sudo >/dev/null 2>&1; then
+    sudo chown -R "$(id -u):$(id -g)" "${uv_cache_dir}"
+  else
+    echo "sudo is unavailable; using a local uv cache for this run"
+    export UV_CACHE_DIR="$PWD/.uv-cache"
+    mkdir -p "${UV_CACHE_DIR}"
+  fi
+}
+
 if [[ -f package-lock.json ]]; then
+  fix_npm_cache_permissions
   fix_node_modules_permissions
   npm ci
+  npm run build
 fi
 
-uv sync --all-extra
+fix_uv_cache_permissions
+uv sync --all-extras
+
+# Keep Scan-o-Matic developer data in-repo for easy persistence and inspection.
+som_data_dir="${SCANOMATIC_DATA:-$PWD/.scan-o-matic}"
+if [[ "$som_data_dir" == *'${'* ]]; then
+  # Guard against unresolved devcontainer placeholders (e.g. ${workspaceFolder}).
+  som_data_dir="$PWD/.scan-o-matic"
+fi
+som_projects_root="$PWD/somprojects"
+mkdir -p "$som_data_dir/config" "$som_projects_root"
+
+# Ensure the app's settings model points projects to the in-repo location.
+SCANOMATIC_DATA="$som_data_dir" uv run python - <<'PY'
+import os
+from scanomatic.io.app_config import Config
+
+cfg = Config()
+cfg.paths.projects_root = os.path.abspath("somprojects")
+cfg.save_current_settings()
+PY
 
 echo "Devcontainer is ready."
 echo "Run backend checks: uvx --with tox-uv tox -e lint,unit,integration"

.gitignore

@@ -40,3 +40,5 @@ geckodriver.log
 *.orig
 .env
 /.*_cache
+somprojects
+.scan-o-matic

Dockerfile

@@ -45,7 +45,6 @@ RUN pip install --no-cache-dir uv \
     && rm -rf /root/.cache /tmp/wheels /tmp/requirements.txt
 
 COPY data/ /tmp/data/
-COPY setup_config.py /opt/setup_config.py
 COPY docker-entrypoint.sh /opt/docker-entrypoint.sh
 RUN chmod +x /opt/docker-entrypoint.sh
 

README.md

@@ -1,5 +1,3 @@
-
-
 # Scan-o-matic (program) and scanomatic (python module)
 
 This project contains the code for the massive microbial phenotyping platform Scan-o-matic.
@@ -8,7 +6,7 @@ Scan-o-matic was published in [G3 September 2016](http://g3journal.org/content/6
 
 Please refer to the general [Scan-o-Matic Wiki](https://github.com/Scan-o-Matic/scanomatic/wiki) for further instructions, though setup and starting should be as described below.
 
-# Setting up and running
+## Setting up and running
 
 1. Clone this library `git@github.com:Scan-o-Matic/scanomatic-standalone.git`
 2. And change directory: `cd scanomatic-standalone`
@@ -17,7 +15,7 @@ Please refer to the general [Scan-o-Matic Wiki](https://github.com/Scan-o-Matic/
 5. Run `docker-compose up -d`. Omit the `-d` if you don't wish to run it in the background.
 6. In your browser navigate to `http://localhost:5000`
 
-## Setting up with prebuilt images (GHCR)
+### Setting up with prebuilt images (GHCR)
 
 If you don't want to build locally, you can run Scan-o-Matic directly from prebuilt container images published to GitHub Container Registry (GHCR).
 
@@ -31,7 +29,7 @@ Tag recommendations:
 2. Use a pinned version tag (for example `3.0.1` or `v3.0.1`) for reproducible deployments.
 3. Avoid prerelease tags in production unless you explicitly want release candidates.
 
-### Example compose.yml
+#### Example compose.yml
 
 ```yaml
 services:
@@ -54,32 +52,32 @@ Then run:
 2. Start with `docker compose up -d`.
 3. Open `http://localhost:5000` in your browser.
 
-## Reporting issues
+### Reporting issues
 
 If you have a problem please create and issue here on the git repository.
 If it relates to a specific project please include the relevant log-files for that project.
 There are also a set of files that probably is relevant: `.project.settings`, `.project.compilation`, `.project.compilation.instructions`, `.scan.instructions`, `.analysis.instructions`...
-Please also include the server and ui-server log files (those will be localized to a hidden folder called `.scan-o-matic/logs` in your users directory.
+Please also include the server and ui-server log files (those will be localized to a hidden folder called `.scan-o-matic/logs` in your users directory).
 
 Do however please note, that if you are doing something super secret, the files will contain some information on what you are doing and it may be needed that you go through them before uploading them publically.
 In this case, only redact the sensitive information, but keep general systematic parts of these lines as intact as possible.
 
-# Developers
+## Developers
 
 This section contains information relevant to those contributing to the source-code.
 
-## Tests and quality assurance
+### Tests and quality assurance
 
 Scan-o-Matic has three kinds of tests: unit, integration, and system tests. In addition the Scan-o-Matic code should be lint free and pass type checking.
 Backend tests, linting and typechecking are run by `tox` while front-end tests are run with `karma` and lint with `eslint`.
 
 The tox environments are `lint`, `ty`, `unit`, `integration` and `system`. Only the latter requires additional dependencies.
 
-### Tox ty type checking
+#### Tox ty type checking
 
 Currently this is not required as the code is still riddled with type errors, however there's a `typecheck-changed.sh` that filters out errors only in files changed in the branch. It should be used to guarantee that no new errors are introduced and it is encouraged to clean up unrelated errors in the files touched.
 
-### System tests
+#### System tests
 
 System tests require `firefox`, `geckodriver`, `google-chrome` and `chromedriver` to be installed on host system.
 After that it run with `tox -e system`.

docker-compose.yml

@@ -6,6 +6,6 @@ services:
     environment:
       - LOGGING_LEVEL=20
     volumes:
-      - ${SOM_SETTINGS:-/tmp/.scan-o-matic}:/root/.scan-o-matic
-      - ${SOM_PROJECTS_ROOT:-/tmp/SoM}:/somprojects
+      - ${SOM_SETTINGS:-./.scan-o-matic}:/root/.scan-o-matic
+      - ${SOM_PROJECTS_ROOT:-./somprojects}:/somprojects
       - /dev/bus/usb:/dev/bus/usb

docker-entrypoint.sh

@@ -1,5 +1,8 @@
 #!/bin/sh
 set -e
 
-python /opt/setup_config.py
+mkdir -p /root/.scan-o-matic/config
+# Copy default config recursively without overwriting existing user config
+cp -an /tmp/data/config/. /root/.scan-o-matic/config/
+
 exec "$@"