docker refactor

.github/workflows/build_container.yml

@@ -1,13 +1,16 @@
-name: Build R container
+name: Build R containers
 
-# The image tag is read from conf/containers.config, so that single line is both what the
-# pipeline pulls and what this workflow publishes -- the two cannot drift.
-# Runs on pushes to a feature branch, whenever the container changes.
+# The image tag of the container is read from conf/containers.config, so that the tag is
+# what the pipeline pulls and what this workflow publishes. Each container is built independently,
+# so updating one tag in conf/containers.config rebuilds only that image:
+#   feature branch - rebuilds its tag on every push, unless main or devel is pinned to that tag
+#   main or devel  - builds only a tag that has never been published, so most merges are a no-op
+#   manual run     - 'force' rebuilds and overwrites the tag on any branch
 
 on:
   push:
     paths:
-      - 'containers/r/**'
+      - 'containers/**'
       - 'conf/containers.config'
       - '.github/workflows/build_container.yml'
   workflow_dispatch:
@@ -22,81 +25,116 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  build:
-    name: Build and publish
+  discover:
+    name: Select containers to build
     runs-on: ubuntu-latest
     permissions:
       contents: read
-      packages: write
+      packages: read
+    outputs:
+      containers: ${{ steps.select.outputs.containers }}
 
     steps:
       - uses: actions/checkout@v4
 
-      - name: Resolve image from conf/containers.config
-        id: image
-        run: |
-          IMAGE=$(sed -n "s|.*withLabel: *'r'.*container *= *\"\([^\"]*\)\".*|\1|p" conf/containers.config)
-          if [ -z "$IMAGE" ]; then
-            echo "::error::could not parse the 'r' container from conf/containers.config"
-            exit 1
-          fi
-          case "$IMAGE" in
-            ghcr.io/goekelab/*) ;;
-            *) echo "::error::refusing to push outside ghcr.io/goekelab: $IMAGE"; exit 1 ;;
-          esac
-          echo "image=$IMAGE" >> "$GITHUB_OUTPUT"
-          echo "Resolved container: $IMAGE"
-
-      - name: Refuse to overwrite a tag main or devel is pinned to
-        if: github.ref_name != 'main' && github.ref_name != 'devel'
+      - uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Select containers to build
+        id: select
         run: |
-          IMAGE="${{ steps.image.outputs.image }}"
-          for BASE in main devel; do
-            git fetch --no-tags --depth=1 origin "$BASE" 2>/dev/null || continue
-            PINNED=$(git show "FETCH_HEAD:conf/containers.config" 2>/dev/null \
-              | sed -n "s|.*withLabel: *'r'.*container *= *\"\([^\"]*\)\".*|\1|p")
-            if [ "$IMAGE" = "$PINNED" ]; then
-              echo "::error::$BASE is pinned to $IMAGE -- bump the tag in conf/containers.config before rebuilding it"
+          RELEASED=false
+          case "${{ github.ref_name }}" in main|devel) RELEASED=true ;; esac
+
+          # Each base branch is fetched into its own ref so both stay readable in the loop
+          BASES=""
+          for BASE in devel main; do
+            if git fetch --no-tags --depth=1 origin "$BASE:refs/base/$BASE" 2>/dev/null; then
+              BASES="$BASES refs/base/$BASE"
+            fi
+          done
+
+          # devel is at or ahead of main, so it is the baseline for "did this container change"
+          DIFF_BASE=refs/base/devel
+
+          # Every containers/<label>/Dockerfile is an image whose <label> is its process label
+          : > selected.txt
+          for DOCKERFILE in containers/*/Dockerfile; do
+            LABEL=$(basename "$(dirname "$DOCKERFILE")")
+            IMAGE=$(sed -n "s|.*withLabel: *'$LABEL'.*container *= *\"\([^\"]*\)\".*|\1|p" conf/containers.config)
+
+            if [ -z "$IMAGE" ]; then
+              echo "::error::could not parse the '$LABEL' container from conf/containers.config"
               exit 1
             fi
+            case "$IMAGE" in
+              ghcr.io/goekelab/*) ;;
+              *) echo "::error::refusing to push outside ghcr.io/goekelab: $IMAGE"; exit 1 ;;
+            esac
+
+            CHANGED=false
+            if ! git diff --quiet "$DIFF_BASE" HEAD -- "containers/$LABEL"; then
+              CHANGED=true
+            fi
+
+            # A tag main or devel is pinned to is immutable, so an edit needs a tag bump first
+            if [ "$RELEASED" = false ] && [ "$CHANGED" = true ]; then
+              for REF in $BASES; do
+                PINNED=$(git show "$REF:conf/containers.config" 2>/dev/null \
+                  | sed -n "s|.*withLabel: *'$LABEL'.*container *= *\"\([^\"]*\)\".*|\1|p")
+                if [ "$IMAGE" = "$PINNED" ]; then
+                  echo "::error::${REF#refs/base/} is pinned to $IMAGE -- bump the tag in conf/containers.config before rebuilding it"
+                  exit 1
+                fi
+              done
+            fi
+
+            # An unpublished tag is always built, since the pipeline cannot pull what is not there
+            if [ "${{ inputs.force }}" != "true" ] && docker manifest inspect "$IMAGE" >/dev/null 2>&1; then
+              if [ "$RELEASED" = true ] || [ "$CHANGED" = false ]; then
+                echo "::notice::$IMAGE is already published and unchanged -- skipping $LABEL"
+                continue
+              fi
+            fi
+
+            echo "$LABEL $IMAGE" >> selected.txt
           done
 
+          CONTAINERS=$(jq -R -s -c 'split("\n") | map(select(length > 0) | split(" ")) | map({label: .[0], image: .[1]})' < selected.txt)
+          echo "containers=$CONTAINERS" >> "$GITHUB_OUTPUT"
+          echo "Building: $CONTAINERS"
+
+  build:
+    name: Build and publish (${{ matrix.container.label }})
+    needs: discover
+    if: needs.discover.outputs.containers != '[]'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    strategy:
+      fail-fast: false
+      matrix:
+        container: ${{ fromJson(needs.discover.outputs.containers) }}
+
+    steps:
+      - uses: actions/checkout@v4
+
       - uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Decide whether to build
-        id: decide
-        run: |
-          IMAGE="${{ steps.image.outputs.image }}"
-          if docker manifest inspect "$IMAGE" >/dev/null 2>&1; then EXISTS=true; else EXISTS=false; fi
-
-          if [ "${{ inputs.force }}" = "true" ]; then
-            BUILD=true
-          elif [ "${{ github.ref_name }}" = "main" ] || [ "${{ github.ref_name }}" = "devel" ]; then
-            # Released tags are immutable: the merge that brought the bump in has already
-            # published it from the feature branch, so there is nothing left to do
-            [ "$EXISTS" = "false" ] && BUILD=true || BUILD=false
-          else
-            # Feature branch: the tag is unreleased (guarded above), so rebuild freely
-            BUILD=true
-          fi
-
-          echo "build=$BUILD" >> "$GITHUB_OUTPUT"
-          if [ "$BUILD" = "false" ]; then
-            echo "::notice::$IMAGE is already published -- skipping build"
-          fi
-
       - uses: docker/setup-buildx-action@v3
-        if: steps.decide.outputs.build == 'true'
 
       - uses: docker/build-push-action@v6
-        if: steps.decide.outputs.build == 'true'
         with:
-          context: containers/r
+          context: containers/${{ matrix.container.label }}
           push: true
-          tags: ${{ steps.image.outputs.image }}
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
+          tags: ${{ matrix.container.image }}
+          cache-from: type=gha,scope=${{ matrix.container.label }}
+          cache-to: type=gha,mode=max,scope=${{ matrix.container.label }}

bin/create_seurat_object.R

@@ -0,0 +1,14 @@
+# createSeuratObject() - build a Seurat object from a bambu count directory, so the
+# clustering never has to read a SummarizedExperiment.
+#   countsDir - path to a count directory written by saveCounts()
+#   metadata  - data.frame of cell metadata, one row per barcode with matching row names
+#   assay, minCells, project - passed through to CreateSeuratObject()
+# Assumes Seurat is loaded by the caller
+
+createSeuratObject <- function(countsDir, assay = "RNA", metadata = NULL, minCells = 0, project = "SeuratProject") {
+    counts <- ReadMtx(mtx      = file.path(countsDir, "counts.mtx.gz"),
+                      cells    = file.path(countsDir, "barcodes.tsv.gz"),
+                      features = file.path(countsDir, "features.tsv.gz"))
+
+    CreateSeuratObject(counts, assay = assay, meta.data = metadata, min.cells = minCells, project = project)
+}

conf/base.config

@@ -19,5 +19,5 @@ process {
     withLabel: 'long'           { time = { 12.h * task.attempt } }
 
     // R processes exit with code 1 on BiocParallel/OOM failures (not signal-based codes)
-    withLabel: 'r'              { errorStrategy = { task.exitStatus in [1, 130, 137, 139, 143] ? 'retry' : 'finish' } }
+    withLabel: 'bambu|seurat'   { errorStrategy = { task.exitStatus in [1, 130, 137, 139, 143] ? 'retry' : 'finish' } }
 }

conf/containers.config

@@ -1,5 +1,6 @@
 process {
-    withLabel: 'r'                  { container = "ghcr.io/goekelab/bambu-pipe-r:1.1.0" }
+    withLabel: 'bambu'              { container = "ghcr.io/goekelab/bambu-pipe-bambu:1.0.0" }
+    withLabel: 'seurat'             { container = "ghcr.io/goekelab/bambu-pipe-seurat:1.0.0" }
     withLabel: 'spaceranger'        { container = "quay.io/nf-core/spaceranger:9c5e7dc93c32448e" }
     withLabel: 'minimap2_samtools'  { container = "community.wave.seqera.io/library/minimap2_samtools:b09096fc890429ce" }
     withLabel: 'preprocess'         { container = "community.wave.seqera.io/library/chopper_cutadapt_flexiplex_pigz:077c3bc67452482c" }

containers/r/Dockerfile → containers/bambu/Dockerfile

@@ -7,28 +7,20 @@ RUN apt-get update && apt-get install -y libcurl4-openssl-dev procps git \
 ARG MAMBA_DOCKERFILE_ACTIVATE=1
 
 # Install bioconductor version of bambu for its dependencies
-# TODO: data.table 1.17.8 installation is a temp fix for clustered_EM 
+# TODO: data.table 1.17.8 installation is a temp fix for clustered_EM
 RUN micromamba install -y -n base -c conda-forge -c bioconda \
     r-base=4.5.3 \
     r-devtools=2.5.2 \
-    r-remotes=2.5.0 \
     r-biocmanager=1.30.27 \
     bioconductor-bambu=3.12.1 \
     bioconductor-dropletutils=1.30.0 \
-    r-seurat=5.4.0 \
-    r-harmony=2.0.2 \
     r-data.table=1.17.8 \
-    r-magick \
-    r-leidenalg \
     gxx=15.2.0 \
     && micromamba clean --all --yes
 
-# Install Seurat Wrappers 
-RUN R -e 'remotes::install_github("satijalab/seurat-wrappers", upgrade = "never")'
-
-# Clone bambu single cell feature branch 
+# Clone bambu single cell feature branch
 RUN git clone --branch devel_pre_v4 \
     https://github.com/GoekeLab/bambu.git /opt/bambu \
     && R -e 'devtools::install("/opt/bambu")'
 
-ENV PATH="$MAMBA_ROOT_PREFIX/bin:$PATH"
+ENV PATH="$MAMBA_ROOT_PREFIX/bin:$PATH"

containers/seurat/Dockerfile

@@ -0,0 +1,13 @@
+# Seurat 5.4.0 and harmony are already installed in the base image
+FROM satijalab/seurat:5.4.0
+
+# procps supplies the ps that Nextflow uses to collect task metrics
+# libmagick++-dev is needed by magick, an import of SpatialExperiment (a Banksy dependency)
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        procps libmagick++-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install Seurat Wrappers, plus Banksy for spatially aware clustering (Banksy is only a
+# suggested dependency of Seurat Wrappers, so it is not installed alongside it)
+RUN R -e 'remotes::install_github("satijalab/seurat-wrappers", upgrade = "never")' \
+    && R -e 'library(SeuratWrappers); library(Banksy); library(harmony)'

main.nf

@@ -92,7 +92,7 @@ workflow STANDARD {
 
         // cluster the cells first, then pool each cluster's cells for the EM
         if (params.quantification_mode == 'EM_clusters') {
-            CLUSTERING(BAMBU_TRANSCRIPT_DISCOVERY.out.se_gene_counts, ch_n_samples)
+            CLUSTERING(BAMBU_TRANSCRIPT_DISCOVERY.out.gene_counts, BAMBU_TRANSCRIPT_DISCOVERY.out.col_data, ch_n_samples)
             BAMBU_CLUSTER_LEVEL_QUANTIFICATION(CLUSTERING.out.clusters, BAMBU_TRANSCRIPT_DISCOVERY.out.quant_data, BAMBU_TRANSCRIPT_DISCOVERY.out.extended_annotations, ch_genome)
         } else if (params.quantification_mode == 'EM') {
             BAMBU_SINGLE_CELL_QUANTIFICATION(BAMBU_TRANSCRIPT_DISCOVERY.out.quant_data, BAMBU_TRANSCRIPT_DISCOVERY.out.extended_annotations, ch_genome)
@@ -133,9 +133,10 @@ workflow VISIUM_HD {
     if (params.quantification_mode == 'EM_clusters') {
         def requested_bin = String.format('%03dum', params.clustering_bin) // convert clustering_bin specified as an integer into Spaceranger format
         // perform clustering at the requested resolution only
-        ch_clustering = AGGREGATE_BINS_VISIUM_HD.out.se_gene_counts
-            .filter { resolution, _se_gene_counts -> resolution == requested_bin }
-            .join(SPOT_BIN_MAPPINGS.out.csv) // [resolution, se_gene_counts, spot_mappings]
+        ch_clustering = AGGREGATE_BINS_VISIUM_HD.out.gene_counts
+            .join(AGGREGATE_BINS_VISIUM_HD.out.col_data)
+            .filter { resolution, _gene_counts, _col_data -> resolution == requested_bin }
+            .join(SPOT_BIN_MAPPINGS.out.csv) // [resolution, gene_counts, col_data, spot_mappings]
         SEURAT_VISIUM_HD(ch_clustering)
         BAMBU_CLUSTER_LEVEL_QUANTIFICATION(SEURAT_VISIUM_HD.out.clusters, ch_quant_data, ch_extended_anno, ch_genome)
 

modules/bambu/shared/cluster_level_quantification.nf

@@ -1,7 +1,7 @@
 process BAMBU_CLUSTER_LEVEL_QUANTIFICATION {
     publishDir "$params.output_dir", mode: 'copy', pattern: 'transcript_counts_clusters'
     publishDir "$params.output_dir", mode: 'copy', pattern: 'gene_counts_clusters'
-    label "r"
+    label "bambu"
     label "low_cpu"
     label "high_mem"
     label "long"

modules/bambu/shared/construct_read_class.nf

@@ -1,6 +1,6 @@
 process BAMBU_CONSTRUCT_READ_CLASS{
     publishDir "$params.output_dir/intermediate_R/read_class", mode: 'copy', pattern: '*_read_class.rds', enabled: params.save_intermediates
-    label "r"
+    label "bambu"
     label "low_cpu"
     label "high_mem"
     label "medium"

modules/bambu/shared/prepare_annotation.nf

@@ -1,5 +1,5 @@
 process BAMBU_PREPARE_ANNOTATION{
-    label "r"
+    label "bambu"
     label "low_cpu"
     label "low_mem"
     label "short"

modules/bambu/standard/single_cell_quantification.nf

@@ -1,6 +1,6 @@
 process BAMBU_SINGLE_CELL_QUANTIFICATION {
     publishDir "$params.output_dir", mode: 'copy', pattern: 'transcript_counts_singlecell'
-    label "r"
+    label "bambu"
     label "low_cpu"
     label "high_mem"
     label "long"

modules/bambu/standard/transcript_discovery.nf

@@ -3,7 +3,7 @@ process BAMBU_TRANSCRIPT_DISCOVERY{
     publishDir "$params.output_dir", mode: 'copy', pattern: 'unique_counts'
     publishDir "$params.output_dir", mode: 'copy', pattern: 'gene_counts'
     publishDir "$params.output_dir/intermediate_R", mode: 'copy', pattern: '*.rds', enabled: params.save_intermediates
-    label "r"
+    label "bambu"
     label "medium_cpu"
     label "high_mem"
     label "medium"
@@ -17,11 +17,11 @@ process BAMBU_TRANSCRIPT_DISCOVERY{
 	output:
     path ('quant_data.rds'), emit: quant_data
     path ('unique_counts/se_unique_counts.rds')
-    path ('gene_counts/se_gene_counts.rds'), emit: se_gene_counts
 	path ('extended_annotations.rds'), emit: extended_annotations
     path ('extended_annotations.gtf')
     path ('unique_counts')
-    path ('gene_counts')
+    path ('gene_counts'), emit: gene_counts
+    path ('col_data.rds'), emit: col_data
     path "versions.yml", topic: 'versions'
 
 	script:
@@ -55,6 +55,10 @@ process BAMBU_TRANSCRIPT_DISCOVERY{
     seDiscoveryGene <- transcriptToGeneExpression(seDiscovery)
     saveCounts(seDiscoveryGene, "gene_counts")
 
+    # colData is saved to create the Seurat object's metadata
+    geneColData <- as.data.frame(colData(seDiscoveryGene))
+    saveRDS(geneColData, "col_data.rds")
+
     writeLines(c('"${task.process}":', paste0('    R: ', R.Version()\$version.string), paste0('    bambu: ', as.character(packageVersion("bambu")))), "versions.yml")
 	"""
 }

modules/bambu/visium_hd/aggregate_bins.nf

@@ -1,7 +1,7 @@
 process AGGREGATE_BINS_VISIUM_HD {
     publishDir "$params.output_dir/unique_counts", mode: 'copy', pattern: 'unique_counts_*'
     publishDir "$params.output_dir/gene_counts", mode: 'copy', pattern: 'gene_counts_*'
-    label "r"
+    label "bambu"
     label "low_cpu"
     label "medium_mem"
     label "short"
@@ -13,7 +13,7 @@ process AGGREGATE_BINS_VISIUM_HD {
     output:
     tuple val(resolution), path("unique_counts_${resolution}"), emit: unique_counts
     tuple val(resolution), path("gene_counts_${resolution}"),   emit: gene_counts
-    tuple val(resolution), path("gene_counts_${resolution}/se_gene_counts_${resolution}.rds"), emit: se_gene_counts
+    tuple val(resolution), path("col_data_${resolution}.rds"),  emit: col_data
     path "versions.yml", topic: 'versions'
 
     script:
@@ -37,6 +37,10 @@ process AGGREGATE_BINS_VISIUM_HD {
     saveCounts(uniqueAgg, "unique_counts_$resolution", "Transcript Expression")
     saveCounts(geneAgg,   "gene_counts_$resolution")
 
+    # colData is saved to create the Seurat object's metadata
+    binColData <- as.data.frame(colData(geneAgg))
+    saveRDS(binColData, "col_data_${resolution}.rds")
+
     writeLines(c('"${task.process}":', paste0('    R: ', R.Version()\$version.string), paste0('    bambu: ', as.character(packageVersion("bambu")))), "versions.yml")
     """
 }

modules/bambu/visium_hd/spot_level_quantification.nf

@@ -1,6 +1,6 @@
 process BAMBU_SPOT_LEVEL_QUANTIFICATION {
     publishDir "$params.output_dir/transcript_counts", mode: 'copy', pattern: 'transcript_counts_*'
-    label "r"
+    label "bambu"
     label "low_cpu"
     label "high_mem"
     label "long"