Merge staging into main branch

.github/workflows/ci-cd.yml

@@ -201,57 +201,6 @@ jobs:
         with:
           ssh-private-key: ${{ secrets.PROD_SERVER_SSH_KEY }}
 
-      - name: Sync GitHub App env vars to Production /opt/appstore/.env
-        env:
-          SERVER_HOST: ${{ secrets.PROD_SERVER_HOST }}
-          GH_APP_ID: ${{ secrets.PROD_GITHUB_APP_ID }}
-          GH_APP_SLUG: ${{ secrets.PROD_GITHUB_APP_SLUG }}
-          GH_APP_PRIVATE_KEY: ${{ secrets.PROD_GITHUB_APP_PRIVATE_KEY }}
-          GH_APP_STATE_SECRET: ${{ secrets.PROD_GITHUB_APP_STATE_SECRET }}
-          FRONTEND_BASE_URL: ${{ secrets.PROD_FRONTEND_BASE_URL }}
-        run: |
-          set -euo pipefail
-          umask 077
-
-          # Build the new block locally so the multi-line PEM stays exact;
-          # piping it through SSH heredoc would risk shell expansion.
-          SNIPPET="$(mktemp)"
-          trap 'rm -f "$SNIPPET"' EXIT
-          {
-            printf '# >>> github-app (managed by ci-cd.yml) >>>\n'
-            printf 'GITHUB_APP_ID=%s\n' "$GH_APP_ID"
-            printf 'GITHUB_APP_SLUG=%s\n' "$GH_APP_SLUG"
-            printf 'GITHUB_APP_STATE_SECRET=%s\n' "$GH_APP_STATE_SECRET"
-            printf 'FRONTEND_BASE_URL=%s\n' "$FRONTEND_BASE_URL"
-            # PEM contains real newlines — wrap in double quotes so docker-compose
-            # / python-dotenv reads it as a single multi-line value.
-            printf 'GITHUB_APP_PRIVATE_KEY="%s"\n' "$GH_APP_PRIVATE_KEY"
-            printf '# <<< github-app <<<\n'
-          } > "$SNIPPET"
-
-          scp -o StrictHostKeyChecking=no "$SNIPPET" ubuntu@$SERVER_HOST:/tmp/appstore-github-env
-
-          ssh -o StrictHostKeyChecking=no ubuntu@$SERVER_HOST bash -s <<'ENDSSH'
-            set -euo pipefail
-            cd /opt/appstore
-            umask 077
-
-            [ -f .env ] || { touch .env; chmod 600 .env; }
-
-            # Drop any prior managed block (delimited by markers); silent no-op
-            # if absent. Multi-line PEM is removed cleanly because the delete
-            # range is line-based on our markers.
-            sed -i '/^# >>> github-app (managed by ci-cd\.yml) >>>$/,/^# <<< github-app <<<$/d' .env
-
-            # Trailing newline before append so we never collide with a
-            # non-newline-terminated last line.
-            [ -s .env ] && [ "$(tail -c1 .env | wc -l)" -eq 0 ] && printf '\n' >> .env
-
-            cat /tmp/appstore-github-env >> .env
-            rm -f /tmp/appstore-github-env
-            chmod 600 .env
-          ENDSSH
-
       - name: Deploy api + celery-worker + celery-beat on Production
         env:
           SERVER_HOST: ${{ secrets.PROD_SERVER_HOST }}

.gitignore

@@ -233,4 +233,6 @@ loki-data/
 grafana-data/
 promtail-positions.yaml
 
-.DS_Store
+.DS_Store
+# Local SQLite databases (test fixtures, scratch DBs)
+*.db

alembic/versions/034d40e1dad3_add_activation_link_to_access_type.py

@@ -0,0 +1,53 @@
+"""add ACTIVATION_LINK to accesstype enum
+
+Revision ID: 034d40e1dad3
+Revises: c8e9d3b7f1a2
+Create Date: 2026-06-29 12:00:00.000000
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision: str = '034d40e1dad3'
+down_revision: Union[str, Sequence[str], None] = 'c8e9d3b7f1a2'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Upgrade schema.
+
+    Adds a new ``ACTIVATION_LINK`` value to the ``accesstype`` Postgres enum
+    that backs ``deployment_instance_access.access_type``. Required for the
+    Overleaf LaTeX Lab app (and any future app that produces one-time
+    activation links during the Ansible run rather than passwords/keys
+    before it): the post-Ansible SSH fetch in ``deploy_tasks`` writes such
+    rows via ``DeploymentCredentialService.persist_activation_links``.
+
+    ``IF NOT EXISTS`` keeps the migration idempotent — safe to re-run if a
+    manual ALTER got there first.
+
+    Note: ``ALTER TYPE ... ADD VALUE`` is allowed inside a transaction since
+    PG 12, but the newly-added label is only usable after commit. The
+    backfill of any rows using this value happens at deploy time, never in
+    this migration, so the default Alembic transaction wrap is fine.
+    """
+    op.execute(
+        "ALTER TYPE accesstype "
+        "ADD VALUE IF NOT EXISTS 'ACTIVATION_LINK'"
+    )
+
+
+def downgrade() -> None:
+    """Downgrade schema.
+
+    Postgres does not support removing values from an enum type without
+    rebuilding the type and all dependent columns. Rolling back the
+    application code is the correct response; the unused enum label is
+    harmless. Left intentionally empty (same approach as
+    ``a1c5e8d2f307_add_expiry_enum_values_to_deployment_log``).
+    """
+    pass

alembic/versions/b5c41a8e7d92_add_group_id_to_deployment_instance_access.py

@@ -0,0 +1,62 @@
+"""add group_id to deployment_instance_access
+
+Revision ID: b5c41a8e7d92
+Revises: e7f3a91d05b8
+Create Date: 2026-06-23 17:00:00.000000
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = 'b5c41a8e7d92'
+down_revision: Union[str, Sequence[str], None] = 'e7f3a91d05b8'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Upgrade schema.
+
+    Adds a nullable ``group_id`` FK on ``deployment_instance_access`` pointing
+    to ``course_groups.id``. This is the missing link that lets the new
+    student self-service endpoint filter credentials down to the rows the
+    student is entitled to see — students see rows where ``group_id`` matches
+    one of their group memberships; lecturer admin credentials use
+    ``group_id IS NULL`` and remain hidden from students.
+
+    Nullable + no default + no backfill in this migration. A separate data
+    migration (``b6d52b9f8ea3_backfill...``) walks existing
+    ``deployments.deployment_parameters`` JSON to retroactively populate the
+    new column for pre-feature deployments.
+    """
+    op.add_column(
+        'deployment_instance_access',
+        sa.Column('group_id', sa.String(length=36), nullable=True),
+    )
+    op.create_foreign_key(
+        'fk_deployment_instance_access_group_id',
+        source_table='deployment_instance_access',
+        referent_table='course_groups',
+        local_cols=['group_id'],
+        remote_cols=['id'],
+    )
+    op.create_index(
+        'ix_deployment_instance_access_group_id',
+        'deployment_instance_access',
+        ['group_id'],
+    )
+
+
+def downgrade() -> None:
+    """Downgrade schema."""
+    op.drop_index('ix_deployment_instance_access_group_id', table_name='deployment_instance_access')
+    op.drop_constraint(
+        'fk_deployment_instance_access_group_id',
+        'deployment_instance_access',
+        type_='foreignkey',
+    )
+    op.drop_column('deployment_instance_access', 'group_id')

alembic/versions/b6d52b9f8ea3_backfill_access_group_id.py

@@ -0,0 +1,138 @@
+"""backfill deployment_instance_access.group_id from existing deployments
+
+Revision ID: b6d52b9f8ea3
+Revises: b5c41a8e7d92
+Create Date: 2026-06-23 17:05:00.000000
+
+Walks every existing deployment's ``deployment_parameters`` JSON (which holds
+the original ``stack_assignments[*].groups[*]`` payload), looks up the
+matching ``course_groups`` row by ``(course_id, name)``, and stamps
+``deployment_instance_access.group_id`` for rows whose ``username`` matches
+the sanitized group name.
+
+Idempotent: only updates rows where ``group_id IS NULL``. Re-runs are no-ops.
+
+Skipped gracefully when:
+- ``deployment_parameters`` is NULL or unparseable
+- No matching ``course_groups`` row exists (lecturer never created groups via
+  the courses API — those access rows stay ``group_id = NULL`` and remain
+  invisible to students; lecturer-side flow is unaffected)
+- The access row's ``username`` doesn't match any group's sanitized name
+  (likely an admin/teacher credential — intentionally stays NULL)
+"""
+from __future__ import annotations
+
+import json
+import re
+
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = 'b6d52b9f8ea3'
+down_revision: Union[str, Sequence[str], None] = 'b5c41a8e7d92'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def _sanitize_username(name: str) -> str:
+    """Mirror of credential_generator_service._sanitize_username.
+
+    Kept inline (not imported) so the migration is self-contained and
+    immune to future service-code refactors. MUST be kept byte-for-byte
+    in sync with the service implementation — otherwise the backfill
+    can't match existing DeploymentInstanceAccess.username rows against
+    GroupMember-derived group names.
+    """
+    username = name.lower().replace(" ", "_").replace(".", "_").replace("-", "_")
+    username = re.sub(r"[^a-z0-9_]", "", username)
+    if username and username[0].isdigit():
+        username = "u" + username
+    return (username or "user")[:32]
+
+
+def upgrade() -> None:
+    """Backfill ``group_id`` on existing ``deployment_instance_access`` rows."""
+    bind = op.get_bind()
+
+    deployments = bind.execute(sa.text(
+        "SELECT id, course_id, deployment_parameters "
+        "FROM deployments "
+        "WHERE deployment_parameters IS NOT NULL"
+    )).fetchall()
+
+    updated = 0
+    for dep in deployments:
+        dep_id = dep[0]
+        course_id = dep[1]
+        params_raw = dep[2]
+        if not params_raw:
+            continue
+        try:
+            params = json.loads(params_raw)
+        except (json.JSONDecodeError, TypeError):
+            continue
+
+        stack_assignments = params.get("stack_assignments") or []
+        for stack in stack_assignments:
+            for group in (stack.get("groups") or []):
+                group_name = group.get("group_name")
+                if not group_name:
+                    continue
+                sanitized = _sanitize_username(group_name)
+
+                # Find the CourseGroup row for this (course_id, group_name).
+                # Older payloads may carry course_group_id directly; prefer it.
+                course_group_id = group.get("course_group_id")
+                if not course_group_id:
+                    row = bind.execute(
+                        sa.text(
+                            "SELECT id FROM course_groups "
+                            "WHERE course_id = :course_id AND name = :name "
+                            "LIMIT 1"
+                        ),
+                        {"course_id": course_id, "name": group_name},
+                    ).fetchone()
+                    if not row:
+                        # No persisted CourseGroup for this group → skip.
+                        # Lecturer-side flow keeps working; students just
+                        # don't see anything for this group.
+                        continue
+                    course_group_id = row[0]
+
+                # Stamp any access row of this deployment whose username
+                # matches the sanitized group name AND is still group_id IS NULL.
+                result = bind.execute(
+                    sa.text(
+                        "UPDATE deployment_instance_access "
+                        "SET group_id = :group_id "
+                        "WHERE id IN ("
+                        "  SELECT dia.id FROM deployment_instance_access dia "
+                        "  JOIN deployment_instances di "
+                        "    ON di.id = dia.deployment_instance_id "
+                        "  WHERE di.deployment_id = :dep_id "
+                        "    AND dia.username = :username "
+                        "    AND dia.group_id IS NULL "
+                        ")"
+                    ),
+                    {
+                        "group_id": course_group_id,
+                        "dep_id": dep_id,
+                        "username": sanitized,
+                    },
+                )
+                updated += result.rowcount or 0
+
+    print(f"backfill complete: stamped group_id on {updated} access row(s)")
+
+
+def downgrade() -> None:
+    """Set all group_id back to NULL — reverses the backfill.
+
+    Idempotent and safe: no data is destroyed; the schema column itself
+    is dropped by the previous migration's downgrade.
+    """
+    op.execute("UPDATE deployment_instance_access SET group_id = NULL")