From aa56f94bd30b1e8016ff7d291934a21669d5efd8 Mon Sep 17 00:00:00 2001 From: nicowre Date: Tue, 30 Jun 2026 15:08:06 +0200 Subject: [PATCH] feat(deployments): redeploy whole deployment or single VM with config overrides MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds two new endpoints that let lecturers re-apply a deployment from scratch — either across every VM in the class or targeted at one wedged group — and accept per-deployment / per-VM template parameter overrides so a config change actually takes effect (Heat + Ansible re-run, not just a Heat update_stack refresh). API POST /deployments/{id}/redeploy iterates sequentially over every DeploymentInstance row and destroy-and-recreates each VM. Body accepts: - deployment_parameter_overrides (apply to every VM) - instance_parameter_overrides (per-VM map, merged on top) - preserve_credentials (default false → fresh creds) POST /deployments/{id}/instances/{instance_id}/redeploy same flow for one VM only. Parent deployment stays RUNNING so sibling VMs remain reachable for students; only the targeted instance flips to the new REDEPLOYING status. Backend - Extracts _provision_one_stack_assignment as the shared 'create stack → wait SSH → run Ansible → persist credentials' helper. Both the initial deploy_stack loop and the new redeploy tasks call it, so drift between first-deploy and redeploy is impossible. - _merge_parameter_layers implements the override order base → deployment → instance (later layers win, backend-managed Heat params like user_json/key_name are always stripped). - _snapshot_instance_credentials / rebind path so preserve_credentials can keep students' logins working across a config change. - Adds DeploymentInstanceStatus.REDEPLOYING + Alembic migration e2a91d05c7b8 to extend the Postgres enum (IF NOT EXISTS, idempotent). Tests - 21 unit tests for the merge logic, vm_name → stack_assignment reconstruction, credential snapshot/rebind, orchestration, and redeploy_deployment fan-out (incl. partial-failure path). - 9 API tests for the two endpoints (success, defaults, 400 in transitional state, 404 paths). Bruno - bruno/Deployments/Redeploy Deployment.bru - bruno/Deployments/Redeploy Instance.bru Both with full docs covering body schema, merge order, error cases, and the preserve_credentials caveat. --- ...c7b8_add_redeploying_to_instance_status.py | 48 + bruno/Deployments/Redeploy Deployment.bru | 160 ++ bruno/Deployments/Redeploy Instance.bru | 141 ++ src/api/deployments.py | 229 +- src/models/deployment_instance.py | 11 +- src/schemas/deployment.py | 78 + src/tasks/deploy_tasks.py | 1944 +++++++++++++---- tests/api/test_deployment_redeploy_routes.py | 303 +++ tests/unit/test_redeploy_tasks.py | 541 +++++ 9 files changed, 3053 insertions(+), 402 deletions(-) create mode 100644 alembic/versions/e2a91d05c7b8_add_redeploying_to_instance_status.py create mode 100644 bruno/Deployments/Redeploy Deployment.bru create mode 100644 bruno/Deployments/Redeploy Instance.bru create mode 100644 tests/api/test_deployment_redeploy_routes.py create mode 100644 tests/unit/test_redeploy_tasks.py diff --git a/alembic/versions/e2a91d05c7b8_add_redeploying_to_instance_status.py b/alembic/versions/e2a91d05c7b8_add_redeploying_to_instance_status.py new file mode 100644 index 0000000..7b45d61 --- /dev/null +++ b/alembic/versions/e2a91d05c7b8_add_redeploying_to_instance_status.py @@ -0,0 +1,48 @@ +"""add REDEPLOYING to deploymentinstancestatus enum + +Revision ID: e2a91d05c7b8 +Revises: f9c2a14e7b80 +Create Date: 2026-06-30 09:00:00.000000 + +""" +from typing import Sequence, Union + +from alembic import op + + +# revision identifiers, used by Alembic. +revision: str = 'e2a91d05c7b8' +down_revision: Union[str, Sequence[str], None] = 'f9c2a14e7b80' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Upgrade schema. + + Adds a new ``REDEPLOYING`` value to the ``deploymentinstancestatus`` + Postgres enum so the ``redeploy_instance`` Celery task can mark a single + DeploymentInstance row as transient while it tears down + recreates its + Heat stack, without flipping the parent Deployment to RESTARTING (which + would block the user from running other ops on its siblings). + + ``IF NOT EXISTS`` keeps the migration idempotent — safe to re-run if a + manual ALTER got there first. Same approach as + ``034d40e1dad3_add_activation_link_to_access_type``. + """ + op.execute( + "ALTER TYPE deploymentinstancestatus " + "ADD VALUE IF NOT EXISTS 'REDEPLOYING'" + ) + + +def downgrade() -> None: + """Downgrade schema. + + Postgres does not support removing values from an enum type without + rebuilding the type and all dependent columns. Rolling back the + application code is the correct response; the unused enum label is + harmless. Left intentionally empty (mirrors + ``034d40e1dad3_add_activation_link_to_access_type``). + """ + pass diff --git a/bruno/Deployments/Redeploy Deployment.bru b/bruno/Deployments/Redeploy Deployment.bru new file mode 100644 index 0000000..a6e6348 --- /dev/null +++ b/bruno/Deployments/Redeploy Deployment.bru @@ -0,0 +1,160 @@ +meta { + name: Redeploy Deployment + type: http + seq: 6 +} + +post { + url: {{base_url}}/api/v1/deployments/{{deployment_id}}/redeploy?openstack_project_id={{openstack_project_local_id}} + body: json + auth: bearer +} + +auth:bearer { + token: {{access_token}} +} + +params:query { + openstack_project_id: {{openstack_project_local_id}} +} + +params:path { + deployment_id: +} + +body:json { + { + "deployment_parameter_overrides": { + "include_example_notebooks": true + }, + "instance_parameter_overrides": { + "instance-uuid-of-group-3": { + "flavor": "gp1.medium" + } + }, + "preserve_credentials": false + } +} + +docs { + # Redeploy Deployment (all VMs) + + Destroy-and-recreate **every VM** (DeploymentInstance) in this deployment, one + after another, optionally with overridden parameters (template / config values). + + Unlike `POST /deployments/{id}/restart` (which only triggers a Heat + `update_stack` on the existing stack), this fully rebuilds each VM from + scratch: + + 1. Heat stack deleted + 2. Heat stack recreated with merged parameters + 3. Ansible re-run + 4. Credentials regenerated (unless `preserve_credentials=true`) + + Use it when a config / template parameter changed and you want the change to + actually take effect. + + ## Authorization + - **Owner (Lecturer)** or **Admin** only + - Lecturers can only redeploy their own deployments + - Admins can redeploy any deployment + + ## Body + + All fields are optional. Empty body = redeploy with the stored + `deployment_parameters` unchanged and fresh credentials. + + | Field | Type | Description | + |----------------------------------|-------------------------------------|-------------| + | `deployment_parameter_overrides` | `dict[str, Any]` | Merged ON TOP of the deployment's stored parameters for every redeployed VM. | + | `instance_parameter_overrides` | `dict[instance_id, dict[str, Any]]` | Per-VM overrides, keyed by `DeploymentInstance.id`. Merged ON TOP of `deployment_parameter_overrides` for that one VM. | + | `preserve_credentials` | `bool` (default `false`) | Keep the existing per-VM credentials instead of regenerating them. Useful to avoid breaking student logins during a quick config change. | + + ### Merge order + + Later layers win: + + template defaults + ↓ + deployment.deployment_parameters + ↓ + deployment_parameter_overrides + ↓ + instance_parameter_overrides[] + + ## Process + + 1. API validates permissions and deployment state. + 2. A single Celery task `redeploy_deployment` is enqueued. + 3. The task iterates **sequentially** over every DeploymentInstance row and + reuses the `redeploy_instance` task per VM (to avoid quota exhaustion in + parallel mode). + 4. For each VM: status flips to `REDEPLOYING` → Heat stack deleted → Heat + stack recreated with merged params → Ansible run → credentials persisted. + 5. The parent deployment stays in `RUNNING` between instances; siblings + remain reachable for students. + + ## Response + + Returns 202 Accepted; the actual redeploy runs asynchronously. + + ## Example Response + + ```json + { + "success": true, + "data": { + "deployment_id": "deploy-123", + "instance_count": 5, + "status": "redeploy_queued", + "preserve_credentials": false + }, + "message": "Redeploy requested for 5 instance(s); operation in progress", + "request_id": "req-456" + } + ``` + + ## Error Responses + + **404 Not Found** + ```json + {"success": false, "message": "Deployment with ID xyz not found"} + ``` + + **403 Forbidden** (lecturer trying to redeploy another lecturer's deployment) + ```json + {"success": false, "message": "You do not have permission to access this deployment"} + ``` + + **400 Bad Request** (deployment in transitional state) + ```json + {"success": false, "message": "Cannot redeploy deployment in creating state. Please wait for the current operation to complete."} + ``` + + **400 Bad Request** (deployment has no instances) + ```json + {"success": false, "message": "Deployment has no instances to redeploy"} + ``` + + **500 Internal Server Error** (task enqueue failure) + ```json + {"success": false, "message": "Failed to enqueue redeploy task"} + ``` + + ## Tracking Progress + + - Per-VM status visible on each `DeploymentInstance.status` (flips through + `REDEPLOYING` → `CREATING` → `RUNNING|FAILED`). + - Live log stream: `GET /deployments/{id}/logs/stream`. + - The deployment's `openstack_stack_id` JSON array is rewritten incrementally + as old stack IDs are dropped and new ones are added — so a follow-up DELETE + still cleans up correctly. + + ## Caveats + + - `preserve_credentials=true` can only carry over credentials that are + persisted in `DeploymentInstanceAccess`. Passwords generated ad-hoc inside + Ansible playbooks (without a credential spec entry) are still regenerated. + - The Heat stack name format stays `-s-`, so + OpenStack tags / dashboards keep working across redeploys. +} diff --git a/bruno/Deployments/Redeploy Instance.bru b/bruno/Deployments/Redeploy Instance.bru new file mode 100644 index 0000000..f1784c8 --- /dev/null +++ b/bruno/Deployments/Redeploy Instance.bru @@ -0,0 +1,141 @@ +meta { + name: Redeploy Instance + type: http + seq: 7 +} + +post { + url: {{base_url}}/api/v1/deployments/{{deployment_id}}/instances/{{instance_id}}/redeploy?openstack_project_id={{openstack_project_local_id}} + body: json + auth: bearer +} + +auth:bearer { + token: {{access_token}} +} + +params:query { + openstack_project_id: {{openstack_project_local_id}} +} + +params:path { + deployment_id: + instance_id: +} + +body:json { + { + "deployment_parameter_overrides": { + "include_example_notebooks": true, + "flavor": "gp1.medium" + }, + "preserve_credentials": false + } +} + +docs { + # Redeploy Instance (single VM) + + Destroy-and-recreate **exactly one VM** (DeploymentInstance) inside an + existing deployment. + + Use this when one VM is wedged, or to apply a config change to a single + group without touching its siblings. The parent deployment stays in + `RUNNING` for the duration — only the targeted instance flips to + `REDEPLOYING`. + + ## Authorization + - **Owner (Lecturer)** or **Admin** only + - Lecturers can only redeploy instances of their own deployments + - Admins can redeploy any instance + + ## Body + + All fields are optional. Empty body = redeploy with the deployment's stored + parameters unchanged and fresh credentials. + + | Field | Type | Description | + |----------------------------------|--------------------------|-------------| + | `deployment_parameter_overrides` | `dict[str, Any]` | Merged ON TOP of the deployment's stored parameters for this VM. Since there's only one VM in scope, treat this as the full override map for this instance. | + | `preserve_credentials` | `bool` (default `false`) | Keep the existing access rows (passwords / SSH keys / activation links) on the freshly-created instance instead of regenerating them. | + + > **Note:** The `instance_parameter_overrides` field accepted by the + > deployment-wide endpoint is **ignored here**. Pass per-VM parameters + > directly in `deployment_parameter_overrides`. + + ## Process + + 1. API validates permissions, deployment state, and instance existence. + 2. Celery task `redeploy_instance` is enqueued. + 3. Task flips instance status to `REDEPLOYING`. + 4. If `preserve_credentials=true`: snapshots the current access rows. + 5. Old Heat stack deleted, old `DeploymentInstance` + access rows wiped. + 6. New Heat stack created with merged parameters (same `-s-` slug + so OpenStack tags stay consistent). + 7. Ansible runs; credentials are persisted (or restored from snapshot). + 8. The deployment's `openstack_stack_id` JSON array is rewritten so the old + ID is dropped and the new one is added — a future DELETE walks the right + list. + + ## Response + + Returns 202 Accepted; the actual redeploy runs asynchronously. + + ## Example Response + + ```json + { + "success": true, + "data": { + "deployment_id": "deploy-123", + "instance_id": "inst-A", + "status": "redeploy_queued", + "preserve_credentials": false + }, + "message": "Redeploy requested for instance; operation in progress", + "request_id": "req-456" + } + ``` + + ## Error Responses + + **404 Not Found** (deployment missing) + ```json + {"success": false, "message": "Deployment with ID xyz not found"} + ``` + + **404 Not Found** (instance missing or belongs to a different deployment) + ```json + {"success": false, "message": "Instance inst-A not found in deployment deploy-123"} + ``` + + **403 Forbidden** (lecturer not the owner) + ```json + {"success": false, "message": "You do not have permission to access this deployment"} + ``` + + **400 Bad Request** (deployment in transitional state) + ```json + {"success": false, "message": "Cannot redeploy instance while deployment is in deleting state."} + ``` + + **500 Internal Server Error** (task enqueue failure) + ```json + {"success": false, "message": "Failed to enqueue redeploy task"} + ``` + + ## Tracking Progress + + - Watch this single instance's `.status` (REDEPLOYING → CREATING → RUNNING|FAILED). + - Live log stream: `GET /deployments/{id}/logs/stream`. + - Sibling VMs remain reachable throughout — no class-wide outage. + + ## Caveats + + - The instance lookup uses the row's stored `vm_name` (`-s-` suffix) + to recover which `stack_assignment` produced it. Manually-renamed VMs may + fail with `Cannot recover stack_assignment for instance ...`. + - `preserve_credentials=true` only restores credentials present in + `DeploymentInstanceAccess`. Passwords generated ad-hoc inside Ansible + playbooks (no credential spec entry) are still regenerated. +} diff --git a/src/api/deployments.py b/src/api/deployments.py index 6b67ed2..c192501 100644 --- a/src/api/deployments.py +++ b/src/api/deployments.py @@ -11,7 +11,7 @@ from src.core.dependencies import RequestID, Pagination, CurrentUser, require_roles from src.repositories.deployment_repository import DeploymentRepository from src.repositories.openstack_project_repository import OpenstackProjectRepository -from src.schemas.deployment import DeploymentResponse, DeploymentCreate, DeploymentExtend +from src.schemas.deployment import DeploymentResponse, DeploymentCreate, DeploymentExtend, DeploymentRedeployRequest from src.services.deployment_service import DeploymentService from src.services.deployment_log_service import DeploymentLogService from src.services.openstack_heat_service import HeatStackService @@ -23,8 +23,10 @@ ) from src.tasks.deploy_tasks import delete_deployment as delete_deployment_task from src.tasks.deploy_tasks import restart_deployment as restart_deployment_task +from src.tasks.deploy_tasks import redeploy_instance as redeploy_instance_task +from src.tasks.deploy_tasks import redeploy_deployment as redeploy_deployment_task from src.models.deployment import DeploymentStatus, Deployment -from src.models.deployment_instance import DeploymentInstance +from src.models.deployment_instance import DeploymentInstance, DeploymentInstanceStatus from src.models.deployment_instance_access import DeploymentInstanceAccess from src.models.template_version import TemplateVersion @@ -877,6 +879,229 @@ async def restart_deployment( ) +@router.post( + "/{deployment_id}/redeploy", + status_code=status.HTTP_202_ACCEPTED, + summary="Redeploy every VM in a deployment", + responses={ + 202: {"description": "Redeploy requested; operation in progress"}, + 400: {"description": "Deployment is in an invalid state or has no instances"}, + 403: {"description": "Forbidden - not owner or insufficient role"}, + 404: {"description": "Deployment not found"}, + 500: {"description": "Failed to enqueue redeploy task"}, + }, +) +async def redeploy_deployment_endpoint( + deployment_id: str, + db: DBSession, + request_id: RequestID, + user: CurrentUser, + payload: DeploymentRedeployRequest | None = None, + openstack_project_id: UUID | None = Query( + None, + description="OpenStack project (local DB id) the deployment must belong to. Required for non-admin users.", + ), +): + """Destroy-and-recreate every VM (``DeploymentInstance``) in this + deployment, one after another, optionally with overridden parameters. + + Unlike :func:`restart_deployment` (which only triggers a Heat + ``update_stack`` on the existing stack), this rebuilds each VM from + scratch: Heat stack deleted → Heat stack recreated → Ansible re-run → + credentials regenerated (unless ``preserve_credentials=true``). Use it + when a config / template parameter changed and you want the change to + actually take effect. + + Sequential by design — running them in parallel risks tripping the + OpenStack project quota mid-class. The parent deployment stays in + ``RUNNING`` between instances so the UI can show per-VM progress and + siblings stay reachable. + + **Authorization:** Owner (lecturer) or Admin only. + """ + deployment_repo = DeploymentRepository(db) + deployment = deployment_repo.get_by_id(deployment_id) + if not deployment: + raise NotFoundException(f"Deployment with ID {deployment_id} not found") + + authorize_deployment_access(deployment, user, openstack_project_id, db) + + transitional_states = [ + DeploymentStatus.CREATING, + DeploymentStatus.DELETING, + DeploymentStatus.RESTARTING, + ] + if deployment.status in transitional_states: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + f"Cannot redeploy deployment in {deployment.status.value} state. " + f"Please wait for the current operation to complete." + ), + ) + + # If any instance is already mid-redeploy, refuse — two redeploy tasks racing + # the same Heat stack would corrupt deployment.openstack_stack_id and leak + # stacks. We only need to find ONE such instance to reject. + in_flight = ( + db.query(DeploymentInstance.id) + .filter( + DeploymentInstance.deployment_id == deployment_id, + DeploymentInstance.status == DeploymentInstanceStatus.REDEPLOYING, + ) + .first() + ) + if in_flight is not None: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + "A redeploy is already in progress for at least one instance " + "in this deployment; wait for it to finish before queuing another." + ), + ) + + instance_count = ( + db.query(DeploymentInstance) + .filter(DeploymentInstance.deployment_id == deployment_id) + .count() + ) + if instance_count == 0: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Deployment has no instances to redeploy", + ) + + body = payload or DeploymentRedeployRequest() + # We deliberately don't wrap .delay() in a broad except: a broker outage or + # a non-serialisable override value should surface as the underlying error + # (Celery's own message) so the caller can diagnose it, instead of being + # masked behind a generic "Failed to enqueue redeploy task". + redeploy_deployment_task.delay( + deployment_id, + deployment_parameter_overrides=body.deployment_parameter_overrides, + instance_parameter_overrides=body.instance_parameter_overrides, + preserve_credentials=body.preserve_credentials, + ) + + return ResponseBuilder.success( + data={ + "deployment_id": deployment_id, + "instance_count": instance_count, + "status": "redeploy_queued", + "preserve_credentials": body.preserve_credentials, + }, + message=f"Redeploy requested for {instance_count} instance(s); operation in progress", + request_id=request_id, + status_code=status.HTTP_202_ACCEPTED, + ) + + +@router.post( + "/{deployment_id}/instances/{instance_id}/redeploy", + status_code=status.HTTP_202_ACCEPTED, + summary="Redeploy a single VM (DeploymentInstance)", + responses={ + 202: {"description": "Redeploy requested; operation in progress"}, + 400: {"description": "Deployment or instance is in an invalid state"}, + 403: {"description": "Forbidden - not owner or insufficient role"}, + 404: {"description": "Deployment or instance not found"}, + 500: {"description": "Failed to enqueue redeploy task"}, + }, +) +async def redeploy_instance_endpoint( + deployment_id: str, + instance_id: str, + db: DBSession, + request_id: RequestID, + user: CurrentUser, + payload: DeploymentRedeployRequest | None = None, + openstack_project_id: UUID | None = Query( + None, + description="OpenStack project (local DB id) the deployment must belong to. Required for non-admin users.", + ), +): + """Destroy-and-recreate exactly one VM inside an existing deployment. + + Use this when one VM is wedged, or to apply a config change to a + single group without touching its siblings. The parent deployment + stays in ``RUNNING`` for the duration — only the target instance + flips to ``REDEPLOYING``. + + Body parameters mirror the deployment-wide endpoint, with one + semantic shift: ``deployment_parameter_overrides`` is treated as the + full override for this one VM (since there's only one in scope). + ``instance_parameter_overrides`` is ignored here — pass per-VM + parameters directly in ``deployment_parameter_overrides``. + + **Authorization:** Owner (lecturer) or Admin only. + """ + deployment_repo = DeploymentRepository(db) + deployment = deployment_repo.get_by_id(deployment_id) + if not deployment: + raise NotFoundException(f"Deployment with ID {deployment_id} not found") + + authorize_deployment_access(deployment, user, openstack_project_id, db) + + transitional_states = [ + DeploymentStatus.CREATING, + DeploymentStatus.DELETING, + DeploymentStatus.RESTARTING, + ] + if deployment.status in transitional_states: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + f"Cannot redeploy instance while deployment is in {deployment.status.value} state." + ), + ) + + instance = ( + db.query(DeploymentInstance) + .filter( + DeploymentInstance.id == instance_id, + DeploymentInstance.deployment_id == deployment_id, + ) + .first() + ) + if not instance: + raise NotFoundException( + f"Instance {instance_id} not found in deployment {deployment_id}" + ) + + # Refuse if this instance is already mid-redeploy — a second task racing the + # first one would delete a stack the first already removed (raises and flips + # status to FAILED) and the survivors would corrupt deployment.openstack_stack_id. + if instance.status == DeploymentInstanceStatus.REDEPLOYING: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=( + f"Instance {instance_id} is already being redeployed; " + "wait for the current operation to finish." + ), + ) + + body = payload or DeploymentRedeployRequest() + # No broad except around .delay() — see redeploy_deployment_endpoint above. + redeploy_instance_task.delay( + deployment_id, + instance_id, + deployment_parameter_overrides=body.deployment_parameter_overrides, + preserve_credentials=body.preserve_credentials, + ) + + return ResponseBuilder.success( + data={ + "deployment_id": deployment_id, + "instance_id": instance_id, + "status": "redeploy_queued", + "preserve_credentials": body.preserve_credentials, + }, + message="Redeploy requested for instance; operation in progress", + request_id=request_id, + status_code=status.HTTP_202_ACCEPTED, + ) + + @router.patch( "/{deployment_id}/extend", summary="Extend a deployment's lifetime", diff --git a/src/models/deployment_instance.py b/src/models/deployment_instance.py index adb83e7..0783db7 100644 --- a/src/models/deployment_instance.py +++ b/src/models/deployment_instance.py @@ -9,9 +9,18 @@ class DeploymentInstanceStatus(str, Enum): - """Deployment instance status values.""" + """Deployment instance status values. + + ``REDEPLOYING`` is set on a single instance while ``redeploy_instance`` + tears down its Heat stack and rebuilds it. The parent ``Deployment`` row + stays in ``RUNNING`` during that — only this one instance is transient, + so siblings remain reachable. When the redeploy succeeds the row is + replaced with a fresh ``DeploymentInstance``; the old row is deleted as + part of the task, mirroring delete_deployment's instance teardown. + """ CREATING = "creating" RUNNING = "running" + REDEPLOYING = "redeploying" FAILED = "failed" DELETED = "deleted" diff --git a/src/schemas/deployment.py b/src/schemas/deployment.py index 690984b..6465aa1 100644 --- a/src/schemas/deployment.py +++ b/src/schemas/deployment.py @@ -196,6 +196,84 @@ class DeploymentResponse(BaseModel): ) +class DeploymentRedeployRequest(BaseModel): + """Body for ``POST /deployments/{id}/redeploy`` and + ``POST /deployments/{id}/instances/{instance_id}/redeploy``. + + Per the product spec, a "Config" in this codebase = the optional template + parameters surfaced by the underlying app (e.g. an on/off toggle "include + example notebooks"). A redeploy may **carry over** the deployment's existing + parameter map unchanged, or **override** it for the whole deployment + (``deployment_parameter_overrides``) and/or for individual VMs + (``instance_parameter_overrides``). + + Merge order during the actual redeploy task:: + + template defaults → deployment.deployment_parameters + → deployment_parameter_overrides + → instance_parameter_overrides[] + + ``preserve_credentials`` controls whether the existing + ``DeploymentInstanceAccess`` rows (passwords, SSH keys, activation links) + are kept and re-bound to the freshly-created instance, or wiped and + regenerated from scratch. Default ``False`` mirrors a clean + destroy-and-recreate; pass ``True`` to keep students' logins working + across the redeploy. + """ + + deployment_parameter_overrides: Optional[dict[str, Any]] = Field( + default=None, + description=( + "Parameters to merge ON TOP of the deployment's stored " + "``deployment_parameters`` for every redeployed VM. Keys absent here " + "fall back to the deployment's stored value (then to template defaults). " + "Pass an empty dict to keep the deployment-level params unchanged." + ), + ) + instance_parameter_overrides: Optional[dict[str, dict[str, Any]]] = Field( + default=None, + description=( + "Per-VM parameter overrides keyed by DeploymentInstance.id. " + "Each entry is merged ON TOP of the deployment-level overrides for " + "that one VM only. Keys not present in any layer fall back to " + "template defaults. Instance IDs not in this map use the " + "deployment-level overrides as-is. Ignored when redeploying a single " + "instance via the per-instance endpoint — pass the override under " + "``deployment_parameter_overrides`` there since there's only one VM " + "in scope." + ), + ) + preserve_credentials: bool = Field( + default=False, + description=( + "When True, the existing per-VM credentials (DeploymentInstanceAccess " + "rows) are re-bound to the freshly-created instance instead of being " + "regenerated. Useful to avoid breaking student logins during a quick " + "config change. Default False = clean destroy-and-recreate, fresh " + "passwords/keys/activation links." + ), + ) + + # ``extra="forbid"`` mirrors the contract recently tightened in PR #178 + # for course-filters: a typo in the request body (e.g. ``preserve_credential``) + # should surface as 422 instead of being silently dropped — silent drops on + # a destructive operation like redeploy are particularly easy to misdiagnose. + model_config = ConfigDict( + extra="forbid", + json_schema_extra={ + "example": { + "deployment_parameter_overrides": { + "include_example_notebooks": True, + }, + "instance_parameter_overrides": { + "instance-uuid-of-group-3": {"flavor": "gp1.medium"}, + }, + "preserve_credentials": False, + } + } + ) + + class DeploymentExtend(BaseModel): """Body for ``PATCH /deployments/{id}/extend``. diff --git a/src/tasks/deploy_tasks.py b/src/tasks/deploy_tasks.py index 7eda454..ac6ffd9 100644 --- a/src/tasks/deploy_tasks.py +++ b/src/tasks/deploy_tasks.py @@ -1,12 +1,45 @@ -"""Deploy tasks for Celery.""" +"""Deploy tasks for Celery. + +This module owns three lifecycle tasks that the API enqueues: + + * :func:`deploy_stack` — initial provisioning. For each ``stack_assignment`` + in ``deployment_parameters`` it creates a Heat stack, waits for SSH, + copies scripts/files, runs the Ansible playbooks and persists credentials + + access rows. + * :func:`delete_deployment` — tears down every Heat stack the deployment + owns and removes the DB row (only when ALL stacks were torn down OK; on + partial failure the row stays so the user can retry). + * :func:`restart_deployment` — triggers a Heat ``update_stack`` to + refresh-in-place. Does NOT recreate or re-run Ansible. + * :func:`redeploy_instance` — destroy-and-recreate a *single* VM + (``DeploymentInstance``) inside an existing deployment, optionally with + overridden parameters. The parent deployment stays RUNNING so the + redeploy doesn't drag siblings down. + * :func:`redeploy_deployment` — same as redeploy_instance, but iterates + over every instance in the deployment. Used when the lecturer wants to + apply a config change across the whole class without re-running the + wizard. + +The single-stack provisioning loop body lives in +:func:`_provision_one_stack_assignment` so both ``deploy_stack`` and +``redeploy_instance`` go through the same code path. That keeps +"credentials → heat → ansible → persist" identical for first-time deploys +and redeploys — drift here would be very expensive to debug. +""" import json import logging import time +from pathlib import Path +from typing import Any, Callable, Optional from uuid import UUID +import yaml as _yaml + from src.celery_app import celery_app from src.core.database import SessionLocal from src.models.deployment import DeploymentStatus +from src.models.deployment_instance import DeploymentInstance, DeploymentInstanceStatus +from src.models.deployment_instance_access import DeploymentInstanceAccess from src.models.deployment_log import DeploymentLogLevel, DeploymentLogEventType from src.models.template_version_file import FileType from src.repositories.deployment_repository import DeploymentRepository @@ -24,6 +57,561 @@ logger = logging.getLogger(__name__) +# Heat parameters the backend ALWAYS owns — must never come from the +# wizard / override dicts. Kept as a module constant so both the initial +# deploy path and the redeploy path strip them out identically. +_BACKEND_MANAGED_HEAT_PARAMS = frozenset({"user_json", "key_name"}) + +# Bookkeeping keys present in ``generated`` but not actual credential +# types — must not leak into the ``applications`` section of user_json. +_NON_APP_KEYS = frozenset({ + "username", "email", "group_name", "group_index", + "course_group_id", "students", "linux", +}) + + +# --------------------------------------------------------------------------- +# Shared template / parameter loading helpers +# --------------------------------------------------------------------------- + + +class _TemplateContext: + """Bundle of everything loaded once per deploy/redeploy task. + + Holds the template files, the parsed credentials spec, the playbooks + sorted in the right order, and the parameter split (Heat vs Ansible) + derived from the heat template's declared parameters. + + Centralising this keeps :func:`deploy_stack` and the redeploy paths in + sync — when a new file type or parameter rule is added it lives here + and both callers pick it up. + """ + + def __init__( + self, + *, + heat_template: str, + files_dict: dict[str, str], + playbooks: list[tuple[str, str]], + scripts: dict[str, str], + template_files: dict[str, str], + credentials_spec: dict[str, list], + heat_defined_params: set[str], + ) -> None: + self.heat_template = heat_template + self.files_dict = files_dict + self.playbooks = playbooks + self.scripts = scripts + self.template_files = template_files + self.credentials_spec = credentials_spec + self.heat_defined_params = heat_defined_params + + def split_parameters( + self, all_parameters: dict[str, Any] + ) -> tuple[dict[str, Any], dict[str, Any]]: + """Split a merged parameter dict into (heat, ansible) by what the + heat template declares. Backend-managed Heat params are stripped so + a malicious override can't bypass ``key_name`` / ``user_json``. + """ + heat_params = { + k: v for k, v in all_parameters.items() + if k in self.heat_defined_params and k not in _BACKEND_MANAGED_HEAT_PARAMS + } + ansible_params = { + k: v for k, v in all_parameters.items() + if k not in self.heat_defined_params + } + return heat_params, ansible_params + + +def _load_template_context( + file_service: TemplateVersionFileService, + template_version_id: str, +) -> _TemplateContext | str: + """Build a :class:`_TemplateContext` for the given template version. + + Returns the context on success, or a string error message on failure + (caller is expected to forward that into ``_fail``). Implemented as a + return-or-error pattern instead of raising so the call sites can stay + aligned with the original loop's ``_fail`` shape. + """ + files = file_service.get_version_files( + template_version_id, include_content=True, skip_access_check=True + ) + if not files: + return f"No files found for template version {template_version_id}" + + heat_file = next((f for f in files if f.is_primary), None) + if not heat_file or not heat_file.content: + return "No primary Heat template found" + + try: + heat_template_parsed = _yaml.safe_load(heat_file.content) + heat_defined_params = set(heat_template_parsed.get("parameters", {}).keys()) + except Exception: + heat_defined_params = set() + + # Parse app.yaml for credentials spec. + app_yaml_file = next((f for f in files if f.file_name == "app.yaml"), None) + credentials_spec: dict[str, list] = {"per_group": [], "teacher": []} + if app_yaml_file and app_yaml_file.content: + manifest = AppManifestParser.parse(app_yaml_file.content) + credentials_spec = manifest.get("credentials", credentials_spec) + + # _common playbooks first (sorted), then template-specific. + playbooks: list[tuple[str, str]] = [] + common_playbooks_dir = Path(__file__).parent.parent / "_common" / "playbooks" + if common_playbooks_dir.exists(): + for common_file in sorted(common_playbooks_dir.glob("*.yml")): + playbooks.append((f"_common/{common_file.name}", common_file.read_text())) + + scripts: dict[str, str] = {} + template_files: dict[str, str] = {} + for f in sorted(files, key=lambda x: (x.order, x.file_name)): + if f.file_type == FileType.ANSIBLE_PLAYBOOK and f.content: + playbooks.append((f.file_name, f.content)) + elif f.file_type == FileType.SHELL_SCRIPT and f.content: + scripts[f.file_name] = f.content + elif f.file_type == FileType.CONFIG_FILE and f.content: + template_files[f.file_name] = f.content + + files_dict: dict[str, str] = {} + for f in files: + if f.file_type == FileType.CLOUD_INIT and f.content: + files_dict["../cloud-init/user-data.yaml"] = f.content + + return _TemplateContext( + heat_template=heat_file.content, + files_dict=files_dict, + playbooks=playbooks, + scripts=scripts, + template_files=template_files, + credentials_spec=credentials_spec, + heat_defined_params=heat_defined_params, + ) + + +# --------------------------------------------------------------------------- +# The single-stack provisioning unit, shared by deploy + redeploy +# --------------------------------------------------------------------------- + + +def _provision_one_stack_assignment( + *, + db, + deployment, + stack_assignment_data: dict, + template_context: _TemplateContext, + heat_service: HeatStackService, + ansible_service_factory: Callable[..., AnsibleService], + log_service: DeploymentLogService, + all_parameters: dict[str, Any], + teacher_info: dict, + stack_name: str, + stack_index: int, + total_stacks: int, + cancel_check: Callable[[], bool] | None = None, + preserved_user_json: dict | None = None, +) -> tuple[Optional[str], Optional[DeploymentInstance]]: + """Run the create-stack → Ansible → persist-credentials cycle for one + stack assignment. + + Extracted verbatim from the original ``deploy_stack`` loop body. Both + the initial deploy and the per-instance redeploy go through this so + drift between the two paths stays impossible. + + Args: + db: SQLAlchemy session. + deployment: Deployment row (kept loaded throughout for tags / name). + stack_assignment_data: One element of + ``deployment_parameters['stack_assignments']``. + template_context: Output of :func:`_load_template_context`. + heat_service: A connected ``HeatStackService`` for the deployment's + OpenStack project. + ansible_service_factory: Callable that, given ``floating_ip`` and + ``cancel_check``, returns an :class:`AnsibleService` bound to + this db/deployment. Injected so tests can stub it out. + log_service: Active ``DeploymentLogService`` instance. + all_parameters: Effective (already-merged) parameter dict for THIS + stack. Callers compute the merge — this function only splits + it into Heat vs Ansible before use. + teacher_info: ``deployment_parameters['teacher']`` payload, used + to construct the credential admin block. + stack_name: Final Heat stack name (already collision-safe). + stack_index: 1-based position used only for log messages. + total_stacks: Total count, also log-only. + cancel_check: Optional predicate; when it returns True after Heat + success the function logs the cancel and returns ``(stack_id, + None)`` so the caller can short-circuit the rest of the loop. + ``None`` (the default) disables cancellation entirely — used + by the redeploy task, which is itself a discrete unit. + preserved_user_json: When set, skip credential GENERATION and reuse + this user_json verbatim. Used by ``redeploy_instance`` with + ``preserve_credentials=True`` so students keep their logins. + + Returns: + Tuple ``(stack_id, instance)``: + * ``stack_id`` is the newly-created Heat stack ID, or ``None`` if + creation failed before Heat reported back a usable ID. + * ``instance`` is the persisted ``DeploymentInstance`` row, or + ``None`` when persistence failed (the caller still gets the + stack id for incremental persistence / cleanup). + + On Heat / Ansible failure the function re-raises after logging, so the + caller's outer ``try/except`` can record the error and decide whether + to continue with the next assignment (deploy_stack) or fail the redeploy. + """ + from src.schemas.deployment import StackAssignment, TeacherInfo + + deployment_id = deployment.id + stack_assignment = StackAssignment(**stack_assignment_data) + teacher = TeacherInfo(**teacher_info) + + heat_parameters, ansible_parameters = template_context.split_parameters(all_parameters) + + # --- Generate credentials --- + # ``preserved_user_json``, when set, is NOT a substitute for the + # ``generated`` dict consumed by Heat / Ansible — it only carries access + # rows we'll rebind onto the new instance after persistence (see + # ``_rebind_preserved_access_rows``). Ansible still receives fresh creds + # so the playbook is free to template them into config; we then overwrite + # the DB-side access rows with the preserved set so students' logins keep + # working in the credentials API. Playbooks that aren't idempotent on + # passwords may still produce drift between what's on the VM and what's + # in the DB — preserve_credentials is a best-effort guarantee. + generated = CredentialGeneratorService.generate( + credentials_spec=template_context.credentials_spec, + stack_assignment=stack_assignment, + teacher=teacher, + ) + + stack_params = {**heat_parameters} + stack_params["key_name"] = get_settings().ansible_ssh_key_name + tags = { + "deployment_id": deployment_id, + "course_id": deployment.course_id, + "template_version_id": deployment.template_version_id, + "stack_index": str(stack_index), + } + + log_service.log( + deployment_id=deployment_id, + event_type=DeploymentLogEventType.DEPLOYMENT_STARTED, + message=f"Creating Heat stack {stack_index}/{total_stacks}: {stack_name}", + level=DeploymentLogLevel.INFO, + details={"stack_index": stack_index, "stack_name": stack_name}, + ) + + # --- 1. Create Heat stack --- + stack_result = heat_service.create_stack( + stack_name=stack_name, + template=template_context.heat_template, + parameters=stack_params, + files=template_context.files_dict or None, + tags=tags, + timeout_mins=60, + ) + stack_id: str = stack_result["stack_id"] + + log_service.log( + deployment_id=deployment_id, + event_type=DeploymentLogEventType.STACK_CREATE, + message=f"Heat stack {stack_index} created: {stack_name}", + level=DeploymentLogLevel.INFO, + details={"stack_id": stack_id, "stack_name": stack_name, "stack_index": stack_index}, + ) + + # --- 2. Build user_json + persist credentials --- + # Credential persistence is load-bearing: without a DeploymentInstance row + # the deployment has no DB record of the new VM, and the caller has no way + # to retry or reach it. Bubble the exception up so the caller can record + # the Heat-stack id as an orphan and report the failure to the user — the + # previous "log and continue to Ansible" path silently produced an orphan + # stack and a "redeployed" success response. + credentials_for_db = _build_user_json(generated) + try: + instance = DeploymentCredentialService(db).persist_credentials_for_stack( + deployment_id=deployment_id, + stack_name=stack_name, + openstack_stack_id=stack_id, + user_json=credentials_for_db, + floating_ip=stack_result.get("floating_ip") or "", + heat_outputs=stack_result.get("outputs") or {}, + flavor=stack_params.get("flavor"), + ) + except Exception as cred_error: + logger.error( + f"Failed to persist credentials for stack {stack_index}: {cred_error}", + exc_info=True, + ) + log_service.log( + deployment_id=deployment_id, + event_type=DeploymentLogEventType.FAILED, + message=f"Stack {stack_index} created but credential persistence failed", + level=DeploymentLogLevel.ERROR, + details={"stack_index": stack_index, "error": str(cred_error)}, + ) + # Stamp the heat stack id onto the exception so the outer task can + # record it as an orphan for later cleanup (same convention Heat's + # create_stack uses on CREATE_FAILED / timeout). + cred_error.stack_id = stack_id # type: ignore[attr-defined] + raise + + # If preserve_credentials is in play, replace the just-generated access + # rows with the snapshot from the OLD instance so logins keep working. + if preserved_user_json is not None: + try: + rebound = _rebind_preserved_access_rows(db, instance, preserved_user_json) + log_service.log( + deployment_id=deployment_id, + event_type=DeploymentLogEventType.DEPLOYMENT_STARTED, + message=( + f"preserve_credentials: rebound {rebound} access row(s) " + f"from snapshot for stack {stack_index}" + ), + level=DeploymentLogLevel.INFO, + details={"stack_index": stack_index, "rebound": rebound}, + ) + except Exception as rebind_err: + # Rebinding is best-effort: failing here would leave fresh + # auto-generated creds in place, which is recoverable. Log and + # carry on rather than dropping the whole redeploy. + logger.error( + f"Failed to rebind preserved credentials for stack {stack_index}: {rebind_err}", + exc_info=True, + ) + log_service.log( + deployment_id=deployment_id, + event_type=DeploymentLogEventType.FAILED, + message=( + f"preserve_credentials: rebind failed for stack {stack_index}; " + "fresh credentials remain in place" + ), + level=DeploymentLogLevel.WARNING, + details={"stack_index": stack_index, "error": str(rebind_err)}, + ) + + # --- 3. Ansible (only if playbooks exist and SSH key available) --- + ssh_private_key = get_settings().ansible_ssh_private_key or "" + playbooks = template_context.playbooks + if playbooks and ssh_private_key: + floating_ip = stack_result.get("floating_ip", "") + if not floating_ip: + log_service.log( + deployment_id=deployment_id, + event_type=DeploymentLogEventType.ANSIBLE_FAILED, + message=f"No floating_ip in stack result for stack {stack_index} — skipping Ansible", + level=DeploymentLogLevel.WARNING, + ) + else: + if cancel_check and cancel_check(): + # Caller's deploy_stack uses the per-iteration checkpoint to + # bail out cleanly; reuse the same shape here so the loop + # outside can decide what to do. + log_service.log( + deployment_id=deployment_id, + event_type=DeploymentLogEventType.DEPLOYMENT_DELETION_REQUESTED, + message=f"Cancel detected after Heat for stack {stack_index}; skipping Ansible", + level=DeploymentLogLevel.INFO, + ) + return stack_id, instance + + ansible = ansible_service_factory( + floating_ip=floating_ip, + cancel_check=cancel_check, + ) + try: + ansible.wait_for_ssh() + ansible.copy_files( + scripts=template_context.scripts, + files=template_context.template_files, + ) + + extra_vars = { + **generated, + **ansible_parameters, + "course_label": deployment.name, + "stack_label": stack_name, + "ssh_allow_users": [ + s["linux"]["username"] + for s in generated.get("deployment_groups", []) + if s.get("linux", {}).get("password") + ], + } + ansible.run_playbooks(playbooks=playbooks, extra_vars=extra_vars) + + _maybe_persist_activation_links( + db=db, + log_service=log_service, + deployment_id=deployment_id, + stack_index=stack_index, + ansible=ansible, + instance=instance, + generated=generated, + ) + except CancelledException: + # Let the caller's outer handler decide whether this is a + # cancel-success (deploy_stack) or a hard fail (redeploy). + raise + elif playbooks and not ssh_private_key: + log_service.log( + deployment_id=deployment_id, + event_type=DeploymentLogEventType.ANSIBLE_FAILED, + message="Playbooks defined but no SSH private key configured — skipping Ansible", + level=DeploymentLogLevel.WARNING, + ) + + return stack_id, instance + + +def _build_user_json(generated: dict) -> dict: + """Translate ``CredentialGeneratorService.generate`` output into the + two-section ``user_json`` consumed by ``persist_credentials_for_stack``. + + Two-section layout: + - ``instance.credentials`` / ``instance.admin_credentials``: SSH + (Linux) credentials. Only emitted for templates whose app.yaml + declares ``per_group.linux``; teacher always has an auto-generated + linux block (admin key), so the admin SSH row is written for every + template. + - ``applications[]``: every NON-linux credential type declared in + app.yaml (postgres, pgadmin, web_url, …). One ``applications`` + entry per credential type, with a ``credentials`` list for the + groups and an ``admin_credentials`` block for the teacher. + + Extracted verbatim from the inlined logic in ``deploy_stack`` so the + initial-deploy and redeploy paths produce identical user_json shapes. + """ + group_entries = generated.get("deployment_groups", []) or [] + teacher_entry = generated.get("teacher", {}) or {} + + ssh_credentials = [ + { + "username": s["linux"]["username"], + "password": s["linux"]["password"], + "ssh_private_key": (s.get("linux", {}).get("ssh_key") or {}).get("private_key"), + "group_id": s.get("course_group_id"), + } + for s in group_entries + if s.get("linux", {}).get("password") + ] + ssh_admin = ( + { + "username": teacher_entry["linux"]["username"], + "password": teacher_entry["linux"]["password"], + "ssh_private_key": (teacher_entry["linux"].get("ssh_key") or {}).get("private_key"), + } + if teacher_entry.get("linux", {}).get("password") + else None + ) + + # App-credentials section — collected per credential type by union of + # keys across all group entries and the teacher entry (minus the + # bookkeeping keys and ``linux``, which has its own SSH section). + app_cred_types: list[str] = [] + for source in (*group_entries, teacher_entry): + for key in source.keys(): + if key in _NON_APP_KEYS or key in app_cred_types: + continue + app_cred_types.append(key) + + applications = [] + for cred_type in app_cred_types: + group_creds = [] + for s in group_entries: + cred = s.get(cred_type) + if not isinstance(cred, dict) or not cred.get("password"): + continue + group_creds.append({ + **cred, + "group_id": s.get("course_group_id"), + }) + admin_cred = teacher_entry.get(cred_type) + admin_block = ( + admin_cred + if isinstance(admin_cred, dict) and admin_cred.get("password") + else None + ) + if not group_creds and not admin_block: + continue + applications.append({ + "name": cred_type, + "credentials": group_creds, + "admin_credentials": admin_block, + }) + + return { + "instance": { + "credentials": ssh_credentials, + "admin_credentials": ssh_admin, + }, + "applications": applications, + } + + +def _maybe_persist_activation_links( + *, + db, + log_service: DeploymentLogService, + deployment_id: str, + stack_index: int, + ansible: AnsibleService, + instance: Optional[DeploymentInstance], + generated: dict, +) -> None: + """Post-Ansible fetch of /opt/dozilab/OVERLEAF_USERS.json and persistence + of activation-link rows. No-op when the file is absent or instance + persistence failed earlier. + + Same hardcoded path as before; the call is wrapped in a broad + try/except so a single failed fetch never escalates to a deployment + failure (the file still lives on the VM for manual recovery). + """ + try: + users_json = ansible.fetch_remote_json("/opt/dozilab/OVERLEAF_USERS.json") + if users_json and instance is not None: + username_to_group_id: dict[str, str | None] = {} + for s in generated.get("deployment_groups", []): + course_group_id = s.get("course_group_id") + linux_username = s.get("linux", {}).get("username") + if linux_username: + username_to_group_id[linux_username] = course_group_id + top_username = s.get("username") + if top_username: + username_to_group_id.setdefault(top_username, course_group_id) + + written = DeploymentCredentialService(db).persist_activation_links( + instance_id=instance.id, + overleaf_users_json=users_json, + username_to_group_id=username_to_group_id, + ) + log_service.log( + deployment_id=deployment_id, + event_type=DeploymentLogEventType.ANSIBLE_COMPLETED, + message=f"Persisted {written} activation-link credential(s) for stack {stack_index}", + level=DeploymentLogLevel.INFO, + details={"stack_index": stack_index, "count": written}, + ) + except Exception as fetch_err: + logger.warning( + "Post-Ansible activation-link fetch failed for " + f"deployment {deployment_id} stack {stack_index}: {fetch_err}", + exc_info=True, + ) + log_service.log( + deployment_id=deployment_id, + event_type=DeploymentLogEventType.ANSIBLE_COMPLETED, + message=f"Stack {stack_index} done; activation-link fetch skipped: {fetch_err}", + level=DeploymentLogLevel.WARNING, + details={"stack_index": stack_index, "error": str(fetch_err)}, + ) + + +# --------------------------------------------------------------------------- +# Tasks +# --------------------------------------------------------------------------- + + @celery_app.task(bind=True) def deploy_stack(self, deployment_id: str) -> dict: """Deploy Heat stacks and run Ansible playbooks for a deployment. @@ -72,61 +660,11 @@ def deploy_stack(self, deployment_id: str) -> dict: if not stack_assignments_raw: return _fail(repo, log_service, deployment_id, "No stack_assignments found") - # --- Load template files from DB --- - files = file_service.get_version_files( - deployment.template_version_id, include_content=True, skip_access_check=True - ) - if not files: - return _fail(repo, log_service, deployment_id, f"No files found for template version {deployment.template_version_id}") - - heat_file = next((f for f in files if f.is_primary), None) - if not heat_file or not heat_file.content: - return _fail(repo, log_service, deployment_id, "No primary Heat template found") - - # Split parameters: Heat gets only what heat/main.yaml defines, Ansible gets the rest - import yaml as _yaml - try: - heat_template_parsed = _yaml.safe_load(heat_file.content) - heat_defined_params = set(heat_template_parsed.get("parameters", {}).keys()) - except Exception: - heat_defined_params = set() - - backend_managed = {"user_json", "key_name"} - base_heat_parameters = { - k: v for k, v in all_parameters.items() - if k in heat_defined_params and k not in backend_managed - } - ansible_parameters = { - k: v for k, v in all_parameters.items() - if k not in heat_defined_params - } - - # Parse app.yaml for credentials spec and playbook list - app_yaml_file = next((f for f in files if f.file_name == "app.yaml"), None) - credentials_spec: dict[str, list] = {"per_group": [], "teacher": []} - playbooks: list[tuple[str, str]] = [] - scripts: dict[str, str] = {} - template_files: dict[str, str] = {} - - if app_yaml_file and app_yaml_file.content: - manifest = AppManifestParser.parse(app_yaml_file.content) - credentials_spec = manifest.get("credentials", credentials_spec) - - # Load _common playbooks first (sorted by filename → 00_, 01_, ...) - from pathlib import Path - common_playbooks_dir = Path(__file__).parent.parent / "_common" / "playbooks" - if common_playbooks_dir.exists(): - for common_file in sorted(common_playbooks_dir.glob("*.yml")): - playbooks.append((f"_common/{common_file.name}", common_file.read_text())) - - # Then template-specific playbooks - for f in sorted(files, key=lambda x: (x.order, x.file_name)): - if f.file_type == FileType.ANSIBLE_PLAYBOOK and f.content: - playbooks.append((f.file_name, f.content)) - elif f.file_type == FileType.SHELL_SCRIPT and f.content: - scripts[f.file_name] = f.content - elif f.file_type == FileType.CONFIG_FILE and f.content: - template_files[f.file_name] = f.content + # --- Load template files / playbooks / etc. --- + ctx_or_error = _load_template_context(file_service, deployment.template_version_id) + if isinstance(ctx_or_error, str): + return _fail(repo, log_service, deployment_id, ctx_or_error) + template_context = ctx_or_error # --- Get OpenStack credentials --- # The OpenStack project this deployment runs against is now persisted @@ -150,18 +688,16 @@ def deploy_stack(self, deployment_id: str) -> dict: except Exception as kp_err: return _fail(repo, log_service, deployment_id, f"Failed to ensure Ansible keypair: {kp_err}") - # SSH private key from settings (shared backend key, not per-project) ssh_private_key = get_settings().ansible_ssh_private_key or "" - # cloud-init files dict for Heat - files_dict = {} - for f in files: - if f.file_type == FileType.CLOUD_INIT and f.content: - files_dict["../cloud-init/user-data.yaml"] = f.content - - # Reconstruct Pydantic objects - from src.schemas.deployment import StackAssignment, TeacherInfo - teacher = TeacherInfo(**teacher_info) + def _ansible_factory(*, floating_ip: str, cancel_check): + return AnsibleService( + db=db, + deployment_id=deployment_id, + floating_ip=floating_ip, + ssh_private_key=ssh_private_key, + cancel_check=cancel_check, + ) created_stack_ids: list[str] = [] failed_stacks: list[dict] = [] @@ -185,327 +721,60 @@ def deploy_stack(self, deployment_id: str) -> dict: ) return {"status": "cancelled", "stack_count": len(created_stack_ids), "stack_ids": created_stack_ids} - stack_id = None # set after Heat succeeds + stack_id: Optional[str] = None try: - stack_assignment = StackAssignment(**stack_assignment_data) - - # --- Generate credentials from app.yaml spec --- - generated = CredentialGeneratorService.generate( - credentials_spec=credentials_spec, - stack_assignment=stack_assignment, - teacher=teacher, - ) - - stack_params = {**base_heat_parameters} - stack_params["key_name"] = get_settings().ansible_ssh_key_name - stack_name = f"{deployment.name}-s{idx}-{deployment_id[:4]}" - stack_name = stack_name.replace(" ", "-").replace("_", "-").lower()[:64] - tags = { - "deployment_id": deployment_id, - "course_id": deployment.course_id, - "template_version_id": deployment.template_version_id, - "stack_index": str(idx), - } - - log_service.log( - deployment_id=deployment_id, - event_type=DeploymentLogEventType.DEPLOYMENT_STARTED, - message=f"Creating Heat stack {idx}/{len(stack_assignments_raw)}: {stack_name}", - level=DeploymentLogLevel.INFO, - details={"stack_index": idx, "stack_name": stack_name}, - ) - - # --- 1. Create Heat stack --- - stack_result = heat_service.create_stack( + stack_name = _build_stack_name(deployment, idx) + cancel_check = lambda: is_cancel_requested(db, deployment_id) # noqa: E731 + + stack_id, _instance = _provision_one_stack_assignment( + db=db, + deployment=deployment, + stack_assignment_data=stack_assignment_data, + template_context=template_context, + heat_service=heat_service, + ansible_service_factory=_ansible_factory, + log_service=log_service, + all_parameters=all_parameters, + teacher_info=teacher_info, stack_name=stack_name, - template=heat_file.content, - parameters=stack_params, - files=files_dict or None, - tags=tags, - timeout_mins=60, + stack_index=idx, + total_stacks=len(stack_assignments_raw), + cancel_check=cancel_check, ) - stack_id = stack_result["stack_id"] - created_stack_ids.append(stack_id) - - # Persist the new stack ID incrementally so a parallel cancel - # (DELETE request → delete_deployment task) can find every - # Heat stack we've created so far. Without this, the cleanup - # would miss stacks created in later loop iterations because - # `openstack_stack_id` is otherwise only flushed at the very - # end of the task. - try: - deployment.openstack_stack_id = json.dumps(created_stack_ids) - db.commit() - except Exception as persist_err: - db.rollback() - logger.warning( - f"Failed to incrementally persist stack id {stack_id}: {persist_err}" - ) + if stack_id: + created_stack_ids.append(stack_id) + + # Persist the new stack ID incrementally so a parallel cancel + # (DELETE request → delete_deployment task) can find every + # Heat stack we've created so far. Without this, the cleanup + # would miss stacks created in later loop iterations because + # `openstack_stack_id` is otherwise only flushed at the very + # end of the task. + try: + deployment.openstack_stack_id = json.dumps(created_stack_ids) + db.commit() + except Exception as persist_err: + db.rollback() + logger.warning( + f"Failed to incrementally persist stack id {stack_id}: {persist_err}" + ) + # If cancel was observed mid-provision, the helper returned a + # stack_id with no further work — fold the cancel up so the + # caller doesn't keep iterating. + if cancel_check(): + return {"status": "cancelled", "stack_count": len(created_stack_ids), "stack_ids": created_stack_ids} + except CancelledException as cancel_err: + # Cancel was observed mid-SSH-wait or mid-playbook. + # Don't escalate to FAILED — log and exit cleanly. log_service.log( deployment_id=deployment_id, - event_type=DeploymentLogEventType.STACK_CREATE, - message=f"Heat stack {idx} created: {stack_name}", + event_type=DeploymentLogEventType.DEPLOYMENT_DELETION_REQUESTED, + message=f"Ansible phase cancelled for stack {idx}: {cancel_err}", level=DeploymentLogLevel.INFO, - details={"stack_id": stack_id, "stack_name": stack_name, "stack_index": idx}, + details={"created_stack_ids": created_stack_ids}, ) - - try: - # Build user_json from `generated` so DB passwords match what Ansible sets. - # - # Two-section layout: - # - ``instance.credentials`` / ``instance.admin_credentials``: - # SSH (Linux) credentials. Only emitted for templates whose - # app.yaml declares ``per_group.linux``; teacher always has - # an auto-generated linux block (admin key), so the admin - # SSH row is written for every template. - # - ``applications[]``: every NON-linux credential type - # declared in app.yaml (postgres, pgadmin, web_url, …). - # One ``applications`` entry per credential type, with - # a ``credentials`` list for the groups and an - # ``admin_credentials`` block for the teacher. Without - # this, templates like ansible-postgres-group-db that - # declare only ``per_group.postgres`` + ``per_group.pgadmin`` - # would produce zero student-visible access rows. - NON_APP_KEYS = { - "username", "email", "group_name", "group_index", - "course_group_id", "students", "linux", - } - - group_entries = generated.get("deployment_groups", []) or [] - teacher_entry = generated.get("teacher", {}) or {} - - # SSH rows — same shape as before. Filter on linux.password - # is preserved: templates without per_group.linux simply - # don't get SSH access rows for their groups, which is - # correct (those students log into the app, not the VM). - ssh_credentials = [ - { - "username": s["linux"]["username"], - "password": s["linux"]["password"], - "ssh_private_key": (s.get("linux", {}).get("ssh_key") or {}).get("private_key"), - # course_groups.id this group corresponds to - # (passed in from the wizard via GroupInfo.course_group_id). - # Stamped onto DeploymentInstanceAccess.group_id so - # student self-service can filter on it. None when - # the wizard didn't supply it — row stays NULL and - # remains invisible to students. - "group_id": s.get("course_group_id"), - } - for s in group_entries - if s.get("linux", {}).get("password") - ] - ssh_admin = ( - { - "username": teacher_entry["linux"]["username"], - "password": teacher_entry["linux"]["password"], - "ssh_private_key": (teacher_entry["linux"].get("ssh_key") or {}).get("private_key"), - } - if teacher_entry.get("linux", {}).get("password") - else None - ) - - # App-credentials section — collected per credential type - # by union of keys across all group entries and the teacher - # entry (minus the bookkeeping keys above and ``linux``, - # which has its own SSH section). - app_cred_types: list[str] = [] - for source in (*group_entries, teacher_entry): - for key in source.keys(): - if key in NON_APP_KEYS or key in app_cred_types: - continue - app_cred_types.append(key) - - applications = [] - for cred_type in app_cred_types: - group_creds = [] - for s in group_entries: - cred = s.get(cred_type) - if not isinstance(cred, dict) or not cred.get("password"): - continue - group_creds.append({ - **cred, - "group_id": s.get("course_group_id"), - }) - admin_cred = teacher_entry.get(cred_type) - admin_block = ( - admin_cred - if isinstance(admin_cred, dict) and admin_cred.get("password") - else None - ) - if not group_creds and not admin_block: - continue - applications.append({ - "name": cred_type, - "credentials": group_creds, - "admin_credentials": admin_block, - }) - - credentials_for_db = { - "instance": { - "credentials": ssh_credentials, - "admin_credentials": ssh_admin, - }, - "applications": applications, - } - # Bind the returned DeploymentInstance so the post-Ansible - # activation-link fetch below can append rows to it. None - # if persistence failed (see except clause); the post- - # Ansible block guards against that. - instance = DeploymentCredentialService(db).persist_credentials_for_stack( - deployment_id=deployment_id, - stack_name=stack_name, - openstack_stack_id=stack_id, - user_json=credentials_for_db, - floating_ip=stack_result.get("floating_ip") or "", - heat_outputs=stack_result.get("outputs") or {}, - flavor=stack_params.get("flavor"), - ) - except Exception as cred_error: - instance = None - logger.error(f"Failed to persist credentials for stack {idx}: {cred_error}", exc_info=True) - log_service.log( - deployment_id=deployment_id, - event_type=DeploymentLogEventType.FAILED, - message=f"Stack {idx} created but credential persistence failed", - level=DeploymentLogLevel.ERROR, - details={"stack_index": idx, "error": str(cred_error)} - ) - - # --- 2+3+4. Ansible (only if playbooks exist and SSH key available) --- - if playbooks and ssh_private_key: - floating_ip = stack_result.get("floating_ip", "") - if not floating_ip: - log_service.log( - deployment_id=deployment_id, - event_type=DeploymentLogEventType.ANSIBLE_FAILED, - message=f"No floating_ip in stack result for stack {idx} — skipping Ansible", - level=DeploymentLogLevel.WARNING, - ) - else: - # Cooperative cancellation checkpoint #2: between Heat - # success and the (long-running) Ansible phase. The - # AnsibleService also gets the same predicate so it - # can bail mid-poll / mid-playbook. - cancel_check = lambda: is_cancel_requested(db, deployment_id) # noqa: E731 - if cancel_check(): - log_service.log( - deployment_id=deployment_id, - event_type=DeploymentLogEventType.DEPLOYMENT_DELETION_REQUESTED, - message=f"Cancel detected after Heat for stack {idx}; skipping Ansible", - level=DeploymentLogLevel.INFO, - details={"created_stack_ids": created_stack_ids}, - ) - return {"status": "cancelled", "stack_count": len(created_stack_ids), "stack_ids": created_stack_ids} - - ansible = AnsibleService( - db=db, - deployment_id=deployment_id, - floating_ip=floating_ip, - ssh_private_key=ssh_private_key, - cancel_check=cancel_check, - ) - try: - ansible.wait_for_ssh() - ansible.copy_files(scripts=scripts, files=template_files) - - extra_vars = { - **generated, - **ansible_parameters, - "course_label": deployment.name, - "stack_label": stack_name, - "ssh_allow_users": [ - s["linux"]["username"] - for s in generated.get("deployment_groups", []) - if s.get("linux", {}).get("password") - ], - } - ansible.run_playbooks(playbooks=playbooks, extra_vars=extra_vars) - - # --- 5. Post-Ansible: collect any per-stack - # output JSON the playbook wrote into - # /opt/dozilab/. Currently only Overleaf - # uses this — it writes activation links - # there because the Overleaf CLI generates - # one-time setup URLs at provisioning time - # (we have no password/SSH key to persist - # pre-Ansible). Any future app may opt in - # by writing the same shape; the fetch is - # a no-op when the file is absent. - # TODO: when a second app needs this, replace - # the hardcoded path with a list declared - # in app.yaml (e.g. ``post_ansible_outputs``). - try: - users_json = ansible.fetch_remote_json( - "/opt/dozilab/OVERLEAF_USERS.json" - ) - if users_json and instance is not None: - # Map the playbook's per-group ``username`` - # (e.g. "gruppe01") to the matching - # course_groups.id stamped onto the - # generated deployment_groups entries. - username_to_group_id: dict[str, str | None] = {} - for s in generated.get("deployment_groups", []): - course_group_id = s.get("course_group_id") - linux_username = s.get("linux", {}).get("username") - if linux_username: - username_to_group_id[linux_username] = course_group_id - # Fallback for credential specs that - # set a top-level username but no - # linux block. - top_username = s.get("username") - if top_username: - username_to_group_id.setdefault( - top_username, course_group_id - ) - - written = DeploymentCredentialService(db).persist_activation_links( - instance_id=instance.id, - overleaf_users_json=users_json, - username_to_group_id=username_to_group_id, - ) - log_service.log( - deployment_id=deployment_id, - event_type=DeploymentLogEventType.ANSIBLE_COMPLETED, - message=f"Persisted {written} activation-link credential(s) for stack {idx}", - level=DeploymentLogLevel.INFO, - details={"stack_index": idx, "count": written}, - ) - except Exception as fetch_err: - # Never fail the deployment because of a - # post-fetch issue — the file still lives on - # the VM at /opt/dozilab/OVERLEAF_USERS.{json,txt} - # for manual recovery. - logger.warning( - "Post-Ansible activation-link fetch failed for " - f"deployment {deployment_id} stack {idx}: {fetch_err}", - exc_info=True, - ) - log_service.log( - deployment_id=deployment_id, - event_type=DeploymentLogEventType.ANSIBLE_COMPLETED, - message=f"Stack {idx} done; activation-link fetch skipped: {fetch_err}", - level=DeploymentLogLevel.WARNING, - details={"stack_index": idx, "error": str(fetch_err)}, - ) - except CancelledException as cancel_err: - # Cancel was observed mid-SSH-wait or mid-playbook. - # Don't escalate to FAILED — log and exit cleanly. - log_service.log( - deployment_id=deployment_id, - event_type=DeploymentLogEventType.DEPLOYMENT_DELETION_REQUESTED, - message=f"Ansible phase cancelled for stack {idx}: {cancel_err}", - level=DeploymentLogLevel.INFO, - details={"created_stack_ids": created_stack_ids}, - ) - return {"status": "cancelled", "stack_count": len(created_stack_ids), "stack_ids": created_stack_ids} - elif playbooks and not ssh_private_key: - log_service.log( - deployment_id=deployment_id, - event_type=DeploymentLogEventType.ANSIBLE_FAILED, - message="Playbooks defined but no SSH private key configured — skipping Ansible", - level=DeploymentLogLevel.WARNING, - ) + return {"status": "cancelled", "stack_count": len(created_stack_ids), "stack_ids": created_stack_ids} except Exception as stack_error: # If create_stack raised AFTER the stack was actually created # (CREATE_FAILED or wait timeout), it stamps the id onto the @@ -581,6 +850,17 @@ def deploy_stack(self, deployment_id: str) -> dict: db.close() +def _build_stack_name(deployment, stack_index: int) -> str: + """Build a Heat-safe stack name from the deployment name + index. + + Pulled out so the redeploy path produces the same naming scheme as the + initial deploy. Heat is allergic to spaces, underscores and >64-char + names, so the same sanitisation rule must apply everywhere. + """ + stack_name = f"{deployment.name}-s{stack_index}-{deployment.id[:4]}" + return stack_name.replace(" ", "-").replace("_", "-").lower()[:64] + + def _fail(repo, log_service, deployment_id: str, error_msg: str) -> dict: """Log error, set status to FAILED and return failure dict.""" logger.error(error_msg) @@ -596,7 +876,6 @@ def _fail(repo, log_service, deployment_id: str, error_msg: str) -> dict: - def _gc_orphan_student_memberships(db, user_ids: set[str]) -> None: """Drop CourseMember / GroupMember / User rows for users that no longer have any deployment behind them. @@ -945,7 +1224,7 @@ def delete_deployment(self, deployment_id: str) -> dict: except Exception as e: logger.warning(f"Failed to delete deployment instances for {deployment_id}: {e}") db.rollback() - + # Finally delete DB record try: deleted = repo.delete(UUID(deployment_id)) @@ -987,33 +1266,33 @@ def delete_deployment(self, deployment_id: str) -> dict: @celery_app.task(bind=True) def restart_deployment(self, deployment_id: str) -> dict: """Restart a deployment by updating the Heat stack. - + This task restarts an existing deployment by triggering a Heat stack update, which can restart VMs or refresh the stack configuration. - + Args: deployment_id: ID of the deployment to restart - + Returns: Result dictionary with status and task information """ task_id = self.request.id logger.info(f"Starting restart task for deployment_id={deployment_id}, task_id={task_id}") - + db = SessionLocal() try: repo = DeploymentRepository(db) log_service = DeploymentLogService(db) - + deployment = repo.get_by_id(deployment_id) - + if not deployment: logger.error(f"Deployment not found: {deployment_id}") return {"status": "failed", "error": "Deployment not found"} - + # Update status to RESTARTING repo.update_status(deployment_id, DeploymentStatus.RESTARTING) - + log_service.log( deployment_id=deployment_id, event_type=DeploymentLogEventType.DEPLOYMENT_STARTED, @@ -1021,7 +1300,7 @@ def restart_deployment(self, deployment_id: str) -> dict: level=DeploymentLogLevel.INFO, details={"task_id": task_id} ) - + # Verify deployment has a stack if not deployment.openstack_stack_id: error_msg = "No OpenStack stack associated with this deployment" @@ -1034,7 +1313,7 @@ def restart_deployment(self, deployment_id: str) -> dict: ) repo.update_status(deployment_id, DeploymentStatus.FAILED) return {"status": "failed", "error": error_msg} - + # Get OpenStack project from the deployment's persisted FK. Previously # this was re-derived from teacher.id at every read site and picked the # user's first OpenstackProject row, which broke whenever a user had @@ -1052,14 +1331,14 @@ def restart_deployment(self, deployment_id: str) -> dict: ) repo.update_status(deployment_id, DeploymentStatus.FAILED) return {"status": "failed", "error": error_msg} - + try: heat_service = HeatStackService(openstack_project) - + # Trigger stack update to restart resources logger.info(f"Updating Heat stack {deployment.openstack_stack_id} to trigger restart") heat_service.update_stack(deployment.openstack_stack_id) - + log_service.log( deployment_id=deployment_id, event_type=DeploymentLogEventType.DEPLOYMENT_STARTED, @@ -1067,24 +1346,24 @@ def restart_deployment(self, deployment_id: str) -> dict: level=DeploymentLogLevel.INFO, details={"stack_id": deployment.openstack_stack_id} ) - + # Update status back to RUNNING (stack update is async in OpenStack) repo.update_status(deployment_id, DeploymentStatus.RUNNING) - + log_service.log( deployment_id=deployment_id, event_type=DeploymentLogEventType.DEPLOYMENT_STARTED, message="Restart completed successfully", level=DeploymentLogLevel.INFO ) - + return { "status": "restarted", "deployment_id": deployment_id, "stack_id": deployment.openstack_stack_id, "task_id": task_id } - + except Exception as e: error_msg = f"Failed to restart deployment: {str(e)}" logger.exception(error_msg) @@ -1097,9 +1376,876 @@ def restart_deployment(self, deployment_id: str) -> dict: ) repo.update_status(deployment_id, DeploymentStatus.FAILED) return {"status": "failed", "deployment_id": deployment_id, "error": str(e)} - + except Exception as e: logger.exception(f"Error during restart task for deployment {deployment_id}: {e}") return {"status": "failed", "deployment_id": deployment_id, "error": str(e)} finally: db.close() + + +# --------------------------------------------------------------------------- +# Redeploy paths +# --------------------------------------------------------------------------- + + +def _merge_parameter_layers( + *, + base: dict[str, Any], + deployment_overrides: dict[str, Any] | None, + instance_overrides: dict[str, Any] | None, +) -> dict[str, Any]: + """Merge parameter layers for a redeploy. + + Order — later layers win:: + + base (deployment.deployment_parameters['parameters']) + ↓ + deployment_overrides (apply to every VM) + ↓ + instance_overrides (apply to THIS one VM) + + Shallow merge by design — template parameters are flat key/value + pairs (per the app.yaml contract). Nested dicts in a parameter value + are replaced, not deep-merged, mirroring how the wizard treats them. + """ + merged = dict(base) + if deployment_overrides: + merged.update(deployment_overrides) + if instance_overrides: + merged.update(instance_overrides) + return merged + + +def _snapshot_instance_credentials( + db, instance: DeploymentInstance +) -> dict[str, Any]: + """Read ``DeploymentInstanceAccess`` rows back into a ``generated``-shaped + dict so the redeploy task can reuse them when ``preserve_credentials=True``. + + Round-tripping the rows back into the ``generated`` shape is intrinsically + lossy — ``_build_user_json``'s ``applications[]`` block was originally + keyed by app.yaml credential type (``postgres``, ``pgadmin``, …), but + once persisted it lives under a single ``DATABASE`` ``AccessType`` row + that merges them. We can't recover the original app name. The snapshot + therefore carries the access rows verbatim under a single ``_preserved_access`` + list per group; ``_rebind_preserved_access_rows`` (called AFTER the new + instance is persisted) then re-points the old rows at the new instance, + keeping the port / connection_url / ssh_private_key fields intact. + + Why not regenerate via the ``generated`` dict + Ansible: + * Server never persists ``ssh_key.public_key`` — re-emitting an empty + public_key would break authorized_keys on the new VM. + * Non-SSH rows lose port / connection_url under the access-type roundtrip. + * Ansible playbooks generate their own per-VM passwords; injecting the + OLD password into extra_vars doesn't actually keep students' logins + working unless the playbook is idempotent about it. + + Returns a dict with two keys (both consumed only inside this module): + * ``_preserved_access``: list of dicts holding every column of each + DeploymentInstanceAccess row, ready for the rebind step. + * ``deployment_groups`` / ``teacher``: kept empty — credentials are + rebound from access rows, NOT pumped through user_json. + """ + # Capture all columns we need to rehydrate the access row against the new + # instance. Fernet-encrypted columns (password, ssh_private_key) are + # decrypted transparently on read via EncryptedString; we re-encrypt on + # the new INSERT. + preserved: list[dict[str, Any]] = [] + for access in list(instance.access_methods or []): + preserved.append({ + "access_type": access.access_type, + "group_id": access.group_id, + "connection_url": access.connection_url, + "username": access.username, + "password": access.password, + "ssh_private_key": access.ssh_private_key, + "port": access.port, + "is_active": access.is_active, + "expires_at": access.expires_at, + }) + + return { + "_preserved_access": preserved, + "deployment_groups": [], + "teacher": {}, + } + + +def _rebind_preserved_access_rows( + db, + new_instance: DeploymentInstance, + preserved_user_json: dict[str, Any], +) -> int: + """Recreate access rows from a snapshot under the new instance's id. + + Companion to :func:`_snapshot_instance_credentials`. ``persist_credentials_for_stack`` + already wrote freshly-generated access rows for the new instance — we drop + those and replace with the preserved set so port / connection_url / SSH + private key persist verbatim across the redeploy. + + Returns the number of preserved rows written. + """ + preserved = (preserved_user_json or {}).get("_preserved_access") or [] + if not preserved: + return 0 + + # Drop the just-generated access rows for the new instance — they're + # superseded by the snapshot. + db.query(DeploymentInstanceAccess).filter( + DeploymentInstanceAccess.deployment_instance_id == new_instance.id + ).delete(synchronize_session=False) + + written = 0 + for row in preserved: + db.add( + DeploymentInstanceAccess( + deployment_instance_id=new_instance.id, + access_type=row["access_type"], + group_id=row.get("group_id"), + connection_url=row.get("connection_url"), + username=row.get("username"), + password=row.get("password"), + ssh_private_key=row.get("ssh_private_key"), + port=row.get("port"), + is_active=row.get("is_active", True), + expires_at=row.get("expires_at"), + ) + ) + written += 1 + db.commit() + return written + + +def _delete_single_instance_resources( + *, + db, + deployment, + instance: DeploymentInstance, + heat_service: HeatStackService, + log_service: DeploymentLogService, +) -> tuple[bool, list[str]]: + """Tear down the Heat stack for ONE instance and clean its DB rows. + + Mirrors the relevant slice of ``delete_deployment`` but for a single + instance, not the whole deployment. Returns ``(success, remaining_stack_ids)``: + * ``success`` is False when Heat refused to delete the stack — in + that case the row is intentionally NOT removed (so the user can + retry), and the parent deployment.openstack_stack_id keeps the + surviving id so a future cleanup can pick it up. + * ``remaining_stack_ids`` is the updated stack-id list for the + parent deployment (caller persists it). + """ + deployment_id = deployment.id + stack_id = instance.openstack_server_id # Heat stack id, same column + + # Parse the deployment's stack-id JSON array so we can rewrite it + # after the per-instance delete. + try: + stack_ids = json.loads(deployment.openstack_stack_id or "[]") + if not isinstance(stack_ids, list): + stack_ids = [stack_ids] + except (json.JSONDecodeError, TypeError): + stack_ids = [deployment.openstack_stack_id] if deployment.openstack_stack_id else [] + + if stack_id: + try: + heat_service.delete_stack(stack_id) + log_service.log( + deployment_id=deployment_id, + event_type=DeploymentLogEventType.DEPLOYMENT_DELETED, + message=f"Redeploy: Heat stack deleted: {stack_id}", + level=DeploymentLogLevel.INFO, + details={"stack_id": stack_id, "instance_id": instance.id}, + ) + except Exception as delete_error: + logger.error( + f"Redeploy: failed to delete stack {stack_id} for instance {instance.id}: {delete_error}", + exc_info=True, + ) + log_service.log( + deployment_id=deployment_id, + event_type=DeploymentLogEventType.FAILED, + message=f"Redeploy: failed to delete Heat stack {stack_id}: {delete_error}", + level=DeploymentLogLevel.ERROR, + details={"stack_id": stack_id, "instance_id": instance.id, "error": str(delete_error)}, + ) + return False, stack_ids + + # Pull the deleted id out of the deployment's stack-id list. + stack_ids = [sid for sid in stack_ids if sid != stack_id] + + # Wipe access rows + the instance row itself; the new instance will + # be persisted by the provisioning helper afterwards. + try: + db.query(DeploymentInstanceAccess).filter( + DeploymentInstanceAccess.deployment_instance_id == instance.id + ).delete(synchronize_session=False) + db.delete(instance) + db.flush() + except Exception as wipe_err: + logger.error( + f"Redeploy: failed to delete DB rows for instance {instance.id}: {wipe_err}", + exc_info=True, + ) + db.rollback() + return False, stack_ids + + return True, stack_ids + + +def _build_redeploy_stack_name(deployment, stack_index: int, instance_id: str) -> str: + """Like :func:`_build_stack_name` but with a per-redeploy suffix to avoid + colliding with the OLD stack still in DELETE_IN_PROGRESS on Heat. + + ``heat_service.delete_stack`` returns as soon as Heat ACK's the request; + the actual stack tear-down (VMs / volumes / floating-IPs) takes 30-120 s + on a busy region. If we tried to ``create_stack`` with the same name + immediately, Heat would reject it with "Stack already exists". We suffix + with ``-r<8-hex-of-old-instance-id>`` — deterministic per redeploy, + short enough not to blow the 64-char cap, and stable across retries of + the same task so an Ansible-only re-run doesn't accidentally fork + another stack. + + The new name is what gets persisted into ``DeploymentInstance.vm_name``, + so the original ``-s{idx}-`` slug stays inside the name and remains + parseable by :func:`_recover_stack_index_for_instance`. + """ + # Strip dashes so the suffix is compact and only hex chars survive (the + # regex used to recover stack_index allows [a-f0-9]+ after the index). + suffix = instance_id.replace("-", "")[:8].lower() + base = f"{deployment.name}-s{stack_index}-{deployment.id[:4]}-r{suffix}" + return base.replace(" ", "-").replace("_", "-").lower()[:64] + + +def _recover_stack_assignment_and_index( + instance: DeploymentInstance, + stack_assignments_raw: list[dict], + db=None, +) -> tuple[dict | None, int | None]: + """Recover the (stack_assignment, 1-based stack_index) tuple for an + instance. Single source of truth for the regex + fallback so the two + pieces of state can't drift. + + Resolution order: + 1. Regex on ``vm_name`` for the original ``-s-`` slug. This is + the cheap path; nothing else needs to happen when the name survived + unmodified. + 2. ``StackAssignment.stack_index`` — the wizard payload itself carries + the 1-based index for each assignment, so we can look up by it. + 3. Fallback: position of this instance among its deployment siblings + when sorted by ``created_at``. This matches the order in which + ``deploy_stack`` iterates ``stack_assignments`` (sequential, no + concurrency), so the Nth-created instance corresponds to the Nth + assignment. Only used when ``db`` is supplied. + + Returns ``(None, None)`` if all three paths fail. + """ + import re + + idx_one_based: int | None = None + if instance.vm_name: + match = re.search(r"-s(\d+)-[a-f0-9]+", instance.vm_name) + if match: + idx_one_based = int(match.group(1)) + + # If the regex worked, try to match the assignment by its stack_index field + # first (most reliable), then fall back to positional index. + if idx_one_based is not None: + for assignment in stack_assignments_raw: + if isinstance(assignment, dict) and assignment.get("stack_index") == idx_one_based: + return assignment, idx_one_based + if 0 <= idx_one_based - 1 < len(stack_assignments_raw): + return stack_assignments_raw[idx_one_based - 1], idx_one_based + # vm_name parsed but no matching assignment (e.g. empty list, or a + # malformed deployment_parameters). Hold onto the parsed index so the + # logging fallback (_stack_index_for_instance) is still useful, but + # signal "no assignment recovered" via the None on the first tuple slot. + return None, idx_one_based + + # Final fallback: positional match by created_at among siblings. Only + # works when we have a session and there's at least one assignment. + if db is not None and stack_assignments_raw: + siblings = ( + db.query(DeploymentInstance.id) + .filter(DeploymentInstance.deployment_id == instance.deployment_id) + .order_by(DeploymentInstance.created_at.asc()) + .all() + ) + ordered_ids = [row[0] for row in siblings] + try: + pos = ordered_ids.index(instance.id) + except ValueError: + pos = -1 + if 0 <= pos < len(stack_assignments_raw): + idx_one_based = pos + 1 + return stack_assignments_raw[pos], idx_one_based + + return None, None + + +def _reconstruct_stack_assignment_for_instance( + instance: DeploymentInstance, + stack_assignments_raw: list[dict], +) -> dict | None: + """Back-compat shim around :func:`_recover_stack_assignment_and_index`. + + Kept for the unit tests that exercise the vm_name parse in isolation + (``tests/unit/test_redeploy_tasks.py``). New callers should use + :func:`_recover_stack_assignment_and_index` so the index + assignment + come back together (no second regex pass). + """ + assignment, _ = _recover_stack_assignment_and_index(instance, stack_assignments_raw) + return assignment + + +def _stack_index_for_instance( + instance: DeploymentInstance, + stack_assignments_raw: list[dict], +) -> int: + """Back-compat shim around :func:`_recover_stack_assignment_and_index`. + + Returns the 1-based index recovered from vm_name, or 1 as a last-resort + fallback. New code in this module should call the combined helper. + """ + _, idx = _recover_stack_assignment_and_index(instance, stack_assignments_raw) + return idx if idx is not None else 1 + + +@celery_app.task(bind=True) +def redeploy_instance( + self, + deployment_id: str, + instance_id: str, + deployment_parameter_overrides: dict | None = None, + preserve_credentials: bool = False, +) -> dict: + """Destroy-and-recreate a single ``DeploymentInstance`` (= one VM). + + The parent deployment stays in ``RUNNING`` so its siblings remain + reachable. Only the targeted instance flips to ``REDEPLOYING`` for the + duration of the task. On success the old row is dropped and a fresh + one (new credentials by default) takes its place; on failure the row + flips to ``FAILED`` and the deployment.openstack_stack_id is rewritten + to reflect surviving stacks (so a future delete cleans up correctly). + + Heat naming is preserved (same ``-s{idx}-{dep4}`` slug) so the new + stack looks identical to a fresh deploy in the OpenStack tags. + + Args: + deployment_id: Parent deployment ID. + instance_id: ID of the ``DeploymentInstance`` row to recreate. + deployment_parameter_overrides: Optional overrides merged on top + of the deployment's stored parameters before splitting into + Heat / Ansible. + preserve_credentials: When True, reuse the existing access rows + on the new instance instead of regenerating. Default False. + """ + task_id = self.request.id + logger.info( + f"Starting redeploy_instance task for deployment_id={deployment_id} " + f"instance_id={instance_id} task_id={task_id}" + ) + + db = SessionLocal() + try: + repo = DeploymentRepository(db) + log_service = DeploymentLogService(db) + file_service = TemplateVersionFileService(db) + + deployment = repo.get_by_id(deployment_id) + if not deployment: + return {"status": "failed", "error": "Deployment not found"} + + instance = ( + db.query(DeploymentInstance) + .filter( + DeploymentInstance.id == instance_id, + DeploymentInstance.deployment_id == deployment_id, + ) + .first() + ) + if not instance: + return {"status": "failed", "error": f"Instance {instance_id} not found"} + + instance.status = DeploymentInstanceStatus.REDEPLOYING + db.commit() + + log_service.log( + deployment_id=deployment_id, + event_type=DeploymentLogEventType.DEPLOYMENT_STARTED, + message=( + f"Redeploying instance {instance.vm_name or instance_id} " + f"(task_id: {task_id}, preserve_credentials={preserve_credentials})" + ), + level=DeploymentLogLevel.INFO, + details={ + "task_id": task_id, + "instance_id": instance_id, + "preserve_credentials": preserve_credentials, + "overrides": deployment_parameter_overrides or {}, + }, + ) + + # --- Parse deployment parameters --- + if not deployment.deployment_parameters: + return _fail_instance(repo, log_service, db, deployment_id, instance, "No deployment_parameters found") + + try: + deployment_params = json.loads(deployment.deployment_parameters) + base_parameters = deployment_params.get("parameters", {}) + stack_assignments_raw = deployment_params.get("stack_assignments", []) + teacher_info = deployment_params.get("teacher", {}) + except json.JSONDecodeError as e: + return _fail_instance(repo, log_service, db, deployment_id, instance, f"Invalid deployment_parameters JSON: {e}") + + # Recover the stack assignment + index that produced THIS instance + # via the consolidated helper (regex first, positional fallback) so + # both pieces come from the same source — no risk of one returning + # None while the other defaults to 1. + stack_assignment_data, stack_idx = _recover_stack_assignment_and_index( + instance, stack_assignments_raw, db=db, + ) + if stack_assignment_data is None or stack_idx is None: + return _fail_instance( + repo, log_service, db, deployment_id, instance, + f"Cannot recover stack_assignment for instance {instance_id} (vm_name={instance.vm_name!r})", + ) + + # Load template once. + ctx_or_error = _load_template_context(file_service, deployment.template_version_id) + if isinstance(ctx_or_error, str): + return _fail_instance(repo, log_service, db, deployment_id, instance, ctx_or_error) + template_context = ctx_or_error + + # Optionally snapshot existing credentials before we wipe the access rows. + preserved_user_json = ( + _snapshot_instance_credentials(db, instance) if preserve_credentials else None + ) + + openstack_project = deployment.openstack_project + if not openstack_project: + return _fail_instance( + repo, log_service, db, deployment_id, instance, + f"Deployment {deployment_id} has no openstack_project_id set", + ) + + heat_service = HeatStackService(openstack_project) + + # --- 1. Delete the old Heat stack + DB rows for this instance --- + ok, remaining_stack_ids = _delete_single_instance_resources( + db=db, + deployment=deployment, + instance=instance, + heat_service=heat_service, + log_service=log_service, + ) + if not ok: + # _delete_single_instance_resources already logged; flip instance + # to FAILED (it may still be in DB if the wipe failed) and bail. + try: + # If the instance row is still around, mark it FAILED. + still_there = db.query(DeploymentInstance).filter( + DeploymentInstance.id == instance_id + ).first() + if still_there: + still_there.status = DeploymentInstanceStatus.FAILED + db.commit() + except Exception: + db.rollback() + return { + "status": "failed", + "deployment_id": deployment_id, + "instance_id": instance_id, + "error": "Failed to delete old Heat stack / DB rows", + } + + # Persist the rewritten stack-id list so a parallel delete sees it. + try: + deployment.openstack_stack_id = json.dumps(remaining_stack_ids) + db.commit() + except Exception as persist_err: + db.rollback() + logger.warning(f"Failed to persist remaining stack ids: {persist_err}") + + # --- 2. Recreate the stack with merged parameters --- + if get_settings().ansible_ssh_private_key: + try: + from src.services.ansible_keypair_service import AnsibleKeypairService + AnsibleKeypairService.ensure_keypair(openstack_project) + except Exception as kp_err: + # The old instance row is already gone; emit a placeholder + # FAILED row carrying the deployment + course_group_id so the + # UI still shows "1 of N VMs broken" rather than silently + # shrinking the class roster. + _record_redeploy_failure_placeholder( + db=db, + deployment_id=deployment_id, + old_instance_id=instance_id, + stack_index=stack_idx, + log_service=log_service, + error_msg=f"Failed to ensure Ansible keypair: {kp_err}", + ) + return { + "status": "failed", + "deployment_id": deployment_id, + "instance_id": instance_id, + "error": f"Failed to ensure Ansible keypair: {kp_err}", + } + + ssh_private_key = get_settings().ansible_ssh_private_key or "" + + def _ansible_factory(*, floating_ip: str, cancel_check): + return AnsibleService( + db=db, + deployment_id=deployment_id, + floating_ip=floating_ip, + ssh_private_key=ssh_private_key, + cancel_check=cancel_check, + ) + + merged_params = _merge_parameter_layers( + base=base_parameters, + deployment_overrides=deployment_parameter_overrides, + instance_overrides=None, + ) + # Use a redeploy-suffixed name to avoid colliding with the OLD stack + # that Heat may still be tearing down (delete_stack is async). + stack_name = _build_redeploy_stack_name(deployment, stack_idx, instance_id) + + try: + new_stack_id, new_instance = _provision_one_stack_assignment( + db=db, + deployment=deployment, + stack_assignment_data=stack_assignment_data, + template_context=template_context, + heat_service=heat_service, + ansible_service_factory=_ansible_factory, + log_service=log_service, + all_parameters=merged_params, + teacher_info=teacher_info, + stack_name=stack_name, + stack_index=stack_idx, + total_stacks=1, + cancel_check=None, + preserved_user_json=preserved_user_json, + ) + except Exception as e: + logger.exception(f"Redeploy of instance {instance_id} failed: {e}") + log_service.log( + deployment_id=deployment_id, + event_type=DeploymentLogEventType.FAILED, + message=f"Redeploy failed: {e}", + level=DeploymentLogLevel.ERROR, + details={"instance_id": instance_id, "error": str(e)}, + ) + # An orphan stack id may have been stamped onto the exception by + # the Heat service (CREATE_FAILED / timeout) or by the credential + # persist step (which keeps the stack id alive so the user can + # retry). Add it back to the deployment so a future delete cleans + # up rather than silently leaking the stack. + orphan = getattr(e, "stack_id", None) + if orphan: + remaining_stack_ids.append(orphan) + try: + deployment.openstack_stack_id = json.dumps(remaining_stack_ids) + db.commit() + except Exception: + db.rollback() + # Make sure the deployment still has a placeholder DeploymentInstance + # row visible — the old row is gone and (depending on where the + # exception hit) the new one may never have been persisted. + _record_redeploy_failure_placeholder( + db=db, + deployment_id=deployment_id, + old_instance_id=instance_id, + stack_index=stack_idx, + log_service=log_service, + error_msg=str(e), + ) + return { + "status": "failed", + "deployment_id": deployment_id, + "instance_id": instance_id, + "error": str(e), + } + + # Persist the new stack id back onto the parent deployment. + if new_stack_id: + remaining_stack_ids.append(new_stack_id) + try: + deployment.openstack_stack_id = json.dumps(remaining_stack_ids) + db.commit() + except Exception as persist_err: + db.rollback() + logger.warning( + f"Failed to persist redeployed stack id {new_stack_id}: {persist_err}" + ) + + log_service.log( + deployment_id=deployment_id, + event_type=DeploymentLogEventType.VM_READY, + message=f"Redeploy completed for instance {new_instance.id if new_instance else instance_id}", + level=DeploymentLogLevel.INFO, + details={ + "old_instance_id": instance_id, + "new_instance_id": new_instance.id if new_instance else None, + "new_stack_id": new_stack_id, + }, + ) + + # Normalise the parent deployment status — when a previously-FAILED + # deployment was recovered via per-instance redeploy, the row should + # come back to RUNNING. Other statuses (CREATING/DELETING/RESTARTING) + # were already gated out by the endpoint, so a redeploy never sees + # them mid-flight. + try: + if deployment.status != DeploymentStatus.RUNNING: + repo.update_status(deployment_id, DeploymentStatus.RUNNING) + except Exception as status_err: + logger.warning( + f"Could not normalise deployment status after redeploy_instance: {status_err}" + ) + + return { + "status": "redeployed", + "deployment_id": deployment_id, + "old_instance_id": instance_id, + "new_instance_id": new_instance.id if new_instance else None, + "new_stack_id": new_stack_id, + "task_id": task_id, + } + + except Exception as e: + logger.exception(f"Error during redeploy_instance task for {instance_id}: {e}") + return {"status": "failed", "deployment_id": deployment_id, "instance_id": instance_id, "error": str(e)} + finally: + db.close() + + +def _fail_instance( + repo: DeploymentRepository, + log_service: DeploymentLogService, + db, + deployment_id: str, + instance: DeploymentInstance | None, + error_msg: str, +) -> dict: + """Log + flip a single instance to FAILED without touching the + parent deployment's status (siblings should stay reachable).""" + logger.error(error_msg) + log_service.log( + deployment_id=deployment_id, + event_type=DeploymentLogEventType.FAILED, + message=error_msg, + level=DeploymentLogLevel.ERROR, + details={"instance_id": instance.id if instance else None}, + ) + if instance is not None: + try: + instance.status = DeploymentInstanceStatus.FAILED + db.commit() + except Exception: + db.rollback() + return {"status": "failed", "error": error_msg, "deployment_id": deployment_id} + + +def _record_redeploy_failure_placeholder( + *, + db, + deployment_id: str, + old_instance_id: str, + stack_index: int, + log_service: DeploymentLogService, + error_msg: str, +) -> None: + """Insert a placeholder ``DeploymentInstance(status=FAILED)`` after a + mid-redeploy crash that left the deployment with no row for this slot. + + By the time we reach this helper the OLD instance has already been + deleted from the DB (and its Heat stack from OpenStack), and the new + one may or may not have been persisted before the failure hit. Without + a placeholder the deployment silently shrinks — the lecturer sees one + fewer VM in the dashboard with no obvious link back to the failed + redeploy. The placeholder keeps the count stable, surfaces the error + via the deployment-logs API, and gives the operator a clear retry target. + + No-op if a row for ``old_instance_id`` is somehow still around (e.g. + rollback un-deleted it) or if a new row was already persisted in the + same task — we never want to double-write. + """ + try: + existing = ( + db.query(DeploymentInstance) + .filter(DeploymentInstance.id == old_instance_id) + .first() + ) + if existing is not None: + # Old row survived (rollback or duplicate path) — just flip its + # status rather than inserting a sibling. + existing.status = DeploymentInstanceStatus.FAILED + db.commit() + return + + # Create a fresh row. We don't know the new Heat stack id; leave + # openstack_server_id NULL so future delete_deployment doesn't try + # to tear down a non-existent stack. The vm_name records which + # slot failed so it shows up in the UI in roughly the right place. + from uuid import uuid4 + placeholder = DeploymentInstance( + id=str(uuid4()), + deployment_id=deployment_id, + vm_name=f"redeploy-failed-s{stack_index}", + openstack_server_id=None, + status=DeploymentInstanceStatus.FAILED, + ) + db.add(placeholder) + db.commit() + log_service.log( + deployment_id=deployment_id, + event_type=DeploymentLogEventType.FAILED, + message=( + f"Redeploy left stack #{stack_index} without a DB row; " + "wrote a FAILED placeholder so the slot stays visible." + ), + level=DeploymentLogLevel.WARNING, + details={ + "old_instance_id": old_instance_id, + "placeholder_id": placeholder.id, + "stack_index": stack_index, + "error": error_msg, + }, + ) + except Exception as placeholder_err: + # Last resort — the placeholder is itself best-effort. Log and let + # the surrounding return run; the operator still has the log row + # explaining what failed. + logger.error( + f"Failed to record redeploy-failure placeholder for {old_instance_id}: {placeholder_err}", + exc_info=True, + ) + try: + db.rollback() + except Exception: + pass + + +@celery_app.task(bind=True) +def redeploy_deployment( + self, + deployment_id: str, + deployment_parameter_overrides: dict | None = None, + instance_parameter_overrides: dict | None = None, + preserve_credentials: bool = False, +) -> dict: + """Redeploy every VM in a deployment, one after another. + + Sequential by design — the parent OpenStack project has finite quota, + so fan-out would risk a quota exhaustion mid-class. The deployment + stays ``RUNNING`` between instances (only the targeted one flips to + REDEPLOYING) so the UI shows per-VM progress instead of "everything + rebooting". + + The endpoint accepts both a deployment-wide override map and a + per-instance map (keyed by ``DeploymentInstance.id``). For each + instance the redeploy merges: + + base (deployment.deployment_parameters['parameters']) + + deployment_parameter_overrides + + instance_parameter_overrides.get(instance_id, {}) + + and recreates that one VM via :func:`redeploy_instance`'s logic. + """ + task_id = self.request.id + logger.info( + f"Starting redeploy_deployment task for deployment_id={deployment_id} task_id={task_id}" + ) + + db = SessionLocal() + try: + repo = DeploymentRepository(db) + deployment = repo.get_by_id(deployment_id) + if not deployment: + return {"status": "failed", "error": "Deployment not found"} + + instances = list( + db.query(DeploymentInstance) + .filter(DeploymentInstance.deployment_id == deployment_id) + .order_by(DeploymentInstance.created_at.asc()) + .all() + ) + if not instances: + return {"status": "failed", "error": "Deployment has no instances to redeploy"} + + # Snapshot instance IDs now — the per-instance redeploy deletes + # the row and creates a new one, which would invalidate any + # ORM-bound list mid-iteration. + instance_ids = [inst.id for inst in instances] + finally: + db.close() + + results: list[dict] = [] + overall_status = "redeployed" + for inst_id in instance_ids: + per_instance_overrides = (instance_parameter_overrides or {}).get(inst_id) + # The per-instance task merges its own params on top — pass the + # per-VM overrides folded into the deployment-wide map for this call. + # We DON'T pass instance_parameter_overrides through to redeploy_instance + # itself because each call already targets one VM. + merged_dep_overrides = _merge_parameter_layers( + base={}, + deployment_overrides=deployment_parameter_overrides, + instance_overrides=per_instance_overrides, + ) + # Wrap the inner call: redeploy_instance is expected to return a + # dict on every code path, but a future regression (or a SQLAlchemy + # error escaping its outer try) must NOT abandon the loop mid-class. + # Without this, instances we never touched are silently skipped, + # the aggregated result is lost, and the operator only sees a generic + # Celery failure. + try: + result = redeploy_instance.run( + deployment_id=deployment_id, + instance_id=inst_id, + deployment_parameter_overrides=merged_dep_overrides or None, + preserve_credentials=preserve_credentials, + ) + except Exception as inner_err: + logger.exception( + f"redeploy_instance.run raised for instance {inst_id}; " + "continuing with remaining instances" + ) + result = { + "status": "failed", + "deployment_id": deployment_id, + "instance_id": inst_id, + "error": f"unhandled exception: {inner_err}", + } + results.append(result) + if result.get("status") != "redeployed": + overall_status = "partial_failure" + + # Normalise the parent deployment's status on success: if it was in + # FAILED (e.g. recovery flow via redeploy), bring it back to RUNNING. + # On partial_failure we leave the status alone — the per-instance rows + # already carry FAILED markers and the operator may want to retry. + if overall_status == "redeployed": + finalize_db = SessionLocal() + try: + finalize_repo = DeploymentRepository(finalize_db) + finalize_dep = finalize_repo.get_by_id(deployment_id) + if finalize_dep is not None and finalize_dep.status != DeploymentStatus.RUNNING: + finalize_repo.update_status(deployment_id, DeploymentStatus.RUNNING) + except Exception as status_err: + logger.warning( + f"Could not normalise deployment status to RUNNING after redeploy: {status_err}" + ) + finally: + finalize_db.close() + + return { + "status": overall_status, + "deployment_id": deployment_id, + "task_id": task_id, + "instance_results": results, + } diff --git a/tests/api/test_deployment_redeploy_routes.py b/tests/api/test_deployment_redeploy_routes.py new file mode 100644 index 0000000..9e70a84 --- /dev/null +++ b/tests/api/test_deployment_redeploy_routes.py @@ -0,0 +1,303 @@ +"""API tests for the redeploy endpoints. + +Two endpoints are covered: + +* ``POST /deployments/{id}/redeploy`` — fan-out over every instance +* ``POST /deployments/{id}/instances/{instance_id}/redeploy`` — single VM + +Both flow through ``authorize_deployment_access`` for ownership, then +enqueue the matching Celery task with the body's override / preserve +flags. The actual task body is unit-tested in +``tests/unit/test_redeploy_tasks.py`` — here we cover only the HTTP +contract: status codes, body forwarding, transitional-state gating, +404s. +""" +from unittest.mock import patch, MagicMock + +from fastapi.testclient import TestClient + +from src.main import app +from src.core.dependencies import get_current_user, get_db +from src.models.user import UserRole +from src.models.deployment import DeploymentStatus + + +TEST_OS_PROJECT_ID = "11111111-1111-1111-1111-111111111111" + + +def mock_lecturer_user(): + return { + "sub": "lecturer-123", "email": "l@x.de", "name": "L", + "preferred_username": "lec", "roles": [UserRole.LECTURER.value], + "user_id": 1, + } + + +def mock_admin_user(): + return { + "sub": "admin-123", "email": "a@x.de", "name": "A", + "preferred_username": "adm", "roles": [UserRole.ADMIN.value], + "user_id": 2, + } + + +client = TestClient(app) + + +def _mock_deployment_owned_by(user_id: int, *, status_=DeploymentStatus.RUNNING): + """Build a deployment whose ``deployment_parameters`` claim ``user_id`` + as the owning lecturer (via the Keycloak ID mapping the API does).""" + d = MagicMock() + d.id = "deploy-123" + d.status = status_ + d.openstack_stack_id = '["stack-1"]' + d.deployment_parameters = '{"teacher": {"id": "lecturer-123"}}' + d.openstack_project_id = TEST_OS_PROJECT_ID + return d + + +def _patch_db_owner_lookup(user_id: int): + """Make the User-table query in ``get_deployment_owner_id`` return a + row with ``id=user_id`` so ownership checks pass for the lecturer. + + Also default the per-deployment "any instance already REDEPLOYING?" query + to None so the deployment-wide endpoint's in-flight guard doesn't trip. + Individual tests that want to assert that guard can override + ``mock_db.query.return_value.filter.return_value.first`` afterwards. + """ + mock_db = MagicMock() + mock_user = MagicMock() + mock_user.id = user_id + # First call → User lookup (mock_user); subsequent .first() calls (e.g. the + # REDEPLOYING-in-flight check) → None so the gate is open by default. + mock_db.query.return_value.filter.return_value.first.side_effect = [ + mock_user, None, None, None, None + ] + return mock_db + + +# --------------------------------------------------------------------------- +# POST /{id}/redeploy — full deployment +# --------------------------------------------------------------------------- + + +@patch("src.api.deployments.redeploy_deployment_task") +@patch("src.api.deployments.DeploymentRepository") +def test_redeploy_deployment_success(mock_repo_class, mock_task): + """Happy path: 202 returned, task enqueued with body overrides.""" + app.dependency_overrides[get_current_user] = mock_lecturer_user + dep = _mock_deployment_owned_by(1) + + db = _patch_db_owner_lookup(1) + # The endpoint counts instances via db.query(DeploymentInstance).filter(...).count() + db.query.return_value.filter.return_value.count.return_value = 3 + app.dependency_overrides[get_db] = lambda: db + + repo = MagicMock() + repo.get_by_id.return_value = dep + mock_repo_class.return_value = repo + + response = client.post( + f"/api/v1/deployments/deploy-123/redeploy?openstack_project_id={TEST_OS_PROJECT_ID}", + json={ + "deployment_parameter_overrides": {"flag": True}, + "instance_parameter_overrides": {"inst-a": {"flag": False}}, + "preserve_credentials": True, + }, + ) + + assert response.status_code == 202 + body = response.json() + assert body["data"]["status"] == "redeploy_queued" + assert body["data"]["instance_count"] == 3 + mock_task.delay.assert_called_once_with( + "deploy-123", + deployment_parameter_overrides={"flag": True}, + instance_parameter_overrides={"inst-a": {"flag": False}}, + preserve_credentials=True, + ) + app.dependency_overrides.clear() + + +@patch("src.api.deployments.redeploy_deployment_task") +@patch("src.api.deployments.DeploymentRepository") +def test_redeploy_deployment_no_body_defaults(mock_repo_class, mock_task): + """Body is optional — defaults are forwarded (no overrides, fresh creds).""" + app.dependency_overrides[get_current_user] = mock_lecturer_user + dep = _mock_deployment_owned_by(1) + + db = _patch_db_owner_lookup(1) + db.query.return_value.filter.return_value.count.return_value = 1 + app.dependency_overrides[get_db] = lambda: db + + repo = MagicMock() + repo.get_by_id.return_value = dep + mock_repo_class.return_value = repo + + response = client.post( + f"/api/v1/deployments/deploy-123/redeploy?openstack_project_id={TEST_OS_PROJECT_ID}" + ) + + assert response.status_code == 202 + mock_task.delay.assert_called_once_with( + "deploy-123", + deployment_parameter_overrides=None, + instance_parameter_overrides=None, + preserve_credentials=False, + ) + app.dependency_overrides.clear() + + +@patch("src.api.deployments.DeploymentRepository") +def test_redeploy_deployment_blocked_in_transitional_state(mock_repo_class): + """A deployment that's CREATING/DELETING/RESTARTING can't be redeployed.""" + app.dependency_overrides[get_current_user] = mock_lecturer_user + dep = _mock_deployment_owned_by(1, status_=DeploymentStatus.CREATING) + app.dependency_overrides[get_db] = lambda: _patch_db_owner_lookup(1) + + repo = MagicMock() + repo.get_by_id.return_value = dep + mock_repo_class.return_value = repo + + response = client.post( + f"/api/v1/deployments/deploy-123/redeploy?openstack_project_id={TEST_OS_PROJECT_ID}" + ) + assert response.status_code == 400 + app.dependency_overrides.clear() + + +@patch("src.api.deployments.DeploymentRepository") +def test_redeploy_deployment_no_instances_400(mock_repo_class): + """A deployment with zero instance rows — nothing to redeploy.""" + app.dependency_overrides[get_current_user] = mock_lecturer_user + dep = _mock_deployment_owned_by(1) + + db = _patch_db_owner_lookup(1) + db.query.return_value.filter.return_value.count.return_value = 0 + app.dependency_overrides[get_db] = lambda: db + + repo = MagicMock() + repo.get_by_id.return_value = dep + mock_repo_class.return_value = repo + + response = client.post( + f"/api/v1/deployments/deploy-123/redeploy?openstack_project_id={TEST_OS_PROJECT_ID}" + ) + assert response.status_code == 400 + app.dependency_overrides.clear() + + +@patch("src.api.deployments.DeploymentRepository") +def test_redeploy_deployment_404(mock_repo_class): + app.dependency_overrides[get_current_user] = mock_lecturer_user + repo = MagicMock() + repo.get_by_id.return_value = None + mock_repo_class.return_value = repo + + response = client.post(f"/api/v1/deployments/nope/redeploy?openstack_project_id={TEST_OS_PROJECT_ID}") + assert response.status_code == 404 + app.dependency_overrides.clear() + + +# --------------------------------------------------------------------------- +# POST /{id}/instances/{instance_id}/redeploy — single VM +# --------------------------------------------------------------------------- + + +@patch("src.api.deployments.redeploy_instance_task") +@patch("src.api.deployments.DeploymentRepository") +def test_redeploy_instance_success(mock_repo_class, mock_task): + """Happy path for the per-VM endpoint.""" + app.dependency_overrides[get_current_user] = mock_lecturer_user + dep = _mock_deployment_owned_by(1) + + db = _patch_db_owner_lookup(1) + # Two .first() calls in flight: first one resolves the User row for + # the ownership check; second resolves the DeploymentInstance row. + # Order the side_effect to match. + mock_user = MagicMock() + mock_user.id = 1 + mock_instance = MagicMock() + mock_instance.id = "inst-A" + mock_instance.deployment_id = "deploy-123" + db.query.return_value.filter.return_value.first.side_effect = [mock_user, mock_instance] + app.dependency_overrides[get_db] = lambda: db + + repo = MagicMock() + repo.get_by_id.return_value = dep + mock_repo_class.return_value = repo + + response = client.post( + f"/api/v1/deployments/deploy-123/instances/inst-A/redeploy?openstack_project_id={TEST_OS_PROJECT_ID}", + json={"deployment_parameter_overrides": {"include_notebooks": False}}, + ) + + assert response.status_code == 202 + body = response.json() + assert body["data"]["status"] == "redeploy_queued" + assert body["data"]["instance_id"] == "inst-A" + mock_task.delay.assert_called_once_with( + "deploy-123", + "inst-A", + deployment_parameter_overrides={"include_notebooks": False}, + preserve_credentials=False, + ) + app.dependency_overrides.clear() + + +@patch("src.api.deployments.DeploymentRepository") +def test_redeploy_instance_instance_not_found(mock_repo_class): + """Deployment exists but the instance doesn't — 404.""" + app.dependency_overrides[get_current_user] = mock_lecturer_user + dep = _mock_deployment_owned_by(1) + + db = _patch_db_owner_lookup(1) + # User lookup returns the owner row, but the instance lookup returns None. + # The ownership check uses .filter.first() (user), and the instance + # lookup is the SAME shape, so we sequence the return values. + db.query.return_value.filter.return_value.first.side_effect = [ + MagicMock(id=1), # owner user + None, # instance + ] + app.dependency_overrides[get_db] = lambda: db + + repo = MagicMock() + repo.get_by_id.return_value = dep + mock_repo_class.return_value = repo + + response = client.post( + f"/api/v1/deployments/deploy-123/instances/missing/redeploy?openstack_project_id={TEST_OS_PROJECT_ID}" + ) + assert response.status_code == 404 + app.dependency_overrides.clear() + + +@patch("src.api.deployments.DeploymentRepository") +def test_redeploy_instance_blocked_in_transitional_state(mock_repo_class): + app.dependency_overrides[get_current_user] = mock_lecturer_user + dep = _mock_deployment_owned_by(1, status_=DeploymentStatus.DELETING) + app.dependency_overrides[get_db] = lambda: _patch_db_owner_lookup(1) + + repo = MagicMock() + repo.get_by_id.return_value = dep + mock_repo_class.return_value = repo + + response = client.post( + f"/api/v1/deployments/deploy-123/instances/inst-A/redeploy?openstack_project_id={TEST_OS_PROJECT_ID}" + ) + assert response.status_code == 400 + app.dependency_overrides.clear() + + +@patch("src.api.deployments.DeploymentRepository") +def test_redeploy_instance_deployment_not_found(mock_repo_class): + app.dependency_overrides[get_current_user] = mock_lecturer_user + repo = MagicMock() + repo.get_by_id.return_value = None + mock_repo_class.return_value = repo + + response = client.post( + f"/api/v1/deployments/nope/instances/inst-A/redeploy?openstack_project_id={TEST_OS_PROJECT_ID}" + ) + assert response.status_code == 404 + app.dependency_overrides.clear() diff --git a/tests/unit/test_redeploy_tasks.py b/tests/unit/test_redeploy_tasks.py new file mode 100644 index 0000000..4bbd956 --- /dev/null +++ b/tests/unit/test_redeploy_tasks.py @@ -0,0 +1,541 @@ +"""Tests for the redeploy_instance / redeploy_deployment Celery tasks +and the parameter-override helpers in src.tasks.deploy_tasks. + +The redeploy paths bundle four pieces of policy worth covering with +unit tests, none of which need a live OpenStack or DB: + +1. ``_merge_parameter_layers`` — order is base → deployment → instance, + with later layers winning. Empty / None layers are no-ops. + +2. ``_reconstruct_stack_assignment_for_instance`` — recovers the + ``StackAssignment`` payload that produced an instance from the + ``-s-`` suffix in ``vm_name``. Mis-named instances + return None (caller treats as fatal). + +3. ``redeploy_instance`` — the orchestration: instance flips to + REDEPLOYING, old stack is deleted, ``_provision_one_stack_assignment`` + is called with the merged params, the deployment's stack-id JSON + array is rewritten on the way through. + +4. ``redeploy_deployment`` — iterates instances sequentially and folds + per-instance overrides into the deployment-wide map before calling + ``redeploy_instance``. +""" +import json +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +from src.models.deployment_instance import DeploymentInstanceStatus +from src.tasks import deploy_tasks + + +_DEPLOYMENT_ID = "00000000-0000-0000-0000-000000000abc" +_INSTANCE_ID = "11111111-1111-1111-1111-111111111111" + + +# --------------------------------------------------------------------------- +# _merge_parameter_layers +# --------------------------------------------------------------------------- + + +def test_merge_layers_later_wins(): + """Deployment override beats base, instance override beats deployment.""" + base = {"a": 1, "b": 2, "c": 3} + dep = {"b": 20, "d": 40} + inst = {"c": 300} + + result = deploy_tasks._merge_parameter_layers( + base=base, deployment_overrides=dep, instance_overrides=inst + ) + + assert result == {"a": 1, "b": 20, "c": 300, "d": 40} + # Inputs are not mutated. + assert base == {"a": 1, "b": 2, "c": 3} + assert dep == {"b": 20, "d": 40} + assert inst == {"c": 300} + + +def test_merge_layers_none_skipped(): + """None / empty layers are no-ops; base survives untouched.""" + base = {"x": 1} + assert deploy_tasks._merge_parameter_layers( + base=base, deployment_overrides=None, instance_overrides=None + ) == {"x": 1} + assert deploy_tasks._merge_parameter_layers( + base=base, deployment_overrides={}, instance_overrides={} + ) == {"x": 1} + + +def test_merge_layers_only_deployment(): + """Deployment-only override paths (used by the per-instance endpoint).""" + base = {"x": 1} + assert deploy_tasks._merge_parameter_layers( + base=base, deployment_overrides={"y": 2}, instance_overrides=None + ) == {"x": 1, "y": 2} + + +# --------------------------------------------------------------------------- +# _reconstruct_stack_assignment_for_instance +# --------------------------------------------------------------------------- + + +def test_reconstruct_picks_correct_assignment_by_suffix(): + """Heat names follow ``-s-`` — we recover ````.""" + raw = [{"stack_index": 1, "tag": "A"}, {"stack_index": 2, "tag": "B"}, {"stack_index": 3, "tag": "C"}] + inst = SimpleNamespace(vm_name="sql-kurs-s2-abcd") + + chosen = deploy_tasks._reconstruct_stack_assignment_for_instance(inst, raw) + assert chosen == {"stack_index": 2, "tag": "B"} + + +def test_reconstruct_handles_dashes_in_deployment_name(): + """Deployment names may contain dashes — the ``-s-`` suffix is + matched right-to-left so the dashes in the prefix don't confuse it.""" + raw = [{"stack_index": 1}, {"stack_index": 2}] + inst = SimpleNamespace(vm_name="sql-kurs-summer-2026-s2-1234") + + chosen = deploy_tasks._reconstruct_stack_assignment_for_instance(inst, raw) + assert chosen == {"stack_index": 2} + + +def test_reconstruct_returns_none_for_bad_name(): + """A wedged / legacy vm_name → None (caller fails the redeploy).""" + raw = [{"stack_index": 1}] + assert deploy_tasks._reconstruct_stack_assignment_for_instance( + SimpleNamespace(vm_name=None), raw + ) is None + assert deploy_tasks._reconstruct_stack_assignment_for_instance( + SimpleNamespace(vm_name="not-a-valid-name"), raw + ) is None + + +def test_reconstruct_returns_none_when_index_out_of_range(): + """A name that parses but points past the assignments list is unsafe.""" + raw = [{"stack_index": 1}] + inst = SimpleNamespace(vm_name="dep-s7-abcd") + assert deploy_tasks._reconstruct_stack_assignment_for_instance(inst, raw) is None + + +def test_stack_index_for_instance_fallback(): + """Fallback to 1 when no index suffix is parseable — keeps the redeploy + able to log even if the name was hand-edited.""" + assert deploy_tasks._stack_index_for_instance( + SimpleNamespace(vm_name="dep-s3-abcd"), [] + ) == 3 + assert deploy_tasks._stack_index_for_instance( + SimpleNamespace(vm_name="bogus"), [] + ) == 1 + + +# --------------------------------------------------------------------------- +# _build_stack_name +# --------------------------------------------------------------------------- + + +def test_build_stack_name_matches_deploy_format(): + """Redeploy must produce the same Heat-safe slug as the initial deploy + so a redeployed VM still matches its stack tag pattern.""" + dep = SimpleNamespace(id="abcd1234-xxxx", name="SQL Kurs Sommer_2026") + assert deploy_tasks._build_stack_name(dep, 3) == "sql-kurs-sommer-2026-s3-abcd" + + +def test_build_stack_name_truncates_to_64_chars(): + dep = SimpleNamespace(id="abcd1234", name="X" * 100) + name = deploy_tasks._build_stack_name(dep, 1) + assert len(name) <= 64 + + +# --------------------------------------------------------------------------- +# _snapshot_instance_credentials (preserve_credentials path) +# +# The snapshot's contract changed: instead of round-tripping access rows +# back into ``generated``-shaped per-group/teacher buckets (which was +# lossy — public_key, port, connection_url, and the original app.yaml +# credential type were all dropped), it now captures every field of each +# ``DeploymentInstanceAccess`` row verbatim under ``_preserved_access``, +# ready for ``_rebind_preserved_access_rows`` to re-attach to the new +# instance after persistence. ``deployment_groups`` and ``teacher`` come +# back empty by design — Ansible still receives fresh creds, the DB rows +# are rebound separately. +# --------------------------------------------------------------------------- + + +def test_snapshot_credentials_captures_all_access_row_fields_verbatim(): + """SSH + non-SSH rows survive the snapshot with port / connection_url + / ssh_private_key intact, ready for rebinding.""" + ssh = SimpleNamespace( + access_type=SimpleNamespace(value="ssh"), + username="g1", password="pw1", ssh_private_key="key1", + connection_url="ssh g1@1.2.3.4", port=22, + group_id="grp-1", is_active=True, expires_at=None, + ) + db_row = SimpleNamespace( + access_type=SimpleNamespace(value="database"), + username="dbu", password="dbpw", ssh_private_key=None, + connection_url="https://pgadmin.example/", port=80, + group_id="grp-1", is_active=True, expires_at=None, + ) + admin = SimpleNamespace( + access_type=SimpleNamespace(value="ssh"), + username="teacher", password="adminpw", ssh_private_key="adminkey", + connection_url="ssh teacher@1.2.3.4", port=22, + group_id=None, is_active=True, expires_at=None, + ) + instance = SimpleNamespace(access_methods=[ssh, db_row, admin]) + + snap = deploy_tasks._snapshot_instance_credentials(MagicMock(), instance) + + # The shim fields stay empty on purpose — preserved data lives under + # _preserved_access and is rebound from there. + assert snap["deployment_groups"] == [] + assert snap["teacher"] == {} + + rows = snap["_preserved_access"] + assert len(rows) == 3 + # Every column we care about for student logins is present verbatim. + assert rows[0]["username"] == "g1" + assert rows[0]["password"] == "pw1" + assert rows[0]["ssh_private_key"] == "key1" + assert rows[0]["port"] == 22 + assert rows[0]["connection_url"] == "ssh g1@1.2.3.4" + assert rows[1]["connection_url"] == "https://pgadmin.example/" + assert rows[1]["port"] == 80 + assert rows[2]["group_id"] is None # teacher row + + +# --------------------------------------------------------------------------- +# redeploy_instance — orchestration +# --------------------------------------------------------------------------- + + +def _make_instance(*, vm_name="dep-s1-abcd", access_methods=None): + """Build a stub ``DeploymentInstance`` row good enough for the redeploy + task. Status is a real enum so the ``.value`` lookup works the same way + the production code expects.""" + inst = MagicMock() + inst.id = _INSTANCE_ID + inst.vm_name = vm_name + inst.openstack_server_id = "old-stack-id" + inst.deployment_id = _DEPLOYMENT_ID + inst.access_methods = access_methods or [] + inst.status = DeploymentInstanceStatus.RUNNING + return inst + + +def _make_deployment(*, stack_ids=("old-stack-id",), parameters=None, stack_assignments=None): + params_payload = { + "parameters": parameters or {"include_notebooks": True}, + "stack_assignments": stack_assignments or [{"stack_index": 1, "groups": [{ + "group_name": "G1", "group_index": 1, "students": [], "course_group_id": "grp-1", + }]}], + "teacher": { + "id": "kc-teacher", "username": "prof", "email": "p@x.de", + "first_name": "Prof", "last_name": "X", + }, + } + return SimpleNamespace( + id=_DEPLOYMENT_ID, + name="dep", + course_id="course-1", + template_version_id="tv-1", + openstack_stack_id=json.dumps(list(stack_ids)), + deployment_parameters=json.dumps(params_payload), + openstack_project=SimpleNamespace(id="proj-1"), + ) + + +def _patch_redeploy_environment(*, deployment, instance, provision_return=None): + """Patch every collaborator of redeploy_instance with MagicMocks and + return them so the test can assert on call args. + + Returns a dict with all the patches' return values for convenience. + """ + session = MagicMock() + + # db.query(DeploymentInstance).filter(...).first() → instance + inst_filter = MagicMock() + inst_filter.first.return_value = instance + inst_query = MagicMock() + inst_query.filter.return_value = inst_filter + + # db.query(DeploymentInstanceAccess).filter(...).delete() + access_filter = MagicMock() + access_filter.delete.return_value = 0 + access_query = MagicMock() + access_query.filter.return_value = access_filter + + # Map .query(Model) to the right stub. + def _query(model): + from src.models.deployment_instance import DeploymentInstance as DI + from src.models.deployment_instance_access import DeploymentInstanceAccess as DIA + if model is DI: + return inst_query + if model is DIA: + return access_query + return MagicMock() + session.query.side_effect = _query + + repo = MagicMock() + repo.get_by_id.return_value = deployment + + log_service = MagicMock() + file_service = MagicMock() + + heat = MagicMock() + heat.delete_stack.return_value = True + + template_context = MagicMock() + template_context.split_parameters.return_value = ({}, {}) + + return { + "session": session, + "repo": repo, + "log_service": log_service, + "file_service": file_service, + "heat": heat, + "template_context": template_context, + "provision_return": provision_return, + } + + +def _run_redeploy_instance(env, *, deployment_overrides=None, preserve_credentials=False): + """Run the redeploy_instance task with a fully patched environment. + + Returns the task result dict. + """ + new_instance = MagicMock(id="new-instance-id") + provision_return = env.get("provision_return") or ("new-stack-id", new_instance) + + with ( + patch.object(deploy_tasks, "SessionLocal", return_value=env["session"]), + patch.object(deploy_tasks, "DeploymentRepository", return_value=env["repo"]), + patch.object(deploy_tasks, "DeploymentLogService", return_value=env["log_service"]), + patch.object(deploy_tasks, "TemplateVersionFileService", return_value=env["file_service"]), + patch.object(deploy_tasks, "HeatStackService", return_value=env["heat"]), + patch.object(deploy_tasks, "_load_template_context", return_value=env["template_context"]), + patch.object(deploy_tasks, "_provision_one_stack_assignment", return_value=provision_return) as provision_mock, + patch.object(deploy_tasks, "get_settings", return_value=SimpleNamespace( + ansible_ssh_private_key="", ansible_ssh_key_name="kp", + )), + ): + result = deploy_tasks.redeploy_instance.run( + _DEPLOYMENT_ID, + _INSTANCE_ID, + deployment_parameter_overrides=deployment_overrides, + preserve_credentials=preserve_credentials, + ) + return result, provision_mock + + +def test_redeploy_instance_happy_path_flips_status_and_calls_provision(): + """REDEPLOYING is set before the old stack is torn down, and the + provision helper is invoked with the recovered stack assignment.""" + deployment = _make_deployment(stack_ids=("old-stack-id", "sibling")) + instance = _make_instance(vm_name="dep-s1-abcd") + + env = _patch_redeploy_environment(deployment=deployment, instance=instance) + result, provision_mock = _run_redeploy_instance(env) + + assert result["status"] == "redeployed" + assert result["new_stack_id"] == "new-stack-id" + assert result["new_instance_id"] == "new-instance-id" + # Status was flipped to REDEPLOYING during the task. + assert instance.status == DeploymentInstanceStatus.REDEPLOYING + # Old Heat stack was deleted. + env["heat"].delete_stack.assert_called_once_with("old-stack-id") + # Provision helper got called with the merged params. + provision_mock.assert_called_once() + kwargs = provision_mock.call_args.kwargs + assert kwargs["stack_index"] == 1 + # No overrides → effective params equal the deployment's stored ones. + assert kwargs["all_parameters"] == {"include_notebooks": True} + # Single-instance redeploy → no preserved user_json. + assert kwargs["preserved_user_json"] is None + + +def test_redeploy_instance_overrides_are_merged_into_effective_params(): + """A deployment-level override wins over the stored parameter.""" + deployment = _make_deployment(parameters={"flag": False, "size": "small"}) + instance = _make_instance(vm_name="dep-s1-abcd") + + env = _patch_redeploy_environment(deployment=deployment, instance=instance) + result, provision_mock = _run_redeploy_instance( + env, deployment_overrides={"flag": True, "extra": 42} + ) + + assert result["status"] == "redeployed" + kwargs = provision_mock.call_args.kwargs + assert kwargs["all_parameters"] == {"flag": True, "size": "small", "extra": 42} + + +def test_redeploy_instance_preserve_credentials_carries_snapshot_in(): + """When preserve_credentials=True the provision helper is handed the + snapshot dict containing every access-row field. The rebind step + (called from inside _provision_one_stack_assignment, which is mocked + here) is what actually re-attaches them — we only verify the snapshot + travels far enough to reach it.""" + ssh = SimpleNamespace( + access_type=SimpleNamespace(value="ssh"), + username="g1", password="pw1", ssh_private_key="k1", + connection_url="ssh g1@1.2.3.4", port=22, + group_id="grp-1", is_active=True, expires_at=None, + ) + instance = _make_instance(vm_name="dep-s1-abcd", access_methods=[ssh]) + deployment = _make_deployment() + + env = _patch_redeploy_environment(deployment=deployment, instance=instance) + _, provision_mock = _run_redeploy_instance(env, preserve_credentials=True) + + preserved = provision_mock.call_args.kwargs["preserved_user_json"] + assert preserved is not None + rows = preserved["_preserved_access"] + assert len(rows) == 1 + assert rows[0]["username"] == "g1" + assert rows[0]["ssh_private_key"] == "k1" + assert rows[0]["port"] == 22 + assert rows[0]["group_id"] == "grp-1" + + +def test_redeploy_instance_rewrites_stack_id_list_on_deployment(): + """The deployment's openstack_stack_id JSON loses the old id and + gains the new one — so a future delete walks the right list.""" + deployment = _make_deployment(stack_ids=("old-stack-id", "sibling-stack")) + instance = _make_instance(vm_name="dep-s1-abcd") + + env = _patch_redeploy_environment(deployment=deployment, instance=instance) + result, _ = _run_redeploy_instance(env) + + assert result["status"] == "redeployed" + final_ids = json.loads(deployment.openstack_stack_id) + assert "old-stack-id" not in final_ids + assert "sibling-stack" in final_ids + assert "new-stack-id" in final_ids + + +def test_redeploy_instance_fails_when_stack_assignment_missing(): + """An instance whose name can't be reconstructed → fatal redeploy. + Status flips to FAILED, no provision helper call, no Heat delete.""" + deployment = _make_deployment(stack_assignments=[]) + instance = _make_instance(vm_name="legacy-no-suffix") + + env = _patch_redeploy_environment(deployment=deployment, instance=instance) + result, provision_mock = _run_redeploy_instance(env) + + assert result["status"] == "failed" + assert "stack_assignment" in result["error"] + provision_mock.assert_not_called() + env["heat"].delete_stack.assert_not_called() + assert instance.status == DeploymentInstanceStatus.FAILED + + +def test_redeploy_instance_heat_delete_failure_marks_failed(): + """Heat refusing to delete the old stack must NOT remove the row; + instance flips to FAILED so the user can retry.""" + deployment = _make_deployment() + instance = _make_instance(vm_name="dep-s1-abcd") + + env = _patch_redeploy_environment(deployment=deployment, instance=instance) + env["heat"].delete_stack.side_effect = RuntimeError("openstack 500") + + result, provision_mock = _run_redeploy_instance(env) + + assert result["status"] == "failed" + provision_mock.assert_not_called() + + +# --------------------------------------------------------------------------- +# redeploy_deployment — fan-out over instances +# --------------------------------------------------------------------------- + + +def test_redeploy_deployment_iterates_instances_with_merged_per_vm_overrides(): + """Per-instance overrides are folded into the deployment-wide map on + the way through, so each redeploy_instance call sees its own merged + override dict.""" + deployment = _make_deployment() + inst_a = SimpleNamespace(id="inst-A", created_at=1) + inst_b = SimpleNamespace(id="inst-B", created_at=2) + + session = MagicMock() + # db.query(DeploymentInstance).filter(...).order_by(...).all() → [inst_a, inst_b] + instances_chain = MagicMock() + instances_chain.filter.return_value.order_by.return_value.all.return_value = [inst_a, inst_b] + session.query.return_value = instances_chain + + repo = MagicMock() + repo.get_by_id.return_value = deployment + + redeploy_calls: list[dict] = [] + + def _fake_redeploy_instance_run(**kwargs): + redeploy_calls.append(kwargs) + return {"status": "redeployed", "instance_id": kwargs.get("instance_id")} + + with ( + patch.object(deploy_tasks, "SessionLocal", return_value=session), + patch.object(deploy_tasks, "DeploymentRepository", return_value=repo), + patch.object(deploy_tasks.redeploy_instance, "run", side_effect=_fake_redeploy_instance_run), + ): + result = deploy_tasks.redeploy_deployment.run( + _DEPLOYMENT_ID, + deployment_parameter_overrides={"flag": True}, + instance_parameter_overrides={"inst-B": {"flag": False, "extra": 1}}, + preserve_credentials=True, + ) + + assert result["status"] == "redeployed" + assert len(redeploy_calls) == 2 + # inst_a inherits only the deployment-wide override. + assert redeploy_calls[0]["instance_id"] == "inst-A" + assert redeploy_calls[0]["deployment_parameter_overrides"] == {"flag": True} + assert redeploy_calls[0]["preserve_credentials"] is True + # inst_b gets the merged override map (instance wins over deployment). + assert redeploy_calls[1]["instance_id"] == "inst-B" + assert redeploy_calls[1]["deployment_parameter_overrides"] == {"flag": False, "extra": 1} + + +def test_redeploy_deployment_reports_partial_failure_when_one_instance_fails(): + """One failed instance → overall status flips to partial_failure but + the loop still hits every instance.""" + deployment = _make_deployment() + inst_a = SimpleNamespace(id="inst-A", created_at=1) + inst_b = SimpleNamespace(id="inst-B", created_at=2) + + session = MagicMock() + session.query.return_value.filter.return_value.order_by.return_value.all.return_value = [inst_a, inst_b] + + repo = MagicMock() + repo.get_by_id.return_value = deployment + + def _fake(*, deployment_id, instance_id, **kwargs): + if instance_id == "inst-A": + return {"status": "failed", "instance_id": "inst-A"} + return {"status": "redeployed", "instance_id": "inst-B"} + + with ( + patch.object(deploy_tasks, "SessionLocal", return_value=session), + patch.object(deploy_tasks, "DeploymentRepository", return_value=repo), + patch.object(deploy_tasks.redeploy_instance, "run", side_effect=_fake), + ): + result = deploy_tasks.redeploy_deployment.run(_DEPLOYMENT_ID) + + assert result["status"] == "partial_failure" + assert len(result["instance_results"]) == 2 + + +def test_redeploy_deployment_returns_failed_when_no_instances(): + """A deployment with zero instance rows can't be redeployed.""" + deployment = _make_deployment() + session = MagicMock() + session.query.return_value.filter.return_value.order_by.return_value.all.return_value = [] + repo = MagicMock() + repo.get_by_id.return_value = deployment + + with ( + patch.object(deploy_tasks, "SessionLocal", return_value=session), + patch.object(deploy_tasks, "DeploymentRepository", return_value=repo), + ): + result = deploy_tasks.redeploy_deployment.run(_DEPLOYMENT_ID) + + assert result["status"] == "failed" + assert "no instances" in result["error"].lower()