OCPEDGE-2498: Add migration support

deploy/Makefile

@@ -66,6 +66,9 @@ full-clean:
 clean-spoke:
 	@./openshift-clusters/scripts/clean-spoke.sh
 
+clean-mutable-topology:
+	@./openshift-clusters/scripts/clean-mutable-topology.sh
+
 ssh:
 	@./aws-hypervisor/scripts/ssh.sh
 
@@ -143,6 +146,9 @@ baremetal-fencing-agent:
 baremetal-wizard:
 	@./openshift-clusters/scripts/baremetal-wizard.sh
 
+sno-to-3node:
+	@./openshift-clusters/scripts/sno-to-3node.sh
+
 patch-nodes:
 	@./openshift-clusters/scripts/patch-nodes.sh
 get-tnf-logs:
@@ -184,6 +190,7 @@ help:
 	@echo "  fencing-assisted     - Deploy hub + spoke TNF cluster via assisted installer"
 	@echo "  sno-ipi              - Deploy Single Node OpenShift (SNO) IPI cluster (non-interactive)"
 	@echo "  sno-agent            - Deploy Single Node OpenShift (SNO) Agent cluster (non-interactive)"
+	@echo "  sno-to-3node         - Transition existing SNO cluster to 3-node HA (platform:none)"
 	@echo ""
 	@echo "OpenShift Cluster Management:"
 	@echo "  redeploy-cluster     - Redeploy OpenShift cluster using dev-scripts make redeploy"
@@ -192,6 +199,7 @@ help:
 	@echo "  clean                - Clean OpenShift cluster using dev-scripts clean target"
 	@echo "  full-clean           - Fully clean instance cache and OpenShift cluster using dev-scripts realclean target"
 	@echo "  clean-spoke          - Clean spoke cluster resources (VMs, network, auth) from assisted installer"
+	@echo "  clean-mutable-topology - Remove master-1/2 VMs, disks, DHCP/DNS entries (sno-to-3node cleanup)"
 	@echo "  patch-nodes          - Build resource-agents RPM and patch cluster nodes (default version: 4.11)"
 	@echo ""
 	@echo "Baremetal Adoption:"

deploy/openshift-clusters/clean-mutable-topology.yml

@@ -0,0 +1,63 @@
+---
+- hosts: metal_machine
+  gather_facts: yes
+
+  pre_tasks:
+    - name: Confirm mutable topology cleanup
+      ansible.builtin.pause:
+        prompt: >-
+          This will destroy the master-1 and master-2 VMs, their disks, DHCP reservations,
+          and DNS entries. master-0 (if still present) will be unaffected.
+          Press Enter to proceed or Ctrl+C to abort.
+      delegate_to: localhost
+      run_once: true
+      when: interactive_mode | default(true) | bool
+
+    - name: Detect cluster domain from kubeconfig (best-effort)
+      shell: |
+        KUBECONFIG={{ dev_scripts_path | default('openshift-metal3/dev-scripts') }}/ocp/{{ sno_cluster_name | default('ostest') }}/auth/kubeconfig \
+          oc get infrastructure cluster -o jsonpath='{.status.apiServerInternalURI}' 2>/dev/null \
+          | sed 's|^https://api-int\.||; s|:6443$||'
+      register: detected_domain
+      changed_when: false
+      failed_when: false
+      ignore_errors: true
+
+    - name: Set cluster domain (use detected, variable override, or default)
+      set_fact:
+        sno_cluster_domain: >-
+          {{ sno_cluster_domain
+             if (sno_cluster_domain is defined and sno_cluster_domain)
+             else (detected_domain.stdout | trim
+                   if (detected_domain.stdout is defined and detected_domain.stdout | trim)
+                   else 'ostest.test.metalkube.org') }}
+
+    - name: Check live cluster topology
+      shell: |
+        KUBECONFIG={{ dev_scripts_path | default('openshift-metal3/dev-scripts') }}/ocp/{{ sno_cluster_name | default('ostest') }}/auth/kubeconfig \
+          oc get infrastructure cluster \
+          -o jsonpath='{.status.controlPlaneTopology}' 2>/dev/null || echo "Unknown"
+      register: live_topology
+      changed_when: false
+      failed_when: false
+      ignore_errors: true
+
+    - name: Abort if cluster is HA and force_cleanup is not set
+      ansible.builtin.fail:
+        msg: >-
+          Cluster controlPlaneTopology is '{{ live_topology.stdout | trim }}'.
+          Refusing to remove master-1 and master-2 from a live HA cluster.
+          Re-run with -e "force_cleanup=true" to override.
+      when:
+        - live_topology.stdout | trim == "HighlyAvailable"
+        - not (force_cleanup | default(false) | bool)
+
+  tasks:
+    - name: Run mutable topology cleanup
+      import_role:
+        name: mutable-topology/sno-to-3node
+        tasks_from: clean.yml
+
+    - name: Cleanup complete
+      ansible.builtin.debug:
+        msg: "Mutable topology cleanup complete. master-1 and master-2 VMs have been removed."

deploy/openshift-clusters/roles/dev-scripts/install-dev/handlers/main.yml

@@ -19,6 +19,18 @@
   failed_when: false
   listen: Set OCP project
 
+- name: Warn if oc project failed after all retries
+  debug:
+    msg: >-
+      WARNING: 'oc project openshift-machine-api' still failing after all retries
+      (rc={{ oc_project_result.rc | default('n/a') }}).
+      stderr: {{ oc_project_result.stderr | default('') | trim }}
+  when:
+    - kubeconfig_path is defined
+    - kubeconfig_stat.stat.exists | default(false)
+    - oc_project_result.rc | default(0) != 0
+  listen: Set OCP project
+
 - name: Warn about missing kubeconfig
   debug:
     msg: >-

deploy/openshift-clusters/roles/dev-scripts/install-dev/tasks/config.yml

@@ -28,8 +28,8 @@
     msg: >-
       Config file {{ config_file[method] }} does not set AGENT_E2E_TEST_SCENARIO,
       which is required for the agent installation method. Add
-      AGENT_E2E_TEST_SCENARIO="TNF_IPV4" (fencing) or "TNA_IPV4" (arbiter) to
-      the config file.
+      AGENT_E2E_TEST_SCENARIO="TNF_IPV4" (fencing), "TNA_IPV4" (arbiter), or
+      "SNO_IPV4" (sno) to the config file.
   when:
     - method == "agent"
     - config_scenario | length == 0

deploy/openshift-clusters/roles/mutable-topology/sno-to-3node/defaults/main.yml

@@ -0,0 +1,40 @@
+---
+# Cluster identity — sno_cluster_domain and sno_master0_ip are auto-detected
+# from the cluster at runtime; sno_cluster_name is a static default used for
+# kubeconfig path and VM naming (override with -e sno_cluster_name=... if needed)
+sno_cluster_name: ostest
+sno_cluster_domain: ""
+sno_infra_id: ""
+
+# Existing master-0 (auto-detected from cluster)
+sno_master0_ip: ""
+
+# New node IPs (static assignments within the dev-scripts DHCP range)
+sno_master1_ip: "192.168.111.21"
+sno_master2_ip: "192.168.111.22"
+
+# VM specs
+sno_vm_vcpus: 6
+sno_vm_ram_mb: 16384
+sno_vm_disk_gb: 50
+
+# Libvirt network (dev-scripts baremetal network)
+sno_libvirt_network: ostestbm
+sno_libvirt_bridge: ostestbm
+
+# RHCOS live ISO path on hypervisor (auto-detected from release image if empty)
+sno_rhcos_live_iso: ""
+
+# Timeouts
+sno_mco_timeout_minutes: 45
+sno_node_join_timeout_minutes: 20
+sno_etcd_timeout_minutes: 15
+
+# Auto-fix MCO drain deadlock during topology transition
+sno_auto_fix_drain: true
+
+# Paths (override if dev-scripts is in a non-standard location)
+sno_kubeconfig: ""
+
+# VM image directory
+sno_vm_image_dir: "/var/lib/libvirt/images"

deploy/openshift-clusters/roles/mutable-topology/sno-to-3node/tasks/boot-nodes.yml

@@ -0,0 +1,154 @@
+---
+- name: "[boot] Check if RHCOS live ISO exists on hypervisor"
+  stat:
+    path: "/var/lib/libvirt/images/rhcos-live.iso"
+  register: iso_stat
+
+- name: "[boot] Find RHCOS live ISO from dev-scripts cache or release payload"
+  shell: |
+    DEVSCRIPTS_ISO=$(find /var/lib/libvirt/images -name 'rhcos-*-live*.iso' 2>/dev/null | head -1)
+    if [ -n "$DEVSCRIPTS_ISO" ]; then
+      echo "Using existing ISO: $DEVSCRIPTS_ISO"
+      sudo ln -sf "$DEVSCRIPTS_ISO" /var/lib/libvirt/images/rhcos-live.iso
+      exit 0
+    fi
+
+    CACHE_ISO=$(find {{ sno_dev_scripts_path }}/ -name 'rhcos-*-live*.iso' 2>/dev/null | head -1)
+    if [ -n "$CACHE_ISO" ]; then
+      echo "Using dev-scripts cached ISO: $CACHE_ISO"
+      sudo ln -sf "$CACHE_ISO" /var/lib/libvirt/images/rhcos-live.iso
+      exit 0
+    fi
+
+    echo "No cached ISO found, pulling from release payload..."
+    PULL_SECRET="{{ sno_dev_scripts_path }}/pull_secret.json"
+    RELEASE_IMAGE=$(KUBECONFIG="{{ sno_kubeconfig_resolved }}" oc get clusterversion version \
+      -o jsonpath='{.status.desired.image}')
+    echo "Release image: $RELEASE_IMAGE"
+    RHCOS_IMAGE=$(oc adm release info \
+      --image-for=machine-os-images \
+      -a "$PULL_SECRET" \
+      "$RELEASE_IMAGE")
+    echo "machine-os-images: $RHCOS_IMAGE"
+    oc image extract "$RHCOS_IMAGE" \
+      -a "$PULL_SECRET" \
+      --path /coreos/coreos-x86_64.iso:/tmp/ \
+      --confirm
+    sudo mv /tmp/coreos-x86_64.iso /var/lib/libvirt/images/rhcos-live.iso
+    echo "ISO pulled from release payload: /var/lib/libvirt/images/rhcos-live.iso"
+  when: not iso_stat.stat.exists and sno_rhcos_live_iso == ""
+
+- name: "[boot] Set ISO path"
+  set_fact:
+    sno_iso_path: "{{ sno_rhcos_live_iso if sno_rhcos_live_iso else '/var/lib/libvirt/images/rhcos-live.iso' }}"
+
+- name: "[boot] Read master.ign content"
+  slurp:
+    src: /tmp/master.ign
+  register: master_ign_content
+
+- name: "[boot] Set base64-encoded master.ign"
+  set_fact:
+    sno_master_ign_b64: "{{ master_ign_content.content }}"
+
+- name: "[boot] Generate auto-install ignition"
+  template:
+    src: auto-install.ign.j2
+    dest: /tmp/auto-install.ign
+    mode: '0644'
+
+- name: "[boot] Ensure coreos-installer is available"
+  shell: |
+    if ! command -v coreos-installer &>/dev/null; then
+      echo "coreos-installer not found, installing via dnf..."
+      sudo dnf install -y coreos-installer
+    else
+      echo "coreos-installer already installed: $(command -v coreos-installer)"
+    fi
+
+- name: "[boot] Create per-node ISO with embedded ignition"
+  shell: |
+    sudo cp {{ sno_iso_path }} /var/lib/libvirt/images/rhcos-{{ item.hostname }}.iso
+    sudo coreos-installer iso ignition embed -i /tmp/auto-install.ign /var/lib/libvirt/images/rhcos-{{ item.hostname }}.iso -f
+  loop: "{{ sno_new_nodes }}"
+
+- name: "[boot] Boot each VM with ignition-embedded ISO"
+  shell: |
+    VM_NAME="{{ item.name }}"
+    MAC="{{ sno_node_macs[item.hostname] }}"
+
+    sudo virsh destroy "$VM_NAME" 2>/dev/null || true
+    sudo virsh undefine "$VM_NAME" --nvram 2>/dev/null || true
+
+    sudo virt-install \
+      --name "$VM_NAME" \
+      --ram {{ sno_vm_ram_mb }} \
+      --vcpus {{ sno_vm_vcpus }} \
+      --disk {{ sno_vm_image_dir }}/${VM_NAME}.qcow2,bus=virtio \
+      --network network={{ sno_libvirt_network }},model=virtio,mac=${MAC} \
+      --cdrom /var/lib/libvirt/images/rhcos-{{ item.hostname }}.iso \
+      --os-variant rhel9.0 \
+      --graphics none \
+      --noautoconsole \
+      --boot loader=/usr/share/edk2/ovmf/OVMF_CODE.fd,loader_ro=yes,loader_type=pflash,nvram_template=/usr/share/edk2/ovmf/OVMF_VARS.fd,loader_secure=no \
+      --boot hd,cdrom \
+      --tpm none
+  loop: "{{ sno_new_nodes }}"
+
+- name: "[boot] Verify VMs are running (initial install boot)"
+  shell: |
+    sudo virsh domstate {{ item.name }}
+  register: vm_state
+  loop: "{{ sno_new_nodes }}"
+  changed_when: false
+  failed_when: "'running' not in vm_state.stdout"
+
+- name: "[boot] Wait for coreos-installer to complete (VM will power off)"
+  # coreos-installer runs ExecStartPost=systemctl reboot, but RHCOS live issues
+  # an ACPI poweroff rather than a reset. libvirt fires on_poweroff=destroy so
+  # the VM shuts off. We poll until shut off, then boot from disk below.
+  shell: |
+    for i in $(seq 50); do
+      STATE=$(sudo virsh domstate {{ item.name }} 2>/dev/null || echo "unknown")
+      if echo "$STATE" | grep -q "shut off"; then
+        echo "{{ item.name }} shut off after $((i * 20))s - install complete"
+        exit 0
+      fi
+      sleep 20
+    done
+    echo "Timeout: {{ item.name }} did not shut off within 1000s"
+    exit 1
+  loop: "{{ sno_new_nodes }}"
+  changed_when: false
+
+- name: "[boot] Remove CDROM from boot order after install"
+  # Prevent coreos-installer loop: strip the cdrom boot entry so UEFI only
+  # tries the hard disk on subsequent boots.
+  shell: |
+    TMPXML=$(mktemp /tmp/vm-XXXXXX.xml)
+    sudo virsh dumpxml {{ item.name }} > "$TMPXML"
+    sudo sed -i "/<boot dev='cdrom'\/>/d" "$TMPXML"
+    sudo virsh define "$TMPXML"
+    sudo rm -f "$TMPXML"
+  loop: "{{ sno_new_nodes }}"
+  changed_when: true
+
+- name: "[boot] Start VMs to boot from installed RHCOS"
+  shell: |
+    sudo virsh start {{ item.name }}
+  loop: "{{ sno_new_nodes }}"
+  changed_when: true
+
+- name: "[boot] Verify VMs are running from installed disk"
+  shell: |
+    sudo virsh domstate {{ item.name }}
+  register: vm_state_disk
+  loop: "{{ sno_new_nodes }}"
+  changed_when: false
+  failed_when: "'running' not in vm_state_disk.stdout"
+
+- name: "[boot] VMs booted with RHCOS"
+  debug:
+    msg: >-
+      {{ sno_new_nodes | length }} VMs installed and started from disk.
+      Waiting for nodes to join the cluster...