diff --git a/deploy/Makefile b/deploy/Makefile
index 306f6a99..c60a9e3d 100644
--- a/deploy/Makefile
+++ b/deploy/Makefile
@@ -66,6 +66,9 @@ full-clean:
clean-spoke:
@./openshift-clusters/scripts/clean-spoke.sh
+clean-mutable-topology:
+ @./openshift-clusters/scripts/clean-mutable-topology.sh $(EXTRA_ARGS)
+
ssh:
@./aws-hypervisor/scripts/ssh.sh
@@ -143,6 +146,9 @@ baremetal-fencing-agent:
baremetal-wizard:
@./openshift-clusters/scripts/baremetal-wizard.sh
+sno-to-3node:
+ @./openshift-clusters/scripts/sno-to-3node.sh $(EXTRA_ARGS)
+
patch-nodes:
@./openshift-clusters/scripts/patch-nodes.sh
get-tnf-logs:
@@ -184,6 +190,7 @@ help:
@echo " fencing-assisted - Deploy hub + spoke TNF cluster via assisted installer"
@echo " sno-ipi - Deploy Single Node OpenShift (SNO) IPI cluster (non-interactive)"
@echo " sno-agent - Deploy Single Node OpenShift (SNO) Agent cluster (non-interactive)"
+ @echo " sno-to-3node - Transition existing SNO cluster to 3-node HA (platform:none)"
@echo ""
@echo "OpenShift Cluster Management:"
@echo " redeploy-cluster - Redeploy OpenShift cluster using dev-scripts make redeploy"
@@ -192,6 +199,7 @@ help:
@echo " clean - Clean OpenShift cluster using dev-scripts clean target"
@echo " full-clean - Fully clean instance cache and OpenShift cluster using dev-scripts realclean target"
@echo " clean-spoke - Clean spoke cluster resources (VMs, network, auth) from assisted installer"
+ @echo " clean-mutable-topology - Remove master-1/2 VMs, disks, DHCP/DNS entries (sno-to-3node cleanup)"
@echo " patch-nodes - Build resource-agents RPM and patch cluster nodes (default version: 4.11)"
@echo ""
@echo "Baremetal Adoption:"
diff --git a/deploy/common.sh b/deploy/common.sh
index b965f6d3..2a2472b5 100755
--- a/deploy/common.sh
+++ b/deploy/common.sh
@@ -66,6 +66,9 @@ function sync_config_files() {
done
}
+# Set USER if not already set (needed by instance.env)
+export USER="${USER:-$(whoami 2>/dev/null || echo 'user')}"
+
if [[ -f "${COMMON_DIR}/aws-hypervisor/instance.env" ]]; then
# shellcheck source=/dev/null
source "${COMMON_DIR}/aws-hypervisor/instance.env"
diff --git a/deploy/openshift-clusters/clean-mutable-topology.yml b/deploy/openshift-clusters/clean-mutable-topology.yml
new file mode 100644
index 00000000..249389d5
--- /dev/null
+++ b/deploy/openshift-clusters/clean-mutable-topology.yml
@@ -0,0 +1,63 @@
+---
+- hosts: metal_machine
+ gather_facts: yes
+
+ pre_tasks:
+ - name: Confirm mutable topology cleanup
+ ansible.builtin.pause:
+ prompt: >-
+ This will destroy the master-1 and master-2 VMs, their disks, DHCP reservations,
+ and DNS entries. master-0 (if still present) will be unaffected.
+ Press Enter to proceed or Ctrl+C to abort.
+ delegate_to: localhost
+ run_once: true
+ when: interactive_mode | default(true) | bool
+
+ - name: Detect cluster domain from kubeconfig (best-effort)
+ shell: |
+ KUBECONFIG={{ dev_scripts_path | default('openshift-metal3/dev-scripts') }}/ocp/{{ sno_cluster_name | default('ostest') }}/auth/kubeconfig \
+ oc get infrastructure cluster -o jsonpath='{.status.apiServerInternalURI}' 2>/dev/null \
+ | sed 's|^https://api-int\.||; s|:6443$||'
+ register: detected_domain
+ changed_when: false
+ failed_when: false
+ ignore_errors: true
+
+ - name: Set cluster domain (use detected, variable override, or default)
+ set_fact:
+ sno_cluster_domain: >-
+ {{ sno_cluster_domain
+ if (sno_cluster_domain is defined and sno_cluster_domain)
+ else (detected_domain.stdout | trim
+ if (detected_domain.stdout is defined and detected_domain.stdout | trim)
+ else 'ostest.test.metalkube.org') }}
+
+ - name: Check live cluster topology
+ shell: |
+ KUBECONFIG={{ dev_scripts_path | default('openshift-metal3/dev-scripts') }}/ocp/{{ sno_cluster_name | default('ostest') }}/auth/kubeconfig \
+ oc get infrastructure cluster \
+ -o jsonpath='{.status.controlPlaneTopology}' 2>/dev/null || echo "Unknown"
+ register: live_topology
+ changed_when: false
+ failed_when: false
+ ignore_errors: true
+
+ - name: Abort if cluster is HA and force_cleanup is not set
+ ansible.builtin.fail:
+ msg: >-
+ Cluster controlPlaneTopology is '{{ live_topology.stdout | trim }}'.
+ Refusing to remove master-1 and master-2 from a live HA cluster.
+ Re-run with -e "force_cleanup=true" to override.
+ when:
+ - live_topology.stdout | trim == "HighlyAvailable"
+ - not (force_cleanup | default(false) | bool)
+
+ tasks:
+ - name: Run mutable topology cleanup
+ import_role:
+ name: mutable-topology/sno-to-3node
+ tasks_from: clean.yml
+
+ - name: Cleanup complete
+ ansible.builtin.debug:
+ msg: "Mutable topology cleanup complete. master-1 and master-2 VMs have been removed."
diff --git a/deploy/openshift-clusters/roles/dev-scripts/install-dev/handlers/main.yml b/deploy/openshift-clusters/roles/dev-scripts/install-dev/handlers/main.yml
index d0a6ea98..3d53ff34 100644
--- a/deploy/openshift-clusters/roles/dev-scripts/install-dev/handlers/main.yml
+++ b/deploy/openshift-clusters/roles/dev-scripts/install-dev/handlers/main.yml
@@ -19,6 +19,18 @@
failed_when: false
listen: Set OCP project
+- name: Warn if oc project failed after all retries
+ debug:
+ msg: >-
+ WARNING: 'oc project openshift-machine-api' still failing after all retries
+ (rc={{ oc_project_result.rc | default('n/a') }}).
+ stderr: {{ oc_project_result.stderr | default('') | trim }}
+ when:
+ - kubeconfig_path is defined
+ - kubeconfig_stat.stat.exists | default(false)
+ - oc_project_result.rc | default(0) != 0
+ listen: Set OCP project
+
- name: Warn about missing kubeconfig
debug:
msg: >-
diff --git a/deploy/openshift-clusters/roles/dev-scripts/install-dev/tasks/config.yml b/deploy/openshift-clusters/roles/dev-scripts/install-dev/tasks/config.yml
index d320495a..8e1a7c41 100644
--- a/deploy/openshift-clusters/roles/dev-scripts/install-dev/tasks/config.yml
+++ b/deploy/openshift-clusters/roles/dev-scripts/install-dev/tasks/config.yml
@@ -28,8 +28,8 @@
msg: >-
Config file {{ config_file[method] }} does not set AGENT_E2E_TEST_SCENARIO,
which is required for the agent installation method. Add
- AGENT_E2E_TEST_SCENARIO="TNF_IPV4" (fencing) or "TNA_IPV4" (arbiter) to
- the config file.
+ AGENT_E2E_TEST_SCENARIO="TNF_IPV4" (fencing), "TNA_IPV4" (arbiter), or
+ "SNO_IPV4" (sno) to the config file.
when:
- method == "agent"
- config_scenario | length == 0
diff --git a/deploy/openshift-clusters/roles/mutable-topology/sno-to-3node/defaults/main.yml b/deploy/openshift-clusters/roles/mutable-topology/sno-to-3node/defaults/main.yml
new file mode 100644
index 00000000..b67ac9fb
--- /dev/null
+++ b/deploy/openshift-clusters/roles/mutable-topology/sno-to-3node/defaults/main.yml
@@ -0,0 +1,41 @@
+---
+# Cluster identity — sno_cluster_domain and sno_master0_ip are auto-detected
+# from the cluster at runtime; sno_cluster_name is a static default used for
+# kubeconfig path and VM naming (override with -e sno_cluster_name=... if needed)
+sno_cluster_name: ostest
+sno_cluster_domain: ""
+sno_infra_id: ""
+
+# Existing master-0 (auto-detected from cluster)
+sno_master0_ip: ""
+
+# New node IPs (static assignments within the dev-scripts DHCP range)
+sno_master1_ip: "192.168.111.21"
+sno_master2_ip: "192.168.111.22"
+
+# VM specs
+sno_vm_vcpus: 6
+sno_vm_ram_mb: 16384
+sno_vm_disk_gb: 50
+
+# Libvirt network (dev-scripts baremetal network)
+sno_libvirt_network: ostestbm
+sno_libvirt_bridge: ostestbm
+
+# RHCOS live ISO path on hypervisor (auto-detected from release image if empty)
+sno_rhcos_live_iso: ""
+
+# Timeouts
+sno_dns_transition_timeout_minutes: 20
+sno_topology_mco_timeout_minutes: 20
+sno_node_join_timeout_minutes: 20
+sno_etcd_timeout_minutes: 15
+
+# Auto-fix MCO drain deadlock during topology transition
+sno_auto_fix_drain: true
+
+# Paths (override if dev-scripts is in a non-standard location)
+sno_kubeconfig: ""
+
+# VM image directory
+sno_vm_image_dir: "/var/lib/libvirt/images"
diff --git a/deploy/openshift-clusters/roles/mutable-topology/sno-to-3node/tasks/boot-nodes.yml b/deploy/openshift-clusters/roles/mutable-topology/sno-to-3node/tasks/boot-nodes.yml
new file mode 100644
index 00000000..f0e97d9b
--- /dev/null
+++ b/deploy/openshift-clusters/roles/mutable-topology/sno-to-3node/tasks/boot-nodes.yml
@@ -0,0 +1,154 @@
+---
+- name: "[boot] Check if RHCOS live ISO exists on hypervisor"
+ stat:
+ path: "/var/lib/libvirt/images/rhcos-live.iso"
+ register: iso_stat
+
+- name: "[boot] Find RHCOS live ISO from dev-scripts cache or release payload"
+ shell: |
+ DEVSCRIPTS_ISO=$(find /var/lib/libvirt/images -name 'rhcos-*-live*.iso' 2>/dev/null | head -1)
+ if [ -n "$DEVSCRIPTS_ISO" ]; then
+ echo "Using existing ISO: $DEVSCRIPTS_ISO"
+ sudo ln -sf "$DEVSCRIPTS_ISO" /var/lib/libvirt/images/rhcos-live.iso
+ exit 0
+ fi
+
+ CACHE_ISO=$(find {{ sno_dev_scripts_path }}/ -name 'rhcos-*-live*.iso' 2>/dev/null | head -1)
+ if [ -n "$CACHE_ISO" ]; then
+ echo "Using dev-scripts cached ISO: $CACHE_ISO"
+ sudo ln -sf "$CACHE_ISO" /var/lib/libvirt/images/rhcos-live.iso
+ exit 0
+ fi
+
+ echo "No cached ISO found, pulling from release payload..."
+ PULL_SECRET="{{ sno_dev_scripts_path }}/pull_secret.json"
+ RELEASE_IMAGE=$(KUBECONFIG="{{ sno_kubeconfig_resolved }}" oc get clusterversion version \
+ -o jsonpath='{.status.desired.image}')
+ echo "Release image: $RELEASE_IMAGE"
+ RHCOS_IMAGE=$(oc adm release info \
+ --image-for=machine-os-images \
+ -a "$PULL_SECRET" \
+ "$RELEASE_IMAGE")
+ echo "machine-os-images: $RHCOS_IMAGE"
+ oc image extract "$RHCOS_IMAGE" \
+ -a "$PULL_SECRET" \
+ --path /coreos/coreos-x86_64.iso:/tmp/ \
+ --confirm
+ sudo mv /tmp/coreos-x86_64.iso /var/lib/libvirt/images/rhcos-live.iso
+ echo "ISO pulled from release payload: /var/lib/libvirt/images/rhcos-live.iso"
+ when: not iso_stat.stat.exists and sno_rhcos_live_iso == ""
+
+- name: "[boot] Set ISO path"
+ set_fact:
+ sno_iso_path: "{{ sno_rhcos_live_iso if sno_rhcos_live_iso else '/var/lib/libvirt/images/rhcos-live.iso' }}"
+
+- name: "[boot] Read master.ign content"
+ slurp:
+ src: /tmp/master.ign
+ register: master_ign_content
+
+- name: "[boot] Set base64-encoded master.ign"
+ set_fact:
+ sno_master_ign_b64: "{{ master_ign_content.content }}"
+
+- name: "[boot] Generate auto-install ignition"
+ template:
+ src: auto-install.ign.j2
+ dest: /tmp/auto-install.ign
+ mode: '0644'
+
+- name: "[boot] Ensure coreos-installer is available"
+ shell: |
+ if ! command -v coreos-installer &>/dev/null; then
+ echo "coreos-installer not found, installing via dnf..."
+ sudo dnf install -y coreos-installer
+ else
+ echo "coreos-installer already installed: $(command -v coreos-installer)"
+ fi
+
+- name: "[boot] Create per-node ISO with embedded ignition"
+ shell: |
+ sudo cp {{ sno_iso_path }} /var/lib/libvirt/images/rhcos-{{ item.hostname }}.iso
+ sudo coreos-installer iso ignition embed -i /tmp/auto-install.ign /var/lib/libvirt/images/rhcos-{{ item.hostname }}.iso -f
+ loop: "{{ sno_new_nodes }}"
+
+- name: "[boot] Boot each VM with ignition-embedded ISO"
+ shell: |
+ VM_NAME="{{ item.name }}"
+ MAC="{{ sno_node_macs[item.hostname] }}"
+
+ sudo virsh destroy "$VM_NAME" 2>/dev/null || true
+ sudo virsh undefine "$VM_NAME" --nvram 2>/dev/null || true
+
+ sudo virt-install \
+ --name "$VM_NAME" \
+ --ram {{ sno_vm_ram_mb }} \
+ --vcpus {{ sno_vm_vcpus }} \
+ --disk {{ sno_vm_image_dir }}/${VM_NAME}.qcow2,bus=virtio \
+ --network network={{ sno_libvirt_network }},model=virtio,mac=${MAC} \
+ --cdrom /var/lib/libvirt/images/rhcos-{{ item.hostname }}.iso \
+ --os-variant rhel9.0 \
+ --graphics none \
+ --noautoconsole \
+ --boot loader=/usr/share/edk2/ovmf/OVMF_CODE.fd,loader_ro=yes,loader_type=pflash,nvram_template=/usr/share/edk2/ovmf/OVMF_VARS.fd,loader_secure=no \
+ --boot hd,cdrom \
+ --tpm none
+ loop: "{{ sno_new_nodes }}"
+
+- name: "[boot] Verify VMs are running (initial install boot)"
+ shell: |
+ sudo virsh domstate {{ item.name }}
+ register: vm_state
+ loop: "{{ sno_new_nodes }}"
+ changed_when: false
+ failed_when: "'running' not in vm_state.stdout"
+
+- name: "[boot] Wait for coreos-installer to complete (VM will power off)"
+ # coreos-installer runs ExecStartPost=systemctl reboot, but RHCOS live issues
+ # an ACPI poweroff rather than a reset. libvirt fires on_poweroff=destroy so
+ # the VM shuts off. We poll until shut off, then boot from disk below.
+ shell: |
+ for i in $(seq 50); do
+ STATE=$(sudo virsh domstate {{ item.name }} 2>/dev/null || echo "unknown")
+ if echo "$STATE" | grep -q "shut off"; then
+ echo "{{ item.name }} shut off after $((i * 20))s - install complete"
+ exit 0
+ fi
+ sleep 20
+ done
+ echo "Timeout: {{ item.name }} did not shut off within 1000s"
+ exit 1
+ loop: "{{ sno_new_nodes }}"
+ changed_when: false
+
+- name: "[boot] Remove CDROM from boot order after install"
+ # Prevent coreos-installer loop: strip the cdrom boot entry so UEFI only
+ # tries the hard disk on subsequent boots.
+ shell: |
+ TMPXML=$(mktemp /tmp/vm-XXXXXX.xml)
+ sudo virsh dumpxml {{ item.name }} > "$TMPXML"
+ sudo sed -i "//d" "$TMPXML"
+ sudo virsh define "$TMPXML"
+ sudo rm -f "$TMPXML"
+ loop: "{{ sno_new_nodes }}"
+ changed_when: true
+
+- name: "[boot] Start VMs to boot from installed RHCOS"
+ shell: |
+ sudo virsh start {{ item.name }}
+ loop: "{{ sno_new_nodes }}"
+ changed_when: true
+
+- name: "[boot] Verify VMs are running from installed disk"
+ shell: |
+ sudo virsh domstate {{ item.name }}
+ register: vm_state_disk
+ loop: "{{ sno_new_nodes }}"
+ changed_when: false
+ failed_when: "'running' not in vm_state_disk.stdout"
+
+- name: "[boot] VMs booted with RHCOS"
+ debug:
+ msg: >-
+ {{ sno_new_nodes | length }} VMs installed and started from disk.
+ Waiting for nodes to join the cluster...
diff --git a/deploy/openshift-clusters/roles/mutable-topology/sno-to-3node/tasks/clean.yml b/deploy/openshift-clusters/roles/mutable-topology/sno-to-3node/tasks/clean.yml
new file mode 100644
index 00000000..575979f8
--- /dev/null
+++ b/deploy/openshift-clusters/roles/mutable-topology/sno-to-3node/tasks/clean.yml
@@ -0,0 +1,72 @@
+---
+- name: "[clean] Stop and undefine new VMs"
+ shell: |
+ sudo virsh destroy {{ item.name }} 2>/dev/null || true
+ sudo virsh undefine {{ item.name }} --remove-all-storage --nvram 2>/dev/null || true
+ loop: "{{ sno_new_nodes }}"
+ failed_when: false
+ changed_when: true
+
+- name: "[clean] Remove disk images"
+ file:
+ path: "{{ sno_vm_image_dir }}/{{ item.name }}.qcow2"
+ state: absent
+ loop: "{{ sno_new_nodes }}"
+ become: true
+ failed_when: false
+
+- name: "[clean] Remove NVRAM files"
+ file:
+ path: "/var/lib/libvirt/qemu/nvram/{{ item.name }}_VARS.fd"
+ state: absent
+ loop: "{{ sno_new_nodes }}"
+ become: true
+ failed_when: false
+
+- name: "[clean] Remove per-node RHCOS ISOs"
+ file:
+ path: "/var/lib/libvirt/images/rhcos-{{ item.hostname }}.iso"
+ state: absent
+ loop: "{{ sno_new_nodes }}"
+ become: true
+ failed_when: false
+
+- name: "[clean] Remove entries from addnhosts"
+ lineinfile:
+ path: "{{ sno_addnhosts_path }}"
+ regexp: "^{{ item.ip }}\\s"
+ state: absent
+ loop: "{{ sno_new_nodes }}"
+ become: true
+ failed_when: false
+
+- name: "[clean] Remove hostname entries from addnhosts"
+ lineinfile:
+ path: "{{ sno_addnhosts_path }}"
+ regexp: "\\s{{ item.hostname }}\\.{{ sno_cluster_domain }}(\\s|$)"
+ state: absent
+ loop: "{{ sno_new_nodes }}"
+ become: true
+ failed_when: false
+
+- name: "[clean] Stop and disable HAProxy service"
+ systemd:
+ name: haproxy
+ state: stopped
+ enabled: false
+ become: true
+ failed_when: false
+
+- name: "[clean] Remove HAProxy configuration"
+ file:
+ path: /etc/haproxy/haproxy.cfg
+ state: absent
+ become: true
+ failed_when: false
+
+- name: "[clean] Cleanup complete"
+ debug:
+ msg: >-
+ Removed new resources: VMs {{ sno_new_nodes | map(attribute='name') | list | join(', ') }},
+ disk images, NVRAM files, ISOs, addnhosts entries, and HAProxy.
+ Use 'make full-clean' to remove the entire cluster including master-0 and libvirt network.
diff --git a/deploy/openshift-clusters/roles/mutable-topology/sno-to-3node/tasks/create-vms.yml b/deploy/openshift-clusters/roles/mutable-topology/sno-to-3node/tasks/create-vms.yml
new file mode 100644
index 00000000..1779afde
--- /dev/null
+++ b/deploy/openshift-clusters/roles/mutable-topology/sno-to-3node/tasks/create-vms.yml
@@ -0,0 +1,162 @@
+---
+- name: "[create-vms] Cleanup stale VMs"
+ shell: |
+ sudo virsh destroy {{ item.name }} 2>/dev/null || true
+ sudo virsh undefine {{ item.name }} --remove-all-storage 2>/dev/null || true
+ sudo rm -f {{ sno_vm_image_dir }}/{{ item.name }}.qcow2
+ loop: "{{ sno_new_nodes }}"
+ failed_when: false
+ changed_when: true
+
+- name: "[create-vms] Remove stale DHCP entries from libvirt network"
+ shell: |
+ NET_XML=$(sudo virsh net-dumpxml {{ sno_libvirt_network }})
+ EXISTING_MAC=$(echo "$NET_XML" | python3 -c "
+ import xml.etree.ElementTree as ET, sys
+ root = ET.parse(sys.stdin).getroot()
+ for host in root.findall('.//dhcp/host'):
+ if host.get('ip') == '{{ item.ip }}':
+ print(host.get('mac',''))
+ " 2>/dev/null || true)
+
+ if [ -n "$EXISTING_MAC" ]; then
+ sudo virsh net-update {{ sno_libvirt_network }} delete ip-dhcp-host \
+ "" --live --config 2>/dev/null || true
+ fi
+
+ sudo virsh net-update {{ sno_libvirt_network }} delete dns-host \
+ "{{ item.hostname }}.{{ sno_cluster_domain }}" \
+ --live --config 2>/dev/null || true
+ sudo virsh net-update {{ sno_libvirt_network }} delete dns-host \
+ "{{ item.hostname }}" \
+ --live --config 2>/dev/null || true
+ loop: "{{ sno_new_nodes }}"
+ failed_when: false
+ changed_when: true
+
+- name: "[create-vms] Clean stale entries from addnhosts"
+ lineinfile:
+ path: "{{ sno_addnhosts_path }}"
+ regexp: "^{{ item.ip }}\\s"
+ state: absent
+ loop: "{{ sno_new_nodes }}"
+ become: true
+ failed_when: false
+
+- name: "[create-vms] Also clean stale hostname entries from addnhosts"
+ lineinfile:
+ path: "{{ sno_addnhosts_path }}"
+ regexp: "\\s{{ item.hostname }}\\.{{ sno_cluster_domain }}(\\s|$)"
+ state: absent
+ loop: "{{ sno_new_nodes }}"
+ become: true
+ failed_when: false
+
+- name: "[create-vms] Flush stale DHCP leases"
+ shell: |
+ python3 -c "
+ import json, os
+ lf = '{{ sno_lease_file }}'
+ if not os.path.exists(lf):
+ print('Lease file does not exist, skipping')
+ exit(0)
+ if os.path.getsize(lf) == 0:
+ print('Lease file is empty, skipping')
+ exit(0)
+ with open(lf) as f:
+ leases = json.load(f)
+ reserved = {{ [sno_master1_ip, sno_master2_ip] | to_json }}
+ before = len(leases)
+ leases = [l for l in leases if l.get('ip-address') not in reserved]
+ after = len(leases)
+ with open(lf, 'w') as f:
+ json.dump(leases, f, indent=2)
+ print(f'Flushed {before - after} stale leases')
+ "
+ become: true
+ changed_when: true
+ failed_when: false
+
+- name: "[create-vms] Generate random MAC addresses"
+ shell: |
+ printf '52:54:00:%02x:%02x:%02x\n' $((RANDOM%256)) $((RANDOM%256)) $((RANDOM%256))
+ register: mac_gen
+ loop: "{{ sno_new_nodes }}"
+ changed_when: false
+
+- name: "[create-vms] Store MAC addresses"
+ set_fact:
+ sno_node_macs: "{{ sno_node_macs | default({}) | combine({item.item.hostname: item.stdout | trim}) }}"
+ loop: "{{ mac_gen.results }}"
+
+- name: "[create-vms] Display VM details"
+ debug:
+ msg: "{{ item.hostname }}: IP={{ item.ip }}, MAC={{ sno_node_macs[item.hostname] }}"
+ loop: "{{ sno_new_nodes }}"
+
+- name: "[create-vms] Create VM disk images"
+ shell: |
+ sudo qemu-img create -f qcow2 {{ sno_vm_image_dir }}/{{ item.name }}.qcow2 {{ sno_vm_disk_gb }}G
+ args:
+ creates: "{{ sno_vm_image_dir }}/{{ item.name }}.qcow2"
+ loop: "{{ sno_new_nodes }}"
+
+- name: "[create-vms] Define VMs"
+ shell: |
+ sudo virt-install \
+ --name {{ item.name }} \
+ --ram {{ sno_vm_ram_mb }} \
+ --vcpus {{ sno_vm_vcpus }} \
+ --disk {{ sno_vm_image_dir }}/{{ item.name }}.qcow2,bus=virtio \
+ --network network={{ sno_libvirt_network }},model=virtio,mac={{ sno_node_macs[item.hostname] }} \
+ --os-variant rhel9.0 \
+ --graphics none \
+ --noautoconsole \
+ --boot loader=/usr/share/edk2/ovmf/OVMF_CODE.fd,loader_ro=yes,loader_type=pflash,nvram_template=/usr/share/edk2/ovmf/OVMF_VARS.fd,loader_secure=no \
+ --boot hd \
+ --tpm none \
+ --noreboot \
+ --import \
+ --print-xml | sudo virsh define /dev/stdin
+ loop: "{{ sno_new_nodes }}"
+
+- name: "[create-vms] Add DHCP reservations to libvirt network"
+ shell: |
+ sudo virsh net-update {{ sno_libvirt_network }} add ip-dhcp-host \
+ "" \
+ --live --config
+ loop: "{{ sno_new_nodes }}"
+
+- name: "[create-vms] Add DNS entries to libvirt network"
+ shell: |
+ sudo virsh net-update {{ sno_libvirt_network }} add dns-host \
+ "{{ item.hostname }}.{{ sno_cluster_domain }}" \
+ --live --config
+ loop: "{{ sno_new_nodes }}"
+ failed_when: false
+
+- name: "[create-vms] Add entries to addnhosts"
+ lineinfile:
+ path: "{{ sno_addnhosts_path }}"
+ line: "{{ item.ip }} {{ item.hostname }}.{{ sno_cluster_domain }} {{ item.hostname }}"
+ regexp: "^{{ item.ip }}\\s"
+ state: present
+ loop: "{{ sno_new_nodes }}"
+ become: true
+
+- name: "[create-vms] Reload libvirt dnsmasq"
+ shell: |
+ sudo kill -HUP $(cat /run/libvirt/network/{{ sno_libvirt_bridge }}.pid) 2>/dev/null || true
+ changed_when: true
+
+- name: "[create-vms] Verify DNS for new nodes"
+ shell: |
+ dig +short {{ item.hostname }}.{{ sno_cluster_domain }} @192.168.111.1
+ register: dns_verify
+ changed_when: false
+ failed_when: dns_verify.stdout | trim == ""
+ loop: "{{ sno_new_nodes }}"
+
+- name: "[create-vms] VMs created and DNS configured"
+ debug:
+ msg: "{{ sno_new_nodes | length }} VMs created with DNS entries."
diff --git a/deploy/openshift-clusters/roles/mutable-topology/sno-to-3node/tasks/dns-transition.yml b/deploy/openshift-clusters/roles/mutable-topology/sno-to-3node/tasks/dns-transition.yml
new file mode 100644
index 00000000..035d0fcf
--- /dev/null
+++ b/deploy/openshift-clusters/roles/mutable-topology/sno-to-3node/tasks/dns-transition.yml
@@ -0,0 +1,52 @@
+---
+- name: "[dns] Delete SNO dnsmasq MachineConfig (switch to HAProxy DNS)"
+ shell: |
+ oc delete mc 50-master-dnsmasq-configuration --ignore-not-found=true
+ environment: "{{ sno_oc_env }}"
+ register: delete_dnsmasq_mc
+ changed_when: "'deleted' in delete_dnsmasq_mc.stdout"
+ failed_when: false
+
+- name: "[dns] Display dnsmasq MC deletion status"
+ debug:
+ msg: "{{ 'MachineConfig 50-master-dnsmasq-configuration deleted' if delete_dnsmasq_mc.changed else 'MachineConfig not found (may have been deleted previously)' }}"
+
+- name: "[dns] Wait for MCO to apply DNS configuration changes"
+ include_tasks: poll-mcp.yml
+ vars:
+ mco_poll_purpose: "DNS transition"
+ mco_poll_timeout_minutes: "{{ sno_dns_transition_timeout_minutes }}"
+ mco_poll_fail_on_timeout: true
+ mco_poll_auto_fix_drain: false
+ mco_poll_check_rendered_config: false # Standalone MC deletion - generation/config won't change
+ when: delete_dnsmasq_mc.changed
+
+- name: "[dns] Verify DNS configuration on nodes"
+ shell: |
+ # Check that nodes are using cluster DNS, not dnsmasq
+ oc get nodes -o json | jq -r '.items[].metadata.name' | while read node; do
+ RESOLV=$(oc debug node/$node -- chroot /host cat /etc/resolv.conf 2>/dev/null | grep "^nameserver" | head -1)
+ echo "$node: $RESOLV"
+
+ # Verify NOT using 192.168.111.80 (dnsmasq on master-0)
+ if echo "$RESOLV" | grep -q "192.168.111.80"; then
+ echo "ERROR: Node $node still using dnsmasq (192.168.111.80)"
+ exit 1
+ fi
+ done
+ environment: "{{ sno_oc_env }}"
+ register: dns_verify
+ changed_when: false
+ failed_when: false
+ when: delete_dnsmasq_mc.changed
+
+- name: "[dns] DNS transition complete"
+ debug:
+ msg: |
+ {% if delete_dnsmasq_mc.changed %}
+ ✓ SNO dnsmasq MachineConfig removed
+ ✓ MCO rollout completed
+ ✓ Nodes using correct DNS resolvers
+ {% else %}
+ ✓ SNO dnsmasq MachineConfig already removed (skipped MCO wait)
+ {% endif %}
diff --git a/deploy/openshift-clusters/roles/mutable-topology/sno-to-3node/tasks/haproxy-setup.yml b/deploy/openshift-clusters/roles/mutable-topology/sno-to-3node/tasks/haproxy-setup.yml
new file mode 100644
index 00000000..e15aaa9f
--- /dev/null
+++ b/deploy/openshift-clusters/roles/mutable-topology/sno-to-3node/tasks/haproxy-setup.yml
@@ -0,0 +1,352 @@
+---
+- name: "[haproxy] Check if haproxy is already installed"
+ shell: rpm -q haproxy
+ register: haproxy_check
+ changed_when: false
+ failed_when: false
+
+- name: "[haproxy] Install haproxy package on hypervisor"
+ package:
+ name: haproxy
+ state: present
+ become: true
+ when: haproxy_check.rc != 0
+
+- name: "[haproxy] Create haproxy runtime directory"
+ file:
+ path: /run/haproxy
+ state: directory
+ owner: haproxy
+ group: haproxy
+ mode: '0755'
+ become: true
+
+- name: "[haproxy] Create haproxy configuration for OpenShift"
+ copy:
+ dest: /etc/haproxy/haproxy.cfg
+ content: |
+ global
+ log /dev/log local0
+ log /dev/log local1 notice
+ chroot /var/lib/haproxy
+ stats socket /run/haproxy/admin.sock mode 660 level admin
+ stats timeout 30s
+ user haproxy
+ group haproxy
+ daemon
+
+ defaults
+ log global
+ mode tcp
+ option tcplog
+ option dontlognull
+ timeout connect 5000
+ timeout client 50000
+ timeout server 50000
+
+ # API server (6443)
+ frontend api_server
+ bind *:6443
+ default_backend api_server_backend
+
+ backend api_server_backend
+ balance roundrobin
+ option tcp-check
+ server master-0 {{ sno_master0_ip }}:6443 check
+ server master-1 {{ sno_master1_ip }}:6443 check
+ server master-2 {{ sno_master2_ip }}:6443 check
+
+ # Ingress HTTP (80)
+ frontend ingress_http
+ bind *:80
+ default_backend ingress_http_backend
+
+ backend ingress_http_backend
+ balance roundrobin
+ option tcp-check
+ server master-0 {{ sno_master0_ip }}:80 check
+ server master-1 {{ sno_master1_ip }}:80 check
+ server master-2 {{ sno_master2_ip }}:80 check
+
+ # Ingress HTTPS (443)
+ frontend ingress_https
+ bind *:443
+ default_backend ingress_https_backend
+
+ backend ingress_https_backend
+ balance roundrobin
+ option tcp-check
+ server master-0 {{ sno_master0_ip }}:443 check
+ server master-1 {{ sno_master1_ip }}:443 check
+ server master-2 {{ sno_master2_ip }}:443 check
+
+ # Machine Config Server (22623)
+ frontend machine_config
+ bind *:22623
+ default_backend machine_config_backend
+
+ backend machine_config_backend
+ balance roundrobin
+ option tcp-check
+ server master-0 {{ sno_master0_ip }}:22623 check
+ server master-1 {{ sno_master1_ip }}:22623 check
+ server master-2 {{ sno_master2_ip }}:22623 check
+ owner: root
+ group: root
+ mode: '0644'
+ become: true
+ register: haproxy_config
+
+- name: "[haproxy] Allow HAProxy to bind to OpenShift ports in SELinux"
+ shell: |
+ if ! semanage port -l | grep -q "^http_port_t.*tcp.*{{ item }}"; then
+ semanage port -a -t http_port_t -p tcp {{ item }}
+ echo "CHANGED"
+ else
+ echo "UNCHANGED"
+ fi
+ loop:
+ - 6443
+ - 22623
+ become: true
+ register: selinux_port_config
+ changed_when: "'CHANGED' in selinux_port_config.stdout"
+
+- name: "[haproxy] Check existing firewall rules (public zone)"
+ shell: |
+ firewall-cmd --zone=public --list-ports | grep -q "{{ item }}"
+ loop:
+ - 6443/tcp
+ - 80/tcp
+ - 443/tcp
+ - 22623/tcp
+ register: fw_public_check
+ failed_when: false
+ changed_when: false
+ become: true
+
+- name: "[haproxy] Configure firewall for HAProxy ports"
+ firewalld:
+ port: "{{ item.item }}"
+ permanent: true
+ state: enabled
+ immediate: true
+ loop: "{{ fw_public_check.results }}"
+ when: item.rc != 0
+ become: true
+ failed_when: false
+
+- name: "[haproxy] Check existing firewall rules (libvirt zone)"
+ shell: |
+ firewall-cmd --zone=libvirt --list-ports | grep -q "{{ item }}"
+ loop:
+ - 6443/tcp
+ - 80/tcp
+ - 443/tcp
+ - 22623/tcp
+ register: fw_libvirt_check
+ failed_when: false
+ changed_when: false
+ become: true
+
+- name: "[haproxy] Allow HAProxy to reach backends through libvirt zone"
+ firewalld:
+ zone: libvirt
+ port: "{{ item.item }}"
+ permanent: true
+ state: enabled
+ immediate: true
+ loop: "{{ fw_libvirt_check.results }}"
+ when: item.rc != 0
+ become: true
+ failed_when: false
+
+- name: "[haproxy] Enable and start haproxy service"
+ systemd:
+ name: haproxy
+ state: started
+ enabled: true
+ daemon_reload: true
+ become: true
+
+- name: "[haproxy] Restart haproxy if config changed"
+ systemd:
+ name: haproxy
+ state: restarted
+ become: true
+ when: haproxy_config.changed
+
+- name: "[haproxy] Wait for haproxy to be ready"
+ wait_for:
+ port: 6443
+ host: 192.168.111.1
+ timeout: 30
+
+- name: "[haproxy] Remove stale apps entries from /etc/hosts before DNS reconfiguration"
+ lineinfile:
+ path: /etc/hosts
+ regexp: "apps\\.{{ sno_cluster_domain | regex_escape() }}"
+ state: absent
+ become: true
+ register: hosts_cleanup
+
+- name: "[haproxy] Display /etc/hosts cleanup"
+ debug:
+ msg: "✓ Removed stale apps routes from /etc/hosts (preparing for wildcard DNS)"
+ when: hosts_cleanup.changed
+
+- name: "[haproxy] Generate libvirt network config for 3-node HA"
+ template:
+ src: libvirt-network-3node.xml.j2
+ dest: /tmp/libvirt-network-3node.xml
+ mode: '0644'
+
+- name: "[haproxy] Update libvirt network with HAProxy DNS configuration"
+ shell: |
+ sudo virsh net-destroy {{ sno_libvirt_network }} 2>/dev/null || true
+ sudo virsh net-undefine {{ sno_libvirt_network }}
+ sudo virsh net-define /tmp/libvirt-network-3node.xml
+ sudo virsh net-start {{ sno_libvirt_network }}
+ register: libvirt_network_update
+ changed_when: true
+
+- name: "[haproxy] Update API entries in addnhosts to point to HAProxy (round robin)"
+ lineinfile:
+ path: "{{ sno_addnhosts_path }}"
+ regexp: "^192\\.168\\.111\\.[0-9]+\\s+api\\.{{ sno_cluster_domain | regex_escape }}"
+ line: "192.168.111.1 api.{{ sno_cluster_domain }} api"
+ state: present
+ become: true
+ failed_when: false
+ register: api_addnhosts
+
+- name: "[haproxy] Update API-INT entries in addnhosts to point to HAProxy (round robin)"
+ lineinfile:
+ path: "{{ sno_addnhosts_path }}"
+ regexp: "^192\\.168\\.111\\.[0-9]+\\s+api-int\\.{{ sno_cluster_domain | regex_escape }}"
+ line: "192.168.111.1 api-int.{{ sno_cluster_domain }} api-int"
+ state: present
+ become: true
+ failed_when: false
+ register: api_int_addnhosts
+
+- name: "[haproxy] Update NetworkManager dnsmasq API entry"
+ lineinfile:
+ path: "/etc/NetworkManager/dnsmasq.d/openshift-{{ sno_cluster_name }}.conf"
+ regexp: "^address=/api\\.{{ sno_cluster_domain | regex_escape }}/"
+ line: "address=/api.{{ sno_cluster_domain }}/192.168.111.1"
+ state: present
+ become: true
+ failed_when: false
+ register: nm_dnsmasq_config
+
+- name: "[haproxy] Restart NetworkManager to flush DNS cache"
+ systemd:
+ name: NetworkManager
+ state: restarted
+ become: true
+ register: nm_restart
+ when: >
+ nm_dnsmasq_config.changed or
+ api_addnhosts.changed or
+ api_int_addnhosts.changed
+
+- name: "[haproxy] Restart libvirtd to restore VM network connectivity"
+ systemd:
+ name: libvirtd
+ state: restarted
+ become: true
+ register: libvirtd_restart
+ when: libvirt_network_update.changed
+
+- name: "[haproxy] Verify DNS configuration"
+ when: >
+ libvirt_network_update.changed or
+ api_addnhosts.changed or
+ api_int_addnhosts.changed or
+ nm_dnsmasq_config.changed
+ block:
+ - name: "[haproxy] Verify API DNS resolution through HAProxy"
+ shell: |
+ RESULT=$(dig +short api.{{ sno_cluster_domain }} @192.168.111.1)
+ echo "DNS result: $RESULT"
+ if echo "$RESULT" | grep -q "192.168.111.1"; then
+ exit 0
+ else
+ exit 1
+ fi
+ register: api_dns_check
+ changed_when: false
+ failed_when: false
+ retries: 5
+ delay: 2
+ until: api_dns_check.rc == 0
+
+ - name: "[haproxy] Check API DNS result"
+ fail:
+ msg: |
+ API DNS not pointing to HAProxy!
+ Expected: 192.168.111.1
+ Got: {{ api_dns_check.stdout }}
+ when: api_dns_check.rc != 0
+
+ - name: "[haproxy] Verify API-INT DNS resolution through HAProxy"
+ shell: |
+ RESULT=$(dig +short api-int.{{ sno_cluster_domain }} @192.168.111.1)
+ echo "DNS result: $RESULT"
+ if echo "$RESULT" | grep -q "192.168.111.1"; then
+ exit 0
+ else
+ exit 1
+ fi
+ register: api_int_dns_check
+ changed_when: false
+ failed_when: false
+ retries: 5
+ delay: 2
+ until: api_int_dns_check.rc == 0
+
+ - name: "[haproxy] Check API-INT DNS result"
+ fail:
+ msg: |
+ API-INT DNS not pointing to HAProxy!
+ Expected: 192.168.111.1
+ Got: {{ api_int_dns_check.stdout }}
+ when: api_int_dns_check.rc != 0
+
+ - name: "[haproxy] Verify wildcard DNS for non-existent apps route"
+ shell: |
+ dig +short test.apps.{{ sno_cluster_domain }} @192.168.111.1
+ register: wildcard_dns_check
+ changed_when: false
+ failed_when: wildcard_dns_check.stdout | trim != "192.168.111.1"
+
+ - name: "[haproxy] Verify DNS for OAuth route (real route)"
+ shell: |
+ dig +short oauth-openshift.apps.{{ sno_cluster_domain }} @192.168.111.1
+ register: oauth_dns_check
+ changed_when: false
+ failed_when: oauth_dns_check.stdout | trim != "192.168.111.1"
+
+ - name: "[haproxy] Verify DNS for Console route (real route)"
+ shell: |
+ dig +short console-openshift-console.apps.{{ sno_cluster_domain }} @192.168.111.1
+ register: console_dns_check
+ changed_when: false
+ failed_when: console_dns_check.stdout | trim != "192.168.111.1"
+
+ - name: "[haproxy] Display apps wildcard DNS verification"
+ debug:
+ msg: |
+ ✓ Wildcard DNS: test.apps → {{ wildcard_dns_check.stdout | trim }}
+ ✓ OAuth route: oauth-openshift.apps → {{ oauth_dns_check.stdout | trim }}
+ ✓ Console route: console-openshift-console.apps → {{ console_dns_check.stdout | trim }}
+
+- name: "[haproxy] HAProxy setup complete"
+ debug:
+ msg: |
+ ✓ HAProxy configured and running on hypervisor (192.168.111.1)
+ ✓ DNS updated to route all traffic through HAProxy (round-robin load balancing)
+ ✓ API/API-INT entries in addnhosts point to HAProxy (.1)
+ ✓ Backends: 3 masters ({{ sno_master0_ip }}, {{ sno_master1_ip }}, {{ sno_master2_ip }})
+ ✓ Services: API (6443), Ingress (80/443), MCS (22623)
diff --git a/deploy/openshift-clusters/roles/mutable-topology/sno-to-3node/tasks/ignition.yml b/deploy/openshift-clusters/roles/mutable-topology/sno-to-3node/tasks/ignition.yml
new file mode 100644
index 00000000..d7147c6a
--- /dev/null
+++ b/deploy/openshift-clusters/roles/mutable-topology/sno-to-3node/tasks/ignition.yml
@@ -0,0 +1,33 @@
+---
+- name: "[ignition] Extract MCS CA certificate"
+ shell: |
+ oc get configmap -n openshift-machine-config-operator machine-config-server-ca \
+ -o jsonpath='{.data.ca-bundle\.crt}' --request-timeout=30s
+ environment: "{{ sno_oc_env }}"
+ register: mcs_ca_raw
+ changed_when: false
+
+- name: "[ignition] Base64-encode MCS CA"
+ set_fact:
+ sno_mcs_ca_b64: "{{ mcs_ca_raw.stdout | b64encode }}"
+
+- name: "[ignition] Generate master.ign from template"
+ template:
+ src: master.ign.j2
+ dest: /tmp/master.ign
+ mode: '0644'
+
+- name: "[ignition] Verify MCS reachability"
+ shell: |
+ curl -sk --max-time 10 https://api-int.{{ sno_cluster_domain }}:22623/healthz
+ register: mcs_health
+ changed_when: false
+ failed_when: false
+
+- name: "[ignition] Display MCS status"
+ debug:
+ msg: "MCS healthz: {{ mcs_health.stdout | default('unreachable') }}"
+
+- name: "[ignition] Ignition config generated"
+ debug:
+ msg: "master.ign written to /tmp/master.ign for domain {{ sno_cluster_domain }}"
diff --git a/deploy/openshift-clusters/roles/mutable-topology/sno-to-3node/tasks/ingress-reconfigure.yml b/deploy/openshift-clusters/roles/mutable-topology/sno-to-3node/tasks/ingress-reconfigure.yml
new file mode 100644
index 00000000..b46c3142
--- /dev/null
+++ b/deploy/openshift-clusters/roles/mutable-topology/sno-to-3node/tasks/ingress-reconfigure.yml
@@ -0,0 +1,121 @@
+---
+# Reconfigure IngressController for HA after MCO rollout and operator sync
+# This must run AFTER Infrastructure.status has been updated AND operators have synced
+# The IngressController operator reads Infrastructure.status to determine default replicas
+
+- name: "[ingress] Check current IngressController replica configuration"
+ shell:
+ cmd: |
+ REPLICAS=$(oc get ingresscontroller/default -n openshift-ingress-operator -o json 2>/dev/null | jq -r '.spec.replicas // empty')
+ echo "replicas=${REPLICAS}"
+
+ # Empty means field not set - defaults to 2 in HA mode, so it's already correct
+ if [ -z "$REPLICAS" ]; then
+ echo "needs_deletion=false (field unset - using HA defaults)"
+ exit 0
+ fi
+
+ # If replicas is 1, we need to delete to trigger HA recreation
+ if [ "$REPLICAS" = "1" ]; then
+ echo "needs_deletion=true (currently replicas=1)"
+ exit 0
+ fi
+
+ # If replicas >= 2, already configured for HA
+ echo "needs_deletion=false (already replicas=$REPLICAS)"
+ environment: "{{ sno_oc_env }}"
+ register: ingress_check
+ changed_when: false
+
+- name: "[ingress] Display current IngressController status"
+ debug:
+ msg: "{{ ingress_check.stdout_lines }}"
+
+- name: "[ingress] Delete IngressController to trigger recreation with HA defaults"
+ shell: |
+ oc delete ingresscontroller/default -n openshift-ingress-operator
+ environment: "{{ sno_oc_env }}"
+ register: delete_ingress
+ changed_when: delete_ingress.rc == 0
+ when: "'needs_deletion=true' in ingress_check.stdout"
+
+- name: "[ingress] Wait for IngressController automatic recreation"
+ shell: |
+ oc get ingresscontroller/default -n openshift-ingress-operator -o json 2>&1
+ environment: "{{ sno_oc_env }}"
+ register: ingress_recreate_check
+ changed_when: false
+ retries: 12
+ delay: 10
+ until: ingress_recreate_check.rc == 0
+ when: "'needs_deletion=true' in ingress_check.stdout"
+
+- name: "[ingress] Verify IngressController has correct replicas for HA"
+ shell:
+ cmd: |
+ REPLICAS=$(oc get ingresscontroller/default -n openshift-ingress-operator -o json 2>/dev/null | jq -r '.spec.replicas // empty')
+ echo "replicas=${REPLICAS}"
+
+ # Empty or >= 2 is correct for HA
+ if [ -z "$REPLICAS" ] || [ "$REPLICAS" -ge 2 ] 2>/dev/null; then
+ exit 0
+ fi
+
+ # If still 1, that's an error
+ if [ "$REPLICAS" = "1" ]; then
+ echo "ERROR: IngressController still has replicas=1 after HA transition"
+ exit 1
+ fi
+ environment: "{{ sno_oc_env }}"
+ register: ingress_config
+ changed_when: false
+ failed_when: ingress_config.rc != 0
+
+- name: "[ingress] Display IngressController configuration"
+ debug:
+ msg: "✓ IngressController configured for HA: {{ ingress_config.stdout }}"
+
+- name: "[ingress] Wait for router pods to be ready"
+ shell: |
+ # Check if router pods are running with hostNetwork
+ oc get pods -n openshift-ingress -l ingresscontroller.operator.openshift.io/deployment-ingresscontroller=default -o json | jq -e '
+ .items[] |
+ select(.spec.hostNetwork != true) |
+ "\(.metadata.name): ERROR - not using hostNetwork"
+ ' && exit 1
+
+ oc get pods -n openshift-ingress -l ingresscontroller.operator.openshift.io/deployment-ingresscontroller=default -o json | jq -e '
+ .items[] |
+ select(.status.phase != "Running") |
+ "\(.metadata.name): phase=\(.status.phase)"
+ ' && exit 1
+
+ oc get pods -n openshift-ingress -l ingresscontroller.operator.openshift.io/deployment-ingresscontroller=default -o json | jq -e '
+ .items[] |
+ select(any(.status.containerStatuses[]?; .ready != true)) |
+ "\(.metadata.name): not ready"
+ ' && exit 1
+
+ echo "All router pods ready"
+ environment: "{{ sno_oc_env }}"
+ register: router_pods_check
+ changed_when: false
+ failed_when: false
+ retries: 24
+ delay: 10
+ until: >-
+ router_pods_check is defined and
+ router_pods_check.rc is defined and
+ router_pods_check.rc == 0 and
+ 'All router pods ready' in router_pods_check.stdout
+
+- name: "[ingress] IngressController reconfiguration complete"
+ debug:
+ msg: |
+ {% if 'needs_deletion=true' in ingress_check.stdout %}
+ ✓ IngressController recreated with HA defaults
+ ✓ Router pods ready and serving traffic
+ {% else %}
+ ✓ IngressController already configured for HA (skipped deletion)
+ ✓ Router pods ready and serving traffic
+ {% endif %}
diff --git a/deploy/openshift-clusters/roles/mutable-topology/sno-to-3node/tasks/main.yml b/deploy/openshift-clusters/roles/mutable-topology/sno-to-3node/tasks/main.yml
new file mode 100644
index 00000000..1342d8f7
--- /dev/null
+++ b/deploy/openshift-clusters/roles/mutable-topology/sno-to-3node/tasks/main.yml
@@ -0,0 +1,63 @@
+---
+- name: "Phase 0: Preflight checks"
+ import_tasks: preflight.yml
+
+- name: "[main] Check if VMs exist"
+ shell: |
+ sudo virsh list --all --name | grep -E '^({{ sno_cluster_name }}_master_1|{{ sno_cluster_name }}_master_2)$' | wc -l
+ register: vm_count_check
+ changed_when: false
+ failed_when: false
+
+- name: "[main] Check if etcd has 3+ members"
+ shell: |
+ oc get pods -n openshift-etcd -l app=etcd --no-headers 2>/dev/null | wc -l
+ environment: "{{ sno_oc_env }}"
+ register: etcd_count_check
+ changed_when: false
+ failed_when: false
+
+- name: "[main] Set phase completion flags"
+ set_fact:
+ sno_needs_vms: "{{ (vm_count_check.stdout | trim | int) < 2 }}"
+ sno_needs_nodes: "{{ not sno_has_three_nodes }}"
+ sno_needs_etcd: "{{ (etcd_count_check.stdout | trim | int) < 3 }}"
+
+- name: "Phase 1: Generate ignition config"
+ import_tasks: ignition.yml
+ when: sno_needs_nodes
+
+- name: "Phase 2: Create new VMs"
+ import_tasks: create-vms.yml
+ when: sno_needs_vms
+
+- name: "Phase 3: Boot nodes with RHCOS"
+ import_tasks: boot-nodes.yml
+ when: sno_needs_vms
+
+- name: "Phase 4: Wait for nodes to join"
+ import_tasks: wait-nodes.yml
+ when: sno_needs_nodes
+
+- name: "Phase 5: Wait for etcd scaling"
+ import_tasks: wait-etcd.yml
+ when: sno_needs_etcd
+
+- name: "Phase 6: Configure HAProxy load balancer"
+ import_tasks: haproxy-setup.yml
+
+- name: "Phase 7: DNS transition to HAProxy"
+ import_tasks: dns-transition.yml
+
+- name: "Phase 8: Topology transition"
+ import_tasks: topology.yml
+
+- name: "Phase 9: MCO rollout after topology transition"
+ import_tasks: mco-rollout.yml
+ when: not sno_already_ha
+
+- name: "Phase 10: IngressController reconfiguration for HA"
+ import_tasks: ingress-reconfigure.yml
+
+- name: "Phase 11: Final verification"
+ import_tasks: verify.yml
diff --git a/deploy/openshift-clusters/roles/mutable-topology/sno-to-3node/tasks/mco-rollout.yml b/deploy/openshift-clusters/roles/mutable-topology/sno-to-3node/tasks/mco-rollout.yml
new file mode 100644
index 00000000..f9338e10
--- /dev/null
+++ b/deploy/openshift-clusters/roles/mutable-topology/sno-to-3node/tasks/mco-rollout.yml
@@ -0,0 +1,12 @@
+---
+- name: "[mco] Display MCO rollout monitoring start"
+ debug:
+ msg: "Monitoring MCO rollout after topology transition (timeout: {{ sno_topology_mco_timeout_minutes }} min)"
+
+- name: "[mco] Wait for MCO rollout after topology transition"
+ include_tasks: poll-mcp.yml
+ vars:
+ mco_poll_purpose: "topology transition"
+ mco_poll_timeout_minutes: "{{ sno_topology_mco_timeout_minutes }}"
+ mco_poll_fail_on_timeout: false
+ mco_poll_auto_fix_drain: "{{ sno_auto_fix_drain | default(true) }}"
diff --git a/deploy/openshift-clusters/roles/mutable-topology/sno-to-3node/tasks/poll-mcp.yml b/deploy/openshift-clusters/roles/mutable-topology/sno-to-3node/tasks/poll-mcp.yml
new file mode 100644
index 00000000..2623966f
--- /dev/null
+++ b/deploy/openshift-clusters/roles/mutable-topology/sno-to-3node/tasks/poll-mcp.yml
@@ -0,0 +1,159 @@
+---
+# Reusable MCP polling task for monitoring MCO rollouts
+# This uses two-phase monitoring to avoid instant completion when MCO is fast
+#
+# Parameters:
+# - mco_poll_purpose: Description of what we're waiting for
+# - mco_poll_timeout_minutes: Timeout in minutes for completion phase
+# - mco_poll_fail_on_timeout: Whether to fail the playbook if timeout (default: true)
+# - mco_poll_auto_fix_drain: Whether to auto-fix drain deadlock (default: false)
+# - mco_poll_check_rendered_config: Whether to check generation/config matching (default: true)
+# Set to false for standalone MC changes
+
+# Phase 1: Wait for MCO to START processing (see Updating=True OR generation change)
+- name: "[mco] Wait for MCO to start processing {{ mco_poll_purpose }} (up to 2 min)"
+ shell: |
+ MCP_JSON=$(oc get mcp master -o json 2>/dev/null) || exit 1
+
+ UPDATING=$(echo "$MCP_JSON" | jq -r '(.status.conditions[] | select(.type=="Updating") | .status) // "Unknown"')
+ OBSERVED_GEN=$(echo "$MCP_JSON" | jq -r '.status.observedGeneration // 0')
+ METADATA_GEN=$(echo "$MCP_JSON" | jq -r '.metadata.generation // 0')
+ UPDATED_COUNT=$(echo "$MCP_JSON" | jq -r '.status.updatedMachineCount // 0')
+ READY_COUNT=$(echo "$MCP_JSON" | jq -r '.status.readyMachineCount // 0')
+ MACHINE_COUNT=$(echo "$MCP_JSON" | jq -r '.status.machineCount // 0')
+
+ echo "STATUS=updating:${UPDATING},gen:${OBSERVED_GEN}/${METADATA_GEN},machines:${MACHINE_COUNT}/${UPDATED_COUNT}/${READY_COUNT}"
+
+ # MCO has started if ANY of these are true:
+ # - Updating=True (actively rolling out)
+ # - observedGeneration < metadata.generation (hasn't seen new spec yet)
+ # - Not all machines updated (updatedMachineCount < machineCount)
+ # - Not all machines ready (readyMachineCount < machineCount)
+
+ if [ "$UPDATING" = "True" ]; then
+ echo "MCO_STARTED=true (Updating=True)"
+ exit 0
+ fi
+
+ if [ "$OBSERVED_GEN" != "$METADATA_GEN" ]; then
+ echo "MCO_STARTED=true (generation mismatch: observed=$OBSERVED_GEN, metadata=$METADATA_GEN)"
+ exit 0
+ fi
+
+ if [ "$UPDATED_COUNT" != "$MACHINE_COUNT" ] || [ "$READY_COUNT" != "$MACHINE_COUNT" ]; then
+ echo "MCO_STARTED=true (machines not all updated/ready)"
+ exit 0
+ fi
+
+ # If we get here, MCO either hasn't started OR already completed
+ # Give it a few more seconds
+ echo "MCO_STARTED=false (still waiting)"
+ exit 1
+ environment: "{{ sno_oc_env }}"
+ register: mco_start_check
+ changed_when: false
+ failed_when: false
+ retries: 12 # 2 minutes (12 * 10s)
+ delay: 10
+ until: mco_start_check.rc == 0
+
+- name: "[mco] Display MCO start status"
+ debug:
+ msg: "{{ mco_start_check.stdout_lines | last }}"
+
+# Phase 2: Wait for MCO to COMPLETE processing
+- name: "[mco] Poll MCP master for {{ mco_poll_purpose }} completion"
+ shell: |
+ MCP_JSON=$(oc get mcp master -o json 2>/dev/null) || {
+ echo "STATUS=api_unreachable"
+ echo "RESULT=pending"
+ exit 0
+ }
+
+ # Extract MCP state using jq
+ UPDATED=$(echo "$MCP_JSON" | jq -r '(.status.conditions[] | select(.type=="Updated") | .status) // "Unknown"')
+ UPDATING=$(echo "$MCP_JSON" | jq -r '(.status.conditions[] | select(.type=="Updating") | .status) // "Unknown"')
+ DEGRADED=$(echo "$MCP_JSON" | jq -r '(.status.conditions[] | select(.type=="Degraded") | .status) // "Unknown"')
+ SPEC_CONFIG=$(echo "$MCP_JSON" | jq -r '.spec.configuration.name // ""')
+ STATUS_CONFIG=$(echo "$MCP_JSON" | jq -r '.status.configuration.name // ""')
+ MACHINE_COUNT=$(echo "$MCP_JSON" | jq -r '.status.machineCount // 0')
+ UPDATED_COUNT=$(echo "$MCP_JSON" | jq -r '.status.updatedMachineCount // 0')
+ READY_COUNT=$(echo "$MCP_JSON" | jq -r '.status.readyMachineCount // 0')
+ OBSERVED_GEN=$(echo "$MCP_JSON" | jq -r '.status.observedGeneration // 0')
+ METADATA_GEN=$(echo "$MCP_JSON" | jq -r '.metadata.generation // 0')
+
+ CORDONED_NODES=$(oc get nodes --no-headers 2>/dev/null | grep SchedulingDisabled | awk '{print $1}' || true)
+
+ echo "STATUS=machines:${MACHINE_COUNT}/${UPDATED_COUNT}/${READY_COUNT},updated:${UPDATED},updating:${UPDATING},degraded:${DEGRADED},gen:${OBSERVED_GEN}/${METADATA_GEN},config_match:$([ "$SPEC_CONFIG" = "$STATUS_CONFIG" ] && echo yes || echo no),cordoned:${CORDONED_NODES:-none}"
+
+ # Common completion criteria (always required):
+ # - All machines are updated and ready
+ # - Conditions show Updated=True, Updating=False, Degraded=False
+ COMMON_READY=false
+ if [ "$MACHINE_COUNT" = "$UPDATED_COUNT" ] && \
+ [ "$MACHINE_COUNT" = "$READY_COUNT" ] && \
+ [ "$UPDATED" = "True" ] && \
+ [ "$UPDATING" = "False" ] && \
+ [ "$DEGRADED" = "False" ]; then
+ COMMON_READY=true
+ fi
+
+ # For rendered config changes: also check generation and config matching
+ # For standalone MC changes: skip generation/config checks (they don't change)
+ CHECK_RENDERED="{{ mco_poll_check_rendered_config | default(true) | bool }}"
+
+ if [ "$CHECK_RENDERED" = "True" ]; then
+ # Rendered config mode: MCO has new spec to reconcile
+ # Additional checks: observedGeneration == generation, spec.config == status.config
+ if [ "$COMMON_READY" = "true" ] && \
+ [ "$OBSERVED_GEN" = "$METADATA_GEN" ] && \
+ [ "$SPEC_CONFIG" = "$STATUS_CONFIG" ]; then
+ echo "RESULT=done"
+ exit 0
+ fi
+ else
+ # Standalone MC mode: MC added/removed outside rendered config
+ # Just check common criteria (generation/config tracking won't change)
+ if [ "$COMMON_READY" = "true" ]; then
+ echo "RESULT=done"
+ exit 0
+ fi
+ fi
+
+ # Auto-fix drain deadlock for any cordoned node.
+ # On SNO→3node, MCD cordons a node before draining. If MCC cannot reach
+ # the node (api-int flap, RBAC timing) the drain annotation never advances
+ # and the node stays cordoned forever. Fix: uncordon, set lastAppliedDrain
+ # to the drain- value MCC is waiting for, restart MCC.
+ if [ -n "$CORDONED_NODES" ] && [ "{{ mco_poll_auto_fix_drain | default(false) | bool }}" = "True" ]; then
+ DESIRED=$(oc get mcp master -o jsonpath='{.spec.configuration.name}' 2>/dev/null || echo "")
+ for NODE_NAME in $CORDONED_NODES; do
+ echo "DRAIN_FIX: Fixing drain deadlock on $NODE_NAME (desired=${DESIRED})..."
+
+ oc adm uncordon "$NODE_NAME" 2>&1 || true
+
+ if [ -n "$DESIRED" ]; then
+ oc annotate node "$NODE_NAME" \
+ "machineconfiguration.openshift.io/lastAppliedDrain=drain-${DESIRED}" \
+ --overwrite 2>&1 || true
+ echo "DRAIN_FIX: Set lastAppliedDrain=drain-${DESIRED} on $NODE_NAME"
+ fi
+ done
+
+ oc delete pod -n openshift-machine-config-operator \
+ -l k8s-app=machine-config-controller 2>&1 || true
+ echo "DRAIN_FIX: Restarted MCC pod"
+ fi
+
+ echo "RESULT=pending"
+ environment: "{{ sno_oc_env }}"
+ register: mcp_poll_result
+ until: "'RESULT=done' in mcp_poll_result.stdout"
+ retries: "{{ (mco_poll_timeout_minutes | int * 60 / 30) | int }}"
+ delay: 30
+ changed_when: false
+ failed_when: mco_poll_fail_on_timeout | default(true) | bool and mcp_poll_result.rc != 0
+
+- name: "[mco] Display MCP poll result"
+ debug:
+ msg: "{{ mcp_poll_result.stdout }}"
diff --git a/deploy/openshift-clusters/roles/mutable-topology/sno-to-3node/tasks/preflight.yml b/deploy/openshift-clusters/roles/mutable-topology/sno-to-3node/tasks/preflight.yml
new file mode 100644
index 00000000..38205659
--- /dev/null
+++ b/deploy/openshift-clusters/roles/mutable-topology/sno-to-3node/tasks/preflight.yml
@@ -0,0 +1,165 @@
+---
+- name: "[preflight] Query Infrastructure CR"
+ shell: |
+ oc get infrastructure cluster -o json
+ environment: "{{ sno_oc_env }}"
+ register: infra_cr_raw
+ changed_when: false
+
+- name: "[preflight] Parse infrastructure data"
+ set_fact:
+ sno_infra_data: "{{ infra_cr_raw.stdout | from_json }}"
+
+- name: "[preflight] Extract cluster identity"
+ set_fact:
+ sno_platform: "{{ sno_infra_data.spec.platformSpec.type }}"
+ sno_cp_topology: "{{ sno_infra_data.status.controlPlaneTopology }}"
+ sno_infra_topology: "{{ sno_infra_data.status.infrastructureTopology }}"
+ sno_infra_id: "{{ sno_infra_id if sno_infra_id else sno_infra_data.status.infrastructureName }}"
+ sno_cluster_domain: >-
+ {{ sno_cluster_domain if sno_cluster_domain
+ else sno_infra_data.status.apiServerInternalURI
+ | regex_replace('^https://api-int\.', '')
+ | regex_replace(':6443$', '') }}
+
+- name: "[preflight] Assert platform is None"
+ assert:
+ that: sno_platform == "None"
+ fail_msg: "Expected platform None, got {{ sno_platform }}"
+
+- name: "[preflight] Display detected cluster info"
+ debug:
+ msg: >-
+ Cluster: {{ sno_cluster_name }} | Domain: {{ sno_cluster_domain }} |
+ InfraID: {{ sno_infra_id }} | Platform: {{ sno_platform }} |
+ Topology: {{ sno_cp_topology }}
+
+- name: "[preflight] Get cluster nodes"
+ shell: |
+ oc get nodes -o json
+ environment: "{{ sno_oc_env }}"
+ register: nodes_raw
+ changed_when: false
+
+- name: "[preflight] Parse node data"
+ set_fact:
+ sno_nodes: "{{ (nodes_raw.stdout | from_json)['items'] }}"
+
+- name: "[preflight] Detect current cluster state"
+ set_fact:
+ sno_node_count: "{{ sno_nodes | length | int }}"
+ sno_has_three_nodes: "{{ (sno_nodes | length) == 3 }}"
+ sno_is_fresh: "{{ (sno_nodes | length) == 1 }}"
+ sno_transition_in_progress: "{{ (sno_nodes | length) == 3 and sno_cp_topology == 'SingleReplica' }}"
+ sno_already_ha: "{{ (sno_nodes | length) == 3 and sno_cp_topology == 'HighlyAvailable' }}"
+
+- name: "[preflight] Assert valid node count (1 or 3)"
+ assert:
+ that:
+ - (sno_node_count | int) in [1, 3]
+ fail_msg: "Expected 1 or 3 nodes, found {{ sno_node_count }}. Cluster is in an unexpected state."
+
+- name: "[preflight] Display detected state"
+ debug:
+ msg: |
+ Cluster state: {{ sno_node_count }} nodes, topology={{ sno_cp_topology }}
+ {% if sno_is_fresh %}
+ → Fresh SNO cluster - will add 2 nodes and transition to HA
+ {% elif sno_transition_in_progress %}
+ → Transition in progress - nodes added but topology not transitioned yet
+ {% elif sno_already_ha %}
+ → Already HighlyAvailable - will verify configuration
+ {% endif %}
+
+- name: "[preflight] Auto-detect first node IP"
+ set_fact:
+ sno_master0_ip: >-
+ {{ sno_master0_ip if sno_master0_ip
+ else (sno_nodes[0].status.addresses
+ | selectattr('type', 'equalto', 'InternalIP')
+ | map(attribute='address') | first) }}
+
+- name: "[preflight] Display first node IP"
+ debug:
+ msg: "First node ({{ sno_nodes[0].metadata.name }}) IP: {{ sno_master0_ip }}"
+
+- name: "[preflight] Check first node is Ready"
+ assert:
+ that: >-
+ sno_nodes[0].status.conditions
+ | selectattr('type', 'equalto', 'Ready')
+ | map(attribute='status') | first == 'True'
+ fail_msg: "First node {{ sno_nodes[0].metadata.name }} is not Ready"
+
+- name: "[preflight] Check etcd pods"
+ shell: |
+ oc get pods -n openshift-etcd -l app=etcd --no-headers \
+ -o custom-columns=NAME:.metadata.name,READY:.status.containerStatuses[*].ready 2>/dev/null || true
+ environment: "{{ sno_oc_env }}"
+ register: etcd_pods_raw
+ changed_when: false
+
+- name: "[preflight] Display etcd status"
+ debug:
+ msg: "{{ etcd_pods_raw.stdout }}"
+
+- name: "[preflight] Check for degraded cluster operators"
+ shell: |
+ DEGRADED=$(oc get co -o json | jq -r '[.items[] | select(.status.conditions[] | select(.type=="Degraded" and .status=="True")) | .metadata.name] | join(", ")')
+ if [ -n "$DEGRADED" ]; then
+ echo "Degraded COs: $DEGRADED"
+ exit 1
+ fi
+ echo "All cluster operators healthy"
+ environment: "{{ sno_oc_env }}"
+ register: co_check
+ changed_when: false
+ failed_when: false
+
+- name: "[preflight] Display CO status"
+ debug:
+ msg: "{{ co_check.stdout }}"
+
+- name: "[preflight] Warn if COs degraded"
+ debug:
+ msg: "WARNING: Some COs are degraded. Proceeding anyway."
+ when: co_check.rc != 0
+
+- name: "[preflight] Check if api-int DNS exists"
+ shell: |
+ dig +short api-int.{{ sno_cluster_domain }} @192.168.111.1
+ register: dns_apiint
+ changed_when: false
+ failed_when: false
+
+- name: "[preflight] Add api-int as additional hostname to master-0 DNS entry"
+ shell: |
+ # Delete existing master-0 entry
+ sudo virsh net-update {{ sno_libvirt_network }} delete dns-host \
+ "master-0" \
+ --live --config 2>/dev/null || true
+
+ # Re-add with both hostnames
+ sudo virsh net-update {{ sno_libvirt_network }} add dns-host \
+ "master-0api-int.{{ sno_cluster_domain }}" \
+ --live --config
+ when: dns_apiint.stdout | trim == ""
+ changed_when: true
+
+- name: "[preflight] Verify api-int DNS resolves"
+ shell: |
+ dig +short api-int.{{ sno_cluster_domain }} @192.168.111.1
+ register: dns_apiint_verify
+ changed_when: false
+ failed_when: dns_apiint_verify.stdout | trim == ""
+ retries: 3
+ delay: 2
+ until: dns_apiint_verify.stdout | trim != ""
+
+- name: "[preflight] Display api-int DNS status"
+ debug:
+ msg: "api-int.{{ sno_cluster_domain }} resolves to {{ dns_apiint_verify.stdout | trim }}"
+
+- name: "[preflight] Preflight checks passed"
+ debug:
+ msg: "All preflight checks passed. Ready for topology transition."
diff --git a/deploy/openshift-clusters/roles/mutable-topology/sno-to-3node/tasks/topology.yml b/deploy/openshift-clusters/roles/mutable-topology/sno-to-3node/tasks/topology.yml
new file mode 100644
index 00000000..25b4bc7b
--- /dev/null
+++ b/deploy/openshift-clusters/roles/mutable-topology/sno-to-3node/tasks/topology.yml
@@ -0,0 +1,233 @@
+---
+- name: "[topology] Check if oc adm transition topology command is available"
+ shell: |
+ oc adm transition topology --help
+ environment: "{{ sno_oc_env }}"
+ register: topology_cmd_check
+ changed_when: false
+ failed_when: false
+
+- name: "[topology] Skip topology transition if command not available"
+ debug:
+ msg: "oc adm transition topology command not available - skipping topology transition tasks"
+ when: topology_cmd_check.rc != 0
+
+- name: "[topology] Display transition status"
+ debug:
+ msg: "{{ 'Topology already HighlyAvailable - will skip transition but ensure DNS and ingress configured' if sno_already_ha else 'Topology is SingleReplica - will perform full transition' }}"
+ when: topology_cmd_check.rc == 0
+
+- name: "[topology] Wait for cluster to be ready for transition (up to 20 minutes)"
+ shell: |
+ # Use the official preflight command to validate cluster readiness
+ # This includes operator stability checks with proper Progressing logic
+ oc adm transition topology --control-plane=HighlyAvailable --infrastructure=HighlyAvailable 2>&1
+ environment: "{{ sno_oc_env }}"
+ register: preflight_check
+ changed_when: false
+ failed_when: false
+ retries: 120
+ delay: 10
+ until: >-
+ preflight_check is defined and
+ preflight_check.rc is defined and
+ preflight_check.rc == 0
+ when:
+ - topology_cmd_check.rc == 0
+ - not sno_already_ha
+
+- name: "[topology] Display readiness validation"
+ debug:
+ msg: "✓ Cluster ready for topology transition - all preflight checks passed"
+ when:
+ - topology_cmd_check.rc == 0
+ - not sno_already_ha
+
+- name: "[topology] Stop before transition confirmation (stop_before_transition_confirm=true)"
+ block:
+ - name: "[topology] Display staged state message"
+ debug:
+ msg: |
+ ✓ Transition staged and ready to confirm
+
+ The cluster is ready for topology transition. To complete manually:
+ $ oc adm transition topology --control-plane=HighlyAvailable --infrastructure=HighlyAvailable --confirm
+
+ To resume and complete the transition automatically, rerun:
+ $ make sno-to-3node
+
+ Playbook stopped as requested (stop_before_transition_confirm=true)
+
+ - name: "[topology] End playbook at staged state"
+ meta: end_play
+ when:
+ - topology_cmd_check.rc == 0
+ - not sno_already_ha
+ - stop_before_transition_confirm | default(false) | bool
+
+- name: "[topology] Initiate topology transition (patches Infrastructure.spec)"
+ shell: |
+ oc adm transition topology --control-plane=HighlyAvailable --infrastructure=HighlyAvailable --confirm
+ environment: "{{ sno_oc_env }}"
+ register: transition_result
+ changed_when: false
+ when:
+ - topology_cmd_check.rc == 0
+ - not sno_already_ha
+
+- name: "[topology] Verify Infrastructure.spec.controlPlaneTopology updated (fast-fail check)"
+ shell: |
+ oc get infrastructure cluster -o jsonpath='{.spec.controlPlaneTopology}' 2>&1
+ environment: "{{ sno_oc_env }}"
+ register: topo_spec_verify
+ changed_when: false
+ retries: 5
+ delay: 2
+ until: >-
+ topo_spec_verify is defined and
+ topo_spec_verify.rc is defined and
+ topo_spec_verify.rc == 0 and
+ topo_spec_verify.stdout == "HighlyAvailable"
+ failed_when: false
+ when:
+ - topology_cmd_check.rc == 0
+ - not sno_already_ha
+
+- name: "[topology] Check if transition command succeeded"
+ fail:
+ msg: |
+ Topology transition command failed.
+
+ Command output:
+ {{ transition_result.stdout if transition_result.stdout else 'No output' }}
+
+ Command errors:
+ {{ transition_result.stderr if transition_result.stderr else 'No errors' }}
+ when:
+ - topology_cmd_check.rc == 0
+ - not sno_already_ha
+ - transition_result.rc != 0
+
+- name: "[topology] Check if Infrastructure.spec was updated"
+ fail:
+ msg: |
+ Topology transition command succeeded but Infrastructure.spec was not updated after 10 seconds.
+
+ Infrastructure.spec.controlPlaneTopology: {{ topo_spec_verify.stdout if topo_spec_verify.stdout else 'unable to read' }}
+ Expected: HighlyAvailable
+
+ This may indicate a timing issue. Check the transition command output above.
+ when:
+ - topology_cmd_check.rc == 0
+ - not sno_already_ha
+ - topo_spec_verify.stdout != "HighlyAvailable"
+
+- name: "[topology] Display transition initiated"
+ debug:
+ msg: "✓ Topology transition initiated - Infrastructure.spec updated to HighlyAvailable. CCO will reconcile status and operators will reconfigure."
+ when:
+ - topology_cmd_check.rc == 0
+ - not sno_already_ha
+
+- name: "[topology] Wait for Infrastructure.status update (up to 5 minutes)"
+ shell: |
+ oc get infrastructure cluster -o jsonpath='{.status.controlPlaneTopology},{.status.infrastructureTopology}' 2>&1
+ environment: "{{ sno_oc_env }}"
+ register: status_check
+ changed_when: false
+ failed_when: false
+ retries: 30
+ delay: 10
+ until: >-
+ status_check is defined and
+ status_check.rc is defined and
+ status_check.rc == 0 and
+ status_check.stdout is defined and
+ status_check.stdout == "HighlyAvailable,HighlyAvailable"
+ when:
+ - topology_cmd_check.rc == 0
+ - not sno_already_ha
+
+- name: "[topology] Check if CCO reconciliation succeeded (both topologies HighlyAvailable)"
+ set_fact:
+ cco_reconciled: "{{ status_check.rc == 0 and status_check.stdout == 'HighlyAvailable,HighlyAvailable' }}"
+ when:
+ - topology_cmd_check.rc == 0
+ - not sno_already_ha
+
+- name: "[topology] ⚠️ WARNING: CCO did not reconcile status - applying manual patch fallback"
+ debug:
+ msg: |
+ ╔═══════════════════════════════════════════════════════════════════════════════╗
+ ║ ⚠️ CRITICAL WARNING ⚠️ ║
+ ╠═══════════════════════════════════════════════════════════════════════════════╣
+ ║ The cluster-config-operator did NOT automatically update Infrastructure ║
+ ║ status within 5 minutes after spec was patched. ║
+ ║ ║
+ ║ This indicates one of the following: ║
+ ║ 1. The CCO topology controller is NOT deployed in this payload ║
+ ║ 2. The CCO controller encountered an error during reconciliation ║
+ ║ 3. The controller loop timing is longer than expected ║
+ ║ ║
+ ║ Applying MANUAL PATCH as fallback to complete the topology transition. ║
+ ║ ║
+ ║ ACTION REQUIRED: ║
+ ║ - Verify CCO logs: oc logs -n openshift-cluster-config-operator ... ║
+ ║ - Check if mutable-topology-controller is present in CCO image ║
+ ║ - Confirm payload includes expected CCO version ║
+ ╚═══════════════════════════════════════════════════════════════════════════════╝
+ when:
+ - topology_cmd_check.rc == 0
+ - not sno_already_ha
+ - not cco_reconciled
+
+- name: "[topology] Apply manual Infrastructure.status patch (fallback)"
+ shell: |
+ oc patch infrastructure cluster --type merge --subresource status \
+ -p '{"status":{"controlPlaneTopology":"HighlyAvailable","infrastructureTopology":"HighlyAvailable"}}'
+ environment: "{{ sno_oc_env }}"
+ when:
+ - topology_cmd_check.rc == 0
+ - not sno_already_ha
+ - not cco_reconciled
+
+- name: "[topology] Verify manual patch applied successfully"
+ shell: |
+ oc get infrastructure cluster -o jsonpath='{.status.controlPlaneTopology},{.status.infrastructureTopology}'
+ environment: "{{ sno_oc_env }}"
+ register: manual_patch_verify
+ changed_when: false
+ failed_when: manual_patch_verify.stdout != "HighlyAvailable,HighlyAvailable"
+ when:
+ - topology_cmd_check.rc == 0
+ - not sno_already_ha
+ - not cco_reconciled
+
+- name: "[topology] Topology transition complete (via CCO)"
+ debug:
+ msg: "✓ Topology spec and status updated to HighlyAvailable by cluster-config-operator. Operators will reconfigure to the new topology."
+ when:
+ - topology_cmd_check.rc == 0
+ - not sno_already_ha
+ - cco_reconciled
+
+- name: "[topology] Topology transition complete (via manual fallback)"
+ debug:
+ msg: "✓ Topology spec and status updated to HighlyAvailable via manual patch. Operators will reconfigure to the new topology. (CCO reconciliation timed out - see warning above)"
+ when:
+ - topology_cmd_check.rc == 0
+ - not sno_already_ha
+ - not cco_reconciled
+
+- name: "[topology] Topology transition complete"
+ debug:
+ msg: |
+ ✓ Infrastructure.spec.controlPlaneTopology = HighlyAvailable
+ ✓ Infrastructure.status.controlPlaneTopology = HighlyAvailable
+ {% if not sno_already_ha %}
+ ✓ MCO will re-render master configs in Phase 9
+ ✓ IngressController will be reconfigured after MCO rollout completes
+ {% else %}
+ ✓ Topology already HighlyAvailable (skipping MCO rollout wait)
+ ✓ IngressController will be reconfigured in Phase 10
+ {% endif %}
diff --git a/deploy/openshift-clusters/roles/mutable-topology/sno-to-3node/tasks/verify.yml b/deploy/openshift-clusters/roles/mutable-topology/sno-to-3node/tasks/verify.yml
new file mode 100644
index 00000000..9a880e87
--- /dev/null
+++ b/deploy/openshift-clusters/roles/mutable-topology/sno-to-3node/tasks/verify.yml
@@ -0,0 +1,73 @@
+---
+- name: "[verify] Wait for topology transition to complete (up to 20 minutes)"
+ shell: |
+ oc adm transition topology status 2>&1
+ environment: "{{ sno_oc_env }}"
+ register: transition_status
+ changed_when: false
+ failed_when: false
+ retries: 120
+ delay: 10
+ until: >-
+ transition_status is defined and
+ transition_status.rc is defined and
+ transition_status.rc == 0 and
+ transition_status.stdout is defined and
+ 'No transition in progress' in transition_status.stdout
+
+- name: "[verify] Display transition status"
+ debug:
+ msg: "{{ transition_status.stdout }}"
+ when: transition_status is defined
+
+- name: "[verify] Check transition completion"
+ fail:
+ msg: "Topology transition did not complete after 20 minutes. See status output above."
+ when:
+ - transition_status is defined
+ - (transition_status.rc != 0 or 'No transition in progress' not in transition_status.stdout)
+
+- name: "[verify] Get node count and status"
+ shell: |
+ oc get nodes --no-headers
+ environment: "{{ sno_oc_env }}"
+ register: verify_nodes
+ changed_when: false
+
+- name: "[verify] Nodes"
+ debug:
+ msg: "{{ verify_nodes.stdout }}"
+
+- name: "[verify] Assert 3 nodes"
+ assert:
+ that: (verify_nodes.stdout_lines | length) >= 3
+ fail_msg: "Expected 3+ nodes, found {{ verify_nodes.stdout_lines | length }}"
+
+- name: "[verify] Assert all nodes Ready"
+ assert:
+ that: "'NotReady' not in verify_nodes.stdout"
+ fail_msg: "One or more nodes are NotReady:\n{{ verify_nodes.stdout }}"
+
+- name: "[verify] Get etcd member count"
+ shell: |
+ oc get pods -n openshift-etcd -l app=etcd --no-headers | wc -l
+ environment: "{{ sno_oc_env }}"
+ register: verify_etcd
+ changed_when: false
+
+- name: "[verify] etcd members"
+ debug:
+ msg: "etcd pods: {{ verify_etcd.stdout | trim }}"
+
+- name: "[verify] Assert 3 etcd members"
+ assert:
+ that: verify_etcd.stdout | trim | int >= 3
+ fail_msg: "Expected 3+ etcd pods, found {{ verify_etcd.stdout | trim }}"
+
+- name: "[verify] SNO to 3-node HA transition complete"
+ debug:
+ msg: >-
+ ✓ Topology transition verified via cluster-config-operator conditions
+ ✓ {{ verify_nodes.stdout_lines | length }} nodes Ready
+ ✓ {{ verify_etcd.stdout | trim }} etcd members
+ ✓ Cluster ready for production use
diff --git a/deploy/openshift-clusters/roles/mutable-topology/sno-to-3node/tasks/wait-etcd.yml b/deploy/openshift-clusters/roles/mutable-topology/sno-to-3node/tasks/wait-etcd.yml
new file mode 100644
index 00000000..e92737a8
--- /dev/null
+++ b/deploy/openshift-clusters/roles/mutable-topology/sno-to-3node/tasks/wait-etcd.yml
@@ -0,0 +1,52 @@
+---
+- name: "[wait-etcd] Poll for etcd scaling to 3 members"
+ shell: |
+ FULLY_READY=$(oc get pods -n openshift-etcd -l app=etcd -o json 2>/dev/null | python3 -c "
+ import json, sys
+ data = json.load(sys.stdin)
+ count = 0
+ for pod in data.get('items', []):
+ containers = pod.get('status', {}).get('containerStatuses', [])
+ if containers and all(c.get('ready', False) for c in containers):
+ count += 1
+ print(count)
+ " 2>/dev/null || echo "0")
+ echo "ETCD_FULLY_READY=$FULLY_READY"
+
+ if [ "$FULLY_READY" -ge 3 ]; then
+ echo "RESULT=done"
+ else
+ echo "RESULT=pending"
+ fi
+ environment: "{{ sno_oc_env }}"
+ register: etcd_poll
+ until: "'RESULT=done' in etcd_poll.stdout"
+ retries: "{{ (sno_etcd_timeout_minutes | int * 60 / 20) | int }}"
+ delay: 20
+ changed_when: false
+
+- name: "[etcd] Display etcd member list"
+ shell: |
+ ETCD_POD=$(oc get pods -n openshift-etcd -l app=etcd -o jsonpath='{.items[0].metadata.name}' 2>/dev/null)
+ oc exec -n openshift-etcd "$ETCD_POD" -c etcdctl -- etcdctl member list -w table 2>/dev/null || echo "Could not query etcd members"
+ environment: "{{ sno_oc_env }}"
+ register: etcd_members
+ changed_when: false
+ failed_when: false
+
+- name: "[etcd] Members"
+ debug:
+ msg: "{{ etcd_members.stdout }}"
+
+- name: "[etcd] Display etcd endpoint health"
+ shell: |
+ ETCD_POD=$(oc get pods -n openshift-etcd -l app=etcd -o jsonpath='{.items[0].metadata.name}' 2>/dev/null)
+ oc exec -n openshift-etcd "$ETCD_POD" -c etcdctl -- etcdctl endpoint health --cluster -w table 2>/dev/null || echo "Could not query endpoint health"
+ environment: "{{ sno_oc_env }}"
+ register: etcd_health
+ changed_when: false
+ failed_when: false
+
+- name: "[etcd] Health"
+ debug:
+ msg: "{{ etcd_health.stdout }}"
diff --git a/deploy/openshift-clusters/roles/mutable-topology/sno-to-3node/tasks/wait-nodes.yml b/deploy/openshift-clusters/roles/mutable-topology/sno-to-3node/tasks/wait-nodes.yml
new file mode 100644
index 00000000..f02335f8
--- /dev/null
+++ b/deploy/openshift-clusters/roles/mutable-topology/sno-to-3node/tasks/wait-nodes.yml
@@ -0,0 +1,48 @@
+---
+- name: "[wait-nodes] Poll for CSR approval and node join"
+ shell: |
+ PENDING=$(oc get csr -o json 2>/dev/null | python3 -c "
+ import json, sys
+ data = json.load(sys.stdin)
+ pending = []
+ for csr in data.get('items', []):
+ status = csr.get('status', {})
+ if not status.get('conditions'):
+ name = csr['metadata']['name']
+ pending.append(name)
+ for name in pending:
+ print(name)
+ " 2>/dev/null || true)
+
+ if [ -n "$PENDING" ]; then
+ for CSR in $PENDING; do
+ oc adm certificate approve "$CSR" 2>/dev/null || true
+ echo "Approved CSR: $CSR"
+ done
+ fi
+
+ READY_COUNT=$(oc get nodes --no-headers 2>/dev/null | grep -c ' Ready' || echo "0")
+ echo "READY_NODES=$READY_COUNT"
+
+ if [ "$READY_COUNT" -ge 3 ]; then
+ echo "RESULT=done"
+ else
+ echo "RESULT=pending"
+ fi
+ environment: "{{ sno_oc_env }}"
+ register: node_poll
+ until: "'RESULT=done' in node_poll.stdout"
+ retries: "{{ (sno_node_join_timeout_minutes | int * 60 / 20) | int }}"
+ delay: 20
+ changed_when: false
+
+- name: "[wait-nodes] Display final node list"
+ shell: |
+ oc get nodes -o wide
+ environment: "{{ sno_oc_env }}"
+ register: node_list
+ changed_when: false
+
+- name: "[wait-nodes] Nodes joined"
+ debug:
+ msg: "{{ node_list.stdout }}"
diff --git a/deploy/openshift-clusters/roles/mutable-topology/sno-to-3node/templates/auto-install.ign.j2 b/deploy/openshift-clusters/roles/mutable-topology/sno-to-3node/templates/auto-install.ign.j2
new file mode 100644
index 00000000..5bdd58e0
--- /dev/null
+++ b/deploy/openshift-clusters/roles/mutable-topology/sno-to-3node/templates/auto-install.ign.j2
@@ -0,0 +1,25 @@
+{
+ "ignition": {
+ "version": "3.2.0"
+ },
+ "storage": {
+ "files": [
+ {
+ "path": "/etc/master.ign",
+ "mode": 420,
+ "contents": {
+ "source": "data:text/plain;charset=utf-8;base64,{{ sno_master_ign_b64 }}"
+ }
+ }
+ ]
+ },
+ "systemd": {
+ "units": [
+ {
+ "name": "auto-install.service",
+ "enabled": true,
+ "contents": "[Unit]\nDescription=Auto-install RHCOS to disk\nAfter=network-online.target\nWants=network-online.target\n\n[Service]\nType=oneshot\nRemainAfterExit=yes\nExecStart=/usr/bin/coreos-installer install /dev/vda --ignition-file /etc/master.ign --insecure\nExecStartPost=/usr/bin/systemctl reboot\n\n[Install]\nWantedBy=multi-user.target\n"
+ }
+ ]
+ }
+}
diff --git a/deploy/openshift-clusters/roles/mutable-topology/sno-to-3node/templates/libvirt-network-3node.xml.j2 b/deploy/openshift-clusters/roles/mutable-topology/sno-to-3node/templates/libvirt-network-3node.xml.j2
new file mode 100644
index 00000000..7e364bb1
--- /dev/null
+++ b/deploy/openshift-clusters/roles/mutable-topology/sno-to-3node/templates/libvirt-network-3node.xml.j2
@@ -0,0 +1,38 @@
+
+ {{ sno_libvirt_network }}
+
+
+
+
+
+
+
+
+
+ ns1
+
+
+ virthost
+ api.{{ sno_cluster_domain }}
+ api-int.{{ sno_cluster_domain }}
+
+
+ master-0.{{ sno_cluster_domain }}
+
+
+ master-1.{{ sno_cluster_domain }}
+
+
+ master-2.{{ sno_cluster_domain }}
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/deploy/openshift-clusters/roles/mutable-topology/sno-to-3node/templates/master.ign.j2 b/deploy/openshift-clusters/roles/mutable-topology/sno-to-3node/templates/master.ign.j2
new file mode 100644
index 00000000..d2c5b010
--- /dev/null
+++ b/deploy/openshift-clusters/roles/mutable-topology/sno-to-3node/templates/master.ign.j2
@@ -0,0 +1,21 @@
+{
+ "ignition": {
+ "version": "3.2.0",
+ "security": {
+ "tls": {
+ "certificateAuthorities": [
+ {
+ "source": "data:text/plain;charset=utf-8;base64,{{ sno_mcs_ca_b64 }}"
+ }
+ ]
+ }
+ },
+ "config": {
+ "merge": [
+ {
+ "source": "https://api-int.{{ sno_cluster_domain }}:22623/config/master"
+ }
+ ]
+ }
+ }
+}
diff --git a/deploy/openshift-clusters/roles/mutable-topology/sno-to-3node/vars/main.yml b/deploy/openshift-clusters/roles/mutable-topology/sno-to-3node/vars/main.yml
new file mode 100644
index 00000000..e8a1722c
--- /dev/null
+++ b/deploy/openshift-clusters/roles/mutable-topology/sno-to-3node/vars/main.yml
@@ -0,0 +1,26 @@
+---
+# Control flags
+stop_before_transition_confirm: false # Set to true to stop before --confirm (for manual testing)
+
+sno_dev_scripts_path: "{{ dev_scripts_path | default('openshift-metal3/dev-scripts') }}"
+sno_kubeconfig_resolved: >-
+ {{ sno_kubeconfig if sno_kubeconfig
+ else sno_dev_scripts_path ~ '/ocp/' ~ sno_cluster_name ~ '/auth/kubeconfig' }}
+sno_addnhosts_path: "/var/lib/libvirt/dnsmasq/{{ sno_libvirt_bridge }}.addnhosts"
+sno_lease_file: "/var/lib/libvirt/dnsmasq/{{ sno_libvirt_bridge }}.status"
+sno_nm_dnsmasq_conf: "/etc/NetworkManager/dnsmasq.d/openshift-{{ sno_cluster_name }}.conf"
+
+sno_oc_env:
+ KUBECONFIG: "{{ sno_kubeconfig_resolved }}"
+ PATH: "/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:{{ ansible_env.PATH | default('/usr/local/bin:/usr/bin') }}"
+ OC_ENABLE_CMD_TRANSITION_TOPOLOGY: "true"
+
+sno_new_nodes:
+ - index: 1
+ name: "{{ sno_cluster_name }}_master_1"
+ hostname: "master-1"
+ ip: "{{ sno_master1_ip }}"
+ - index: 2
+ name: "{{ sno_cluster_name }}_master_2"
+ hostname: "master-2"
+ ip: "{{ sno_master2_ip }}"
diff --git a/deploy/openshift-clusters/scripts/clean-mutable-topology.sh b/deploy/openshift-clusters/scripts/clean-mutable-topology.sh
new file mode 100755
index 00000000..03036bc6
--- /dev/null
+++ b/deploy/openshift-clusters/scripts/clean-mutable-topology.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+set -euo pipefail
+
+SCRIPT_DIR=$(dirname "$0")
+DEPLOY_DIR="$(cd "${SCRIPT_DIR}/../.." && pwd)"
+
+# shellcheck source=/dev/null
+source "${DEPLOY_DIR}/common.sh"
+
+if [[ ! -f "$(get_node_dir)/aws-instance-id" ]]; then
+ echo "Error: No instance found. Run 'make deploy' first."
+ exit 1
+fi
+
+if [[ ! -f "${DEPLOY_DIR}/openshift-clusters/inventory.ini" ]]; then
+ echo "Error: inventory.ini not found. Run 'make inventory' first."
+ exit 1
+fi
+
+cd "${DEPLOY_DIR}/openshift-clusters"
+ansible-playbook clean-mutable-topology.yml -e "interactive_mode=false" -i inventory.ini "$@"
diff --git a/deploy/openshift-clusters/scripts/sno-to-3node.sh b/deploy/openshift-clusters/scripts/sno-to-3node.sh
new file mode 100755
index 00000000..13f59447
--- /dev/null
+++ b/deploy/openshift-clusters/scripts/sno-to-3node.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+set -euo pipefail
+
+SCRIPT_DIR=$(dirname "$0")
+DEPLOY_DIR="$(cd "${SCRIPT_DIR}/../.." && pwd)"
+
+# shellcheck source=/dev/null
+source "${DEPLOY_DIR}/common.sh"
+
+if [[ ! -f "$(get_node_dir)/aws-instance-id" ]]; then
+ echo "Error: No instance found. Run 'make deploy' first."
+ exit 1
+fi
+
+if [[ ! -f "${DEPLOY_DIR}/openshift-clusters/inventory.ini" ]]; then
+ echo "Error: inventory.ini not found. Run 'make inventory' first."
+ exit 1
+fi
+
+cd "${DEPLOY_DIR}/openshift-clusters"
+ansible-playbook sno-to-3node.yml -e "interactive_mode=false" -i inventory.ini "$@"
diff --git a/deploy/openshift-clusters/sno-to-3node.yml b/deploy/openshift-clusters/sno-to-3node.yml
new file mode 100644
index 00000000..f742a88d
--- /dev/null
+++ b/deploy/openshift-clusters/sno-to-3node.yml
@@ -0,0 +1,23 @@
+---
+- hosts: metal_machine
+ gather_facts: no
+ force_handlers: yes
+
+ pre_tasks:
+ - name: Confirm SNO to 3-node transition
+ ansible.builtin.pause:
+ prompt: >-
+ This will transition the SNO cluster to a 3-node HA cluster.
+ 2 new VMs will be created and the cluster topology will be updated.
+ Press Enter to proceed or Ctrl+C to abort.
+ delegate_to: localhost
+ run_once: true
+ when: interactive_mode | default(true) | bool
+
+ roles:
+ - mutable-topology/sno-to-3node
+
+ tasks:
+ - name: Transition complete
+ ansible.builtin.debug:
+ msg: "SNO to 3-node HA transition completed successfully."