diff --git a/arch/riscv/include/asm/kvm_gstage.h b/arch/riscv/include/asm/kvm_gstage.h index aaf080ba1b77a2..c61a1bbb272f48 100644 --- a/arch/riscv/include/asm/kvm_gstage.h +++ b/arch/riscv/include/asm/kvm_gstage.h @@ -8,7 +8,7 @@ #define __RISCV_KVM_GSTAGE_H_ #include - +#include struct kvm_gstage { struct kvm *kvm; unsigned long flags; @@ -108,7 +108,8 @@ static inline void kvm_riscv_gstage_init(struct kvm_gstage *gstage, struct kvm * { gstage->kvm = kvm; gstage->flags = 0; - gstage->vmid = READ_ONCE(kvm->arch.vmid.vmid); + gstage->vmid = + kvm_riscv_gstage_vmid_hwid(atomic64_read(&kvm->arch.vmid.id)); gstage->pgd = kvm->arch.pgd; gstage->pgd_levels = kvm->arch.pgd_levels; } diff --git a/arch/riscv/include/asm/kvm_mmu.h b/arch/riscv/include/asm/kvm_mmu.h index 5439e76f0a9606..aae9b8ac0878fc 100644 --- a/arch/riscv/include/asm/kvm_mmu.h +++ b/arch/riscv/include/asm/kvm_mmu.h @@ -16,6 +16,7 @@ int kvm_riscv_mmu_map(struct kvm_vcpu *vcpu, struct kvm_memory_slot *memslot, struct kvm_gstage_mapping *out_map); int kvm_riscv_mmu_alloc_pgd(struct kvm *kvm); void kvm_riscv_mmu_free_pgd(struct kvm *kvm); +void kvm_riscv_mmu_update_hgatp_vmid(struct kvm_vcpu *vcpu, u64 vmid); void kvm_riscv_mmu_update_hgatp(struct kvm_vcpu *vcpu); #endif diff --git a/arch/riscv/include/asm/kvm_vmid.h b/arch/riscv/include/asm/kvm_vmid.h index db61b0525a8d0f..ddd7cc125c3f60 100644 --- a/arch/riscv/include/asm/kvm_vmid.h +++ b/arch/riscv/include/asm/kvm_vmid.h @@ -6,21 +6,42 @@ #ifndef __RISCV_KVM_VMID_H_ #define __RISCV_KVM_VMID_H_ +#include #include struct kvm_vmid { /* - * Writes to vmid_version and vmid happen with vmid_lock held - * whereas reads happen without any lock held. + * Software VMID: + * + * [ generation | hardware VMID ] + * + * Keep the identity 64-bit on RV32 so that the generation does not + * wrap at the native word size. Zero denotes an unallocated VMID. + * + * Only the low hardware VMID bits may be written to HGATP + * or used as a hardware fence VMID. */ - unsigned long vmid_version; - unsigned long vmid; + atomic64_t id; }; void __init kvm_riscv_gstage_vmid_detect(void); unsigned long kvm_riscv_gstage_vmid_bits(void); + +int __init kvm_riscv_gstage_vmid_alloc_init(void); +void kvm_riscv_gstage_vmid_alloc_free(void); + int kvm_riscv_gstage_vmid_init(struct kvm *kvm); + +unsigned long kvm_riscv_gstage_vmid_hwid(u64 vmid); + bool kvm_riscv_gstage_vmid_ver_changed(struct kvm_vmid *vmid); + void kvm_riscv_gstage_vmid_update(struct kvm_vcpu *vcpu); +/* + * Invalidate the current CPU's fast-path VMID state while preserving + * the old identity conservatively in reserved_vmids. + */ +void kvm_riscv_gstage_vmid_cpu_invalidate(void); + #endif diff --git a/arch/riscv/kvm/main.c b/arch/riscv/kvm/main.c index 89568ccce01de5..d6c6be4df61b17 100644 --- a/arch/riscv/kvm/main.c +++ b/arch/riscv/kvm/main.c @@ -16,7 +16,7 @@ #include #include #include - +#include static DEFINE_PER_CPU(bool, kvm_riscv_virtualization_enabled); DEFINE_STATIC_KEY_FALSE(kvm_riscv_vsstage_tlb_no_gpa); @@ -84,6 +84,7 @@ int kvm_arch_enable_virtualization_cpu(void) void kvm_arch_disable_virtualization_cpu(void) { + kvm_riscv_gstage_vmid_cpu_invalidate(); kvm_riscv_aia_disable(); kvm_riscv_csr_cleanup(); kvm_riscv_nacl_disable(); @@ -112,6 +113,7 @@ static int kvm_riscv_cpu_pm_notifier(struct notifier_block *self, unsigned long * is enabled on this CPU. */ if (__this_cpu_read(kvm_riscv_virtualization_enabled)) { + kvm_riscv_gstage_vmid_cpu_invalidate(); kvm_riscv_aia_pm_enter(); kvm_riscv_csr_cleanup(); } @@ -130,6 +132,7 @@ static struct notifier_block kvm_riscv_cpu_pm_nb = { static void kvm_riscv_teardown(void) { kvm_riscv_aia_exit(); + kvm_riscv_gstage_vmid_alloc_free(); kvm_riscv_nacl_exit(); kvm_riscv_v_exit(); kvm_unregister_perf_callbacks(); @@ -181,8 +184,15 @@ static int __init riscv_kvm_init(void) kvm_riscv_gstage_vmid_detect(); + rc = kvm_riscv_gstage_vmid_alloc_init(); + if (rc) { + kvm_riscv_nacl_exit(); + return rc; + } + rc = kvm_riscv_aia_init(); if (rc && rc != -ENODEV) { + kvm_riscv_gstage_vmid_alloc_free(); kvm_riscv_nacl_exit(); return rc; } diff --git a/arch/riscv/kvm/mmu.c b/arch/riscv/kvm/mmu.c index 6035b5ec950398..9b597092f8ec60 100644 --- a/arch/riscv/kvm/mmu.c +++ b/arch/riscv/kvm/mmu.c @@ -15,7 +15,7 @@ #include #include #include - +#include static bool __read_mostly eager_page_split = true; module_param(eager_page_split, bool, 0644); @@ -796,16 +796,23 @@ void kvm_riscv_mmu_free_pgd(struct kvm *kvm) kvm_mmu_free_memory_cache(&kvm->arch.pgd_split_page_cache); } -void kvm_riscv_mmu_update_hgatp(struct kvm_vcpu *vcpu) +void kvm_riscv_mmu_update_hgatp_vmid(struct kvm_vcpu *vcpu, u64 vmid) { struct kvm_arch *ka = &vcpu->kvm->arch; unsigned long hgatp = kvm_riscv_gstage_mode(ka->pgd_levels) << HGATP_MODE_SHIFT; - hgatp |= (READ_ONCE(ka->vmid.vmid) << HGATP_VMID_SHIFT) & HGATP_VMID; + hgatp |= (kvm_riscv_gstage_vmid_hwid(vmid) << HGATP_VMID_SHIFT) & HGATP_VMID; hgatp |= (ka->pgd_phys >> PAGE_SHIFT) & HGATP_PPN; ncsr_write(CSR_HGATP, hgatp); +} + +void kvm_riscv_mmu_update_hgatp(struct kvm_vcpu *vcpu) +{ + u64 vmid = atomic64_read(&vcpu->kvm->arch.vmid.id); + + kvm_riscv_mmu_update_hgatp_vmid(vcpu, vmid); if (!kvm_riscv_gstage_vmid_bits()) kvm_riscv_local_hfence_gvma_all(); diff --git a/arch/riscv/kvm/tlb.c b/arch/riscv/kvm/tlb.c index 2ae34632cdcb13..b85e5ba2198aa7 100644 --- a/arch/riscv/kvm/tlb.c +++ b/arch/riscv/kvm/tlb.c @@ -224,7 +224,7 @@ void kvm_riscv_local_tlb_sanitize(struct kvm_vcpu *vcpu) * entries by VMID whenever underlying Host CPU changes for a VCPU. */ - vmid = READ_ONCE(vcpu->kvm->arch.vmid.vmid); + vmid = kvm_riscv_gstage_vmid_hwid(atomic64_read(&vcpu->kvm->arch.vmid.id)); kvm_riscv_local_hfence_gvma_vmid_all(vmid); /* @@ -244,7 +244,7 @@ void kvm_riscv_fence_i_process(struct kvm_vcpu *vcpu) void kvm_riscv_tlb_flush_process(struct kvm_vcpu *vcpu) { struct kvm_vmid *v = &vcpu->kvm->arch.vmid; - unsigned long vmid = READ_ONCE(v->vmid); + unsigned long vmid = kvm_riscv_gstage_vmid_hwid(atomic64_read(&v->id)); if (kvm_riscv_nacl_available()) nacl_hfence_gvma_vmid_all(nacl_shmem(), vmid); @@ -255,7 +255,7 @@ void kvm_riscv_tlb_flush_process(struct kvm_vcpu *vcpu) void kvm_riscv_hfence_vvma_all_process(struct kvm_vcpu *vcpu) { struct kvm_vmid *v = &vcpu->kvm->arch.vmid; - unsigned long vmid = READ_ONCE(v->vmid); + unsigned long vmid = kvm_riscv_gstage_vmid_hwid(atomic64_read(&v->id)); if (kvm_riscv_nacl_available()) nacl_hfence_vvma_all(nacl_shmem(), vmid); @@ -531,8 +531,11 @@ void kvm_riscv_hfence_vvma_all(struct kvm *kvm, int kvm_arch_flush_remote_tlbs_range(struct kvm *kvm, gfn_t gfn, u64 nr_pages) { + unsigned long vmid = + kvm_riscv_gstage_vmid_hwid(atomic64_read(&kvm->arch.vmid.id)); + kvm_riscv_hfence_gvma_vmid_gpa(kvm, -1UL, 0, gfn << PAGE_SHIFT, nr_pages << PAGE_SHIFT, - PAGE_SHIFT, READ_ONCE(kvm->arch.vmid.vmid)); + PAGE_SHIFT, vmid); return 0; } diff --git a/arch/riscv/kvm/vcpu.c b/arch/riscv/kvm/vcpu.c index e062ca19f9d8f3..85542c1a5ec3fa 100644 --- a/arch/riscv/kvm/vcpu.c +++ b/arch/riscv/kvm/vcpu.c @@ -943,14 +943,14 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) continue; ret = 1; - kvm_riscv_gstage_vmid_update(vcpu); - ret = kvm_riscv_check_vcpu_requests(vcpu); if (ret <= 0) continue; preempt_disable(); + kvm_riscv_gstage_vmid_update(vcpu); + /* Update AIA HW state before entering guest */ ret = kvm_riscv_vcpu_aia_update(vcpu); if (ret <= 0) { diff --git a/arch/riscv/kvm/vcpu_sbi_replace.c b/arch/riscv/kvm/vcpu_sbi_replace.c index 506a510b6bff3b..1be386a78e27f6 100644 --- a/arch/riscv/kvm/vcpu_sbi_replace.c +++ b/arch/riscv/kvm/vcpu_sbi_replace.c @@ -104,7 +104,7 @@ static int kvm_sbi_ext_rfence_handler(struct kvm_vcpu *vcpu, struct kvm_run *run kvm_riscv_vcpu_pmu_incr_fw(vcpu, SBI_PMU_FW_FENCE_I_SENT); break; case SBI_EXT_RFENCE_REMOTE_SFENCE_VMA: - vmid = READ_ONCE(vcpu->kvm->arch.vmid.vmid); + vmid = kvm_riscv_gstage_vmid_hwid(atomic64_read(&vcpu->kvm->arch.vmid.id)); if ((cp->a2 == 0 && cp->a3 == 0) || cp->a3 == -1UL) kvm_riscv_hfence_vvma_all(vcpu->kvm, hbase, hmask, vmid); else @@ -113,7 +113,7 @@ static int kvm_sbi_ext_rfence_handler(struct kvm_vcpu *vcpu, struct kvm_run *run kvm_riscv_vcpu_pmu_incr_fw(vcpu, SBI_PMU_FW_HFENCE_VVMA_SENT); break; case SBI_EXT_RFENCE_REMOTE_SFENCE_VMA_ASID: - vmid = READ_ONCE(vcpu->kvm->arch.vmid.vmid); + vmid = kvm_riscv_gstage_vmid_hwid(atomic64_read(&vcpu->kvm->arch.vmid.id)); if ((cp->a2 == 0 && cp->a3 == 0) || cp->a3 == -1UL) kvm_riscv_hfence_vvma_asid_all(vcpu->kvm, hbase, hmask, cp->a4, vmid); diff --git a/arch/riscv/kvm/vcpu_sbi_v01.c b/arch/riscv/kvm/vcpu_sbi_v01.c index de544ea3f28dc3..ca3efe63c1cf19 100644 --- a/arch/riscv/kvm/vcpu_sbi_v01.c +++ b/arch/riscv/kvm/vcpu_sbi_v01.c @@ -88,14 +88,14 @@ static int kvm_sbi_ext_v01_handler(struct kvm_vcpu *vcpu, struct kvm_run *run, if (cp->a7 == SBI_EXT_0_1_REMOTE_FENCE_I) kvm_riscv_fence_i(vcpu->kvm, hbase, hmask); else if (cp->a7 == SBI_EXT_0_1_REMOTE_SFENCE_VMA) { - vmid = READ_ONCE(vcpu->kvm->arch.vmid.vmid); + vmid = kvm_riscv_gstage_vmid_hwid(atomic64_read(&vcpu->kvm->arch.vmid.id)); if (cp->a1 == 0 && cp->a2 == 0) kvm_riscv_hfence_vvma_all(vcpu->kvm, hbase, hmask, vmid); else kvm_riscv_hfence_vvma_gva(vcpu->kvm, hbase, hmask, cp->a1, cp->a2, PAGE_SHIFT, vmid); } else { - vmid = READ_ONCE(vcpu->kvm->arch.vmid.vmid); + vmid = kvm_riscv_gstage_vmid_hwid(atomic64_read(&vcpu->kvm->arch.vmid.id)); if (cp->a1 == 0 && cp->a2 == 0) kvm_riscv_hfence_vvma_asid_all(vcpu->kvm, hbase, hmask, cp->a3, vmid); diff --git a/arch/riscv/kvm/vmid.c b/arch/riscv/kvm/vmid.c index c15bdb1dd8bef0..eee8306c12707b 100644 --- a/arch/riscv/kvm/vmid.c +++ b/arch/riscv/kvm/vmid.c @@ -6,11 +6,15 @@ * Anup Patel */ +#include #include #include #include #include #include +#include +#include +#include #include #include #include @@ -18,26 +22,54 @@ #include #include -static unsigned long vmid_version = 1; -static unsigned long vmid_next; +static atomic64_t vmid_generation; +static unsigned long *vmid_map; static unsigned long vmid_bits __ro_after_init; -static DEFINE_SPINLOCK(vmid_lock); +static unsigned long vmid_cur_idx = 1; + +static DEFINE_RAW_SPINLOCK(vmid_lock); + +static DEFINE_PER_CPU(atomic64_t, active_vmids); +static DEFINE_PER_CPU(u64, reserved_vmids); + +static cpumask_t tlb_flush_pending; + +#define VMID_FIRST_VERSION BIT_ULL(vmid_bits) +#define NUM_VMIDS BIT(vmid_bits) +#define VMID_HW_MASK (VMID_FIRST_VERSION - 1) + +static bool vmid_gen_match(u64 vmid) +{ + return !((vmid ^ atomic64_read(&vmid_generation)) >> vmid_bits); +} + +unsigned long kvm_riscv_gstage_vmid_hwid(u64 vmid) +{ + if (!vmid_bits) + return 0; + + return (unsigned long)(vmid & VMID_HW_MASK); +} void __init kvm_riscv_gstage_vmid_detect(void) { - /* Figure-out number of VMID bits in HW */ + /* Figure out the number of VMID bits supported by hardware. */ csr_write(CSR_HGATP, (kvm_riscv_gstage_mode(kvm_riscv_gstage_max_pgd_levels) << - HGATP_MODE_SHIFT) | HGATP_VMID); + HGATP_MODE_SHIFT) | HGATP_VMID); vmid_bits = csr_read(CSR_HGATP); vmid_bits = (vmid_bits & HGATP_VMID) >> HGATP_VMID_SHIFT; vmid_bits = fls_long(vmid_bits); csr_write(CSR_HGATP, 0); - /* We polluted local TLB so flush all guest TLB */ + /* Flush the local guest TLB after probing HGATP. */ kvm_riscv_local_hfence_gvma_all(); - /* We don't use VMID bits if they are not sufficient */ - if ((1UL << vmid_bits) < num_possible_cpus()) + /* + * VMID 0 is reserved. During rollover every possible CPU may + * reserve one hardware VMID, and we still need at least one VMID + * available for a new allocation. + */ + if (vmid_bits && NUM_VMIDS - 1 <= num_possible_cpus()) vmid_bits = 0; } @@ -46,80 +78,295 @@ unsigned long kvm_riscv_gstage_vmid_bits(void) return vmid_bits; } +int __init kvm_riscv_gstage_vmid_alloc_init(void) +{ + int cpu; + + if (!vmid_bits) + return 0; + + vmid_map = bitmap_zalloc(NUM_VMIDS, GFP_KERNEL); + if (!vmid_map) + return -ENOMEM; + + vmid_cur_idx = 1; + + /* Hardware VMID 0 is reserved. */ + __set_bit(0, vmid_map); + + atomic64_set(&vmid_generation, VMID_FIRST_VERSION); + + for_each_possible_cpu(cpu) { + atomic64_set(&per_cpu(active_vmids, cpu), 0); + per_cpu(reserved_vmids, cpu) = 0; + } + + /* + * Every CPU performs an initial local invalidation before its + * first VMID activation. + */ + cpumask_copy(&tlb_flush_pending, cpu_possible_mask); + + return 0; +} + +void kvm_riscv_gstage_vmid_alloc_free(void) +{ + bitmap_free(vmid_map); + vmid_map = NULL; +} + int kvm_riscv_gstage_vmid_init(struct kvm *kvm) { - /* Mark the initial VMID and VMID version invalid */ - kvm->arch.vmid.vmid_version = 0; - kvm->arch.vmid.vmid = 0; + atomic64_set(&kvm->arch.vmid.id, 0); return 0; } bool kvm_riscv_gstage_vmid_ver_changed(struct kvm_vmid *vmid) { + u64 id; + if (!vmid_bits) return false; - return unlikely(READ_ONCE(vmid->vmid_version) != - READ_ONCE(vmid_version)); + id = atomic64_read(&vmid->id); + + return unlikely(!vmid_gen_match(id)); } -static void __local_hfence_gvma_all(void *info) +/* + * Called with vmid_lock held after vmid_generation has already been + * advanced. + * + * No remote CPU is interrupted here. Instead, preserve every VMID + * which may still be used by a CPU and queue a local invalidation for + * that CPU's next VMID activation. + */ +static void flush_context(void) { - kvm_riscv_local_hfence_gvma_all(); + u64 vmid; + int cpu; + + bitmap_zero(vmid_map, NUM_VMIDS); + __set_bit(0, vmid_map); + + for_each_possible_cpu(cpu) { + vmid = atomic64_xchg(&per_cpu(active_vmids, cpu), 0); + + /* + * The CPU may already have been caught by an earlier rollover + * without performing another activation since then. In that + * case reserved_vmids is the only record of the old context. + */ + if (!vmid) + vmid = per_cpu(reserved_vmids, cpu); + + if (vmid) + __set_bit(kvm_riscv_gstage_vmid_hwid(vmid), + vmid_map); + + per_cpu(reserved_vmids, cpu) = vmid; + } + + cpumask_copy(&tlb_flush_pending, cpu_possible_mask); } -void kvm_riscv_gstage_vmid_update(struct kvm_vcpu *vcpu) +/* + * Update every reserved copy of an old software VMID. + * + * Do not stop after the first match: the same VM may have been active + * on more than one CPU when rollover occurred. + */ +static bool check_update_reserved_vmid(u64 old_vmid, u64 new_vmid) { - unsigned long i; - struct kvm_vcpu *v; - struct kvm_vmid *vmid = &vcpu->kvm->arch.vmid; + bool hit = false; + int cpu; - if (!kvm_riscv_gstage_vmid_ver_changed(vmid)) - return; + for_each_possible_cpu(cpu) { + if (per_cpu(reserved_vmids, cpu) == old_vmid) { + per_cpu(reserved_vmids, cpu) = new_vmid; + hit = true; + } + } + + return hit; +} + +/* + * Allocate/promote a software VMID. + * + * vmid_lock must be held by the caller. + */ +static u64 new_vmid_locked(struct kvm_vmid *kvm_vmid) +{ + u64 vmid = atomic64_read(&kvm_vmid->id); + u64 generation = atomic64_read(&vmid_generation); + u64 new_vmid; + unsigned long idx; - spin_lock(&vmid_lock); + if (vmid) { + new_vmid = generation | + kvm_riscv_gstage_vmid_hwid(vmid); + + /* + * The old VMID is still protected by one or more CPUs. + * Keep the same hardware VMID and only promote generation. + */ + if (check_update_reserved_vmid(vmid, new_vmid)) + return new_vmid; + + /* + * The VM had a VMID in an older generation. Reuse the same + * hardware number if it has not already been claimed. + */ + idx = kvm_riscv_gstage_vmid_hwid(vmid); + if (!__test_and_set_bit(idx, vmid_map)) + return new_vmid; + } + + /* + * Find a free VMID in the current generation. + */ + idx = find_next_zero_bit(vmid_map, NUM_VMIDS, vmid_cur_idx); + if (idx != NUM_VMIDS) + goto set_vmid; + + /* + * No free VMID. Start a new generation, preserve all CPU-local + * users, and defer each CPU's flush until its next activation. + */ + generation += VMID_FIRST_VERSION; + + /* Software VMID 0 is reserved as the invalid identifier. */ + if (unlikely(!generation)) + generation = VMID_FIRST_VERSION; + + atomic64_xchg(&vmid_generation, generation); + flush_context(); /* - * We need to re-check the vmid_version here to ensure that if - * another vcpu already allocated a valid vmid for this vm. + * NUM_VMIDS - 1 > num_possible_cpus(), therefore rollover must + * leave at least one allocatable hardware VMID. */ - if (!kvm_riscv_gstage_vmid_ver_changed(vmid)) { - spin_unlock(&vmid_lock); + idx = find_next_zero_bit(vmid_map, NUM_VMIDS, 1); + +set_vmid: + __set_bit(idx, vmid_map); + vmid_cur_idx = idx; + + return generation | idx; +} + +void kvm_riscv_gstage_vmid_update(struct kvm_vcpu *vcpu) +{ + struct kvm_vmid *kvm_vmid = &vcpu->kvm->arch.vmid; + atomic64_t *active; + unsigned long flags; + u64 vmid; + u64 old_active_vmid; + unsigned int cpu; + + if (!vmid_bits) return; - } - /* First user of a new VMID version? */ - if (unlikely(vmid_next == 0)) { - WRITE_ONCE(vmid_version, READ_ONCE(vmid_version) + 1); - vmid_next = 1; + /* + * active_vmids and tlb_flush_pending are per-CPU state. Keep the + * complete VMID activation on the same CPU. + */ + lockdep_assert_preemption_disabled(); + + cpu = smp_processor_id(); + active = this_cpu_ptr(&active_vmids); + + vmid = atomic64_read(&kvm_vmid->id); + old_active_vmid = atomic64_read(active); + /* + * Fast path. + * + * The cmpxchg races with flush_context()'s xchg on the same + * per-CPU atomic. Either this activation is captured by rollover, + * or rollover clears active first and the cmpxchg fails. + */ + if (old_active_vmid && vmid_gen_match(vmid)) { /* - * We ran out of VMIDs so we increment vmid_version and - * start assigning VMIDs from 1. - * - * This also means existing VMIDs assignment to all Guest - * instances is invalid and we have force VMID re-assignement - * for all Guest instances. The Guest instances that were not - * running will automatically pick-up new VMIDs because will - * call kvm_riscv_gstage_vmid_update() whenever they enter - * in-kernel run loop. For Guest instances that are already - * running, we force VM exits on all host CPUs using IPI and - * flush all Guest TLBs. + * HGATP must contain the same hardware VMID that is published + * in active_vmids. */ - on_each_cpu_mask(cpu_online_mask, __local_hfence_gvma_all, - NULL, 1); + if (old_active_vmid != vmid) + kvm_riscv_mmu_update_hgatp_vmid(vcpu, vmid); + + if (atomic64_cmpxchg(active, old_active_vmid, vmid) == + old_active_vmid) + return; } - vmid->vmid = vmid_next; - vmid_next++; - vmid_next &= (1 << vmid_bits) - 1; + raw_spin_lock_irqsave(&vmid_lock, flags); - WRITE_ONCE(vmid->vmid_version, READ_ONCE(vmid_version)); + /* + * Re-read under the allocator lock because another vCPU of this VM + * may already have promoted or allocated the shared VMID. + */ + vmid = atomic64_read(&kvm_vmid->id); + + if (!vmid_gen_match(vmid)) { + vmid = new_vmid_locked(kvm_vmid); + atomic64_set(&kvm_vmid->id, vmid); + } + + /* + * Complete this CPU's deferred rollover invalidation before + * installing the new hardware VMID. + */ + if (cpumask_test_and_clear_cpu(cpu, &tlb_flush_pending)) + kvm_riscv_local_hfence_gvma_all(); + + /* + * Install the current hardware VMID before publishing this CPU as + * active with the new software VMID. + */ + kvm_riscv_mmu_update_hgatp_vmid(vcpu, vmid); + + /* + * Once active_vmids is visible, the required local invalidation and + * HGATP update are complete. + */ + atomic64_set(active, vmid); + + raw_spin_unlock_irqrestore(&vmid_lock, flags); +} - spin_unlock(&vmid_lock); +/* + * CPU virtualization/CSR state is being discarded. + * + * Withdraw fast-path eligibility, but conservatively preserve the last + * identity instead of making its hardware VMID immediately reusable. + */ +void kvm_riscv_gstage_vmid_cpu_invalidate(void) +{ + unsigned long flags; + u64 vmid; + unsigned int cpu; + + if (!vmid_bits) + return; + + lockdep_assert_preemption_disabled(); + + cpu = smp_processor_id(); + + raw_spin_lock_irqsave(&vmid_lock, flags); + + vmid = atomic64_xchg(this_cpu_ptr(&active_vmids), 0); + + if (vmid) + per_cpu(reserved_vmids, cpu) = vmid; + + /* + * If active was already zero, retain the old reserved entry. + */ + cpumask_set_cpu(cpu, &tlb_flush_pending); - /* Request G-stage page table update for all VCPUs */ - kvm_for_each_vcpu(i, v, vcpu->kvm) - kvm_make_request(KVM_REQ_UPDATE_HGATP, v); + raw_spin_unlock_irqrestore(&vmid_lock, flags); }