Skip to content

[PW_SID:1163048] [v3] RISC-V: KVM: Avoid synchronous IPIs on VMID rollover #2644

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
linux-riscv-bot wants to merge 1 commit into
base: workflow__riscv__fixes
Choose a base branch
from
Open

[PW_SID:1163048] [v3] RISC-V: KVM: Avoid synchronous IPIs on VMID rollover #2644

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions arch/riscv/include/asm/kvm_gstage.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
#define __RISCV_KVM_GSTAGE_H_

#include <linux/kvm_types.h>

#include <asm/kvm_vmid.h>
struct kvm_gstage {
struct kvm *kvm;
unsigned long flags;
Expand Down Expand Up @@ -108,7 +108,8 @@ static inline void kvm_riscv_gstage_init(struct kvm_gstage *gstage, struct kvm *
{
gstage->kvm = kvm;
gstage->flags = 0;
gstage->vmid = READ_ONCE(kvm->arch.vmid.vmid);
gstage->vmid =
kvm_riscv_gstage_vmid_hwid(atomic64_read(&kvm->arch.vmid.id));
gstage->pgd = kvm->arch.pgd;
gstage->pgd_levels = kvm->arch.pgd_levels;
}
Expand Down
1 change: 1 addition & 0 deletions arch/riscv/include/asm/kvm_mmu.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ int kvm_riscv_mmu_map(struct kvm_vcpu *vcpu, struct kvm_memory_slot *memslot,
struct kvm_gstage_mapping *out_map);
int kvm_riscv_mmu_alloc_pgd(struct kvm *kvm);
void kvm_riscv_mmu_free_pgd(struct kvm *kvm);
void kvm_riscv_mmu_update_hgatp_vmid(struct kvm_vcpu *vcpu, u64 vmid);
void kvm_riscv_mmu_update_hgatp(struct kvm_vcpu *vcpu);

#endif
29 changes: 25 additions & 4 deletions arch/riscv/include/asm/kvm_vmid.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,21 +6,42 @@
#ifndef __RISCV_KVM_VMID_H_
#define __RISCV_KVM_VMID_H_

#include <linux/atomic.h>
#include <linux/kvm_types.h>

struct kvm_vmid {
/*
* Writes to vmid_version and vmid happen with vmid_lock held
* whereas reads happen without any lock held.
* Software VMID:
*
* [ generation | hardware VMID ]
*
* Keep the identity 64-bit on RV32 so that the generation does not
* wrap at the native word size. Zero denotes an unallocated VMID.
*
* Only the low hardware VMID bits may be written to HGATP
* or used as a hardware fence VMID.
*/
unsigned long vmid_version;
unsigned long vmid;
atomic64_t id;
};

void __init kvm_riscv_gstage_vmid_detect(void);
unsigned long kvm_riscv_gstage_vmid_bits(void);

int __init kvm_riscv_gstage_vmid_alloc_init(void);
void kvm_riscv_gstage_vmid_alloc_free(void);

int kvm_riscv_gstage_vmid_init(struct kvm *kvm);

unsigned long kvm_riscv_gstage_vmid_hwid(u64 vmid);

bool kvm_riscv_gstage_vmid_ver_changed(struct kvm_vmid *vmid);

void kvm_riscv_gstage_vmid_update(struct kvm_vcpu *vcpu);

/*
* Invalidate the current CPU's fast-path VMID state while preserving
* the old identity conservatively in reserved_vmids.
*/
void kvm_riscv_gstage_vmid_cpu_invalidate(void);

#endif
12 changes: 11 additions & 1 deletion arch/riscv/kvm/main.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
#include <asm/kvm_nacl.h>
#include <asm/sbi.h>
#include <asm/kvm_vcpu_vector.h>

#include <asm/kvm_vmid.h>
static DEFINE_PER_CPU(bool, kvm_riscv_virtualization_enabled);

DEFINE_STATIC_KEY_FALSE(kvm_riscv_vsstage_tlb_no_gpa);
Expand Down Expand Up @@ -84,6 +84,7 @@ int kvm_arch_enable_virtualization_cpu(void)

void kvm_arch_disable_virtualization_cpu(void)
{
kvm_riscv_gstage_vmid_cpu_invalidate();
kvm_riscv_aia_disable();
kvm_riscv_csr_cleanup();
kvm_riscv_nacl_disable();
Expand Down Expand Up @@ -112,6 +113,7 @@ static int kvm_riscv_cpu_pm_notifier(struct notifier_block *self, unsigned long
* is enabled on this CPU.
*/
if (__this_cpu_read(kvm_riscv_virtualization_enabled)) {
kvm_riscv_gstage_vmid_cpu_invalidate();
kvm_riscv_aia_pm_enter();
kvm_riscv_csr_cleanup();
}
Expand All @@ -130,6 +132,7 @@ static struct notifier_block kvm_riscv_cpu_pm_nb = {
static void kvm_riscv_teardown(void)
{
kvm_riscv_aia_exit();
kvm_riscv_gstage_vmid_alloc_free();
kvm_riscv_nacl_exit();
kvm_riscv_v_exit();
kvm_unregister_perf_callbacks();
Expand Down Expand Up @@ -181,8 +184,15 @@ static int __init riscv_kvm_init(void)

kvm_riscv_gstage_vmid_detect();

rc = kvm_riscv_gstage_vmid_alloc_init();
if (rc) {
kvm_riscv_nacl_exit();
return rc;
}

rc = kvm_riscv_aia_init();
if (rc && rc != -ENODEV) {
kvm_riscv_gstage_vmid_alloc_free();
kvm_riscv_nacl_exit();
return rc;
}
Expand Down
13 changes: 10 additions & 3 deletions arch/riscv/kvm/mmu.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
#include <linux/sched/signal.h>
#include <asm/kvm_mmu.h>
#include <asm/kvm_nacl.h>

#include <asm/kvm_vmid.h>
static bool __read_mostly eager_page_split = true;
module_param(eager_page_split, bool, 0644);

Expand Down Expand Up @@ -796,16 +796,23 @@ void kvm_riscv_mmu_free_pgd(struct kvm *kvm)
kvm_mmu_free_memory_cache(&kvm->arch.pgd_split_page_cache);
}

void kvm_riscv_mmu_update_hgatp(struct kvm_vcpu *vcpu)
void kvm_riscv_mmu_update_hgatp_vmid(struct kvm_vcpu *vcpu, u64 vmid)
{
struct kvm_arch *ka = &vcpu->kvm->arch;
unsigned long hgatp = kvm_riscv_gstage_mode(ka->pgd_levels)
<< HGATP_MODE_SHIFT;

hgatp |= (READ_ONCE(ka->vmid.vmid) << HGATP_VMID_SHIFT) & HGATP_VMID;
hgatp |= (kvm_riscv_gstage_vmid_hwid(vmid) << HGATP_VMID_SHIFT) & HGATP_VMID;
hgatp |= (ka->pgd_phys >> PAGE_SHIFT) & HGATP_PPN;

ncsr_write(CSR_HGATP, hgatp);
}

void kvm_riscv_mmu_update_hgatp(struct kvm_vcpu *vcpu)
{
u64 vmid = atomic64_read(&vcpu->kvm->arch.vmid.id);

kvm_riscv_mmu_update_hgatp_vmid(vcpu, vmid);

if (!kvm_riscv_gstage_vmid_bits())
kvm_riscv_local_hfence_gvma_all();
Expand Down
11 changes: 7 additions & 4 deletions arch/riscv/kvm/tlb.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -224,7 +224,7 @@ void kvm_riscv_local_tlb_sanitize(struct kvm_vcpu *vcpu)
* entries by VMID whenever underlying Host CPU changes for a VCPU.
*/

vmid = READ_ONCE(vcpu->kvm->arch.vmid.vmid);
vmid = kvm_riscv_gstage_vmid_hwid(atomic64_read(&vcpu->kvm->arch.vmid.id));
kvm_riscv_local_hfence_gvma_vmid_all(vmid);

/*
Expand All @@ -244,7 +244,7 @@ void kvm_riscv_fence_i_process(struct kvm_vcpu *vcpu)
void kvm_riscv_tlb_flush_process(struct kvm_vcpu *vcpu)
{
struct kvm_vmid *v = &vcpu->kvm->arch.vmid;
unsigned long vmid = READ_ONCE(v->vmid);
unsigned long vmid = kvm_riscv_gstage_vmid_hwid(atomic64_read(&v->id));

if (kvm_riscv_nacl_available())
nacl_hfence_gvma_vmid_all(nacl_shmem(), vmid);
Expand All @@ -255,7 +255,7 @@ void kvm_riscv_tlb_flush_process(struct kvm_vcpu *vcpu)
void kvm_riscv_hfence_vvma_all_process(struct kvm_vcpu *vcpu)
{
struct kvm_vmid *v = &vcpu->kvm->arch.vmid;
unsigned long vmid = READ_ONCE(v->vmid);
unsigned long vmid = kvm_riscv_gstage_vmid_hwid(atomic64_read(&v->id));

if (kvm_riscv_nacl_available())
nacl_hfence_vvma_all(nacl_shmem(), vmid);
Expand Down Expand Up @@ -531,8 +531,11 @@ void kvm_riscv_hfence_vvma_all(struct kvm *kvm,

int kvm_arch_flush_remote_tlbs_range(struct kvm *kvm, gfn_t gfn, u64 nr_pages)
{
unsigned long vmid =
kvm_riscv_gstage_vmid_hwid(atomic64_read(&kvm->arch.vmid.id));

kvm_riscv_hfence_gvma_vmid_gpa(kvm, -1UL, 0,
gfn << PAGE_SHIFT, nr_pages << PAGE_SHIFT,
PAGE_SHIFT, READ_ONCE(kvm->arch.vmid.vmid));
PAGE_SHIFT, vmid);
return 0;
}
4 changes: 2 additions & 2 deletions arch/riscv/kvm/vcpu.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -943,14 +943,14 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
continue;
ret = 1;

kvm_riscv_gstage_vmid_update(vcpu);

ret = kvm_riscv_check_vcpu_requests(vcpu);
if (ret <= 0)
continue;

preempt_disable();

kvm_riscv_gstage_vmid_update(vcpu);

/* Update AIA HW state before entering guest */
ret = kvm_riscv_vcpu_aia_update(vcpu);
if (ret <= 0) {
Expand Down
4 changes: 2 additions & 2 deletions arch/riscv/kvm/vcpu_sbi_replace.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ static int kvm_sbi_ext_rfence_handler(struct kvm_vcpu *vcpu, struct kvm_run *run
kvm_riscv_vcpu_pmu_incr_fw(vcpu, SBI_PMU_FW_FENCE_I_SENT);
break;
case SBI_EXT_RFENCE_REMOTE_SFENCE_VMA:
vmid = READ_ONCE(vcpu->kvm->arch.vmid.vmid);
vmid = kvm_riscv_gstage_vmid_hwid(atomic64_read(&vcpu->kvm->arch.vmid.id));
if ((cp->a2 == 0 && cp->a3 == 0) || cp->a3 == -1UL)
kvm_riscv_hfence_vvma_all(vcpu->kvm, hbase, hmask, vmid);
else
Expand All @@ -113,7 +113,7 @@ static int kvm_sbi_ext_rfence_handler(struct kvm_vcpu *vcpu, struct kvm_run *run
kvm_riscv_vcpu_pmu_incr_fw(vcpu, SBI_PMU_FW_HFENCE_VVMA_SENT);
break;
case SBI_EXT_RFENCE_REMOTE_SFENCE_VMA_ASID:
vmid = READ_ONCE(vcpu->kvm->arch.vmid.vmid);
vmid = kvm_riscv_gstage_vmid_hwid(atomic64_read(&vcpu->kvm->arch.vmid.id));
if ((cp->a2 == 0 && cp->a3 == 0) || cp->a3 == -1UL)
kvm_riscv_hfence_vvma_asid_all(vcpu->kvm, hbase, hmask,
cp->a4, vmid);
Expand Down
4 changes: 2 additions & 2 deletions arch/riscv/kvm/vcpu_sbi_v01.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,14 +88,14 @@ static int kvm_sbi_ext_v01_handler(struct kvm_vcpu *vcpu, struct kvm_run *run,
if (cp->a7 == SBI_EXT_0_1_REMOTE_FENCE_I)
kvm_riscv_fence_i(vcpu->kvm, hbase, hmask);
else if (cp->a7 == SBI_EXT_0_1_REMOTE_SFENCE_VMA) {
vmid = READ_ONCE(vcpu->kvm->arch.vmid.vmid);
vmid = kvm_riscv_gstage_vmid_hwid(atomic64_read(&vcpu->kvm->arch.vmid.id));
if (cp->a1 == 0 && cp->a2 == 0)
kvm_riscv_hfence_vvma_all(vcpu->kvm, hbase, hmask, vmid);
else
kvm_riscv_hfence_vvma_gva(vcpu->kvm, hbase, hmask, cp->a1,
cp->a2, PAGE_SHIFT, vmid);
} else {
vmid = READ_ONCE(vcpu->kvm->arch.vmid.vmid);
vmid = kvm_riscv_gstage_vmid_hwid(atomic64_read(&vcpu->kvm->arch.vmid.id));
if (cp->a1 == 0 && cp->a2 == 0)
kvm_riscv_hfence_vvma_asid_all(vcpu->kvm, hbase, hmask,
cp->a3, vmid);
Expand Down
Loading
Loading