From d62dae95a93692c705f33ec48c1bcebc5d0262f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20L=C3=A9ger?= Date: Fri, 11 Sep 2026 20:14:48 +0800 Subject: [PATCH 1/9] riscv: add SBI SSE extension definitions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add definitions for the SBI Supervisor Software Events extension [1]. This extension enables the SBI to inject events into supervisor software much like ARM SDEI. [1] https://lists.riscv.org/g/tech-prs/message/515 Signed-off-by: Clément Léger Co-developed-by: Himanshu Chauhan Signed-off-by: Himanshu Chauhan Co-developed-by: Zhanpeng Zhang Signed-off-by: Zhanpeng Zhang Signed-off-by: Linux RISC-V bot --- arch/riscv/include/asm/sbi.h | 63 ++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/arch/riscv/include/asm/sbi.h b/arch/riscv/include/asm/sbi.h index 5725e0ca4dda3e..4e54d79ba543a9 100644 --- a/arch/riscv/include/asm/sbi.h +++ b/arch/riscv/include/asm/sbi.h @@ -38,6 +38,7 @@ enum sbi_ext_id { SBI_EXT_FWFT = 0x46574654, SBI_EXT_MPXY = 0x4D505859, SBI_EXT_DBTR = 0x44425452, + SBI_EXT_SSE = 0x535345, /* Experimentals extensions must lie within this range */ SBI_EXT_EXPERIMENTAL_START = 0x08000000, @@ -506,6 +507,68 @@ enum sbi_mpxy_rpmi_attribute_id { #define SBI_MPXY_CHAN_CAP_SEND_WITHOUT_RESP BIT(4) #define SBI_MPXY_CHAN_CAP_GET_NOTIFICATIONS BIT(5) +enum sbi_ext_sse_fid { + SBI_SSE_EVENT_ATTR_READ = 0, + SBI_SSE_EVENT_ATTR_WRITE, + SBI_SSE_EVENT_REGISTER, + SBI_SSE_EVENT_UNREGISTER, + SBI_SSE_EVENT_ENABLE, + SBI_SSE_EVENT_DISABLE, + SBI_SSE_EVENT_COMPLETE, + SBI_SSE_EVENT_INJECT, + SBI_SSE_HART_UNMASK, + SBI_SSE_HART_MASK, +}; + +enum sbi_sse_state { + SBI_SSE_STATE_UNUSED = 0, + SBI_SSE_STATE_REGISTERED = 1, + SBI_SSE_STATE_ENABLED = 2, + SBI_SSE_STATE_RUNNING = 3, +}; + +/* SBI SSE Event Attributes. */ +enum sbi_sse_attr_id { + SBI_SSE_ATTR_STATUS = 0x00000000, + SBI_SSE_ATTR_PRIO = 0x00000001, + SBI_SSE_ATTR_CONFIG = 0x00000002, + SBI_SSE_ATTR_PREFERRED_HART = 0x00000003, + SBI_SSE_ATTR_ENTRY_PC = 0x00000004, + SBI_SSE_ATTR_ENTRY_ARG = 0x00000005, + SBI_SSE_ATTR_INTERRUPTED_SEPC = 0x00000006, + SBI_SSE_ATTR_INTERRUPTED_FLAGS = 0x00000007, + SBI_SSE_ATTR_INTERRUPTED_A6 = 0x00000008, + SBI_SSE_ATTR_INTERRUPTED_A7 = 0x00000009, + + SBI_SSE_ATTR_MAX = 0x0000000A +}; + +#define SBI_SSE_ATTR_STATUS_STATE_OFFSET 0 +#define SBI_SSE_ATTR_STATUS_STATE_MASK 0x3 +#define SBI_SSE_ATTR_STATUS_PENDING_OFFSET 2 +#define SBI_SSE_ATTR_STATUS_INJECT_OFFSET 3 + +#define SBI_SSE_ATTR_CONFIG_ONESHOT BIT(0) + +#define SBI_SSE_ATTR_INTERRUPTED_FLAGS_SSTATUS_SPP BIT(0) +#define SBI_SSE_ATTR_INTERRUPTED_FLAGS_SSTATUS_SPIE BIT(1) +#define SBI_SSE_ATTR_INTERRUPTED_FLAGS_HSTATUS_SPV BIT(2) +#define SBI_SSE_ATTR_INTERRUPTED_FLAGS_HSTATUS_SPVP BIT(3) +#define SBI_SSE_ATTR_INTERRUPTED_FLAGS_SSTATUS_SPELP BIT(4) +#define SBI_SSE_ATTR_INTERRUPTED_FLAGS_SSTATUS_SDT BIT(5) + +#define SBI_SSE_EVENT_LOCAL_HIGH_PRIO_RAS 0x00000000 +#define SBI_SSE_EVENT_LOCAL_DOUBLE_TRAP 0x00000001 +#define SBI_SSE_EVENT_GLOBAL_HIGH_PRIO_RAS 0x00008000 +#define SBI_SSE_EVENT_LOCAL_PMU_OVERFLOW 0x00010000 +#define SBI_SSE_EVENT_LOCAL_LOW_PRIO_RAS 0x00100000 +#define SBI_SSE_EVENT_GLOBAL_LOW_PRIO_RAS 0x00108000 +#define SBI_SSE_EVENT_LOCAL_SOFTWARE_INJECTED 0xffff0000 +#define SBI_SSE_EVENT_GLOBAL_SOFTWARE_INJECTED 0xffff8000 + +#define SBI_SSE_EVENT_PLATFORM BIT(14) +#define SBI_SSE_EVENT_GLOBAL BIT(15) + /* SBI debug triggers function IDs */ enum sbi_ext_dbtr_fid { SBI_EXT_DBTR_NUM_TRIGGERS = 0, From cec0277b0fe4bce4a1921785bbae6925af274ee4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20L=C3=A9ger?= Date: Fri, 11 Sep 2026 20:14:49 +0800 Subject: [PATCH 2/9] riscv: add support for SBI Supervisor Software Events extension MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The SBI SSE extension allows firmware to notify supervisor software of events that must be delivered independently of normal S-mode interrupts. Firmware saves the minimal state required to enter the supervisor handler, and Linux builds the synthetic handler context around it. SSE can arrive while Linux is already in an exception entry path. At that point sscratch and tp may be in the middle of the normal trap-entry exchange, so they cannot always identify current. Store current in a per-CPU slot and use the hart ID passed by firmware to recover it. Give each event, including each CPU instance of a local event, a dedicated stack and shadow call stack. Synchronize vmapped stack ranges before unmasking events so that the handler cannot take a vmalloc fault while running in an NMI-like context. The handler is a synthetic supervisor episode, but completion must resume the context interrupted by the SSE. Preserve stvec and, when the hypervisor extension is present, hstatus across the handler. Read the interrupted a6 and a7 values from the SSE attributes, construct pt_regs for the interrupted context, and write back any changes made by the handler. Nested exceptions on the SSE event stack temporarily replace both TASK_TI_KERNEL_SP and TASK_TI_USER_SP. Preserve their original values across the handler and restore them before completing the event, so a nested exception cannot leave the interrupted task referring to the event stack. Keep an explicit EVENT_REGISTER not-supported result distinct from other firmware failures. This lets clients select another delivery mechanism only when firmware has positively rejected the requested event. Signed-off-by: Clément Léger Co-developed-by: Himanshu Chauhan Signed-off-by: Himanshu Chauhan Co-developed-by: Zhanpeng Zhang Signed-off-by: Zhanpeng Zhang Signed-off-by: Linux RISC-V bot --- MAINTAINERS | 12 ++ arch/riscv/include/asm/asm.h | 14 +- arch/riscv/include/asm/scs.h | 7 + arch/riscv/include/asm/sse.h | 82 +++++++++ arch/riscv/include/asm/thread_info.h | 1 + arch/riscv/kernel/Makefile | 1 + arch/riscv/kernel/asm-offsets.c | 14 ++ arch/riscv/kernel/entry.S | 14 ++ arch/riscv/kernel/sbi_sse.c | 246 +++++++++++++++++++++++++++ arch/riscv/kernel/sbi_sse_entry.S | 226 ++++++++++++++++++++++++ 10 files changed, 614 insertions(+), 3 deletions(-) create mode 100644 arch/riscv/include/asm/sse.h create mode 100644 arch/riscv/kernel/sbi_sse.c create mode 100644 arch/riscv/kernel/sbi_sse_entry.S diff --git a/MAINTAINERS b/MAINTAINERS index a830d3b252e2d9..f6a2ed9990dd06 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -23424,6 +23424,18 @@ F: arch/riscv/boot/dts/spacemit/ N: spacemit K: spacemit +RISC-V SUPERVISOR SOFTWARE EVENTS +M: Zhanpeng Zhang +M: Himanshu Chauhan +R: Yunhui Cui +L: linux-riscv@lists.infradead.org +S: Maintained +F: arch/riscv/include/asm/sse.h +F: arch/riscv/kernel/sbi_sse.c +F: arch/riscv/kernel/sbi_sse_entry.S +F: drivers/firmware/riscv/riscv_sbi_sse.c +F: include/linux/riscv_sbi_sse.h + RISC-V TENSTORRENT SoC SUPPORT M: Drew Fustini M: Joel Stanley diff --git a/arch/riscv/include/asm/asm.h b/arch/riscv/include/asm/asm.h index b8bf842d4c1367..e1196caea02dcb 100644 --- a/arch/riscv/include/asm/asm.h +++ b/arch/riscv/include/asm/asm.h @@ -91,16 +91,24 @@ .endm #ifdef CONFIG_SMP -.macro asm_per_cpu dst sym tmp - lw \tmp, TASK_TI_CPU_NUM(tp) - slli \tmp, \tmp, RISCV_LGPTR +.macro asm_per_cpu_with_cpu dst sym tmp cpu + slli \tmp, \cpu, RISCV_LGPTR la \dst, __per_cpu_offset add \dst, \dst, \tmp REG_L \tmp, 0(\dst) la \dst, \sym add \dst, \dst, \tmp .endm + +.macro asm_per_cpu dst sym tmp + lw \tmp, TASK_TI_CPU_NUM(tp) + asm_per_cpu_with_cpu \dst \sym \tmp \tmp +.endm #else /* CONFIG_SMP */ +.macro asm_per_cpu_with_cpu dst sym tmp cpu + la \dst, \sym +.endm + .macro asm_per_cpu dst sym tmp la \dst, \sym .endm diff --git a/arch/riscv/include/asm/scs.h b/arch/riscv/include/asm/scs.h index 023a412fe38d37..0d70a35bc01ab8 100644 --- a/arch/riscv/include/asm/scs.h +++ b/arch/riscv/include/asm/scs.h @@ -17,6 +17,11 @@ load_per_cpu gp, irq_shadow_call_stack_ptr, \tmp .endm +/* Load the per-CPU IRQ shadow call stack to gp. */ +.macro scs_load_sse_stack reg_evt + REG_L gp, SSE_REG_EVT_SHADOW_STACK(\reg_evt) +.endm + /* Load task_scs_sp(current) to gp. */ .macro scs_load_current REG_L gp, TASK_TI_SCS_SP(tp) @@ -40,6 +45,8 @@ .endm .macro scs_load_irq_stack tmp .endm +.macro scs_load_sse_stack reg_evt +.endm .macro scs_load_current .endm .macro scs_load_current_if_task_changed prev diff --git a/arch/riscv/include/asm/sse.h b/arch/riscv/include/asm/sse.h new file mode 100644 index 00000000000000..cbd8618c6e00e3 --- /dev/null +++ b/arch/riscv/include/asm/sse.h @@ -0,0 +1,82 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2024 Rivos Inc. + */ +#ifndef __ASM_SSE_H +#define __ASM_SSE_H + +#include +#include + +#include + +static inline bool riscv_sse_available(void) +{ +#ifdef CONFIG_RISCV_SBI + return sbi_probe_extension(SBI_EXT_SSE) > 0; +#else + return false; +#endif +} + +static inline void riscv_sse_mask_current_hart(void) +{ +#ifdef CONFIG_RISCV_SBI + struct sbiret ret; + + if (!riscv_sse_available()) + return; + + ret = sbi_ecall(SBI_EXT_SSE, SBI_SSE_HART_MASK, 0, 0, 0, 0, 0, 0); + if (ret.error && ret.error != SBI_ERR_ALREADY_STOPPED) + pr_emerg("SSE hart mask failed: %ld\n", ret.error); +#endif +} + +#ifdef CONFIG_RISCV_SBI_SSE + +struct sse_event_interrupted_state { + unsigned long a6; + unsigned long a7; +}; + +struct sse_event_arch_data { + void *stack; + void *shadow_stack; + unsigned long tmp; + struct sse_event_interrupted_state *interrupted; + phys_addr_t interrupted_phys; + u32 evt_id; + unsigned long hart_id; + unsigned int cpu_id; +}; + +struct riscv_sse_interrupted_context { + struct pt_regs *regs; + unsigned long hstatus; +}; + +static inline bool sse_event_is_global(u32 evt) +{ + return !!(evt & SBI_SSE_EVENT_GLOBAL); +} + +void arch_sse_event_update_cpu(struct sse_event_arch_data *arch_evt, int cpu); +int arch_sse_init_event(struct sse_event_arch_data *arch_evt, u32 evt_id, + int cpu); +void arch_sse_free_event(struct sse_event_arch_data *arch_evt); +int arch_sse_register_event(struct sse_event_arch_data *arch_evt); +void arch_sse_init_cpu(void); + +void sse_handle_event(struct sse_event_arch_data *arch_evt, + struct pt_regs *regs); +asmlinkage void handle_sse(void); +asmlinkage void noinstr do_sse(struct sse_event_arch_data *arch_evt, + struct pt_regs *regs, unsigned long hstatus); + +const struct riscv_sse_interrupted_context * +riscv_sse_get_interrupted_context(void); + +#endif + +#endif diff --git a/arch/riscv/include/asm/thread_info.h b/arch/riscv/include/asm/thread_info.h index 55019fdfa9ecaa..d14b45610c73a2 100644 --- a/arch/riscv/include/asm/thread_info.h +++ b/arch/riscv/include/asm/thread_info.h @@ -36,6 +36,7 @@ #define OVERFLOW_STACK_SIZE SZ_4K #define IRQ_STACK_SIZE THREAD_SIZE +#define SSE_STACK_SIZE THREAD_SIZE #ifndef __ASSEMBLER__ diff --git a/arch/riscv/kernel/Makefile b/arch/riscv/kernel/Makefile index ebe1c3588177b4..1e34f87e97c09a 100644 --- a/arch/riscv/kernel/Makefile +++ b/arch/riscv/kernel/Makefile @@ -101,6 +101,7 @@ obj-$(CONFIG_DYNAMIC_FTRACE) += mcount-dyn.o obj-$(CONFIG_PERF_EVENTS) += perf_callchain.o obj-$(CONFIG_HAVE_PERF_REGS) += perf_regs.o obj-$(CONFIG_RISCV_SBI) += sbi.o sbi_ecall.o +obj-$(CONFIG_RISCV_SBI_SSE) += sbi_sse.o sbi_sse_entry.o ifeq ($(CONFIG_RISCV_SBI), y) obj-$(CONFIG_SMP) += sbi-ipi.o obj-$(CONFIG_SMP) += cpu_ops_sbi.o diff --git a/arch/riscv/kernel/asm-offsets.c b/arch/riscv/kernel/asm-offsets.c index a75f0cfea1e9fa..15363703cdd6d0 100644 --- a/arch/riscv/kernel/asm-offsets.c +++ b/arch/riscv/kernel/asm-offsets.c @@ -15,6 +15,8 @@ #include #include #include +#include +#include #include void asm_offsets(void); @@ -533,6 +535,18 @@ void asm_offsets(void) DEFINE(FREGS_A6, offsetof(struct __arch_ftrace_regs, a6)); DEFINE(FREGS_A7, offsetof(struct __arch_ftrace_regs, a7)); #endif + +#ifdef CONFIG_RISCV_SBI_SSE + OFFSET(SSE_REG_EVT_STACK, sse_event_arch_data, stack); + OFFSET(SSE_REG_EVT_SHADOW_STACK, sse_event_arch_data, shadow_stack); + OFFSET(SSE_REG_EVT_TMP, sse_event_arch_data, tmp); + OFFSET(SSE_REG_HART_ID, sse_event_arch_data, hart_id); + OFFSET(SSE_REG_CPU_ID, sse_event_arch_data, cpu_id); + + DEFINE(SBI_EXT_SSE, SBI_EXT_SSE); + DEFINE(SBI_SSE_EVENT_COMPLETE, SBI_SSE_EVENT_COMPLETE); + DEFINE(ASM_NR_CPUS, CONFIG_NR_CPUS); +#endif #ifdef CONFIG_RISCV_SBI DEFINE(SBI_EXT_FWFT, SBI_EXT_FWFT); DEFINE(SBI_EXT_FWFT_SET, SBI_EXT_FWFT_SET); diff --git a/arch/riscv/kernel/entry.S b/arch/riscv/kernel/entry.S index d799c4e56f8049..0b79fa7241ea63 100644 --- a/arch/riscv/kernel/entry.S +++ b/arch/riscv/kernel/entry.S @@ -424,6 +424,15 @@ SYM_FUNC_END(call_on_irq_stack) * arguments are passed to schedule_tail. */ SYM_FUNC_START(__switch_to) +#ifdef CONFIG_RISCV_SBI_SSE + /* + * Mark the interval where tp changes from prev to next. SSE entry uses + * the interrupted tp while this per-CPU pointer is NULL. + */ + asm_per_cpu t0, __sbi_sse_entry_task, t1 + REG_S zero, 0(t0) +#endif + /* Save context into prev->thread */ li a4, TASK_THREAD_RA add a3, a0, a4 @@ -470,6 +479,11 @@ SYM_FUNC_START(__switch_to) REG_L s11, TASK_THREAD_S11_RA(a4) /* The offset of thread_info in task_struct is zero. */ move tp, a1 +#ifdef CONFIG_RISCV_SBI_SSE + /* Publish next only after tp contains its task_struct pointer. */ + asm_per_cpu t0, __sbi_sse_entry_task, t1 + REG_S tp, 0(t0) +#endif /* Switch to the next shadow call stack */ scs_load_current ret diff --git a/arch/riscv/kernel/sbi_sse.c b/arch/riscv/kernel/sbi_sse.c new file mode 100644 index 00000000000000..7dd496e2bdb0ef --- /dev/null +++ b/arch/riscv/kernel/sbi_sse.c @@ -0,0 +1,246 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2025 Rivos Inc. + */ +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +DEFINE_PER_CPU(struct task_struct *, __sbi_sse_entry_task); +static DEFINE_PER_CPU(struct riscv_sse_interrupted_context *, + riscv_sse_interrupted_context); + +const struct riscv_sse_interrupted_context * +riscv_sse_get_interrupted_context(void) +{ + return this_cpu_read(riscv_sse_interrupted_context); +} + +void __weak sse_handle_event(struct sse_event_arch_data *arch_evt, struct pt_regs *regs) +{ +} + +void noinstr do_sse(struct sse_event_arch_data *arch_evt, + struct pt_regs *regs, unsigned long hstatus) +{ + struct riscv_sse_interrupted_context context = { regs, hstatus }; + struct riscv_sse_interrupted_context *previous; + struct sbiret sret; + + nmi_enter(); + instrumentation_begin(); + + /* Retrieve missing GPRs from SBI */ + sret = sbi_ecall(SBI_EXT_SSE, SBI_SSE_EVENT_ATTR_READ, arch_evt->evt_id, + SBI_SSE_ATTR_INTERRUPTED_A6, + (SBI_SSE_ATTR_INTERRUPTED_A7 - + SBI_SSE_ATTR_INTERRUPTED_A6) + 1, + (unsigned long)arch_evt->interrupted_phys, 0, 0); + if (sret.error) { + pr_warn("Failed to read interrupted registers for event %x: %ld\n", + arch_evt->evt_id, sret.error); + /* Let the client quiesce its source without using incomplete regs. */ + sse_handle_event(arch_evt, NULL); + goto out; + } + + memcpy(®s->a6, arch_evt->interrupted, + sizeof(*arch_evt->interrupted)); + + /* Make the interrupted frame visible while clients handle this event. */ + previous = this_cpu_read(riscv_sse_interrupted_context); + this_cpu_write(riscv_sse_interrupted_context, &context); + sse_handle_event(arch_evt, regs); + this_cpu_write(riscv_sse_interrupted_context, previous); + + if (memcmp(®s->a6, arch_evt->interrupted, + sizeof(*arch_evt->interrupted))) { + memcpy(arch_evt->interrupted, ®s->a6, + sizeof(*arch_evt->interrupted)); + sret = sbi_ecall(SBI_EXT_SSE, SBI_SSE_EVENT_ATTR_WRITE, + arch_evt->evt_id, SBI_SSE_ATTR_INTERRUPTED_A6, + (SBI_SSE_ATTR_INTERRUPTED_A7 - + SBI_SSE_ATTR_INTERRUPTED_A6) + 1, + (unsigned long)arch_evt->interrupted_phys, 0, 0); + /* + * If writeback fails, COMPLETE resumes with firmware's original + * a6/a7 rather than treating the shared buffer as committed. + */ + if (sret.error) + pr_warn("Failed to write interrupted registers for event %x: %ld\n", + arch_evt->evt_id, sret.error); + } + +out: + instrumentation_end(); + nmi_exit(); +} + +static void *alloc_to_stack_pointer(void *alloc) +{ + return alloc ? alloc + SSE_STACK_SIZE : NULL; +} + +static void *stack_pointer_to_alloc(void *stack) +{ + return stack ? stack - SSE_STACK_SIZE : NULL; +} + +static void arch_sse_flush_tlb_range(struct sse_event_arch_data *arch_evt, + unsigned long start, unsigned long size) +{ + unsigned long end = start + size; + + if (sse_event_is_global(arch_evt->evt_id)) + flush_tlb_kernel_range(start, end); + else + local_flush_tlb_kernel_range(start, end); +} + +static void arch_sse_shadow_stack_cpu_sync(struct sse_event_arch_data *arch_evt) +{ +#ifdef CONFIG_SHADOW_CALL_STACK + if (arch_evt->shadow_stack) + arch_sse_flush_tlb_range(arch_evt, + (unsigned long)arch_evt->shadow_stack, + SCS_SIZE); +#endif +} + +#ifdef CONFIG_VMAP_STACK +static void *sse_stack_alloc(unsigned int cpu) +{ + void *stack = arch_alloc_vmap_stack(SSE_STACK_SIZE, cpu_to_node(cpu)); + + return alloc_to_stack_pointer(stack); +} + +static void sse_stack_free(void *stack) +{ + vfree(stack_pointer_to_alloc(stack)); +} + +static void arch_sse_stack_cpu_sync(struct sse_event_arch_data *arch_evt) +{ + void *p_stack = arch_evt->stack; + unsigned long stack = (unsigned long)stack_pointer_to_alloc(p_stack); + + /* + * Flush the tlb to avoid taking any exception when accessing the + * vmapped stack inside the SSE handler + */ + arch_sse_flush_tlb_range(arch_evt, stack, SSE_STACK_SIZE); + + arch_sse_shadow_stack_cpu_sync(arch_evt); +} +#else /* CONFIG_VMAP_STACK */ +static void *sse_stack_alloc(unsigned int cpu) +{ + void *stack = kmalloc(SSE_STACK_SIZE, GFP_KERNEL); + + return alloc_to_stack_pointer(stack); +} + +static void sse_stack_free(void *stack) +{ + kfree(stack_pointer_to_alloc(stack)); +} + +static void arch_sse_stack_cpu_sync(struct sse_event_arch_data *arch_evt) +{ + arch_sse_shadow_stack_cpu_sync(arch_evt); +} +#endif /* CONFIG_VMAP_STACK */ + +static int sse_init_scs(int cpu, struct sse_event_arch_data *arch_evt) +{ + void *stack; + + if (!scs_is_enabled()) + return 0; + + stack = scs_alloc(cpu_to_node(cpu)); + if (!stack) + return -ENOMEM; + + arch_evt->shadow_stack = stack; + + return 0; +} + +void arch_sse_event_update_cpu(struct sse_event_arch_data *arch_evt, int cpu) +{ + arch_evt->cpu_id = cpu; + arch_evt->hart_id = cpuid_to_hartid_map(cpu); +} + +void arch_sse_init_cpu(void) +{ + __this_cpu_write(__sbi_sse_entry_task, current); +} + +int arch_sse_init_event(struct sse_event_arch_data *arch_evt, u32 evt_id, + int cpu) +{ + void *stack; + + arch_evt->interrupted = kmalloc_obj(*arch_evt->interrupted, GFP_KERNEL); + if (!arch_evt->interrupted) + return -ENOMEM; + + arch_evt->evt_id = evt_id; + stack = sse_stack_alloc(cpu); + if (!stack) + goto err_free_interrupted; + + arch_evt->stack = stack; + + if (sse_init_scs(cpu, arch_evt)) { + sse_stack_free(arch_evt->stack); + goto err_free_interrupted; + } + + /* kmalloc keeps the two adjacent SBI attribute words contiguous. */ + arch_evt->interrupted_phys = virt_to_phys(arch_evt->interrupted); + + arch_sse_event_update_cpu(arch_evt, cpu); + + return 0; + +err_free_interrupted: + kfree(arch_evt->interrupted); + arch_evt->interrupted = NULL; + return -ENOMEM; +} + +void arch_sse_free_event(struct sse_event_arch_data *arch_evt) +{ + scs_free(arch_evt->shadow_stack); + sse_stack_free(arch_evt->stack); + kfree(arch_evt->interrupted); +} + +int arch_sse_register_event(struct sse_event_arch_data *arch_evt) +{ + struct sbiret sret; + + arch_sse_stack_cpu_sync(arch_evt); + + sret = sbi_ecall(SBI_EXT_SSE, SBI_SSE_EVENT_REGISTER, arch_evt->evt_id, + (unsigned long)handle_sse, (unsigned long)arch_evt, 0, + 0, 0); + if (sret.error == SBI_ERR_NOT_SUPPORTED) + return -EOPNOTSUPP; + + return sbi_err_map_linux_errno(sret.error); +} diff --git a/arch/riscv/kernel/sbi_sse_entry.S b/arch/riscv/kernel/sbi_sse_entry.S new file mode 100644 index 00000000000000..e0e8efba12ddbb --- /dev/null +++ b/arch/riscv/kernel/sbi_sse_entry.S @@ -0,0 +1,226 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2025 Rivos Inc. + */ + +#include +#include + +#include +#include +#include +#include +#include + +/* When entering handle_sse, the following registers are set: + * a6: contains the hartid + * a7: contains a sse_event_arch_data struct pointer + */ +SYM_CODE_START(handle_sse) + /* Save stack temporarily */ + REG_S sp, SSE_REG_EVT_TMP(a7) + /* Set entry stack */ + REG_L sp, SSE_REG_EVT_STACK(a7) + + addi sp, sp, -(PT_SIZE_ON_STACK) + REG_S ra, PT_RA(sp) + REG_S s0, PT_S0(sp) + REG_S s1, PT_S1(sp) + REG_S s2, PT_S2(sp) + REG_S s3, PT_S3(sp) + REG_S s4, PT_S4(sp) + REG_S s5, PT_S5(sp) + REG_S s6, PT_S6(sp) + REG_S s7, PT_S7(sp) + REG_S s8, PT_S8(sp) + REG_S s9, PT_S9(sp) + REG_S s10, PT_S10(sp) + REG_S s11, PT_S11(sp) + REG_S tp, PT_TP(sp) + REG_S t0, PT_T0(sp) + REG_S t1, PT_T1(sp) + REG_S t2, PT_T2(sp) + REG_S t3, PT_T3(sp) + REG_S t4, PT_T4(sp) + REG_S t5, PT_T5(sp) + REG_S t6, PT_T6(sp) + REG_S gp, PT_GP(sp) + REG_S a0, PT_A0(sp) + REG_S a1, PT_A1(sp) + REG_S a2, PT_A2(sp) + REG_S a3, PT_A3(sp) + REG_S a4, PT_A4(sp) + REG_S a5, PT_A5(sp) + + /* Retrieve entry sp */ + REG_L a4, SSE_REG_EVT_TMP(a7) + /* Save CSRs */ + csrr a0, CSR_EPC + csrr a1, CSR_SSTATUS + csrr a2, CSR_STVAL + csrr a3, CSR_SCAUSE + + REG_S a0, PT_EPC(sp) + REG_S a1, PT_STATUS(sp) + REG_S a2, PT_BADADDR(sp) + REG_S a3, PT_CAUSE(sp) + REG_S a4, PT_SP(sp) + + /* Disable user memory access and floating/vector computing */ + li t0, SR_SUM | SR_FS_VS + csrc CSR_STATUS, t0 + + load_global_pointer + scs_load_sse_stack a7 + +#ifdef CONFIG_SMP + REG_L t4, SSE_REG_HART_ID(a7) + lw t3, SSE_REG_CPU_ID(a7) + + bne t4, a6, .Lfind_hart_id_slowpath + +.Lcpu_id_found: +#else + mv t3, zero +#endif + + asm_per_cpu_with_cpu t2 __sbi_sse_entry_task t1 t3 + REG_L tp, 0(t2) + bnez tp, .Lcurrent_task_found + + /* __switch_to() marks its transition window with a NULL entry task. */ + REG_L tp, PT_TP(sp) + +.Lcurrent_task_found: + /* Nested exceptions temporarily replace these with the SSE stack. */ + REG_L s6, TASK_TI_KERNEL_SP(tp) + REG_L s7, TASK_TI_USER_SP(tp) + + mv a1, sp /* pt_regs on stack */ + + /* + * Run the SSE handler with the normal exception vector, but restore the + * interrupted stvec before completing the event. SSE can arrive while + * the kernel is using a temporary trap vector in a sensitive entry path. + */ + csrr s3, CSR_STVEC + la t0, handle_exception + csrw CSR_STVEC, t0 + + /* + * Preserve the full HS-mode virtualization state across the handler. + * hstatus is live supervisor state rather than an SSE interrupted + * attribute, and OpenSBI consumes hstatus.SPV during event completion. + * Saving the whole CSR keeps the handler episode transparent to KVM and + * avoids having to infer which hstatus bits may matter to a guest resume. + */ + li s5, 0 + ALTERNATIVE("nop", "csrr s5, hstatus", 0, RISCV_ISA_EXT_H, 1) + + /* + * Save sscratch for restoration since we might have interrupted the + * kernel in early exception path and thus, we don't know the content of + * sscratch. + */ + csrrw s4, CSR_SSCRATCH, x0 + + mv a0, a7 + mv a2, s5 + + call do_sse + + /* Leave no reference to the event stack in the interrupted task. */ + REG_S s7, TASK_TI_USER_SP(tp) + REG_S s6, TASK_TI_KERNEL_SP(tp) + + csrw CSR_SSCRATCH, s4 + ALTERNATIVE("nop", "csrw hstatus, s5", 0, RISCV_ISA_EXT_H, 1) + csrw CSR_STVEC, s3 + + REG_L a0, PT_STATUS(sp) + REG_L a1, PT_EPC(sp) + REG_L a2, PT_BADADDR(sp) + REG_L a3, PT_CAUSE(sp) + csrw CSR_SSTATUS, a0 + csrw CSR_EPC, a1 + csrw CSR_STVAL, a2 + csrw CSR_SCAUSE, a3 + + REG_L ra, PT_RA(sp) + REG_L s0, PT_S0(sp) + REG_L s1, PT_S1(sp) + REG_L s2, PT_S2(sp) + REG_L s3, PT_S3(sp) + REG_L s4, PT_S4(sp) + REG_L s5, PT_S5(sp) + REG_L s6, PT_S6(sp) + REG_L s7, PT_S7(sp) + REG_L s8, PT_S8(sp) + REG_L s9, PT_S9(sp) + REG_L s10, PT_S10(sp) + REG_L s11, PT_S11(sp) + REG_L tp, PT_TP(sp) + REG_L t0, PT_T0(sp) + REG_L t1, PT_T1(sp) + REG_L t2, PT_T2(sp) + REG_L t3, PT_T3(sp) + REG_L t4, PT_T4(sp) + REG_L t5, PT_T5(sp) + REG_L t6, PT_T6(sp) + REG_L gp, PT_GP(sp) + REG_L a0, PT_A0(sp) + REG_L a1, PT_A1(sp) + REG_L a2, PT_A2(sp) + REG_L a3, PT_A3(sp) + REG_L a4, PT_A4(sp) + REG_L a5, PT_A5(sp) + + REG_L sp, PT_SP(sp) + + li a7, SBI_EXT_SSE + li a6, SBI_SSE_EVENT_COMPLETE + ecall + + /* + * COMPLETE must resume the interrupted context and never return. Trap + * through the normal vector instead of falling into adjacent assembly. + */ + la t0, handle_exception + csrw CSR_STVEC, t0 + ebreak + /* The fatal trap must not return; execution should never reach here. */ + +#ifdef CONFIG_SMP +.Lfind_hart_id_slowpath: + + /* Restore current task struct from __sbi_sse_entry_task */ + li t1, ASM_NR_CPUS + /* Slowpath to find the CPU id associated to the hart id */ + la t0, __cpuid_to_hartid_map + li t3, 0 + +.Lhart_id_loop: + REG_L t2, 0(t0) + beq t2, a6, .Lcpu_id_found + + /* Increment pointer and CPU number */ + addi t3, t3, 1 + addi t0, t0, RISCV_SZPTR + bltu t3, t1, .Lhart_id_loop + + /* + * This should never happen since we expect the hart_id to match one + * of our CPU, but better be safe than sorry + */ + la tp, init_task + la a0, sse_hart_id_panic_string + la t0, panic + jalr t0 +#endif + +SYM_CODE_END(handle_sse) +ASM_NOKPROBE(handle_sse) + +SYM_DATA_START_LOCAL(sse_hart_id_panic_string) + .ascii "Unable to match hart_id with cpu\0" +SYM_DATA_END(sse_hart_id_panic_string) From 6bd96431b355a583d425990318542f3b5a2d3307 Mon Sep 17 00:00:00 2001 From: Zhanpeng Zhang Date: Fri, 11 Sep 2026 20:14:50 +0800 Subject: [PATCH 3/9] riscv: sse: mask events during shutdown and kexec SSE delivery is independent of normal S-mode interrupts. Firmware may also retain an event registration until Linux explicitly unregisters it. A hart must therefore stop accepting SSE events before Linux stops servicing the registered handler. Mask SSE on the local hart before panic stop, CPU stop, restart, poweroff, and crash shutdown paths. This prevents firmware from entering Linux-owned handler state after the corresponding CPU or kernel context is no longer valid. A crash kernel cannot identify or take ownership of registrations inherited from the crashed kernel. Reject a later normal kexec while such SSE state may still exist, rather than transferring unknown firmware state to another kernel. Signed-off-by: Zhanpeng Zhang Signed-off-by: Linux RISC-V bot --- arch/riscv/kernel/machine_kexec.c | 11 +++++++++++ arch/riscv/kernel/reset.c | 18 ++++++++++++++++++ arch/riscv/kernel/smp.c | 17 +++++++++++++++++ 3 files changed, 46 insertions(+) diff --git a/arch/riscv/kernel/machine_kexec.c b/arch/riscv/kernel/machine_kexec.c index 738df176ff6f16..24e7affae70bf4 100644 --- a/arch/riscv/kernel/machine_kexec.c +++ b/arch/riscv/kernel/machine_kexec.c @@ -14,10 +14,13 @@ #include /* For set_memory_x() */ #include /* For unreachable() */ #include /* For cpu_down() */ +#include #include #include #include +#include + /* * machine_kexec_prepare - Initialize kexec * @@ -36,6 +39,13 @@ machine_kexec_prepare(struct kimage *image) unsigned int control_code_buffer_sz = 0; int i = 0; + /* A crash kernel cannot tear down registrations inherited from firmware. */ + if (is_kdump_kernel() && image->type != KEXEC_TYPE_CRASH && + riscv_sse_available()) { + pr_err("Normal kexec from a crash kernel is unsupported with SSE\n"); + return -EOPNOTSUPP; + } + /* Find the Flattened Device Tree and save its physical address */ for (i = 0; i < image->nr_segments; i++) { if (image->segment[i].memsz <= sizeof(fdt)) @@ -127,6 +137,7 @@ void machine_crash_shutdown(struct pt_regs *regs) { local_irq_disable(); + riscv_sse_mask_current_hart(); /* shutdown non-crashing cpus */ crash_smp_send_stop(); diff --git a/arch/riscv/kernel/reset.c b/arch/riscv/kernel/reset.c index 14eb08a6db8555..fdab37e7ae52dc 100644 --- a/arch/riscv/kernel/reset.c +++ b/arch/riscv/kernel/reset.c @@ -6,6 +6,20 @@ #include #include #include +#include + +#include + +#ifndef CONFIG_SMP +void __noreturn panic_smp_self_stop(void) +{ + riscv_sse_mask_current_hart(); + local_irq_disable(); + + for (;;) + cpu_relax(); +} +#endif static void __noreturn default_power_off(void) { @@ -18,6 +32,8 @@ EXPORT_SYMBOL(pm_power_off); void machine_restart(char *cmd) { + riscv_sse_mask_current_hart(); + /* * UpdateCapsule() depends on the system being reset via ResetSystem(). */ @@ -30,12 +46,14 @@ void machine_restart(char *cmd) void machine_halt(void) { + riscv_sse_mask_current_hart(); do_kernel_power_off(); default_power_off(); } void machine_power_off(void) { + riscv_sse_mask_current_hart(); do_kernel_power_off(); default_power_off(); } diff --git a/arch/riscv/kernel/smp.c b/arch/riscv/kernel/smp.c index fa66f9c97d748d..7f0d4e7332f155 100644 --- a/arch/riscv/kernel/smp.c +++ b/arch/riscv/kernel/smp.c @@ -23,10 +23,13 @@ #include #include #include +#include #include #include #include +#include +#include enum ipi_message_type { IPI_RESCHEDULE, @@ -79,8 +82,18 @@ int riscv_hartid_to_cpuid(unsigned long hartid) return -ENOENT; } +void __noreturn panic_smp_self_stop(void) +{ + riscv_sse_mask_current_hart(); + local_irq_disable(); + + for (;;) + cpu_relax(); +} + static void ipi_stop(void) { + riscv_sse_mask_current_hart(); set_cpu_online(smp_processor_id(), false); while (1) wait_for_interrupt(); @@ -91,6 +104,7 @@ static atomic_t waiting_for_crash_ipi = ATOMIC_INIT(0); static inline void ipi_cpu_crash_stop(unsigned int cpu, struct pt_regs *regs) { + riscv_sse_mask_current_hart(); crash_save_cpu(regs, cpu); atomic_dec(&waiting_for_crash_ipi); @@ -254,6 +268,8 @@ void smp_send_stop(void) { unsigned long timeout; + riscv_sse_mask_current_hart(); + if (num_online_cpus() > 1) { cpumask_t mask; @@ -301,6 +317,7 @@ void crash_smp_send_stop(void) return; cpus_stopped = 1; + riscv_sse_mask_current_hart(); /* * If this cpu is the only one alive at this point in time, online or From 3760250dd1acb18cec3d8cce21d76a9f9cc5bf5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20L=C3=A9ger?= Date: Fri, 11 Sep 2026 20:14:51 +0800 Subject: [PATCH 4/9] drivers: firmware: add riscv SSE support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a driver-level interface for RISC-V SSE. Linux clients can register handlers, select a target CPU for global events, and enable, disable or unregister events. The architecture entry wrapper completes an event after the registered handler returns. PMU and GHES drivers can use this interface. Represent global events with one firmware registration and local events with one registration per possible CPU. Keep registration and enable state stable across CPU hotplug, and validate firmware-provided hart IDs before converting them to Linux CPU IDs. Use phys_addr_t for attribute buffers to match the physical addresses passed to firmware. Require MMU support because the current event-stack implementation relies on vmapped memory and TLB synchronization. Serialize client list updates with the SSE mutex and the CPU read lock. Normal CPU hotplug callbacks provide the matching write-side exclusion, and CPUHP state removal holds the SSE mutex. These rules avoid holding an additional spinlock across firmware calls. Local event operations select a per-CPU registration, so require callers to remain on the current CPU and use lockdep assertions to verify that contract. Propagate firmware failures from register, disable and unregister operations. Update Linux state only after successful firmware operations, and release an event only after all registrations are gone. Preserve the difference between SBI_ERR_NOT_SUPPORTED and a generic SBI failure so clients only select a fallback after firmware explicitly rejects an SSE operation. If setup fails partway through a local event, roll back only the CPU instances changed by that invocation. Keep non-fallback errors across CPUs. This lets a client distinguish an event rejected as unsupported by every failing hart from an unknown firmware failure. A failed registration rollback can leave firmware state without a client handle. Retain these events on a driver-owned cleanup list with a no-op handler. This lets CPU hotplug and shutdown retry cleanup without relying on client callback lifetime. Mask SSE before CPU teardown. If a normal CPU-offline teardown fails after partially changing local events, reconstruct their requested registration and enable state. Unmask the hart, then return the original error to abort the offline operation. Shutdown and CPUHP state removal cannot fail, so record incomplete cleanup and refuse an unsafe normal kexec instead. Close enable admission before reboot or kexec teardown. Protect the shutdown check and firmware enable operation with RCU, then drain existing enable calls before CPUHP removes registrations. This prevents a client from re-enabling an event between the teardown disable and unregister operations. A crash kernel cannot identify registrations inherited from the crashed kernel. Leave SSE masked, report the retained firmware state separately from an unavailable extension, and do not let clients select a delivery path that may still be owned by the old kernel. Signed-off-by: Clément Léger Co-developed-by: Himanshu Chauhan Signed-off-by: Himanshu Chauhan Co-developed-by: Zhanpeng Zhang Signed-off-by: Zhanpeng Zhang Acked-by: Conor Dooley Signed-off-by: Linux RISC-V bot --- MAINTAINERS | 7 + drivers/firmware/Kconfig | 1 + drivers/firmware/Makefile | 1 + drivers/firmware/riscv/Kconfig | 18 + drivers/firmware/riscv/Makefile | 3 + drivers/firmware/riscv/riscv_sbi_sse.c | 1214 ++++++++++++++++++++++++ include/linux/cpuhotplug.h | 1 + include/linux/riscv_sbi_sse.h | 89 ++ 8 files changed, 1334 insertions(+) create mode 100644 drivers/firmware/riscv/Kconfig create mode 100644 drivers/firmware/riscv/Makefile create mode 100644 drivers/firmware/riscv/riscv_sbi_sse.c create mode 100644 include/linux/riscv_sbi_sse.h diff --git a/MAINTAINERS b/MAINTAINERS index f6a2ed9990dd06..d11cd44d24498f 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -23330,6 +23330,13 @@ T: git git://git.kernel.org/pub/scm/linux/kernel/git/iommu/linux.git F: Documentation/devicetree/bindings/iommu/riscv,iommu.yaml F: drivers/iommu/riscv/ +RISC-V FIRMWARE DRIVERS +M: Conor Dooley +L: linux-riscv@lists.infradead.org +S: Maintained +T: git git://git.kernel.org/pub/scm/linux/kernel/git/conor/linux.git +F: drivers/firmware/riscv/ + RISC-V MICROCHIP SUPPORT M: Conor Dooley M: Daire McNamara diff --git a/drivers/firmware/Kconfig b/drivers/firmware/Kconfig index dbaca140a1b0a4..47eac61b915056 100644 --- a/drivers/firmware/Kconfig +++ b/drivers/firmware/Kconfig @@ -318,6 +318,7 @@ source "drivers/firmware/meson/Kconfig" source "drivers/firmware/microchip/Kconfig" source "drivers/firmware/psci/Kconfig" source "drivers/firmware/qcom/Kconfig" +source "drivers/firmware/riscv/Kconfig" source "drivers/firmware/samsung/Kconfig" source "drivers/firmware/smccc/Kconfig" source "drivers/firmware/tegra/Kconfig" diff --git a/drivers/firmware/Makefile b/drivers/firmware/Makefile index be46f1e1dc77fa..879953c316dca0 100644 --- a/drivers/firmware/Makefile +++ b/drivers/firmware/Makefile @@ -35,6 +35,7 @@ obj-y += efi/ obj-y += imx/ obj-y += psci/ obj-y += qcom/ +obj-y += riscv/ obj-y += samsung/ obj-y += smccc/ obj-y += tegra/ diff --git a/drivers/firmware/riscv/Kconfig b/drivers/firmware/riscv/Kconfig new file mode 100644 index 00000000000000..d15ff84e1258cd --- /dev/null +++ b/drivers/firmware/riscv/Kconfig @@ -0,0 +1,18 @@ +# SPDX-License-Identifier: GPL-2.0-only +menu "Risc-V Specific firmware drivers" +depends on RISCV + +config RISCV_SBI_SSE + bool "Enable SBI Supervisor Software Events support" + depends on RISCV_SBI && MMU && !HIBERNATION + default y + help + The Supervisor Software Events support allows the SBI to deliver + NMI-like notifications to the supervisor mode software. When enabled, + this option provides support to register callbacks on specific SSE + events. + + Hibernation is not supported because firmware registrations do not + survive restoring a kernel image. + +endmenu diff --git a/drivers/firmware/riscv/Makefile b/drivers/firmware/riscv/Makefile new file mode 100644 index 00000000000000..c8795d4bbb2eab --- /dev/null +++ b/drivers/firmware/riscv/Makefile @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0 + +obj-$(CONFIG_RISCV_SBI_SSE) += riscv_sbi_sse.o diff --git a/drivers/firmware/riscv/riscv_sbi_sse.c b/drivers/firmware/riscv/riscv_sbi_sse.c new file mode 100644 index 00000000000000..e81cfb4b0c563c --- /dev/null +++ b/drivers/firmware/riscv/riscv_sbi_sse.c @@ -0,0 +1,1214 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2025 Rivos Inc. + */ + +#define pr_fmt(fmt) "sse: " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +struct sse_event { + struct list_head list; + u32 evt_id; + u32 priority; + sse_event_handler_fn __rcu *handler; + void *handler_arg; + /* Only valid for global events */ + unsigned int cpu; + /* + * Desired state requested by the client. Firmware state is tracked per + * instance because a failed transition can leave a partial result. + */ + bool enable_requested; + /* Registration failed, but firmware state still needs driver cleanup. */ + bool cleanup_pending; + + union { + struct sse_registered_event *global; + struct sse_registered_event __percpu *local; + }; +}; + +static bool sse_available __ro_after_init; +static bool sse_fw_state_retained __ro_after_init; +static bool sse_shutting_down; +static atomic_t sse_teardown_failed = ATOMIC_INIT(0); +/* + * Client-side updates hold sse_mutex and the CPU read lock. Normal CPU hotplug + * callbacks exclude them through the CPUHP write side. Initialization replay + * runs before clients can register, and state removal holds sse_mutex. + */ +static LIST_HEAD(events); +static DEFINE_MUTEX(sse_mutex); + +/* + * A registration rollback can fail before the client receives an event + * handle. Keep the retained event independent of client-owned callback text + * and data while the driver retries firmware cleanup. + */ +static int sse_cleanup_event_handler(u32 evt, void *arg, struct pt_regs *regs) +{ + return 0; +} + +struct sse_registered_event { + struct sse_event_arch_data arch; + struct sse_event *event; + unsigned long attr; + /* + * Actual firmware state for one global or per-CPU instance. A retry can + * then skip instances that already completed a partial transition. + */ + bool is_registered; + bool is_enabled; +}; + +void sse_handle_event(struct sse_event_arch_data *arch_event, + struct pt_regs *regs) +{ + sse_event_handler_fn *handler; + int ret; + struct sse_registered_event *reg_evt = + container_of(arch_event, struct sse_registered_event, arch); + struct sse_event *evt = reg_evt->event; + + rcu_read_lock(); + handler = rcu_dereference(evt->handler); + ret = handler(evt->evt_id, evt->handler_arg, regs); + rcu_read_unlock(); + if (ret) + pr_warn("event %x handler failed with error %d\n", evt->evt_id, ret); +} + +static struct sse_event *sse_event_get(u32 evt) +{ + struct sse_event *event; + + lockdep_assert_held(&sse_mutex); + + list_for_each_entry(event, &events, list) { + if (event->evt_id == evt) + return event; + } + + return NULL; +} + +static phys_addr_t sse_event_get_attr_phys(struct sse_registered_event *reg_evt) +{ + phys_addr_t phys; + void *addr = ®_evt->attr; + + if (sse_event_is_global(reg_evt->event->evt_id)) + phys = virt_to_phys(addr); + else + phys = per_cpu_ptr_to_phys(addr); + + return phys; +} + +static struct sse_registered_event *sse_get_reg_evt(struct sse_event *event) +{ + if (sse_event_is_global(event->evt_id)) + return event->global; + else + return per_cpu_ptr(event->local, smp_processor_id()); +} + +static int sse_err_map_linux_errno(long err) +{ + if (err == SBI_ERR_NOT_SUPPORTED) + return -EOPNOTSUPP; + + return sbi_err_map_linux_errno(err); +} + +static int sse_sbi_event_func(struct sse_event *event, unsigned long func) +{ + struct sbiret ret; + u32 evt = event->evt_id; + struct sse_registered_event *reg_evt = sse_get_reg_evt(event); + + ret = sbi_ecall(SBI_EXT_SSE, func, evt, 0, 0, 0, 0, 0); + if (ret.error) { + pr_warn("Failed to execute func %lx, event %x, error %ld\n", + func, evt, ret.error); + return sse_err_map_linux_errno(ret.error); + } + + if (func == SBI_SSE_EVENT_DISABLE) + reg_evt->is_enabled = false; + else if (func == SBI_SSE_EVENT_ENABLE) + reg_evt->is_enabled = true; + + return 0; +} + +int sse_event_disable_local(struct sse_event *event) +{ + if (!sse_event_is_global(event->evt_id)) + lockdep_assert_preemption_disabled(); + + if (!sse_get_reg_evt(event)->is_enabled) + return 0; + + return sse_sbi_event_func(event, SBI_SSE_EVENT_DISABLE); +} +EXPORT_SYMBOL_GPL(sse_event_disable_local); + +int sse_event_enable_local(struct sse_event *event) +{ + struct sse_registered_event *reg_evt = sse_get_reg_evt(event); + int ret; + + if (!sse_event_is_global(event->evt_id)) + lockdep_assert_preemption_disabled(); + + rcu_read_lock(); + if (READ_ONCE(sse_shutting_down)) { + ret = -ESHUTDOWN; + goto out; + } + + if (!reg_evt->is_registered) { + ret = -EINVAL; + goto out; + } + + if (reg_evt->is_enabled) { + ret = 0; + goto out; + } + + ret = sse_sbi_event_func(event, SBI_SSE_EVENT_ENABLE); +out: + rcu_read_unlock(); + + return ret; +} +EXPORT_SYMBOL_GPL(sse_event_enable_local); + +static int sse_event_attr_get_no_lock(struct sse_registered_event *reg_evt, + unsigned long attr_id, unsigned long *val) +{ + struct sbiret sret; + u32 evt = reg_evt->event->evt_id; + phys_addr_t phys; + + phys = sse_event_get_attr_phys(reg_evt); + + sret = sbi_ecall(SBI_EXT_SSE, SBI_SSE_EVENT_ATTR_READ, evt, attr_id, 1, + (unsigned long)phys, 0, 0); + if (sret.error) { + pr_debug("Failed to get event %x attr %lx, error %ld\n", evt, + attr_id, sret.error); + return sse_err_map_linux_errno(sret.error); + } + + *val = reg_evt->attr; + + return 0; +} + +static int sse_event_attr_set_nolock(struct sse_registered_event *reg_evt, + unsigned long attr_id, unsigned long val) +{ + struct sbiret sret; + u32 evt = reg_evt->event->evt_id; + phys_addr_t phys; + + reg_evt->attr = val; + phys = sse_event_get_attr_phys(reg_evt); + + sret = sbi_ecall(SBI_EXT_SSE, SBI_SSE_EVENT_ATTR_WRITE, evt, attr_id, 1, + (unsigned long)phys, 0, 0); + if (sret.error) + pr_debug("Failed to set event %x attr %lx, error %ld\n", evt, + attr_id, sret.error); + + return sse_err_map_linux_errno(sret.error); +} + +static void sse_global_event_update_cpu(struct sse_event *event, + unsigned int cpu) +{ + struct sse_registered_event *reg_evt = event->global; + + event->cpu = cpu; + arch_sse_event_update_cpu(®_evt->arch, cpu); +} + +static int sse_event_set_target_cpu_nolock(struct sse_event *event, + unsigned int cpu) +{ + unsigned long hart_id, old_hart_id; + struct sse_registered_event *reg_evt = event->global; + u32 evt = event->evt_id; + unsigned int old_cpu; + bool was_enabled; + int ret; + + if (!sse_event_is_global(evt)) + return -EINVAL; + + if (cpu >= nr_cpu_ids || !cpu_online(cpu)) + return -EINVAL; + hart_id = cpuid_to_hartid_map(cpu); + old_cpu = event->cpu; + old_hart_id = cpuid_to_hartid_map(old_cpu); + + was_enabled = reg_evt->is_enabled; + if (was_enabled) { + ret = sse_event_disable_local(event); + if (ret) + return ret; + } + + ret = sse_event_attr_set_nolock(reg_evt, SBI_SSE_ATTR_PREFERRED_HART, + hart_id); + if (ret == 0) + sse_global_event_update_cpu(event, cpu); + + if (was_enabled) { + int enable_ret; + + enable_ret = sse_event_enable_local(event); + if (enable_ret) { + int rollback_ret; + + /* + * The preferred hart was already changed. Restore both the + * firmware attribute and Linux's cached target before reporting + * the failed migration. + */ + rollback_ret = sse_event_attr_set_nolock(reg_evt, + SBI_SSE_ATTR_PREFERRED_HART, + old_hart_id); + if (!rollback_ret) { + sse_global_event_update_cpu(event, old_cpu); + rollback_ret = sse_event_enable_local(event); + } + if (!rollback_ret) + return enable_ret; + + pr_warn("Failed to restore global event %x to CPU %u: %d\n", + evt, old_cpu, rollback_ret); + event->enable_requested = false; + return enable_ret; + } + } + + return ret; +} + +int sse_event_set_target_cpu(struct sse_event *event, unsigned int cpu) +{ + int ret; + + if (cpu >= nr_cpu_ids) + return -EINVAL; + + scoped_guard(mutex, &sse_mutex) { + if (READ_ONCE(sse_shutting_down)) + return -ESHUTDOWN; + + scoped_guard(cpus_read_lock) { + if (!cpu_online(cpu)) + return -EINVAL; + + ret = sse_event_set_target_cpu_nolock(event, cpu); + } + } + + return ret; +} +EXPORT_SYMBOL_GPL(sse_event_set_target_cpu); + +static int sse_event_init_registered(unsigned int cpu, + struct sse_registered_event *reg_evt, + struct sse_event *event) +{ + reg_evt->event = event; + reg_evt->is_registered = false; + reg_evt->is_enabled = false; + + return arch_sse_init_event(®_evt->arch, event->evt_id, cpu); +} + +static void sse_event_free_registered(struct sse_registered_event *reg_evt) +{ + arch_sse_free_event(®_evt->arch); +} + +static int sse_event_alloc_global(struct sse_event *event) +{ + unsigned int cpu; + int err; + struct sse_registered_event *reg_evt; + + reg_evt = kzalloc_obj(*reg_evt, GFP_KERNEL); + if (!reg_evt) + return -ENOMEM; + + event->global = reg_evt; + cpu = cpumask_first(cpu_possible_mask); + if (cpu >= nr_cpu_ids) { + kfree(reg_evt); + return -ENODEV; + } + + err = sse_event_init_registered(cpu, reg_evt, event); + if (err) + kfree(reg_evt); + + return err; +} + +static int sse_event_alloc_local(struct sse_event *event) +{ + int err; + unsigned int cpu, err_cpu; + struct sse_registered_event *reg_evt; + struct sse_registered_event __percpu *reg_evts; + + reg_evts = alloc_percpu(struct sse_registered_event); + if (!reg_evts) + return -ENOMEM; + + event->local = reg_evts; + + for_each_possible_cpu(cpu) { + reg_evt = per_cpu_ptr(reg_evts, cpu); + err = sse_event_init_registered(cpu, reg_evt, event); + if (err) { + err_cpu = cpu; + goto err_free_per_cpu; + } + } + + return 0; + +err_free_per_cpu: + for_each_possible_cpu(cpu) { + if (cpu == err_cpu) + break; + reg_evt = per_cpu_ptr(reg_evts, cpu); + sse_event_free_registered(reg_evt); + } + + free_percpu(reg_evts); + + return err; +} + +static struct sse_event *sse_event_alloc(u32 evt, u32 priority, + sse_event_handler_fn *handler, + void *arg) +{ + int err; + struct sse_event *event; + + event = kzalloc_obj(*event, GFP_KERNEL); + if (!event) + return ERR_PTR(-ENOMEM); + + event->evt_id = evt; + event->priority = priority; + event->handler_arg = arg; + RCU_INIT_POINTER(event->handler, handler); + + if (sse_event_is_global(evt)) + err = sse_event_alloc_global(event); + else + err = sse_event_alloc_local(event); + + if (err) { + kfree(event); + return ERR_PTR(err); + } + + return event; +} + +static int sse_sbi_register_event(struct sse_event *event, + struct sse_registered_event *reg_evt) +{ + int ret; + + if (reg_evt->is_registered) + return 0; + + ret = sse_event_attr_set_nolock(reg_evt, SBI_SSE_ATTR_PRIO, + event->priority); + if (ret) + return ret; + + ret = arch_sse_register_event(®_evt->arch); + if (!ret) + reg_evt->is_registered = true; + + return ret; +} + +static int sse_event_register_local(struct sse_event *event) +{ + int ret; + struct sse_registered_event *reg_evt; + + reg_evt = per_cpu_ptr(event->local, smp_processor_id()); + ret = sse_sbi_register_event(event, reg_evt); + if (ret) + pr_debug("Failed to register event %x: err %d\n", event->evt_id, + ret); + + return ret; +} + +static int sse_sbi_unregister_event(struct sse_event *event) +{ + struct sse_registered_event *reg_evt = sse_get_reg_evt(event); + int ret; + + if (!reg_evt->is_registered) + return 0; + + ret = sse_sbi_event_func(event, SBI_SSE_EVENT_UNREGISTER); + if (!ret) { + reg_evt->is_registered = false; + reg_evt->is_enabled = false; + } + + return ret; +} + +struct sse_per_cpu_evt { + struct sse_event *event; + unsigned long func; + atomic_t first_error; + atomic_t nonfallback_error; + cpumask_t changed; +}; + +static void sse_event_per_cpu_func(void *info) +{ + struct sse_per_cpu_evt *cpu_evt = info; + struct sse_registered_event *reg_evt; + bool changed; + int ret; + + reg_evt = sse_get_reg_evt(cpu_evt->event); + + if (cpu_evt->func == SBI_SSE_EVENT_REGISTER) { + changed = !reg_evt->is_registered; + ret = sse_event_register_local(cpu_evt->event); + } else if (cpu_evt->func == SBI_SSE_EVENT_UNREGISTER) { + changed = reg_evt->is_registered; + ret = sse_sbi_unregister_event(cpu_evt->event); + } else if (cpu_evt->func == SBI_SSE_EVENT_ENABLE) { + changed = !reg_evt->is_enabled; + ret = sse_event_enable_local(cpu_evt->event); + } else if (cpu_evt->func == SBI_SSE_EVENT_DISABLE) { + changed = reg_evt->is_enabled; + ret = sse_event_disable_local(cpu_evt->event); + } else { + changed = false; + ret = -EINVAL; + } + + if (ret) { + atomic_cmpxchg(&cpu_evt->first_error, 0, ret); + if (ret != -EOPNOTSUPP) + atomic_cmpxchg(&cpu_evt->nonfallback_error, 0, ret); + } else if (changed) { + cpumask_set_cpu(smp_processor_id(), &cpu_evt->changed); + } +} + +static bool sse_event_is_registered(struct sse_event *event) +{ + unsigned int cpu; + + if (sse_event_is_global(event->evt_id)) + return event->global->is_registered; + + for_each_possible_cpu(cpu) { + if (per_cpu_ptr(event->local, cpu)->is_registered) + return true; + } + + return false; +} + +static bool sse_event_is_enabled(struct sse_event *event) +{ + unsigned int cpu; + + if (sse_event_is_global(event->evt_id)) + return event->global->is_enabled; + + for_each_possible_cpu(cpu) { + if (per_cpu_ptr(event->local, cpu)->is_enabled) + return true; + } + + return false; +} + +static void sse_event_free(struct sse_event *event) +{ + unsigned int cpu; + struct sse_registered_event *reg_evt; + + if (WARN_ON_ONCE(sse_event_is_registered(event))) + return; + + if (sse_event_is_global(event->evt_id)) { + sse_event_free_registered(event->global); + kfree(event->global); + } else { + for_each_possible_cpu(cpu) { + reg_evt = per_cpu_ptr(event->local, cpu); + sse_event_free_registered(reg_evt); + } + free_percpu(event->local); + } + + kfree(event); +} + +static struct sse_event *sse_register_failed(struct sse_event *event, int ret) +{ + /* + * Keep failed rollback state visible to CPU hotplug and shutdown. The + * core-owned handler also makes the retained registration independent of + * the client whose registration request failed. + */ + if (sse_event_is_registered(event)) { + event->cleanup_pending = true; + rcu_assign_pointer(event->handler, sse_cleanup_event_handler); + synchronize_rcu(); + list_add(&event->list, &events); + pr_err("Event %x remains registered after rollback; cleanup retained\n", + event->evt_id); + ret = -EUCLEAN; + } else { + sse_event_free(event); + } + + return ERR_PTR(ret); +} + +void sse_event_cleanup(struct sse_event *event) +{ + guard(mutex)(&sse_mutex); + guard(cpus_read_lock)(); + + if (event->cleanup_pending) + return; + + /* + * Firmware may still enter the old callback after disable or unregister + * fails. Publish a core-owned callback, then wait before the client frees + * its callback data. + */ + event->cleanup_pending = true; + rcu_assign_pointer(event->handler, sse_cleanup_event_handler); + synchronize_rcu(); +} +EXPORT_SYMBOL_GPL(sse_event_cleanup); + +static void sse_release_cleanup_event(struct sse_event *event) +{ + if (!event->cleanup_pending || sse_event_is_registered(event)) + return; + + list_del(&event->list); + sse_event_free(event); +} + +static int sse_event_setup_all_cpus(struct sse_event *event, + unsigned long func, + unsigned long rollback_func) +{ + struct sse_per_cpu_evt cpu_evt; + int rollback_ret; + int ret; + + cpu_evt.event = event; + atomic_set(&cpu_evt.first_error, 0); + atomic_set(&cpu_evt.nonfallback_error, 0); + cpumask_clear(&cpu_evt.changed); + cpu_evt.func = func; + on_each_cpu(sse_event_per_cpu_func, &cpu_evt, 1); + /* IRQ fallback is safe only if every failing CPU reports unsupported. */ + ret = atomic_read(&cpu_evt.nonfallback_error); + if (!ret) + ret = atomic_read(&cpu_evt.first_error); + /* + * A previous attempt may already have changed some CPUs. Roll back only + * instances changed by this invocation. + */ + if (ret) { + cpu_evt.func = rollback_func; + atomic_set(&cpu_evt.first_error, 0); + atomic_set(&cpu_evt.nonfallback_error, 0); + on_each_cpu_mask(&cpu_evt.changed, sse_event_per_cpu_func, &cpu_evt, 1); + + rollback_ret = atomic_read(&cpu_evt.nonfallback_error); + if (!rollback_ret) + rollback_ret = atomic_read(&cpu_evt.first_error); + + /* A rollback failure leaves the firmware state uncertain. */ + return rollback_ret ?: ret; + } + + return 0; +} + +static int sse_event_teardown_all_cpus(struct sse_event *event, + unsigned long func) +{ + struct sse_per_cpu_evt cpu_evt; + + cpu_evt.event = event; + atomic_set(&cpu_evt.first_error, 0); + atomic_set(&cpu_evt.nonfallback_error, 0); + cpumask_clear(&cpu_evt.changed); + cpu_evt.func = func; + on_each_cpu(sse_event_per_cpu_func, &cpu_evt, 1); + + return atomic_read(&cpu_evt.first_error); +} + +int sse_event_enable(struct sse_event *event) +{ + int ret = 0; + + scoped_guard(mutex, &sse_mutex) { + if (READ_ONCE(sse_shutting_down)) + return -ESHUTDOWN; + + scoped_guard(cpus_read_lock) { + if (sse_event_is_global(event->evt_id)) { + ret = sse_event_enable_local(event); + } else { + ret = sse_event_setup_all_cpus(event, + SBI_SSE_EVENT_ENABLE, + SBI_SSE_EVENT_DISABLE); + } + event->enable_requested = !ret; + } + } + + return ret; +} +EXPORT_SYMBOL_GPL(sse_event_enable); + +static int sse_events_mask(void) +{ + struct sbiret ret; + + ret = sbi_ecall(SBI_EXT_SSE, SBI_SSE_HART_MASK, 0, 0, 0, 0, 0, 0); + if (ret.error == SBI_ERR_ALREADY_STOPPED) + return 0; + + return sse_err_map_linux_errno(ret.error); +} + +static int sse_events_unmask(void) +{ + struct sbiret ret; + + ret = sbi_ecall(SBI_EXT_SSE, SBI_SSE_HART_UNMASK, 0, 0, 0, 0, 0, 0); + if (ret.error == SBI_ERR_ALREADY_STARTED) + return 0; + + return sse_err_map_linux_errno(ret.error); +} + +static int sse_event_disable_nolock(struct sse_event *event) +{ + if (sse_event_is_global(event->evt_id)) + return sse_event_disable_local(event); + + return sse_event_teardown_all_cpus(event, SBI_SSE_EVENT_DISABLE); +} + +int sse_event_disable(struct sse_event *event) +{ + int ret = 0; + + scoped_guard(mutex, &sse_mutex) { + if (READ_ONCE(sse_shutting_down)) + return -ESHUTDOWN; + + scoped_guard(cpus_read_lock) { + if (!event->enable_requested && !sse_event_is_enabled(event)) + return 0; + + event->enable_requested = false; + ret = sse_event_disable_nolock(event); + if (!ret && sse_event_is_enabled(event)) + ret = -EIO; + } + } + + return ret; +} +EXPORT_SYMBOL_GPL(sse_event_disable); + +struct sse_event *sse_event_register(u32 evt, u32 priority, + sse_event_handler_fn *handler, void *arg) +{ + struct sse_event *event; + int cpu; + int ret = 0; + + if (sse_fw_state_retained) + return ERR_PTR(-EUCLEAN); + if (!sse_available) + return ERR_PTR(-EOPNOTSUPP); + + guard(mutex)(&sse_mutex); + if (READ_ONCE(sse_shutting_down)) + return ERR_PTR(-ESHUTDOWN); + + guard(cpus_read_lock)(); + + if (sse_event_get(evt)) + return ERR_PTR(-EEXIST); + + event = sse_event_alloc(evt, priority, handler, arg); + if (IS_ERR(event)) + return event; + + if (sse_event_is_global(evt)) { + unsigned long preferred_hart; + + ret = sse_event_attr_get_no_lock(event->global, + SBI_SSE_ATTR_PREFERRED_HART, + &preferred_hart); + if (ret) + return sse_register_failed(event, ret); + + cpu = riscv_hartid_to_cpuid(preferred_hart); + if (cpu < 0 || !cpu_online(cpu)) { + cpu = cpumask_first(cpu_online_mask); + if (cpu >= nr_cpu_ids) + return sse_register_failed(event, -ENODEV); + + ret = sse_event_set_target_cpu_nolock(event, cpu); + if (ret) + return sse_register_failed(event, ret); + } else { + sse_global_event_update_cpu(event, cpu); + } + + ret = sse_sbi_register_event(event, event->global); + if (ret) + return sse_register_failed(event, ret); + } else { + ret = sse_event_setup_all_cpus(event, SBI_SSE_EVENT_REGISTER, + SBI_SSE_EVENT_UNREGISTER); + if (ret) + return sse_register_failed(event, ret); + } + + list_add(&event->list, &events); + + return event; +} +EXPORT_SYMBOL_GPL(sse_event_register); + +static int sse_event_unregister_nolock(struct sse_event *event) +{ + if (sse_event_is_global(event->evt_id)) + return sse_sbi_unregister_event(event); + + return sse_event_teardown_all_cpus(event, SBI_SSE_EVENT_UNREGISTER); +} + +int sse_event_unregister(struct sse_event *event) +{ + int ret = 0; + + scoped_guard(mutex, &sse_mutex) { + if (READ_ONCE(sse_shutting_down)) + return -ESHUTDOWN; + + scoped_guard(cpus_read_lock) { + ret = sse_event_unregister_nolock(event); + if (ret) + return ret; + if (sse_event_is_registered(event)) + return -EBUSY; + + list_del(&event->list); + + sse_event_free(event); + } + } + + return ret; +} +EXPORT_SYMBOL_GPL(sse_event_unregister); + +static int sse_teardown_event(struct sse_event *event, unsigned int cpu); + +static int sse_cpu_online(unsigned int cpu) +{ + int ret, rollback_ret; + struct sse_event *event, *tmp; + struct sse_registered_event *reg_evt; + + arch_sse_init_cpu(); + + list_for_each_entry_safe(event, tmp, &events, list) { + if (sse_event_is_global(event->evt_id)) + continue; + if (event->cleanup_pending) { + ret = sse_teardown_event(event, cpu); + if (ret) + goto rollback; + sse_release_cleanup_event(event); + continue; + } + + ret = sse_event_register_local(event); + if (ret) + goto rollback; + if (event->enable_requested) + ret = sse_event_enable_local(event); + else + ret = sse_event_disable_local(event); + if (ret) + goto rollback; + } + + /* Only unmask after every cached per-CPU state is reconstructed. */ + ret = sse_events_unmask(); + if (!ret) + return 0; + +rollback: + /* A failed startup callback is not followed by this state's teardown. */ + list_for_each_entry_safe(event, tmp, &events, list) { + if (sse_event_is_global(event->evt_id)) + continue; + + reg_evt = sse_get_reg_evt(event); + rollback_ret = reg_evt->is_enabled ? + sse_event_disable_local(event) : 0; + if (rollback_ret) { + pr_warn("Failed to disable event %x while rolling back CPU %u: %d\n", + event->evt_id, cpu, rollback_ret); + atomic_set(&sse_teardown_failed, 1); + continue; + } + + rollback_ret = reg_evt->is_registered ? + sse_sbi_unregister_event(event) : 0; + if (rollback_ret) { + pr_warn("Failed to unregister event %x while rolling back CPU %u: %d\n", + event->evt_id, cpu, rollback_ret); + atomic_set(&sse_teardown_failed, 1); + } + sse_release_cleanup_event(event); + } + + return ret; +} + +static int sse_teardown_event(struct sse_event *event, unsigned int cpu) +{ + struct sse_registered_event *reg_evt = sse_get_reg_evt(event); + int ret; + + if (reg_evt->is_enabled) { + ret = sse_event_disable_local(event); + if (ret) { + pr_warn("Failed to disable event %x on CPU %u: %d\n", + event->evt_id, cpu, ret); + return ret; + } + } + + ret = sse_sbi_unregister_event(event); + if (ret) + pr_warn("Failed to unregister event %x on CPU %u: %d\n", + event->evt_id, cpu, ret); + + return ret; +} + +static int sse_restore_local_events(unsigned int cpu) +{ + struct sse_event *event; + int first_error = 0; + int ret; + + list_for_each_entry(event, &events, list) { + if (sse_event_is_global(event->evt_id) || event->cleanup_pending) + continue; + + ret = sse_event_register_local(event); + if (!ret) { + ret = event->enable_requested ? + sse_event_enable_local(event) : + sse_event_disable_local(event); + } + if (ret) { + pr_warn("Failed to restore event %x on CPU %u: %d\n", + event->evt_id, cpu, ret); + if (!first_error) + first_error = ret; + } + } + + return first_error; +} + +static int sse_cpu_teardown(unsigned int cpu) +{ + /* Only a regular CPU-offline callback may abort the CPUHP operation. */ + bool regular_offline = READ_ONCE(sse_available) && + !READ_ONCE(sse_shutting_down); + unsigned int next_cpu; + struct sse_event *event, *tmp; + int first_error = 0; + int ret; + + /* Do not dismantle CPU state while firmware can still deliver SSE. */ + ret = sse_events_mask(); + if (ret) { + pr_warn("Failed to mask SSE on CPU %u during teardown: %d\n", + cpu, ret); + if (READ_ONCE(sse_shutting_down)) + atomic_set(&sse_teardown_failed, 1); + /* CPUHP installation rollback and state removal cannot fail. */ + return regular_offline ? ret : 0; + } + + list_for_each_entry_safe(event, tmp, &events, list) { + if (sse_event_is_global(event->evt_id)) + continue; + + ret = sse_teardown_event(event, cpu); + if (ret && !first_error) + first_error = ret; + sse_release_cleanup_event(event); + if (ret && regular_offline) + goto restore_cpu; + } + + list_for_each_entry_safe(event, tmp, &events, list) { + if (!sse_event_is_global(event->evt_id) || event->cpu != cpu) + continue; + + /* + * cpuhp_remove_state() invokes teardown while every CPU remains + * online. Do not migrate to a CPU whose callback may have run. + */ + if (READ_ONCE(sse_shutting_down) || event->cleanup_pending) { + ret = sse_teardown_event(event, cpu); + } else { + next_cpu = cpumask_any_but(cpu_online_mask, cpu); + if (next_cpu >= nr_cpu_ids) { + ret = sse_teardown_event(event, cpu); + } else { + ret = sse_event_set_target_cpu_nolock(event, next_cpu); + if (ret) + pr_warn("Failed to migrate global event %x from CPU %u: %d\n", + event->evt_id, cpu, ret); + } + } + + if (ret && !first_error) + first_error = ret; + sse_release_cleanup_event(event); + if (ret && regular_offline) + goto restore_cpu; + } + + if (first_error) { + /* An offline CPU is not revisited when this CPUHP state is removed. */ + atomic_set(&sse_teardown_failed, 1); + } + + return 0; + +restore_cpu: + /* + * CPUHP leaves this CPU online when teardown returns an error. Restore + * every client-owned local event before making SSE delivery visible again. + */ + ret = sse_restore_local_events(cpu); + if (!ret) + ret = sse_events_unmask(); + if (ret) { + atomic_set(&sse_teardown_failed, 1); + pr_warn("Failed to restore SSE after aborting CPU %u offline: %d\n", + cpu, ret); + } else { + pr_warn("Aborted CPU %u offline after SSE teardown failed: %d\n", + cpu, first_error); + } + + return first_error; +} + +static int sse_pm_notifier(struct notifier_block *nb, unsigned long action, + void *data) +{ + int ret; + + WARN_ON_ONCE(preemptible()); + + switch (action) { + case CPU_PM_ENTER: + ret = sse_events_mask(); + break; + case CPU_PM_EXIT: + case CPU_PM_ENTER_FAILED: + if (READ_ONCE(sse_shutting_down)) + return NOTIFY_OK; + ret = sse_events_unmask(); + break; + default: + return NOTIFY_DONE; + } + + if (ret) + return notifier_from_errno(ret); + + return NOTIFY_OK; +} + +static struct notifier_block sse_pm_nb = { + .notifier_call = sse_pm_notifier, +}; + +static int sse_panic_notifier(struct notifier_block *nb, unsigned long action, + void *data) +{ + riscv_sse_mask_current_hart(); + + return NOTIFY_OK; +} + +static struct notifier_block sse_panic_nb = { + .notifier_call = sse_panic_notifier, + .priority = INT_MAX, +}; + +/* + * Mask all CPUs and unregister all events on reboot or kexec. + */ +static int sse_reboot_notifier(struct notifier_block *nb, unsigned long action, + void *data) +{ + int ret; + + scoped_guard(mutex, &sse_mutex) { + if (!sse_shutting_down) { + WRITE_ONCE(sse_shutting_down, true); + ret = cpu_pm_unregister_notifier(&sse_pm_nb); + if (ret) { + pr_warn("Failed to unregister CPU PM notifier: %d\n", + ret); + atomic_set(&sse_teardown_failed, 1); + } + /* Drain CPU PM callbacks and client enables before teardown. */ + synchronize_rcu(); + cpuhp_remove_state(CPUHP_AP_RISCV_SSE_ONLINE); + } + } + + /* Normal kexec preserves firmware state but discards old kernel memory. */ + if (kexec_in_progress && atomic_read(&sse_teardown_failed)) + panic("SSE teardown failed; refusing unsafe kexec"); + + return NOTIFY_OK; +} + +static struct notifier_block sse_reboot_nb = { + .notifier_call = sse_reboot_notifier, +}; + +static int __init sse_init(void) +{ + int ret; + + /* + * A kdump kernel cannot identify registrations left by the crashed + * kernel. Keep them masked by not initializing SSE again. + */ + if (is_kdump_kernel() && riscv_sse_available()) { + sse_fw_state_retained = true; + pr_info("SSE remains disabled in the crash kernel\n"); + return -EOPNOTSUPP; + } + + if (sbi_probe_extension(SBI_EXT_SSE) <= 0) { + pr_info("Missing SBI SSE extension\n"); + return -EOPNOTSUPP; + } + pr_info("SBI SSE extension detected\n"); + + ret = cpu_pm_register_notifier(&sse_pm_nb); + if (ret) { + pr_warn("Failed to register CPU PM notifier...\n"); + return ret; + } + + ret = register_reboot_notifier(&sse_reboot_nb); + if (ret) { + pr_warn("Failed to register reboot notifier...\n"); + goto remove_cpupm; + } + + ret = atomic_notifier_chain_register(&panic_notifier_list, &sse_panic_nb); + if (ret) { + pr_warn("Failed to register panic notifier...\n"); + goto remove_reboot; + } + + /* Tear down perf events before dismantling their SSE delivery path. */ + ret = cpuhp_setup_state(CPUHP_AP_RISCV_SSE_ONLINE, "riscv/sse:online", + sse_cpu_online, sse_cpu_teardown); + if (ret < 0) + goto remove_panic; + + sse_available = true; + + return 0; + +remove_panic: + atomic_notifier_chain_unregister(&panic_notifier_list, &sse_panic_nb); + +remove_reboot: + unregister_reboot_notifier(&sse_reboot_nb); + +remove_cpupm: + cpu_pm_unregister_notifier(&sse_pm_nb); + + return ret; +} +arch_initcall(sse_init); diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h index feb32949aeea6a..d13af475508d5f 100644 --- a/include/linux/cpuhotplug.h +++ b/include/linux/cpuhotplug.h @@ -200,6 +200,7 @@ enum cpuhp_state { CPUHP_AP_ARM_MVEBU_SYNC_CLOCKS, CPUHP_AP_ARM_CORESIGHT_ONLINE, CPUHP_AP_X86_INTEL_EPB_ONLINE, + CPUHP_AP_RISCV_SSE_ONLINE, CPUHP_AP_PERF_ONLINE, CPUHP_AP_PERF_X86_ONLINE, CPUHP_AP_PERF_X86_UNCORE_ONLINE, diff --git a/include/linux/riscv_sbi_sse.h b/include/linux/riscv_sbi_sse.h new file mode 100644 index 00000000000000..774a782e556d2f --- /dev/null +++ b/include/linux/riscv_sbi_sse.h @@ -0,0 +1,89 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2025 Rivos Inc. + */ + +#ifndef __LINUX_RISCV_SBI_SSE_H +#define __LINUX_RISCV_SBI_SSE_H + +#include +#include +#include +#include + +struct sse_event; +struct pt_regs; + +typedef int (sse_event_handler_fn)(u32 event_num, void *arg, + struct pt_regs *regs); + +#ifdef CONFIG_RISCV_SBI_SSE + +/* + * The callback and its argument must remain valid until unregister succeeds. + * The callback runs in NMI context and must not sleep. + * regs is NULL if firmware cannot provide a complete interrupted context. + */ +struct sse_event *sse_event_register(u32 event_num, u32 priority, + sse_event_handler_fn *handler, void *arg); + +int sse_event_unregister(struct sse_event *evt); + +/* + * Transfer a retained event to the SSE core for deferred cleanup. The caller + * must not access the event or its callback data after this function returns. + */ +void sse_event_cleanup(struct sse_event *evt); + +int sse_event_set_target_cpu(struct sse_event *sse_evt, unsigned int cpu); + +int sse_event_enable(struct sse_event *sse_evt); + +int sse_event_disable(struct sse_event *sse_evt); + +/* Local events require the caller to remain on the current CPU. */ +int sse_event_enable_local(struct sse_event *sse_evt); +int sse_event_disable_local(struct sse_event *sse_evt); + +#else +static inline struct sse_event *sse_event_register(u32 event_num, u32 priority, + sse_event_handler_fn *handler, + void *arg) +{ + return ERR_PTR(-EOPNOTSUPP); +} + +static inline int sse_event_unregister(struct sse_event *evt) +{ + return -EOPNOTSUPP; +} + +static inline void sse_event_cleanup(struct sse_event *evt) { } + +static inline int sse_event_set_target_cpu(struct sse_event *sse_evt, + unsigned int cpu) +{ + return -EOPNOTSUPP; +} + +static inline int sse_event_enable(struct sse_event *sse_evt) +{ + return -EOPNOTSUPP; +} + +static inline int sse_event_disable(struct sse_event *sse_evt) +{ + return -EOPNOTSUPP; +} + +static inline int sse_event_enable_local(struct sse_event *sse_evt) +{ + return -EOPNOTSUPP; +} + +static inline int sse_event_disable_local(struct sse_event *sse_evt) +{ + return -EOPNOTSUPP; +} +#endif +#endif /* __LINUX_RISCV_SBI_SSE_H */ From 53ebfd9781195530b904ecb1d39dc267647ea1ca Mon Sep 17 00:00:00 2001 From: Zhanpeng Zhang Date: Fri, 11 Sep 2026 20:14:52 +0800 Subject: [PATCH 5/9] riscv: mm: avoid enabling interrupts for nofault page faults RISC-V enables interrupts in handle_page_fault() before checking whether the fault occurred with fault handling disabled. A nofault access from an atomic context can therefore run tracepoints and open an interrupt window before reaching the exception-table fixup. Handle an exception-table entry before entering the generic fault path when fault handling is disabled. Also keep interrupts disabled until such a fault has been resolved. This makes RISC-V consistent with the expectation that an in-atomic nofault access does not enter the normal fault-handling path. It also removes one source of re-entry when perf sampling is delivered through an SBI Supervisor Software Event (SSE). Signed-off-by: Zhanpeng Zhang Signed-off-by: Linux RISC-V bot --- arch/riscv/mm/fault.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/arch/riscv/mm/fault.c b/arch/riscv/mm/fault.c index 04ed6f8acae4fd..520495420462fe 100644 --- a/arch/riscv/mm/fault.c +++ b/arch/riscv/mm/fault.c @@ -294,6 +294,13 @@ void handle_page_fault(struct pt_regs *regs) if (kprobe_page_fault(regs, cause)) return; + /* + * Nofault accesses must be resolved through the exception table before + * entering the generic fault path or enabling interrupts. + */ + if (unlikely(faulthandler_disabled()) && fixup_exception(regs)) + return; + if (user_mode(regs)) trace_page_fault_user(addr, regs, cause); else @@ -314,8 +321,8 @@ void handle_page_fault(struct pt_regs *regs) return; } - /* Enable interrupts if they were enabled in the parent context. */ - if (!regs_irqs_disabled(regs)) + /* Do not open an interrupt window before a nofault fixup completes. */ + if (!regs_irqs_disabled(regs) && !faulthandler_disabled()) local_irq_enable(); /* From 31423cdbe637bf8780b40823380db52709ae53bd Mon Sep 17 00:00:00 2001 From: Zhanpeng Zhang Date: Fri, 11 Sep 2026 20:14:53 +0800 Subject: [PATCH 6/9] perf: RISC-V: support callchains with SSE delivery PMU overflow delivery through SSE enters Linux with a synthetic supervisor context on a dedicated event stack. The perf unwinder must use the context interrupted by the overflow rather than treating the SSE handler frame as the sampled frame. Use the interrupted pt_regs published by the RISC-V SSE entry path. Walk a kernel callchain only when the interrupted PC, SP, and frame pointer are consistent with the current task stack. For sensitive entry windows and IRQ stacks whose bounds cannot be proven, retain the interrupted PC without following an unsafe frame chain. User callchains continue through the existing nofault RISC-V user unwinder. If hstatus.SPV says the interrupted context was a guest, do not interpret the guest stack through the host address space. DWARF callchains additionally copy a raw user stack. The generic arch_perf_out_copy_user() implementation can take an exception-table handled fault when a source page is not resident. Repeated nested faults from the SSE handler can corrupt the interrupted kernel context under load. Provide an SSE-specific RISC-V copy path. Verify that the active page table belongs to current, use fast-only GUP to acquire each resident source page without falling back to a faulting slow path, and copy through the kernel mapping while holding the page reference. Stop at the first unavailable page and preserve perf's existing truncated-user-stack semantics. Keep the generic in-atomic user copy unchanged outside an SSE handler. Susheng Yang reported this failure with perf callchain workloads. Reported-by: Susheng Yang Signed-off-by: Zhanpeng Zhang Signed-off-by: Linux RISC-V bot --- arch/riscv/include/asm/perf_event.h | 10 ++ arch/riscv/kernel/perf_callchain.c | 142 ++++++++++++++++++++++++++++ 2 files changed, 152 insertions(+) diff --git a/arch/riscv/include/asm/perf_event.h b/arch/riscv/include/asm/perf_event.h index bcc928fd3785c1..ddc40479475195 100644 --- a/arch/riscv/include/asm/perf_event.h +++ b/arch/riscv/include/asm/perf_event.h @@ -18,6 +18,16 @@ (regs)->sp = current_stack_pointer; \ (regs)->status = SR_PP; \ } + +#ifdef CONFIG_RISCV_SBI_SSE +/* + * Raw user-stack sampling can run in the NMI-like SSE context. Route it + * through an implementation that does not fault on a non-resident page. + */ +unsigned long riscv_perf_out_copy_user(void *dst, const void *src, + unsigned long n); +#define arch_perf_out_copy_user riscv_perf_out_copy_user +#endif #endif #endif /* _ASM_RISCV_PERF_EVENT_H */ diff --git a/arch/riscv/kernel/perf_callchain.c b/arch/riscv/kernel/perf_callchain.c index b465bc9eb870ec..ec75689c7aecbc 100644 --- a/arch/riscv/kernel/perf_callchain.c +++ b/arch/riscv/kernel/perf_callchain.c @@ -1,9 +1,15 @@ // SPDX-License-Identifier: GPL-2.0 /* Copyright (C) 2019 Hangzhou C-SKY Microsystems co.,ltd. */ +#include +#include #include +#include +#include #include +#include +#include #include static bool fill_callchain(void *entry, unsigned long pc) @@ -11,6 +17,128 @@ static bool fill_callchain(void *entry, unsigned long pc) return perf_callchain_store(entry, pc) == 0; } +#ifdef CONFIG_RISCV_SBI_SSE +static bool sse_addr_on_task_stack(unsigned long addr, unsigned long size) +{ + unsigned long end = addr + size; + unsigned long stack; + + if (end < addr) + return false; + + stack = (unsigned long)task_stack_page(current); + if (addr >= stack && end <= stack + THREAD_SIZE) + return true; + + return false; +} + +static bool sse_kernel_regs_safe(struct pt_regs *regs) +{ + unsigned long fp = frame_pointer(regs); + unsigned long pc = instruction_pointer(regs); + unsigned long sp = user_stack_pointer(regs); + + if (!__kernel_text_address(pc)) + return false; + if (!sse_addr_on_task_stack(sp, sizeof(unsigned long))) + return false; + if (fp < sizeof(struct stackframe)) + return false; + + return sse_addr_on_task_stack(fp - sizeof(struct stackframe), + sizeof(struct stackframe)); +} + +static bool sse_callchain_is_guest(const struct riscv_sse_interrupted_context *context) +{ + return context && (context->hstatus & HSTATUS_SPV); +} + +static bool sse_callchain_kernel(struct perf_callchain_entry_ctx *entry, + struct pt_regs *regs) +{ + const struct riscv_sse_interrupted_context *context; + unsigned long pc; + + context = riscv_sse_get_interrupted_context(); + if (!context || context->regs != regs) + return false; + + /* A guest stack cannot be walked using the host kernel address space. */ + if (sse_callchain_is_guest(context)) + return true; + + if (user_mode(regs)) + return true; + + if (sse_kernel_regs_safe(regs)) { + walk_stackframe(NULL, regs, fill_callchain, entry); + return true; + } + + /* + * Keep the sample useful for sensitive entry paths and IRQ stacks. The + * generic walker does not take explicit IRQ stack bounds, and its + * THREAD_SIZE alignment assumption fails for non-vmapped IRQ stacks. + * Conservatively avoid walking IRQ stacks in every configuration. + */ + pc = instruction_pointer(regs); + if (__kernel_text_address(pc)) + perf_callchain_store(entry, pc); + + return true; +} + +unsigned long riscv_perf_out_copy_user(void *dst, const void *src, + unsigned long n) +{ + unsigned long addr = (unsigned long)src; + unsigned long copied = 0; + + /* Keep the generic fast path unchanged outside an SSE handler. */ + if (!riscv_sse_get_interrupted_context()) { + unsigned long ret; + + pagefault_disable(); + ret = __copy_from_user_inatomic(dst, src, n); + pagefault_enable(); + return ret; + } + + if (!access_ok(src, n)) + return n; + + /* Do not sample user memory through an unrelated active page table. */ + if (!current->mm || + (csr_read(CSR_SATP) & SATP_PPN) != virt_to_pfn(current->mm->pgd)) + return n; + + while (copied < n) { + unsigned long offset = offset_in_page(addr); + unsigned long chunk = min(n - copied, PAGE_SIZE - offset); + struct page *page; + + /* + * Fast-only GUP cannot fault. This follows perf_virt_to_phys(): + * local interrupts remain disabled throughout SSE processing, so a + * concurrent unmap cannot complete its TLB teardown before this + * temporary reference is put. + */ + if (!get_user_page_fast_only(addr, 0, &page)) + break; + + memcpy((char *)dst + copied, + (char *)page_address(page) + offset, chunk); + put_page(page); + addr += chunk; + copied += chunk; + } + + return n - copied; +} +#endif + /* * This will be called when the target is in user mode * This function will only be called when we use @@ -28,6 +156,15 @@ static bool fill_callchain(void *entry, unsigned long pc) void perf_callchain_user(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs) { +#ifdef CONFIG_RISCV_SBI_SSE + const struct riscv_sse_interrupted_context *context; + + context = riscv_sse_get_interrupted_context(); + /* A guest stack cannot be walked using the host address space. */ + if (sse_callchain_is_guest(context)) + return; +#endif + if (perf_guest_state()) { /* TODO: We don't support guest os callchain now */ return; @@ -39,6 +176,11 @@ void perf_callchain_user(struct perf_callchain_entry_ctx *entry, void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs) { +#ifdef CONFIG_RISCV_SBI_SSE + if (sse_callchain_kernel(entry, regs)) + return; +#endif + if (perf_guest_state()) { /* TODO: We don't support guest os callchain now */ return; From f48967f69b65932fba341dc14380416ba5e054dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20L=C3=A9ger?= Date: Fri, 11 Sep 2026 20:14:54 +0800 Subject: [PATCH 7/9] perf: RISC-V: add support for SSE event MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Register a handler for the local PMU overflow SSE event so that RISC-V perf can receive overflows even when normal S-mode interrupts are masked. Reuse the existing overflow handler and pass it the interrupted pt_regs rebuilt by the architecture SSE entry path. Select the delivery mechanism once during PMU probe. Prefer SSE when its event can be registered and enabled. If the extension or PMU event is explicitly unsupported, use the ordinary PMU interrupt. Do not enable the interrupt after any other SSE setup failure or when a crash kernel may have inherited firmware state. Install the PMU enable and disable callbacks only after SSE delivery becomes active. Keep the local PMU SSE event disabled across CPU power management. On entry, the generic SSE notifier masks the hart before the lower-priority PMU notifier disables the event and stops the counters. On exit, the SSE notifier first unmasks the hart while the event remains disabled. The PMU notifier then restores counters and event userpage state before enabling the event. An unmask failure stops the notifier chain and leaves the counters stopped. Ordinary PMU interrupts retain their existing notifier ordering. An SSE overflow can arrive as soon as the event is enabled during probe. Publish the counter mask before SSE setup, so an early handler can stop the counter source even before perf starts admitting normal samples. After a real overflow, restart only counters whose perf state is still running. Honor a non-zero return from perf_event_overflow() and leave throttled events stopped. Guest attribution is not part of this version. Detect an interrupted guest from hstatus.SPV and skip its sample while still updating the period and counter state, rather than exposing guest state as a host sample. The perf PMU callbacks cannot return errors. If an SSE transition or interrupted-context read fails, latch the failure per CPU and stop its mapped events through the normal perf state transitions. Do not reset the firmware counter mapping behind perf, restart a failed event, or attempt a runtime switch to IRQ delivery. During cleanup, close callback admission and synchronously drain each CPU before disabling and unregistering the SSE event. If firmware refuses the cleanup, stop the counter source and transfer the event to the SSE core so later CPU hotplug or shutdown processing can retry without using freed PMU callback state. Signed-off-by: Clément Léger Co-developed-by: Himanshu Chauhan Signed-off-by: Himanshu Chauhan Co-developed-by: Zhanpeng Zhang Signed-off-by: Zhanpeng Zhang Signed-off-by: Linux RISC-V bot --- Documentation/arch/riscv/index.rst | 1 + Documentation/arch/riscv/pmu-sse.rst | 55 +++ MAINTAINERS | 1 + drivers/firmware/riscv/riscv_sbi_sse.c | 9 + drivers/perf/Kconfig | 11 + drivers/perf/riscv_pmu.c | 14 +- drivers/perf/riscv_pmu_sbi.c | 544 ++++++++++++++++++++----- include/linux/perf/riscv_pmu.h | 20 +- include/linux/riscv_sbi_sse.h | 6 + 9 files changed, 566 insertions(+), 95 deletions(-) create mode 100644 Documentation/arch/riscv/pmu-sse.rst diff --git a/Documentation/arch/riscv/index.rst b/Documentation/arch/riscv/index.rst index ac535c52d509c7..5cb909c83a882c 100644 --- a/Documentation/arch/riscv/index.rst +++ b/Documentation/arch/riscv/index.rst @@ -11,6 +11,7 @@ RISC-V architecture vm-layout hwprobe patch-acceptance + pmu-sse uabi vector cmodx diff --git a/Documentation/arch/riscv/pmu-sse.rst b/Documentation/arch/riscv/pmu-sse.rst new file mode 100644 index 00000000000000..b7458a9d0116c1 --- /dev/null +++ b/Documentation/arch/riscv/pmu-sse.rst @@ -0,0 +1,55 @@ +.. SPDX-License-Identifier: GPL-2.0 + +======================================== +RISC-V PMU overflow delivery through SSE +======================================== + +When ``CONFIG_RISCV_PMU_SBI_SSE`` is enabled and firmware provides the local +PMU overflow event, the RISC-V SBI PMU driver uses Supervisor Software Events +(SSE) to deliver counter overflows. + +Delivery selection +================== + +The delivery mechanism is selected once while the PMU device is probed. The +driver first tries to register and enable the local PMU overflow SSE event. It +tries the ordinary PMU interrupt only when the SSE extension or the local PMU +event is explicitly unsupported. It does not change the delivery mechanism +after the PMU has been registered. + +Other SSE setup errors do not prove that firmware released the overflow route, +so the driver does not enable the ordinary PMU interrupt. The PMU remains +available for counting but not sampling. If setup retained an SSE event, the +PMU keeps quiescing that event around perf scheduling changes without rearming +it. This preserves callback ownership without creating two possible delivery +mechanisms for the same hardware overflow. + +Interrupted context +=================== + +SSE enters Linux through a supervisor handler context constructed by firmware. +The entry contract preserves the interrupted GPRs except ``a6`` and ``a7``, +which Linux reads from the interrupted-register event attributes. Linux then +reconstructs the interrupted ``pt_regs`` and publishes it while dispatching the +event. + +Perf uses that context for register samples and callchains. Kernel stack +walking verifies that the interrupted frame belongs to the current task stack +before dereferencing it. For sensitive entry paths and IRQ stacks, Linux +records the interrupted PC without walking a stack whose bounds cannot be +proved. User callchains use the existing no-fault user unwinder. A DWARF +raw user-stack copy from an SSE handler must not take a page fault, so it +walks the current task's page tables with fast-only GUP, copies each resident +page through its kernel mapping, and stops at the first non-resident page, +preserving perf's truncated-user-stack semantics. + +CPU power management +==================== + +The PMU and SSE CPU power-management callbacks are ordered according to the +selected delivery mechanism. With SSE delivery, SSE events are masked before +the lower-priority PMU callback disables the local PMU event and stops the +counters on entry. On exit, the hart is unmasked while the local PMU event is +still disabled. The PMU callback then restores the counters and enables the +event, so an unmask failure leaves the counters stopped. The ordinary +interrupt path retains the existing PMU ordering. diff --git a/MAINTAINERS b/MAINTAINERS index d11cd44d24498f..91b4e32a1fb21e 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -23392,6 +23392,7 @@ M: Atish Patra R: Anup Patel L: linux-riscv@lists.infradead.org S: Supported +F: Documentation/arch/riscv/pmu-sse.rst F: drivers/perf/riscv_pmu.c F: drivers/perf/riscv_pmu_legacy.c F: drivers/perf/riscv_pmu_sbi.c diff --git a/drivers/firmware/riscv/riscv_sbi_sse.c b/drivers/firmware/riscv/riscv_sbi_sse.c index e81cfb4b0c563c..c7d2a2def00a8d 100644 --- a/drivers/firmware/riscv/riscv_sbi_sse.c +++ b/drivers/firmware/riscv/riscv_sbi_sse.c @@ -161,6 +161,15 @@ static int sse_sbi_event_func(struct sse_event *event, unsigned long func) return 0; } +bool sse_event_is_enabled_local(struct sse_event *event) +{ + if (!sse_event_is_global(event->evt_id)) + lockdep_assert_preemption_disabled(); + + return sse_get_reg_evt(event)->is_enabled; +} +EXPORT_SYMBOL_GPL(sse_event_is_enabled_local); + int sse_event_disable_local(struct sse_event *event) { if (!sse_event_is_global(event->evt_id)) diff --git a/drivers/perf/Kconfig b/drivers/perf/Kconfig index 245e7bb763b994..10b047856121dc 100644 --- a/drivers/perf/Kconfig +++ b/drivers/perf/Kconfig @@ -105,6 +105,17 @@ config RISCV_PMU_SBI full perf feature support i.e. counter overflow, privilege mode filtering, counter configuration. +config RISCV_PMU_SBI_SSE + depends on RISCV_PMU_SBI && RISCV_SBI_SSE + bool "RISC-V PMU SSE events" + default n + help + Say y if you want to use SSE events to deliver PMU interrupts. This + provides a way to profile the kernel at any level by using NMI-like + SSE events. SSE events being really intrusive, this option allows + to select it only if needed. See Documentation/arch/riscv/pmu-sse.rst + for interrupted-context limitations. + config STARFIVE_STARLINK_PMU depends on ARCH_STARFIVE || COMPILE_TEST depends on 64BIT diff --git a/drivers/perf/riscv_pmu.c b/drivers/perf/riscv_pmu.c index 8e3cd0f35336fa..1f65c640b69398 100644 --- a/drivers/perf/riscv_pmu.c +++ b/drivers/perf/riscv_pmu.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include @@ -247,6 +248,11 @@ void riscv_pmu_start(struct perf_event *event, int flags) if (flags & PERF_EF_RELOAD) WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE)); +#ifdef CONFIG_RISCV_PMU_SBI_SSE + if (unlikely(this_cpu_ptr(rvpmu->hw_events)->sse_failed)) + return; +#endif + hwc->state = 0; riscv_pmu_event_set_period(event); init_val = local64_read(&hwc->prev_count) & max_period; @@ -306,6 +312,7 @@ static int riscv_pmu_event_init(struct perf_event *event) struct hw_perf_event *hwc = &event->hw; struct riscv_pmu *rvpmu = to_riscv_pmu(event->pmu); int mapped_event; + int ret; u64 event_config = 0; uint64_t cmask; @@ -331,8 +338,11 @@ static int riscv_pmu_event_init(struct perf_event *event) hwc->idx = -1; hwc->event_base = mapped_event; - if (rvpmu->event_init) - rvpmu->event_init(event); + if (rvpmu->event_init) { + ret = rvpmu->event_init(event); + if (ret) + return ret; + } if (!is_sampling_event(event)) { /* diff --git a/drivers/perf/riscv_pmu_sbi.c b/drivers/perf/riscv_pmu_sbi.c index 50220f7b46d9b0..7e971b47730c41 100644 --- a/drivers/perf/riscv_pmu_sbi.c +++ b/drivers/perf/riscv_pmu_sbi.c @@ -16,13 +16,16 @@ #include #include #include +#include #include +#include #include #include #include #include #include +#include #include #include #include @@ -95,7 +98,6 @@ static bool riscv_pmu_use_irq; static unsigned int riscv_pmu_irq_num; static unsigned int riscv_pmu_irq_mask; static unsigned int riscv_pmu_irq; - /* Cache the available counters in a bitmask */ static unsigned long cmask; @@ -896,14 +898,24 @@ static int pmu_sbi_get_ctrinfo(int nctr, unsigned long *mask) return 0; } -static inline void pmu_sbi_stop_all(struct riscv_pmu *pmu) +static inline void pmu_sbi_stop_all_mask(unsigned long ctr_mask) { /* * No need to check the error because we are disabling all the counters * which may include counters that are not enabled yet. */ sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_STOP, - 0, pmu->cmask, SBI_PMU_STOP_FLAG_RESET, 0, 0, 0); + 0, ctr_mask, SBI_PMU_STOP_FLAG_RESET, 0, 0, 0); +} + +static inline void pmu_sbi_stop_all(struct riscv_pmu *pmu) +{ + pmu_sbi_stop_all_mask(pmu->cmask); +} + +static void pmu_sbi_stop_all_cpu(void *info) +{ + pmu_sbi_stop_all(info); } static inline void pmu_sbi_stop_hw_ctrs(struct riscv_pmu *pmu) @@ -953,7 +965,7 @@ static inline void pmu_sbi_stop_hw_ctrs(struct riscv_pmu *pmu) static inline void pmu_sbi_start_ovf_ctrs_sbi(struct cpu_hw_events *cpu_hw_evt, u64 ctr_ovf_mask) { - int idx = 0, i; + int idx, i; struct perf_event *event; unsigned long flag = SBI_PMU_START_FLAG_SET_INIT_VALUE; unsigned long ctr_start_mask = 0; @@ -962,7 +974,19 @@ static inline void pmu_sbi_start_ovf_ctrs_sbi(struct cpu_hw_events *cpu_hw_evt, u64 init_val = 0; for (i = 0; i < BITS_TO_LONGS(RISCV_MAX_COUNTERS); i++) { - ctr_start_mask = cpu_hw_evt->used_hw_ctrs[i] & ~ctr_ovf_mask; + unsigned long ctr_ovf_mask_word; + int lidx; + + ctr_ovf_mask_word = ctr_ovf_mask >> (i * BITS_PER_LONG); + ctr_start_mask = 0; + for_each_set_bit(idx, &cpu_hw_evt->used_hw_ctrs[i], BITS_PER_LONG) { + lidx = idx + i * BITS_PER_LONG; + event = cpu_hw_evt->events[lidx]; + if (event && !(event->hw.state & PERF_HES_STOPPED)) + ctr_start_mask |= BIT(idx); + } + ctr_start_mask &= ~ctr_ovf_mask_word; + /* Start all the counters that did not overflow in a single shot */ if (ctr_start_mask) { sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_START, i * BITS_PER_LONG, @@ -971,23 +995,25 @@ static inline void pmu_sbi_start_ovf_ctrs_sbi(struct cpu_hw_events *cpu_hw_evt, } /* Reinitialize and start all the counter that overflowed */ - while (ctr_ovf_mask) { - if (ctr_ovf_mask & 0x01) { - event = cpu_hw_evt->events[idx]; - hwc = &event->hw; - max_period = riscv_pmu_ctr_get_width_mask(event); - init_val = local64_read(&hwc->prev_count) & max_period; + for (idx = 0; idx < RISCV_MAX_COUNTERS; idx++) { + if (!(ctr_ovf_mask & BIT_ULL(idx))) + continue; + + event = cpu_hw_evt->events[idx]; + if (!event || event->hw.state & PERF_HES_STOPPED) + continue; + + hwc = &event->hw; + max_period = riscv_pmu_ctr_get_width_mask(event); + init_val = local64_read(&hwc->prev_count) & max_period; #if defined(CONFIG_32BIT) - sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_START, idx, 1, - flag, init_val, init_val >> 32, 0); + sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_START, idx, 1, + flag, init_val, init_val >> 32, 0); #else - sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_START, idx, 1, - flag, init_val, 0, 0); + sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_START, idx, 1, + flag, init_val, 0, 0); #endif - perf_event_update_userpage(event); - } - ctr_ovf_mask = ctr_ovf_mask >> 1; - idx++; + perf_event_update_userpage(event); } } @@ -1002,7 +1028,7 @@ static inline void pmu_sbi_start_ovf_ctrs_snapshot(struct cpu_hw_events *cpu_hw_ struct riscv_pmu_snapshot_data *sdata = cpu_hw_evt->snapshot_addr; for_each_set_bit(idx, cpu_hw_evt->used_hw_ctrs, RISCV_MAX_COUNTERS) { - if (ctr_ovf_mask & BIT(idx)) { + if (ctr_ovf_mask & BIT_ULL(idx)) { event = cpu_hw_evt->events[idx]; hwc = &event->hw; max_period = riscv_pmu_ctr_get_width_mask(event); @@ -1016,31 +1042,111 @@ static inline void pmu_sbi_start_ovf_ctrs_snapshot(struct cpu_hw_events *cpu_hw_ } for (i = 0; i < BITS_TO_LONGS(RISCV_MAX_COUNTERS); i++) { + unsigned long ctr_start_mask = 0; + int lidx; + /* Restore the counter values to relative indices for used hw counters */ - for_each_set_bit(idx, &cpu_hw_evt->used_hw_ctrs[i], BITS_PER_LONG) - sdata->ctr_values[idx] = - cpu_hw_evt->snapshot_cval_shcopy[idx + i * BITS_PER_LONG]; + for_each_set_bit(idx, &cpu_hw_evt->used_hw_ctrs[i], BITS_PER_LONG) { + lidx = idx + i * BITS_PER_LONG; + event = cpu_hw_evt->events[lidx]; + if (event && !(event->hw.state & PERF_HES_STOPPED)) + ctr_start_mask |= BIT(idx); + + sdata->ctr_values[idx] = cpu_hw_evt->snapshot_cval_shcopy[lidx]; + } + /* Start all the counters in a single shot */ - sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_START, idx * BITS_PER_LONG, - cpu_hw_evt->used_hw_ctrs[i], flag, 0, 0, 0); + if (ctr_start_mask) + sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_START, + i * BITS_PER_LONG, ctr_start_mask, flag, 0, 0, 0); } } +static bool pmu_sbi_sse_failed(struct cpu_hw_events *cpu_hw_evt) +{ +#ifdef CONFIG_RISCV_PMU_SBI_SSE + return READ_ONCE(cpu_hw_evt->sse_failed); +#else + return false; +#endif +} + static void pmu_sbi_start_overflow_mask(struct riscv_pmu *pmu, u64 ctr_ovf_mask) { struct cpu_hw_events *cpu_hw_evt = this_cpu_ptr(pmu->hw_events); + if (unlikely(pmu_sbi_sse_failed(cpu_hw_evt))) + return; + if (sbi_pmu_snapshot_available()) pmu_sbi_start_ovf_ctrs_snapshot(cpu_hw_evt, ctr_ovf_mask); else pmu_sbi_start_ovf_ctrs_sbi(cpu_hw_evt, ctr_ovf_mask); } -static irqreturn_t pmu_sbi_ovf_handler(int irq, void *dev) +#ifdef CONFIG_RISCV_PMU_SBI_SSE +/* + * A local SSE delivery failure makes the current PMU state unsafe to resume. + * Latch the failure before stopping mapped events so the SSE transition and + * overflow restart paths cannot undo the fail-safe while they are quiesced. + */ +static void pmu_sbi_fail_sse(struct riscv_pmu *pmu, const char *op, int ret) +{ + struct cpu_hw_events *cpu_hw_evt = this_cpu_ptr(pmu->hw_events); + struct perf_event *event; + int idx; + + if (READ_ONCE(cpu_hw_evt->sse_failed)) + return; + + WRITE_ONCE(cpu_hw_evt->sse_failed, true); + pr_err_ratelimited("failed to %s local PMU SSE event: %d; stopping counters\n", + op, ret); + + for (idx = 0; idx < RISCV_MAX_COUNTERS; idx++) { + event = cpu_hw_evt->events[idx]; + if (event) + riscv_pmu_stop(event, PERF_EF_UPDATE); + } +} + +static void pmu_sbi_sse_disable(struct pmu *pmu) +{ + struct riscv_pmu *rvpmu = to_riscv_pmu(pmu); + struct cpu_hw_events *cpu_hw_evt = this_cpu_ptr(rvpmu->hw_events); + int ret; + + if (!READ_ONCE(rvpmu->sse_active) || + READ_ONCE(cpu_hw_evt->sse_failed)) + return; + + ret = sse_event_disable_local(rvpmu->sse_evt); + if (unlikely(ret)) + pmu_sbi_fail_sse(rvpmu, "disable", ret); +} + +static void pmu_sbi_sse_enable(struct pmu *pmu) +{ + struct riscv_pmu *rvpmu = to_riscv_pmu(pmu); + struct cpu_hw_events *cpu_hw_evt = this_cpu_ptr(rvpmu->hw_events); + int ret; + + if (!READ_ONCE(rvpmu->sse_active) || + READ_ONCE(cpu_hw_evt->sse_failed)) + return; + + ret = sse_event_enable_local(rvpmu->sse_evt); + if (unlikely(ret)) + pmu_sbi_fail_sse(rvpmu, "enable", ret); +} +#endif + +static irqreturn_t pmu_sbi_ovf_handler(struct cpu_hw_events *cpu_hw_evt, + struct pt_regs *regs, bool from_sse, + bool from_guest) { struct perf_sample_data data; - struct pt_regs *regs; struct hw_perf_event *hw_evt; union sbi_pmu_ctr_info *info; int lidx, hidx, fidx; @@ -1048,28 +1154,38 @@ static irqreturn_t pmu_sbi_ovf_handler(int irq, void *dev) struct perf_event *event; u64 overflow; u64 overflowed_ctrs = 0; - struct cpu_hw_events *cpu_hw_evt = dev; u64 start_clock = sched_clock(); struct riscv_pmu_snapshot_data *sdata = cpu_hw_evt->snapshot_addr; if (WARN_ON_ONCE(!cpu_hw_evt)) return IRQ_NONE; - /* Firmware counter don't support overflow yet */ + /* + * SSE can arrive before perf installs an event. The early exits below + * must stop the PMU source before firmware completes the SSE. + */ fidx = find_first_bit(cpu_hw_evt->used_hw_ctrs, RISCV_MAX_COUNTERS); if (fidx == RISCV_MAX_COUNTERS) { - csr_clear(CSR_SIP, BIT(riscv_pmu_irq_num)); + if (from_sse) + pmu_sbi_stop_all_mask(cmask); + else + csr_clear(CSR_SIP, BIT(riscv_pmu_irq_num)); return IRQ_NONE; } event = cpu_hw_evt->events[fidx]; if (!event) { - ALT_SBI_PMU_OVF_CLEAR_PENDING(riscv_pmu_irq_mask); + if (from_sse) + pmu_sbi_stop_all_mask(cmask); + else + ALT_SBI_PMU_OVF_CLEAR_PENDING(riscv_pmu_irq_mask); return IRQ_NONE; } pmu = to_riscv_pmu(event->pmu); pmu_sbi_stop_hw_ctrs(pmu); + if (unlikely(pmu_sbi_sse_failed(cpu_hw_evt))) + return IRQ_NONE; /* Overflow status register should only be read after counter are stopped */ if (sbi_pmu_snapshot_available()) @@ -1079,15 +1195,17 @@ static irqreturn_t pmu_sbi_ovf_handler(int irq, void *dev) /* * Overflow interrupt pending bit should only be cleared after stopping - * all the counters to avoid any race condition. + * all the counters to avoid any race condition. When using SSE, + * interrupt is cleared when stopping counters. */ - ALT_SBI_PMU_OVF_CLEAR_PENDING(riscv_pmu_irq_mask); + if (!from_sse) + ALT_SBI_PMU_OVF_CLEAR_PENDING(riscv_pmu_irq_mask); /* No overflow bit is set */ - if (!overflow) + if (!overflow) { + pmu_sbi_start_overflow_mask(pmu, 0); return IRQ_NONE; - - regs = get_irq_regs(); + } for_each_set_bit(lidx, cpu_hw_evt->used_hw_ctrs, RISCV_MAX_COUNTERS) { struct perf_event *event = cpu_hw_evt->events[lidx]; @@ -1109,14 +1227,20 @@ static irqreturn_t pmu_sbi_ovf_handler(int irq, void *dev) hidx = info->csr - CSR_CYCLE; /* check if the corresponding bit is set in scountovf or overflow mask in shmem */ - if (!(overflow & BIT(hidx))) + if (!(overflow & BIT_ULL(hidx))) continue; +#ifdef CONFIG_CPU_PM + /* Do not let CPU-PM resume override this overflow decision. */ + if (from_sse) + clear_bit(lidx, cpu_hw_evt->pm_resume_hw_ctrs); +#endif + /* * Keep a track of overflowed counters so that they can be started * with updated initial value. */ - overflowed_ctrs |= BIT(lidx); + overflowed_ctrs |= BIT_ULL(lidx); hw_evt = &event->hw; /* Update the event states here so that we know the state while reading */ hw_evt->state |= PERF_HES_STOPPED; @@ -1124,6 +1248,14 @@ static irqreturn_t pmu_sbi_ovf_handler(int irq, void *dev) hw_evt->state |= PERF_HES_UPTODATE; perf_sample_data_init(&data, 0, hw_evt->last_period); if (riscv_pmu_event_set_period(event)) { + int overflow_ret; + + /* Guest attribution and guest stack sampling are not supported yet. */ + if (from_guest) { + hw_evt->state = 0; + continue; + } + /* * Unlike other ISAs, RISC-V don't have to disable interrupts * to avoid throttling here. As per the specification, the @@ -1132,128 +1264,291 @@ static irqreturn_t pmu_sbi_ovf_handler(int irq, void *dev) * TODO: We will need to stop the guest counters once * virtualization support is added. */ - perf_event_overflow(event, &data, regs); + overflow_ret = perf_event_overflow(event, &data, regs); + if (!overflow_ret) + hw_evt->state = 0; + } else { + hw_evt->state = 0; } - /* Reset the state as we are going to start the counter after the loop */ - hw_evt->state = 0; } pmu_sbi_start_overflow_mask(pmu, overflowed_ctrs); + perf_sample_event_took(sched_clock() - start_clock); return IRQ_HANDLED; } -static int pmu_sbi_starting_cpu(unsigned int cpu, struct hlist_node *node) +static irqreturn_t pmu_sbi_ovf_irq_handler(int irq, void *dev) { - struct riscv_pmu *pmu = hlist_entry_safe(node, struct riscv_pmu, node); - struct cpu_hw_events *cpu_hw_evt = this_cpu_ptr(pmu->hw_events); - - /* - * We keep enabling userspace access to CYCLE, TIME and INSTRET via the - * legacy option but that will be removed in the future. - */ - if (sysctl_perf_user_access == SYSCTL_LEGACY) - csr_write(CSR_SCOUNTEREN, 0x7); - else - csr_write(CSR_SCOUNTEREN, 0x2); + return pmu_sbi_ovf_handler(dev, get_irq_regs(), false, false); +} - /* Stop all the counters so that they can be enabled from perf */ - pmu_sbi_stop_all(pmu); +#ifdef CONFIG_RISCV_PMU_SBI_SSE +static int pmu_sbi_ovf_sse_handler(u32 evt, void *arg, struct pt_regs *regs) +{ + const struct riscv_sse_interrupted_context *context; + struct riscv_pmu *pmu = arg; + struct cpu_hw_events *hw_event = raw_cpu_ptr(pmu->hw_events); + bool from_guest; + + if (unlikely(!READ_ONCE(pmu->sse_active))) { + pmu_sbi_stop_all(pmu); + return -EIO; + } - if (riscv_pmu_use_irq) { - cpu_hw_evt->irq = riscv_pmu_irq; - ALT_SBI_PMU_OVF_CLEAR_PENDING(riscv_pmu_irq_mask); - enable_percpu_irq(riscv_pmu_irq, IRQ_TYPE_NONE); + if (unlikely(!regs)) { + pmu_sbi_fail_sse(pmu, "read interrupted context for", -EIO); + return -EIO; } - if (sbi_pmu_snapshot_available()) - return pmu_sbi_snapshot_setup(pmu, cpu); + context = riscv_sse_get_interrupted_context(); + from_guest = context && context->regs == regs && + (context->hstatus & HSTATUS_SPV); + pmu_sbi_ovf_handler(hw_event, regs, true, from_guest); return 0; } -static int pmu_sbi_dying_cpu(unsigned int cpu, struct hlist_node *node) +static int pmu_sbi_setup_sse(struct riscv_pmu *pmu) { - if (riscv_pmu_use_irq) { - disable_percpu_irq(riscv_pmu_irq); - } + int ret; + struct sse_event *evt; - /* Disable all counters access for user mode now */ - csr_write(CSR_SCOUNTEREN, 0x0); + evt = sse_event_register(SBI_SSE_EVENT_LOCAL_PMU_OVERFLOW, 0, + pmu_sbi_ovf_sse_handler, pmu); + if (IS_ERR(evt)) + return PTR_ERR(evt); + pmu->sse_evt = evt; - if (sbi_pmu_snapshot_available()) - return pmu_sbi_snapshot_disable(); + ret = sse_event_enable(evt); + if (ret) { + int cleanup_ret; - return 0; + cleanup_ret = sse_event_disable(evt); + if (cleanup_ret) { + pr_warn("failed to disable SSE event after setup error: %d\n", + cleanup_ret); + return cleanup_ret; + } + + cleanup_ret = sse_event_unregister(evt); + + if (cleanup_ret) { + pr_warn("failed to unregister SSE event: %d\n", + cleanup_ret); + } else { + pmu->sse_evt = NULL; + } + return cleanup_ret ?: ret; + } + + WRITE_ONCE(pmu->sse_active, true); + pr_info("using SSE for PMU event delivery\n"); + + return ret; } -static int pmu_sbi_setup_irqs(struct riscv_pmu *pmu, struct platform_device *pdev) +static void pmu_sbi_cleanup_sse(struct riscv_pmu *pmu) { + struct sse_event *sse_evt; int ret; - struct cpu_hw_events __percpu *hw_events = pmu->hw_events; - struct irq_domain *domain = NULL; + sse_evt = pmu->sse_evt; + if (!sse_evt) + return; + /* + * Close callback admission before draining each CPU. PMU callbacks run + * with local interrupts disabled, so the synchronous IPI cannot complete + * until a callback that observed the old state has returned. + */ + WRITE_ONCE(pmu->sse_active, false); + on_each_cpu(pmu_sbi_stop_all_cpu, pmu, 1); + + ret = sse_event_disable(sse_evt); + if (ret) { + pr_warn("failed to disable SSE event: %d\n", ret); + goto retain; + } + + ret = sse_event_unregister(sse_evt); + if (ret) { + pr_warn("failed to unregister SSE event: %d\n", ret); + goto retain; + } + + pmu->sse_evt = NULL; + return; + +retain: + sse_event_cleanup(sse_evt); + pmu->sse_evt = NULL; +} + +static bool pmu_sbi_sse_state_retained(struct riscv_pmu *pmu) +{ + return pmu->sse_evt; +} +#else +static int pmu_sbi_setup_sse(struct riscv_pmu *pmu) +{ + return -EOPNOTSUPP; +} + +static void pmu_sbi_cleanup_sse(struct riscv_pmu *pmu) {} + +static bool pmu_sbi_sse_state_retained(struct riscv_pmu *pmu) +{ + return false; +} +#endif + +static bool pmu_sbi_select_irq(void) +{ if (riscv_isa_extension_available(NULL, SSCOFPMF)) { riscv_pmu_irq_num = RV_IRQ_PMU; - riscv_pmu_use_irq = true; + return true; } else if (IS_ENABLED(CONFIG_ERRATA_THEAD_PMU) && riscv_cached_mvendorid(0) == THEAD_VENDOR_ID && riscv_cached_marchid(0) == 0 && riscv_cached_mimpid(0) == 0) { riscv_pmu_irq_num = THEAD_C9XX_RV_IRQ_PMU; - riscv_pmu_use_irq = true; + return true; } else if (riscv_has_vendor_extension_unlikely(ANDES_VENDOR_ID, RISCV_ISA_VENDOR_EXT_XANDESPMU) && IS_ENABLED(CONFIG_ANDES_CUSTOM_PMU)) { riscv_pmu_irq_num = ANDES_SLI_CAUSE_BASE + ANDES_RV_IRQ_PMOVI; - riscv_pmu_use_irq = true; + return true; } - riscv_pmu_irq_mask = BIT(riscv_pmu_irq_num % BITS_PER_LONG); + return false; +} - if (!riscv_pmu_use_irq) +static int pmu_sbi_setup_irq(struct riscv_pmu *pmu) +{ + struct cpu_hw_events __percpu *hw_events = pmu->hw_events; + struct irq_domain *domain; + int ret; + + if (!pmu_sbi_select_irq()) return -EOPNOTSUPP; + riscv_pmu_irq_mask = BIT(riscv_pmu_irq_num % BITS_PER_LONG); + domain = irq_find_matching_fwnode(riscv_get_intc_hwnode(), DOMAIN_BUS_ANY); if (!domain) { pr_err("Failed to find INTC IRQ root domain\n"); - ret = -ENODEV; - goto err; + return -ENODEV; } riscv_pmu_irq = irq_create_mapping(domain, riscv_pmu_irq_num); if (!riscv_pmu_irq) { pr_err("Failed to map PMU interrupt for node\n"); - ret = -ENODEV; - goto err; + return -ENODEV; } - ret = request_percpu_irq(riscv_pmu_irq, pmu_sbi_ovf_handler, "riscv-pmu", hw_events); + ret = request_percpu_irq(riscv_pmu_irq, pmu_sbi_ovf_irq_handler, + "riscv-pmu", hw_events); if (ret) { pr_err("registering percpu irq failed [%d]\n", ret); irq_dispose_mapping(riscv_pmu_irq); riscv_pmu_irq = 0; - goto err; + return ret; } return 0; -err: - riscv_pmu_use_irq = false; - return ret; +} + +static int pmu_sbi_starting_cpu(unsigned int cpu, struct hlist_node *node) +{ + struct riscv_pmu *pmu = hlist_entry_safe(node, struct riscv_pmu, node); + struct cpu_hw_events *cpu_hw_evt = this_cpu_ptr(pmu->hw_events); + + /* + * We keep enabling userspace access to CYCLE, TIME and INSTRET via the + * legacy option but that will be removed in the future. + */ + if (sysctl_perf_user_access == SYSCTL_LEGACY) + csr_write(CSR_SCOUNTEREN, 0x7); + else + csr_write(CSR_SCOUNTEREN, 0x2); + + /* Stop all the counters so that they can be enabled from perf */ + pmu_sbi_stop_all(pmu); + + if (riscv_pmu_use_irq) { + cpu_hw_evt->irq = riscv_pmu_irq; + ALT_SBI_PMU_OVF_CLEAR_PENDING(riscv_pmu_irq_mask); + enable_percpu_irq(riscv_pmu_irq, IRQ_TYPE_NONE); + } + + if (sbi_pmu_snapshot_available()) + return pmu_sbi_snapshot_setup(pmu, cpu); + + return 0; +} + +static int pmu_sbi_dying_cpu(unsigned int cpu, struct hlist_node *node) +{ + if (riscv_pmu_use_irq) + disable_percpu_irq(riscv_pmu_irq); + + /* Disable all counters access for user mode now */ + csr_write(CSR_SCOUNTEREN, 0x0); + + if (sbi_pmu_snapshot_available()) + return pmu_sbi_snapshot_disable(); + + return 0; +} + +static int pmu_sbi_setup_irqs(struct riscv_pmu *pmu, struct platform_device *pdev) +{ + int irq_ret; + int sse_ret; + + /* Do not claim an IRQ route while a crash kernel leaves SSE state intact. */ + if (is_kdump_kernel() && riscv_sse_available()) { + pr_warn("PMU delivery unavailable with retained crash-kernel SSE state\n"); + return -EUCLEAN; + } + + sse_ret = pmu_sbi_setup_sse(pmu); + if (!sse_ret) { + riscv_pmu_use_irq = false; + return 0; + } + if (pmu_sbi_sse_state_retained(pmu)) { + pr_err("PMU-SSE setup failed with firmware state retained: %d\n", + sse_ret); + return -EUCLEAN; + } + /* Only an explicitly unsupported SSE path proves IRQ fallback is safe. */ + if (sse_ret != -EOPNOTSUPP) { + pr_err("PMU-SSE setup failed: %d\n", sse_ret); + return sse_ret; + } + + irq_ret = pmu_sbi_setup_irq(pmu); + if (!irq_ret) { + riscv_pmu_use_irq = true; + return 0; + } + return irq_ret; } #ifdef CONFIG_CPU_PM -static int riscv_pm_pmu_notify(struct notifier_block *b, unsigned long cmd, - void *v) +static int riscv_pm_pmu_update(struct riscv_pmu *rvpmu, unsigned long cmd) { - struct riscv_pmu *rvpmu = container_of(b, struct riscv_pmu, riscv_pm_nb); struct cpu_hw_events *cpuc = this_cpu_ptr(rvpmu->hw_events); bool enabled = !bitmap_empty(cpuc->used_hw_ctrs, RISCV_MAX_COUNTERS); struct perf_event *event; int idx; + if (cmd == CPU_PM_ENTER) + bitmap_zero(cpuc->pm_resume_hw_ctrs, RISCV_MAX_COUNTERS); + if (!enabled) return NOTIFY_OK; @@ -1264,6 +1559,8 @@ static int riscv_pm_pmu_notify(struct notifier_block *b, unsigned long cmd, switch (cmd) { case CPU_PM_ENTER: + if (!(event->hw.state & PERF_HES_STOPPED)) + set_bit(idx, cpuc->pm_resume_hw_ctrs); /* * Stop and update the counter */ @@ -1271,6 +1568,8 @@ static int riscv_pm_pmu_notify(struct notifier_block *b, unsigned long cmd, break; case CPU_PM_EXIT: case CPU_PM_ENTER_FAILED: + if (!test_and_clear_bit(idx, cpuc->pm_resume_hw_ctrs)) + break; /* * Restore and enable the counter. */ @@ -1284,9 +1583,59 @@ static int riscv_pm_pmu_notify(struct notifier_block *b, unsigned long cmd, return NOTIFY_OK; } +static int riscv_pm_pmu_notify(struct notifier_block *b, + unsigned long cmd, void *v) +{ + struct riscv_pmu *rvpmu = container_of(b, struct riscv_pmu, + riscv_pm_nb); + +#ifdef CONFIG_RISCV_PMU_SBI_SSE + struct cpu_hw_events *cpuc = this_cpu_ptr(rvpmu->hw_events); + int ret; + + if (!riscv_pmu_use_irq && READ_ONCE(rvpmu->sse_active)) { + switch (cmd) { + case CPU_PM_ENTER: + cpuc->pm_resume_sse = + sse_event_is_enabled_local(rvpmu->sse_evt); + if (cpuc->pm_resume_sse) { + ret = sse_event_disable_local(rvpmu->sse_evt); + if (ret) { + cpuc->pm_resume_sse = false; + pmu_sbi_fail_sse(rvpmu, "disable for CPU PM", + ret); + return notifier_from_errno(ret); + } + } + break; + case CPU_PM_EXIT: + case CPU_PM_ENTER_FAILED: + ret = riscv_pm_pmu_update(rvpmu, cmd); + if (!cpuc->pm_resume_sse) + return ret; + + cpuc->pm_resume_sse = false; + ret = sse_event_enable_local(rvpmu->sse_evt); + if (ret) { + pmu_sbi_fail_sse(rvpmu, "enable after CPU PM", ret); + return notifier_from_errno(ret); + } + return NOTIFY_OK; + default: + break; + } + } +#endif + + return riscv_pm_pmu_update(rvpmu, cmd); +} + static int riscv_pm_pmu_register(struct riscv_pmu *pmu) { pmu->riscv_pm_nb.notifier_call = riscv_pm_pmu_notify; + /* Keep PMU-SSE disabled until counters and userpage state are restored. */ + pmu->riscv_pm_nb.priority = riscv_pmu_use_irq ? 1 : -1; + return cpu_pm_register_notifier(&pmu->riscv_pm_nb); } @@ -1301,6 +1650,8 @@ static inline void riscv_pm_pmu_unregister(struct riscv_pmu *pmu) { } static void riscv_pmu_destroy(struct riscv_pmu *pmu) { + pmu_sbi_cleanup_sse(pmu); + if (sbi_v2_available) { if (sbi_pmu_snapshot_available()) { pmu_sbi_snapshot_disable(); @@ -1312,7 +1663,7 @@ static void riscv_pmu_destroy(struct riscv_pmu *pmu) cpuhp_state_remove_instance(CPUHP_AP_PERF_RISCV_STARTING, &pmu->node); } -static void pmu_sbi_event_init(struct perf_event *event) +static int pmu_sbi_event_init(struct perf_event *event) { /* * The permissions are set at event_init so that we do not depend @@ -1324,6 +1675,8 @@ static void pmu_sbi_event_init(struct perf_event *event) event->hw.flags |= PERF_EVENT_FLAG_USER_ACCESS; else event->hw.flags |= PERF_EVENT_FLAG_LEGACY; + + return 0; } static void pmu_sbi_event_mapped(struct perf_event *event, struct mm_struct *mm) @@ -1453,6 +1806,7 @@ static int pmu_sbi_device_probe(struct platform_device *pdev) /* cache all the information about counters now */ if (pmu_sbi_get_ctrinfo(num_counters, &cmask)) goto out_free; + pmu->cmask = cmask; ret = pmu_sbi_setup_irqs(pmu, pdev); if (ret < 0) { @@ -1462,9 +1816,15 @@ static int pmu_sbi_device_probe(struct platform_device *pdev) } irq_requested = (ret == 0); +#ifdef CONFIG_RISCV_PMU_SBI_SSE + if (pmu->sse_active) { + pmu->pmu.pmu_enable = pmu_sbi_sse_enable; + pmu->pmu.pmu_disable = pmu_sbi_sse_disable; + } +#endif + pmu->pmu.attr_groups = riscv_pmu_attr_groups; pmu->pmu.parent = &pdev->dev; - pmu->cmask = cmask; pmu->ctr_start = pmu_sbi_ctr_start; pmu->ctr_stop = pmu_sbi_ctr_stop; pmu->event_map = pmu_sbi_event_map; diff --git a/include/linux/perf/riscv_pmu.h b/include/linux/perf/riscv_pmu.h index f82a28040594bd..fb913ce7f65e5f 100644 --- a/include/linux/perf/riscv_pmu.h +++ b/include/linux/perf/riscv_pmu.h @@ -28,6 +28,8 @@ #define RISCV_PMU_CONFIG1_GUEST_EVENTS 0x1 +struct sse_event; + struct cpu_hw_events { /* currently enabled events */ int n_events; @@ -39,6 +41,18 @@ struct cpu_hw_events { DECLARE_BITMAP(used_hw_ctrs, RISCV_MAX_COUNTERS); /* currently enabled firmware counters */ DECLARE_BITMAP(used_fw_ctrs, RISCV_MAX_COUNTERS); +#ifdef CONFIG_RISCV_PMU_SBI_SSE + /* Keep counters stopped after an unrecoverable SSE transition failure. */ + bool sse_failed; +#endif +#ifdef CONFIG_CPU_PM + /* Counters stopped by CPU PM and still waiting to be restored. */ + DECLARE_BITMAP(pm_resume_hw_ctrs, RISCV_MAX_COUNTERS); +#ifdef CONFIG_RISCV_PMU_SBI_SSE + /* Restore the local PMU SSE event after counters and userpage state. */ + bool pm_resume_sse; +#endif +#endif /* The virtual address of the shared memory where counter snapshot will be taken */ void *snapshot_addr; /* The physical address of the shared memory where counter snapshot will be taken */ @@ -54,6 +68,10 @@ struct riscv_pmu { char *name; irqreturn_t (*handle_irq)(int irq_num, void *dev); +#ifdef CONFIG_RISCV_PMU_SBI_SSE + struct sse_event *sse_evt; + bool sse_active; +#endif unsigned long cmask; u64 (*ctr_read)(struct perf_event *event); @@ -63,7 +81,7 @@ struct riscv_pmu { void (*ctr_start)(struct perf_event *event, u64 init_val); void (*ctr_stop)(struct perf_event *event, unsigned long flag); int (*event_map)(struct perf_event *event, u64 *config); - void (*event_init)(struct perf_event *event); + int (*event_init)(struct perf_event *event); void (*event_mapped)(struct perf_event *event, struct mm_struct *mm); void (*event_unmapped)(struct perf_event *event, struct mm_struct *mm); uint8_t (*csr_index)(struct perf_event *event); diff --git a/include/linux/riscv_sbi_sse.h b/include/linux/riscv_sbi_sse.h index 774a782e556d2f..f6b3881c8884bb 100644 --- a/include/linux/riscv_sbi_sse.h +++ b/include/linux/riscv_sbi_sse.h @@ -42,6 +42,7 @@ int sse_event_enable(struct sse_event *sse_evt); int sse_event_disable(struct sse_event *sse_evt); /* Local events require the caller to remain on the current CPU. */ +bool sse_event_is_enabled_local(struct sse_event *sse_evt); int sse_event_enable_local(struct sse_event *sse_evt); int sse_event_disable_local(struct sse_event *sse_evt); @@ -76,6 +77,11 @@ static inline int sse_event_disable(struct sse_event *sse_evt) return -EOPNOTSUPP; } +static inline bool sse_event_is_enabled_local(struct sse_event *sse_evt) +{ + return false; +} + static inline int sse_event_enable_local(struct sse_event *sse_evt) { return -EOPNOTSUPP; From 442499878d89303338f07fda30862d07259df277 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20L=C3=A9ger?= Date: Fri, 11 Sep 2026 20:14:55 +0800 Subject: [PATCH 8/9] selftests/riscv: add SSE test module MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add an SSE selftest module and runner. Loading the module executes smoke tests for the SSE framework, and the runner reports any error emitted by the module. Add stress={0,1,2} modes to exercise repeated handler entry and completion, single and multiple read-only SBI calls from a handler, and self re-injection. Check the SBI return values so handler execution alone cannot produce a false pass, and avoid touching an unreserved PMU counter owned by another user. Harden the test against false failures and leaks by using TEST_PROGS for the runner, using phys_addr_t for SBI attribute buffers, adding timeouts to busy waits, pinning each priority chain with migrate_disable(), and holding the CPU read lock while a fast-test target is selected, injected, and completed. Unregister all registered events on error, check teardown failures, and let kthread_stop() drive monitor-thread exit. Track handler progress across fast, priority, and stress paths. If firmware reports injectable events but the test cannot acquire or handle any of them, report SKIP instead of silently passing a capability-only run. Signed-off-by: Clément Léger Co-developed-by: Himanshu Chauhan Signed-off-by: Himanshu Chauhan Co-developed-by: Zhanpeng Zhang Signed-off-by: Zhanpeng Zhang Signed-off-by: Linux RISC-V bot --- MAINTAINERS | 2 + tools/testing/selftests/riscv/Makefile | 2 +- tools/testing/selftests/riscv/sse/Makefile | 5 + .../selftests/riscv/sse/module/Makefile | 22 + .../riscv/sse/module/riscv_sse_test.c | 1154 +++++++++++++++++ .../selftests/riscv/sse/run_sse_test.sh | 59 + 6 files changed, 1243 insertions(+), 1 deletion(-) create mode 100644 tools/testing/selftests/riscv/sse/Makefile create mode 100644 tools/testing/selftests/riscv/sse/module/Makefile create mode 100644 tools/testing/selftests/riscv/sse/module/riscv_sse_test.c create mode 100644 tools/testing/selftests/riscv/sse/run_sse_test.sh diff --git a/MAINTAINERS b/MAINTAINERS index 91b4e32a1fb21e..ca27ca62f2988c 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -23318,6 +23318,7 @@ C: irc://irc.libera.chat/riscv P: Documentation/arch/riscv/patch-acceptance.rst T: git git://git.kernel.org/pub/scm/linux/kernel/git/riscv/linux.git F: arch/riscv/ +F: tools/testing/selftests/riscv/ N: riscv K: riscv @@ -23443,6 +23444,7 @@ F: arch/riscv/kernel/sbi_sse.c F: arch/riscv/kernel/sbi_sse_entry.S F: drivers/firmware/riscv/riscv_sbi_sse.c F: include/linux/riscv_sbi_sse.h +F: tools/testing/selftests/riscv/sse/ RISC-V TENSTORRENT SoC SUPPORT M: Drew Fustini diff --git a/tools/testing/selftests/riscv/Makefile b/tools/testing/selftests/riscv/Makefile index 5671b4405a1294..43c7c8f97676f8 100644 --- a/tools/testing/selftests/riscv/Makefile +++ b/tools/testing/selftests/riscv/Makefile @@ -5,7 +5,7 @@ ARCH ?= $(shell uname -m 2>/dev/null || echo not) ifneq (,$(filter $(ARCH),riscv)) -RISCV_SUBTARGETS ?= abi hwprobe mm sigreturn vector cfi +RISCV_SUBTARGETS ?= abi hwprobe mm sigreturn vector cfi sse else RISCV_SUBTARGETS := endif diff --git a/tools/testing/selftests/riscv/sse/Makefile b/tools/testing/selftests/riscv/sse/Makefile new file mode 100644 index 00000000000000..7e2677fdce09a6 --- /dev/null +++ b/tools/testing/selftests/riscv/sse/Makefile @@ -0,0 +1,5 @@ +TEST_GEN_MODS_DIR := module + +TEST_PROGS := run_sse_test.sh + +include ../../lib.mk diff --git a/tools/testing/selftests/riscv/sse/module/Makefile b/tools/testing/selftests/riscv/sse/module/Makefile new file mode 100644 index 00000000000000..eac4b1c6228bb6 --- /dev/null +++ b/tools/testing/selftests/riscv/sse/module/Makefile @@ -0,0 +1,22 @@ +ifneq ($(CONFIG_RISCV_SBI_SSE),) +obj-m += riscv_sse_test.o +endif + +ifndef KERNELRELEASE + +TESTMODS_DIR := $(realpath $(dir $(abspath $(lastword $(MAKEFILE_LIST))))) +KDIR ?= /lib/modules/$(shell uname -r)/build + +# Ensure that KDIR exists, otherwise skip the compilation +modules: +ifneq ("$(wildcard $(KDIR))", "") + $(Q)$(MAKE) -C $(KDIR) modules KBUILD_EXTMOD=$(TESTMODS_DIR) +endif + +# Ensure that KDIR exists, otherwise skip the clean target +clean: +ifneq ("$(wildcard $(KDIR))", "") + $(Q)$(MAKE) -C $(KDIR) clean KBUILD_EXTMOD=$(TESTMODS_DIR) +endif + +endif diff --git a/tools/testing/selftests/riscv/sse/module/riscv_sse_test.c b/tools/testing/selftests/riscv/sse/module/riscv_sse_test.c new file mode 100644 index 00000000000000..cc5c2e46f2fdb4 --- /dev/null +++ b/tools/testing/selftests/riscv/sse/module/riscv_sse_test.c @@ -0,0 +1,1154 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2025 Rivos Inc. + */ + +#define pr_fmt(fmt) "riscv_sse_test: " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#define RUN_LOOP_COUNT 1000 +#define SSE_FAILED_PREFIX "FAILED: " +#define SSE_SKIP_PREFIX "SKIP: " +#define STRESS_DURATION_MS 3000 +#define STRESS_INJECT_NS 10000 +#define STRESS_REINJECT_DEPTH 10 +#define sse_err(...) pr_err(SSE_FAILED_PREFIX __VA_ARGS__) +#define sse_skip(...) pr_info(SSE_SKIP_PREFIX __VA_ARGS__) + +enum sse_stress_mode { + SSE_STRESS_OFF, + SSE_STRESS_AFTER_SMOKE, + SSE_STRESS_ONLY, +}; + +static int stress; +module_param(stress, int, 0444); +MODULE_PARM_DESC(stress, "Stress mode: 0=off, 1=after smoke, 2=stress only"); + +static char *run_id = "unknown"; +module_param(run_id, charp, 0444); +MODULE_PARM_DESC(run_id, "Unique identifier used to delimit one test run"); + +/* Do not report PASS for a capability-only run that handled no event. */ +static atomic_t sse_test_handler_count = ATOMIC_INIT(0); +static bool sse_stress_event_can_inject; + +struct sse_event_desc { + u32 evt_id; + const char *name; + bool can_inject; +}; + +static struct sse_event_desc sse_event_descs[] = { + { + .evt_id = SBI_SSE_EVENT_LOCAL_HIGH_PRIO_RAS, + .name = "local_high_prio_ras", + }, + { + .evt_id = SBI_SSE_EVENT_LOCAL_DOUBLE_TRAP, + .name = "local_double_trap", + }, + { + .evt_id = SBI_SSE_EVENT_GLOBAL_HIGH_PRIO_RAS, + .name = "global_high_prio_ras", + }, + { + .evt_id = SBI_SSE_EVENT_LOCAL_PMU_OVERFLOW, + .name = "local_pmu_overflow", + }, + { + .evt_id = SBI_SSE_EVENT_LOCAL_LOW_PRIO_RAS, + .name = "local_low_prio_ras", + }, + { + .evt_id = SBI_SSE_EVENT_GLOBAL_LOW_PRIO_RAS, + .name = "global_low_prio_ras", + }, + { + .evt_id = SBI_SSE_EVENT_LOCAL_SOFTWARE_INJECTED, + .name = "local_software_injected", + }, + { + .evt_id = SBI_SSE_EVENT_GLOBAL_SOFTWARE_INJECTED, + .name = "global_software_injected", + } +}; + +static DEFINE_MUTEX(sse_test_cleanup_lock); +/* Firmware permits only one registration for each event ID. */ +static struct sse_event *sse_test_cleanup_events[ARRAY_SIZE(sse_event_descs)]; + +static void sse_test_cleanup_workfn(struct work_struct *work); +static DECLARE_DELAYED_WORK(sse_test_cleanup_work, sse_test_cleanup_workfn); + +static void sse_test_queue_cleanup(struct sse_event *event) +{ + int i, free_slot = -1; + + mutex_lock(&sse_test_cleanup_lock); + for (i = 0; i < ARRAY_SIZE(sse_test_cleanup_events); i++) { + if (sse_test_cleanup_events[i] == event) + goto out_schedule; + if (!sse_test_cleanup_events[i] && free_slot < 0) + free_slot = i; + } + + if (WARN_ON_ONCE(free_slot < 0)) + goto out_unlock; + + sse_test_cleanup_events[free_slot] = event; + +out_schedule: + mod_delayed_work(system_wq, &sse_test_cleanup_work, + msecs_to_jiffies(100)); +out_unlock: + mutex_unlock(&sse_test_cleanup_lock); +} + +static struct sse_event_desc *sse_get_evt_desc(u32 evt) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(sse_event_descs); i++) { + if (sse_event_descs[i].evt_id == evt) + return &sse_event_descs[i]; + } + + return NULL; +} + +static const char *sse_evt_name(u32 evt) +{ + struct sse_event_desc *desc = sse_get_evt_desc(evt); + + return desc ? desc->name : NULL; +} + +static bool sse_test_can_inject_event(u32 evt) +{ + struct sse_event_desc *desc = sse_get_evt_desc(evt); + + return desc ? desc->can_inject : false; +} + +/* + * Firmware can invoke the callback until unregister succeeds. Pin the module + * so its handler text cannot disappear first. + */ +static struct sse_event *sse_test_event_register(u32 evt, u32 priority, + sse_event_handler_fn *handler, + void *arg) +{ + struct sse_event *event; + + event = sse_event_register(evt, priority, handler, arg); + if (!IS_ERR(event)) + __module_get(THIS_MODULE); + + return event; +} + +static int sse_test_event_unregister(struct sse_event *event) +{ + int ret; + + ret = sse_event_unregister(event); + if (!ret) + module_put(THIS_MODULE); + else + sse_test_queue_cleanup(event); + + return ret; +} + +static void sse_test_cleanup_workfn(struct work_struct *work) +{ + bool retry = false; + int i, ret; + + mutex_lock(&sse_test_cleanup_lock); + for (i = 0; i < ARRAY_SIZE(sse_test_cleanup_events); i++) { + struct sse_event *event = sse_test_cleanup_events[i]; + + if (!event) + continue; + + ret = sse_event_disable(event); + if (!ret) + ret = sse_event_unregister(event); + if (ret) { + retry = true; + continue; + } + + sse_test_cleanup_events[i] = NULL; + module_put(THIS_MODULE); + } + mutex_unlock(&sse_test_cleanup_lock); + + if (retry) + mod_delayed_work(system_wq, &sse_test_cleanup_work, + msecs_to_jiffies(100)); +} + +static struct sbiret sbi_sse_ecall(int fid, unsigned long arg0, unsigned long arg1) +{ + return sbi_ecall(SBI_EXT_SSE, fid, arg0, arg1, 0, 0, 0, 0); +} + +static int sse_event_attr_read(u32 evt, unsigned long attr_id, + unsigned long *attr_buf) +{ + struct sbiret sret; + phys_addr_t phys; + + phys = virt_to_phys(attr_buf); + + sret = sbi_ecall(SBI_EXT_SSE, SBI_SSE_EVENT_ATTR_READ, evt, attr_id, 1, + (unsigned long)phys, 0, 0); + if (sret.error) + return sbi_err_map_linux_errno(sret.error); + + return 0; +} + +static int sse_event_attr_get(u32 evt, unsigned long attr_id, + unsigned long *val) +{ + unsigned long *attr_buf; + int ret; + + attr_buf = kmalloc_obj(*attr_buf, GFP_KERNEL); + if (!attr_buf) + return -ENOMEM; + + ret = sse_event_attr_read(evt, attr_id, attr_buf); + if (!ret) + *val = *attr_buf; + kfree(attr_buf); + + return ret; +} + +static int sse_test_signal(u32 evt, unsigned int cpu) +{ + unsigned long hart_id = cpuid_to_hartid_map(cpu); + struct sbiret ret; + + ret = sbi_sse_ecall(SBI_SSE_EVENT_INJECT, evt, hart_id); + if (ret.error) { + sse_err("Failed to signal event %x, error %ld\n", evt, ret.error); + return sbi_err_map_linux_errno(ret.error); + } + + return 0; +} + +static int sse_test_wait_not_running(u32 evt) +{ + unsigned long timeout = jiffies + HZ; + unsigned long status; + int ret; + + do { + ret = sse_event_attr_get(evt, SBI_SSE_ATTR_STATUS, &status); + if (ret) { + sse_err("Failed to get status for evt %x, error %d\n", evt, ret); + return ret; + } + status &= SBI_SSE_ATTR_STATUS_STATE_MASK; + cpu_relax(); + } while (status == SBI_SSE_STATE_RUNNING && time_before(jiffies, timeout)); + + if (status == SBI_SSE_STATE_RUNNING) { + sse_err("Timed out waiting for event %x to leave RUNNING state\n", evt); + return -ETIMEDOUT; + } + + return 0; +} + +struct sse_test_wait_status { + u32 evt; + unsigned long *attr_buf; + unsigned long status; + int ret; +}; + +static void sse_test_read_status_local(void *info) +{ + struct sse_test_wait_status *wait = info; + + wait->ret = sse_event_attr_read(wait->evt, SBI_SSE_ATTR_STATUS, + wait->attr_buf); + if (!wait->ret) + wait->status = *wait->attr_buf & SBI_SSE_ATTR_STATUS_STATE_MASK; +} + +static int sse_test_wait_not_running_on_cpu(u32 evt, unsigned int cpu) +{ + struct sse_test_wait_status wait = { .evt = evt }; + unsigned long timeout; + int ret = 0; + + if (sse_event_is_global(evt)) + return sse_test_wait_not_running(evt); + + wait.attr_buf = kmalloc_obj(*wait.attr_buf, GFP_KERNEL); + if (!wait.attr_buf) + return -ENOMEM; + + timeout = jiffies + HZ; + do { + ret = smp_call_function_single(cpu, sse_test_read_status_local, + &wait, true); + if (ret || wait.ret) { + ret = ret ?: wait.ret; + break; + } + if (wait.status != SBI_SSE_STATE_RUNNING) + break; + usleep_range(100, 200); + } while (time_before(jiffies, timeout)); + + if (!ret && wait.status == SBI_SSE_STATE_RUNNING) { + sse_err("Timed out waiting for event %x on CPU %u\n", evt, cpu); + ret = -ETIMEDOUT; + } + + kfree(wait.attr_buf); + + return ret; +} + +static int sse_test_inject_event(struct sse_event *event, u32 evt, unsigned int cpu) +{ + int res; + + if (sse_event_is_global(evt)) { + /* + * Due to the fact the completion might happen faster than + * the call to SBI_SSE_COMPLETE in the handler, if the event was + * running on another CPU, we need to wait for the event status + * to be !RUNNING. + */ + res = sse_test_wait_not_running(evt); + if (res) + return res; + + res = sse_event_set_target_cpu(event, cpu); + if (res) { + sse_err("Failed to set cpu for evt %x, error %d\n", evt, res); + return res; + } + } + + return sse_test_signal(evt, cpu); +} + +struct fast_test_arg { + u32 evt; + int cpu; + bool args_ready; + bool completion; +}; + +/* A failed unregister may leave firmware holding this handler argument. */ +static struct fast_test_arg fast_test_arg; + +static int sse_test_handler(u32 evt, void *arg, struct pt_regs *regs) +{ + int ret = 0; + struct fast_test_arg *targ = arg; + u32 test_evt; + int cpu; + + atomic_inc(&sse_test_handler_count); + + /* Pairs with the argument publication in sse_run_fast_test_cpu(). */ + if (!smp_load_acquire(&targ->args_ready)) { + sse_err("Received SSE event %x before its test arguments were published\n", + evt); + ret = -EINVAL; + goto complete; + } + + test_evt = READ_ONCE(targ->evt); + cpu = READ_ONCE(targ->cpu); + + if (evt != test_evt) { + sse_err("Received SSE event id %x instead of %x\n", test_evt, evt); + ret = -EINVAL; + } + + if (!sse_event_is_global(evt) && cpu != smp_processor_id()) { + sse_err("Received SSE event %d on CPU %d instead of %d\n", evt, smp_processor_id(), + cpu); + ret = -EINVAL; + } + +complete: + WRITE_ONCE(targ->args_ready, false); + /* Publish handler-side checks before waking the waiting CPU. */ + smp_store_release(&targ->completion, true); + + return ret; +} + +static int sse_run_fast_test_cpu(struct fast_test_arg *test_arg, + struct sse_event *event, u32 evt, int cpu) +{ + unsigned long timeout; + int ret; + + WRITE_ONCE(test_arg->completion, false); + WRITE_ONCE(test_arg->args_ready, false); + WRITE_ONCE(test_arg->evt, evt); + WRITE_ONCE(test_arg->cpu, cpu); + /* Publish all arguments before firmware can inject on another hart. */ + smp_store_release(&test_arg->args_ready, true); + + ret = sse_test_inject_event(event, evt, cpu); + if (ret) { + sse_err("event %s injection failed, err %d\n", + sse_evt_name(evt), ret); + return ret; + } + + timeout = jiffies + HZ / 100; + /* We can not use since they are not NMI safe */ + /* Pairs with the handler's completion publication. */ + while (!smp_load_acquire(&test_arg->completion) && + time_before(jiffies, timeout)) + cpu_relax(); + /* Acquire the handler's checks even if the loop observed a timeout. */ + if (!smp_load_acquire(&test_arg->completion)) { + sse_err("Failed to wait for event %s completion on CPU %d\n", + sse_evt_name(evt), cpu); + return -ETIMEDOUT; + } + + return sse_test_wait_not_running_on_cpu(evt, cpu); +} + +static void sse_run_fast_test(struct fast_test_arg *test_arg, + struct sse_event *event, u32 evt) +{ + int cpu; + + if (sse_event_is_global(evt)) { + /* Keep the selected target online through injection and completion. */ + cpu_hotplug_disable(); + for_each_online_cpu(cpu) { + if (sse_run_fast_test_cpu(test_arg, event, evt, cpu)) + break; + } + cpu_hotplug_enable(); + return; + } + + guard(cpus_read_lock)(); + for_each_online_cpu(cpu) { + if (sse_run_fast_test_cpu(test_arg, event, evt, cpu)) + return; + } +} + +static void sse_test_injection_fast(void) +{ + int i, ret = 0, j; + u32 evt; + struct sse_event *event; + + pr_info("Starting SSE test (fast)\n"); + + for (i = 0; i < ARRAY_SIZE(sse_event_descs); i++) { + evt = sse_event_descs[i].evt_id; + WRITE_ONCE(fast_test_arg.evt, evt); + WRITE_ONCE(fast_test_arg.cpu, -1); + WRITE_ONCE(fast_test_arg.args_ready, false); + WRITE_ONCE(fast_test_arg.completion, false); + + if (!sse_event_descs[i].can_inject) + continue; + + event = sse_test_event_register(evt, 0, sse_test_handler, + (void *)&fast_test_arg); + if (IS_ERR(event)) { + if (PTR_ERR(event) == -EEXIST) { + pr_info("Event %s already registered, skipping\n", + sse_evt_name(evt)); + continue; + } + sse_err("Failed to register event %s, err %ld\n", sse_evt_name(evt), + PTR_ERR(event)); + continue; + } + + ret = sse_event_enable(event); + if (ret) { + sse_err("Failed to enable event %s, err %d\n", sse_evt_name(evt), ret); + goto err_disable; + } + + pr_info("Starting testing event %s\n", sse_evt_name(evt)); + + for (j = 0; j < RUN_LOOP_COUNT; j++) + sse_run_fast_test(&fast_test_arg, event, evt); + pr_info("Finished testing event %s\n", sse_evt_name(evt)); + +err_disable: + ret = sse_event_disable(event); + if (ret) + sse_err("Failed to disable event %s, err %d\n", + sse_evt_name(evt), ret); + ret = sse_test_event_unregister(event); + if (ret) { + sse_err("Failed to unregister event %s, err %d\n", + sse_evt_name(evt), ret); + return; + } + } + pr_info("Finished SSE test (fast)\n"); +} + +struct priority_test_arg { + unsigned long evt; + struct sse_event *event; + bool called; + bool enable_attempted; + u32 prio; + struct priority_test_arg *next_evt_arg; + void (*check_func)(struct priority_test_arg *arg); +}; + +/* A failed unregister may leave firmware holding these handler arguments. */ +static struct priority_test_arg default_hi_prio_args[] = { + { .evt = SBI_SSE_EVENT_GLOBAL_SOFTWARE_INJECTED }, + { .evt = SBI_SSE_EVENT_LOCAL_SOFTWARE_INJECTED }, + { .evt = SBI_SSE_EVENT_GLOBAL_LOW_PRIO_RAS }, + { .evt = SBI_SSE_EVENT_LOCAL_LOW_PRIO_RAS }, + { .evt = SBI_SSE_EVENT_LOCAL_PMU_OVERFLOW }, + { .evt = SBI_SSE_EVENT_GLOBAL_HIGH_PRIO_RAS }, + { .evt = SBI_SSE_EVENT_LOCAL_DOUBLE_TRAP }, + { .evt = SBI_SSE_EVENT_LOCAL_HIGH_PRIO_RAS }, +}; + +static struct priority_test_arg default_low_prio_args[] = { + { .evt = SBI_SSE_EVENT_LOCAL_HIGH_PRIO_RAS }, + { .evt = SBI_SSE_EVENT_LOCAL_DOUBLE_TRAP }, + { .evt = SBI_SSE_EVENT_GLOBAL_HIGH_PRIO_RAS }, + { .evt = SBI_SSE_EVENT_LOCAL_PMU_OVERFLOW }, + { .evt = SBI_SSE_EVENT_LOCAL_LOW_PRIO_RAS }, + { .evt = SBI_SSE_EVENT_GLOBAL_LOW_PRIO_RAS }, + { .evt = SBI_SSE_EVENT_LOCAL_SOFTWARE_INJECTED }, + { .evt = SBI_SSE_EVENT_GLOBAL_SOFTWARE_INJECTED }, +}; + +static struct priority_test_arg set_prio_args[] = { + { .evt = SBI_SSE_EVENT_GLOBAL_SOFTWARE_INJECTED, .prio = 5 }, + { .evt = SBI_SSE_EVENT_LOCAL_SOFTWARE_INJECTED, .prio = 10 }, + { .evt = SBI_SSE_EVENT_GLOBAL_LOW_PRIO_RAS, .prio = 15 }, + { .evt = SBI_SSE_EVENT_LOCAL_LOW_PRIO_RAS, .prio = 20 }, + { .evt = SBI_SSE_EVENT_LOCAL_PMU_OVERFLOW, .prio = 25 }, + { .evt = SBI_SSE_EVENT_GLOBAL_HIGH_PRIO_RAS, .prio = 30 }, + { .evt = SBI_SSE_EVENT_LOCAL_DOUBLE_TRAP, .prio = 35 }, + { .evt = SBI_SSE_EVENT_LOCAL_HIGH_PRIO_RAS, .prio = 40 }, +}; + +static struct priority_test_arg same_prio_args[] = { + { .evt = SBI_SSE_EVENT_LOCAL_PMU_OVERFLOW, .prio = 0 }, + { .evt = SBI_SSE_EVENT_LOCAL_HIGH_PRIO_RAS, .prio = 10 }, + { .evt = SBI_SSE_EVENT_LOCAL_SOFTWARE_INJECTED, .prio = 10 }, + { .evt = SBI_SSE_EVENT_GLOBAL_SOFTWARE_INJECTED, .prio = 10 }, + { .evt = SBI_SSE_EVENT_GLOBAL_HIGH_PRIO_RAS, .prio = 20 }, +}; + +static int sse_hi_priority_test_handler(u32 evt, void *arg, + struct pt_regs *regs) +{ + struct priority_test_arg *targ = arg; + struct priority_test_arg *next = READ_ONCE(targ->next_evt_arg); + + atomic_inc(&sse_test_handler_count); + WRITE_ONCE(targ->called, 1); + + if (next) { + sse_test_signal(next->evt, smp_processor_id()); + if (!READ_ONCE(next->called)) { + sse_err("Higher priority event %s was not handled %s\n", + sse_evt_name(next->evt), sse_evt_name(evt)); + } + } + + return 0; +} + +static int sse_low_priority_test_handler(u32 evt, void *arg, struct pt_regs *regs) +{ + struct priority_test_arg *targ = arg; + struct priority_test_arg *next = READ_ONCE(targ->next_evt_arg); + + atomic_inc(&sse_test_handler_count); + WRITE_ONCE(targ->called, 1); + + if (next) { + sse_test_signal(next->evt, smp_processor_id()); + if (READ_ONCE(next->called)) { + sse_err("Lower priority event %s was handle before %s\n", + sse_evt_name(next->evt), sse_evt_name(evt)); + } + } + + return 0; +} + +static void sse_test_injection_priority_arg(struct priority_test_arg *args, unsigned int args_size, + sse_event_handler_fn handler, const char *test_name) +{ + unsigned int i; + unsigned long timeout; + int ret; + int target_cpu; + struct sse_event *event; + struct priority_test_arg *arg, *first_arg = NULL, *prev_arg = NULL; + + pr_info("Starting SSE priority test (%s)\n", test_name); + /* Keep the complete priority chain on one CPU. */ + migrate_disable(); + target_cpu = smp_processor_id(); + + for (i = 0; i < args_size; i++) { + arg = &args[i]; + + if (!sse_test_can_inject_event(arg->evt)) + continue; + + WRITE_ONCE(arg->called, false); + WRITE_ONCE(arg->next_evt_arg, NULL); + WRITE_ONCE(arg->event, NULL); + WRITE_ONCE(arg->enable_attempted, false); + + event = sse_test_event_register(arg->evt, arg->prio, handler, + (void *)arg); + if (IS_ERR(event)) { + if (PTR_ERR(event) == -EEXIST) { + pr_info("Event %s already registered, skipping\n", + sse_evt_name(arg->evt)); + continue; + } + sse_err("Failed to register event %s, err %ld\n", sse_evt_name(arg->evt), + PTR_ERR(event)); + goto release_events; + } + arg->event = event; + + if (sse_event_is_global(arg->evt)) { + /* Keep the chain on one stable CPU. */ + ret = sse_event_set_target_cpu(event, target_cpu); + if (ret) { + sse_err("Failed to set event %s target CPU, err %d\n", + sse_evt_name(arg->evt), ret); + goto release_events; + } + } + + WRITE_ONCE(arg->enable_attempted, true); + ret = sse_event_enable(event); + if (ret) { + sse_err("Failed to enable event %s, err %d\n", sse_evt_name(arg->evt), ret); + goto release_events; + } + + if (prev_arg) + WRITE_ONCE(prev_arg->next_evt_arg, arg); + + prev_arg = arg; + + if (!first_arg) + first_arg = arg; + } + + if (!first_arg) { + pr_info("No injectable event available for %s priority test\n", + test_name); + goto out; + } + + /* Inject first event, handler should trigger the others in chain. */ + ret = sse_test_inject_event(first_arg->event, first_arg->evt, target_cpu); + if (ret) { + sse_err("SSE event %s injection failed\n", sse_evt_name(first_arg->evt)); + goto release_events; + } + + /* Lower-priority events run after the handler that injected them completes. */ + arg = first_arg; + while (arg) { + timeout = jiffies + HZ; + while (!READ_ONCE(arg->called) && time_before(jiffies, timeout)) + cpu_relax(); + + if (!READ_ONCE(arg->called)) { + sse_err("Event %s handler was not called\n", + sse_evt_name(arg->evt)); + ret = -EINVAL; + } + + event = arg->event; + arg = READ_ONCE(arg->next_evt_arg); + } + +release_events: + + for (i = 0; i < args_size; i++) { + arg = &args[i]; + event = arg->event; + if (!event) + continue; + + ret = sse_test_wait_not_running_on_cpu(arg->evt, target_cpu); + if (ret) + sse_err("Event %s did not complete, err %d\n", + sse_evt_name(arg->evt), ret); + + if (arg->enable_attempted) { + ret = sse_event_disable(event); + if (ret) { + sse_err("Failed to disable event %s, err %d\n", + sse_evt_name(arg->evt), ret); + sse_test_queue_cleanup(event); + WRITE_ONCE(arg->event, NULL); + WRITE_ONCE(arg->enable_attempted, false); + continue; + } + } + + ret = sse_test_event_unregister(event); + if (ret) { + sse_err("Failed to unregister event %s, err %d\n", + sse_evt_name(arg->evt), ret); + continue; + } + + WRITE_ONCE(arg->event, NULL); + WRITE_ONCE(arg->enable_attempted, false); + } + + pr_info("Finished SSE priority test (%s)\n", test_name); +out: + migrate_enable(); +} + +static void sse_test_injection_priority(void) +{ + sse_test_injection_priority_arg(default_hi_prio_args, ARRAY_SIZE(default_hi_prio_args), + sse_hi_priority_test_handler, "high"); + + sse_test_injection_priority_arg(default_low_prio_args, ARRAY_SIZE(default_low_prio_args), + sse_low_priority_test_handler, "low"); + + sse_test_injection_priority_arg(set_prio_args, ARRAY_SIZE(set_prio_args), + sse_low_priority_test_handler, "set"); + + sse_test_injection_priority_arg(same_prio_args, ARRAY_SIZE(same_prio_args), + sse_low_priority_test_handler, "same_prio_args"); +} + +static int sse_get_inject_status(u32 evt, bool *can_inject) +{ + int ret; + unsigned long val; + + /* Check if injection is supported */ + ret = sse_event_attr_get(evt, SBI_SSE_ATTR_STATUS, &val); + if (ret == sbi_err_map_linux_errno(SBI_ERR_NOT_SUPPORTED) || + ret == sbi_err_map_linux_errno(SBI_ERR_INVALID_PARAM)) { + *can_inject = false; + return 0; + } + if (ret) + return ret; + + *can_inject = !!(val & BIT(SBI_SSE_ATTR_STATUS_INJECT_OFFSET)); + + return 0; +} + +static int sse_init_events(void) +{ + int i, injectable = 0, ret; + + for (i = 0; i < ARRAY_SIZE(sse_event_descs); i++) { + struct sse_event_desc *desc = &sse_event_descs[i]; + + ret = sse_get_inject_status(desc->evt_id, &desc->can_inject); + if (ret) { + sse_err("Failed to read injection status for %s, err %d\n", + desc->name, ret); + return ret; + } + + if (desc->can_inject) + injectable++; + else + pr_info("Can not inject event %s, tests using this event will be skipped\n", + desc->name); + + if (desc->evt_id == SBI_SSE_EVENT_LOCAL_SOFTWARE_INJECTED) + sse_stress_event_can_inject = desc->can_inject; + } + + return injectable; +} + +struct stress_test_ctx { + struct sse_event *event; + struct hrtimer timer; + struct hrtimer stop_timer; + struct task_struct *monitor_task; + wait_queue_head_t wait_q; + atomic_t inject_count; + atomic_t handler_count; + atomic_t handler_errors; + u32 evt_id; + int layer; + bool running; + bool test_done; +}; + +static struct stress_test_ctx stress_ctx; +static DEFINE_PER_CPU(int, stress_reinject_cpu_depth); + +static int stress_handler_empty(u32 evt, void *arg, struct pt_regs *regs) +{ + struct stress_test_ctx *ctx = arg; + + atomic_inc(&sse_test_handler_count); + atomic_inc(&ctx->handler_count); + + return 0; +} + +static int stress_handler_ecall(u32 evt, void *arg, struct pt_regs *regs) +{ + struct stress_test_ctx *ctx = arg; + struct sbiret ret; + + ret = sbi_ecall(SBI_EXT_BASE, SBI_EXT_BASE_GET_SPEC_VERSION, + 0, 0, 0, 0, 0, 0); + if (ret.error) + atomic_inc(&ctx->handler_errors); + atomic_inc(&sse_test_handler_count); + atomic_inc(&ctx->handler_count); + + return 0; +} + +static int stress_handler_multi_ecall(u32 evt, void *arg, struct pt_regs *regs) +{ + struct stress_test_ctx *ctx = arg; + struct sbiret ret; + + ret = sbi_ecall(SBI_EXT_BASE, SBI_EXT_BASE_GET_SPEC_VERSION, + 0, 0, 0, 0, 0, 0); + if (ret.error) + atomic_inc(&ctx->handler_errors); + ret = sbi_ecall(SBI_EXT_BASE, SBI_EXT_BASE_GET_IMP_ID, + 0, 0, 0, 0, 0, 0); + if (ret.error) + atomic_inc(&ctx->handler_errors); + ret = sbi_ecall(SBI_EXT_BASE, SBI_EXT_BASE_GET_IMP_VERSION, + 0, 0, 0, 0, 0, 0); + if (ret.error) + atomic_inc(&ctx->handler_errors); + atomic_inc(&sse_test_handler_count); + atomic_inc(&ctx->handler_count); + + return 0; +} + +static int stress_handler_reinject(u32 evt, void *arg, struct pt_regs *regs) +{ + struct stress_test_ctx *ctx = arg; + int *depth = this_cpu_ptr(&stress_reinject_cpu_depth); + + (*depth)++; + if (*depth < STRESS_REINJECT_DEPTH) + sse_test_signal(evt, smp_processor_id()); + else + *depth = 0; + + atomic_inc(&sse_test_handler_count); + atomic_inc(&ctx->handler_count); + + return 0; +} + +static sse_event_handler_fn *stress_handlers[] = { + stress_handler_empty, + stress_handler_ecall, + stress_handler_multi_ecall, + stress_handler_reinject, +}; + +static const char * const stress_layer_names[] = { + "empty handler", + "single SBI ecall in handler", + "multiple SBI ecalls in handler", + "self re-inject", +}; + +static enum hrtimer_restart stress_timer_callback(struct hrtimer *timer) +{ + struct stress_test_ctx *ctx = container_of(timer, struct stress_test_ctx, timer); + + if (!READ_ONCE(ctx->running)) + return HRTIMER_NORESTART; + + if (!sse_test_signal(ctx->evt_id, smp_processor_id())) + atomic_inc(&ctx->inject_count); + hrtimer_forward_now(timer, ns_to_ktime(STRESS_INJECT_NS)); + + return HRTIMER_RESTART; +} + +static enum hrtimer_restart stress_stop_timer_callback(struct hrtimer *timer) +{ + struct stress_test_ctx *ctx; + + ctx = container_of(timer, struct stress_test_ctx, stop_timer); + WRITE_ONCE(ctx->test_done, true); + wake_up(&ctx->wait_q); + + return HRTIMER_NORESTART; +} + +static int stress_monitor_thread(void *data) +{ + struct stress_test_ctx *ctx = data; + unsigned long last_inject = 0, last_handler = 0; + + while (!kthread_should_stop()) { + unsigned long inject = atomic_read(&ctx->inject_count); + unsigned long handler = atomic_read(&ctx->handler_count); + + pr_info("stress layer %d: inject=%lu (+%lu), handler=%lu (+%lu)\n", + ctx->layer, inject, inject - last_inject, + handler, handler - last_handler); + + last_inject = inject; + last_handler = handler; + + schedule_timeout_interruptible(HZ); + } + + return 0; +} + +static int sse_stress_test_layer(int layer) +{ + struct sse_event *event; + int inject_count, handler_count; + int ret, target_cpu, unregister_ret; + + if (layer < 0 || layer >= ARRAY_SIZE(stress_handlers)) + return -EINVAL; + + pr_info("Starting SSE stress layer %d (%s)\n", + layer, stress_layer_names[layer]); + + memset(&stress_ctx, 0, sizeof(stress_ctx)); + stress_ctx.evt_id = SBI_SSE_EVENT_LOCAL_SOFTWARE_INJECTED; + stress_ctx.layer = layer; + WRITE_ONCE(stress_ctx.running, true); + atomic_set(&stress_ctx.inject_count, 0); + atomic_set(&stress_ctx.handler_count, 0); + atomic_set(&stress_ctx.handler_errors, 0); + init_waitqueue_head(&stress_ctx.wait_q); + + event = sse_test_event_register(stress_ctx.evt_id, 0, + stress_handlers[layer], &stress_ctx); + if (IS_ERR(event)) { + sse_err("Failed to register stress event, err %ld\n", + PTR_ERR(event)); + return PTR_ERR(event); + } + + stress_ctx.event = event; + + ret = sse_event_enable(event); + if (ret) { + sse_err("Failed to enable stress event, err %d\n", ret); + goto err_disable; + } + + stress_ctx.monitor_task = kthread_run(stress_monitor_thread, + &stress_ctx, "sse_stress_mon"); + if (IS_ERR(stress_ctx.monitor_task)) { + ret = PTR_ERR(stress_ctx.monitor_task); + sse_err("Failed to create stress monitor thread, err %d\n", ret); + goto err_disable; + } + + /* Keep the pinned timer and its local event on the selected CPU. */ + cpus_read_lock(); + migrate_disable(); + target_cpu = smp_processor_id(); + hrtimer_setup(&stress_ctx.timer, stress_timer_callback, + CLOCK_MONOTONIC, HRTIMER_MODE_PINNED); + hrtimer_start(&stress_ctx.timer, ns_to_ktime(STRESS_INJECT_NS), + HRTIMER_MODE_REL_PINNED); + migrate_enable(); + + hrtimer_setup(&stress_ctx.stop_timer, stress_stop_timer_callback, + CLOCK_MONOTONIC, HRTIMER_MODE_REL); + hrtimer_start(&stress_ctx.stop_timer, ms_to_ktime(STRESS_DURATION_MS), + HRTIMER_MODE_REL); + + wait_event(stress_ctx.wait_q, READ_ONCE(stress_ctx.test_done)); + + WRITE_ONCE(stress_ctx.running, false); + hrtimer_cancel(&stress_ctx.timer); + hrtimer_cancel(&stress_ctx.stop_timer); + kthread_stop(stress_ctx.monitor_task); + + pr_info("Finished SSE stress layer %d (%s): inject=%d, handler=%d\n", + layer, stress_layer_names[layer], + atomic_read(&stress_ctx.inject_count), + atomic_read(&stress_ctx.handler_count)); + + inject_count = atomic_read(&stress_ctx.inject_count); + handler_count = atomic_read(&stress_ctx.handler_count); + if (!inject_count || !handler_count) { + sse_err("Stress layer %d made no progress: inject=%d, handler=%d\n", + layer, inject_count, handler_count); + ret = -EIO; + } + if (atomic_read(&stress_ctx.handler_errors)) { + sse_err("Stress layer %d observed %d SBI call errors\n", layer, + atomic_read(&stress_ctx.handler_errors)); + ret = -EIO; + } + if (sse_test_wait_not_running_on_cpu(stress_ctx.evt_id, target_cpu)) { + sse_err("Stress event did not complete on CPU %d\n", target_cpu); + if (!ret) + ret = -ETIMEDOUT; + } + cpus_read_unlock(); + +err_disable: + if (sse_event_disable(event)) { + sse_err("Failed to disable stress event\n"); + if (!ret) + ret = -EIO; + } + unregister_ret = sse_test_event_unregister(event); + if (unregister_ret) { + sse_err("Failed to unregister stress event\n"); + if (!ret) + ret = unregister_ret; + } + stress_ctx.event = NULL; + + return ret; +} + +static void sse_stress_test_all_layers(void) +{ + int i, ret; + + pr_info("Starting SSE stress tests: duration=%d ms, interval=%d ns\n", + STRESS_DURATION_MS, STRESS_INJECT_NS); + + for (i = 0; i < ARRAY_SIZE(stress_handlers); i++) { + ret = sse_stress_test_layer(i); + if (ret) { + sse_err("Stress layer %d failed, err %d\n", i, ret); + break; + } + + msleep(100); + } + + pr_info("Finished SSE stress tests\n"); +} + +static int __init sse_test_init(void) +{ + int ret; + + pr_info("RUN %s BEGIN\n", run_id); + atomic_set(&sse_test_handler_count, 0); + + if (stress < SSE_STRESS_OFF || stress > SSE_STRESS_ONLY) { + sse_err("Invalid stress mode %d\n", stress); + pr_info("RUN %s END\n", run_id); + return -EINVAL; + } + + ret = sse_init_events(); + if (ret < 0) { + pr_info("RUN %s END\n", run_id); + return ret; + } + if (!ret) { + sse_skip("No injectable SSE event is available\n"); + pr_info("RUN %s END\n", run_id); + return 0; + } + if (stress == SSE_STRESS_ONLY && !sse_stress_event_can_inject) { + sse_skip("Local software-injected event is unavailable for stress\n"); + pr_info("RUN %s END\n", run_id); + return 0; + } + + if (stress != SSE_STRESS_ONLY) { + sse_test_injection_fast(); + sse_test_injection_priority(); + } + + if (stress == SSE_STRESS_AFTER_SMOKE && !sse_stress_event_can_inject) + sse_skip("Local software-injected event is unavailable for stress\n"); + else if (stress != SSE_STRESS_OFF) + sse_stress_test_all_layers(); + if (!atomic_read(&sse_test_handler_count)) + sse_skip("No SSE event was handled\n"); + + pr_info("RUN %s END\n", run_id); + + return 0; +} + +static void __exit sse_test_exit(void) +{ + cancel_delayed_work_sync(&sse_test_cleanup_work); +} + +module_init(sse_test_init); +module_exit(sse_test_exit); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Clément Léger "); +MODULE_DESCRIPTION("Test module for SSE"); diff --git a/tools/testing/selftests/riscv/sse/run_sse_test.sh b/tools/testing/selftests/riscv/sse/run_sse_test.sh new file mode 100644 index 00000000000000..e70a2fd14b05e7 --- /dev/null +++ b/tools/testing/selftests/riscv/sse/run_sse_test.sh @@ -0,0 +1,59 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 +# +# Copyright (C) 2025 Rivos Inc. + +MODULE_NAME=riscv_sse_test +DRIVER="./module/${MODULE_NAME}.ko" +ksft_skip=4 + +check_test_requirements() +{ + uid=$(id -u) + if [ $uid -ne 0 ]; then + echo "$0: Must be run as root" + exit $ksft_skip + fi + + if ! which insmod > /dev/null 2>&1; then + echo "$0: You need insmod installed" + exit $ksft_skip + fi + + if [ ! -f "$DRIVER" ]; then + echo "$0: SSE is disabled or ${MODULE_NAME} is not built" + exit $ksft_skip + fi +} + +check_test_requirements +run_id="$$-$(date +%s)" + +if ! insmod "$DRIVER" run_id="$run_id" "$@" > /dev/null 2>&1; then + echo "${MODULE_NAME}: failed to load, please check dmesg" + exit 1 +fi + +if ! rmmod "$MODULE_NAME"; then + echo "${MODULE_NAME}: failed to unload, please check dmesg" + exit 1 +fi + +run_log=$(dmesg | sed -n \ + "/${MODULE_NAME}: RUN ${run_id} BEGIN/,/${MODULE_NAME}: RUN ${run_id} END/p") +if [ -z "$run_log" ]; then + echo "${MODULE_NAME}: unable to find log for run ${run_id}" + exit 1 +fi + +if echo "$run_log" | grep -q "${MODULE_NAME}: FAILED:"; then + echo "${MODULE_NAME} failed, please check dmesg" + exit 1 +fi + +if echo "$run_log" | grep -q "${MODULE_NAME}: SKIP:"; then + echo "${MODULE_NAME}: no injectable SSE event" + exit $ksft_skip +fi + +exit 0 From fe081f5ee2e44c4035f277dd20e8d0d5464fae8a Mon Sep 17 00:00:00 2001 From: Zhanpeng Zhang Date: Fri, 11 Sep 2026 20:14:56 +0800 Subject: [PATCH 9/9] selftests/riscv: add perf user-stack SSE copy regression test On RISC-V, PMU overflow interrupts can be delivered through the SBI Supervisor Software Events (SSE) mechanism. A perf event that samples the raw user stack (PERF_SAMPLE_STACK_USER, as perf record --call-graph dwarf does) then copies a chunk of the interrupted task's user stack from an NMI-like context. If that copy is allowed to take a nested page fault it can corrupt the interrupted task's kernel state and hang or crash the machine under load; this is what riscv_perf_out_copy_user() and the nofault page-fault change fix. The existing SSE selftest module exercises the framework (register, enable, inject, complete, priorities, stress) but never drives the perf user-stack copy that motivated the no-fault path. Add a userspace test that closes that gap: - Open a sampling hardware PMU event with PERF_SAMPLE_STACK_USER over a deep, partially non-resident user stack, drain the ring buffer, and verify every SAMPLE record is well formed and never reports more dumped bytes than were requested. This checks that a non-resident page truncates the dump cleanly instead of faulting or overrunning. - Drive a multi-CPU unix-socket + deep-recursion workload under high-frequency DWARF sampling; the pass criterion is simply that the machine survives, since the original bug took it down. The test reports SKIP when hardware PMU sampling is unavailable or perf_event_paranoid forbids it, so it is safe to run unprivileged or in constrained environments. It is placed under the RISC-V SSE selftests because SSE delivery is the RISC-V-specific condition it protects, and is wired into the sse subtarget Makefile alongside the module runner. Signed-off-by: Zhanpeng Zhang Signed-off-by: Linux RISC-V bot --- tools/testing/selftests/riscv/sse/Makefile | 5 + .../selftests/riscv/sse/sse_perf_ustack.c | 564 ++++++++++++++++++ 2 files changed, 569 insertions(+) create mode 100644 tools/testing/selftests/riscv/sse/sse_perf_ustack.c diff --git a/tools/testing/selftests/riscv/sse/Makefile b/tools/testing/selftests/riscv/sse/Makefile index 7e2677fdce09a6..646b418de9ffb9 100644 --- a/tools/testing/selftests/riscv/sse/Makefile +++ b/tools/testing/selftests/riscv/sse/Makefile @@ -1,5 +1,10 @@ +CFLAGS += -I$(top_srcdir)/tools/testing/selftests +LDLIBS += -lpthread + TEST_GEN_MODS_DIR := module +TEST_GEN_PROGS := sse_perf_ustack + TEST_PROGS := run_sse_test.sh include ../../lib.mk diff --git a/tools/testing/selftests/riscv/sse/sse_perf_ustack.c b/tools/testing/selftests/riscv/sse/sse_perf_ustack.c new file mode 100644 index 00000000000000..9535b6d7ba4c18 --- /dev/null +++ b/tools/testing/selftests/riscv/sse/sse_perf_ustack.c @@ -0,0 +1,564 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Regression test for the RISC-V perf user-stack copy taken in SSE + * (NMI-like) context. + * + * On RISC-V, PMU overflow interrupts can be delivered through the SBI + * Supervisor Software Events (SSE) mechanism. A perf event that samples the + * raw user stack (PERF_SAMPLE_STACK_USER, as perf record --call-graph dwarf + * does) then copies a large chunk of the interrupted task's user stack from + * that context. If that copy is allowed to take a nested page fault it can + * corrupt the interrupted task's kernel state and hang or crash the machine + * under load. + * + * This test exercises that exact path: + * - It opens a sampling hardware PMU event with PERF_SAMPLE_STACK_USER. + * - It samples a child running on a controlled user stack followed by an + * inaccessible page, so the copy must truncate at that page boundary. + * - It checks that every user-stack sample record is well formed and that + * the dumped size never exceeds the requested size (i.e. the copy stops + * cleanly rather than faulting on). + * - It then drives a multi-threaded unix-socket + deep-recursion workload + * under high-frequency per-CPU sampling and requires every active sampler + * to make progress without taking the machine down. + * + * The test is architecture independent in what it drives; it is placed under + * the RISC-V SSE selftests because SSE delivery is the RISC-V-specific + * condition it is meant to protect. + */ +#define _GNU_SOURCE + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "../../kselftest.h" + +#ifndef noinline +#define noinline __attribute__((noinline)) +#endif + +#define STACK_DUMP_SIZE 8192 /* 8 KiB, 8-byte aligned */ +#define RB_DATA_PAGES 64 /* power of two */ +#define SELF_SAMPLE_FREQ 4000 +#define STRESS_SAMPLE_FREQ 5000 +#define STRESS_SECONDS 5 +#define RECURSE_DEPTH 512 +#define TRUNCATION_RUN_MS 250 + +static long page_size; + +static int perf_event_open(struct perf_event_attr *attr, pid_t pid, int cpu, + int group_fd, unsigned long flags) +{ + return syscall(__NR_perf_event_open, attr, pid, cpu, group_fd, flags); +} + +/* Prevent the compiler from optimizing away a stack buffer. */ +static void keep_alive(void *p) +{ + __asm__ __volatile__("" : : "r"(p) : "memory"); +} + +/* + * Consume a deep user stack and keep it live, so a raw user-stack sample has + * many pages to copy. Returns a value derived from the stack so the compiler + * cannot elide the frames. + */ +static noinline unsigned long burn_stack(int depth, unsigned long *sink) +{ + unsigned long frame[32]; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(frame); i++) + frame[i] = (unsigned long)depth * i + *sink; + + if (depth > 0) + frame[depth & 31] += burn_stack(depth - 1, sink); + + for (i = 0; i < ARRAY_SIZE(frame); i++) + *sink += frame[i]; + + keep_alive(frame); + return *sink; +} + +static struct perf_event_attr sampling_attr(unsigned long freq) +{ + struct perf_event_attr attr = { + .type = PERF_TYPE_HARDWARE, + .size = sizeof(attr), + .config = PERF_COUNT_HW_INSTRUCTIONS, + .sample_type = PERF_SAMPLE_STACK_USER, + .sample_stack_user = STACK_DUMP_SIZE, + .freq = 1, + .sample_freq = freq, + .disabled = 1, + .exclude_kernel = 1, + .exclude_hv = 1, + }; + + return attr; +} + +static bool open_skip_reason(int err, const char **why) +{ + switch (err) { + case EACCES: + case EPERM: + *why = "insufficient privilege for PMU sampling (perf_event_paranoid)"; + return true; + case ENOENT: + case ENODEV: + case EOPNOTSUPP: + *why = "hardware PMU sampling event not available"; + return true; + default: + return false; + } +} + +static bool pmu_sse_route_testable(const char **why) +{ + char *line = NULL; + size_t line_size = 0; + FILE *interrupts; + bool testable = true; + + interrupts = fopen("/proc/interrupts", "re"); + if (!interrupts) { + *why = "cannot inspect the active PMU delivery route"; + return false; + } + + /* The SBI PMU driver registers this name only for ordinary IRQ delivery. */ + while (getline(&line, &line_size, interrupts) >= 0) { + if (strstr(line, "riscv-pmu")) { + *why = "ordinary RISC-V PMU IRQ delivery is active"; + testable = false; + break; + } + } + + free(line); + fclose(interrupts); + return testable; +} + +static bool ring_copy_from(void *dst, const void *rb, size_t rb_bytes, + uint64_t pos, size_t size) +{ + size_t offset = pos % rb_bytes; + size_t first; + + if (size > rb_bytes) + return false; + + first = size < rb_bytes - offset ? size : rb_bytes - offset; + memcpy(dst, (const char *)rb + offset, first); + if (first != size) + memcpy((char *)dst + first, rb, size - first); + + return true; +} + +static int truncation_child(void *arg) +{ + int ready_fd = (intptr_t)arg; + char ready = 1; + + if (write(ready_fd, &ready, sizeof(ready)) != 1) + return 1; + + for (;;) + __asm__ __volatile__("" : : : "memory"); +} + +static pid_t start_truncation_child(void **stack_mapping) +{ + struct pollfd pfd = { .events = POLLIN }; + size_t mapping_size = 2 * page_size; + char ready; + void *stack; + pid_t pid; + int pipefd[2]; + int saved_errno; + + stack = mmap(NULL, mapping_size, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (stack == MAP_FAILED) + return -1; + if (mprotect(stack, page_size, PROT_READ | PROT_WRITE)) + goto err_unmap; + if (pipe(pipefd)) + goto err_unmap; + + /* clone() starts the child below the inaccessible second page. */ + pid = clone(truncation_child, (char *)stack + page_size, SIGCHLD, + (void *)(intptr_t)pipefd[1]); + if (pid < 0) + goto err_pipe; + + close(pipefd[1]); + pfd.fd = pipefd[0]; + if (poll(&pfd, 1, 1000) != 1 || + read(pipefd[0], &ready, sizeof(ready)) != sizeof(ready)) { + saved_errno = ETIMEDOUT; + kill(pid, SIGKILL); + waitpid(pid, NULL, 0); + close(pipefd[0]); + errno = saved_errno; + goto err_unmap; + } + close(pipefd[0]); + + *stack_mapping = stack; + return pid; + +err_pipe: + saved_errno = errno; + close(pipefd[0]); + close(pipefd[1]); + errno = saved_errno; +err_unmap: + saved_errno = errno; + munmap(stack, mapping_size); + errno = saved_errno; + return -1; +} + +static void stop_truncation_child(pid_t pid, void *stack_mapping) +{ + kill(pid, SIGKILL); + while (waitpid(pid, NULL, 0) < 0 && errno == EINTR) + ; + munmap(stack_mapping, 2 * page_size); +} + +/* + * Subtest 1: sample a child whose stack is followed by an inaccessible page. + * Every record must be well formed and at least one stack copy must truncate + * at the controlled page boundary rather than fault or overrun. + */ +static void test_ustack_records_wellformed(void) +{ + struct perf_event_attr attr = sampling_attr(SELF_SAMPLE_FREQ); + size_t rb_bytes = (size_t)RB_DATA_PAGES * page_size; + struct perf_event_mmap_page *meta; + unsigned long samples = 0, truncated = 0; + void *child_stack; + const char *why; + void *rb; + pid_t child; + int fd; + + child = start_truncation_child(&child_stack); + if (child < 0) { + ksft_test_result_fail("ustack records: create guarded stack child: %s\n", + strerror(errno)); + return; + } + + fd = perf_event_open(&attr, child, -1, -1, PERF_FLAG_FD_CLOEXEC); + if (fd < 0) { + if (open_skip_reason(errno, &why)) + ksft_test_result_skip("ustack records: %s\n", why); + else + ksft_test_result_fail("ustack records: perf_event_open: %s\n", + strerror(errno)); + goto out_child; + } + + meta = mmap(NULL, page_size + rb_bytes, PROT_READ | PROT_WRITE, + MAP_SHARED, fd, 0); + if (meta == MAP_FAILED) { + ksft_test_result_fail("ustack records: mmap ring buffer: %s\n", + strerror(errno)); + close(fd); + goto out_child; + } + rb = (char *)meta + page_size; + + ioctl(fd, PERF_EVENT_IOC_RESET, 0); + ioctl(fd, PERF_EVENT_IOC_ENABLE, 0); + usleep(TRUNCATION_RUN_MS * 1000); + ioctl(fd, PERF_EVENT_IOC_DISABLE, 0); + + /* Drain the ring buffer and validate every SAMPLE record. */ + { + uint64_t head = __atomic_load_n(&meta->data_head, __ATOMIC_ACQUIRE); + uint64_t tail = meta->data_tail; + bool ok = true; + + if (head < tail || head - tail > rb_bytes) + ok = false; + + while (ok && tail < head) { + struct perf_event_header hdr; + uint64_t available = head - tail; + + if (available < sizeof(hdr) || + !ring_copy_from(&hdr, rb, rb_bytes, tail, sizeof(hdr)) || + hdr.size < sizeof(hdr) || hdr.size > available || + hdr.size > rb_bytes) { + ok = false; + break; + } + + if (hdr.type == PERF_RECORD_SAMPLE) { + uint64_t dump_size, dyn_size; + size_t cursor = sizeof(hdr); + + if (sizeof(dump_size) > hdr.size - cursor || + !ring_copy_from(&dump_size, rb, rb_bytes, + tail + cursor, sizeof(dump_size)) || + dump_size > STACK_DUMP_SIZE) { + ok = false; + break; + } + cursor += sizeof(dump_size); + samples++; + if (dump_size) { + /* data blob then trailing dynamic size */ + if (dump_size > hdr.size - cursor) { + ok = false; + break; + } + cursor += dump_size; + if (sizeof(dyn_size) > hdr.size - cursor || + !ring_copy_from(&dyn_size, rb, rb_bytes, + tail + cursor, + sizeof(dyn_size))) { + ok = false; + break; + } + if (dyn_size > dump_size) { + ok = false; + break; + } + if (dyn_size < dump_size) + truncated++; + } + } + tail += hdr.size; + } + __atomic_store_n(&meta->data_tail, head, __ATOMIC_RELEASE); + + if (!ok) + ksft_test_result_fail("ustack records: malformed sample record\n"); + else if (samples == 0) + ksft_test_result_skip("ustack records: no samples collected\n"); + else if (truncated == 0) + ksft_test_result_fail("ustack records: no guarded-stack truncation\n"); + else + ksft_test_result_pass("ustack records: %lu samples, %lu truncated\n", + samples, truncated); + } + + munmap(meta, page_size + rb_bytes); + close(fd); +out_child: + stop_truncation_child(child, child_stack); +} + +/* ---- Subtest 2: multi-threaded per-CPU sampling stress ---- */ + +struct stress_thread { + pthread_t tid; + int cpu; + int *stop; + int fd; + void *rb; + size_t rb_bytes; +}; + +static void *stress_worker(void *arg) +{ + struct stress_thread *st = arg; + unsigned long sink = 1; + int sv[2]; + char buf[64]; + + if (socketpair(AF_UNIX, SOCK_STREAM, 0, sv) == 0) { + while (!__atomic_load_n(st->stop, __ATOMIC_RELAXED)) { + /* unix-socket ping-pong: takes the socket locks the + * original bug corrupted, while sampling nests. + */ + if (write(sv[0], buf, sizeof(buf)) > 0) + (void)read(sv[1], buf, sizeof(buf)); + burn_stack(RECURSE_DEPTH, &sink); + /* Periodically consume the ring buffer so sampling + * keeps delivering rather than filling up and stopping. + */ + if (st->rb) { + struct perf_event_mmap_page *m = st->rb; + uint64_t h = __atomic_load_n(&m->data_head, + __ATOMIC_ACQUIRE); + __atomic_store_n(&m->data_tail, h, + __ATOMIC_RELEASE); + } + } + close(sv[0]); + close(sv[1]); + } + + return (void *)sink; +} + +static void test_sse_stress_no_crash(void) +{ + struct perf_event_attr attr = sampling_attr(STRESS_SAMPLE_FREQ); + size_t rb_bytes = (size_t)RB_DATA_PAGES * page_size; + struct stress_thread *threads; + cpu_set_t available; + long progressed = 0; + int stop = 0; + const char *why = NULL; + long started = 0; + long nproc; + long slot; + int cpu; + + if (sched_getaffinity(0, sizeof(available), &available)) { + ksft_test_result_fail("sse stress: sched_getaffinity: %s\n", + strerror(errno)); + return; + } + nproc = CPU_COUNT(&available); + if (nproc < 1) { + ksft_test_result_skip("sse stress: no available CPUs\n"); + return; + } + + threads = calloc(nproc, sizeof(*threads)); + if (!threads) { + ksft_test_result_fail("sse stress: out of memory\n"); + return; + } + + slot = 0; + for (cpu = 0; cpu < CPU_SETSIZE; cpu++) { + struct stress_thread *st; + pthread_attr_t thread_attr; + cpu_set_t set; + void *map; + int ret; + + if (!CPU_ISSET(cpu, &available)) + continue; + st = &threads[slot++]; + + st->fd = -1; + st->cpu = cpu; + st->stop = &stop; + st->rb_bytes = rb_bytes; + st->fd = perf_event_open(&attr, -1, cpu, -1, + PERF_FLAG_FD_CLOEXEC); + if (st->fd < 0) { + if (!started && open_skip_reason(errno, &why)) + break; + continue; + } + + map = mmap(NULL, page_size + rb_bytes, PROT_READ | PROT_WRITE, + MAP_SHARED, st->fd, 0); + if (map == MAP_FAILED) { + close(st->fd); + st->fd = -1; + continue; + } + st->rb = map; + + CPU_ZERO(&set); + CPU_SET(cpu, &set); + pthread_attr_init(&thread_attr); + ret = pthread_attr_setaffinity_np(&thread_attr, sizeof(set), &set); + if (!ret) + ret = pthread_create(&st->tid, &thread_attr, + stress_worker, st); + pthread_attr_destroy(&thread_attr); + + if (ret) { + munmap(st->rb, page_size + rb_bytes); + close(st->fd); + st->rb = NULL; + st->fd = -1; + continue; + } + + ioctl(st->fd, PERF_EVENT_IOC_RESET, 0); + ioctl(st->fd, PERF_EVENT_IOC_ENABLE, 0); + started++; + } + + if (started == 0) { + free(threads); + if (why) + ksft_test_result_skip("sse stress: %s\n", why); + else + ksft_test_result_skip("sse stress: could not start any sampler\n"); + return; + } + + sleep(STRESS_SECONDS); + __atomic_store_n(&stop, 1, __ATOMIC_RELAXED); + + for (slot = 0; slot < nproc; slot++) { + struct stress_thread *st = &threads[slot]; + struct perf_event_mmap_page *meta; + + if (st->fd < 0) + continue; + pthread_join(st->tid, NULL); + ioctl(st->fd, PERF_EVENT_IOC_DISABLE, 0); + meta = st->rb; + if (__atomic_load_n(&meta->data_head, __ATOMIC_ACQUIRE)) + progressed++; + munmap(st->rb, page_size + rb_bytes); + close(st->fd); + } + + free(threads); + if (progressed != started) + ksft_test_result_fail("sse stress: %ld/%ld samplers made progress\n", + progressed, started); + else + ksft_test_result_pass("sse stress: %ld samplers x %ds made progress\n", + started, STRESS_SECONDS); +} + +int main(void) +{ + const char *why; + + page_size = sysconf(_SC_PAGESIZE); + + ksft_print_header(); + ksft_set_plan(2); + if (!pmu_sse_route_testable(&why)) { + ksft_test_result_skip("ustack records: %s\n", why); + ksft_test_result_skip("sse stress: %s\n", why); + ksft_finished(); + } + + test_ustack_records_wellformed(); + test_sse_stress_no_crash(); + + ksft_finished(); +}