diff --git a/Documentation/arch/riscv/index.rst b/Documentation/arch/riscv/index.rst index ac535c52d509c7..5cb909c83a882c 100644 --- a/Documentation/arch/riscv/index.rst +++ b/Documentation/arch/riscv/index.rst @@ -11,6 +11,7 @@ RISC-V architecture vm-layout hwprobe patch-acceptance + pmu-sse uabi vector cmodx diff --git a/Documentation/arch/riscv/pmu-sse.rst b/Documentation/arch/riscv/pmu-sse.rst new file mode 100644 index 00000000000000..b7458a9d0116c1 --- /dev/null +++ b/Documentation/arch/riscv/pmu-sse.rst @@ -0,0 +1,55 @@ +.. SPDX-License-Identifier: GPL-2.0 + +======================================== +RISC-V PMU overflow delivery through SSE +======================================== + +When ``CONFIG_RISCV_PMU_SBI_SSE`` is enabled and firmware provides the local +PMU overflow event, the RISC-V SBI PMU driver uses Supervisor Software Events +(SSE) to deliver counter overflows. + +Delivery selection +================== + +The delivery mechanism is selected once while the PMU device is probed. The +driver first tries to register and enable the local PMU overflow SSE event. It +tries the ordinary PMU interrupt only when the SSE extension or the local PMU +event is explicitly unsupported. It does not change the delivery mechanism +after the PMU has been registered. + +Other SSE setup errors do not prove that firmware released the overflow route, +so the driver does not enable the ordinary PMU interrupt. The PMU remains +available for counting but not sampling. If setup retained an SSE event, the +PMU keeps quiescing that event around perf scheduling changes without rearming +it. This preserves callback ownership without creating two possible delivery +mechanisms for the same hardware overflow. + +Interrupted context +=================== + +SSE enters Linux through a supervisor handler context constructed by firmware. +The entry contract preserves the interrupted GPRs except ``a6`` and ``a7``, +which Linux reads from the interrupted-register event attributes. Linux then +reconstructs the interrupted ``pt_regs`` and publishes it while dispatching the +event. + +Perf uses that context for register samples and callchains. Kernel stack +walking verifies that the interrupted frame belongs to the current task stack +before dereferencing it. For sensitive entry paths and IRQ stacks, Linux +records the interrupted PC without walking a stack whose bounds cannot be +proved. User callchains use the existing no-fault user unwinder. A DWARF +raw user-stack copy from an SSE handler must not take a page fault, so it +walks the current task's page tables with fast-only GUP, copies each resident +page through its kernel mapping, and stops at the first non-resident page, +preserving perf's truncated-user-stack semantics. + +CPU power management +==================== + +The PMU and SSE CPU power-management callbacks are ordered according to the +selected delivery mechanism. With SSE delivery, SSE events are masked before +the lower-priority PMU callback disables the local PMU event and stops the +counters on entry. On exit, the hart is unmasked while the local PMU event is +still disabled. The PMU callback then restores the counters and enables the +event, so an unmask failure leaves the counters stopped. The ordinary +interrupt path retains the existing PMU ordering. diff --git a/MAINTAINERS b/MAINTAINERS index a830d3b252e2d9..ca27ca62f2988c 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -23318,6 +23318,7 @@ C: irc://irc.libera.chat/riscv P: Documentation/arch/riscv/patch-acceptance.rst T: git git://git.kernel.org/pub/scm/linux/kernel/git/riscv/linux.git F: arch/riscv/ +F: tools/testing/selftests/riscv/ N: riscv K: riscv @@ -23330,6 +23331,13 @@ T: git git://git.kernel.org/pub/scm/linux/kernel/git/iommu/linux.git F: Documentation/devicetree/bindings/iommu/riscv,iommu.yaml F: drivers/iommu/riscv/ +RISC-V FIRMWARE DRIVERS +M: Conor Dooley +L: linux-riscv@lists.infradead.org +S: Maintained +T: git git://git.kernel.org/pub/scm/linux/kernel/git/conor/linux.git +F: drivers/firmware/riscv/ + RISC-V MICROCHIP SUPPORT M: Conor Dooley M: Daire McNamara @@ -23385,6 +23393,7 @@ M: Atish Patra R: Anup Patel L: linux-riscv@lists.infradead.org S: Supported +F: Documentation/arch/riscv/pmu-sse.rst F: drivers/perf/riscv_pmu.c F: drivers/perf/riscv_pmu_legacy.c F: drivers/perf/riscv_pmu_sbi.c @@ -23424,6 +23433,19 @@ F: arch/riscv/boot/dts/spacemit/ N: spacemit K: spacemit +RISC-V SUPERVISOR SOFTWARE EVENTS +M: Zhanpeng Zhang +M: Himanshu Chauhan +R: Yunhui Cui +L: linux-riscv@lists.infradead.org +S: Maintained +F: arch/riscv/include/asm/sse.h +F: arch/riscv/kernel/sbi_sse.c +F: arch/riscv/kernel/sbi_sse_entry.S +F: drivers/firmware/riscv/riscv_sbi_sse.c +F: include/linux/riscv_sbi_sse.h +F: tools/testing/selftests/riscv/sse/ + RISC-V TENSTORRENT SoC SUPPORT M: Drew Fustini M: Joel Stanley diff --git a/arch/riscv/include/asm/asm.h b/arch/riscv/include/asm/asm.h index b8bf842d4c1367..e1196caea02dcb 100644 --- a/arch/riscv/include/asm/asm.h +++ b/arch/riscv/include/asm/asm.h @@ -91,16 +91,24 @@ .endm #ifdef CONFIG_SMP -.macro asm_per_cpu dst sym tmp - lw \tmp, TASK_TI_CPU_NUM(tp) - slli \tmp, \tmp, RISCV_LGPTR +.macro asm_per_cpu_with_cpu dst sym tmp cpu + slli \tmp, \cpu, RISCV_LGPTR la \dst, __per_cpu_offset add \dst, \dst, \tmp REG_L \tmp, 0(\dst) la \dst, \sym add \dst, \dst, \tmp .endm + +.macro asm_per_cpu dst sym tmp + lw \tmp, TASK_TI_CPU_NUM(tp) + asm_per_cpu_with_cpu \dst \sym \tmp \tmp +.endm #else /* CONFIG_SMP */ +.macro asm_per_cpu_with_cpu dst sym tmp cpu + la \dst, \sym +.endm + .macro asm_per_cpu dst sym tmp la \dst, \sym .endm diff --git a/arch/riscv/include/asm/perf_event.h b/arch/riscv/include/asm/perf_event.h index bcc928fd3785c1..ddc40479475195 100644 --- a/arch/riscv/include/asm/perf_event.h +++ b/arch/riscv/include/asm/perf_event.h @@ -18,6 +18,16 @@ (regs)->sp = current_stack_pointer; \ (regs)->status = SR_PP; \ } + +#ifdef CONFIG_RISCV_SBI_SSE +/* + * Raw user-stack sampling can run in the NMI-like SSE context. Route it + * through an implementation that does not fault on a non-resident page. + */ +unsigned long riscv_perf_out_copy_user(void *dst, const void *src, + unsigned long n); +#define arch_perf_out_copy_user riscv_perf_out_copy_user +#endif #endif #endif /* _ASM_RISCV_PERF_EVENT_H */ diff --git a/arch/riscv/include/asm/sbi.h b/arch/riscv/include/asm/sbi.h index 5725e0ca4dda3e..4e54d79ba543a9 100644 --- a/arch/riscv/include/asm/sbi.h +++ b/arch/riscv/include/asm/sbi.h @@ -38,6 +38,7 @@ enum sbi_ext_id { SBI_EXT_FWFT = 0x46574654, SBI_EXT_MPXY = 0x4D505859, SBI_EXT_DBTR = 0x44425452, + SBI_EXT_SSE = 0x535345, /* Experimentals extensions must lie within this range */ SBI_EXT_EXPERIMENTAL_START = 0x08000000, @@ -506,6 +507,68 @@ enum sbi_mpxy_rpmi_attribute_id { #define SBI_MPXY_CHAN_CAP_SEND_WITHOUT_RESP BIT(4) #define SBI_MPXY_CHAN_CAP_GET_NOTIFICATIONS BIT(5) +enum sbi_ext_sse_fid { + SBI_SSE_EVENT_ATTR_READ = 0, + SBI_SSE_EVENT_ATTR_WRITE, + SBI_SSE_EVENT_REGISTER, + SBI_SSE_EVENT_UNREGISTER, + SBI_SSE_EVENT_ENABLE, + SBI_SSE_EVENT_DISABLE, + SBI_SSE_EVENT_COMPLETE, + SBI_SSE_EVENT_INJECT, + SBI_SSE_HART_UNMASK, + SBI_SSE_HART_MASK, +}; + +enum sbi_sse_state { + SBI_SSE_STATE_UNUSED = 0, + SBI_SSE_STATE_REGISTERED = 1, + SBI_SSE_STATE_ENABLED = 2, + SBI_SSE_STATE_RUNNING = 3, +}; + +/* SBI SSE Event Attributes. */ +enum sbi_sse_attr_id { + SBI_SSE_ATTR_STATUS = 0x00000000, + SBI_SSE_ATTR_PRIO = 0x00000001, + SBI_SSE_ATTR_CONFIG = 0x00000002, + SBI_SSE_ATTR_PREFERRED_HART = 0x00000003, + SBI_SSE_ATTR_ENTRY_PC = 0x00000004, + SBI_SSE_ATTR_ENTRY_ARG = 0x00000005, + SBI_SSE_ATTR_INTERRUPTED_SEPC = 0x00000006, + SBI_SSE_ATTR_INTERRUPTED_FLAGS = 0x00000007, + SBI_SSE_ATTR_INTERRUPTED_A6 = 0x00000008, + SBI_SSE_ATTR_INTERRUPTED_A7 = 0x00000009, + + SBI_SSE_ATTR_MAX = 0x0000000A +}; + +#define SBI_SSE_ATTR_STATUS_STATE_OFFSET 0 +#define SBI_SSE_ATTR_STATUS_STATE_MASK 0x3 +#define SBI_SSE_ATTR_STATUS_PENDING_OFFSET 2 +#define SBI_SSE_ATTR_STATUS_INJECT_OFFSET 3 + +#define SBI_SSE_ATTR_CONFIG_ONESHOT BIT(0) + +#define SBI_SSE_ATTR_INTERRUPTED_FLAGS_SSTATUS_SPP BIT(0) +#define SBI_SSE_ATTR_INTERRUPTED_FLAGS_SSTATUS_SPIE BIT(1) +#define SBI_SSE_ATTR_INTERRUPTED_FLAGS_HSTATUS_SPV BIT(2) +#define SBI_SSE_ATTR_INTERRUPTED_FLAGS_HSTATUS_SPVP BIT(3) +#define SBI_SSE_ATTR_INTERRUPTED_FLAGS_SSTATUS_SPELP BIT(4) +#define SBI_SSE_ATTR_INTERRUPTED_FLAGS_SSTATUS_SDT BIT(5) + +#define SBI_SSE_EVENT_LOCAL_HIGH_PRIO_RAS 0x00000000 +#define SBI_SSE_EVENT_LOCAL_DOUBLE_TRAP 0x00000001 +#define SBI_SSE_EVENT_GLOBAL_HIGH_PRIO_RAS 0x00008000 +#define SBI_SSE_EVENT_LOCAL_PMU_OVERFLOW 0x00010000 +#define SBI_SSE_EVENT_LOCAL_LOW_PRIO_RAS 0x00100000 +#define SBI_SSE_EVENT_GLOBAL_LOW_PRIO_RAS 0x00108000 +#define SBI_SSE_EVENT_LOCAL_SOFTWARE_INJECTED 0xffff0000 +#define SBI_SSE_EVENT_GLOBAL_SOFTWARE_INJECTED 0xffff8000 + +#define SBI_SSE_EVENT_PLATFORM BIT(14) +#define SBI_SSE_EVENT_GLOBAL BIT(15) + /* SBI debug triggers function IDs */ enum sbi_ext_dbtr_fid { SBI_EXT_DBTR_NUM_TRIGGERS = 0, diff --git a/arch/riscv/include/asm/scs.h b/arch/riscv/include/asm/scs.h index 023a412fe38d37..0d70a35bc01ab8 100644 --- a/arch/riscv/include/asm/scs.h +++ b/arch/riscv/include/asm/scs.h @@ -17,6 +17,11 @@ load_per_cpu gp, irq_shadow_call_stack_ptr, \tmp .endm +/* Load the per-CPU IRQ shadow call stack to gp. */ +.macro scs_load_sse_stack reg_evt + REG_L gp, SSE_REG_EVT_SHADOW_STACK(\reg_evt) +.endm + /* Load task_scs_sp(current) to gp. */ .macro scs_load_current REG_L gp, TASK_TI_SCS_SP(tp) @@ -40,6 +45,8 @@ .endm .macro scs_load_irq_stack tmp .endm +.macro scs_load_sse_stack reg_evt +.endm .macro scs_load_current .endm .macro scs_load_current_if_task_changed prev diff --git a/arch/riscv/include/asm/sse.h b/arch/riscv/include/asm/sse.h new file mode 100644 index 00000000000000..cbd8618c6e00e3 --- /dev/null +++ b/arch/riscv/include/asm/sse.h @@ -0,0 +1,82 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2024 Rivos Inc. + */ +#ifndef __ASM_SSE_H +#define __ASM_SSE_H + +#include +#include + +#include + +static inline bool riscv_sse_available(void) +{ +#ifdef CONFIG_RISCV_SBI + return sbi_probe_extension(SBI_EXT_SSE) > 0; +#else + return false; +#endif +} + +static inline void riscv_sse_mask_current_hart(void) +{ +#ifdef CONFIG_RISCV_SBI + struct sbiret ret; + + if (!riscv_sse_available()) + return; + + ret = sbi_ecall(SBI_EXT_SSE, SBI_SSE_HART_MASK, 0, 0, 0, 0, 0, 0); + if (ret.error && ret.error != SBI_ERR_ALREADY_STOPPED) + pr_emerg("SSE hart mask failed: %ld\n", ret.error); +#endif +} + +#ifdef CONFIG_RISCV_SBI_SSE + +struct sse_event_interrupted_state { + unsigned long a6; + unsigned long a7; +}; + +struct sse_event_arch_data { + void *stack; + void *shadow_stack; + unsigned long tmp; + struct sse_event_interrupted_state *interrupted; + phys_addr_t interrupted_phys; + u32 evt_id; + unsigned long hart_id; + unsigned int cpu_id; +}; + +struct riscv_sse_interrupted_context { + struct pt_regs *regs; + unsigned long hstatus; +}; + +static inline bool sse_event_is_global(u32 evt) +{ + return !!(evt & SBI_SSE_EVENT_GLOBAL); +} + +void arch_sse_event_update_cpu(struct sse_event_arch_data *arch_evt, int cpu); +int arch_sse_init_event(struct sse_event_arch_data *arch_evt, u32 evt_id, + int cpu); +void arch_sse_free_event(struct sse_event_arch_data *arch_evt); +int arch_sse_register_event(struct sse_event_arch_data *arch_evt); +void arch_sse_init_cpu(void); + +void sse_handle_event(struct sse_event_arch_data *arch_evt, + struct pt_regs *regs); +asmlinkage void handle_sse(void); +asmlinkage void noinstr do_sse(struct sse_event_arch_data *arch_evt, + struct pt_regs *regs, unsigned long hstatus); + +const struct riscv_sse_interrupted_context * +riscv_sse_get_interrupted_context(void); + +#endif + +#endif diff --git a/arch/riscv/include/asm/thread_info.h b/arch/riscv/include/asm/thread_info.h index 55019fdfa9ecaa..d14b45610c73a2 100644 --- a/arch/riscv/include/asm/thread_info.h +++ b/arch/riscv/include/asm/thread_info.h @@ -36,6 +36,7 @@ #define OVERFLOW_STACK_SIZE SZ_4K #define IRQ_STACK_SIZE THREAD_SIZE +#define SSE_STACK_SIZE THREAD_SIZE #ifndef __ASSEMBLER__ diff --git a/arch/riscv/kernel/Makefile b/arch/riscv/kernel/Makefile index ebe1c3588177b4..1e34f87e97c09a 100644 --- a/arch/riscv/kernel/Makefile +++ b/arch/riscv/kernel/Makefile @@ -101,6 +101,7 @@ obj-$(CONFIG_DYNAMIC_FTRACE) += mcount-dyn.o obj-$(CONFIG_PERF_EVENTS) += perf_callchain.o obj-$(CONFIG_HAVE_PERF_REGS) += perf_regs.o obj-$(CONFIG_RISCV_SBI) += sbi.o sbi_ecall.o +obj-$(CONFIG_RISCV_SBI_SSE) += sbi_sse.o sbi_sse_entry.o ifeq ($(CONFIG_RISCV_SBI), y) obj-$(CONFIG_SMP) += sbi-ipi.o obj-$(CONFIG_SMP) += cpu_ops_sbi.o diff --git a/arch/riscv/kernel/asm-offsets.c b/arch/riscv/kernel/asm-offsets.c index a75f0cfea1e9fa..15363703cdd6d0 100644 --- a/arch/riscv/kernel/asm-offsets.c +++ b/arch/riscv/kernel/asm-offsets.c @@ -15,6 +15,8 @@ #include #include #include +#include +#include #include void asm_offsets(void); @@ -533,6 +535,18 @@ void asm_offsets(void) DEFINE(FREGS_A6, offsetof(struct __arch_ftrace_regs, a6)); DEFINE(FREGS_A7, offsetof(struct __arch_ftrace_regs, a7)); #endif + +#ifdef CONFIG_RISCV_SBI_SSE + OFFSET(SSE_REG_EVT_STACK, sse_event_arch_data, stack); + OFFSET(SSE_REG_EVT_SHADOW_STACK, sse_event_arch_data, shadow_stack); + OFFSET(SSE_REG_EVT_TMP, sse_event_arch_data, tmp); + OFFSET(SSE_REG_HART_ID, sse_event_arch_data, hart_id); + OFFSET(SSE_REG_CPU_ID, sse_event_arch_data, cpu_id); + + DEFINE(SBI_EXT_SSE, SBI_EXT_SSE); + DEFINE(SBI_SSE_EVENT_COMPLETE, SBI_SSE_EVENT_COMPLETE); + DEFINE(ASM_NR_CPUS, CONFIG_NR_CPUS); +#endif #ifdef CONFIG_RISCV_SBI DEFINE(SBI_EXT_FWFT, SBI_EXT_FWFT); DEFINE(SBI_EXT_FWFT_SET, SBI_EXT_FWFT_SET); diff --git a/arch/riscv/kernel/entry.S b/arch/riscv/kernel/entry.S index d799c4e56f8049..0b79fa7241ea63 100644 --- a/arch/riscv/kernel/entry.S +++ b/arch/riscv/kernel/entry.S @@ -424,6 +424,15 @@ SYM_FUNC_END(call_on_irq_stack) * arguments are passed to schedule_tail. */ SYM_FUNC_START(__switch_to) +#ifdef CONFIG_RISCV_SBI_SSE + /* + * Mark the interval where tp changes from prev to next. SSE entry uses + * the interrupted tp while this per-CPU pointer is NULL. + */ + asm_per_cpu t0, __sbi_sse_entry_task, t1 + REG_S zero, 0(t0) +#endif + /* Save context into prev->thread */ li a4, TASK_THREAD_RA add a3, a0, a4 @@ -470,6 +479,11 @@ SYM_FUNC_START(__switch_to) REG_L s11, TASK_THREAD_S11_RA(a4) /* The offset of thread_info in task_struct is zero. */ move tp, a1 +#ifdef CONFIG_RISCV_SBI_SSE + /* Publish next only after tp contains its task_struct pointer. */ + asm_per_cpu t0, __sbi_sse_entry_task, t1 + REG_S tp, 0(t0) +#endif /* Switch to the next shadow call stack */ scs_load_current ret diff --git a/arch/riscv/kernel/machine_kexec.c b/arch/riscv/kernel/machine_kexec.c index 738df176ff6f16..24e7affae70bf4 100644 --- a/arch/riscv/kernel/machine_kexec.c +++ b/arch/riscv/kernel/machine_kexec.c @@ -14,10 +14,13 @@ #include /* For set_memory_x() */ #include /* For unreachable() */ #include /* For cpu_down() */ +#include #include #include #include +#include + /* * machine_kexec_prepare - Initialize kexec * @@ -36,6 +39,13 @@ machine_kexec_prepare(struct kimage *image) unsigned int control_code_buffer_sz = 0; int i = 0; + /* A crash kernel cannot tear down registrations inherited from firmware. */ + if (is_kdump_kernel() && image->type != KEXEC_TYPE_CRASH && + riscv_sse_available()) { + pr_err("Normal kexec from a crash kernel is unsupported with SSE\n"); + return -EOPNOTSUPP; + } + /* Find the Flattened Device Tree and save its physical address */ for (i = 0; i < image->nr_segments; i++) { if (image->segment[i].memsz <= sizeof(fdt)) @@ -127,6 +137,7 @@ void machine_crash_shutdown(struct pt_regs *regs) { local_irq_disable(); + riscv_sse_mask_current_hart(); /* shutdown non-crashing cpus */ crash_smp_send_stop(); diff --git a/arch/riscv/kernel/perf_callchain.c b/arch/riscv/kernel/perf_callchain.c index b465bc9eb870ec..ec75689c7aecbc 100644 --- a/arch/riscv/kernel/perf_callchain.c +++ b/arch/riscv/kernel/perf_callchain.c @@ -1,9 +1,15 @@ // SPDX-License-Identifier: GPL-2.0 /* Copyright (C) 2019 Hangzhou C-SKY Microsystems co.,ltd. */ +#include +#include #include +#include +#include #include +#include +#include #include static bool fill_callchain(void *entry, unsigned long pc) @@ -11,6 +17,128 @@ static bool fill_callchain(void *entry, unsigned long pc) return perf_callchain_store(entry, pc) == 0; } +#ifdef CONFIG_RISCV_SBI_SSE +static bool sse_addr_on_task_stack(unsigned long addr, unsigned long size) +{ + unsigned long end = addr + size; + unsigned long stack; + + if (end < addr) + return false; + + stack = (unsigned long)task_stack_page(current); + if (addr >= stack && end <= stack + THREAD_SIZE) + return true; + + return false; +} + +static bool sse_kernel_regs_safe(struct pt_regs *regs) +{ + unsigned long fp = frame_pointer(regs); + unsigned long pc = instruction_pointer(regs); + unsigned long sp = user_stack_pointer(regs); + + if (!__kernel_text_address(pc)) + return false; + if (!sse_addr_on_task_stack(sp, sizeof(unsigned long))) + return false; + if (fp < sizeof(struct stackframe)) + return false; + + return sse_addr_on_task_stack(fp - sizeof(struct stackframe), + sizeof(struct stackframe)); +} + +static bool sse_callchain_is_guest(const struct riscv_sse_interrupted_context *context) +{ + return context && (context->hstatus & HSTATUS_SPV); +} + +static bool sse_callchain_kernel(struct perf_callchain_entry_ctx *entry, + struct pt_regs *regs) +{ + const struct riscv_sse_interrupted_context *context; + unsigned long pc; + + context = riscv_sse_get_interrupted_context(); + if (!context || context->regs != regs) + return false; + + /* A guest stack cannot be walked using the host kernel address space. */ + if (sse_callchain_is_guest(context)) + return true; + + if (user_mode(regs)) + return true; + + if (sse_kernel_regs_safe(regs)) { + walk_stackframe(NULL, regs, fill_callchain, entry); + return true; + } + + /* + * Keep the sample useful for sensitive entry paths and IRQ stacks. The + * generic walker does not take explicit IRQ stack bounds, and its + * THREAD_SIZE alignment assumption fails for non-vmapped IRQ stacks. + * Conservatively avoid walking IRQ stacks in every configuration. + */ + pc = instruction_pointer(regs); + if (__kernel_text_address(pc)) + perf_callchain_store(entry, pc); + + return true; +} + +unsigned long riscv_perf_out_copy_user(void *dst, const void *src, + unsigned long n) +{ + unsigned long addr = (unsigned long)src; + unsigned long copied = 0; + + /* Keep the generic fast path unchanged outside an SSE handler. */ + if (!riscv_sse_get_interrupted_context()) { + unsigned long ret; + + pagefault_disable(); + ret = __copy_from_user_inatomic(dst, src, n); + pagefault_enable(); + return ret; + } + + if (!access_ok(src, n)) + return n; + + /* Do not sample user memory through an unrelated active page table. */ + if (!current->mm || + (csr_read(CSR_SATP) & SATP_PPN) != virt_to_pfn(current->mm->pgd)) + return n; + + while (copied < n) { + unsigned long offset = offset_in_page(addr); + unsigned long chunk = min(n - copied, PAGE_SIZE - offset); + struct page *page; + + /* + * Fast-only GUP cannot fault. This follows perf_virt_to_phys(): + * local interrupts remain disabled throughout SSE processing, so a + * concurrent unmap cannot complete its TLB teardown before this + * temporary reference is put. + */ + if (!get_user_page_fast_only(addr, 0, &page)) + break; + + memcpy((char *)dst + copied, + (char *)page_address(page) + offset, chunk); + put_page(page); + addr += chunk; + copied += chunk; + } + + return n - copied; +} +#endif + /* * This will be called when the target is in user mode * This function will only be called when we use @@ -28,6 +156,15 @@ static bool fill_callchain(void *entry, unsigned long pc) void perf_callchain_user(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs) { +#ifdef CONFIG_RISCV_SBI_SSE + const struct riscv_sse_interrupted_context *context; + + context = riscv_sse_get_interrupted_context(); + /* A guest stack cannot be walked using the host address space. */ + if (sse_callchain_is_guest(context)) + return; +#endif + if (perf_guest_state()) { /* TODO: We don't support guest os callchain now */ return; @@ -39,6 +176,11 @@ void perf_callchain_user(struct perf_callchain_entry_ctx *entry, void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs) { +#ifdef CONFIG_RISCV_SBI_SSE + if (sse_callchain_kernel(entry, regs)) + return; +#endif + if (perf_guest_state()) { /* TODO: We don't support guest os callchain now */ return; diff --git a/arch/riscv/kernel/reset.c b/arch/riscv/kernel/reset.c index 14eb08a6db8555..fdab37e7ae52dc 100644 --- a/arch/riscv/kernel/reset.c +++ b/arch/riscv/kernel/reset.c @@ -6,6 +6,20 @@ #include #include #include +#include + +#include + +#ifndef CONFIG_SMP +void __noreturn panic_smp_self_stop(void) +{ + riscv_sse_mask_current_hart(); + local_irq_disable(); + + for (;;) + cpu_relax(); +} +#endif static void __noreturn default_power_off(void) { @@ -18,6 +32,8 @@ EXPORT_SYMBOL(pm_power_off); void machine_restart(char *cmd) { + riscv_sse_mask_current_hart(); + /* * UpdateCapsule() depends on the system being reset via ResetSystem(). */ @@ -30,12 +46,14 @@ void machine_restart(char *cmd) void machine_halt(void) { + riscv_sse_mask_current_hart(); do_kernel_power_off(); default_power_off(); } void machine_power_off(void) { + riscv_sse_mask_current_hart(); do_kernel_power_off(); default_power_off(); } diff --git a/arch/riscv/kernel/sbi_sse.c b/arch/riscv/kernel/sbi_sse.c new file mode 100644 index 00000000000000..7dd496e2bdb0ef --- /dev/null +++ b/arch/riscv/kernel/sbi_sse.c @@ -0,0 +1,246 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2025 Rivos Inc. + */ +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +DEFINE_PER_CPU(struct task_struct *, __sbi_sse_entry_task); +static DEFINE_PER_CPU(struct riscv_sse_interrupted_context *, + riscv_sse_interrupted_context); + +const struct riscv_sse_interrupted_context * +riscv_sse_get_interrupted_context(void) +{ + return this_cpu_read(riscv_sse_interrupted_context); +} + +void __weak sse_handle_event(struct sse_event_arch_data *arch_evt, struct pt_regs *regs) +{ +} + +void noinstr do_sse(struct sse_event_arch_data *arch_evt, + struct pt_regs *regs, unsigned long hstatus) +{ + struct riscv_sse_interrupted_context context = { regs, hstatus }; + struct riscv_sse_interrupted_context *previous; + struct sbiret sret; + + nmi_enter(); + instrumentation_begin(); + + /* Retrieve missing GPRs from SBI */ + sret = sbi_ecall(SBI_EXT_SSE, SBI_SSE_EVENT_ATTR_READ, arch_evt->evt_id, + SBI_SSE_ATTR_INTERRUPTED_A6, + (SBI_SSE_ATTR_INTERRUPTED_A7 - + SBI_SSE_ATTR_INTERRUPTED_A6) + 1, + (unsigned long)arch_evt->interrupted_phys, 0, 0); + if (sret.error) { + pr_warn("Failed to read interrupted registers for event %x: %ld\n", + arch_evt->evt_id, sret.error); + /* Let the client quiesce its source without using incomplete regs. */ + sse_handle_event(arch_evt, NULL); + goto out; + } + + memcpy(®s->a6, arch_evt->interrupted, + sizeof(*arch_evt->interrupted)); + + /* Make the interrupted frame visible while clients handle this event. */ + previous = this_cpu_read(riscv_sse_interrupted_context); + this_cpu_write(riscv_sse_interrupted_context, &context); + sse_handle_event(arch_evt, regs); + this_cpu_write(riscv_sse_interrupted_context, previous); + + if (memcmp(®s->a6, arch_evt->interrupted, + sizeof(*arch_evt->interrupted))) { + memcpy(arch_evt->interrupted, ®s->a6, + sizeof(*arch_evt->interrupted)); + sret = sbi_ecall(SBI_EXT_SSE, SBI_SSE_EVENT_ATTR_WRITE, + arch_evt->evt_id, SBI_SSE_ATTR_INTERRUPTED_A6, + (SBI_SSE_ATTR_INTERRUPTED_A7 - + SBI_SSE_ATTR_INTERRUPTED_A6) + 1, + (unsigned long)arch_evt->interrupted_phys, 0, 0); + /* + * If writeback fails, COMPLETE resumes with firmware's original + * a6/a7 rather than treating the shared buffer as committed. + */ + if (sret.error) + pr_warn("Failed to write interrupted registers for event %x: %ld\n", + arch_evt->evt_id, sret.error); + } + +out: + instrumentation_end(); + nmi_exit(); +} + +static void *alloc_to_stack_pointer(void *alloc) +{ + return alloc ? alloc + SSE_STACK_SIZE : NULL; +} + +static void *stack_pointer_to_alloc(void *stack) +{ + return stack ? stack - SSE_STACK_SIZE : NULL; +} + +static void arch_sse_flush_tlb_range(struct sse_event_arch_data *arch_evt, + unsigned long start, unsigned long size) +{ + unsigned long end = start + size; + + if (sse_event_is_global(arch_evt->evt_id)) + flush_tlb_kernel_range(start, end); + else + local_flush_tlb_kernel_range(start, end); +} + +static void arch_sse_shadow_stack_cpu_sync(struct sse_event_arch_data *arch_evt) +{ +#ifdef CONFIG_SHADOW_CALL_STACK + if (arch_evt->shadow_stack) + arch_sse_flush_tlb_range(arch_evt, + (unsigned long)arch_evt->shadow_stack, + SCS_SIZE); +#endif +} + +#ifdef CONFIG_VMAP_STACK +static void *sse_stack_alloc(unsigned int cpu) +{ + void *stack = arch_alloc_vmap_stack(SSE_STACK_SIZE, cpu_to_node(cpu)); + + return alloc_to_stack_pointer(stack); +} + +static void sse_stack_free(void *stack) +{ + vfree(stack_pointer_to_alloc(stack)); +} + +static void arch_sse_stack_cpu_sync(struct sse_event_arch_data *arch_evt) +{ + void *p_stack = arch_evt->stack; + unsigned long stack = (unsigned long)stack_pointer_to_alloc(p_stack); + + /* + * Flush the tlb to avoid taking any exception when accessing the + * vmapped stack inside the SSE handler + */ + arch_sse_flush_tlb_range(arch_evt, stack, SSE_STACK_SIZE); + + arch_sse_shadow_stack_cpu_sync(arch_evt); +} +#else /* CONFIG_VMAP_STACK */ +static void *sse_stack_alloc(unsigned int cpu) +{ + void *stack = kmalloc(SSE_STACK_SIZE, GFP_KERNEL); + + return alloc_to_stack_pointer(stack); +} + +static void sse_stack_free(void *stack) +{ + kfree(stack_pointer_to_alloc(stack)); +} + +static void arch_sse_stack_cpu_sync(struct sse_event_arch_data *arch_evt) +{ + arch_sse_shadow_stack_cpu_sync(arch_evt); +} +#endif /* CONFIG_VMAP_STACK */ + +static int sse_init_scs(int cpu, struct sse_event_arch_data *arch_evt) +{ + void *stack; + + if (!scs_is_enabled()) + return 0; + + stack = scs_alloc(cpu_to_node(cpu)); + if (!stack) + return -ENOMEM; + + arch_evt->shadow_stack = stack; + + return 0; +} + +void arch_sse_event_update_cpu(struct sse_event_arch_data *arch_evt, int cpu) +{ + arch_evt->cpu_id = cpu; + arch_evt->hart_id = cpuid_to_hartid_map(cpu); +} + +void arch_sse_init_cpu(void) +{ + __this_cpu_write(__sbi_sse_entry_task, current); +} + +int arch_sse_init_event(struct sse_event_arch_data *arch_evt, u32 evt_id, + int cpu) +{ + void *stack; + + arch_evt->interrupted = kmalloc_obj(*arch_evt->interrupted, GFP_KERNEL); + if (!arch_evt->interrupted) + return -ENOMEM; + + arch_evt->evt_id = evt_id; + stack = sse_stack_alloc(cpu); + if (!stack) + goto err_free_interrupted; + + arch_evt->stack = stack; + + if (sse_init_scs(cpu, arch_evt)) { + sse_stack_free(arch_evt->stack); + goto err_free_interrupted; + } + + /* kmalloc keeps the two adjacent SBI attribute words contiguous. */ + arch_evt->interrupted_phys = virt_to_phys(arch_evt->interrupted); + + arch_sse_event_update_cpu(arch_evt, cpu); + + return 0; + +err_free_interrupted: + kfree(arch_evt->interrupted); + arch_evt->interrupted = NULL; + return -ENOMEM; +} + +void arch_sse_free_event(struct sse_event_arch_data *arch_evt) +{ + scs_free(arch_evt->shadow_stack); + sse_stack_free(arch_evt->stack); + kfree(arch_evt->interrupted); +} + +int arch_sse_register_event(struct sse_event_arch_data *arch_evt) +{ + struct sbiret sret; + + arch_sse_stack_cpu_sync(arch_evt); + + sret = sbi_ecall(SBI_EXT_SSE, SBI_SSE_EVENT_REGISTER, arch_evt->evt_id, + (unsigned long)handle_sse, (unsigned long)arch_evt, 0, + 0, 0); + if (sret.error == SBI_ERR_NOT_SUPPORTED) + return -EOPNOTSUPP; + + return sbi_err_map_linux_errno(sret.error); +} diff --git a/arch/riscv/kernel/sbi_sse_entry.S b/arch/riscv/kernel/sbi_sse_entry.S new file mode 100644 index 00000000000000..e0e8efba12ddbb --- /dev/null +++ b/arch/riscv/kernel/sbi_sse_entry.S @@ -0,0 +1,226 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2025 Rivos Inc. + */ + +#include +#include + +#include +#include +#include +#include +#include + +/* When entering handle_sse, the following registers are set: + * a6: contains the hartid + * a7: contains a sse_event_arch_data struct pointer + */ +SYM_CODE_START(handle_sse) + /* Save stack temporarily */ + REG_S sp, SSE_REG_EVT_TMP(a7) + /* Set entry stack */ + REG_L sp, SSE_REG_EVT_STACK(a7) + + addi sp, sp, -(PT_SIZE_ON_STACK) + REG_S ra, PT_RA(sp) + REG_S s0, PT_S0(sp) + REG_S s1, PT_S1(sp) + REG_S s2, PT_S2(sp) + REG_S s3, PT_S3(sp) + REG_S s4, PT_S4(sp) + REG_S s5, PT_S5(sp) + REG_S s6, PT_S6(sp) + REG_S s7, PT_S7(sp) + REG_S s8, PT_S8(sp) + REG_S s9, PT_S9(sp) + REG_S s10, PT_S10(sp) + REG_S s11, PT_S11(sp) + REG_S tp, PT_TP(sp) + REG_S t0, PT_T0(sp) + REG_S t1, PT_T1(sp) + REG_S t2, PT_T2(sp) + REG_S t3, PT_T3(sp) + REG_S t4, PT_T4(sp) + REG_S t5, PT_T5(sp) + REG_S t6, PT_T6(sp) + REG_S gp, PT_GP(sp) + REG_S a0, PT_A0(sp) + REG_S a1, PT_A1(sp) + REG_S a2, PT_A2(sp) + REG_S a3, PT_A3(sp) + REG_S a4, PT_A4(sp) + REG_S a5, PT_A5(sp) + + /* Retrieve entry sp */ + REG_L a4, SSE_REG_EVT_TMP(a7) + /* Save CSRs */ + csrr a0, CSR_EPC + csrr a1, CSR_SSTATUS + csrr a2, CSR_STVAL + csrr a3, CSR_SCAUSE + + REG_S a0, PT_EPC(sp) + REG_S a1, PT_STATUS(sp) + REG_S a2, PT_BADADDR(sp) + REG_S a3, PT_CAUSE(sp) + REG_S a4, PT_SP(sp) + + /* Disable user memory access and floating/vector computing */ + li t0, SR_SUM | SR_FS_VS + csrc CSR_STATUS, t0 + + load_global_pointer + scs_load_sse_stack a7 + +#ifdef CONFIG_SMP + REG_L t4, SSE_REG_HART_ID(a7) + lw t3, SSE_REG_CPU_ID(a7) + + bne t4, a6, .Lfind_hart_id_slowpath + +.Lcpu_id_found: +#else + mv t3, zero +#endif + + asm_per_cpu_with_cpu t2 __sbi_sse_entry_task t1 t3 + REG_L tp, 0(t2) + bnez tp, .Lcurrent_task_found + + /* __switch_to() marks its transition window with a NULL entry task. */ + REG_L tp, PT_TP(sp) + +.Lcurrent_task_found: + /* Nested exceptions temporarily replace these with the SSE stack. */ + REG_L s6, TASK_TI_KERNEL_SP(tp) + REG_L s7, TASK_TI_USER_SP(tp) + + mv a1, sp /* pt_regs on stack */ + + /* + * Run the SSE handler with the normal exception vector, but restore the + * interrupted stvec before completing the event. SSE can arrive while + * the kernel is using a temporary trap vector in a sensitive entry path. + */ + csrr s3, CSR_STVEC + la t0, handle_exception + csrw CSR_STVEC, t0 + + /* + * Preserve the full HS-mode virtualization state across the handler. + * hstatus is live supervisor state rather than an SSE interrupted + * attribute, and OpenSBI consumes hstatus.SPV during event completion. + * Saving the whole CSR keeps the handler episode transparent to KVM and + * avoids having to infer which hstatus bits may matter to a guest resume. + */ + li s5, 0 + ALTERNATIVE("nop", "csrr s5, hstatus", 0, RISCV_ISA_EXT_H, 1) + + /* + * Save sscratch for restoration since we might have interrupted the + * kernel in early exception path and thus, we don't know the content of + * sscratch. + */ + csrrw s4, CSR_SSCRATCH, x0 + + mv a0, a7 + mv a2, s5 + + call do_sse + + /* Leave no reference to the event stack in the interrupted task. */ + REG_S s7, TASK_TI_USER_SP(tp) + REG_S s6, TASK_TI_KERNEL_SP(tp) + + csrw CSR_SSCRATCH, s4 + ALTERNATIVE("nop", "csrw hstatus, s5", 0, RISCV_ISA_EXT_H, 1) + csrw CSR_STVEC, s3 + + REG_L a0, PT_STATUS(sp) + REG_L a1, PT_EPC(sp) + REG_L a2, PT_BADADDR(sp) + REG_L a3, PT_CAUSE(sp) + csrw CSR_SSTATUS, a0 + csrw CSR_EPC, a1 + csrw CSR_STVAL, a2 + csrw CSR_SCAUSE, a3 + + REG_L ra, PT_RA(sp) + REG_L s0, PT_S0(sp) + REG_L s1, PT_S1(sp) + REG_L s2, PT_S2(sp) + REG_L s3, PT_S3(sp) + REG_L s4, PT_S4(sp) + REG_L s5, PT_S5(sp) + REG_L s6, PT_S6(sp) + REG_L s7, PT_S7(sp) + REG_L s8, PT_S8(sp) + REG_L s9, PT_S9(sp) + REG_L s10, PT_S10(sp) + REG_L s11, PT_S11(sp) + REG_L tp, PT_TP(sp) + REG_L t0, PT_T0(sp) + REG_L t1, PT_T1(sp) + REG_L t2, PT_T2(sp) + REG_L t3, PT_T3(sp) + REG_L t4, PT_T4(sp) + REG_L t5, PT_T5(sp) + REG_L t6, PT_T6(sp) + REG_L gp, PT_GP(sp) + REG_L a0, PT_A0(sp) + REG_L a1, PT_A1(sp) + REG_L a2, PT_A2(sp) + REG_L a3, PT_A3(sp) + REG_L a4, PT_A4(sp) + REG_L a5, PT_A5(sp) + + REG_L sp, PT_SP(sp) + + li a7, SBI_EXT_SSE + li a6, SBI_SSE_EVENT_COMPLETE + ecall + + /* + * COMPLETE must resume the interrupted context and never return. Trap + * through the normal vector instead of falling into adjacent assembly. + */ + la t0, handle_exception + csrw CSR_STVEC, t0 + ebreak + /* The fatal trap must not return; execution should never reach here. */ + +#ifdef CONFIG_SMP +.Lfind_hart_id_slowpath: + + /* Restore current task struct from __sbi_sse_entry_task */ + li t1, ASM_NR_CPUS + /* Slowpath to find the CPU id associated to the hart id */ + la t0, __cpuid_to_hartid_map + li t3, 0 + +.Lhart_id_loop: + REG_L t2, 0(t0) + beq t2, a6, .Lcpu_id_found + + /* Increment pointer and CPU number */ + addi t3, t3, 1 + addi t0, t0, RISCV_SZPTR + bltu t3, t1, .Lhart_id_loop + + /* + * This should never happen since we expect the hart_id to match one + * of our CPU, but better be safe than sorry + */ + la tp, init_task + la a0, sse_hart_id_panic_string + la t0, panic + jalr t0 +#endif + +SYM_CODE_END(handle_sse) +ASM_NOKPROBE(handle_sse) + +SYM_DATA_START_LOCAL(sse_hart_id_panic_string) + .ascii "Unable to match hart_id with cpu\0" +SYM_DATA_END(sse_hart_id_panic_string) diff --git a/arch/riscv/kernel/smp.c b/arch/riscv/kernel/smp.c index fa66f9c97d748d..7f0d4e7332f155 100644 --- a/arch/riscv/kernel/smp.c +++ b/arch/riscv/kernel/smp.c @@ -23,10 +23,13 @@ #include #include #include +#include #include #include #include +#include +#include enum ipi_message_type { IPI_RESCHEDULE, @@ -79,8 +82,18 @@ int riscv_hartid_to_cpuid(unsigned long hartid) return -ENOENT; } +void __noreturn panic_smp_self_stop(void) +{ + riscv_sse_mask_current_hart(); + local_irq_disable(); + + for (;;) + cpu_relax(); +} + static void ipi_stop(void) { + riscv_sse_mask_current_hart(); set_cpu_online(smp_processor_id(), false); while (1) wait_for_interrupt(); @@ -91,6 +104,7 @@ static atomic_t waiting_for_crash_ipi = ATOMIC_INIT(0); static inline void ipi_cpu_crash_stop(unsigned int cpu, struct pt_regs *regs) { + riscv_sse_mask_current_hart(); crash_save_cpu(regs, cpu); atomic_dec(&waiting_for_crash_ipi); @@ -254,6 +268,8 @@ void smp_send_stop(void) { unsigned long timeout; + riscv_sse_mask_current_hart(); + if (num_online_cpus() > 1) { cpumask_t mask; @@ -301,6 +317,7 @@ void crash_smp_send_stop(void) return; cpus_stopped = 1; + riscv_sse_mask_current_hart(); /* * If this cpu is the only one alive at this point in time, online or diff --git a/arch/riscv/mm/fault.c b/arch/riscv/mm/fault.c index 04ed6f8acae4fd..520495420462fe 100644 --- a/arch/riscv/mm/fault.c +++ b/arch/riscv/mm/fault.c @@ -294,6 +294,13 @@ void handle_page_fault(struct pt_regs *regs) if (kprobe_page_fault(regs, cause)) return; + /* + * Nofault accesses must be resolved through the exception table before + * entering the generic fault path or enabling interrupts. + */ + if (unlikely(faulthandler_disabled()) && fixup_exception(regs)) + return; + if (user_mode(regs)) trace_page_fault_user(addr, regs, cause); else @@ -314,8 +321,8 @@ void handle_page_fault(struct pt_regs *regs) return; } - /* Enable interrupts if they were enabled in the parent context. */ - if (!regs_irqs_disabled(regs)) + /* Do not open an interrupt window before a nofault fixup completes. */ + if (!regs_irqs_disabled(regs) && !faulthandler_disabled()) local_irq_enable(); /* diff --git a/drivers/firmware/Kconfig b/drivers/firmware/Kconfig index dbaca140a1b0a4..47eac61b915056 100644 --- a/drivers/firmware/Kconfig +++ b/drivers/firmware/Kconfig @@ -318,6 +318,7 @@ source "drivers/firmware/meson/Kconfig" source "drivers/firmware/microchip/Kconfig" source "drivers/firmware/psci/Kconfig" source "drivers/firmware/qcom/Kconfig" +source "drivers/firmware/riscv/Kconfig" source "drivers/firmware/samsung/Kconfig" source "drivers/firmware/smccc/Kconfig" source "drivers/firmware/tegra/Kconfig" diff --git a/drivers/firmware/Makefile b/drivers/firmware/Makefile index be46f1e1dc77fa..879953c316dca0 100644 --- a/drivers/firmware/Makefile +++ b/drivers/firmware/Makefile @@ -35,6 +35,7 @@ obj-y += efi/ obj-y += imx/ obj-y += psci/ obj-y += qcom/ +obj-y += riscv/ obj-y += samsung/ obj-y += smccc/ obj-y += tegra/ diff --git a/drivers/firmware/riscv/Kconfig b/drivers/firmware/riscv/Kconfig new file mode 100644 index 00000000000000..d15ff84e1258cd --- /dev/null +++ b/drivers/firmware/riscv/Kconfig @@ -0,0 +1,18 @@ +# SPDX-License-Identifier: GPL-2.0-only +menu "Risc-V Specific firmware drivers" +depends on RISCV + +config RISCV_SBI_SSE + bool "Enable SBI Supervisor Software Events support" + depends on RISCV_SBI && MMU && !HIBERNATION + default y + help + The Supervisor Software Events support allows the SBI to deliver + NMI-like notifications to the supervisor mode software. When enabled, + this option provides support to register callbacks on specific SSE + events. + + Hibernation is not supported because firmware registrations do not + survive restoring a kernel image. + +endmenu diff --git a/drivers/firmware/riscv/Makefile b/drivers/firmware/riscv/Makefile new file mode 100644 index 00000000000000..c8795d4bbb2eab --- /dev/null +++ b/drivers/firmware/riscv/Makefile @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0 + +obj-$(CONFIG_RISCV_SBI_SSE) += riscv_sbi_sse.o diff --git a/drivers/firmware/riscv/riscv_sbi_sse.c b/drivers/firmware/riscv/riscv_sbi_sse.c new file mode 100644 index 00000000000000..c7d2a2def00a8d --- /dev/null +++ b/drivers/firmware/riscv/riscv_sbi_sse.c @@ -0,0 +1,1223 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2025 Rivos Inc. + */ + +#define pr_fmt(fmt) "sse: " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +struct sse_event { + struct list_head list; + u32 evt_id; + u32 priority; + sse_event_handler_fn __rcu *handler; + void *handler_arg; + /* Only valid for global events */ + unsigned int cpu; + /* + * Desired state requested by the client. Firmware state is tracked per + * instance because a failed transition can leave a partial result. + */ + bool enable_requested; + /* Registration failed, but firmware state still needs driver cleanup. */ + bool cleanup_pending; + + union { + struct sse_registered_event *global; + struct sse_registered_event __percpu *local; + }; +}; + +static bool sse_available __ro_after_init; +static bool sse_fw_state_retained __ro_after_init; +static bool sse_shutting_down; +static atomic_t sse_teardown_failed = ATOMIC_INIT(0); +/* + * Client-side updates hold sse_mutex and the CPU read lock. Normal CPU hotplug + * callbacks exclude them through the CPUHP write side. Initialization replay + * runs before clients can register, and state removal holds sse_mutex. + */ +static LIST_HEAD(events); +static DEFINE_MUTEX(sse_mutex); + +/* + * A registration rollback can fail before the client receives an event + * handle. Keep the retained event independent of client-owned callback text + * and data while the driver retries firmware cleanup. + */ +static int sse_cleanup_event_handler(u32 evt, void *arg, struct pt_regs *regs) +{ + return 0; +} + +struct sse_registered_event { + struct sse_event_arch_data arch; + struct sse_event *event; + unsigned long attr; + /* + * Actual firmware state for one global or per-CPU instance. A retry can + * then skip instances that already completed a partial transition. + */ + bool is_registered; + bool is_enabled; +}; + +void sse_handle_event(struct sse_event_arch_data *arch_event, + struct pt_regs *regs) +{ + sse_event_handler_fn *handler; + int ret; + struct sse_registered_event *reg_evt = + container_of(arch_event, struct sse_registered_event, arch); + struct sse_event *evt = reg_evt->event; + + rcu_read_lock(); + handler = rcu_dereference(evt->handler); + ret = handler(evt->evt_id, evt->handler_arg, regs); + rcu_read_unlock(); + if (ret) + pr_warn("event %x handler failed with error %d\n", evt->evt_id, ret); +} + +static struct sse_event *sse_event_get(u32 evt) +{ + struct sse_event *event; + + lockdep_assert_held(&sse_mutex); + + list_for_each_entry(event, &events, list) { + if (event->evt_id == evt) + return event; + } + + return NULL; +} + +static phys_addr_t sse_event_get_attr_phys(struct sse_registered_event *reg_evt) +{ + phys_addr_t phys; + void *addr = ®_evt->attr; + + if (sse_event_is_global(reg_evt->event->evt_id)) + phys = virt_to_phys(addr); + else + phys = per_cpu_ptr_to_phys(addr); + + return phys; +} + +static struct sse_registered_event *sse_get_reg_evt(struct sse_event *event) +{ + if (sse_event_is_global(event->evt_id)) + return event->global; + else + return per_cpu_ptr(event->local, smp_processor_id()); +} + +static int sse_err_map_linux_errno(long err) +{ + if (err == SBI_ERR_NOT_SUPPORTED) + return -EOPNOTSUPP; + + return sbi_err_map_linux_errno(err); +} + +static int sse_sbi_event_func(struct sse_event *event, unsigned long func) +{ + struct sbiret ret; + u32 evt = event->evt_id; + struct sse_registered_event *reg_evt = sse_get_reg_evt(event); + + ret = sbi_ecall(SBI_EXT_SSE, func, evt, 0, 0, 0, 0, 0); + if (ret.error) { + pr_warn("Failed to execute func %lx, event %x, error %ld\n", + func, evt, ret.error); + return sse_err_map_linux_errno(ret.error); + } + + if (func == SBI_SSE_EVENT_DISABLE) + reg_evt->is_enabled = false; + else if (func == SBI_SSE_EVENT_ENABLE) + reg_evt->is_enabled = true; + + return 0; +} + +bool sse_event_is_enabled_local(struct sse_event *event) +{ + if (!sse_event_is_global(event->evt_id)) + lockdep_assert_preemption_disabled(); + + return sse_get_reg_evt(event)->is_enabled; +} +EXPORT_SYMBOL_GPL(sse_event_is_enabled_local); + +int sse_event_disable_local(struct sse_event *event) +{ + if (!sse_event_is_global(event->evt_id)) + lockdep_assert_preemption_disabled(); + + if (!sse_get_reg_evt(event)->is_enabled) + return 0; + + return sse_sbi_event_func(event, SBI_SSE_EVENT_DISABLE); +} +EXPORT_SYMBOL_GPL(sse_event_disable_local); + +int sse_event_enable_local(struct sse_event *event) +{ + struct sse_registered_event *reg_evt = sse_get_reg_evt(event); + int ret; + + if (!sse_event_is_global(event->evt_id)) + lockdep_assert_preemption_disabled(); + + rcu_read_lock(); + if (READ_ONCE(sse_shutting_down)) { + ret = -ESHUTDOWN; + goto out; + } + + if (!reg_evt->is_registered) { + ret = -EINVAL; + goto out; + } + + if (reg_evt->is_enabled) { + ret = 0; + goto out; + } + + ret = sse_sbi_event_func(event, SBI_SSE_EVENT_ENABLE); +out: + rcu_read_unlock(); + + return ret; +} +EXPORT_SYMBOL_GPL(sse_event_enable_local); + +static int sse_event_attr_get_no_lock(struct sse_registered_event *reg_evt, + unsigned long attr_id, unsigned long *val) +{ + struct sbiret sret; + u32 evt = reg_evt->event->evt_id; + phys_addr_t phys; + + phys = sse_event_get_attr_phys(reg_evt); + + sret = sbi_ecall(SBI_EXT_SSE, SBI_SSE_EVENT_ATTR_READ, evt, attr_id, 1, + (unsigned long)phys, 0, 0); + if (sret.error) { + pr_debug("Failed to get event %x attr %lx, error %ld\n", evt, + attr_id, sret.error); + return sse_err_map_linux_errno(sret.error); + } + + *val = reg_evt->attr; + + return 0; +} + +static int sse_event_attr_set_nolock(struct sse_registered_event *reg_evt, + unsigned long attr_id, unsigned long val) +{ + struct sbiret sret; + u32 evt = reg_evt->event->evt_id; + phys_addr_t phys; + + reg_evt->attr = val; + phys = sse_event_get_attr_phys(reg_evt); + + sret = sbi_ecall(SBI_EXT_SSE, SBI_SSE_EVENT_ATTR_WRITE, evt, attr_id, 1, + (unsigned long)phys, 0, 0); + if (sret.error) + pr_debug("Failed to set event %x attr %lx, error %ld\n", evt, + attr_id, sret.error); + + return sse_err_map_linux_errno(sret.error); +} + +static void sse_global_event_update_cpu(struct sse_event *event, + unsigned int cpu) +{ + struct sse_registered_event *reg_evt = event->global; + + event->cpu = cpu; + arch_sse_event_update_cpu(®_evt->arch, cpu); +} + +static int sse_event_set_target_cpu_nolock(struct sse_event *event, + unsigned int cpu) +{ + unsigned long hart_id, old_hart_id; + struct sse_registered_event *reg_evt = event->global; + u32 evt = event->evt_id; + unsigned int old_cpu; + bool was_enabled; + int ret; + + if (!sse_event_is_global(evt)) + return -EINVAL; + + if (cpu >= nr_cpu_ids || !cpu_online(cpu)) + return -EINVAL; + hart_id = cpuid_to_hartid_map(cpu); + old_cpu = event->cpu; + old_hart_id = cpuid_to_hartid_map(old_cpu); + + was_enabled = reg_evt->is_enabled; + if (was_enabled) { + ret = sse_event_disable_local(event); + if (ret) + return ret; + } + + ret = sse_event_attr_set_nolock(reg_evt, SBI_SSE_ATTR_PREFERRED_HART, + hart_id); + if (ret == 0) + sse_global_event_update_cpu(event, cpu); + + if (was_enabled) { + int enable_ret; + + enable_ret = sse_event_enable_local(event); + if (enable_ret) { + int rollback_ret; + + /* + * The preferred hart was already changed. Restore both the + * firmware attribute and Linux's cached target before reporting + * the failed migration. + */ + rollback_ret = sse_event_attr_set_nolock(reg_evt, + SBI_SSE_ATTR_PREFERRED_HART, + old_hart_id); + if (!rollback_ret) { + sse_global_event_update_cpu(event, old_cpu); + rollback_ret = sse_event_enable_local(event); + } + if (!rollback_ret) + return enable_ret; + + pr_warn("Failed to restore global event %x to CPU %u: %d\n", + evt, old_cpu, rollback_ret); + event->enable_requested = false; + return enable_ret; + } + } + + return ret; +} + +int sse_event_set_target_cpu(struct sse_event *event, unsigned int cpu) +{ + int ret; + + if (cpu >= nr_cpu_ids) + return -EINVAL; + + scoped_guard(mutex, &sse_mutex) { + if (READ_ONCE(sse_shutting_down)) + return -ESHUTDOWN; + + scoped_guard(cpus_read_lock) { + if (!cpu_online(cpu)) + return -EINVAL; + + ret = sse_event_set_target_cpu_nolock(event, cpu); + } + } + + return ret; +} +EXPORT_SYMBOL_GPL(sse_event_set_target_cpu); + +static int sse_event_init_registered(unsigned int cpu, + struct sse_registered_event *reg_evt, + struct sse_event *event) +{ + reg_evt->event = event; + reg_evt->is_registered = false; + reg_evt->is_enabled = false; + + return arch_sse_init_event(®_evt->arch, event->evt_id, cpu); +} + +static void sse_event_free_registered(struct sse_registered_event *reg_evt) +{ + arch_sse_free_event(®_evt->arch); +} + +static int sse_event_alloc_global(struct sse_event *event) +{ + unsigned int cpu; + int err; + struct sse_registered_event *reg_evt; + + reg_evt = kzalloc_obj(*reg_evt, GFP_KERNEL); + if (!reg_evt) + return -ENOMEM; + + event->global = reg_evt; + cpu = cpumask_first(cpu_possible_mask); + if (cpu >= nr_cpu_ids) { + kfree(reg_evt); + return -ENODEV; + } + + err = sse_event_init_registered(cpu, reg_evt, event); + if (err) + kfree(reg_evt); + + return err; +} + +static int sse_event_alloc_local(struct sse_event *event) +{ + int err; + unsigned int cpu, err_cpu; + struct sse_registered_event *reg_evt; + struct sse_registered_event __percpu *reg_evts; + + reg_evts = alloc_percpu(struct sse_registered_event); + if (!reg_evts) + return -ENOMEM; + + event->local = reg_evts; + + for_each_possible_cpu(cpu) { + reg_evt = per_cpu_ptr(reg_evts, cpu); + err = sse_event_init_registered(cpu, reg_evt, event); + if (err) { + err_cpu = cpu; + goto err_free_per_cpu; + } + } + + return 0; + +err_free_per_cpu: + for_each_possible_cpu(cpu) { + if (cpu == err_cpu) + break; + reg_evt = per_cpu_ptr(reg_evts, cpu); + sse_event_free_registered(reg_evt); + } + + free_percpu(reg_evts); + + return err; +} + +static struct sse_event *sse_event_alloc(u32 evt, u32 priority, + sse_event_handler_fn *handler, + void *arg) +{ + int err; + struct sse_event *event; + + event = kzalloc_obj(*event, GFP_KERNEL); + if (!event) + return ERR_PTR(-ENOMEM); + + event->evt_id = evt; + event->priority = priority; + event->handler_arg = arg; + RCU_INIT_POINTER(event->handler, handler); + + if (sse_event_is_global(evt)) + err = sse_event_alloc_global(event); + else + err = sse_event_alloc_local(event); + + if (err) { + kfree(event); + return ERR_PTR(err); + } + + return event; +} + +static int sse_sbi_register_event(struct sse_event *event, + struct sse_registered_event *reg_evt) +{ + int ret; + + if (reg_evt->is_registered) + return 0; + + ret = sse_event_attr_set_nolock(reg_evt, SBI_SSE_ATTR_PRIO, + event->priority); + if (ret) + return ret; + + ret = arch_sse_register_event(®_evt->arch); + if (!ret) + reg_evt->is_registered = true; + + return ret; +} + +static int sse_event_register_local(struct sse_event *event) +{ + int ret; + struct sse_registered_event *reg_evt; + + reg_evt = per_cpu_ptr(event->local, smp_processor_id()); + ret = sse_sbi_register_event(event, reg_evt); + if (ret) + pr_debug("Failed to register event %x: err %d\n", event->evt_id, + ret); + + return ret; +} + +static int sse_sbi_unregister_event(struct sse_event *event) +{ + struct sse_registered_event *reg_evt = sse_get_reg_evt(event); + int ret; + + if (!reg_evt->is_registered) + return 0; + + ret = sse_sbi_event_func(event, SBI_SSE_EVENT_UNREGISTER); + if (!ret) { + reg_evt->is_registered = false; + reg_evt->is_enabled = false; + } + + return ret; +} + +struct sse_per_cpu_evt { + struct sse_event *event; + unsigned long func; + atomic_t first_error; + atomic_t nonfallback_error; + cpumask_t changed; +}; + +static void sse_event_per_cpu_func(void *info) +{ + struct sse_per_cpu_evt *cpu_evt = info; + struct sse_registered_event *reg_evt; + bool changed; + int ret; + + reg_evt = sse_get_reg_evt(cpu_evt->event); + + if (cpu_evt->func == SBI_SSE_EVENT_REGISTER) { + changed = !reg_evt->is_registered; + ret = sse_event_register_local(cpu_evt->event); + } else if (cpu_evt->func == SBI_SSE_EVENT_UNREGISTER) { + changed = reg_evt->is_registered; + ret = sse_sbi_unregister_event(cpu_evt->event); + } else if (cpu_evt->func == SBI_SSE_EVENT_ENABLE) { + changed = !reg_evt->is_enabled; + ret = sse_event_enable_local(cpu_evt->event); + } else if (cpu_evt->func == SBI_SSE_EVENT_DISABLE) { + changed = reg_evt->is_enabled; + ret = sse_event_disable_local(cpu_evt->event); + } else { + changed = false; + ret = -EINVAL; + } + + if (ret) { + atomic_cmpxchg(&cpu_evt->first_error, 0, ret); + if (ret != -EOPNOTSUPP) + atomic_cmpxchg(&cpu_evt->nonfallback_error, 0, ret); + } else if (changed) { + cpumask_set_cpu(smp_processor_id(), &cpu_evt->changed); + } +} + +static bool sse_event_is_registered(struct sse_event *event) +{ + unsigned int cpu; + + if (sse_event_is_global(event->evt_id)) + return event->global->is_registered; + + for_each_possible_cpu(cpu) { + if (per_cpu_ptr(event->local, cpu)->is_registered) + return true; + } + + return false; +} + +static bool sse_event_is_enabled(struct sse_event *event) +{ + unsigned int cpu; + + if (sse_event_is_global(event->evt_id)) + return event->global->is_enabled; + + for_each_possible_cpu(cpu) { + if (per_cpu_ptr(event->local, cpu)->is_enabled) + return true; + } + + return false; +} + +static void sse_event_free(struct sse_event *event) +{ + unsigned int cpu; + struct sse_registered_event *reg_evt; + + if (WARN_ON_ONCE(sse_event_is_registered(event))) + return; + + if (sse_event_is_global(event->evt_id)) { + sse_event_free_registered(event->global); + kfree(event->global); + } else { + for_each_possible_cpu(cpu) { + reg_evt = per_cpu_ptr(event->local, cpu); + sse_event_free_registered(reg_evt); + } + free_percpu(event->local); + } + + kfree(event); +} + +static struct sse_event *sse_register_failed(struct sse_event *event, int ret) +{ + /* + * Keep failed rollback state visible to CPU hotplug and shutdown. The + * core-owned handler also makes the retained registration independent of + * the client whose registration request failed. + */ + if (sse_event_is_registered(event)) { + event->cleanup_pending = true; + rcu_assign_pointer(event->handler, sse_cleanup_event_handler); + synchronize_rcu(); + list_add(&event->list, &events); + pr_err("Event %x remains registered after rollback; cleanup retained\n", + event->evt_id); + ret = -EUCLEAN; + } else { + sse_event_free(event); + } + + return ERR_PTR(ret); +} + +void sse_event_cleanup(struct sse_event *event) +{ + guard(mutex)(&sse_mutex); + guard(cpus_read_lock)(); + + if (event->cleanup_pending) + return; + + /* + * Firmware may still enter the old callback after disable or unregister + * fails. Publish a core-owned callback, then wait before the client frees + * its callback data. + */ + event->cleanup_pending = true; + rcu_assign_pointer(event->handler, sse_cleanup_event_handler); + synchronize_rcu(); +} +EXPORT_SYMBOL_GPL(sse_event_cleanup); + +static void sse_release_cleanup_event(struct sse_event *event) +{ + if (!event->cleanup_pending || sse_event_is_registered(event)) + return; + + list_del(&event->list); + sse_event_free(event); +} + +static int sse_event_setup_all_cpus(struct sse_event *event, + unsigned long func, + unsigned long rollback_func) +{ + struct sse_per_cpu_evt cpu_evt; + int rollback_ret; + int ret; + + cpu_evt.event = event; + atomic_set(&cpu_evt.first_error, 0); + atomic_set(&cpu_evt.nonfallback_error, 0); + cpumask_clear(&cpu_evt.changed); + cpu_evt.func = func; + on_each_cpu(sse_event_per_cpu_func, &cpu_evt, 1); + /* IRQ fallback is safe only if every failing CPU reports unsupported. */ + ret = atomic_read(&cpu_evt.nonfallback_error); + if (!ret) + ret = atomic_read(&cpu_evt.first_error); + /* + * A previous attempt may already have changed some CPUs. Roll back only + * instances changed by this invocation. + */ + if (ret) { + cpu_evt.func = rollback_func; + atomic_set(&cpu_evt.first_error, 0); + atomic_set(&cpu_evt.nonfallback_error, 0); + on_each_cpu_mask(&cpu_evt.changed, sse_event_per_cpu_func, &cpu_evt, 1); + + rollback_ret = atomic_read(&cpu_evt.nonfallback_error); + if (!rollback_ret) + rollback_ret = atomic_read(&cpu_evt.first_error); + + /* A rollback failure leaves the firmware state uncertain. */ + return rollback_ret ?: ret; + } + + return 0; +} + +static int sse_event_teardown_all_cpus(struct sse_event *event, + unsigned long func) +{ + struct sse_per_cpu_evt cpu_evt; + + cpu_evt.event = event; + atomic_set(&cpu_evt.first_error, 0); + atomic_set(&cpu_evt.nonfallback_error, 0); + cpumask_clear(&cpu_evt.changed); + cpu_evt.func = func; + on_each_cpu(sse_event_per_cpu_func, &cpu_evt, 1); + + return atomic_read(&cpu_evt.first_error); +} + +int sse_event_enable(struct sse_event *event) +{ + int ret = 0; + + scoped_guard(mutex, &sse_mutex) { + if (READ_ONCE(sse_shutting_down)) + return -ESHUTDOWN; + + scoped_guard(cpus_read_lock) { + if (sse_event_is_global(event->evt_id)) { + ret = sse_event_enable_local(event); + } else { + ret = sse_event_setup_all_cpus(event, + SBI_SSE_EVENT_ENABLE, + SBI_SSE_EVENT_DISABLE); + } + event->enable_requested = !ret; + } + } + + return ret; +} +EXPORT_SYMBOL_GPL(sse_event_enable); + +static int sse_events_mask(void) +{ + struct sbiret ret; + + ret = sbi_ecall(SBI_EXT_SSE, SBI_SSE_HART_MASK, 0, 0, 0, 0, 0, 0); + if (ret.error == SBI_ERR_ALREADY_STOPPED) + return 0; + + return sse_err_map_linux_errno(ret.error); +} + +static int sse_events_unmask(void) +{ + struct sbiret ret; + + ret = sbi_ecall(SBI_EXT_SSE, SBI_SSE_HART_UNMASK, 0, 0, 0, 0, 0, 0); + if (ret.error == SBI_ERR_ALREADY_STARTED) + return 0; + + return sse_err_map_linux_errno(ret.error); +} + +static int sse_event_disable_nolock(struct sse_event *event) +{ + if (sse_event_is_global(event->evt_id)) + return sse_event_disable_local(event); + + return sse_event_teardown_all_cpus(event, SBI_SSE_EVENT_DISABLE); +} + +int sse_event_disable(struct sse_event *event) +{ + int ret = 0; + + scoped_guard(mutex, &sse_mutex) { + if (READ_ONCE(sse_shutting_down)) + return -ESHUTDOWN; + + scoped_guard(cpus_read_lock) { + if (!event->enable_requested && !sse_event_is_enabled(event)) + return 0; + + event->enable_requested = false; + ret = sse_event_disable_nolock(event); + if (!ret && sse_event_is_enabled(event)) + ret = -EIO; + } + } + + return ret; +} +EXPORT_SYMBOL_GPL(sse_event_disable); + +struct sse_event *sse_event_register(u32 evt, u32 priority, + sse_event_handler_fn *handler, void *arg) +{ + struct sse_event *event; + int cpu; + int ret = 0; + + if (sse_fw_state_retained) + return ERR_PTR(-EUCLEAN); + if (!sse_available) + return ERR_PTR(-EOPNOTSUPP); + + guard(mutex)(&sse_mutex); + if (READ_ONCE(sse_shutting_down)) + return ERR_PTR(-ESHUTDOWN); + + guard(cpus_read_lock)(); + + if (sse_event_get(evt)) + return ERR_PTR(-EEXIST); + + event = sse_event_alloc(evt, priority, handler, arg); + if (IS_ERR(event)) + return event; + + if (sse_event_is_global(evt)) { + unsigned long preferred_hart; + + ret = sse_event_attr_get_no_lock(event->global, + SBI_SSE_ATTR_PREFERRED_HART, + &preferred_hart); + if (ret) + return sse_register_failed(event, ret); + + cpu = riscv_hartid_to_cpuid(preferred_hart); + if (cpu < 0 || !cpu_online(cpu)) { + cpu = cpumask_first(cpu_online_mask); + if (cpu >= nr_cpu_ids) + return sse_register_failed(event, -ENODEV); + + ret = sse_event_set_target_cpu_nolock(event, cpu); + if (ret) + return sse_register_failed(event, ret); + } else { + sse_global_event_update_cpu(event, cpu); + } + + ret = sse_sbi_register_event(event, event->global); + if (ret) + return sse_register_failed(event, ret); + } else { + ret = sse_event_setup_all_cpus(event, SBI_SSE_EVENT_REGISTER, + SBI_SSE_EVENT_UNREGISTER); + if (ret) + return sse_register_failed(event, ret); + } + + list_add(&event->list, &events); + + return event; +} +EXPORT_SYMBOL_GPL(sse_event_register); + +static int sse_event_unregister_nolock(struct sse_event *event) +{ + if (sse_event_is_global(event->evt_id)) + return sse_sbi_unregister_event(event); + + return sse_event_teardown_all_cpus(event, SBI_SSE_EVENT_UNREGISTER); +} + +int sse_event_unregister(struct sse_event *event) +{ + int ret = 0; + + scoped_guard(mutex, &sse_mutex) { + if (READ_ONCE(sse_shutting_down)) + return -ESHUTDOWN; + + scoped_guard(cpus_read_lock) { + ret = sse_event_unregister_nolock(event); + if (ret) + return ret; + if (sse_event_is_registered(event)) + return -EBUSY; + + list_del(&event->list); + + sse_event_free(event); + } + } + + return ret; +} +EXPORT_SYMBOL_GPL(sse_event_unregister); + +static int sse_teardown_event(struct sse_event *event, unsigned int cpu); + +static int sse_cpu_online(unsigned int cpu) +{ + int ret, rollback_ret; + struct sse_event *event, *tmp; + struct sse_registered_event *reg_evt; + + arch_sse_init_cpu(); + + list_for_each_entry_safe(event, tmp, &events, list) { + if (sse_event_is_global(event->evt_id)) + continue; + if (event->cleanup_pending) { + ret = sse_teardown_event(event, cpu); + if (ret) + goto rollback; + sse_release_cleanup_event(event); + continue; + } + + ret = sse_event_register_local(event); + if (ret) + goto rollback; + if (event->enable_requested) + ret = sse_event_enable_local(event); + else + ret = sse_event_disable_local(event); + if (ret) + goto rollback; + } + + /* Only unmask after every cached per-CPU state is reconstructed. */ + ret = sse_events_unmask(); + if (!ret) + return 0; + +rollback: + /* A failed startup callback is not followed by this state's teardown. */ + list_for_each_entry_safe(event, tmp, &events, list) { + if (sse_event_is_global(event->evt_id)) + continue; + + reg_evt = sse_get_reg_evt(event); + rollback_ret = reg_evt->is_enabled ? + sse_event_disable_local(event) : 0; + if (rollback_ret) { + pr_warn("Failed to disable event %x while rolling back CPU %u: %d\n", + event->evt_id, cpu, rollback_ret); + atomic_set(&sse_teardown_failed, 1); + continue; + } + + rollback_ret = reg_evt->is_registered ? + sse_sbi_unregister_event(event) : 0; + if (rollback_ret) { + pr_warn("Failed to unregister event %x while rolling back CPU %u: %d\n", + event->evt_id, cpu, rollback_ret); + atomic_set(&sse_teardown_failed, 1); + } + sse_release_cleanup_event(event); + } + + return ret; +} + +static int sse_teardown_event(struct sse_event *event, unsigned int cpu) +{ + struct sse_registered_event *reg_evt = sse_get_reg_evt(event); + int ret; + + if (reg_evt->is_enabled) { + ret = sse_event_disable_local(event); + if (ret) { + pr_warn("Failed to disable event %x on CPU %u: %d\n", + event->evt_id, cpu, ret); + return ret; + } + } + + ret = sse_sbi_unregister_event(event); + if (ret) + pr_warn("Failed to unregister event %x on CPU %u: %d\n", + event->evt_id, cpu, ret); + + return ret; +} + +static int sse_restore_local_events(unsigned int cpu) +{ + struct sse_event *event; + int first_error = 0; + int ret; + + list_for_each_entry(event, &events, list) { + if (sse_event_is_global(event->evt_id) || event->cleanup_pending) + continue; + + ret = sse_event_register_local(event); + if (!ret) { + ret = event->enable_requested ? + sse_event_enable_local(event) : + sse_event_disable_local(event); + } + if (ret) { + pr_warn("Failed to restore event %x on CPU %u: %d\n", + event->evt_id, cpu, ret); + if (!first_error) + first_error = ret; + } + } + + return first_error; +} + +static int sse_cpu_teardown(unsigned int cpu) +{ + /* Only a regular CPU-offline callback may abort the CPUHP operation. */ + bool regular_offline = READ_ONCE(sse_available) && + !READ_ONCE(sse_shutting_down); + unsigned int next_cpu; + struct sse_event *event, *tmp; + int first_error = 0; + int ret; + + /* Do not dismantle CPU state while firmware can still deliver SSE. */ + ret = sse_events_mask(); + if (ret) { + pr_warn("Failed to mask SSE on CPU %u during teardown: %d\n", + cpu, ret); + if (READ_ONCE(sse_shutting_down)) + atomic_set(&sse_teardown_failed, 1); + /* CPUHP installation rollback and state removal cannot fail. */ + return regular_offline ? ret : 0; + } + + list_for_each_entry_safe(event, tmp, &events, list) { + if (sse_event_is_global(event->evt_id)) + continue; + + ret = sse_teardown_event(event, cpu); + if (ret && !first_error) + first_error = ret; + sse_release_cleanup_event(event); + if (ret && regular_offline) + goto restore_cpu; + } + + list_for_each_entry_safe(event, tmp, &events, list) { + if (!sse_event_is_global(event->evt_id) || event->cpu != cpu) + continue; + + /* + * cpuhp_remove_state() invokes teardown while every CPU remains + * online. Do not migrate to a CPU whose callback may have run. + */ + if (READ_ONCE(sse_shutting_down) || event->cleanup_pending) { + ret = sse_teardown_event(event, cpu); + } else { + next_cpu = cpumask_any_but(cpu_online_mask, cpu); + if (next_cpu >= nr_cpu_ids) { + ret = sse_teardown_event(event, cpu); + } else { + ret = sse_event_set_target_cpu_nolock(event, next_cpu); + if (ret) + pr_warn("Failed to migrate global event %x from CPU %u: %d\n", + event->evt_id, cpu, ret); + } + } + + if (ret && !first_error) + first_error = ret; + sse_release_cleanup_event(event); + if (ret && regular_offline) + goto restore_cpu; + } + + if (first_error) { + /* An offline CPU is not revisited when this CPUHP state is removed. */ + atomic_set(&sse_teardown_failed, 1); + } + + return 0; + +restore_cpu: + /* + * CPUHP leaves this CPU online when teardown returns an error. Restore + * every client-owned local event before making SSE delivery visible again. + */ + ret = sse_restore_local_events(cpu); + if (!ret) + ret = sse_events_unmask(); + if (ret) { + atomic_set(&sse_teardown_failed, 1); + pr_warn("Failed to restore SSE after aborting CPU %u offline: %d\n", + cpu, ret); + } else { + pr_warn("Aborted CPU %u offline after SSE teardown failed: %d\n", + cpu, first_error); + } + + return first_error; +} + +static int sse_pm_notifier(struct notifier_block *nb, unsigned long action, + void *data) +{ + int ret; + + WARN_ON_ONCE(preemptible()); + + switch (action) { + case CPU_PM_ENTER: + ret = sse_events_mask(); + break; + case CPU_PM_EXIT: + case CPU_PM_ENTER_FAILED: + if (READ_ONCE(sse_shutting_down)) + return NOTIFY_OK; + ret = sse_events_unmask(); + break; + default: + return NOTIFY_DONE; + } + + if (ret) + return notifier_from_errno(ret); + + return NOTIFY_OK; +} + +static struct notifier_block sse_pm_nb = { + .notifier_call = sse_pm_notifier, +}; + +static int sse_panic_notifier(struct notifier_block *nb, unsigned long action, + void *data) +{ + riscv_sse_mask_current_hart(); + + return NOTIFY_OK; +} + +static struct notifier_block sse_panic_nb = { + .notifier_call = sse_panic_notifier, + .priority = INT_MAX, +}; + +/* + * Mask all CPUs and unregister all events on reboot or kexec. + */ +static int sse_reboot_notifier(struct notifier_block *nb, unsigned long action, + void *data) +{ + int ret; + + scoped_guard(mutex, &sse_mutex) { + if (!sse_shutting_down) { + WRITE_ONCE(sse_shutting_down, true); + ret = cpu_pm_unregister_notifier(&sse_pm_nb); + if (ret) { + pr_warn("Failed to unregister CPU PM notifier: %d\n", + ret); + atomic_set(&sse_teardown_failed, 1); + } + /* Drain CPU PM callbacks and client enables before teardown. */ + synchronize_rcu(); + cpuhp_remove_state(CPUHP_AP_RISCV_SSE_ONLINE); + } + } + + /* Normal kexec preserves firmware state but discards old kernel memory. */ + if (kexec_in_progress && atomic_read(&sse_teardown_failed)) + panic("SSE teardown failed; refusing unsafe kexec"); + + return NOTIFY_OK; +} + +static struct notifier_block sse_reboot_nb = { + .notifier_call = sse_reboot_notifier, +}; + +static int __init sse_init(void) +{ + int ret; + + /* + * A kdump kernel cannot identify registrations left by the crashed + * kernel. Keep them masked by not initializing SSE again. + */ + if (is_kdump_kernel() && riscv_sse_available()) { + sse_fw_state_retained = true; + pr_info("SSE remains disabled in the crash kernel\n"); + return -EOPNOTSUPP; + } + + if (sbi_probe_extension(SBI_EXT_SSE) <= 0) { + pr_info("Missing SBI SSE extension\n"); + return -EOPNOTSUPP; + } + pr_info("SBI SSE extension detected\n"); + + ret = cpu_pm_register_notifier(&sse_pm_nb); + if (ret) { + pr_warn("Failed to register CPU PM notifier...\n"); + return ret; + } + + ret = register_reboot_notifier(&sse_reboot_nb); + if (ret) { + pr_warn("Failed to register reboot notifier...\n"); + goto remove_cpupm; + } + + ret = atomic_notifier_chain_register(&panic_notifier_list, &sse_panic_nb); + if (ret) { + pr_warn("Failed to register panic notifier...\n"); + goto remove_reboot; + } + + /* Tear down perf events before dismantling their SSE delivery path. */ + ret = cpuhp_setup_state(CPUHP_AP_RISCV_SSE_ONLINE, "riscv/sse:online", + sse_cpu_online, sse_cpu_teardown); + if (ret < 0) + goto remove_panic; + + sse_available = true; + + return 0; + +remove_panic: + atomic_notifier_chain_unregister(&panic_notifier_list, &sse_panic_nb); + +remove_reboot: + unregister_reboot_notifier(&sse_reboot_nb); + +remove_cpupm: + cpu_pm_unregister_notifier(&sse_pm_nb); + + return ret; +} +arch_initcall(sse_init); diff --git a/drivers/perf/Kconfig b/drivers/perf/Kconfig index 245e7bb763b994..10b047856121dc 100644 --- a/drivers/perf/Kconfig +++ b/drivers/perf/Kconfig @@ -105,6 +105,17 @@ config RISCV_PMU_SBI full perf feature support i.e. counter overflow, privilege mode filtering, counter configuration. +config RISCV_PMU_SBI_SSE + depends on RISCV_PMU_SBI && RISCV_SBI_SSE + bool "RISC-V PMU SSE events" + default n + help + Say y if you want to use SSE events to deliver PMU interrupts. This + provides a way to profile the kernel at any level by using NMI-like + SSE events. SSE events being really intrusive, this option allows + to select it only if needed. See Documentation/arch/riscv/pmu-sse.rst + for interrupted-context limitations. + config STARFIVE_STARLINK_PMU depends on ARCH_STARFIVE || COMPILE_TEST depends on 64BIT diff --git a/drivers/perf/riscv_pmu.c b/drivers/perf/riscv_pmu.c index 8e3cd0f35336fa..1f65c640b69398 100644 --- a/drivers/perf/riscv_pmu.c +++ b/drivers/perf/riscv_pmu.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include @@ -247,6 +248,11 @@ void riscv_pmu_start(struct perf_event *event, int flags) if (flags & PERF_EF_RELOAD) WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE)); +#ifdef CONFIG_RISCV_PMU_SBI_SSE + if (unlikely(this_cpu_ptr(rvpmu->hw_events)->sse_failed)) + return; +#endif + hwc->state = 0; riscv_pmu_event_set_period(event); init_val = local64_read(&hwc->prev_count) & max_period; @@ -306,6 +312,7 @@ static int riscv_pmu_event_init(struct perf_event *event) struct hw_perf_event *hwc = &event->hw; struct riscv_pmu *rvpmu = to_riscv_pmu(event->pmu); int mapped_event; + int ret; u64 event_config = 0; uint64_t cmask; @@ -331,8 +338,11 @@ static int riscv_pmu_event_init(struct perf_event *event) hwc->idx = -1; hwc->event_base = mapped_event; - if (rvpmu->event_init) - rvpmu->event_init(event); + if (rvpmu->event_init) { + ret = rvpmu->event_init(event); + if (ret) + return ret; + } if (!is_sampling_event(event)) { /* diff --git a/drivers/perf/riscv_pmu_sbi.c b/drivers/perf/riscv_pmu_sbi.c index 50220f7b46d9b0..7e971b47730c41 100644 --- a/drivers/perf/riscv_pmu_sbi.c +++ b/drivers/perf/riscv_pmu_sbi.c @@ -16,13 +16,16 @@ #include #include #include +#include #include +#include #include #include #include #include #include +#include #include #include #include @@ -95,7 +98,6 @@ static bool riscv_pmu_use_irq; static unsigned int riscv_pmu_irq_num; static unsigned int riscv_pmu_irq_mask; static unsigned int riscv_pmu_irq; - /* Cache the available counters in a bitmask */ static unsigned long cmask; @@ -896,14 +898,24 @@ static int pmu_sbi_get_ctrinfo(int nctr, unsigned long *mask) return 0; } -static inline void pmu_sbi_stop_all(struct riscv_pmu *pmu) +static inline void pmu_sbi_stop_all_mask(unsigned long ctr_mask) { /* * No need to check the error because we are disabling all the counters * which may include counters that are not enabled yet. */ sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_STOP, - 0, pmu->cmask, SBI_PMU_STOP_FLAG_RESET, 0, 0, 0); + 0, ctr_mask, SBI_PMU_STOP_FLAG_RESET, 0, 0, 0); +} + +static inline void pmu_sbi_stop_all(struct riscv_pmu *pmu) +{ + pmu_sbi_stop_all_mask(pmu->cmask); +} + +static void pmu_sbi_stop_all_cpu(void *info) +{ + pmu_sbi_stop_all(info); } static inline void pmu_sbi_stop_hw_ctrs(struct riscv_pmu *pmu) @@ -953,7 +965,7 @@ static inline void pmu_sbi_stop_hw_ctrs(struct riscv_pmu *pmu) static inline void pmu_sbi_start_ovf_ctrs_sbi(struct cpu_hw_events *cpu_hw_evt, u64 ctr_ovf_mask) { - int idx = 0, i; + int idx, i; struct perf_event *event; unsigned long flag = SBI_PMU_START_FLAG_SET_INIT_VALUE; unsigned long ctr_start_mask = 0; @@ -962,7 +974,19 @@ static inline void pmu_sbi_start_ovf_ctrs_sbi(struct cpu_hw_events *cpu_hw_evt, u64 init_val = 0; for (i = 0; i < BITS_TO_LONGS(RISCV_MAX_COUNTERS); i++) { - ctr_start_mask = cpu_hw_evt->used_hw_ctrs[i] & ~ctr_ovf_mask; + unsigned long ctr_ovf_mask_word; + int lidx; + + ctr_ovf_mask_word = ctr_ovf_mask >> (i * BITS_PER_LONG); + ctr_start_mask = 0; + for_each_set_bit(idx, &cpu_hw_evt->used_hw_ctrs[i], BITS_PER_LONG) { + lidx = idx + i * BITS_PER_LONG; + event = cpu_hw_evt->events[lidx]; + if (event && !(event->hw.state & PERF_HES_STOPPED)) + ctr_start_mask |= BIT(idx); + } + ctr_start_mask &= ~ctr_ovf_mask_word; + /* Start all the counters that did not overflow in a single shot */ if (ctr_start_mask) { sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_START, i * BITS_PER_LONG, @@ -971,23 +995,25 @@ static inline void pmu_sbi_start_ovf_ctrs_sbi(struct cpu_hw_events *cpu_hw_evt, } /* Reinitialize and start all the counter that overflowed */ - while (ctr_ovf_mask) { - if (ctr_ovf_mask & 0x01) { - event = cpu_hw_evt->events[idx]; - hwc = &event->hw; - max_period = riscv_pmu_ctr_get_width_mask(event); - init_val = local64_read(&hwc->prev_count) & max_period; + for (idx = 0; idx < RISCV_MAX_COUNTERS; idx++) { + if (!(ctr_ovf_mask & BIT_ULL(idx))) + continue; + + event = cpu_hw_evt->events[idx]; + if (!event || event->hw.state & PERF_HES_STOPPED) + continue; + + hwc = &event->hw; + max_period = riscv_pmu_ctr_get_width_mask(event); + init_val = local64_read(&hwc->prev_count) & max_period; #if defined(CONFIG_32BIT) - sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_START, idx, 1, - flag, init_val, init_val >> 32, 0); + sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_START, idx, 1, + flag, init_val, init_val >> 32, 0); #else - sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_START, idx, 1, - flag, init_val, 0, 0); + sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_START, idx, 1, + flag, init_val, 0, 0); #endif - perf_event_update_userpage(event); - } - ctr_ovf_mask = ctr_ovf_mask >> 1; - idx++; + perf_event_update_userpage(event); } } @@ -1002,7 +1028,7 @@ static inline void pmu_sbi_start_ovf_ctrs_snapshot(struct cpu_hw_events *cpu_hw_ struct riscv_pmu_snapshot_data *sdata = cpu_hw_evt->snapshot_addr; for_each_set_bit(idx, cpu_hw_evt->used_hw_ctrs, RISCV_MAX_COUNTERS) { - if (ctr_ovf_mask & BIT(idx)) { + if (ctr_ovf_mask & BIT_ULL(idx)) { event = cpu_hw_evt->events[idx]; hwc = &event->hw; max_period = riscv_pmu_ctr_get_width_mask(event); @@ -1016,31 +1042,111 @@ static inline void pmu_sbi_start_ovf_ctrs_snapshot(struct cpu_hw_events *cpu_hw_ } for (i = 0; i < BITS_TO_LONGS(RISCV_MAX_COUNTERS); i++) { + unsigned long ctr_start_mask = 0; + int lidx; + /* Restore the counter values to relative indices for used hw counters */ - for_each_set_bit(idx, &cpu_hw_evt->used_hw_ctrs[i], BITS_PER_LONG) - sdata->ctr_values[idx] = - cpu_hw_evt->snapshot_cval_shcopy[idx + i * BITS_PER_LONG]; + for_each_set_bit(idx, &cpu_hw_evt->used_hw_ctrs[i], BITS_PER_LONG) { + lidx = idx + i * BITS_PER_LONG; + event = cpu_hw_evt->events[lidx]; + if (event && !(event->hw.state & PERF_HES_STOPPED)) + ctr_start_mask |= BIT(idx); + + sdata->ctr_values[idx] = cpu_hw_evt->snapshot_cval_shcopy[lidx]; + } + /* Start all the counters in a single shot */ - sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_START, idx * BITS_PER_LONG, - cpu_hw_evt->used_hw_ctrs[i], flag, 0, 0, 0); + if (ctr_start_mask) + sbi_ecall(SBI_EXT_PMU, SBI_EXT_PMU_COUNTER_START, + i * BITS_PER_LONG, ctr_start_mask, flag, 0, 0, 0); } } +static bool pmu_sbi_sse_failed(struct cpu_hw_events *cpu_hw_evt) +{ +#ifdef CONFIG_RISCV_PMU_SBI_SSE + return READ_ONCE(cpu_hw_evt->sse_failed); +#else + return false; +#endif +} + static void pmu_sbi_start_overflow_mask(struct riscv_pmu *pmu, u64 ctr_ovf_mask) { struct cpu_hw_events *cpu_hw_evt = this_cpu_ptr(pmu->hw_events); + if (unlikely(pmu_sbi_sse_failed(cpu_hw_evt))) + return; + if (sbi_pmu_snapshot_available()) pmu_sbi_start_ovf_ctrs_snapshot(cpu_hw_evt, ctr_ovf_mask); else pmu_sbi_start_ovf_ctrs_sbi(cpu_hw_evt, ctr_ovf_mask); } -static irqreturn_t pmu_sbi_ovf_handler(int irq, void *dev) +#ifdef CONFIG_RISCV_PMU_SBI_SSE +/* + * A local SSE delivery failure makes the current PMU state unsafe to resume. + * Latch the failure before stopping mapped events so the SSE transition and + * overflow restart paths cannot undo the fail-safe while they are quiesced. + */ +static void pmu_sbi_fail_sse(struct riscv_pmu *pmu, const char *op, int ret) +{ + struct cpu_hw_events *cpu_hw_evt = this_cpu_ptr(pmu->hw_events); + struct perf_event *event; + int idx; + + if (READ_ONCE(cpu_hw_evt->sse_failed)) + return; + + WRITE_ONCE(cpu_hw_evt->sse_failed, true); + pr_err_ratelimited("failed to %s local PMU SSE event: %d; stopping counters\n", + op, ret); + + for (idx = 0; idx < RISCV_MAX_COUNTERS; idx++) { + event = cpu_hw_evt->events[idx]; + if (event) + riscv_pmu_stop(event, PERF_EF_UPDATE); + } +} + +static void pmu_sbi_sse_disable(struct pmu *pmu) +{ + struct riscv_pmu *rvpmu = to_riscv_pmu(pmu); + struct cpu_hw_events *cpu_hw_evt = this_cpu_ptr(rvpmu->hw_events); + int ret; + + if (!READ_ONCE(rvpmu->sse_active) || + READ_ONCE(cpu_hw_evt->sse_failed)) + return; + + ret = sse_event_disable_local(rvpmu->sse_evt); + if (unlikely(ret)) + pmu_sbi_fail_sse(rvpmu, "disable", ret); +} + +static void pmu_sbi_sse_enable(struct pmu *pmu) +{ + struct riscv_pmu *rvpmu = to_riscv_pmu(pmu); + struct cpu_hw_events *cpu_hw_evt = this_cpu_ptr(rvpmu->hw_events); + int ret; + + if (!READ_ONCE(rvpmu->sse_active) || + READ_ONCE(cpu_hw_evt->sse_failed)) + return; + + ret = sse_event_enable_local(rvpmu->sse_evt); + if (unlikely(ret)) + pmu_sbi_fail_sse(rvpmu, "enable", ret); +} +#endif + +static irqreturn_t pmu_sbi_ovf_handler(struct cpu_hw_events *cpu_hw_evt, + struct pt_regs *regs, bool from_sse, + bool from_guest) { struct perf_sample_data data; - struct pt_regs *regs; struct hw_perf_event *hw_evt; union sbi_pmu_ctr_info *info; int lidx, hidx, fidx; @@ -1048,28 +1154,38 @@ static irqreturn_t pmu_sbi_ovf_handler(int irq, void *dev) struct perf_event *event; u64 overflow; u64 overflowed_ctrs = 0; - struct cpu_hw_events *cpu_hw_evt = dev; u64 start_clock = sched_clock(); struct riscv_pmu_snapshot_data *sdata = cpu_hw_evt->snapshot_addr; if (WARN_ON_ONCE(!cpu_hw_evt)) return IRQ_NONE; - /* Firmware counter don't support overflow yet */ + /* + * SSE can arrive before perf installs an event. The early exits below + * must stop the PMU source before firmware completes the SSE. + */ fidx = find_first_bit(cpu_hw_evt->used_hw_ctrs, RISCV_MAX_COUNTERS); if (fidx == RISCV_MAX_COUNTERS) { - csr_clear(CSR_SIP, BIT(riscv_pmu_irq_num)); + if (from_sse) + pmu_sbi_stop_all_mask(cmask); + else + csr_clear(CSR_SIP, BIT(riscv_pmu_irq_num)); return IRQ_NONE; } event = cpu_hw_evt->events[fidx]; if (!event) { - ALT_SBI_PMU_OVF_CLEAR_PENDING(riscv_pmu_irq_mask); + if (from_sse) + pmu_sbi_stop_all_mask(cmask); + else + ALT_SBI_PMU_OVF_CLEAR_PENDING(riscv_pmu_irq_mask); return IRQ_NONE; } pmu = to_riscv_pmu(event->pmu); pmu_sbi_stop_hw_ctrs(pmu); + if (unlikely(pmu_sbi_sse_failed(cpu_hw_evt))) + return IRQ_NONE; /* Overflow status register should only be read after counter are stopped */ if (sbi_pmu_snapshot_available()) @@ -1079,15 +1195,17 @@ static irqreturn_t pmu_sbi_ovf_handler(int irq, void *dev) /* * Overflow interrupt pending bit should only be cleared after stopping - * all the counters to avoid any race condition. + * all the counters to avoid any race condition. When using SSE, + * interrupt is cleared when stopping counters. */ - ALT_SBI_PMU_OVF_CLEAR_PENDING(riscv_pmu_irq_mask); + if (!from_sse) + ALT_SBI_PMU_OVF_CLEAR_PENDING(riscv_pmu_irq_mask); /* No overflow bit is set */ - if (!overflow) + if (!overflow) { + pmu_sbi_start_overflow_mask(pmu, 0); return IRQ_NONE; - - regs = get_irq_regs(); + } for_each_set_bit(lidx, cpu_hw_evt->used_hw_ctrs, RISCV_MAX_COUNTERS) { struct perf_event *event = cpu_hw_evt->events[lidx]; @@ -1109,14 +1227,20 @@ static irqreturn_t pmu_sbi_ovf_handler(int irq, void *dev) hidx = info->csr - CSR_CYCLE; /* check if the corresponding bit is set in scountovf or overflow mask in shmem */ - if (!(overflow & BIT(hidx))) + if (!(overflow & BIT_ULL(hidx))) continue; +#ifdef CONFIG_CPU_PM + /* Do not let CPU-PM resume override this overflow decision. */ + if (from_sse) + clear_bit(lidx, cpu_hw_evt->pm_resume_hw_ctrs); +#endif + /* * Keep a track of overflowed counters so that they can be started * with updated initial value. */ - overflowed_ctrs |= BIT(lidx); + overflowed_ctrs |= BIT_ULL(lidx); hw_evt = &event->hw; /* Update the event states here so that we know the state while reading */ hw_evt->state |= PERF_HES_STOPPED; @@ -1124,6 +1248,14 @@ static irqreturn_t pmu_sbi_ovf_handler(int irq, void *dev) hw_evt->state |= PERF_HES_UPTODATE; perf_sample_data_init(&data, 0, hw_evt->last_period); if (riscv_pmu_event_set_period(event)) { + int overflow_ret; + + /* Guest attribution and guest stack sampling are not supported yet. */ + if (from_guest) { + hw_evt->state = 0; + continue; + } + /* * Unlike other ISAs, RISC-V don't have to disable interrupts * to avoid throttling here. As per the specification, the @@ -1132,128 +1264,291 @@ static irqreturn_t pmu_sbi_ovf_handler(int irq, void *dev) * TODO: We will need to stop the guest counters once * virtualization support is added. */ - perf_event_overflow(event, &data, regs); + overflow_ret = perf_event_overflow(event, &data, regs); + if (!overflow_ret) + hw_evt->state = 0; + } else { + hw_evt->state = 0; } - /* Reset the state as we are going to start the counter after the loop */ - hw_evt->state = 0; } pmu_sbi_start_overflow_mask(pmu, overflowed_ctrs); + perf_sample_event_took(sched_clock() - start_clock); return IRQ_HANDLED; } -static int pmu_sbi_starting_cpu(unsigned int cpu, struct hlist_node *node) +static irqreturn_t pmu_sbi_ovf_irq_handler(int irq, void *dev) { - struct riscv_pmu *pmu = hlist_entry_safe(node, struct riscv_pmu, node); - struct cpu_hw_events *cpu_hw_evt = this_cpu_ptr(pmu->hw_events); - - /* - * We keep enabling userspace access to CYCLE, TIME and INSTRET via the - * legacy option but that will be removed in the future. - */ - if (sysctl_perf_user_access == SYSCTL_LEGACY) - csr_write(CSR_SCOUNTEREN, 0x7); - else - csr_write(CSR_SCOUNTEREN, 0x2); + return pmu_sbi_ovf_handler(dev, get_irq_regs(), false, false); +} - /* Stop all the counters so that they can be enabled from perf */ - pmu_sbi_stop_all(pmu); +#ifdef CONFIG_RISCV_PMU_SBI_SSE +static int pmu_sbi_ovf_sse_handler(u32 evt, void *arg, struct pt_regs *regs) +{ + const struct riscv_sse_interrupted_context *context; + struct riscv_pmu *pmu = arg; + struct cpu_hw_events *hw_event = raw_cpu_ptr(pmu->hw_events); + bool from_guest; + + if (unlikely(!READ_ONCE(pmu->sse_active))) { + pmu_sbi_stop_all(pmu); + return -EIO; + } - if (riscv_pmu_use_irq) { - cpu_hw_evt->irq = riscv_pmu_irq; - ALT_SBI_PMU_OVF_CLEAR_PENDING(riscv_pmu_irq_mask); - enable_percpu_irq(riscv_pmu_irq, IRQ_TYPE_NONE); + if (unlikely(!regs)) { + pmu_sbi_fail_sse(pmu, "read interrupted context for", -EIO); + return -EIO; } - if (sbi_pmu_snapshot_available()) - return pmu_sbi_snapshot_setup(pmu, cpu); + context = riscv_sse_get_interrupted_context(); + from_guest = context && context->regs == regs && + (context->hstatus & HSTATUS_SPV); + pmu_sbi_ovf_handler(hw_event, regs, true, from_guest); return 0; } -static int pmu_sbi_dying_cpu(unsigned int cpu, struct hlist_node *node) +static int pmu_sbi_setup_sse(struct riscv_pmu *pmu) { - if (riscv_pmu_use_irq) { - disable_percpu_irq(riscv_pmu_irq); - } + int ret; + struct sse_event *evt; - /* Disable all counters access for user mode now */ - csr_write(CSR_SCOUNTEREN, 0x0); + evt = sse_event_register(SBI_SSE_EVENT_LOCAL_PMU_OVERFLOW, 0, + pmu_sbi_ovf_sse_handler, pmu); + if (IS_ERR(evt)) + return PTR_ERR(evt); + pmu->sse_evt = evt; - if (sbi_pmu_snapshot_available()) - return pmu_sbi_snapshot_disable(); + ret = sse_event_enable(evt); + if (ret) { + int cleanup_ret; - return 0; + cleanup_ret = sse_event_disable(evt); + if (cleanup_ret) { + pr_warn("failed to disable SSE event after setup error: %d\n", + cleanup_ret); + return cleanup_ret; + } + + cleanup_ret = sse_event_unregister(evt); + + if (cleanup_ret) { + pr_warn("failed to unregister SSE event: %d\n", + cleanup_ret); + } else { + pmu->sse_evt = NULL; + } + return cleanup_ret ?: ret; + } + + WRITE_ONCE(pmu->sse_active, true); + pr_info("using SSE for PMU event delivery\n"); + + return ret; } -static int pmu_sbi_setup_irqs(struct riscv_pmu *pmu, struct platform_device *pdev) +static void pmu_sbi_cleanup_sse(struct riscv_pmu *pmu) { + struct sse_event *sse_evt; int ret; - struct cpu_hw_events __percpu *hw_events = pmu->hw_events; - struct irq_domain *domain = NULL; + sse_evt = pmu->sse_evt; + if (!sse_evt) + return; + /* + * Close callback admission before draining each CPU. PMU callbacks run + * with local interrupts disabled, so the synchronous IPI cannot complete + * until a callback that observed the old state has returned. + */ + WRITE_ONCE(pmu->sse_active, false); + on_each_cpu(pmu_sbi_stop_all_cpu, pmu, 1); + + ret = sse_event_disable(sse_evt); + if (ret) { + pr_warn("failed to disable SSE event: %d\n", ret); + goto retain; + } + + ret = sse_event_unregister(sse_evt); + if (ret) { + pr_warn("failed to unregister SSE event: %d\n", ret); + goto retain; + } + + pmu->sse_evt = NULL; + return; + +retain: + sse_event_cleanup(sse_evt); + pmu->sse_evt = NULL; +} + +static bool pmu_sbi_sse_state_retained(struct riscv_pmu *pmu) +{ + return pmu->sse_evt; +} +#else +static int pmu_sbi_setup_sse(struct riscv_pmu *pmu) +{ + return -EOPNOTSUPP; +} + +static void pmu_sbi_cleanup_sse(struct riscv_pmu *pmu) {} + +static bool pmu_sbi_sse_state_retained(struct riscv_pmu *pmu) +{ + return false; +} +#endif + +static bool pmu_sbi_select_irq(void) +{ if (riscv_isa_extension_available(NULL, SSCOFPMF)) { riscv_pmu_irq_num = RV_IRQ_PMU; - riscv_pmu_use_irq = true; + return true; } else if (IS_ENABLED(CONFIG_ERRATA_THEAD_PMU) && riscv_cached_mvendorid(0) == THEAD_VENDOR_ID && riscv_cached_marchid(0) == 0 && riscv_cached_mimpid(0) == 0) { riscv_pmu_irq_num = THEAD_C9XX_RV_IRQ_PMU; - riscv_pmu_use_irq = true; + return true; } else if (riscv_has_vendor_extension_unlikely(ANDES_VENDOR_ID, RISCV_ISA_VENDOR_EXT_XANDESPMU) && IS_ENABLED(CONFIG_ANDES_CUSTOM_PMU)) { riscv_pmu_irq_num = ANDES_SLI_CAUSE_BASE + ANDES_RV_IRQ_PMOVI; - riscv_pmu_use_irq = true; + return true; } - riscv_pmu_irq_mask = BIT(riscv_pmu_irq_num % BITS_PER_LONG); + return false; +} - if (!riscv_pmu_use_irq) +static int pmu_sbi_setup_irq(struct riscv_pmu *pmu) +{ + struct cpu_hw_events __percpu *hw_events = pmu->hw_events; + struct irq_domain *domain; + int ret; + + if (!pmu_sbi_select_irq()) return -EOPNOTSUPP; + riscv_pmu_irq_mask = BIT(riscv_pmu_irq_num % BITS_PER_LONG); + domain = irq_find_matching_fwnode(riscv_get_intc_hwnode(), DOMAIN_BUS_ANY); if (!domain) { pr_err("Failed to find INTC IRQ root domain\n"); - ret = -ENODEV; - goto err; + return -ENODEV; } riscv_pmu_irq = irq_create_mapping(domain, riscv_pmu_irq_num); if (!riscv_pmu_irq) { pr_err("Failed to map PMU interrupt for node\n"); - ret = -ENODEV; - goto err; + return -ENODEV; } - ret = request_percpu_irq(riscv_pmu_irq, pmu_sbi_ovf_handler, "riscv-pmu", hw_events); + ret = request_percpu_irq(riscv_pmu_irq, pmu_sbi_ovf_irq_handler, + "riscv-pmu", hw_events); if (ret) { pr_err("registering percpu irq failed [%d]\n", ret); irq_dispose_mapping(riscv_pmu_irq); riscv_pmu_irq = 0; - goto err; + return ret; } return 0; -err: - riscv_pmu_use_irq = false; - return ret; +} + +static int pmu_sbi_starting_cpu(unsigned int cpu, struct hlist_node *node) +{ + struct riscv_pmu *pmu = hlist_entry_safe(node, struct riscv_pmu, node); + struct cpu_hw_events *cpu_hw_evt = this_cpu_ptr(pmu->hw_events); + + /* + * We keep enabling userspace access to CYCLE, TIME and INSTRET via the + * legacy option but that will be removed in the future. + */ + if (sysctl_perf_user_access == SYSCTL_LEGACY) + csr_write(CSR_SCOUNTEREN, 0x7); + else + csr_write(CSR_SCOUNTEREN, 0x2); + + /* Stop all the counters so that they can be enabled from perf */ + pmu_sbi_stop_all(pmu); + + if (riscv_pmu_use_irq) { + cpu_hw_evt->irq = riscv_pmu_irq; + ALT_SBI_PMU_OVF_CLEAR_PENDING(riscv_pmu_irq_mask); + enable_percpu_irq(riscv_pmu_irq, IRQ_TYPE_NONE); + } + + if (sbi_pmu_snapshot_available()) + return pmu_sbi_snapshot_setup(pmu, cpu); + + return 0; +} + +static int pmu_sbi_dying_cpu(unsigned int cpu, struct hlist_node *node) +{ + if (riscv_pmu_use_irq) + disable_percpu_irq(riscv_pmu_irq); + + /* Disable all counters access for user mode now */ + csr_write(CSR_SCOUNTEREN, 0x0); + + if (sbi_pmu_snapshot_available()) + return pmu_sbi_snapshot_disable(); + + return 0; +} + +static int pmu_sbi_setup_irqs(struct riscv_pmu *pmu, struct platform_device *pdev) +{ + int irq_ret; + int sse_ret; + + /* Do not claim an IRQ route while a crash kernel leaves SSE state intact. */ + if (is_kdump_kernel() && riscv_sse_available()) { + pr_warn("PMU delivery unavailable with retained crash-kernel SSE state\n"); + return -EUCLEAN; + } + + sse_ret = pmu_sbi_setup_sse(pmu); + if (!sse_ret) { + riscv_pmu_use_irq = false; + return 0; + } + if (pmu_sbi_sse_state_retained(pmu)) { + pr_err("PMU-SSE setup failed with firmware state retained: %d\n", + sse_ret); + return -EUCLEAN; + } + /* Only an explicitly unsupported SSE path proves IRQ fallback is safe. */ + if (sse_ret != -EOPNOTSUPP) { + pr_err("PMU-SSE setup failed: %d\n", sse_ret); + return sse_ret; + } + + irq_ret = pmu_sbi_setup_irq(pmu); + if (!irq_ret) { + riscv_pmu_use_irq = true; + return 0; + } + return irq_ret; } #ifdef CONFIG_CPU_PM -static int riscv_pm_pmu_notify(struct notifier_block *b, unsigned long cmd, - void *v) +static int riscv_pm_pmu_update(struct riscv_pmu *rvpmu, unsigned long cmd) { - struct riscv_pmu *rvpmu = container_of(b, struct riscv_pmu, riscv_pm_nb); struct cpu_hw_events *cpuc = this_cpu_ptr(rvpmu->hw_events); bool enabled = !bitmap_empty(cpuc->used_hw_ctrs, RISCV_MAX_COUNTERS); struct perf_event *event; int idx; + if (cmd == CPU_PM_ENTER) + bitmap_zero(cpuc->pm_resume_hw_ctrs, RISCV_MAX_COUNTERS); + if (!enabled) return NOTIFY_OK; @@ -1264,6 +1559,8 @@ static int riscv_pm_pmu_notify(struct notifier_block *b, unsigned long cmd, switch (cmd) { case CPU_PM_ENTER: + if (!(event->hw.state & PERF_HES_STOPPED)) + set_bit(idx, cpuc->pm_resume_hw_ctrs); /* * Stop and update the counter */ @@ -1271,6 +1568,8 @@ static int riscv_pm_pmu_notify(struct notifier_block *b, unsigned long cmd, break; case CPU_PM_EXIT: case CPU_PM_ENTER_FAILED: + if (!test_and_clear_bit(idx, cpuc->pm_resume_hw_ctrs)) + break; /* * Restore and enable the counter. */ @@ -1284,9 +1583,59 @@ static int riscv_pm_pmu_notify(struct notifier_block *b, unsigned long cmd, return NOTIFY_OK; } +static int riscv_pm_pmu_notify(struct notifier_block *b, + unsigned long cmd, void *v) +{ + struct riscv_pmu *rvpmu = container_of(b, struct riscv_pmu, + riscv_pm_nb); + +#ifdef CONFIG_RISCV_PMU_SBI_SSE + struct cpu_hw_events *cpuc = this_cpu_ptr(rvpmu->hw_events); + int ret; + + if (!riscv_pmu_use_irq && READ_ONCE(rvpmu->sse_active)) { + switch (cmd) { + case CPU_PM_ENTER: + cpuc->pm_resume_sse = + sse_event_is_enabled_local(rvpmu->sse_evt); + if (cpuc->pm_resume_sse) { + ret = sse_event_disable_local(rvpmu->sse_evt); + if (ret) { + cpuc->pm_resume_sse = false; + pmu_sbi_fail_sse(rvpmu, "disable for CPU PM", + ret); + return notifier_from_errno(ret); + } + } + break; + case CPU_PM_EXIT: + case CPU_PM_ENTER_FAILED: + ret = riscv_pm_pmu_update(rvpmu, cmd); + if (!cpuc->pm_resume_sse) + return ret; + + cpuc->pm_resume_sse = false; + ret = sse_event_enable_local(rvpmu->sse_evt); + if (ret) { + pmu_sbi_fail_sse(rvpmu, "enable after CPU PM", ret); + return notifier_from_errno(ret); + } + return NOTIFY_OK; + default: + break; + } + } +#endif + + return riscv_pm_pmu_update(rvpmu, cmd); +} + static int riscv_pm_pmu_register(struct riscv_pmu *pmu) { pmu->riscv_pm_nb.notifier_call = riscv_pm_pmu_notify; + /* Keep PMU-SSE disabled until counters and userpage state are restored. */ + pmu->riscv_pm_nb.priority = riscv_pmu_use_irq ? 1 : -1; + return cpu_pm_register_notifier(&pmu->riscv_pm_nb); } @@ -1301,6 +1650,8 @@ static inline void riscv_pm_pmu_unregister(struct riscv_pmu *pmu) { } static void riscv_pmu_destroy(struct riscv_pmu *pmu) { + pmu_sbi_cleanup_sse(pmu); + if (sbi_v2_available) { if (sbi_pmu_snapshot_available()) { pmu_sbi_snapshot_disable(); @@ -1312,7 +1663,7 @@ static void riscv_pmu_destroy(struct riscv_pmu *pmu) cpuhp_state_remove_instance(CPUHP_AP_PERF_RISCV_STARTING, &pmu->node); } -static void pmu_sbi_event_init(struct perf_event *event) +static int pmu_sbi_event_init(struct perf_event *event) { /* * The permissions are set at event_init so that we do not depend @@ -1324,6 +1675,8 @@ static void pmu_sbi_event_init(struct perf_event *event) event->hw.flags |= PERF_EVENT_FLAG_USER_ACCESS; else event->hw.flags |= PERF_EVENT_FLAG_LEGACY; + + return 0; } static void pmu_sbi_event_mapped(struct perf_event *event, struct mm_struct *mm) @@ -1453,6 +1806,7 @@ static int pmu_sbi_device_probe(struct platform_device *pdev) /* cache all the information about counters now */ if (pmu_sbi_get_ctrinfo(num_counters, &cmask)) goto out_free; + pmu->cmask = cmask; ret = pmu_sbi_setup_irqs(pmu, pdev); if (ret < 0) { @@ -1462,9 +1816,15 @@ static int pmu_sbi_device_probe(struct platform_device *pdev) } irq_requested = (ret == 0); +#ifdef CONFIG_RISCV_PMU_SBI_SSE + if (pmu->sse_active) { + pmu->pmu.pmu_enable = pmu_sbi_sse_enable; + pmu->pmu.pmu_disable = pmu_sbi_sse_disable; + } +#endif + pmu->pmu.attr_groups = riscv_pmu_attr_groups; pmu->pmu.parent = &pdev->dev; - pmu->cmask = cmask; pmu->ctr_start = pmu_sbi_ctr_start; pmu->ctr_stop = pmu_sbi_ctr_stop; pmu->event_map = pmu_sbi_event_map; diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h index feb32949aeea6a..d13af475508d5f 100644 --- a/include/linux/cpuhotplug.h +++ b/include/linux/cpuhotplug.h @@ -200,6 +200,7 @@ enum cpuhp_state { CPUHP_AP_ARM_MVEBU_SYNC_CLOCKS, CPUHP_AP_ARM_CORESIGHT_ONLINE, CPUHP_AP_X86_INTEL_EPB_ONLINE, + CPUHP_AP_RISCV_SSE_ONLINE, CPUHP_AP_PERF_ONLINE, CPUHP_AP_PERF_X86_ONLINE, CPUHP_AP_PERF_X86_UNCORE_ONLINE, diff --git a/include/linux/perf/riscv_pmu.h b/include/linux/perf/riscv_pmu.h index f82a28040594bd..fb913ce7f65e5f 100644 --- a/include/linux/perf/riscv_pmu.h +++ b/include/linux/perf/riscv_pmu.h @@ -28,6 +28,8 @@ #define RISCV_PMU_CONFIG1_GUEST_EVENTS 0x1 +struct sse_event; + struct cpu_hw_events { /* currently enabled events */ int n_events; @@ -39,6 +41,18 @@ struct cpu_hw_events { DECLARE_BITMAP(used_hw_ctrs, RISCV_MAX_COUNTERS); /* currently enabled firmware counters */ DECLARE_BITMAP(used_fw_ctrs, RISCV_MAX_COUNTERS); +#ifdef CONFIG_RISCV_PMU_SBI_SSE + /* Keep counters stopped after an unrecoverable SSE transition failure. */ + bool sse_failed; +#endif +#ifdef CONFIG_CPU_PM + /* Counters stopped by CPU PM and still waiting to be restored. */ + DECLARE_BITMAP(pm_resume_hw_ctrs, RISCV_MAX_COUNTERS); +#ifdef CONFIG_RISCV_PMU_SBI_SSE + /* Restore the local PMU SSE event after counters and userpage state. */ + bool pm_resume_sse; +#endif +#endif /* The virtual address of the shared memory where counter snapshot will be taken */ void *snapshot_addr; /* The physical address of the shared memory where counter snapshot will be taken */ @@ -54,6 +68,10 @@ struct riscv_pmu { char *name; irqreturn_t (*handle_irq)(int irq_num, void *dev); +#ifdef CONFIG_RISCV_PMU_SBI_SSE + struct sse_event *sse_evt; + bool sse_active; +#endif unsigned long cmask; u64 (*ctr_read)(struct perf_event *event); @@ -63,7 +81,7 @@ struct riscv_pmu { void (*ctr_start)(struct perf_event *event, u64 init_val); void (*ctr_stop)(struct perf_event *event, unsigned long flag); int (*event_map)(struct perf_event *event, u64 *config); - void (*event_init)(struct perf_event *event); + int (*event_init)(struct perf_event *event); void (*event_mapped)(struct perf_event *event, struct mm_struct *mm); void (*event_unmapped)(struct perf_event *event, struct mm_struct *mm); uint8_t (*csr_index)(struct perf_event *event); diff --git a/include/linux/riscv_sbi_sse.h b/include/linux/riscv_sbi_sse.h new file mode 100644 index 00000000000000..f6b3881c8884bb --- /dev/null +++ b/include/linux/riscv_sbi_sse.h @@ -0,0 +1,95 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2025 Rivos Inc. + */ + +#ifndef __LINUX_RISCV_SBI_SSE_H +#define __LINUX_RISCV_SBI_SSE_H + +#include +#include +#include +#include + +struct sse_event; +struct pt_regs; + +typedef int (sse_event_handler_fn)(u32 event_num, void *arg, + struct pt_regs *regs); + +#ifdef CONFIG_RISCV_SBI_SSE + +/* + * The callback and its argument must remain valid until unregister succeeds. + * The callback runs in NMI context and must not sleep. + * regs is NULL if firmware cannot provide a complete interrupted context. + */ +struct sse_event *sse_event_register(u32 event_num, u32 priority, + sse_event_handler_fn *handler, void *arg); + +int sse_event_unregister(struct sse_event *evt); + +/* + * Transfer a retained event to the SSE core for deferred cleanup. The caller + * must not access the event or its callback data after this function returns. + */ +void sse_event_cleanup(struct sse_event *evt); + +int sse_event_set_target_cpu(struct sse_event *sse_evt, unsigned int cpu); + +int sse_event_enable(struct sse_event *sse_evt); + +int sse_event_disable(struct sse_event *sse_evt); + +/* Local events require the caller to remain on the current CPU. */ +bool sse_event_is_enabled_local(struct sse_event *sse_evt); +int sse_event_enable_local(struct sse_event *sse_evt); +int sse_event_disable_local(struct sse_event *sse_evt); + +#else +static inline struct sse_event *sse_event_register(u32 event_num, u32 priority, + sse_event_handler_fn *handler, + void *arg) +{ + return ERR_PTR(-EOPNOTSUPP); +} + +static inline int sse_event_unregister(struct sse_event *evt) +{ + return -EOPNOTSUPP; +} + +static inline void sse_event_cleanup(struct sse_event *evt) { } + +static inline int sse_event_set_target_cpu(struct sse_event *sse_evt, + unsigned int cpu) +{ + return -EOPNOTSUPP; +} + +static inline int sse_event_enable(struct sse_event *sse_evt) +{ + return -EOPNOTSUPP; +} + +static inline int sse_event_disable(struct sse_event *sse_evt) +{ + return -EOPNOTSUPP; +} + +static inline bool sse_event_is_enabled_local(struct sse_event *sse_evt) +{ + return false; +} + +static inline int sse_event_enable_local(struct sse_event *sse_evt) +{ + return -EOPNOTSUPP; +} + +static inline int sse_event_disable_local(struct sse_event *sse_evt) +{ + return -EOPNOTSUPP; +} +#endif +#endif /* __LINUX_RISCV_SBI_SSE_H */ diff --git a/tools/testing/selftests/riscv/Makefile b/tools/testing/selftests/riscv/Makefile index 5671b4405a1294..43c7c8f97676f8 100644 --- a/tools/testing/selftests/riscv/Makefile +++ b/tools/testing/selftests/riscv/Makefile @@ -5,7 +5,7 @@ ARCH ?= $(shell uname -m 2>/dev/null || echo not) ifneq (,$(filter $(ARCH),riscv)) -RISCV_SUBTARGETS ?= abi hwprobe mm sigreturn vector cfi +RISCV_SUBTARGETS ?= abi hwprobe mm sigreturn vector cfi sse else RISCV_SUBTARGETS := endif diff --git a/tools/testing/selftests/riscv/sse/Makefile b/tools/testing/selftests/riscv/sse/Makefile new file mode 100644 index 00000000000000..646b418de9ffb9 --- /dev/null +++ b/tools/testing/selftests/riscv/sse/Makefile @@ -0,0 +1,10 @@ +CFLAGS += -I$(top_srcdir)/tools/testing/selftests +LDLIBS += -lpthread + +TEST_GEN_MODS_DIR := module + +TEST_GEN_PROGS := sse_perf_ustack + +TEST_PROGS := run_sse_test.sh + +include ../../lib.mk diff --git a/tools/testing/selftests/riscv/sse/module/Makefile b/tools/testing/selftests/riscv/sse/module/Makefile new file mode 100644 index 00000000000000..eac4b1c6228bb6 --- /dev/null +++ b/tools/testing/selftests/riscv/sse/module/Makefile @@ -0,0 +1,22 @@ +ifneq ($(CONFIG_RISCV_SBI_SSE),) +obj-m += riscv_sse_test.o +endif + +ifndef KERNELRELEASE + +TESTMODS_DIR := $(realpath $(dir $(abspath $(lastword $(MAKEFILE_LIST))))) +KDIR ?= /lib/modules/$(shell uname -r)/build + +# Ensure that KDIR exists, otherwise skip the compilation +modules: +ifneq ("$(wildcard $(KDIR))", "") + $(Q)$(MAKE) -C $(KDIR) modules KBUILD_EXTMOD=$(TESTMODS_DIR) +endif + +# Ensure that KDIR exists, otherwise skip the clean target +clean: +ifneq ("$(wildcard $(KDIR))", "") + $(Q)$(MAKE) -C $(KDIR) clean KBUILD_EXTMOD=$(TESTMODS_DIR) +endif + +endif diff --git a/tools/testing/selftests/riscv/sse/module/riscv_sse_test.c b/tools/testing/selftests/riscv/sse/module/riscv_sse_test.c new file mode 100644 index 00000000000000..cc5c2e46f2fdb4 --- /dev/null +++ b/tools/testing/selftests/riscv/sse/module/riscv_sse_test.c @@ -0,0 +1,1154 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2025 Rivos Inc. + */ + +#define pr_fmt(fmt) "riscv_sse_test: " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#define RUN_LOOP_COUNT 1000 +#define SSE_FAILED_PREFIX "FAILED: " +#define SSE_SKIP_PREFIX "SKIP: " +#define STRESS_DURATION_MS 3000 +#define STRESS_INJECT_NS 10000 +#define STRESS_REINJECT_DEPTH 10 +#define sse_err(...) pr_err(SSE_FAILED_PREFIX __VA_ARGS__) +#define sse_skip(...) pr_info(SSE_SKIP_PREFIX __VA_ARGS__) + +enum sse_stress_mode { + SSE_STRESS_OFF, + SSE_STRESS_AFTER_SMOKE, + SSE_STRESS_ONLY, +}; + +static int stress; +module_param(stress, int, 0444); +MODULE_PARM_DESC(stress, "Stress mode: 0=off, 1=after smoke, 2=stress only"); + +static char *run_id = "unknown"; +module_param(run_id, charp, 0444); +MODULE_PARM_DESC(run_id, "Unique identifier used to delimit one test run"); + +/* Do not report PASS for a capability-only run that handled no event. */ +static atomic_t sse_test_handler_count = ATOMIC_INIT(0); +static bool sse_stress_event_can_inject; + +struct sse_event_desc { + u32 evt_id; + const char *name; + bool can_inject; +}; + +static struct sse_event_desc sse_event_descs[] = { + { + .evt_id = SBI_SSE_EVENT_LOCAL_HIGH_PRIO_RAS, + .name = "local_high_prio_ras", + }, + { + .evt_id = SBI_SSE_EVENT_LOCAL_DOUBLE_TRAP, + .name = "local_double_trap", + }, + { + .evt_id = SBI_SSE_EVENT_GLOBAL_HIGH_PRIO_RAS, + .name = "global_high_prio_ras", + }, + { + .evt_id = SBI_SSE_EVENT_LOCAL_PMU_OVERFLOW, + .name = "local_pmu_overflow", + }, + { + .evt_id = SBI_SSE_EVENT_LOCAL_LOW_PRIO_RAS, + .name = "local_low_prio_ras", + }, + { + .evt_id = SBI_SSE_EVENT_GLOBAL_LOW_PRIO_RAS, + .name = "global_low_prio_ras", + }, + { + .evt_id = SBI_SSE_EVENT_LOCAL_SOFTWARE_INJECTED, + .name = "local_software_injected", + }, + { + .evt_id = SBI_SSE_EVENT_GLOBAL_SOFTWARE_INJECTED, + .name = "global_software_injected", + } +}; + +static DEFINE_MUTEX(sse_test_cleanup_lock); +/* Firmware permits only one registration for each event ID. */ +static struct sse_event *sse_test_cleanup_events[ARRAY_SIZE(sse_event_descs)]; + +static void sse_test_cleanup_workfn(struct work_struct *work); +static DECLARE_DELAYED_WORK(sse_test_cleanup_work, sse_test_cleanup_workfn); + +static void sse_test_queue_cleanup(struct sse_event *event) +{ + int i, free_slot = -1; + + mutex_lock(&sse_test_cleanup_lock); + for (i = 0; i < ARRAY_SIZE(sse_test_cleanup_events); i++) { + if (sse_test_cleanup_events[i] == event) + goto out_schedule; + if (!sse_test_cleanup_events[i] && free_slot < 0) + free_slot = i; + } + + if (WARN_ON_ONCE(free_slot < 0)) + goto out_unlock; + + sse_test_cleanup_events[free_slot] = event; + +out_schedule: + mod_delayed_work(system_wq, &sse_test_cleanup_work, + msecs_to_jiffies(100)); +out_unlock: + mutex_unlock(&sse_test_cleanup_lock); +} + +static struct sse_event_desc *sse_get_evt_desc(u32 evt) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(sse_event_descs); i++) { + if (sse_event_descs[i].evt_id == evt) + return &sse_event_descs[i]; + } + + return NULL; +} + +static const char *sse_evt_name(u32 evt) +{ + struct sse_event_desc *desc = sse_get_evt_desc(evt); + + return desc ? desc->name : NULL; +} + +static bool sse_test_can_inject_event(u32 evt) +{ + struct sse_event_desc *desc = sse_get_evt_desc(evt); + + return desc ? desc->can_inject : false; +} + +/* + * Firmware can invoke the callback until unregister succeeds. Pin the module + * so its handler text cannot disappear first. + */ +static struct sse_event *sse_test_event_register(u32 evt, u32 priority, + sse_event_handler_fn *handler, + void *arg) +{ + struct sse_event *event; + + event = sse_event_register(evt, priority, handler, arg); + if (!IS_ERR(event)) + __module_get(THIS_MODULE); + + return event; +} + +static int sse_test_event_unregister(struct sse_event *event) +{ + int ret; + + ret = sse_event_unregister(event); + if (!ret) + module_put(THIS_MODULE); + else + sse_test_queue_cleanup(event); + + return ret; +} + +static void sse_test_cleanup_workfn(struct work_struct *work) +{ + bool retry = false; + int i, ret; + + mutex_lock(&sse_test_cleanup_lock); + for (i = 0; i < ARRAY_SIZE(sse_test_cleanup_events); i++) { + struct sse_event *event = sse_test_cleanup_events[i]; + + if (!event) + continue; + + ret = sse_event_disable(event); + if (!ret) + ret = sse_event_unregister(event); + if (ret) { + retry = true; + continue; + } + + sse_test_cleanup_events[i] = NULL; + module_put(THIS_MODULE); + } + mutex_unlock(&sse_test_cleanup_lock); + + if (retry) + mod_delayed_work(system_wq, &sse_test_cleanup_work, + msecs_to_jiffies(100)); +} + +static struct sbiret sbi_sse_ecall(int fid, unsigned long arg0, unsigned long arg1) +{ + return sbi_ecall(SBI_EXT_SSE, fid, arg0, arg1, 0, 0, 0, 0); +} + +static int sse_event_attr_read(u32 evt, unsigned long attr_id, + unsigned long *attr_buf) +{ + struct sbiret sret; + phys_addr_t phys; + + phys = virt_to_phys(attr_buf); + + sret = sbi_ecall(SBI_EXT_SSE, SBI_SSE_EVENT_ATTR_READ, evt, attr_id, 1, + (unsigned long)phys, 0, 0); + if (sret.error) + return sbi_err_map_linux_errno(sret.error); + + return 0; +} + +static int sse_event_attr_get(u32 evt, unsigned long attr_id, + unsigned long *val) +{ + unsigned long *attr_buf; + int ret; + + attr_buf = kmalloc_obj(*attr_buf, GFP_KERNEL); + if (!attr_buf) + return -ENOMEM; + + ret = sse_event_attr_read(evt, attr_id, attr_buf); + if (!ret) + *val = *attr_buf; + kfree(attr_buf); + + return ret; +} + +static int sse_test_signal(u32 evt, unsigned int cpu) +{ + unsigned long hart_id = cpuid_to_hartid_map(cpu); + struct sbiret ret; + + ret = sbi_sse_ecall(SBI_SSE_EVENT_INJECT, evt, hart_id); + if (ret.error) { + sse_err("Failed to signal event %x, error %ld\n", evt, ret.error); + return sbi_err_map_linux_errno(ret.error); + } + + return 0; +} + +static int sse_test_wait_not_running(u32 evt) +{ + unsigned long timeout = jiffies + HZ; + unsigned long status; + int ret; + + do { + ret = sse_event_attr_get(evt, SBI_SSE_ATTR_STATUS, &status); + if (ret) { + sse_err("Failed to get status for evt %x, error %d\n", evt, ret); + return ret; + } + status &= SBI_SSE_ATTR_STATUS_STATE_MASK; + cpu_relax(); + } while (status == SBI_SSE_STATE_RUNNING && time_before(jiffies, timeout)); + + if (status == SBI_SSE_STATE_RUNNING) { + sse_err("Timed out waiting for event %x to leave RUNNING state\n", evt); + return -ETIMEDOUT; + } + + return 0; +} + +struct sse_test_wait_status { + u32 evt; + unsigned long *attr_buf; + unsigned long status; + int ret; +}; + +static void sse_test_read_status_local(void *info) +{ + struct sse_test_wait_status *wait = info; + + wait->ret = sse_event_attr_read(wait->evt, SBI_SSE_ATTR_STATUS, + wait->attr_buf); + if (!wait->ret) + wait->status = *wait->attr_buf & SBI_SSE_ATTR_STATUS_STATE_MASK; +} + +static int sse_test_wait_not_running_on_cpu(u32 evt, unsigned int cpu) +{ + struct sse_test_wait_status wait = { .evt = evt }; + unsigned long timeout; + int ret = 0; + + if (sse_event_is_global(evt)) + return sse_test_wait_not_running(evt); + + wait.attr_buf = kmalloc_obj(*wait.attr_buf, GFP_KERNEL); + if (!wait.attr_buf) + return -ENOMEM; + + timeout = jiffies + HZ; + do { + ret = smp_call_function_single(cpu, sse_test_read_status_local, + &wait, true); + if (ret || wait.ret) { + ret = ret ?: wait.ret; + break; + } + if (wait.status != SBI_SSE_STATE_RUNNING) + break; + usleep_range(100, 200); + } while (time_before(jiffies, timeout)); + + if (!ret && wait.status == SBI_SSE_STATE_RUNNING) { + sse_err("Timed out waiting for event %x on CPU %u\n", evt, cpu); + ret = -ETIMEDOUT; + } + + kfree(wait.attr_buf); + + return ret; +} + +static int sse_test_inject_event(struct sse_event *event, u32 evt, unsigned int cpu) +{ + int res; + + if (sse_event_is_global(evt)) { + /* + * Due to the fact the completion might happen faster than + * the call to SBI_SSE_COMPLETE in the handler, if the event was + * running on another CPU, we need to wait for the event status + * to be !RUNNING. + */ + res = sse_test_wait_not_running(evt); + if (res) + return res; + + res = sse_event_set_target_cpu(event, cpu); + if (res) { + sse_err("Failed to set cpu for evt %x, error %d\n", evt, res); + return res; + } + } + + return sse_test_signal(evt, cpu); +} + +struct fast_test_arg { + u32 evt; + int cpu; + bool args_ready; + bool completion; +}; + +/* A failed unregister may leave firmware holding this handler argument. */ +static struct fast_test_arg fast_test_arg; + +static int sse_test_handler(u32 evt, void *arg, struct pt_regs *regs) +{ + int ret = 0; + struct fast_test_arg *targ = arg; + u32 test_evt; + int cpu; + + atomic_inc(&sse_test_handler_count); + + /* Pairs with the argument publication in sse_run_fast_test_cpu(). */ + if (!smp_load_acquire(&targ->args_ready)) { + sse_err("Received SSE event %x before its test arguments were published\n", + evt); + ret = -EINVAL; + goto complete; + } + + test_evt = READ_ONCE(targ->evt); + cpu = READ_ONCE(targ->cpu); + + if (evt != test_evt) { + sse_err("Received SSE event id %x instead of %x\n", test_evt, evt); + ret = -EINVAL; + } + + if (!sse_event_is_global(evt) && cpu != smp_processor_id()) { + sse_err("Received SSE event %d on CPU %d instead of %d\n", evt, smp_processor_id(), + cpu); + ret = -EINVAL; + } + +complete: + WRITE_ONCE(targ->args_ready, false); + /* Publish handler-side checks before waking the waiting CPU. */ + smp_store_release(&targ->completion, true); + + return ret; +} + +static int sse_run_fast_test_cpu(struct fast_test_arg *test_arg, + struct sse_event *event, u32 evt, int cpu) +{ + unsigned long timeout; + int ret; + + WRITE_ONCE(test_arg->completion, false); + WRITE_ONCE(test_arg->args_ready, false); + WRITE_ONCE(test_arg->evt, evt); + WRITE_ONCE(test_arg->cpu, cpu); + /* Publish all arguments before firmware can inject on another hart. */ + smp_store_release(&test_arg->args_ready, true); + + ret = sse_test_inject_event(event, evt, cpu); + if (ret) { + sse_err("event %s injection failed, err %d\n", + sse_evt_name(evt), ret); + return ret; + } + + timeout = jiffies + HZ / 100; + /* We can not use since they are not NMI safe */ + /* Pairs with the handler's completion publication. */ + while (!smp_load_acquire(&test_arg->completion) && + time_before(jiffies, timeout)) + cpu_relax(); + /* Acquire the handler's checks even if the loop observed a timeout. */ + if (!smp_load_acquire(&test_arg->completion)) { + sse_err("Failed to wait for event %s completion on CPU %d\n", + sse_evt_name(evt), cpu); + return -ETIMEDOUT; + } + + return sse_test_wait_not_running_on_cpu(evt, cpu); +} + +static void sse_run_fast_test(struct fast_test_arg *test_arg, + struct sse_event *event, u32 evt) +{ + int cpu; + + if (sse_event_is_global(evt)) { + /* Keep the selected target online through injection and completion. */ + cpu_hotplug_disable(); + for_each_online_cpu(cpu) { + if (sse_run_fast_test_cpu(test_arg, event, evt, cpu)) + break; + } + cpu_hotplug_enable(); + return; + } + + guard(cpus_read_lock)(); + for_each_online_cpu(cpu) { + if (sse_run_fast_test_cpu(test_arg, event, evt, cpu)) + return; + } +} + +static void sse_test_injection_fast(void) +{ + int i, ret = 0, j; + u32 evt; + struct sse_event *event; + + pr_info("Starting SSE test (fast)\n"); + + for (i = 0; i < ARRAY_SIZE(sse_event_descs); i++) { + evt = sse_event_descs[i].evt_id; + WRITE_ONCE(fast_test_arg.evt, evt); + WRITE_ONCE(fast_test_arg.cpu, -1); + WRITE_ONCE(fast_test_arg.args_ready, false); + WRITE_ONCE(fast_test_arg.completion, false); + + if (!sse_event_descs[i].can_inject) + continue; + + event = sse_test_event_register(evt, 0, sse_test_handler, + (void *)&fast_test_arg); + if (IS_ERR(event)) { + if (PTR_ERR(event) == -EEXIST) { + pr_info("Event %s already registered, skipping\n", + sse_evt_name(evt)); + continue; + } + sse_err("Failed to register event %s, err %ld\n", sse_evt_name(evt), + PTR_ERR(event)); + continue; + } + + ret = sse_event_enable(event); + if (ret) { + sse_err("Failed to enable event %s, err %d\n", sse_evt_name(evt), ret); + goto err_disable; + } + + pr_info("Starting testing event %s\n", sse_evt_name(evt)); + + for (j = 0; j < RUN_LOOP_COUNT; j++) + sse_run_fast_test(&fast_test_arg, event, evt); + pr_info("Finished testing event %s\n", sse_evt_name(evt)); + +err_disable: + ret = sse_event_disable(event); + if (ret) + sse_err("Failed to disable event %s, err %d\n", + sse_evt_name(evt), ret); + ret = sse_test_event_unregister(event); + if (ret) { + sse_err("Failed to unregister event %s, err %d\n", + sse_evt_name(evt), ret); + return; + } + } + pr_info("Finished SSE test (fast)\n"); +} + +struct priority_test_arg { + unsigned long evt; + struct sse_event *event; + bool called; + bool enable_attempted; + u32 prio; + struct priority_test_arg *next_evt_arg; + void (*check_func)(struct priority_test_arg *arg); +}; + +/* A failed unregister may leave firmware holding these handler arguments. */ +static struct priority_test_arg default_hi_prio_args[] = { + { .evt = SBI_SSE_EVENT_GLOBAL_SOFTWARE_INJECTED }, + { .evt = SBI_SSE_EVENT_LOCAL_SOFTWARE_INJECTED }, + { .evt = SBI_SSE_EVENT_GLOBAL_LOW_PRIO_RAS }, + { .evt = SBI_SSE_EVENT_LOCAL_LOW_PRIO_RAS }, + { .evt = SBI_SSE_EVENT_LOCAL_PMU_OVERFLOW }, + { .evt = SBI_SSE_EVENT_GLOBAL_HIGH_PRIO_RAS }, + { .evt = SBI_SSE_EVENT_LOCAL_DOUBLE_TRAP }, + { .evt = SBI_SSE_EVENT_LOCAL_HIGH_PRIO_RAS }, +}; + +static struct priority_test_arg default_low_prio_args[] = { + { .evt = SBI_SSE_EVENT_LOCAL_HIGH_PRIO_RAS }, + { .evt = SBI_SSE_EVENT_LOCAL_DOUBLE_TRAP }, + { .evt = SBI_SSE_EVENT_GLOBAL_HIGH_PRIO_RAS }, + { .evt = SBI_SSE_EVENT_LOCAL_PMU_OVERFLOW }, + { .evt = SBI_SSE_EVENT_LOCAL_LOW_PRIO_RAS }, + { .evt = SBI_SSE_EVENT_GLOBAL_LOW_PRIO_RAS }, + { .evt = SBI_SSE_EVENT_LOCAL_SOFTWARE_INJECTED }, + { .evt = SBI_SSE_EVENT_GLOBAL_SOFTWARE_INJECTED }, +}; + +static struct priority_test_arg set_prio_args[] = { + { .evt = SBI_SSE_EVENT_GLOBAL_SOFTWARE_INJECTED, .prio = 5 }, + { .evt = SBI_SSE_EVENT_LOCAL_SOFTWARE_INJECTED, .prio = 10 }, + { .evt = SBI_SSE_EVENT_GLOBAL_LOW_PRIO_RAS, .prio = 15 }, + { .evt = SBI_SSE_EVENT_LOCAL_LOW_PRIO_RAS, .prio = 20 }, + { .evt = SBI_SSE_EVENT_LOCAL_PMU_OVERFLOW, .prio = 25 }, + { .evt = SBI_SSE_EVENT_GLOBAL_HIGH_PRIO_RAS, .prio = 30 }, + { .evt = SBI_SSE_EVENT_LOCAL_DOUBLE_TRAP, .prio = 35 }, + { .evt = SBI_SSE_EVENT_LOCAL_HIGH_PRIO_RAS, .prio = 40 }, +}; + +static struct priority_test_arg same_prio_args[] = { + { .evt = SBI_SSE_EVENT_LOCAL_PMU_OVERFLOW, .prio = 0 }, + { .evt = SBI_SSE_EVENT_LOCAL_HIGH_PRIO_RAS, .prio = 10 }, + { .evt = SBI_SSE_EVENT_LOCAL_SOFTWARE_INJECTED, .prio = 10 }, + { .evt = SBI_SSE_EVENT_GLOBAL_SOFTWARE_INJECTED, .prio = 10 }, + { .evt = SBI_SSE_EVENT_GLOBAL_HIGH_PRIO_RAS, .prio = 20 }, +}; + +static int sse_hi_priority_test_handler(u32 evt, void *arg, + struct pt_regs *regs) +{ + struct priority_test_arg *targ = arg; + struct priority_test_arg *next = READ_ONCE(targ->next_evt_arg); + + atomic_inc(&sse_test_handler_count); + WRITE_ONCE(targ->called, 1); + + if (next) { + sse_test_signal(next->evt, smp_processor_id()); + if (!READ_ONCE(next->called)) { + sse_err("Higher priority event %s was not handled %s\n", + sse_evt_name(next->evt), sse_evt_name(evt)); + } + } + + return 0; +} + +static int sse_low_priority_test_handler(u32 evt, void *arg, struct pt_regs *regs) +{ + struct priority_test_arg *targ = arg; + struct priority_test_arg *next = READ_ONCE(targ->next_evt_arg); + + atomic_inc(&sse_test_handler_count); + WRITE_ONCE(targ->called, 1); + + if (next) { + sse_test_signal(next->evt, smp_processor_id()); + if (READ_ONCE(next->called)) { + sse_err("Lower priority event %s was handle before %s\n", + sse_evt_name(next->evt), sse_evt_name(evt)); + } + } + + return 0; +} + +static void sse_test_injection_priority_arg(struct priority_test_arg *args, unsigned int args_size, + sse_event_handler_fn handler, const char *test_name) +{ + unsigned int i; + unsigned long timeout; + int ret; + int target_cpu; + struct sse_event *event; + struct priority_test_arg *arg, *first_arg = NULL, *prev_arg = NULL; + + pr_info("Starting SSE priority test (%s)\n", test_name); + /* Keep the complete priority chain on one CPU. */ + migrate_disable(); + target_cpu = smp_processor_id(); + + for (i = 0; i < args_size; i++) { + arg = &args[i]; + + if (!sse_test_can_inject_event(arg->evt)) + continue; + + WRITE_ONCE(arg->called, false); + WRITE_ONCE(arg->next_evt_arg, NULL); + WRITE_ONCE(arg->event, NULL); + WRITE_ONCE(arg->enable_attempted, false); + + event = sse_test_event_register(arg->evt, arg->prio, handler, + (void *)arg); + if (IS_ERR(event)) { + if (PTR_ERR(event) == -EEXIST) { + pr_info("Event %s already registered, skipping\n", + sse_evt_name(arg->evt)); + continue; + } + sse_err("Failed to register event %s, err %ld\n", sse_evt_name(arg->evt), + PTR_ERR(event)); + goto release_events; + } + arg->event = event; + + if (sse_event_is_global(arg->evt)) { + /* Keep the chain on one stable CPU. */ + ret = sse_event_set_target_cpu(event, target_cpu); + if (ret) { + sse_err("Failed to set event %s target CPU, err %d\n", + sse_evt_name(arg->evt), ret); + goto release_events; + } + } + + WRITE_ONCE(arg->enable_attempted, true); + ret = sse_event_enable(event); + if (ret) { + sse_err("Failed to enable event %s, err %d\n", sse_evt_name(arg->evt), ret); + goto release_events; + } + + if (prev_arg) + WRITE_ONCE(prev_arg->next_evt_arg, arg); + + prev_arg = arg; + + if (!first_arg) + first_arg = arg; + } + + if (!first_arg) { + pr_info("No injectable event available for %s priority test\n", + test_name); + goto out; + } + + /* Inject first event, handler should trigger the others in chain. */ + ret = sse_test_inject_event(first_arg->event, first_arg->evt, target_cpu); + if (ret) { + sse_err("SSE event %s injection failed\n", sse_evt_name(first_arg->evt)); + goto release_events; + } + + /* Lower-priority events run after the handler that injected them completes. */ + arg = first_arg; + while (arg) { + timeout = jiffies + HZ; + while (!READ_ONCE(arg->called) && time_before(jiffies, timeout)) + cpu_relax(); + + if (!READ_ONCE(arg->called)) { + sse_err("Event %s handler was not called\n", + sse_evt_name(arg->evt)); + ret = -EINVAL; + } + + event = arg->event; + arg = READ_ONCE(arg->next_evt_arg); + } + +release_events: + + for (i = 0; i < args_size; i++) { + arg = &args[i]; + event = arg->event; + if (!event) + continue; + + ret = sse_test_wait_not_running_on_cpu(arg->evt, target_cpu); + if (ret) + sse_err("Event %s did not complete, err %d\n", + sse_evt_name(arg->evt), ret); + + if (arg->enable_attempted) { + ret = sse_event_disable(event); + if (ret) { + sse_err("Failed to disable event %s, err %d\n", + sse_evt_name(arg->evt), ret); + sse_test_queue_cleanup(event); + WRITE_ONCE(arg->event, NULL); + WRITE_ONCE(arg->enable_attempted, false); + continue; + } + } + + ret = sse_test_event_unregister(event); + if (ret) { + sse_err("Failed to unregister event %s, err %d\n", + sse_evt_name(arg->evt), ret); + continue; + } + + WRITE_ONCE(arg->event, NULL); + WRITE_ONCE(arg->enable_attempted, false); + } + + pr_info("Finished SSE priority test (%s)\n", test_name); +out: + migrate_enable(); +} + +static void sse_test_injection_priority(void) +{ + sse_test_injection_priority_arg(default_hi_prio_args, ARRAY_SIZE(default_hi_prio_args), + sse_hi_priority_test_handler, "high"); + + sse_test_injection_priority_arg(default_low_prio_args, ARRAY_SIZE(default_low_prio_args), + sse_low_priority_test_handler, "low"); + + sse_test_injection_priority_arg(set_prio_args, ARRAY_SIZE(set_prio_args), + sse_low_priority_test_handler, "set"); + + sse_test_injection_priority_arg(same_prio_args, ARRAY_SIZE(same_prio_args), + sse_low_priority_test_handler, "same_prio_args"); +} + +static int sse_get_inject_status(u32 evt, bool *can_inject) +{ + int ret; + unsigned long val; + + /* Check if injection is supported */ + ret = sse_event_attr_get(evt, SBI_SSE_ATTR_STATUS, &val); + if (ret == sbi_err_map_linux_errno(SBI_ERR_NOT_SUPPORTED) || + ret == sbi_err_map_linux_errno(SBI_ERR_INVALID_PARAM)) { + *can_inject = false; + return 0; + } + if (ret) + return ret; + + *can_inject = !!(val & BIT(SBI_SSE_ATTR_STATUS_INJECT_OFFSET)); + + return 0; +} + +static int sse_init_events(void) +{ + int i, injectable = 0, ret; + + for (i = 0; i < ARRAY_SIZE(sse_event_descs); i++) { + struct sse_event_desc *desc = &sse_event_descs[i]; + + ret = sse_get_inject_status(desc->evt_id, &desc->can_inject); + if (ret) { + sse_err("Failed to read injection status for %s, err %d\n", + desc->name, ret); + return ret; + } + + if (desc->can_inject) + injectable++; + else + pr_info("Can not inject event %s, tests using this event will be skipped\n", + desc->name); + + if (desc->evt_id == SBI_SSE_EVENT_LOCAL_SOFTWARE_INJECTED) + sse_stress_event_can_inject = desc->can_inject; + } + + return injectable; +} + +struct stress_test_ctx { + struct sse_event *event; + struct hrtimer timer; + struct hrtimer stop_timer; + struct task_struct *monitor_task; + wait_queue_head_t wait_q; + atomic_t inject_count; + atomic_t handler_count; + atomic_t handler_errors; + u32 evt_id; + int layer; + bool running; + bool test_done; +}; + +static struct stress_test_ctx stress_ctx; +static DEFINE_PER_CPU(int, stress_reinject_cpu_depth); + +static int stress_handler_empty(u32 evt, void *arg, struct pt_regs *regs) +{ + struct stress_test_ctx *ctx = arg; + + atomic_inc(&sse_test_handler_count); + atomic_inc(&ctx->handler_count); + + return 0; +} + +static int stress_handler_ecall(u32 evt, void *arg, struct pt_regs *regs) +{ + struct stress_test_ctx *ctx = arg; + struct sbiret ret; + + ret = sbi_ecall(SBI_EXT_BASE, SBI_EXT_BASE_GET_SPEC_VERSION, + 0, 0, 0, 0, 0, 0); + if (ret.error) + atomic_inc(&ctx->handler_errors); + atomic_inc(&sse_test_handler_count); + atomic_inc(&ctx->handler_count); + + return 0; +} + +static int stress_handler_multi_ecall(u32 evt, void *arg, struct pt_regs *regs) +{ + struct stress_test_ctx *ctx = arg; + struct sbiret ret; + + ret = sbi_ecall(SBI_EXT_BASE, SBI_EXT_BASE_GET_SPEC_VERSION, + 0, 0, 0, 0, 0, 0); + if (ret.error) + atomic_inc(&ctx->handler_errors); + ret = sbi_ecall(SBI_EXT_BASE, SBI_EXT_BASE_GET_IMP_ID, + 0, 0, 0, 0, 0, 0); + if (ret.error) + atomic_inc(&ctx->handler_errors); + ret = sbi_ecall(SBI_EXT_BASE, SBI_EXT_BASE_GET_IMP_VERSION, + 0, 0, 0, 0, 0, 0); + if (ret.error) + atomic_inc(&ctx->handler_errors); + atomic_inc(&sse_test_handler_count); + atomic_inc(&ctx->handler_count); + + return 0; +} + +static int stress_handler_reinject(u32 evt, void *arg, struct pt_regs *regs) +{ + struct stress_test_ctx *ctx = arg; + int *depth = this_cpu_ptr(&stress_reinject_cpu_depth); + + (*depth)++; + if (*depth < STRESS_REINJECT_DEPTH) + sse_test_signal(evt, smp_processor_id()); + else + *depth = 0; + + atomic_inc(&sse_test_handler_count); + atomic_inc(&ctx->handler_count); + + return 0; +} + +static sse_event_handler_fn *stress_handlers[] = { + stress_handler_empty, + stress_handler_ecall, + stress_handler_multi_ecall, + stress_handler_reinject, +}; + +static const char * const stress_layer_names[] = { + "empty handler", + "single SBI ecall in handler", + "multiple SBI ecalls in handler", + "self re-inject", +}; + +static enum hrtimer_restart stress_timer_callback(struct hrtimer *timer) +{ + struct stress_test_ctx *ctx = container_of(timer, struct stress_test_ctx, timer); + + if (!READ_ONCE(ctx->running)) + return HRTIMER_NORESTART; + + if (!sse_test_signal(ctx->evt_id, smp_processor_id())) + atomic_inc(&ctx->inject_count); + hrtimer_forward_now(timer, ns_to_ktime(STRESS_INJECT_NS)); + + return HRTIMER_RESTART; +} + +static enum hrtimer_restart stress_stop_timer_callback(struct hrtimer *timer) +{ + struct stress_test_ctx *ctx; + + ctx = container_of(timer, struct stress_test_ctx, stop_timer); + WRITE_ONCE(ctx->test_done, true); + wake_up(&ctx->wait_q); + + return HRTIMER_NORESTART; +} + +static int stress_monitor_thread(void *data) +{ + struct stress_test_ctx *ctx = data; + unsigned long last_inject = 0, last_handler = 0; + + while (!kthread_should_stop()) { + unsigned long inject = atomic_read(&ctx->inject_count); + unsigned long handler = atomic_read(&ctx->handler_count); + + pr_info("stress layer %d: inject=%lu (+%lu), handler=%lu (+%lu)\n", + ctx->layer, inject, inject - last_inject, + handler, handler - last_handler); + + last_inject = inject; + last_handler = handler; + + schedule_timeout_interruptible(HZ); + } + + return 0; +} + +static int sse_stress_test_layer(int layer) +{ + struct sse_event *event; + int inject_count, handler_count; + int ret, target_cpu, unregister_ret; + + if (layer < 0 || layer >= ARRAY_SIZE(stress_handlers)) + return -EINVAL; + + pr_info("Starting SSE stress layer %d (%s)\n", + layer, stress_layer_names[layer]); + + memset(&stress_ctx, 0, sizeof(stress_ctx)); + stress_ctx.evt_id = SBI_SSE_EVENT_LOCAL_SOFTWARE_INJECTED; + stress_ctx.layer = layer; + WRITE_ONCE(stress_ctx.running, true); + atomic_set(&stress_ctx.inject_count, 0); + atomic_set(&stress_ctx.handler_count, 0); + atomic_set(&stress_ctx.handler_errors, 0); + init_waitqueue_head(&stress_ctx.wait_q); + + event = sse_test_event_register(stress_ctx.evt_id, 0, + stress_handlers[layer], &stress_ctx); + if (IS_ERR(event)) { + sse_err("Failed to register stress event, err %ld\n", + PTR_ERR(event)); + return PTR_ERR(event); + } + + stress_ctx.event = event; + + ret = sse_event_enable(event); + if (ret) { + sse_err("Failed to enable stress event, err %d\n", ret); + goto err_disable; + } + + stress_ctx.monitor_task = kthread_run(stress_monitor_thread, + &stress_ctx, "sse_stress_mon"); + if (IS_ERR(stress_ctx.monitor_task)) { + ret = PTR_ERR(stress_ctx.monitor_task); + sse_err("Failed to create stress monitor thread, err %d\n", ret); + goto err_disable; + } + + /* Keep the pinned timer and its local event on the selected CPU. */ + cpus_read_lock(); + migrate_disable(); + target_cpu = smp_processor_id(); + hrtimer_setup(&stress_ctx.timer, stress_timer_callback, + CLOCK_MONOTONIC, HRTIMER_MODE_PINNED); + hrtimer_start(&stress_ctx.timer, ns_to_ktime(STRESS_INJECT_NS), + HRTIMER_MODE_REL_PINNED); + migrate_enable(); + + hrtimer_setup(&stress_ctx.stop_timer, stress_stop_timer_callback, + CLOCK_MONOTONIC, HRTIMER_MODE_REL); + hrtimer_start(&stress_ctx.stop_timer, ms_to_ktime(STRESS_DURATION_MS), + HRTIMER_MODE_REL); + + wait_event(stress_ctx.wait_q, READ_ONCE(stress_ctx.test_done)); + + WRITE_ONCE(stress_ctx.running, false); + hrtimer_cancel(&stress_ctx.timer); + hrtimer_cancel(&stress_ctx.stop_timer); + kthread_stop(stress_ctx.monitor_task); + + pr_info("Finished SSE stress layer %d (%s): inject=%d, handler=%d\n", + layer, stress_layer_names[layer], + atomic_read(&stress_ctx.inject_count), + atomic_read(&stress_ctx.handler_count)); + + inject_count = atomic_read(&stress_ctx.inject_count); + handler_count = atomic_read(&stress_ctx.handler_count); + if (!inject_count || !handler_count) { + sse_err("Stress layer %d made no progress: inject=%d, handler=%d\n", + layer, inject_count, handler_count); + ret = -EIO; + } + if (atomic_read(&stress_ctx.handler_errors)) { + sse_err("Stress layer %d observed %d SBI call errors\n", layer, + atomic_read(&stress_ctx.handler_errors)); + ret = -EIO; + } + if (sse_test_wait_not_running_on_cpu(stress_ctx.evt_id, target_cpu)) { + sse_err("Stress event did not complete on CPU %d\n", target_cpu); + if (!ret) + ret = -ETIMEDOUT; + } + cpus_read_unlock(); + +err_disable: + if (sse_event_disable(event)) { + sse_err("Failed to disable stress event\n"); + if (!ret) + ret = -EIO; + } + unregister_ret = sse_test_event_unregister(event); + if (unregister_ret) { + sse_err("Failed to unregister stress event\n"); + if (!ret) + ret = unregister_ret; + } + stress_ctx.event = NULL; + + return ret; +} + +static void sse_stress_test_all_layers(void) +{ + int i, ret; + + pr_info("Starting SSE stress tests: duration=%d ms, interval=%d ns\n", + STRESS_DURATION_MS, STRESS_INJECT_NS); + + for (i = 0; i < ARRAY_SIZE(stress_handlers); i++) { + ret = sse_stress_test_layer(i); + if (ret) { + sse_err("Stress layer %d failed, err %d\n", i, ret); + break; + } + + msleep(100); + } + + pr_info("Finished SSE stress tests\n"); +} + +static int __init sse_test_init(void) +{ + int ret; + + pr_info("RUN %s BEGIN\n", run_id); + atomic_set(&sse_test_handler_count, 0); + + if (stress < SSE_STRESS_OFF || stress > SSE_STRESS_ONLY) { + sse_err("Invalid stress mode %d\n", stress); + pr_info("RUN %s END\n", run_id); + return -EINVAL; + } + + ret = sse_init_events(); + if (ret < 0) { + pr_info("RUN %s END\n", run_id); + return ret; + } + if (!ret) { + sse_skip("No injectable SSE event is available\n"); + pr_info("RUN %s END\n", run_id); + return 0; + } + if (stress == SSE_STRESS_ONLY && !sse_stress_event_can_inject) { + sse_skip("Local software-injected event is unavailable for stress\n"); + pr_info("RUN %s END\n", run_id); + return 0; + } + + if (stress != SSE_STRESS_ONLY) { + sse_test_injection_fast(); + sse_test_injection_priority(); + } + + if (stress == SSE_STRESS_AFTER_SMOKE && !sse_stress_event_can_inject) + sse_skip("Local software-injected event is unavailable for stress\n"); + else if (stress != SSE_STRESS_OFF) + sse_stress_test_all_layers(); + if (!atomic_read(&sse_test_handler_count)) + sse_skip("No SSE event was handled\n"); + + pr_info("RUN %s END\n", run_id); + + return 0; +} + +static void __exit sse_test_exit(void) +{ + cancel_delayed_work_sync(&sse_test_cleanup_work); +} + +module_init(sse_test_init); +module_exit(sse_test_exit); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Clément Léger "); +MODULE_DESCRIPTION("Test module for SSE"); diff --git a/tools/testing/selftests/riscv/sse/run_sse_test.sh b/tools/testing/selftests/riscv/sse/run_sse_test.sh new file mode 100644 index 00000000000000..e70a2fd14b05e7 --- /dev/null +++ b/tools/testing/selftests/riscv/sse/run_sse_test.sh @@ -0,0 +1,59 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 +# +# Copyright (C) 2025 Rivos Inc. + +MODULE_NAME=riscv_sse_test +DRIVER="./module/${MODULE_NAME}.ko" +ksft_skip=4 + +check_test_requirements() +{ + uid=$(id -u) + if [ $uid -ne 0 ]; then + echo "$0: Must be run as root" + exit $ksft_skip + fi + + if ! which insmod > /dev/null 2>&1; then + echo "$0: You need insmod installed" + exit $ksft_skip + fi + + if [ ! -f "$DRIVER" ]; then + echo "$0: SSE is disabled or ${MODULE_NAME} is not built" + exit $ksft_skip + fi +} + +check_test_requirements +run_id="$$-$(date +%s)" + +if ! insmod "$DRIVER" run_id="$run_id" "$@" > /dev/null 2>&1; then + echo "${MODULE_NAME}: failed to load, please check dmesg" + exit 1 +fi + +if ! rmmod "$MODULE_NAME"; then + echo "${MODULE_NAME}: failed to unload, please check dmesg" + exit 1 +fi + +run_log=$(dmesg | sed -n \ + "/${MODULE_NAME}: RUN ${run_id} BEGIN/,/${MODULE_NAME}: RUN ${run_id} END/p") +if [ -z "$run_log" ]; then + echo "${MODULE_NAME}: unable to find log for run ${run_id}" + exit 1 +fi + +if echo "$run_log" | grep -q "${MODULE_NAME}: FAILED:"; then + echo "${MODULE_NAME} failed, please check dmesg" + exit 1 +fi + +if echo "$run_log" | grep -q "${MODULE_NAME}: SKIP:"; then + echo "${MODULE_NAME}: no injectable SSE event" + exit $ksft_skip +fi + +exit 0 diff --git a/tools/testing/selftests/riscv/sse/sse_perf_ustack.c b/tools/testing/selftests/riscv/sse/sse_perf_ustack.c new file mode 100644 index 00000000000000..9535b6d7ba4c18 --- /dev/null +++ b/tools/testing/selftests/riscv/sse/sse_perf_ustack.c @@ -0,0 +1,564 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Regression test for the RISC-V perf user-stack copy taken in SSE + * (NMI-like) context. + * + * On RISC-V, PMU overflow interrupts can be delivered through the SBI + * Supervisor Software Events (SSE) mechanism. A perf event that samples the + * raw user stack (PERF_SAMPLE_STACK_USER, as perf record --call-graph dwarf + * does) then copies a large chunk of the interrupted task's user stack from + * that context. If that copy is allowed to take a nested page fault it can + * corrupt the interrupted task's kernel state and hang or crash the machine + * under load. + * + * This test exercises that exact path: + * - It opens a sampling hardware PMU event with PERF_SAMPLE_STACK_USER. + * - It samples a child running on a controlled user stack followed by an + * inaccessible page, so the copy must truncate at that page boundary. + * - It checks that every user-stack sample record is well formed and that + * the dumped size never exceeds the requested size (i.e. the copy stops + * cleanly rather than faulting on). + * - It then drives a multi-threaded unix-socket + deep-recursion workload + * under high-frequency per-CPU sampling and requires every active sampler + * to make progress without taking the machine down. + * + * The test is architecture independent in what it drives; it is placed under + * the RISC-V SSE selftests because SSE delivery is the RISC-V-specific + * condition it is meant to protect. + */ +#define _GNU_SOURCE + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "../../kselftest.h" + +#ifndef noinline +#define noinline __attribute__((noinline)) +#endif + +#define STACK_DUMP_SIZE 8192 /* 8 KiB, 8-byte aligned */ +#define RB_DATA_PAGES 64 /* power of two */ +#define SELF_SAMPLE_FREQ 4000 +#define STRESS_SAMPLE_FREQ 5000 +#define STRESS_SECONDS 5 +#define RECURSE_DEPTH 512 +#define TRUNCATION_RUN_MS 250 + +static long page_size; + +static int perf_event_open(struct perf_event_attr *attr, pid_t pid, int cpu, + int group_fd, unsigned long flags) +{ + return syscall(__NR_perf_event_open, attr, pid, cpu, group_fd, flags); +} + +/* Prevent the compiler from optimizing away a stack buffer. */ +static void keep_alive(void *p) +{ + __asm__ __volatile__("" : : "r"(p) : "memory"); +} + +/* + * Consume a deep user stack and keep it live, so a raw user-stack sample has + * many pages to copy. Returns a value derived from the stack so the compiler + * cannot elide the frames. + */ +static noinline unsigned long burn_stack(int depth, unsigned long *sink) +{ + unsigned long frame[32]; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(frame); i++) + frame[i] = (unsigned long)depth * i + *sink; + + if (depth > 0) + frame[depth & 31] += burn_stack(depth - 1, sink); + + for (i = 0; i < ARRAY_SIZE(frame); i++) + *sink += frame[i]; + + keep_alive(frame); + return *sink; +} + +static struct perf_event_attr sampling_attr(unsigned long freq) +{ + struct perf_event_attr attr = { + .type = PERF_TYPE_HARDWARE, + .size = sizeof(attr), + .config = PERF_COUNT_HW_INSTRUCTIONS, + .sample_type = PERF_SAMPLE_STACK_USER, + .sample_stack_user = STACK_DUMP_SIZE, + .freq = 1, + .sample_freq = freq, + .disabled = 1, + .exclude_kernel = 1, + .exclude_hv = 1, + }; + + return attr; +} + +static bool open_skip_reason(int err, const char **why) +{ + switch (err) { + case EACCES: + case EPERM: + *why = "insufficient privilege for PMU sampling (perf_event_paranoid)"; + return true; + case ENOENT: + case ENODEV: + case EOPNOTSUPP: + *why = "hardware PMU sampling event not available"; + return true; + default: + return false; + } +} + +static bool pmu_sse_route_testable(const char **why) +{ + char *line = NULL; + size_t line_size = 0; + FILE *interrupts; + bool testable = true; + + interrupts = fopen("/proc/interrupts", "re"); + if (!interrupts) { + *why = "cannot inspect the active PMU delivery route"; + return false; + } + + /* The SBI PMU driver registers this name only for ordinary IRQ delivery. */ + while (getline(&line, &line_size, interrupts) >= 0) { + if (strstr(line, "riscv-pmu")) { + *why = "ordinary RISC-V PMU IRQ delivery is active"; + testable = false; + break; + } + } + + free(line); + fclose(interrupts); + return testable; +} + +static bool ring_copy_from(void *dst, const void *rb, size_t rb_bytes, + uint64_t pos, size_t size) +{ + size_t offset = pos % rb_bytes; + size_t first; + + if (size > rb_bytes) + return false; + + first = size < rb_bytes - offset ? size : rb_bytes - offset; + memcpy(dst, (const char *)rb + offset, first); + if (first != size) + memcpy((char *)dst + first, rb, size - first); + + return true; +} + +static int truncation_child(void *arg) +{ + int ready_fd = (intptr_t)arg; + char ready = 1; + + if (write(ready_fd, &ready, sizeof(ready)) != 1) + return 1; + + for (;;) + __asm__ __volatile__("" : : : "memory"); +} + +static pid_t start_truncation_child(void **stack_mapping) +{ + struct pollfd pfd = { .events = POLLIN }; + size_t mapping_size = 2 * page_size; + char ready; + void *stack; + pid_t pid; + int pipefd[2]; + int saved_errno; + + stack = mmap(NULL, mapping_size, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (stack == MAP_FAILED) + return -1; + if (mprotect(stack, page_size, PROT_READ | PROT_WRITE)) + goto err_unmap; + if (pipe(pipefd)) + goto err_unmap; + + /* clone() starts the child below the inaccessible second page. */ + pid = clone(truncation_child, (char *)stack + page_size, SIGCHLD, + (void *)(intptr_t)pipefd[1]); + if (pid < 0) + goto err_pipe; + + close(pipefd[1]); + pfd.fd = pipefd[0]; + if (poll(&pfd, 1, 1000) != 1 || + read(pipefd[0], &ready, sizeof(ready)) != sizeof(ready)) { + saved_errno = ETIMEDOUT; + kill(pid, SIGKILL); + waitpid(pid, NULL, 0); + close(pipefd[0]); + errno = saved_errno; + goto err_unmap; + } + close(pipefd[0]); + + *stack_mapping = stack; + return pid; + +err_pipe: + saved_errno = errno; + close(pipefd[0]); + close(pipefd[1]); + errno = saved_errno; +err_unmap: + saved_errno = errno; + munmap(stack, mapping_size); + errno = saved_errno; + return -1; +} + +static void stop_truncation_child(pid_t pid, void *stack_mapping) +{ + kill(pid, SIGKILL); + while (waitpid(pid, NULL, 0) < 0 && errno == EINTR) + ; + munmap(stack_mapping, 2 * page_size); +} + +/* + * Subtest 1: sample a child whose stack is followed by an inaccessible page. + * Every record must be well formed and at least one stack copy must truncate + * at the controlled page boundary rather than fault or overrun. + */ +static void test_ustack_records_wellformed(void) +{ + struct perf_event_attr attr = sampling_attr(SELF_SAMPLE_FREQ); + size_t rb_bytes = (size_t)RB_DATA_PAGES * page_size; + struct perf_event_mmap_page *meta; + unsigned long samples = 0, truncated = 0; + void *child_stack; + const char *why; + void *rb; + pid_t child; + int fd; + + child = start_truncation_child(&child_stack); + if (child < 0) { + ksft_test_result_fail("ustack records: create guarded stack child: %s\n", + strerror(errno)); + return; + } + + fd = perf_event_open(&attr, child, -1, -1, PERF_FLAG_FD_CLOEXEC); + if (fd < 0) { + if (open_skip_reason(errno, &why)) + ksft_test_result_skip("ustack records: %s\n", why); + else + ksft_test_result_fail("ustack records: perf_event_open: %s\n", + strerror(errno)); + goto out_child; + } + + meta = mmap(NULL, page_size + rb_bytes, PROT_READ | PROT_WRITE, + MAP_SHARED, fd, 0); + if (meta == MAP_FAILED) { + ksft_test_result_fail("ustack records: mmap ring buffer: %s\n", + strerror(errno)); + close(fd); + goto out_child; + } + rb = (char *)meta + page_size; + + ioctl(fd, PERF_EVENT_IOC_RESET, 0); + ioctl(fd, PERF_EVENT_IOC_ENABLE, 0); + usleep(TRUNCATION_RUN_MS * 1000); + ioctl(fd, PERF_EVENT_IOC_DISABLE, 0); + + /* Drain the ring buffer and validate every SAMPLE record. */ + { + uint64_t head = __atomic_load_n(&meta->data_head, __ATOMIC_ACQUIRE); + uint64_t tail = meta->data_tail; + bool ok = true; + + if (head < tail || head - tail > rb_bytes) + ok = false; + + while (ok && tail < head) { + struct perf_event_header hdr; + uint64_t available = head - tail; + + if (available < sizeof(hdr) || + !ring_copy_from(&hdr, rb, rb_bytes, tail, sizeof(hdr)) || + hdr.size < sizeof(hdr) || hdr.size > available || + hdr.size > rb_bytes) { + ok = false; + break; + } + + if (hdr.type == PERF_RECORD_SAMPLE) { + uint64_t dump_size, dyn_size; + size_t cursor = sizeof(hdr); + + if (sizeof(dump_size) > hdr.size - cursor || + !ring_copy_from(&dump_size, rb, rb_bytes, + tail + cursor, sizeof(dump_size)) || + dump_size > STACK_DUMP_SIZE) { + ok = false; + break; + } + cursor += sizeof(dump_size); + samples++; + if (dump_size) { + /* data blob then trailing dynamic size */ + if (dump_size > hdr.size - cursor) { + ok = false; + break; + } + cursor += dump_size; + if (sizeof(dyn_size) > hdr.size - cursor || + !ring_copy_from(&dyn_size, rb, rb_bytes, + tail + cursor, + sizeof(dyn_size))) { + ok = false; + break; + } + if (dyn_size > dump_size) { + ok = false; + break; + } + if (dyn_size < dump_size) + truncated++; + } + } + tail += hdr.size; + } + __atomic_store_n(&meta->data_tail, head, __ATOMIC_RELEASE); + + if (!ok) + ksft_test_result_fail("ustack records: malformed sample record\n"); + else if (samples == 0) + ksft_test_result_skip("ustack records: no samples collected\n"); + else if (truncated == 0) + ksft_test_result_fail("ustack records: no guarded-stack truncation\n"); + else + ksft_test_result_pass("ustack records: %lu samples, %lu truncated\n", + samples, truncated); + } + + munmap(meta, page_size + rb_bytes); + close(fd); +out_child: + stop_truncation_child(child, child_stack); +} + +/* ---- Subtest 2: multi-threaded per-CPU sampling stress ---- */ + +struct stress_thread { + pthread_t tid; + int cpu; + int *stop; + int fd; + void *rb; + size_t rb_bytes; +}; + +static void *stress_worker(void *arg) +{ + struct stress_thread *st = arg; + unsigned long sink = 1; + int sv[2]; + char buf[64]; + + if (socketpair(AF_UNIX, SOCK_STREAM, 0, sv) == 0) { + while (!__atomic_load_n(st->stop, __ATOMIC_RELAXED)) { + /* unix-socket ping-pong: takes the socket locks the + * original bug corrupted, while sampling nests. + */ + if (write(sv[0], buf, sizeof(buf)) > 0) + (void)read(sv[1], buf, sizeof(buf)); + burn_stack(RECURSE_DEPTH, &sink); + /* Periodically consume the ring buffer so sampling + * keeps delivering rather than filling up and stopping. + */ + if (st->rb) { + struct perf_event_mmap_page *m = st->rb; + uint64_t h = __atomic_load_n(&m->data_head, + __ATOMIC_ACQUIRE); + __atomic_store_n(&m->data_tail, h, + __ATOMIC_RELEASE); + } + } + close(sv[0]); + close(sv[1]); + } + + return (void *)sink; +} + +static void test_sse_stress_no_crash(void) +{ + struct perf_event_attr attr = sampling_attr(STRESS_SAMPLE_FREQ); + size_t rb_bytes = (size_t)RB_DATA_PAGES * page_size; + struct stress_thread *threads; + cpu_set_t available; + long progressed = 0; + int stop = 0; + const char *why = NULL; + long started = 0; + long nproc; + long slot; + int cpu; + + if (sched_getaffinity(0, sizeof(available), &available)) { + ksft_test_result_fail("sse stress: sched_getaffinity: %s\n", + strerror(errno)); + return; + } + nproc = CPU_COUNT(&available); + if (nproc < 1) { + ksft_test_result_skip("sse stress: no available CPUs\n"); + return; + } + + threads = calloc(nproc, sizeof(*threads)); + if (!threads) { + ksft_test_result_fail("sse stress: out of memory\n"); + return; + } + + slot = 0; + for (cpu = 0; cpu < CPU_SETSIZE; cpu++) { + struct stress_thread *st; + pthread_attr_t thread_attr; + cpu_set_t set; + void *map; + int ret; + + if (!CPU_ISSET(cpu, &available)) + continue; + st = &threads[slot++]; + + st->fd = -1; + st->cpu = cpu; + st->stop = &stop; + st->rb_bytes = rb_bytes; + st->fd = perf_event_open(&attr, -1, cpu, -1, + PERF_FLAG_FD_CLOEXEC); + if (st->fd < 0) { + if (!started && open_skip_reason(errno, &why)) + break; + continue; + } + + map = mmap(NULL, page_size + rb_bytes, PROT_READ | PROT_WRITE, + MAP_SHARED, st->fd, 0); + if (map == MAP_FAILED) { + close(st->fd); + st->fd = -1; + continue; + } + st->rb = map; + + CPU_ZERO(&set); + CPU_SET(cpu, &set); + pthread_attr_init(&thread_attr); + ret = pthread_attr_setaffinity_np(&thread_attr, sizeof(set), &set); + if (!ret) + ret = pthread_create(&st->tid, &thread_attr, + stress_worker, st); + pthread_attr_destroy(&thread_attr); + + if (ret) { + munmap(st->rb, page_size + rb_bytes); + close(st->fd); + st->rb = NULL; + st->fd = -1; + continue; + } + + ioctl(st->fd, PERF_EVENT_IOC_RESET, 0); + ioctl(st->fd, PERF_EVENT_IOC_ENABLE, 0); + started++; + } + + if (started == 0) { + free(threads); + if (why) + ksft_test_result_skip("sse stress: %s\n", why); + else + ksft_test_result_skip("sse stress: could not start any sampler\n"); + return; + } + + sleep(STRESS_SECONDS); + __atomic_store_n(&stop, 1, __ATOMIC_RELAXED); + + for (slot = 0; slot < nproc; slot++) { + struct stress_thread *st = &threads[slot]; + struct perf_event_mmap_page *meta; + + if (st->fd < 0) + continue; + pthread_join(st->tid, NULL); + ioctl(st->fd, PERF_EVENT_IOC_DISABLE, 0); + meta = st->rb; + if (__atomic_load_n(&meta->data_head, __ATOMIC_ACQUIRE)) + progressed++; + munmap(st->rb, page_size + rb_bytes); + close(st->fd); + } + + free(threads); + if (progressed != started) + ksft_test_result_fail("sse stress: %ld/%ld samplers made progress\n", + progressed, started); + else + ksft_test_result_pass("sse stress: %ld samplers x %ds made progress\n", + started, STRESS_SECONDS); +} + +int main(void) +{ + const char *why; + + page_size = sysconf(_SC_PAGESIZE); + + ksft_print_header(); + ksft_set_plan(2); + if (!pmu_sse_route_testable(&why)) { + ksft_test_result_skip("ustack records: %s\n", why); + ksft_test_result_skip("sse stress: %s\n", why); + ksft_finished(); + } + + test_ustack_records_wellformed(); + test_sse_stress_no_crash(); + + ksft_finished(); +}