From af984d1f37d8bfc670ed210e5345fc07a7abda31 Mon Sep 17 00:00:00 2001 From: Andy Chiu Date: Fri, 11 Sep 2026 13:23:59 -0500 Subject: [PATCH 1/2] riscv: introduce assembly version of jump table This is the introductary patch for us to call the user copy that supports fast unaligned access after the runtime detection. Signed-off-by: Andy Chiu Signed-off-by: Linux RISC-V bot --- arch/riscv/include/asm/jump_label.h | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/arch/riscv/include/asm/jump_label.h b/arch/riscv/include/asm/jump_label.h index 3ab5f2e3212bec..d75961ed33246c 100644 --- a/arch/riscv/include/asm/jump_label.h +++ b/arch/riscv/include/asm/jump_label.h @@ -65,6 +65,32 @@ static __always_inline bool arch_static_branch_jump(struct static_key * const ke label: return true; } +#else +#include + +#define JUMP_TABLE_ENTRY(key, label) \ + .pushsection __jump_table, "aw"; \ + .align RISCV_LGPTR; \ + .long 1b - ., label - .; \ + RISCV_PTR key - .; \ + .popsection + +#define ARCH_STATIC_BRANCH_ASM(key, label) \ + .align 2; \ + .option push; \ + .option norelax; \ + .option norvc; \ +1: nop; \ + .option pop; \ + JUMP_TABLE_ENTRY(key, label) +#define ARCH_STATIC_BRANCH_JUMP_ASM(key, label) \ + .align 2; \ + .option push; \ + .option norelax; \ + .option norvc; \ +1: j label; \ + .option pop; \ + JUMP_TABLE_ENTRY(key, label) #endif /* __ASSEMBLER__ */ #endif /* __ASM_JUMP_LABEL_H */ From e89b294b6552ae1d1ca7b8ab59aee364dc03c59f Mon Sep 17 00:00:00 2001 From: Andy Chiu Date: Fri, 11 Sep 2026 13:24:00 -0500 Subject: [PATCH 2/2] riscv: optmize ucopy for has_fast_unaligned_accesses Skipping the software aligning code increases the ucopy bandwidth by up to 11.4%. Commit b94cec5761d2 ("riscv: skip software algning code for HAVE_EFFICIENT_UNALIGNED_ACCESS") provides this optimization at compile time, which depends on NONPORTABLE. This patch extends it such that the optimization can be enabled at runtime, after the unaligned access speed is resolved. To do so, we utilize the assembly version of static key and connect it to fast_unaligned_access_speed_key. The kernel jumps to the software aligning code as the default behavior, then the jump is optimized to an nop if the platform supports native misaligned access. Signed-off-by: Andy Chiu Signed-off-by: Linux RISC-V bot --- arch/riscv/lib/uaccess.S | 61 ++++++++++++++++++++++------------------ 1 file changed, 34 insertions(+), 27 deletions(-) diff --git a/arch/riscv/lib/uaccess.S b/arch/riscv/lib/uaccess.S index cf8586a937de8c..ad1457e789fb4a 100644 --- a/arch/riscv/lib/uaccess.S +++ b/arch/riscv/lib/uaccess.S @@ -5,6 +5,7 @@ #include #include #include +#include .macro fixup op reg addr lbl 100: @@ -77,33 +78,11 @@ SYM_FUNC_START(fallback_scalar_usercopy_sum_enabled) bltu a2, a3, .Lbyte_copy_tail #if !defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) - /* - * Copy first bytes until dst is aligned to word boundary. - * a0 - start of dst - * t1 - start of aligned dst - */ - addi t1, a0, SZREG-1 - andi t1, t1, ~(SZREG-1) - /* dst is already aligned, skip */ - beq a0, t1, .Lskip_align_dst -1: - /* a5 - one byte for copying data */ - fixup lb a5, 0(a1), 10f - addi a1, a1, 1 /* src */ - fixup sb a5, 0(a0), 10f - addi a0, a0, 1 /* dst */ - bltu a0, t1, 1b /* t1 - start of aligned dst */ - -.Lskip_align_dst: - /* - * Now dst is aligned. - * Use shift-copy if src is misaligned. - * Use word-copy if both src and dst are aligned because - * can not use shift-copy which do not require shifting - */ - /* a1 - start of src */ - andi a3, a1, SZREG-1 - bnez a3, .Lshift_copy +#if defined(CONFIG_RISCV_PROBE_UNALIGNED_ACCESS) && defined(CONFIG_JUMP_LABEL) + ARCH_STATIC_BRANCH_JUMP_ASM(fast_unaligned_access_speed_key + 1, .Lalign_dst) +#else + j .Lalign_dst +#endif #endif .Lword_copy: /* @@ -139,6 +118,34 @@ SYM_FUNC_START(fallback_scalar_usercopy_sum_enabled) j .Lbyte_copy_tail #if !defined(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) +.Lalign_dst: + /* + * Copy first bytes until dst is aligned to word boundary. + * a0 - start of dst + * t1 - start of aligned dst + */ + addi t1, a0, SZREG-1 + andi t1, t1, ~(SZREG-1) + /* dst is already aligned, skip */ + beq a0, t1, .Lskip_align_dst +1: + /* a5 - one byte for copying data */ + fixup lb a5, 0(a1), 10f + addi a1, a1, 1 /* src */ + fixup sb a5, 0(a0), 10f + addi a0, a0, 1 /* dst */ + bltu a0, t1, 1b /* t1 - start of aligned dst */ + +.Lskip_align_dst: + /* + * Now dst is aligned. + * Use shift-copy if src is misaligned. + * Use word-copy if both src and dst are aligned because + * can not use shift-copy which do not require shifting + */ + /* a1 - start of src */ + andi a3, a1, SZREG-1 + beqz a3, .Lword_copy .Lshift_copy: /*