diff --git a/arch/riscv/include/asm/word-at-a-time.h b/arch/riscv/include/asm/word-at-a-time.h index 3802cda71ab7ee..31240133eccdef 100644 --- a/arch/riscv/include/asm/word-at-a-time.h +++ b/arch/riscv/include/asm/word-at-a-time.h @@ -35,17 +35,57 @@ static inline unsigned long prep_zero_mask(unsigned long val, static inline unsigned long create_zero_mask(unsigned long bits) { + if (IS_ENABLED(CONFIG_RISCV_ISA_ZBB) && IS_ENABLED(CONFIG_TOOLCHAIN_HAS_ZBB) && + riscv_has_extension_likely(RISCV_ISA_EXT_ZBB)) + return bits; + bits = (bits - 1) & ~bits; return bits >> 7; } +#ifdef CONFIG_64BIT +/* + * Jan Achrenius on G+: microoptimized version of + * the simpler "(mask & ONEBYTES) * ONEBYTES >> 56" + * that works for the bytemasks without having to + * mask them first. + */ +static inline long count_masked_bytes(unsigned long mask) +{ + return mask*0x0001020304050608ul >> 56; +} + +#else /* 32-bit case */ + +/* Carl Chatfield / Jan Achrenius G+ version for 32-bit */ +static inline long count_masked_bytes(long mask) +{ + /* (000000 0000ff 00ffff ffffff) -> ( 1 1 2 3 ) */ + long a = (0x0ff0001+mask) >> 23; + /* Fix the 1 for 00 case */ + return a & mask; +} +#endif + static inline unsigned long find_zero(unsigned long mask) { - return fls64(mask) >> 3; + if (IS_ENABLED(CONFIG_RISCV_ISA_ZBB) && IS_ENABLED(CONFIG_TOOLCHAIN_HAS_ZBB) && + riscv_has_extension_likely(RISCV_ISA_EXT_ZBB)) + return __ffs(mask) >> 3; + + return count_masked_bytes(mask); } -/* The mask we created is directly usable as a bytemask */ -#define zero_bytemask(mask) (mask) +static inline unsigned long zero_bytemask(unsigned long bits) +{ + if (IS_ENABLED(CONFIG_RISCV_ISA_ZBB) && IS_ENABLED(CONFIG_TOOLCHAIN_HAS_ZBB) && + riscv_has_extension_likely(RISCV_ISA_EXT_ZBB)) { + bits = (bits - 1) & ~bits; + return bits >> 7; + } + + return bits; +} #ifdef CONFIG_DCACHE_WORD_ACCESS