Skip to content

[PW_SID:1160924] riscv: word-at-a-time: improve find_zero() #2626

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
linux-riscv-bot wants to merge 3 commits into
base: workflow__riscv__fixes
Choose a base branch
from
Open

[PW_SID:1160924] riscv: word-at-a-time: improve find_zero() #2626

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
e1a35d4
riscv: word-at-a-time: improve find_zero() for !RISCV_ISA_ZBB
xhackerustc Sep 8, 2026
16ceb82
riscv: word-at-a-time: improve find_zero() without Zbb
xhackerustc Sep 8, 2026
d797e20
riscv: word-at-a-time: improve find_zero() for Zbb
xhackerustc Sep 8, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 43 additions & 3 deletions arch/riscv/include/asm/word-at-a-time.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,17 +35,57 @@ static inline unsigned long prep_zero_mask(unsigned long val,

static inline unsigned long create_zero_mask(unsigned long bits)
{
if (IS_ENABLED(CONFIG_RISCV_ISA_ZBB) && IS_ENABLED(CONFIG_TOOLCHAIN_HAS_ZBB) &&
riscv_has_extension_likely(RISCV_ISA_EXT_ZBB))
return bits;

bits = (bits - 1) & ~bits;
return bits >> 7;
}

#ifdef CONFIG_64BIT
/*
* Jan Achrenius on G+: microoptimized version of
* the simpler "(mask & ONEBYTES) * ONEBYTES >> 56"
* that works for the bytemasks without having to
* mask them first.
*/
static inline long count_masked_bytes(unsigned long mask)
{
return mask*0x0001020304050608ul >> 56;
}

#else /* 32-bit case */

/* Carl Chatfield / Jan Achrenius G+ version for 32-bit */
static inline long count_masked_bytes(long mask)
{
/* (000000 0000ff 00ffff ffffff) -> ( 1 1 2 3 ) */
long a = (0x0ff0001+mask) >> 23;
/* Fix the 1 for 00 case */
return a & mask;
}
#endif

static inline unsigned long find_zero(unsigned long mask)
{
return fls64(mask) >> 3;
if (IS_ENABLED(CONFIG_RISCV_ISA_ZBB) && IS_ENABLED(CONFIG_TOOLCHAIN_HAS_ZBB) &&
riscv_has_extension_likely(RISCV_ISA_EXT_ZBB))
return __ffs(mask) >> 3;

return count_masked_bytes(mask);
}

/* The mask we created is directly usable as a bytemask */
#define zero_bytemask(mask) (mask)
static inline unsigned long zero_bytemask(unsigned long bits)
{
if (IS_ENABLED(CONFIG_RISCV_ISA_ZBB) && IS_ENABLED(CONFIG_TOOLCHAIN_HAS_ZBB) &&
riscv_has_extension_likely(RISCV_ISA_EXT_ZBB)) {
bits = (bits - 1) & ~bits;
return bits >> 7;
}

return bits;
}

#ifdef CONFIG_DCACHE_WORD_ACCESS

Expand Down
Loading