diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index d6c2dbf8455ced..d41af95d9a76e7 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -218,7 +218,7 @@ config RISCV select PCI_ECAM if (ACPI && PCI) select PCI_MSI if PCI select RELOCATABLE if !MMU && !PHYS_RAM_BASE_FIXED - select RISCV_ALTERNATIVE + select RISCV_ALTERNATIVE_EARLY if 64BIT select RISCV_APLIC select RISCV_IMSIC select RISCV_INTC @@ -539,17 +539,8 @@ config RISCV_COMBO_SPINLOCKS endchoice -config RISCV_ALTERNATIVE - bool - help - This Kconfig allows the kernel to automatically patch the - erratum or cpufeature required by the execution platform at run - time. The code patching overhead is minimal, as it's only done - once at boot and once on each module load. - config RISCV_ALTERNATIVE_EARLY bool - depends on RISCV_ALTERNATIVE help Allows early patching of the kernel for special errata @@ -578,7 +569,6 @@ config RISCV_ISA_SUPM config RISCV_ISA_SVNAPOT bool "Svnapot extension support for supervisor mode NAPOT pages" depends on 64BIT && MMU - depends on RISCV_ALTERNATIVE default y help Enable support for the Svnapot ISA-extension when it is detected @@ -612,10 +602,33 @@ config RISCV_ISA_SSQOSID cache partitioning scheme and use the MCID to track how much cache a process, or a group of processes, is using. +config RISCV_ISA_SV48 + bool "Page-Based 48-bit Virtual-Memory System" + depends on 64BIT + default y + help + Add support for the Page-Based 48-bit Virtual-Memory System. + + Enable this option for systems for which a 39-bit virtual address + space is insufficient. But if 39-bit is enough, disabling this + option will result in smaller kernel size and slighly better + performance. + +config RISCV_ISA_SV57 + bool "Page-Based 57-bit Virtual-Memory System" + depends on 64BIT + default y + help + Add support for the Page-Based 57-bit Virtual-Memory System. + + Enable this option for systems for which a 48-bit virtual address + space is insufficient. But if 48-bit is enough, disabling this + option will result in smaller kernel size and slighly better + performance. + config RISCV_ISA_SVPBMT bool "Svpbmt extension support for supervisor mode page-based memory types" depends on 64BIT && MMU - depends on RISCV_ALTERNATIVE default y help Add support for the Svpbmt ISA-extension (Supervisor-mode: @@ -685,7 +698,6 @@ config RISCV_ISA_V_PREEMPTIVE config RISCV_ISA_ZAWRS bool "Zawrs extension support for more efficient busy waiting" - depends on RISCV_ALTERNATIVE default y help The Zawrs extension defines instructions to be used in polling loops @@ -704,7 +716,6 @@ config TOOLCHAIN_HAS_ZABHA config RISCV_ISA_ZABHA bool "Zabha extension support for atomic byte/halfword operations" depends on TOOLCHAIN_HAS_ZABHA - depends on RISCV_ALTERNATIVE default y help Enable the use of the Zabha ISA-extension to implement kernel @@ -721,7 +732,6 @@ config TOOLCHAIN_HAS_ZACAS config RISCV_ISA_ZACAS bool "Zacas extension support for atomic CAS" - depends on RISCV_ALTERNATIVE default y help Enable the use of the Zacas ISA-extension to implement kernel atomic @@ -766,7 +776,6 @@ config RISCV_ISA_ZBA config RISCV_ISA_ZBB bool "Zbb extension support for bit manipulation instructions" - depends on RISCV_ALTERNATIVE default y help Add support for enabling optimisations in the kernel when the @@ -791,7 +800,6 @@ config RISCV_ISA_ZBC bool "Zbc extension support for carry-less multiplication instructions" depends on TOOLCHAIN_HAS_ZBC depends on MMU - depends on RISCV_ALTERNATIVE default y help Adds support to dynamically detect the presence of the Zbc @@ -813,7 +821,6 @@ config TOOLCHAIN_HAS_ZBKB config RISCV_ISA_ZBKB bool "Zbkb extension support for bit manipulation instructions" depends on TOOLCHAIN_HAS_ZBKB - depends on RISCV_ALTERNATIVE default y help Adds support to dynamically detect the presence of the ZBKB @@ -827,7 +834,6 @@ config RISCV_ISA_ZBKB config RISCV_ISA_ZICBOM bool "Zicbom extension support for non-coherent DMA operation" depends on MMU - depends on RISCV_ALTERNATIVE default y select RISCV_DMA_NONCOHERENT select DMA_DIRECT_REMAP @@ -843,7 +849,6 @@ config RISCV_ISA_ZICBOM config RISCV_ISA_ZICBOZ bool "Zicboz extension support for faster zeroing of memory" - depends on RISCV_ALTERNATIVE default y help Enable the use of the Zicboz extension (cbo.zero instruction) @@ -856,7 +861,6 @@ config RISCV_ISA_ZICBOZ config RISCV_ISA_ZICBOP bool "Zicbop extension support for cache block prefetch" depends on MMU - depends on RISCV_ALTERNATIVE default y help Adds support to dynamically detect the presence of the ZICBOP @@ -871,7 +875,6 @@ config RISCV_ISA_ZICBOP config RISCV_ISA_SVRSW60T59B bool "Svrsw60t59b extension support for using PTE bits 60 and 59" depends on MMU && 64BIT - depends on RISCV_ALTERNATIVE default y help Adds support to dynamically detect the presence of the Svrsw60t59b @@ -1204,7 +1207,6 @@ config RISCV_USER_CFI bool "riscv userspace control flow integrity" depends on 64BIT && MMU && \ $(cc-option,-mabi=lp64 -march=rv64ima_zicfiss_zicfilp -fcf-protection=full) - depends on RISCV_ALTERNATIVE select RISCV_SBI select ARCH_HAS_USER_SHADOW_STACK select ARCH_USES_HIGH_VMA_FLAGS diff --git a/arch/riscv/Kconfig.errata b/arch/riscv/Kconfig.errata index 3c945d086c7d02..1a66e5b6f7d113 100644 --- a/arch/riscv/Kconfig.errata +++ b/arch/riscv/Kconfig.errata @@ -2,7 +2,7 @@ menu "CPU errata selection" config ERRATA_ANDES bool "Andes AX45MP errata" - depends on RISCV_ALTERNATIVE && RISCV_SBI + depends on RISCV_SBI help All Andes errata Kconfig depend on this Kconfig. Disabling this Kconfig will disable all Andes errata. Please say "Y" @@ -23,7 +23,6 @@ config ERRATA_ANDES_CMO config ERRATA_MIPS bool "MIPS errata" - depends on RISCV_ALTERNATIVE help All MIPS errata Kconfig depend on this Kconfig. Disabling this Kconfig will disable all MIPS errata. Please say "Y" @@ -46,7 +45,6 @@ config ERRATA_MIPS_P8700_PAUSE_OPCODE config ERRATA_SIFIVE bool "SiFive errata" - depends on RISCV_ALTERNATIVE help All SiFive errata Kconfig depend on this Kconfig. Disabling this Kconfig will disable all SiFive errata. Please say "Y" @@ -98,7 +96,6 @@ config ERRATA_STARFIVE_JH7100 config ERRATA_THEAD bool "T-HEAD errata" - depends on RISCV_ALTERNATIVE help All T-HEAD errata Kconfig depend on this Kconfig. Disabling this Kconfig will disable all T-HEAD errata. Please say "Y" diff --git a/arch/riscv/include/asm/alternative-macros.h b/arch/riscv/include/asm/alternative-macros.h index 9619bd5c8ebaa3..629d884578daff 100644 --- a/arch/riscv/include/asm/alternative-macros.h +++ b/arch/riscv/include/asm/alternative-macros.h @@ -2,8 +2,6 @@ #ifndef __ASM_ALTERNATIVE_MACROS_H #define __ASM_ALTERNATIVE_MACROS_H -#ifdef CONFIG_RISCV_ALTERNATIVE - #ifdef __ASSEMBLER__ .macro ALT_ENTRY oldptr newptr vendor_id patch_id new_len @@ -108,28 +106,6 @@ __ALTERNATIVE_CFG_2(old_c, new_c_1, vendor_id_1, patch_id_1, IS_ENABLED(CONFIG_k_1), \ new_c_2, vendor_id_2, patch_id_2, IS_ENABLED(CONFIG_k_2)) -#else /* CONFIG_RISCV_ALTERNATIVE */ -#ifdef __ASSEMBLER__ - -.macro ALTERNATIVE_CFG old_c - \old_c -.endm - -#define __ALTERNATIVE_CFG(old_c, ...) ALTERNATIVE_CFG old_c -#define __ALTERNATIVE_CFG_2(old_c, ...) ALTERNATIVE_CFG old_c - -#else /* !__ASSEMBLER__ */ - -#define __ALTERNATIVE_CFG(old_c, ...) old_c "\n" -#define __ALTERNATIVE_CFG_2(old_c, ...) old_c "\n" - -#endif /* __ASSEMBLER__ */ - -#define _ALTERNATIVE_CFG(old_c, ...) __ALTERNATIVE_CFG(old_c) -#define _ALTERNATIVE_CFG_2(old_c, ...) __ALTERNATIVE_CFG_2(old_c) - -#endif /* CONFIG_RISCV_ALTERNATIVE */ - /* * Usage: * ALTERNATIVE(old_content, new_content, vendor_id, patch_id, CONFIG_k) diff --git a/arch/riscv/include/asm/alternative.h b/arch/riscv/include/asm/alternative.h index 8407d1d535b852..6be7b2b6ade9ee 100644 --- a/arch/riscv/include/asm/alternative.h +++ b/arch/riscv/include/asm/alternative.h @@ -10,8 +10,6 @@ #ifndef __ASSEMBLER__ -#ifdef CONFIG_RISCV_ALTERNATIVE - #include #include #include @@ -35,7 +33,7 @@ void __init apply_early_boot_alternatives(void); void apply_module_alternatives(void *start, size_t length); void riscv_alternative_fix_offsets(void *alt_ptr, unsigned int len, - int patch_offset); + int patch_offset, bool early); struct alt_entry { s32 old_offset; /* offset relative to original instruction or data */ @@ -61,13 +59,5 @@ void thead_errata_patch_func(struct alt_entry *begin, struct alt_entry *end, void riscv_cpufeature_patch_func(struct alt_entry *begin, struct alt_entry *end, unsigned int stage); -#else /* CONFIG_RISCV_ALTERNATIVE */ - -static inline void apply_boot_alternatives(void) { } -static inline void apply_early_boot_alternatives(void) { } -static inline void apply_module_alternatives(void *start, size_t length) { } - -#endif /* CONFIG_RISCV_ALTERNATIVE */ - #endif #endif diff --git a/arch/riscv/include/asm/cpufeature-macros.h b/arch/riscv/include/asm/cpufeature-macros.h index a8103edbf51f82..adaf9e3fb25c0b 100644 --- a/arch/riscv/include/asm/cpufeature-macros.h +++ b/arch/riscv/include/asm/cpufeature-macros.h @@ -47,20 +47,14 @@ static __always_inline bool riscv_has_extension_unlikely(const unsigned long ext { compiletime_assert(ext < RISCV_ISA_EXT_MAX, "ext must be < RISCV_ISA_EXT_MAX"); - if (IS_ENABLED(CONFIG_RISCV_ALTERNATIVE)) - return __riscv_has_extension_unlikely(STANDARD_EXT, ext); - - return __riscv_isa_extension_available(NULL, ext); + return __riscv_has_extension_unlikely(STANDARD_EXT, ext); } static __always_inline bool riscv_has_extension_likely(const unsigned long ext) { compiletime_assert(ext < RISCV_ISA_EXT_MAX, "ext must be < RISCV_ISA_EXT_MAX"); - if (IS_ENABLED(CONFIG_RISCV_ALTERNATIVE)) - return __riscv_has_extension_likely(STANDARD_EXT, ext); - - return __riscv_isa_extension_available(NULL, ext); + return __riscv_has_extension_likely(STANDARD_EXT, ext); } #endif /* _ASM_CPUFEATURE_MACROS_H */ diff --git a/arch/riscv/include/asm/cpufeature.h b/arch/riscv/include/asm/cpufeature.h index 739fcc84bf7b28..af18ba93f58052 100644 --- a/arch/riscv/include/asm/cpufeature.h +++ b/arch/riscv/include/asm/cpufeature.h @@ -36,6 +36,8 @@ extern const struct seq_operations cpuinfo_op; /* Per-cpu ISA extensions. */ extern struct riscv_isainfo hart_isa[NR_CPUS]; +extern DECLARE_BITMAP(riscv_isa, RISCV_ISA_EXT_MAX); + extern u32 thead_vlenb_of; void __init riscv_user_isa_enable(void); @@ -134,8 +136,7 @@ static __always_inline bool riscv_cpu_has_extension_likely(int cpu, const unsign { compiletime_assert(ext < RISCV_ISA_EXT_MAX, "ext must be < RISCV_ISA_EXT_MAX"); - if (IS_ENABLED(CONFIG_RISCV_ALTERNATIVE) && - __riscv_has_extension_likely(STANDARD_EXT, ext)) + if (__riscv_has_extension_likely(STANDARD_EXT, ext)) return true; return __riscv_isa_extension_available(hart_isa[cpu].isa, ext); @@ -145,8 +146,7 @@ static __always_inline bool riscv_cpu_has_extension_unlikely(int cpu, const unsi { compiletime_assert(ext < RISCV_ISA_EXT_MAX, "ext must be < RISCV_ISA_EXT_MAX"); - if (IS_ENABLED(CONFIG_RISCV_ALTERNATIVE) && - __riscv_has_extension_unlikely(STANDARD_EXT, ext)) + if (__riscv_has_extension_unlikely(STANDARD_EXT, ext)) return true; return __riscv_isa_extension_available(hart_isa[cpu].isa, ext); diff --git a/arch/riscv/include/asm/hwcap.h b/arch/riscv/include/asm/hwcap.h index f8db798b2654bb..b6d50025cb793c 100644 --- a/arch/riscv/include/asm/hwcap.h +++ b/arch/riscv/include/asm/hwcap.h @@ -122,6 +122,8 @@ #define RISCV_ISA_EXT_ZICCAMOA 113 #define RISCV_ISA_EXT_ZICCIF 114 #define RISCV_ISA_EXT_ZA64RS 115 +#define RISCV_ISA_EXT_SV48 116 +#define RISCV_ISA_EXT_SV57 117 #define RISCV_ISA_EXT_XLINUXENVCFG 127 diff --git a/arch/riscv/include/asm/pgalloc.h b/arch/riscv/include/asm/pgalloc.h index 770ce18a7328b8..0133f7ba41d7d0 100644 --- a/arch/riscv/include/asm/pgalloc.h +++ b/arch/riscv/include/asm/pgalloc.h @@ -41,7 +41,7 @@ static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd) static inline void p4d_populate(struct mm_struct *mm, p4d_t *p4d, pud_t *pud) { - if (pgtable_l4_enabled) { + if (pgtable_l4_enabled()) { unsigned long pfn = virt_to_pfn(pud); set_p4d(p4d, __p4d((pfn << _PAGE_PFN_SHIFT) | _PAGE_TABLE)); @@ -51,7 +51,7 @@ static inline void p4d_populate(struct mm_struct *mm, p4d_t *p4d, pud_t *pud) static inline void p4d_populate_safe(struct mm_struct *mm, p4d_t *p4d, pud_t *pud) { - if (pgtable_l4_enabled) { + if (pgtable_l4_enabled()) { unsigned long pfn = virt_to_pfn(pud); set_p4d_safe(p4d, @@ -61,7 +61,7 @@ static inline void p4d_populate_safe(struct mm_struct *mm, p4d_t *p4d, static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, p4d_t *p4d) { - if (pgtable_l5_enabled) { + if (pgtable_l5_enabled()) { unsigned long pfn = virt_to_pfn(p4d); set_pgd(pgd, __pgd((pfn << _PAGE_PFN_SHIFT) | _PAGE_TABLE)); @@ -71,7 +71,7 @@ static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, p4d_t *p4d) static inline void pgd_populate_safe(struct mm_struct *mm, pgd_t *pgd, p4d_t *p4d) { - if (pgtable_l5_enabled) { + if (pgtable_l5_enabled()) { unsigned long pfn = virt_to_pfn(p4d); set_pgd_safe(pgd, @@ -82,21 +82,21 @@ static inline void pgd_populate_safe(struct mm_struct *mm, pgd_t *pgd, #define pud_free pud_free static inline void pud_free(struct mm_struct *mm, pud_t *pud) { - if (pgtable_l4_enabled) + if (pgtable_l4_enabled()) __pud_free(mm, pud); } static inline void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pud, unsigned long addr) { - if (pgtable_l4_enabled) + if (pgtable_l4_enabled()) tlb_remove_ptdesc(tlb, virt_to_ptdesc(pud)); } static inline void __p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4d, unsigned long addr) { - if (pgtable_l5_enabled) + if (pgtable_l5_enabled()) tlb_remove_ptdesc(tlb, virt_to_ptdesc(p4d)); } #endif /* __PAGETABLE_PMD_FOLDED */ diff --git a/arch/riscv/include/asm/pgtable-32.h b/arch/riscv/include/asm/pgtable-32.h index 00f3369570a836..5003a7e0f0b06e 100644 --- a/arch/riscv/include/asm/pgtable-32.h +++ b/arch/riscv/include/asm/pgtable-32.h @@ -33,7 +33,7 @@ _PAGE_WRITE | _PAGE_EXEC | \ _PAGE_USER | _PAGE_GLOBAL)) -static const __maybe_unused int pgtable_l4_enabled; -static const __maybe_unused int pgtable_l5_enabled; +static inline bool pgtable_l4_enabled(void) { return false; } +static inline bool pgtable_l5_enabled(void) { return false; } #endif /* _ASM_RISCV_PGTABLE_32_H */ diff --git a/arch/riscv/include/asm/pgtable-64.h b/arch/riscv/include/asm/pgtable-64.h index 6e789fa58514c7..caf8458b344c78 100644 --- a/arch/riscv/include/asm/pgtable-64.h +++ b/arch/riscv/include/asm/pgtable-64.h @@ -10,14 +10,48 @@ #include #include -extern bool pgtable_l4_enabled; -extern bool pgtable_l5_enabled; +extern bool _pgtable_l4_enabled; +extern bool _pgtable_l5_enabled; + +#ifdef USE_EARLY_PGTABLE_LEVELS +static __always_inline bool pgtable_l5_enabled(void) +{ + if (!IS_ENABLED(CONFIG_RISCV_ISA_SV57)) + return false; + + return _pgtable_l5_enabled; +} + +static __always_inline bool pgtable_l4_enabled(void) +{ + if (!IS_ENABLED(CONFIG_RISCV_ISA_SV48)) + return false; + + return _pgtable_l4_enabled; +} +#else +static __always_inline bool pgtable_l5_enabled(void) +{ + if (!IS_ENABLED(CONFIG_RISCV_ISA_SV57)) + return false; + + return riscv_has_extension_likely(RISCV_ISA_EXT_SV57); +} + +static __always_inline bool pgtable_l4_enabled(void) +{ + if (!IS_ENABLED(CONFIG_RISCV_ISA_SV48)) + return false; + + return riscv_has_extension_likely(RISCV_ISA_EXT_SV48); +} +#endif #define PGDIR_SHIFT_L3 30 #define PGDIR_SHIFT_L4 39 #define PGDIR_SHIFT_L5 48 -#define PGDIR_SHIFT (pgtable_l5_enabled ? PGDIR_SHIFT_L5 : \ - (pgtable_l4_enabled ? PGDIR_SHIFT_L4 : PGDIR_SHIFT_L3)) +#define PGDIR_SHIFT (pgtable_l5_enabled() ? PGDIR_SHIFT_L5 : \ + (pgtable_l4_enabled() ? PGDIR_SHIFT_L4 : PGDIR_SHIFT_L3)) /* Size of region mapped by a page global directory */ #define PGDIR_SIZE (_AC(1, UL) << PGDIR_SHIFT) #define PGDIR_MASK (~(PGDIR_SIZE - 1)) @@ -26,8 +60,8 @@ extern bool pgtable_l5_enabled; #define P4D_SHIFT_L3 30 #define P4D_SHIFT_L4 39 #define P4D_SHIFT_L5 39 -#define P4D_SHIFT (pgtable_l5_enabled ? P4D_SHIFT_L5 : \ - (pgtable_l4_enabled ? P4D_SHIFT_L4 : P4D_SHIFT_L3)) +#define P4D_SHIFT (pgtable_l5_enabled() ? P4D_SHIFT_L5 : \ + (pgtable_l4_enabled() ? P4D_SHIFT_L4 : P4D_SHIFT_L3)) #define P4D_SIZE (_AC(1, UL) << P4D_SHIFT) #define P4D_MASK (~(P4D_SIZE - 1)) @@ -233,7 +267,7 @@ static inline struct page *pud_page(pud_t pud) #define mm_p4d_folded mm_p4d_folded static inline bool mm_p4d_folded(struct mm_struct *mm) { - if (pgtable_l5_enabled) + if (pgtable_l5_enabled()) return false; return true; @@ -242,7 +276,7 @@ static inline bool mm_p4d_folded(struct mm_struct *mm) #define mm_pud_folded mm_pud_folded static inline bool mm_pud_folded(struct mm_struct *mm) { - if (pgtable_l4_enabled) + if (pgtable_l4_enabled()) return false; return true; @@ -275,7 +309,7 @@ static inline unsigned long _pmd_pfn(pmd_t pmd) static inline void set_p4d(p4d_t *p4dp, p4d_t p4d) { - if (pgtable_l4_enabled) + if (pgtable_l4_enabled()) WRITE_ONCE(*p4dp, p4d); else set_pud((pud_t *)p4dp, (pud_t){ p4d_val(p4d) }); @@ -283,7 +317,7 @@ static inline void set_p4d(p4d_t *p4dp, p4d_t p4d) static inline int p4d_none(p4d_t p4d) { - if (pgtable_l4_enabled) + if (pgtable_l4_enabled()) return (p4d_val(p4d) == 0); return 0; @@ -291,7 +325,7 @@ static inline int p4d_none(p4d_t p4d) static inline int p4d_present(p4d_t p4d) { - if (pgtable_l4_enabled) + if (pgtable_l4_enabled()) return (p4d_val(p4d) & _PAGE_PRESENT); return 1; @@ -299,7 +333,7 @@ static inline int p4d_present(p4d_t p4d) static inline int p4d_bad(p4d_t p4d) { - if (pgtable_l4_enabled) + if (pgtable_l4_enabled()) return !p4d_present(p4d); return 0; @@ -307,7 +341,7 @@ static inline int p4d_bad(p4d_t p4d) static inline void p4d_clear(p4d_t *p4d) { - if (pgtable_l4_enabled) + if (pgtable_l4_enabled()) set_p4d(p4d, __p4d(0)); } @@ -323,7 +357,7 @@ static inline unsigned long _p4d_pfn(p4d_t p4d) static inline pud_t *p4d_pgtable(p4d_t p4d) { - if (pgtable_l4_enabled) + if (pgtable_l4_enabled()) return (pud_t *)pfn_to_virt(__page_val_to_pfn(p4d_val(p4d))); return (pud_t *)pud_pgtable((pud_t) { p4d_val(p4d) }); @@ -342,7 +376,7 @@ pud_t *pud_offset(p4d_t *p4d, unsigned long address); static inline void set_pgd(pgd_t *pgdp, pgd_t pgd) { - if (pgtable_l5_enabled) + if (pgtable_l5_enabled()) WRITE_ONCE(*pgdp, pgd); else set_p4d((p4d_t *)pgdp, (p4d_t){ pgd_val(pgd) }); @@ -350,7 +384,7 @@ static inline void set_pgd(pgd_t *pgdp, pgd_t pgd) static inline int pgd_none(pgd_t pgd) { - if (pgtable_l5_enabled) + if (pgtable_l5_enabled()) return (pgd_val(pgd) == 0); return 0; @@ -358,7 +392,7 @@ static inline int pgd_none(pgd_t pgd) static inline int pgd_present(pgd_t pgd) { - if (pgtable_l5_enabled) + if (pgtable_l5_enabled()) return (pgd_val(pgd) & _PAGE_PRESENT); return 1; @@ -366,7 +400,7 @@ static inline int pgd_present(pgd_t pgd) static inline int pgd_bad(pgd_t pgd) { - if (pgtable_l5_enabled) + if (pgtable_l5_enabled()) return !pgd_present(pgd); return 0; @@ -374,13 +408,13 @@ static inline int pgd_bad(pgd_t pgd) static inline void pgd_clear(pgd_t *pgd) { - if (pgtable_l5_enabled) + if (pgtable_l5_enabled()) set_pgd(pgd, __pgd(0)); } static inline p4d_t *pgd_pgtable(pgd_t pgd) { - if (pgtable_l5_enabled) + if (pgtable_l5_enabled()) return (p4d_t *)pfn_to_virt(__page_val_to_pfn(pgd_val(pgd))); return (p4d_t *)p4d_pgtable((p4d_t) { pgd_val(pgd) }); diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h index 40b1ed4f3ea893..a71671166852a8 100644 --- a/arch/riscv/include/asm/pgtable.h +++ b/arch/riscv/include/asm/pgtable.h @@ -75,8 +75,8 @@ #define VA_BITS_SV48 48 #define VA_BITS_SV57 57 -#define VA_BITS (pgtable_l5_enabled ? \ - VA_BITS_SV57 : (pgtable_l4_enabled ? VA_BITS_SV48 : VA_BITS_SV39)) +#define VA_BITS (pgtable_l5_enabled() ? \ + VA_BITS_SV57 : (pgtable_l4_enabled() ? VA_BITS_SV48 : VA_BITS_SV39)) #else #define VA_BITS VA_BITS_SV32 #endif diff --git a/arch/riscv/include/asm/vendor_extensions.h b/arch/riscv/include/asm/vendor_extensions.h index 7437304a71b911..b00149be262730 100644 --- a/arch/riscv/include/asm/vendor_extensions.h +++ b/arch/riscv/include/asm/vendor_extensions.h @@ -54,11 +54,9 @@ static __always_inline bool riscv_has_vendor_extension_likely(const unsigned lon if (!IS_ENABLED(CONFIG_RISCV_ISA_VENDOR_EXT)) return false; - if (IS_ENABLED(CONFIG_RISCV_ALTERNATIVE)) - return __riscv_has_extension_likely(vendor, - ext + RISCV_VENDOR_EXT_ALTERNATIVES_BASE); + return __riscv_has_extension_likely(vendor, + ext + RISCV_VENDOR_EXT_ALTERNATIVES_BASE); - return __riscv_isa_vendor_extension_available(VENDOR_EXT_ALL_CPUS, vendor, ext); } static __always_inline bool riscv_has_vendor_extension_unlikely(const unsigned long vendor, @@ -67,11 +65,9 @@ static __always_inline bool riscv_has_vendor_extension_unlikely(const unsigned l if (!IS_ENABLED(CONFIG_RISCV_ISA_VENDOR_EXT)) return false; - if (IS_ENABLED(CONFIG_RISCV_ALTERNATIVE)) - return __riscv_has_extension_unlikely(vendor, - ext + RISCV_VENDOR_EXT_ALTERNATIVES_BASE); + return __riscv_has_extension_unlikely(vendor, + ext + RISCV_VENDOR_EXT_ALTERNATIVES_BASE); - return __riscv_isa_vendor_extension_available(VENDOR_EXT_ALL_CPUS, vendor, ext); } static __always_inline bool riscv_cpu_has_vendor_extension_likely(const unsigned long vendor, @@ -80,8 +76,7 @@ static __always_inline bool riscv_cpu_has_vendor_extension_likely(const unsigned if (!IS_ENABLED(CONFIG_RISCV_ISA_VENDOR_EXT)) return false; - if (IS_ENABLED(CONFIG_RISCV_ALTERNATIVE) && - __riscv_has_extension_likely(vendor, ext + RISCV_VENDOR_EXT_ALTERNATIVES_BASE)) + if (__riscv_has_extension_likely(vendor, ext + RISCV_VENDOR_EXT_ALTERNATIVES_BASE)) return true; return __riscv_isa_vendor_extension_available(cpu, vendor, ext); @@ -94,8 +89,7 @@ static __always_inline bool riscv_cpu_has_vendor_extension_unlikely(const unsign if (!IS_ENABLED(CONFIG_RISCV_ISA_VENDOR_EXT)) return false; - if (IS_ENABLED(CONFIG_RISCV_ALTERNATIVE) && - __riscv_has_extension_unlikely(vendor, ext + RISCV_VENDOR_EXT_ALTERNATIVES_BASE)) + if (__riscv_has_extension_unlikely(vendor, ext + RISCV_VENDOR_EXT_ALTERNATIVES_BASE)) return true; return __riscv_isa_vendor_extension_available(cpu, vendor, ext); diff --git a/arch/riscv/include/asm/vmalloc.h b/arch/riscv/include/asm/vmalloc.h index fefe94dc98e20d..4d8b80ce570b42 100644 --- a/arch/riscv/include/asm/vmalloc.h +++ b/arch/riscv/include/asm/vmalloc.h @@ -3,15 +3,14 @@ #define _ASM_RISCV_VMALLOC_H #ifdef CONFIG_HAVE_ARCH_HUGE_VMAP - -extern bool pgtable_l4_enabled, pgtable_l5_enabled; +#include #define IOREMAP_MAX_ORDER (PUD_SHIFT) #define arch_vmap_pud_supported arch_vmap_pud_supported static inline bool arch_vmap_pud_supported(pgprot_t prot) { - return pgtable_l4_enabled || pgtable_l5_enabled; + return pgtable_l4_enabled() || pgtable_l5_enabled(); } #define arch_vmap_pmd_supported arch_vmap_pmd_supported diff --git a/arch/riscv/kernel/Makefile b/arch/riscv/kernel/Makefile index ebe1c3588177b4..91353c94bfabf4 100644 --- a/arch/riscv/kernel/Makefile +++ b/arch/riscv/kernel/Makefile @@ -48,7 +48,7 @@ always-$(KBUILD_BUILTIN) += vmlinux.lds obj-y += head.o obj-y += soc.o -obj-$(CONFIG_RISCV_ALTERNATIVE) += alternative.o +obj-y += alternative.o obj-y += cpu.o obj-y += cpufeature.o obj-y += entry.o diff --git a/arch/riscv/kernel/alternative.c b/arch/riscv/kernel/alternative.c index c0c9306022c5f8..bbb215349452d2 100644 --- a/arch/riscv/kernel/alternative.c +++ b/arch/riscv/kernel/alternative.c @@ -75,7 +75,8 @@ static u32 riscv_instruction_at(void *p) } static void riscv_alternative_fix_auipc_jalr(void *ptr, u32 auipc_insn, - u32 jalr_insn, int patch_offset) + u32 jalr_insn, int patch_offset, + bool early) { u32 call[2] = { auipc_insn, jalr_insn }; s32 imm; @@ -88,10 +89,15 @@ static void riscv_alternative_fix_auipc_jalr(void *ptr, u32 auipc_insn, riscv_insn_insert_utype_itype_imm(&call[0], &call[1], imm); /* patch the call place again */ - patch_text_nosync(ptr, call, sizeof(u32) * 2); + if (early) { + memcpy(ptr, call, sizeof(call)); + } else { + patch_text_nosync(ptr, call, sizeof(call)); + } } -static void riscv_alternative_fix_jal(void *ptr, u32 jal_insn, int patch_offset) +static void riscv_alternative_fix_jal(void *ptr, u32 jal_insn, int patch_offset, + bool early) { s32 imm; @@ -103,11 +109,14 @@ static void riscv_alternative_fix_jal(void *ptr, u32 jal_insn, int patch_offset) riscv_insn_insert_jtype_imm(&jal_insn, imm); /* patch the call place again */ - patch_text_nosync(ptr, &jal_insn, sizeof(u32)); + if (early) + memcpy(ptr, &jal_insn, sizeof(u32)); + else + patch_text_nosync(ptr, &jal_insn, sizeof(u32)); } void riscv_alternative_fix_offsets(void *alt_ptr, unsigned int len, - int patch_offset) + int patch_offset, bool early) { int num_insn = len / sizeof(u32); int i; @@ -131,7 +140,8 @@ void riscv_alternative_fix_offsets(void *alt_ptr, unsigned int len, continue; riscv_alternative_fix_auipc_jalr(alt_ptr + i * sizeof(u32), - insn, insn2, patch_offset); + insn, insn2, patch_offset, + early); i++; } @@ -144,7 +154,7 @@ void riscv_alternative_fix_offsets(void *alt_ptr, unsigned int len, continue; riscv_alternative_fix_jal(alt_ptr + i * sizeof(u32), - insn, patch_offset); + insn, patch_offset, early); } } } diff --git a/arch/riscv/kernel/cpu.c b/arch/riscv/kernel/cpu.c index 3dbc8cc557dd1d..af84b97af3a19e 100644 --- a/arch/riscv/kernel/cpu.c +++ b/arch/riscv/kernel/cpu.c @@ -292,9 +292,9 @@ static void print_mmu(struct seq_file *f) #if defined(CONFIG_32BIT) sv_type = "sv32"; #elif defined(CONFIG_64BIT) - if (pgtable_l5_enabled) + if (pgtable_l5_enabled()) sv_type = "sv57"; - else if (pgtable_l4_enabled) + else if (pgtable_l4_enabled()) sv_type = "sv48"; else sv_type = "sv39"; diff --git a/arch/riscv/kernel/cpufeature.c b/arch/riscv/kernel/cpufeature.c index 61d21f7148305c..90a5de58260d9f 100644 --- a/arch/riscv/kernel/cpufeature.c +++ b/arch/riscv/kernel/cpufeature.c @@ -39,7 +39,7 @@ static bool any_cpu_has_zicbom; unsigned long elf_hwcap __read_mostly; /* Host ISA bitmap */ -static DECLARE_BITMAP(riscv_isa, RISCV_ISA_EXT_MAX) __read_mostly; +DECLARE_BITMAP(riscv_isa, RISCV_ISA_EXT_MAX) __read_mostly; /* Per-cpu ISA extensions. */ struct riscv_isainfo hart_isa[NR_CPUS]; @@ -640,6 +640,8 @@ const struct riscv_isa_ext_data riscv_isa_ext[] = { __RISCV_ISA_EXT_DATA(svpbmt, RISCV_ISA_EXT_SVPBMT), __RISCV_ISA_EXT_DATA(svrsw60t59b, RISCV_ISA_EXT_SVRSW60T59B), __RISCV_ISA_EXT_DATA(svvptc, RISCV_ISA_EXT_SVVPTC), + __RISCV_ISA_EXT_DATA(sv48, RISCV_ISA_EXT_SV48), + __RISCV_ISA_EXT_DATA(sv57, RISCV_ISA_EXT_SV57), }; const size_t riscv_isa_ext_count = ARRAY_SIZE(riscv_isa_ext); @@ -938,6 +940,11 @@ static void __init riscv_fill_hwcap_from_isa_string(unsigned long *isa2hwcap) set_bit(RISCV_ISA_EXT_ZIHPM, source_isa); } + if (_pgtable_l4_enabled) + set_bit(RISCV_ISA_EXT_SV48, source_isa); + if (_pgtable_l5_enabled) + set_bit(RISCV_ISA_EXT_SV57, source_isa); + /* * "V" in ISA strings is ambiguous in practice: it should mean * just the standard V-1.0 but vendors aren't well behaved. @@ -1098,6 +1105,11 @@ static int __init riscv_fill_hwcap_from_ext_list(unsigned long *isa2hwcap) riscv_isa_set_ext(ext, source_isa); } + if (_pgtable_l4_enabled) + set_bit(RISCV_ISA_EXT_SV48, source_isa); + if (_pgtable_l5_enabled) + set_bit(RISCV_ISA_EXT_SV57, source_isa); + riscv_resolve_isa(source_isa, isainfo->isa, &this_hwcap, isa2hwcap); riscv_fill_cpu_vendor_ext(cpu_node, cpu); @@ -1163,6 +1175,8 @@ void __init riscv_fill_hwcap(void) isa2hwcap[RISCV_ISA_EXT_C] = COMPAT_HWCAP_ISA_C; isa2hwcap[RISCV_ISA_EXT_V] = COMPAT_HWCAP_ISA_V; + bitmap_zero(riscv_isa, RISCV_ISA_EXT_MAX); + if (!acpi_disabled) { riscv_fill_hwcap_from_isa_string(isa2hwcap); } else { @@ -1233,7 +1247,6 @@ void __init riscv_user_isa_enable(void) pr_warn("Zicbop disabled as it is unavailable on some harts\n"); } -#ifdef CONFIG_RISCV_ALTERNATIVE /* * Alternative patch sites consider 48 bits when determining when to patch * the old instruction sequence with the new. These bits are broken into a @@ -1267,6 +1280,17 @@ static bool riscv_cpufeature_patch_check(u16 id, u16 value) return false; } +static bool __init_or_module riscv_is_isa_ext_early_id(u16 id) +{ + switch (id) { + case RISCV_ISA_EXT_SV48: + case RISCV_ISA_EXT_SV57: + return true; + } + + return false; +} + void __init_or_module riscv_cpufeature_patch_func(struct alt_entry *begin, struct alt_entry *end, unsigned int stage) @@ -1274,9 +1298,7 @@ void __init_or_module riscv_cpufeature_patch_func(struct alt_entry *begin, struct alt_entry *alt; void *oldptr, *altptr; u16 id, value, vendor; - - if (stage == RISCV_ALTERNATIVES_EARLY_BOOT) - return; + bool early = stage == RISCV_ALTERNATIVES_EARLY_BOOT; for (alt = begin; alt < end; alt++) { id = PATCH_ID_CPUFEATURE_ID(alt->patch_id); @@ -1291,6 +1313,8 @@ void __init_or_module riscv_cpufeature_patch_func(struct alt_entry *begin, * vendor extension. */ if (id < RISCV_ISA_EXT_MAX) { + if (early && !riscv_is_isa_ext_early_id(id)) + continue; /* * This patch should be treated as errata so skip * processing here. @@ -1305,6 +1329,8 @@ void __init_or_module riscv_cpufeature_patch_func(struct alt_entry *begin, if (!riscv_cpufeature_patch_check(id, value)) continue; } else if (id >= RISCV_VENDOR_EXT_ALTERNATIVES_BASE) { + if (early) + continue; if (!__riscv_isa_vendor_extension_available(VENDOR_EXT_ALL_CPUS, vendor, id - RISCV_VENDOR_EXT_ALTERNATIVES_BASE)) continue; @@ -1316,10 +1342,20 @@ void __init_or_module riscv_cpufeature_patch_func(struct alt_entry *begin, oldptr = ALT_OLD_PTR(alt); altptr = ALT_ALT_PTR(alt); - mutex_lock(&text_mutex); - patch_text_nosync(oldptr, altptr, alt->alt_len); - riscv_alternative_fix_offsets(oldptr, alt->alt_len, oldptr - altptr); - mutex_unlock(&text_mutex); + if (early) { + /* oldptr is writable through the MMU-off kernel mapping. */ + memcpy(oldptr, altptr, alt->alt_len); + riscv_alternative_fix_offsets(oldptr, alt->alt_len, + oldptr - altptr, true); + } else { + mutex_lock(&text_mutex); + patch_text_nosync(oldptr, altptr, alt->alt_len); + riscv_alternative_fix_offsets(oldptr, alt->alt_len, + oldptr - altptr, false); + mutex_unlock(&text_mutex); + } } + + if (early) + local_flush_icache_all(); } -#endif diff --git a/arch/riscv/mm/init.c b/arch/riscv/mm/init.c index fb37b0b67efec6..e2b738ca90cf32 100644 --- a/arch/riscv/mm/init.c +++ b/arch/riscv/mm/init.c @@ -6,6 +6,11 @@ * Nick Kossifidis */ +#ifdef CONFIG_64BIT +/* riscv_has_extension_likely() cannot be used this early */ +#define USE_EARLY_PGTABLE_LEVELS +#endif + #include #include #include @@ -52,10 +57,8 @@ u64 satp_mode __ro_after_init = SATP_MODE_32; EXPORT_SYMBOL(satp_mode); #ifdef CONFIG_64BIT -bool pgtable_l4_enabled __ro_after_init = true; -bool pgtable_l5_enabled __ro_after_init = true; -EXPORT_SYMBOL(pgtable_l4_enabled); -EXPORT_SYMBOL(pgtable_l5_enabled); +bool _pgtable_l4_enabled __ro_after_init = true; +bool _pgtable_l5_enabled __ro_after_init = true; #endif phys_addr_t phys_ram_base __ro_after_init; @@ -652,23 +655,23 @@ static void __meminit create_p4d_mapping(p4d_t *p4dp, uintptr_t va, phys_addr_t } #define pgd_next_t p4d_t -#define alloc_pgd_next(__va) (pgtable_l5_enabled ? \ - pt_ops.alloc_p4d(__va) : (pgtable_l4_enabled ? \ +#define alloc_pgd_next(__va) (pgtable_l5_enabled() ? \ + pt_ops.alloc_p4d(__va) : (pgtable_l4_enabled() ? \ pt_ops.alloc_pud(__va) : pt_ops.alloc_pmd(__va))) -#define get_pgd_next_virt(__pa) (pgtable_l5_enabled ? \ - pt_ops.get_p4d_virt(__pa) : (pgd_next_t *)(pgtable_l4_enabled ? \ +#define get_pgd_next_virt(__pa) (pgtable_l5_enabled() ? \ + pt_ops.get_p4d_virt(__pa) : (pgd_next_t *)(pgtable_l4_enabled() ? \ pt_ops.get_pud_virt(__pa) : (pud_t *)pt_ops.get_pmd_virt(__pa))) #define create_pgd_next_mapping(__nextp, __va, __pa, __sz, __prot) \ - (pgtable_l5_enabled ? \ + (pgtable_l5_enabled() ? \ create_p4d_mapping(__nextp, __va, __pa, __sz, __prot) : \ - (pgtable_l4_enabled ? \ + (pgtable_l4_enabled() ? \ create_pud_mapping((pud_t *)__nextp, __va, __pa, __sz, __prot) : \ create_pmd_mapping((pmd_t *)__nextp, __va, __pa, __sz, __prot))) -#define fixmap_pgd_next (pgtable_l5_enabled ? \ - (uintptr_t)fixmap_p4d : (pgtable_l4_enabled ? \ +#define fixmap_pgd_next (pgtable_l5_enabled() ? \ + (uintptr_t)fixmap_p4d : (pgtable_l4_enabled() ? \ (uintptr_t)fixmap_pud : (uintptr_t)fixmap_pmd)) -#define trampoline_pgd_next (pgtable_l5_enabled ? \ - (uintptr_t)trampoline_p4d : (pgtable_l4_enabled ? \ +#define trampoline_pgd_next (pgtable_l5_enabled() ? \ + (uintptr_t)trampoline_p4d : (pgtable_l4_enabled() ? \ (uintptr_t)trampoline_pud : (uintptr_t)trampoline_pmd)) #else #define pgd_next_t pte_t @@ -713,11 +716,11 @@ static uintptr_t __meminit best_map_size(phys_addr_t pa, uintptr_t va, phys_addr if (debug_pagealloc_enabled()) return PAGE_SIZE; - if (pgtable_l5_enabled && + if (pgtable_l5_enabled() && !(pa & (P4D_SIZE - 1)) && !(va & (P4D_SIZE - 1)) && size >= P4D_SIZE) return P4D_SIZE; - if (pgtable_l4_enabled && + if (pgtable_l4_enabled() && !(pa & (PUD_SIZE - 1)) && !(va & (PUD_SIZE - 1)) && size >= PUD_SIZE) return PUD_SIZE; @@ -769,14 +772,14 @@ u64 __pi_set_satp_mode_from_fdt(uintptr_t dtb_pa); static void __init disable_pgtable_l5(void) { - pgtable_l5_enabled = false; + _pgtable_l5_enabled = false; kernel_map.page_offset = PAGE_OFFSET_L4; satp_mode = SATP_MODE_48; } static void __init disable_pgtable_l4(void) { - pgtable_l4_enabled = false; + _pgtable_l4_enabled = false; kernel_map.page_offset = PAGE_OFFSET_L3; satp_mode = SATP_MODE_39; } @@ -839,6 +842,14 @@ static __init void set_satp_mode(uintptr_t dtb_pa) kernel_map.page_offset = PAGE_OFFSET_L5; + /* Fall back to SV48 if RISCV_ISA_SV57 isn't enabled */ + if (!IS_ENABLED(CONFIG_RISCV_ISA_SV57) && satp_mode_limit == 0) + satp_mode_limit = SATP_MODE_48; + + /* Fall back to SV39 if RISCV_ISA_SV48 isn't enabled */ + if (!IS_ENABLED(CONFIG_RISCV_ISA_SV48) && satp_mode_limit == SATP_MODE_48) + satp_mode_limit = SATP_MODE_39; + if (satp_mode_limit == SATP_MODE_48) { disable_pgtable_l5(); } else if (satp_mode_limit == SATP_MODE_39) { @@ -867,7 +878,7 @@ static __init void set_satp_mode(uintptr_t dtb_pa) create_pgd_mapping(early_pg_dir, set_satp_mode_pmd, - pgtable_l5_enabled ? + pgtable_l5_enabled() ? (uintptr_t)early_p4d : (uintptr_t)early_pud, PGDIR_SIZE, PAGE_TABLE); @@ -879,7 +890,7 @@ static __init void set_satp_mode(uintptr_t dtb_pa) local_flush_tlb_all(); if (hw_satp != identity_satp) { - if (pgtable_l5_enabled) { + if (pgtable_l5_enabled()) { disable_pgtable_l5(); memset(early_pg_dir, 0, PAGE_SIZE); goto retry; @@ -892,6 +903,10 @@ static __init void set_satp_mode(uintptr_t dtb_pa) memset(early_p4d, 0, PAGE_SIZE); memset(early_pud, 0, PAGE_SIZE); memset(early_pmd, 0, PAGE_SIZE); + if (pgtable_l4_enabled()) + set_bit(RISCV_ISA_EXT_SV48, riscv_isa); + if (pgtable_l5_enabled()) + set_bit(RISCV_ISA_EXT_SV57, riscv_isa); } #endif @@ -1120,11 +1135,11 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa) #ifndef __PAGETABLE_PMD_FOLDED /* Setup fixmap P4D and PUD */ - if (pgtable_l5_enabled) + if (pgtable_l5_enabled()) create_p4d_mapping(fixmap_p4d, FIXADDR_START, (uintptr_t)fixmap_pud, P4D_SIZE, PAGE_TABLE); /* Setup fixmap PUD and PMD */ - if (pgtable_l4_enabled) + if (pgtable_l4_enabled()) create_pud_mapping(fixmap_pud, FIXADDR_START, (uintptr_t)fixmap_pmd, PUD_SIZE, PAGE_TABLE); create_pmd_mapping(fixmap_pmd, FIXADDR_START, @@ -1132,10 +1147,10 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa) /* Setup trampoline PGD and PMD */ create_pgd_mapping(trampoline_pg_dir, kernel_map.virt_addr, trampoline_pgd_next, PGDIR_SIZE, PAGE_TABLE); - if (pgtable_l5_enabled) + if (pgtable_l5_enabled()) create_p4d_mapping(trampoline_p4d, kernel_map.virt_addr, (uintptr_t)trampoline_pud, P4D_SIZE, PAGE_TABLE); - if (pgtable_l4_enabled) + if (pgtable_l4_enabled()) create_pud_mapping(trampoline_pud, kernel_map.virt_addr, (uintptr_t)trampoline_pmd, PUD_SIZE, PAGE_TABLE); create_pmd_mapping(trampoline_pmd, kernel_map.virt_addr, @@ -1406,7 +1421,7 @@ static void __init preallocate_pgd_pages_range(unsigned long start, unsigned lon if (!p4d) goto failed; - if (pgtable_l5_enabled) + if (pgtable_l5_enabled()) continue; lvl = "pud"; @@ -1414,7 +1429,7 @@ static void __init preallocate_pgd_pages_range(unsigned long start, unsigned lon if (!pud) goto failed; - if (pgtable_l4_enabled) + if (pgtable_l4_enabled()) continue; lvl = "pmd"; @@ -1626,13 +1641,13 @@ static void __meminit remove_pud_mapping(pud_t *pud_base, unsigned long addr, un for (; addr < end; addr = next) { next = pud_addr_end(addr, end); - pudp = pgtable_l4_enabled ? pud_base + pud_index(addr) : pud_base; + pudp = pgtable_l4_enabled() ? pud_base + pud_index(addr) : pud_base; pud = pudp_get(pudp); if (!pud_present(pud)) continue; if (pud_leaf(pud)) { - if (pgtable_l4_enabled) { + if (pgtable_l4_enabled()) { pud_clear(pudp); if (is_vmemmap) free_vmemmap_storage(pud_page(pud), PUD_SIZE, altmap); @@ -1643,7 +1658,7 @@ static void __meminit remove_pud_mapping(pud_t *pud_base, unsigned long addr, un pmd_base = pmd_offset(pudp, 0); remove_pmd_mapping(pmd_base, addr, next, is_vmemmap, altmap); - if (pgtable_l4_enabled) + if (pgtable_l4_enabled()) free_pmd_table(pmd_base, pudp, is_vmemmap); } } @@ -1657,13 +1672,13 @@ static void __meminit remove_p4d_mapping(p4d_t *p4d_base, unsigned long addr, un for (; addr < end; addr = next) { next = p4d_addr_end(addr, end); - p4dp = pgtable_l5_enabled ? p4d_base + p4d_index(addr) : p4d_base; + p4dp = pgtable_l5_enabled() ? p4d_base + p4d_index(addr) : p4d_base; p4d = p4dp_get(p4dp); if (!p4d_present(p4d)) continue; if (p4d_leaf(p4d)) { - if (pgtable_l5_enabled) { + if (pgtable_l5_enabled()) { p4d_clear(p4dp); if (is_vmemmap) free_vmemmap_storage(p4d_page(p4d), P4D_SIZE, altmap); @@ -1674,7 +1689,7 @@ static void __meminit remove_p4d_mapping(p4d_t *p4d_base, unsigned long addr, un pud_base = pud_offset(p4dp, 0); remove_pud_mapping(pud_base, addr, next, is_vmemmap, altmap); - if (pgtable_l5_enabled) + if (pgtable_l5_enabled()) free_pud_table(pud_base, p4dp); } } diff --git a/arch/riscv/mm/kasan_init.c b/arch/riscv/mm/kasan_init.c index 1f3aa9611187f6..84f87c8277c3f2 100644 --- a/arch/riscv/mm/kasan_init.c +++ b/arch/riscv/mm/kasan_init.c @@ -168,7 +168,7 @@ static void __init kasan_early_clear_pud(p4d_t *p4dp, pud_t *pudp, *base_pud; unsigned long next; - if (!pgtable_l4_enabled) { + if (!pgtable_l4_enabled()) { pudp = (pud_t *)p4dp; } else { base_pud = pt_ops.get_pud_virt(pfn_to_phys(_p4d_pfn(p4dp_get(p4dp)))); @@ -193,7 +193,7 @@ static void __init kasan_early_clear_p4d(pgd_t *pgdp, p4d_t *p4dp, *base_p4d; unsigned long next; - if (!pgtable_l5_enabled) { + if (!pgtable_l5_enabled()) { p4dp = (p4d_t *)pgdp; } else { base_p4d = pt_ops.get_p4d_virt(pfn_to_phys(_pgd_pfn(pgdp_get(pgdp)))); @@ -203,7 +203,7 @@ static void __init kasan_early_clear_p4d(pgd_t *pgdp, do { next = p4d_addr_end(vaddr, end); - if (pgtable_l4_enabled && IS_ALIGNED(vaddr, P4D_SIZE) && + if (pgtable_l4_enabled() && IS_ALIGNED(vaddr, P4D_SIZE) && (next - vaddr) >= P4D_SIZE) { p4d_clear(p4dp); continue; @@ -221,7 +221,7 @@ static void __init kasan_early_clear_pgd(pgd_t *pgdp, do { next = pgd_addr_end(vaddr, end); - if (pgtable_l5_enabled && IS_ALIGNED(vaddr, PGDIR_SIZE) && + if (pgtable_l5_enabled() && IS_ALIGNED(vaddr, PGDIR_SIZE) && (next - vaddr) >= PGDIR_SIZE) { pgd_clear(pgdp); continue; @@ -239,7 +239,7 @@ static void __init kasan_early_populate_pud(p4d_t *p4dp, phys_addr_t phys_addr; unsigned long next; - if (!pgtable_l4_enabled) { + if (!pgtable_l4_enabled()) { pudp = (pud_t *)p4dp; } else { base_pud = pt_ops.get_pud_virt(pfn_to_phys(_p4d_pfn(p4dp_get(p4dp)))); @@ -277,7 +277,7 @@ static void __init kasan_early_populate_p4d(pgd_t *pgdp, * Note that this test is then completely equivalent to * p4dp = p4d_offset(pgdp, vaddr) */ - if (!pgtable_l5_enabled) { + if (!pgtable_l5_enabled()) { p4dp = (p4d_t *)pgdp; } else { base_p4d = pt_ops.get_p4d_virt(pfn_to_phys(_pgd_pfn(pgdp_get(pgdp)))); @@ -336,7 +336,7 @@ asmlinkage void __init kasan_early_init(void) (__pa((uintptr_t)kasan_early_shadow_pte)), PAGE_TABLE)); - if (pgtable_l4_enabled) { + if (pgtable_l4_enabled()) { for (i = 0; i < PTRS_PER_PUD; ++i) set_pud(kasan_early_shadow_pud + i, pfn_pud(PFN_DOWN @@ -344,7 +344,7 @@ asmlinkage void __init kasan_early_init(void) PAGE_TABLE)); } - if (pgtable_l5_enabled) { + if (pgtable_l5_enabled()) { for (i = 0; i < PTRS_PER_P4D; ++i) set_p4d(kasan_early_shadow_p4d + i, pfn_p4d(PFN_DOWN @@ -461,7 +461,7 @@ static void __init create_tmp_mapping(void) memcpy(tmp_pg_dir, swapper_pg_dir, sizeof(pgd_t) * PTRS_PER_PGD); /* Copy the last p4d since it is shared with the kernel mapping. */ - if (pgtable_l5_enabled) { + if (pgtable_l5_enabled()) { ptr = (p4d_t *)pgd_page_vaddr(pgdp_get(pgd_offset_k(KASAN_SHADOW_END))); memcpy(tmp_p4d, ptr, sizeof(p4d_t) * PTRS_PER_P4D); set_pgd(&tmp_pg_dir[pgd_index(KASAN_SHADOW_END)], @@ -472,7 +472,7 @@ static void __init create_tmp_mapping(void) } /* Copy the last pud since it is shared with the kernel mapping. */ - if (pgtable_l4_enabled) { + if (pgtable_l4_enabled()) { ptr = (pud_t *)p4d_page_vaddr(p4dp_get(base_p4d + p4d_index(KASAN_SHADOW_END))); memcpy(tmp_pud, ptr, sizeof(pud_t) * PTRS_PER_PUD); set_p4d(&base_p4d[p4d_index(KASAN_SHADOW_END)], diff --git a/arch/riscv/mm/pgtable.c b/arch/riscv/mm/pgtable.c index 9c4427d0b1874e..cd180f9abb5e3c 100644 --- a/arch/riscv/mm/pgtable.c +++ b/arch/riscv/mm/pgtable.c @@ -41,7 +41,7 @@ EXPORT_SYMBOL_GPL(ptep_test_and_clear_young); #ifdef CONFIG_64BIT pud_t *pud_offset(p4d_t *p4d, unsigned long address) { - if (pgtable_l4_enabled) + if (pgtable_l4_enabled()) return p4d_pgtable(p4dp_get(p4d)) + pud_index(address); return (pud_t *)p4d; @@ -50,7 +50,7 @@ EXPORT_SYMBOL_GPL(pud_offset); p4d_t *p4d_offset(pgd_t *pgd, unsigned long address) { - if (pgtable_l5_enabled) + if (pgtable_l5_enabled()) return pgd_pgtable(pgdp_get(pgd)) + p4d_index(address); return (p4d_t *)pgd; diff --git a/arch/riscv/mm/ptdump.c b/arch/riscv/mm/ptdump.c index f4b4a9fcbbd886..f765a63e2ca7db 100644 --- a/arch/riscv/mm/ptdump.c +++ b/arch/riscv/mm/ptdump.c @@ -446,8 +446,8 @@ static int __init ptdump_init(void) kernel_ptd_info.base_addr = KERN_VIRT_START; - pg_level[1].name = pgtable_l5_enabled ? "P4D" : "PGD"; - pg_level[2].name = pgtable_l4_enabled ? "PUD" : "PGD"; + pg_level[1].name = pgtable_l5_enabled() ? "P4D" : "PGD"; + pg_level[2].name = pgtable_l4_enabled() ? "PUD" : "PGD"; for (i = 0; i < ARRAY_SIZE(pg_level); i++) for (j = 0; j < ARRAY_SIZE(pte_bits); j++)