diff --git a/arch/arm/mm/fault.c b/arch/arm/mm/fault.c index 0a09d4ff771853..70472744f5c5aa 100644 --- a/arch/arm/mm/fault.c +++ b/arch/arm/mm/fault.c @@ -344,6 +344,7 @@ do_page_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs) vm_fault_t fault; unsigned int flags = FAULT_FLAG_DEFAULT; vm_flags_t vm_flags = VM_ACCESS_FLAGS; + bool vma_lock_retried = false; if (kprobe_page_fault(regs, fsr)) return 0; @@ -395,6 +396,7 @@ do_page_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs) if (!(flags & FAULT_FLAG_USER)) goto lock_mmap; +lock_vma: vma = lock_vma_under_rcu(mm, addr); if (!vma) goto lock_mmap; @@ -424,6 +426,12 @@ do_page_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs) goto no_context; return 0; } + + if (!vma_lock_retried) { + vma_lock_retried = true; + goto lock_vma; + } + lock_mmap: retry: diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c index 0b52557652be66..b8633863380e80 100644 --- a/arch/arm64/mm/fault.c +++ b/arch/arm64/mm/fault.c @@ -610,6 +610,7 @@ static int __kprobes do_page_fault(unsigned long far, unsigned long esr, struct vm_area_struct *vma; int si_code; int pkey = -1; + bool vma_lock_retried = false; if (kprobe_page_fault(regs, esr)) return 0; @@ -678,6 +679,7 @@ static int __kprobes do_page_fault(unsigned long far, unsigned long esr, if (!(mm_flags & FAULT_FLAG_USER)) goto lock_mmap; +lock_vma: vma = lock_vma_under_rcu(mm, addr); if (!vma) goto lock_mmap; @@ -724,6 +726,12 @@ static int __kprobes do_page_fault(unsigned long far, unsigned long esr, goto no_context; return 0; } + + if (!vma_lock_retried) { + vma_lock_retried = true; + goto lock_vma; + } + lock_mmap: retry: diff --git a/arch/loongarch/mm/fault.c b/arch/loongarch/mm/fault.c index 2c93d33356e57b..ef6ea847b1e05a 100644 --- a/arch/loongarch/mm/fault.c +++ b/arch/loongarch/mm/fault.c @@ -181,6 +181,7 @@ static void __kprobes __do_page_fault(struct pt_regs *regs, struct mm_struct *mm = tsk->mm; struct vm_area_struct *vma = NULL; vm_fault_t fault; + bool vma_lock_retried = false; if (kprobe_page_fault(regs, current->thread.trap_nr)) return; @@ -219,6 +220,7 @@ static void __kprobes __do_page_fault(struct pt_regs *regs, if (!(flags & FAULT_FLAG_USER)) goto lock_mmap; +lock_vma: vma = lock_vma_under_rcu(mm, address); if (!vma) goto lock_mmap; @@ -265,6 +267,12 @@ static void __kprobes __do_page_fault(struct pt_regs *regs, no_context(regs, write, address); return; } + + if (!vma_lock_retried) { + vma_lock_retried = true; + goto lock_vma; + } + lock_mmap: retry: diff --git a/arch/powerpc/mm/fault.c b/arch/powerpc/mm/fault.c index 806c74e0d5ab70..06018b6d7086ff 100644 --- a/arch/powerpc/mm/fault.c +++ b/arch/powerpc/mm/fault.c @@ -422,6 +422,7 @@ static int ___do_page_fault(struct pt_regs *regs, unsigned long address, int is_write = page_fault_is_write(error_code); vm_fault_t fault, major = 0; bool kprobe_fault = kprobe_page_fault(regs, 11); + bool vma_lock_retried = false; if (unlikely(debugger_fault_handler(regs) || kprobe_fault)) return 0; @@ -487,6 +488,7 @@ static int ___do_page_fault(struct pt_regs *regs, unsigned long address, if (!(flags & FAULT_FLAG_USER)) goto lock_mmap; +lock_vma: vma = lock_vma_under_rcu(mm, address); if (!vma) goto lock_mmap; @@ -517,6 +519,11 @@ static int ___do_page_fault(struct pt_regs *regs, unsigned long address, if (fault_signal_pending(fault, regs)) return user_mode(regs) ? 0 : SIGBUS; + if (!vma_lock_retried) { + vma_lock_retried = true; + goto lock_vma; + } + lock_mmap: /* When running in the kernel we expect faults to occur only to diff --git a/arch/riscv/mm/fault.c b/arch/riscv/mm/fault.c index 04ed6f8acae4fd..ff861793dba973 100644 --- a/arch/riscv/mm/fault.c +++ b/arch/riscv/mm/fault.c @@ -284,6 +284,7 @@ void handle_page_fault(struct pt_regs *regs) unsigned int flags = FAULT_FLAG_DEFAULT; int code = SEGV_MAPERR; vm_fault_t fault; + bool vma_lock_retried = false; cause = regs->cause; addr = regs->badaddr; @@ -347,6 +348,7 @@ void handle_page_fault(struct pt_regs *regs) if (!(flags & FAULT_FLAG_USER)) goto lock_mmap; +lock_vma: vma = lock_vma_under_rcu(mm, addr); if (!vma) goto lock_mmap; @@ -376,6 +378,12 @@ void handle_page_fault(struct pt_regs *regs) no_context(regs, addr); return; } + + if (!vma_lock_retried) { + vma_lock_retried = true; + goto lock_vma; + } + lock_mmap: retry: diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c index 46d828926009bf..dcd1ba24497f8a 100644 --- a/arch/s390/mm/fault.c +++ b/arch/s390/mm/fault.c @@ -271,6 +271,7 @@ static void do_exception(struct pt_regs *regs, int access) unsigned int flags; vm_fault_t fault; bool is_write; + bool vma_lock_retried = false; /* * The instruction that caused the program check has @@ -294,6 +295,7 @@ static void do_exception(struct pt_regs *regs, int access) flags |= FAULT_FLAG_WRITE; if (!(flags & FAULT_FLAG_USER)) goto lock_mmap; +lock_vma: vma = lock_vma_under_rcu(mm, address); if (!vma) goto lock_mmap; @@ -318,6 +320,10 @@ static void do_exception(struct pt_regs *regs, int access) handle_fault_error_nolock(regs, 0); return; } + if (!vma_lock_retried) { + vma_lock_retried = true; + goto lock_vma; + } lock_mmap: retry: vma = lock_mm_and_find_vma(mm, address, regs); diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c index aa88370ce73943..df10d5cea4eec5 100644 --- a/arch/x86/mm/fault.c +++ b/arch/x86/mm/fault.c @@ -1222,6 +1222,7 @@ void do_user_addr_fault(struct pt_regs *regs, struct mm_struct *mm; vm_fault_t fault; unsigned int flags = FAULT_FLAG_DEFAULT; + bool vma_lock_retried = false; tsk = current; mm = tsk->mm; @@ -1331,6 +1332,7 @@ void do_user_addr_fault(struct pt_regs *regs, if (!(flags & FAULT_FLAG_USER)) goto lock_mmap; +lock_vma: vma = lock_vma_under_rcu(mm, address); if (!vma) goto lock_mmap; @@ -1360,6 +1362,12 @@ void do_user_addr_fault(struct pt_regs *regs, ARCH_DEFAULT_PKEY); return; } + + if (!vma_lock_retried) { + vma_lock_retried = true; + goto lock_vma; + } + lock_mmap: retry: