diff --git a/arch/arm64/include/asm/uaccess.h b/arch/arm64/include/asm/uaccess.h index 9f5bd9c69c24..1a14eb2a51dc 100644 --- a/arch/arm64/include/asm/uaccess.h +++ b/arch/arm64/include/asm/uaccess.h @@ -270,28 +270,43 @@ do { \ #define get_user __get_user /* - * We must not call into the scheduler between __mte_enable_tco_async() and - * __mte_disable_tco_async(). As `dst` and `src` may contain blocking - * functions, we must evaluate these outside of the critical section. + * Nofault load without TCO management for use inside a + * __begin/__end_kernel_nofault_bare() region. */ -#define __get_kernel_nofault(dst, src, type, err_label) \ +#define __get_kernel_nofault_bare(dst, src, type, err_label) \ do { \ __typeof__(dst) __gkn_dst = (dst); \ __typeof__(src) __gkn_src = (src); \ do { \ __label__ __gkn_label; \ - \ - __mte_enable_tco_async(); \ __raw_get_mem("ldr", *((type *)(__gkn_dst)), \ (__force type *)(__gkn_src), __gkn_label, K); \ - __mte_disable_tco_async(); \ break; \ __gkn_label: \ - __mte_disable_tco_async(); \ goto err_label; \ } while (0); \ } while (0) +/* + * We must not call into the scheduler between __mte_enable_tco_async() and + * __mte_disable_tco_async(). As dst and src may contain blocking functions, + * evaluate them before overriding TCO. + */ +#define __get_kernel_nofault(dst, src, type, err_label) \ +do { \ + __label__ __gkn_tco_err; \ + __typeof__(dst) __gkn_tco_dst = (dst); \ + __typeof__(src) __gkn_tco_src = (src); \ + __mte_enable_tco_async(); \ + __get_kernel_nofault_bare(__gkn_tco_dst, __gkn_tco_src, type, \ + __gkn_tco_err); \ + __mte_disable_tco_async(); \ + break; \ +__gkn_tco_err: \ + __mte_disable_tco_async(); \ + goto err_label; \ +} while (0) + #define __put_mem_asm(store, reg, x, addr, label, type) \ asm goto( \ "1: " store " " reg "0, [%1]\n" \ @@ -366,28 +381,48 @@ do { \ #define put_user __put_user -/* - * We must not call into the scheduler between __mte_enable_tco_async() and - * __mte_disable_tco_async(). As `dst` and `src` may contain blocking - * functions, we must evaluate these outside of the critical section. - */ -#define __put_kernel_nofault(dst, src, type, err_label) \ +/* Nofault store without TCO management; see __get_kernel_nofault_bare. */ +#define __put_kernel_nofault_bare(dst, src, type, err_label) \ do { \ __typeof__(dst) __pkn_dst = (dst); \ __typeof__(src) __pkn_src = (src); \ \ do { \ __label__ __pkn_err; \ - __mte_enable_tco_async(); \ __raw_put_mem("str", *((type *)(__pkn_src)), \ (__force type *)(__pkn_dst), __pkn_err, K); \ - __mte_disable_tco_async(); \ break; \ __pkn_err: \ - __mte_disable_tco_async(); \ goto err_label; \ } while (0); \ -} while(0) +} while (0) + +/* + * We must not call into the scheduler between __mte_enable_tco_async() and + * __mte_disable_tco_async(). As `dst` and `src` may contain blocking + * functions, we must evaluate these outside of the critical section. + */ +#define __put_kernel_nofault(dst, src, type, err_label) \ +do { \ + __label__ __pkn_tco_err; \ + __typeof__(dst) __pkn_tco_dst = (dst); \ + __typeof__(src) __pkn_tco_src = (src); \ + __mte_enable_tco_async(); \ + __put_kernel_nofault_bare(__pkn_tco_dst, __pkn_tco_src, type, \ + __pkn_tco_err); \ + __mte_disable_tco_async(); \ + break; \ +__pkn_tco_err: \ + __mte_disable_tco_async(); \ + goto err_label; \ +} while (0) + +/* + * A context switch re-enables tag checking, hence the no-scheduling + * requirement for a bare nofault region. + */ +#define __begin_kernel_nofault_bare() __mte_enable_tco_async() +#define __end_kernel_nofault_bare() __mte_disable_tco_async() extern unsigned long __must_check __arch_copy_from_user(void *to, const void __user *from, unsigned long n); #define raw_copy_from_user(to, from, n) \ diff --git a/include/linux/uaccess.h b/include/linux/uaccess.h index eddbbb65ccc4..b25e778ddc38 100644 --- a/include/linux/uaccess.h +++ b/include/linux/uaccess.h @@ -637,6 +637,26 @@ do { \ #endif /* !__get_kernel_nofault */ +/* + * Architectures may use the begin/end hooks to establish state shared by a + * sequence of bare nofault accesses. Every path out of the region must call + * the end hook. The region, including expressions passed to the bare + * accessors, must not call into the scheduler. + */ +#ifndef __get_kernel_nofault_bare +#define __get_kernel_nofault_bare __get_kernel_nofault +#define __put_kernel_nofault_bare __put_kernel_nofault +#endif + +#ifndef __begin_kernel_nofault_bare +#define __begin_kernel_nofault_bare() do {} while (0) +#define __end_kernel_nofault_bare() do {} while (0) +#endif + +DEFINE_LOCK_GUARD_0(__kernel_nofault_bare, + __begin_kernel_nofault_bare(), + __end_kernel_nofault_bare()) + /** * get_kernel_nofault(): safely attempt to read from a location * @val: read into this variable diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c index b3cc5c8fc875..f8c557161584 100644 --- a/kernel/bpf/helpers.c +++ b/kernel/bpf/helpers.c @@ -3726,7 +3726,8 @@ __bpf_kfunc void __bpf_trap(void) * * Since strings are not necessarily %NUL-terminated, we cannot directly call * in-kernel implementations. Instead, we open-code the implementations using - * __get_kernel_nofault instead of plain dereference to make them safe. + * __get_kernel_nofault_bare under guard(__kernel_nofault_bare)() to make them + * safe. */ static int __bpf_strncasecmp(const char *s1, const char *s2, bool ignore_case, size_t len) @@ -3738,11 +3739,14 @@ static int __bpf_strncasecmp(const char *s1, const char *s2, bool ignore_case, s !copy_from_kernel_nofault_allowed(s2, 1)) { return -ERANGE; } + if (!len) + return 0; guard(pagefault)(); + guard(__kernel_nofault_bare)(); for (i = 0; i < len && i < XATTR_SIZE_MAX; i++) { - __get_kernel_nofault(&c1, s1, char, err_out); - __get_kernel_nofault(&c2, s2, char, err_out); + __get_kernel_nofault_bare(&c1, s1, char, err_out); + __get_kernel_nofault_bare(&c2, s2, char, err_out); if (ignore_case) { c1 = tolower(c1); c2 = tolower(c2); @@ -3837,10 +3841,13 @@ __bpf_kfunc int bpf_strnchr(const char *s__ign, size_t count, char c) if (!copy_from_kernel_nofault_allowed(s__ign, 1)) return -ERANGE; + if (!count) + return -ENOENT; guard(pagefault)(); + guard(__kernel_nofault_bare)(); for (i = 0; i < count && i < XATTR_SIZE_MAX; i++) { - __get_kernel_nofault(&sc, s__ign, char, err_out); + __get_kernel_nofault_bare(&sc, s__ign, char, err_out); if (sc == c) return i; if (sc == '\0') @@ -3893,8 +3900,9 @@ __bpf_kfunc int bpf_strchrnul(const char *s__ign, char c) return -ERANGE; guard(pagefault)(); + guard(__kernel_nofault_bare)(); for (i = 0; i < XATTR_SIZE_MAX; i++) { - __get_kernel_nofault(&sc, s__ign, char, err_out); + __get_kernel_nofault_bare(&sc, s__ign, char, err_out); if (sc == '\0' || sc == c) return i; s__ign++; @@ -3925,8 +3933,9 @@ __bpf_kfunc int bpf_strrchr(const char *s__ign, int c) return -ERANGE; guard(pagefault)(); + guard(__kernel_nofault_bare)(); for (i = 0; i < XATTR_SIZE_MAX; i++) { - __get_kernel_nofault(&sc, s__ign, char, err_out); + __get_kernel_nofault_bare(&sc, s__ign, char, err_out); if (sc == c) last = i; if (sc == '\0') @@ -3956,10 +3965,13 @@ __bpf_kfunc int bpf_strnlen(const char *s__ign, size_t count) if (!copy_from_kernel_nofault_allowed(s__ign, 1)) return -ERANGE; + if (!count) + return 0; guard(pagefault)(); + guard(__kernel_nofault_bare)(); for (i = 0; i < count && i < XATTR_SIZE_MAX; i++) { - __get_kernel_nofault(&c, s__ign, char, err_out); + __get_kernel_nofault_bare(&c, s__ign, char, err_out); if (c == '\0') return i; s__ign++; @@ -4008,12 +4020,13 @@ __bpf_kfunc int bpf_strspn(const char *s__ign, const char *accept__ign) } guard(pagefault)(); + guard(__kernel_nofault_bare)(); for (i = 0; i < XATTR_SIZE_MAX; i++) { - __get_kernel_nofault(&cs, s__ign, char, err_out); + __get_kernel_nofault_bare(&cs, s__ign, char, err_out); if (cs == '\0') return i; for (j = 0; j < XATTR_SIZE_MAX; j++) { - __get_kernel_nofault(&ca, accept__ign + j, char, err_out); + __get_kernel_nofault_bare(&ca, accept__ign + j, char, err_out); if (cs == ca || ca == '\0') break; } @@ -4052,12 +4065,13 @@ __bpf_kfunc int bpf_strcspn(const char *s__ign, const char *reject__ign) } guard(pagefault)(); + guard(__kernel_nofault_bare)(); for (i = 0; i < XATTR_SIZE_MAX; i++) { - __get_kernel_nofault(&cs, s__ign, char, err_out); + __get_kernel_nofault_bare(&cs, s__ign, char, err_out); if (cs == '\0') return i; for (j = 0; j < XATTR_SIZE_MAX; j++) { - __get_kernel_nofault(&cr, reject__ign + j, char, err_out); + __get_kernel_nofault_bare(&cr, reject__ign + j, char, err_out); if (cs == cr || cr == '\0') break; } @@ -4084,9 +4098,10 @@ static int __bpf_strnstr(const char *s1, const char *s2, size_t len, } guard(pagefault)(); + guard(__kernel_nofault_bare)(); for (i = 0; i < XATTR_SIZE_MAX; i++) { for (j = 0; i + j <= len && j < XATTR_SIZE_MAX; j++) { - __get_kernel_nofault(&c2, s2 + j, char, err_out); + __get_kernel_nofault_bare(&c2, s2 + j, char, err_out); if (c2 == '\0') return i; /* @@ -4096,7 +4111,7 @@ static int __bpf_strnstr(const char *s1, const char *s2, size_t len, */ if (i + j == len) break; - __get_kernel_nofault(&c1, s1 + j, char, err_out); + __get_kernel_nofault_bare(&c1, s1 + j, char, err_out); if (ignore_case) { c1 = tolower(c1); diff --git a/mm/maccess.c b/mm/maccess.c index 486559d68858..87486bf98d15 100644 --- a/mm/maccess.c +++ b/mm/maccess.c @@ -19,7 +19,7 @@ bool __weak copy_from_kernel_nofault_allowed(const void *unsafe_src, */ #define copy_from_kernel_nofault_loop(dst, src, len, type, err_label) \ while (len >= sizeof(type)) { \ - __get_kernel_nofault(dst, src, type, err_label); \ + __get_kernel_nofault_bare(dst, src, type, err_label); \ kmsan_check_memory(src, sizeof(type)); \ dst += sizeof(type); \ src += sizeof(type); \ @@ -35,26 +35,29 @@ long copy_from_kernel_nofault(void *dst, const void *src, size_t size) if (!copy_from_kernel_nofault_allowed(src, size)) return -ERANGE; + if (!size) + return 0; - pagefault_disable(); - if (!(align & 7)) - copy_from_kernel_nofault_loop(dst, src, size, u64, Efault); - if (!(align & 3)) - copy_from_kernel_nofault_loop(dst, src, size, u32, Efault); - if (!(align & 1)) - copy_from_kernel_nofault_loop(dst, src, size, u16, Efault); - copy_from_kernel_nofault_loop(dst, src, size, u8, Efault); - pagefault_enable(); + scoped_guard(pagefault) { + scoped_guard(__kernel_nofault_bare) { + if (!(align & 7)) + copy_from_kernel_nofault_loop(dst, src, size, u64, Efault); + if (!(align & 3)) + copy_from_kernel_nofault_loop(dst, src, size, u32, Efault); + if (!(align & 1)) + copy_from_kernel_nofault_loop(dst, src, size, u16, Efault); + copy_from_kernel_nofault_loop(dst, src, size, u8, Efault); + } + } return 0; Efault: - pagefault_enable(); return -EFAULT; } EXPORT_SYMBOL_GPL(copy_from_kernel_nofault); #define copy_to_kernel_nofault_loop(dst, src, len, type, err_label) \ while (len >= sizeof(type)) { \ - __put_kernel_nofault(dst, src, type, err_label); \ + __put_kernel_nofault_bare(dst, src, type, err_label); \ instrument_write(dst, sizeof(type)); \ dst += sizeof(type); \ src += sizeof(type); \ @@ -65,21 +68,25 @@ long copy_to_kernel_nofault(void *dst, const void *src, size_t size) { unsigned long align = 0; + if (!size) + return 0; + if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)) align = (unsigned long)dst | (unsigned long)src; - pagefault_disable(); - if (!(align & 7)) - copy_to_kernel_nofault_loop(dst, src, size, u64, Efault); - if (!(align & 3)) - copy_to_kernel_nofault_loop(dst, src, size, u32, Efault); - if (!(align & 1)) - copy_to_kernel_nofault_loop(dst, src, size, u16, Efault); - copy_to_kernel_nofault_loop(dst, src, size, u8, Efault); - pagefault_enable(); + scoped_guard(pagefault) { + scoped_guard(__kernel_nofault_bare) { + if (!(align & 7)) + copy_to_kernel_nofault_loop(dst, src, size, u64, Efault); + if (!(align & 3)) + copy_to_kernel_nofault_loop(dst, src, size, u32, Efault); + if (!(align & 1)) + copy_to_kernel_nofault_loop(dst, src, size, u16, Efault); + copy_to_kernel_nofault_loop(dst, src, size, u8, Efault); + } + } return 0; Efault: - pagefault_enable(); return -EFAULT; } @@ -92,18 +99,19 @@ long strncpy_from_kernel_nofault(char *dst, const void *unsafe_addr, long count) if (!copy_from_kernel_nofault_allowed(unsafe_addr, count)) return -ERANGE; - pagefault_disable(); - do { - __get_kernel_nofault(dst, src, u8, Efault); - dst++; - src++; - } while (dst[-1] && src - unsafe_addr < count); - pagefault_enable(); + scoped_guard(pagefault) { + scoped_guard(__kernel_nofault_bare) { + do { + __get_kernel_nofault_bare(dst, src, u8, Efault); + dst++; + src++; + } while (dst[-1] && src - unsafe_addr < count); + } + } dst[-1] = '\0'; return src - unsafe_addr; Efault: - pagefault_enable(); dst[0] = '\0'; return -EFAULT; }