diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index 004b06785521c..be3ec883c08f6 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -1488,8 +1488,15 @@ int bpf_jmp_offset(struct bpf_insn *insn); struct bpf_iarray *bpf_insn_successors(struct bpf_verifier_env *env, u32 idx); void bpf_fmt_stack_mask(char *buf, ssize_t buf_sz, u64 stack_mask); bool bpf_subprog_is_global(const struct bpf_verifier_env *env, int subprog); -bool btf_type_is_scalar_struct(struct bpf_verifier_env *env, const struct btf *btf, - const struct btf_type *t, int rec); + +/* Kinds of member a by-value struct or union may be composed of. */ +enum btf_member_kind { + BTF_MEMBER_SCALAR = BIT(0), /* an int or an enum, or an array of them */ + BTF_MEMBER_ARENA_PTR = BIT(1), /* a pointer carrying the "arena" type tag */ +}; + +bool btf_struct_is_composed_of(struct bpf_verifier_env *env, const struct btf *btf, + const struct btf_type *t, u32 member_kinds); int bpf_find_subprog(struct bpf_verifier_env *env, int off); bool bpf_is_throw_kfunc(struct bpf_insn *insn); diff --git a/include/linux/btf.h b/include/linux/btf.h index 89d5a5c4f1171..ddd0f4f32d24b 100644 --- a/include/linux/btf.h +++ b/include/linux/btf.h @@ -235,6 +235,7 @@ struct btf_record *btf_parse_fields(const struct btf *btf, const struct btf_type u32 field_mask, u32 value_size); int btf_check_and_fixup_fields(const struct btf *btf, struct btf_record *rec); bool btf_type_is_void(const struct btf_type *t); +bool btf_type_is_arena_ptr(const struct btf *btf, const struct btf_type *t); s32 btf_find_by_name_kind(const struct btf *btf, const char *name, u8 kind); s32 bpf_find_btf_id(const char *name, u32 kind, struct btf **btf_p); struct btf *btf_get_module_btf(const struct module *module); diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 91b8ce77f699a..70481fadacc0f 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -3523,6 +3523,22 @@ static int btf_type_tag_walk(const struct btf *btf, return 0; } +bool btf_type_is_arena_ptr(const struct btf *btf, const struct btf_type *t) +{ + if (!btf_type_is_ptr(t)) + return false; + + for (t = btf_type_by_id(btf, t->type); btf_type_is_modifier(t); + t = btf_type_by_id(btf, t->type)) { + if (!btf_type_is_type_tag(t) || btf_type_kflag(t)) + continue; + if (!strcmp(__btf_name_by_offset(btf, t->name_off), "arena")) + return true; + } + + return false; +} + static int btf_find_kptr(const struct btf *btf, const struct btf_type *t, u32 off, int sz, struct btf_field_info *info, u32 field_mask) { @@ -7927,51 +7943,19 @@ static int btf_scan_decl_tags(struct bpf_verifier_env *env, return 0; } -static int btf_scan_type_tags(struct bpf_verifier_env *env, - const struct btf *btf, u32 type_id, - u32 *tags) +static void btf_scan_type_tags(const struct btf *btf, u32 type_id, u32 *tags) { - static const struct btf_type_tag_match func_type_tags[] = { - { "arena", ARG_TAG_ARENA }, - }; - struct btf_type_tag_walk_ctx ctx; - const struct btf_type *t; - int err; - /* Find the first pointer type in the chain. */ - t = btf_type_skip_modifiers(btf, type_id, NULL); + const struct btf_type *t = btf_type_skip_modifiers(btf, type_id, NULL); - /* - * We currently reject type tags on non-pointer types, - * which neither LLVM nor GCC support anyway. - */ - if (!t || !btf_type_is_ptr(t)) - return 0; - - ctx.t = t; - err = btf_type_tag_walk(btf, &ctx, func_type_tags, - ARRAY_SIZE(func_type_tags)); - if (err) { - bpf_log(&env->log, - "function signature member has multiple type tags\n"); - return err; - } - *tags |= ctx.res; - - return 0; + if (btf_type_is_arena_ptr(btf, t)) + *tags |= ARG_TAG_ARENA; } /* Check whether the type is a valid return type. */ static int btf_validate_return_type(struct bpf_verifier_env *env, struct btf *btf, const struct btf_type *t, int subprog, bool is_global) { - u32 tags = 0; - int err; - - err = btf_scan_type_tags(env, btf, t->type, &tags); - if (err) - return err; - t = btf_type_skip_modifiers(btf, t->type, NULL); /* @@ -7979,7 +7963,7 @@ static int btf_validate_return_type(struct bpf_verifier_env *env, struct btf *bt * General arena variables are not allowed, since it makes no sense to return by value * a variable that's on the heap in the first place. */ - if (subprog && (tags & ARG_TAG_ARENA) && btf_type_is_ptr(t)) + if (subprog && btf_type_is_arena_ptr(btf, t)) return 0; /* We always accept void or scalars. */ @@ -7988,14 +7972,18 @@ static int btf_validate_return_type(struct bpf_verifier_env *env, struct btf *bt if (btf_type_is_struct(t) && t->size <= 16) { /* - * A global function's caller models the return as an opaque - * scalar pair, so it may only return scalars by value. A local - * function is verified inline, so a pointer field stays tracked - * and needs no such restriction. + * A global function may return a struct with scalar(s) or arena + * pointer(s) as its members. A local function is verified inline, + * so its caller receives the real register state and any member + * is fine. */ bool local_func = subprog && !is_global; + u32 member_kinds = BTF_MEMBER_SCALAR; + + if (subprog) + member_kinds |= BTF_MEMBER_ARENA_PTR; - if (local_func || btf_type_is_scalar_struct(env, btf, t, 0)) + if (local_func || btf_struct_is_composed_of(env, btf, t, member_kinds)) return 0; } @@ -8091,7 +8079,8 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) if (is_global) { bpf_log(log, "Global function %s() has unsupported return type. " - "Only void, scalar, or a scalar-only struct/union up to 16 bytes is supported.\n", + "Only void, a scalar, an arena pointer, or a struct/union of " + "those up to 16 bytes is supported.\n", tname); } return err; @@ -8106,9 +8095,7 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) if (err) return err; - err = btf_scan_type_tags(env, btf, args[i].type, &tags); - if (err) - return err; + btf_scan_type_tags(btf, args[i].type, &tags); t = btf_type_by_id(btf, args[i].type); while (btf_type_is_modifier(t)) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index e036ae20bf6b9..0c2181d587489 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -10403,10 +10403,14 @@ static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx) * return to the caller whatever the callee had in the * return register(s) */ - bpf_diag_mod_begin(env, &caller->regs[BPF_REG_0], r0, BPF_DIAG_MOD_WRITE); - for (i = 0; i < nregs; i++) - caller->regs[ret_regs[i]] = callee->regs[ret_regs[i]]; - bpf_diag_mod_end(env); + for (i = 0; i < nregs; i++) { + u32 regno = ret_regs[i]; + + bpf_diag_mod_begin(env, &caller->regs[regno], &callee->regs[regno], + BPF_DIAG_MOD_WRITE); + caller->regs[regno] = callee->regs[regno]; + bpf_diag_mod_end(env); + } } /* for callbacks like bpf_loop or bpf_for_each_map_elem go back to callsite, @@ -11619,10 +11623,32 @@ static bool is_kfunc_arg_implicit(const struct bpf_call_arg_meta *meta, u32 arg_ return argn <= arg_idx; } -/* Returns true if struct is composed of scalars, 4 levels of nesting allowed */ -bool btf_type_is_scalar_struct(struct bpf_verifier_env *env, - const struct btf *btf, - const struct btf_type *t, int rec) +#define BTF_MEMBER_MAX_DEPTH 4 +#define BTF_MEMBER_PATH_LEN 64 + +struct btf_member_path { + const struct btf_member *member[BTF_MEMBER_MAX_DEPTH]; + int depth; + bool too_deep; +}; + +static bool btf_member_kind_allowed(const struct btf *btf, const struct btf_type *t, + u32 member_kinds) +{ + if ((member_kinds & BTF_MEMBER_SCALAR) && btf_type_is_scalar(t)) + return true; + if ((member_kinds & BTF_MEMBER_ARENA_PTR) && btf_type_is_arena_ptr(btf, t)) + return true; + return false; +} + +/* + * Returns true if every member of struct @t is of a kind listed in + * @member_kinds, BTF_MEMBER_MAX_DEPTH levels of nesting allowed. + */ +static bool btf_struct_member_walk(struct bpf_verifier_env *env, const struct btf *btf, + const struct btf_type *t, u32 member_kinds, int rec, + struct btf_member_path *path) { const struct btf_type *member_type; const struct btf_member *member; @@ -11636,27 +11662,61 @@ bool btf_type_is_scalar_struct(struct bpf_verifier_env *env, member_type = btf_type_skip_modifiers(btf, member->type, NULL); if (btf_type_is_struct(member_type)) { - if (rec >= 3) { + if (rec >= BTF_MEMBER_MAX_DEPTH - 1) { verbose(env, "max struct nesting depth exceeded\n"); + if (path) + path->too_deep = true; return false; } - if (!btf_type_is_scalar_struct(env, btf, member_type, rec + 1)) - return false; + if (!btf_struct_member_walk(env, btf, member_type, member_kinds, + rec + 1, path)) + goto bad_path; continue; } if (btf_type_is_array(member_type)) { array = btf_array(member_type); if (!array->nelems) - return false; + goto bad_member; member_type = btf_type_skip_modifiers(btf, array->type, NULL); - if (!btf_type_is_scalar(member_type)) - return false; - continue; } - if (!btf_type_is_scalar(member_type)) - return false; + if (!btf_member_kind_allowed(btf, member_type, member_kinds)) + goto bad_member; } return true; + +bad_member: + if (path) + path->depth = rec + 1; +bad_path: + if (path && path->depth) + path->member[rec] = member; + return false; +} + +bool btf_struct_is_composed_of(struct bpf_verifier_env *env, + const struct btf *btf, + const struct btf_type *t, u32 member_kinds) +{ + return btf_struct_member_walk(env, btf, t, member_kinds, 0, NULL); +} + +static bool btf_type_is_scalar_struct(struct bpf_verifier_env *env, + const struct btf *btf, + const struct btf_type *t) +{ + return btf_struct_is_composed_of(env, btf, t, BTF_MEMBER_SCALAR); +} + +static void btf_member_path_str(const struct btf *btf, const struct btf_member_path *path, + char *buf, size_t buf_sz) +{ + size_t len = 0; + int i; + + buf[0] = '\0'; + for (i = 0; i < path->depth; i++) + len += scnprintf(buf + len, buf_sz - len, "%s%s", i ? "." : "", + btf_name_by_offset(btf, path->member[i]->name_off)); } enum kfunc_ptr_arg_type { @@ -12039,7 +12099,7 @@ get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, (is_kfunc_arg_mem_size(meta->btf, &args[arg + 1]) || is_kfunc_arg_const_mem_size(meta->btf, &args[arg + 1]))) { if (!btf_type_is_void(ref_t) && !btf_type_is_scalar(ref_t) && - !btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) { + !btf_type_is_scalar_struct(env, meta->btf, ref_t)) { verbose(env, "%s pointer type %s %s must point to void, scalar, or struct with scalar\n", reg_arg_name(env, argno), btf_type_str(ref_t), ref_tname); return -EINVAL; @@ -12055,7 +12115,7 @@ get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, * scalars. The access size is derived from the pointed-to BTF type. */ if (!btf_type_is_scalar(ref_t) && - !btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) { + !btf_type_is_scalar_struct(env, meta->btf, ref_t)) { verbose(env, "%s pointer type %s %s must point to scalar, or struct with scalar\n", reg_arg_name(env, argno), btf_type_str(ref_t), ref_tname); return -EINVAL; @@ -13111,7 +13171,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me break; } - if (!btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) { + if (!btf_type_is_scalar_struct(env, meta->btf, ref_t)) { enum bpf_reg_type reg2btf_type = lookup_reg2btf_ids(ref_id); const char *expected_type; @@ -13653,7 +13713,7 @@ static int check_special_kfunc(struct bpf_verifier_env *env, struct bpf_call_arg struct_meta = btf_find_struct_meta(ret_btf, ret_btf_id); if (is_bpf_percpu_obj_new_kfunc(meta->func_id)) { - if (!btf_type_is_scalar_struct(env, ret_btf, ret_t, 0)) { + if (!btf_type_is_scalar_struct(env, ret_btf, ret_t)) { verbose(env, "bpf_percpu_obj_new type ID argument must be of a struct of scalars\n"); return -EINVAL; } @@ -14026,17 +14086,47 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, meta.func_id == special_kfunc_list[KF_bpf_res_spin_lock_irqsave])) __mark_reg_const_zero(env, ®s[BPF_REG_0]); } else if (btf_type_is_struct(t)) { + struct btf_member_path path = {}; + const char *member_note = ""; + /* - * The returned struct comes back as raw register bits modeled - * as an unknown scalar, so it must contain only scalars: - * otherwise a pointer field would be laundered into a scalar - * and escape provenance and reference tracking. + * The returned struct may only contain scalars and arena pointers + * as its members. Otherwise, any other pointer would be laundered + * into a scalar and escape provenance and reference tracking. */ - if (!btf_type_is_scalar_struct(env, desc_btf, t, 0)) { - verbose(env, - "kernel function %s returns %s %s that is not composed of scalars\n", + if (!btf_struct_member_walk(env, desc_btf, t, + BTF_MEMBER_SCALAR | BTF_MEMBER_ARENA_PTR, 0, &path)) { + if (path.too_deep) { + member_note = bpf_diag_fmt( + env, " It nests structs more than %d levels deep.", + BTF_MEMBER_MAX_DEPTH); + } else if (path.depth) { + const struct btf_member *bad = path.member[path.depth - 1]; + char bad_name[BTF_MEMBER_PATH_LEN]; + const struct btf_type *bad_type; + + verbose(env, + "kernel function %s returns %s %s that is not composed of scalars or arena pointers\n", + func_name, btf_type_str(t), + btf_name_by_offset(desc_btf, t->name_off)); + btf_member_path_str(desc_btf, &path, bad_name, sizeof(bad_name)); + bad_type = btf_type_skip_modifiers(desc_btf, bad->type, NULL); + verbose(env, "member '%s' has type %s\n", bad_name, + btf_type_str(bad_type)); + member_note = bpf_diag_fmt( + env, + " Its member '%s' is %s, not a scalar or an arena pointer.", + bad_name, btf_type_str(bad_type)); + } + bpf_diag_program_structure( + env, insn_idx, "unsupported kernel function return type", + "Call a kernel function that returns only scalars or arena pointers by value.", + "%s() returns %s %s by value.%s " + "Only kfuncs returning scalar values or arena pointers, or " + "structures composed of scalar values and arena pointers are " + "supported.", func_name, btf_type_str(t), - btf_name_by_offset(desc_btf, t->name_off)); + btf_name_by_offset(desc_btf, t->name_off), member_note); return -EINVAL; } mark_kfunc_ret_regs(env, regs, t->size); diff --git a/tools/testing/selftests/bpf/prog_tests/aggregate_ret.c b/tools/testing/selftests/bpf/prog_tests/aggregate_ret.c index e0b94ed10f941..07d9d6e1d6b88 100644 --- a/tools/testing/selftests/bpf/prog_tests/aggregate_ret.c +++ b/tools/testing/selftests/bpf/prog_tests/aggregate_ret.c @@ -1,11 +1,53 @@ // SPDX-License-Identifier: GPL-2.0 /* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ #include +#include #include "aggregate_ret_func.skel.h" #include "aggregate_ret_kfunc.skel.h" +#include "aggregate_ret_kfunc_arena.skel.h" + +static bool testmod_has_arena_tagged_member(void) +{ + struct btf *vmlinux_btf, *module_btf = NULL; + const struct btf_type *t; + bool tagged = false; + __s32 id; + + vmlinux_btf = btf__load_vmlinux_btf(); + if (!vmlinux_btf) + return false; + + module_btf = btf__load_module_btf("bpf_testmod", vmlinux_btf); + if (!module_btf) + goto out; + + /* prog_test_ret_arena::a is 'void __arena_tag *': PTR -> TYPE_TAG -> void */ + id = btf__find_by_name_kind(module_btf, "prog_test_ret_arena", BTF_KIND_STRUCT); + if (id <= 0) + goto out; + + t = btf__type_by_id(module_btf, btf_members(btf__type_by_id(module_btf, id))[0].type); + if (!t || !btf_is_ptr(t)) + goto out; + + t = btf__type_by_id(module_btf, t->type); + tagged = t && btf_is_type_tag(t) && + !strcmp(btf__name_by_offset(module_btf, t->name_off), "arena"); + +out: + btf__free(module_btf); + btf__free(vmlinux_btf); + + return tagged; +} void test_aggregate_ret(void) { RUN_TESTS(aggregate_ret_func); RUN_TESTS(aggregate_ret_kfunc); + + if (testmod_has_arena_tagged_member()) + RUN_TESTS(aggregate_ret_kfunc_arena); + else + test__skip(); } diff --git a/tools/testing/selftests/bpf/progs/aggregate_ret_func.c b/tools/testing/selftests/bpf/progs/aggregate_ret_func.c index 6f66fc822cede..237adb8e5ee1e 100644 --- a/tools/testing/selftests/bpf/progs/aggregate_ret_func.c +++ b/tools/testing/selftests/bpf/progs/aggregate_ret_func.c @@ -2,6 +2,7 @@ /* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ #include #include +#include #include "bpf_misc.h" typedef unsigned __int128 u128; @@ -234,4 +235,121 @@ __naked int aggregate_ret_global_union_ptr_fail(void) #endif +/* + * gcc returns a by-value struct through a hidden pointer, and emits the + * 'r0 = r1' returning it after the __naked body's exit, leaving the + * subprogram falling through. Build these with clang only. + */ +#if defined(__clang__) + +struct arena_pair { + void __arena *lo; + void __arena *hi; +}; + +struct arena_and_scalar { + void __arena *p; + __u64 x; +}; + +struct arena_array { + void __arena *p[2]; +}; + +struct arena_single { + void __arena *p; +}; + +__naked struct arena_pair global_ret_arena_pair(void) +{ + asm volatile ( + "r0 = 0;" + "r2 = 0;" + "exit;" + ); +} + +SEC("tc") +__load_if_JITed() +__success __retval(0) +__naked int aggregate_ret_global_arena_pair(void) +{ + asm volatile ( + "call %[global_ret_arena_pair];" + "r0 = 0;" + "exit;" + : + : __imm(global_ret_arena_pair) + : __clobber_all); +} + +__naked struct arena_and_scalar global_ret_arena_and_scalar(void) +{ + asm volatile ( + "r0 = 0;" + "r2 = 0;" + "exit;" + ); +} + +SEC("tc") +__load_if_JITed() +__success __retval(0) +__naked int aggregate_ret_global_arena_and_scalar(void) +{ + asm volatile ( + "call %[global_ret_arena_and_scalar];" + "r0 = 0;" + "exit;" + : + : __imm(global_ret_arena_and_scalar) + : __clobber_all); +} + +__naked struct arena_array global_ret_arena_array(void) +{ + asm volatile ( + "r0 = 0;" + "r2 = 0;" + "exit;" + ); +} + +SEC("tc") +__load_if_JITed() +__success __retval(0) +__naked int aggregate_ret_global_arena_array(void) +{ + asm volatile ( + "call %[global_ret_arena_array];" + "r0 = 0;" + "exit;" + : + : __imm(global_ret_arena_array) + : __clobber_all); +} + +__naked struct arena_single global_ret_arena_single(void) +{ + asm volatile ( + "r0 = 0;" + "exit;" + ); +} + +SEC("tc") +__success __retval(0) +__naked int aggregate_ret_global_arena_single(void) +{ + asm volatile ( + "call %[global_ret_arena_single];" + "r0 = 0;" + "exit;" + : + : __imm(global_ret_arena_single) + : __clobber_all); +} + +#endif + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/aggregate_ret_kfunc.c b/tools/testing/selftests/bpf/progs/aggregate_ret_kfunc.c index d6b422ae9784b..e9c82df8efb26 100644 --- a/tools/testing/selftests/bpf/progs/aggregate_ret_kfunc.c +++ b/tools/testing/selftests/bpf/progs/aggregate_ret_kfunc.c @@ -18,6 +18,8 @@ void __kfunc_btf_root(void) : "r"(&bpf_kfunc_call_test_i128), "r"(&bpf_kfunc_call_test_ret_fastcall), "r"(&bpf_kfunc_call_test_ret_ptr), + "r"(&bpf_kfunc_call_test_ret_nested), + "r"(&bpf_kfunc_call_test_ret_deep), "r"(&bpf_kfunc_call_test_ret_ii), "r"(&bpf_kfunc_call_test_ret_big)); } @@ -71,7 +73,8 @@ __naked int aggregate_ret_kfunc_fastcall_fail(void) SEC("tc") __arch_x86_64 __arch_arm64 -__failure __msg("is not composed of scalars") +__failure __msg("is not composed of scalars or arena pointers") +__msg("member 'p' has type PTR") __naked int aggregate_ret_kfunc_ptr_fail(void) { asm volatile ( @@ -84,6 +87,37 @@ __naked int aggregate_ret_kfunc_ptr_fail(void) : __clobber_all); } +SEC("tc") +__arch_x86_64 __arch_arm64 +__failure __msg("is not composed of scalars or arena pointers") +__msg("member 'in.p' has type PTR") +__naked int aggregate_ret_kfunc_nested_ptr_fail(void) +{ + asm volatile ( + "r1 = 0;" + "call %[bpf_kfunc_call_test_ret_nested];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_kfunc_call_test_ret_nested) + : __clobber_all); +} + +SEC("tc") +__arch_x86_64 __arch_arm64 +__failure __msg("max struct nesting depth exceeded") +__naked int aggregate_ret_kfunc_too_deep_fail(void) +{ + asm volatile ( + "r1 = 0;" + "call %[bpf_kfunc_call_test_ret_deep];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_kfunc_call_test_ret_deep) + : __clobber_all); +} + SEC("tc") __arch_x86_64 __arch_arm64 __failure __msg("R2 !read_ok") diff --git a/tools/testing/selftests/bpf/progs/aggregate_ret_kfunc_arena.c b/tools/testing/selftests/bpf/progs/aggregate_ret_kfunc_arena.c new file mode 100644 index 0000000000000..f68deae6c900e --- /dev/null +++ b/tools/testing/selftests/bpf/progs/aggregate_ret_kfunc_arena.c @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include +#include +#include "bpf_misc.h" +#include "../test_kmods/bpf_testmod_kfunc.h" + +void __kfunc_btf_root(void) +{ + asm volatile ("" + : + : "r"(&bpf_kfunc_call_test_ret_arena), + "r"(&bpf_kfunc_call_test_ret_arena_mixed)); +} + +SEC("tc") +__arch_x86_64 __arch_arm64 +__load_if_JITed() +__success __retval(0) +__naked int aggregate_ret_kfunc_arena(void) +{ + asm volatile ( + "call %[bpf_kfunc_call_test_ret_arena];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_kfunc_call_test_ret_arena) + : __clobber_all); +} + +SEC("tc") +__arch_x86_64 __arch_arm64 +__load_if_JITed() +__success __retval(0) +__naked int aggregate_ret_kfunc_arena_mixed(void) +{ + asm volatile ( + "r1 = 0;" + "call %[bpf_kfunc_call_test_ret_arena_mixed];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_kfunc_call_test_ret_arena_mixed) + : __clobber_all); +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/exceptions_fail.c b/tools/testing/selftests/bpf/progs/exceptions_fail.c index 9708efb93683b..35794329640b7 100644 --- a/tools/testing/selftests/bpf/progs/exceptions_fail.c +++ b/tools/testing/selftests/bpf/progs/exceptions_fail.c @@ -60,7 +60,7 @@ __noinline int exception_cb_ok_arg_small(int a) SEC("?tc") __exception_cb(exception_cb_bad_ret_type1) -__failure __msg("Only void, scalar, or a scalar-only struct/union up to 16 bytes is supported.") +__failure __msg("Only void, a scalar, an arena pointer, or a struct/union of those") int reject_exception_cb_type_1(struct __sk_buff *ctx) { bpf_throw(0); diff --git a/tools/testing/selftests/bpf/progs/verifier_arena.c b/tools/testing/selftests/bpf/progs/verifier_arena.c index 815f342eb4b05..d37424d1161a5 100644 --- a/tools/testing/selftests/bpf/progs/verifier_arena.c +++ b/tools/testing/selftests/bpf/progs/verifier_arena.c @@ -734,4 +734,41 @@ int check_arena_arg_ret(void *ctx) return 0; } +#if defined(__clang_major__) && __clang_major__ >= 23 + +struct arena_page_pair { + u32 __arena *first; + u32 __arena *second; +}; + +__weak struct arena_page_pair split_arena_page(u32 __arena *page) +{ + struct arena_page_pair pair; + + pair.first = page; + pair.second = page + 1; + + return pair; +} + +SEC("syscall") +__load_if_JITed() +__success __retval(0) +int check_arena_struct_ret(void *ctx) +{ + u32 __arena *page = bpf_arena_alloc_pages(&arena, NULL, 1, NUMA_NO_NODE, 0); + struct arena_page_pair pair; + + if (!page) + return 1; + + pair = split_arena_page(page); + if (!pair.first || !pair.second) + return 2; + + return 0; +} + +#endif + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c index 850cf4f830c48..81fb93ea466e5 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c @@ -981,6 +981,34 @@ __bpf_kfunc struct prog_test_ret_ptr bpf_kfunc_call_test_ret_ptr(u64 tag) return r; } +__bpf_kfunc struct prog_test_ret_arena bpf_kfunc_call_test_ret_arena(void) +{ + struct prog_test_ret_arena r = { .a = NULL, .b = NULL }; + + return r; +} + +__bpf_kfunc struct prog_test_ret_arena_mixed bpf_kfunc_call_test_ret_arena_mixed(u64 tag) +{ + struct prog_test_ret_arena_mixed r = { .p = NULL, .tag = tag }; + + return r; +} + +__bpf_kfunc struct prog_test_ret_nested bpf_kfunc_call_test_ret_nested(u64 tag) +{ + struct prog_test_ret_nested r = { .in = { .p = NULL }, .tag = tag }; + + return r; +} + +__bpf_kfunc struct prog_test_ret_deep bpf_kfunc_call_test_ret_deep(u64 v) +{ + struct prog_test_ret_deep r = { .l1 = { .l2 = { .l3 = { .l4 = { .v = v } } } } }; + + return r; +} + __bpf_kfunc struct prog_test_ret_ii bpf_kfunc_call_test_ret_ii(int a, int b) { struct prog_test_ret_ii r = { .a = a, .b = b }; @@ -1539,6 +1567,10 @@ BTF_ID_FLAGS(func, bpf_kfunc_call_test_i128) BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_pair) BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_fastcall, KF_FASTCALL) BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_ptr) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_arena) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_arena_mixed) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_nested) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_deep) BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_ii) #endif BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_big) diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h b/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h index 65e693ada7364..aebf88102dc6b 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h @@ -26,6 +26,12 @@ struct prog_test_ref_kfunc { }; #endif +#if __has_attribute(btf_type_tag) +#define __arena_tag __attribute__((btf_type_tag("arena"))) +#else +#define __arena_tag +#endif + struct bpf_iter_testmod_seq; struct prog_test_pass1 { @@ -70,6 +76,35 @@ struct prog_test_ret_ptr { /* 16 bytes: contains a pointer */ __u64 tag; }; +struct prog_test_ret_arena { /* 16 bytes: two arena pointers */ + void __arena_tag *a; + void __arena_tag *b; +}; + +struct prog_test_ret_arena_mixed { /* 16 bytes: an arena pointer and a scalar */ + void __arena_tag *p; + __u64 tag; +}; + +struct prog_test_ret_nested { /* 16 bytes: the pointer hides one level down */ + struct { + void *p; + } in; + __u64 tag; +}; + +struct prog_test_ret_deep { /* 8 bytes, but nested past the 4-level walk limit */ + struct { + struct { + struct { + struct { + __u64 v; + } l4; + } l3; + } l2; + } l1; +}; + struct prog_test_ret_big { /* 24 bytes: too large for R0:R2 */ __u64 a; __u64 b; @@ -159,6 +194,10 @@ struct prog_test_ret_pair bpf_kfunc_call_test_ret_pair(__u64 a, __u64 b) __ksym; struct prog_test_ret_pair bpf_kfunc_call_test_ret_fastcall(__u64 a, __u64 b) __ksym; struct prog_test_ret_ii bpf_kfunc_call_test_ret_ii(int a, int b) __ksym; struct prog_test_ret_ptr bpf_kfunc_call_test_ret_ptr(__u64 tag) __ksym; +struct prog_test_ret_nested bpf_kfunc_call_test_ret_nested(__u64 tag) __ksym; +struct prog_test_ret_arena bpf_kfunc_call_test_ret_arena(void) __ksym; +struct prog_test_ret_arena_mixed bpf_kfunc_call_test_ret_arena_mixed(__u64 tag) __ksym; +struct prog_test_ret_deep bpf_kfunc_call_test_ret_deep(__u64 v) __ksym; struct prog_test_ret_big bpf_kfunc_call_test_ret_big(void) __ksym; __u64 bpf_kfunc_call_stack_arg(__u64 a, __u64 b, __u64 c, __u64 d, __u64 e, __u64 f, __u64 g, __u64 h,