diff --git a/arch/x86/net/bpf_jit_comp32.c b/arch/x86/net/bpf_jit_comp32.c index 852baf2e4db4..017cc9e7e4e2 100644 --- a/arch/x86/net/bpf_jit_comp32.c +++ b/arch/x86/net/bpf_jit_comp32.c @@ -1620,13 +1620,25 @@ static int emit_kfunc_call(const struct bpf_prog *bpf_prog, u8 *end_addr, /* add esp,"bytes_in_stack" */ end_addr -= 3; - /* mov dword ptr [ebp+off],edx */ - if (fm->ret_size > sizeof(u32)) - end_addr -= 3; - - /* mov dword ptr [ebp+off],eax */ - if (fm->ret_size) + switch (fm->ret_size) { + case 0: + break; + case sizeof(u8): + case sizeof(u16): + /* mov[s|z]x eax,[a]x */ end_addr -= 3; + fallthrough; + case sizeof(u32): + /* cdq or xor edx,edx */ + end_addr -= fm->ret_flags & BTF_FMODEL_SIGNED_ARG ? 1 : 2; + fallthrough; + case sizeof(u64): + /* Store EDX:EAX in R0. */ + end_addr -= 6; + break; + default: + return -EINVAL; + } jmp_offset = (u8 *)__bpf_call_base + insn->imm - end_addr; if (!is_simm32(jmp_offset)) { @@ -1637,15 +1649,38 @@ static int emit_kfunc_call(const struct bpf_prog *bpf_prog, u8 *end_addr, EMIT1_off32(0xE8, jmp_offset); - if (fm->ret_size) + switch (fm->ret_size) { + case sizeof(u8): + /* mov[s|z]x eax,al */ + EMIT3(0x0F, fm->ret_flags & BTF_FMODEL_SIGNED_ARG ? 0xBE : 0xB6, + add_2reg(0xC0, IA32_EAX, IA32_EAX)); + break; + case sizeof(u16): + /* mov[s|z]x eax,ax */ + EMIT3(0x0F, fm->ret_flags & BTF_FMODEL_SIGNED_ARG ? 0xBF : 0xB7, + add_2reg(0xC0, IA32_EAX, IA32_EAX)); + break; + default: + break; + } + + if (fm->ret_size && fm->ret_size <= sizeof(u32)) { + if (fm->ret_flags & BTF_FMODEL_SIGNED_ARG) + /* cdq */ + EMIT1(0x99); + else + /* xor edx,edx */ + EMIT2(0x31, add_2reg(0xC0, IA32_EDX, IA32_EDX)); + } + + if (fm->ret_size) { /* mov dword ptr [ebp+off],eax */ EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EAX), STACK_VAR(bpf2ia32[BPF_REG_0][0])); - - if (fm->ret_size > sizeof(u32)) /* mov dword ptr [ebp+off],edx */ EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EDX), STACK_VAR(bpf2ia32[BPF_REG_0][1])); + } if (bytes_in_stack) /* add esp,"bytes_in_stack" */ diff --git a/tools/testing/selftests/bpf/prog_tests/verifier.c b/tools/testing/selftests/bpf/prog_tests/verifier.c index f7f94ccebce2..730884b0bb7d 100644 --- a/tools/testing/selftests/bpf/prog_tests/verifier.c +++ b/tools/testing/selftests/bpf/prog_tests/verifier.c @@ -129,6 +129,7 @@ #include "verifier_set_retval.skel.h" #include "verifier_lsm.skel.h" #include "verifier_jit_inline.skel.h" +#include "verifier_kfunc_return.skel.h" #include "irq.skel.h" #include "verifier_ctx_ptr_param.skel.h" #include "verifier_zext.skel.h" @@ -293,6 +294,7 @@ void test_verifier_lsm(void) { RUN(verifier_lsm); } void test_irq(void) { RUN(irq); } void test_verifier_mtu(void) { RUN(verifier_mtu); } void test_verifier_jit_inline(void) { RUN(verifier_jit_inline); } +void test_verifier_kfunc_return(void) { RUN(verifier_kfunc_return); } void test_verifier_ctx_ptr_param(void) { RUN(verifier_ctx_ptr_param); } void test_verifier_zext(void) { RUN_TESTS(verifier_zext); } diff --git a/tools/testing/selftests/bpf/progs/bpf_misc.h b/tools/testing/selftests/bpf/progs/bpf_misc.h index eb88d9ce6c34..ed32d8cee190 100644 --- a/tools/testing/selftests/bpf/progs/bpf_misc.h +++ b/tools/testing/selftests/bpf/progs/bpf_misc.h @@ -161,6 +161,7 @@ #define __btf_func_path(path) __test_tag("test_btf_func_path=" path) #define __arch(arch) __test_tag("test_arch=" arch) #define __arch_x86_64 __arch("X86_64") +#define __arch_x86_32 __arch("X86_32") #define __arch_arm64 __arch("ARM64") #define __arch_riscv64 __arch("RISCV64") #define __arch_s390x __arch("s390x") diff --git a/tools/testing/selftests/bpf/progs/verifier_kfunc_return.c b/tools/testing/selftests/bpf/progs/verifier_kfunc_return.c new file mode 100644 index 000000000000..52a94daf0392 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_kfunc_return.c @@ -0,0 +1,84 @@ +// SPDX-License-Identifier: GPL-2.0 + +#define BPF_NO_KFUNC_PROTOTYPES +#include +#include +#include "bpf_misc.h" + +extern int bpf_strcmp(const char *s1__ign, const char *s2__ign) __ksym; +extern bool bpf_dynptr_is_null(const struct bpf_dynptr *p) __ksym; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, __u32); + __type(value, __u64); +} kfunc_return_map SEC(".maps"); + +static const char string_a[] = "a"; +static const char string_b[] = "b"; + +void __kfunc_btf_root(void) +{ + struct bpf_dynptr ptr = {}; + + bpf_strcmp(string_a, string_b); + bpf_dynptr_is_null(&ptr); +} + +SEC("syscall") +__flag(BPF_F_SLEEPABLE) +__success __retval(0) +__arch_x86_32 +__naked void kfunc_bool_return_zero_extended(void) +{ + asm volatile (" \ + r0 = 0; \ + *(u32 *)(r10 - 4) = r0; \ + r2 = r10; \ + r2 += -4; \ + r1 = %[kfunc_return_map] ll; \ + call %[bpf_map_lookup_elem]; \ + if r0 == 0 goto 1f; \ + r1 = r0; \ + r2 = 8; \ + r3 = 0; \ + r4 = r10; \ + r4 += -24; \ + call %[bpf_dynptr_from_mem]; \ + if r0 != 0 goto 1f; \ + r0 = 0x100000000 ll; \ + r1 = r10; \ + r1 += -24; \ + call bpf_dynptr_is_null; \ + r0 >>= 32; \ + exit; \ +1: r0 = 2; \ + exit; \ +" : + : __imm_addr(kfunc_return_map), + __imm(bpf_map_lookup_elem), + __imm(bpf_dynptr_from_mem) + : __clobber_all); +} + +SEC("syscall") +__flag(BPF_F_SLEEPABLE) +__success __retval(-1) +__arch_x86_32 +__naked void kfunc_int_return_sign_extended(void) +{ + asm volatile (" \ + r0 = 0; \ + r1 = %[string_a] ll; \ + r2 = %[string_b] ll; \ + call bpf_strcmp; \ + r0 >>= 32; \ + exit; \ +" : + : __imm_addr(string_a), + __imm_addr(string_b) + : __clobber_all); +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/test_loader.c b/tools/testing/selftests/bpf/test_loader.c index 794a7dfb0579..59079f4ea799 100644 --- a/tools/testing/selftests/bpf/test_loader.c +++ b/tools/testing/selftests/bpf/test_loader.c @@ -375,16 +375,19 @@ static const char **collect_decl_tags(struct btf *btf, int id, int *cnt) enum arch { ARCH_UNKNOWN = 0x1, ARCH_X86_64 = 0x2, - ARCH_ARM64 = 0x4, - ARCH_RISCV64 = 0x8, - ARCH_S390X = 0x10, - ARCH_LOONGARCH = 0x20, + ARCH_X86_32 = 0x4, + ARCH_ARM64 = 0x8, + ARCH_RISCV64 = 0x10, + ARCH_S390X = 0x20, + ARCH_LOONGARCH = 0x40, }; static int get_current_arch(void) { #if defined(__x86_64__) return ARCH_X86_64; +#elif defined(__i386__) + return ARCH_X86_32; #elif defined(__aarch64__) return ARCH_ARM64; #elif defined(__riscv) && __riscv_xlen == 64 @@ -580,6 +583,8 @@ static int parse_test_spec(struct test_loader *tester, } else if ((val = str_has_pfx(s, "test_arch="))) { if (strcmp(val, "X86_64") == 0) { arch = ARCH_X86_64; + } else if (strcmp(val, "X86_32") == 0) { + arch = ARCH_X86_32; } else if (strcmp(val, "ARM64") == 0) { arch = ARCH_ARM64; } else if (strcmp(val, "RISCV64") == 0) {