Add a program to exceptions_cleanup that keeps a record in a hidden slot and passes it to a callee through a function pointer. The caller throws with the record in its slot, and the callee throws before and after its discard, when the slot of the caller is already cleared. Check in verifier_frame_desc the slots and instructions that this takes, and that the caller clears the hidden slot of a lock before a callx to a subprog that unlocks. Add a program to exceptions_fail in which a callx can call one subprog that discards the record and another that leaves it with the caller, which keeps it in a hidden slot. Signed-off-by: Kumar Kartikeya Dwivedi --- .../bpf/prog_tests/exceptions_cleanup.c | 9 ++ .../selftests/bpf/progs/exceptions_cleanup.c | 44 ++++++++++ .../selftests/bpf/progs/exceptions_fail.c | 70 ++++++++++++++++ .../selftests/bpf/progs/verifier_frame_desc.c | 83 +++++++++++++++++++ 4 files changed, 206 insertions(+) diff --git a/tools/testing/selftests/bpf/prog_tests/exceptions_cleanup.c b/tools/testing/selftests/bpf/prog_tests/exceptions_cleanup.c index 5ef5f01a6be1..07d3e8cd7046 100644 --- a/tools/testing/selftests/bpf/prog_tests/exceptions_cleanup.c +++ b/tools/testing/selftests/bpf/prog_tests/exceptions_cleanup.c @@ -253,6 +253,15 @@ static void run_suite(const char *prefix) 1, "", .param = 10); RUN_AS(throw_callee_consumes_from_slot, "throw_callee_consumes_from_slot_return", 0, 1, "", .param = 8); + /* a callx hands over like a call, and its caller empties the slot in front of it */ + RUN_AS(throw_callx_consumes_from_slot, "throw_callx_consumes_from_slot_caller", THROWN, + 1, "0", .param = 3); + RUN_AS(throw_callx_consumes_from_slot, "throw_callx_consumes_from_slot_callee", THROWN, + 1, "0", .param = 1); + RUN_AS(throw_callx_consumes_from_slot, "throw_callx_consumes_from_slot_after", THROWN, + 1, "", .param = 2); + RUN_AS(throw_callx_consumes_from_slot, "throw_callx_consumes_from_slot_return", 0, + 1, "", .param = 0); /* the callee has it nowhere at its throw and keeps it in a slot of its own */ RUN_AS(throw_callee_consumes_slot, "throw_callee_consumes_slot_throw", THROWN, 1, "0", .param = 1); diff --git a/tools/testing/selftests/bpf/progs/exceptions_cleanup.c b/tools/testing/selftests/bpf/progs/exceptions_cleanup.c index d6bb893f9da2..13cb476dffe6 100644 --- a/tools/testing/selftests/bpf/progs/exceptions_cleanup.c +++ b/tools/testing/selftests/bpf/progs/exceptions_cleanup.c @@ -1730,6 +1730,50 @@ __naked int throw_callee_consumes_from_slot(void) : __clobber_all); } +/* + * The same through the address of the callee, with the record in no register + * at the throw of the caller: the insns in front of the callx empty the slot + * of the caller, so a throw of the callee in front of its discard releases + * the record once. + */ +SEC("tc") +__naked int throw_callx_consumes_from_slot(void) +{ + asm volatile ( + "r1 = %[ringbuf] ll;" + "r2 = 8;" + "r3 = 0;" + "call %[bpf_ringbuf_reserve];" + "if r0 == 0 goto 2f;" + "r6 = r0;" + "r1 = %[acquired] ll;" + "*(u64 *)(r1 + 0) = r0;" + "r2 = 1;" + "r1 = %[acquired_cnt] ll;" + "*(u32 *)(r1 + 0) = r2;" + "r0 = 0;" + "r8 = %[param] ll;" + "r8 = *(u32 *)(r8 + 0);" + "if r8 != 3 goto 1f;" + "r6 = 0;" + "r1 = 0xeB9F;" + "call %[bpf_throw];" + "1: r1 = r6;" + "r2 = %[consume_in_callee] ll;" + "callx r2;" + "2: r0 = 0;" + "exit;" + : + : __imm_addr(ringbuf), + __imm_addr(acquired), + __imm_addr(acquired_cnt), + __imm_addr(param), + __imm_addr(consume_in_callee), + __imm(bpf_ringbuf_reserve), + __imm(bpf_throw) + : __clobber_all); +} + /* * Discards the record its caller hands it. When it throws instead the record * is in no register: the cookie has taken its place in R1. diff --git a/tools/testing/selftests/bpf/progs/exceptions_fail.c b/tools/testing/selftests/bpf/progs/exceptions_fail.c index 9b47345d5fd0..630ae7b9e857 100644 --- a/tools/testing/selftests/bpf/progs/exceptions_fail.c +++ b/tools/testing/selftests/bpf/progs/exceptions_fail.c @@ -561,6 +561,76 @@ __naked int reject_callee_releases_sometimes(void) : __clobber_all); } +/* Discards the record in R1. */ +__used __naked static __noinline int discard_rec(void) +{ + asm volatile ( + "r2 = 0;" + "call %[bpf_ringbuf_discard];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_ringbuf_discard) + : __clobber_all); +} + +/* Leaves the record in R1 with its caller. */ +__used __naked static __noinline int keep_rec(void) +{ + asm volatile ( + "r0 = 0;" + "exit;" + ::: __clobber_all); +} + +/* + * The record is in no register at the throw, so it is in a slot. The insns in + * front of the callx empty the slot for the subprog that is handed the + * record, and would for the one that leaves it with the caller as well, which + * discards it after the call. + */ +SEC("?tc") +__arch_x86_64 __arch_arm64 +__failure __msg("to a subprog that releases it and to one that does not") +__naked int reject_callx_releases_sometimes(void) +{ + asm volatile ( + "r7 = *(u32 *)(r1 + %[len]);" + "r1 = %[fail_ringbuf] ll;" + "r2 = 8;" + "r3 = 0;" + "call %[bpf_ringbuf_reserve];" + "if r0 == 0 goto 3f;" + "r6 = r0;" + "r0 = 0;" + "if r7 != 3 goto 1f;" + "r6 = 0;" + "r1 = 0;" + "call %[bpf_throw];" + "1: r2 = %[discard_rec] ll;" + "r8 = 0;" + "if r7 != 1 goto 2f;" + "r2 = %[keep_rec] ll;" + "r8 = 1;" + "2: r1 = r6;" + "callx r2;" + "if r8 == 0 goto 3f;" + "r1 = r6;" + "r2 = 0;" + "call %[bpf_ringbuf_discard];" + "3: r0 = 0;" + "exit;" + : + : __imm_addr(fail_ringbuf), + __imm_addr(discard_rec), + __imm_addr(keep_rec), + __imm(bpf_ringbuf_reserve), + __imm(bpf_ringbuf_discard), + __imm(bpf_throw), + [len]"i"(offsetof(struct __sk_buff, len)) + : __clobber_all); +} + __used static __noinline void leave_rcu(void) { bpf_rcu_read_unlock(); diff --git a/tools/testing/selftests/bpf/progs/verifier_frame_desc.c b/tools/testing/selftests/bpf/progs/verifier_frame_desc.c index e0456a929126..489cdd897c6c 100644 --- a/tools/testing/selftests/bpf/progs/verifier_frame_desc.c +++ b/tools/testing/selftests/bpf/progs/verifier_frame_desc.c @@ -799,6 +799,89 @@ __naked int callee_unlocks_slot(void) : __clobber_all); } +/* + * The same through the address of the callee: the callx finds the slot of the + * caller emptied as the call does, since the only subprog it calls unlocks. + */ +SEC("tc") +__description("frame_desc: the caller empties the slot of a lock in front of a callx") +__arch_x86_64 __arch_arm64 +__success __log_level(5) +__msg("frame descriptors: slots 1 insns 3 sites 0 classes 1") +__naked int callx_unlocks_slot(void) +{ + asm volatile ( + "r6 = *(u32 *)(r1 + %[len]);" + "r1 = 0;" + "*(u32 *)(r10 - 4) = r1;" + "r1 = %[lock_values] ll;" + "r2 = r10;" + "r2 += -4;" + "call %[bpf_map_lookup_elem];" + "if r0 == 0 goto 2f;" + "r7 = r0;" + "r1 = r7;" + "r1 += %[lock_off];" + "call %[bpf_spin_lock];" + "if r6 != 1 goto 1f;" + "r7 = 0;" + "r1 = 0xeB9F;" + "call %[bpf_throw];" + "1: r1 = r7;" + "r2 = %[unlock_value] ll;" + "callx r2;" + "2: r0 = 0;" + "exit;" + : + : __imm_addr(lock_values), + __imm_addr(unlock_value), + __imm(bpf_map_lookup_elem), + __imm(bpf_spin_lock), + __imm(bpf_throw), + [len]"i"(offsetof(struct __sk_buff, len)), + [lock_off]"i"(offsetof(struct lock_value, lock)) + : __clobber_all); +} + +/* + * The caller has the record nowhere at its throw, so it keeps it in a slot: + * filled after the reserve, empty at the entry, and emptied in front of the + * callx that hands the record to the callee, which holds it in R6. + */ +SEC("tc") +__description("frame_desc: a callx hands a record over from the slot of the caller") +__arch_x86_64 __arch_arm64 +__success __log_level(5) +__msg("frame descriptors: slots 1 insns 5 sites 1 classes 1") +__naked int callx_consumes_record(void) +{ + asm volatile ( + "r7 = *(u32 *)(r1 + %[len]);" + "r1 = %[ringbuf] ll;" + "r2 = 8;" + "r3 = 0;" + "call %[bpf_ringbuf_reserve];" + "if r0 == 0 goto 2f;" + "r6 = r0;" + "r0 = 0;" + "if r7 != 3 goto 1f;" + "r6 = 0;" + "r1 = 0xeB9F;" + "call %[bpf_throw];" + "1: r1 = r6;" + "r2 = %[consume_record] ll;" + "callx r2;" + "2: r0 = 0;" + "exit;" + : + : __imm_addr(ringbuf), + __imm_addr(consume_record), + __imm(bpf_ringbuf_reserve), + __imm(bpf_throw), + [len]"i"(offsetof(struct __sk_buff, len)) + : __clobber_all); +} + static __noinline void take_slock(void) { bpf_spin_lock(&slock); -- 2.53.0