Extend the stack argument mechanism to kfunc calls, allowing kfuncs with more than 5 parameters to receive additional arguments via the r11-based stack arg area. For kfuncs, the caller is a BPF program and the callee is a kernel function. The BPF program writes outgoing args at negative r11 offsets, following the same convention as BPF-to-BPF calls: Outgoing: r11 - 8 (arg6), ..., r11 - N*8 (last arg) The following is an example: int foo(int a1, int a2, int a3, int a4, int a5, int a6, int a7) { ... kfunc1(a1, a2, a3, a4, a5, a6, a7, a8); ... kfunc2(a1, a2, a3, a4, a5, a6, a7, a8, a9); ... } Caller (foo), generated by llvm =============================== Incoming (positive offsets): r11+8: [incoming arg 6] r11+16: [incoming arg 7] Outgoing for kfunc1 (negative offsets): r11-8: [outgoing arg 6] r11-16: [outgoing arg 7] r11-24: [outgoing arg 8] Outgoing for kfunc2 (negative offsets): r11-8: [outgoing arg 6] r11-16: [outgoing arg 7] r11-24: [outgoing arg 8] r11-32: [outgoing arg 9] Later JIT will marshal outgoing arguments to the native calling convention for kfunc1() and kfunc2(). There are two places where meta->release_regno needs to keep regno for later releasing the reference. Also, 'cur_aux(env)->arg_prog = regno' is also keeping regno for later fixup. Since stack arguments don't have a valid register number (regno is set to -1), these three cases are rejected for now if the argument is on the stack. Signed-off-by: Yonghong Song --- kernel/bpf/verifier.c | 127 ++++++++++++++++++++++++++++++++++-------- 1 file changed, 103 insertions(+), 24 deletions(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 2a346e4f28e1..13f1fd788092 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -4502,6 +4502,14 @@ static int check_stack_arg_access(struct bpf_verifier_env *env, return 0; } +/* Check whether a stack arg slot has been properly initialized. */ +static bool is_stack_arg_slot_initialized(struct bpf_func_state *state, int spi) +{ + if (spi >= (int)(state->stack_arg_depth / BPF_REG_SIZE)) + return false; + return state->stack_arg_regs[spi].type != NOT_INIT; +} + static int out_arg_idx_from_off(int off) { return -off / BPF_REG_SIZE - 1; @@ -7314,8 +7322,12 @@ static int check_mem_size_reg(struct bpf_verifier_env *env, } err = check_helper_mem_access(env, mem_reg, mem_argno, size_reg->umax_value, access_type, zero_size_allowed, meta); - if (!err && !is_stack_argno(size_argno)) - err = mark_chain_precision(env, size_argno); + if (!err) { + if (is_stack_argno(size_argno)) + size_reg->precise = true; + else + err = mark_chain_precision(env, size_argno); + } return err; } @@ -7358,8 +7370,6 @@ static int check_kfunc_mem_size_reg(struct bpf_verifier_env *env, struct bpf_reg u32 argno = make_argno(mem_argno); int err; - WARN_ON_ONCE(mem_argno > BPF_REG_3); - memset(&meta, 0, sizeof(meta)); if (may_be_null) { @@ -11667,6 +11677,21 @@ bool bpf_is_kfunc_pkt_changing(struct bpf_kfunc_call_arg_meta *meta) return meta->func_id == special_kfunc_list[KF_bpf_xdp_pull_data]; } +static struct bpf_reg_state *get_kfunc_arg_reg(struct bpf_verifier_env *env, int argno) +{ + struct bpf_func_state *caller; + int spi; + + if (argno < MAX_BPF_FUNC_REG_ARGS) + return &cur_regs(env)[argno + 1]; + + caller = cur_func(env); + spi = out_arg_spi(caller, argno - MAX_BPF_FUNC_REG_ARGS); + if (spi >= caller->stack_arg_depth / BPF_REG_SIZE) + return NULL; + return &caller->stack_arg_regs[spi]; +} + static enum kfunc_ptr_arg_type get_kfunc_ptr_arg_type(struct bpf_verifier_env *env, struct bpf_kfunc_call_arg_meta *meta, @@ -11674,8 +11699,6 @@ get_kfunc_ptr_arg_type(struct bpf_verifier_env *env, const char *ref_tname, const struct btf_param *args, int argno, int nargs, struct bpf_reg_state *reg) { - u32 regno = argno + 1; - struct bpf_reg_state *regs = cur_regs(env); bool arg_mem_size = false; if (meta->func_id == special_kfunc_list[KF_bpf_cast_to_kern_ctx] || @@ -11683,10 +11706,14 @@ get_kfunc_ptr_arg_type(struct bpf_verifier_env *env, meta->func_id == special_kfunc_list[KF_bpf_session_cookie]) return KF_ARG_PTR_TO_CTX; - if (argno + 1 < nargs && - (is_kfunc_arg_mem_size(meta->btf, &args[argno + 1], ®s[regno + 1]) || - is_kfunc_arg_const_mem_size(meta->btf, &args[argno + 1], ®s[regno + 1]))) - arg_mem_size = true; + if (argno + 1 < nargs) { + struct bpf_reg_state *next_reg = get_kfunc_arg_reg(env, argno + 1); + + if (next_reg && + (is_kfunc_arg_mem_size(meta->btf, &args[argno + 1], next_reg) || + is_kfunc_arg_const_mem_size(meta->btf, &args[argno + 1], next_reg))) + arg_mem_size = true; + } /* In this function, we verify the kfunc's BTF as per the argument type, * leaving the rest of the verification with respect to the register @@ -12358,9 +12385,9 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ args = (const struct btf_param *)(meta->func_proto + 1); nargs = btf_type_vlen(meta->func_proto); - if (nargs > MAX_BPF_FUNC_REG_ARGS) { + if (nargs > MAX_BPF_FUNC_ARGS) { verbose(env, "Function %s has %d > %d args\n", func_name, nargs, - MAX_BPF_FUNC_REG_ARGS); + MAX_BPF_FUNC_ARGS); return -EINVAL; } if (nargs > MAX_BPF_FUNC_REG_ARGS && !bpf_jit_supports_stack_args()) { @@ -12373,20 +12400,44 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ * verifier sees. */ for (i = 0; i < nargs; i++) { - struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[i + 1]; + struct bpf_reg_state *regs = cur_regs(env), *reg; const struct btf_type *t, *ref_t, *resolve_ret; enum bpf_arg_type arg_type = ARG_DONTCARE; u32 argno = make_argno(i); - u32 regno = i + 1, ref_id, type_size; + struct bpf_reg_state tmp_reg; + int regno = i + 1; + u32 ref_id, type_size; bool is_ret_buf_sz = false; int kf_arg_type; + if (i < MAX_BPF_FUNC_REG_ARGS) { + reg = ®s[i + 1]; + } else { + /* Retrieve the reg state from the outgoing stack arg slot. */ + struct bpf_func_state *caller = cur_func(env); + int spi = out_arg_spi(caller, i - MAX_BPF_FUNC_REG_ARGS); + + if (!is_stack_arg_slot_initialized(caller, spi)) { + verbose(env, "stack %s not properly initialized\n", + reg_arg_name(env, argno)); + return -EINVAL; + } + + tmp_reg = caller->stack_arg_regs[spi]; + reg = &tmp_reg; + regno = -1; + } + if (is_kfunc_arg_prog_aux(btf, &args[i])) { /* Reject repeated use bpf_prog_aux */ if (meta->arg_prog) { verifier_bug(env, "Only 1 prog->aux argument supported per-kfunc"); return -EFAULT; } + if (regno < 0) { + verbose(env, "%s prog->aux cannot be a stack argument\n", reg_arg_name(env, argno)); + return -EINVAL; + } meta->arg_prog = true; cur_aux(env)->arg_prog = regno; continue; @@ -12413,9 +12464,13 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ reg_arg_name(env, argno)); return -EINVAL; } - ret = mark_chain_precision(env, regno); - if (ret < 0) - return ret; + if (regno < 0) { + reg->precise = true; + } else { + ret = mark_chain_precision(env, regno); + if (ret < 0) + return ret; + } meta->arg_constant.found = true; meta->arg_constant.value = reg->var_off.value; } else if (is_kfunc_arg_scalar_with_name(btf, &args[i], "rdonly_buf_size")) { @@ -12438,9 +12493,13 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ } meta->r0_size = reg->var_off.value; - ret = mark_chain_precision(env, regno); - if (ret) - return ret; + if (regno < 0) { + reg->precise = true; + } else { + ret = mark_chain_precision(env, regno); + if (ret) + return ret; + } } continue; } @@ -12466,8 +12525,13 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ return -EFAULT; } meta->ref_obj_id = reg->ref_obj_id; - if (is_kfunc_release(meta)) + if (is_kfunc_release(meta)) { + if (regno < 0) { + verbose(env, "%s release arg cannot be a stack argument\n", reg_arg_name(env, argno)); + return -EINVAL; + } meta->release_regno = regno; + } } ref_t = btf_type_skip_modifiers(btf, t->type, &ref_id); @@ -12626,6 +12690,10 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ dynptr_arg_type |= DYNPTR_TYPE_FILE; } else if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_file_discard]) { dynptr_arg_type |= DYNPTR_TYPE_FILE; + if (regno < 0) { + verbose(env, "%s release arg cannot be a stack argument\n", reg_arg_name(env, argno)); + return -EINVAL; + } meta->release_regno = regno; } else if (meta->func_id == special_kfunc_list[KF_bpf_dynptr_clone] && (dynptr_arg_type & MEM_UNINIT)) { @@ -12780,9 +12848,9 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_kfunc_call_ break; case KF_ARG_PTR_TO_MEM_SIZE: { - struct bpf_reg_state *buff_reg = ®s[regno]; + struct bpf_reg_state *buff_reg = reg; const struct btf_param *buff_arg = &args[i]; - struct bpf_reg_state *size_reg = ®s[regno + 1]; + struct bpf_reg_state *size_reg = get_kfunc_arg_reg(env, i + 1); const struct btf_param *size_arg = &args[i + 1]; if (!bpf_register_is_null(buff_reg) || !is_kfunc_arg_nullable(meta->btf, buff_arg)) { @@ -13686,7 +13754,7 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, nargs = btf_type_vlen(meta.func_proto); args = (const struct btf_param *)(meta.func_proto + 1); - for (i = 0; i < nargs; i++) { + for (i = 0; i < min_t(int, nargs, MAX_BPF_FUNC_REG_ARGS); i++) { u32 regno = i + 1; t = btf_type_skip_modifiers(desc_btf, args[i].type, NULL); @@ -13697,6 +13765,17 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, mark_btf_func_reg_size(env, regno, t->size); } + /* Track outgoing stack arg depth for kfuncs with >5 args */ + if (nargs > MAX_BPF_FUNC_REG_ARGS) { + struct bpf_func_state *caller = cur_func(env); + struct bpf_subprog_info *caller_info = &env->subprog_info[caller->subprogno]; + u16 kfunc_stack_arg_depth = (nargs - MAX_BPF_FUNC_REG_ARGS) * BPF_REG_SIZE; + + if (kfunc_stack_arg_depth > caller_info->outgoing_stack_arg_depth) + caller_info->outgoing_stack_arg_depth = kfunc_stack_arg_depth; + invalidate_outgoing_stack_args(caller); + } + if (bpf_is_iter_next_kfunc(&meta)) { err = process_iter_next_call(env, insn_idx, &meta); if (err) -- 2.52.0