A private stack keeps its frame pointer in %r9 on x86-64, saved and restored around every call. An unwind skips the restore: a frame resumed at its pad then addresses its stack through a stale pointer, and one sent to its epilogue pops its callee-saved registers one slot off. So force NO_PRIV_STACK for a program that has a table or can unwind, on every arch for now. Signed-off-by: Yonghong Song --- kernel/bpf/exception.c | 10 ++++++++++ kernel/bpf/exception.h | 1 + kernel/bpf/verifier.c | 11 +++++++++++ 3 files changed, 22 insertions(+) diff --git a/kernel/bpf/exception.c b/kernel/bpf/exception.c index 43e01d0be44f..5945fdf5f75c 100644 --- a/kernel/bpf/exception.c +++ b/kernel/bpf/exception.c @@ -183,6 +183,16 @@ void bpf_exc_mark_call_sites(struct bpf_verifier_env *env) } } +bool bpf_prog_may_unwind(const struct bpf_verifier_env *env) +{ + u32 i; + + for (i = 0; i < env->subprog_cnt; i++) + if (env->subprog_info[i].might_unwind) + return true; + return false; +} + int bpf_exc_check_prog(struct bpf_verifier_env *env) { int err; diff --git a/kernel/bpf/exception.h b/kernel/bpf/exception.h index 6d6550daa98e..ecbca25dabef 100644 --- a/kernel/bpf/exception.h +++ b/kernel/bpf/exception.h @@ -14,6 +14,7 @@ int bpf_exc_check_info(struct bpf_verifier_env *env, const union bpf_attr *attr, bpfptr_t uattr); void bpf_exc_mark_call_sites(struct bpf_verifier_env *env); int bpf_exc_check_prog(struct bpf_verifier_env *env); +bool bpf_prog_may_unwind(const struct bpf_verifier_env *env); int bpf_exc_pad_of_call(struct bpf_verifier_env *env, u32 idx); bool bpf_is_unwind_kfunc(const struct bpf_insn *insn); bool bpf_is_unwind_resume_kfunc(const struct bpf_insn *insn); diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index ec5496aa12f7..0392d013b3b7 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -5956,6 +5956,17 @@ static int check_max_stack_depth(struct bpf_verifier_env *env) } } + /* + * A private stack keeps its frame pointer in %r9 on x86-64, restored + * by a pop after the call that an unwind skips. A frame resumed at a + * pad then addresses its stack through a stale pointer, and a frame + * sent to its epilogue instead pops its callee-saved registers one + * slot off. Do not use a private stack for any program that can + * unwind, on every arch for now. + */ + if (env->cleanup_info_cnt || bpf_prog_may_unwind(env)) + priv_stack_mode = NO_PRIV_STACK; + if (priv_stack_mode == PRIV_STACK_UNKNOWN) priv_stack_mode = bpf_enable_priv_stack(env->prog); -- 2.53.0-Meta