From: Alexei Starovoitov The number of maps has no limit now. - The dump of a program looks for the map of each ld_imm64. Put the addresses of the maps into a tree. - The JIT of each subprog needs the insn arrays only. - The jump table of a subprog doubles. Signed-off-by: Alexei Starovoitov --- kernel/bpf/cfg.c | 3 ++- kernel/bpf/fixups.c | 4 +-- kernel/bpf/syscall.c | 64 +++++++++++++++++++++----------------------- 3 files changed, 35 insertions(+), 36 deletions(-) diff --git a/kernel/bpf/cfg.c b/kernel/bpf/cfg.c index c16b4a283dbc..92dc8d0dc22a 100644 --- a/kernel/bpf/cfg.c +++ b/kernel/bpf/cfg.c @@ -366,13 +366,14 @@ static int __compute_subprog_jts(struct bpf_verifier_env *env) } old_cnt = subprog->jt ? subprog->jt->cnt : 0; - jt = bpf_iarray_realloc(subprog->jt, old_cnt + jt_cur->cnt); + jt = bpf_iarray_realloc(subprog->jt, roundup_pow_of_two(old_cnt + jt_cur->cnt)); if (!jt) { subprog->jt = NULL; kvfree(jt_cur); return -ENOMEM; } memcpy(jt->items + old_cnt, jt_cur->items, jt_cur->cnt << 2); + jt->cnt = old_cnt + jt_cur->cnt; subprog->jt = jt; kvfree(jt_cur); diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c index d1eecb95545f..20697eba9c5f 100644 --- a/kernel/bpf/fixups.c +++ b/kernel/bpf/fixups.c @@ -1950,8 +1950,8 @@ static int jit_subprogs(struct bpf_verifier_env *env) func[i]->aux->arena_stack_size = prog->aux->arena_stack_size; func[i]->aux->arena_stack_entry = env->subprog_info[i].arena_stack_entry; func[i]->aux->uses_arena_insns = bpf_prog_has_arena_insns(func[i]); - func[i]->aux->used_maps = env->used_maps; - func[i]->aux->used_map_cnt = env->used_map_cnt; + func[i]->aux->used_maps = env->insn_array_maps; + func[i]->aux->used_map_cnt = env->insn_array_map_cnt; num_exentries = 0; insn = func[i]->insnsi; for (j = 0; j < func[i]->len; j++, insn++) { diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 67878c6b0cc0..ad24973413f4 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -5015,32 +5016,23 @@ static int bpf_map_get_fd_by_id(const union bpf_attr *attr) return fd; } -static const struct bpf_map *bpf_map_from_imm(const struct bpf_prog *prog, - unsigned long addr, u32 *off, - u32 *type) +/* address of a map, addresses of its value -> map */ +static int bpf_prog_map_addrs(const struct bpf_prog *prog, struct maple_tree *mt) { - const struct bpf_map *map; - int i; + struct bpf_map *map; + int i, err = 0; + u64 addr; - mutex_lock(&prog->aux->used_maps_mutex); - for (i = 0, *off = 0; i < prog->aux->used_map_cnt; i++) { + guard(mutex)(&prog->aux->used_maps_mutex); + for (i = 0; i < prog->aux->used_map_cnt && !err; i++) { map = prog->aux->used_maps[i]; - if (map == (void *)addr) { - *type = BPF_PSEUDO_MAP_FD; - goto out; - } - if (!map->ops->map_direct_value_meta) - continue; - if (!map->ops->map_direct_value_meta(map, addr, off)) { - *type = BPF_PSEUDO_MAP_VALUE; - goto out; - } + err = mtree_store(mt, (unsigned long)map, map, GFP_USER); + if (!err && map->ops->map_direct_value_meta && + !map->ops->map_direct_value_addr(map, &addr, 0)) + err = mtree_store_range(mt, addr, addr + map->value_size - 1, map, + GFP_USER); } - map = NULL; - -out: - mutex_unlock(&prog->aux->used_maps_mutex); - return map; + return err; } static void prepare_dump_pseudo_call(struct bpf_insn *insn) @@ -5069,16 +5061,19 @@ static void prepare_dump_pseudo_call(struct bpf_insn *insn) static struct bpf_insn *bpf_insn_prepare_dump(const struct bpf_prog *prog, const struct cred *f_cred) { + struct bpf_insn *insns = NULL; const struct bpf_map *map; - struct bpf_insn *insns; - u32 off, type; + struct maple_tree maps; + u32 off; u64 imm; u8 code; int i; - insns = kvmemdup(prog->insnsi, bpf_prog_insn_size(prog), GFP_USER | __GFP_NOWARN); + mt_init(&maps); + if (!bpf_prog_map_addrs(prog, &maps)) + insns = kvmemdup(prog->insnsi, bpf_prog_insn_size(prog), GFP_USER | __GFP_NOWARN); if (!insns) - return insns; + goto out; for (i = 0; i < prog->len; i++) { code = insns[i].code; @@ -5120,15 +5115,18 @@ static struct bpf_insn *bpf_insn_prepare_dump(const struct bpf_prog *prog, continue; imm = ((u64)insns[i + 1].imm << 32) | (u32)insns[i].imm; - map = bpf_map_from_imm(prog, imm, &off, &type); - if (map) { - insns[i].src_reg = type; - insns[i].imm = map->id; - insns[i + 1].imm = off; + map = mtree_load(&maps, imm); + if (!map || imm > ULONG_MAX) continue; - } + off = 0; + insns[i].src_reg = BPF_PSEUDO_MAP_FD; + if (imm != (unsigned long)map && !map->ops->map_direct_value_meta(map, imm, &off)) + insns[i].src_reg = BPF_PSEUDO_MAP_VALUE; + insns[i].imm = map->id; + insns[i + 1].imm = off; } - +out: + mtree_destroy(&maps); return insns; } -- 2.55.0