There are many adjacent blank lines in the verifier that have accumulated over time. Drop them for cleanup. No functional changes intended. Signed-off-by: Leon Hwang --- kernel/bpf/verifier.c | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index ed7ba0e6a9ce..2690d063a240 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -651,7 +651,6 @@ static void __mark_dynptr_reg(struct bpf_reg_state *reg, enum bpf_dynptr_type type, bool first_slot, int id, int parent_id); - static void mark_dynptr_stack_regs(struct bpf_verifier_env *env, struct bpf_reg_state *sreg1, struct bpf_reg_state *sreg2, @@ -1688,7 +1687,6 @@ static bool same_callsites(struct bpf_verifier_state *a, struct bpf_verifier_sta return true; } - void bpf_free_backedges(struct bpf_scc_visit *visit) { struct bpf_scc_backedge *backedge, *next; @@ -2308,7 +2306,6 @@ static struct bpf_verifier_state *push_async_cb(struct bpf_verifier_env *env, return &elem->st; } - static int cmp_subprogs(const void *a, const void *b) { return ((struct bpf_subprog_info *)a)->start - @@ -3325,7 +3322,6 @@ static bool is_spillable_regtype(enum bpf_reg_type type) } } - /* check if register is a constant scalar value */ static bool is_reg_const(struct bpf_reg_state *reg, bool subreg32) { @@ -3987,7 +3983,6 @@ static int check_stack_read(struct bpf_verifier_env *env, return err; } - /* check_stack_write dispatches to check_stack_write_fixed_off or * check_stack_write_var_off. * @@ -4782,7 +4777,6 @@ static int check_sock_access(struct bpf_verifier_env *env, int insn_idx, valid = false; } - if (valid) { env->insn_aux_data[insn_idx].ctx_field_size = info.ctx_field_size; @@ -6602,7 +6596,6 @@ static int check_stack_range_initialized( if (err) return err; - if (tnum_is_const(reg->var_off)) { min_off = max_off = reg->var_off.value + off; } else { @@ -7316,7 +7309,6 @@ static bool is_iter_new_kfunc(struct bpf_kfunc_call_arg_meta *meta) return meta->kfunc_flags & KF_ITER_NEW; } - static bool is_iter_destroy_kfunc(struct bpf_kfunc_call_arg_meta *meta) { return meta->kfunc_flags & KF_ITER_DESTROY; @@ -11459,7 +11451,6 @@ static int process_irq_flag(struct bpf_verifier_env *env, struct bpf_reg_state * return 0; } - static int ref_set_non_owning(struct bpf_verifier_env *env, struct bpf_reg_state *reg) { struct btf_record *rec = reg_btf_record(reg); @@ -16335,7 +16326,6 @@ static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn) return 0; } - static bool return_retval_range(struct bpf_verifier_env *env, struct bpf_retval_range *range) { enum bpf_prog_type prog_type = resolve_prog_type(env->prog); @@ -18228,8 +18218,6 @@ static void release_insn_arrays(struct bpf_verifier_env *env) bpf_insn_array_release(env->insn_array_maps[i]); } - - /* The verifier does more data flow analysis than llvm and will not * explore branches that are dead at run time. Malicious programs can * have dead code too. Therefore replace all dead at-run-time code @@ -18257,8 +18245,6 @@ static void sanitize_dead_code(struct bpf_verifier_env *env) } } - - static void free_states(struct bpf_verifier_env *env) { struct bpf_verifier_state_list *sl; @@ -18540,7 +18526,6 @@ static int do_check_main(struct bpf_verifier_env *env) return ret; } - static void print_verification_stats(struct bpf_verifier_env *env) { /* Skip over hidden subprogs which are not verified. */ -- 2.54.0 Introduce global percpu data, inspired by the commit 6316f78306c1 ("Merge branch 'support-global-data'"). It enables the definition of global percpu variables in BPF, similar to the include/linux/percpu-defs.h::DEFINE_PER_CPU() macro. For example, in BPF, it is able to define a global percpu variable like: int data SEC(".percpu"); With this patch, tools like retsnoop [1] and bpfsnoop [2] can simplify their BPF code for handling LBRs. The code can be updated from static struct perf_branch_entry lbrs[1][MAX_LBR_ENTRIES] SEC(".data.lbrs"); to static struct perf_branch_entry lbrs[MAX_LBR_ENTRIES] SEC(".percpu.lbrs"); This eliminates the need to retrieve the CPU ID using the bpf_get_smp_processor_id() helper. Additionally, by reusing global percpu data map, sharing information between tail callers and callees or freplace callers and callees becomes simpler compared to reusing percpu_array maps. Links: [1] https://github.com/anakryiko/retsnoop [2] https://github.com/bpfsnoop/bpfsnoop Assisted-by: Codex:gpt-5.5-xhigh Signed-off-by: Leon Hwang --- kernel/bpf/arraymap.c | 39 +++++++++++++++++++++++++++++++++++++-- kernel/bpf/const_fold.c | 1 - kernel/bpf/fixups.c | 32 ++++++++++++++++++++++++++++++++ kernel/bpf/verifier.c | 14 ++++++++++++++ 4 files changed, 83 insertions(+), 3 deletions(-) diff --git a/kernel/bpf/arraymap.c b/kernel/bpf/arraymap.c index e6271a2bf6d6..e6b5d8e89723 100644 --- a/kernel/bpf/arraymap.c +++ b/kernel/bpf/arraymap.c @@ -259,6 +259,38 @@ static void *percpu_array_map_lookup_elem(struct bpf_map *map, void *key) return this_cpu_ptr(array->pptrs[index & array->index_mask]); } +static int percpu_array_map_direct_value_addr(const struct bpf_map *map, u64 *imm, u32 off) +{ + struct bpf_array *array = container_of(map, struct bpf_array, map); + + if (map->max_entries != 1) + return -EOPNOTSUPP; + if (off >= map->value_size) + return -EINVAL; + if (!bpf_jit_supports_percpu_insn()) + return -EOPNOTSUPP; + + *imm = (u64) array->pptrs[0]; + return 0; +} + +static int percpu_array_map_direct_value_meta(const struct bpf_map *map, u64 imm, u32 *off) +{ + struct bpf_array *array = container_of(map, struct bpf_array, map); + u64 base = (u64) array->pptrs[0]; + u64 range = array->elem_size; + + if (map->max_entries != 1) + return -EOPNOTSUPP; + if (imm < base || imm >= base + range) + return -ENOENT; + if (!bpf_jit_supports_percpu_insn()) + return -EOPNOTSUPP; + + *off = imm - base; + return 0; +} + /* emit BPF instructions equivalent to C code of percpu_array_map_lookup_elem() */ static int percpu_array_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn_buf) { @@ -551,9 +583,10 @@ static int array_map_check_btf(struct bpf_map *map, const struct btf_type *key_type, const struct btf_type *value_type) { - /* One exception for keyless BTF: .bss/.data/.rodata map */ + /* One exception for keyless BTF: .bss/.data/.rodata/.percpu map */ if (btf_type_is_void(key_type)) { - if (map->map_type != BPF_MAP_TYPE_ARRAY || + if ((map->map_type != BPF_MAP_TYPE_ARRAY && + map->map_type != BPF_MAP_TYPE_PERCPU_ARRAY) || map->max_entries != 1) return -EINVAL; @@ -832,6 +865,8 @@ const struct bpf_map_ops percpu_array_map_ops = { .map_get_next_key = bpf_array_get_next_key, .map_lookup_elem = percpu_array_map_lookup_elem, .map_gen_lookup = percpu_array_map_gen_lookup, + .map_direct_value_addr = percpu_array_map_direct_value_addr, + .map_direct_value_meta = percpu_array_map_direct_value_meta, .map_update_elem = array_map_update_elem, .map_delete_elem = array_map_delete_elem, .map_lookup_percpu_elem = percpu_array_map_lookup_percpu_elem, diff --git a/kernel/bpf/const_fold.c b/kernel/bpf/const_fold.c index b2a19acadb91..5787246bef30 100644 --- a/kernel/bpf/const_fold.c +++ b/kernel/bpf/const_fold.c @@ -182,7 +182,6 @@ static void const_reg_xfer(struct bpf_verifier_env *env, struct const_arg_info * u64 val = 0; if (!bpf_map_is_rdonly(map) || !map->ops->map_direct_value_addr || - map->map_type == BPF_MAP_TYPE_INSN_ARRAY || off < 0 || off + size > map->value_size || bpf_map_direct_read(map, off, size, &val, is_ldsx)) { *dst = unknown; diff --git a/kernel/bpf/fixups.c b/kernel/bpf/fixups.c index 3cf2cc6e3ab6..4f84d087ca69 100644 --- a/kernel/bpf/fixups.c +++ b/kernel/bpf/fixups.c @@ -1819,6 +1819,38 @@ int bpf_do_misc_fixups(struct bpf_verifier_env *env) goto next_insn; } + if (env->prog->jit_requested && + bpf_jit_supports_percpu_insn() && + insn->code == (BPF_LD | BPF_IMM | BPF_DW) && + (insn->src_reg == BPF_PSEUDO_MAP_VALUE || + insn->src_reg == BPF_PSEUDO_MAP_IDX_VALUE)) { + struct bpf_map *map; + + aux = &env->insn_aux_data[i + delta]; + map = env->used_maps[aux->map_index]; + if (map->map_type != BPF_MAP_TYPE_PERCPU_ARRAY) + goto next_insn; + + /* + * Reuse the original ld_imm64 insn, and add one + * mov64_percpu_reg insn. + */ + + insn_buf[0] = insn[1]; + insn_buf[1] = BPF_MOV64_PERCPU_REG(insn->dst_reg, insn->dst_reg); + cnt = 2; + + i++; + new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); + if (!new_prog) + return -ENOMEM; + + delta += cnt - 1; + env->prog = prog = new_prog; + insn = new_prog->insnsi + i + delta; + goto next_insn; + } + if (insn->code != (BPF_JMP | BPF_CALL)) goto next_insn; if (insn->src_reg == BPF_PSEUDO_CALL) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 2690d063a240..eecd9ab82e91 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -5556,6 +5556,8 @@ int bpf_map_direct_read(struct bpf_map *map, int off, int size, u64 *val, u64 addr; int err; + if (map->map_type == BPF_MAP_TYPE_INSN_ARRAY || map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) + return -EINVAL; err = map->ops->map_direct_value_addr(map, &addr, off); if (err) return err; @@ -8074,6 +8076,12 @@ static int check_arg_const_str(struct bpf_verifier_env *env, return -EACCES; } + if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) { + verbose(env, "%s points to percpu_array map which cannot be used as const string\n", + reg_arg_name(env, argno)); + return -EACCES; + } + if (!bpf_map_is_rdonly(map)) { verbose(env, "%s does not point to a readonly map'\n", reg_arg_name(env, argno)); return -EACCES; @@ -18143,6 +18151,12 @@ static int check_and_resolve_insns(struct bpf_verifier_env *env) return -EINVAL; } + if (map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY && + !env->prog->jit_requested) { + verbose(env, "JIT is required to use global percpu data\n"); + return -EOPNOTSUPP; + } + err = map->ops->map_direct_value_addr(map, &addr, off); if (err) { verbose(env, "invalid access to map value pointer, value_size=%u off=%u\n", -- 2.54.0 libbpf needs a reliable way to distinguish kernels that can support global percpu data from those that cannot. Add a dedicated feature probe, so libbpf can make capability decisions early and fail predictably when global percpu data is unavailable. Signed-off-by: Leon Hwang --- tools/lib/bpf/features.c | 35 +++++++++++++++++++++++++++++++++ tools/lib/bpf/libbpf_internal.h | 2 ++ 2 files changed, 37 insertions(+) diff --git a/tools/lib/bpf/features.c b/tools/lib/bpf/features.c index b7e388f99d0b..ef9581c11303 100644 --- a/tools/lib/bpf/features.c +++ b/tools/lib/bpf/features.c @@ -620,6 +620,38 @@ static int probe_bpf_syscall_common_attrs(int token_fd) return probe_sys_bpf_ext(); } +static int probe_kern_percpu_data(int token_fd) +{ + struct bpf_insn insns[] = { + BPF_LD_MAP_VALUE(BPF_REG_1, 0, 0), + BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, 0), + BPF_EXIT_INSN(), + }; + LIBBPF_OPTS(bpf_map_create_opts, map_opts, + .token_fd = token_fd, + .map_flags = token_fd ? BPF_F_TOKEN_FD : 0, + ); + LIBBPF_OPTS(bpf_prog_load_opts, prog_opts, + .token_fd = token_fd, + .prog_flags = token_fd ? BPF_F_TOKEN_FD : 0, + ); + int ret, map, insn_cnt = ARRAY_SIZE(insns); + + map = bpf_map_create(BPF_MAP_TYPE_PERCPU_ARRAY, "libbpf_percpu", sizeof(int), 8, 1, + &map_opts); + if (map < 0) { + pr_warn("Error in %s(): %s. Couldn't create simple percpu_array map.\n", + __func__, errstr(map)); + return map; + } + + insns[0].imm = map; + + ret = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, &prog_opts); + close(map); + return probe_fd(ret); +} + typedef int (*feature_probe_fn)(int /* token_fd */); static struct kern_feature_cache feature_cache; @@ -707,6 +739,9 @@ static struct kern_feature_desc { [FEAT_BPF_SYSCALL_COMMON_ATTRS] = { "BPF syscall common attributes support", probe_bpf_syscall_common_attrs, }, + [FEAT_PERCPU_DATA] = { + "kernel supports percpu data", probe_kern_percpu_data, + }, }; bool feat_supported(struct kern_feature_cache *cache, enum kern_feature_id feat_id) diff --git a/tools/lib/bpf/libbpf_internal.h b/tools/lib/bpf/libbpf_internal.h index 04cd303fb5a8..47ae39125f68 100644 --- a/tools/lib/bpf/libbpf_internal.h +++ b/tools/lib/bpf/libbpf_internal.h @@ -401,6 +401,8 @@ enum kern_feature_id { FEAT_BTF_LAYOUT, /* Kernel supports BPF syscall common attributes */ FEAT_BPF_SYSCALL_COMMON_ATTRS, + /* Kernel supports percpu data */ + FEAT_PERCPU_DATA, __FEAT_CNT, }; -- 2.54.0 Add support for global percpu data in libbpf by adding a new ".percpu" section, similar to ".data". It enables efficient handling of percpu global variables in bpf programs. When generating loader for lightweight skeleton, update the percpu_array map used for global percpu data using BPF_F_ALL_CPUS, in order to update values across all CPUs using one value slot. Unlike global data, the mmaped data for global percpu data will be marked as read-only after populating the percpu_array map. Thereafter, users can read those initialized percpu data after loading prog. If they want to update the percpu data after loading prog, they have to update the percpu_array map using key=0 instead. Signed-off-by: Leon Hwang --- tools/lib/bpf/bpf_gen_internal.h | 3 +- tools/lib/bpf/gen_loader.c | 3 +- tools/lib/bpf/libbpf.c | 68 ++++++++++++++++++++++++++------ 3 files changed, 60 insertions(+), 14 deletions(-) diff --git a/tools/lib/bpf/bpf_gen_internal.h b/tools/lib/bpf/bpf_gen_internal.h index 49af4260b8e6..5ea8383805d3 100644 --- a/tools/lib/bpf/bpf_gen_internal.h +++ b/tools/lib/bpf/bpf_gen_internal.h @@ -66,7 +66,8 @@ void bpf_gen__prog_load(struct bpf_gen *gen, enum bpf_prog_type prog_type, const char *prog_name, const char *license, struct bpf_insn *insns, size_t insn_cnt, struct bpf_prog_load_opts *load_attr, int prog_idx); -void bpf_gen__map_update_elem(struct bpf_gen *gen, int map_idx, void *value, __u32 value_size); +void bpf_gen__map_update_elem(struct bpf_gen *gen, int map_idx, void *value, __u32 value_size, + __u64 flags); void bpf_gen__map_freeze(struct bpf_gen *gen, int map_idx); void bpf_gen__record_attach_target(struct bpf_gen *gen, const char *name, enum bpf_attach_type type); void bpf_gen__record_extern(struct bpf_gen *gen, const char *name, bool is_weak, diff --git a/tools/lib/bpf/gen_loader.c b/tools/lib/bpf/gen_loader.c index d79695f01c87..7ceb096f18ec 100644 --- a/tools/lib/bpf/gen_loader.c +++ b/tools/lib/bpf/gen_loader.c @@ -1190,7 +1190,7 @@ void bpf_gen__prog_load(struct bpf_gen *gen, } void bpf_gen__map_update_elem(struct bpf_gen *gen, int map_idx, void *pvalue, - __u32 value_size) + __u32 value_size, __u64 flags) { int attr_size = offsetofend(union bpf_attr, flags); int map_update_attr, value, key; @@ -1198,6 +1198,7 @@ void bpf_gen__map_update_elem(struct bpf_gen *gen, int map_idx, void *pvalue, int zero = 0; memset(&attr, 0, attr_size); + attr.flags = flags; value = add_data(gen, pvalue, value_size); key = add_data(gen, &zero, sizeof(zero)); diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c index 1368752aa13c..199d08d3cdcd 100644 --- a/tools/lib/bpf/libbpf.c +++ b/tools/lib/bpf/libbpf.c @@ -541,6 +541,7 @@ struct bpf_struct_ops { }; #define DATA_SEC ".data" +#define PERCPU_SEC ".percpu" #define BSS_SEC ".bss" #define RODATA_SEC ".rodata" #define KCONFIG_SEC ".kconfig" @@ -555,6 +556,7 @@ enum libbpf_map_type { LIBBPF_MAP_BSS, LIBBPF_MAP_RODATA, LIBBPF_MAP_KCONFIG, + LIBBPF_MAP_PERCPU, }; struct bpf_map_def { @@ -666,6 +668,7 @@ enum sec_type { SEC_DATA, SEC_RODATA, SEC_ST_OPS, + SEC_PERCPU, }; struct elf_sec_desc { @@ -1839,6 +1842,8 @@ static size_t bpf_map_mmap_sz(const struct bpf_map *map) switch (map->def.type) { case BPF_MAP_TYPE_ARRAY: return array_map_mmap_sz(map->def.value_size, map->def.max_entries); + case BPF_MAP_TYPE_PERCPU_ARRAY: + return map->def.value_size; case BPF_MAP_TYPE_ARENA: return page_sz * map->def.max_entries; default: @@ -1938,7 +1943,7 @@ static bool map_is_mmapable(struct bpf_object *obj, struct bpf_map *map) struct btf_var_secinfo *vsi; int i, n; - if (!map->btf_value_type_id) + if (!map->btf_value_type_id || map->libbpf_type == LIBBPF_MAP_PERCPU) return false; t = btf__type_by_id(obj->btf, map->btf_value_type_id); @@ -1962,6 +1967,7 @@ static int bpf_object__init_internal_map(struct bpf_object *obj, enum libbpf_map_type type, const char *real_name, int sec_idx, void *data, size_t data_sz) { + bool is_percpu = type == LIBBPF_MAP_PERCPU; struct bpf_map_def *def; struct bpf_map *map; size_t mmap_sz; @@ -1983,7 +1989,7 @@ bpf_object__init_internal_map(struct bpf_object *obj, enum libbpf_map_type type, } def = &map->def; - def->type = BPF_MAP_TYPE_ARRAY; + def->type = is_percpu ? BPF_MAP_TYPE_PERCPU_ARRAY : BPF_MAP_TYPE_ARRAY; def->key_size = sizeof(int); def->value_size = data_sz; def->max_entries = 1; @@ -1996,8 +2002,9 @@ bpf_object__init_internal_map(struct bpf_object *obj, enum libbpf_map_type type, if (map_is_mmapable(obj, map)) def->map_flags |= BPF_F_MMAPABLE; - pr_debug("map '%s' (global data): at sec_idx %d, offset %zu, flags %x.\n", - map->name, map->sec_idx, map->sec_offset, def->map_flags); + pr_debug("map '%s' (global %sdata): at sec_idx %d, offset %zu, flags %x.\n", + map->name, is_percpu ? "percpu " : "", map->sec_idx, + map->sec_offset, def->map_flags); mmap_sz = bpf_map_mmap_sz(map); map->mmaped = mmap(NULL, mmap_sz, PROT_READ | PROT_WRITE, @@ -2057,6 +2064,13 @@ static int bpf_object__init_global_data_maps(struct bpf_object *obj) NULL, sec_desc->data->d_size); break; + case SEC_PERCPU: + sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, sec_idx)); + err = bpf_object__init_internal_map(obj, LIBBPF_MAP_PERCPU, + sec_name, sec_idx, + sec_desc->data->d_buf, + sec_desc->data->d_size); + break; default: /* skip */ break; @@ -4016,6 +4030,11 @@ static int bpf_object__elf_collect(struct bpf_object *obj) sec_desc->sec_type = SEC_RODATA; sec_desc->shdr = sh; sec_desc->data = data; + } else if (strcmp(name, PERCPU_SEC) == 0 || + str_has_pfx(name, PERCPU_SEC ".")) { + sec_desc->sec_type = SEC_PERCPU; + sec_desc->shdr = sh; + sec_desc->data = data; } else if (strcmp(name, STRUCT_OPS_SEC) == 0 || strcmp(name, STRUCT_OPS_LINK_SEC) == 0 || strcmp(name, "?" STRUCT_OPS_SEC) == 0 || @@ -4544,6 +4563,7 @@ static bool bpf_object__shndx_is_data(const struct bpf_object *obj, case SEC_BSS: case SEC_DATA: case SEC_RODATA: + case SEC_PERCPU: return true; default: return false; @@ -4569,6 +4589,8 @@ bpf_object__section_to_libbpf_map_type(const struct bpf_object *obj, int shndx) return LIBBPF_MAP_DATA; case SEC_RODATA: return LIBBPF_MAP_RODATA; + case SEC_PERCPU: + return LIBBPF_MAP_PERCPU; default: return LIBBPF_MAP_UNSPEC; } @@ -4944,7 +4966,7 @@ static int map_fill_btf_type_info(struct bpf_object *obj, struct bpf_map *map) /* * LLVM annotates global data differently in BTF, that is, - * only as '.data', '.bss' or '.rodata'. + * only as '.data', '.bss', '.percpu' or '.rodata'. */ if (!bpf_map__is_internal(map)) return -ENOENT; @@ -5297,18 +5319,30 @@ static int bpf_object__populate_internal_map(struct bpf_object *obj, struct bpf_map *map) { enum libbpf_map_type map_type = map->libbpf_type; + bool is_percpu = map_type == LIBBPF_MAP_PERCPU; + __u64 update_flags = 0; int err, zero = 0; size_t mmap_sz; + if (is_percpu) { + if (!obj->gen_loader && !kernel_supports(obj, FEAT_PERCPU_DATA)) { + pr_warn("map '%s': kernel does not support percpu data.\n", + bpf_map__name(map)); + return -EOPNOTSUPP; + } + + update_flags = BPF_F_ALL_CPUS; + } + if (obj->gen_loader) { bpf_gen__map_update_elem(obj->gen_loader, map - obj->maps, - map->mmaped, map->def.value_size); + map->mmaped, map->def.value_size, update_flags); if (map_type == LIBBPF_MAP_RODATA || map_type == LIBBPF_MAP_KCONFIG) bpf_gen__map_freeze(obj->gen_loader, map - obj->maps); return 0; } - err = bpf_map_update_elem(map->fd, &zero, map->mmaped, 0); + err = bpf_map_update_elem(map->fd, &zero, map->mmaped, update_flags); if (err) { err = -errno; pr_warn("map '%s': failed to set initial contents: %s\n", @@ -5353,6 +5387,13 @@ bpf_object__populate_internal_map(struct bpf_object *obj, struct bpf_map *map) return err; } map->mmaped = mmaped; + } else if (is_percpu) { + if (mprotect(map->mmaped, mmap_sz, PROT_READ)) { + err = -errno; + pr_warn("map '%s': failed to mprotect() contents: %s\n", + bpf_map__name(map), errstr(err)); + return err; + } } else if (map->mmaped) { munmap(map->mmaped, mmap_sz); map->mmaped = NULL; @@ -10806,16 +10847,19 @@ int bpf_map__fd(const struct bpf_map *map) static bool map_uses_real_name(const struct bpf_map *map) { - /* Since libbpf started to support custom .data.* and .rodata.* maps, - * their user-visible name differs from kernel-visible name. Users see - * such map's corresponding ELF section name as a map name. - * This check distinguishes .data/.rodata from .data.* and .rodata.* - * maps to know which name has to be returned to the user. + /* + * Since libbpf started to support custom .data.*, .rodata.* and + * .percpu.* maps, their user-visible name differs from + * kernel-visible name. Users see such map's corresponding ELF section + * name as a map name. This check distinguishes plain .data/.rodata/.percpu + * from .data.*, .rodata.* and .percpu.* to choose which name to return. */ if (map->libbpf_type == LIBBPF_MAP_DATA && strcmp(map->real_name, DATA_SEC) != 0) return true; if (map->libbpf_type == LIBBPF_MAP_RODATA && strcmp(map->real_name, RODATA_SEC) != 0) return true; + if (map->libbpf_type == LIBBPF_MAP_PERCPU && strcmp(map->real_name, PERCPU_SEC) != 0) + return true; return false; } -- 2.54.0 Enhance bpftool to generate skeletons that properly handle global percpu variables. The generated skeleton now includes a dedicated structure for percpu data, allowing users to initialize and access percpu variables more efficiently. For global percpu variables, the skeleton now includes a nested structure, e.g.: struct test_global_percpu_data { struct bpf_object_skeleton *skeleton; struct bpf_object *obj; struct { struct bpf_map *percpu; } maps; // ... struct test_global_percpu_data__percpu { int data; char run; struct { char set; int i; int nums[7]; } struct_data; int nums[7]; } *percpu; // ... }; * The "struct test_global_percpu_data__percpu *percpu" points to initialized data, which is actually "maps.percpu->mmaped". * Before loading the skeleton, updating the "struct test_global_percpu_data__percpu *percpu" modifies the initial value of the corresponding global percpu variables. * After loading the skeleton, "maps.percpu->mmaped" has been marked as read-only in libbpf. If users want to update the global percpu variables, they have to update the "maps.percpu" map instead. Signed-off-by: Leon Hwang --- tools/bpf/bpftool/gen.c | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/tools/bpf/bpftool/gen.c b/tools/bpf/bpftool/gen.c index 6ae7262ebe0c..e0c5c643fe9e 100644 --- a/tools/bpf/bpftool/gen.c +++ b/tools/bpf/bpftool/gen.c @@ -92,7 +92,7 @@ static void get_header_guard(char *guard, const char *obj_name, const char *suff static bool get_map_ident(const struct bpf_map *map, char *buf, size_t buf_sz) { - static const char *sfxs[] = { ".data", ".rodata", ".bss", ".kconfig" }; + static const char *sfxs[] = { ".data", ".rodata", ".bss", ".percpu", ".kconfig" }; const char *name = bpf_map__name(map); int i, n; @@ -117,7 +117,7 @@ static bool get_map_ident(const struct bpf_map *map, char *buf, size_t buf_sz) static bool get_datasec_ident(const char *sec_name, char *buf, size_t buf_sz) { - static const char *pfxs[] = { ".data", ".rodata", ".bss", ".kconfig" }; + static const char *pfxs[] = { ".data", ".rodata", ".bss", ".percpu", ".kconfig" }; int i, n; /* recognize hard coded LLVM section name */ @@ -254,6 +254,11 @@ static const struct btf_type *find_type_for_map(struct btf *btf, const char *map return NULL; } +static bool bpf_map_is_percpu_data(const struct bpf_map *map) +{ + return bpf_map__is_internal(map) && bpf_map__type(map) == BPF_MAP_TYPE_PERCPU_ARRAY; +} + static bool is_mmapable_map(const struct bpf_map *map, char *buf, size_t sz) { size_t tmp_sz; @@ -263,13 +268,12 @@ static bool is_mmapable_map(const struct bpf_map *map, char *buf, size_t sz) return true; } - if (!bpf_map__is_internal(map) || !(bpf_map__map_flags(map) & BPF_F_MMAPABLE)) - return false; - - if (!get_map_ident(map, buf, sz)) - return false; + if (bpf_map__is_internal(map) && + ((bpf_map__map_flags(map) & BPF_F_MMAPABLE) || bpf_map_is_percpu_data(map)) && + get_map_ident(map, buf, sz)) + return true; - return true; + return false; } static int codegen_datasecs(struct bpf_object *obj, const char *obj_name) @@ -669,7 +673,7 @@ static void codegen_destroy(struct bpf_object *obj, const char *obj_name) if (!get_map_ident(map, ident, sizeof(ident))) continue; if (bpf_map__is_internal(map) && - (bpf_map__map_flags(map) & BPF_F_MMAPABLE)) + ((bpf_map__map_flags(map) & BPF_F_MMAPABLE) || bpf_map_is_percpu_data(map))) printf("\tskel_free_map_data(skel->%1$s, skel->maps.%1$s.initial_value, %2$zu);\n", ident, bpf_map_mmap_sz(map)); codegen("\ @@ -847,7 +851,7 @@ static int gen_trace(struct bpf_object *obj, const char *obj_name, const char *h bpf_object__for_each_map(map, obj) { const char *mmap_flags; - if (!is_mmapable_map(map, ident, sizeof(ident))) + if (!is_mmapable_map(map, ident, sizeof(ident)) || bpf_map_is_percpu_data(map)) continue; if (bpf_map__map_flags(map) & BPF_F_RDONLY_PROG) -- 2.54.0 If the arch, like s390x, does not support percpu insn, these cases won't test global percpu data by checking FEAT_PERCPU_DATA support. The following APIs have been tested for global percpu data: 1. bpf_map__set_initial_value() 2. bpf_map__initial_value() 3. generated percpu struct pointer pointing to internal map's mmaped data 4. bpf_map__lookup_elem() for global percpu data map At the same time, the case is also tested with 'bpftool gen skeleton -L'. Signed-off-by: Leon Hwang --- tools/testing/selftests/bpf/Makefile | 2 +- .../bpf/prog_tests/global_data_init.c | 196 ++++++++++++++++++ .../bpf/progs/test_global_percpu_data.c | 31 +++ 3 files changed, 228 insertions(+), 1 deletion(-) create mode 100644 tools/testing/selftests/bpf/progs/test_global_percpu_data.c diff --git a/tools/testing/selftests/bpf/Makefile b/tools/testing/selftests/bpf/Makefile index d53b7e496ac9..d21064cfc066 100644 --- a/tools/testing/selftests/bpf/Makefile +++ b/tools/testing/selftests/bpf/Makefile @@ -526,7 +526,7 @@ LSKELS_SIGNED := fentry_test.c fexit_test.c atomics.c # Generate both light skeleton and libbpf skeleton for these LSKELS_EXTRA := test_ksyms_module.c test_ksyms_weak.c kfunc_call_test.c \ - kfunc_call_test_subprog.c + kfunc_call_test_subprog.c test_global_percpu_data.c SKEL_BLACKLIST += $$(LSKELS) $$(LSKELS_SIGNED) test_static_linked.skel.h-deps := test_static_linked1.bpf.o test_static_linked2.bpf.o diff --git a/tools/testing/selftests/bpf/prog_tests/global_data_init.c b/tools/testing/selftests/bpf/prog_tests/global_data_init.c index 8466332d7406..5abdb0b8b8b3 100644 --- a/tools/testing/selftests/bpf/prog_tests/global_data_init.c +++ b/tools/testing/selftests/bpf/prog_tests/global_data_init.c @@ -1,5 +1,8 @@ // SPDX-License-Identifier: GPL-2.0 #include +#include "bpf/libbpf_internal.h" +#include "test_global_percpu_data.skel.h" +#include "test_global_percpu_data.lskel.h" void test_global_data_init(void) { @@ -60,3 +63,196 @@ void test_global_data_init(void) free(newval); bpf_object__close(obj); } + +static void test_global_percpu_data_init(void) +{ + struct test_global_percpu_data__percpu *init_data, *data = NULL; + struct test_global_percpu_data__percpu init_value = {}; + int key, prog_fd, err, num_cpus, num_online, i; + struct test_global_percpu_data *skel = NULL; + __u64 args[2] = {0x1234ULL, 0x5678ULL}; + size_t elem_sz, init_data_sz; + struct bpf_map *map; + bool *online; + LIBBPF_OPTS(bpf_test_run_opts, topts, + .ctx_in = args, + .ctx_size_in = sizeof(args), + .flags = BPF_F_TEST_RUN_ON_CPU, + ); + + num_cpus = libbpf_num_possible_cpus(); + if (!ASSERT_GT(num_cpus, 0, "libbpf_num_possible_cpus")) + return; + + err = parse_cpu_mask_file("/sys/devices/system/cpu/online", &online, &num_online); + if (!ASSERT_OK(err, "parse_cpu_mask_file")) + return; + + elem_sz = roundup(sizeof(*data), 8); + data = calloc(1, elem_sz); + if (!ASSERT_OK_PTR(data, "calloc data")) + goto out; + + skel = test_global_percpu_data__open(); + if (!ASSERT_OK_PTR(skel, "test_global_percpu_data__open")) + goto out; + if (!ASSERT_OK_PTR(skel->percpu, "skel->percpu")) + goto out; + + ASSERT_EQ(skel->percpu->data, -1, "skel->percpu->data"); + ASSERT_FALSE(skel->percpu->run, "skel->percpu->run"); + ASSERT_EQ(skel->percpu->nums[6], 0, "skel->percpu->nums[6]"); + ASSERT_EQ(skel->percpu->struct_data.i, -1, "struct_data.i"); + ASSERT_FALSE(skel->percpu->struct_data.set, "struct_data.set"); + ASSERT_EQ(skel->percpu->struct_data.nums[6], 0, "struct_data.nums[6]"); + + map = skel->maps.percpu; + if (!ASSERT_EQ(bpf_map__type(map), BPF_MAP_TYPE_PERCPU_ARRAY, "bpf_map__type")) + goto out; + + init_value.data = 2; + init_value.nums[6] = -1; + init_value.struct_data.i = 2; + init_value.struct_data.nums[6] = -1; + err = bpf_map__set_initial_value(map, &init_value, sizeof(init_value)); + if (!ASSERT_OK(err, "bpf_map__set_initial_value")) + goto out; + + init_data = bpf_map__initial_value(map, &init_data_sz); + if (!ASSERT_OK_PTR(init_data, "bpf_map__initial_value")) + goto out; + + ASSERT_EQ(init_data->data, init_value.data, "init_value data"); + ASSERT_EQ(init_data->run, init_value.run, "init_value run"); + ASSERT_EQ(init_data->struct_data.i, init_value.struct_data.i, "init_value struct_data.i"); + ASSERT_EQ(init_data->struct_data.nums[6], init_value.struct_data.nums[6], + "init_value struct_data.nums[6]"); + ASSERT_EQ(init_data_sz, sizeof(init_value), "init_value size"); + ASSERT_EQ((void *) init_data, (void *) skel->percpu, "skel->percpu eq init_data"); + ASSERT_EQ(skel->percpu->data, init_value.data, "skel->percpu->data"); + ASSERT_EQ(skel->percpu->run, init_value.run, "skel->percpu->run"); + ASSERT_EQ(skel->percpu->struct_data.i, init_value.struct_data.i, + "skel->percpu->struct_data.i"); + ASSERT_EQ(skel->percpu->struct_data.nums[6], init_value.struct_data.nums[6], + "skel->percpu->struct_data.nums[6]"); + + err = test_global_percpu_data__load(skel); + if (!ASSERT_OK(err, "test_global_percpu_data__load")) + goto out; + + ASSERT_OK_PTR(skel->percpu, "skel->percpu"); + + key = 0; + prog_fd = bpf_program__fd(skel->progs.update_percpu_data); + + /* run on every CPU */ + for (i = 0; i < num_online; i++) { + __u64 flags; + + if (!online[i]) + continue; + + topts.cpu = i; + topts.retval = 0; + err = bpf_prog_test_run_opts(prog_fd, &topts); + ASSERT_OK(err, "bpf_prog_test_run_opts"); + ASSERT_EQ(topts.retval, 0, "bpf_prog_test_run_opts retval"); + + flags = ((__u64) i << 32) | BPF_F_CPU; + err = bpf_map__lookup_elem(map, &key, sizeof(key), data, elem_sz, flags); + if (!ASSERT_OK(err, "bpf_map__lookup_elem")) + goto out; + + ASSERT_EQ(data->data, 1, "data->data"); + ASSERT_TRUE(data->run, "data->run"); + ASSERT_EQ(data->nums[6], 0xc0de, "data->nums[6]"); + ASSERT_EQ(data->struct_data.i, 1, "struct_data.i"); + ASSERT_TRUE(data->struct_data.set, "struct_data.set"); + ASSERT_EQ(data->struct_data.nums[6], 0xc0de, "struct_data.nums[6]"); + } + +out: + test_global_percpu_data__destroy(skel); + free(data); + free(online); +} + +static void test_global_percpu_data_lskel(void) +{ + int key, prog_fd, map_fd, err, num_cpus, num_online, i; + struct test_global_percpu_data__percpu *data = NULL; + struct test_global_percpu_data_lskel *lskel = NULL; + __u64 args[2] = {0x1234ULL, 0x5678ULL}; + size_t elem_sz; + bool *online; + LIBBPF_OPTS(bpf_test_run_opts, topts, + .ctx_in = args, + .ctx_size_in = sizeof(args), + .flags = BPF_F_TEST_RUN_ON_CPU, + ); + + num_cpus = libbpf_num_possible_cpus(); + if (!ASSERT_GT(num_cpus, 0, "libbpf_num_possible_cpus")) + return; + + err = parse_cpu_mask_file("/sys/devices/system/cpu/online", &online, &num_online); + if (!ASSERT_OK(err, "parse_cpu_mask_file")) + return; + + elem_sz = roundup(sizeof(*data), 8); + data = calloc(1, elem_sz); + if (!ASSERT_OK_PTR(data, "calloc data")) + goto out; + + lskel = test_global_percpu_data_lskel__open_and_load(); + if (!ASSERT_OK_PTR(lskel, "test_global_percpu_data_lskel__open_and_load")) + goto out; + + key = 0; + map_fd = lskel->maps.percpu.map_fd; + prog_fd = lskel->progs.update_percpu_data.prog_fd; + + /* run on every CPU */ + for (i = 0; i < num_online; i++) { + __u64 flags; + + if (!online[i]) + continue; + + topts.cpu = i; + topts.retval = 0; + err = bpf_prog_test_run_opts(prog_fd, &topts); + ASSERT_OK(err, "bpf_prog_test_run_opts"); + ASSERT_EQ(topts.retval, 0, "bpf_prog_test_run_opts retval"); + + flags = ((__u64) i << 32) | BPF_F_CPU; + err = bpf_map_lookup_elem_flags(map_fd, &key, data, flags); + if (!ASSERT_OK(err, "bpf_map_lookup_elem_flags")) + goto out; + + ASSERT_EQ(data->data, 1, "data->data"); + ASSERT_TRUE(data->run, "data->run"); + ASSERT_EQ(data->nums[6], 0xc0de, "data->nums[6]"); + ASSERT_EQ(data->struct_data.i, 1, "struct_data.i"); + ASSERT_TRUE(data->struct_data.set, "struct_data.set"); + ASSERT_EQ(data->struct_data.nums[6], 0xc0de, "struct_data.nums[6]"); + } + +out: + test_global_percpu_data_lskel__destroy(lskel); + free(data); + free(online); +} + +void test_global_percpu_data(void) +{ + if (!feat_supported(NULL, FEAT_PERCPU_DATA)) { + test__skip(); + return; + } + + if (test__start_subtest("init")) + test_global_percpu_data_init(); + if (test__start_subtest("lskel")) + test_global_percpu_data_lskel(); +} diff --git a/tools/testing/selftests/bpf/progs/test_global_percpu_data.c b/tools/testing/selftests/bpf/progs/test_global_percpu_data.c new file mode 100644 index 000000000000..ba92ffb0ca49 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/test_global_percpu_data.c @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include "bpf_misc.h" + +int data SEC(".percpu") = -1; +int nums[7] SEC(".percpu"); +char run SEC(".percpu") = 0; +struct { + char set; + int i; + int nums[7]; +} struct_data SEC(".percpu") = { + .set = 0, + .i = -1, +}; + +SEC("raw_tp/task_rename") +__auxiliary +int update_percpu_data(void *ctx) +{ + struct_data.nums[6] = 0xc0de; + struct_data.set = 1; + struct_data.i = 1; + nums[6] = 0xc0de; + data = 1; + run = 1; + return 0; +} + +char _license[] SEC("license") = "GPL"; -- 2.54.0 Add two tests to verify the verifier log "R%d points to percpu_array map which cannot be used as const string\n". Signed-off-by: Leon Hwang --- .../bpf/prog_tests/global_data_init.c | 7 ++++++ .../bpf/progs/test_global_percpu_data.c | 23 +++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/tools/testing/selftests/bpf/prog_tests/global_data_init.c b/tools/testing/selftests/bpf/prog_tests/global_data_init.c index 5abdb0b8b8b3..729878d5d3b3 100644 --- a/tools/testing/selftests/bpf/prog_tests/global_data_init.c +++ b/tools/testing/selftests/bpf/prog_tests/global_data_init.c @@ -244,6 +244,11 @@ static void test_global_percpu_data_lskel(void) free(online); } +static void test_global_percpu_data_verifier_log(void) +{ + RUN_TESTS(test_global_percpu_data); +} + void test_global_percpu_data(void) { if (!feat_supported(NULL, FEAT_PERCPU_DATA)) { @@ -255,4 +260,6 @@ void test_global_percpu_data(void) test_global_percpu_data_init(); if (test__start_subtest("lskel")) test_global_percpu_data_lskel(); + if (test__start_subtest("verifier_log")) + test_global_percpu_data_verifier_log(); } diff --git a/tools/testing/selftests/bpf/progs/test_global_percpu_data.c b/tools/testing/selftests/bpf/progs/test_global_percpu_data.c index ba92ffb0ca49..2222ad3a49bd 100644 --- a/tools/testing/selftests/bpf/progs/test_global_percpu_data.c +++ b/tools/testing/selftests/bpf/progs/test_global_percpu_data.c @@ -28,4 +28,27 @@ int update_percpu_data(void *ctx) return 0; } +static const char fmt[] SEC(".percpu.fmt") = "data %d\n"; + +SEC("?kprobe") +__failure __msg("R{{[0-9]+}} points to percpu_array map which cannot be used as const string") +int verifier_strncmp(void *ctx) +{ + return bpf_strncmp("test", 5, fmt); +} + +SEC("?kprobe") +__failure __msg("R{{[0-9]+}} points to percpu_array map which cannot be used as const string") +int verifier_snprintf(void *ctx) +{ + u64 args[] = { data }; + char buf[128]; + int len; + + len = bpf_snprintf(buf, sizeof(buf), fmt, args, 1); + if (len > 0) + bpf_printk("snprintf: %s\n", buf); + return 0; +} + char _license[] SEC("license") = "GPL"; -- 2.54.0 Add a test to verify global percpu data related xlated insns: 1. ld_imm64: compare xlated one with the one in ELF object file. 2. mov64_percpu_reg: it is added by verifier. Signed-off-by: Leon Hwang --- .../bpf/prog_tests/global_data_init.c | 86 +++++++++++++++++++ .../bpf/progs/test_global_percpu_data.c | 11 +++ 2 files changed, 97 insertions(+) diff --git a/tools/testing/selftests/bpf/prog_tests/global_data_init.c b/tools/testing/selftests/bpf/prog_tests/global_data_init.c index 729878d5d3b3..c3b9423246b5 100644 --- a/tools/testing/selftests/bpf/prog_tests/global_data_init.c +++ b/tools/testing/selftests/bpf/prog_tests/global_data_init.c @@ -249,6 +249,90 @@ static void test_global_percpu_data_verifier_log(void) RUN_TESTS(test_global_percpu_data); } +static int find_ld_imm64(const struct bpf_insn *insns, size_t insn_cnt, struct bpf_insn *ld_imm64) +{ + size_t i; + + for (i = 0; i < insn_cnt; i++) { + if (insns[i].code == (BPF_LD | BPF_IMM | BPF_DW)) { + ld_imm64[0] = insns[i]; + ld_imm64[1] = insns[i + 1]; + return i; + } + } + + return -ENOENT; +} + +/* + * Special (internal-only) form of mov, used to resolve per-CPU addrs: + * dst_reg = src_reg + + * BPF_ADDR_PERCPU is used as a special insn->off value. + */ +#define BPF_ADDR_PERCPU (-1) + +#define BPF_MOV64_PERCPU_REG(DST, SRC) \ + ((struct bpf_insn) { \ + .code = BPF_ALU64 | BPF_MOV | BPF_X, \ + .dst_reg = DST, \ + .src_reg = SRC, \ + .off = BPF_ADDR_PERCPU, \ + .imm = 0 }) + +static __u64 ld_imm64_to_u64(const struct bpf_insn *insn) +{ + return ((__u64) insn[1].imm << 32) | (__u64) insn[0].imm; +} + +static void test_global_percpu_data_xlated(void) +{ + struct bpf_insn ld_imm64_raw[2], ld_imm64_xlated[2], mov64_percpu_reg, *insns = NULL; + size_t insn_sz = sizeof(struct bpf_insn); + struct test_global_percpu_data *skel; + struct bpf_program *prog; + int idx, err; + __u32 cnt; + + skel = test_global_percpu_data__open(); + if (!ASSERT_OK_PTR(skel, "test_global_percpu_data__open")) + return; + + prog = skel->progs.verifier_percpu_read; + idx = find_ld_imm64(bpf_program__insns(prog), bpf_program__insn_cnt(prog), ld_imm64_raw); + if (!ASSERT_GE(idx, 0, "find_ld_imm64 raw")) + goto out; + + err = test_global_percpu_data__load(skel); + if (!ASSERT_OK(err, "test_global_percpu_data__load")) + goto out; + + err = get_xlated_program(bpf_program__fd(prog), &insns, &cnt); + if (!ASSERT_OK(err, "get_xlated_program")) + goto out; + if (!ASSERT_GT(cnt, idx + 2, "xlated insn count")) + goto out; + + idx = find_ld_imm64(insns, cnt, ld_imm64_xlated); + if (!ASSERT_GE(idx, 0, "find_ld_imm64 xlated")) + goto out; + + ASSERT_EQ(ld_imm64_xlated[0].code, ld_imm64_raw[0].code, "ld_imm64 opcode"); + ASSERT_TRUE(ld_imm64_xlated[0].dst_reg == ld_imm64_raw[0].dst_reg, "ld_imm64 dst_reg"); + /* + * The xlated instruction has the map ID in imm and the offset + * in the next instruction's imm. The raw instruction just has + * the offset in its imm. + */ + ASSERT_EQ(ld_imm64_xlated[1].imm, ld_imm64_to_u64(ld_imm64_raw), "ld_imm64 off"); + + mov64_percpu_reg = BPF_MOV64_PERCPU_REG(ld_imm64_raw[0].dst_reg, ld_imm64_raw[0].dst_reg); + ASSERT_MEMEQ(&insns[idx + 2], &mov64_percpu_reg, insn_sz, "mov64_percpu_reg"); + +out: + test_global_percpu_data__destroy(skel); + free(insns); +} + void test_global_percpu_data(void) { if (!feat_supported(NULL, FEAT_PERCPU_DATA)) { @@ -262,4 +346,6 @@ void test_global_percpu_data(void) test_global_percpu_data_lskel(); if (test__start_subtest("verifier_log")) test_global_percpu_data_verifier_log(); + if (test__start_subtest("xlated")) + test_global_percpu_data_xlated(); } diff --git a/tools/testing/selftests/bpf/progs/test_global_percpu_data.c b/tools/testing/selftests/bpf/progs/test_global_percpu_data.c index 2222ad3a49bd..9a1b1a314c2f 100644 --- a/tools/testing/selftests/bpf/progs/test_global_percpu_data.c +++ b/tools/testing/selftests/bpf/progs/test_global_percpu_data.c @@ -51,4 +51,15 @@ int verifier_snprintf(void *ctx) return 0; } +static volatile const char fmt2[] SEC(".percpu.fmt") = "data %d\n"; + +SEC("kprobe") +__auxiliary +int verifier_percpu_read(void *ctx) +{ + char c = fmt2[4]; + + return c == ' '; +} + char _license[] SEC("license") = "GPL"; -- 2.54.0 Add a test to verify that it is OK to iter the percpu_array map used for global percpu data. Signed-off-by: Leon Hwang --- .../bpf/prog_tests/global_data_init.c | 52 +++++++++++++++++++ .../bpf/progs/test_global_percpu_data.c | 25 +++++++++ 2 files changed, 77 insertions(+) diff --git a/tools/testing/selftests/bpf/prog_tests/global_data_init.c b/tools/testing/selftests/bpf/prog_tests/global_data_init.c index c3b9423246b5..6c1813ffe5f3 100644 --- a/tools/testing/selftests/bpf/prog_tests/global_data_init.c +++ b/tools/testing/selftests/bpf/prog_tests/global_data_init.c @@ -333,6 +333,56 @@ static void test_global_percpu_data_xlated(void) free(insns); } +static void test_global_percpu_data_iter(void) +{ + DECLARE_LIBBPF_OPTS(bpf_iter_attach_opts, opts); + struct test_global_percpu_data *skel; + union bpf_iter_link_info linfo = {}; + struct bpf_link *link = NULL; + int fd, num_cpus, len, err; + char buf[16]; + + num_cpus = libbpf_num_possible_cpus(); + if (!ASSERT_GT(num_cpus, 0, "libbpf_num_possible_cpus")) + return; + + skel = test_global_percpu_data__open(); + if (!ASSERT_OK_PTR(skel, "test_global_percpu_data__open")) + return; + + skel->rodata->num_cpus = num_cpus; + skel->rodata->offsetof_num = offsetof(struct test_global_percpu_data__percpu, struct_data); + skel->rodata->offsetof_num += sizeof(skel->percpu->struct_data) - sizeof(int); + skel->rodata->elem_sz = roundup(sizeof(struct test_global_percpu_data__percpu), 8); + skel->percpu->struct_data.nums[6] = 0xc0de; + + err = test_global_percpu_data__load(skel); + if (!ASSERT_OK(err, "test_global_percpu_data__load")) + goto out; + + linfo.map.map_fd = bpf_map__fd(skel->maps.percpu); + opts.link_info = &linfo; + opts.link_info_len = sizeof(linfo); + link = bpf_program__attach_iter(skel->progs.dump_percpu_data, &opts); + if (!ASSERT_OK_PTR(link, "bpf_program__attach_iter")) + goto out; + + fd = bpf_iter_create(bpf_link__fd(link)); + if (!ASSERT_GE(fd, 0, "bpf_iter_create")) + goto out; + + while ((len = read(fd, buf, sizeof(buf))) > 0) + do { } while (0); + ASSERT_EQ(len, 0, "read iter"); + ASSERT_TRUE(skel->bss->run_iter, "run_iter"); + ASSERT_EQ(skel->bss->percpu_data_sum, 0xc0de * num_cpus, "percpu_data_sum"); + + close(fd); +out: + bpf_link__destroy(link); + test_global_percpu_data__destroy(skel); +} + void test_global_percpu_data(void) { if (!feat_supported(NULL, FEAT_PERCPU_DATA)) { @@ -348,4 +398,6 @@ void test_global_percpu_data(void) test_global_percpu_data_verifier_log(); if (test__start_subtest("xlated")) test_global_percpu_data_xlated(); + if (test__start_subtest("iter")) + test_global_percpu_data_iter(); } diff --git a/tools/testing/selftests/bpf/progs/test_global_percpu_data.c b/tools/testing/selftests/bpf/progs/test_global_percpu_data.c index 9a1b1a314c2f..037c8a05be4b 100644 --- a/tools/testing/selftests/bpf/progs/test_global_percpu_data.c +++ b/tools/testing/selftests/bpf/progs/test_global_percpu_data.c @@ -62,4 +62,29 @@ int verifier_percpu_read(void *ctx) return c == ' '; } +volatile const __u32 num_cpus = 0; +volatile const int offsetof_num; +volatile const int elem_sz; +__u32 percpu_data_sum = 0; +bool run_iter = false; + +SEC("iter/bpf_map_elem") +__auxiliary +int dump_percpu_data(struct bpf_iter__bpf_map_elem *ctx) +{ + void *pptr = ctx->value; + int i; + + if (!pptr) + return 0; + + run_iter = true; + + for (i = 0; i < num_cpus; i++) { + percpu_data_sum += *(int *) (pptr + offsetof_num); + pptr += elem_sz; + } + return 0; +} + char _license[] SEC("license") = "GPL"; -- 2.54.0