Parse the compiler-emitted .bpf_cleanup section and hand the table to BPF_PROG_LOAD: - At open, each record field, a byte offset into a code section, is resolved through its .rel.bpf_cleanup relocation: R_BPF_64_NODYLD32 from LLVM or R_BPF_64_ABS32 from GNU as. - At relocation, once a main program's subprogs are appended, the records in the main program and in those subprogs get final instruction indices and are sorted by begin_off, as the kernel wants. A record in code no function claims belongs to a weak function the static linker kept after overriding it, and is skipped, as that code's relocations are. - At load, the table goes through the bpf_prog_load() options, from bpf_object_load_prog() and from bpf_program__clone(), the path veristat uses. Signed-off-by: Yonghong Song --- tools/lib/bpf/libbpf.c | 287 +++++++++++++++++++++++++++++++- tools/lib/bpf/libbpf_internal.h | 3 + 2 files changed, 288 insertions(+), 2 deletions(-) diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c index b3cfd7c58776..d227dcc7ef0b 100644 --- a/tools/lib/bpf/libbpf.c +++ b/tools/lib/bpf/libbpf.c @@ -516,6 +516,11 @@ struct bpf_program { void *line_info; __u32 line_info_rec_size; __u32 line_info_cnt; + + struct bpf_cleanup_info *cleanup_info; + __u32 cleanup_info_rec_size; + __u32 cleanup_info_cnt; + __u32 prog_flags; __u8 hash[SHA256_DIGEST_LENGTH]; @@ -572,6 +577,7 @@ struct bpf_struct_ops { * are loaded with BPF_F_ARENA_SCALAR. */ #define ARENA_DATA_SEC ".arena.data" +#define CLEANUP_SEC ".bpf_cleanup" enum libbpf_map_type { LIBBPF_MAP_UNSPEC, @@ -710,6 +716,25 @@ struct elf_sec_desc { Elf_Data *data; }; +#define CLEANUP_REC_FIELDS (sizeof(struct bpf_cleanup_info) / sizeof(__u32)) + +/* Index of each field of struct bpf_cleanup_info, read as an array of __u32. */ +enum { + CLEANUP_REC_BEGIN, + CLEANUP_REC_END, + CLEANUP_REC_PAD, +}; + +/* + * One (begin, end, landing_pad) triple from .bpf_cleanup, each field resolved + * from its relocation to a section and an instruction index within it. Final + * indices wait for subprogram placement, which differs per main program. + */ +struct cleanup_raw_rec { + int sec_idx[CLEANUP_REC_FIELDS]; + size_t insn_idx[CLEANUP_REC_FIELDS]; +}; + struct elf_state { int fd; const void *obj_buf; @@ -729,6 +754,8 @@ struct elf_state { bool has_st_ops; int arena_data_shndx; int jumptables_data_shndx; + Elf_Data *cleanup_data; + int cleanup_shndx; }; struct usdt_manager; @@ -817,6 +844,9 @@ struct bpf_object { void *jumptables_data; size_t jumptables_data_sz; + struct cleanup_raw_rec *cleanup_recs; + size_t cleanup_rec_cnt; + struct { struct bpf_program *prog; unsigned int sym_off; @@ -874,6 +904,7 @@ void bpf_program__unload(struct bpf_program *prog) zfree(&prog->func_info); zfree(&prog->line_info); zfree(&prog->subprogs); + zfree(&prog->cleanup_info); } static void bpf_program__exit(struct bpf_program *prog) @@ -1639,6 +1670,7 @@ static struct bpf_object *bpf_object__new(const char *path, obj->efile.obj_buf_sz = obj_buf_sz; obj->efile.btf_maps_shndx = -1; obj->efile.arena_data_shndx = -1; + obj->efile.cleanup_shndx = -1; obj->kconfig_map_idx = -1; obj->arena_map_idx = -1; @@ -1658,6 +1690,7 @@ static void bpf_object__elf_finish(struct bpf_object *obj) obj->efile.ehdr = NULL; obj->efile.symbols = NULL; obj->efile.arena_data = NULL; + obj->efile.cleanup_data = NULL; zfree(&obj->efile.secs); obj->efile.sec_cnt = 0; @@ -4275,6 +4308,9 @@ static int bpf_object__elf_collect(struct bpf_object *obj) sec_desc->shdr = sh; sec_desc->data = data; obj->efile.has_st_ops = true; + } else if (strcmp(name, CLEANUP_SEC) == 0) { + obj->efile.cleanup_data = data; + obj->efile.cleanup_shndx = idx; } else if (strcmp(name, ARENA_SEC) == 0) { obj->efile.arena_data = data; obj->efile.arena_data_shndx = idx; @@ -4300,8 +4336,9 @@ static int bpf_object__elf_collect(struct bpf_object *obj) /* * Only do relo for section with exec instructions, - * struct_ops, maps, and read-only data that might - * have pointers to functions. + * struct_ops, maps, read-only data that might have + * pointers to functions, and the exception cleanup + * table. */ if (!section_have_execinstr(obj, targ_sec_idx) && !(obj->data_in_arena && @@ -4315,6 +4352,7 @@ static int bpf_object__elf_collect(struct bpf_object *obj) strcmp(name, ".rel" STRUCT_OPS_LINK_SEC) && strcmp(name, ".rel?" STRUCT_OPS_SEC) && strcmp(name, ".rel?" STRUCT_OPS_LINK_SEC) && + strcmp(name, ".rel" CLEANUP_SEC) && strcmp(name, ".rel" MAPS_ELF_SEC)) { pr_info("elf: skipping relo section(%d) %s for section(%d) %s\n", idx, name, targ_sec_idx, @@ -5099,6 +5137,219 @@ static struct bpf_program *find_prog_by_sec_insn(const struct bpf_object *obj, return NULL; } +static int bpf_object_init_cleanup_info(struct bpf_object *obj) +{ + Elf_Data *data = obj->efile.cleanup_data; + size_t i, nrels, nslots, nrecs; + struct cleanup_raw_rec *recs; + Elf_Data *relo = NULL; + const char *vals; + int ret = 0; + bool native; + + if (!data || obj->efile.cleanup_shndx < 0 || !data->d_size) + return 0; + + native = is_native_endianness(obj); + + for (i = 0; i < obj->efile.sec_cnt; i++) { + struct elf_sec_desc *sd = &obj->efile.secs[i]; + + if (sd->sec_type == SEC_RELO && sd->shdr && + sd->shdr->sh_info == (Elf64_Word)obj->efile.cleanup_shndx) { + relo = sd->data; + break; + } + } + if (!relo) { + pr_warn("%s present without relocations\n", CLEANUP_SEC); + return -LIBBPF_ERRNO__FORMAT; + } + if (data->d_size % sizeof(struct bpf_cleanup_info)) { + pr_warn("%s size %zu is not a multiple of the record size %zu\n", + CLEANUP_SEC, data->d_size, sizeof(struct bpf_cleanup_info)); + return -LIBBPF_ERRNO__FORMAT; + } + + vals = data->d_buf; + nslots = data->d_size / sizeof(__u32); + nrecs = data->d_size / sizeof(struct bpf_cleanup_info); + + recs = calloc(nrecs, sizeof(*recs)); + if (!recs) + return -ENOMEM; + for (i = 0; i < nslots; i++) + recs[i / CLEANUP_REC_FIELDS].sec_idx[i % CLEANUP_REC_FIELDS] = -1; + + /* One relocation per 4-byte field, naming the section it points into. */ + nrels = relo->d_size / sizeof(Elf64_Rel); + for (i = 0; i < nrels; i++) { + Elf64_Rel *rel = elf_rel_by_idx(relo, i); + Elf64_Sym *sym = elf_sym_by_idx(obj, ELF64_R_SYM(rel->r_info)); + size_t type = ELF64_R_TYPE(rel->r_info); + size_t slot = rel->r_offset / sizeof(__u32); + struct cleanup_raw_rec *rec; + size_t off; + __u32 val; + + if (type != R_BPF_64_NODYLD32 && type != R_BPF_64_ABS32) { + pr_warn("%s: relocation %zu has unexpected type %zu\n", + CLEANUP_SEC, i, type); + ret = -LIBBPF_ERRNO__FORMAT; + goto out; + } + if (!sym || slot >= nslots || rel->r_offset % sizeof(__u32)) { + pr_warn("%s: bad relocation %zu\n", CLEANUP_SEC, i); + ret = -LIBBPF_ERRNO__FORMAT; + goto out; + } + /* + * The addend lives in the section data, which libelf leaves in + * the object's byte order; a non-section symbol additionally + * contributes its own value. The section is only byte aligned. + */ + memcpy(&val, vals + slot * sizeof(val), sizeof(val)); + off = (native ? val : bswap_32(val)) + sym->st_value; + if (off % BPF_INSN_SZ) { + pr_warn("%s: field %zu offset %zu is not instruction aligned\n", + CLEANUP_SEC, slot, off); + ret = -LIBBPF_ERRNO__FORMAT; + goto out; + } + rec = &recs[slot / CLEANUP_REC_FIELDS]; + if (rec->sec_idx[slot % CLEANUP_REC_FIELDS] >= 0) { + pr_warn("%s: field %zu has more than one relocation\n", + CLEANUP_SEC, slot); + ret = -LIBBPF_ERRNO__FORMAT; + goto out; + } + rec->sec_idx[slot % CLEANUP_REC_FIELDS] = sym->st_shndx; + rec->insn_idx[slot % CLEANUP_REC_FIELDS] = off / BPF_INSN_SZ; + } + + for (i = 0; i < nslots; i++) { + if (recs[i / CLEANUP_REC_FIELDS].sec_idx[i % CLEANUP_REC_FIELDS] < 0) { + pr_warn("%s: field %zu has no relocation\n", CLEANUP_SEC, i); + ret = -LIBBPF_ERRNO__FORMAT; + goto out; + } + } + + obj->cleanup_recs = recs; + obj->cleanup_rec_cnt = nrecs; + return 0; +out: + free(recs); + return ret; +} + +static int cmp_cleanup_info(const void *a, const void *b) +{ + const struct bpf_cleanup_info *x = a, *y = b; + + if (x->begin_off == y->begin_off) + return 0; + return x->begin_off < y->begin_off ? -1 : 1; +} + +static int bpf_prog_collect_cleanup_info(struct bpf_object *obj, + struct bpf_program *prog) +{ + size_t i; + int j; + + for (i = 0; i < obj->cleanup_rec_cnt; i++) { + struct cleanup_raw_rec *raw = &obj->cleanup_recs[i]; + struct bpf_program *owner = NULL; + struct bpf_cleanup_info ci = {}; + __u32 fields[CLEANUP_REC_FIELDS]; + void *tmp; + + for (j = 0; j < CLEANUP_REC_FIELDS; j++) { + size_t idx = raw->insn_idx[j], final; + struct bpf_program *p; + + /* + * An exclusive end may name the instruction past the + * last of a function, so ask about the last one the + * range covers, the way the kernel does. + */ + if (j == CLEANUP_REC_END) { + if (!idx) { + pr_warn("%s: record %zu ends at instruction 0\n", + CLEANUP_SEC, i); + return -LIBBPF_ERRNO__FORMAT; + } + idx--; + } + + p = find_prog_by_sec_insn(obj, raw->sec_idx[j], idx); + if (!p && j == CLEANUP_REC_BEGIN) { + /* + * Code no function claims is an overridden weak + * function the linker kept: skip its records, + * as its relocations are. + */ + pr_debug("%s: record %zu is in no function, probably an overridden weak function, skipping\n", + CLEANUP_SEC, i); + break; + } + if (!p) { + pr_warn("%s: record %zu field %d is not inside a function\n", + CLEANUP_SEC, i, j); + return -LIBBPF_ERRNO__FORMAT; + } + if (!owner) { + owner = p; + } else if (owner != p) { + pr_warn("%s: record %zu spans functions '%s' and '%s'\n", + CLEANUP_SEC, i, owner->name, p->name); + return -LIBBPF_ERRNO__FORMAT; + } + + if (owner == prog) { + final = raw->insn_idx[j] - prog->sec_insn_off; + } else if (prog_is_subprog(obj, owner) && owner->sub_insn_off) { + /* + * sub_insn_off is where this subprogram was + * appended to the main program being relocated; + * zero means it is not part of it. + */ + final = owner->sub_insn_off + + raw->insn_idx[j] - owner->sec_insn_off; + } else { + owner = NULL; + break; + } + fields[j] = final; + } + if (!owner) + continue; + + ci.begin_off = fields[CLEANUP_REC_BEGIN]; + ci.end_off = fields[CLEANUP_REC_END]; + ci.landing_pad_off = fields[CLEANUP_REC_PAD]; + + tmp = libbpf_reallocarray(prog->cleanup_info, prog->cleanup_info_cnt + 1, + sizeof(*prog->cleanup_info)); + if (!tmp) + return -ENOMEM; + prog->cleanup_info = tmp; + prog->cleanup_info_rec_size = sizeof(struct bpf_cleanup_info); + prog->cleanup_info[prog->cleanup_info_cnt++] = ci; + + pr_debug("prog '%s': cleanup region [%u,%u) -> landing pad %u\n", + prog->name, ci.begin_off, ci.end_off, ci.landing_pad_off); + } + + if (!prog->cleanup_info_cnt) + return 0; + + qsort(prog->cleanup_info, prog->cleanup_info_cnt, + sizeof(*prog->cleanup_info), cmp_cleanup_info); + return 0; +} + static int bpf_object__collect_prog_relos(struct bpf_object *obj, Elf64_Shdr *shdr, Elf_Data *data) { @@ -8197,6 +8448,13 @@ static int bpf_object__relocate(struct bpf_object *obj, const char *targ_btf_pat return err; } } + + err = bpf_prog_collect_cleanup_info(obj, prog); + if (err) { + pr_warn("prog '%s': failed to collect cleanup info: %s\n", + prog->name, errstr(err)); + return err; + } } for (i = 0; i < obj->nr_programs; i++) { prog = &obj->programs[i]; @@ -8561,6 +8819,9 @@ static int bpf_object__collect_relos(struct bpf_object *obj) return -LIBBPF_ERRNO__INTERNAL; } + if (idx == obj->efile.cleanup_shndx) + continue; + if (obj->efile.secs[idx].sec_type == SEC_RODATA) err = bpf_object__collect_rodata_relos(obj, shdr, data); else if (obj->efile.secs[idx].sec_type == SEC_DATA) @@ -8856,6 +9117,11 @@ static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog load_attr.line_info_rec_size = prog->line_info_rec_size; load_attr.line_info_cnt = prog->line_info_cnt; } + if (prog->cleanup_info_cnt) { + load_attr.cleanup_info = prog->cleanup_info; + load_attr.cleanup_info_cnt = prog->cleanup_info_cnt; + load_attr.cleanup_info_rec_size = prog->cleanup_info_rec_size; + } load_attr.log_level = log_level; load_attr.prog_flags = prog->prog_flags; /* the program accesses its data in arena through plain numbers */ @@ -9460,6 +9726,7 @@ static struct bpf_object *bpf_object_open(const char *path, const void *obj_buf, err = err ? : bpf_object__init_maps(obj, opts); err = err ? : bpf_object_init_progs(obj, opts); err = err ? : bpf_object__collect_relos(obj); + err = err ? : bpf_object_init_cleanup_info(obj); if (err) goto out; @@ -10602,6 +10869,9 @@ void bpf_object__close(struct bpf_object *obj) zfree(&obj->jumptables_data); obj->jumptables_data_sz = 0; + zfree(&obj->cleanup_recs); + obj->cleanup_rec_cnt = 0; + for (i = 0; i < obj->jumptable_map_cnt; i++) close(obj->jumptable_maps[i].fd); zfree(&obj->jumptable_maps); @@ -11011,6 +11281,19 @@ int bpf_program__clone(struct bpf_program *prog, const struct bpf_prog_load_opts attr.line_info_rec_size = info ? info_rec_size : prog->line_info_rec_size; } + /* exception cleanup table */ + info = OPTS_GET(opts, cleanup_info, NULL); + info_cnt = OPTS_GET(opts, cleanup_info_cnt, 0); + info_rec_size = OPTS_GET(opts, cleanup_info_rec_size, 0); + if (!!info != !!info_cnt || !!info != !!info_rec_size) { + pr_warn("prog '%s': cleanup_info, cleanup_info_cnt, and cleanup_info_rec_size must all be specified or all omitted\n", + prog->name); + return libbpf_err(-EINVAL); + } + attr.cleanup_info = info ?: prog->cleanup_info; + attr.cleanup_info_cnt = info ? info_cnt : prog->cleanup_info_cnt; + attr.cleanup_info_rec_size = info ? info_rec_size : prog->cleanup_info_rec_size; + /* Logging is caller-controlled; no fallback to prog/obj log settings */ attr.log_buf = OPTS_GET(opts, log_buf, NULL); attr.log_size = OPTS_GET(opts, log_size, 0); diff --git a/tools/lib/bpf/libbpf_internal.h b/tools/lib/bpf/libbpf_internal.h index 4e45dfacb548..ecbdb0d7dee1 100644 --- a/tools/lib/bpf/libbpf_internal.h +++ b/tools/lib/bpf/libbpf_internal.h @@ -56,6 +56,9 @@ #ifndef R_BPF_64_ABS32 #define R_BPF_64_ABS32 3 #endif +#ifndef R_BPF_64_NODYLD32 +#define R_BPF_64_NODYLD32 4 +#endif #ifndef R_BPF_64_32 #define R_BPF_64_32 10 #endif -- 2.53.0-Meta