Parse the compiler-emitted .bpf_cleanup section and hand the resulting table to BPF_PROG_LOAD. Each record is three 4-byte fields, and each field is a byte offset into some code section named by a matching .rel.bpf_cleanup relocation. bpf_object__init_cleanup_info() resolves both halves once at open time and keeps (section, instruction index) pairs; it rejects a record whose field has no relocation, whose relocation is not one of the two 32-bit types that spell a data reference to a code section -- R_BPF_64_NODYLD32 from LLVM, R_BPF_64_ABS32 from GNU as -- or whose offset is not instruction aligned. The records are sorted by begin_off once the offsets are final: the kernel wants the table sorted with disjoint ranges so that it can find the record covering a call site with a binary search, and records arrive in .bpf_cleanup order, which says nothing about where the subprograms they describe were appended. Overlapping ranges are reported here, where the program name and both regions are still at hand. bpf_object_load_prog() then passes the per-program table through the bpf_prog_load() options added in the previous patch, with the record size carried on the program the way func_info and line_info carry theirs rather than taken from a sizeof() at the call site. bpf_program__clone() carries it too. That is a second load path -- the one veristat uses -- and without the table the kernel sees landing pads nothing reaches and refuses the program with "unreachable insn". Signed-off-by: Yonghong Song --- tools/lib/bpf/libbpf.c | 309 ++++++++++++++++++++++++++++++++ tools/lib/bpf/libbpf_internal.h | 3 + 2 files changed, 312 insertions(+) diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c index 2d0081784445..d0e601b3499c 100644 --- a/tools/lib/bpf/libbpf.c +++ b/tools/lib/bpf/libbpf.c @@ -514,6 +514,11 @@ struct bpf_program { void *line_info; __u32 line_info_rec_size; __u32 line_info_cnt; + + struct bpf_cleanup_info *cleanup_info; + __u32 cleanup_info_rec_size; + __u32 cleanup_info_cnt; + __u32 prog_flags; __u8 hash[SHA256_DIGEST_LENGTH]; @@ -549,6 +554,7 @@ struct bpf_struct_ops { #define STRUCT_OPS_SEC ".struct_ops" #define STRUCT_OPS_LINK_SEC ".struct_ops.link" #define ARENA_SEC ".addr_space.1" +#define CLEANUP_SEC ".bpf_cleanup" enum libbpf_map_type { LIBBPF_MAP_UNSPEC, @@ -677,6 +683,26 @@ struct elf_sec_desc { Elf_Data *data; }; +#define CLEANUP_REC_FIELDS (sizeof(struct bpf_cleanup_info) / sizeof(__u32)) + +/* Index of each field of struct bpf_cleanup_info, read as an array of __u32. */ +enum { + CLEANUP_REC_BEGIN, + CLEANUP_REC_END, + CLEANUP_REC_PAD, +}; + +/* + * One (begin, end, landing_pad) triple from .bpf_cleanup, with each field + * resolved from its relocation to an ELF section plus a section-relative + * instruction index. The mapping to final program instruction indices can only + * happen after subprogram placement, which differs per main program. + */ +struct cleanup_raw_rec { + int sec_idx[CLEANUP_REC_FIELDS]; + size_t insn_idx[CLEANUP_REC_FIELDS]; +}; + struct elf_state { int fd; const void *obj_buf; @@ -696,6 +722,8 @@ struct elf_state { bool has_st_ops; int arena_data_shndx; int jumptables_data_shndx; + Elf_Data *cleanup_data; + int cleanup_shndx; }; struct usdt_manager; @@ -773,6 +801,9 @@ struct bpf_object { void *jumptables_data; size_t jumptables_data_sz; + struct cleanup_raw_rec *cleanup_recs; + size_t cleanup_rec_cnt; + struct { struct bpf_program *prog; unsigned int sym_off; @@ -819,7 +850,10 @@ static void bpf_program__exit(struct bpf_program *prog) zfree(&prog->sec_name); zfree(&prog->insns); zfree(&prog->reloc_desc); + zfree(&prog->cleanup_info); + prog->cleanup_info_rec_size = 0; + prog->cleanup_info_cnt = 0; prog->nr_reloc = 0; prog->insns_cnt = 0; prog->sec_idx = -1; @@ -1561,6 +1595,7 @@ static struct bpf_object *bpf_object__new(const char *path, obj->efile.obj_buf = obj_buf; obj->efile.obj_buf_sz = obj_buf_sz; obj->efile.btf_maps_shndx = -1; + obj->efile.cleanup_shndx = -1; obj->kconfig_map_idx = -1; obj->arena_map_idx = -1; @@ -4047,6 +4082,9 @@ static int bpf_object__elf_collect(struct bpf_object *obj) sec_desc->shdr = sh; sec_desc->data = data; obj->efile.has_st_ops = true; + } else if (strcmp(name, CLEANUP_SEC) == 0) { + obj->efile.cleanup_data = data; + obj->efile.cleanup_shndx = idx; } else if (strcmp(name, ARENA_SEC) == 0) { obj->efile.arena_data = data; obj->efile.arena_data_shndx = idx; @@ -4074,6 +4112,7 @@ static int bpf_object__elf_collect(struct bpf_object *obj) strcmp(name, ".rel" STRUCT_OPS_LINK_SEC) && strcmp(name, ".rel?" STRUCT_OPS_SEC) && strcmp(name, ".rel?" STRUCT_OPS_LINK_SEC) && + strcmp(name, ".rel" CLEANUP_SEC) && strcmp(name, ".rel" MAPS_ELF_SEC)) { pr_info("elf: skipping relo section(%d) %s for section(%d) %s\n", idx, name, targ_sec_idx, @@ -4854,6 +4893,238 @@ static struct bpf_program *find_prog_by_sec_insn(const struct bpf_object *obj, return NULL; } +static int bpf_object__init_cleanup_info(struct bpf_object *obj) +{ + Elf_Data *data = obj->efile.cleanup_data; + Elf_Data *relo = NULL; + size_t i, nrels, nslots, nrecs; + struct cleanup_raw_rec *recs; + int *slot_sec, ret = 0; + size_t *slot_val; + const __u32 *vals; + bool native; + + if (!data || obj->efile.cleanup_shndx < 0) + return 0; + + native = is_native_endianness(obj); + + for (i = 0; i < obj->efile.sec_cnt; i++) { + struct elf_sec_desc *sd = &obj->efile.secs[i]; + + if (sd->sec_type == SEC_RELO && sd->shdr && + sd->shdr->sh_info == (Elf64_Word)obj->efile.cleanup_shndx) { + relo = sd->data; + break; + } + } + if (!relo) { + pr_warn("%s present without relocations\n", CLEANUP_SEC); + return -LIBBPF_ERRNO__FORMAT; + } + if (data->d_size % sizeof(struct bpf_cleanup_info)) { + pr_warn("%s size %zu is not a multiple of the record size %zu\n", + CLEANUP_SEC, data->d_size, sizeof(struct bpf_cleanup_info)); + return -LIBBPF_ERRNO__FORMAT; + } + + vals = data->d_buf; + nslots = data->d_size / sizeof(__u32); + nrecs = data->d_size / sizeof(struct bpf_cleanup_info); + + slot_sec = calloc(nslots, sizeof(*slot_sec)); + slot_val = calloc(nslots, sizeof(*slot_val)); + recs = calloc(nrecs ?: 1, sizeof(*recs)); + if (!slot_sec || !slot_val || !recs) { + ret = -ENOMEM; + goto out; + } + for (i = 0; i < nslots; i++) + slot_sec[i] = -1; + + /* One relocation per 4-byte field, naming the section it points into. */ + nrels = relo->d_size / sizeof(Elf64_Rel); + for (i = 0; i < nrels; i++) { + Elf64_Rel *rel = elf_rel_by_idx(relo, i); + Elf64_Sym *sym = elf_sym_by_idx(obj, ELF64_R_SYM(rel->r_info)); + size_t type = ELF64_R_TYPE(rel->r_info); + size_t slot = rel->r_offset / sizeof(__u32); + + if (type != R_BPF_64_NODYLD32 && type != R_BPF_64_ABS32) { + pr_warn("%s: relocation %zu has unexpected type %zu\n", + CLEANUP_SEC, i, type); + ret = -LIBBPF_ERRNO__FORMAT; + goto out; + } + if (!sym || slot >= nslots || rel->r_offset % sizeof(__u32)) { + pr_warn("%s: bad relocation %zu\n", CLEANUP_SEC, i); + ret = -LIBBPF_ERRNO__FORMAT; + goto out; + } + slot_sec[slot] = sym->st_shndx; + /* + * The addend lives in the section data, which libelf leaves in + * the object's byte order; a non-section symbol additionally + * contributes its own value. + */ + slot_val[slot] = (native ? vals[slot] : bswap_32(vals[slot])) + + sym->st_value; + } + + for (i = 0; i < nslots; i++) { + struct cleanup_raw_rec *rec = &recs[i / CLEANUP_REC_FIELDS]; + size_t field = i % CLEANUP_REC_FIELDS; + + if (slot_sec[i] < 0) { + pr_warn("%s: field %zu has no relocation\n", CLEANUP_SEC, i); + ret = -LIBBPF_ERRNO__FORMAT; + goto out; + } + if (slot_val[i] % BPF_INSN_SZ) { + pr_warn("%s: field %zu offset %zu is not instruction aligned\n", + CLEANUP_SEC, i, slot_val[i]); + ret = -LIBBPF_ERRNO__FORMAT; + goto out; + } + rec->sec_idx[field] = slot_sec[i]; + rec->insn_idx[field] = slot_val[i] / BPF_INSN_SZ; + } + + obj->cleanup_recs = recs; + obj->cleanup_rec_cnt = nrecs; + recs = NULL; +out: + free(recs); + free(slot_val); + free(slot_sec); + return ret; +} + +static int cmp_cleanup_info(const void *a, const void *b) +{ + const struct bpf_cleanup_info *x = a, *y = b; + + if (x->begin_off == y->begin_off) + return 0; + return x->begin_off < y->begin_off ? -1 : 1; +} + +static int bpf_prog_collect_cleanup_info(struct bpf_object *obj, + struct bpf_program *prog) +{ + size_t i; + int j; + + for (i = 0; i < obj->cleanup_rec_cnt; i++) { + struct cleanup_raw_rec *raw = &obj->cleanup_recs[i]; + struct bpf_program *owner = NULL; + struct bpf_cleanup_info ci = {}; + __u32 fields[CLEANUP_REC_FIELDS]; + void *tmp; + + if (raw->sec_idx[CLEANUP_REC_BEGIN] == raw->sec_idx[CLEANUP_REC_END] && + raw->insn_idx[CLEANUP_REC_BEGIN] >= raw->insn_idx[CLEANUP_REC_END]) { + pr_warn("%s: record %zu is an empty range [%zu,%zu)\n", + CLEANUP_SEC, i, raw->insn_idx[CLEANUP_REC_BEGIN], + raw->insn_idx[CLEANUP_REC_END]); + return -LIBBPF_ERRNO__FORMAT; + } + + for (j = 0; j < CLEANUP_REC_FIELDS; j++) { + size_t idx = raw->insn_idx[j], final; + struct bpf_program *p; + + /* + * The end of a range is exclusive, so it may name the + * instruction just past the last one of a function, + * which belongs to the next function or to nothing at + * all. Ask about the last instruction the range covers, + * the way the kernel does. + */ + if (j == CLEANUP_REC_END) { + if (!idx) { + pr_warn("%s: record %zu ends at instruction 0\n", + CLEANUP_SEC, i); + return -LIBBPF_ERRNO__FORMAT; + } + idx--; + } + + p = find_prog_by_sec_insn(obj, raw->sec_idx[j], idx); + if (!p) { + pr_warn("%s: record %zu field %d is not inside a function\n", + CLEANUP_SEC, i, j); + return -LIBBPF_ERRNO__FORMAT; + } + if (!owner) { + owner = p; + } else if (owner != p) { + pr_warn("%s: record %zu spans functions '%s' and '%s'\n", + CLEANUP_SEC, i, owner->name, p->name); + return -LIBBPF_ERRNO__FORMAT; + } + + if (owner == prog) { + final = raw->insn_idx[j] - prog->sec_insn_off; + } else if (prog_is_subprog(obj, owner) && owner->sub_insn_off) { + /* + * sub_insn_off is where this subprogram was + * appended to the main program being relocated; + * zero means it is not part of it. + */ + final = owner->sub_insn_off + + raw->insn_idx[j] - owner->sec_insn_off; + } else { + owner = NULL; + break; + } + fields[j] = final; + } + if (!owner) + continue; + + ci.begin_off = fields[CLEANUP_REC_BEGIN]; + ci.end_off = fields[CLEANUP_REC_END]; + ci.landing_pad_off = fields[CLEANUP_REC_PAD]; + + tmp = libbpf_reallocarray(prog->cleanup_info, prog->cleanup_info_cnt + 1, + sizeof(*prog->cleanup_info)); + if (!tmp) + return -ENOMEM; + prog->cleanup_info = tmp; + prog->cleanup_info_rec_size = sizeof(struct bpf_cleanup_info); + prog->cleanup_info[prog->cleanup_info_cnt++] = ci; + + pr_debug("prog '%s': cleanup region [%u,%u) -> landing pad %u\n", + prog->name, ci.begin_off, ci.end_off, ci.landing_pad_off); + } + + if (!prog->cleanup_info_cnt) + return 0; + + if (prog->cleanup_info_cnt > INT32_MAX / sizeof(struct bpf_cleanup_info)) { + pr_warn("prog '%s': too many cleanup records: %u\n", + prog->name, prog->cleanup_info_cnt); + return -LIBBPF_ERRNO__FORMAT; + } + + qsort(prog->cleanup_info, prog->cleanup_info_cnt, + sizeof(*prog->cleanup_info), cmp_cleanup_info); + for (i = 1; i < prog->cleanup_info_cnt; i++) { + struct bpf_cleanup_info *prev = &prog->cleanup_info[i - 1]; + struct bpf_cleanup_info *cur = &prog->cleanup_info[i]; + + if (cur->begin_off < prev->end_off) { + pr_warn("prog '%s': overlapping cleanup regions [%u,%u) and [%u,%u)\n", + prog->name, prev->begin_off, prev->end_off, + cur->begin_off, cur->end_off); + return -LIBBPF_ERRNO__FORMAT; + } + } + + return 0; +} + static int bpf_object__collect_prog_relos(struct bpf_object *obj, Elf64_Shdr *shdr, Elf_Data *data) { @@ -7589,6 +7860,13 @@ static int bpf_object__relocate(struct bpf_object *obj, const char *targ_btf_pat return err; } } + + err = bpf_prog_collect_cleanup_info(obj, prog); + if (err) { + pr_warn("prog '%s': failed to collect cleanup info: %s\n", + prog->name, errstr(err)); + return err; + } } for (i = 0; i < obj->nr_programs; i++) { prog = &obj->programs[i]; @@ -7779,6 +8057,9 @@ static int bpf_object__collect_relos(struct bpf_object *obj) return -LIBBPF_ERRNO__INTERNAL; } + if (idx == obj->efile.cleanup_shndx) + continue; + if (obj->efile.secs[idx].sec_type == SEC_ST_OPS) err = bpf_object__collect_st_ops_relos(obj, shdr, data); else if (idx == obj->efile.btf_maps_shndx) @@ -8051,6 +8332,11 @@ static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog load_attr.line_info_rec_size = prog->line_info_rec_size; load_attr.line_info_cnt = prog->line_info_cnt; } + if (prog->cleanup_info_cnt) { + load_attr.cleanup_info = prog->cleanup_info; + load_attr.cleanup_info_cnt = prog->cleanup_info_cnt; + load_attr.cleanup_info_rec_size = prog->cleanup_info_rec_size; + } load_attr.log_level = log_level; load_attr.prog_flags = prog->prog_flags; load_attr.fd_array = obj->fd_array; @@ -8644,6 +8930,7 @@ static struct bpf_object *bpf_object_open(const char *path, const void *obj_buf, err = err ? : bpf_object__init_maps(obj, opts); err = err ? : bpf_object_init_progs(obj, opts); err = err ? : bpf_object__collect_relos(obj); + err = err ? : bpf_object__init_cleanup_info(obj); if (err) goto out; @@ -9758,6 +10045,9 @@ void bpf_object__close(struct bpf_object *obj) zfree(&obj->jumptables_data); obj->jumptables_data_sz = 0; + zfree(&obj->cleanup_recs); + obj->cleanup_rec_cnt = 0; + for (i = 0; i < obj->jumptable_map_cnt; i++) close(obj->jumptable_maps[i].fd); zfree(&obj->jumptable_maps); @@ -10156,6 +10446,25 @@ int bpf_program__clone(struct bpf_program *prog, const struct bpf_prog_load_opts attr.line_info_rec_size = info ? info_rec_size : prog->line_info_rec_size; } + /* exception cleanup table */ + info = OPTS_GET(opts, cleanup_info, NULL); + info_cnt = OPTS_GET(opts, cleanup_info_cnt, 0); + info_rec_size = OPTS_GET(opts, cleanup_info_rec_size, 0); + if (!!info != !!info_cnt || !!info != !!info_rec_size) { + pr_warn("prog '%s': cleanup_info, cleanup_info_cnt, and cleanup_info_rec_size must all be specified or all omitted\n", + prog->name); + return libbpf_err(-EINVAL); + } + if (info) { + attr.cleanup_info = info; + attr.cleanup_info_cnt = info_cnt; + attr.cleanup_info_rec_size = info_rec_size; + } else if (prog->cleanup_info_cnt) { + attr.cleanup_info = prog->cleanup_info; + attr.cleanup_info_cnt = prog->cleanup_info_cnt; + attr.cleanup_info_rec_size = prog->cleanup_info_rec_size; + } + /* Logging is caller-controlled; no fallback to prog/obj log settings */ attr.log_buf = OPTS_GET(opts, log_buf, NULL); attr.log_size = OPTS_GET(opts, log_size, 0); diff --git a/tools/lib/bpf/libbpf_internal.h b/tools/lib/bpf/libbpf_internal.h index cb4d96233844..3ba6d9090368 100644 --- a/tools/lib/bpf/libbpf_internal.h +++ b/tools/lib/bpf/libbpf_internal.h @@ -56,6 +56,9 @@ #ifndef R_BPF_64_ABS32 #define R_BPF_64_ABS32 3 #endif +#ifndef R_BPF_64_NODYLD32 +#define R_BPF_64_NODYLD32 4 +#endif #ifndef R_BPF_64_32 #define R_BPF_64_32 10 #endif -- 2.53.0-Meta