For a BPF_CORE_ENUMVAL_VALUE relocation, bpf_core_calc_enumval_relo() reads the value of a 32-bit enumerator from btf_enum->val, which is __s32, so the value is sign-extended to 64 bits even if the enum is unsigned. A value in [0x80000000, 0xffffffff] of an unsigned enum becomes 0xffffffff80000000 or above. Clang emits such a value zero-extended in the ld_imm64 of the relocation, so the check of the original value fails, and the whole object is refused even if the target value is the same: enum dentry_flags { DCACHE_PRIVATE = 0x80000000 }; out = bpf_core_enum_value(enum dentry_flags, DCACHE_PRIVATE); libbpf: prog 'p': relo #0: unexpected insn #0 (LDIMM64) value: got 2147483648, exp 18446744071562067968 -> 18446744071562067968 libbpf: prog 'p': relo #0: failed to patch insn #0: -EINVAL And a target value in that range is patched sign-extended, which is wrong wherever the program uses it as a 64-bit value. The x86 vmlinux has 29 enums with such values, e.g., dentry_flags, init_ns_ino and e820_type. Commit 89e7a42e3030 ("libbpf: Normalize enum values according to signedness") added btf_enum_value(), which extends the value as given by the kflag of the enum, for btf_dump and map fields, but not for CO-RE relocations, whose relo_core.c is also built into the kernel. Use it in bpf_core_calc_enumval_relo() as well, and add it to the kernel's include/linux/btf.h, so that the kernel gets the same fix. GCC, however, sign-extends every 32-bit enum value, since its __builtin_preserve_enum_value() returns int. For the example above: clang: r1 = 0x80000000 ll gcc: lddw %r1,-2147483648 An object built by GCC is accepted today, because the original value is sign-extended too, and the change above alone would refuse it: libbpf: prog 'p': relo #0: unexpected insn #0 (LDIMM64) value: got 18446744071562067968, exp 2147483648 -> 2147483648 The two forms differ only for an unsigned value in [0x80000000, 0xffffffff]. For such a value, accept the sign-extended form as well. In that case, also sign-extend the target value, since GCC code uses the ld_imm64 as the int value widened to 64 bits. An original value below 0x80000000 is the same in both forms, so the instruction doesn't show which compiler emitted it. The target value then gets the extension given by the kflag of the target enum, which is the form clang emits, as its builtin returns long. A target value that doesn't fit in 32 bits, e.g., from an enum64 in the target kernel, is patched unchanged. The kflag 0 of the target enum doesn't always mean unsigned, though. Before commit 6089fb325cf7 ("bpf: Add btf enum64 support") in v6.0, the kernel refused an enum with the kflag set, so the BTF of an older kernel has the kflag 0 on every enum, signed or not. pahole doesn't record the signedness before v1.24 either, and the kernel still accepts pahole v1.22. On such a kernel, a negative target value, which has been sign-extended so far, would become, e.g., 0xffffffff instead of -1: long v = bpf_core_enum_value(enum perf_event_state, PERF_EVENT_STATE_OFF); The local enum has the kflag 1, so the original value matches the ld_imm64, and nothing would warn. So if the target enum has the kflag 0 but the local enum is signed, take the 32-bit target value as signed too. A target value is then zero-extended only if neither enum is signed. In summary, a 32-bit target value is extended as follows, where kflag 1 is signed: local enum target enum clang GCC ---------- ----------- ----- ---------- kflag 1 kflag 1 sign sign kflag 1 kflag 0 sign sign kflag 0 kflag 1 sign sign kflag 0 kflag 0 zero zero, or sign (*) (*) sign if the local value is in [0x80000000, 0xffffffff], since GCC emits it sign-extended in the ld_imm64 Fixes: 776281652ddc ("libbpf: Permit 64bit relocation value") Signed-off-by: Yonghong Song --- include/linux/btf.h | 5 ++++ tools/lib/bpf/relo_core.c | 59 ++++++++++++++++++++++++++++++++++++--- 2 files changed, 60 insertions(+), 4 deletions(-) diff --git a/include/linux/btf.h b/include/linux/btf.h index 87fad4076f0e..ceb27984df14 100644 --- a/include/linux/btf.h +++ b/include/linux/btf.h @@ -525,6 +525,11 @@ static inline struct btf_enum *btf_enum(const struct btf_type *t) return (struct btf_enum *)(t + 1); } +static inline u64 btf_enum_value(const struct btf_type *t, const struct btf_enum *e) +{ + return btf_type_kflag(t) ? (s64)e->val : (u32)e->val; +} + static inline struct btf_enum64 *btf_enum64(const struct btf_type *t) { return (struct btf_enum64 *)(t + 1); diff --git a/tools/lib/bpf/relo_core.c b/tools/lib/bpf/relo_core.c index 2672623a4198..b83d7b72ab05 100644 --- a/tools/lib/bpf/relo_core.c +++ b/tools/lib/bpf/relo_core.c @@ -13,6 +13,11 @@ static const char *btf_kind_str(const struct btf_type *t) return btf_type_str(t); } +static bool btf_kflag(const struct btf_type *t) +{ + return btf_type_kflag(t); +} + static bool is_ldimm64_insn(struct bpf_insn *insn) { return insn->code == (BPF_LD | BPF_IMM | BPF_DW); @@ -861,6 +866,13 @@ static int bpf_core_calc_type_relo(const struct bpf_core_relo *relo, return 0; } +static __u64 bpf_core_enumval(const struct btf_type *t, int idx) +{ + if (btf_is_enum(t)) + return btf_enum_value(t, btf_enum(t) + idx); + return btf_enum64_value(btf_enum64(t) + idx); +} + static int bpf_core_calc_enumval_relo(const struct bpf_core_relo *relo, const struct bpf_core_spec *spec, __u64 *val) @@ -875,10 +887,7 @@ static int bpf_core_calc_enumval_relo(const struct bpf_core_relo *relo, if (!spec) return -EUCLEAN; /* request instruction poisoning */ t = btf_type_by_id(spec->btf, spec->spec[0].type_id); - if (btf_is_enum(t)) - *val = btf_enum(t)[spec->spec[0].idx].val; - else - *val = btf_enum64_value(btf_enum64(t) + spec->spec[0].idx); + *val = bpf_core_enumval(t, spec->spec[0].idx); break; default: return -EOPNOTSUPP; @@ -887,6 +896,27 @@ static int bpf_core_calc_enumval_relo(const struct bpf_core_relo *relo, return 0; } +/* + * The kflag 0 of a 32-bit enum doesn't always mean unsigned: the BTF of a + * kernel before v6.0, or generated by pahole before v1.24, has the kflag 0 + * on every enum, signed or not. If the local enum is signed, take a 32-bit + * target value with the kflag 0 as signed too, so that, e.g., -1 stays -1 + * on such a kernel. + */ +static bool bpf_core_enumval_targ_is_signed(const struct bpf_core_relo *relo, + const struct bpf_core_spec *local_spec, + const struct bpf_core_spec *targ_spec) +{ + const struct btf_type *local_t, *targ_t; + + if (relo->kind != BPF_CORE_ENUMVAL_VALUE) + return false; + + local_t = btf_type_by_id(local_spec->btf, local_spec->spec[0].type_id); + targ_t = btf_type_by_id(targ_spec->btf, targ_spec->spec[0].type_id); + return btf_is_enum(targ_t) && !btf_kflag(targ_t) && btf_kflag(local_t); +} + /* Calculate original and target relocation values, given local and target * specs and relocation kind. These values are calculated for each candidate. * If there are multiple candidates, resulting values should all be consistent @@ -962,6 +992,8 @@ static int bpf_core_calc_relo(const char *prog_name, } else if (core_relo_is_enumval_based(relo->kind)) { err = bpf_core_calc_enumval_relo(relo, local_spec, &res->orig_val); err = err ?: bpf_core_calc_enumval_relo(relo, targ_spec, &res->new_val); + if (!err && bpf_core_enumval_targ_is_signed(relo, local_spec, targ_spec)) + res->new_val = (__s32)res->new_val; } done: @@ -1028,6 +1060,20 @@ static int insn_bytes_to_bpf_size(__u32 sz) } } +/* + * A 32-bit enum value is in the low 32 bits of the ld_imm64, extended as + * the compiler chooses: clang zero-extends an unsigned value, GCC + * sign-extends every value, as its __builtin_preserve_enum_value() returns + * int. Check whether the instruction has the sign-extended form of an + * unsigned value, so that the target value gets that form too. + */ +static bool bpf_core_enumval_is_sext(const struct bpf_core_relo *relo, + __u64 imm, __u64 orig_val) +{ + return relo->kind == BPF_CORE_ENUMVAL_VALUE && orig_val <= UINT_MAX && + imm != orig_val && imm == (__u64)(__s32)orig_val; +} + /* * Patch relocatable BPF instruction. * @@ -1144,6 +1190,11 @@ int bpf_core_patch_insn(const char *prog_name, struct bpf_insn *insn, return bpf_core_poison_insn(prog_name, relo_idx, insn, insn_idx); imm = (__u32)insn[0].imm | ((__u64)insn[1].imm << 32); + if (res->validate && bpf_core_enumval_is_sext(relo, imm, orig_val)) { + orig_val = imm; + if (new_val <= UINT_MAX) + new_val = (__s32)new_val; + } if (res->validate && imm != orig_val) { pr_warn("prog '%s': relo #%d: unexpected insn #%d (LDIMM64) value: got %llu, exp %llu -> %llu\n", prog_name, relo_idx, -- 2.53.1-Meta Test the extension of unsigned 32-bit enum values in CO-RE relocations from the previous patch. test_core_reloc_enumval gets an optional local enum with values out of the range of int, i.e., an unsigned 4-byte enum, whose values clang emits zero-extended in the ld_imm64 of the relocations. The enumval target has the same enum with different values, 0x80000001 and 0xffffffff, which are expected zero-extended in the 64-bit outputs. GCC's __builtin_preserve_enum_value() returns int, so storing the value of enum named_uint_enum into a long field sign-extends it, and uint_val1 would read back as 0xffffffff80000001. Convert the result to unsigned int for GCC to get the enumerator value. Clang still stores the 64-bit relocated value, so its upper 32 bits keep being checked. Without the fix, the check of the original value fails, and the whole object is refused: libbpf: prog 'test_core_enumval': relo #11: unexpected insn #53 (LDIMM64) value: got 2147483648, exp 18446744071562067968 -> 18446744071562067969 libbpf: prog 'test_core_enumval': relo #11: failed to patch insn #53: -EINVAL #122/71 core_reloc/enumval:FAIL Signed-off-by: Yonghong Song --- .../selftests/bpf/prog_tests/core_reloc.c | 2 ++ .../selftests/bpf/progs/core_reloc_types.h | 8 +++++++ .../bpf/progs/test_core_reloc_enumval.c | 21 +++++++++++++++++++ 3 files changed, 31 insertions(+) diff --git a/tools/testing/selftests/bpf/prog_tests/core_reloc.c b/tools/testing/selftests/bpf/prog_tests/core_reloc.c index 80432c4656b5..98b3c5fb1028 100644 --- a/tools/testing/selftests/bpf/prog_tests/core_reloc.c +++ b/tools/testing/selftests/bpf/prog_tests/core_reloc.c @@ -904,6 +904,8 @@ static const struct core_reloc_test_case test_cases[] = { .named_val2 = 2, .anon_val1 = 0x10, .anon_val2 = 0x20, + .uint_val1 = 0x80000001, + .uint_val2 = 0xffffffff, }), ENUMVAL_CASE(enumval___diff, { .named_val1_exists = true, diff --git a/tools/testing/selftests/bpf/progs/core_reloc_types.h b/tools/testing/selftests/bpf/progs/core_reloc_types.h index 5760ae015e09..6d758e74bd58 100644 --- a/tools/testing/selftests/bpf/progs/core_reloc_types.h +++ b/tools/testing/selftests/bpf/progs/core_reloc_types.h @@ -1215,6 +1215,8 @@ struct core_reloc_enumval_output { int named_val2; int anon_val1; int anon_val2; + long uint_val1; + long uint_val2; }; struct core_reloc_enum64val_output { @@ -1243,9 +1245,15 @@ typedef enum { ANON_ENUM_VAL3 = 0x30, } anon_enum; +enum named_uint_enum { + UINT_ENUM_VAL1 = 0x80000001, + UINT_ENUM_VAL2 = 0xffffffff, +}; + struct core_reloc_enumval { enum named_enum f1; anon_enum f2; + enum named_uint_enum f3; }; enum named_unsigned_enum64 { diff --git a/tools/testing/selftests/bpf/progs/test_core_reloc_enumval.c b/tools/testing/selftests/bpf/progs/test_core_reloc_enumval.c index e7ef3dada2bf..8b8abbaaf533 100644 --- a/tools/testing/selftests/bpf/progs/test_core_reloc_enumval.c +++ b/tools/testing/selftests/bpf/progs/test_core_reloc_enumval.c @@ -27,6 +27,12 @@ typedef enum { ANON_ENUM_VAL3 = 0x30, } anon_enum; +/* 4-byte enum with values out of the range of int, so it is unsigned */ +enum named_uint_enum { + UINT_ENUM_VAL1 = 0x80000000, + UINT_ENUM_VAL2 = 0xfffffff0, +}; + struct core_reloc_enumval_output { bool named_val1_exists; bool named_val2_exists; @@ -39,8 +45,17 @@ struct core_reloc_enumval_output { int named_val2; int anon_val1; int anon_val2; + long uint_val1; + long uint_val2; }; +#ifdef __clang__ +#define UINT_ENUM_VALUE(v) bpf_core_enum_value(enum named_uint_enum, v) +#else +/* GCC's __builtin_preserve_enum_value() returns int */ +#define UINT_ENUM_VALUE(v) ((unsigned int)bpf_core_enum_value(enum named_uint_enum, v)) +#endif + SEC("raw_tracepoint/sys_enter") int test_core_enumval(void *ctx) { @@ -64,6 +79,12 @@ int test_core_enumval(void *ctx) out->anon_val1 = bpf_core_enum_value(anon, ANON_ENUM_VAL1); out->anon_val2 = bpf_core_enum_value(anon, ANON_ENUM_VAL2); /* ANON_ENUM_VAL3 value is optional */ + + /* enum named_uint_enum is optional */ + if (bpf_core_enum_value_exists(enum named_uint_enum, UINT_ENUM_VAL1)) + out->uint_val1 = UINT_ENUM_VALUE(UINT_ENUM_VAL1); + if (bpf_core_enum_value_exists(enum named_uint_enum, UINT_ENUM_VAL2)) + out->uint_val2 = UINT_ENUM_VALUE(UINT_ENUM_VAL2); #else data.skip = true; #endif -- 2.53.1-Meta The libbpf fix takes a 32-bit target enum value with the kflag 0 as signed if the local enum is signed, since the BTF of a kernel before v6.0 has the kflag 0 on every enum, signed or not. Test it. test_core_reloc_enumval gets an optional signed local enum, named_int_enum, with the values -1 and 1, so the enum is signed while one of its values is not negative. The enumval target has the same enum with the values -2 and -3, which are expected sign-extended in the 64-bit outputs. The new test case enumval___no_kflag has the same target types, but its target BTF gets the kflag of every enum cleared, as in the BTF of a kernel before v6.0. The signed enum then gets its signedness from the local enum. The test writes the modified BTF to a temporary file: one it creates itself in core_reloc, or the one btfgen already created in core_reloc_btfgen. The cleanup now removes the temporary file if the test case clears the kflag, not only if btfgen runs. GCC's __builtin_preserve_enum_value() returns int, so with GCC the stores sign-extend the values anyway. Without taking the signedness from the local enum, the upper 32 bits of both values are zero in enumval___no_kflag: run_core_reloc_tests:FAIL:check_result input/output data don't match output byte #44: EXP 0xff GOT 0x00 ... output byte #55: EXP 0xff GOT 0x00 #122/72 core_reloc/enumval___no_kflag:FAIL Signed-off-by: Yonghong Song --- .../selftests/bpf/prog_tests/core_reloc.c | 90 ++++++++++++++++++- .../btf__core_reloc_enumval___no_kflag.c | 3 + .../selftests/bpf/progs/core_reloc_types.h | 16 ++++ .../bpf/progs/test_core_reloc_enumval.c | 14 +++ 4 files changed, 121 insertions(+), 2 deletions(-) create mode 100644 tools/testing/selftests/bpf/progs/btf__core_reloc_enumval___no_kflag.c diff --git a/tools/testing/selftests/bpf/prog_tests/core_reloc.c b/tools/testing/selftests/bpf/prog_tests/core_reloc.c index 98b3c5fb1028..1c502e7f0c60 100644 --- a/tools/testing/selftests/bpf/prog_tests/core_reloc.c +++ b/tools/testing/selftests/bpf/prog_tests/core_reloc.c @@ -370,6 +370,14 @@ static int duration = 0; .fails = true, \ } +#define ENUMVAL_NO_KFLAG_CASE(name, ...) { \ + ENUMVAL_CASE_COMMON(name), \ + .output = STRUCT_TO_CHAR_PTR(core_reloc_enumval_output) \ + __VA_ARGS__, \ + .output_len = sizeof(struct core_reloc_enumval_output), \ + .clear_enum_kflag = true, \ +} + #define ENUM64VAL_CASE_COMMON(name) \ .case_name = #name, \ .bpf_obj_file = "test_core_reloc_enum64val.bpf.o", \ @@ -406,6 +414,7 @@ struct core_reloc_test_case { bool run_btfgen_fails; bool needs_testmod; bool relaxed_core_relocs; + bool clear_enum_kflag; const char *prog_name; const char *raw_tp_name; setup_test_fn setup; @@ -906,6 +915,24 @@ static const struct core_reloc_test_case test_cases[] = { .anon_val2 = 0x20, .uint_val1 = 0x80000001, .uint_val2 = 0xffffffff, + .int_val1 = -2, + .int_val2 = -3, + }), + ENUMVAL_NO_KFLAG_CASE(enumval___no_kflag, { + .named_val1_exists = true, + .named_val2_exists = true, + .named_val3_exists = true, + .anon_val1_exists = true, + .anon_val2_exists = true, + .anon_val3_exists = true, + .named_val1 = 1, + .named_val2 = 2, + .anon_val1 = 0x10, + .anon_val2 = 0x20, + .uint_val1 = 0x80000001, + .uint_val2 = 0xffffffff, + .int_val1 = -2, + .int_val2 = -3, }), ENUMVAL_CASE(enumval___diff, { .named_val1_exists = true, @@ -999,6 +1026,46 @@ static int run_btfgen(const char *src_btf, const char *dst_btf, const char *objp return run_bpftool_command(args); } +/* + * Write the BTF of src to dst with the kflag of every enum cleared, as in the + * BTF of a kernel before v6.0, where the kflag 0 doesn't mean unsigned. + */ +static int write_btf_no_enum_kflag(const char *src, const char *dst) +{ + const void *raw_data; + struct btf_type *t; + struct btf *btf; + __u32 i, n, raw_sz; + FILE *f = NULL; + int err = -1; + + btf = btf__parse(src, NULL); + if (!ASSERT_OK_PTR(btf, "btf_parse")) + return -1; + + n = btf__type_cnt(btf); + for (i = 1; i < n; i++) { + t = (struct btf_type *)btf__type_by_id(btf, i); + if (btf_is_enum(t)) + t->info &= ~(1U << 31); /* the kflag */ + } + + raw_data = btf__raw_data(btf, &raw_sz); + if (!ASSERT_OK_PTR(raw_data, "btf_raw_data")) + goto out; + f = fopen(dst, "w"); + if (!ASSERT_OK_PTR(f, "btf_fopen")) + goto out; + if (!ASSERT_EQ(fwrite(raw_data, 1, raw_sz, f), raw_sz, "btf_write")) + goto out; + err = 0; +out: + if (f) + fclose(f); + btf__free(btf); + return err; +} + static void run_core_reloc_tests(bool use_btfgen) { const size_t mmap_sz = roundup_page(sizeof(struct data)); @@ -1050,6 +1117,25 @@ static void run_core_reloc_tests(bool use_btfgen) test_case->btf_src_file = btf_file; } + /* + * Without btfgen, create the temporary file here; with btfgen, + * rewrite the one btfgen made. + */ + if (test_case->clear_enum_kflag) { + if (!use_btfgen) { + fd = mkstemp(btf_file); + if (!ASSERT_GE(fd, 0, "btf_tmp")) + continue; + close(fd); /* we only need the path */ + } + err = write_btf_no_enum_kflag(test_case->btf_src_file, btf_file); + if (err) { + remove(btf_file); + continue; + } + test_case->btf_src_file = btf_file; + } + if (test_case->setup) { err = test_case->setup(test_case); if (CHECK(err, "test_setup", "test #%d setup failed: %d\n", i, err)) @@ -1141,8 +1227,8 @@ static void run_core_reloc_tests(bool use_btfgen) CHECK_FAIL(munmap(mmap_data, mmap_sz)); mmap_data = NULL; } - if (use_btfgen) - remove(test_case->btf_src_file); + if (use_btfgen || test_case->clear_enum_kflag) + remove(btf_file); bpf_link__destroy(link); link = NULL; bpf_object__close(obj); diff --git a/tools/testing/selftests/bpf/progs/btf__core_reloc_enumval___no_kflag.c b/tools/testing/selftests/bpf/progs/btf__core_reloc_enumval___no_kflag.c new file mode 100644 index 000000000000..f34e3e342efb --- /dev/null +++ b/tools/testing/selftests/bpf/progs/btf__core_reloc_enumval___no_kflag.c @@ -0,0 +1,3 @@ +#include "core_reloc_types.h" + +void f(struct core_reloc_enumval___no_kflag x) {} diff --git a/tools/testing/selftests/bpf/progs/core_reloc_types.h b/tools/testing/selftests/bpf/progs/core_reloc_types.h index 6d758e74bd58..35beb76068bd 100644 --- a/tools/testing/selftests/bpf/progs/core_reloc_types.h +++ b/tools/testing/selftests/bpf/progs/core_reloc_types.h @@ -1217,6 +1217,8 @@ struct core_reloc_enumval_output { int anon_val2; long uint_val1; long uint_val2; + long int_val1; + long int_val2; }; struct core_reloc_enum64val_output { @@ -1250,10 +1252,24 @@ enum named_uint_enum { UINT_ENUM_VAL2 = 0xffffffff, }; +enum named_int_enum { + INT_ENUM_VAL1 = -2, + INT_ENUM_VAL2 = -3, +}; + struct core_reloc_enumval { enum named_enum f1; anon_enum f2; enum named_uint_enum f3; + enum named_int_enum f4; +}; + +/* the same types, whose BTF gets the kflag of every enum cleared */ +struct core_reloc_enumval___no_kflag { + enum named_enum f1; + anon_enum f2; + enum named_uint_enum f3; + enum named_int_enum f4; }; enum named_unsigned_enum64 { diff --git a/tools/testing/selftests/bpf/progs/test_core_reloc_enumval.c b/tools/testing/selftests/bpf/progs/test_core_reloc_enumval.c index 8b8abbaaf533..a524e5028c77 100644 --- a/tools/testing/selftests/bpf/progs/test_core_reloc_enumval.c +++ b/tools/testing/selftests/bpf/progs/test_core_reloc_enumval.c @@ -33,6 +33,12 @@ enum named_uint_enum { UINT_ENUM_VAL2 = 0xfffffff0, }; +/* signed 4-byte enum, with a non-negative value too */ +enum named_int_enum { + INT_ENUM_VAL1 = -1, + INT_ENUM_VAL2 = 1, +}; + struct core_reloc_enumval_output { bool named_val1_exists; bool named_val2_exists; @@ -47,6 +53,8 @@ struct core_reloc_enumval_output { int anon_val2; long uint_val1; long uint_val2; + long int_val1; + long int_val2; }; #ifdef __clang__ @@ -85,6 +93,12 @@ int test_core_enumval(void *ctx) out->uint_val1 = UINT_ENUM_VALUE(UINT_ENUM_VAL1); if (bpf_core_enum_value_exists(enum named_uint_enum, UINT_ENUM_VAL2)) out->uint_val2 = UINT_ENUM_VALUE(UINT_ENUM_VAL2); + + /* enum named_int_enum is optional */ + if (bpf_core_enum_value_exists(enum named_int_enum, INT_ENUM_VAL1)) + out->int_val1 = bpf_core_enum_value(enum named_int_enum, INT_ENUM_VAL1); + if (bpf_core_enum_value_exists(enum named_int_enum, INT_ENUM_VAL2)) + out->int_val2 = bpf_core_enum_value(enum named_int_enum, INT_ENUM_VAL2); #else data.skip = true; #endif -- 2.53.1-Meta