The BPF memory allocator prefixes each object with a header that holds struct llist_node while the object sits on a free list, and a pointer to the owning bpf_mem_cache while it is allocated. The header size was sizeof(struct llist_node), so on 32-bit architectures the memory handed out was offset by 4 bytes from the 8-byte aligned kmalloc allocation. The verifier assumes that allocated objects are 8-byte aligned and allows BPF_DW atomic instructions on their u64 fields, e.g. on values of non-preallocated hash maps, objects from bpf_obj_new() and local storage. On arm32 the JIT does not implement atomics, and the interpreter executes them via atomic64_*(), which are implemented with LDREXD/STREXD on ARMv6K and later. Those instructions require 8-byte aligned addresses; a misaligned address raises an alignment fault and crashes the kernel. Pad the header to 8 bytes to preserve the alignment provided by kmalloc. Store the percpu pointer of percpu objects at the new header offset instead of assuming the header is pointer-sized, and update the copy of the header size used by the hash map memory usage accounting. Fixes: 7c8199e24fa0 ("bpf: Introduce any context BPF specific memory allocator.") Reported-by: Thibaut VARÈNE Reported-by: Stijn Tintel Signed-off-by: Felix Fietkau --- kernel/bpf/hashtab.c | 2 +- kernel/bpf/memalloc.c | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/kernel/bpf/hashtab.c b/kernel/bpf/hashtab.c index 3dd9b4924ae4..900c1851d264 100644 --- a/kernel/bpf/hashtab.c +++ b/kernel/bpf/hashtab.c @@ -2337,7 +2337,7 @@ static u64 htab_map_mem_usage(const struct bpf_map *map) else if (!lru) usage += sizeof(struct htab_elem *) * num_possible_cpus(); } else { -#define LLIST_NODE_SZ sizeof(struct llist_node) +#define LLIST_NODE_SZ ALIGN(sizeof(struct llist_node), 8) num_entries = htab->use_percpu_counter ? percpu_counter_sum(&htab->pcount) : diff --git a/kernel/bpf/memalloc.c b/kernel/bpf/memalloc.c index e9662db7198f..ca7555b1da34 100644 --- a/kernel/bpf/memalloc.c +++ b/kernel/bpf/memalloc.c @@ -33,7 +33,7 @@ * Every allocated objected is padded with extra 8 bytes that contains * struct llist_node. */ -#define LLIST_NODE_SZ sizeof(struct llist_node) +#define LLIST_NODE_SZ ALIGN(sizeof(struct llist_node), 8) #define BPF_MEM_ALLOC_SIZE_MAX 4096 @@ -142,7 +142,7 @@ static struct llist_node notrace *__llist_del_first(struct llist_head *head) static void *__alloc(struct bpf_mem_cache *c, int node, gfp_t flags) { if (c->percpu_size) { - void __percpu **obj = kmalloc_node(c->percpu_size, flags, node); + void *obj = kmalloc_node(c->percpu_size, flags, node); void __percpu *pptr = __alloc_percpu_gfp(c->unit_size, 8, flags); if (!obj || !pptr) { @@ -150,7 +150,7 @@ static void *__alloc(struct bpf_mem_cache *c, int node, gfp_t flags) kfree(obj); return NULL; } - obj[1] = pptr; + *(void __percpu **)(obj + LLIST_NODE_SZ) = pptr; return obj; } @@ -257,7 +257,7 @@ static void alloc_bulk(struct bpf_mem_cache *c, int cnt, int node, bool atomic) static void free_one(void *obj, bool percpu) { if (percpu) - free_percpu(((void __percpu **)obj)[1]); + free_percpu(*(void __percpu **)(obj + LLIST_NODE_SZ)); kfree(obj); } -- 2.51.0