SLUB's internal bulk allocation __kmem_cache_alloc_bulk() can currently allocate some objects from KFENCE, i.e. when refilling a sheaf. It works but it's conceptually the wrong layer, as KFENCE allocations should only happen when objects are actually handed out from slab to its users. Currently for sheaf-enabled caches, slab_alloc_node() can return KFENCE object via kfence_alloc(), but also via alloc_from_pcs() when a sheaf was refilled with KFENCE objects. Continuing like this would also complicate the upcoming sheaf refill changes. Thus remove KFENCE allocation from __kmem_cache_alloc_bulk() and move it to the places that return slab objects to users. slab_alloc_node() is already covered (see above). Add kfence_alloc() to kmem_cache_alloc_from_sheaf() to handle KFENCE allocations from prefilled sheafs, with a comment that the caller should not expect the sheaf size to decrease after every allocation because of this possibility. For kmem_cache_alloc_bulk() implement a different strategy to handle KFENCE upfront and rely on internal batched operations afterwards. Assume there will be at most once KFENCE allocation per bulk allocation and then assign its index in the array of objects randomly. Cc: Alexander Potapenko Cc: Marco Elver Cc: Dmitry Vyukov Signed-off-by: Vlastimil Babka --- mm/slub.c | 44 ++++++++++++++++++++++++++++++++++++-------- 1 file changed, 36 insertions(+), 8 deletions(-) diff --git a/mm/slub.c b/mm/slub.c index 87a1d2f9de0d..4731b9e461c2 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -5530,6 +5530,9 @@ int kmem_cache_refill_sheaf(struct kmem_cache *s, gfp_t gfp, * * The gfp parameter is meant only to specify __GFP_ZERO or __GFP_ACCOUNT * memcg charging is forced over limit if necessary, to avoid failure. + * + * It is possible that the allocation comes from kfence and then the sheaf + * size is not decreased. */ void * kmem_cache_alloc_from_sheaf_noprof(struct kmem_cache *s, gfp_t gfp, @@ -5541,7 +5544,10 @@ kmem_cache_alloc_from_sheaf_noprof(struct kmem_cache *s, gfp_t gfp, if (sheaf->size == 0) goto out; - ret = sheaf->objects[--sheaf->size]; + ret = kfence_alloc(s, s->object_size, gfp); + + if (likely(!ret)) + ret = sheaf->objects[--sheaf->size]; init = slab_want_init_on_alloc(gfp, s); @@ -7361,14 +7367,8 @@ int __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size, local_lock_irqsave(&s->cpu_slab->lock, irqflags); for (i = 0; i < size; i++) { - void *object = kfence_alloc(s, s->object_size, flags); - - if (unlikely(object)) { - p[i] = object; - continue; - } + void *object = c->freelist; - object = c->freelist; if (unlikely(!object)) { /* * We may have removed an object from c->freelist using @@ -7449,6 +7449,7 @@ int kmem_cache_alloc_bulk_noprof(struct kmem_cache *s, gfp_t flags, size_t size, void **p) { unsigned int i = 0; + void *kfence_obj; if (!size) return 0; @@ -7457,6 +7458,20 @@ int kmem_cache_alloc_bulk_noprof(struct kmem_cache *s, gfp_t flags, size_t size, if (unlikely(!s)) return 0; + /* + * to make things simpler, only assume at most once kfence allocated + * object per bulk allocation and choose its index randomly + */ + kfence_obj = kfence_alloc(s, s->object_size, flags); + + if (unlikely(kfence_obj)) { + if (unlikely(size == 1)) { + p[0] = kfence_obj; + goto out; + } + size--; + } + if (s->cpu_sheaves) i = alloc_from_pcs_bulk(s, size, p); @@ -7468,10 +7483,23 @@ int kmem_cache_alloc_bulk_noprof(struct kmem_cache *s, gfp_t flags, size_t size, if (unlikely(__kmem_cache_alloc_bulk(s, flags, size - i, p + i) == 0)) { if (i > 0) __kmem_cache_free_bulk(s, i, p); + if (kfence_obj) + __kfence_free(kfence_obj); return 0; } } + if (unlikely(kfence_obj)) { + int idx = get_random_u32_below(size + 1); + + if (idx != size) + p[size] = p[idx]; + p[idx] = kfence_obj; + + size++; + } + +out: /* * memcg and kmem_cache debug support and memory initialization. * Done outside of the IRQ disabled fastpath loop. -- 2.51.1