slab_free_hook() takes two bools, init and after_rcu_delay, and a later patch needs to tell it whether spinning is allowed. Instead of adding a third bool, make it take free_flags. Add SLAB_FREE_INIT and SLAB_FREE_AFTER_RCU_DELAY, and convert the callers. No functional change intended. Suggested-by: Vlastimil Babka (SUSE) Link: https://lore.kernel.org/f31885cd-57c2-428d-9760-a76a0137f989@kernel.org Signed-off-by: Harry Yoo (Meta) --- mm/slab.h | 6 ++++-- mm/slub.c | 47 ++++++++++++++++++++++++++++++++++------------- 2 files changed, 38 insertions(+), 15 deletions(-) diff --git a/mm/slab.h b/mm/slab.h index e94ac63d10dc..15b0d98ae1d4 100644 --- a/mm/slab.h +++ b/mm/slab.h @@ -24,8 +24,10 @@ #define SLAB_ALLOC_NO_RECURSE 0x04 /* prevent kmalloc() recursion */ #define SLAB_ALLOC_NO_OBJ_EXT 0x08 /* prevent obj_exts array allocation */ -#define SLAB_FREE_DEFAULT 0x00 /* no flags */ -#define SLAB_FREE_NOLOCK 0x01 /* spinning not allowed */ +#define SLAB_FREE_DEFAULT 0x00 /* no flags */ +#define SLAB_FREE_NOLOCK 0x01 /* spinning not allowed */ +#define SLAB_FREE_INIT 0x02 /* zero the object on free */ +#define SLAB_FREE_AFTER_RCU_DELAY 0x04 /* RCU grace period has elapsed */ static inline unsigned int to_alloc_flags(unsigned int free_flags) { diff --git a/mm/slub.c b/mm/slub.c index 86259c32cf59..761dc2affea2 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -2694,9 +2694,10 @@ struct rcu_delayed_free { * - __kcsan_check_access() */ static __always_inline -bool slab_free_hook(struct kmem_cache *s, void *x, bool init, - bool after_rcu_delay) +bool slab_free_hook(struct kmem_cache *s, void *x, unsigned int free_flags) { + bool init = free_flags & SLAB_FREE_INIT; + bool after_rcu_delay = free_flags & SLAB_FREE_AFTER_RCU_DELAY; /* Are the object contents still accessible? */ bool still_accessible = (s->flags & SLAB_TYPESAFE_BY_RCU) && !after_rcu_delay; @@ -2786,10 +2787,10 @@ bool slab_free_freelist_hook(struct kmem_cache *s, void **head, void **tail, void *object; void *next = *head; void *old_tail = *tail; - bool init; + unsigned int free_flags = SLAB_FREE_DEFAULT; if (is_kfence_address(next)) { - slab_free_hook(s, next, false, false); + slab_free_hook(s, next, free_flags); return false; } @@ -2797,14 +2798,15 @@ bool slab_free_freelist_hook(struct kmem_cache *s, void **head, void **tail, *head = NULL; *tail = NULL; - init = slab_want_init_on_free(s); + if (slab_want_init_on_free(s)) + free_flags |= SLAB_FREE_INIT; do { object = next; next = get_freepointer(s, object); /* If object's reuse doesn't have to be delayed */ - if (likely(slab_free_hook(s, object, init, false))) { + if (likely(slab_free_hook(s, object, free_flags))) { /* Move object to the new freelist */ set_freepointer(s, object, *head); *head = object; @@ -3025,18 +3027,21 @@ static void sheaf_flush_unused(struct kmem_cache *s, struct slab_sheaf *sheaf) static bool __rcu_free_sheaf_prepare(struct kmem_cache *s, struct slab_sheaf *sheaf) { - bool init = slab_want_init_on_free(s); + unsigned int free_flags = SLAB_FREE_AFTER_RCU_DELAY; void **p = &sheaf->objects[0]; unsigned int i = 0; bool pfmemalloc = false; + if (slab_want_init_on_free(s)) + free_flags |= SLAB_FREE_INIT; + while (i < sheaf->size) { struct slab *slab = virt_to_slab(p[i]); memcg_slab_free_hook(s, slab, p + i, 1); alloc_tagging_slab_free_hook(s, slab, p + i, 1); - if (unlikely(!slab_free_hook(s, p[i], init, true))) { + if (unlikely(!slab_free_hook(s, p[i], free_flags))) { p[i] = p[--sheaf->size]; continue; } @@ -6458,10 +6463,13 @@ static unsigned int __free_to_pcs_batch(struct kmem_cache *s, size_t size, void */ static void free_to_pcs_bulk(struct kmem_cache *s, size_t size, void **p) { - bool init = slab_want_init_on_free(s); + unsigned int free_flags = SLAB_FREE_DEFAULT; void **remote_objects = p; unsigned int remote_nr = 0; + if (slab_want_init_on_free(s)) + free_flags |= SLAB_FREE_INIT; + /* * Process the free hooks and separate out remote objects by * partitioning the 'p' array in place: @@ -6476,7 +6484,7 @@ static void free_to_pcs_bulk(struct kmem_cache *s, size_t size, void **p) memcg_slab_free_hook(s, slab, p + i, 1); alloc_tagging_slab_free_hook(s, slab, p + i, 1); - if (unlikely(!slab_free_hook(s, p[i], init, false))) { + if (unlikely(!slab_free_hook(s, p[i], free_flags))) { p[i] = p[--size]; continue; } @@ -6600,10 +6608,15 @@ static __fastpath_inline void slab_free(struct kmem_cache *s, struct slab *slab, void *object, unsigned long addr) { + unsigned int free_flags = SLAB_FREE_DEFAULT; + + if (slab_want_init_on_free(s)) + free_flags |= SLAB_FREE_INIT; + memcg_slab_free_hook(s, slab, &object, 1); alloc_tagging_slab_free_hook(s, slab, &object, 1); - if (unlikely(!slab_free_hook(s, object, slab_want_init_on_free(s), false))) + if (unlikely(!slab_free_hook(s, object, free_flags))) return; if (likely(can_free_to_pcs(slab)) && likely(free_to_pcs(s, object, true))) @@ -6619,10 +6632,14 @@ static noinline void memcg_alloc_abort_single(struct kmem_cache *s, void *object) { struct slab *slab = virt_to_slab(object); + unsigned int free_flags = SLAB_FREE_DEFAULT; + + if (slab_want_init_on_free(s)) + free_flags |= SLAB_FREE_INIT; alloc_tagging_slab_free_hook(s, slab, &object, 1); - if (likely(slab_free_hook(s, object, slab_want_init_on_free(s), false))) + if (likely(slab_free_hook(s, object, free_flags))) __slab_free(s, slab, object, object, 1, _RET_IP_); } #endif @@ -6651,6 +6668,7 @@ static void slab_free_after_rcu_debug(struct rcu_head *rcu_head) void *object = delayed_free->object; struct slab *slab = virt_to_slab(object); struct kmem_cache *s; + unsigned int free_flags = SLAB_FREE_AFTER_RCU_DELAY; kfree(delayed_free); @@ -6664,8 +6682,11 @@ static void slab_free_after_rcu_debug(struct rcu_head *rcu_head) if (WARN_ON(!(s->flags & SLAB_TYPESAFE_BY_RCU))) return; + if (slab_want_init_on_free(s)) + free_flags |= SLAB_FREE_INIT; + /* resume freeing */ - if (slab_free_hook(s, object, slab_want_init_on_free(s), true)) { + if (slab_free_hook(s, object, free_flags)) { __slab_free(s, slab, object, object, 1, _THIS_IP_); stat(s, FREE_SLOWPATH); } -- 2.54.0