The function takes all the parameters that exist as fields in slab_alloc_context, except alloc_flags. Replace them with a single pointer. This moves slab_alloc_context initialization to a number of callers, which is more verbose, but arguably also more clear than a long list of parameters, and most do not use the 'lru' field. This will also allow kmalloc_nolock() to call slab_alloc_node() and reduce the special open-coding it currently has. Signed-off-by: Vlastimil Babka (SUSE) --- mm/slub.c | 75 ++++++++++++++++++++++++++++++++++++++++++++------------------- 1 file changed, 53 insertions(+), 22 deletions(-) diff --git a/mm/slub.c b/mm/slub.c index e634137b67fa..0b9974bfcb24 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -4921,30 +4921,23 @@ unsigned int alloc_from_pcs_bulk(struct kmem_cache *s, gfp_t gfp, size_t size, * * Otherwise we can simply pick the next object from the lockless free list. */ -static __fastpath_inline void *slab_alloc_node(struct kmem_cache *s, struct list_lru *lru, - gfp_t gfpflags, int node, unsigned long addr, size_t orig_size) +static __fastpath_inline void *slab_alloc_node(struct kmem_cache *s, + gfp_t gfpflags, int node, struct slab_alloc_context *ac) { - const unsigned int alloc_flags = SLAB_ALLOC_DEFAULT; void *object; - struct slab_alloc_context ac = { - .caller_addr = addr, - .orig_size = orig_size, - .alloc_flags = alloc_flags, - .lru = lru, - }; s = slab_pre_alloc_hook(s, gfpflags); if (unlikely(!s)) return NULL; - object = kfence_alloc(s, orig_size, gfpflags); + object = kfence_alloc(s, ac->orig_size, gfpflags); if (unlikely(object)) goto out; - object = alloc_from_pcs(s, gfpflags, alloc_flags, node); + object = alloc_from_pcs(s, gfpflags, ac->alloc_flags, node); if (!object) - object = __slab_alloc_node(s, gfpflags, node, &ac); + object = __slab_alloc_node(s, gfpflags, node, ac); maybe_wipe_obj_freeptr(s, object); @@ -4953,15 +4946,21 @@ static __fastpath_inline void *slab_alloc_node(struct kmem_cache *s, struct list * In case this fails due to memcg_slab_post_alloc_hook(), * object is set to NULL */ - slab_post_alloc_hook(s, gfpflags, 1, &object, &ac); + slab_post_alloc_hook(s, gfpflags, 1, &object, ac); return object; } void *kmem_cache_alloc_noprof(struct kmem_cache *s, gfp_t gfpflags) { - void *ret = slab_alloc_node(s, NULL, gfpflags, NUMA_NO_NODE, _RET_IP_, - s->object_size); + void *ret; + struct slab_alloc_context ac = { + .caller_addr = _RET_IP_, + .orig_size = s->object_size, + .alloc_flags = SLAB_ALLOC_DEFAULT, + }; + + ret = slab_alloc_node(s, gfpflags, NUMA_NO_NODE, &ac); trace_kmem_cache_alloc(_RET_IP_, ret, s, gfpflags, NUMA_NO_NODE); @@ -4972,8 +4971,15 @@ EXPORT_SYMBOL(kmem_cache_alloc_noprof); void *kmem_cache_alloc_lru_noprof(struct kmem_cache *s, struct list_lru *lru, gfp_t gfpflags) { - void *ret = slab_alloc_node(s, lru, gfpflags, NUMA_NO_NODE, _RET_IP_, - s->object_size); + void *ret; + struct slab_alloc_context ac = { + .caller_addr = _RET_IP_, + .orig_size = s->object_size, + .alloc_flags = SLAB_ALLOC_DEFAULT, + .lru = lru, + }; + + ret = slab_alloc_node(s, gfpflags, NUMA_NO_NODE, &ac); trace_kmem_cache_alloc(_RET_IP_, ret, s, gfpflags, NUMA_NO_NODE); @@ -5005,7 +5011,14 @@ EXPORT_SYMBOL(kmem_cache_charge); */ void *kmem_cache_alloc_node_noprof(struct kmem_cache *s, gfp_t gfpflags, int node) { - void *ret = slab_alloc_node(s, NULL, gfpflags, node, _RET_IP_, s->object_size); + void *ret; + struct slab_alloc_context ac = { + .caller_addr = _RET_IP_, + .orig_size = s->object_size, + .alloc_flags = SLAB_ALLOC_DEFAULT, + }; + + ret = slab_alloc_node(s, gfpflags, node, &ac); trace_kmem_cache_alloc(_RET_IP_, ret, s, gfpflags, node); @@ -5335,6 +5348,11 @@ void *__do_kmalloc_node(size_t size, kmem_buckets *b, gfp_t flags, int node, { struct kmem_cache *s; void *ret; + struct slab_alloc_context ac = { + .caller_addr = caller, + .orig_size = size, + .alloc_flags = SLAB_ALLOC_DEFAULT, + }; if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) { ret = __kmalloc_large_node_noprof(size, flags, node); @@ -5348,7 +5366,7 @@ void *__do_kmalloc_node(size_t size, kmem_buckets *b, gfp_t flags, int node, s = kmalloc_slab(size, b, flags, token); - ret = slab_alloc_node(s, NULL, flags, node, caller, size); + ret = slab_alloc_node(s, flags, node, &ac); ret = kasan_kmalloc(s, ret, size, flags); trace_kmalloc(caller, ret, size, s->size, flags, node); return ret; @@ -5467,8 +5485,14 @@ EXPORT_SYMBOL(__kmalloc_node_track_caller_noprof); void *__kmalloc_cache_noprof(struct kmem_cache *s, gfp_t gfpflags, size_t size) { - void *ret = slab_alloc_node(s, NULL, gfpflags, NUMA_NO_NODE, - _RET_IP_, size); + void *ret; + struct slab_alloc_context ac = { + .caller_addr = _RET_IP_, + .orig_size = size, + .alloc_flags = SLAB_ALLOC_DEFAULT, + }; + + ret = slab_alloc_node(s, gfpflags, NUMA_NO_NODE, &ac); trace_kmalloc(_RET_IP_, ret, size, s->size, gfpflags, NUMA_NO_NODE); @@ -5480,7 +5504,14 @@ EXPORT_SYMBOL(__kmalloc_cache_noprof); void *__kmalloc_cache_node_noprof(struct kmem_cache *s, gfp_t gfpflags, int node, size_t size) { - void *ret = slab_alloc_node(s, NULL, gfpflags, node, _RET_IP_, size); + void *ret; + struct slab_alloc_context ac = { + .caller_addr = _RET_IP_, + .orig_size = size, + .alloc_flags = SLAB_ALLOC_DEFAULT, + }; + + ret = slab_alloc_node(s, gfpflags, node, &ac); trace_kmalloc(_RET_IP_, ret, size, s->size, gfpflags, node); -- 2.54.0