From: Alexei Starovoitov Add kmem_cache_alloc_nolock_noprof() and its kmem_cache_alloc_nolock() alloc_hooks() wrapper, mirroring kmalloc_nolock(). Allocates one object from a specific kmem_cache and is safe from any context, including NMI and IRQ-off, returning NULL on transient trylock failure. The new function is needed by the upcoming SLAB_BPF_ARENA kfunc path so arena object allocation can run from BPF program context. Signed-off-by: Alexei Starovoitov --- include/linux/slab.h | 5 +++++ mm/slub.c | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/include/linux/slab.h b/include/linux/slab.h index 2b5ab488e96b..7ce9125a6a2c 100644 --- a/include/linux/slab.h +++ b/include/linux/slab.h @@ -958,6 +958,11 @@ static __always_inline __alloc_size(1) void *kmalloc_noprof(size_t size, gfp_t f void *kmalloc_nolock_noprof(size_t size, gfp_t gfp_flags, int node); #define kmalloc_nolock(...) alloc_hooks(kmalloc_nolock_noprof(__VA_ARGS__)) +void *kmem_cache_alloc_nolock_noprof(struct kmem_cache *s, gfp_t gfp_flags, + int node); +#define kmem_cache_alloc_nolock(...) \ + alloc_hooks(kmem_cache_alloc_nolock_noprof(__VA_ARGS__)) + /** * __alloc_objs - Allocate objects of a given type using * @KMALLOC: which size-based kmalloc wrapper to allocate with. diff --git a/mm/slub.c b/mm/slub.c index a2bf3756ca7d..601986aaebdf 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -5402,6 +5402,45 @@ void *kmalloc_nolock_noprof(size_t size, gfp_t gfp_flags, int node) } EXPORT_SYMBOL_GPL(kmalloc_nolock_noprof); +/** + * kmem_cache_alloc_nolock - Allocate one object from a specific cache, + * safe from any context (including NMI/IRQ-off), like kmalloc_nolock(). + * + * Returns NULL on failure (including the trylock paths that may transiently + * fail under contention). + */ +void *kmem_cache_alloc_nolock_noprof(struct kmem_cache *s, gfp_t gfp_flags, + int node) +{ + gfp_t alloc_gfp = __GFP_NOWARN | __GFP_NOMEMALLOC | gfp_flags; + void *ret; + + VM_WARN_ON_ONCE(gfp_flags & ~(__GFP_ACCOUNT | __GFP_ZERO | + __GFP_NO_OBJ_EXT)); + + if (IS_ENABLED(CONFIG_PREEMPT_RT) && (in_nmi() || in_hardirq())) + return NULL; + if (!IS_ENABLED(CONFIG_SMP) && in_nmi()) + return NULL; + + if (!(s->flags & __CMPXCHG_DOUBLE) && !kmem_cache_debug(s)) + return NULL; + + ret = alloc_from_pcs(s, alloc_gfp, node); + if (!ret) + ret = __slab_alloc_node(s, alloc_gfp, node, _RET_IP_, + s->object_size); + if (!ret) + return NULL; + + maybe_wipe_obj_freeptr(s, ret); + slab_post_alloc_hook(s, NULL, alloc_gfp, 1, &ret, + slab_want_init_on_alloc(alloc_gfp, s), + s->object_size); + return kasan_kmalloc(s, ret, s->object_size, alloc_gfp); +} +EXPORT_SYMBOL_GPL(kmem_cache_alloc_nolock_noprof); + void *__kmalloc_node_track_caller_noprof(DECL_BUCKET_PARAMS(size, b), gfp_t flags, int node, unsigned long caller) { -- 2.53.0-Meta