In deferred_percpu_work_fn() we have a bunch of single-use local variables for the various llists. Remove them and access the lists directly. In defer_free() make it more obvious and documented what we are doing. Also restrict guard(preempt) to only the necessary part. Signed-off-by: Vlastimil Babka (SUSE) --- mm/slub.c | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/mm/slub.c b/mm/slub.c index b9aeb02a880f..044db93d64a0 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -6371,16 +6371,12 @@ static void free_to_pcs_bulk(struct kmem_cache *s, size_t size, void **p) static void deferred_percpu_work_fn(struct irq_work *work) { struct deferred_percpu_work *dpw; - struct llist_head *objs, *objs_by_rcu, *rcu_sheaves; struct llist_node *llnode, *pos, *t; struct slab_sheaf *sheaf, *next; dpw = container_of(work, struct deferred_percpu_work, work); - rcu_sheaves = &dpw->rcu_sheaves; - objs = &dpw->objects; - objs_by_rcu = &dpw->objects_by_rcu; - llnode = llist_del_all(objs); + llnode = llist_del_all(&dpw->objects); llist_for_each_safe(pos, t, llnode) { struct kmem_cache *s; struct slab *slab; @@ -6403,7 +6399,7 @@ static void deferred_percpu_work_fn(struct irq_work *work) stat(s, FREE_SLOWPATH); } - llnode = llist_del_all(objs_by_rcu); + llnode = llist_del_all(&dpw->objects_by_rcu); llist_for_each_safe(pos, t, llnode) { void *head = pos; void *objp = kvmalloc_obj_start_addr(head); @@ -6411,21 +6407,27 @@ static void deferred_percpu_work_fn(struct irq_work *work) kvfree_call_rcu(head, objp); } - llnode = llist_del_all(rcu_sheaves); + llnode = llist_del_all(&dpw->rcu_sheaves); llist_for_each_entry_safe(sheaf, next, llnode, llnode) call_rcu(&sheaf->rcu_head, rcu_free_sheaf); } -static void defer_free(struct kmem_cache *s, void *head) +static void defer_free(struct kmem_cache *s, void *obj) { struct deferred_percpu_work *dpw; + struct llist_node *llnode; - guard(preempt)(); + /* + * Place the llist node where the freepointer would be if we freed the + * object immediately. That means we can write there safely, only need + * to remove kasan tag first. + */ + llnode = kasan_reset_tag(obj) + s->offset; - head = kasan_reset_tag(head); + guard(preempt)(); dpw = this_cpu_ptr(&deferred_percpu_work); - if (llist_add(head + s->offset, &dpw->objects)) + if (llist_add(llnode, &dpw->objects)) irq_work_queue(&dpw->work); } -- 2.55.0 KFENCE objects are one of the reasons why kfree_nolock() cannot currently handle kmalloc() objects. They are however rare so we can simply defer their freeing to irq_work. The only complication is where to put the llist node. We cannot use the freepointer location like in defer_free() because for some caches it may be outside the object area and KFENCE would detect writes there. Since KFENCE already solves a similar situation when freeing objects from SLAB_TYPESAFE_BY_RCU caches with an rcu_head in its internal metadata, reuse that rcu_head also for the llist node. Introduce kfence_obj_to_llnode() and kfence_llnode_to_obj() so SLAB can work with this llist node without being exposed to KFENCE internals. Signed-off-by: Vlastimil Babka (SUSE) --- include/linux/kfence.h | 5 +++++ mm/kfence/core.c | 14 ++++++++++++++ mm/kfence/kfence.h | 5 ++++- mm/slub.c | 39 ++++++++++++++++++++++++++++++++++++--- 4 files changed, 59 insertions(+), 4 deletions(-) diff --git a/include/linux/kfence.h b/include/linux/kfence.h index e5822f6e7f27..00721c85258d 100644 --- a/include/linux/kfence.h +++ b/include/linux/kfence.h @@ -188,6 +188,9 @@ static __always_inline __must_check bool kfence_free(void *addr) return true; } +struct llist_node *kfence_obj_to_llnode(void *addr); +void *kfence_llnode_to_obj(struct llist_node *llnode); + /** * kfence_handle_page_fault() - perform page fault handling for KFENCE pages * @addr: faulting address @@ -235,6 +238,8 @@ static inline size_t kfence_ksize(const void *addr) { return 0; } static inline void *kfence_object_start(const void *addr) { return NULL; } static inline void __kfence_free(void *addr) { } static inline bool __must_check kfence_free(void *addr) { return false; } +static inline struct llist_node *kfence_obj_to_llnode(void *addr) { return NULL; } +static inline void *kfence_llnode_to_obj(struct llist_node *llnode) { return NULL; } static inline bool __must_check kfence_handle_page_fault(unsigned long addr, bool is_write, struct pt_regs *regs) { diff --git a/mm/kfence/core.c b/mm/kfence/core.c index 6577bd76954e..42519d24687f 100644 --- a/mm/kfence/core.c +++ b/mm/kfence/core.c @@ -1271,6 +1271,20 @@ void __kfence_free(void *addr) } } +struct llist_node *kfence_obj_to_llnode(void *addr) +{ + struct kfence_metadata *meta = addr_to_metadata((unsigned long)addr); + + return &meta->llnode; +} + +void *kfence_llnode_to_obj(struct llist_node *llnode) +{ + struct kfence_metadata *meta = container_of(llnode, struct kfence_metadata, llnode); + + return (void *)meta->addr; +} + bool kfence_handle_page_fault(unsigned long addr, bool is_write, struct pt_regs *regs) { const int page_index = (addr - (unsigned long)__kfence_pool) / PAGE_SIZE; diff --git a/mm/kfence/kfence.h b/mm/kfence/kfence.h index 1f618f9b0d12..0fca1dc2c794 100644 --- a/mm/kfence/kfence.h +++ b/mm/kfence/kfence.h @@ -58,7 +58,10 @@ struct kfence_track { /* KFENCE metadata per guarded allocation. */ struct kfence_metadata { struct list_head list __guarded_by(&kfence_freelist_lock); /* Freelist node. */ - struct rcu_head rcu_head; /* For delayed freeing. */ + union { + struct rcu_head rcu_head; /* For delayed freeing. */ + struct llist_node llnode; /* For kfree_nolock(). */ + }; /* * Lock protecting below data; to ensure consistency of the below data, diff --git a/mm/slub.c b/mm/slub.c index 044db93d64a0..2d7648b96bfa 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -4050,6 +4050,7 @@ static void flush_all(struct kmem_cache *s) struct deferred_percpu_work { struct llist_head objects; + struct llist_head objects_kfence; struct llist_head objects_by_rcu; struct llist_head rcu_sheaves; struct irq_work work; @@ -4059,6 +4060,7 @@ static void deferred_percpu_work_fn(struct irq_work *work); static DEFINE_PER_CPU(struct deferred_percpu_work, deferred_percpu_work) = { .objects = LLIST_HEAD_INIT(objects), + .objects_kfence = LLIST_HEAD_INIT(objects_kfence), .objects_by_rcu = LLIST_HEAD_INIT(objects_by_rcu), .rcu_sheaves = LLIST_HEAD_INIT(rcu_sheaves), .work = IRQ_WORK_INIT(deferred_percpu_work_fn), @@ -6399,6 +6401,13 @@ static void deferred_percpu_work_fn(struct irq_work *work) stat(s, FREE_SLOWPATH); } + llnode = llist_del_all(&dpw->objects_kfence); + llist_for_each_safe(pos, t, llnode) { + void *obj = kfence_llnode_to_obj(pos); + + __kfence_free(obj); + } + llnode = llist_del_all(&dpw->objects_by_rcu); llist_for_each_safe(pos, t, llnode) { void *head = pos; @@ -6431,6 +6440,21 @@ static void defer_free(struct kmem_cache *s, void *obj) irq_work_queue(&dpw->work); } +static void defer_free_kfence(void *obj) +{ + struct deferred_percpu_work *dpw; + struct llist_node *llnode; + + /* kasan_reset_tag() is not necessary, kfence objects are not tagged */ + llnode = kfence_obj_to_llnode(obj); + + guard(preempt)(); + + dpw = this_cpu_ptr(&deferred_percpu_work); + if (llist_add(llnode, &dpw->objects_kfence)) + irq_work_queue(&dpw->work); +} + void defer_kfree_rcu(struct kvfree_rcu_head *head) { struct deferred_percpu_work *dpw; @@ -6758,10 +6782,13 @@ EXPORT_SYMBOL(kfree); /* * Can be called while holding raw_spinlock_t or from IRQ and NMI, * but ONLY for objects allocated by kmalloc_nolock(). - * Debug checks (like kmemleak and kfence) were skipped on allocation, - * hence + * + * In case kmemleak is enabled, + * * obj = kmalloc(); kfree_nolock(obj); - * will miss kmemleak/kfence book keeping and will cause false positives. + * + * will miss kmemleak book keeping and will cause false positives. + * * large_kmalloc is not supported either. */ void kfree_nolock(const void *object) @@ -6793,6 +6820,12 @@ void kfree_nolock(const void *object) * since they take spinlocks or not safe from any context. */ kmsan_slab_free(s, x); + + if (is_kfence_address(x)) { + defer_free_kfence(x); + return; + } + /* * If KASAN finds a kernel bug it will do kasan_report_invalid_free() * which will call raw_spin_lock_irqsave() which is technically -- 2.55.0 Kmemleak handling is one of the reasons why kfree_nolock() cannot currently handle kmalloc() objects, because calling kmemleak_free() would involve spinning on its internal raw spinlocks. Kmemleak is a debugging mechanism so we could simply defer all kfree_nolock() to irq_work if it's enabled, and eat the extra cost. But that would be unnecessary pessimistic. We expect kfree_nolock() will be still mostly called on objects from kmalloc_nolock() that are not registered in kmemleak so they still don't need any deferred freeing. Thus introduce kmemleak_may_need_free() that can check if the object is registered. This is done using __lookup_object() performed under a raw_spin_trylock_irqsave(), which is safe to attempt from kfree_nolock() (except from a NMI on a !CONFIG_SMP system). When that trylock fails or can't be attempted, we however must assume the object might be registered, and defer the freeing. The ordering of kmsan/kasan handling and kmemleak is also different from what kfree() is doing, but as explained in the comment, it should be OK. Signed-off-by: Vlastimil Babka (SUSE) --- include/linux/kmemleak.h | 17 +++++++++++++++++ mm/kmemleak.c | 42 ++++++++++++++++++++++++++++++++++++++++++ mm/slub.c | 34 ++++++++++++++++++++++++++-------- 3 files changed, 85 insertions(+), 8 deletions(-) diff --git a/include/linux/kmemleak.h b/include/linux/kmemleak.h index fbd424b2abb1..52f75f10a9ce 100644 --- a/include/linux/kmemleak.h +++ b/include/linux/kmemleak.h @@ -22,6 +22,7 @@ extern void kmemleak_alloc_percpu(const void __percpu *ptr, size_t size, extern void kmemleak_vmalloc(const struct vm_struct *area, size_t size, gfp_t gfp) __ref; extern void kmemleak_free(const void *ptr) __ref; +bool kmemleak_may_need_free(const void *ptr) __ref; extern void kmemleak_free_part(const void *ptr, size_t size) __ref; extern void kmemleak_free_percpu(const void __percpu *ptr) __ref; extern void kmemleak_update_trace(const void *ptr) __ref; @@ -50,6 +51,14 @@ static inline void kmemleak_free_recursive(const void *ptr, slab_flags_t flags) kmemleak_free(ptr); } +static inline bool kmemleak_may_need_free_recursive(const void *ptr, slab_flags_t flags) +{ + if (!(flags & SLAB_NOLEAKTRACE)) + return kmemleak_may_need_free(ptr); + + return false; +} + static inline void kmemleak_erase(void **ptr) { *ptr = NULL; @@ -86,6 +95,14 @@ static inline void kmemleak_free_part(const void *ptr, size_t size) static inline void kmemleak_free_recursive(const void *ptr, slab_flags_t flags) { } +static inline bool kmemleak_may_need_free(const void *ptr) +{ + return false; +} +static inline bool kmemleak_may_need_free_recursive(const void *ptr, slab_flags_t flags) +{ + return false; +} static inline void kmemleak_free_percpu(const void __percpu *ptr) { } diff --git a/mm/kmemleak.c b/mm/kmemleak.c index 7c7ba17ce7af..e3560ce82632 100644 --- a/mm/kmemleak.c +++ b/mm/kmemleak.c @@ -1168,6 +1168,48 @@ void __ref kmemleak_free(const void *ptr) } EXPORT_SYMBOL_GPL(kmemleak_free); +/** + * kmemleak_may_need_free - check if object is registered + * @ptr: pointer to beginning of the object + * + * This function is called from the kernel allocator when an object should be + * freed but the caller context might be unsafe to spin on the internal locks. + * + * It will therefore only use trylock and thus might return a false positive + * if the trylock fails and the status cannot be determined. + * + * For objects that (might) need free, the allocator has to defer the actual + * freeing to a safe context. + * + * The assumption is that most objects freed from the unsafe context are also + * allocated in such context and thus are not registered in kmemleak, so it's + * unlikely the defered freeing will be necessary just because kmemleak is + * enabled. + */ +bool __ref kmemleak_may_need_free(const void *ptr) +{ + unsigned long flags; + struct kmemleak_object *object; + + pr_debug("%s(0x%px)\n", __func__, ptr); + + if (!kmemleak_free_enabled || !ptr || IS_ERR(ptr)) + return false; + + /* On UP, raw_spin_trylock() always succeeds even when it is locked */ + if (!IS_ENABLED(CONFIG_SMP) && in_nmi()) + return true; + + if (!raw_spin_trylock_irqsave(&kmemleak_lock, flags)) + return true; + + object = __lookup_object((unsigned long)ptr, 0, 0); + + raw_spin_unlock_irqrestore(&kmemleak_lock, flags); + + return !!object; +} + /** * kmemleak_free_part - partially unregister a previously registered object * @ptr: pointer to the beginning or inside the object. This also diff --git a/mm/slub.c b/mm/slub.c index 2d7648b96bfa..423b5bdb910b 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -6390,6 +6390,8 @@ static void deferred_percpu_work_fn(struct irq_work *work) /* Point 'x' back to the beginning of allocated object */ x -= s->offset; + kmemleak_free_recursive(x, s->flags); + /* * We used freepointer in 'x' to link 'x' into df->objects. * Clear it to NULL to avoid false positive detection @@ -6403,8 +6405,15 @@ static void deferred_percpu_work_fn(struct irq_work *work) llnode = llist_del_all(&dpw->objects_kfence); llist_for_each_safe(pos, t, llnode) { + struct kmem_cache *s; + struct slab *slab; void *obj = kfence_llnode_to_obj(pos); + slab = virt_to_slab(obj); + s = slab->slab_cache; + + kmemleak_free_recursive(obj, s->flags); + __kfence_free(obj); } @@ -6781,15 +6790,10 @@ EXPORT_SYMBOL(kfree); /* * Can be called while holding raw_spinlock_t or from IRQ and NMI, - * but ONLY for objects allocated by kmalloc_nolock(). - * - * In case kmemleak is enabled, + * but may defer freeing to irq_work() in some cases. * - * obj = kmalloc(); kfree_nolock(obj); - * - * will miss kmemleak book keeping and will cause false positives. - * - * large_kmalloc is not supported either. + * Intended mainly for objects allocated from kmalloc_nolock(), but can handle + * also kmem_cache_alloc() and kmalloc() objects, except large_kmalloc. */ void kfree_nolock(const void *object) { @@ -6844,9 +6848,23 @@ void kfree_nolock(const void *object) */ kasan_slab_free(s, x, false, false, /* skip quarantine */true); + /* + * with kfree() the kmemleak handling happens much sooner, but for + * defering we need to write llnode to the object's freepointer so + * we should have it in the state when it's no longer treated as + * allocated by kasan etc. + * + * defer_free will also reset the pointer tag, but it's ok to do a + * deferred kmemleak_free() using the untagged pointer, because + * __lookup_object() resets the tag anyway + */ + if (unlikely(kmemleak_may_need_free_recursive(x, s->flags))) + goto defer; + if (likely(can_free_to_pcs(slab)) && likely(free_to_pcs(s, x, false))) return; +defer: /* * __slab_free() can locklessly cmpxchg16 into a slab, but then it might * need to take spin_lock for further processing. -- 2.55.0 Large kmalloc objects is the only remaining case that kfree_nolock() cannot handle from kmalloc() allocations. Note kmalloc_nolock() does not return large kmalloc objects. Supporting them is however mostly straigtforward. free_large_kmalloc() calls kmsan and kasan hooks that should be safe and similar to those called in kfree_nolock(). Freeing the pages can be handled by free_frozen_pages_nolock(). The only obstacle is kmemleak_free(), which we can solve by deferring when necessary, the same way as done for small kmalloc objects. Signed-off-by: Vlastimil Babka (SUSE) --- mm/slub.c | 74 ++++++++++++++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 62 insertions(+), 12 deletions(-) diff --git a/mm/slub.c b/mm/slub.c index 423b5bdb910b..3be98faa9f0d 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -4051,6 +4051,7 @@ static void flush_all(struct kmem_cache *s) struct deferred_percpu_work { struct llist_head objects; struct llist_head objects_kfence; + struct llist_head objects_large_kmalloc; struct llist_head objects_by_rcu; struct llist_head rcu_sheaves; struct irq_work work; @@ -4061,6 +4062,7 @@ static void deferred_percpu_work_fn(struct irq_work *work); static DEFINE_PER_CPU(struct deferred_percpu_work, deferred_percpu_work) = { .objects = LLIST_HEAD_INIT(objects), .objects_kfence = LLIST_HEAD_INIT(objects_kfence), + .objects_large_kmalloc = LLIST_HEAD_INIT(objects_large_kmalloc), .objects_by_rcu = LLIST_HEAD_INIT(objects_by_rcu), .rcu_sheaves = LLIST_HEAD_INIT(rcu_sheaves), .work = IRQ_WORK_INIT(deferred_percpu_work_fn), @@ -6365,6 +6367,21 @@ static void free_to_pcs_bulk(struct kmem_cache *s, size_t size, void **p) } } +static inline void +__free_large_kmalloc_page(struct page *page, unsigned int free_flags) +{ + unsigned int order = compound_order(page); + + mod_lruvec_page_state(page, NR_SLAB_UNRECLAIMABLE_B, + -(PAGE_SIZE << order)); + __ClearPageLargeKmalloc(page); + + if (free_flags & SLAB_FREE_NOLOCK) + free_frozen_pages_nolock(page, order); + else + free_frozen_pages(page, order); +} + /* * In PREEMPT_RT irq_work runs in per-cpu kthread, so it's safe * to take sleeping spin_locks from __slab_free(). @@ -6417,6 +6434,15 @@ static void deferred_percpu_work_fn(struct irq_work *work) __kfence_free(obj); } + llnode = llist_del_all(&dpw->objects_large_kmalloc); + llist_for_each_safe(pos, t, llnode) { + struct page *page = virt_to_page(pos); + + kmemleak_free(pos); + + __free_large_kmalloc_page(page, SLAB_FREE_DEFAULT); + } + llnode = llist_del_all(&dpw->objects_by_rcu); llist_for_each_safe(pos, t, llnode) { void *head = pos; @@ -6464,6 +6490,24 @@ static void defer_free_kfence(void *obj) irq_work_queue(&dpw->work); } +static void defer_free_large_kmalloc(void *obj) +{ + struct deferred_percpu_work *dpw; + struct llist_node *llnode; + + /* + * we can simply use the first word of the large kmalloc object + * for the llnode, as there's no ctor or TYPESAFE_BY_RCU + */ + llnode = kasan_reset_tag(obj); + + guard(preempt)(); + + dpw = this_cpu_ptr(&deferred_percpu_work); + if (llist_add(llnode, &dpw->objects_large_kmalloc)) + irq_work_queue(&dpw->work); +} + void defer_kfree_rcu(struct kvfree_rcu_head *head) { struct deferred_percpu_work *dpw; @@ -6712,9 +6756,11 @@ size_t ksize(const void *objp) } EXPORT_SYMBOL(ksize); -static void free_large_kmalloc(struct page *page, void *object) +static void free_large_kmalloc(struct page *page, void *object, + unsigned int free_flags) { unsigned int order = compound_order(page); + bool nolock = free_flags & SLAB_FREE_NOLOCK; if (WARN_ON_ONCE(!PageLargeKmalloc(page))) { dump_page(page, "Not a kmalloc allocation"); @@ -6724,14 +6770,16 @@ static void free_large_kmalloc(struct page *page, void *object) if (WARN_ON_ONCE(order == 0)) pr_warn_once("object pointer: 0x%p\n", object); - kmemleak_free(object); + if (!nolock) + kmemleak_free(object); + kasan_kfree_large(object); kmsan_kfree_large(object); - mod_lruvec_page_state(page, NR_SLAB_UNRECLAIMABLE_B, - -(PAGE_SIZE << order)); - __ClearPageLargeKmalloc(page); - free_frozen_pages(page, order); + if (unlikely(nolock && kmemleak_may_need_free(object))) + defer_free_large_kmalloc(object); + else + __free_large_kmalloc_page(page, free_flags); } /* @@ -6753,7 +6801,7 @@ void kvfree_rcu_cb(struct rcu_head *head) if (slab) slab_free(slab->slab_cache, slab, obj, _RET_IP_); else - free_large_kmalloc(page, obj); + free_large_kmalloc(page, obj, SLAB_FREE_DEFAULT); } } @@ -6779,7 +6827,7 @@ void kfree(const void *object) slab = page_slab(page); if (!slab) { /* kmalloc_nolock() doesn't support large kmalloc */ - free_large_kmalloc(page, (void *)object); + free_large_kmalloc(page, (void *)object, SLAB_FREE_DEFAULT); return; } @@ -6793,10 +6841,11 @@ EXPORT_SYMBOL(kfree); * but may defer freeing to irq_work() in some cases. * * Intended mainly for objects allocated from kmalloc_nolock(), but can handle - * also kmem_cache_alloc() and kmalloc() objects, except large_kmalloc. + * also kmem_cache_alloc() and kmalloc() objects, including large_kmalloc. */ void kfree_nolock(const void *object) { + struct page *page; struct slab *slab; struct kmem_cache *s; void *x = (void *)object; @@ -6804,9 +6853,10 @@ void kfree_nolock(const void *object) if (unlikely(ZERO_OR_NULL_PTR(object))) return; - slab = virt_to_slab(object); + page = virt_to_page(object); + slab = page_slab(page); if (unlikely(!slab)) { - WARN_ONCE(1, "large_kmalloc is not supported by kfree_nolock()"); + free_large_kmalloc(page, (void *)object, SLAB_FREE_NOLOCK); return; } @@ -7167,7 +7217,7 @@ int build_detached_freelist(struct kmem_cache *s, size_t size, if (!s) { /* Handle kalloc'ed objects */ if (!slab) { - free_large_kmalloc(page, object); + free_large_kmalloc(page, object, SLAB_FREE_DEFAULT); df->slab = NULL; return size; } -- 2.55.0 In set_cpus_allowed_force() we use kfree_rcu() because kfree() is unsafe under p->pi_lock. With kfree_nolock() now being able to free arbitrary kmalloc() objects, we can switch to kfree_nolock() and avoid the unnecessary rcu grace period delay. Only in some cases the freeing might be deferred to irq_work(). Signed-off-by: Vlastimil Babka (SUSE) --- kernel/sched/core.c | 9 ++------- kernel/sched/sched.h | 7 +------ 2 files changed, 3 insertions(+), 13 deletions(-) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 96226707c2f6..d2929e4e23f1 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -2807,20 +2807,15 @@ void set_cpus_allowed_force(struct task_struct *p, const struct cpumask *new_mas .user_mask = NULL, .flags = SCA_USER, /* clear the user requested mask */ }; - union cpumask_rcuhead { - cpumask_t cpumask; - struct rcu_head rcu; - }; scoped_guard (__task_rq_lock, p) do_set_cpus_allowed(p, &ac); /* * Because this is called with p->pi_lock held, it is not possible - * to use kfree() here (when PREEMPT_RT=y), therefore punt to using - * kfree_rcu(). + * to use kfree() here (when PREEMPT_RT=y), thus use kfree_nolock() */ - kfree_rcu((union cpumask_rcuhead *)ac.user_mask, rcu); + kfree_nolock(ac.user_mask); } int dup_user_cpus_ptr(struct task_struct *dst, struct task_struct *src, diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 56acf502ba26..6a8d0578e963 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -2886,12 +2886,7 @@ static inline bool task_allowed_on_cpu(struct task_struct *p, int cpu) static inline cpumask_t *alloc_user_cpus_ptr(int node) { - /* - * See set_cpus_allowed_force() above for the rcu_head usage. - */ - int size = max_t(int, cpumask_size(), sizeof(struct rcu_head)); - - return kmalloc_node(size, GFP_KERNEL, node); + return kmalloc_node(cpumask_size(), GFP_KERNEL, node); } static inline struct task_struct *get_push_task(struct rq *rq) -- 2.55.0