AI Trajectory

Trajectory: (Expand/Collapse All)
Seq	Timestamp	Type	Name	Duration
0/0	2026/08/07 08:50	flow	patch-triage	0m Args: null Results: { "EnableConfigs": [ "MEMCG", "CGROUPS" ], "FocusSymbols": [ "bpf_thread_wq_init", "bpf_thread_wq_set_callback", "bpf_thread_wq_start", "bpf_try_to_free_mem_cgroup_pages", "cgroup_kthread_attach" ], "Reasoning": "The patch introduces a new BPF thread workqueue feature (`bpf_thread_wq`) that allows BPF programs to schedule work in a kthread context, optionally attached to a cgroup. It also adds a new BPF kfunc `bpf_try_to_free_mem_cgroup_pages` for memory cgroup reclaim. These are functional changes to the core BPF and memory control subsystems, reachable from userspace via BPF programs, and should be fuzzed.", "WorthFuzzing": true }
1/1	2026/08/07 08:50	action	read-patch-diff	0m Args: null Results: { "PatchDiff": "commit e72e2c008c951fa5d07abaad9d32e877879e2ea7\nAuthor: syz-cluster \u003ctriage@syzkaller.com\u003e\nDate: Fri Aug 7 08:50:40 2026 +0000\n\n syz-cluster: applied patch under review\n\ndiff --git a/include/linux/bpf.h b/include/linux/bpf.h\nindex 73bacfc6444df..d63ce83198696 100644\n--- a/include/linux/bpf.h\n+++ b/include/linux/bpf.h\n@@ -213,6 +213,7 @@ enum btf_field_type {\n \tBPF_UPTR = (1 \u003c\u003c 11),\n \tBPF_RES_SPIN_LOCK = (1 \u003c\u003c 12),\n \tBPF_TASK_WORK = (1 \u003c\u003c 13),\n+\tBPF_THREAD_WQ = (1 \u003c\u003c 14),\n };\n \n enum bpf_cgroup_storage_type {\n@@ -267,6 +268,7 @@ struct btf_record {\n \tint wq_off;\n \tint refcount_off;\n \tint task_work_off;\n+\tint thread_wq_off;\n \tstruct btf_field fields[];\n };\n \n@@ -372,6 +374,8 @@ static inline const char btf_field_type_name(enum btf_field_type type)\n \t\treturn \"bpf_refcount\";\n \tcase BPF_TASK_WORK:\n \t\treturn \"bpf_task_work\";\n+\tcase BPF_THREAD_WQ:\n+\t\treturn \"bpf_thread_wq\";\n \tdefault:\n \t\tWARN_ON_ONCE(1);\n \t\treturn \"unknown\";\n@@ -412,6 +416,8 @@ static inline u32 btf_field_type_size(enum btf_field_type type)\n \t\treturn sizeof(struct bpf_refcount);\n \tcase BPF_TASK_WORK:\n \t\treturn sizeof(struct bpf_task_work);\n+\tcase BPF_THREAD_WQ:\n+\t\treturn sizeof(struct bpf_thread_wq);\n \tdefault:\n \t\tWARN_ON_ONCE(1);\n \t\treturn 0;\n@@ -446,6 +452,8 @@ static inline u32 btf_field_type_align(enum btf_field_type type)\n \t\treturn __alignof__(struct bpf_refcount);\n \tcase BPF_TASK_WORK:\n \t\treturn __alignof__(struct bpf_task_work);\n+\tcase BPF_THREAD_WQ:\n+\t\treturn __alignof__(struct bpf_thread_wq);\n \tdefault:\n \t\tWARN_ON_ONCE(1);\n \t\treturn 0;\n@@ -478,6 +486,7 @@ static inline void bpf_obj_init_field(const struct btf_field field, void addr)\n \tcase BPF_KPTR_PERCPU:\n \tcase BPF_UPTR:\n \tcase BPF_TASK_WORK:\n+\tcase BPF_THREAD_WQ:\n \t\tbreak;\n \tdefault:\n \t\tWARN_ON_ONCE(1);\n@@ -502,6 +511,7 @@ static inline bool btf_field_is_nmi_safe(enum btf_field_type type)\n \tcase BPF_TASK_WORK:\n \tcase BPF_KPTR_UNREF:\n \tcase BPF_REFCOUNT:\n+\tcase BPF_THREAD_WQ:\n \t\treturn true;\n \tdefault:\n \t\treturn false;\n@@ -644,6 +654,7 @@ void copy_map_value_locked(struct bpf_map map, void dst, void src,\n void bpf_timer_cancel_and_free(void timer);\n void bpf_wq_cancel_and_free(void timer);\n void bpf_task_work_cancel_and_free(void timer);\n+void bpf_thread_wq_cancel_and_free(void val);\n void bpf_list_head_free(const struct btf_field field, void list_head,\n \t\t\tstruct bpf_spin_lock spin_lock);\n void bpf_rb_root_free(const struct btf_field field, void rb_root,\n@@ -701,7 +712,8 @@ bool bpf_map_meta_equal(const struct bpf_map meta0,\n \n static inline bool bpf_map_has_internal_structs(struct bpf_map map)\n {\n-\treturn btf_record_has_field(map-\u003erecord, BPF_TIMER \| BPF_WORKQUEUE \| BPF_TASK_WORK);\n+\treturn btf_record_has_field(map-\u003erecord, BPF_TIMER \| BPF_WORKQUEUE \|\n+\t\t\t\t\t\t BPF_TASK_WORK \| BPF_THREAD_WQ);\n }\n \n void bpf_map_free_internal_structs(struct bpf_map map, void obj);\n@@ -2725,6 +2737,7 @@ bool btf_record_equal(const struct btf_record rec_a, const struct btf_record r\n void bpf_obj_free_timer(const struct btf_record rec, void obj);\n void bpf_obj_free_workqueue(const struct btf_record rec, void obj);\n void bpf_obj_free_task_work(const struct btf_record rec, void obj);\n+void bpf_obj_free_thread_wq(const struct btf_record rec, void obj);\n void bpf_obj_cancel_fields(struct bpf_map map, void obj);\n void bpf_obj_free_fields(const struct btf_record rec, void obj);\n void __bpf_obj_drop_impl(void p, const struct btf_record rec, bool percpu);\ndiff --git a/include/linux/cgroup.h b/include/linux/cgroup.h\nindex f2aa46a4f871e..9b4a8dc748ace 100644\n--- a/include/linux/cgroup.h\n+++ b/include/linux/cgroup.h\n@@ -923,4 +923,6 @@ struct cgroup task_get_cgroup1(struct task_struct tsk, int hierarchy_id);\n \n struct cgroup_of_peak of_peak(struct kernfs_open_file of);\n \n+int cgroup_kthread_attach(struct cgroup cgrp, struct task_struct task);\n+\n #endif / _LINUX_CGROUP_H /\ndiff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h\nindex ffd96e8b920bd..0558520f67fe4 100644\n--- a/include/uapi/linux/bpf.h\n+++ b/include/uapi/linux/bpf.h\n@@ -7574,6 +7574,10 @@ struct bpf_wq {\n \t__u64 __opaque[2];\n } __attribute__((aligned(8)));\n \n+struct bpf_thread_wq {\n+\t__u64 __opaque[2];\n+} __attribute__((aligned(8)));\n+\n struct bpf_dynptr {\n \t__u64 __opaque[2];\n } __attribute__((aligned(8)));\ndiff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c\nindex 42414633cf263..bf6fb5f51d21a 100644\n--- a/kernel/bpf/btf.c\n+++ b/kernel/bpf/btf.c\n@@ -3665,6 +3665,7 @@ static int btf_get_field_type(const struct btf btf, const struct btf_type var_\n \t\t{ BPF_TIMER, \"bpf_timer\", true },\n \t\t{ BPF_WORKQUEUE, \"bpf_wq\", true },\n \t\t{ BPF_TASK_WORK, \"bpf_task_work\", true },\n+\t\t{ BPF_THREAD_WQ, \"bpf_thread_wq\", true },\n \t\t{ BPF_LIST_HEAD, \"bpf_list_head\", false },\n \t\t{ BPF_LIST_NODE, \"bpf_list_node\", false },\n \t\t{ BPF_RB_ROOT, \"bpf_rb_root\", false },\n@@ -3850,6 +3851,7 @@ static int btf_find_field_one(const struct btf btf,\n \tcase BPF_RB_NODE:\n \tcase BPF_REFCOUNT:\n \tcase BPF_TASK_WORK:\n+\tcase BPF_THREAD_WQ:\n \t\tret = btf_find_struct(btf, var_type, off, sz, field_type,\n \t\t\t\t info_cnt ? \u0026info[0] : \u0026tmp);\n \t\tif (ret \u003c 0)\n@@ -4145,6 +4147,7 @@ struct btf_record btf_parse_fields(const struct btf btf, const struct btf_type\n \trec-\u003ewq_off = -EINVAL;\n \trec-\u003erefcount_off = -EINVAL;\n \trec-\u003etask_work_off = -EINVAL;\n+\trec-\u003ethread_wq_off = -EINVAL;\n \tfor (i = 0; i \u003c cnt; i++) {\n \t\tfield_type_size = btf_field_type_size(info_arr[i].type);\n \t\tif (info_arr[i].off + field_type_size \u003e value_size) {\n@@ -4188,6 +4191,10 @@ struct btf_record btf_parse_fields(const struct btf btf, const struct btf_type\n \t\t\tWARN_ON_ONCE(rec-\u003etask_work_off \u003e= 0);\n \t\t\trec-\u003etask_work_off = rec-\u003efields[i].offset;\n \t\t\tbreak;\n+\t\tcase BPF_THREAD_WQ:\n+\t\t\tWARN_ON_ONCE(rec-\u003ethread_wq_off \u003e= 0);\n+\t\t\trec-\u003ethread_wq_off = rec-\u003efields[i].offset;\n+\t\t\tbreak;\n \t\tcase BPF_REFCOUNT:\n \t\t\tWARN_ON_ONCE(rec-\u003erefcount_off \u003e= 0);\n \t\t\t/* Cache offset for faster lookup at runtime /\ndiff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c\nindex 6388b6b23e490..8e01bad1b16a0 100644\n--- a/kernel/bpf/helpers.c\n+++ b/kernel/bpf/helpers.c\n@@ -29,6 +29,8 @@\n #include \u003clinux/task_work.h\u003e\n #include \u003clinux/irq_work.h\u003e\n #include \u003clinux/buildid.h\u003e\n+#include \u003clinux/kthread.h\u003e\n+#include \u003clinux/jhash.h\u003e\n \n #include \"../../lib/kstrtox.h\"\n \n@@ -1110,6 +1112,17 @@ static void map_key_from_value(struct bpf_map map, void value, u32 arr_idx)\n \treturn (void )value - round_up(map-\u003ekey_size, 8);\n }\n \n+static u32 bpf_map_elem_id_from_value(struct bpf_map map, void value)\n+{\n+\tu32 arr_idx;\n+\tvoid key;\n+\n+\tkey = map_key_from_value(map, value, \u0026arr_idx);\n+\tif (map-\u003emap_type == BPF_MAP_TYPE_ARRAY)\n+\t\treturn arr_idx;\n+\treturn jhash(key, map-\u003ekey_size, 0);\n+}\n+\n enum bpf_async_type {\n \tBPF_ASYNC_TYPE_TIMER = 0,\n \tBPF_ASYNC_TYPE_WQ,\n@@ -4769,6 +4782,397 @@ __bpf_kfunc int bpf_timer_cancel_async(struct bpf_timer timer)\n \t}\n }\n \n+/\n+ BPF thread workqueue (kthread_worker based) implementation\n+ \n+ Why bpf_thread_wq does NOT use the bpf_async infrastructure:\n+ \n+ bpf_timer and bpf_wq share a common cleanup path via bpf_async:\n+ \n+ bpf_async_cancel_and_free()\n+ * -\u003e bpf_async_schedule_op()\n+ * -\u003e irq_work_queue() // schedule from any context\n+ * -\u003e bpf_async_process_op() // runs in hardirq context\n+ * -\u003e bpf_wq_work() / timer cb // atomic, non-sleepable\n+ \n+ This works for timer and workqueue because their callbacks and\n+ * cancellation (hrtimer_cancel / bpf_wq_cancel_and_free) complete\n+ * synchronously and need not sleep. The hardirq context is sufficient.\n+ \n+ bpf_thread_wq is different: cancellation and final cleanup may need to\n+ * sleep:\n+ \n+ kthread_cancel_work_sync() - waits for the work to finish, can\n+ * schedule out if the work is running.\n+ * kthread_destroy_worker() - stops the kthread, internally\n+ * synchronizes with kthread exit.\n+ * cgroup_put() (final put) - may acquire cgroup_mutex and other\n+ * sleeping locks during offline.\n+ \n+ None of these can be called from hardirq context (irq_work). Doing so\n+ * would trigger might_sleep() warnings or deadlock.\n+ \n+ Therefore bpf_thread_wq implements its own RCU-based cleanup:\n+ \n+ bpf_thread_wq_cancel_and_free() // sleepable (called from\n+ * // map free / elem delete)\n+ * -\u003e xchg(ctx, NULL) // detach atomically\n+ * -\u003e kthread_cancel_work_sync() // ok to sleep\n+ * -\u003e bpf_thread_wq_ctx_put()\n+ * -\u003e call_rcu_tasks_trace() // wait for BPF callbacks\n+ * -\u003e schedule_work() // switch to sleepable context\n+ * -\u003e bpf_thread_wq_destroy_work_fn()\n+ * -\u003e bpf_prog_put()\n+ * -\u003e cgroup_put()\n+ * -\u003e kthread_destroy_worker()\n+ * -\u003e kfree(ctx)\n+ \n+ The same design choice was made for bpf_task_work, which also avoids\n+ * bpf_async because task_work cancellation synchronizes with the task\n+ * and may sleep.\n+ /\n+\n+struct bpf_thread_wq_ctx {\n+\tstruct kthread_worker worker;\n+\tstruct kthread_work work;\n+\tstruct bpf_prog prog;\n+\tbpf_callback_t callback_fn;\n+\tstruct bpf_map map;\n+\tvoid value;\n+\tstruct cgroup cgrp;\n+\trefcount_t refcnt;\n+\tstruct rcu_head rcu;\n+\tstruct work_struct destroy_work;\n+};\n+\n+/* Kernel-internal representation that fits in struct bpf_thread_wq /\n+struct bpf_thread_wq_kern {\n+\tstruct bpf_thread_wq_ctx ctx;\n+} __aligned(8);\n+\n+static void bpf_thread_wq_destroy_work_fn(struct work_struct work)\n+{\n+\tstruct bpf_thread_wq_ctx ctx = container_of(work,\n+\t\t\t\t\t\t struct bpf_thread_wq_ctx,\n+\t\t\t\t\t\t destroy_work);\n+\n+\tif (ctx-\u003eprog)\n+\t\tbpf_prog_put(ctx-\u003eprog);\n+\tif (ctx-\u003ecgrp)\n+\t\tcgroup_put(ctx-\u003ecgrp);\n+\tif (ctx-\u003eworker)\n+\t\tkthread_destroy_worker(ctx-\u003eworker);\n+\tkfree(ctx);\n+}\n+\n+static void bpf_thread_wq_ctx_free_rcu(struct rcu_head rcu)\n+{\n+\tstruct bpf_thread_wq_ctx ctx = container_of(rcu,\n+\t\t\t\t\t\t struct bpf_thread_wq_ctx,\n+\t\t\t\t\t\t rcu);\n+\n+\tINIT_WORK(\u0026ctx-\u003edestroy_work, bpf_thread_wq_destroy_work_fn);\n+\tschedule_work(\u0026ctx-\u003edestroy_work);\n+}\n+\n+static void bpf_thread_wq_ctx_put(struct bpf_thread_wq_ctx ctx)\n+{\n+\tif (!refcount_dec_and_test(\u0026ctx-\u003erefcnt))\n+\t\treturn;\n+\tcall_rcu_tasks_trace(\u0026ctx-\u003ercu, bpf_thread_wq_ctx_free_rcu);\n+}\n+\n+static void bpf_thread_wq_work_fn(struct kthread_work work)\n+{\n+\tstruct bpf_thread_wq_ctx ctx = container_of(work,\n+\t\t\t\t\t\t struct bpf_thread_wq_ctx,\n+\t\t\t\t\t\t work);\n+\tbpf_callback_t callback_fn;\n+\tvoid value = ctx-\u003evalue;\n+\tstruct bpf_map map = ctx-\u003emap;\n+\tvoid key;\n+\tu32 idx;\n+\n+\tBTF_TYPE_EMIT(struct bpf_thread_wq);\n+\n+\tcallback_fn = READ_ONCE(ctx-\u003ecallback_fn);\n+\tif (!callback_fn)\n+\t\tgoto out;\n+\tkey = map_key_from_value(map, value, \u0026idx);\n+\n+\trcu_read_lock_trace();\n+\tmigrate_disable();\n+\n+\tcallback_fn = READ_ONCE(ctx-\u003ecallback_fn);\n+\tif (callback_fn)\n+\t\tcallback_fn((u64)(long)map, (u64)(long)key, (u64)(long)value,\n+\t\t\t 0, 0);\n+\n+\tmigrate_enable();\n+\trcu_read_unlock_trace();\n+\n+out:\n+\tbpf_thread_wq_ctx_put(ctx);\n+}\n+\n+/\n+ bpf_thread_wq_init() creates a kthread worker and may attach it to a cgroup.\n+ * The helpers used here can sleep and acquire several locks through kthread\n+ * creation/destruction, cgroup lookup and cgroup kthread attachment. Keep this\n+ * kfunc available only to BPF_PROG_TYPE_SYSCALL programs so it is not invoked\n+ * from BPF program contexts that already hold locks which could deadlock with\n+ * those paths.\n+ /\n+__bpf_kfunc int bpf_thread_wq_init(struct bpf_thread_wq twq, void p__map,\n+\t\t\t\t u64 cgroup_id, unsigned int flags)\n+{\n+\tstruct bpf_thread_wq_kern twk = (struct bpf_thread_wq_kern )twq;\n+\tstruct bpf_map map = p__map;\n+\tstruct bpf_thread_wq_ctx ctx, old_ctx;\n+\tstruct kthread_worker worker;\n+\tstruct cgroup cgrp = NULL;\n+\tvoid value;\n+\tu32 elem_id;\n+\tint err;\n+\n+\tBUILD_BUG_ON(sizeof(struct bpf_thread_wq_kern)\n+\t\t\t\u003e sizeof(struct bpf_thread_wq));\n+\tBUILD_BUG_ON(__alignof__(struct bpf_thread_wq_kern)\n+\t\t\t!= __alignof__(struct bpf_thread_wq));\n+\n+\tif (flags)\n+\t\treturn -EINVAL;\n+\n+\told_ctx = READ_ONCE(twk-\u003ectx);\n+\tif (old_ctx)\n+\t\treturn -EBUSY;\n+\n+\tvalue = (void )twq - map-\u003erecord-\u003ethread_wq_off;\n+\telem_id = bpf_map_elem_id_from_value(map, value);\n+\tworker = kthread_run_worker(0, \"bpf_twq/%d/%x\", map-\u003eid, elem_id);\n+\tif (IS_ERR(worker))\n+\t\treturn PTR_ERR(worker);\n+\n+\t/* Setup ctx. /\n+\tctx = bpf_map_kmalloc_nolock(map, sizeof(ctx), GFP_KERNEL,\n+\t\t\t\t map-\u003enuma_node);\n+\tif (!ctx) {\n+\t\terr = -ENOMEM;\n+\t\tgoto destroy_worker;\n+\t}\n+\tmemset(ctx, 0, sizeof(ctx));\n+\tctx-\u003eworker = worker;\n+\tctx-\u003emap = map;\n+\tctx-\u003evalue = value;\n+\trefcount_set(\u0026ctx-\u003erefcnt, 1);\n+\tkthread_init_work(\u0026ctx-\u003ework, bpf_thread_wq_work_fn);\n+\n+\tif (cgroup_id) {\n+#ifdef CONFIG_CGROUPS\n+\t\tcgrp = cgroup_get_from_id(cgroup_id);\n+\t\tif (IS_ERR(cgrp)) {\n+\t\t\terr = PTR_ERR(cgrp);\n+\t\t\tgoto kfree_ctx;\n+\t\t}\n+\t\tctx-\u003ecgrp = cgrp;\n+\n+\t\t/\n+\t\t * kthread_run_worker() wakes the kthread, but it may not have\n+\t\t * executed cgroup_kthread_ready() yet, which clears\n+\t\t * no_cgroup_migration.\n+\t\t * Do a queue work and flush to wait the kthread run.\n+\t\t /\n+\t\trefcount_inc(\u0026ctx-\u003erefcnt);\n+\t\tif (!kthread_queue_work(ctx-\u003eworker, \u0026ctx-\u003ework)) {\n+\t\t\trefcount_dec(\u0026ctx-\u003erefcnt);\n+\t\t\terr = -EBUSY;\n+\t\t\tgoto cgroup_put;\n+\t\t}\n+\t\tkthread_flush_work(\u0026ctx-\u003ework);\n+\n+\t\tif (worker-\u003etask-\u003eno_cgroup_migration) {\n+\t\t\terr = -EAGAIN;\n+\t\t\tgoto cgroup_put;\n+\t\t}\n+\n+\t\terr = cgroup_kthread_attach(cgrp, worker-\u003etask);\n+\t\tif (err)\n+\t\t\tgoto cgroup_put;\n+#else\n+\t\terr = -EOPNOTSUPP;\n+\t\tgoto kfree_ctx;\n+#endif\n+\t}\n+\n+\told_ctx = cmpxchg(\u0026twk-\u003ectx, NULL, ctx);\n+\tif (old_ctx) {\n+\t\terr = -EBUSY;\n+\t\tgoto cgroup_put;\n+\t}\n+\n+\t/\n+\t * Paired with the map destruction path. Ensures that ctx is globally\n+\t * visible before we check map-\u003eusercnt.\n+\t * If usercnt has dropped to zero, the destruction path will either see\n+\t * the ctx (and cancel it) or we see usercnt == 0 here and cancel\n+\t * ourselves.\n+\t * Without this barrier, a CPU could reorder the load of usercnt before\n+\t * the cmpxchg store becomes visible, breaking the mutual exclusion\n+\t * guarantee.\n+\t /\n+\tsmp_mb();\n+\n+\tif (!atomic64_read(\u0026map-\u003eusercnt)) {\n+\t\tbpf_thread_wq_cancel_and_free(twq);\n+\t\treturn -EPERM;\n+\t}\n+\n+\treturn 0;\n+\n+cgroup_put:\n+#ifdef CONFIG_CGROUPS\n+\tif (cgrp)\n+\t\tcgroup_put(cgrp);\n+#endif\n+kfree_ctx:\n+\t/\n+\t * Not use bpf_thread_wq_ctx_put because ctx has not yet entered\n+\t * the running state.\n+\t /\n+\tkfree(ctx);\n+destroy_worker:\n+\tkthread_destroy_worker(worker);\n+\treturn err;\n+}\n+\n+__bpf_kfunc int bpf_thread_wq_set_callback(struct bpf_thread_wq twq,\n+\t\t\t\t\t int (callback_fn)(void map,\n+\t\t\t\t\t\t\t int key,\n+\t\t\t\t\t\t\t void value),\n+\t\t\t\t\t unsigned int flags,\n+\t\t\t\t\t struct bpf_prog_aux aux)\n+{\n+\tstruct bpf_thread_wq_kern twk = (struct bpf_thread_wq_kern )twq;\n+\tstruct bpf_thread_wq_ctx ctx;\n+\tstruct bpf_prog prog;\n+\n+\tif (flags)\n+\t\treturn -EINVAL;\n+\n+\tctx = READ_ONCE(twk-\u003ectx);\n+\tif (!ctx)\n+\t\treturn -EINVAL;\n+\n+\tprog = bpf_prog_inc_not_zero(aux-\u003eprog);\n+\tif (IS_ERR(prog))\n+\t\treturn PTR_ERR(prog);\n+\n+\t/\n+\t Allow set_callback only once to prevent UAF: a concurrent\n+\t * set_callback could bpf_prog_put() the prog while the worker\n+\t * kthread is still executing its callback.\n+\t /\n+\tif (cmpxchg(\u0026ctx-\u003eprog, NULL, prog) != NULL) {\n+\t\tbpf_prog_put(prog);\n+\t\treturn -EBUSY;\n+\t}\n+\t/\n+\t * Safe to set callback_fn after prog: bpf_thread_wq_start() and\n+\t * bpf_thread_wq_work_fn() both check callback_fn with READ_ONCE()\n+\t * and bail out if it is still NULL.\n+\t /\n+\tWRITE_ONCE(ctx-\u003ecallback_fn, (void )callback_fn);\n+\n+\treturn 0;\n+}\n+\n+__bpf_kfunc int\n+bpf_thread_wq_start(struct bpf_thread_wq twq, unsigned int flags)\n+{\n+\tstruct bpf_thread_wq_kern twk = (struct bpf_thread_wq_kern )twq;\n+\tstruct bpf_thread_wq_ctx ctx;\n+\tint err;\n+\n+\tif (flags)\n+\t\treturn -EINVAL;\n+\n+\trcu_read_lock_trace();\n+\n+\terr = 0;\n+\n+\tctx = READ_ONCE(twk-\u003ectx);\n+\tif (!ctx \|\| !READ_ONCE(ctx-\u003ecallback_fn)) {\n+\t\terr = -EINVAL;\n+\t\tgoto unlock;\n+\t}\n+\n+\tif (!refcount_inc_not_zero(\u0026ctx-\u003erefcnt))\n+\t\terr = -ENOENT;\n+\n+unlock:\n+\trcu_read_unlock_trace();\n+\tif (err)\n+\t\treturn err;\n+\n+\tif (!kthread_queue_work(ctx-\u003eworker, \u0026ctx-\u003ework)) {\n+\t\tbpf_thread_wq_ctx_put(ctx);\n+\t\treturn -EBUSY;\n+\t}\n+\n+\treturn 0;\n+}\n+\n+void bpf_thread_wq_cancel_and_free(void val)\n+{\n+\tstruct bpf_thread_wq_kern twk = val;\n+\tstruct bpf_thread_wq_ctx ctx;\n+\n+\tctx = xchg(\u0026twk-\u003ectx, NULL);\n+\tif (!ctx)\n+\t\treturn;\n+\n+\tmight_sleep();\n+\n+\t/\n+\t * Prevent future callbacks from running and wait for any\n+\t * in-progress execution to finish.\n+\t /\n+\tWRITE_ONCE(ctx-\u003ecallback_fn, NULL);\n+\t/\n+\t * kthread_cancel_work_sync() returns true when it dequeues a pending\n+\t * work item from the work_list without executing it. Each successful\n+\t * bpf_thread_wq_start() call increments ctx-\u003erefcnt and relies on the\n+\t * subsequent bpf_thread_wq_work_fn() execution to release that\n+\t * reference via bpf_thread_wq_ctx_put(). If the work was pending and\n+\t * got cancelled here, work_fn will never run for that queued instance,\n+\t * so we must drop the reference ourselves to avoid a permanent refcount\n+\t * leak.\n+\t \n+\t This covers two scenarios uniformly:\n+\t * 1. The work is purely pending (not currently executing) - e.g. a\n+\t * normal bpf_thread_wq_start() call queued it but the worker\n+\t * thread hasn't picked it up yet.\n+\t * 2. The work is currently in-flight AND was self-rescheduled from\n+\t * within the callback - kthread_cancel_work_sync() dequeues the\n+\t * re-queued pending node and then waits for the in-flight\n+\t * execution to complete.\n+\t * In both cases the return value is true, indicating one orphaned\n+\t * reference that needs to be released here.\n+\t /\n+\tif (kthread_cancel_work_sync(\u0026ctx-\u003ework))\n+\t\tbpf_thread_wq_ctx_put(ctx);\n+\n+\t/\n+\t * Drop our own reference. If the work was still in-flight above,\n+\t * the refcount won't hit zero here - it will reach zero when the\n+\t * work path calls bpf_thread_wq_ctx_put() upon completion. Either\n+\t * way, final cleanup (worker destruction, prog put, cgroup put,\n+\t * kfree) happens exclusively in the RCU callback to keep the\n+\t * teardown path single-threaded.\n+\t /\n+\tbpf_thread_wq_ctx_put(ctx);\n+}\n+\n __bpf_kfunc_end_defs();\n \n static void bpf_task_work_cancel_scheduled(struct irq_work irq_work)\n@@ -4917,6 +5321,8 @@ BTF_ID_FLAGS(func, bpf_modify_return_test_tp)\n BTF_ID_FLAGS(func, bpf_wq_init)\n BTF_ID_FLAGS(func, bpf_wq_set_callback, KF_IMPLICIT_ARGS)\n BTF_ID_FLAGS(func, bpf_wq_start)\n+BTF_ID_FLAGS(func, bpf_thread_wq_set_callback, KF_IMPLICIT_ARGS)\n+BTF_ID_FLAGS(func, bpf_thread_wq_start)\n BTF_ID_FLAGS(func, bpf_preempt_disable)\n BTF_ID_FLAGS(func, bpf_preempt_enable)\n BTF_ID_FLAGS(func, bpf_iter_bits_new, KF_ITER_NEW)\n@@ -4978,6 +5384,15 @@ static const struct btf_kfunc_id_set common_kfunc_set = {\n \t.set = \u0026common_btf_ids,\n };\n \n+BTF_KFUNCS_START(syscall_btf_ids)\n+BTF_ID_FLAGS(func, bpf_thread_wq_init, KF_SLEEPABLE)\n+BTF_KFUNCS_END(syscall_btf_ids)\n+\n+static const struct btf_kfunc_id_set syscall_kfunc_set = {\n+\t.owner = THIS_MODULE,\n+\t.set = \u0026syscall_btf_ids,\n+};\n+\n static int __init kfunc_init(void)\n {\n \tint ret;\n@@ -5000,6 +5415,7 @@ static int __init kfunc_init(void)\n \tret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, \u0026generic_kfunc_set);\n \tret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL, \u0026generic_kfunc_set);\n \tret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_CGROUP_SKB, \u0026generic_kfunc_set);\n+\tret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL, \u0026syscall_kfunc_set);\n \tret = ret ?: register_btf_id_dtor_kfuncs(generic_dtors,\n \t\t\t\t\t\t ARRAY_SIZE(generic_dtors),\n \t\t\t\t\t\t THIS_MODULE);\n@@ -5037,4 +5453,6 @@ void bpf_map_free_internal_structs(struct bpf_map map, void val)\n \t\tbpf_obj_free_workqueue(map-\u003erecord, val);\n \tif (btf_record_has_field(map-\u003erecord, BPF_TASK_WORK))\n \t\tbpf_obj_free_task_work(map-\u003erecord, val);\n+\tif (btf_record_has_field(map-\u003erecord, BPF_THREAD_WQ))\n+\t\tbpf_obj_free_thread_wq(map-\u003erecord, val);\n }\ndiff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c\nindex 8d111da886553..dea14823bacd2 100644\n--- a/kernel/bpf/syscall.c\n+++ b/kernel/bpf/syscall.c\n@@ -688,6 +688,7 @@ void btf_record_free(struct btf_record rec)\n \t\tcase BPF_REFCOUNT:\n \t\tcase BPF_WORKQUEUE:\n \t\tcase BPF_TASK_WORK:\n+\t\tcase BPF_THREAD_WQ:\n \t\t\t/ Nothing to release /\n \t\t\tbreak;\n \t\tdefault:\n@@ -742,6 +743,7 @@ struct btf_record btf_record_dup(const struct btf_record rec)\n \t\tcase BPF_REFCOUNT:\n \t\tcase BPF_WORKQUEUE:\n \t\tcase BPF_TASK_WORK:\n+\t\tcase BPF_THREAD_WQ:\n \t\t\t/ Nothing to acquire /\n \t\t\tbreak;\n \t\tdefault:\n@@ -807,6 +809,13 @@ void bpf_obj_free_task_work(const struct btf_record rec, void obj)\n \tbpf_task_work_cancel_and_free(obj + rec-\u003etask_work_off);\n }\n \n+void bpf_obj_free_thread_wq(const struct btf_record rec, void obj)\n+{\n+\tif (WARN_ON_ONCE(!btf_record_has_field(rec, BPF_THREAD_WQ)))\n+\t\treturn;\n+\tbpf_thread_wq_cancel_and_free(obj + rec-\u003ethread_wq_off);\n+}\n+\n void bpf_obj_cancel_fields(struct bpf_map map, void obj)\n {\n \tbpf_map_free_internal_structs(map, obj);\n@@ -839,6 +848,9 @@ void bpf_obj_free_fields(const struct btf_record rec, void obj)\n \t\tcase BPF_TASK_WORK:\n \t\t\tbpf_task_work_cancel_and_free(field_ptr);\n \t\t\tbreak;\n+\t\tcase BPF_THREAD_WQ:\n+\t\t\tbpf_thread_wq_cancel_and_free(field_ptr);\n+\t\t\tbreak;\n \t\tcase BPF_KPTR_UNREF:\n \t\t\tWRITE_ONCE((u64 )field_ptr, 0);\n \t\t\tbreak;\n@@ -1265,7 +1277,7 @@ static int map_check_btf(struct bpf_map map, struct bpf_token token,\n \tmap-\u003erecord = btf_parse_fields(btf, value_type,\n \t\t\t\t BPF_SPIN_LOCK \| BPF_RES_SPIN_LOCK \| BPF_TIMER \| BPF_KPTR \| BPF_LIST_HEAD \|\n \t\t\t\t BPF_RB_ROOT \| BPF_REFCOUNT \| BPF_WORKQUEUE \| BPF_UPTR \|\n-\t\t\t\t BPF_TASK_WORK,\n+\t\t\t\t BPF_TASK_WORK \| BPF_THREAD_WQ,\n \t\t\t\t map-\u003evalue_size);\n \tif (!IS_ERR_OR_NULL(map-\u003erecord)) {\n \t\tint i;\n@@ -1299,6 +1311,7 @@ static int map_check_btf(struct bpf_map map, struct bpf_token token,\n \t\t\tcase BPF_TIMER:\n \t\t\tcase BPF_WORKQUEUE:\n \t\t\tcase BPF_TASK_WORK:\n+\t\t\tcase BPF_THREAD_WQ:\n \t\t\t\tif (map-\u003emap_type != BPF_MAP_TYPE_HASH \u0026\u0026\n \t\t\t\t map-\u003emap_type != BPF_MAP_TYPE_RHASH \u0026\u0026\n \t\t\t\t map-\u003emap_type != BPF_MAP_TYPE_LRU_HASH \u0026\u0026\ndiff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c\nindex e6233c0081d10..44e79fe480b6b 100644\n--- a/kernel/bpf/verifier.c\n+++ b/kernel/bpf/verifier.c\n@@ -477,6 +477,7 @@ static bool is_async_callback_calling_kfunc(u32 btf_id);\n static bool is_callback_calling_kfunc(u32 btf_id);\n \n static bool is_bpf_wq_set_callback_kfunc(u32 btf_id);\n+static bool is_bpf_thread_wq_set_callback_kfunc(u32 btf_id);\n static bool is_task_work_add_kfunc(u32 func_id);\n \n static bool is_sync_callback_calling_function(enum bpf_func_id func_id)\n@@ -516,9 +517,11 @@ static bool is_async_cb_sleepable(struct bpf_verifier_env env, struct bpf_insn\n \tif (bpf_helper_call(insn) \u0026\u0026 insn-\u003eimm == BPF_FUNC_timer_set_callback)\n \t\treturn false;\n \n-\t/* bpf_wq and bpf_task_work callbacks are always sleepable. /\n+\t/ bpf_wq, bpf_thread_wq and bpf_task_work callbacks are always sleepable. /\n \tif (bpf_pseudo_kfunc_call(insn) \u0026\u0026 insn-\u003eoff == 0 \u0026\u0026\n-\t (is_bpf_wq_set_callback_kfunc(insn-\u003eimm) \|\| is_task_work_add_kfunc(insn-\u003eimm)))\n+\t (is_bpf_wq_set_callback_kfunc(insn-\u003eimm) \|\|\n+\t is_bpf_thread_wq_set_callback_kfunc(insn-\u003eimm) \|\|\n+\t is_task_work_add_kfunc(insn-\u003eimm)))\n \t\treturn true;\n \n \tverifier_bug(env, \"unhandled async callback in is_async_cb_sleepable\");\n@@ -1871,7 +1874,10 @@ static void refine_map_lookup_value(struct bpf_reg_state reg)\n \t\t * as UID of the inner map.\n \t\t /\n \t\tif (btf_record_has_field(map-\u003einner_map_meta-\u003erecord,\n-\t\t\t\t\t BPF_TIMER \| BPF_WORKQUEUE \| BPF_TASK_WORK))\n+\t\t\t\t\t BPF_TIMER \|\n+\t\t\t\t\t BPF_WORKQUEUE \|\n+\t\t\t\t\t BPF_TASK_WORK \|\n+\t\t\t\t\t BPF_THREAD_WQ))\n \t\t\treg-\u003emap_uid = reg-\u003eid;\n \t} else if (map-\u003emap_type == BPF_MAP_TYPE_XSKMAP) {\n \t\treg-\u003etype = PTR_TO_XDP_SOCK \| maybe_null;\n@@ -7243,6 +7249,9 @@ static int check_map_field_pointer(struct bpf_verifier_env env, struct bpf_reg_\n \tcase BPF_WORKQUEUE:\n \t\tfield_off = map-\u003erecord-\u003ewq_off;\n \t\tbreak;\n+\tcase BPF_THREAD_WQ:\n+\t\tfield_off = map-\u003erecord-\u003ethread_wq_off;\n+\t\tbreak;\n \tdefault:\n \t\tverifier_bug(env, \"unsupported BTF field type: %s\\n\", struct_name);\n \t\treturn -EINVAL;\n@@ -10940,6 +10949,7 @@ enum {\n \tKF_ARG_WORKQUEUE_ID,\n \tKF_ARG_RES_SPIN_LOCK_ID,\n \tKF_ARG_TASK_WORK_ID,\n+\tKF_ARG_THREAD_WQ_ID,\n \tKF_ARG_PROG_AUX_ID,\n \tKF_ARG_TIMER_ID\n };\n@@ -10953,6 +10963,7 @@ BTF_ID(struct, bpf_rb_node)\n BTF_ID(struct, bpf_wq)\n BTF_ID(struct, bpf_res_spin_lock)\n BTF_ID(struct, bpf_task_work)\n+BTF_ID(struct, bpf_thread_wq)\n BTF_ID(struct, bpf_prog_aux)\n BTF_ID(struct, bpf_timer)\n \n@@ -11013,6 +11024,11 @@ static bool is_kfunc_arg_task_work(const struct btf btf, const struct btf_param\n \treturn __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_TASK_WORK_ID);\n }\n \n+static bool is_kfunc_arg_thread_wq(const struct btf btf, const struct btf_param arg)\n+{\n+\treturn __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_THREAD_WQ_ID);\n+}\n+\n static bool is_kfunc_arg_res_spin_lock(const struct btf btf, const struct btf_param arg)\n {\n \treturn __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_RES_SPIN_LOCK_ID);\n@@ -11132,6 +11148,7 @@ enum kfunc_ptr_arg_type {\n \tKF_ARG_PTR_TO_IRQ_FLAG,\n \tKF_ARG_PTR_TO_RES_SPIN_LOCK,\n \tKF_ARG_PTR_TO_TASK_WORK,\n+\tKF_ARG_PTR_TO_THREAD_WQ,\n };\n \n enum special_kfunc_type {\n@@ -11178,6 +11195,7 @@ enum special_kfunc_type {\n \tKF_bpf_percpu_obj_drop,\n \tKF_bpf_throw,\n \tKF_bpf_wq_set_callback,\n+\tKF_bpf_thread_wq_set_callback,\n \tKF_bpf_preempt_disable,\n \tKF_bpf_preempt_enable,\n \tKF_bpf_iter_css_task_new,\n@@ -11258,6 +11276,7 @@ BTF_ID(func, bpf_percpu_obj_drop_impl)\n BTF_ID(func, bpf_percpu_obj_drop)\n BTF_ID(func, bpf_throw)\n BTF_ID(func, bpf_wq_set_callback)\n+BTF_ID(func, bpf_thread_wq_set_callback)\n BTF_ID(func, bpf_preempt_disable)\n BTF_ID(func, bpf_preempt_enable)\n #ifdef CONFIG_CGROUPS\n@@ -11460,6 +11479,8 @@ get_kfunc_arg_type(struct bpf_verifier_env env, struct bpf_call_arg_meta meta,\n \t\targ_type = KF_ARG_PTR_TO_TIMER;\n \telse if (is_kfunc_arg_task_work(meta-\u003ebtf, \u0026args[arg]))\n \t\targ_type = KF_ARG_PTR_TO_TASK_WORK;\n+\telse if (is_kfunc_arg_thread_wq(meta-\u003ebtf, \u0026args[arg]))\n+\t\targ_type = KF_ARG_PTR_TO_THREAD_WQ;\n \telse if (is_kfunc_arg_irq_flag(meta-\u003ebtf, \u0026args[arg]))\n \t\targ_type = KF_ARG_PTR_TO_IRQ_FLAG;\n \telse if (is_kfunc_arg_res_spin_lock(meta-\u003ebtf, \u0026args[arg]))\n@@ -11833,6 +11854,7 @@ static bool is_sync_callback_calling_kfunc(u32 btf_id)\n static bool is_async_callback_calling_kfunc(u32 btf_id)\n {\n \treturn is_bpf_wq_set_callback_kfunc(btf_id) \|\|\n+\t is_bpf_thread_wq_set_callback_kfunc(btf_id) \|\|\n \t is_task_work_add_kfunc(btf_id);\n }\n \n@@ -11847,6 +11869,11 @@ static bool is_bpf_wq_set_callback_kfunc(u32 btf_id)\n \treturn btf_id == special_kfunc_list[KF_bpf_wq_set_callback];\n }\n \n+static bool is_bpf_thread_wq_set_callback_kfunc(u32 btf_id)\n+{\n+\treturn btf_id == special_kfunc_list[KF_bpf_thread_wq_set_callback];\n+}\n+\n static bool is_callback_calling_kfunc(u32 btf_id)\n {\n \treturn is_sync_callback_calling_kfunc(btf_id) \|\|\n@@ -12192,6 +12219,7 @@ static int check_kfunc_args(struct bpf_verifier_env env, struct bpf_call_arg_me\n \t\tcase KF_ARG_PTR_TO_WORKQUEUE:\n \t\tcase KF_ARG_PTR_TO_TIMER:\n \t\tcase KF_ARG_PTR_TO_TASK_WORK:\n+\t\tcase KF_ARG_PTR_TO_THREAD_WQ:\n \t\tcase KF_ARG_PTR_TO_IRQ_FLAG:\n \t\tcase KF_ARG_PTR_TO_RES_SPIN_LOCK:\n \t\t\tbreak;\n@@ -13139,6 +13167,16 @@ static int check_kfunc_call(struct bpf_verifier_env env, struct bpf_insn insn,\n \t\t}\n \t}\n \n+\tif (is_bpf_thread_wq_set_callback_kfunc(meta.func_id)) {\n+\t\terr = push_callback_call(env, insn, insn_idx, meta.subprogno,\n+\t\t\t\t\t set_timer_callback_state);\n+\t\tif (err) {\n+\t\t\tverbose(env, \"kfunc %s#%d failed callback verification\\n\",\n+\t\t\t\tfunc_name, meta.func_id);\n+\t\t\treturn err;\n+\t\t}\n+\t}\n+\n \tif (is_task_work_add_kfunc(meta.func_id)) {\n \t\terr = push_callback_call(env, insn, insn_idx, meta.subprogno,\n \t\t\t\t\t set_task_work_schedule_callback_state);\ndiff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c\nindex 38f8d9df8fbc4..164e069c75762 100644\n--- a/kernel/cgroup/cgroup.c\n+++ b/kernel/cgroup/cgroup.c\n@@ -3042,6 +3042,19 @@ int cgroup_attach_task(struct cgroup dst_cgrp, struct task_struct leader,\n \treturn ret;\n }\n \n+int cgroup_kthread_attach(struct cgroup cgrp, struct task_struct task)\n+{\n+\tint ret;\n+\n+\tcgroup_lock();\n+\tcgroup_attach_lock(CGRP_ATTACH_LOCK_GLOBAL, NULL);\n+\tret = cgroup_attach_task(cgrp, task, false);\n+\tcgroup_attach_unlock(CGRP_ATTACH_LOCK_GLOBAL, NULL);\n+\tcgroup_unlock();\n+\n+\treturn ret;\n+}\n+\n struct task_struct cgroup_procs_write_start(char buf, bool threadgroup,\n \t\t\t\t\t enum cgroup_attach_lock_mode lock_mode)\n {\ndiff --git a/mm/bpf_memcontrol.c b/mm/bpf_memcontrol.c\nindex 716df49d76477..3f7a5c97e1355 100644\n--- a/mm/bpf_memcontrol.c\n+++ b/mm/bpf_memcontrol.c\n@@ -6,6 +6,7 @@\n /\n \n #include \u003clinux/memcontrol.h\u003e\n+#include \u003clinux/swap.h\u003e\n #include \u003clinux/bpf.h\u003e\n \n __bpf_kfunc_start_defs();\n@@ -159,6 +160,61 @@ __bpf_kfunc void bpf_mem_cgroup_flush_stats(struct mem_cgroup memcg)\n \tmem_cgroup_flush_stats(memcg);\n }\n \n+/\n+ bpf_try_to_free_mem_cgroup_pages - attempt to reclaim pages from\n+ * a memory cgroup\n+ * @memcg: the target memory cgroup to reclaim from\n+ * @nr_pages: the number of pages to reclaim\n+ * @gfp_mask: GFP flags controlling the reclaim behavior\n+ * @reclaim_options: bitmask of MEMCG_RECLAIM_* flags to tune\n+ * reclaim strategy\n+ * @swappiness: swappiness override value, or a sentinel to use\n+ * the default\n+ \n+ BPF-facing wrapper around try_to_free_mem_cgroup_pages() that\n+ * validates and translates the @swappiness argument before\n+ * delegating to the core reclaim path.\n+ \n+ The @swappiness parameter follows these semantics:\n+ * - Values in [MIN_SWAPPINESS, SWAPPINESS_ANON_ONLY] are passed\n+ * through as an explicit swappiness override.\n+ * - Values below MIN_SWAPPINESS are treated as \"use the system\n+ * default\"; the override pointer is set to NULL and the cgroup's\n+ * own swappiness setting takes effect.\n+ * - Values above SWAPPINESS_ANON_ONLY are rejected as invalid.\n+ * - If @reclaim_options does not include MEMCG_RECLAIM_PROACTIVE,\n+ * the @swappiness override is ignored entirely by the core\n+ * reclaim path and the system default is used regardless.\n+ \n+ Swap usage during reclaim is gated on @reclaim_options: swap is\n+ * considered only when MEMCG_RECLAIM_MAY_SWAP is set. Without this\n+ * flag, reclaim is restricted to file-backed pages regardless of the\n+ * @swappiness value or the cgroup's swappiness setting.\n+ \n+ Return:\n+ * The number of pages actually reclaimed on success, or 0\n+ * if @swappiness exceeds SWAPPINESS_ANON_ONLY.\n+ /\n+__bpf_kfunc unsigned long\n+bpf_try_to_free_mem_cgroup_pages(struct mem_cgroup memcg,\n+\t\t\t\t unsigned long nr_pages,\n+\t\t\t\t gfp_t gfp_mask,\n+\t\t\t\t unsigned int reclaim_options,\n+\t\t\t\t int swappiness)\n+{\n+\tint swapiness_ptr;\n+\n+\tif (swappiness \u003e SWAPPINESS_ANON_ONLY)\n+\t\treturn 0;\n+\telse if (swappiness \u003c MIN_SWAPPINESS)\n+\t\tswapiness_ptr = NULL;\n+\telse\n+\t\tswapiness_ptr = \u0026swappiness;\n+\n+\treturn try_to_free_mem_cgroup_pages(memcg, nr_pages, gfp_mask,\n+\t\t\t\t\t reclaim_options, swapiness_ptr);\n+}\n+\n __bpf_kfunc_end_defs();\n \n BTF_KFUNCS_START(bpf_memcontrol_kfuncs)\n@@ -172,6 +228,8 @@ BTF_ID_FLAGS(func, bpf_mem_cgroup_usage)\n BTF_ID_FLAGS(func, bpf_mem_cgroup_page_state)\n BTF_ID_FLAGS(func, bpf_mem_cgroup_flush_stats, KF_SLEEPABLE)\n \n+BTF_ID_FLAGS(func, bpf_try_to_free_mem_cgroup_pages, KF_SLEEPABLE)\n+\n BTF_KFUNCS_END(bpf_memcontrol_kfuncs)\n \n static const struct btf_kfunc_id_set bpf_memcontrol_kfunc_set = {\ndiff --git a/tools/testing/selftests/bpf/bpf_experimental.h b/tools/testing/selftests/bpf/bpf_experimental.h\nindex ff37ae5a113dc..ea905ea5603c2 100644\n--- a/tools/testing/selftests/bpf/bpf_experimental.h\n+++ b/tools/testing/selftests/bpf/bpf_experimental.h\n@@ -351,6 +351,13 @@ extern void bpf_iter_css_destroy(struct bpf_iter_css it) __weak __ksym;\n extern int bpf_wq_init(struct bpf_wq wq, void p__map, unsigned int flags) __weak __ksym;\n extern int bpf_wq_start(struct bpf_wq wq, unsigned int flags) __weak __ksym;\n \n+struct bpf_thread_wq;\n+extern int bpf_thread_wq_init(struct bpf_thread_wq twq, void p__map,\n+\t\t\t __u64 cgroup_id,\n+\t\t\t unsigned int flags) __weak __ksym;\n+extern int bpf_thread_wq_start(struct bpf_thread_wq twq,\n+\t\t\t unsigned int flags) __weak __ksym;\n+\n struct bpf_iter_kmem_cache;\n extern int bpf_iter_kmem_cache_new(struct bpf_iter_kmem_cache it) __weak __ksym;\n extern struct kmem_cache bpf_iter_kmem_cache_next(struct bpf_iter_kmem_cache it) __weak __ksym;\ndiff --git a/tools/testing/selftests/bpf/prog_tests/memcg_async_reclaim.c b/tools/testing/selftests/bpf/prog_tests/memcg_async_reclaim.c\nnew file mode 100644\nindex 0000000000000..40a5fe62cb384\n--- /dev/null\n+++ b/tools/testing/selftests/bpf/prog_tests/memcg_async_reclaim.c\n@@ -0,0 +1,479 @@\n+// SPDX-License-Identifier: GPL-2.0\n+/\n+ * Memory controller eBPF async reclaim test\n+ /\n+\n+#include \u003ctest_progs.h\u003e\n+#include \u003csys/mman.h\u003e\n+#include \u003csys/stat.h\u003e\n+#include \u003csys/time.h\u003e\n+#include \u003csys/vfs.h\u003e\n+#include \u003csys/wait.h\u003e\n+#include \u003cfcntl.h\u003e\n+#include \u003clinux/magic.h\u003e\n+#include \u003cunistd.h\u003e\n+#include \u003cstdio.h\u003e\n+#include \u003cstdlib.h\u003e\n+#include \u003cstring.h\u003e\n+\n+#include \"cgroup_helpers.h\"\n+\n+struct bpf_args_s {\n+\tu64 high_cgroup_id;\n+\tu64 low_cgroup_id;\n+\tu64 event_delta_threshold;\n+\tu64 check_ns;\n+};\n+\n+#include \"memcg_async_reclaim.skel.h\"\n+\n+#define FILE_SIZE (32 1024 * 1024ul)\n+#define BUFFER_SIZE (4096)\n+#define CG_LIMIT (32 * 1024 * 1024ul)\n+#define READ_TIMES 16\n+\n+#define CG_DIR \"/memcg_async_reclaim\"\n+#define CG_HIGH_DIR CG_DIR \"/high\"\n+#define CG_LOW_DIR CG_DIR \"/low\"\n+\n+#define CHECK_PERIOD_NS (2 * 1000 * 1000ull)\n+#define EVENT_DELTA_THRESHOLD 1\n+\n+/\n+ Test files must reside on a filesystem that supports page reclaim without\n+ * swap (e.g. ext4). If /tmp is on tmpfs, the file pages are shmem-backed\n+ * and can only be reclaimed through swap. But the test disables swap\n+ * (memory.swap.max=0), making reclaim impossible and causing OOM.\n+ \n+ Pick a directory on a non-tmpfs filesystem: try $TMPDIR first, then /tmp,\n+ * and fall back to the current directory if the chosen path is on tmpfs.\n+ /\n+static int get_test_dir(char buf, size_t size)\n+{\n+\tstatic const char * const candidates[] = { \"/tmp\", \".\" };\n+\tconst char tmpdir = getenv(\"TMPDIR\");\n+\tstruct statfs sfs;\n+\tsize_t i;\n+\n+\tif (tmpdir \u0026\u0026 tmpdir[0] \u0026\u0026 statfs(tmpdir, \u0026sfs) == 0 \u0026\u0026\n+\t sfs.f_type != TMPFS_MAGIC) {\n+\t\tsnprintf(buf, size, \"%s\", tmpdir);\n+\t\treturn 0;\n+\t}\n+\n+\tfor (i = 0; i \u003c ARRAY_SIZE(candidates); i++) {\n+\t\tif (statfs(candidates[i], \u0026sfs) == 0 \u0026\u0026\n+\t\t sfs.f_type != TMPFS_MAGIC) {\n+\t\t\tsnprintf(buf, size, \"%s\", candidates[i]);\n+\t\t\treturn 0;\n+\t\t}\n+\t}\n+\n+\treturn -1;\n+}\n+\n+static int setup_high_low_cgroups(u64 high_cgroup_id, u64 low_cgroup_id)\n+{\n+\tint ret;\n+\tchar limit_buf[20];\n+\n+\tret = setup_cgroup_environment();\n+\tif (!ASSERT_OK(ret, \"setup_cgroup_environment\"))\n+\t\tgoto cleanup;\n+\n+\tret = create_and_get_cgroup(CG_DIR);\n+\tif (!ASSERT_GE(ret, 0, \"create_and_get_cgroup \" CG_DIR))\n+\t\tgoto cleanup;\n+\tclose(ret);\n+\n+\tret = enable_controllers(CG_DIR, \"memory\");\n+\tif (!ASSERT_OK(ret, \"enable_controllers\"))\n+\t\tgoto cleanup;\n+\n+\tsnprintf(limit_buf, sizeof(limit_buf), \"%lu\", CG_LIMIT);\n+\tret = write_cgroup_file(CG_DIR, \"memory.max\", limit_buf);\n+\tif (!ASSERT_OK(ret, \"write_cgroup_file memory.max\"))\n+\t\tgoto cleanup;\n+\n+\tret = write_cgroup_file(CG_DIR, \"memory.swap.max\", \"0\");\n+\tif (!ASSERT_OK(ret, \"write_cgroup_file memory.swap.max\"))\n+\t\tgoto cleanup;\n+\n+\tret = create_and_get_cgroup(CG_HIGH_DIR);\n+\tif (!ASSERT_GE(ret, 0, \"create_and_get_cgroup \" CG_HIGH_DIR))\n+\t\tgoto cleanup;\n+\tclose(ret);\n+\n+\thigh_cgroup_id = get_cgroup_id(CG_HIGH_DIR);\n+\tif (!ASSERT_GT(high_cgroup_id, 0, \"get_cgroup_id\"))\n+\t\tgoto cleanup;\n+\n+\tret = create_and_get_cgroup(CG_LOW_DIR);\n+\tif (!ASSERT_GE(ret, 0, \"create_and_get_cgroup \" CG_LOW_DIR))\n+\t\tgoto cleanup;\n+\tclose(ret);\n+\n+\tlow_cgroup_id = get_cgroup_id(CG_LOW_DIR);\n+\tif (!ASSERT_GT(high_cgroup_id, 0, \"get_cgroup_id\"))\n+\t\tgoto cleanup;\n+\n+\treturn 0;\n+\n+cleanup:\n+\tcleanup_cgroup_environment();\n+\treturn -1;\n+}\n+\n+static int write_file(const char filename)\n+{\n+\tint ret = -1;\n+\tsize_t written = 0;\n+\tchar buffer;\n+\tFILE fp;\n+\n+\tfp = fopen(filename, \"wb\");\n+\tif (!fp)\n+\t\tgoto out;\n+\n+\tbuffer = malloc(BUFFER_SIZE);\n+\tif (!buffer)\n+\t\tgoto cleanup_fp;\n+\n+\tmemset(buffer, 'A', BUFFER_SIZE);\n+\n+\twhile (written \u003c FILE_SIZE) {\n+\t\tsize_t to_write = FILE_SIZE - written \u003c BUFFER_SIZE ?\n+\t\t\t\t FILE_SIZE - written : BUFFER_SIZE;\n+\n+\t\tif (fwrite(buffer, 1, to_write, fp) != to_write)\n+\t\t\tgoto cleanup;\n+\t\twritten += to_write;\n+\t}\n+\n+\tret = 0;\n+cleanup:\n+\tfree(buffer);\n+cleanup_fp:\n+\tfclose(fp);\n+out:\n+\treturn ret;\n+}\n+\n+static int read_file(const char filename, int iterations)\n+{\n+\tint ret = -1;\n+\tlong page_size = sysconf(_SC_PAGESIZE);\n+\tchar map;\n+\tsize_t i;\n+\tint fd;\n+\tstruct stat sb;\n+\n+\tfd = open(filename, O_RDONLY);\n+\tif (fd == -1)\n+\t\tgoto out;\n+\n+\tif (fstat(fd, \u0026sb) == -1)\n+\t\tgoto cleanup_fd;\n+\n+\tif (sb.st_size != FILE_SIZE) {\n+\t\tfprintf(stderr, \"File size mismatch: expected %lu, got %lu\\n\",\n+\t\t\t(unsigned long)FILE_SIZE, (unsigned long)sb.st_size);\n+\t\tgoto cleanup_fd;\n+\t}\n+\n+\tmap = mmap(NULL, FILE_SIZE, PROT_READ, MAP_PRIVATE, fd, 0);\n+\tif (map == MAP_FAILED)\n+\t\tgoto cleanup_fd;\n+\n+\tfor (int iter = 0; iter \u003c iterations; iter++) {\n+\t\tfor (i = 0; i \u003c FILE_SIZE; i += page_size) {\n+\t\t\t/* access a byte to trigger page fault /\n+\t\t\tasm volatile(\"\" :: \"r\"(map[i]) : \"memory\");\n+\t\t}\n+\t}\n+\n+\tif (munmap(map, FILE_SIZE) == -1)\n+\t\tgoto cleanup_fd;\n+\n+\tret = 0;\n+\n+cleanup_fd:\n+\tclose(fd);\n+out:\n+\treturn ret;\n+}\n+\n+static int real_test_child_work(const char cgroup_path, char data_filename,\n+\t\t\t\tchar time_filename, int read_times)\n+{\n+\tstruct timeval start, end;\n+\tdouble elapsed;\n+\tFILE fp;\n+\n+\tif (!ASSERT_OK(join_parent_cgroup(cgroup_path), \"join_parent_cgroup\"))\n+\t\treturn -1;\n+\n+\tgettimeofday(\u0026start, NULL);\n+\n+\tif (!ASSERT_OK(write_file(data_filename), \"write_file\"))\n+\t\treturn -1;\n+\n+\tif (!ASSERT_OK(read_file(data_filename, read_times), \"read_file\"))\n+\t\treturn -1;\n+\n+\tgettimeofday(\u0026end, NULL);\n+\n+\tif (!time_filename)\n+\t\treturn 0;\n+\n+\telapsed = (end.tv_sec - start.tv_sec) +\n+\t\t (end.tv_usec - start.tv_usec) / 1000000.0;\n+\tprintf(\"%.6f\\n\", elapsed);\n+\n+\tfp = fopen(time_filename, \"w\");\n+\tif (!ASSERT_OK_PTR(fp, \"fopen\"))\n+\t\treturn -1;\n+\tfprintf(fp, \"%.6f\", elapsed);\n+\tfclose(fp);\n+\n+\treturn 0;\n+}\n+\n+static int get_time(char time_filename, double time)\n+{\n+\tint ret = -1;\n+\tFILE fp;\n+\tchar buf[64];\n+\n+\tfp = fopen(time_filename, \"r\");\n+\tif (!ASSERT_OK_PTR(fp, \"fopen\"))\n+\t\tgoto out;\n+\n+\tif (!ASSERT_OK_PTR(fgets(buf, sizeof(buf), fp), \"fgets\"))\n+\t\tgoto cleanup;\n+\n+\tif (sscanf(buf, \"%lf\", time) != 1) {\n+\t\tPRINT_FAIL(\"sscanf %s\", buf);\n+\t\tgoto cleanup;\n+\t}\n+\n+\tret = 0;\n+cleanup:\n+\tfclose(fp);\n+out:\n+\treturn ret;\n+}\n+\n+static int\n+run_high_low_workload(double high_elapsed, double low_elapsed, int read_times)\n+{\n+\tchar test_dir[PATH_MAX], high_data_file[PATH_MAX], low_data_file[PATH_MAX];\n+\tchar high_time_file[PATH_MAX], low_time_file[PATH_MAX];\n+\tint ret, fd;\n+\tpid_t high_pid, low_pid;\n+\tint status;\n+\n+\tret = get_test_dir(test_dir, sizeof(test_dir));\n+\tif (!ASSERT_OK(ret, \"get_test_dir: no non-tmpfs directory found\"))\n+\t\treturn -1;\n+\n+\tfd = snprintf(high_data_file, sizeof(high_data_file),\n+\t\t \"%s/memcg_async_high_data_XXXXXX\", test_dir);\n+\tif (!ASSERT_LT(fd, sizeof(high_data_file), \"high_data_file path\"))\n+\t\treturn -1;\n+\n+\tfd = snprintf(low_data_file, sizeof(low_data_file),\n+\t\t \"%s/memcg_async_low_data_XXXXXX\", test_dir);\n+\tif (!ASSERT_LT(fd, sizeof(low_data_file), \"low_data_file path\"))\n+\t\treturn -1;\n+\n+\tfd = snprintf(high_time_file, sizeof(high_time_file),\n+\t\t \"%s/memcg_async_high_time_XXXXXX\", test_dir);\n+\tif (!ASSERT_LT(fd, sizeof(high_time_file), \"high_time_file path\"))\n+\t\treturn -1;\n+\n+\tfd = snprintf(low_time_file, sizeof(low_time_file),\n+\t\t \"%s/memcg_async_low_time_XXXXXX\", test_dir);\n+\tif (!ASSERT_LT(fd, sizeof(low_time_file), \"low_time_file path\"))\n+\t\treturn -1;\n+\n+\tfd = mkstemp(high_data_file);\n+\tif (!ASSERT_GE(fd, 0, \"mkstemp\"))\n+\t\treturn -1;\n+\tclose(fd);\n+\n+\tfd = mkstemp(low_data_file);\n+\tif (!ASSERT_GE(fd, 0, \"mkstemp\"))\n+\t\tgoto cleanup_high_data;\n+\tclose(fd);\n+\n+\tfd = mkstemp(high_time_file);\n+\tif (!ASSERT_GE(fd, 0, \"mkstemp\"))\n+\t\tgoto cleanup_low_data;\n+\tclose(fd);\n+\n+\tfd = mkstemp(low_time_file);\n+\tif (!ASSERT_GE(fd, 0, \"mkstemp\"))\n+\t\tgoto cleanup_high_time;\n+\tclose(fd);\n+\n+\tlow_pid = fork();\n+\tif (!ASSERT_GE(low_pid, 0, \"fork low\"))\n+\t\tgoto cleanup_low_time;\n+\tif (low_pid == 0)\n+\t\texit(real_test_child_work(CG_LOW_DIR, low_data_file,\n+\t\t\t\t\t low_time_file, read_times));\n+\n+\thigh_pid = fork();\n+\tif (!ASSERT_GE(high_pid, 0, \"fork high\")) {\n+\t\t(void)waitpid(low_pid, NULL, 0);\n+\t\tgoto cleanup_low_time;\n+\t}\n+\tif (high_pid == 0)\n+\t\texit(real_test_child_work(CG_HIGH_DIR, high_data_file,\n+\t\t\t\t\t high_time_file, read_times));\n+\n+\tret = waitpid(low_pid, \u0026status, 0);\n+\tif (!ASSERT_GT(ret, 0, \"low waitpid\"))\n+\t\tgoto cleanup_low_time;\n+\tif (!ASSERT_TRUE(WIFEXITED(status), \"low exited\"))\n+\t\tgoto cleanup_low_time;\n+\tif (!ASSERT_EQ(WEXITSTATUS(status), 0, \"low exit status\"))\n+\t\tgoto cleanup_low_time;\n+\n+\tret = waitpid(high_pid, \u0026status, 0);\n+\tif (!ASSERT_GT(ret, 0, \"high waitpid\"))\n+\t\tgoto cleanup_low_time;\n+\tif (!ASSERT_TRUE(WIFEXITED(status), \"high exited\"))\n+\t\tgoto cleanup_low_time;\n+\tif (!ASSERT_EQ(WEXITSTATUS(status), 0, \"high exit status\"))\n+\t\tgoto cleanup_low_time;\n+\n+\tif (get_time(high_time_file, high_elapsed))\n+\t\tgoto cleanup_low_time;\n+\tif (get_time(low_time_file, low_elapsed))\n+\t\tgoto cleanup_low_time;\n+\n+\tret = 0;\n+\n+cleanup_low_time:\n+\tunlink(low_time_file);\n+cleanup_high_time:\n+\tunlink(high_time_file);\n+cleanup_low_data:\n+\tunlink(low_data_file);\n+cleanup_high_data:\n+\tunlink(high_data_file);\n+\treturn ret;\n+}\n+\n+static int\n+setup_bpf(u64 high_cgroup_id, u64 low_cgroup_id,\n+\t struct memcg_async_reclaim *skel_ptr, bool use_thread_wq)\n+{\n+\tstruct memcg_async_reclaim skel;\n+\tstruct bpf_args_s bpf_args = {\n+\t\t.high_cgroup_id = high_cgroup_id,\n+\t\t.low_cgroup_id = low_cgroup_id,\n+\t\t.event_delta_threshold = EVENT_DELTA_THRESHOLD,\n+\t\t.check_ns = CHECK_PERIOD_NS,\n+\t};\n+\tLIBBPF_OPTS(bpf_test_run_opts, run_opts,\n+\t\t.ctx_in = \u0026bpf_args,\n+\t\t.ctx_size_in = sizeof(bpf_args));\n+\tint prog_init_fd, err;\n+\n+\tskel = memcg_async_reclaim__open_and_load();\n+\tif (!ASSERT_OK_PTR(skel, \"memcg_async_reclaim__open_and_load\"))\n+\t\treturn -1;\n+\n+\tif (use_thread_wq)\n+\t\tprog_init_fd = bpf_program__fd(skel-\u003eprogs.thread_wq_prog_init);\n+\telse\n+\t\tprog_init_fd = bpf_program__fd(skel-\u003eprogs.wq_prog_init);\n+\tif (!ASSERT_GE(prog_init_fd, 0, \"bpf_program__fd\"))\n+\t\tgoto error_out;\n+\n+\terr = bpf_prog_test_run_opts(prog_init_fd, \u0026run_opts);\n+\tif (!ASSERT_OK(err, \"bpf_prog_test_run_opts\"))\n+\t\tgoto error_out;\n+\tif (!ASSERT_EQ(run_opts.retval, 0, \"prog_init retval\"))\n+\t\tgoto error_out;\n+\n+\tskel_ptr = skel;\n+\treturn 0;\n+\n+error_out:\n+\tmemcg_async_reclaim__destroy(skel);\n+\treturn -1;\n+}\n+\n+void test_memcg_wq_async_reclaim(void)\n+{\n+\tu64 high_cgroup_id, low_cgroup_id;\n+\tint err;\n+\tdouble high_time = 0.0, low_time = 0.0;\n+\tstruct memcg_async_reclaim skel = NULL;\n+\n+\terr = setup_high_low_cgroups(\u0026high_cgroup_id, \u0026low_cgroup_id);\n+\tif (!ASSERT_OK(err, \"setup_high_low_cgroups reclaim\"))\n+\t\treturn;\n+\n+\terr = setup_bpf(high_cgroup_id, low_cgroup_id, \u0026skel, false);\n+\tif (!ASSERT_OK(err, \"setup_bpf\"))\n+\t\tgoto out;\n+\n+\terr = run_high_low_workload(\u0026high_time, \u0026low_time, READ_TIMES);\n+\tif (!ASSERT_OK(err, \"run_high_low_workload reclaim\"))\n+\t\tgoto out;\n+\n+\tif (high_time \u003e= low_time) {\n+\t\tPRINT_FAIL(\"high cgroup not improved with async reclaim: high_time=%f low_time=%f\",\n+\t\t\t high_time, low_time);\n+\t}\n+\n+out:\n+\tif (skel)\n+\t\tmemcg_async_reclaim__destroy(skel);\n+\t/\n+\t Wait for bpf_wq to release the reference to cgroup\n+\t * to ensure the successful deletion of cgroup.\n+\t /\n+\tsleep(1);\n+\tcleanup_cgroup_environment();\n+}\n+\n+void test_memcg_thread_wq_async_reclaim(void)\n+{\n+\tu64 high_cgroup_id, low_cgroup_id;\n+\tint err;\n+\tdouble high_time = 0.0, low_time = 0.0;\n+\tstruct memcg_async_reclaim skel = NULL;\n+\n+\terr = setup_high_low_cgroups(\u0026high_cgroup_id, \u0026low_cgroup_id);\n+\tif (!ASSERT_OK(err, \"setup_high_low_cgroups reclaim\"))\n+\t\treturn;\n+\n+\terr = setup_bpf(high_cgroup_id, low_cgroup_id, \u0026skel, true);\n+\tif (!ASSERT_OK(err, \"setup_bpf\"))\n+\t\tgoto out;\n+\n+\terr = run_high_low_workload(\u0026high_time, \u0026low_time, READ_TIMES);\n+\tif (!ASSERT_OK(err, \"run_high_low_workload reclaim\"))\n+\t\tgoto out;\n+\n+\tif (high_time \u003e= low_time) {\n+\t\tPRINT_FAIL(\"high cgroup not improved with async reclaim: high_time=%f low_time=%f\",\n+\t\t\t high_time, low_time);\n+\t}\n+\n+out:\n+\tif (skel)\n+\t\tmemcg_async_reclaim__destroy(skel);\n+\t/\n+\t Wait for bpf_thread_wq to release the reference to cgroup\n+\t * to ensure the successful deletion of cgroup.\n+\t /\n+\tsleep(1);\n+\tcleanup_cgroup_environment();\n+}\ndiff --git a/tools/testing/selftests/bpf/prog_tests/thread_wq_cgroup.c b/tools/testing/selftests/bpf/prog_tests/thread_wq_cgroup.c\nnew file mode 100644\nindex 0000000000000..7537b03f17e2a\n--- /dev/null\n+++ b/tools/testing/selftests/bpf/prog_tests/thread_wq_cgroup.c\n@@ -0,0 +1,87 @@\n+// SPDX-License-Identifier: GPL-2.0\n+#include \u003ctest_progs.h\u003e\n+#include \u003cunistd.h\u003e\n+#include \"cgroup_helpers.h\"\n+#include \"thread_wq_cgroup.skel.h\"\n+\n+#define TEST_CGROUP \"/thread_wq_test\"\n+#define WAIT_TIMEOUT_SECS 30\n+\n+void test_thread_wq_cgroup(void)\n+{\n+\tstruct thread_wq_cgroup skel = NULL;\n+\tint err, prog_fd, cg_fd = -1;\n+\tunsigned long long cg_id;\n+\tint waited_secs;\n+\n+\tLIBBPF_OPTS(bpf_test_run_opts, topts);\n+\n+\terr = setup_cgroup_environment();\n+\tif (!ASSERT_OK(err, \"setup_cgroup_environment\"))\n+\t\treturn;\n+\tcg_fd = create_and_get_cgroup(TEST_CGROUP);\n+\tif (!ASSERT_GE(cg_fd, 0, \"create_and_get_cgroup\"))\n+\t\tgoto cleanup;\n+\tcg_id = get_cgroup_id(TEST_CGROUP);\n+\tif (!ASSERT_GT(cg_id, 0ULL, \"get_cgroup_id\"))\n+\t\tgoto cleanup;\n+\n+\tskel = thread_wq_cgroup__open_and_load();\n+\tif (!ASSERT_OK_PTR(skel, \"open_and_load\"))\n+\t\tgoto cleanup;\n+\n+\tprog_fd = bpf_program__fd(skel-\u003eprogs.start_thread_wq);\n+\n+\t/* Run bpf_thread_wq in the specified cgroup. /\n+\tskel-\u003ebss-\u003etest_key = 0;\n+\tskel-\u003ebss-\u003etarget_cgroup_id = cg_id;\n+\tskel-\u003ebss-\u003ecallback_cgroup_id = 0;\n+\tskel-\u003ebss-\u003etwq_done = 0;\n+\tif (!ASSERT_OK(bpf_prog_test_run_opts(prog_fd, \u0026topts),\n+\t\t \"bpf_prog_test_run_opts in cgroup\"))\n+\t\tgoto cleanup;\n+\tif (!ASSERT_OK(topts.retval, \"retval in cgroup\"))\n+\t\tgoto cleanup;\n+\tfor (waited_secs = 0; waited_secs \u003c WAIT_TIMEOUT_SECS; waited_secs++) {\n+\t\tif (skel-\u003ebss-\u003etwq_done)\n+\t\t\tbreak;\n+\t\tsleep(1);\n+\t}\n+\tif (!ASSERT_TRUE(skel-\u003ebss-\u003etwq_done, \"twq_done in cgroup\"))\n+\t\tgoto cleanup;\n+\tif (!ASSERT_EQ(skel-\u003ebss-\u003ecallback_cgroup_id, cg_id,\n+\t\t \"callback_cgroup_id in cgroup\"))\n+\t\tgoto cleanup;\n+\n+\t/ Run bpf_thread_wq without cgroup attachment (cgroup_id = 0). /\n+\tLIBBPF_OPTS_RESET(topts);\n+\tskel-\u003ebss-\u003etest_key = 1;\n+\tskel-\u003ebss-\u003etarget_cgroup_id = 0;\n+\tskel-\u003ebss-\u003ecallback_cgroup_id = 0;\n+\tskel-\u003ebss-\u003etwq_done = 0;\n+\tif (!ASSERT_OK(bpf_prog_test_run_opts(prog_fd, \u0026topts),\n+\t\t \"bpf_prog_test_run_opts without cgroup\"))\n+\t\tgoto cleanup;\n+\tif (!ASSERT_OK(topts.retval, \"retval without cgroup\"))\n+\t\tgoto cleanup;\n+\tfor (waited_secs = 0; waited_secs \u003c WAIT_TIMEOUT_SECS; waited_secs++) {\n+\t\tif (skel-\u003ebss-\u003etwq_done)\n+\t\t\tbreak;\n+\t\tsleep(1);\n+\t}\n+\tif (!ASSERT_TRUE(skel-\u003ebss-\u003etwq_done, \"twq_done without cgroup\"))\n+\t\tgoto cleanup;\n+\tif (!ASSERT_NEQ(skel-\u003ebss-\u003ecallback_cgroup_id, cg_id,\n+\t\t\t\"callback_cgroup_id without cgroup\"))\n+\t\tgoto cleanup;\n+\n+cleanup:\n+\tif (skel) {\n+\t\tthread_wq_cgroup__destroy(skel);\n+\t\t/ Wait thread_wq kthread quit. /\n+\t\tsleep(2);\n+\t}\n+\tif (cg_fd \u003e= 0)\n+\t\tclose(cg_fd);\n+\tcleanup_cgroup_environment();\n+}\ndiff --git a/tools/testing/selftests/bpf/progs/memcg_async_reclaim.c b/tools/testing/selftests/bpf/progs/memcg_async_reclaim.c\nnew file mode 100644\nindex 0000000000000..eaccc8a37388b\n--- /dev/null\n+++ b/tools/testing/selftests/bpf/progs/memcg_async_reclaim.c\n@@ -0,0 +1,255 @@\n+// SPDX-License-Identifier: GPL-2.0\n+\n+#include \"vmlinux.h\"\n+#include \"bpf_experimental.h\"\n+#include \u003cbpf/bpf_helpers.h\u003e\n+#include \u003cbpf/bpf_tracing.h\u003e\n+\n+#define BIT(nr)\t\t\t(1UL \u003c\u003c (nr))\n+\n+#define ___GFP_IO\t\tBIT(___GFP_IO_BIT)\n+#define ___GFP_FS\t\tBIT(___GFP_FS_BIT)\n+#define ___GFP_DIRECT_RECLAIM\tBIT(___GFP_DIRECT_RECLAIM_BIT)\n+#define ___GFP_KSWAPD_RECLAIM\tBIT(___GFP_KSWAPD_RECLAIM_BIT)\n+\n+#define __GFP_IO\t\t((gfp_t)___GFP_IO)\n+#define __GFP_FS\t\t((gfp_t)___GFP_FS)\n+#define __GFP_DIRECT_RECLAIM\t((gfp_t)___GFP_DIRECT_RECLAIM)\n+#define __GFP_KSWAPD_RECLAIM\t((gfp_t)___GFP_KSWAPD_RECLAIM)\n+#define __GFP_RECLAIM\t((gfp_t)(___GFP_DIRECT_RECLAIM \| ___GFP_KSWAPD_RECLAIM))\n+\n+#define GFP_KERNEL\t(__GFP_RECLAIM \| __GFP_IO \| __GFP_FS)\n+#define CLOCK_MONOTONIC_ID\t1\n+#define RECLAIM_PAGES\t\t32\n+#define RECLAIM_MAX_ITER\t32\n+\n+struct bpf_args_s {\n+\tu64 high_cgroup_id;\n+\tu64 low_cgroup_id;\n+\tu64 event_delta_threshold;\n+\tu64 check_ns;\n+};\n+\n+struct cgroup_memcg {\n+\tstruct cgroup cgrp;\n+\tstruct mem_cgroup memcg;\n+};\n+\n+static u64 wq_high_cgroup_id;\n+static u64 wq_low_cgroup_id;\n+\n+static int get_cgroup_memcg_from_id(u64 cgroup_id, struct cgroup_memcg cm)\n+{\n+\tcm-\u003ecgrp = bpf_cgroup_from_id(cgroup_id);\n+\tif (!cm-\u003ecgrp)\n+\t\treturn -1;\n+\n+\tcm-\u003ememcg = bpf_get_mem_cgroup(\u0026cm-\u003ecgrp-\u003eself);\n+\tif (!cm-\u003ememcg) {\n+\t\tbpf_cgroup_release(cm-\u003ecgrp);\n+\t\treturn -1;\n+\t}\n+\n+\treturn 0;\n+}\n+\n+static void put_cgroup_memcg(struct cgroup_memcg cm)\n+{\n+\tbpf_put_mem_cgroup(cm-\u003ememcg);\n+\tbpf_cgroup_release(cm-\u003ecgrp);\n+}\n+\n+static int get_cgroup_event(u64 cgroup_id, u64 val)\n+{\n+\tstruct cgroup_memcg cm;\n+\n+\tif (get_cgroup_memcg_from_id(cgroup_id, \u0026cm))\n+\t\treturn -1;\n+\tbpf_mem_cgroup_flush_stats(cm.memcg);\n+\tval = bpf_mem_cgroup_page_state(cm.memcg, WORKINGSET_REFAULT_FILE);\n+\tput_cgroup_memcg(\u0026cm);\n+\n+\treturn 0;\n+}\n+\n+static bool\n+should_reclaim_cgroup(u64 cgroup_id, u64 prev_event, u64 event_delta_threshold)\n+{\n+\tu64 cur, delta;\n+\n+\tif (get_cgroup_event(cgroup_id, \u0026cur))\n+\t\treturn false;\n+\n+\tdelta = cur - prev_event;\n+\tprev_event = cur;\n+\n+\treturn delta \u003e= event_delta_threshold;\n+}\n+\n+static int reclaim_cgroup(u64 cgroup_id)\n+{\n+\tstruct cgroup_memcg cm;\n+\tint i;\n+\n+\tif (get_cgroup_memcg_from_id(cgroup_id, \u0026cm))\n+\t\treturn 0;\n+\n+\tfor (i = 0; i \u003c RECLAIM_MAX_ITER; i++) {\n+\t\tif (!bpf_try_to_free_mem_cgroup_pages(cm.memcg, RECLAIM_PAGES,\n+\t\t\t\t\t\t GFP_KERNEL, 0, -1))\n+\t\t\tbreak;\n+\t}\n+\n+\tput_cgroup_memcg(\u0026cm);\n+\n+\treturn 0;\n+}\n+\n+struct wq_elem {\n+\tstruct bpf_timer timer;\n+\tstruct bpf_wq work;\n+\tu64 prev_event;\n+\tu64 event_delta_threshold;\n+\tu64 check_ns;\n+};\n+\n+struct {\n+\t__uint(type, BPF_MAP_TYPE_ARRAY);\n+\t__uint(max_entries, 1);\n+\t__type(key, __u32);\n+\t__type(value, struct wq_elem);\n+} wq_map SEC(\".maps\");\n+\n+static int async_free(void map, int key, void value)\n+{\n+\tstruct wq_elem elem = value;\n+\n+\tif (should_reclaim_cgroup(wq_high_cgroup_id, \u0026elem-\u003eprev_event,\n+\t\telem-\u003eevent_delta_threshold)) {\n+\t\treclaim_cgroup(wq_low_cgroup_id);\n+\t\tbpf_wq_start(\u0026elem-\u003ework, 0);\n+\t}\n+\n+\treturn 0;\n+}\n+\n+static int wq_timer_cb(void map, int key, struct wq_elem elem)\n+{\n+\tbpf_wq_start(\u0026elem-\u003ework, 0);\n+\tbpf_timer_start(\u0026elem-\u003etimer, elem-\u003echeck_ns, 0);\n+\n+\treturn 0;\n+}\n+\n+SEC(\"syscall\")\n+int wq_prog_init(struct bpf_args_s ctx)\n+{\n+\tstruct wq_elem elem;\n+\t__u32 key = 0;\n+\tint ret;\n+\n+\telem = bpf_map_lookup_elem(\u0026wq_map, \u0026key);\n+\tif (!elem)\n+\t\treturn -1;\n+\n+\tret = bpf_wq_init(\u0026elem-\u003ework, \u0026wq_map, 0);\n+\tif (ret)\n+\t\treturn ret;\n+\n+\tret = bpf_wq_set_callback(\u0026elem-\u003ework, async_free, 0);\n+\tif (ret)\n+\t\treturn ret;\n+\n+\tret = bpf_timer_init(\u0026elem-\u003etimer, \u0026wq_map, CLOCK_MONOTONIC_ID);\n+\tif (ret)\n+\t\treturn ret;\n+\n+\tret = bpf_timer_set_callback(\u0026elem-\u003etimer, wq_timer_cb);\n+\tif (ret)\n+\t\treturn ret;\n+\n+\telem-\u003eprev_event = 0;\n+\telem-\u003eevent_delta_threshold = ctx-\u003eevent_delta_threshold;\n+\telem-\u003echeck_ns = ctx-\u003echeck_ns;\n+\n+\twq_high_cgroup_id = ctx-\u003ehigh_cgroup_id;\n+\twq_low_cgroup_id = ctx-\u003elow_cgroup_id;\n+\n+\treturn bpf_timer_start(\u0026elem-\u003etimer, elem-\u003echeck_ns, 0);\n+}\n+\n+struct thread_wq_elem {\n+\tstruct bpf_timer timer;\n+\tstruct bpf_thread_wq work;\n+\tu64 prev_event;\n+\tu64 event_delta_threshold;\n+\tu64 check_ns;\n+};\n+\n+struct {\n+\t__uint(type, BPF_MAP_TYPE_ARRAY);\n+\t__uint(max_entries, 1);\n+\t__type(key, __u32);\n+\t__type(value, struct thread_wq_elem);\n+} thread_wq_map SEC(\".maps\");\n+\n+static int thread_async_free(void map, int key, void value)\n+{\n+\tstruct thread_wq_elem elem = value;\n+\n+\tif (should_reclaim_cgroup(wq_high_cgroup_id, \u0026elem-\u003eprev_event,\n+\t\telem-\u003eevent_delta_threshold)) {\n+\t\treclaim_cgroup(wq_low_cgroup_id);\n+\t\tbpf_thread_wq_start(\u0026elem-\u003ework, 0);\n+\t}\n+\n+\treturn 0;\n+}\n+\n+static int thread_wq_timer_cb(void map, int key, struct thread_wq_elem elem)\n+{\n+\tbpf_thread_wq_start(\u0026elem-\u003ework, 0);\n+\tbpf_timer_start(\u0026elem-\u003etimer, elem-\u003echeck_ns, 0);\n+\n+\treturn 0;\n+}\n+\n+SEC(\"syscall\")\n+int thread_wq_prog_init(struct bpf_args_s ctx)\n+{\n+\tstruct thread_wq_elem elem;\n+\t__u32 key = 0;\n+\tint ret;\n+\n+\telem = bpf_map_lookup_elem(\u0026thread_wq_map, \u0026key);\n+\tif (!elem)\n+\t\treturn -1;\n+\n+\tret = bpf_thread_wq_init(\u0026elem-\u003ework, \u0026thread_wq_map,\n+\t\t\t\t ctx-\u003elow_cgroup_id, 0);\n+\tif (ret)\n+\t\treturn ret;\n+\n+\tret = bpf_thread_wq_set_callback(\u0026elem-\u003ework, thread_async_free, 0);\n+\tif (ret)\n+\t\treturn ret;\n+\n+\tret = bpf_timer_init(\u0026elem-\u003etimer, \u0026thread_wq_map, CLOCK_MONOTONIC_ID);\n+\tif (ret)\n+\t\treturn ret;\n+\n+\tret = bpf_timer_set_callback(\u0026elem-\u003etimer, thread_wq_timer_cb);\n+\tif (ret)\n+\t\treturn ret;\n+\n+\telem-\u003eprev_event = 0;\n+\telem-\u003eevent_delta_threshold = ctx-\u003eevent_delta_threshold;\n+\telem-\u003echeck_ns = ctx-\u003echeck_ns;\n+\n+\twq_high_cgroup_id = ctx-\u003ehigh_cgroup_id;\n+\twq_low_cgroup_id = ctx-\u003elow_cgroup_id;\n+\n+\treturn bpf_timer_start(\u0026elem-\u003etimer, elem-\u003echeck_ns, 0);\n+}\n+\n+char LICENSE[] SEC(\"license\") = \"GPL\";\ndiff --git a/tools/testing/selftests/bpf/progs/thread_wq_cgroup.c b/tools/testing/selftests/bpf/progs/thread_wq_cgroup.c\nnew file mode 100644\nindex 0000000000000..c70a37f55397d\n--- /dev/null\n+++ b/tools/testing/selftests/bpf/progs/thread_wq_cgroup.c\n@@ -0,0 +1,56 @@\n+// SPDX-License-Identifier: GPL-2.0\n+/* Copyright (c) 2026 /\n+\n+#include \"bpf_experimental.h\"\n+#include \u003cbpf/bpf_helpers.h\u003e\n+#include \"bpf_misc.h\"\n+\n+char _license[] SEC(\"license\") = \"GPL\";\n+\n+struct elem {\n+\tstruct bpf_thread_wq twq;\n+};\n+\n+struct {\n+\t__uint(type, BPF_MAP_TYPE_ARRAY);\n+\t__uint(max_entries, 2);\n+\t__type(key, int);\n+\t__type(value, struct elem);\n+} map_arr SEC(\".maps\");\n+\n+__u64 target_cgroup_id;\n+__u64 callback_cgroup_id;\n+int twq_done;\n+int test_key;\n+\n+static int twq_callback(void map, int key, void value)\n+{\n+\tcallback_cgroup_id = bpf_get_current_cgroup_id();\n+\ttwq_done = 1;\n+\treturn 0;\n+}\n+\n+SEC(\"syscall\")\n+int start_thread_wq(void ctx)\n+{\n+\tstruct elem val;\n+\tint key = test_key;\n+\tint ret;\n+\n+\tval = bpf_map_lookup_elem(\u0026map_arr, \u0026key);\n+\tif (!val)\n+\t\treturn -1;\n+\n+\tret = bpf_thread_wq_init(\u0026val-\u003etwq, \u0026map_arr, target_cgroup_id, 0);\n+\tif (ret)\n+\t\tgoto out;\n+\n+\tret = bpf_thread_wq_set_callback(\u0026val-\u003etwq, twq_callback, 0);\n+\tif (ret)\n+\t\tgoto out;\n+\n+\tret = bpf_thread_wq_start(\u0026val-\u003etwq, 0);\n+\n+out:\n+\treturn ret;\n+}\n" }
2/1	2026/08/07 08:50	agent	patch-evaluator	0m Model: best-expensive Args: null Results: { "EnableConfigs": [ "MEMCG", "CGROUPS" ], "FocusSymbols": [ "bpf_thread_wq_init", "bpf_thread_wq_set_callback", "bpf_thread_wq_start", "bpf_try_to_free_mem_cgroup_pages", "cgroup_kthread_attach" ], "Reasoning": "The patch introduces a new BPF thread workqueue feature (`bpf_thread_wq`) that allows BPF programs to schedule work in a kthread context, optionally attached to a cgroup. It also adds a new BPF kfunc `bpf_try_to_free_mem_cgroup_pages` for memory cgroup reclaim. These are functional changes to the core BPF and memory control subsystems, reachable from userspace via BPF programs, and should be fuzzed.", "WorthFuzzing": true } Instruction: You are an expert Linux kernel maintainer. Your job is to review a provided patch series and determine if it makes functional changes to the kernel that should be fuzzed. IMPORTANT: The changes have ALREADY been applied and committed as the HEAD commit in your workspace. Do NOT rely on your internal knowledge of the kernel. You must actively use your code access tools to examine the actual source code and confirm any assumptions. Return WorthFuzzing=false if the patch only contains: - Modifications to Documentation/, Kconfig files, or code comments. - Purely decorative changes, such as logging (e.g., pr_err, printk) or tracepoints. - Changes to numeric constants or macros that do not functionally alter execution flow. - Code paths that are impossible to reach in virtualized environments like GCE or QEMU, even when utilizing software-emulated hardware (e.g., usb gadget, mac80211_hwsim). - Code in vendor-specific PCIe switch, SmartNIC, or GPU drivers (e.g., mlxsw, pds_core, qed, ionic, amdgpu) that require physical PCIe hardware cards not emulated in standard QEMU. - Driver .remove, .shutdown, or pci_unregister_driver teardown callbacks (e.g., igb_remove) that are executed only during PCI hot-unplug or sysfs driver unbind operations. If it modifies reachable core kernel logic, drivers, or architectures, use your code search tools to verify the code can be executed, then return WorthFuzzing=true. When returning WorthFuzzing=true, you MUST ALSO: 1. Extract any specific kernel functions that should be heavily fuzzed into FocusSymbols. Avoid listing generic hot-path functions to prevent skewed test distributions. Prefer non-static, non-inlined API entrypoint functions over internal static helper functions (which are inlined by the compiler and do not have distinct symbol addresses). 2. Identify any specific CONFIG_ options required to properly test this new/modified feature. Go and look into the Kconfig files and check for ifdefs around the code, do not make assumptions. Also check "depends on" lines in Kconfig to include any non-standard parent subsystem configs needed for Kbuild to compile the code statically into vmlinux. List them in the EnableConfigs output array, and DO NOT add a 'CONFIG_' prefix (e.g., return "NET_IPV4" instead of "CONFIG_NET_IPV4"). Prefer calling several tools at the same time to save round-trips. Use set-results tool to provide results of the analysis. It must be called exactly once before the final reply. Ignore results of this tool. Prompt: For your convenience, here is the diff of the changes: commit e72e2c008c951fa5d07abaad9d32e877879e2ea7 Author: syz-cluster <triage@syzkaller.com> Date: Fri Aug 7 08:50:40 2026 +0000 syz-cluster: applied patch under review diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 73bacfc6444df..d63ce83198696 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -213,6 +213,7 @@ enum btf_field_type { BPF_UPTR = (1 << 11), BPF_RES_SPIN_LOCK = (1 << 12), BPF_TASK_WORK = (1 << 13), + BPF_THREAD_WQ = (1 << 14), }; enum bpf_cgroup_storage_type { @@ -267,6 +268,7 @@ struct btf_record { int wq_off; int refcount_off; int task_work_off; + int thread_wq_off; struct btf_field fields[]; }; @@ -372,6 +374,8 @@ static inline const char btf_field_type_name(enum btf_field_type type) return "bpf_refcount"; case BPF_TASK_WORK: return "bpf_task_work"; + case BPF_THREAD_WQ: + return "bpf_thread_wq"; default: WARN_ON_ONCE(1); return "unknown"; @@ -412,6 +416,8 @@ static inline u32 btf_field_type_size(enum btf_field_type type) return sizeof(struct bpf_refcount); case BPF_TASK_WORK: return sizeof(struct bpf_task_work); + case BPF_THREAD_WQ: + return sizeof(struct bpf_thread_wq); default: WARN_ON_ONCE(1); return 0; @@ -446,6 +452,8 @@ static inline u32 btf_field_type_align(enum btf_field_type type) return __alignof__(struct bpf_refcount); case BPF_TASK_WORK: return __alignof__(struct bpf_task_work); + case BPF_THREAD_WQ: + return __alignof__(struct bpf_thread_wq); default: WARN_ON_ONCE(1); return 0; @@ -478,6 +486,7 @@ static inline void bpf_obj_init_field(const struct btf_field field, void addr) case BPF_KPTR_PERCPU: case BPF_UPTR: case BPF_TASK_WORK: + case BPF_THREAD_WQ: break; default: WARN_ON_ONCE(1); @@ -502,6 +511,7 @@ static inline bool btf_field_is_nmi_safe(enum btf_field_type type) case BPF_TASK_WORK: case BPF_KPTR_UNREF: case BPF_REFCOUNT: + case BPF_THREAD_WQ: return true; default: return false; @@ -644,6 +654,7 @@ void copy_map_value_locked(struct bpf_map map, void dst, void src, void bpf_timer_cancel_and_free(void timer); void bpf_wq_cancel_and_free(void timer); void bpf_task_work_cancel_and_free(void timer); +void bpf_thread_wq_cancel_and_free(void val); void bpf_list_head_free(const struct btf_field field, void list_head, struct bpf_spin_lock spin_lock); void bpf_rb_root_free(const struct btf_field field, void rb_root, @@ -701,7 +712,8 @@ bool bpf_map_meta_equal(const struct bpf_map meta0, static inline bool bpf_map_has_internal_structs(struct bpf_map map) { - return btf_record_has_field(map->record, BPF_TIMER \| BPF_WORKQUEUE \| BPF_TASK_WORK); + return btf_record_has_field(map->record, BPF_TIMER \| BPF_WORKQUEUE \| + BPF_TASK_WORK \| BPF_THREAD_WQ); } void bpf_map_free_internal_structs(struct bpf_map map, void obj); @@ -2725,6 +2737,7 @@ bool btf_record_equal(const struct btf_record rec_a, const struct btf_record r void bpf_obj_free_timer(const struct btf_record rec, void obj); void bpf_obj_free_workqueue(const struct btf_record rec, void obj); void bpf_obj_free_task_work(const struct btf_record rec, void obj); +void bpf_obj_free_thread_wq(const struct btf_record rec, void obj); void bpf_obj_cancel_fields(struct bpf_map map, void obj); void bpf_obj_free_fields(const struct btf_record rec, void obj); void __bpf_obj_drop_impl(void p, const struct btf_record rec, bool percpu); diff --git a/include/linux/cgroup.h b/include/linux/cgroup.h index f2aa46a4f871e..9b4a8dc748ace 100644 --- a/include/linux/cgroup.h +++ b/include/linux/cgroup.h @@ -923,4 +923,6 @@ struct cgroup task_get_cgroup1(struct task_struct tsk, int hierarchy_id); struct cgroup_of_peak of_peak(struct kernfs_open_file of); +int cgroup_kthread_attach(struct cgroup cgrp, struct task_struct task); + #endif / _LINUX_CGROUP_H / diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index ffd96e8b920bd..0558520f67fe4 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -7574,6 +7574,10 @@ struct bpf_wq { __u64 __opaque[2]; } __attribute__((aligned(8))); +struct bpf_thread_wq { + __u64 __opaque[2]; +} __attribute__((aligned(8))); + struct bpf_dynptr { __u64 __opaque[2]; } __attribute__((aligned(8))); diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 42414633cf263..bf6fb5f51d21a 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -3665,6 +3665,7 @@ static int btf_get_field_type(const struct btf btf, const struct btf_type var_ { BPF_TIMER, "bpf_timer", true }, { BPF_WORKQUEUE, "bpf_wq", true }, { BPF_TASK_WORK, "bpf_task_work", true }, + { BPF_THREAD_WQ, "bpf_thread_wq", true }, { BPF_LIST_HEAD, "bpf_list_head", false }, { BPF_LIST_NODE, "bpf_list_node", false }, { BPF_RB_ROOT, "bpf_rb_root", false }, @@ -3850,6 +3851,7 @@ static int btf_find_field_one(const struct btf btf, case BPF_RB_NODE: case BPF_REFCOUNT: case BPF_TASK_WORK: + case BPF_THREAD_WQ: ret = btf_find_struct(btf, var_type, off, sz, field_type, info_cnt ? &info[0] : &tmp); if (ret < 0) @@ -4145,6 +4147,7 @@ struct btf_record btf_parse_fields(const struct btf btf, const struct btf_type rec->wq_off = -EINVAL; rec->refcount_off = -EINVAL; rec->task_work_off = -EINVAL; + rec->thread_wq_off = -EINVAL; for (i = 0; i < cnt; i++) { field_type_size = btf_field_type_size(info_arr[i].type); if (info_arr[i].off + field_type_size > value_size) { @@ -4188,6 +4191,10 @@ struct btf_record btf_parse_fields(const struct btf btf, const struct btf_type WARN_ON_ONCE(rec->task_work_off >= 0); rec->task_work_off = rec->fields[i].offset; break; + case BPF_THREAD_WQ: + WARN_ON_ONCE(rec->thread_wq_off >= 0); + rec->thread_wq_off = rec->fields[i].offset; + break; case BPF_REFCOUNT: WARN_ON_ONCE(rec->refcount_off >= 0); /* Cache offset for faster lookup at runtime / diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c index 6388b6b23e490..8e01bad1b16a0 100644 --- a/kernel/bpf/helpers.c +++ b/kernel/bpf/helpers.c @@ -29,6 +29,8 @@ #include <linux/task_work.h> #include <linux/irq_work.h> #include <linux/buildid.h> +#include <linux/kthread.h> +#include <linux/jhash.h> #include "../../lib/kstrtox.h" @@ -1110,6 +1112,17 @@ static void map_key_from_value(struct bpf_map map, void value, u32 arr_idx) return (void )value - round_up(map->key_size, 8); } +static u32 bpf_map_elem_id_from_value(struct bpf_map map, void value) +{ + u32 arr_idx; + void key; + + key = map_key_from_value(map, value, &arr_idx); + if (map->map_type == BPF_MAP_TYPE_ARRAY) + return arr_idx; + return jhash(key, map->key_size, 0); +} + enum bpf_async_type { BPF_ASYNC_TYPE_TIMER = 0, BPF_ASYNC_TYPE_WQ, @@ -4769,6 +4782,397 @@ __bpf_kfunc int bpf_timer_cancel_async(struct bpf_timer timer) } } +/* + * BPF thread workqueue (kthread_worker based) implementation + * + * Why bpf_thread_wq does NOT use the bpf_async infrastructure: + * + * bpf_timer and bpf_wq share a common cleanup path via bpf_async: + * + * bpf_async_cancel_and_free() + * -> bpf_async_schedule_op() + * -> irq_work_queue() // schedule from any context + * -> bpf_async_process_op() // runs in hardirq context + * -> bpf_wq_work() / timer cb // atomic, non-sleepable + * + * This works for timer and workqueue because their callbacks and + * cancellation (hrtimer_cancel / bpf_wq_cancel_and_free) complete + * synchronously and need not sleep. The hardirq context is sufficient. + * + * bpf_thread_wq is different: cancellation and final cleanup may need to + * sleep: + * + * kthread_cancel_work_sync() - waits for the work to finish, can + * schedule out if the work is running. + * kthread_destroy_worker() - stops the kthread, internally + * synchronizes with kthread exit. + * cgroup_put() (final put) - may acquire cgroup_mutex and other + * sleeping locks during offline. + * + * None of these can be called from hardirq context (irq_work). Doing so + * would trigger might_sleep() warnings or deadlock. + * + * Therefore bpf_thread_wq implements its own RCU-based cleanup: + * + * bpf_thread_wq_cancel_and_free() // sleepable (called from + * // map free / elem delete) + * -> xchg(ctx, NULL) // detach atomically + * -> kthread_cancel_work_sync() // ok to sleep + * -> bpf_thread_wq_ctx_put() + * -> call_rcu_tasks_trace() // wait for BPF callbacks + * -> schedule_work() // switch to sleepable context + * -> bpf_thread_wq_destroy_work_fn() + * -> bpf_prog_put() + * -> cgroup_put() + * -> kthread_destroy_worker() + * -> kfree(ctx) + * + * The same design choice was made for bpf_task_work, which also avoids + * bpf_async because task_work cancellation synchronizes with the task + * and may sleep. + / + +struct bpf_thread_wq_ctx { + struct kthread_worker worker; + struct kthread_work work; + struct bpf_prog prog; + bpf_callback_t callback_fn; + struct bpf_map map; + void value; + struct cgroup cgrp; + refcount_t refcnt; + struct rcu_head rcu; + struct work_struct destroy_work; +}; + +/* Kernel-internal representation that fits in struct bpf_thread_wq / +struct bpf_thread_wq_kern { + struct bpf_thread_wq_ctx ctx; +} __aligned(8); + +static void bpf_thread_wq_destroy_work_fn(struct work_struct work) +{ + struct bpf_thread_wq_ctx ctx = container_of(work, + struct bpf_thread_wq_ctx, + destroy_work); + + if (ctx->prog) + bpf_prog_put(ctx->prog); + if (ctx->cgrp) + cgroup_put(ctx->cgrp); + if (ctx->worker) + kthread_destroy_worker(ctx->worker); + kfree(ctx); +} + +static void bpf_thread_wq_ctx_free_rcu(struct rcu_head rcu) +{ + struct bpf_thread_wq_ctx ctx = container_of(rcu, + struct bpf_thread_wq_ctx, + rcu); + + INIT_WORK(&ctx->destroy_work, bpf_thread_wq_destroy_work_fn); + schedule_work(&ctx->destroy_work); +} + +static void bpf_thread_wq_ctx_put(struct bpf_thread_wq_ctx ctx) +{ + if (!refcount_dec_and_test(&ctx->refcnt)) + return; + call_rcu_tasks_trace(&ctx->rcu, bpf_thread_wq_ctx_free_rcu); +} + +static void bpf_thread_wq_work_fn(struct kthread_work work) +{ + struct bpf_thread_wq_ctx ctx = container_of(work, + struct bpf_thread_wq_ctx, + work); + bpf_callback_t callback_fn; + void value = ctx->value; + struct bpf_map map = ctx->map; + void key; + u32 idx; + + BTF_TYPE_EMIT(struct bpf_thread_wq); + + callback_fn = READ_ONCE(ctx->callback_fn); + if (!callback_fn) + goto out; + key = map_key_from_value(map, value, &idx); + + rcu_read_lock_trace(); + migrate_disable(); + + callback_fn = READ_ONCE(ctx->callback_fn); + if (callback_fn) + callback_fn((u64)(long)map, (u64)(long)key, (u64)(long)value, + 0, 0); + + migrate_enable(); + rcu_read_unlock_trace(); + +out: + bpf_thread_wq_ctx_put(ctx); +} + +/* + * bpf_thread_wq_init() creates a kthread worker and may attach it to a cgroup. + * The helpers used here can sleep and acquire several locks through kthread + * creation/destruction, cgroup lookup and cgroup kthread attachment. Keep this + * kfunc available only to BPF_PROG_TYPE_SYSCALL programs so it is not invoked + * from BPF program contexts that already hold locks which could deadlock with + * those paths. + / +__bpf_kfunc int bpf_thread_wq_init(struct bpf_thread_wq twq, void p__map, + u64 cgroup_id, unsigned int flags) +{ + struct bpf_thread_wq_kern twk = (struct bpf_thread_wq_kern )twq; + struct bpf_map map = p__map; + struct bpf_thread_wq_ctx ctx, old_ctx; + struct kthread_worker worker; + struct cgroup cgrp = NULL; + void value; + u32 elem_id; + int err; + + BUILD_BUG_ON(sizeof(struct bpf_thread_wq_kern) + > sizeof(struct bpf_thread_wq)); + BUILD_BUG_ON(__alignof__(struct bpf_thread_wq_kern) + != __alignof__(struct bpf_thread_wq)); + + if (flags) + return -EINVAL; + + old_ctx = READ_ONCE(twk->ctx); + if (old_ctx) + return -EBUSY; + + value = (void )twq - map->record->thread_wq_off; + elem_id = bpf_map_elem_id_from_value(map, value); + worker = kthread_run_worker(0, "bpf_twq/%d/%x", map->id, elem_id); + if (IS_ERR(worker)) + return PTR_ERR(worker); + + /* Setup ctx. / + ctx = bpf_map_kmalloc_nolock(map, sizeof(ctx), GFP_KERNEL, + map->numa_node); + if (!ctx) { + err = -ENOMEM; + goto destroy_worker; + } + memset(ctx, 0, sizeof(ctx)); + ctx->worker = worker; + ctx->map = map; + ctx->value = value; + refcount_set(&ctx->refcnt, 1); + kthread_init_work(&ctx->work, bpf_thread_wq_work_fn); + + if (cgroup_id) { +#ifdef CONFIG_CGROUPS + cgrp = cgroup_get_from_id(cgroup_id); + if (IS_ERR(cgrp)) { + err = PTR_ERR(cgrp); + goto kfree_ctx; + } + ctx->cgrp = cgrp; + + / + * kthread_run_worker() wakes the kthread, but it may not have + * executed cgroup_kthread_ready() yet, which clears + * no_cgroup_migration. + * Do a queue work and flush to wait the kthread run. + / + refcount_inc(&ctx->refcnt); + if (!kthread_queue_work(ctx->worker, &ctx->work)) { + refcount_dec(&ctx->refcnt); + err = -EBUSY; + goto cgroup_put; + } + kthread_flush_work(&ctx->work); + + if (worker->task->no_cgroup_migration) { + err = -EAGAIN; + goto cgroup_put; + } + + err = cgroup_kthread_attach(cgrp, worker->task); + if (err) + goto cgroup_put; +#else + err = -EOPNOTSUPP; + goto kfree_ctx; +#endif + } + + old_ctx = cmpxchg(&twk->ctx, NULL, ctx); + if (old_ctx) { + err = -EBUSY; + goto cgroup_put; + } + + / + * Paired with the map destruction path. Ensures that ctx is globally + * visible before we check map->usercnt. + * If usercnt has dropped to zero, the destruction path will either see + * the ctx (and cancel it) or we see usercnt == 0 here and cancel + * ourselves. + * Without this barrier, a CPU could reorder the load of usercnt before + * the cmpxchg store becomes visible, breaking the mutual exclusion + * guarantee. + / + smp_mb(); + + if (!atomic64_read(&map->usercnt)) { + bpf_thread_wq_cancel_and_free(twq); + return -EPERM; + } + + return 0; + +cgroup_put: +#ifdef CONFIG_CGROUPS + if (cgrp) + cgroup_put(cgrp); +#endif +kfree_ctx: + / + * Not use bpf_thread_wq_ctx_put because ctx has not yet entered + * the running state. + / + kfree(ctx); +destroy_worker: + kthread_destroy_worker(worker); + return err; +} + +__bpf_kfunc int bpf_thread_wq_set_callback(struct bpf_thread_wq twq, + int (callback_fn)(void map, + int key, + void value), + unsigned int flags, + struct bpf_prog_aux aux) +{ + struct bpf_thread_wq_kern twk = (struct bpf_thread_wq_kern )twq; + struct bpf_thread_wq_ctx ctx; + struct bpf_prog prog; + + if (flags) + return -EINVAL; + + ctx = READ_ONCE(twk->ctx); + if (!ctx) + return -EINVAL; + + prog = bpf_prog_inc_not_zero(aux->prog); + if (IS_ERR(prog)) + return PTR_ERR(prog); + + /* + * Allow set_callback only once to prevent UAF: a concurrent + * set_callback could bpf_prog_put() the prog while the worker + * kthread is still executing its callback. + / + if (cmpxchg(&ctx->prog, NULL, prog) != NULL) { + bpf_prog_put(prog); + return -EBUSY; + } + / + * Safe to set callback_fn after prog: bpf_thread_wq_start() and + * bpf_thread_wq_work_fn() both check callback_fn with READ_ONCE() + * and bail out if it is still NULL. + / + WRITE_ONCE(ctx->callback_fn, (void )callback_fn); + + return 0; +} + +__bpf_kfunc int +bpf_thread_wq_start(struct bpf_thread_wq twq, unsigned int flags) +{ + struct bpf_thread_wq_kern twk = (struct bpf_thread_wq_kern )twq; + struct bpf_thread_wq_ctx ctx; + int err; + + if (flags) + return -EINVAL; + + rcu_read_lock_trace(); + + err = 0; + + ctx = READ_ONCE(twk->ctx); + if (!ctx \|\| !READ_ONCE(ctx->callback_fn)) { + err = -EINVAL; + goto unlock; + } + + if (!refcount_inc_not_zero(&ctx->refcnt)) + err = -ENOENT; + +unlock: + rcu_read_unlock_trace(); + if (err) + return err; + + if (!kthread_queue_work(ctx->worker, &ctx->work)) { + bpf_thread_wq_ctx_put(ctx); + return -EBUSY; + } + + return 0; +} + +void bpf_thread_wq_cancel_and_free(void val) +{ + struct bpf_thread_wq_kern twk = val; + struct bpf_thread_wq_ctx ctx; + + ctx = xchg(&twk->ctx, NULL); + if (!ctx) + return; + + might_sleep(); + + / + * Prevent future callbacks from running and wait for any + * in-progress execution to finish. + / + WRITE_ONCE(ctx->callback_fn, NULL); + / + * kthread_cancel_work_sync() returns true when it dequeues a pending + * work item from the work_list without executing it. Each successful + * bpf_thread_wq_start() call increments ctx->refcnt and relies on the + * subsequent bpf_thread_wq_work_fn() execution to release that + * reference via bpf_thread_wq_ctx_put(). If the work was pending and + * got cancelled here, work_fn will never run for that queued instance, + * so we must drop the reference ourselves to avoid a permanent refcount + * leak. + * + * This covers two scenarios uniformly: + * 1. The work is purely pending (not currently executing) - e.g. a + * normal bpf_thread_wq_start() call queued it but the worker + * thread hasn't picked it up yet. + * 2. The work is currently in-flight AND was self-rescheduled from + * within the callback - kthread_cancel_work_sync() dequeues the + * re-queued pending node and then waits for the in-flight + * execution to complete. + * In both cases the return value is true, indicating one orphaned + * reference that needs to be released here. + / + if (kthread_cancel_work_sync(&ctx->work)) + bpf_thread_wq_ctx_put(ctx); + + / + * Drop our own reference. If the work was still in-flight above, + * the refcount won't hit zero here - it will reach zero when the + * work path calls bpf_thread_wq_ctx_put() upon completion. Either + * way, final cleanup (worker destruction, prog put, cgroup put, + * kfree) happens exclusively in the RCU callback to keep the + * teardown path single-threaded. + / + bpf_thread_wq_ctx_put(ctx); +} + __bpf_kfunc_end_defs(); static void bpf_task_work_cancel_scheduled(struct irq_work irq_work) @@ -4917,6 +5321,8 @@ BTF_ID_FLAGS(func, bpf_modify_return_test_tp) BTF_ID_FLAGS(func, bpf_wq_init) BTF_ID_FLAGS(func, bpf_wq_set_callback, KF_IMPLICIT_ARGS) BTF_ID_FLAGS(func, bpf_wq_start) +BTF_ID_FLAGS(func, bpf_thread_wq_set_callback, KF_IMPLICIT_ARGS) +BTF_ID_FLAGS(func, bpf_thread_wq_start) BTF_ID_FLAGS(func, bpf_preempt_disable) BTF_ID_FLAGS(func, bpf_preempt_enable) BTF_ID_FLAGS(func, bpf_iter_bits_new, KF_ITER_NEW) @@ -4978,6 +5384,15 @@ static const struct btf_kfunc_id_set common_kfunc_set = { .set = &common_btf_ids, }; +BTF_KFUNCS_START(syscall_btf_ids) +BTF_ID_FLAGS(func, bpf_thread_wq_init, KF_SLEEPABLE) +BTF_KFUNCS_END(syscall_btf_ids) + +static const struct btf_kfunc_id_set syscall_kfunc_set = { + .owner = THIS_MODULE, + .set = &syscall_btf_ids, +}; + static int __init kfunc_init(void) { int ret; @@ -5000,6 +5415,7 @@ static int __init kfunc_init(void) ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, &generic_kfunc_set); ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL, &generic_kfunc_set); ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_CGROUP_SKB, &generic_kfunc_set); + ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL, &syscall_kfunc_set); ret = ret ?: register_btf_id_dtor_kfuncs(generic_dtors, ARRAY_SIZE(generic_dtors), THIS_MODULE); @@ -5037,4 +5453,6 @@ void bpf_map_free_internal_structs(struct bpf_map map, void val) bpf_obj_free_workqueue(map->record, val); if (btf_record_has_field(map->record, BPF_TASK_WORK)) bpf_obj_free_task_work(map->record, val); + if (btf_record_has_field(map->record, BPF_THREAD_WQ)) + bpf_obj_free_thread_wq(map->record, val); } diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 8d111da886553..dea14823bacd2 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -688,6 +688,7 @@ void btf_record_free(struct btf_record rec) case BPF_REFCOUNT: case BPF_WORKQUEUE: case BPF_TASK_WORK: + case BPF_THREAD_WQ: / Nothing to release / break; default: @@ -742,6 +743,7 @@ struct btf_record btf_record_dup(const struct btf_record rec) case BPF_REFCOUNT: case BPF_WORKQUEUE: case BPF_TASK_WORK: + case BPF_THREAD_WQ: / Nothing to acquire / break; default: @@ -807,6 +809,13 @@ void bpf_obj_free_task_work(const struct btf_record rec, void obj) bpf_task_work_cancel_and_free(obj + rec->task_work_off); } +void bpf_obj_free_thread_wq(const struct btf_record rec, void obj) +{ + if (WARN_ON_ONCE(!btf_record_has_field(rec, BPF_THREAD_WQ))) + return; + bpf_thread_wq_cancel_and_free(obj + rec->thread_wq_off); +} + void bpf_obj_cancel_fields(struct bpf_map map, void obj) { bpf_map_free_internal_structs(map, obj); @@ -839,6 +848,9 @@ void bpf_obj_free_fields(const struct btf_record rec, void obj) case BPF_TASK_WORK: bpf_task_work_cancel_and_free(field_ptr); break; + case BPF_THREAD_WQ: + bpf_thread_wq_cancel_and_free(field_ptr); + break; case BPF_KPTR_UNREF: WRITE_ONCE((u64 )field_ptr, 0); break; @@ -1265,7 +1277,7 @@ static int map_check_btf(struct bpf_map map, struct bpf_token token, map->record = btf_parse_fields(btf, value_type, BPF_SPIN_LOCK \| BPF_RES_SPIN_LOCK \| BPF_TIMER \| BPF_KPTR \| BPF_LIST_HEAD \| BPF_RB_ROOT \| BPF_REFCOUNT \| BPF_WORKQUEUE \| BPF_UPTR \| - BPF_TASK_WORK, + BPF_TASK_WORK \| BPF_THREAD_WQ, map->value_size); if (!IS_ERR_OR_NULL(map->record)) { int i; @@ -1299,6 +1311,7 @@ static int map_check_btf(struct bpf_map map, struct bpf_token token, case BPF_TIMER: case BPF_WORKQUEUE: case BPF_TASK_WORK: + case BPF_THREAD_WQ: if (map->map_type != BPF_MAP_TYPE_HASH && map->map_type != BPF_MAP_TYPE_RHASH && map->map_type != BPF_MAP_TYPE_LRU_HASH && diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index e6233c0081d10..44e79fe480b6b 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -477,6 +477,7 @@ static bool is_async_callback_calling_kfunc(u32 btf_id); static bool is_callback_calling_kfunc(u32 btf_id); static bool is_bpf_wq_set_callback_kfunc(u32 btf_id); +static bool is_bpf_thread_wq_set_callback_kfunc(u32 btf_id); static bool is_task_work_add_kfunc(u32 func_id); static bool is_sync_callback_calling_function(enum bpf_func_id func_id) @@ -516,9 +517,11 @@ static bool is_async_cb_sleepable(struct bpf_verifier_env env, struct bpf_insn if (bpf_helper_call(insn) && insn->imm == BPF_FUNC_timer_set_callback) return false; - /* bpf_wq and bpf_task_work callbacks are always sleepable. / + / bpf_wq, bpf_thread_wq and bpf_task_work callbacks are always sleepable. / if (bpf_pseudo_kfunc_call(insn) && insn->off == 0 && - (is_bpf_wq_set_callback_kfunc(insn->imm) \|\| is_task_work_add_kfunc(insn->imm))) + (is_bpf_wq_set_callback_kfunc(insn->imm) \|\| + is_bpf_thread_wq_set_callback_kfunc(insn->imm) \|\| + is_task_work_add_kfunc(insn->imm))) return true; verifier_bug(env, "unhandled async callback in is_async_cb_sleepable"); @@ -1871,7 +1874,10 @@ static void refine_map_lookup_value(struct bpf_reg_state reg) * as UID of the inner map. / if (btf_record_has_field(map->inner_map_meta->record, - BPF_TIMER \| BPF_WORKQUEUE \| BPF_TASK_WORK)) + BPF_TIMER \| + BPF_WORKQUEUE \| + BPF_TASK_WORK \| + BPF_THREAD_WQ)) reg->map_uid = reg->id; } else if (map->map_type == BPF_MAP_TYPE_XSKMAP) { reg->type = PTR_TO_XDP_SOCK \| maybe_null; @@ -7243,6 +7249,9 @@ static int check_map_field_pointer(struct bpf_verifier_env env, struct bpf_reg_ case BPF_WORKQUEUE: field_off = map->record->wq_off; break; + case BPF_THREAD_WQ: + field_off = map->record->thread_wq_off; + break; default: verifier_bug(env, "unsupported BTF field type: %s\n", struct_name); return -EINVAL; @@ -10940,6 +10949,7 @@ enum { KF_ARG_WORKQUEUE_ID, KF_ARG_RES_SPIN_LOCK_ID, KF_ARG_TASK_WORK_ID, + KF_ARG_THREAD_WQ_ID, KF_ARG_PROG_AUX_ID, KF_ARG_TIMER_ID }; @@ -10953,6 +10963,7 @@ BTF_ID(struct, bpf_rb_node) BTF_ID(struct, bpf_wq) BTF_ID(struct, bpf_res_spin_lock) BTF_ID(struct, bpf_task_work) +BTF_ID(struct, bpf_thread_wq) BTF_ID(struct, bpf_prog_aux) BTF_ID(struct, bpf_timer) @@ -11013,6 +11024,11 @@ static bool is_kfunc_arg_task_work(const struct btf btf, const struct btf_param return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_TASK_WORK_ID); } +static bool is_kfunc_arg_thread_wq(const struct btf btf, const struct btf_param arg) +{ + return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_THREAD_WQ_ID); +} + static bool is_kfunc_arg_res_spin_lock(const struct btf btf, const struct btf_param arg) { return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_RES_SPIN_LOCK_ID); @@ -11132,6 +11148,7 @@ enum kfunc_ptr_arg_type { KF_ARG_PTR_TO_IRQ_FLAG, KF_ARG_PTR_TO_RES_SPIN_LOCK, KF_ARG_PTR_TO_TASK_WORK, + KF_ARG_PTR_TO_THREAD_WQ, }; enum special_kfunc_type { @@ -11178,6 +11195,7 @@ enum special_kfunc_type { KF_bpf_percpu_obj_drop, KF_bpf_throw, KF_bpf_wq_set_callback, + KF_bpf_thread_wq_set_callback, KF_bpf_preempt_disable, KF_bpf_preempt_enable, KF_bpf_iter_css_task_new, @@ -11258,6 +11276,7 @@ BTF_ID(func, bpf_percpu_obj_drop_impl) BTF_ID(func, bpf_percpu_obj_drop) BTF_ID(func, bpf_throw) BTF_ID(func, bpf_wq_set_callback) +BTF_ID(func, bpf_thread_wq_set_callback) BTF_ID(func, bpf_preempt_disable) BTF_ID(func, bpf_preempt_enable) #ifdef CONFIG_CGROUPS @@ -11460,6 +11479,8 @@ get_kfunc_arg_type(struct bpf_verifier_env env, struct bpf_call_arg_meta meta, arg_type = KF_ARG_PTR_TO_TIMER; else if (is_kfunc_arg_task_work(meta->btf, &args[arg])) arg_type = KF_ARG_PTR_TO_TASK_WORK; + else if (is_kfunc_arg_thread_wq(meta->btf, &args[arg])) + arg_type = KF_ARG_PTR_TO_THREAD_WQ; else if (is_kfunc_arg_irq_flag(meta->btf, &args[arg])) arg_type = KF_ARG_PTR_TO_IRQ_FLAG; else if (is_kfunc_arg_res_spin_lock(meta->btf, &args[arg])) @@ -11833,6 +11854,7 @@ static bool is_sync_callback_calling_kfunc(u32 btf_id) static bool is_async_callback_calling_kfunc(u32 btf_id) { return is_bpf_wq_set_callback_kfunc(btf_id) \|\| + is_bpf_thread_wq_set_callback_kfunc(btf_id) \|\| is_task_work_add_kfunc(btf_id); } @@ -11847,6 +11869,11 @@ static bool is_bpf_wq_set_callback_kfunc(u32 btf_id) return btf_id == special_kfunc_list[KF_bpf_wq_set_callback]; } +static bool is_bpf_thread_wq_set_callback_kfunc(u32 btf_id) +{ + return btf_id == special_kfunc_list[KF_bpf_thread_wq_set_callback]; +} + static bool is_callback_calling_kfunc(u32 btf_id) { return is_sync_callback_calling_kfunc(btf_id) \|\| @@ -12192,6 +12219,7 @@ static int check_kfunc_args(struct bpf_verifier_env env, struct bpf_call_arg_me case KF_ARG_PTR_TO_WORKQUEUE: case KF_ARG_PTR_TO_TIMER: case KF_ARG_PTR_TO_TASK_WORK: + case KF_ARG_PTR_TO_THREAD_WQ: case KF_ARG_PTR_TO_IRQ_FLAG: case KF_ARG_PTR_TO_RES_SPIN_LOCK: break; @@ -13139,6 +13167,16 @@ static int check_kfunc_call(struct bpf_verifier_env env, struct bpf_insn insn, } } + if (is_bpf_thread_wq_set_callback_kfunc(meta.func_id)) { + err = push_callback_call(env, insn, insn_idx, meta.subprogno, + set_timer_callback_state); + if (err) { + verbose(env, "kfunc %s#%d failed callback verification\n", + func_name, meta.func_id); + return err; + } + } + if (is_task_work_add_kfunc(meta.func_id)) { err = push_callback_call(env, insn, insn_idx, meta.subprogno, set_task_work_schedule_callback_state); diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c index 38f8d9df8fbc4..164e069c75762 100644 --- a/kernel/cgroup/cgroup.c +++ b/kernel/cgroup/cgroup.c @@ -3042,6 +3042,19 @@ int cgroup_attach_task(struct cgroup dst_cgrp, struct task_struct leader, return ret; } +int cgroup_kthread_attach(struct cgroup cgrp, struct task_struct task) +{ + int ret; + + cgroup_lock(); + cgroup_attach_lock(CGRP_ATTACH_LOCK_GLOBAL, NULL); + ret = cgroup_attach_task(cgrp, task, false); + cgroup_attach_unlock(CGRP_ATTACH_LOCK_GLOBAL, NULL); + cgroup_unlock(); + + return ret; +} + struct task_struct cgroup_procs_write_start(char buf, bool threadgroup, enum cgroup_attach_lock_mode lock_mode) { diff --git a/mm/bpf_memcontrol.c b/mm/bpf_memcontrol.c index 716df49d76477..3f7a5c97e1355 100644 --- a/mm/bpf_memcontrol.c +++ b/mm/bpf_memcontrol.c @@ -6,6 +6,7 @@ / #include <linux/memcontrol.h> +#include <linux/swap.h> #include <linux/bpf.h> __bpf_kfunc_start_defs(); @@ -159,6 +160,61 @@ __bpf_kfunc void bpf_mem_cgroup_flush_stats(struct mem_cgroup memcg) mem_cgroup_flush_stats(memcg); } +/* + * bpf_try_to_free_mem_cgroup_pages - attempt to reclaim pages from + * a memory cgroup + * @memcg: the target memory cgroup to reclaim from + * @nr_pages: the number of pages to reclaim + * @gfp_mask: GFP flags controlling the reclaim behavior + * @reclaim_options: bitmask of MEMCG_RECLAIM_* flags to tune + * reclaim strategy + * @swappiness: swappiness override value, or a sentinel to use + * the default + * + * BPF-facing wrapper around try_to_free_mem_cgroup_pages() that + * validates and translates the @swappiness argument before + * delegating to the core reclaim path. + * + * The @swappiness parameter follows these semantics: + * - Values in [MIN_SWAPPINESS, SWAPPINESS_ANON_ONLY] are passed + * through as an explicit swappiness override. + * - Values below MIN_SWAPPINESS are treated as "use the system + * default"; the override pointer is set to NULL and the cgroup's + * own swappiness setting takes effect. + * - Values above SWAPPINESS_ANON_ONLY are rejected as invalid. + * - If @reclaim_options does not include MEMCG_RECLAIM_PROACTIVE, + * the @swappiness override is ignored entirely by the core + * reclaim path and the system default is used regardless. + * + * Swap usage during reclaim is gated on @reclaim_options: swap is + * considered only when MEMCG_RECLAIM_MAY_SWAP is set. Without this + * flag, reclaim is restricted to file-backed pages regardless of the + * @swappiness value or the cgroup's swappiness setting. + * + * Return: + * The number of pages actually reclaimed on success, or 0 + * if @swappiness exceeds SWAPPINESS_ANON_ONLY. + / +__bpf_kfunc unsigned long +bpf_try_to_free_mem_cgroup_pages(struct mem_cgroup memcg, + unsigned long nr_pages, + gfp_t gfp_mask, + unsigned int reclaim_options, + int swappiness) +{ + int swapiness_ptr; + + if (swappiness > SWAPPINESS_ANON_ONLY) + return 0; + else if (swappiness < MIN_SWAPPINESS) + swapiness_ptr = NULL; + else + swapiness_ptr = &swappiness; + + return try_to_free_mem_cgroup_pages(memcg, nr_pages, gfp_mask, + reclaim_options, swapiness_ptr); +} + __bpf_kfunc_end_defs(); BTF_KFUNCS_START(bpf_memcontrol_kfuncs) @@ -172,6 +228,8 @@ BTF_ID_FLAGS(func, bpf_mem_cgroup_usage) BTF_ID_FLAGS(func, bpf_mem_cgroup_page_state) BTF_ID_FLAGS(func, bpf_mem_cgroup_flush_stats, KF_SLEEPABLE) +BTF_ID_FLAGS(func, bpf_try_to_free_mem_cgroup_pages, KF_SLEEPABLE) + BTF_KFUNCS_END(bpf_memcontrol_kfuncs) static const struct btf_kfunc_id_set bpf_memcontrol_kfunc_set = { diff --git a/tools/testing/selftests/bpf/bpf_experimental.h b/tools/testing/selftests/bpf/bpf_experimental.h index ff37ae5a113dc..ea905ea5603c2 100644 --- a/tools/testing/selftests/bpf/bpf_experimental.h +++ b/tools/testing/selftests/bpf/bpf_experimental.h @@ -351,6 +351,13 @@ extern void bpf_iter_css_destroy(struct bpf_iter_css it) __weak __ksym; extern int bpf_wq_init(struct bpf_wq wq, void p__map, unsigned int flags) __weak __ksym; extern int bpf_wq_start(struct bpf_wq wq, unsigned int flags) __weak __ksym; +struct bpf_thread_wq; +extern int bpf_thread_wq_init(struct bpf_thread_wq twq, void p__map, + __u64 cgroup_id, + unsigned int flags) __weak __ksym; +extern int bpf_thread_wq_start(struct bpf_thread_wq twq, + unsigned int flags) __weak __ksym; + struct bpf_iter_kmem_cache; extern int bpf_iter_kmem_cache_new(struct bpf_iter_kmem_cache it) __weak __ksym; extern struct kmem_cache bpf_iter_kmem_cache_next(struct bpf_iter_kmem_cache it) __weak __ksym; diff --git a/tools/testing/selftests/bpf/prog_tests/memcg_async_reclaim.c b/tools/testing/selftests/bpf/prog_tests/memcg_async_reclaim.c new file mode 100644 index 0000000000000..40a5fe62cb384 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/memcg_async_reclaim.c @@ -0,0 +1,479 @@ +// SPDX-License-Identifier: GPL-2.0 +/ + * Memory controller eBPF async reclaim test + / + +#include <test_progs.h> +#include <sys/mman.h> +#include <sys/stat.h> +#include <sys/time.h> +#include <sys/vfs.h> +#include <sys/wait.h> +#include <fcntl.h> +#include <linux/magic.h> +#include <unistd.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#include "cgroup_helpers.h" + +struct bpf_args_s { + u64 high_cgroup_id; + u64 low_cgroup_id; + u64 event_delta_threshold; + u64 check_ns; +}; + +#include "memcg_async_reclaim.skel.h" + +#define FILE_SIZE (32 1024 * 1024ul) +#define BUFFER_SIZE (4096) +#define CG_LIMIT (32 * 1024 * 1024ul) +#define READ_TIMES 16 + +#define CG_DIR "/memcg_async_reclaim" +#define CG_HIGH_DIR CG_DIR "/high" +#define CG_LOW_DIR CG_DIR "/low" + +#define CHECK_PERIOD_NS (2 * 1000 * 1000ull) +#define EVENT_DELTA_THRESHOLD 1 + +/* + * Test files must reside on a filesystem that supports page reclaim without + * swap (e.g. ext4). If /tmp is on tmpfs, the file pages are shmem-backed + * and can only be reclaimed through swap. But the test disables swap + * (memory.swap.max=0), making reclaim impossible and causing OOM. + * + * Pick a directory on a non-tmpfs filesystem: try $TMPDIR first, then /tmp, + * and fall back to the current directory if the chosen path is on tmpfs. + / +static int get_test_dir(char buf, size_t size) +{ + static const char * const candidates[] = { "/tmp", "." }; + const char tmpdir = getenv("TMPDIR"); + struct statfs sfs; + size_t i; + + if (tmpdir && tmpdir[0] && statfs(tmpdir, &sfs) == 0 && + sfs.f_type != TMPFS_MAGIC) { + snprintf(buf, size, "%s", tmpdir); + return 0; + } + + for (i = 0; i < ARRAY_SIZE(candidates); i++) { + if (statfs(candidates[i], &sfs) == 0 && + sfs.f_type != TMPFS_MAGIC) { + snprintf(buf, size, "%s", candidates[i]); + return 0; + } + } + + return -1; +} + +static int setup_high_low_cgroups(u64 high_cgroup_id, u64 low_cgroup_id) +{ + int ret; + char limit_buf[20]; + + ret = setup_cgroup_environment(); + if (!ASSERT_OK(ret, "setup_cgroup_environment")) + goto cleanup; + + ret = create_and_get_cgroup(CG_DIR); + if (!ASSERT_GE(ret, 0, "create_and_get_cgroup " CG_DIR)) + goto cleanup; + close(ret); + + ret = enable_controllers(CG_DIR, "memory"); + if (!ASSERT_OK(ret, "enable_controllers")) + goto cleanup; + + snprintf(limit_buf, sizeof(limit_buf), "%lu", CG_LIMIT); + ret = write_cgroup_file(CG_DIR, "memory.max", limit_buf); + if (!ASSERT_OK(ret, "write_cgroup_file memory.max")) + goto cleanup; + + ret = write_cgroup_file(CG_DIR, "memory.swap.max", "0"); + if (!ASSERT_OK(ret, "write_cgroup_file memory.swap.max")) + goto cleanup; + + ret = create_and_get_cgroup(CG_HIGH_DIR); + if (!ASSERT_GE(ret, 0, "create_and_get_cgroup " CG_HIGH_DIR)) + goto cleanup; + close(ret); + + high_cgroup_id = get_cgroup_id(CG_HIGH_DIR); + if (!ASSERT_GT(high_cgroup_id, 0, "get_cgroup_id")) + goto cleanup; + + ret = create_and_get_cgroup(CG_LOW_DIR); + if (!ASSERT_GE(ret, 0, "create_and_get_cgroup " CG_LOW_DIR)) + goto cleanup; + close(ret); + + low_cgroup_id = get_cgroup_id(CG_LOW_DIR); + if (!ASSERT_GT(high_cgroup_id, 0, "get_cgroup_id")) + goto cleanup; + + return 0; + +cleanup: + cleanup_cgroup_environment(); + return -1; +} + +static int write_file(const char filename) +{ + int ret = -1; + size_t written = 0; + char buffer; + FILE fp; + + fp = fopen(filename, "wb"); + if (!fp) + goto out; + + buffer = malloc(BUFFER_SIZE); + if (!buffer) + goto cleanup_fp; + + memset(buffer, 'A', BUFFER_SIZE); + + while (written < FILE_SIZE) { + size_t to_write = FILE_SIZE - written < BUFFER_SIZE ? + FILE_SIZE - written : BUFFER_SIZE; + + if (fwrite(buffer, 1, to_write, fp) != to_write) + goto cleanup; + written += to_write; + } + + ret = 0; +cleanup: + free(buffer); +cleanup_fp: + fclose(fp); +out: + return ret; +} + +static int read_file(const char filename, int iterations) +{ + int ret = -1; + long page_size = sysconf(_SC_PAGESIZE); + char map; + size_t i; + int fd; + struct stat sb; + + fd = open(filename, O_RDONLY); + if (fd == -1) + goto out; + + if (fstat(fd, &sb) == -1) + goto cleanup_fd; + + if (sb.st_size != FILE_SIZE) { + fprintf(stderr, "File size mismatch: expected %lu, got %lu\n", + (unsigned long)FILE_SIZE, (unsigned long)sb.st_size); + goto cleanup_fd; + } + + map = mmap(NULL, FILE_SIZE, PROT_READ, MAP_PRIVATE, fd, 0); + if (map == MAP_FAILED) + goto cleanup_fd; + + for (int iter = 0; iter < iterations; iter++) { + for (i = 0; i < FILE_SIZE; i += page_size) { + /* access a byte to trigger page fault / + asm volatile("" :: "r"(map[i]) : "memory"); + } + } + + if (munmap(map, FILE_SIZE) == -1) + goto cleanup_fd; + + ret = 0; + +cleanup_fd: + close(fd); +out: + return ret; +} + +static int real_test_child_work(const char cgroup_path, char data_filename, + char time_filename, int read_times) +{ + struct timeval start, end; + double elapsed; + FILE fp; + + if (!ASSERT_OK(join_parent_cgroup(cgroup_path), "join_parent_cgroup")) + return -1; + + gettimeofday(&start, NULL); + + if (!ASSERT_OK(write_file(data_filename), "write_file")) + return -1; + + if (!ASSERT_OK(read_file(data_filename, read_times), "read_file")) + return -1; + + gettimeofday(&end, NULL); + + if (!time_filename) + return 0; + + elapsed = (end.tv_sec - start.tv_sec) + + (end.tv_usec - start.tv_usec) / 1000000.0; + printf("%.6f\n", elapsed); + + fp = fopen(time_filename, "w"); + if (!ASSERT_OK_PTR(fp, "fopen")) + return -1; + fprintf(fp, "%.6f", elapsed); + fclose(fp); + + return 0; +} + +static int get_time(char time_filename, double time) +{ + int ret = -1; + FILE fp; + char buf[64]; + + fp = fopen(time_filename, "r"); + if (!ASSERT_OK_PTR(fp, "fopen")) + goto out; + + if (!ASSERT_OK_PTR(fgets(buf, sizeof(buf), fp), "fgets")) + goto cleanup; + + if (sscanf(buf, "%lf", time) != 1) { + PRINT_FAIL("sscanf %s", buf); + goto cleanup; + } + + ret = 0; +cleanup: + fclose(fp); +out: + return ret; +} + +static int +run_high_low_workload(double high_elapsed, double low_elapsed, int read_times) +{ + char test_dir[PATH_MAX], high_data_file[PATH_MAX], low_data_file[PATH_MAX]; + char high_time_file[PATH_MAX], low_time_file[PATH_MAX]; + int ret, fd; + pid_t high_pid, low_pid; + int status; + + ret = get_test_dir(test_dir, sizeof(test_dir)); + if (!ASSERT_OK(ret, "get_test_dir: no non-tmpfs directory found")) + return -1; + + fd = snprintf(high_data_file, sizeof(high_data_file), + "%s/memcg_async_high_data_XXXXXX", test_dir); + if (!ASSERT_LT(fd, sizeof(high_data_file), "high_data_file path")) + return -1; + + fd = snprintf(low_data_file, sizeof(low_data_file), + "%s/memcg_async_low_data_XXXXXX", test_dir); + if (!ASSERT_LT(fd, sizeof(low_data_file), "low_data_file path")) + return -1; + + fd = snprintf(high_time_file, sizeof(high_time_file), + "%s/memcg_async_high_time_XXXXXX", test_dir); + if (!ASSERT_LT(fd, sizeof(high_time_file), "high_time_file path")) + return -1; + + fd = snprintf(low_time_file, sizeof(low_time_file), + "%s/memcg_async_low_time_XXXXXX", test_dir); + if (!ASSERT_LT(fd, sizeof(low_time_file), "low_time_file path")) + return -1; + + fd = mkstemp(high_data_file); + if (!ASSERT_GE(fd, 0, "mkstemp")) + return -1; + close(fd); + + fd = mkstemp(low_data_file); + if (!ASSERT_GE(fd, 0, "mkstemp")) + goto cleanup_high_data; + close(fd); + + fd = mkstemp(high_time_file); + if (!ASSERT_GE(fd, 0, "mkstemp")) + goto cleanup_low_data; + close(fd); + + fd = mkstemp(low_time_file); + if (!ASSERT_GE(fd, 0, "mkstemp")) + goto cleanup_high_time; + close(fd); + + low_pid = fork(); + if (!ASSERT_GE(low_pid, 0, "fork low")) + goto cleanup_low_time; + if (low_pid == 0) + exit(real_test_child_work(CG_LOW_DIR, low_data_file, + low_time_file, read_times)); + + high_pid = fork(); + if (!ASSERT_GE(high_pid, 0, "fork high")) { + (void)waitpid(low_pid, NULL, 0); + goto cleanup_low_time; + } + if (high_pid == 0) + exit(real_test_child_work(CG_HIGH_DIR, high_data_file, + high_time_file, read_times)); + + ret = waitpid(low_pid, &status, 0); + if (!ASSERT_GT(ret, 0, "low waitpid")) + goto cleanup_low_time; + if (!ASSERT_TRUE(WIFEXITED(status), "low exited")) + goto cleanup_low_time; + if (!ASSERT_EQ(WEXITSTATUS(status), 0, "low exit status")) + goto cleanup_low_time; + + ret = waitpid(high_pid, &status, 0); + if (!ASSERT_GT(ret, 0, "high waitpid")) + goto cleanup_low_time; + if (!ASSERT_TRUE(WIFEXITED(status), "high exited")) + goto cleanup_low_time; + if (!ASSERT_EQ(WEXITSTATUS(status), 0, "high exit status")) + goto cleanup_low_time; + + if (get_time(high_time_file, high_elapsed)) + goto cleanup_low_time; + if (get_time(low_time_file, low_elapsed)) + goto cleanup_low_time; + + ret = 0; + +cleanup_low_time: + unlink(low_time_file); +cleanup_high_time: + unlink(high_time_file); +cleanup_low_data: + unlink(low_data_file); +cleanup_high_data: + unlink(high_data_file); + return ret; +} + +static int +setup_bpf(u64 high_cgroup_id, u64 low_cgroup_id, + struct memcg_async_reclaim *skel_ptr, bool use_thread_wq) +{ + struct memcg_async_reclaim skel; + struct bpf_args_s bpf_args = { + .high_cgroup_id = high_cgroup_id, + .low_cgroup_id = low_cgroup_id, + .event_delta_threshold = EVENT_DELTA_THRESHOLD, + .check_ns = CHECK_PERIOD_NS, + }; + LIBBPF_OPTS(bpf_test_run_opts, run_opts, + .ctx_in = &bpf_args, + .ctx_size_in = sizeof(bpf_args)); + int prog_init_fd, err; + + skel = memcg_async_reclaim__open_and_load(); + if (!ASSERT_OK_PTR(skel, "memcg_async_reclaim__open_and_load")) + return -1; + + if (use_thread_wq) + prog_init_fd = bpf_program__fd(skel->progs.thread_wq_prog_init); + else + prog_init_fd = bpf_program__fd(skel->progs.wq_prog_init); + if (!ASSERT_GE(prog_init_fd, 0, "bpf_program__fd")) + goto error_out; + + err = bpf_prog_test_run_opts(prog_init_fd, &run_opts); + if (!ASSERT_OK(err, "bpf_prog_test_run_opts")) + goto error_out; + if (!ASSERT_EQ(run_opts.retval, 0, "prog_init retval")) + goto error_out; + + skel_ptr = skel; + return 0; + +error_out: + memcg_async_reclaim__destroy(skel); + return -1; +} + +void test_memcg_wq_async_reclaim(void) +{ + u64 high_cgroup_id, low_cgroup_id; + int err; + double high_time = 0.0, low_time = 0.0; + struct memcg_async_reclaim skel = NULL; + + err = setup_high_low_cgroups(&high_cgroup_id, &low_cgroup_id); + if (!ASSERT_OK(err, "setup_high_low_cgroups reclaim")) + return; + + err = setup_bpf(high_cgroup_id, low_cgroup_id, &skel, false); + if (!ASSERT_OK(err, "setup_bpf")) + goto out; + + err = run_high_low_workload(&high_time, &low_time, READ_TIMES); + if (!ASSERT_OK(err, "run_high_low_workload reclaim")) + goto out; + + if (high_time >= low_time) { + PRINT_FAIL("high cgroup not improved with async reclaim: high_time=%f low_time=%f", + high_time, low_time); + } + +out: + if (skel) + memcg_async_reclaim__destroy(skel); + /* + * Wait for bpf_wq to release the reference to cgroup + * to ensure the successful deletion of cgroup. + / + sleep(1); + cleanup_cgroup_environment(); +} + +void test_memcg_thread_wq_async_reclaim(void) +{ + u64 high_cgroup_id, low_cgroup_id; + int err; + double high_time = 0.0, low_time = 0.0; + struct memcg_async_reclaim skel = NULL; + + err = setup_high_low_cgroups(&high_cgroup_id, &low_cgroup_id); + if (!ASSERT_OK(err, "setup_high_low_cgroups reclaim")) + return; + + err = setup_bpf(high_cgroup_id, low_cgroup_id, &skel, true); + if (!ASSERT_OK(err, "setup_bpf")) + goto out; + + err = run_high_low_workload(&high_time, &low_time, READ_TIMES); + if (!ASSERT_OK(err, "run_high_low_workload reclaim")) + goto out; + + if (high_time >= low_time) { + PRINT_FAIL("high cgroup not improved with async reclaim: high_time=%f low_time=%f", + high_time, low_time); + } + +out: + if (skel) + memcg_async_reclaim__destroy(skel); + /* + * Wait for bpf_thread_wq to release the reference to cgroup + * to ensure the successful deletion of cgroup. + / + sleep(1); + cleanup_cgroup_environment(); +} diff --git a/tools/testing/selftests/bpf/prog_tests/thread_wq_cgroup.c b/tools/testing/selftests/bpf/prog_tests/thread_wq_cgroup.c new file mode 100644 index 0000000000000..7537b03f17e2a --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/thread_wq_cgroup.c @@ -0,0 +1,87 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <test_progs.h> +#include <unistd.h> +#include "cgroup_helpers.h" +#include "thread_wq_cgroup.skel.h" + +#define TEST_CGROUP "/thread_wq_test" +#define WAIT_TIMEOUT_SECS 30 + +void test_thread_wq_cgroup(void) +{ + struct thread_wq_cgroup skel = NULL; + int err, prog_fd, cg_fd = -1; + unsigned long long cg_id; + int waited_secs; + + LIBBPF_OPTS(bpf_test_run_opts, topts); + + err = setup_cgroup_environment(); + if (!ASSERT_OK(err, "setup_cgroup_environment")) + return; + cg_fd = create_and_get_cgroup(TEST_CGROUP); + if (!ASSERT_GE(cg_fd, 0, "create_and_get_cgroup")) + goto cleanup; + cg_id = get_cgroup_id(TEST_CGROUP); + if (!ASSERT_GT(cg_id, 0ULL, "get_cgroup_id")) + goto cleanup; + + skel = thread_wq_cgroup__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + goto cleanup; + + prog_fd = bpf_program__fd(skel->progs.start_thread_wq); + + /* Run bpf_thread_wq in the specified cgroup. / + skel->bss->test_key = 0; + skel->bss->target_cgroup_id = cg_id; + skel->bss->callback_cgroup_id = 0; + skel->bss->twq_done = 0; + if (!ASSERT_OK(bpf_prog_test_run_opts(prog_fd, &topts), + "bpf_prog_test_run_opts in cgroup")) + goto cleanup; + if (!ASSERT_OK(topts.retval, "retval in cgroup")) + goto cleanup; + for (waited_secs = 0; waited_secs < WAIT_TIMEOUT_SECS; waited_secs++) { + if (skel->bss->twq_done) + break; + sleep(1); + } + if (!ASSERT_TRUE(skel->bss->twq_done, "twq_done in cgroup")) + goto cleanup; + if (!ASSERT_EQ(skel->bss->callback_cgroup_id, cg_id, + "callback_cgroup_id in cgroup")) + goto cleanup; + + / Run bpf_thread_wq without cgroup attachment (cgroup_id = 0). / + LIBBPF_OPTS_RESET(topts); + skel->bss->test_key = 1; + skel->bss->target_cgroup_id = 0; + skel->bss->callback_cgroup_id = 0; + skel->bss->twq_done = 0; + if (!ASSERT_OK(bpf_prog_test_run_opts(prog_fd, &topts), + "bpf_prog_test_run_opts without cgroup")) + goto cleanup; + if (!ASSERT_OK(topts.retval, "retval without cgroup")) + goto cleanup; + for (waited_secs = 0; waited_secs < WAIT_TIMEOUT_SECS; waited_secs++) { + if (skel->bss->twq_done) + break; + sleep(1); + } + if (!ASSERT_TRUE(skel->bss->twq_done, "twq_done without cgroup")) + goto cleanup; + if (!ASSERT_NEQ(skel->bss->callback_cgroup_id, cg_id, + "callback_cgroup_id without cgroup")) + goto cleanup; + +cleanup: + if (skel) { + thread_wq_cgroup__destroy(skel); + / Wait thread_wq kthread quit. / + sleep(2); + } + if (cg_fd >= 0) + close(cg_fd); + cleanup_cgroup_environment(); +} diff --git a/tools/testing/selftests/bpf/progs/memcg_async_reclaim.c b/tools/testing/selftests/bpf/progs/memcg_async_reclaim.c new file mode 100644 index 0000000000000..eaccc8a37388b --- /dev/null +++ b/tools/testing/selftests/bpf/progs/memcg_async_reclaim.c @@ -0,0 +1,255 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include "vmlinux.h" +#include "bpf_experimental.h" +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> + +#define BIT(nr) (1UL << (nr)) + +#define ___GFP_IO BIT(___GFP_IO_BIT) +#define ___GFP_FS BIT(___GFP_FS_BIT) +#define ___GFP_DIRECT_RECLAIM BIT(___GFP_DIRECT_RECLAIM_BIT) +#define ___GFP_KSWAPD_RECLAIM BIT(___GFP_KSWAPD_RECLAIM_BIT) + +#define __GFP_IO ((gfp_t)___GFP_IO) +#define __GFP_FS ((gfp_t)___GFP_FS) +#define __GFP_DIRECT_RECLAIM ((gfp_t)___GFP_DIRECT_RECLAIM) +#define __GFP_KSWAPD_RECLAIM ((gfp_t)___GFP_KSWAPD_RECLAIM) +#define __GFP_RECLAIM ((gfp_t)(___GFP_DIRECT_RECLAIM \| ___GFP_KSWAPD_RECLAIM)) + +#define GFP_KERNEL (__GFP_RECLAIM \| __GFP_IO \| __GFP_FS) +#define CLOCK_MONOTONIC_ID 1 +#define RECLAIM_PAGES 32 +#define RECLAIM_MAX_ITER 32 + +struct bpf_args_s { + u64 high_cgroup_id; + u64 low_cgroup_id; + u64 event_delta_threshold; + u64 check_ns; +}; + +struct cgroup_memcg { + struct cgroup cgrp; + struct mem_cgroup memcg; +}; + +static u64 wq_high_cgroup_id; +static u64 wq_low_cgroup_id; + +static int get_cgroup_memcg_from_id(u64 cgroup_id, struct cgroup_memcg cm) +{ + cm->cgrp = bpf_cgroup_from_id(cgroup_id); + if (!cm->cgrp) + return -1; + + cm->memcg = bpf_get_mem_cgroup(&cm->cgrp->self); + if (!cm->memcg) { + bpf_cgroup_release(cm->cgrp); + return -1; + } + + return 0; +} + +static void put_cgroup_memcg(struct cgroup_memcg cm) +{ + bpf_put_mem_cgroup(cm->memcg); + bpf_cgroup_release(cm->cgrp); +} + +static int get_cgroup_event(u64 cgroup_id, u64 val) +{ + struct cgroup_memcg cm; + + if (get_cgroup_memcg_from_id(cgroup_id, &cm)) + return -1; + bpf_mem_cgroup_flush_stats(cm.memcg); + val = bpf_mem_cgroup_page_state(cm.memcg, WORKINGSET_REFAULT_FILE); + put_cgroup_memcg(&cm); + + return 0; +} + +static bool +should_reclaim_cgroup(u64 cgroup_id, u64 prev_event, u64 event_delta_threshold) +{ + u64 cur, delta; + + if (get_cgroup_event(cgroup_id, &cur)) + return false; + + delta = cur - prev_event; + prev_event = cur; + + return delta >= event_delta_threshold; +} + +static int reclaim_cgroup(u64 cgroup_id) +{ + struct cgroup_memcg cm; + int i; + + if (get_cgroup_memcg_from_id(cgroup_id, &cm)) + return 0; + + for (i = 0; i < RECLAIM_MAX_ITER; i++) { + if (!bpf_try_to_free_mem_cgroup_pages(cm.memcg, RECLAIM_PAGES, + GFP_KERNEL, 0, -1)) + break; + } + + put_cgroup_memcg(&cm); + + return 0; +} + +struct wq_elem { + struct bpf_timer timer; + struct bpf_wq work; + u64 prev_event; + u64 event_delta_threshold; + u64 check_ns; +}; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, __u32); + __type(value, struct wq_elem); +} wq_map SEC(".maps"); + +static int async_free(void map, int key, void value) +{ + struct wq_elem elem = value; + + if (should_reclaim_cgroup(wq_high_cgroup_id, &elem->prev_event, + elem->event_delta_threshold)) { + reclaim_cgroup(wq_low_cgroup_id); + bpf_wq_start(&elem->work, 0); + } + + return 0; +} + +static int wq_timer_cb(void map, int key, struct wq_elem elem) +{ + bpf_wq_start(&elem->work, 0); + bpf_timer_start(&elem->timer, elem->check_ns, 0); + + return 0; +} + +SEC("syscall") +int wq_prog_init(struct bpf_args_s ctx) +{ + struct wq_elem elem; + __u32 key = 0; + int ret; + + elem = bpf_map_lookup_elem(&wq_map, &key); + if (!elem) + return -1; + + ret = bpf_wq_init(&elem->work, &wq_map, 0); + if (ret) + return ret; + + ret = bpf_wq_set_callback(&elem->work, async_free, 0); + if (ret) + return ret; + + ret = bpf_timer_init(&elem->timer, &wq_map, CLOCK_MONOTONIC_ID); + if (ret) + return ret; + + ret = bpf_timer_set_callback(&elem->timer, wq_timer_cb); + if (ret) + return ret; + + elem->prev_event = 0; + elem->event_delta_threshold = ctx->event_delta_threshold; + elem->check_ns = ctx->check_ns; + + wq_high_cgroup_id = ctx->high_cgroup_id; + wq_low_cgroup_id = ctx->low_cgroup_id; + + return bpf_timer_start(&elem->timer, elem->check_ns, 0); +} + +struct thread_wq_elem { + struct bpf_timer timer; + struct bpf_thread_wq work; + u64 prev_event; + u64 event_delta_threshold; + u64 check_ns; +}; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 1); + __type(key, __u32); + __type(value, struct thread_wq_elem); +} thread_wq_map SEC(".maps"); + +static int thread_async_free(void map, int key, void value) +{ + struct thread_wq_elem elem = value; + + if (should_reclaim_cgroup(wq_high_cgroup_id, &elem->prev_event, + elem->event_delta_threshold)) { + reclaim_cgroup(wq_low_cgroup_id); + bpf_thread_wq_start(&elem->work, 0); + } + + return 0; +} + +static int thread_wq_timer_cb(void map, int key, struct thread_wq_elem elem) +{ + bpf_thread_wq_start(&elem->work, 0); + bpf_timer_start(&elem->timer, elem->check_ns, 0); + + return 0; +} + +SEC("syscall") +int thread_wq_prog_init(struct bpf_args_s ctx) +{ + struct thread_wq_elem elem; + __u32 key = 0; + int ret; + + elem = bpf_map_lookup_elem(&thread_wq_map, &key); + if (!elem) + return -1; + + ret = bpf_thread_wq_init(&elem->work, &thread_wq_map, + ctx->low_cgroup_id, 0); + if (ret) + return ret; + + ret = bpf_thread_wq_set_callback(&elem->work, thread_async_free, 0); + if (ret) + return ret; + + ret = bpf_timer_init(&elem->timer, &thread_wq_map, CLOCK_MONOTONIC_ID); + if (ret) + return ret; + + ret = bpf_timer_set_callback(&elem->timer, thread_wq_timer_cb); + if (ret) + return ret; + + elem->prev_event = 0; + elem->event_delta_threshold = ctx->event_delta_threshold; + elem->check_ns = ctx->check_ns; + + wq_high_cgroup_id = ctx->high_cgroup_id; + wq_low_cgroup_id = ctx->low_cgroup_id; + + return bpf_timer_start(&elem->timer, elem->check_ns, 0); +} + +char LICENSE[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/thread_wq_cgroup.c b/tools/testing/selftests/bpf/progs/thread_wq_cgroup.c new file mode 100644 index 0000000000000..c70a37f55397d --- /dev/null +++ b/tools/testing/selftests/bpf/progs/thread_wq_cgroup.c @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 / + +#include "bpf_experimental.h" +#include <bpf/bpf_helpers.h> +#include "bpf_misc.h" + +char _license[] SEC("license") = "GPL"; + +struct elem { + struct bpf_thread_wq twq; +}; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 2); + __type(key, int); + __type(value, struct elem); +} map_arr SEC(".maps"); + +__u64 target_cgroup_id; +__u64 callback_cgroup_id; +int twq_done; +int test_key; + +static int twq_callback(void map, int key, void value) +{ + callback_cgroup_id = bpf_get_current_cgroup_id(); + twq_done = 1; + return 0; +} + +SEC("syscall") +int start_thread_wq(void ctx) +{ + struct elem val; + int key = test_key; + int ret; + + val = bpf_map_lookup_elem(&map_arr, &key); + if (!val) + return -1; + + ret = bpf_thread_wq_init(&val->twq, &map_arr, target_cgroup_id, 0); + if (ret) + goto out; + + ret = bpf_thread_wq_set_callback(&val->twq, twq_callback, 0); + if (ret) + goto out; + + ret = bpf_thread_wq_start(&val->twq, 0); + +out: + return ret; +}
3/2	2026/08/07 08:50	llm	patch-evaluator	0m Model: gemini-3.1-pro-preview Args: null Results: null Tokens: input: 25554 output: 29 thoughts: 361 Thoughts: Analyzing the New BPF Thread Workqueue Feature Okay, so this patch is introducing something interesting: a new BPF feature called `bpf_thread_wq`. Right off the bat, I see the addition of `BPF_THREAD_WQ` to `enum btf_field_type`, which tells me this is a pretty fundamental change to the BPF landscape. We're looking at four new functions: `bpf_thread_wq_init`, `bpf_thread_wq_set_callback`, `bpf_thread_wq_start`, and `bpf_thread_wq_cancel_and_free`. Then there's `bpf_try_to_free_mem_cgroup_pages` as well. This all looks like the framework to allow BPF programs to schedule tasks on kthread workers, and, as an optional feature, attach those tasks to a specific cgroup. Definitely sounds like a candidate for some serious fuzzing – gotta see where the edge cases lie. Let's nail down the key symbols we need to investigate: `bpf_thread_wq_init`, `bpf_thread_wq_set_callback`, `bpf_thread_wq_start`, and `bpf_try_to_free_mem_cgroup_pages`. Those are the entry points here. Next, I need to check for any critical `CONFIG` options that might be required. From the look of it, `bpf_thread_wq_init` will use `CONFIG_CGROUPS` to attach to a cgroup, but it looks like it will gracefully degrade with `-EOPNOTSUPP` if cgroups aren't enabled. That's good, less of a hard dependency. Now, `bpf_try_to_free_mem_cgroup_pages` – that's located in `mm/bpf_memcontrol.c`. Since that file is dealing with memory control, it makes sense that `CONFIG_MEMCG` and `CONFIG_BPF_SYSCALL` are likely dependencies, but I should probably check the `mm/Makefile` to confirm the details.
4/2	2026/08/07 08:50	tool	grepper	0m Args: { "Expression": "bpf_memcontrol.o", "PathPrefix": "mm/" } Results: { "Output": "mm/Makefile=106=ifdef CONFIG_BPF_SYSCALL\nmm/Makefile:107:obj-$(CONFIG_MEMCG) += bpf_memcontrol.o\nmm/Makefile-108-endif\n" }
5/2	2026/08/07 08:50	llm	patch-evaluator	0m Model: gemini-3.1-pro-preview Args: null Results: null Tokens: input: 26466 output: 205 thoughts: 832 Thoughts: Analyzing the New BPF Thread Workqueue Feature Okay, so I'm digging into this new BPF thread workqueue patch. First things first: it looks like we need `CONFIG_MEMCG` for `bpf_try_to_free_mem_cgroup_pages`, which makes sense for memory control, and `CONFIG_CGROUPS` is required for `bpf_thread_wq_init` to function fully with cgroups, and then of course `CONFIG_BPF_SYSCALL` is standard for BPF, so let's mark that as understood. Let me confirm that `bpf_thread_wq_init` is actually available. Yep, it's defined right there in `kernel/bpf/helpers.c`. Good. Alright, what else is in play? Hmm, `bpf_thread_wq_cancel_and_free` is interesting, it is called internally when maps are freed. It's not a direct kfunc used by BPF programs, but definitely part of the lifecycle. And `bpf_obj_free_thread_wq` is also internal. Okay, so the kfuncs we do need to focus on are: `bpf_thread_wq_init`, `bpf_thread_wq_set_callback`, `bpf_thread_wq_start`, and `bpf_try_to_free_mem_cgroup_pages`. Now, let's verify that these kfuncs are actually exposed. `bpf_thread_wq_init` is added to `syscall_btf_ids` which is registered for `BPF_PROG_TYPE_SYSCALL`. Good. `bpf_thread_wq_set_callback` and `bpf_thread_wq_start` are in `generic_btf_ids`, which is registered for a bunch of program types, so that's fine. And `bpf_try_to_free_mem_cgroup_pages` is in `bpf_memcontrol_kfuncs`. Confirmed, confirmed, confirmed. So, the key symbols to keep an eye on are: `bpf_thread_wq_init`, `bpf_thread_wq_set_callback`, `bpf_thread_wq_start`, and `bpf_try_to_free_mem_cgroup_pages`. And then the relevant config options are `MEMCG` and `CGROUPS`. Just double-checking... `bpf_thread_wq_cancel_and_free`. No, that's just a global function, internal to `kernel/bpf/helpers.c`. Wait...but `bpf_obj_free_thread_wq` and `bpf_obj_free_fields` call it. Oh, one more thing to consider: `cgroup_kthread_attach`, which is added in `kernel/cgroup/cgroup.c`. It's a global function, and a helper for `bpf_thread_wq_init`. Okay, I think that should be included in the list of focus symbols, even though it's technically a helper function. Alright, let's recap the reasoning: this patch introduces a new BPF thread workqueue, letting BPF programs schedule work in kthread context, optionally tied to a cgroup, and also adds `bpf_try_to_free_mem_cgroup_pages` for memory cgroup reclaim. These are functional changes to the BPF core and memory control, accessible from userspace through BPF programs, so definitely fuzzable. Final Focus Symbols: * `bpf_thread_wq_init` * `bpf_thread_wq_set_callback` * `bpf_thread_wq_start` * `bpf_try_to_free_mem_cgroup_pages` * `cgroup_kthread_attach` Relevant Configs: * `MEMCG` * `CGROUPS` Okay, that's everything. Let's send these results over.
6/2	2026/08/07 08:51	tool	set-results	0m Args: { "EnableConfigs": [ "MEMCG", "CGROUPS" ], "FocusSymbols": [ "bpf_thread_wq_init", "bpf_thread_wq_set_callback", "bpf_thread_wq_start", "bpf_try_to_free_mem_cgroup_pages", "cgroup_kthread_attach" ], "Reasoning": "The patch introduces a new BPF thread workqueue feature (`bpf_thread_wq`) that allows BPF programs to schedule work in a kthread context, optionally attached to a cgroup. It also adds a new BPF kfunc `bpf_try_to_free_mem_cgroup_pages` for memory cgroup reclaim. These are functional changes to the core BPF and memory control subsystems, reachable from userspace via BPF programs, and should be fuzzed.", "WorthFuzzing": true } Results: { "EnableConfigs": [ "MEMCG", "CGROUPS" ], "FocusSymbols": [ "bpf_thread_wq_init", "bpf_thread_wq_set_callback", "bpf_thread_wq_start", "bpf_try_to_free_mem_cgroup_pages", "cgroup_kthread_attach" ], "Reasoning": "The patch introduces a new BPF thread workqueue feature (`bpf_thread_wq`) that allows BPF programs to schedule work in a kthread context, optionally attached to a cgroup. It also adds a new BPF kfunc `bpf_try_to_free_mem_cgroup_pages` for memory cgroup reclaim. These are functional changes to the core BPF and memory control subsystems, reachable from userspace via BPF programs, and should be fuzzed.", "WorthFuzzing": true }

Trajectory: (Expand/Collapse All)

Seq

Timestamp

Type

Name

Duration

0/0

2026/08/07 08:50

flow

patch-triage

Args:

null

Results:

{
  "EnableConfigs": [
    "MEMCG",
    "CGROUPS"
  ],
  "FocusSymbols": [
    "bpf_thread_wq_init",
    "bpf_thread_wq_set_callback",
    "bpf_thread_wq_start",
    "bpf_try_to_free_mem_cgroup_pages",
    "cgroup_kthread_attach"
  ],
  "Reasoning": "The patch introduces a new BPF thread workqueue feature (`bpf_thread_wq`) that allows BPF programs to schedule work in a kthread context, optionally attached to a cgroup. It also adds a new BPF kfunc `bpf_try_to_free_mem_cgroup_pages` for memory cgroup reclaim. These are functional changes to the core BPF and memory control subsystems, reachable from userspace via BPF programs, and should be fuzzed.",
  "WorthFuzzing": true
}

1/1

2026/08/07 08:50

action

read-patch-diff

Args:

null

Results:

{
  "PatchDiff": "commit e72e2c008c951fa5d07abaad9d32e877879e2ea7\nAuthor: syz-cluster \u003ctriage@syzkaller.com\u003e\nDate:   Fri Aug 7 08:50:40 2026 +0000\n\n    syz-cluster: applied patch under review\n\ndiff --git a/include/linux/bpf.h b/include/linux/bpf.h\nindex 73bacfc6444df..d63ce83198696 100644\n--- a/include/linux/bpf.h\n+++ b/include/linux/bpf.h\n@@ -213,6 +213,7 @@ enum btf_field_type {\n \tBPF_UPTR       = (1 \u003c\u003c 11),\n \tBPF_RES_SPIN_LOCK = (1 \u003c\u003c 12),\n \tBPF_TASK_WORK  = (1 \u003c\u003c 13),\n+\tBPF_THREAD_WQ  = (1 \u003c\u003c 14),\n };\n \n enum bpf_cgroup_storage_type {\n@@ -267,6 +268,7 @@ struct btf_record {\n \tint wq_off;\n \tint refcount_off;\n \tint task_work_off;\n+\tint thread_wq_off;\n \tstruct btf_field fields[];\n };\n \n@@ -372,6 +374,8 @@ static inline const char *btf_field_type_name(enum btf_field_type type)\n \t\treturn \"bpf_refcount\";\n \tcase BPF_TASK_WORK:\n \t\treturn \"bpf_task_work\";\n+\tcase BPF_THREAD_WQ:\n+\t\treturn \"bpf_thread_wq\";\n \tdefault:\n \t\tWARN_ON_ONCE(1);\n \t\treturn \"unknown\";\n@@ -412,6 +416,8 @@ static inline u32 btf_field_type_size(enum btf_field_type type)\n \t\treturn sizeof(struct bpf_refcount);\n \tcase BPF_TASK_WORK:\n \t\treturn sizeof(struct bpf_task_work);\n+\tcase BPF_THREAD_WQ:\n+\t\treturn sizeof(struct bpf_thread_wq);\n \tdefault:\n \t\tWARN_ON_ONCE(1);\n \t\treturn 0;\n@@ -446,6 +452,8 @@ static inline u32 btf_field_type_align(enum btf_field_type type)\n \t\treturn __alignof__(struct bpf_refcount);\n \tcase BPF_TASK_WORK:\n \t\treturn __alignof__(struct bpf_task_work);\n+\tcase BPF_THREAD_WQ:\n+\t\treturn __alignof__(struct bpf_thread_wq);\n \tdefault:\n \t\tWARN_ON_ONCE(1);\n \t\treturn 0;\n@@ -478,6 +486,7 @@ static inline void bpf_obj_init_field(const struct btf_field *field, void *addr)\n \tcase BPF_KPTR_PERCPU:\n \tcase BPF_UPTR:\n \tcase BPF_TASK_WORK:\n+\tcase BPF_THREAD_WQ:\n \t\tbreak;\n \tdefault:\n \t\tWARN_ON_ONCE(1);\n@@ -502,6 +511,7 @@ static inline bool btf_field_is_nmi_safe(enum btf_field_type type)\n \tcase BPF_TASK_WORK:\n \tcase BPF_KPTR_UNREF:\n \tcase BPF_REFCOUNT:\n+\tcase BPF_THREAD_WQ:\n \t\treturn true;\n \tdefault:\n \t\treturn false;\n@@ -644,6 +654,7 @@ void copy_map_value_locked(struct bpf_map *map, void *dst, void *src,\n void bpf_timer_cancel_and_free(void *timer);\n void bpf_wq_cancel_and_free(void *timer);\n void bpf_task_work_cancel_and_free(void *timer);\n+void bpf_thread_wq_cancel_and_free(void *val);\n void bpf_list_head_free(const struct btf_field *field, void *list_head,\n \t\t\tstruct bpf_spin_lock *spin_lock);\n void bpf_rb_root_free(const struct btf_field *field, void *rb_root,\n@@ -701,7 +712,8 @@ bool bpf_map_meta_equal(const struct bpf_map *meta0,\n \n static inline bool bpf_map_has_internal_structs(struct bpf_map *map)\n {\n-\treturn btf_record_has_field(map-\u003erecord, BPF_TIMER | BPF_WORKQUEUE | BPF_TASK_WORK);\n+\treturn btf_record_has_field(map-\u003erecord, BPF_TIMER | BPF_WORKQUEUE |\n+\t\t\t\t\t\t BPF_TASK_WORK | BPF_THREAD_WQ);\n }\n \n void bpf_map_free_internal_structs(struct bpf_map *map, void *obj);\n@@ -2725,6 +2737,7 @@ bool btf_record_equal(const struct btf_record *rec_a, const struct btf_record *r\n void bpf_obj_free_timer(const struct btf_record *rec, void *obj);\n void bpf_obj_free_workqueue(const struct btf_record *rec, void *obj);\n void bpf_obj_free_task_work(const struct btf_record *rec, void *obj);\n+void bpf_obj_free_thread_wq(const struct btf_record *rec, void *obj);\n void bpf_obj_cancel_fields(struct bpf_map *map, void *obj);\n void bpf_obj_free_fields(const struct btf_record *rec, void *obj);\n void __bpf_obj_drop_impl(void *p, const struct btf_record *rec, bool percpu);\ndiff --git a/include/linux/cgroup.h b/include/linux/cgroup.h\nindex f2aa46a4f871e..9b4a8dc748ace 100644\n--- a/include/linux/cgroup.h\n+++ b/include/linux/cgroup.h\n@@ -923,4 +923,6 @@ struct cgroup *task_get_cgroup1(struct task_struct *tsk, int hierarchy_id);\n \n struct cgroup_of_peak *of_peak(struct kernfs_open_file *of);\n \n+int cgroup_kthread_attach(struct cgroup *cgrp, struct task_struct *task);\n+\n #endif /* _LINUX_CGROUP_H */\ndiff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h\nindex ffd96e8b920bd..0558520f67fe4 100644\n--- a/include/uapi/linux/bpf.h\n+++ b/include/uapi/linux/bpf.h\n@@ -7574,6 +7574,10 @@ struct bpf_wq {\n \t__u64 __opaque[2];\n } __attribute__((aligned(8)));\n \n+struct bpf_thread_wq {\n+\t__u64 __opaque[2];\n+} __attribute__((aligned(8)));\n+\n struct bpf_dynptr {\n \t__u64 __opaque[2];\n } __attribute__((aligned(8)));\ndiff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c\nindex 42414633cf263..bf6fb5f51d21a 100644\n--- a/kernel/bpf/btf.c\n+++ b/kernel/bpf/btf.c\n@@ -3665,6 +3665,7 @@ static int btf_get_field_type(const struct btf *btf, const struct btf_type *var_\n \t\t{ BPF_TIMER, \"bpf_timer\", true },\n \t\t{ BPF_WORKQUEUE, \"bpf_wq\", true },\n \t\t{ BPF_TASK_WORK, \"bpf_task_work\", true },\n+\t\t{ BPF_THREAD_WQ, \"bpf_thread_wq\", true },\n \t\t{ BPF_LIST_HEAD, \"bpf_list_head\", false },\n \t\t{ BPF_LIST_NODE, \"bpf_list_node\", false },\n \t\t{ BPF_RB_ROOT, \"bpf_rb_root\", false },\n@@ -3850,6 +3851,7 @@ static int btf_find_field_one(const struct btf *btf,\n \tcase BPF_RB_NODE:\n \tcase BPF_REFCOUNT:\n \tcase BPF_TASK_WORK:\n+\tcase BPF_THREAD_WQ:\n \t\tret = btf_find_struct(btf, var_type, off, sz, field_type,\n \t\t\t\t      info_cnt ? \u0026info[0] : \u0026tmp);\n \t\tif (ret \u003c 0)\n@@ -4145,6 +4147,7 @@ struct btf_record *btf_parse_fields(const struct btf *btf, const struct btf_type\n \trec-\u003ewq_off = -EINVAL;\n \trec-\u003erefcount_off = -EINVAL;\n \trec-\u003etask_work_off = -EINVAL;\n+\trec-\u003ethread_wq_off = -EINVAL;\n \tfor (i = 0; i \u003c cnt; i++) {\n \t\tfield_type_size = btf_field_type_size(info_arr[i].type);\n \t\tif (info_arr[i].off + field_type_size \u003e value_size) {\n@@ -4188,6 +4191,10 @@ struct btf_record *btf_parse_fields(const struct btf *btf, const struct btf_type\n \t\t\tWARN_ON_ONCE(rec-\u003etask_work_off \u003e= 0);\n \t\t\trec-\u003etask_work_off = rec-\u003efields[i].offset;\n \t\t\tbreak;\n+\t\tcase BPF_THREAD_WQ:\n+\t\t\tWARN_ON_ONCE(rec-\u003ethread_wq_off \u003e= 0);\n+\t\t\trec-\u003ethread_wq_off = rec-\u003efields[i].offset;\n+\t\t\tbreak;\n \t\tcase BPF_REFCOUNT:\n \t\t\tWARN_ON_ONCE(rec-\u003erefcount_off \u003e= 0);\n \t\t\t/* Cache offset for faster lookup at runtime */\ndiff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c\nindex 6388b6b23e490..8e01bad1b16a0 100644\n--- a/kernel/bpf/helpers.c\n+++ b/kernel/bpf/helpers.c\n@@ -29,6 +29,8 @@\n #include \u003clinux/task_work.h\u003e\n #include \u003clinux/irq_work.h\u003e\n #include \u003clinux/buildid.h\u003e\n+#include \u003clinux/kthread.h\u003e\n+#include \u003clinux/jhash.h\u003e\n \n #include \"../../lib/kstrtox.h\"\n \n@@ -1110,6 +1112,17 @@ static void *map_key_from_value(struct bpf_map *map, void *value, u32 *arr_idx)\n \treturn (void *)value - round_up(map-\u003ekey_size, 8);\n }\n \n+static u32 bpf_map_elem_id_from_value(struct bpf_map *map, void *value)\n+{\n+\tu32 arr_idx;\n+\tvoid *key;\n+\n+\tkey = map_key_from_value(map, value, \u0026arr_idx);\n+\tif (map-\u003emap_type == BPF_MAP_TYPE_ARRAY)\n+\t\treturn arr_idx;\n+\treturn jhash(key, map-\u003ekey_size, 0);\n+}\n+\n enum bpf_async_type {\n \tBPF_ASYNC_TYPE_TIMER = 0,\n \tBPF_ASYNC_TYPE_WQ,\n@@ -4769,6 +4782,397 @@ __bpf_kfunc int bpf_timer_cancel_async(struct bpf_timer *timer)\n \t}\n }\n \n+/*\n+ * BPF thread workqueue (kthread_worker based) implementation\n+ *\n+ * Why bpf_thread_wq does NOT use the bpf_async infrastructure:\n+ *\n+ * bpf_timer and bpf_wq share a common cleanup path via bpf_async:\n+ *\n+ *   bpf_async_cancel_and_free()\n+ *     -\u003e bpf_async_schedule_op()\n+ *       -\u003e irq_work_queue()                // schedule from any context\n+ *         -\u003e bpf_async_process_op()        // runs in hardirq context\n+ *           -\u003e bpf_wq_work() / timer cb    // atomic, non-sleepable\n+ *\n+ * This works for timer and workqueue because their callbacks and\n+ * cancellation (hrtimer_cancel / bpf_wq_cancel_and_free) complete\n+ * synchronously and need not sleep.  The hardirq context is sufficient.\n+ *\n+ * bpf_thread_wq is different: cancellation and final cleanup may need to\n+ * sleep:\n+ *\n+ *   kthread_cancel_work_sync()    - waits for the work to finish, can\n+ *                                    schedule out if the work is running.\n+ *   kthread_destroy_worker()      - stops the kthread, internally\n+ *                                    synchronizes with kthread exit.\n+ *   cgroup_put() (final put)      - may acquire cgroup_mutex and other\n+ *                                    sleeping locks during offline.\n+ *\n+ * None of these can be called from hardirq context (irq_work). Doing so\n+ * would trigger might_sleep() warnings or deadlock.\n+ *\n+ * Therefore bpf_thread_wq implements its own RCU-based cleanup:\n+ *\n+ *   bpf_thread_wq_cancel_and_free()        // sleepable (called from\n+ *                                          // map free / elem delete)\n+ *     -\u003e xchg(ctx, NULL)                   // detach atomically\n+ *     -\u003e kthread_cancel_work_sync()        // ok to sleep\n+ *     -\u003e bpf_thread_wq_ctx_put()\n+ *       -\u003e call_rcu_tasks_trace()          // wait for BPF callbacks\n+ *         -\u003e schedule_work()               // switch to sleepable context\n+ *           -\u003e bpf_thread_wq_destroy_work_fn()\n+ *             -\u003e bpf_prog_put()\n+ *             -\u003e cgroup_put()\n+ *             -\u003e kthread_destroy_worker()\n+ *             -\u003e kfree(ctx)\n+ *\n+ * The same design choice was made for bpf_task_work, which also avoids\n+ * bpf_async because task_work cancellation synchronizes with the task\n+ * and may sleep.\n+ */\n+\n+struct bpf_thread_wq_ctx {\n+\tstruct kthread_worker *worker;\n+\tstruct kthread_work work;\n+\tstruct bpf_prog *prog;\n+\tbpf_callback_t callback_fn;\n+\tstruct bpf_map *map;\n+\tvoid *value;\n+\tstruct cgroup *cgrp;\n+\trefcount_t refcnt;\n+\tstruct rcu_head rcu;\n+\tstruct work_struct destroy_work;\n+};\n+\n+/* Kernel-internal representation that fits in struct bpf_thread_wq */\n+struct bpf_thread_wq_kern {\n+\tstruct bpf_thread_wq_ctx *ctx;\n+} __aligned(8);\n+\n+static void bpf_thread_wq_destroy_work_fn(struct work_struct *work)\n+{\n+\tstruct bpf_thread_wq_ctx *ctx = container_of(work,\n+\t\t\t\t\t\t     struct bpf_thread_wq_ctx,\n+\t\t\t\t\t\t     destroy_work);\n+\n+\tif (ctx-\u003eprog)\n+\t\tbpf_prog_put(ctx-\u003eprog);\n+\tif (ctx-\u003ecgrp)\n+\t\tcgroup_put(ctx-\u003ecgrp);\n+\tif (ctx-\u003eworker)\n+\t\tkthread_destroy_worker(ctx-\u003eworker);\n+\tkfree(ctx);\n+}\n+\n+static void bpf_thread_wq_ctx_free_rcu(struct rcu_head *rcu)\n+{\n+\tstruct bpf_thread_wq_ctx *ctx = container_of(rcu,\n+\t\t\t\t\t\t     struct bpf_thread_wq_ctx,\n+\t\t\t\t\t\t     rcu);\n+\n+\tINIT_WORK(\u0026ctx-\u003edestroy_work, bpf_thread_wq_destroy_work_fn);\n+\tschedule_work(\u0026ctx-\u003edestroy_work);\n+}\n+\n+static void bpf_thread_wq_ctx_put(struct bpf_thread_wq_ctx *ctx)\n+{\n+\tif (!refcount_dec_and_test(\u0026ctx-\u003erefcnt))\n+\t\treturn;\n+\tcall_rcu_tasks_trace(\u0026ctx-\u003ercu, bpf_thread_wq_ctx_free_rcu);\n+}\n+\n+static void bpf_thread_wq_work_fn(struct kthread_work *work)\n+{\n+\tstruct bpf_thread_wq_ctx *ctx = container_of(work,\n+\t\t\t\t\t\t     struct bpf_thread_wq_ctx,\n+\t\t\t\t\t\t     work);\n+\tbpf_callback_t callback_fn;\n+\tvoid *value = ctx-\u003evalue;\n+\tstruct bpf_map *map = ctx-\u003emap;\n+\tvoid *key;\n+\tu32 idx;\n+\n+\tBTF_TYPE_EMIT(struct bpf_thread_wq);\n+\n+\tcallback_fn = READ_ONCE(ctx-\u003ecallback_fn);\n+\tif (!callback_fn)\n+\t\tgoto out;\n+\tkey = map_key_from_value(map, value, \u0026idx);\n+\n+\trcu_read_lock_trace();\n+\tmigrate_disable();\n+\n+\tcallback_fn = READ_ONCE(ctx-\u003ecallback_fn);\n+\tif (callback_fn)\n+\t\tcallback_fn((u64)(long)map, (u64)(long)key, (u64)(long)value,\n+\t\t\t    0, 0);\n+\n+\tmigrate_enable();\n+\trcu_read_unlock_trace();\n+\n+out:\n+\tbpf_thread_wq_ctx_put(ctx);\n+}\n+\n+/*\n+ * bpf_thread_wq_init() creates a kthread worker and may attach it to a cgroup.\n+ * The helpers used here can sleep and acquire several locks through kthread\n+ * creation/destruction, cgroup lookup and cgroup kthread attachment. Keep this\n+ * kfunc available only to BPF_PROG_TYPE_SYSCALL programs so it is not invoked\n+ * from BPF program contexts that already hold locks which could deadlock with\n+ * those paths.\n+ */\n+__bpf_kfunc int bpf_thread_wq_init(struct bpf_thread_wq *twq, void *p__map,\n+\t\t\t\t   u64 cgroup_id, unsigned int flags)\n+{\n+\tstruct bpf_thread_wq_kern *twk = (struct bpf_thread_wq_kern *)twq;\n+\tstruct bpf_map *map = p__map;\n+\tstruct bpf_thread_wq_ctx *ctx, *old_ctx;\n+\tstruct kthread_worker *worker;\n+\tstruct cgroup *cgrp = NULL;\n+\tvoid *value;\n+\tu32 elem_id;\n+\tint err;\n+\n+\tBUILD_BUG_ON(sizeof(struct bpf_thread_wq_kern)\n+\t\t\t\u003e sizeof(struct bpf_thread_wq));\n+\tBUILD_BUG_ON(__alignof__(struct bpf_thread_wq_kern)\n+\t\t\t!= __alignof__(struct bpf_thread_wq));\n+\n+\tif (flags)\n+\t\treturn -EINVAL;\n+\n+\told_ctx = READ_ONCE(twk-\u003ectx);\n+\tif (old_ctx)\n+\t\treturn -EBUSY;\n+\n+\tvalue = (void *)twq - map-\u003erecord-\u003ethread_wq_off;\n+\telem_id = bpf_map_elem_id_from_value(map, value);\n+\tworker = kthread_run_worker(0, \"bpf_twq/%d/%x\", map-\u003eid, elem_id);\n+\tif (IS_ERR(worker))\n+\t\treturn PTR_ERR(worker);\n+\n+\t/* Setup ctx. */\n+\tctx = bpf_map_kmalloc_nolock(map, sizeof(*ctx), GFP_KERNEL,\n+\t\t\t\t     map-\u003enuma_node);\n+\tif (!ctx) {\n+\t\terr = -ENOMEM;\n+\t\tgoto destroy_worker;\n+\t}\n+\tmemset(ctx, 0, sizeof(*ctx));\n+\tctx-\u003eworker = worker;\n+\tctx-\u003emap = map;\n+\tctx-\u003evalue = value;\n+\trefcount_set(\u0026ctx-\u003erefcnt, 1);\n+\tkthread_init_work(\u0026ctx-\u003ework, bpf_thread_wq_work_fn);\n+\n+\tif (cgroup_id) {\n+#ifdef CONFIG_CGROUPS\n+\t\tcgrp = cgroup_get_from_id(cgroup_id);\n+\t\tif (IS_ERR(cgrp)) {\n+\t\t\terr = PTR_ERR(cgrp);\n+\t\t\tgoto kfree_ctx;\n+\t\t}\n+\t\tctx-\u003ecgrp = cgrp;\n+\n+\t\t/*\n+\t\t * kthread_run_worker() wakes the kthread, but it may not have\n+\t\t * executed cgroup_kthread_ready() yet, which clears\n+\t\t * no_cgroup_migration.\n+\t\t * Do a queue work and flush to wait the kthread run.\n+\t\t */\n+\t\trefcount_inc(\u0026ctx-\u003erefcnt);\n+\t\tif (!kthread_queue_work(ctx-\u003eworker, \u0026ctx-\u003ework)) {\n+\t\t\trefcount_dec(\u0026ctx-\u003erefcnt);\n+\t\t\terr = -EBUSY;\n+\t\t\tgoto cgroup_put;\n+\t\t}\n+\t\tkthread_flush_work(\u0026ctx-\u003ework);\n+\n+\t\tif (worker-\u003etask-\u003eno_cgroup_migration) {\n+\t\t\terr = -EAGAIN;\n+\t\t\tgoto cgroup_put;\n+\t\t}\n+\n+\t\terr = cgroup_kthread_attach(cgrp, worker-\u003etask);\n+\t\tif (err)\n+\t\t\tgoto cgroup_put;\n+#else\n+\t\terr = -EOPNOTSUPP;\n+\t\tgoto kfree_ctx;\n+#endif\n+\t}\n+\n+\told_ctx = cmpxchg(\u0026twk-\u003ectx, NULL, ctx);\n+\tif (old_ctx) {\n+\t\terr = -EBUSY;\n+\t\tgoto cgroup_put;\n+\t}\n+\n+\t/*\n+\t * Paired with the map destruction path.  Ensures that ctx is globally\n+\t * visible before we check map-\u003eusercnt.\n+\t * If usercnt has dropped to zero, the destruction path will either see\n+\t * the ctx (and cancel it) or we see usercnt == 0 here and cancel\n+\t * ourselves.\n+\t * Without this barrier, a CPU could reorder the load of usercnt before\n+\t * the cmpxchg store becomes visible, breaking the mutual exclusion\n+\t * guarantee.\n+\t */\n+\tsmp_mb();\n+\n+\tif (!atomic64_read(\u0026map-\u003eusercnt)) {\n+\t\tbpf_thread_wq_cancel_and_free(twq);\n+\t\treturn -EPERM;\n+\t}\n+\n+\treturn 0;\n+\n+cgroup_put:\n+#ifdef CONFIG_CGROUPS\n+\tif (cgrp)\n+\t\tcgroup_put(cgrp);\n+#endif\n+kfree_ctx:\n+\t/*\n+\t * Not use bpf_thread_wq_ctx_put because ctx has not yet entered\n+\t * the running state.\n+\t */\n+\tkfree(ctx);\n+destroy_worker:\n+\tkthread_destroy_worker(worker);\n+\treturn err;\n+}\n+\n+__bpf_kfunc int bpf_thread_wq_set_callback(struct bpf_thread_wq *twq,\n+\t\t\t\t\t   int (callback_fn)(void *map,\n+\t\t\t\t\t\t\t     int *key,\n+\t\t\t\t\t\t\t     void *value),\n+\t\t\t\t\t   unsigned int flags,\n+\t\t\t\t\t   struct bpf_prog_aux *aux)\n+{\n+\tstruct bpf_thread_wq_kern *twk = (struct bpf_thread_wq_kern *)twq;\n+\tstruct bpf_thread_wq_ctx *ctx;\n+\tstruct bpf_prog *prog;\n+\n+\tif (flags)\n+\t\treturn -EINVAL;\n+\n+\tctx = READ_ONCE(twk-\u003ectx);\n+\tif (!ctx)\n+\t\treturn -EINVAL;\n+\n+\tprog = bpf_prog_inc_not_zero(aux-\u003eprog);\n+\tif (IS_ERR(prog))\n+\t\treturn PTR_ERR(prog);\n+\n+\t/*\n+\t * Allow set_callback only once to prevent UAF: a concurrent\n+\t * set_callback could bpf_prog_put() the prog while the worker\n+\t * kthread is still executing its callback.\n+\t */\n+\tif (cmpxchg(\u0026ctx-\u003eprog, NULL, prog) != NULL) {\n+\t\tbpf_prog_put(prog);\n+\t\treturn -EBUSY;\n+\t}\n+\t/*\n+\t * Safe to set callback_fn after prog: bpf_thread_wq_start() and\n+\t * bpf_thread_wq_work_fn() both check callback_fn with READ_ONCE()\n+\t * and bail out if it is still NULL.\n+\t */\n+\tWRITE_ONCE(ctx-\u003ecallback_fn, (void *)callback_fn);\n+\n+\treturn 0;\n+}\n+\n+__bpf_kfunc int\n+bpf_thread_wq_start(struct bpf_thread_wq *twq, unsigned int flags)\n+{\n+\tstruct bpf_thread_wq_kern *twk = (struct bpf_thread_wq_kern *)twq;\n+\tstruct bpf_thread_wq_ctx *ctx;\n+\tint err;\n+\n+\tif (flags)\n+\t\treturn -EINVAL;\n+\n+\trcu_read_lock_trace();\n+\n+\terr = 0;\n+\n+\tctx = READ_ONCE(twk-\u003ectx);\n+\tif (!ctx || !READ_ONCE(ctx-\u003ecallback_fn)) {\n+\t\terr = -EINVAL;\n+\t\tgoto unlock;\n+\t}\n+\n+\tif (!refcount_inc_not_zero(\u0026ctx-\u003erefcnt))\n+\t\terr = -ENOENT;\n+\n+unlock:\n+\trcu_read_unlock_trace();\n+\tif (err)\n+\t\treturn err;\n+\n+\tif (!kthread_queue_work(ctx-\u003eworker, \u0026ctx-\u003ework)) {\n+\t\tbpf_thread_wq_ctx_put(ctx);\n+\t\treturn -EBUSY;\n+\t}\n+\n+\treturn 0;\n+}\n+\n+void bpf_thread_wq_cancel_and_free(void *val)\n+{\n+\tstruct bpf_thread_wq_kern *twk = val;\n+\tstruct bpf_thread_wq_ctx *ctx;\n+\n+\tctx = xchg(\u0026twk-\u003ectx, NULL);\n+\tif (!ctx)\n+\t\treturn;\n+\n+\tmight_sleep();\n+\n+\t/*\n+\t * Prevent future callbacks from running and wait for any\n+\t * in-progress execution to finish.\n+\t */\n+\tWRITE_ONCE(ctx-\u003ecallback_fn, NULL);\n+\t/*\n+\t * kthread_cancel_work_sync() returns true when it dequeues a pending\n+\t * work item from the work_list without executing it.  Each successful\n+\t * bpf_thread_wq_start() call increments ctx-\u003erefcnt and relies on the\n+\t * subsequent bpf_thread_wq_work_fn() execution to release that\n+\t * reference via bpf_thread_wq_ctx_put().  If the work was pending and\n+\t * got cancelled here, work_fn will never run for that queued instance,\n+\t * so we must drop the reference ourselves to avoid a permanent refcount\n+\t * leak.\n+\t *\n+\t * This covers two scenarios uniformly:\n+\t *  1. The work is purely pending (not currently executing) - e.g. a\n+\t *     normal bpf_thread_wq_start() call queued it but the worker\n+\t *     thread hasn't picked it up yet.\n+\t *  2. The work is currently in-flight AND was self-rescheduled from\n+\t *     within the callback - kthread_cancel_work_sync() dequeues the\n+\t *     re-queued pending node and then waits for the in-flight\n+\t *     execution to complete.\n+\t * In both cases the return value is true, indicating one orphaned\n+\t * reference that needs to be released here.\n+\t */\n+\tif (kthread_cancel_work_sync(\u0026ctx-\u003ework))\n+\t\tbpf_thread_wq_ctx_put(ctx);\n+\n+\t/*\n+\t * Drop our own reference.  If the work was still in-flight above,\n+\t * the refcount won't hit zero here - it will reach zero when the\n+\t * work path calls bpf_thread_wq_ctx_put() upon completion.  Either\n+\t * way, final cleanup (worker destruction, prog put, cgroup put,\n+\t * kfree) happens exclusively in the RCU callback to keep the\n+\t * teardown path single-threaded.\n+\t */\n+\tbpf_thread_wq_ctx_put(ctx);\n+}\n+\n __bpf_kfunc_end_defs();\n \n static void bpf_task_work_cancel_scheduled(struct irq_work *irq_work)\n@@ -4917,6 +5321,8 @@ BTF_ID_FLAGS(func, bpf_modify_return_test_tp)\n BTF_ID_FLAGS(func, bpf_wq_init)\n BTF_ID_FLAGS(func, bpf_wq_set_callback, KF_IMPLICIT_ARGS)\n BTF_ID_FLAGS(func, bpf_wq_start)\n+BTF_ID_FLAGS(func, bpf_thread_wq_set_callback, KF_IMPLICIT_ARGS)\n+BTF_ID_FLAGS(func, bpf_thread_wq_start)\n BTF_ID_FLAGS(func, bpf_preempt_disable)\n BTF_ID_FLAGS(func, bpf_preempt_enable)\n BTF_ID_FLAGS(func, bpf_iter_bits_new, KF_ITER_NEW)\n@@ -4978,6 +5384,15 @@ static const struct btf_kfunc_id_set common_kfunc_set = {\n \t.set   = \u0026common_btf_ids,\n };\n \n+BTF_KFUNCS_START(syscall_btf_ids)\n+BTF_ID_FLAGS(func, bpf_thread_wq_init, KF_SLEEPABLE)\n+BTF_KFUNCS_END(syscall_btf_ids)\n+\n+static const struct btf_kfunc_id_set syscall_kfunc_set = {\n+\t.owner = THIS_MODULE,\n+\t.set   = \u0026syscall_btf_ids,\n+};\n+\n static int __init kfunc_init(void)\n {\n \tint ret;\n@@ -5000,6 +5415,7 @@ static int __init kfunc_init(void)\n \tret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, \u0026generic_kfunc_set);\n \tret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL, \u0026generic_kfunc_set);\n \tret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_CGROUP_SKB, \u0026generic_kfunc_set);\n+\tret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL, \u0026syscall_kfunc_set);\n \tret = ret ?: register_btf_id_dtor_kfuncs(generic_dtors,\n \t\t\t\t\t\t  ARRAY_SIZE(generic_dtors),\n \t\t\t\t\t\t  THIS_MODULE);\n@@ -5037,4 +5453,6 @@ void bpf_map_free_internal_structs(struct bpf_map *map, void *val)\n \t\tbpf_obj_free_workqueue(map-\u003erecord, val);\n \tif (btf_record_has_field(map-\u003erecord, BPF_TASK_WORK))\n \t\tbpf_obj_free_task_work(map-\u003erecord, val);\n+\tif (btf_record_has_field(map-\u003erecord, BPF_THREAD_WQ))\n+\t\tbpf_obj_free_thread_wq(map-\u003erecord, val);\n }\ndiff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c\nindex 8d111da886553..dea14823bacd2 100644\n--- a/kernel/bpf/syscall.c\n+++ b/kernel/bpf/syscall.c\n@@ -688,6 +688,7 @@ void btf_record_free(struct btf_record *rec)\n \t\tcase BPF_REFCOUNT:\n \t\tcase BPF_WORKQUEUE:\n \t\tcase BPF_TASK_WORK:\n+\t\tcase BPF_THREAD_WQ:\n \t\t\t/* Nothing to release */\n \t\t\tbreak;\n \t\tdefault:\n@@ -742,6 +743,7 @@ struct btf_record *btf_record_dup(const struct btf_record *rec)\n \t\tcase BPF_REFCOUNT:\n \t\tcase BPF_WORKQUEUE:\n \t\tcase BPF_TASK_WORK:\n+\t\tcase BPF_THREAD_WQ:\n \t\t\t/* Nothing to acquire */\n \t\t\tbreak;\n \t\tdefault:\n@@ -807,6 +809,13 @@ void bpf_obj_free_task_work(const struct btf_record *rec, void *obj)\n \tbpf_task_work_cancel_and_free(obj + rec-\u003etask_work_off);\n }\n \n+void bpf_obj_free_thread_wq(const struct btf_record *rec, void *obj)\n+{\n+\tif (WARN_ON_ONCE(!btf_record_has_field(rec, BPF_THREAD_WQ)))\n+\t\treturn;\n+\tbpf_thread_wq_cancel_and_free(obj + rec-\u003ethread_wq_off);\n+}\n+\n void bpf_obj_cancel_fields(struct bpf_map *map, void *obj)\n {\n \tbpf_map_free_internal_structs(map, obj);\n@@ -839,6 +848,9 @@ void bpf_obj_free_fields(const struct btf_record *rec, void *obj)\n \t\tcase BPF_TASK_WORK:\n \t\t\tbpf_task_work_cancel_and_free(field_ptr);\n \t\t\tbreak;\n+\t\tcase BPF_THREAD_WQ:\n+\t\t\tbpf_thread_wq_cancel_and_free(field_ptr);\n+\t\t\tbreak;\n \t\tcase BPF_KPTR_UNREF:\n \t\t\tWRITE_ONCE(*(u64 *)field_ptr, 0);\n \t\t\tbreak;\n@@ -1265,7 +1277,7 @@ static int map_check_btf(struct bpf_map *map, struct bpf_token *token,\n \tmap-\u003erecord = btf_parse_fields(btf, value_type,\n \t\t\t\t       BPF_SPIN_LOCK | BPF_RES_SPIN_LOCK | BPF_TIMER | BPF_KPTR | BPF_LIST_HEAD |\n \t\t\t\t       BPF_RB_ROOT | BPF_REFCOUNT | BPF_WORKQUEUE | BPF_UPTR |\n-\t\t\t\t       BPF_TASK_WORK,\n+\t\t\t\t       BPF_TASK_WORK | BPF_THREAD_WQ,\n \t\t\t\t       map-\u003evalue_size);\n \tif (!IS_ERR_OR_NULL(map-\u003erecord)) {\n \t\tint i;\n@@ -1299,6 +1311,7 @@ static int map_check_btf(struct bpf_map *map, struct bpf_token *token,\n \t\t\tcase BPF_TIMER:\n \t\t\tcase BPF_WORKQUEUE:\n \t\t\tcase BPF_TASK_WORK:\n+\t\t\tcase BPF_THREAD_WQ:\n \t\t\t\tif (map-\u003emap_type != BPF_MAP_TYPE_HASH \u0026\u0026\n \t\t\t\t    map-\u003emap_type != BPF_MAP_TYPE_RHASH \u0026\u0026\n \t\t\t\t    map-\u003emap_type != BPF_MAP_TYPE_LRU_HASH \u0026\u0026\ndiff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c\nindex e6233c0081d10..44e79fe480b6b 100644\n--- a/kernel/bpf/verifier.c\n+++ b/kernel/bpf/verifier.c\n@@ -477,6 +477,7 @@ static bool is_async_callback_calling_kfunc(u32 btf_id);\n static bool is_callback_calling_kfunc(u32 btf_id);\n \n static bool is_bpf_wq_set_callback_kfunc(u32 btf_id);\n+static bool is_bpf_thread_wq_set_callback_kfunc(u32 btf_id);\n static bool is_task_work_add_kfunc(u32 func_id);\n \n static bool is_sync_callback_calling_function(enum bpf_func_id func_id)\n@@ -516,9 +517,11 @@ static bool is_async_cb_sleepable(struct bpf_verifier_env *env, struct bpf_insn\n \tif (bpf_helper_call(insn) \u0026\u0026 insn-\u003eimm == BPF_FUNC_timer_set_callback)\n \t\treturn false;\n \n-\t/* bpf_wq and bpf_task_work callbacks are always sleepable. */\n+\t/* bpf_wq, bpf_thread_wq and bpf_task_work callbacks are always sleepable. */\n \tif (bpf_pseudo_kfunc_call(insn) \u0026\u0026 insn-\u003eoff == 0 \u0026\u0026\n-\t    (is_bpf_wq_set_callback_kfunc(insn-\u003eimm) || is_task_work_add_kfunc(insn-\u003eimm)))\n+\t    (is_bpf_wq_set_callback_kfunc(insn-\u003eimm) ||\n+\t     is_bpf_thread_wq_set_callback_kfunc(insn-\u003eimm) ||\n+\t     is_task_work_add_kfunc(insn-\u003eimm)))\n \t\treturn true;\n \n \tverifier_bug(env, \"unhandled async callback in is_async_cb_sleepable\");\n@@ -1871,7 +1874,10 @@ static void refine_map_lookup_value(struct bpf_reg_state *reg)\n \t\t * as UID of the inner map.\n \t\t */\n \t\tif (btf_record_has_field(map-\u003einner_map_meta-\u003erecord,\n-\t\t\t\t\t BPF_TIMER | BPF_WORKQUEUE | BPF_TASK_WORK))\n+\t\t\t\t\t BPF_TIMER |\n+\t\t\t\t\t BPF_WORKQUEUE |\n+\t\t\t\t\t BPF_TASK_WORK |\n+\t\t\t\t\t BPF_THREAD_WQ))\n \t\t\treg-\u003emap_uid = reg-\u003eid;\n \t} else if (map-\u003emap_type == BPF_MAP_TYPE_XSKMAP) {\n \t\treg-\u003etype = PTR_TO_XDP_SOCK | maybe_null;\n@@ -7243,6 +7249,9 @@ static int check_map_field_pointer(struct bpf_verifier_env *env, struct bpf_reg_\n \tcase BPF_WORKQUEUE:\n \t\tfield_off = map-\u003erecord-\u003ewq_off;\n \t\tbreak;\n+\tcase BPF_THREAD_WQ:\n+\t\tfield_off = map-\u003erecord-\u003ethread_wq_off;\n+\t\tbreak;\n \tdefault:\n \t\tverifier_bug(env, \"unsupported BTF field type: %s\\n\", struct_name);\n \t\treturn -EINVAL;\n@@ -10940,6 +10949,7 @@ enum {\n \tKF_ARG_WORKQUEUE_ID,\n \tKF_ARG_RES_SPIN_LOCK_ID,\n \tKF_ARG_TASK_WORK_ID,\n+\tKF_ARG_THREAD_WQ_ID,\n \tKF_ARG_PROG_AUX_ID,\n \tKF_ARG_TIMER_ID\n };\n@@ -10953,6 +10963,7 @@ BTF_ID(struct, bpf_rb_node)\n BTF_ID(struct, bpf_wq)\n BTF_ID(struct, bpf_res_spin_lock)\n BTF_ID(struct, bpf_task_work)\n+BTF_ID(struct, bpf_thread_wq)\n BTF_ID(struct, bpf_prog_aux)\n BTF_ID(struct, bpf_timer)\n \n@@ -11013,6 +11024,11 @@ static bool is_kfunc_arg_task_work(const struct btf *btf, const struct btf_param\n \treturn __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_TASK_WORK_ID);\n }\n \n+static bool is_kfunc_arg_thread_wq(const struct btf *btf, const struct btf_param *arg)\n+{\n+\treturn __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_THREAD_WQ_ID);\n+}\n+\n static bool is_kfunc_arg_res_spin_lock(const struct btf *btf, const struct btf_param *arg)\n {\n \treturn __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_RES_SPIN_LOCK_ID);\n@@ -11132,6 +11148,7 @@ enum kfunc_ptr_arg_type {\n \tKF_ARG_PTR_TO_IRQ_FLAG,\n \tKF_ARG_PTR_TO_RES_SPIN_LOCK,\n \tKF_ARG_PTR_TO_TASK_WORK,\n+\tKF_ARG_PTR_TO_THREAD_WQ,\n };\n \n enum special_kfunc_type {\n@@ -11178,6 +11195,7 @@ enum special_kfunc_type {\n \tKF_bpf_percpu_obj_drop,\n \tKF_bpf_throw,\n \tKF_bpf_wq_set_callback,\n+\tKF_bpf_thread_wq_set_callback,\n \tKF_bpf_preempt_disable,\n \tKF_bpf_preempt_enable,\n \tKF_bpf_iter_css_task_new,\n@@ -11258,6 +11276,7 @@ BTF_ID(func, bpf_percpu_obj_drop_impl)\n BTF_ID(func, bpf_percpu_obj_drop)\n BTF_ID(func, bpf_throw)\n BTF_ID(func, bpf_wq_set_callback)\n+BTF_ID(func, bpf_thread_wq_set_callback)\n BTF_ID(func, bpf_preempt_disable)\n BTF_ID(func, bpf_preempt_enable)\n #ifdef CONFIG_CGROUPS\n@@ -11460,6 +11479,8 @@ get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta,\n \t\targ_type = KF_ARG_PTR_TO_TIMER;\n \telse if (is_kfunc_arg_task_work(meta-\u003ebtf, \u0026args[arg]))\n \t\targ_type = KF_ARG_PTR_TO_TASK_WORK;\n+\telse if (is_kfunc_arg_thread_wq(meta-\u003ebtf, \u0026args[arg]))\n+\t\targ_type = KF_ARG_PTR_TO_THREAD_WQ;\n \telse if (is_kfunc_arg_irq_flag(meta-\u003ebtf, \u0026args[arg]))\n \t\targ_type = KF_ARG_PTR_TO_IRQ_FLAG;\n \telse if (is_kfunc_arg_res_spin_lock(meta-\u003ebtf, \u0026args[arg]))\n@@ -11833,6 +11854,7 @@ static bool is_sync_callback_calling_kfunc(u32 btf_id)\n static bool is_async_callback_calling_kfunc(u32 btf_id)\n {\n \treturn is_bpf_wq_set_callback_kfunc(btf_id) ||\n+\t       is_bpf_thread_wq_set_callback_kfunc(btf_id) ||\n \t       is_task_work_add_kfunc(btf_id);\n }\n \n@@ -11847,6 +11869,11 @@ static bool is_bpf_wq_set_callback_kfunc(u32 btf_id)\n \treturn btf_id == special_kfunc_list[KF_bpf_wq_set_callback];\n }\n \n+static bool is_bpf_thread_wq_set_callback_kfunc(u32 btf_id)\n+{\n+\treturn btf_id == special_kfunc_list[KF_bpf_thread_wq_set_callback];\n+}\n+\n static bool is_callback_calling_kfunc(u32 btf_id)\n {\n \treturn is_sync_callback_calling_kfunc(btf_id) ||\n@@ -12192,6 +12219,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me\n \t\tcase KF_ARG_PTR_TO_WORKQUEUE:\n \t\tcase KF_ARG_PTR_TO_TIMER:\n \t\tcase KF_ARG_PTR_TO_TASK_WORK:\n+\t\tcase KF_ARG_PTR_TO_THREAD_WQ:\n \t\tcase KF_ARG_PTR_TO_IRQ_FLAG:\n \t\tcase KF_ARG_PTR_TO_RES_SPIN_LOCK:\n \t\t\tbreak;\n@@ -13139,6 +13167,16 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,\n \t\t}\n \t}\n \n+\tif (is_bpf_thread_wq_set_callback_kfunc(meta.func_id)) {\n+\t\terr = push_callback_call(env, insn, insn_idx, meta.subprogno,\n+\t\t\t\t\t set_timer_callback_state);\n+\t\tif (err) {\n+\t\t\tverbose(env, \"kfunc %s#%d failed callback verification\\n\",\n+\t\t\t\tfunc_name, meta.func_id);\n+\t\t\treturn err;\n+\t\t}\n+\t}\n+\n \tif (is_task_work_add_kfunc(meta.func_id)) {\n \t\terr = push_callback_call(env, insn, insn_idx, meta.subprogno,\n \t\t\t\t\t set_task_work_schedule_callback_state);\ndiff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c\nindex 38f8d9df8fbc4..164e069c75762 100644\n--- a/kernel/cgroup/cgroup.c\n+++ b/kernel/cgroup/cgroup.c\n@@ -3042,6 +3042,19 @@ int cgroup_attach_task(struct cgroup *dst_cgrp, struct task_struct *leader,\n \treturn ret;\n }\n \n+int cgroup_kthread_attach(struct cgroup *cgrp, struct task_struct *task)\n+{\n+\tint ret;\n+\n+\tcgroup_lock();\n+\tcgroup_attach_lock(CGRP_ATTACH_LOCK_GLOBAL, NULL);\n+\tret = cgroup_attach_task(cgrp, task, false);\n+\tcgroup_attach_unlock(CGRP_ATTACH_LOCK_GLOBAL, NULL);\n+\tcgroup_unlock();\n+\n+\treturn ret;\n+}\n+\n struct task_struct *cgroup_procs_write_start(char *buf, bool threadgroup,\n \t\t\t\t\t     enum cgroup_attach_lock_mode *lock_mode)\n {\ndiff --git a/mm/bpf_memcontrol.c b/mm/bpf_memcontrol.c\nindex 716df49d76477..3f7a5c97e1355 100644\n--- a/mm/bpf_memcontrol.c\n+++ b/mm/bpf_memcontrol.c\n@@ -6,6 +6,7 @@\n  */\n \n #include \u003clinux/memcontrol.h\u003e\n+#include \u003clinux/swap.h\u003e\n #include \u003clinux/bpf.h\u003e\n \n __bpf_kfunc_start_defs();\n@@ -159,6 +160,61 @@ __bpf_kfunc void bpf_mem_cgroup_flush_stats(struct mem_cgroup *memcg)\n \tmem_cgroup_flush_stats(memcg);\n }\n \n+/**\n+ * bpf_try_to_free_mem_cgroup_pages - attempt to reclaim pages from\n+ *                                    a memory cgroup\n+ * @memcg:           the target memory cgroup to reclaim from\n+ * @nr_pages:        the number of pages to reclaim\n+ * @gfp_mask:        GFP flags controlling the reclaim behavior\n+ * @reclaim_options: bitmask of MEMCG_RECLAIM_* flags to tune\n+ *                   reclaim strategy\n+ * @swappiness:      swappiness override value, or a sentinel to use\n+ *                   the default\n+ *\n+ * BPF-facing wrapper around try_to_free_mem_cgroup_pages() that\n+ * validates and translates the @swappiness argument before\n+ * delegating to the core reclaim path.\n+ *\n+ * The @swappiness parameter follows these semantics:\n+ *   - Values in [MIN_SWAPPINESS, SWAPPINESS_ANON_ONLY] are passed\n+ *     through as an explicit swappiness override.\n+ *   - Values below MIN_SWAPPINESS are treated as \"use the system\n+ *     default\"; the override pointer is set to NULL and the cgroup's\n+ *     own swappiness setting takes effect.\n+ *   - Values above SWAPPINESS_ANON_ONLY are rejected as invalid.\n+ *   - If @reclaim_options does not include MEMCG_RECLAIM_PROACTIVE,\n+ *     the @swappiness override is ignored entirely by the core\n+ *     reclaim path and the system default is used regardless.\n+ *\n+ * Swap usage during reclaim is gated on @reclaim_options: swap is\n+ * considered only when MEMCG_RECLAIM_MAY_SWAP is set.  Without this\n+ * flag, reclaim is restricted to file-backed pages regardless of the\n+ * @swappiness value or the cgroup's swappiness setting.\n+ *\n+ * Return:\n+ *   The number of pages actually reclaimed on success, or 0\n+ *   if @swappiness exceeds SWAPPINESS_ANON_ONLY.\n+ */\n+__bpf_kfunc unsigned long\n+bpf_try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg,\n+\t\t\t\t unsigned long nr_pages,\n+\t\t\t\t gfp_t gfp_mask,\n+\t\t\t\t unsigned int reclaim_options,\n+\t\t\t\t int swappiness)\n+{\n+\tint *swapiness_ptr;\n+\n+\tif (swappiness \u003e SWAPPINESS_ANON_ONLY)\n+\t\treturn 0;\n+\telse if (swappiness \u003c MIN_SWAPPINESS)\n+\t\tswapiness_ptr = NULL;\n+\telse\n+\t\tswapiness_ptr = \u0026swappiness;\n+\n+\treturn try_to_free_mem_cgroup_pages(memcg, nr_pages, gfp_mask,\n+\t\t\t\t\t    reclaim_options, swapiness_ptr);\n+}\n+\n __bpf_kfunc_end_defs();\n \n BTF_KFUNCS_START(bpf_memcontrol_kfuncs)\n@@ -172,6 +228,8 @@ BTF_ID_FLAGS(func, bpf_mem_cgroup_usage)\n BTF_ID_FLAGS(func, bpf_mem_cgroup_page_state)\n BTF_ID_FLAGS(func, bpf_mem_cgroup_flush_stats, KF_SLEEPABLE)\n \n+BTF_ID_FLAGS(func, bpf_try_to_free_mem_cgroup_pages, KF_SLEEPABLE)\n+\n BTF_KFUNCS_END(bpf_memcontrol_kfuncs)\n \n static const struct btf_kfunc_id_set bpf_memcontrol_kfunc_set = {\ndiff --git a/tools/testing/selftests/bpf/bpf_experimental.h b/tools/testing/selftests/bpf/bpf_experimental.h\nindex ff37ae5a113dc..ea905ea5603c2 100644\n--- a/tools/testing/selftests/bpf/bpf_experimental.h\n+++ b/tools/testing/selftests/bpf/bpf_experimental.h\n@@ -351,6 +351,13 @@ extern void bpf_iter_css_destroy(struct bpf_iter_css *it) __weak __ksym;\n extern int bpf_wq_init(struct bpf_wq *wq, void *p__map, unsigned int flags) __weak __ksym;\n extern int bpf_wq_start(struct bpf_wq *wq, unsigned int flags) __weak __ksym;\n \n+struct bpf_thread_wq;\n+extern int bpf_thread_wq_init(struct bpf_thread_wq *twq, void *p__map,\n+\t\t\t      __u64 cgroup_id,\n+\t\t\t      unsigned int flags) __weak __ksym;\n+extern int bpf_thread_wq_start(struct bpf_thread_wq *twq,\n+\t\t\t       unsigned int flags) __weak __ksym;\n+\n struct bpf_iter_kmem_cache;\n extern int bpf_iter_kmem_cache_new(struct bpf_iter_kmem_cache *it) __weak __ksym;\n extern struct kmem_cache *bpf_iter_kmem_cache_next(struct bpf_iter_kmem_cache *it) __weak __ksym;\ndiff --git a/tools/testing/selftests/bpf/prog_tests/memcg_async_reclaim.c b/tools/testing/selftests/bpf/prog_tests/memcg_async_reclaim.c\nnew file mode 100644\nindex 0000000000000..40a5fe62cb384\n--- /dev/null\n+++ b/tools/testing/selftests/bpf/prog_tests/memcg_async_reclaim.c\n@@ -0,0 +1,479 @@\n+// SPDX-License-Identifier: GPL-2.0\n+/*\n+ * Memory controller eBPF async reclaim test\n+ */\n+\n+#include \u003ctest_progs.h\u003e\n+#include \u003csys/mman.h\u003e\n+#include \u003csys/stat.h\u003e\n+#include \u003csys/time.h\u003e\n+#include \u003csys/vfs.h\u003e\n+#include \u003csys/wait.h\u003e\n+#include \u003cfcntl.h\u003e\n+#include \u003clinux/magic.h\u003e\n+#include \u003cunistd.h\u003e\n+#include \u003cstdio.h\u003e\n+#include \u003cstdlib.h\u003e\n+#include \u003cstring.h\u003e\n+\n+#include \"cgroup_helpers.h\"\n+\n+struct bpf_args_s {\n+\tu64 high_cgroup_id;\n+\tu64 low_cgroup_id;\n+\tu64 event_delta_threshold;\n+\tu64 check_ns;\n+};\n+\n+#include \"memcg_async_reclaim.skel.h\"\n+\n+#define FILE_SIZE (32 * 1024 * 1024ul)\n+#define BUFFER_SIZE (4096)\n+#define CG_LIMIT (32 * 1024 * 1024ul)\n+#define READ_TIMES 16\n+\n+#define CG_DIR \"/memcg_async_reclaim\"\n+#define CG_HIGH_DIR CG_DIR \"/high\"\n+#define CG_LOW_DIR CG_DIR \"/low\"\n+\n+#define CHECK_PERIOD_NS (2 * 1000 * 1000ull)\n+#define EVENT_DELTA_THRESHOLD 1\n+\n+/*\n+ * Test files must reside on a filesystem that supports page reclaim without\n+ * swap (e.g. ext4). If /tmp is on tmpfs, the file pages are shmem-backed\n+ * and can only be reclaimed through swap. But the test disables swap\n+ * (memory.swap.max=0), making reclaim impossible and causing OOM.\n+ *\n+ * Pick a directory on a non-tmpfs filesystem: try $TMPDIR first, then /tmp,\n+ * and fall back to the current directory if the chosen path is on tmpfs.\n+ */\n+static int get_test_dir(char *buf, size_t size)\n+{\n+\tstatic const char * const candidates[] = { \"/tmp\", \".\" };\n+\tconst char *tmpdir = getenv(\"TMPDIR\");\n+\tstruct statfs sfs;\n+\tsize_t i;\n+\n+\tif (tmpdir \u0026\u0026 tmpdir[0] \u0026\u0026 statfs(tmpdir, \u0026sfs) == 0 \u0026\u0026\n+\t    sfs.f_type != TMPFS_MAGIC) {\n+\t\tsnprintf(buf, size, \"%s\", tmpdir);\n+\t\treturn 0;\n+\t}\n+\n+\tfor (i = 0; i \u003c ARRAY_SIZE(candidates); i++) {\n+\t\tif (statfs(candidates[i], \u0026sfs) == 0 \u0026\u0026\n+\t\t    sfs.f_type != TMPFS_MAGIC) {\n+\t\t\tsnprintf(buf, size, \"%s\", candidates[i]);\n+\t\t\treturn 0;\n+\t\t}\n+\t}\n+\n+\treturn -1;\n+}\n+\n+static int setup_high_low_cgroups(u64 *high_cgroup_id, u64 *low_cgroup_id)\n+{\n+\tint ret;\n+\tchar limit_buf[20];\n+\n+\tret = setup_cgroup_environment();\n+\tif (!ASSERT_OK(ret, \"setup_cgroup_environment\"))\n+\t\tgoto cleanup;\n+\n+\tret = create_and_get_cgroup(CG_DIR);\n+\tif (!ASSERT_GE(ret, 0, \"create_and_get_cgroup \" CG_DIR))\n+\t\tgoto cleanup;\n+\tclose(ret);\n+\n+\tret = enable_controllers(CG_DIR, \"memory\");\n+\tif (!ASSERT_OK(ret, \"enable_controllers\"))\n+\t\tgoto cleanup;\n+\n+\tsnprintf(limit_buf, sizeof(limit_buf), \"%lu\", CG_LIMIT);\n+\tret = write_cgroup_file(CG_DIR, \"memory.max\", limit_buf);\n+\tif (!ASSERT_OK(ret, \"write_cgroup_file memory.max\"))\n+\t\tgoto cleanup;\n+\n+\tret = write_cgroup_file(CG_DIR, \"memory.swap.max\", \"0\");\n+\tif (!ASSERT_OK(ret, \"write_cgroup_file memory.swap.max\"))\n+\t\tgoto cleanup;\n+\n+\tret = create_and_get_cgroup(CG_HIGH_DIR);\n+\tif (!ASSERT_GE(ret, 0, \"create_and_get_cgroup \" CG_HIGH_DIR))\n+\t\tgoto cleanup;\n+\tclose(ret);\n+\n+\t*high_cgroup_id = get_cgroup_id(CG_HIGH_DIR);\n+\tif (!ASSERT_GT(*high_cgroup_id, 0, \"get_cgroup_id\"))\n+\t\tgoto cleanup;\n+\n+\tret = create_and_get_cgroup(CG_LOW_DIR);\n+\tif (!ASSERT_GE(ret, 0, \"create_and_get_cgroup \" CG_LOW_DIR))\n+\t\tgoto cleanup;\n+\tclose(ret);\n+\n+\t*low_cgroup_id = get_cgroup_id(CG_LOW_DIR);\n+\tif (!ASSERT_GT(*high_cgroup_id, 0, \"get_cgroup_id\"))\n+\t\tgoto cleanup;\n+\n+\treturn 0;\n+\n+cleanup:\n+\tcleanup_cgroup_environment();\n+\treturn -1;\n+}\n+\n+static int write_file(const char *filename)\n+{\n+\tint ret = -1;\n+\tsize_t written = 0;\n+\tchar *buffer;\n+\tFILE *fp;\n+\n+\tfp = fopen(filename, \"wb\");\n+\tif (!fp)\n+\t\tgoto out;\n+\n+\tbuffer = malloc(BUFFER_SIZE);\n+\tif (!buffer)\n+\t\tgoto cleanup_fp;\n+\n+\tmemset(buffer, 'A', BUFFER_SIZE);\n+\n+\twhile (written \u003c FILE_SIZE) {\n+\t\tsize_t to_write = FILE_SIZE - written \u003c BUFFER_SIZE ?\n+\t\t\t\t  FILE_SIZE - written : BUFFER_SIZE;\n+\n+\t\tif (fwrite(buffer, 1, to_write, fp) != to_write)\n+\t\t\tgoto cleanup;\n+\t\twritten += to_write;\n+\t}\n+\n+\tret = 0;\n+cleanup:\n+\tfree(buffer);\n+cleanup_fp:\n+\tfclose(fp);\n+out:\n+\treturn ret;\n+}\n+\n+static int read_file(const char *filename, int iterations)\n+{\n+\tint ret = -1;\n+\tlong page_size = sysconf(_SC_PAGESIZE);\n+\tchar *map;\n+\tsize_t i;\n+\tint fd;\n+\tstruct stat sb;\n+\n+\tfd = open(filename, O_RDONLY);\n+\tif (fd == -1)\n+\t\tgoto out;\n+\n+\tif (fstat(fd, \u0026sb) == -1)\n+\t\tgoto cleanup_fd;\n+\n+\tif (sb.st_size != FILE_SIZE) {\n+\t\tfprintf(stderr, \"File size mismatch: expected %lu, got %lu\\n\",\n+\t\t\t(unsigned long)FILE_SIZE, (unsigned long)sb.st_size);\n+\t\tgoto cleanup_fd;\n+\t}\n+\n+\tmap = mmap(NULL, FILE_SIZE, PROT_READ, MAP_PRIVATE, fd, 0);\n+\tif (map == MAP_FAILED)\n+\t\tgoto cleanup_fd;\n+\n+\tfor (int iter = 0; iter \u003c iterations; iter++) {\n+\t\tfor (i = 0; i \u003c FILE_SIZE; i += page_size) {\n+\t\t\t/* access a byte to trigger page fault */\n+\t\t\tasm volatile(\"\" :: \"r\"(map[i]) : \"memory\");\n+\t\t}\n+\t}\n+\n+\tif (munmap(map, FILE_SIZE) == -1)\n+\t\tgoto cleanup_fd;\n+\n+\tret = 0;\n+\n+cleanup_fd:\n+\tclose(fd);\n+out:\n+\treturn ret;\n+}\n+\n+static int real_test_child_work(const char *cgroup_path, char *data_filename,\n+\t\t\t\tchar *time_filename, int read_times)\n+{\n+\tstruct timeval start, end;\n+\tdouble elapsed;\n+\tFILE *fp;\n+\n+\tif (!ASSERT_OK(join_parent_cgroup(cgroup_path), \"join_parent_cgroup\"))\n+\t\treturn -1;\n+\n+\tgettimeofday(\u0026start, NULL);\n+\n+\tif (!ASSERT_OK(write_file(data_filename), \"write_file\"))\n+\t\treturn -1;\n+\n+\tif (!ASSERT_OK(read_file(data_filename, read_times), \"read_file\"))\n+\t\treturn -1;\n+\n+\tgettimeofday(\u0026end, NULL);\n+\n+\tif (!time_filename)\n+\t\treturn 0;\n+\n+\telapsed = (end.tv_sec - start.tv_sec) +\n+\t\t  (end.tv_usec - start.tv_usec) / 1000000.0;\n+\tprintf(\"%.6f\\n\", elapsed);\n+\n+\tfp = fopen(time_filename, \"w\");\n+\tif (!ASSERT_OK_PTR(fp, \"fopen\"))\n+\t\treturn -1;\n+\tfprintf(fp, \"%.6f\", elapsed);\n+\tfclose(fp);\n+\n+\treturn 0;\n+}\n+\n+static int get_time(char *time_filename, double *time)\n+{\n+\tint ret = -1;\n+\tFILE *fp;\n+\tchar buf[64];\n+\n+\tfp = fopen(time_filename, \"r\");\n+\tif (!ASSERT_OK_PTR(fp, \"fopen\"))\n+\t\tgoto out;\n+\n+\tif (!ASSERT_OK_PTR(fgets(buf, sizeof(buf), fp), \"fgets\"))\n+\t\tgoto cleanup;\n+\n+\tif (sscanf(buf, \"%lf\", time) != 1) {\n+\t\tPRINT_FAIL(\"sscanf %s\", buf);\n+\t\tgoto cleanup;\n+\t}\n+\n+\tret = 0;\n+cleanup:\n+\tfclose(fp);\n+out:\n+\treturn ret;\n+}\n+\n+static int\n+run_high_low_workload(double *high_elapsed, double *low_elapsed, int read_times)\n+{\n+\tchar test_dir[PATH_MAX], high_data_file[PATH_MAX], low_data_file[PATH_MAX];\n+\tchar high_time_file[PATH_MAX], low_time_file[PATH_MAX];\n+\tint ret, fd;\n+\tpid_t high_pid, low_pid;\n+\tint status;\n+\n+\tret = get_test_dir(test_dir, sizeof(test_dir));\n+\tif (!ASSERT_OK(ret, \"get_test_dir: no non-tmpfs directory found\"))\n+\t\treturn -1;\n+\n+\tfd = snprintf(high_data_file, sizeof(high_data_file),\n+\t\t      \"%s/memcg_async_high_data_XXXXXX\", test_dir);\n+\tif (!ASSERT_LT(fd, sizeof(high_data_file), \"high_data_file path\"))\n+\t\treturn -1;\n+\n+\tfd = snprintf(low_data_file, sizeof(low_data_file),\n+\t\t      \"%s/memcg_async_low_data_XXXXXX\", test_dir);\n+\tif (!ASSERT_LT(fd, sizeof(low_data_file), \"low_data_file path\"))\n+\t\treturn -1;\n+\n+\tfd = snprintf(high_time_file, sizeof(high_time_file),\n+\t\t      \"%s/memcg_async_high_time_XXXXXX\", test_dir);\n+\tif (!ASSERT_LT(fd, sizeof(high_time_file), \"high_time_file path\"))\n+\t\treturn -1;\n+\n+\tfd = snprintf(low_time_file, sizeof(low_time_file),\n+\t\t      \"%s/memcg_async_low_time_XXXXXX\", test_dir);\n+\tif (!ASSERT_LT(fd, sizeof(low_time_file), \"low_time_file path\"))\n+\t\treturn -1;\n+\n+\tfd = mkstemp(high_data_file);\n+\tif (!ASSERT_GE(fd, 0, \"mkstemp\"))\n+\t\treturn -1;\n+\tclose(fd);\n+\n+\tfd = mkstemp(low_data_file);\n+\tif (!ASSERT_GE(fd, 0, \"mkstemp\"))\n+\t\tgoto cleanup_high_data;\n+\tclose(fd);\n+\n+\tfd = mkstemp(high_time_file);\n+\tif (!ASSERT_GE(fd, 0, \"mkstemp\"))\n+\t\tgoto cleanup_low_data;\n+\tclose(fd);\n+\n+\tfd = mkstemp(low_time_file);\n+\tif (!ASSERT_GE(fd, 0, \"mkstemp\"))\n+\t\tgoto cleanup_high_time;\n+\tclose(fd);\n+\n+\tlow_pid = fork();\n+\tif (!ASSERT_GE(low_pid, 0, \"fork low\"))\n+\t\tgoto cleanup_low_time;\n+\tif (low_pid == 0)\n+\t\texit(real_test_child_work(CG_LOW_DIR, low_data_file,\n+\t\t\t\t\t  low_time_file, read_times));\n+\n+\thigh_pid = fork();\n+\tif (!ASSERT_GE(high_pid, 0, \"fork high\")) {\n+\t\t(void)waitpid(low_pid, NULL, 0);\n+\t\tgoto cleanup_low_time;\n+\t}\n+\tif (high_pid == 0)\n+\t\texit(real_test_child_work(CG_HIGH_DIR, high_data_file,\n+\t\t\t\t\t  high_time_file, read_times));\n+\n+\tret = waitpid(low_pid, \u0026status, 0);\n+\tif (!ASSERT_GT(ret, 0, \"low waitpid\"))\n+\t\tgoto cleanup_low_time;\n+\tif (!ASSERT_TRUE(WIFEXITED(status), \"low exited\"))\n+\t\tgoto cleanup_low_time;\n+\tif (!ASSERT_EQ(WEXITSTATUS(status), 0, \"low exit status\"))\n+\t\tgoto cleanup_low_time;\n+\n+\tret = waitpid(high_pid, \u0026status, 0);\n+\tif (!ASSERT_GT(ret, 0, \"high waitpid\"))\n+\t\tgoto cleanup_low_time;\n+\tif (!ASSERT_TRUE(WIFEXITED(status), \"high exited\"))\n+\t\tgoto cleanup_low_time;\n+\tif (!ASSERT_EQ(WEXITSTATUS(status), 0, \"high exit status\"))\n+\t\tgoto cleanup_low_time;\n+\n+\tif (get_time(high_time_file, high_elapsed))\n+\t\tgoto cleanup_low_time;\n+\tif (get_time(low_time_file, low_elapsed))\n+\t\tgoto cleanup_low_time;\n+\n+\tret = 0;\n+\n+cleanup_low_time:\n+\tunlink(low_time_file);\n+cleanup_high_time:\n+\tunlink(high_time_file);\n+cleanup_low_data:\n+\tunlink(low_data_file);\n+cleanup_high_data:\n+\tunlink(high_data_file);\n+\treturn ret;\n+}\n+\n+static int\n+setup_bpf(u64 high_cgroup_id, u64 low_cgroup_id,\n+\t  struct memcg_async_reclaim **skel_ptr, bool use_thread_wq)\n+{\n+\tstruct memcg_async_reclaim *skel;\n+\tstruct bpf_args_s bpf_args = {\n+\t\t.high_cgroup_id = high_cgroup_id,\n+\t\t.low_cgroup_id = low_cgroup_id,\n+\t\t.event_delta_threshold = EVENT_DELTA_THRESHOLD,\n+\t\t.check_ns = CHECK_PERIOD_NS,\n+\t};\n+\tLIBBPF_OPTS(bpf_test_run_opts, run_opts,\n+\t\t.ctx_in = \u0026bpf_args,\n+\t\t.ctx_size_in = sizeof(bpf_args));\n+\tint prog_init_fd, err;\n+\n+\tskel = memcg_async_reclaim__open_and_load();\n+\tif (!ASSERT_OK_PTR(skel, \"memcg_async_reclaim__open_and_load\"))\n+\t\treturn -1;\n+\n+\tif (use_thread_wq)\n+\t\tprog_init_fd = bpf_program__fd(skel-\u003eprogs.thread_wq_prog_init);\n+\telse\n+\t\tprog_init_fd = bpf_program__fd(skel-\u003eprogs.wq_prog_init);\n+\tif (!ASSERT_GE(prog_init_fd, 0, \"bpf_program__fd\"))\n+\t\tgoto error_out;\n+\n+\terr = bpf_prog_test_run_opts(prog_init_fd, \u0026run_opts);\n+\tif (!ASSERT_OK(err, \"bpf_prog_test_run_opts\"))\n+\t\tgoto error_out;\n+\tif (!ASSERT_EQ(run_opts.retval, 0, \"prog_init retval\"))\n+\t\tgoto error_out;\n+\n+\t*skel_ptr = skel;\n+\treturn 0;\n+\n+error_out:\n+\tmemcg_async_reclaim__destroy(skel);\n+\treturn -1;\n+}\n+\n+void test_memcg_wq_async_reclaim(void)\n+{\n+\tu64 high_cgroup_id, low_cgroup_id;\n+\tint err;\n+\tdouble high_time = 0.0, low_time = 0.0;\n+\tstruct memcg_async_reclaim *skel = NULL;\n+\n+\terr = setup_high_low_cgroups(\u0026high_cgroup_id, \u0026low_cgroup_id);\n+\tif (!ASSERT_OK(err, \"setup_high_low_cgroups reclaim\"))\n+\t\treturn;\n+\n+\terr = setup_bpf(high_cgroup_id, low_cgroup_id, \u0026skel, false);\n+\tif (!ASSERT_OK(err, \"setup_bpf\"))\n+\t\tgoto out;\n+\n+\terr = run_high_low_workload(\u0026high_time, \u0026low_time, READ_TIMES);\n+\tif (!ASSERT_OK(err, \"run_high_low_workload reclaim\"))\n+\t\tgoto out;\n+\n+\tif (high_time \u003e= low_time) {\n+\t\tPRINT_FAIL(\"high cgroup not improved with async reclaim: high_time=%f low_time=%f\",\n+\t\t\t   high_time, low_time);\n+\t}\n+\n+out:\n+\tif (skel)\n+\t\tmemcg_async_reclaim__destroy(skel);\n+\t/*\n+\t * Wait for bpf_wq to release the reference to cgroup\n+\t * to ensure the successful deletion of cgroup.\n+\t */\n+\tsleep(1);\n+\tcleanup_cgroup_environment();\n+}\n+\n+void test_memcg_thread_wq_async_reclaim(void)\n+{\n+\tu64 high_cgroup_id, low_cgroup_id;\n+\tint err;\n+\tdouble high_time = 0.0, low_time = 0.0;\n+\tstruct memcg_async_reclaim *skel = NULL;\n+\n+\terr = setup_high_low_cgroups(\u0026high_cgroup_id, \u0026low_cgroup_id);\n+\tif (!ASSERT_OK(err, \"setup_high_low_cgroups reclaim\"))\n+\t\treturn;\n+\n+\terr = setup_bpf(high_cgroup_id, low_cgroup_id, \u0026skel, true);\n+\tif (!ASSERT_OK(err, \"setup_bpf\"))\n+\t\tgoto out;\n+\n+\terr = run_high_low_workload(\u0026high_time, \u0026low_time, READ_TIMES);\n+\tif (!ASSERT_OK(err, \"run_high_low_workload reclaim\"))\n+\t\tgoto out;\n+\n+\tif (high_time \u003e= low_time) {\n+\t\tPRINT_FAIL(\"high cgroup not improved with async reclaim: high_time=%f low_time=%f\",\n+\t\t\t   high_time, low_time);\n+\t}\n+\n+out:\n+\tif (skel)\n+\t\tmemcg_async_reclaim__destroy(skel);\n+\t/*\n+\t * Wait for bpf_thread_wq to release the reference to cgroup\n+\t * to ensure the successful deletion of cgroup.\n+\t */\n+\tsleep(1);\n+\tcleanup_cgroup_environment();\n+}\ndiff --git a/tools/testing/selftests/bpf/prog_tests/thread_wq_cgroup.c b/tools/testing/selftests/bpf/prog_tests/thread_wq_cgroup.c\nnew file mode 100644\nindex 0000000000000..7537b03f17e2a\n--- /dev/null\n+++ b/tools/testing/selftests/bpf/prog_tests/thread_wq_cgroup.c\n@@ -0,0 +1,87 @@\n+// SPDX-License-Identifier: GPL-2.0\n+#include \u003ctest_progs.h\u003e\n+#include \u003cunistd.h\u003e\n+#include \"cgroup_helpers.h\"\n+#include \"thread_wq_cgroup.skel.h\"\n+\n+#define TEST_CGROUP \"/thread_wq_test\"\n+#define WAIT_TIMEOUT_SECS 30\n+\n+void test_thread_wq_cgroup(void)\n+{\n+\tstruct thread_wq_cgroup *skel = NULL;\n+\tint err, prog_fd, cg_fd = -1;\n+\tunsigned long long cg_id;\n+\tint waited_secs;\n+\n+\tLIBBPF_OPTS(bpf_test_run_opts, topts);\n+\n+\terr = setup_cgroup_environment();\n+\tif (!ASSERT_OK(err, \"setup_cgroup_environment\"))\n+\t\treturn;\n+\tcg_fd = create_and_get_cgroup(TEST_CGROUP);\n+\tif (!ASSERT_GE(cg_fd, 0, \"create_and_get_cgroup\"))\n+\t\tgoto cleanup;\n+\tcg_id = get_cgroup_id(TEST_CGROUP);\n+\tif (!ASSERT_GT(cg_id, 0ULL, \"get_cgroup_id\"))\n+\t\tgoto cleanup;\n+\n+\tskel = thread_wq_cgroup__open_and_load();\n+\tif (!ASSERT_OK_PTR(skel, \"open_and_load\"))\n+\t\tgoto cleanup;\n+\n+\tprog_fd = bpf_program__fd(skel-\u003eprogs.start_thread_wq);\n+\n+\t/* Run bpf_thread_wq in the specified cgroup. */\n+\tskel-\u003ebss-\u003etest_key = 0;\n+\tskel-\u003ebss-\u003etarget_cgroup_id = cg_id;\n+\tskel-\u003ebss-\u003ecallback_cgroup_id = 0;\n+\tskel-\u003ebss-\u003etwq_done = 0;\n+\tif (!ASSERT_OK(bpf_prog_test_run_opts(prog_fd, \u0026topts),\n+\t\t       \"bpf_prog_test_run_opts in cgroup\"))\n+\t\tgoto cleanup;\n+\tif (!ASSERT_OK(topts.retval, \"retval in cgroup\"))\n+\t\tgoto cleanup;\n+\tfor (waited_secs = 0; waited_secs \u003c WAIT_TIMEOUT_SECS; waited_secs++) {\n+\t\tif (skel-\u003ebss-\u003etwq_done)\n+\t\t\tbreak;\n+\t\tsleep(1);\n+\t}\n+\tif (!ASSERT_TRUE(skel-\u003ebss-\u003etwq_done, \"twq_done in cgroup\"))\n+\t\tgoto cleanup;\n+\tif (!ASSERT_EQ(skel-\u003ebss-\u003ecallback_cgroup_id, cg_id,\n+\t\t       \"callback_cgroup_id in cgroup\"))\n+\t\tgoto cleanup;\n+\n+\t/* Run bpf_thread_wq without cgroup attachment (cgroup_id = 0). */\n+\tLIBBPF_OPTS_RESET(topts);\n+\tskel-\u003ebss-\u003etest_key = 1;\n+\tskel-\u003ebss-\u003etarget_cgroup_id = 0;\n+\tskel-\u003ebss-\u003ecallback_cgroup_id = 0;\n+\tskel-\u003ebss-\u003etwq_done = 0;\n+\tif (!ASSERT_OK(bpf_prog_test_run_opts(prog_fd, \u0026topts),\n+\t\t       \"bpf_prog_test_run_opts without cgroup\"))\n+\t\tgoto cleanup;\n+\tif (!ASSERT_OK(topts.retval, \"retval without cgroup\"))\n+\t\tgoto cleanup;\n+\tfor (waited_secs = 0; waited_secs \u003c WAIT_TIMEOUT_SECS; waited_secs++) {\n+\t\tif (skel-\u003ebss-\u003etwq_done)\n+\t\t\tbreak;\n+\t\tsleep(1);\n+\t}\n+\tif (!ASSERT_TRUE(skel-\u003ebss-\u003etwq_done, \"twq_done without cgroup\"))\n+\t\tgoto cleanup;\n+\tif (!ASSERT_NEQ(skel-\u003ebss-\u003ecallback_cgroup_id, cg_id,\n+\t\t\t\"callback_cgroup_id without cgroup\"))\n+\t\tgoto cleanup;\n+\n+cleanup:\n+\tif (skel) {\n+\t\tthread_wq_cgroup__destroy(skel);\n+\t\t/* Wait thread_wq kthread quit. */\n+\t\tsleep(2);\n+\t}\n+\tif (cg_fd \u003e= 0)\n+\t\tclose(cg_fd);\n+\tcleanup_cgroup_environment();\n+}\ndiff --git a/tools/testing/selftests/bpf/progs/memcg_async_reclaim.c b/tools/testing/selftests/bpf/progs/memcg_async_reclaim.c\nnew file mode 100644\nindex 0000000000000..eaccc8a37388b\n--- /dev/null\n+++ b/tools/testing/selftests/bpf/progs/memcg_async_reclaim.c\n@@ -0,0 +1,255 @@\n+// SPDX-License-Identifier: GPL-2.0\n+\n+#include \"vmlinux.h\"\n+#include \"bpf_experimental.h\"\n+#include \u003cbpf/bpf_helpers.h\u003e\n+#include \u003cbpf/bpf_tracing.h\u003e\n+\n+#define BIT(nr)\t\t\t(1UL \u003c\u003c (nr))\n+\n+#define ___GFP_IO\t\tBIT(___GFP_IO_BIT)\n+#define ___GFP_FS\t\tBIT(___GFP_FS_BIT)\n+#define ___GFP_DIRECT_RECLAIM\tBIT(___GFP_DIRECT_RECLAIM_BIT)\n+#define ___GFP_KSWAPD_RECLAIM\tBIT(___GFP_KSWAPD_RECLAIM_BIT)\n+\n+#define __GFP_IO\t\t((gfp_t)___GFP_IO)\n+#define __GFP_FS\t\t((gfp_t)___GFP_FS)\n+#define __GFP_DIRECT_RECLAIM\t((gfp_t)___GFP_DIRECT_RECLAIM)\n+#define __GFP_KSWAPD_RECLAIM\t((gfp_t)___GFP_KSWAPD_RECLAIM)\n+#define __GFP_RECLAIM\t((gfp_t)(___GFP_DIRECT_RECLAIM | ___GFP_KSWAPD_RECLAIM))\n+\n+#define GFP_KERNEL\t(__GFP_RECLAIM | __GFP_IO | __GFP_FS)\n+#define CLOCK_MONOTONIC_ID\t1\n+#define RECLAIM_PAGES\t\t32\n+#define RECLAIM_MAX_ITER\t32\n+\n+struct bpf_args_s {\n+\tu64 high_cgroup_id;\n+\tu64 low_cgroup_id;\n+\tu64 event_delta_threshold;\n+\tu64 check_ns;\n+};\n+\n+struct cgroup_memcg {\n+\tstruct cgroup *cgrp;\n+\tstruct mem_cgroup *memcg;\n+};\n+\n+static u64 wq_high_cgroup_id;\n+static u64 wq_low_cgroup_id;\n+\n+static int get_cgroup_memcg_from_id(u64 cgroup_id, struct cgroup_memcg *cm)\n+{\n+\tcm-\u003ecgrp = bpf_cgroup_from_id(cgroup_id);\n+\tif (!cm-\u003ecgrp)\n+\t\treturn -1;\n+\n+\tcm-\u003ememcg = bpf_get_mem_cgroup(\u0026cm-\u003ecgrp-\u003eself);\n+\tif (!cm-\u003ememcg) {\n+\t\tbpf_cgroup_release(cm-\u003ecgrp);\n+\t\treturn -1;\n+\t}\n+\n+\treturn 0;\n+}\n+\n+static void put_cgroup_memcg(struct cgroup_memcg *cm)\n+{\n+\tbpf_put_mem_cgroup(cm-\u003ememcg);\n+\tbpf_cgroup_release(cm-\u003ecgrp);\n+}\n+\n+static int get_cgroup_event(u64 cgroup_id, u64 *val)\n+{\n+\tstruct cgroup_memcg cm;\n+\n+\tif (get_cgroup_memcg_from_id(cgroup_id, \u0026cm))\n+\t\treturn -1;\n+\tbpf_mem_cgroup_flush_stats(cm.memcg);\n+\t*val = bpf_mem_cgroup_page_state(cm.memcg, WORKINGSET_REFAULT_FILE);\n+\tput_cgroup_memcg(\u0026cm);\n+\n+\treturn 0;\n+}\n+\n+static bool\n+should_reclaim_cgroup(u64 cgroup_id, u64 *prev_event, u64 event_delta_threshold)\n+{\n+\tu64 cur, delta;\n+\n+\tif (get_cgroup_event(cgroup_id, \u0026cur))\n+\t\treturn false;\n+\n+\tdelta = cur - *prev_event;\n+\t*prev_event = cur;\n+\n+\treturn delta \u003e= event_delta_threshold;\n+}\n+\n+static int reclaim_cgroup(u64 cgroup_id)\n+{\n+\tstruct cgroup_memcg cm;\n+\tint i;\n+\n+\tif (get_cgroup_memcg_from_id(cgroup_id, \u0026cm))\n+\t\treturn 0;\n+\n+\tfor (i = 0; i \u003c RECLAIM_MAX_ITER; i++) {\n+\t\tif (!bpf_try_to_free_mem_cgroup_pages(cm.memcg, RECLAIM_PAGES,\n+\t\t\t\t\t\t      GFP_KERNEL, 0, -1))\n+\t\t\tbreak;\n+\t}\n+\n+\tput_cgroup_memcg(\u0026cm);\n+\n+\treturn 0;\n+}\n+\n+struct wq_elem {\n+\tstruct bpf_timer timer;\n+\tstruct bpf_wq work;\n+\tu64 prev_event;\n+\tu64 event_delta_threshold;\n+\tu64 check_ns;\n+};\n+\n+struct {\n+\t__uint(type, BPF_MAP_TYPE_ARRAY);\n+\t__uint(max_entries, 1);\n+\t__type(key, __u32);\n+\t__type(value, struct wq_elem);\n+} wq_map SEC(\".maps\");\n+\n+static int async_free(void *map, int *key, void *value)\n+{\n+\tstruct wq_elem *elem = value;\n+\n+\tif (should_reclaim_cgroup(wq_high_cgroup_id, \u0026elem-\u003eprev_event,\n+\t\telem-\u003eevent_delta_threshold)) {\n+\t\treclaim_cgroup(wq_low_cgroup_id);\n+\t\tbpf_wq_start(\u0026elem-\u003ework, 0);\n+\t}\n+\n+\treturn 0;\n+}\n+\n+static int wq_timer_cb(void *map, int *key, struct wq_elem *elem)\n+{\n+\tbpf_wq_start(\u0026elem-\u003ework, 0);\n+\tbpf_timer_start(\u0026elem-\u003etimer, elem-\u003echeck_ns, 0);\n+\n+\treturn 0;\n+}\n+\n+SEC(\"syscall\")\n+int wq_prog_init(struct bpf_args_s *ctx)\n+{\n+\tstruct wq_elem *elem;\n+\t__u32 key = 0;\n+\tint ret;\n+\n+\telem = bpf_map_lookup_elem(\u0026wq_map, \u0026key);\n+\tif (!elem)\n+\t\treturn -1;\n+\n+\tret = bpf_wq_init(\u0026elem-\u003ework, \u0026wq_map, 0);\n+\tif (ret)\n+\t\treturn ret;\n+\n+\tret = bpf_wq_set_callback(\u0026elem-\u003ework, async_free, 0);\n+\tif (ret)\n+\t\treturn ret;\n+\n+\tret = bpf_timer_init(\u0026elem-\u003etimer, \u0026wq_map, CLOCK_MONOTONIC_ID);\n+\tif (ret)\n+\t\treturn ret;\n+\n+\tret = bpf_timer_set_callback(\u0026elem-\u003etimer, wq_timer_cb);\n+\tif (ret)\n+\t\treturn ret;\n+\n+\telem-\u003eprev_event = 0;\n+\telem-\u003eevent_delta_threshold = ctx-\u003eevent_delta_threshold;\n+\telem-\u003echeck_ns = ctx-\u003echeck_ns;\n+\n+\twq_high_cgroup_id = ctx-\u003ehigh_cgroup_id;\n+\twq_low_cgroup_id = ctx-\u003elow_cgroup_id;\n+\n+\treturn bpf_timer_start(\u0026elem-\u003etimer, elem-\u003echeck_ns, 0);\n+}\n+\n+struct thread_wq_elem {\n+\tstruct bpf_timer timer;\n+\tstruct bpf_thread_wq work;\n+\tu64 prev_event;\n+\tu64 event_delta_threshold;\n+\tu64 check_ns;\n+};\n+\n+struct {\n+\t__uint(type, BPF_MAP_TYPE_ARRAY);\n+\t__uint(max_entries, 1);\n+\t__type(key, __u32);\n+\t__type(value, struct thread_wq_elem);\n+} thread_wq_map SEC(\".maps\");\n+\n+static int thread_async_free(void *map, int *key, void *value)\n+{\n+\tstruct thread_wq_elem *elem = value;\n+\n+\tif (should_reclaim_cgroup(wq_high_cgroup_id, \u0026elem-\u003eprev_event,\n+\t\telem-\u003eevent_delta_threshold)) {\n+\t\treclaim_cgroup(wq_low_cgroup_id);\n+\t\tbpf_thread_wq_start(\u0026elem-\u003ework, 0);\n+\t}\n+\n+\treturn 0;\n+}\n+\n+static int thread_wq_timer_cb(void *map, int *key, struct thread_wq_elem *elem)\n+{\n+\tbpf_thread_wq_start(\u0026elem-\u003ework, 0);\n+\tbpf_timer_start(\u0026elem-\u003etimer, elem-\u003echeck_ns, 0);\n+\n+\treturn 0;\n+}\n+\n+SEC(\"syscall\")\n+int thread_wq_prog_init(struct bpf_args_s *ctx)\n+{\n+\tstruct thread_wq_elem *elem;\n+\t__u32 key = 0;\n+\tint ret;\n+\n+\telem = bpf_map_lookup_elem(\u0026thread_wq_map, \u0026key);\n+\tif (!elem)\n+\t\treturn -1;\n+\n+\tret = bpf_thread_wq_init(\u0026elem-\u003ework, \u0026thread_wq_map,\n+\t\t\t\t ctx-\u003elow_cgroup_id, 0);\n+\tif (ret)\n+\t\treturn ret;\n+\n+\tret = bpf_thread_wq_set_callback(\u0026elem-\u003ework, thread_async_free, 0);\n+\tif (ret)\n+\t\treturn ret;\n+\n+\tret = bpf_timer_init(\u0026elem-\u003etimer, \u0026thread_wq_map, CLOCK_MONOTONIC_ID);\n+\tif (ret)\n+\t\treturn ret;\n+\n+\tret = bpf_timer_set_callback(\u0026elem-\u003etimer, thread_wq_timer_cb);\n+\tif (ret)\n+\t\treturn ret;\n+\n+\telem-\u003eprev_event = 0;\n+\telem-\u003eevent_delta_threshold = ctx-\u003eevent_delta_threshold;\n+\telem-\u003echeck_ns = ctx-\u003echeck_ns;\n+\n+\twq_high_cgroup_id = ctx-\u003ehigh_cgroup_id;\n+\twq_low_cgroup_id = ctx-\u003elow_cgroup_id;\n+\n+\treturn bpf_timer_start(\u0026elem-\u003etimer, elem-\u003echeck_ns, 0);\n+}\n+\n+char LICENSE[] SEC(\"license\") = \"GPL\";\ndiff --git a/tools/testing/selftests/bpf/progs/thread_wq_cgroup.c b/tools/testing/selftests/bpf/progs/thread_wq_cgroup.c\nnew file mode 100644\nindex 0000000000000..c70a37f55397d\n--- /dev/null\n+++ b/tools/testing/selftests/bpf/progs/thread_wq_cgroup.c\n@@ -0,0 +1,56 @@\n+// SPDX-License-Identifier: GPL-2.0\n+/* Copyright (c) 2026 */\n+\n+#include \"bpf_experimental.h\"\n+#include \u003cbpf/bpf_helpers.h\u003e\n+#include \"bpf_misc.h\"\n+\n+char _license[] SEC(\"license\") = \"GPL\";\n+\n+struct elem {\n+\tstruct bpf_thread_wq twq;\n+};\n+\n+struct {\n+\t__uint(type, BPF_MAP_TYPE_ARRAY);\n+\t__uint(max_entries, 2);\n+\t__type(key, int);\n+\t__type(value, struct elem);\n+} map_arr SEC(\".maps\");\n+\n+__u64 target_cgroup_id;\n+__u64 callback_cgroup_id;\n+int twq_done;\n+int test_key;\n+\n+static int twq_callback(void *map, int *key, void *value)\n+{\n+\tcallback_cgroup_id = bpf_get_current_cgroup_id();\n+\ttwq_done = 1;\n+\treturn 0;\n+}\n+\n+SEC(\"syscall\")\n+int start_thread_wq(void *ctx)\n+{\n+\tstruct elem *val;\n+\tint key = test_key;\n+\tint ret;\n+\n+\tval = bpf_map_lookup_elem(\u0026map_arr, \u0026key);\n+\tif (!val)\n+\t\treturn -1;\n+\n+\tret = bpf_thread_wq_init(\u0026val-\u003etwq, \u0026map_arr, target_cgroup_id, 0);\n+\tif (ret)\n+\t\tgoto out;\n+\n+\tret = bpf_thread_wq_set_callback(\u0026val-\u003etwq, twq_callback, 0);\n+\tif (ret)\n+\t\tgoto out;\n+\n+\tret = bpf_thread_wq_start(\u0026val-\u003etwq, 0);\n+\n+out:\n+\treturn ret;\n+}\n"
}

2/1

2026/08/07 08:50

agent

patch-evaluator

Model:

best-expensive

Args:

null

Results:

{
  "EnableConfigs": [
    "MEMCG",
    "CGROUPS"
  ],
  "FocusSymbols": [
    "bpf_thread_wq_init",
    "bpf_thread_wq_set_callback",
    "bpf_thread_wq_start",
    "bpf_try_to_free_mem_cgroup_pages",
    "cgroup_kthread_attach"
  ],
  "Reasoning": "The patch introduces a new BPF thread workqueue feature (`bpf_thread_wq`) that allows BPF programs to schedule work in a kthread context, optionally attached to a cgroup. It also adds a new BPF kfunc `bpf_try_to_free_mem_cgroup_pages` for memory cgroup reclaim. These are functional changes to the core BPF and memory control subsystems, reachable from userspace via BPF programs, and should be fuzzed.",
  "WorthFuzzing": true
}

Instruction:

You are an expert Linux kernel maintainer.
Your job is to review a provided patch series and determine
if it makes functional changes to the kernel that should be fuzzed.

IMPORTANT: The changes have ALREADY been applied and committed as the HEAD commit in
your workspace. Do NOT rely on your internal knowledge of the kernel. You must actively
use your code access tools to examine the actual source code and confirm any assumptions.

Return WorthFuzzing=false if the patch only contains:
- Modifications to Documentation/, Kconfig files, or code comments.
- Purely decorative changes, such as logging (e.g., pr_err, printk) or tracepoints.
- Changes to numeric constants or macros that do not functionally alter execution flow.
- Code paths that are impossible to reach in virtualized environments like GCE or QEMU,
  even when utilizing software-emulated hardware (e.g., usb gadget, mac80211_hwsim).
- Code in vendor-specific PCIe switch, SmartNIC, or GPU drivers (e.g., mlxsw, pds_core, qed,
  ionic, amdgpu) that require physical PCIe hardware cards not emulated in standard QEMU.
- Driver .remove, .shutdown, or pci_unregister_driver teardown callbacks (e.g., igb_remove)
  that are executed only during PCI hot-unplug or sysfs driver unbind operations.

If it modifies reachable core kernel logic, drivers, or architectures, use your code search
tools to verify the code can be executed, then return WorthFuzzing=true.

When returning WorthFuzzing=true, you MUST ALSO:
1. Extract any specific kernel functions that should be heavily fuzzed into FocusSymbols.
   Avoid listing generic hot-path functions to prevent skewed test distributions.
   Prefer non-static, non-inlined API entrypoint functions over internal static helper functions
   (which are inlined by the compiler and do not have distinct symbol addresses).
2. Identify any specific CONFIG_ options required to properly test this new/modified feature.
   Go and look into the Kconfig files and check for ifdefs around the code, do not make assumptions.
   Also check "depends on" lines in Kconfig to include any non-standard parent subsystem configs
   needed for Kbuild to compile the code statically into vmlinux. List them in the EnableConfigs
   output array, and DO NOT add a 'CONFIG_' prefix (e.g., return "NET_IPV4" instead of "CONFIG_NET_IPV4").
Prefer calling several tools at the same time to save round-trips.


Use set-results tool to provide results of the analysis.
It must be called exactly once before the final reply.
Ignore results of this tool.

Prompt:

For your convenience, here is the diff of the changes:
commit e72e2c008c951fa5d07abaad9d32e877879e2ea7
Author: syz-cluster <triage@syzkaller.com>
Date:   Fri Aug 7 08:50:40 2026 +0000

    syz-cluster: applied patch under review

diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index 73bacfc6444df..d63ce83198696 100644
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -213,6 +213,7 @@ enum btf_field_type {
 	BPF_UPTR       = (1 << 11),
 	BPF_RES_SPIN_LOCK = (1 << 12),
 	BPF_TASK_WORK  = (1 << 13),
+	BPF_THREAD_WQ  = (1 << 14),
 };
 
 enum bpf_cgroup_storage_type {
@@ -267,6 +268,7 @@ struct btf_record {
 	int wq_off;
 	int refcount_off;
 	int task_work_off;
+	int thread_wq_off;
 	struct btf_field fields[];
 };
 
@@ -372,6 +374,8 @@ static inline const char *btf_field_type_name(enum btf_field_type type)
 		return "bpf_refcount";
 	case BPF_TASK_WORK:
 		return "bpf_task_work";
+	case BPF_THREAD_WQ:
+		return "bpf_thread_wq";
 	default:
 		WARN_ON_ONCE(1);
 		return "unknown";
@@ -412,6 +416,8 @@ static inline u32 btf_field_type_size(enum btf_field_type type)
 		return sizeof(struct bpf_refcount);
 	case BPF_TASK_WORK:
 		return sizeof(struct bpf_task_work);
+	case BPF_THREAD_WQ:
+		return sizeof(struct bpf_thread_wq);
 	default:
 		WARN_ON_ONCE(1);
 		return 0;
@@ -446,6 +452,8 @@ static inline u32 btf_field_type_align(enum btf_field_type type)
 		return __alignof__(struct bpf_refcount);
 	case BPF_TASK_WORK:
 		return __alignof__(struct bpf_task_work);
+	case BPF_THREAD_WQ:
+		return __alignof__(struct bpf_thread_wq);
 	default:
 		WARN_ON_ONCE(1);
 		return 0;
@@ -478,6 +486,7 @@ static inline void bpf_obj_init_field(const struct btf_field *field, void *addr)
 	case BPF_KPTR_PERCPU:
 	case BPF_UPTR:
 	case BPF_TASK_WORK:
+	case BPF_THREAD_WQ:
 		break;
 	default:
 		WARN_ON_ONCE(1);
@@ -502,6 +511,7 @@ static inline bool btf_field_is_nmi_safe(enum btf_field_type type)
 	case BPF_TASK_WORK:
 	case BPF_KPTR_UNREF:
 	case BPF_REFCOUNT:
+	case BPF_THREAD_WQ:
 		return true;
 	default:
 		return false;
@@ -644,6 +654,7 @@ void copy_map_value_locked(struct bpf_map *map, void *dst, void *src,
 void bpf_timer_cancel_and_free(void *timer);
 void bpf_wq_cancel_and_free(void *timer);
 void bpf_task_work_cancel_and_free(void *timer);
+void bpf_thread_wq_cancel_and_free(void *val);
 void bpf_list_head_free(const struct btf_field *field, void *list_head,
 			struct bpf_spin_lock *spin_lock);
 void bpf_rb_root_free(const struct btf_field *field, void *rb_root,
@@ -701,7 +712,8 @@ bool bpf_map_meta_equal(const struct bpf_map *meta0,
 
 static inline bool bpf_map_has_internal_structs(struct bpf_map *map)
 {
-	return btf_record_has_field(map->record, BPF_TIMER | BPF_WORKQUEUE | BPF_TASK_WORK);
+	return btf_record_has_field(map->record, BPF_TIMER | BPF_WORKQUEUE |
+						 BPF_TASK_WORK | BPF_THREAD_WQ);
 }
 
 void bpf_map_free_internal_structs(struct bpf_map *map, void *obj);
@@ -2725,6 +2737,7 @@ bool btf_record_equal(const struct btf_record *rec_a, const struct btf_record *r
 void bpf_obj_free_timer(const struct btf_record *rec, void *obj);
 void bpf_obj_free_workqueue(const struct btf_record *rec, void *obj);
 void bpf_obj_free_task_work(const struct btf_record *rec, void *obj);
+void bpf_obj_free_thread_wq(const struct btf_record *rec, void *obj);
 void bpf_obj_cancel_fields(struct bpf_map *map, void *obj);
 void bpf_obj_free_fields(const struct btf_record *rec, void *obj);
 void __bpf_obj_drop_impl(void *p, const struct btf_record *rec, bool percpu);
diff --git a/include/linux/cgroup.h b/include/linux/cgroup.h
index f2aa46a4f871e..9b4a8dc748ace 100644
--- a/include/linux/cgroup.h
+++ b/include/linux/cgroup.h
@@ -923,4 +923,6 @@ struct cgroup *task_get_cgroup1(struct task_struct *tsk, int hierarchy_id);
 
 struct cgroup_of_peak *of_peak(struct kernfs_open_file *of);
 
+int cgroup_kthread_attach(struct cgroup *cgrp, struct task_struct *task);
+
 #endif /* _LINUX_CGROUP_H */
diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h
index ffd96e8b920bd..0558520f67fe4 100644
--- a/include/uapi/linux/bpf.h
+++ b/include/uapi/linux/bpf.h
@@ -7574,6 +7574,10 @@ struct bpf_wq {
 	__u64 __opaque[2];
 } __attribute__((aligned(8)));
 
+struct bpf_thread_wq {
+	__u64 __opaque[2];
+} __attribute__((aligned(8)));
+
 struct bpf_dynptr {
 	__u64 __opaque[2];
 } __attribute__((aligned(8)));
diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c
index 42414633cf263..bf6fb5f51d21a 100644
--- a/kernel/bpf/btf.c
+++ b/kernel/bpf/btf.c
@@ -3665,6 +3665,7 @@ static int btf_get_field_type(const struct btf *btf, const struct btf_type *var_
 		{ BPF_TIMER, "bpf_timer", true },
 		{ BPF_WORKQUEUE, "bpf_wq", true },
 		{ BPF_TASK_WORK, "bpf_task_work", true },
+		{ BPF_THREAD_WQ, "bpf_thread_wq", true },
 		{ BPF_LIST_HEAD, "bpf_list_head", false },
 		{ BPF_LIST_NODE, "bpf_list_node", false },
 		{ BPF_RB_ROOT, "bpf_rb_root", false },
@@ -3850,6 +3851,7 @@ static int btf_find_field_one(const struct btf *btf,
 	case BPF_RB_NODE:
 	case BPF_REFCOUNT:
 	case BPF_TASK_WORK:
+	case BPF_THREAD_WQ:
 		ret = btf_find_struct(btf, var_type, off, sz, field_type,
 				      info_cnt ? &info[0] : &tmp);
 		if (ret < 0)
@@ -4145,6 +4147,7 @@ struct btf_record *btf_parse_fields(const struct btf *btf, const struct btf_type
 	rec->wq_off = -EINVAL;
 	rec->refcount_off = -EINVAL;
 	rec->task_work_off = -EINVAL;
+	rec->thread_wq_off = -EINVAL;
 	for (i = 0; i < cnt; i++) {
 		field_type_size = btf_field_type_size(info_arr[i].type);
 		if (info_arr[i].off + field_type_size > value_size) {
@@ -4188,6 +4191,10 @@ struct btf_record *btf_parse_fields(const struct btf *btf, const struct btf_type
 			WARN_ON_ONCE(rec->task_work_off >= 0);
 			rec->task_work_off = rec->fields[i].offset;
 			break;
+		case BPF_THREAD_WQ:
+			WARN_ON_ONCE(rec->thread_wq_off >= 0);
+			rec->thread_wq_off = rec->fields[i].offset;
+			break;
 		case BPF_REFCOUNT:
 			WARN_ON_ONCE(rec->refcount_off >= 0);
 			/* Cache offset for faster lookup at runtime */
diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c
index 6388b6b23e490..8e01bad1b16a0 100644
--- a/kernel/bpf/helpers.c
+++ b/kernel/bpf/helpers.c
@@ -29,6 +29,8 @@
 #include <linux/task_work.h>
 #include <linux/irq_work.h>
 #include <linux/buildid.h>
+#include <linux/kthread.h>
+#include <linux/jhash.h>
 
 #include "../../lib/kstrtox.h"
 
@@ -1110,6 +1112,17 @@ static void *map_key_from_value(struct bpf_map *map, void *value, u32 *arr_idx)
 	return (void *)value - round_up(map->key_size, 8);
 }
 
+static u32 bpf_map_elem_id_from_value(struct bpf_map *map, void *value)
+{
+	u32 arr_idx;
+	void *key;
+
+	key = map_key_from_value(map, value, &arr_idx);
+	if (map->map_type == BPF_MAP_TYPE_ARRAY)
+		return arr_idx;
+	return jhash(key, map->key_size, 0);
+}
+
 enum bpf_async_type {
 	BPF_ASYNC_TYPE_TIMER = 0,
 	BPF_ASYNC_TYPE_WQ,
@@ -4769,6 +4782,397 @@ __bpf_kfunc int bpf_timer_cancel_async(struct bpf_timer *timer)
 	}
 }
 
+/*
+ * BPF thread workqueue (kthread_worker based) implementation
+ *
+ * Why bpf_thread_wq does NOT use the bpf_async infrastructure:
+ *
+ * bpf_timer and bpf_wq share a common cleanup path via bpf_async:
+ *
+ *   bpf_async_cancel_and_free()
+ *     -> bpf_async_schedule_op()
+ *       -> irq_work_queue()                // schedule from any context
+ *         -> bpf_async_process_op()        // runs in hardirq context
+ *           -> bpf_wq_work() / timer cb    // atomic, non-sleepable
+ *
+ * This works for timer and workqueue because their callbacks and
+ * cancellation (hrtimer_cancel / bpf_wq_cancel_and_free) complete
+ * synchronously and need not sleep.  The hardirq context is sufficient.
+ *
+ * bpf_thread_wq is different: cancellation and final cleanup may need to
+ * sleep:
+ *
+ *   kthread_cancel_work_sync()    - waits for the work to finish, can
+ *                                    schedule out if the work is running.
+ *   kthread_destroy_worker()      - stops the kthread, internally
+ *                                    synchronizes with kthread exit.
+ *   cgroup_put() (final put)      - may acquire cgroup_mutex and other
+ *                                    sleeping locks during offline.
+ *
+ * None of these can be called from hardirq context (irq_work). Doing so
+ * would trigger might_sleep() warnings or deadlock.
+ *
+ * Therefore bpf_thread_wq implements its own RCU-based cleanup:
+ *
+ *   bpf_thread_wq_cancel_and_free()        // sleepable (called from
+ *                                          // map free / elem delete)
+ *     -> xchg(ctx, NULL)                   // detach atomically
+ *     -> kthread_cancel_work_sync()        // ok to sleep
+ *     -> bpf_thread_wq_ctx_put()
+ *       -> call_rcu_tasks_trace()          // wait for BPF callbacks
+ *         -> schedule_work()               // switch to sleepable context
+ *           -> bpf_thread_wq_destroy_work_fn()
+ *             -> bpf_prog_put()
+ *             -> cgroup_put()
+ *             -> kthread_destroy_worker()
+ *             -> kfree(ctx)
+ *
+ * The same design choice was made for bpf_task_work, which also avoids
+ * bpf_async because task_work cancellation synchronizes with the task
+ * and may sleep.
+ */
+
+struct bpf_thread_wq_ctx {
+	struct kthread_worker *worker;
+	struct kthread_work work;
+	struct bpf_prog *prog;
+	bpf_callback_t callback_fn;
+	struct bpf_map *map;
+	void *value;
+	struct cgroup *cgrp;
+	refcount_t refcnt;
+	struct rcu_head rcu;
+	struct work_struct destroy_work;
+};
+
+/* Kernel-internal representation that fits in struct bpf_thread_wq */
+struct bpf_thread_wq_kern {
+	struct bpf_thread_wq_ctx *ctx;
+} __aligned(8);
+
+static void bpf_thread_wq_destroy_work_fn(struct work_struct *work)
+{
+	struct bpf_thread_wq_ctx *ctx = container_of(work,
+						     struct bpf_thread_wq_ctx,
+						     destroy_work);
+
+	if (ctx->prog)
+		bpf_prog_put(ctx->prog);
+	if (ctx->cgrp)
+		cgroup_put(ctx->cgrp);
+	if (ctx->worker)
+		kthread_destroy_worker(ctx->worker);
+	kfree(ctx);
+}
+
+static void bpf_thread_wq_ctx_free_rcu(struct rcu_head *rcu)
+{
+	struct bpf_thread_wq_ctx *ctx = container_of(rcu,
+						     struct bpf_thread_wq_ctx,
+						     rcu);
+
+	INIT_WORK(&ctx->destroy_work, bpf_thread_wq_destroy_work_fn);
+	schedule_work(&ctx->destroy_work);
+}
+
+static void bpf_thread_wq_ctx_put(struct bpf_thread_wq_ctx *ctx)
+{
+	if (!refcount_dec_and_test(&ctx->refcnt))
+		return;
+	call_rcu_tasks_trace(&ctx->rcu, bpf_thread_wq_ctx_free_rcu);
+}
+
+static void bpf_thread_wq_work_fn(struct kthread_work *work)
+{
+	struct bpf_thread_wq_ctx *ctx = container_of(work,
+						     struct bpf_thread_wq_ctx,
+						     work);
+	bpf_callback_t callback_fn;
+	void *value = ctx->value;
+	struct bpf_map *map = ctx->map;
+	void *key;
+	u32 idx;
+
+	BTF_TYPE_EMIT(struct bpf_thread_wq);
+
+	callback_fn = READ_ONCE(ctx->callback_fn);
+	if (!callback_fn)
+		goto out;
+	key = map_key_from_value(map, value, &idx);
+
+	rcu_read_lock_trace();
+	migrate_disable();
+
+	callback_fn = READ_ONCE(ctx->callback_fn);
+	if (callback_fn)
+		callback_fn((u64)(long)map, (u64)(long)key, (u64)(long)value,
+			    0, 0);
+
+	migrate_enable();
+	rcu_read_unlock_trace();
+
+out:
+	bpf_thread_wq_ctx_put(ctx);
+}
+
+/*
+ * bpf_thread_wq_init() creates a kthread worker and may attach it to a cgroup.
+ * The helpers used here can sleep and acquire several locks through kthread
+ * creation/destruction, cgroup lookup and cgroup kthread attachment. Keep this
+ * kfunc available only to BPF_PROG_TYPE_SYSCALL programs so it is not invoked
+ * from BPF program contexts that already hold locks which could deadlock with
+ * those paths.
+ */
+__bpf_kfunc int bpf_thread_wq_init(struct bpf_thread_wq *twq, void *p__map,
+				   u64 cgroup_id, unsigned int flags)
+{
+	struct bpf_thread_wq_kern *twk = (struct bpf_thread_wq_kern *)twq;
+	struct bpf_map *map = p__map;
+	struct bpf_thread_wq_ctx *ctx, *old_ctx;
+	struct kthread_worker *worker;
+	struct cgroup *cgrp = NULL;
+	void *value;
+	u32 elem_id;
+	int err;
+
+	BUILD_BUG_ON(sizeof(struct bpf_thread_wq_kern)
+			> sizeof(struct bpf_thread_wq));
+	BUILD_BUG_ON(__alignof__(struct bpf_thread_wq_kern)
+			!= __alignof__(struct bpf_thread_wq));
+
+	if (flags)
+		return -EINVAL;
+
+	old_ctx = READ_ONCE(twk->ctx);
+	if (old_ctx)
+		return -EBUSY;
+
+	value = (void *)twq - map->record->thread_wq_off;
+	elem_id = bpf_map_elem_id_from_value(map, value);
+	worker = kthread_run_worker(0, "bpf_twq/%d/%x", map->id, elem_id);
+	if (IS_ERR(worker))
+		return PTR_ERR(worker);
+
+	/* Setup ctx. */
+	ctx = bpf_map_kmalloc_nolock(map, sizeof(*ctx), GFP_KERNEL,
+				     map->numa_node);
+	if (!ctx) {
+		err = -ENOMEM;
+		goto destroy_worker;
+	}
+	memset(ctx, 0, sizeof(*ctx));
+	ctx->worker = worker;
+	ctx->map = map;
+	ctx->value = value;
+	refcount_set(&ctx->refcnt, 1);
+	kthread_init_work(&ctx->work, bpf_thread_wq_work_fn);
+
+	if (cgroup_id) {
+#ifdef CONFIG_CGROUPS
+		cgrp = cgroup_get_from_id(cgroup_id);
+		if (IS_ERR(cgrp)) {
+			err = PTR_ERR(cgrp);
+			goto kfree_ctx;
+		}
+		ctx->cgrp = cgrp;
+
+		/*
+		 * kthread_run_worker() wakes the kthread, but it may not have
+		 * executed cgroup_kthread_ready() yet, which clears
+		 * no_cgroup_migration.
+		 * Do a queue work and flush to wait the kthread run.
+		 */
+		refcount_inc(&ctx->refcnt);
+		if (!kthread_queue_work(ctx->worker, &ctx->work)) {
+			refcount_dec(&ctx->refcnt);
+			err = -EBUSY;
+			goto cgroup_put;
+		}
+		kthread_flush_work(&ctx->work);
+
+		if (worker->task->no_cgroup_migration) {
+			err = -EAGAIN;
+			goto cgroup_put;
+		}
+
+		err = cgroup_kthread_attach(cgrp, worker->task);
+		if (err)
+			goto cgroup_put;
+#else
+		err = -EOPNOTSUPP;
+		goto kfree_ctx;
+#endif
+	}
+
+	old_ctx = cmpxchg(&twk->ctx, NULL, ctx);
+	if (old_ctx) {
+		err = -EBUSY;
+		goto cgroup_put;
+	}
+
+	/*
+	 * Paired with the map destruction path.  Ensures that ctx is globally
+	 * visible before we check map->usercnt.
+	 * If usercnt has dropped to zero, the destruction path will either see
+	 * the ctx (and cancel it) or we see usercnt == 0 here and cancel
+	 * ourselves.
+	 * Without this barrier, a CPU could reorder the load of usercnt before
+	 * the cmpxchg store becomes visible, breaking the mutual exclusion
+	 * guarantee.
+	 */
+	smp_mb();
+
+	if (!atomic64_read(&map->usercnt)) {
+		bpf_thread_wq_cancel_and_free(twq);
+		return -EPERM;
+	}
+
+	return 0;
+
+cgroup_put:
+#ifdef CONFIG_CGROUPS
+	if (cgrp)
+		cgroup_put(cgrp);
+#endif
+kfree_ctx:
+	/*
+	 * Not use bpf_thread_wq_ctx_put because ctx has not yet entered
+	 * the running state.
+	 */
+	kfree(ctx);
+destroy_worker:
+	kthread_destroy_worker(worker);
+	return err;
+}
+
+__bpf_kfunc int bpf_thread_wq_set_callback(struct bpf_thread_wq *twq,
+					   int (callback_fn)(void *map,
+							     int *key,
+							     void *value),
+					   unsigned int flags,
+					   struct bpf_prog_aux *aux)
+{
+	struct bpf_thread_wq_kern *twk = (struct bpf_thread_wq_kern *)twq;
+	struct bpf_thread_wq_ctx *ctx;
+	struct bpf_prog *prog;
+
+	if (flags)
+		return -EINVAL;
+
+	ctx = READ_ONCE(twk->ctx);
+	if (!ctx)
+		return -EINVAL;
+
+	prog = bpf_prog_inc_not_zero(aux->prog);
+	if (IS_ERR(prog))
+		return PTR_ERR(prog);
+
+	/*
+	 * Allow set_callback only once to prevent UAF: a concurrent
+	 * set_callback could bpf_prog_put() the prog while the worker
+	 * kthread is still executing its callback.
+	 */
+	if (cmpxchg(&ctx->prog, NULL, prog) != NULL) {
+		bpf_prog_put(prog);
+		return -EBUSY;
+	}
+	/*
+	 * Safe to set callback_fn after prog: bpf_thread_wq_start() and
+	 * bpf_thread_wq_work_fn() both check callback_fn with READ_ONCE()
+	 * and bail out if it is still NULL.
+	 */
+	WRITE_ONCE(ctx->callback_fn, (void *)callback_fn);
+
+	return 0;
+}
+
+__bpf_kfunc int
+bpf_thread_wq_start(struct bpf_thread_wq *twq, unsigned int flags)
+{
+	struct bpf_thread_wq_kern *twk = (struct bpf_thread_wq_kern *)twq;
+	struct bpf_thread_wq_ctx *ctx;
+	int err;
+
+	if (flags)
+		return -EINVAL;
+
+	rcu_read_lock_trace();
+
+	err = 0;
+
+	ctx = READ_ONCE(twk->ctx);
+	if (!ctx || !READ_ONCE(ctx->callback_fn)) {
+		err = -EINVAL;
+		goto unlock;
+	}
+
+	if (!refcount_inc_not_zero(&ctx->refcnt))
+		err = -ENOENT;
+
+unlock:
+	rcu_read_unlock_trace();
+	if (err)
+		return err;
+
+	if (!kthread_queue_work(ctx->worker, &ctx->work)) {
+		bpf_thread_wq_ctx_put(ctx);
+		return -EBUSY;
+	}
+
+	return 0;
+}
+
+void bpf_thread_wq_cancel_and_free(void *val)
+{
+	struct bpf_thread_wq_kern *twk = val;
+	struct bpf_thread_wq_ctx *ctx;
+
+	ctx = xchg(&twk->ctx, NULL);
+	if (!ctx)
+		return;
+
+	might_sleep();
+
+	/*
+	 * Prevent future callbacks from running and wait for any
+	 * in-progress execution to finish.
+	 */
+	WRITE_ONCE(ctx->callback_fn, NULL);
+	/*
+	 * kthread_cancel_work_sync() returns true when it dequeues a pending
+	 * work item from the work_list without executing it.  Each successful
+	 * bpf_thread_wq_start() call increments ctx->refcnt and relies on the
+	 * subsequent bpf_thread_wq_work_fn() execution to release that
+	 * reference via bpf_thread_wq_ctx_put().  If the work was pending and
+	 * got cancelled here, work_fn will never run for that queued instance,
+	 * so we must drop the reference ourselves to avoid a permanent refcount
+	 * leak.
+	 *
+	 * This covers two scenarios uniformly:
+	 *  1. The work is purely pending (not currently executing) - e.g. a
+	 *     normal bpf_thread_wq_start() call queued it but the worker
+	 *     thread hasn't picked it up yet.
+	 *  2. The work is currently in-flight AND was self-rescheduled from
+	 *     within the callback - kthread_cancel_work_sync() dequeues the
+	 *     re-queued pending node and then waits for the in-flight
+	 *     execution to complete.
+	 * In both cases the return value is true, indicating one orphaned
+	 * reference that needs to be released here.
+	 */
+	if (kthread_cancel_work_sync(&ctx->work))
+		bpf_thread_wq_ctx_put(ctx);
+
+	/*
+	 * Drop our own reference.  If the work was still in-flight above,
+	 * the refcount won't hit zero here - it will reach zero when the
+	 * work path calls bpf_thread_wq_ctx_put() upon completion.  Either
+	 * way, final cleanup (worker destruction, prog put, cgroup put,
+	 * kfree) happens exclusively in the RCU callback to keep the
+	 * teardown path single-threaded.
+	 */
+	bpf_thread_wq_ctx_put(ctx);
+}
+
 __bpf_kfunc_end_defs();
 
 static void bpf_task_work_cancel_scheduled(struct irq_work *irq_work)
@@ -4917,6 +5321,8 @@ BTF_ID_FLAGS(func, bpf_modify_return_test_tp)
 BTF_ID_FLAGS(func, bpf_wq_init)
 BTF_ID_FLAGS(func, bpf_wq_set_callback, KF_IMPLICIT_ARGS)
 BTF_ID_FLAGS(func, bpf_wq_start)
+BTF_ID_FLAGS(func, bpf_thread_wq_set_callback, KF_IMPLICIT_ARGS)
+BTF_ID_FLAGS(func, bpf_thread_wq_start)
 BTF_ID_FLAGS(func, bpf_preempt_disable)
 BTF_ID_FLAGS(func, bpf_preempt_enable)
 BTF_ID_FLAGS(func, bpf_iter_bits_new, KF_ITER_NEW)
@@ -4978,6 +5384,15 @@ static const struct btf_kfunc_id_set common_kfunc_set = {
 	.set   = &common_btf_ids,
 };
 
+BTF_KFUNCS_START(syscall_btf_ids)
+BTF_ID_FLAGS(func, bpf_thread_wq_init, KF_SLEEPABLE)
+BTF_KFUNCS_END(syscall_btf_ids)
+
+static const struct btf_kfunc_id_set syscall_kfunc_set = {
+	.owner = THIS_MODULE,
+	.set   = &syscall_btf_ids,
+};
+
 static int __init kfunc_init(void)
 {
 	int ret;
@@ -5000,6 +5415,7 @@ static int __init kfunc_init(void)
 	ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, &generic_kfunc_set);
 	ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL, &generic_kfunc_set);
 	ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_CGROUP_SKB, &generic_kfunc_set);
+	ret = ret ?: register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL, &syscall_kfunc_set);
 	ret = ret ?: register_btf_id_dtor_kfuncs(generic_dtors,
 						  ARRAY_SIZE(generic_dtors),
 						  THIS_MODULE);
@@ -5037,4 +5453,6 @@ void bpf_map_free_internal_structs(struct bpf_map *map, void *val)
 		bpf_obj_free_workqueue(map->record, val);
 	if (btf_record_has_field(map->record, BPF_TASK_WORK))
 		bpf_obj_free_task_work(map->record, val);
+	if (btf_record_has_field(map->record, BPF_THREAD_WQ))
+		bpf_obj_free_thread_wq(map->record, val);
 }
diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index 8d111da886553..dea14823bacd2 100644
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -688,6 +688,7 @@ void btf_record_free(struct btf_record *rec)
 		case BPF_REFCOUNT:
 		case BPF_WORKQUEUE:
 		case BPF_TASK_WORK:
+		case BPF_THREAD_WQ:
 			/* Nothing to release */
 			break;
 		default:
@@ -742,6 +743,7 @@ struct btf_record *btf_record_dup(const struct btf_record *rec)
 		case BPF_REFCOUNT:
 		case BPF_WORKQUEUE:
 		case BPF_TASK_WORK:
+		case BPF_THREAD_WQ:
 			/* Nothing to acquire */
 			break;
 		default:
@@ -807,6 +809,13 @@ void bpf_obj_free_task_work(const struct btf_record *rec, void *obj)
 	bpf_task_work_cancel_and_free(obj + rec->task_work_off);
 }
 
+void bpf_obj_free_thread_wq(const struct btf_record *rec, void *obj)
+{
+	if (WARN_ON_ONCE(!btf_record_has_field(rec, BPF_THREAD_WQ)))
+		return;
+	bpf_thread_wq_cancel_and_free(obj + rec->thread_wq_off);
+}
+
 void bpf_obj_cancel_fields(struct bpf_map *map, void *obj)
 {
 	bpf_map_free_internal_structs(map, obj);
@@ -839,6 +848,9 @@ void bpf_obj_free_fields(const struct btf_record *rec, void *obj)
 		case BPF_TASK_WORK:
 			bpf_task_work_cancel_and_free(field_ptr);
 			break;
+		case BPF_THREAD_WQ:
+			bpf_thread_wq_cancel_and_free(field_ptr);
+			break;
 		case BPF_KPTR_UNREF:
 			WRITE_ONCE(*(u64 *)field_ptr, 0);
 			break;
@@ -1265,7 +1277,7 @@ static int map_check_btf(struct bpf_map *map, struct bpf_token *token,
 	map->record = btf_parse_fields(btf, value_type,
 				       BPF_SPIN_LOCK | BPF_RES_SPIN_LOCK | BPF_TIMER | BPF_KPTR | BPF_LIST_HEAD |
 				       BPF_RB_ROOT | BPF_REFCOUNT | BPF_WORKQUEUE | BPF_UPTR |
-				       BPF_TASK_WORK,
+				       BPF_TASK_WORK | BPF_THREAD_WQ,
 				       map->value_size);
 	if (!IS_ERR_OR_NULL(map->record)) {
 		int i;
@@ -1299,6 +1311,7 @@ static int map_check_btf(struct bpf_map *map, struct bpf_token *token,
 			case BPF_TIMER:
 			case BPF_WORKQUEUE:
 			case BPF_TASK_WORK:
+			case BPF_THREAD_WQ:
 				if (map->map_type != BPF_MAP_TYPE_HASH &&
 				    map->map_type != BPF_MAP_TYPE_RHASH &&
 				    map->map_type != BPF_MAP_TYPE_LRU_HASH &&
diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
index e6233c0081d10..44e79fe480b6b 100644
--- a/kernel/bpf/verifier.c
+++ b/kernel/bpf/verifier.c
@@ -477,6 +477,7 @@ static bool is_async_callback_calling_kfunc(u32 btf_id);
 static bool is_callback_calling_kfunc(u32 btf_id);
 
 static bool is_bpf_wq_set_callback_kfunc(u32 btf_id);
+static bool is_bpf_thread_wq_set_callback_kfunc(u32 btf_id);
 static bool is_task_work_add_kfunc(u32 func_id);
 
 static bool is_sync_callback_calling_function(enum bpf_func_id func_id)
@@ -516,9 +517,11 @@ static bool is_async_cb_sleepable(struct bpf_verifier_env *env, struct bpf_insn
 	if (bpf_helper_call(insn) && insn->imm == BPF_FUNC_timer_set_callback)
 		return false;
 
-	/* bpf_wq and bpf_task_work callbacks are always sleepable. */
+	/* bpf_wq, bpf_thread_wq and bpf_task_work callbacks are always sleepable. */
 	if (bpf_pseudo_kfunc_call(insn) && insn->off == 0 &&
-	    (is_bpf_wq_set_callback_kfunc(insn->imm) || is_task_work_add_kfunc(insn->imm)))
+	    (is_bpf_wq_set_callback_kfunc(insn->imm) ||
+	     is_bpf_thread_wq_set_callback_kfunc(insn->imm) ||
+	     is_task_work_add_kfunc(insn->imm)))
 		return true;
 
 	verifier_bug(env, "unhandled async callback in is_async_cb_sleepable");
@@ -1871,7 +1874,10 @@ static void refine_map_lookup_value(struct bpf_reg_state *reg)
 		 * as UID of the inner map.
 		 */
 		if (btf_record_has_field(map->inner_map_meta->record,
-					 BPF_TIMER | BPF_WORKQUEUE | BPF_TASK_WORK))
+					 BPF_TIMER |
+					 BPF_WORKQUEUE |
+					 BPF_TASK_WORK |
+					 BPF_THREAD_WQ))
 			reg->map_uid = reg->id;
 	} else if (map->map_type == BPF_MAP_TYPE_XSKMAP) {
 		reg->type = PTR_TO_XDP_SOCK | maybe_null;
@@ -7243,6 +7249,9 @@ static int check_map_field_pointer(struct bpf_verifier_env *env, struct bpf_reg_
 	case BPF_WORKQUEUE:
 		field_off = map->record->wq_off;
 		break;
+	case BPF_THREAD_WQ:
+		field_off = map->record->thread_wq_off;
+		break;
 	default:
 		verifier_bug(env, "unsupported BTF field type: %s\n", struct_name);
 		return -EINVAL;
@@ -10940,6 +10949,7 @@ enum {
 	KF_ARG_WORKQUEUE_ID,
 	KF_ARG_RES_SPIN_LOCK_ID,
 	KF_ARG_TASK_WORK_ID,
+	KF_ARG_THREAD_WQ_ID,
 	KF_ARG_PROG_AUX_ID,
 	KF_ARG_TIMER_ID
 };
@@ -10953,6 +10963,7 @@ BTF_ID(struct, bpf_rb_node)
 BTF_ID(struct, bpf_wq)
 BTF_ID(struct, bpf_res_spin_lock)
 BTF_ID(struct, bpf_task_work)
+BTF_ID(struct, bpf_thread_wq)
 BTF_ID(struct, bpf_prog_aux)
 BTF_ID(struct, bpf_timer)
 
@@ -11013,6 +11024,11 @@ static bool is_kfunc_arg_task_work(const struct btf *btf, const struct btf_param
 	return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_TASK_WORK_ID);
 }
 
+static bool is_kfunc_arg_thread_wq(const struct btf *btf, const struct btf_param *arg)
+{
+	return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_THREAD_WQ_ID);
+}
+
 static bool is_kfunc_arg_res_spin_lock(const struct btf *btf, const struct btf_param *arg)
 {
 	return __is_kfunc_ptr_arg_type(btf, arg, KF_ARG_RES_SPIN_LOCK_ID);
@@ -11132,6 +11148,7 @@ enum kfunc_ptr_arg_type {
 	KF_ARG_PTR_TO_IRQ_FLAG,
 	KF_ARG_PTR_TO_RES_SPIN_LOCK,
 	KF_ARG_PTR_TO_TASK_WORK,
+	KF_ARG_PTR_TO_THREAD_WQ,
 };
 
 enum special_kfunc_type {
@@ -11178,6 +11195,7 @@ enum special_kfunc_type {
 	KF_bpf_percpu_obj_drop,
 	KF_bpf_throw,
 	KF_bpf_wq_set_callback,
+	KF_bpf_thread_wq_set_callback,
 	KF_bpf_preempt_disable,
 	KF_bpf_preempt_enable,
 	KF_bpf_iter_css_task_new,
@@ -11258,6 +11276,7 @@ BTF_ID(func, bpf_percpu_obj_drop_impl)
 BTF_ID(func, bpf_percpu_obj_drop)
 BTF_ID(func, bpf_throw)
 BTF_ID(func, bpf_wq_set_callback)
+BTF_ID(func, bpf_thread_wq_set_callback)
 BTF_ID(func, bpf_preempt_disable)
 BTF_ID(func, bpf_preempt_enable)
 #ifdef CONFIG_CGROUPS
@@ -11460,6 +11479,8 @@ get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta,
 		arg_type = KF_ARG_PTR_TO_TIMER;
 	else if (is_kfunc_arg_task_work(meta->btf, &args[arg]))
 		arg_type = KF_ARG_PTR_TO_TASK_WORK;
+	else if (is_kfunc_arg_thread_wq(meta->btf, &args[arg]))
+		arg_type = KF_ARG_PTR_TO_THREAD_WQ;
 	else if (is_kfunc_arg_irq_flag(meta->btf, &args[arg]))
 		arg_type = KF_ARG_PTR_TO_IRQ_FLAG;
 	else if (is_kfunc_arg_res_spin_lock(meta->btf, &args[arg]))
@@ -11833,6 +11854,7 @@ static bool is_sync_callback_calling_kfunc(u32 btf_id)
 static bool is_async_callback_calling_kfunc(u32 btf_id)
 {
 	return is_bpf_wq_set_callback_kfunc(btf_id) ||
+	       is_bpf_thread_wq_set_callback_kfunc(btf_id) ||
 	       is_task_work_add_kfunc(btf_id);
 }
 
@@ -11847,6 +11869,11 @@ static bool is_bpf_wq_set_callback_kfunc(u32 btf_id)
 	return btf_id == special_kfunc_list[KF_bpf_wq_set_callback];
 }
 
+static bool is_bpf_thread_wq_set_callback_kfunc(u32 btf_id)
+{
+	return btf_id == special_kfunc_list[KF_bpf_thread_wq_set_callback];
+}
+
 static bool is_callback_calling_kfunc(u32 btf_id)
 {
 	return is_sync_callback_calling_kfunc(btf_id) ||
@@ -12192,6 +12219,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me
 		case KF_ARG_PTR_TO_WORKQUEUE:
 		case KF_ARG_PTR_TO_TIMER:
 		case KF_ARG_PTR_TO_TASK_WORK:
+		case KF_ARG_PTR_TO_THREAD_WQ:
 		case KF_ARG_PTR_TO_IRQ_FLAG:
 		case KF_ARG_PTR_TO_RES_SPIN_LOCK:
 			break;
@@ -13139,6 +13167,16 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
 		}
 	}
 
+	if (is_bpf_thread_wq_set_callback_kfunc(meta.func_id)) {
+		err = push_callback_call(env, insn, insn_idx, meta.subprogno,
+					 set_timer_callback_state);
+		if (err) {
+			verbose(env, "kfunc %s#%d failed callback verification\n",
+				func_name, meta.func_id);
+			return err;
+		}
+	}
+
 	if (is_task_work_add_kfunc(meta.func_id)) {
 		err = push_callback_call(env, insn, insn_idx, meta.subprogno,
 					 set_task_work_schedule_callback_state);
diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c
index 38f8d9df8fbc4..164e069c75762 100644
--- a/kernel/cgroup/cgroup.c
+++ b/kernel/cgroup/cgroup.c
@@ -3042,6 +3042,19 @@ int cgroup_attach_task(struct cgroup *dst_cgrp, struct task_struct *leader,
 	return ret;
 }
 
+int cgroup_kthread_attach(struct cgroup *cgrp, struct task_struct *task)
+{
+	int ret;
+
+	cgroup_lock();
+	cgroup_attach_lock(CGRP_ATTACH_LOCK_GLOBAL, NULL);
+	ret = cgroup_attach_task(cgrp, task, false);
+	cgroup_attach_unlock(CGRP_ATTACH_LOCK_GLOBAL, NULL);
+	cgroup_unlock();
+
+	return ret;
+}
+
 struct task_struct *cgroup_procs_write_start(char *buf, bool threadgroup,
 					     enum cgroup_attach_lock_mode *lock_mode)
 {
diff --git a/mm/bpf_memcontrol.c b/mm/bpf_memcontrol.c
index 716df49d76477..3f7a5c97e1355 100644
--- a/mm/bpf_memcontrol.c
+++ b/mm/bpf_memcontrol.c
@@ -6,6 +6,7 @@
  */
 
 #include <linux/memcontrol.h>
+#include <linux/swap.h>
 #include <linux/bpf.h>
 
 __bpf_kfunc_start_defs();
@@ -159,6 +160,61 @@ __bpf_kfunc void bpf_mem_cgroup_flush_stats(struct mem_cgroup *memcg)
 	mem_cgroup_flush_stats(memcg);
 }
 
+/**
+ * bpf_try_to_free_mem_cgroup_pages - attempt to reclaim pages from
+ *                                    a memory cgroup
+ * @memcg:           the target memory cgroup to reclaim from
+ * @nr_pages:        the number of pages to reclaim
+ * @gfp_mask:        GFP flags controlling the reclaim behavior
+ * @reclaim_options: bitmask of MEMCG_RECLAIM_* flags to tune
+ *                   reclaim strategy
+ * @swappiness:      swappiness override value, or a sentinel to use
+ *                   the default
+ *
+ * BPF-facing wrapper around try_to_free_mem_cgroup_pages() that
+ * validates and translates the @swappiness argument before
+ * delegating to the core reclaim path.
+ *
+ * The @swappiness parameter follows these semantics:
+ *   - Values in [MIN_SWAPPINESS, SWAPPINESS_ANON_ONLY] are passed
+ *     through as an explicit swappiness override.
+ *   - Values below MIN_SWAPPINESS are treated as "use the system
+ *     default"; the override pointer is set to NULL and the cgroup's
+ *     own swappiness setting takes effect.
+ *   - Values above SWAPPINESS_ANON_ONLY are rejected as invalid.
+ *   - If @reclaim_options does not include MEMCG_RECLAIM_PROACTIVE,
+ *     the @swappiness override is ignored entirely by the core
+ *     reclaim path and the system default is used regardless.
+ *
+ * Swap usage during reclaim is gated on @reclaim_options: swap is
+ * considered only when MEMCG_RECLAIM_MAY_SWAP is set.  Without this
+ * flag, reclaim is restricted to file-backed pages regardless of the
+ * @swappiness value or the cgroup's swappiness setting.
+ *
+ * Return:
+ *   The number of pages actually reclaimed on success, or 0
+ *   if @swappiness exceeds SWAPPINESS_ANON_ONLY.
+ */
+__bpf_kfunc unsigned long
+bpf_try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg,
+				 unsigned long nr_pages,
+				 gfp_t gfp_mask,
+				 unsigned int reclaim_options,
+				 int swappiness)
+{
+	int *swapiness_ptr;
+
+	if (swappiness > SWAPPINESS_ANON_ONLY)
+		return 0;
+	else if (swappiness < MIN_SWAPPINESS)
+		swapiness_ptr = NULL;
+	else
+		swapiness_ptr = &swappiness;
+
+	return try_to_free_mem_cgroup_pages(memcg, nr_pages, gfp_mask,
+					    reclaim_options, swapiness_ptr);
+}
+
 __bpf_kfunc_end_defs();
 
 BTF_KFUNCS_START(bpf_memcontrol_kfuncs)
@@ -172,6 +228,8 @@ BTF_ID_FLAGS(func, bpf_mem_cgroup_usage)
 BTF_ID_FLAGS(func, bpf_mem_cgroup_page_state)
 BTF_ID_FLAGS(func, bpf_mem_cgroup_flush_stats, KF_SLEEPABLE)
 
+BTF_ID_FLAGS(func, bpf_try_to_free_mem_cgroup_pages, KF_SLEEPABLE)
+
 BTF_KFUNCS_END(bpf_memcontrol_kfuncs)
 
 static const struct btf_kfunc_id_set bpf_memcontrol_kfunc_set = {
diff --git a/tools/testing/selftests/bpf/bpf_experimental.h b/tools/testing/selftests/bpf/bpf_experimental.h
index ff37ae5a113dc..ea905ea5603c2 100644
--- a/tools/testing/selftests/bpf/bpf_experimental.h
+++ b/tools/testing/selftests/bpf/bpf_experimental.h
@@ -351,6 +351,13 @@ extern void bpf_iter_css_destroy(struct bpf_iter_css *it) __weak __ksym;
 extern int bpf_wq_init(struct bpf_wq *wq, void *p__map, unsigned int flags) __weak __ksym;
 extern int bpf_wq_start(struct bpf_wq *wq, unsigned int flags) __weak __ksym;
 
+struct bpf_thread_wq;
+extern int bpf_thread_wq_init(struct bpf_thread_wq *twq, void *p__map,
+			      __u64 cgroup_id,
+			      unsigned int flags) __weak __ksym;
+extern int bpf_thread_wq_start(struct bpf_thread_wq *twq,
+			       unsigned int flags) __weak __ksym;
+
 struct bpf_iter_kmem_cache;
 extern int bpf_iter_kmem_cache_new(struct bpf_iter_kmem_cache *it) __weak __ksym;
 extern struct kmem_cache *bpf_iter_kmem_cache_next(struct bpf_iter_kmem_cache *it) __weak __ksym;
diff --git a/tools/testing/selftests/bpf/prog_tests/memcg_async_reclaim.c b/tools/testing/selftests/bpf/prog_tests/memcg_async_reclaim.c
new file mode 100644
index 0000000000000..40a5fe62cb384
--- /dev/null
+++ b/tools/testing/selftests/bpf/prog_tests/memcg_async_reclaim.c
@@ -0,0 +1,479 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Memory controller eBPF async reclaim test
+ */
+
+#include <test_progs.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <sys/time.h>
+#include <sys/vfs.h>
+#include <sys/wait.h>
+#include <fcntl.h>
+#include <linux/magic.h>
+#include <unistd.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "cgroup_helpers.h"
+
+struct bpf_args_s {
+	u64 high_cgroup_id;
+	u64 low_cgroup_id;
+	u64 event_delta_threshold;
+	u64 check_ns;
+};
+
+#include "memcg_async_reclaim.skel.h"
+
+#define FILE_SIZE (32 * 1024 * 1024ul)
+#define BUFFER_SIZE (4096)
+#define CG_LIMIT (32 * 1024 * 1024ul)
+#define READ_TIMES 16
+
+#define CG_DIR "/memcg_async_reclaim"
+#define CG_HIGH_DIR CG_DIR "/high"
+#define CG_LOW_DIR CG_DIR "/low"
+
+#define CHECK_PERIOD_NS (2 * 1000 * 1000ull)
+#define EVENT_DELTA_THRESHOLD 1
+
+/*
+ * Test files must reside on a filesystem that supports page reclaim without
+ * swap (e.g. ext4). If /tmp is on tmpfs, the file pages are shmem-backed
+ * and can only be reclaimed through swap. But the test disables swap
+ * (memory.swap.max=0), making reclaim impossible and causing OOM.
+ *
+ * Pick a directory on a non-tmpfs filesystem: try $TMPDIR first, then /tmp,
+ * and fall back to the current directory if the chosen path is on tmpfs.
+ */
+static int get_test_dir(char *buf, size_t size)
+{
+	static const char * const candidates[] = { "/tmp", "." };
+	const char *tmpdir = getenv("TMPDIR");
+	struct statfs sfs;
+	size_t i;
+
+	if (tmpdir && tmpdir[0] && statfs(tmpdir, &sfs) == 0 &&
+	    sfs.f_type != TMPFS_MAGIC) {
+		snprintf(buf, size, "%s", tmpdir);
+		return 0;
+	}
+
+	for (i = 0; i < ARRAY_SIZE(candidates); i++) {
+		if (statfs(candidates[i], &sfs) == 0 &&
+		    sfs.f_type != TMPFS_MAGIC) {
+			snprintf(buf, size, "%s", candidates[i]);
+			return 0;
+		}
+	}
+
+	return -1;
+}
+
+static int setup_high_low_cgroups(u64 *high_cgroup_id, u64 *low_cgroup_id)
+{
+	int ret;
+	char limit_buf[20];
+
+	ret = setup_cgroup_environment();
+	if (!ASSERT_OK(ret, "setup_cgroup_environment"))
+		goto cleanup;
+
+	ret = create_and_get_cgroup(CG_DIR);
+	if (!ASSERT_GE(ret, 0, "create_and_get_cgroup " CG_DIR))
+		goto cleanup;
+	close(ret);
+
+	ret = enable_controllers(CG_DIR, "memory");
+	if (!ASSERT_OK(ret, "enable_controllers"))
+		goto cleanup;
+
+	snprintf(limit_buf, sizeof(limit_buf), "%lu", CG_LIMIT);
+	ret = write_cgroup_file(CG_DIR, "memory.max", limit_buf);
+	if (!ASSERT_OK(ret, "write_cgroup_file memory.max"))
+		goto cleanup;
+
+	ret = write_cgroup_file(CG_DIR, "memory.swap.max", "0");
+	if (!ASSERT_OK(ret, "write_cgroup_file memory.swap.max"))
+		goto cleanup;
+
+	ret = create_and_get_cgroup(CG_HIGH_DIR);
+	if (!ASSERT_GE(ret, 0, "create_and_get_cgroup " CG_HIGH_DIR))
+		goto cleanup;
+	close(ret);
+
+	*high_cgroup_id = get_cgroup_id(CG_HIGH_DIR);
+	if (!ASSERT_GT(*high_cgroup_id, 0, "get_cgroup_id"))
+		goto cleanup;
+
+	ret = create_and_get_cgroup(CG_LOW_DIR);
+	if (!ASSERT_GE(ret, 0, "create_and_get_cgroup " CG_LOW_DIR))
+		goto cleanup;
+	close(ret);
+
+	*low_cgroup_id = get_cgroup_id(CG_LOW_DIR);
+	if (!ASSERT_GT(*high_cgroup_id, 0, "get_cgroup_id"))
+		goto cleanup;
+
+	return 0;
+
+cleanup:
+	cleanup_cgroup_environment();
+	return -1;
+}
+
+static int write_file(const char *filename)
+{
+	int ret = -1;
+	size_t written = 0;
+	char *buffer;
+	FILE *fp;
+
+	fp = fopen(filename, "wb");
+	if (!fp)
+		goto out;
+
+	buffer = malloc(BUFFER_SIZE);
+	if (!buffer)
+		goto cleanup_fp;
+
+	memset(buffer, 'A', BUFFER_SIZE);
+
+	while (written < FILE_SIZE) {
+		size_t to_write = FILE_SIZE - written < BUFFER_SIZE ?
+				  FILE_SIZE - written : BUFFER_SIZE;
+
+		if (fwrite(buffer, 1, to_write, fp) != to_write)
+			goto cleanup;
+		written += to_write;
+	}
+
+	ret = 0;
+cleanup:
+	free(buffer);
+cleanup_fp:
+	fclose(fp);
+out:
+	return ret;
+}
+
+static int read_file(const char *filename, int iterations)
+{
+	int ret = -1;
+	long page_size = sysconf(_SC_PAGESIZE);
+	char *map;
+	size_t i;
+	int fd;
+	struct stat sb;
+
+	fd = open(filename, O_RDONLY);
+	if (fd == -1)
+		goto out;
+
+	if (fstat(fd, &sb) == -1)
+		goto cleanup_fd;
+
+	if (sb.st_size != FILE_SIZE) {
+		fprintf(stderr, "File size mismatch: expected %lu, got %lu\n",
+			(unsigned long)FILE_SIZE, (unsigned long)sb.st_size);
+		goto cleanup_fd;
+	}
+
+	map = mmap(NULL, FILE_SIZE, PROT_READ, MAP_PRIVATE, fd, 0);
+	if (map == MAP_FAILED)
+		goto cleanup_fd;
+
+	for (int iter = 0; iter < iterations; iter++) {
+		for (i = 0; i < FILE_SIZE; i += page_size) {
+			/* access a byte to trigger page fault */
+			asm volatile("" :: "r"(map[i]) : "memory");
+		}
+	}
+
+	if (munmap(map, FILE_SIZE) == -1)
+		goto cleanup_fd;
+
+	ret = 0;
+
+cleanup_fd:
+	close(fd);
+out:
+	return ret;
+}
+
+static int real_test_child_work(const char *cgroup_path, char *data_filename,
+				char *time_filename, int read_times)
+{
+	struct timeval start, end;
+	double elapsed;
+	FILE *fp;
+
+	if (!ASSERT_OK(join_parent_cgroup(cgroup_path), "join_parent_cgroup"))
+		return -1;
+
+	gettimeofday(&start, NULL);
+
+	if (!ASSERT_OK(write_file(data_filename), "write_file"))
+		return -1;
+
+	if (!ASSERT_OK(read_file(data_filename, read_times), "read_file"))
+		return -1;
+
+	gettimeofday(&end, NULL);
+
+	if (!time_filename)
+		return 0;
+
+	elapsed = (end.tv_sec - start.tv_sec) +
+		  (end.tv_usec - start.tv_usec) / 1000000.0;
+	printf("%.6f\n", elapsed);
+
+	fp = fopen(time_filename, "w");
+	if (!ASSERT_OK_PTR(fp, "fopen"))
+		return -1;
+	fprintf(fp, "%.6f", elapsed);
+	fclose(fp);
+
+	return 0;
+}
+
+static int get_time(char *time_filename, double *time)
+{
+	int ret = -1;
+	FILE *fp;
+	char buf[64];
+
+	fp = fopen(time_filename, "r");
+	if (!ASSERT_OK_PTR(fp, "fopen"))
+		goto out;
+
+	if (!ASSERT_OK_PTR(fgets(buf, sizeof(buf), fp), "fgets"))
+		goto cleanup;
+
+	if (sscanf(buf, "%lf", time) != 1) {
+		PRINT_FAIL("sscanf %s", buf);
+		goto cleanup;
+	}
+
+	ret = 0;
+cleanup:
+	fclose(fp);
+out:
+	return ret;
+}
+
+static int
+run_high_low_workload(double *high_elapsed, double *low_elapsed, int read_times)
+{
+	char test_dir[PATH_MAX], high_data_file[PATH_MAX], low_data_file[PATH_MAX];
+	char high_time_file[PATH_MAX], low_time_file[PATH_MAX];
+	int ret, fd;
+	pid_t high_pid, low_pid;
+	int status;
+
+	ret = get_test_dir(test_dir, sizeof(test_dir));
+	if (!ASSERT_OK(ret, "get_test_dir: no non-tmpfs directory found"))
+		return -1;
+
+	fd = snprintf(high_data_file, sizeof(high_data_file),
+		      "%s/memcg_async_high_data_XXXXXX", test_dir);
+	if (!ASSERT_LT(fd, sizeof(high_data_file), "high_data_file path"))
+		return -1;
+
+	fd = snprintf(low_data_file, sizeof(low_data_file),
+		      "%s/memcg_async_low_data_XXXXXX", test_dir);
+	if (!ASSERT_LT(fd, sizeof(low_data_file), "low_data_file path"))
+		return -1;
+
+	fd = snprintf(high_time_file, sizeof(high_time_file),
+		      "%s/memcg_async_high_time_XXXXXX", test_dir);
+	if (!ASSERT_LT(fd, sizeof(high_time_file), "high_time_file path"))
+		return -1;
+
+	fd = snprintf(low_time_file, sizeof(low_time_file),
+		      "%s/memcg_async_low_time_XXXXXX", test_dir);
+	if (!ASSERT_LT(fd, sizeof(low_time_file), "low_time_file path"))
+		return -1;
+
+	fd = mkstemp(high_data_file);
+	if (!ASSERT_GE(fd, 0, "mkstemp"))
+		return -1;
+	close(fd);
+
+	fd = mkstemp(low_data_file);
+	if (!ASSERT_GE(fd, 0, "mkstemp"))
+		goto cleanup_high_data;
+	close(fd);
+
+	fd = mkstemp(high_time_file);
+	if (!ASSERT_GE(fd, 0, "mkstemp"))
+		goto cleanup_low_data;
+	close(fd);
+
+	fd = mkstemp(low_time_file);
+	if (!ASSERT_GE(fd, 0, "mkstemp"))
+		goto cleanup_high_time;
+	close(fd);
+
+	low_pid = fork();
+	if (!ASSERT_GE(low_pid, 0, "fork low"))
+		goto cleanup_low_time;
+	if (low_pid == 0)
+		exit(real_test_child_work(CG_LOW_DIR, low_data_file,
+					  low_time_file, read_times));
+
+	high_pid = fork();
+	if (!ASSERT_GE(high_pid, 0, "fork high")) {
+		(void)waitpid(low_pid, NULL, 0);
+		goto cleanup_low_time;
+	}
+	if (high_pid == 0)
+		exit(real_test_child_work(CG_HIGH_DIR, high_data_file,
+					  high_time_file, read_times));
+
+	ret = waitpid(low_pid, &status, 0);
+	if (!ASSERT_GT(ret, 0, "low waitpid"))
+		goto cleanup_low_time;
+	if (!ASSERT_TRUE(WIFEXITED(status), "low exited"))
+		goto cleanup_low_time;
+	if (!ASSERT_EQ(WEXITSTATUS(status), 0, "low exit status"))
+		goto cleanup_low_time;
+
+	ret = waitpid(high_pid, &status, 0);
+	if (!ASSERT_GT(ret, 0, "high waitpid"))
+		goto cleanup_low_time;
+	if (!ASSERT_TRUE(WIFEXITED(status), "high exited"))
+		goto cleanup_low_time;
+	if (!ASSERT_EQ(WEXITSTATUS(status), 0, "high exit status"))
+		goto cleanup_low_time;
+
+	if (get_time(high_time_file, high_elapsed))
+		goto cleanup_low_time;
+	if (get_time(low_time_file, low_elapsed))
+		goto cleanup_low_time;
+
+	ret = 0;
+
+cleanup_low_time:
+	unlink(low_time_file);
+cleanup_high_time:
+	unlink(high_time_file);
+cleanup_low_data:
+	unlink(low_data_file);
+cleanup_high_data:
+	unlink(high_data_file);
+	return ret;
+}
+
+static int
+setup_bpf(u64 high_cgroup_id, u64 low_cgroup_id,
+	  struct memcg_async_reclaim **skel_ptr, bool use_thread_wq)
+{
+	struct memcg_async_reclaim *skel;
+	struct bpf_args_s bpf_args = {
+		.high_cgroup_id = high_cgroup_id,
+		.low_cgroup_id = low_cgroup_id,
+		.event_delta_threshold = EVENT_DELTA_THRESHOLD,
+		.check_ns = CHECK_PERIOD_NS,
+	};
+	LIBBPF_OPTS(bpf_test_run_opts, run_opts,
+		.ctx_in = &bpf_args,
+		.ctx_size_in = sizeof(bpf_args));
+	int prog_init_fd, err;
+
+	skel = memcg_async_reclaim__open_and_load();
+	if (!ASSERT_OK_PTR(skel, "memcg_async_reclaim__open_and_load"))
+		return -1;
+
+	if (use_thread_wq)
+		prog_init_fd = bpf_program__fd(skel->progs.thread_wq_prog_init);
+	else
+		prog_init_fd = bpf_program__fd(skel->progs.wq_prog_init);
+	if (!ASSERT_GE(prog_init_fd, 0, "bpf_program__fd"))
+		goto error_out;
+
+	err = bpf_prog_test_run_opts(prog_init_fd, &run_opts);
+	if (!ASSERT_OK(err, "bpf_prog_test_run_opts"))
+		goto error_out;
+	if (!ASSERT_EQ(run_opts.retval, 0, "prog_init retval"))
+		goto error_out;
+
+	*skel_ptr = skel;
+	return 0;
+
+error_out:
+	memcg_async_reclaim__destroy(skel);
+	return -1;
+}
+
+void test_memcg_wq_async_reclaim(void)
+{
+	u64 high_cgroup_id, low_cgroup_id;
+	int err;
+	double high_time = 0.0, low_time = 0.0;
+	struct memcg_async_reclaim *skel = NULL;
+
+	err = setup_high_low_cgroups(&high_cgroup_id, &low_cgroup_id);
+	if (!ASSERT_OK(err, "setup_high_low_cgroups reclaim"))
+		return;
+
+	err = setup_bpf(high_cgroup_id, low_cgroup_id, &skel, false);
+	if (!ASSERT_OK(err, "setup_bpf"))
+		goto out;
+
+	err = run_high_low_workload(&high_time, &low_time, READ_TIMES);
+	if (!ASSERT_OK(err, "run_high_low_workload reclaim"))
+		goto out;
+
+	if (high_time >= low_time) {
+		PRINT_FAIL("high cgroup not improved with async reclaim: high_time=%f low_time=%f",
+			   high_time, low_time);
+	}
+
+out:
+	if (skel)
+		memcg_async_reclaim__destroy(skel);
+	/*
+	 * Wait for bpf_wq to release the reference to cgroup
+	 * to ensure the successful deletion of cgroup.
+	 */
+	sleep(1);
+	cleanup_cgroup_environment();
+}
+
+void test_memcg_thread_wq_async_reclaim(void)
+{
+	u64 high_cgroup_id, low_cgroup_id;
+	int err;
+	double high_time = 0.0, low_time = 0.0;
+	struct memcg_async_reclaim *skel = NULL;
+
+	err = setup_high_low_cgroups(&high_cgroup_id, &low_cgroup_id);
+	if (!ASSERT_OK(err, "setup_high_low_cgroups reclaim"))
+		return;
+
+	err = setup_bpf(high_cgroup_id, low_cgroup_id, &skel, true);
+	if (!ASSERT_OK(err, "setup_bpf"))
+		goto out;
+
+	err = run_high_low_workload(&high_time, &low_time, READ_TIMES);
+	if (!ASSERT_OK(err, "run_high_low_workload reclaim"))
+		goto out;
+
+	if (high_time >= low_time) {
+		PRINT_FAIL("high cgroup not improved with async reclaim: high_time=%f low_time=%f",
+			   high_time, low_time);
+	}
+
+out:
+	if (skel)
+		memcg_async_reclaim__destroy(skel);
+	/*
+	 * Wait for bpf_thread_wq to release the reference to cgroup
+	 * to ensure the successful deletion of cgroup.
+	 */
+	sleep(1);
+	cleanup_cgroup_environment();
+}
diff --git a/tools/testing/selftests/bpf/prog_tests/thread_wq_cgroup.c b/tools/testing/selftests/bpf/prog_tests/thread_wq_cgroup.c
new file mode 100644
index 0000000000000..7537b03f17e2a
--- /dev/null
+++ b/tools/testing/selftests/bpf/prog_tests/thread_wq_cgroup.c
@@ -0,0 +1,87 @@
+// SPDX-License-Identifier: GPL-2.0
+#include <test_progs.h>
+#include <unistd.h>
+#include "cgroup_helpers.h"
+#include "thread_wq_cgroup.skel.h"
+
+#define TEST_CGROUP "/thread_wq_test"
+#define WAIT_TIMEOUT_SECS 30
+
+void test_thread_wq_cgroup(void)
+{
+	struct thread_wq_cgroup *skel = NULL;
+	int err, prog_fd, cg_fd = -1;
+	unsigned long long cg_id;
+	int waited_secs;
+
+	LIBBPF_OPTS(bpf_test_run_opts, topts);
+
+	err = setup_cgroup_environment();
+	if (!ASSERT_OK(err, "setup_cgroup_environment"))
+		return;
+	cg_fd = create_and_get_cgroup(TEST_CGROUP);
+	if (!ASSERT_GE(cg_fd, 0, "create_and_get_cgroup"))
+		goto cleanup;
+	cg_id = get_cgroup_id(TEST_CGROUP);
+	if (!ASSERT_GT(cg_id, 0ULL, "get_cgroup_id"))
+		goto cleanup;
+
+	skel = thread_wq_cgroup__open_and_load();
+	if (!ASSERT_OK_PTR(skel, "open_and_load"))
+		goto cleanup;
+
+	prog_fd = bpf_program__fd(skel->progs.start_thread_wq);
+
+	/* Run bpf_thread_wq in the specified cgroup. */
+	skel->bss->test_key = 0;
+	skel->bss->target_cgroup_id = cg_id;
+	skel->bss->callback_cgroup_id = 0;
+	skel->bss->twq_done = 0;
+	if (!ASSERT_OK(bpf_prog_test_run_opts(prog_fd, &topts),
+		       "bpf_prog_test_run_opts in cgroup"))
+		goto cleanup;
+	if (!ASSERT_OK(topts.retval, "retval in cgroup"))
+		goto cleanup;
+	for (waited_secs = 0; waited_secs < WAIT_TIMEOUT_SECS; waited_secs++) {
+		if (skel->bss->twq_done)
+			break;
+		sleep(1);
+	}
+	if (!ASSERT_TRUE(skel->bss->twq_done, "twq_done in cgroup"))
+		goto cleanup;
+	if (!ASSERT_EQ(skel->bss->callback_cgroup_id, cg_id,
+		       "callback_cgroup_id in cgroup"))
+		goto cleanup;
+
+	/* Run bpf_thread_wq without cgroup attachment (cgroup_id = 0). */
+	LIBBPF_OPTS_RESET(topts);
+	skel->bss->test_key = 1;
+	skel->bss->target_cgroup_id = 0;
+	skel->bss->callback_cgroup_id = 0;
+	skel->bss->twq_done = 0;
+	if (!ASSERT_OK(bpf_prog_test_run_opts(prog_fd, &topts),
+		       "bpf_prog_test_run_opts without cgroup"))
+		goto cleanup;
+	if (!ASSERT_OK(topts.retval, "retval without cgroup"))
+		goto cleanup;
+	for (waited_secs = 0; waited_secs < WAIT_TIMEOUT_SECS; waited_secs++) {
+		if (skel->bss->twq_done)
+			break;
+		sleep(1);
+	}
+	if (!ASSERT_TRUE(skel->bss->twq_done, "twq_done without cgroup"))
+		goto cleanup;
+	if (!ASSERT_NEQ(skel->bss->callback_cgroup_id, cg_id,
+			"callback_cgroup_id without cgroup"))
+		goto cleanup;
+
+cleanup:
+	if (skel) {
+		thread_wq_cgroup__destroy(skel);
+		/* Wait thread_wq kthread quit. */
+		sleep(2);
+	}
+	if (cg_fd >= 0)
+		close(cg_fd);
+	cleanup_cgroup_environment();
+}
diff --git a/tools/testing/selftests/bpf/progs/memcg_async_reclaim.c b/tools/testing/selftests/bpf/progs/memcg_async_reclaim.c
new file mode 100644
index 0000000000000..eaccc8a37388b
--- /dev/null
+++ b/tools/testing/selftests/bpf/progs/memcg_async_reclaim.c
@@ -0,0 +1,255 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include "vmlinux.h"
+#include "bpf_experimental.h"
+#include <bpf/bpf_helpers.h>
+#include <bpf/bpf_tracing.h>
+
+#define BIT(nr)			(1UL << (nr))
+
+#define ___GFP_IO		BIT(___GFP_IO_BIT)
+#define ___GFP_FS		BIT(___GFP_FS_BIT)
+#define ___GFP_DIRECT_RECLAIM	BIT(___GFP_DIRECT_RECLAIM_BIT)
+#define ___GFP_KSWAPD_RECLAIM	BIT(___GFP_KSWAPD_RECLAIM_BIT)
+
+#define __GFP_IO		((gfp_t)___GFP_IO)
+#define __GFP_FS		((gfp_t)___GFP_FS)
+#define __GFP_DIRECT_RECLAIM	((gfp_t)___GFP_DIRECT_RECLAIM)
+#define __GFP_KSWAPD_RECLAIM	((gfp_t)___GFP_KSWAPD_RECLAIM)
+#define __GFP_RECLAIM	((gfp_t)(___GFP_DIRECT_RECLAIM | ___GFP_KSWAPD_RECLAIM))
+
+#define GFP_KERNEL	(__GFP_RECLAIM | __GFP_IO | __GFP_FS)
+#define CLOCK_MONOTONIC_ID	1
+#define RECLAIM_PAGES		32
+#define RECLAIM_MAX_ITER	32
+
+struct bpf_args_s {
+	u64 high_cgroup_id;
+	u64 low_cgroup_id;
+	u64 event_delta_threshold;
+	u64 check_ns;
+};
+
+struct cgroup_memcg {
+	struct cgroup *cgrp;
+	struct mem_cgroup *memcg;
+};
+
+static u64 wq_high_cgroup_id;
+static u64 wq_low_cgroup_id;
+
+static int get_cgroup_memcg_from_id(u64 cgroup_id, struct cgroup_memcg *cm)
+{
+	cm->cgrp = bpf_cgroup_from_id(cgroup_id);
+	if (!cm->cgrp)
+		return -1;
+
+	cm->memcg = bpf_get_mem_cgroup(&cm->cgrp->self);
+	if (!cm->memcg) {
+		bpf_cgroup_release(cm->cgrp);
+		return -1;
+	}
+
+	return 0;
+}
+
+static void put_cgroup_memcg(struct cgroup_memcg *cm)
+{
+	bpf_put_mem_cgroup(cm->memcg);
+	bpf_cgroup_release(cm->cgrp);
+}
+
+static int get_cgroup_event(u64 cgroup_id, u64 *val)
+{
+	struct cgroup_memcg cm;
+
+	if (get_cgroup_memcg_from_id(cgroup_id, &cm))
+		return -1;
+	bpf_mem_cgroup_flush_stats(cm.memcg);
+	*val = bpf_mem_cgroup_page_state(cm.memcg, WORKINGSET_REFAULT_FILE);
+	put_cgroup_memcg(&cm);
+
+	return 0;
+}
+
+static bool
+should_reclaim_cgroup(u64 cgroup_id, u64 *prev_event, u64 event_delta_threshold)
+{
+	u64 cur, delta;
+
+	if (get_cgroup_event(cgroup_id, &cur))
+		return false;
+
+	delta = cur - *prev_event;
+	*prev_event = cur;
+
+	return delta >= event_delta_threshold;
+}
+
+static int reclaim_cgroup(u64 cgroup_id)
+{
+	struct cgroup_memcg cm;
+	int i;
+
+	if (get_cgroup_memcg_from_id(cgroup_id, &cm))
+		return 0;
+
+	for (i = 0; i < RECLAIM_MAX_ITER; i++) {
+		if (!bpf_try_to_free_mem_cgroup_pages(cm.memcg, RECLAIM_PAGES,
+						      GFP_KERNEL, 0, -1))
+			break;
+	}
+
+	put_cgroup_memcg(&cm);
+
+	return 0;
+}
+
+struct wq_elem {
+	struct bpf_timer timer;
+	struct bpf_wq work;
+	u64 prev_event;
+	u64 event_delta_threshold;
+	u64 check_ns;
+};
+
+struct {
+	__uint(type, BPF_MAP_TYPE_ARRAY);
+	__uint(max_entries, 1);
+	__type(key, __u32);
+	__type(value, struct wq_elem);
+} wq_map SEC(".maps");
+
+static int async_free(void *map, int *key, void *value)
+{
+	struct wq_elem *elem = value;
+
+	if (should_reclaim_cgroup(wq_high_cgroup_id, &elem->prev_event,
+		elem->event_delta_threshold)) {
+		reclaim_cgroup(wq_low_cgroup_id);
+		bpf_wq_start(&elem->work, 0);
+	}
+
+	return 0;
+}
+
+static int wq_timer_cb(void *map, int *key, struct wq_elem *elem)
+{
+	bpf_wq_start(&elem->work, 0);
+	bpf_timer_start(&elem->timer, elem->check_ns, 0);
+
+	return 0;
+}
+
+SEC("syscall")
+int wq_prog_init(struct bpf_args_s *ctx)
+{
+	struct wq_elem *elem;
+	__u32 key = 0;
+	int ret;
+
+	elem = bpf_map_lookup_elem(&wq_map, &key);
+	if (!elem)
+		return -1;
+
+	ret = bpf_wq_init(&elem->work, &wq_map, 0);
+	if (ret)
+		return ret;
+
+	ret = bpf_wq_set_callback(&elem->work, async_free, 0);
+	if (ret)
+		return ret;
+
+	ret = bpf_timer_init(&elem->timer, &wq_map, CLOCK_MONOTONIC_ID);
+	if (ret)
+		return ret;
+
+	ret = bpf_timer_set_callback(&elem->timer, wq_timer_cb);
+	if (ret)
+		return ret;
+
+	elem->prev_event = 0;
+	elem->event_delta_threshold = ctx->event_delta_threshold;
+	elem->check_ns = ctx->check_ns;
+
+	wq_high_cgroup_id = ctx->high_cgroup_id;
+	wq_low_cgroup_id = ctx->low_cgroup_id;
+
+	return bpf_timer_start(&elem->timer, elem->check_ns, 0);
+}
+
+struct thread_wq_elem {
+	struct bpf_timer timer;
+	struct bpf_thread_wq work;
+	u64 prev_event;
+	u64 event_delta_threshold;
+	u64 check_ns;
+};
+
+struct {
+	__uint(type, BPF_MAP_TYPE_ARRAY);
+	__uint(max_entries, 1);
+	__type(key, __u32);
+	__type(value, struct thread_wq_elem);
+} thread_wq_map SEC(".maps");
+
+static int thread_async_free(void *map, int *key, void *value)
+{
+	struct thread_wq_elem *elem = value;
+
+	if (should_reclaim_cgroup(wq_high_cgroup_id, &elem->prev_event,
+		elem->event_delta_threshold)) {
+		reclaim_cgroup(wq_low_cgroup_id);
+		bpf_thread_wq_start(&elem->work, 0);
+	}
+
+	return 0;
+}
+
+static int thread_wq_timer_cb(void *map, int *key, struct thread_wq_elem *elem)
+{
+	bpf_thread_wq_start(&elem->work, 0);
+	bpf_timer_start(&elem->timer, elem->check_ns, 0);
+
+	return 0;
+}
+
+SEC("syscall")
+int thread_wq_prog_init(struct bpf_args_s *ctx)
+{
+	struct thread_wq_elem *elem;
+	__u32 key = 0;
+	int ret;
+
+	elem = bpf_map_lookup_elem(&thread_wq_map, &key);
+	if (!elem)
+		return -1;
+
+	ret = bpf_thread_wq_init(&elem->work, &thread_wq_map,
+				 ctx->low_cgroup_id, 0);
+	if (ret)
+		return ret;
+
+	ret = bpf_thread_wq_set_callback(&elem->work, thread_async_free, 0);
+	if (ret)
+		return ret;
+
+	ret = bpf_timer_init(&elem->timer, &thread_wq_map, CLOCK_MONOTONIC_ID);
+	if (ret)
+		return ret;
+
+	ret = bpf_timer_set_callback(&elem->timer, thread_wq_timer_cb);
+	if (ret)
+		return ret;
+
+	elem->prev_event = 0;
+	elem->event_delta_threshold = ctx->event_delta_threshold;
+	elem->check_ns = ctx->check_ns;
+
+	wq_high_cgroup_id = ctx->high_cgroup_id;
+	wq_low_cgroup_id = ctx->low_cgroup_id;
+
+	return bpf_timer_start(&elem->timer, elem->check_ns, 0);
+}
+
+char LICENSE[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/thread_wq_cgroup.c b/tools/testing/selftests/bpf/progs/thread_wq_cgroup.c
new file mode 100644
index 0000000000000..c70a37f55397d
--- /dev/null
+++ b/tools/testing/selftests/bpf/progs/thread_wq_cgroup.c
@@ -0,0 +1,56 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (c) 2026 */
+
+#include "bpf_experimental.h"
+#include <bpf/bpf_helpers.h>
+#include "bpf_misc.h"
+
+char _license[] SEC("license") = "GPL";
+
+struct elem {
+	struct bpf_thread_wq twq;
+};
+
+struct {
+	__uint(type, BPF_MAP_TYPE_ARRAY);
+	__uint(max_entries, 2);
+	__type(key, int);
+	__type(value, struct elem);
+} map_arr SEC(".maps");
+
+__u64 target_cgroup_id;
+__u64 callback_cgroup_id;
+int twq_done;
+int test_key;
+
+static int twq_callback(void *map, int *key, void *value)
+{
+	callback_cgroup_id = bpf_get_current_cgroup_id();
+	twq_done = 1;
+	return 0;
+}
+
+SEC("syscall")
+int start_thread_wq(void *ctx)
+{
+	struct elem *val;
+	int key = test_key;
+	int ret;
+
+	val = bpf_map_lookup_elem(&map_arr, &key);
+	if (!val)
+		return -1;
+
+	ret = bpf_thread_wq_init(&val->twq, &map_arr, target_cgroup_id, 0);
+	if (ret)
+		goto out;
+
+	ret = bpf_thread_wq_set_callback(&val->twq, twq_callback, 0);
+	if (ret)
+		goto out;
+
+	ret = bpf_thread_wq_start(&val->twq, 0);
+
+out:
+	return ret;
+}

3/2

2026/08/07 08:50

llm

patch-evaluator

Model:

gemini-3.1-pro-preview

Args:

null

Results:

null

Tokens:

							input: 25554
							output: 29
							thoughts: 361

Thoughts:

**Analyzing the New BPF Thread Workqueue Feature**

Okay, so this patch is introducing something interesting: a new BPF feature called `bpf_thread_wq`. Right off the bat, I see the addition of `BPF_THREAD_WQ` to `enum btf_field_type`, which tells me this is a pretty fundamental change to the BPF landscape. We're looking at four new functions: `bpf_thread_wq_init`, `bpf_thread_wq_set_callback`, `bpf_thread_wq_start`, and `bpf_thread_wq_cancel_and_free`. Then there's `bpf_try_to_free_mem_cgroup_pages` as well. This all looks like the framework to allow BPF programs to schedule tasks on kthread workers, and, as an optional feature, attach those tasks to a specific cgroup. Definitely sounds like a candidate for some serious fuzzing – gotta see where the edge cases lie.

Let's nail down the key symbols we need to investigate: `bpf_thread_wq_init`, `bpf_thread_wq_set_callback`, `bpf_thread_wq_start`, and `bpf_try_to_free_mem_cgroup_pages`. Those are the entry points here.

Next, I need to check for any critical `CONFIG` options that might be required. From the look of it, `bpf_thread_wq_init` will use `CONFIG_CGROUPS` to attach to a cgroup, but it looks like it will gracefully degrade with `-EOPNOTSUPP` if cgroups aren't enabled. That's good, less of a hard dependency. Now, `bpf_try_to_free_mem_cgroup_pages` – that's located in `mm/bpf_memcontrol.c`. Since that file is dealing with memory control, it makes sense that `CONFIG_MEMCG` and `CONFIG_BPF_SYSCALL` are likely dependencies, but I should probably check the `mm/Makefile` to confirm the details.

4/2

2026/08/07 08:50

tool

grepper

Args:

{
  "Expression": "bpf_memcontrol.o",
  "PathPrefix": "mm/"
}

Results:

{
  "Output": "mm/Makefile=106=ifdef CONFIG_BPF_SYSCALL\nmm/Makefile:107:obj-$(CONFIG_MEMCG) += bpf_memcontrol.o\nmm/Makefile-108-endif\n"
}

5/2

2026/08/07 08:50

llm

patch-evaluator

Model:

gemini-3.1-pro-preview

Args:

null

Results:

null

Tokens:

							input: 26466
							output: 205
							thoughts: 832

Thoughts:

**Analyzing the New BPF Thread Workqueue Feature**

Okay, so I'm digging into this new BPF thread workqueue patch. First things first: it looks like we need `CONFIG_MEMCG` for `bpf_try_to_free_mem_cgroup_pages`, which makes sense for memory control, and `CONFIG_CGROUPS` is required for `bpf_thread_wq_init` to function fully with cgroups, and then of course `CONFIG_BPF_SYSCALL` is standard for BPF, so let's mark that as understood.

Let me confirm that `bpf_thread_wq_init` is actually available. Yep, it's defined right there in `kernel/bpf/helpers.c`. Good. Alright, what else is in play? Hmm, `bpf_thread_wq_cancel_and_free` is interesting, it is called internally when maps are freed. It's not a direct kfunc used by BPF programs, but definitely part of the lifecycle. And `bpf_obj_free_thread_wq` is also internal.

Okay, so the kfuncs we *do* need to focus on are: `bpf_thread_wq_init`, `bpf_thread_wq_set_callback`, `bpf_thread_wq_start`, and `bpf_try_to_free_mem_cgroup_pages`. Now, let's verify that these kfuncs are actually exposed. `bpf_thread_wq_init` is added to `syscall_btf_ids` which is registered for `BPF_PROG_TYPE_SYSCALL`. Good. `bpf_thread_wq_set_callback` and `bpf_thread_wq_start` are in `generic_btf_ids`, which is registered for a bunch of program types, so that's fine. And `bpf_try_to_free_mem_cgroup_pages` is in `bpf_memcontrol_kfuncs`. Confirmed, confirmed, confirmed.

So, the key symbols to keep an eye on are: `bpf_thread_wq_init`, `bpf_thread_wq_set_callback`, `bpf_thread_wq_start`, and `bpf_try_to_free_mem_cgroup_pages`. And then the relevant config options are `MEMCG` and `CGROUPS`.

Just double-checking... `bpf_thread_wq_cancel_and_free`. No, that's just a global function, internal to `kernel/bpf/helpers.c`. Wait...but `bpf_obj_free_thread_wq` and `bpf_obj_free_fields` call it.

Oh, one more thing to consider: `cgroup_kthread_attach`, which is added in `kernel/cgroup/cgroup.c`. It's a global function, and a helper for `bpf_thread_wq_init`. Okay, I think that *should* be included in the list of focus symbols, even though it's technically a helper function.

Alright, let's recap the reasoning: this patch introduces a new BPF thread workqueue, letting BPF programs schedule work in kthread context, optionally tied to a cgroup, and also adds `bpf_try_to_free_mem_cgroup_pages` for memory cgroup reclaim. These are functional changes to the BPF core and memory control, accessible from userspace through BPF programs, so definitely fuzzable.

**Final Focus Symbols:**

*   `bpf_thread_wq_init`
*   `bpf_thread_wq_set_callback`
*   `bpf_thread_wq_start`
*   `bpf_try_to_free_mem_cgroup_pages`
*   `cgroup_kthread_attach`

**Relevant Configs:**

*   `MEMCG`
*   `CGROUPS`

Okay, that's everything. Let's send these results over.

6/2

2026/08/07 08:51

tool

set-results

Args:

{
  "EnableConfigs": [
    "MEMCG",
    "CGROUPS"
  ],
  "FocusSymbols": [
    "bpf_thread_wq_init",
    "bpf_thread_wq_set_callback",
    "bpf_thread_wq_start",
    "bpf_try_to_free_mem_cgroup_pages",
    "cgroup_kthread_attach"
  ],
  "Reasoning": "The patch introduces a new BPF thread workqueue feature (`bpf_thread_wq`) that allows BPF programs to schedule work in a kthread context, optionally attached to a cgroup. It also adds a new BPF kfunc `bpf_try_to_free_mem_cgroup_pages` for memory cgroup reclaim. These are functional changes to the core BPF and memory control subsystems, reachable from userspace via BPF programs, and should be fuzzed.",
  "WorthFuzzing": true
}

Results:

{
  "EnableConfigs": [
    "MEMCG",
    "CGROUPS"
  ],
  "FocusSymbols": [
    "bpf_thread_wq_init",
    "bpf_thread_wq_set_callback",
    "bpf_thread_wq_start",
    "bpf_try_to_free_mem_cgroup_pages",
    "cgroup_kthread_attach"
  ],
  "Reasoning": "The patch introduces a new BPF thread workqueue feature (`bpf_thread_wq`) that allows BPF programs to schedule work in a kthread context, optionally attached to a cgroup. It also adds a new BPF kfunc `bpf_try_to_free_mem_cgroup_pages` for memory cgroup reclaim. These are functional changes to the core BPF and memory control subsystems, reachable from userspace via BPF programs, and should be fuzzed.",
  "WorthFuzzing": true
}

LLM Calls Summary:
Total Calls	Total Tokens	Avg Tokens	Total Duration (Seconds)	Avg Duration (Seconds)

Tool Calls Summary:
Total Calls	Total Duration (Seconds)	Avg Duration (Seconds)