kmem_cache_create() and friends create new instances of struct kmem_cache and return pointers to those. Quite a few things in core kernel are allocated from such caches; each allocation involves dereferencing an assign-once pointer and for sufficiently hot ones that dereferencing does show in profiles. There had been patches floating around switching some of those to runtime_const infrastructure. Unfortunately, it's arch-specific and most of the architectures lack it. There's an alternative approach applicable at least to the caches that are never destroyed, which covers a lot of them. No matter what, runtime_const for pointers is not going to be faster than plain &, so if we had struct kmem_cache instances with static storage duration, we would be at least no worse off than we are with runtime_const variants. There are obstacles to doing that, but they turn out to be easy to deal with. 1) as it is, struct kmem_cache is opaque for anything outside of a few files in mm/*; that avoids serious headache with header dependencies, etc., and it's not something we want to lose. Solution: struct kmem_cache_opaque, with the size and alignment identical to struct kmem_cache. Calculation of size and alignment can be done via the same mechanism we use for asm-offsets.h and rq-offsets.h, with build-time check for mismatches. With that done, we get an opaque type defined in linux/slab-static.h that can be used for declaring those caches. In linux/slab.h we add a forward declaration of kmem_cache_opaque + helper (to_kmem_cache()) converting a pointer to kmem_cache_opaque into pointer to kmem_cache. 2) real constructor of kmem_cache needs to be taught to deal with preallocated instances. That turns out to be easy - we already pass an obscene amount of optional arguments via struct kmem_cache_args, so we can stash the pointer to preallocated instance in there. Changes in mm/slab_common.c are very minor - we should treat preallocated caches as unmergable, use the instance passed to us instead of allocating a new one and we should not free them. That's it. Note that slab-static.h is needed only in places that create such instances; all users need only slab.h (and they can be modular, unlike runtime_const-based approach). That covers the instances that never get destroyed. Quite a few fall into that category, but there's a major exception - anything in modules must be destroyed before the module gets removed. For example, filesystems that have their inodes allocated from a private kmem_cache can't make use of that technics for their inode allocations, etc. It's not that hard to deal with, but for now let's just ban including slab-static.h from modules. Signed-off-by: Al Viro --- Kbuild | 13 +++++++- include/linux/slab-static.h | 65 +++++++++++++++++++++++++++++++++++++ include/linux/slab.h | 7 ++++ mm/kmem_cache_size.c | 20 ++++++++++++ mm/slab_common.c | 30 ++++++++--------- mm/slub.c | 7 ++++ 6 files changed, 126 insertions(+), 16 deletions(-) create mode 100644 include/linux/slab-static.h create mode 100644 mm/kmem_cache_size.c diff --git a/Kbuild b/Kbuild index 13324b4bbe23..eb985a6614eb 100644 --- a/Kbuild +++ b/Kbuild @@ -45,13 +45,24 @@ kernel/sched/rq-offsets.s: $(offsets-file) $(rq-offsets-file): kernel/sched/rq-offsets.s FORCE $(call filechk,offsets,__RQ_OFFSETS_H__) +# generate kmem_cache_size.h + +kmem_cache_size-file := include/generated/kmem_cache_size.h + +targets += mm/kmem_cache_size.s + +mm/kmem_cache_size.s: $(rq-offsets-file) + +$(kmem_cache_size-file): mm/kmem_cache_size.s FORCE + $(call filechk,offsets,__KMEM_CACHE_SIZE_H__) + # Check for missing system calls quiet_cmd_syscalls = CALL $< cmd_syscalls = $(CONFIG_SHELL) $< $(CC) $(c_flags) $(missing_syscalls_flags) PHONY += missing-syscalls -missing-syscalls: scripts/checksyscalls.sh $(rq-offsets-file) +missing-syscalls: scripts/checksyscalls.sh $(kmem_cache_size-file) $(call cmd,syscalls) # Check the manual modification of atomic headers diff --git a/include/linux/slab-static.h b/include/linux/slab-static.h new file mode 100644 index 000000000000..47b2220b4988 --- /dev/null +++ b/include/linux/slab-static.h @@ -0,0 +1,65 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_SLAB_STATIC_H +#define _LINUX_SLAB_STATIC_H + +#ifdef MODULE +#error "can't use that in modules" +#endif + +#include + +/* same size and alignment as struct kmem_cache: */ +struct kmem_cache_opaque { + unsigned char opaque[KMEM_CACHE_SIZE]; +} __aligned(KMEM_CACHE_ALIGN); + +#define __KMEM_CACHE_SETUP(cache, name, size, flags, ...) \ + __kmem_cache_create_args((name), (size), \ + &(struct kmem_cache_args) { \ + .preallocated = (cache), \ + __VA_ARGS__}, (flags)) + +static inline int +kmem_cache_setup_usercopy(struct kmem_cache *s, + const char *name, unsigned int size, + unsigned int align, slab_flags_t flags, + unsigned int useroffset, unsigned int usersize, + void (*ctor)(void *)) +{ + struct kmem_cache *res; + res = __KMEM_CACHE_SETUP(s, name, size, flags, + .align = align, + .ctor = ctor, + .useroffset = useroffset, + .usersize = usersize); + if (IS_ERR(res)) + return PTR_ERR(res); + return 0; +} + +static inline int +kmem_cache_setup(struct kmem_cache *s, + const char *name, unsigned int size, + unsigned int align, slab_flags_t flags, + void (*ctor)(void *)) +{ + struct kmem_cache *res; + res = __KMEM_CACHE_SETUP(s, name, size, flags, + .align = align, + .ctor = ctor); + if (IS_ERR(res)) + return PTR_ERR(res); + return 0; +} + +#define KMEM_CACHE_SETUP(s, __struct, __flags) \ + __KMEM_CACHE_SETUP((s), #__struct, sizeof(struct __struct), (__flags), \ + .align = __alignof__(struct __struct)) + +#define KMEM_CACHE_SETUP_USERCOPY(s, __struct, __flags, __field) \ + __KMEM_CACHE_SETUP((s), #__struct, sizeof(struct __struct), (__flags), \ + .align = __alignof__(struct __struct), \ + .useroffset = offsetof(struct __struct, __field), \ + .usersize = sizeof_field(struct __struct, __field)) + +#endif diff --git a/include/linux/slab.h b/include/linux/slab.h index 2482992248dc..f16c784148b4 100644 --- a/include/linux/slab.h +++ b/include/linux/slab.h @@ -261,11 +261,17 @@ enum _slab_flag_bits { struct list_lru; struct mem_cgroup; +struct kmem_cache_opaque; /* * struct kmem_cache related prototypes */ bool slab_is_available(void); +static inline struct kmem_cache *to_kmem_cache(struct kmem_cache_opaque *p) +{ + return (struct kmem_cache *)p; +} + /** * struct kmem_cache_args - Less common arguments for kmem_cache_create() * @@ -366,6 +372,7 @@ struct kmem_cache_args { * %0 means no sheaves will be created. */ unsigned int sheaf_capacity; + struct kmem_cache *preallocated; }; struct kmem_cache *__kmem_cache_create_args(const char *name, diff --git a/mm/kmem_cache_size.c b/mm/kmem_cache_size.c new file mode 100644 index 000000000000..1ddbfa41a507 --- /dev/null +++ b/mm/kmem_cache_size.c @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Generate definitions needed by the preprocessor. + * This code generates raw asm output which is post-processed + * to extract and format the required data. + */ + +#define COMPILE_OFFSETS +#include +#include "slab.h" + +int main(void) +{ + /* The constants to put into include/generated/kmem_cache_size.h */ + DEFINE(KMEM_CACHE_SIZE, sizeof(struct kmem_cache)); + DEFINE(KMEM_CACHE_ALIGN, __alignof(struct kmem_cache)); + /* End of constants */ + + return 0; +} diff --git a/mm/slab_common.c b/mm/slab_common.c index eed7ea556cb1..81a413b44afb 100644 --- a/mm/slab_common.c +++ b/mm/slab_common.c @@ -224,33 +224,30 @@ static struct kmem_cache *create_cache(const char *name, struct kmem_cache_args *args, slab_flags_t flags) { - struct kmem_cache *s; + struct kmem_cache *s = args->preallocated; int err; /* If a custom freelist pointer is requested make sure it's sane. */ - err = -EINVAL; if (args->use_freeptr_offset && (args->freeptr_offset >= object_size || !(flags & SLAB_TYPESAFE_BY_RCU) || !IS_ALIGNED(args->freeptr_offset, __alignof__(freeptr_t)))) - goto out; + return ERR_PTR(-EINVAL); - err = -ENOMEM; - s = kmem_cache_zalloc(kmem_cache, GFP_KERNEL); - if (!s) - goto out; + if (!s) { + s = kmem_cache_zalloc(kmem_cache, GFP_KERNEL); + if (!s) + return ERR_PTR(-ENOMEM); + } err = do_kmem_cache_create(s, name, object_size, args, flags); - if (err) - goto out_free_cache; - + if (unlikely(err)) { + if (!args->preallocated) + kmem_cache_free(kmem_cache, s); + return ERR_PTR(err); + } s->refcount = 1; list_add(&s->list, &slab_caches); return s; - -out_free_cache: - kmem_cache_free(kmem_cache, s); -out: - return ERR_PTR(err); } /** @@ -324,6 +321,9 @@ struct kmem_cache *__kmem_cache_create_args(const char *name, object_size - args->usersize < args->useroffset)) args->usersize = args->useroffset = 0; + if (args->preallocated) + flags |= SLAB_NO_MERGE; + if (!args->usersize && !args->sheaf_capacity) s = __kmem_cache_alias(name, object_size, args->align, flags, args->ctor); diff --git a/mm/slub.c b/mm/slub.c index 861592ac5425..41fe79b3f055 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -47,6 +47,7 @@ #include #include #include +#include #include #include "internal.h" @@ -8491,6 +8492,12 @@ void __init kmem_cache_init(void) boot_kmem_cache_node; int node; + /* verify that kmem_cache_opaque is correct */ + BUILD_BUG_ON(sizeof(struct kmem_cache) != + sizeof(struct kmem_cache_opaque)); + BUILD_BUG_ON(__alignof(struct kmem_cache) != + __alignof(struct kmem_cache_opaque)); + if (debug_guardpage_minorder()) slub_max_order = 0; -- 2.47.3 We need to make sure that instance in a module will get to slab_kmem_cache_release() before the module data gets freed. That's only a problem on sysfs setups - otherwise it'll definitely be finished before kmem_cache_destroy() returns. Note that modules themselves have sysfs-exposed attributes, so a similar problem already exists there. That's dealt with by having mod_sysfs_teardown() wait for refcount of module->mkobj.kobj reaching zero. Let's make use of that - have static-duration-in-module kmem_cache instances grab a reference to that kobject upon setup and drop it in the end of slab_kmem_cache_release(). Let setup helpers store the kobjetct to be pinned in kmem_cache_args->owner (for preallocated; if somebody manually sets it for non-preallocated case, it'll be ignored). That would be &THIS_MODULE->mkobj.kobj for a module and NULL in built-in. If sysfs is enabled and we are dealing with preallocated instance, let create_cache() grab and stash that reference in kmem_cache->owner and let slab_kmem_cache_release() drop it instead of freeing kmem_cache instance. Signed-off-by: Al Viro --- include/linux/slab-static.h | 12 ++++++++---- include/linux/slab.h | 4 ++++ mm/slab.h | 1 + mm/slab_common.c | 16 ++++++++++++++-- 4 files changed, 27 insertions(+), 6 deletions(-) diff --git a/include/linux/slab-static.h b/include/linux/slab-static.h index 47b2220b4988..16d1564b4a4b 100644 --- a/include/linux/slab-static.h +++ b/include/linux/slab-static.h @@ -2,10 +2,7 @@ #ifndef _LINUX_SLAB_STATIC_H #define _LINUX_SLAB_STATIC_H -#ifdef MODULE -#error "can't use that in modules" -#endif - +#include #include /* same size and alignment as struct kmem_cache: */ @@ -13,9 +10,16 @@ struct kmem_cache_opaque { unsigned char opaque[KMEM_CACHE_SIZE]; } __aligned(KMEM_CACHE_ALIGN); +#ifdef MODULE +#define THIS_MODULE_KOBJ &THIS_MODULE->mkobj.kobj +#else +#define THIS_MODULE_KOBJ NULL +#endif + #define __KMEM_CACHE_SETUP(cache, name, size, flags, ...) \ __kmem_cache_create_args((name), (size), \ &(struct kmem_cache_args) { \ + .owner = THIS_MODULE_KOBJ, \ .preallocated = (cache), \ __VA_ARGS__}, (flags)) diff --git a/include/linux/slab.h b/include/linux/slab.h index f16c784148b4..dc1aeb14a12b 100644 --- a/include/linux/slab.h +++ b/include/linux/slab.h @@ -60,6 +60,7 @@ enum _slab_flag_bits { #ifdef CONFIG_SLAB_OBJ_EXT _SLAB_NO_OBJ_EXT, #endif + _SLAB_PREALLOCATED, _SLAB_FLAGS_LAST_BIT }; @@ -244,6 +245,8 @@ enum _slab_flag_bits { #define SLAB_NO_OBJ_EXT __SLAB_FLAG_UNUSED #endif +#define SLAB_PREALLOCATED __SLAB_FLAG_BIT(_SLAB_PREALLOCATED) + /* * ZERO_SIZE_PTR will be returned for zero sized kmalloc requests. * @@ -373,6 +376,7 @@ struct kmem_cache_args { */ unsigned int sheaf_capacity; struct kmem_cache *preallocated; + struct kobject *owner; }; struct kmem_cache *__kmem_cache_create_args(const char *name, diff --git a/mm/slab.h b/mm/slab.h index e767aa7e91b0..9ff9a0a3b164 100644 --- a/mm/slab.h +++ b/mm/slab.h @@ -249,6 +249,7 @@ struct kmem_cache { struct list_head list; /* List of slab caches */ #ifdef CONFIG_SYSFS struct kobject kobj; /* For sysfs */ + struct kobject *owner; /* keep that pinned while alive */ #endif #ifdef CONFIG_SLAB_FREELIST_HARDENED unsigned long random; diff --git a/mm/slab_common.c b/mm/slab_common.c index 81a413b44afb..a854e6872acd 100644 --- a/mm/slab_common.c +++ b/mm/slab_common.c @@ -245,6 +245,12 @@ static struct kmem_cache *create_cache(const char *name, kmem_cache_free(kmem_cache, s); return ERR_PTR(err); } +#ifdef CONFIG_SYSFS + if (flags & SLAB_PREALLOCATED) { + s->owner = args->owner; + kobject_get(s->owner); + } +#endif s->refcount = 1; list_add(&s->list, &slab_caches); return s; @@ -322,7 +328,7 @@ struct kmem_cache *__kmem_cache_create_args(const char *name, args->usersize = args->useroffset = 0; if (args->preallocated) - flags |= SLAB_NO_MERGE; + flags |= SLAB_NO_MERGE | SLAB_PREALLOCATED; if (!args->usersize && !args->sheaf_capacity) s = __kmem_cache_alias(name, object_size, args->align, flags, @@ -481,7 +487,13 @@ void slab_kmem_cache_release(struct kmem_cache *s) { __kmem_cache_release(s); kfree_const(s->name); - kmem_cache_free(kmem_cache, s); + if (!(s->flags & SLAB_PREALLOCATED)) { + kmem_cache_free(kmem_cache, s); + return; + } +#ifdef CONFIG_SYSFS + kobject_put(s->owner); +#endif } void kmem_cache_destroy(struct kmem_cache *s) -- 2.47.3 Signed-off-by: Al Viro --- fs/namespace.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/fs/namespace.c b/fs/namespace.c index c58674a20cad..9a9882df463d 100644 --- a/fs/namespace.c +++ b/fs/namespace.c @@ -34,6 +34,7 @@ #include #include #include +#include #include "pnode.h" #include "internal.h" @@ -85,7 +86,8 @@ static u64 mnt_id_ctr = MNT_UNIQUE_ID_OFFSET; static struct hlist_head *mount_hashtable __ro_after_init; static struct hlist_head *mountpoint_hashtable __ro_after_init; -static struct kmem_cache *mnt_cache __ro_after_init; +static struct kmem_cache_opaque __mnt_cache; +#define mnt_cache to_kmem_cache(&__mnt_cache) static DECLARE_RWSEM(namespace_sem); static HLIST_HEAD(unmounted); /* protected by namespace_sem */ static LIST_HEAD(ex_mountpoints); /* protected by namespace_sem */ @@ -5997,7 +5999,7 @@ void __init mnt_init(void) { int err; - mnt_cache = kmem_cache_create("mnt_cache", sizeof(struct mount), + kmem_cache_setup(mnt_cache, "mnt_cache", sizeof(struct mount), 0, SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_ACCOUNT, NULL); mount_hashtable = alloc_large_system_hash("Mount-cache", -- 2.47.3 ... and make is SLAB_PANIC instead of simulating it with BUG_ON() - the boot is not going to get to kernel threads, nevermind userland... Signed-off-by: Al Viro --- kernel/fork.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/kernel/fork.c b/kernel/fork.c index b1f3915d5f8e..ddd2558f9431 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -108,6 +108,7 @@ #include #include #include +#include #include #include @@ -422,7 +423,8 @@ static void free_thread_stack(struct task_struct *tsk) #else /* !(THREAD_SIZE >= PAGE_SIZE) */ -static struct kmem_cache *thread_stack_cache; +static struct kmem_cache_opaque __thread_stack_cache; +#define thread_stack_cache to_kmem_cache(&__thread_stack_cache) static void thread_stack_free_rcu(struct rcu_head *rh) { @@ -453,10 +455,10 @@ static void free_thread_stack(struct task_struct *tsk) void thread_stack_cache_init(void) { - thread_stack_cache = kmem_cache_create_usercopy("thread_stack", - THREAD_SIZE, THREAD_SIZE, 0, 0, + kmem_cache_setup_usercopy(thread_stack_cache, "thread_stack", + THREAD_SIZE, THREAD_SIZE, + SLAB_PANIC, 0, THREAD_SIZE, NULL); - BUG_ON(thread_stack_cache == NULL); } #endif /* THREAD_SIZE >= PAGE_SIZE */ -- 2.47.3 Signed-off-by: Al Viro --- kernel/fork.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/kernel/fork.c b/kernel/fork.c index ddd2558f9431..23ed80d0d6d0 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -465,7 +465,8 @@ void thread_stack_cache_init(void) #endif /* CONFIG_VMAP_STACK */ /* SLAB cache for signal_struct structures (tsk->signal) */ -static struct kmem_cache *signal_cachep; +static struct kmem_cache_opaque signal_cache; +#define signal_cachep to_kmem_cache(&signal_cache) /* SLAB cache for sighand_struct structures (tsk->sighand) */ struct kmem_cache *sighand_cachep; @@ -3024,7 +3025,7 @@ void __init proc_caches_init(void) sizeof(struct sighand_struct), 0, SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_TYPESAFE_BY_RCU| SLAB_ACCOUNT, sighand_ctor); - signal_cachep = kmem_cache_create("signal_cache", + kmem_cache_setup(signal_cachep, "signal_cache", sizeof(struct signal_struct), 0, SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_ACCOUNT, NULL); -- 2.47.3 Signed-off-by: Al Viro --- fs/buffer.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/fs/buffer.c b/fs/buffer.c index 838c0c571022..c8ec1b440880 100644 --- a/fs/buffer.c +++ b/fs/buffer.c @@ -50,6 +50,7 @@ #include #include #include +#include #include "internal.h" @@ -2990,7 +2991,8 @@ EXPORT_SYMBOL(try_to_free_buffers); /* * Buffer-head allocation */ -static struct kmem_cache *bh_cachep __ro_after_init; +static struct kmem_cache_opaque bh_cache; +#define bh_cachep to_kmem_cache(&bh_cache) /* * Once the number of bh's in the machine exceeds this level, we start @@ -3149,7 +3151,7 @@ void __init buffer_init(void) unsigned long nrpages; int ret; - bh_cachep = KMEM_CACHE(buffer_head, + KMEM_CACHE_SETUP(bh_cachep, buffer_head, SLAB_RECLAIM_ACCOUNT|SLAB_PANIC); /* * Limit the bh occupancy to 10% of ZONE_NORMAL -- 2.47.3 No need to bother with runtime_const() for it anymore... Signed-off-by: Al Viro --- fs/dcache.c | 8 ++++---- include/asm-generic/vmlinux.lds.h | 3 +-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/fs/dcache.c b/fs/dcache.c index dc2fff4811d1..43d3b4fbedcc 100644 --- a/fs/dcache.c +++ b/fs/dcache.c @@ -32,6 +32,7 @@ #include #include #include +#include #include "internal.h" #include "mount.h" @@ -86,8 +87,8 @@ __cacheline_aligned_in_smp DEFINE_SEQLOCK(rename_lock); EXPORT_SYMBOL(rename_lock); -static struct kmem_cache *__dentry_cache __ro_after_init; -#define dentry_cache runtime_const_ptr(__dentry_cache) +static struct kmem_cache_opaque __dentry_cache; +#define dentry_cache to_kmem_cache(&__dentry_cache) const struct qstr empty_name = QSTR_INIT("", 0); EXPORT_SYMBOL(empty_name); @@ -3265,10 +3266,9 @@ static void __init dcache_init(void) * but it is probably not worth it because of the cache nature * of the dcache. */ - __dentry_cache = KMEM_CACHE_USERCOPY(dentry, + KMEM_CACHE_SETUP_USERCOPY(dentry_cache, dentry, SLAB_RECLAIM_ACCOUNT|SLAB_PANIC|SLAB_ACCOUNT, d_shortname.string); - runtime_const_init(ptr, __dentry_cache); /* Hash may have been set up in dcache_init_early */ if (!hashdist) diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h index 8ca130af301f..6997f6301260 100644 --- a/include/asm-generic/vmlinux.lds.h +++ b/include/asm-generic/vmlinux.lds.h @@ -971,8 +971,7 @@ #define RUNTIME_CONST_VARIABLES \ RUNTIME_CONST(shift, d_hash_shift) \ - RUNTIME_CONST(ptr, dentry_hashtable) \ - RUNTIME_CONST(ptr, __dentry_cache) + RUNTIME_CONST(ptr, dentry_hashtable) /* Alignment must be consistent with (kunit_suite *) in include/kunit/test.h */ #define KUNIT_TABLE() \ -- 2.47.3 Signed-off-by: Al Viro --- include/linux/fdtable.h | 3 ++- kernel/fork.c | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/include/linux/fdtable.h b/include/linux/fdtable.h index c45306a9f007..f2d553f99c58 100644 --- a/include/linux/fdtable.h +++ b/include/linux/fdtable.h @@ -113,6 +113,7 @@ int iterate_fd(struct files_struct *, unsigned, extern int close_fd(unsigned int fd); extern struct file *file_close_fd(unsigned int fd); -extern struct kmem_cache *files_cachep; +extern struct kmem_cache_opaque files_cache; +#define files_cachep to_kmem_cache(&files_cache) #endif /* __LINUX_FDTABLE_H */ diff --git a/kernel/fork.c b/kernel/fork.c index 23ed80d0d6d0..8c4d9a81ef42 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -472,7 +472,7 @@ static struct kmem_cache_opaque signal_cache; struct kmem_cache *sighand_cachep; /* SLAB cache for files_struct structures (tsk->files) */ -struct kmem_cache *files_cachep; +struct kmem_cache_opaque files_cache; /* SLAB cache for fs_struct structures (tsk->fs) */ struct kmem_cache *fs_cachep; @@ -3029,7 +3029,7 @@ void __init proc_caches_init(void) sizeof(struct signal_struct), 0, SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_ACCOUNT, NULL); - files_cachep = kmem_cache_create("files_cache", + kmem_cache_setup(files_cachep, "files_cache", sizeof(struct files_struct), 0, SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_ACCOUNT, NULL); -- 2.47.3 As much as I hate it, the name "filp" is a part of userland ABI at this point - scripts grepping for it in /proc/slabinfo do exist ;-/ Signed-off-by: Al Viro --- fs/file_table.c | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/fs/file_table.c b/fs/file_table.c index cd4a3db4659a..18a992b40109 100644 --- a/fs/file_table.c +++ b/fs/file_table.c @@ -27,6 +27,7 @@ #include #include #include +#include #include @@ -38,8 +39,10 @@ static struct files_stat_struct files_stat = { }; /* SLAB cache for file structures */ -static struct kmem_cache *filp_cachep __ro_after_init; -static struct kmem_cache *bfilp_cachep __ro_after_init; +static struct kmem_cache_opaque file_cache; +static struct kmem_cache_opaque backing_file_cache; +#define filp_cachep to_kmem_cache(&file_cache) +#define bfilp_cachep to_kmem_cache(&backing_file_cache) static struct percpu_counter nr_files __cacheline_aligned_in_smp; @@ -587,19 +590,20 @@ void fput_close(struct file *file) void __init files_init(void) { - struct kmem_cache_args args = { - .use_freeptr_offset = true, - .freeptr_offset = offsetof(struct file, f_freeptr), - }; - - filp_cachep = kmem_cache_create("filp", sizeof(struct file), &args, - SLAB_HWCACHE_ALIGN | SLAB_PANIC | - SLAB_ACCOUNT | SLAB_TYPESAFE_BY_RCU); + __KMEM_CACHE_SETUP(filp_cachep, "filp", sizeof(struct file), + SLAB_HWCACHE_ALIGN | SLAB_PANIC | + SLAB_ACCOUNT | SLAB_TYPESAFE_BY_RCU, + .use_freeptr_offset = true, + .freeptr_offset = offsetof(struct file, + f_freeptr)); + + __KMEM_CACHE_SETUP(bfilp_cachep, "bfilp", sizeof(struct backing_file), + SLAB_HWCACHE_ALIGN | SLAB_PANIC | + SLAB_ACCOUNT | SLAB_TYPESAFE_BY_RCU, + .use_freeptr_offset = true, + .freeptr_offset = offsetof(struct backing_file, + bf_freeptr)); - args.freeptr_offset = offsetof(struct backing_file, bf_freeptr); - bfilp_cachep = kmem_cache_create("bfilp", sizeof(struct backing_file), - &args, SLAB_HWCACHE_ALIGN | SLAB_PANIC | - SLAB_ACCOUNT | SLAB_TYPESAFE_BY_RCU); percpu_counter_init(&nr_files, 0, GFP_KERNEL); } -- 2.47.3 Signed-off-by: Al Viro --- include/linux/signal.h | 3 ++- kernel/fork.c | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/include/linux/signal.h b/include/linux/signal.h index f19816832f05..a0c7fee8b22a 100644 --- a/include/linux/signal.h +++ b/include/linux/signal.h @@ -323,7 +323,8 @@ static inline void disallow_signal(int sig) kernel_sigaction(sig, SIG_IGN); } -extern struct kmem_cache *sighand_cachep; +extern struct kmem_cache_opaque sighand_cache; +#define sighand_cachep to_kmem_cache(&sighand_cache) extern bool unhandled_signal(struct task_struct *tsk, int sig); diff --git a/kernel/fork.c b/kernel/fork.c index 8c4d9a81ef42..d5b7e4d51596 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -469,7 +469,7 @@ static struct kmem_cache_opaque signal_cache; #define signal_cachep to_kmem_cache(&signal_cache) /* SLAB cache for sighand_struct structures (tsk->sighand) */ -struct kmem_cache *sighand_cachep; +struct kmem_cache_opaque sighand_cache; /* SLAB cache for files_struct structures (tsk->files) */ struct kmem_cache_opaque files_cache; @@ -3021,7 +3021,7 @@ void __init mm_cache_init(void) void __init proc_caches_init(void) { - sighand_cachep = kmem_cache_create("sighand_cache", + kmem_cache_setup(sighand_cachep, "sighand_cache", sizeof(struct sighand_struct), 0, SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_TYPESAFE_BY_RCU| SLAB_ACCOUNT, sighand_ctor); -- 2.47.3 Signed-off-by: Al Viro --- kernel/fork.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/kernel/fork.c b/kernel/fork.c index d5b7e4d51596..f83ca2f5826f 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -478,7 +478,8 @@ struct kmem_cache_opaque files_cache; struct kmem_cache *fs_cachep; /* SLAB cache for mm_struct structures (tsk->mm) */ -static struct kmem_cache *mm_cachep; +static struct kmem_cache_opaque mm_cache; +#define mm_cachep to_kmem_cache(&mm_cache) static void account_kernel_stack(struct task_struct *tsk, int account) { @@ -3011,7 +3012,7 @@ void __init mm_cache_init(void) */ mm_size = sizeof(struct mm_struct) + cpumask_size() + mm_cid_size(); - mm_cachep = kmem_cache_create_usercopy("mm_struct", + kmem_cache_setup_usercopy(mm_cachep, "mm_struct", mm_size, ARCH_MIN_MMSTRUCT_ALIGN, SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_ACCOUNT, offsetof(struct mm_struct, saved_auxv), -- 2.47.3 Signed-off-by: Al Viro --- kernel/fork.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/kernel/fork.c b/kernel/fork.c index f83ca2f5826f..8f0dfefd82f0 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -178,7 +178,8 @@ void __weak arch_release_task_struct(struct task_struct *tsk) { } -static struct kmem_cache *task_struct_cachep; +static struct kmem_cache_opaque task_struct_cache; +#define task_struct_cachep to_kmem_cache(&task_struct_cache) static inline struct task_struct *alloc_task_struct_node(int node) { @@ -860,7 +861,7 @@ void __init fork_init(void) /* create a slab on which task_structs can be allocated */ task_struct_whitelist(&useroffset, &usersize); - task_struct_cachep = kmem_cache_create_usercopy("task_struct", + kmem_cache_setup_usercopy(task_struct_cachep, "task_struct", arch_task_struct_size, align, SLAB_PANIC|SLAB_ACCOUNT, useroffset, usersize, NULL); -- 2.47.3 Signed-off-by: Al Viro --- include/linux/fs_struct.h | 3 ++- kernel/fork.c | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/include/linux/fs_struct.h b/include/linux/fs_struct.h index 0070764b790a..e8c9fac5b7b7 100644 --- a/include/linux/fs_struct.h +++ b/include/linux/fs_struct.h @@ -15,7 +15,8 @@ struct fs_struct { struct path root, pwd; } __randomize_layout; -extern struct kmem_cache *fs_cachep; +extern struct kmem_cache_opaque fs_struct_cache; +#define fs_cachep to_kmem_cache(&fs_struct_cache) extern void exit_fs(struct task_struct *); extern void set_fs_root(struct fs_struct *, const struct path *); diff --git a/kernel/fork.c b/kernel/fork.c index 8f0dfefd82f0..7262abd0d2a4 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -476,7 +476,7 @@ struct kmem_cache_opaque sighand_cache; struct kmem_cache_opaque files_cache; /* SLAB cache for fs_struct structures (tsk->fs) */ -struct kmem_cache *fs_cachep; +struct kmem_cache_opaque fs_struct_cache; /* SLAB cache for mm_struct structures (tsk->mm) */ static struct kmem_cache_opaque mm_cache; @@ -3035,7 +3035,7 @@ void __init proc_caches_init(void) sizeof(struct files_struct), 0, SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_ACCOUNT, NULL); - fs_cachep = kmem_cache_create("fs_cache", + kmem_cache_setup(fs_cachep, "fs_cache", sizeof(struct fs_struct), 0, SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_ACCOUNT, NULL); -- 2.47.3 Signed-off-by: Al Viro --- fs/inode.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/fs/inode.c b/fs/inode.c index 521383223d8a..7c212696ba67 100644 --- a/fs/inode.c +++ b/fs/inode.c @@ -23,6 +23,7 @@ #include #include #include +#include #include #define CREATE_TRACE_POINTS #include @@ -76,7 +77,8 @@ EXPORT_SYMBOL(empty_aops); static DEFINE_PER_CPU(unsigned long, nr_inodes); static DEFINE_PER_CPU(unsigned long, nr_unused); -static struct kmem_cache *inode_cachep __ro_after_init; +static struct kmem_cache_opaque inode_cache; +#define inode_cachep to_kmem_cache(&inode_cache) static long get_nr_inodes(void) { @@ -2564,7 +2566,7 @@ void __init inode_init_early(void) void __init inode_init(void) { /* inode slab cache */ - inode_cachep = kmem_cache_create("inode_cache", + kmem_cache_setup(inode_cachep, "inode_cache", sizeof(struct inode), 0, (SLAB_RECLAIM_ACCOUNT|SLAB_PANIC| -- 2.47.3 A modular example I used for testing... Signed-off-by: Al Viro --- fs/ufs/super.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/fs/ufs/super.c b/fs/ufs/super.c index 6e4585169f94..440229a5b6c9 100644 --- a/fs/ufs/super.c +++ b/fs/ufs/super.c @@ -90,6 +90,7 @@ #include #include #include +#include #include "ufs_fs.h" #include "ufs.h" @@ -1354,7 +1355,8 @@ static int ufs_statfs(struct dentry *dentry, struct kstatfs *buf) return 0; } -static struct kmem_cache * ufs_inode_cachep; +static struct kmem_cache_opaque ufs_inode_cache; +#define ufs_inode_cachep to_kmem_cache(&ufs_inode_cache) static struct inode *ufs_alloc_inode(struct super_block *sb) { @@ -1384,16 +1386,13 @@ static void init_once(void *foo) static int __init init_inodecache(void) { - ufs_inode_cachep = kmem_cache_create_usercopy("ufs_inode_cache", + return kmem_cache_setup_usercopy(ufs_inode_cachep, "ufs_inode_cache", sizeof(struct ufs_inode_info), 0, (SLAB_RECLAIM_ACCOUNT | SLAB_ACCOUNT), offsetof(struct ufs_inode_info, i_u1.i_symlink), sizeof_field(struct ufs_inode_info, i_u1.i_symlink), init_once); - if (ufs_inode_cachep == NULL) - return -ENOMEM; - return 0; } static void destroy_inodecache(void) -- 2.47.3