zsmalloc handle is an unsigned long that holds an address of a handle_cachep object, so zsmalloc and its users have to cast it back and forth. Introduce struct zs_handle and switch zsmalloc, zram and zswap to struct zs_handle pointers. Functions that only read the handle take a const pointer. This is a preparation for embedding struct zs_handle into zram and zswap entries. No functional change. Reviewed-by: Yosry Ahmed Reviewed-by: Nhat Pham Signed-off-by: Sergey Senozhatsky --- drivers/block/zram/zram_drv.c | 28 +++++------ drivers/block/zram/zram_drv.h | 2 +- include/linux/zsmalloc.h | 13 +++--- mm/zsmalloc.c | 87 +++++++++++++++++++---------------- mm/zswap.c | 8 ++-- 5 files changed, 74 insertions(+), 64 deletions(-) diff --git a/drivers/block/zram/zram_drv.c b/drivers/block/zram/zram_drv.c index 5c13addd22f6..2c2e11b2b663 100644 --- a/drivers/block/zram/zram_drv.c +++ b/drivers/block/zram/zram_drv.c @@ -2030,7 +2030,7 @@ static bool zram_meta_alloc(struct zram *zram, u64 disksize) static void slot_free(struct zram *zram, unsigned long index) { - unsigned long handle; + struct zs_handle *handle; #ifdef CONFIG_ZRAM_TRACK_ENTRY_ACTIME zram->table[index].attr.ac_time = 0; @@ -2096,7 +2096,7 @@ static int read_incompressible_page(struct zram *zram, struct page *page, unsigned long index) { struct scatterlist sg[2]; - unsigned long handle; + struct zs_handle *handle; void *src, *dst; handle = zram->table[index].handle; @@ -2118,7 +2118,7 @@ static int read_compressed_page(struct zram *zram, struct page *page, enum zcomp_dstrm_mode mode = ZCOMP_DSTRM_EXCLUSIVE; struct zcomp_dstrm *zstrm; struct scatterlist sg[2]; - unsigned long handle; + struct zs_handle *handle; unsigned int size; void *dst; int ret, prio; @@ -2146,7 +2146,7 @@ static int read_from_zspool_raw(struct zram *zram, struct page *page, unsigned long index) { struct scatterlist sg[2]; - unsigned long handle; + struct zs_handle *handle; unsigned int size; void *dst; @@ -2257,14 +2257,14 @@ static int write_same_filled_page(struct zram *zram, unsigned long fill, static int write_incompressible_page(struct zram *zram, struct page *page, unsigned long index) { - unsigned long handle; + struct zs_handle *handle; void *src; handle = zs_malloc(zram->mem_pool, PAGE_SIZE, GFP_NOIO | __GFP_NOWARN | __GFP_HIGHMEM | __GFP_MOVABLE, page_to_nid(page)); - if (IS_ERR_VALUE(handle)) - return PTR_ERR((void *)handle); + if (IS_ERR(handle)) + return PTR_ERR(handle); if (!zram_can_store_page(zram)) { zs_free(zram->mem_pool, handle); @@ -2295,7 +2295,7 @@ static int zram_write_page(struct zram *zram, struct page *page, unsigned long index) { int ret = 0; - unsigned long handle; + struct zs_handle *handle; unsigned int comp_len; void *mem; struct zcomp_cstrm *zstrm; @@ -2328,9 +2328,9 @@ static int zram_write_page(struct zram *zram, struct page *page, handle = zs_malloc(zram->mem_pool, comp_len, GFP_NOIO | __GFP_NOWARN | __GFP_HIGHMEM | __GFP_MOVABLE, page_to_nid(page)); - if (IS_ERR_VALUE(handle)) { + if (IS_ERR(handle)) { zcomp_cstrm_put(zstrm); - return PTR_ERR((void *)handle); + return PTR_ERR(handle); } if (!zram_can_store_page(zram)) { @@ -2451,8 +2451,8 @@ static int recompress_slot(struct zram *zram, unsigned long index, u32 threshold, u32 prio) { struct zcomp_cstrm *zstrm = NULL; - unsigned long handle_old; - unsigned long handle_new; + struct zs_handle *handle_old; + struct zs_handle *handle_new; unsigned int comp_len_old; unsigned int comp_len_new; unsigned int class_index_old; @@ -2534,9 +2534,9 @@ static int recompress_slot(struct zram *zram, unsigned long index, GFP_NOIO | __GFP_NOWARN | __GFP_HIGHMEM | __GFP_MOVABLE, page_to_nid(page)); - if (IS_ERR_VALUE(handle_new)) { + if (IS_ERR(handle_new)) { zcomp_cstrm_put(zstrm); - return PTR_ERR((void *)handle_new); + return PTR_ERR(handle_new); } zs_obj_write(zram->mem_pool, handle_new, zstrm->buffer, comp_len_new); diff --git a/drivers/block/zram/zram_drv.h b/drivers/block/zram/zram_drv.h index 5416e25e64e8..5b211a9f314c 100644 --- a/drivers/block/zram/zram_drv.h +++ b/drivers/block/zram/zram_drv.h @@ -79,7 +79,7 @@ enum zram_pageflags { */ struct zram_table_entry { union { - unsigned long handle; + struct zs_handle *handle; unsigned long element; unsigned long blk_idx; }; diff --git a/include/linux/zsmalloc.h b/include/linux/zsmalloc.h index 5b7298a5026b..7fa500ae2653 100644 --- a/include/linux/zsmalloc.h +++ b/include/linux/zsmalloc.h @@ -22,14 +22,15 @@ struct zs_pool_stats { }; struct zs_pool; +struct zs_handle; struct scatterlist; struct zs_pool *zs_create_pool(const char *name); void zs_destroy_pool(struct zs_pool *pool); -unsigned long zs_malloc(struct zs_pool *pool, size_t size, gfp_t flags, - const int nid); -void zs_free(struct zs_pool *pool, unsigned long obj); +struct zs_handle *zs_malloc(struct zs_pool *pool, size_t size, gfp_t flags, + const int nid); +void zs_free(struct zs_pool *pool, struct zs_handle *handle); size_t zs_huge_class_size(struct zs_pool *pool); @@ -40,10 +41,10 @@ unsigned int zs_lookup_class_index(struct zs_pool *pool, unsigned int size); void zs_pool_stats(struct zs_pool *pool, struct zs_pool_stats *stats); -void zs_obj_read_sg_begin(struct zs_pool *pool, unsigned long handle, +void zs_obj_read_sg_begin(struct zs_pool *pool, const struct zs_handle *handle, struct scatterlist *sg, size_t mem_len); -void zs_obj_read_sg_end(struct zs_pool *pool, unsigned long handle); -void zs_obj_write(struct zs_pool *pool, unsigned long handle, +void zs_obj_read_sg_end(struct zs_pool *pool, const struct zs_handle *handle); +void zs_obj_write(struct zs_pool *pool, const struct zs_handle *handle, void *handle_mem, size_t mem_len); extern const struct movable_operations zsmalloc_mops; diff --git a/mm/zsmalloc.c b/mm/zsmalloc.c index dab71f582103..2385c1951ca7 100644 --- a/mm/zsmalloc.c +++ b/mm/zsmalloc.c @@ -55,7 +55,7 @@ */ #define ZS_ALIGN 8 -#define ZS_HANDLE_SIZE (sizeof(unsigned long)) +#define ZS_HANDLE_SIZE sizeof(struct zs_handle *) /* * Object location (, ) is encoded as @@ -264,6 +264,10 @@ struct link_free { }; }; +struct zs_handle { + unsigned long __private obj; +}; + static struct kmem_cache *handle_cachep; static struct kmem_cache *zspage_cachep; @@ -438,16 +442,16 @@ static void init_deferred_free(struct zs_pool *pool) {} static void SetZsPageMovable(struct zs_pool *pool, struct zspage *zspage) {} #endif -static unsigned long cache_alloc_handle(gfp_t gfp) +static struct zs_handle *cache_alloc_handle(gfp_t gfp) { gfp = gfp & ~(__GFP_HIGHMEM | __GFP_MOVABLE); - return (unsigned long)kmem_cache_alloc(handle_cachep, gfp); + return kmem_cache_alloc(handle_cachep, gfp); } -static void cache_free_handle(unsigned long handle) +static void cache_free_handle(struct zs_handle *handle) { - kmem_cache_free(handle_cachep, (void *)handle); + kmem_cache_free(handle_cachep, handle); } static struct zspage *cache_alloc_zspage(gfp_t gfp) @@ -466,9 +470,9 @@ static void cache_free_zspage(struct zspage *zspage) * Pairs with READ_ONCE() in handle_to_obj(): zs_free() may read the * handle locklessly, so prevent store tearing here. */ -static void record_obj(unsigned long handle, unsigned long obj) +static void record_obj(struct zs_handle *handle, unsigned long obj) { - WRITE_ONCE(*(unsigned long *)handle, obj); + WRITE_ONCE(ACCESS_PRIVATE(handle, obj), obj); } /* Protected by class->lock */ @@ -817,12 +821,12 @@ static unsigned long location_to_obj(struct zpdesc *zpdesc, unsigned int obj_idx return obj; } -static unsigned long handle_to_obj(unsigned long handle) +static unsigned long handle_to_obj(const struct zs_handle *handle) { - return READ_ONCE(*(unsigned long *)handle); + return READ_ONCE(ACCESS_PRIVATE(handle, obj)); } -static inline unsigned long obj_allocated(struct zpdesc *zpdesc, void *obj) +static inline struct zs_handle *obj_allocated(struct zpdesc *zpdesc, void *obj) { unsigned long handle; struct zspage *zspage = get_zspage(zpdesc); @@ -835,10 +839,10 @@ static inline unsigned long obj_allocated(struct zpdesc *zpdesc, void *obj) handle = *(unsigned long *)obj; if (!(handle & OBJ_ALLOCATED_TAG)) - return 0; + return NULL; /* Clear all tags before returning the handle */ - return handle & ~OBJ_TAG_MASK; + return (struct zs_handle *)(handle & ~OBJ_TAG_MASK); } static void reset_zpdesc(struct zpdesc *zpdesc) @@ -1120,7 +1124,7 @@ unsigned long zs_get_total_pages(struct zs_pool *pool) } EXPORT_SYMBOL_GPL(zs_get_total_pages); -void zs_obj_read_sg_begin(struct zs_pool *pool, unsigned long handle, +void zs_obj_read_sg_begin(struct zs_pool *pool, const struct zs_handle *handle, struct scatterlist *sg, size_t mem_len) { struct zspage *zspage; @@ -1167,7 +1171,7 @@ void zs_obj_read_sg_begin(struct zs_pool *pool, unsigned long handle, } EXPORT_SYMBOL_GPL(zs_obj_read_sg_begin); -void zs_obj_read_sg_end(struct zs_pool *pool, unsigned long handle) +void zs_obj_read_sg_end(struct zs_pool *pool, const struct zs_handle *handle) { struct zspage *zspage; struct zpdesc *zpdesc; @@ -1182,7 +1186,7 @@ void zs_obj_read_sg_end(struct zs_pool *pool, unsigned long handle) } EXPORT_SYMBOL_GPL(zs_obj_read_sg_end); -void zs_obj_write(struct zs_pool *pool, unsigned long handle, +void zs_obj_write(struct zs_pool *pool, const struct zs_handle *handle, void *handle_mem, size_t mem_len) { struct zspage *zspage; @@ -1251,7 +1255,7 @@ size_t zs_huge_class_size(struct zs_pool *pool) EXPORT_SYMBOL_GPL(zs_huge_class_size); static unsigned long obj_malloc(struct zs_pool *pool, - struct zspage *zspage, unsigned long handle) + struct zspage *zspage, struct zs_handle *handle) { int i, nr_zpdesc, offset; unsigned long obj; @@ -1278,9 +1282,10 @@ static unsigned long obj_malloc(struct zs_pool *pool, set_freeobj(zspage, link->next >> OBJ_TAG_BITS); if (likely(!ZsHugePage(zspage))) /* record handle in the header of allocated chunk */ - link->handle = handle | OBJ_ALLOCATED_TAG; + link->handle = (unsigned long)handle | OBJ_ALLOCATED_TAG; else - zspage->first_zpdesc->handle = handle | OBJ_ALLOCATED_TAG; + zspage->first_zpdesc->handle = + (unsigned long)handle | OBJ_ALLOCATED_TAG; kunmap_local(vaddr); mod_zspage_inuse(zspage, 1); @@ -1303,23 +1308,23 @@ static unsigned long obj_malloc(struct zs_pool *pool, * otherwise an ERR_PTR(). * Allocation requests with size > ZS_MAX_ALLOC_SIZE will fail. */ -unsigned long zs_malloc(struct zs_pool *pool, size_t size, gfp_t gfp, - const int nid) +struct zs_handle *zs_malloc(struct zs_pool *pool, size_t size, gfp_t gfp, + const int nid) { - unsigned long handle; + struct zs_handle *handle; struct size_class *class; int newfg; struct zspage *zspage; if (unlikely(!size)) - return (unsigned long)ERR_PTR(-EINVAL); + return ERR_PTR(-EINVAL); if (unlikely(size > ZS_MAX_ALLOC_SIZE)) - return (unsigned long)ERR_PTR(-ENOSPC); + return ERR_PTR(-ENOSPC); handle = cache_alloc_handle(gfp); if (!handle) - return (unsigned long)ERR_PTR(-ENOMEM); + return ERR_PTR(-ENOMEM); class = lookup_size_class(pool, size); @@ -1340,7 +1345,7 @@ unsigned long zs_malloc(struct zs_pool *pool, size_t size, gfp_t gfp, zspage = alloc_zspage(pool, class, gfp, nid); if (!zspage) { cache_free_handle(handle); - return (unsigned long)ERR_PTR(-ENOMEM); + return ERR_PTR(-ENOMEM); } spin_lock(&class->lock); @@ -1408,9 +1413,11 @@ static unsigned int obj_to_class_idx(unsigned long obj) * encoding is disabled), fall back to pool->lock for the lookup. */ #if ZS_OBJ_CLASS_BITS > 0 -static inline void obj_class_get_and_lock(struct zs_pool *pool, unsigned long handle, - unsigned long *objp, struct zspage **zspagep, - struct size_class **classp) +static inline void obj_class_get_and_lock(struct zs_pool *pool, + const struct zs_handle *handle, + unsigned long *objp, + struct zspage **zspagep, + struct size_class **classp) __acquires(&(*classp)->lock) { struct zpdesc *f_zpdesc; @@ -1426,9 +1433,11 @@ static inline void obj_class_get_and_lock(struct zs_pool *pool, unsigned long ha *objp = obj; } #else -static inline void obj_class_get_and_lock(struct zs_pool *pool, unsigned long handle, - unsigned long *objp, struct zspage **zspagep, - struct size_class **classp) +static inline void obj_class_get_and_lock(struct zs_pool *pool, + const struct zs_handle *handle, + unsigned long *objp, + struct zspage **zspagep, + struct size_class **classp) __acquires(&(*classp)->lock) { struct zpdesc *f_zpdesc; @@ -1445,7 +1454,7 @@ static inline void obj_class_get_and_lock(struct zs_pool *pool, unsigned long ha } #endif -void zs_free(struct zs_pool *pool, unsigned long handle) +void zs_free(struct zs_pool *pool, struct zs_handle *handle) { struct zspage *zspage; unsigned long obj; @@ -1453,7 +1462,7 @@ void zs_free(struct zs_pool *pool, unsigned long handle) int fullness; struct zspage *zspage_to_free = NULL; - if (IS_ERR_OR_NULL((void *)handle)) + if (IS_ERR_OR_NULL(handle)) return; obj_class_get_and_lock(pool, handle, &obj, &zspage, &class); @@ -1557,12 +1566,12 @@ static void zs_object_copy(struct size_class *class, unsigned long dst, * Find alloced object in zspage from index object and * return handle. */ -static unsigned long find_alloced_obj(struct size_class *class, - struct zpdesc *zpdesc, int *obj_idx) +static struct zs_handle *find_alloced_obj(struct size_class *class, + struct zpdesc *zpdesc, int *obj_idx) { unsigned int offset; int index = *obj_idx; - unsigned long handle = 0; + struct zs_handle *handle = NULL; void *addr = kmap_local_zpdesc(zpdesc); offset = get_first_obj_offset(zpdesc); @@ -1588,7 +1597,7 @@ static void migrate_zspage(struct zs_pool *pool, struct zspage *src_zspage, struct zspage *dst_zspage) { unsigned long used_obj, free_obj; - unsigned long handle; + struct zs_handle *handle; int obj_idx = 0; struct zpdesc *s_zpdesc = get_first_zpdesc(src_zspage); struct size_class *class = pool->size_class[src_zspage->class]; @@ -1759,7 +1768,7 @@ static int zs_page_migrate(struct page *newpage, struct page *page, struct zpdesc *zpdesc = page_zpdesc(page); void *s_addr, *d_addr, *addr; unsigned int offset; - unsigned long handle; + struct zs_handle *handle; unsigned long old_obj, new_obj; unsigned int obj_idx; @@ -2290,7 +2299,7 @@ static void zs_destroy_caches(void) static int __init zs_init_caches(void) { - handle_cachep = kmem_cache_create("zs_handle", ZS_HANDLE_SIZE, + handle_cachep = kmem_cache_create("zs_handle", sizeof(struct zs_handle), 0, 0, NULL); zspage_cachep = kmem_cache_create("zspage", sizeof(struct zspage), 0, 0, NULL); diff --git a/mm/zswap.c b/mm/zswap.c index 37d217860a6b..1160bd34e289 100644 --- a/mm/zswap.c +++ b/mm/zswap.c @@ -216,7 +216,7 @@ struct zswap_entry { unsigned int length; bool referenced; u8 pool_idx; - unsigned long handle; + struct zs_handle *handle; struct obj_cgroup *objcg; struct list_head lru; }; @@ -889,7 +889,7 @@ static bool zswap_compress(struct folio *folio, long index, struct scatterlist input, output; int comp_ret = 0, alloc_ret = 0; unsigned int dlen = PAGE_SIZE; - unsigned long handle; + struct zs_handle *handle; gfp_t gfp; u8 *dst; bool mapped = false; @@ -950,8 +950,8 @@ static bool zswap_compress(struct folio *folio, long index, gfp = GFP_NOWAIT | __GFP_NORETRY | __GFP_HIGHMEM | __GFP_MOVABLE; handle = zs_malloc(pool->zs_pool, dlen, gfp, folio_nid(folio)); - if (IS_ERR_VALUE(handle)) { - alloc_ret = PTR_ERR((void *)handle); + if (IS_ERR(handle)) { + alloc_ret = PTR_ERR(handle); goto unlock; } -- 2.56.0.542.g3ef89b7020-goog