When storing a folio in zswap, save the folio's node id in the zswap entry instead of relying on the zswap_entry being allocated on the same node. This enables optimizations such as batch allocation of zswap entries for large folios using kmem_cache_alloc_bulk() that does not support specifying a node id. Accordingly, replace the SLUB allocation in zswap_entry_cache_alloc() to be node-agnostic. The size of the zswap_entry remains unchanged as 'referenced' is lumped in with the 'length' (as it doesn't need a full unsigned int anyway). A build time assertion checks for the zswap_entry's 'length' bitfield being large enough to represent PAGE_SIZE, which can be 256KB on some architectures. Suggested-by: Nhat Pham Suggested-by: Yosry Ahmed Signed-off-by: Kanchana P Sridhar Signed-off-by: Kanchana P. Sridhar --- mm/zswap.c | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/mm/zswap.c b/mm/zswap.c index 9a323862bd59..78d9694f75b5 100644 --- a/mm/zswap.c +++ b/mm/zswap.c @@ -181,6 +181,7 @@ struct zswap_pool { */ #define ZSWAP_FIRST_POOL_ID 1 #define ZSWAP_MAX_POOL_ID U8_MAX +#define ZSWAP_MAX_COMPR_LENGTH_SHIFT 19U static DEFINE_XARRAY_FLAGS(zswap_pools, XA_FLAGS_ALLOC1 | XA_FLAGS_LOCK_BH); static struct zswap_pool __rcu *zswap_current_pool; @@ -207,15 +208,19 @@ static struct shrinker *zswap_shrinker; * logic if referenced is unset. See comments in the shrinker * section for context. * pool_idx - id of the zswap_pool that the entry's data is in. + * nid - NUMA node id of the page for which this is the zswap entry. * handle - zsmalloc allocation handle that stores the compressed page data * objcg - the obj_cgroup that the compressed memory is charged to * lru - handle to the pool's lru used to evict pages. */ struct zswap_entry { swp_entry_t swpentry; - unsigned int length; - bool referenced; - u8 pool_idx; + struct { + unsigned int length:ZSWAP_MAX_COMPR_LENGTH_SHIFT; + bool referenced:1U; + u8 pool_idx; + }; + int nid; struct zs_handle handle; struct obj_cgroup *objcg; struct list_head lru; @@ -680,14 +685,8 @@ static int zswap_enabled_param_set(const char *val, * lru functions **********************************/ -static inline int entry_to_nid(struct zswap_entry *entry) -{ - return page_to_nid(virt_to_page(entry)); -} - static void zswap_lru_add(struct zswap_entry *entry) { - int nid = entry_to_nid(entry); struct mem_cgroup *memcg; /* @@ -704,19 +703,18 @@ static void zswap_lru_add(struct zswap_entry *entry) rcu_read_lock(); memcg = obj_cgroup_memcg(entry->objcg); /* will always succeed */ - list_lru_add(&zswap_list_lru, &entry->lru, nid, memcg); + list_lru_add(&zswap_list_lru, &entry->lru, entry->nid, memcg); rcu_read_unlock(); } static void zswap_lru_del(struct zswap_entry *entry) { - int nid = entry_to_nid(entry); struct mem_cgroup *memcg; rcu_read_lock(); memcg = obj_cgroup_memcg(entry->objcg); /* will always succeed */ - list_lru_del(&zswap_list_lru, &entry->lru, nid, memcg); + list_lru_del(&zswap_list_lru, &entry->lru, entry->nid, memcg); rcu_read_unlock(); } @@ -764,10 +762,11 @@ void zswap_memcg_offline_cleanup(struct mem_cgroup *memcg) **********************************/ static struct kmem_cache *zswap_entry_cache; -static struct zswap_entry *zswap_entry_cache_alloc(gfp_t gfp, int nid) +static struct zswap_entry *zswap_entry_cache_alloc(gfp_t gfp) { struct zswap_entry *entry; - entry = kmem_cache_alloc_node(zswap_entry_cache, gfp, nid); + + entry = kmem_cache_alloc(zswap_entry_cache, gfp); if (!entry) return NULL; zs_handle_init(&entry->handle); @@ -1537,9 +1536,10 @@ static bool zswap_store_page(struct folio *folio, long index, { swp_entry_t page_swpentry = folio_swap_entry(folio, index); struct zswap_entry *entry, *old; + int nid = folio_nid(folio); /* allocate entry */ - entry = zswap_entry_cache_alloc(GFP_KERNEL, folio_nid(folio)); + entry = zswap_entry_cache_alloc(GFP_KERNEL); if (!entry) { zswap_reject_kmemcache_fail++; return false; @@ -1554,6 +1554,7 @@ static bool zswap_store_page(struct folio *folio, long index, * to an unrelated live pool. */ entry->pool_idx = pool->idx; + entry->nid = nid; old = xa_store(swap_zswap_tree(page_swpentry), swp_offset(page_swpentry), @@ -1930,6 +1931,8 @@ static int zswap_setup(void) struct zswap_pool *pool; int ret; + BUILD_BUG_ON(ZSWAP_MAX_COMPR_LENGTH_SHIFT <= PAGE_SHIFT); + zswap_entry_cache = KMEM_CACHE(zswap_entry, 0); if (!zswap_entry_cache) { pr_err("entry cache creation failed\n"); -- 2.39.5