Secondary compressors are only used by recompress_store(), which is serialized under zram->dev_lock, so we only ever use one secondary compressor at a time, yet we still allocate them on each CPU, which is just a plain memory wastage. However, we do need per-CPU decompressors in order to support concurrent reads. Now that Read and Write contexts are split, we can allocate just a single compression stream (recomp_cstream) for secondary algorithms instead of per-CPU streams, which saves N - 1 compression contexts and 2 * PAGE_SIZE buffers. For example, on an 8-CPU x86_64 system (4K PAGE_SIZE), this saves: - zstd (level 3, default): ~700 KB (~100 KB/CPU) - zstd (levels 12-22, no dict): ~1.91 MB (~280 KB/CPU) - zstd (levels 14-22, 110K dict): ~5.33 MB (~780 KB/CPU, lazy) - deflate (winbits=-11, default): ~1.04 MB (~152 KB/CPU) - deflate (winbits=15): ~1.86 MB (~272 KB/CPU) - lz4hc: ~1.84 MB (~269 KB/CPU) Signed-off-by: Sergey Senozhatsky --- drivers/block/zram/zcomp.c | 75 ++++++++++++++++++++++++++--------- drivers/block/zram/zcomp.h | 9 +++-- drivers/block/zram/zram_drv.c | 5 ++- 3 files changed, 65 insertions(+), 24 deletions(-) diff --git a/drivers/block/zram/zcomp.c b/drivers/block/zram/zcomp.c index da122bf4dd15..39d1f9384939 100644 --- a/drivers/block/zram/zcomp.c +++ b/drivers/block/zram/zcomp.c @@ -151,6 +151,11 @@ ssize_t zcomp_available_show(const char *comp, char *buf, ssize_t at) struct zcomp_cstrm *zcomp_cstrm_get(struct zcomp *comp) { + if (comp->recomp_cstream) { + mutex_lock(&comp->recomp_cstream->lock); + return comp->recomp_cstream; + } + for (;;) { struct zcomp_cstrm *zstrm = raw_cpu_ptr(comp->cstream); @@ -274,22 +279,25 @@ int zcomp_decompress(struct zcomp *comp, struct zcomp_dstrm *zstrm, int zcomp_cpu_up_prepare(unsigned int cpu, struct hlist_node *node) { struct zcomp *comp = hlist_entry(node, struct zcomp, node); - struct zcomp_cstrm *cstrm; + struct zcomp_cstrm *cstrm = NULL; struct zcomp_dstrm *dstrm; int ret; - cstrm = per_cpu_ptr(comp->cstream, cpu); - ret = zcomp_cstrm_init(comp, cstrm); - if (ret) { - pr_err("Can't allocate a compression stream\n"); - return ret; + if (comp->cstream) { + cstrm = per_cpu_ptr(comp->cstream, cpu); + ret = zcomp_cstrm_init(comp, cstrm); + if (ret) { + pr_err("Can't allocate a compression stream\n"); + return ret; + } } dstrm = per_cpu_ptr(comp->dstream, cpu); ret = zcomp_dstrm_init(comp, dstrm); if (ret) { pr_err("Can't allocate a decompression stream\n"); - zcomp_cstrm_free(comp, cstrm); + if (cstrm) + zcomp_cstrm_free(comp, cstrm); return ret; } @@ -302,10 +310,12 @@ int zcomp_cpu_dead(unsigned int cpu, struct hlist_node *node) struct zcomp_cstrm *cstrm; struct zcomp_dstrm *dstrm; - cstrm = per_cpu_ptr(comp->cstream, cpu); - mutex_lock(&cstrm->lock); - zcomp_cstrm_free(comp, cstrm); - mutex_unlock(&cstrm->lock); + if (comp->cstream) { + cstrm = per_cpu_ptr(comp->cstream, cpu); + mutex_lock(&cstrm->lock); + zcomp_cstrm_free(comp, cstrm); + mutex_unlock(&cstrm->lock); + } dstrm = per_cpu_ptr(comp->dstream, cpu); down_write(&dstrm->lock); @@ -315,7 +325,8 @@ int zcomp_cpu_dead(unsigned int cpu, struct hlist_node *node) return 0; } -static int zcomp_init(struct zcomp *comp, struct zcomp_params *params) +static int zcomp_init(struct zcomp *comp, struct zcomp_params *params, + bool recomp) { int ret, cpu; @@ -324,10 +335,28 @@ static int zcomp_init(struct zcomp *comp, struct zcomp_params *params) if (ret) goto cleanup; - comp->cstream = alloc_percpu(struct zcomp_cstrm); - if (!comp->cstream) { - ret = -ENOMEM; - goto cleanup; + if (recomp) { + comp->recomp_cstream = kzalloc_obj(*comp->recomp_cstream); + if (!comp->recomp_cstream) { + ret = -ENOMEM; + goto cleanup; + } + mutex_init(&comp->recomp_cstream->lock); + ret = zcomp_cstrm_init(comp, comp->recomp_cstream); + if (ret) { + /* zcomp_cstrm_init() cleans up after itself */ + kfree(comp->recomp_cstream); + comp->recomp_cstream = NULL; + goto cleanup; + } + } else { + comp->cstream = alloc_percpu(struct zcomp_cstrm); + if (!comp->cstream) { + ret = -ENOMEM; + goto cleanup; + } + for_each_possible_cpu(cpu) + mutex_init(&per_cpu_ptr(comp->cstream, cpu)->lock); } comp->dstream = alloc_percpu(struct zcomp_dstrm); @@ -339,7 +368,6 @@ static int zcomp_init(struct zcomp *comp, struct zcomp_params *params) for_each_possible_cpu(cpu) { struct zcomp_dstrm *zstrm = per_cpu_ptr(comp->dstream, cpu); - mutex_init(&per_cpu_ptr(comp->cstream, cpu)->lock); init_rwsem(&zstrm->lock); mutex_init(&zstrm->excl_lock); } @@ -351,6 +379,10 @@ static int zcomp_init(struct zcomp *comp, struct zcomp_params *params) return 0; cleanup: + if (comp->recomp_cstream) { + zcomp_cstrm_free(comp, comp->recomp_cstream); + kfree(comp->recomp_cstream); + } comp->ops->release_params(comp->params); free_percpu(comp->dstream); free_percpu(comp->cstream); @@ -360,13 +392,18 @@ static int zcomp_init(struct zcomp *comp, struct zcomp_params *params) void zcomp_destroy(struct zcomp *comp) { cpuhp_state_remove_instance(CPUHP_ZCOMP_PREPARE, &comp->node); + if (comp->recomp_cstream) { + zcomp_cstrm_free(comp, comp->recomp_cstream); + kfree(comp->recomp_cstream); + } comp->ops->release_params(comp->params); free_percpu(comp->dstream); free_percpu(comp->cstream); kfree(comp); } -struct zcomp *zcomp_create(const char *alg, struct zcomp_params *params) +struct zcomp *zcomp_create(const char *alg, struct zcomp_params *params, + bool recomp) { struct zcomp *comp; int error; @@ -389,7 +426,7 @@ struct zcomp *zcomp_create(const char *alg, struct zcomp_params *params) return ERR_PTR(-EINVAL); } - error = zcomp_init(comp, params); + error = zcomp_init(comp, params, recomp); if (error) { kfree(comp); return ERR_PTR(error); diff --git a/drivers/block/zram/zcomp.h b/drivers/block/zram/zcomp.h index bf41b27958c4..924c4185abe9 100644 --- a/drivers/block/zram/zcomp.h +++ b/drivers/block/zram/zcomp.h @@ -39,7 +39,8 @@ struct zcomp_params { /* * Run-time driver context - scratch buffers, etc. It is modified during * request execution (compression/decompression), cannot be shared, so - * it's in per-CPU area. + * it's in per-CPU area (except for recompression, which uses a single + * per-device compression stream). */ struct zcomp_ctx { void *context; @@ -94,8 +95,9 @@ struct zcomp_ops { /* dynamic per-device compression frontend */ struct zcomp { - struct zcomp_cstrm __percpu *cstream; struct zcomp_dstrm __percpu *dstream; + struct zcomp_cstrm __percpu *cstream; + struct zcomp_cstrm *recomp_cstream; const struct zcomp_ops *ops; struct zcomp_params *params; struct hlist_node node; @@ -106,7 +108,8 @@ int zcomp_cpu_dead(unsigned int cpu, struct hlist_node *node); ssize_t zcomp_available_show(const char *comp, char *buf, ssize_t at); const char *zcomp_lookup_backend_name(const char *comp); -struct zcomp *zcomp_create(const char *alg, struct zcomp_params *params); +struct zcomp *zcomp_create(const char *alg, struct zcomp_params *params, + bool recomp); void zcomp_destroy(struct zcomp *comp); struct zcomp_cstrm *zcomp_cstrm_get(struct zcomp *comp); diff --git a/drivers/block/zram/zram_drv.c b/drivers/block/zram/zram_drv.c index a34859ff0461..ad244624d4a5 100644 --- a/drivers/block/zram/zram_drv.c +++ b/drivers/block/zram/zram_drv.c @@ -2526,7 +2526,7 @@ static int recompress_slot(struct zram *zram, unsigned long index, } /* - * We are holding per-CPU stream mutex and entry lock so better + * We are holding compression stream mutex and entry lock so better * avoid direct reclaim. Allocation error is not fatal since * we still have the old object in the mem_pool. * @@ -2928,7 +2928,8 @@ static ssize_t disksize_store(struct device *dev, struct device_attribute *attr, continue; comp = zcomp_create(zram->comp_algs[prio], - &zram->params[prio]); + &zram->params[prio], + prio != ZRAM_PRIMARY_COMP); if (IS_ERR(comp)) { pr_err("Cannot initialise %s compressing backend\n", zram->comp_algs[prio]); -- 2.56.0.385.gd3acb90ef8-goog