In zs_malloc(), the fast path (if (likely(zspage))) allocates from an existing zspage via obj_malloc(), while the slow path allocates a new zspage via alloc_zspage(). Because obj_malloc() has three call sites across zs_malloc() and migrate_zspage(), Clang at -O2 leaves obj_malloc() out-of-line, adding a function call on every fast-path allocation. Conversely, because the 362-instruction slow-path alloc_zspage() is static with a single caller, single-caller inlining under CONFIG_AUTOFDO_CLANG (e.g. with CONFIG_ZSMALLOC=m) inlines alloc_zspage() into zs_malloc(), growing it from 254 to 691 instructions. Mark obj_malloc() and obj_free() __always_inline and mark alloc_zspage() noinline to keep the fast path inline and the slow-path page allocation out-of-line. Signed-off-by: Jenhao Chen --- mm/zsmalloc.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/mm/zsmalloc.c b/mm/zsmalloc.c index 6258a4f5e9fce..5d3443f14a48f 100644 --- a/mm/zsmalloc.c +++ b/mm/zsmalloc.c @@ -960,9 +960,9 @@ static void create_page_chain(struct size_class *class, struct zspage *zspage, /* * Allocate a zspage for the given size class */ -static struct zspage *alloc_zspage(struct zs_pool *pool, - struct size_class *class, - gfp_t gfp, const int nid) +static noinline struct zspage *alloc_zspage(struct zs_pool *pool, + struct size_class *class, + gfp_t gfp, const int nid) { int i; struct zpdesc *zpdescs[ZS_MAX_PAGES_PER_ZSPAGE]; @@ -1209,8 +1209,9 @@ size_t zs_huge_class_size(struct zs_pool *pool) } EXPORT_SYMBOL_GPL(zs_huge_class_size); -static unsigned long obj_malloc(struct zs_pool *pool, - struct zspage *zspage, unsigned long handle) +static __always_inline unsigned long obj_malloc(struct zs_pool *pool, + struct zspage *zspage, + unsigned long handle) { int i, nr_zpdesc, offset; unsigned long obj; @@ -1321,7 +1322,7 @@ unsigned long zs_malloc(struct zs_pool *pool, size_t size, gfp_t gfp, } EXPORT_SYMBOL_GPL(zs_malloc); -static void obj_free(int class_size, unsigned long obj) +static __always_inline void obj_free(int class_size, unsigned long obj) { struct link_free *link; struct zspage *zspage; -- 2.56.0.317.g454f8302e1-goog