When refilling PCP with whole pageblocks for movable allocations, prefer pageblocks from the fullest clean (only free + movable) superpageblock. This packs movable allocations into already-partial superpageblocks, preserving empty superpageblocks for potential 1GB hugepage allocation. Add sb_preferred_for_movable() which walks the clean superpageblock lists from SB_FULL toward SB_ALMOST_EMPTY to find the fullest clean superpageblock with available free pageblocks. Add __rmqueue_from_sb() which scans the buddy free list for a page within a specific superpageblock's PFN range, with a bounded scan limit (8 entries) to avoid excessive latency. Hook into rmqueue_bulk() phase 1 (whole pageblock grab for PCP refill) to try the preferred superpageblock before falling back to the normal __rmqueue() path. This is the primary steering point for movable allocations without per-superpageblock free lists. Also fix an ALLOC_NOFRAGMENT propagation oversight in alloc_pages_bulk_noprof(): the bulk allocator's preferred_zoneref is computed locally, so it must also call alloc_flags_nofragment() to match the protection that the single-page fastpath gets via prepare_alloc_pages(). Without this, bulk folio refills for the page cache could taint clean SPBs that the single-page path would have left alone. Signed-off-by: Rik van Riel Assisted-by: Claude:claude-opus-4.7 syzkaller --- mm/page_alloc.c | 89 +++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 86 insertions(+), 3 deletions(-) diff --git a/mm/page_alloc.c b/mm/page_alloc.c index a17c4cd9a788..9dc65bf93e71 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -2330,6 +2330,73 @@ static void prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags /* Bounded scan limit when searching free lists for tainted superpageblock pages */ #define SPB_SCAN_LIMIT 8 +/** + * sb_preferred_for_movable - Find the fullest clean superpageblock for movable + * @zone: zone to search + * + * Walk spb_lists[CLEAN] from nearly full toward emptiest -- pack movable + * allocations into already-partial superpageblocks before starting new ones. + * Skip SB_FULL since those have no free pageblocks. + * Returns NULL if no suitable superpageblock found. + */ +static struct superpageblock *sb_preferred_for_movable(struct zone *zone) +{ + int full; + struct superpageblock *sb; + + for (full = SB_FULL_75; full < __NR_SB_FULLNESS; full++) { + list_for_each_entry(sb, &zone->spb_lists[SB_CLEAN][full], list) { + if (sb->nr_free) + return sb; + } + } + /* Fall back to empty superpageblocks -- no clean partials available */ + return NULL; +} + +/** + * __rmqueue_from_sb - Try to allocate a page from a specific superpageblock + * @zone: zone to allocate from + * @order: allocation order + * @migratetype: type to allocate + * @sb: preferred superpageblock + * + * Scan the free list at the given order for a page within the superpageblock's + * PFN range. Bounded scan to avoid excessive latency. Returns NULL if + * no suitable page found. + */ +static struct page *__rmqueue_from_sb(struct zone *zone, unsigned int order, + int migratetype, struct superpageblock *sb) +{ + unsigned int current_order; + unsigned long sb_start = sb->start_pfn; + unsigned long sb_end = sb_start + (1UL << SUPERPAGEBLOCK_ORDER); + struct free_area *area; + struct page *page; + int scanned; + + for (current_order = order; current_order < NR_PAGE_ORDERS; + ++current_order) { + area = &zone->free_area[current_order]; + scanned = 0; + + list_for_each_entry(page, &area->free_list[migratetype], + buddy_list) { + unsigned long pfn = page_to_pfn(page); + + if (pfn >= sb_start && pfn < sb_end) { + page_del_and_expand(zone, page, order, + current_order, + migratetype); + return page; + } + if (++scanned >= SPB_SCAN_LIMIT) + break; + } + } + return NULL; +} + /* * Go through the free lists for the given migratetype and remove * the smallest available page from the freelists @@ -3119,12 +3186,26 @@ static bool rmqueue_bulk(struct zone *zone, unsigned int order, * small zones, pages_needed can be less than a whole * pageblock; skip to smaller blocks or individual pages to * avoid overshooting the PCP high watermark. + * + * For movable allocations, prefer pageblocks from the + * fullest clean superpageblock to pack allocations and + * preserve empty superpageblocks for 1GB hugepages. */ while (refilled + pageblock_nr_pages <= pages_needed) { - struct page *page; + struct page *page = NULL; - page = __rmqueue(zone, pageblock_order, - migratetype, alloc_flags, &rmqm); + if (migratetype == MIGRATE_MOVABLE) { + struct superpageblock *sb; + + sb = sb_preferred_for_movable(zone); + if (sb) + page = __rmqueue_from_sb(zone, pageblock_order, + migratetype, sb); + } + if (!page) + page = __rmqueue(zone, pageblock_order, + migratetype, + alloc_flags, &rmqm); if (!page) break; @@ -5843,6 +5924,8 @@ unsigned long alloc_pages_bulk_noprof(gfp_t gfp, int preferred_nid, goto out; gfp = alloc_gfp; + alloc_flags |= alloc_flags_nofragment(zonelist_zone(ac.preferred_zoneref), gfp); + /* Find an allowed local zone that meets the low watermark. */ z = ac.preferred_zoneref; for_next_zone_zonelist_nodemask(zone, z, ac.highest_zoneidx, ac.nodemask) { -- 2.54.0