Add MPOL_F_PRIVATE, the internal flag noting that a policy contains a nodemask with a private node, and making private node memory reachable via standard mempolicies. Plumb zonelist selection into the mpol allocator interfaces via the allocator's alloc_flags, using the alloc_flags-carrying page_alloc interfaces. mpol_alloc_flags() maps MPOL_F_PRIVATE to the alloc_flags an allocation uses: ALLOC_DEFAULT - normal allocation ALLOC_ZONELIST_PRIVATE - private node allocation With this, a VMA with (mpol->flags & MPOL_F_PRIVATE) can successfully services faults like any other mbind() from the private node: buf = mmap(..., MAP_ANON); mbind(buf, ..., {private_node}); buf[0] = 0xdeadbeef; // Page faulted from private node memory Like any other bind, the nodemask relaxes if it is unsatisfiable. For example: apply_policy_zone() allows an unmovable allocation targeting the VMA to fallback to a viable node instead of failing. cpuset rebinding does not affect private nodes - the nodes in the original mask are preserved, and we enforce two remap rules: 1) never allow N_MEMORY nodes to remap to N_MEMORY_PRIVATE 2) never remap N_MEMORY_PRIVATE nodes at all (they stay in place) In the case of an empty nodemask as a result of rebind, revert to cpuset - which never includes N_MEMORY_PRIVATE and always guarantees at least one N_MEMORY node. As of this patch, nothing can actually sets MPOL_F_PRIVATE, so this is a no-op that simply adds the plumbing throughout mempolicy. Signed-off-by: Gregory Price --- include/uapi/linux/mempolicy.h | 1 + mm/mempolicy.c | 79 +++++++++++++++++++++++----------- 2 files changed, 55 insertions(+), 25 deletions(-) diff --git a/include/uapi/linux/mempolicy.h b/include/uapi/linux/mempolicy.h index 7f6fc9599693b..87af18c84b947 100644 --- a/include/uapi/linux/mempolicy.h +++ b/include/uapi/linux/mempolicy.h @@ -67,6 +67,7 @@ enum mempolicy_mode { #define MPOL_F_SHARED (1 << 0) /* identify shared policies */ #define MPOL_F_MOF (1 << 3) /* this policy wants migrate on fault */ #define MPOL_F_MORON (1 << 4) /* Migrate On protnone Reference On Node */ +#define MPOL_F_PRIVATE (1 << 5) /* nodemask contains private nodes */ /* * Enabling zone reclaim means the page allocator will attempt to fulfill diff --git a/mm/mempolicy.c b/mm/mempolicy.c index 2b76c57a460c9..90110e9761122 100644 --- a/mm/mempolicy.c +++ b/mm/mempolicy.c @@ -432,6 +432,10 @@ static int mpol_set_nodemask(struct mempolicy *pol, else pol->w.cpuset_mems_allowed = cpuset_current_mems_allowed; + /* If any private nodes left in the nodemask - add the private flag */ + if (nodes_intersects(nsc->mask2, node_states[N_MEMORY_PRIVATE])) + pol->flags |= MPOL_F_PRIVATE; + ret = mpol_ops[pol->mode].create(pol, &nsc->mask2); return ret; } @@ -505,22 +509,33 @@ static void mpol_rebind_default(struct mempolicy *pol, const nodemask_t *nodes) static void mpol_rebind_nodemask(struct mempolicy *pol, const nodemask_t *nodes) { - nodemask_t tmp; + nodemask_t tmp, priv; + + /* preserve online private nodes to re-add later */ + nodes_and(priv, pol->nodes, node_states[N_MEMORY_PRIVATE]); if (pol->flags & MPOL_F_STATIC_NODES) nodes_and(tmp, pol->w.user_nodemask, *nodes); - else if (pol->flags & MPOL_F_RELATIVE_NODES) - mpol_relative_nodemask(&tmp, &pol->w.user_nodemask, nodes); - else { + else if (pol->flags & MPOL_F_RELATIVE_NODES) { + /* fold only the public part: a private node must not take a slot */ + nodes_and(tmp, pol->w.user_nodemask, node_states[N_MEMORY]); + mpol_relative_nodemask(&tmp, &tmp, nodes); + } else { nodes_remap(tmp, pol->nodes, pol->w.cpuset_mems_allowed, *nodes); pol->w.cpuset_mems_allowed = *nodes; } - if (nodes_empty(tmp)) + /* private nodes are identity-mapped during remap, drop them here */ + nodes_and(tmp, tmp, node_states[N_MEMORY]); + if (nodes_empty(tmp) && nodes_empty(priv)) tmp = *nodes; - pol->nodes = tmp; + /* If any online private nodes remain, add them back */ + nodes_or(pol->nodes, tmp, priv); + /* If no online private nodes remain, strip the private flag */ + if (nodes_empty(priv)) + pol->flags &= ~MPOL_F_PRIVATE; } static void mpol_rebind_preferred(struct mempolicy *pol, @@ -2411,7 +2426,8 @@ bool mempolicy_in_oom_domain(struct task_struct *tsk, } static struct page *alloc_pages_preferred_many(gfp_t gfp, unsigned int order, - int nid, nodemask_t *nodemask) + int nid, nodemask_t *nodemask, + unsigned int aflags) { struct page *page; gfp_t preferred_gfp; @@ -2425,14 +2441,21 @@ static struct page *alloc_pages_preferred_many(gfp_t gfp, unsigned int order, preferred_gfp = gfp | __GFP_NOWARN; preferred_gfp &= ~(__GFP_DIRECT_RECLAIM | __GFP_NOFAIL); page = __alloc_frozen_pages_noprof(preferred_gfp, order, nid, nodemask, - ALLOC_DEFAULT); + aflags); if (!page) page = __alloc_frozen_pages_noprof(gfp, order, nid, NULL, - ALLOC_DEFAULT); + aflags); return page; } +/* A private policy allocates from the private zonelist. */ +static inline unsigned int mpol_alloc_flags(struct mempolicy *pol) +{ + return (pol->flags & MPOL_F_PRIVATE) ? ALLOC_ZONELIST_PRIVATE : + ALLOC_DEFAULT; +} + /** * alloc_pages_mpol - Allocate pages according to NUMA mempolicy. * @gfp: GFP flags. @@ -2448,11 +2471,13 @@ static struct page *alloc_pages_mpol(gfp_t gfp, unsigned int order, { nodemask_t *nodemask; struct page *page; + unsigned int aflags = mpol_alloc_flags(pol); nodemask = policy_nodemask(gfp, pol, ilx, &nid); if (pol->mode == MPOL_PREFERRED_MANY) - return alloc_pages_preferred_many(gfp, order, nid, nodemask); + return alloc_pages_preferred_many(gfp, order, nid, nodemask, + aflags); if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) && /* filter "hugepage" allocation, unless from alloc_pages() */ @@ -2476,7 +2501,7 @@ static struct page *alloc_pages_mpol(gfp_t gfp, unsigned int order, */ page = __alloc_frozen_pages_noprof( gfp | __GFP_THISNODE | __GFP_NORETRY, order, - nid, NULL, ALLOC_DEFAULT); + nid, NULL, aflags); if (page || !(gfp & __GFP_DIRECT_RECLAIM)) return page; /* @@ -2488,7 +2513,7 @@ static struct page *alloc_pages_mpol(gfp_t gfp, unsigned int order, } } - page = __alloc_frozen_pages_noprof(gfp, order, nid, nodemask, ALLOC_DEFAULT); + page = __alloc_frozen_pages_noprof(gfp, order, nid, nodemask, aflags); if (unlikely(pol->mode == MPOL_INTERLEAVE || pol->mode == MPOL_WEIGHTED_INTERLEAVE) && page) { @@ -2597,6 +2622,7 @@ static unsigned long alloc_pages_bulk_interleave(gfp_t gfp, struct mempolicy *pol, unsigned long nr_pages, struct page **page_array) { + unsigned int aflags = mpol_alloc_flags(pol); int nodes; unsigned long nr_pages_per_node; int delta; @@ -2610,14 +2636,14 @@ static unsigned long alloc_pages_bulk_interleave(gfp_t gfp, for (i = 0; i < nodes; i++) { if (delta) { - nr_allocated = alloc_pages_bulk_noprof(gfp, - interleave_nodes(pol), NULL, + nr_allocated = __alloc_pages_bulk_noprof(gfp, + aflags, interleave_nodes(pol), NULL, nr_pages_per_node + 1, page_array); delta--; } else { - nr_allocated = alloc_pages_bulk_noprof(gfp, - interleave_nodes(pol), NULL, + nr_allocated = __alloc_pages_bulk_noprof(gfp, + aflags, interleave_nodes(pol), NULL, nr_pages_per_node, page_array); } @@ -2632,6 +2658,7 @@ static unsigned long alloc_pages_bulk_weighted_interleave(gfp_t gfp, struct mempolicy *pol, unsigned long nr_pages, struct page **page_array) { + unsigned int aflags = mpol_alloc_flags(pol); struct weighted_interleave_state *state; struct task_struct *me = current; unsigned int cpuset_mems_cookie; @@ -2667,8 +2694,8 @@ static unsigned long alloc_pages_bulk_weighted_interleave(gfp_t gfp, weight = me->il_weight; if (weight && node_isset(node, nodes)) { node_pages = min(rem_pages, weight); - nr_allocated = __alloc_pages_bulk(gfp, node, NULL, node_pages, - page_array); + nr_allocated = __alloc_pages_bulk_noprof(gfp, aflags, node, + NULL, node_pages, page_array); page_array += nr_allocated; total_allocated += nr_allocated; /* if that's all the pages, no need to interleave */ @@ -2732,8 +2759,8 @@ static unsigned long alloc_pages_bulk_weighted_interleave(gfp_t gfp, /* node_pages can be 0 if an allocation fails and rounds == 0 */ if (!node_pages) break; - nr_allocated = __alloc_pages_bulk(gfp, node, NULL, node_pages, - page_array); + nr_allocated = __alloc_pages_bulk_noprof(gfp, aflags, node, + NULL, node_pages, page_array); page_array += nr_allocated; total_allocated += nr_allocated; if (total_allocated == nr_pages) @@ -2750,17 +2777,19 @@ static unsigned long alloc_pages_bulk_preferred_many(gfp_t gfp, int nid, struct mempolicy *pol, unsigned long nr_pages, struct page **page_array) { + unsigned int aflags = mpol_alloc_flags(pol); gfp_t preferred_gfp; unsigned long nr_allocated = 0; preferred_gfp = gfp | __GFP_NOWARN; preferred_gfp &= ~(__GFP_DIRECT_RECLAIM | __GFP_NOFAIL); - nr_allocated = alloc_pages_bulk_noprof(preferred_gfp, nid, &pol->nodes, - nr_pages, page_array); + nr_allocated = __alloc_pages_bulk_noprof(preferred_gfp, aflags, + nid, &pol->nodes, nr_pages, page_array); if (nr_allocated < nr_pages) - nr_allocated += alloc_pages_bulk_noprof(gfp, numa_node_id(), NULL, + nr_allocated += __alloc_pages_bulk_noprof(gfp, aflags, + numa_node_id(), NULL, nr_pages - nr_allocated, page_array + nr_allocated); return nr_allocated; @@ -2796,8 +2825,8 @@ unsigned long alloc_pages_bulk_mempolicy_noprof(gfp_t gfp, nid = numa_node_id(); nodemask = policy_nodemask(gfp, pol, NO_INTERLEAVE_INDEX, &nid); - return alloc_pages_bulk_noprof(gfp, nid, nodemask, - nr_pages, page_array); + return __alloc_pages_bulk_noprof(gfp, mpol_alloc_flags(pol), nid, + nodemask, nr_pages, page_array); } int vma_dup_policy(struct vm_area_struct *src, struct vm_area_struct *dst) -- 2.53.0-Meta