From: Yuan-Hao Hsu Add support for calling folio_pte_batch_flags() in GUP-fast context when we are not holding the PT lock. Instead of reading PTEs through ptep_get(), we have to use ptep_get_lockless(). Specify the new behavior through a new flag FPB_LOCKLESS. We'll make use of this functionality for batching PTEs in gup_fast_pte_range() next. Reviewed-by: David Hildenbrand (Arm) Reviewed-by: Rik van Riel Acked-by: David Hildenbrand (Arm) Co-developed-by: Yeoreum Yun Signed-off-by: Yuan-Hao Hsu --- mm/internal.h | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/mm/internal.h b/mm/internal.h index 177bd47045d1..4db3381238d0 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -373,6 +373,12 @@ typedef int __bitwise fpb_t; */ #define FPB_MERGE_YOUNG_DIRTY ((__force fpb_t)BIT(4)) +/* + * Read the page table entries with ptep_get_lockless(): the caller does not + * hold the page table lock (e.g, GUP-fast). + */ +#define FPB_LOCKLESS ((__force fpb_t)BIT(5)) + static inline pte_t __pte_batch_clear_ignored(pte_t pte, fpb_t flags) { if (!(flags & FPB_RESPECT_DIRTY)) @@ -441,7 +447,7 @@ static inline unsigned int folio_pte_batch_flags(struct folio *folio, ptep = ptep + nr; while (nr < max_nr) { - pte = ptep_get(ptep); + pte = (flags & FPB_LOCKLESS) ? ptep_get_lockless(ptep) : ptep_get(ptep); if (!pte_same(__pte_batch_clear_ignored(pte, flags), expected_pte)) break; -- 2.43.0