Move ~150 lines of folio bit-lock and wait queue infrastructure from pagemap.h to folio_wait.h. pagemap.h includes the new header so existing users don't break. Signed-off-by: Tal Zussman --- include/linux/folio_wait.h | 181 +++++++++++++++++++++++++++++++++++++++++++++ include/linux/pagemap.h | 172 +----------------------------------------- mm/folio_wait.c | 2 +- 3 files changed, 183 insertions(+), 172 deletions(-) diff --git a/include/linux/folio_wait.h b/include/linux/folio_wait.h new file mode 100644 index 000000000000..80ddf1ffcae4 --- /dev/null +++ b/include/linux/folio_wait.h @@ -0,0 +1,181 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_FOLIO_WAIT_H +#define _LINUX_FOLIO_WAIT_H + +#include +#include +#include + +struct wait_page_key { + struct folio *folio; + int bit_nr; + int page_match; +}; + +struct wait_page_queue { + struct folio *folio; + int bit_nr; + wait_queue_entry_t wait; +}; + +static inline bool wake_page_match(struct wait_page_queue *wait_page, + struct wait_page_key *key) +{ + if (wait_page->folio != key->folio) + return false; + key->page_match = 1; + + if (wait_page->bit_nr != key->bit_nr) + return false; + + return true; +} + +void __folio_lock(struct folio *folio); +int __folio_lock_killable(struct folio *folio); +vm_fault_t __folio_lock_or_retry(struct folio *folio, struct vm_fault *vmf); +void unlock_page(struct page *page); +void folio_unlock(struct folio *folio); + +/** + * folio_trylock() - Attempt to lock a folio. + * @folio: The folio to attempt to lock. + * + * Sometimes it is undesirable to wait for a folio to be unlocked (eg + * when the locks are being taken in the wrong order, or if making + * progress through a batch of folios is more important than processing + * them in order). Usually folio_lock() is the correct function to call. + * + * Context: Any context. + * Return: Whether the lock was successfully acquired. + */ +static inline bool folio_trylock(struct folio *folio) +{ + return likely(!test_and_set_bit_lock(PG_locked, folio_flags(folio, 0))); +} + +/* + * Return true if the page was successfully locked + */ +static inline bool trylock_page(struct page *page) +{ + return folio_trylock(page_folio(page)); +} + +/** + * folio_lock() - Lock this folio. + * @folio: The folio to lock. + * + * The folio lock protects against many things, probably more than it + * should. It is primarily held while a folio is being brought uptodate, + * either from its backing file or from swap. It is also held while a + * folio is being truncated from its address_space, so holding the lock + * is sufficient to keep folio->mapping stable. + * + * The folio lock is also held while write() is modifying the page to + * provide POSIX atomicity guarantees (as long as the write does not + * cross a page boundary). Other modifications to the data in the folio + * do not hold the folio lock and can race with writes, eg DMA and stores + * to mapped pages. + * + * Context: May sleep. If you need to acquire the locks of two or + * more folios, they must be in order of ascending index, if they are + * in the same address_space. If they are in different address_spaces, + * acquire the lock of the folio which belongs to the address_space which + * has the lowest address in memory first. + */ +static inline void folio_lock(struct folio *folio) +{ + might_sleep(); + if (!folio_trylock(folio)) + __folio_lock(folio); +} + +/** + * lock_page() - Lock the folio containing this page. + * @page: The page to lock. + * + * See folio_lock() for a description of what the lock protects. + * This is a legacy function and new code should probably use folio_lock() + * instead. + * + * Context: May sleep. Pages in the same folio share a lock, so do not + * attempt to lock two pages which share a folio. + */ +static inline void lock_page(struct page *page) +{ + struct folio *folio; + might_sleep(); + + folio = page_folio(page); + if (!folio_trylock(folio)) + __folio_lock(folio); +} + +/** + * folio_lock_killable() - Lock this folio, interruptible by a fatal signal. + * @folio: The folio to lock. + * + * Attempts to lock the folio, like folio_lock(), except that the sleep + * to acquire the lock is interruptible by a fatal signal. + * + * Context: May sleep; see folio_lock(). + * Return: 0 if the lock was acquired; -EINTR if a fatal signal was received. + */ +static inline int folio_lock_killable(struct folio *folio) +{ + might_sleep(); + if (!folio_trylock(folio)) + return __folio_lock_killable(folio); + return 0; +} + +/* + * folio_lock_or_retry - Lock the folio, unless this would block and the + * caller indicated that it can handle a retry. + * + * Return value and mmap_lock implications depend on flags; see + * __folio_lock_or_retry(). + */ +static inline vm_fault_t folio_lock_or_retry(struct folio *folio, + struct vm_fault *vmf) +{ + might_sleep(); + if (!folio_trylock(folio)) + return __folio_lock_or_retry(folio, vmf); + return 0; +} + +/* + * This is exported only for folio_wait_locked/folio_wait_writeback, etc., + * and should not be used directly. + */ +void folio_wait_bit(struct folio *folio, int bit_nr); +int folio_wait_bit_killable(struct folio *folio, int bit_nr); + +/* + * Wait for a folio to be unlocked. + * + * This must be called with the caller "holding" the folio, + * ie with increased folio reference count so that the folio won't + * go away during the wait. + */ +static inline void folio_wait_locked(struct folio *folio) +{ + if (folio_test_locked(folio)) + folio_wait_bit(folio, PG_locked); +} + +static inline int folio_wait_locked_killable(struct folio *folio) +{ + if (!folio_test_locked(folio)) + return 0; + return folio_wait_bit_killable(folio, PG_locked); +} + +void folio_end_read(struct folio *folio, bool success); +void folio_end_private_2(struct folio *folio); +void folio_wait_private_2(struct folio *folio); +int folio_wait_private_2_killable(struct folio *folio); + +#endif /* _LINUX_FOLIO_WAIT_H */ diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h index 627771e82eb1..7f65c2b0097b 100644 --- a/include/linux/pagemap.h +++ b/include/linux/pagemap.h @@ -15,6 +15,7 @@ #include #include /* for in_interrupt() */ #include +#include struct folio_batch; @@ -1072,174 +1073,6 @@ static inline pgoff_t linear_page_index(const struct vm_area_struct *vma, return pgoff; } -struct wait_page_key { - struct folio *folio; - int bit_nr; - int page_match; -}; - -struct wait_page_queue { - struct folio *folio; - int bit_nr; - wait_queue_entry_t wait; -}; - -static inline bool wake_page_match(struct wait_page_queue *wait_page, - struct wait_page_key *key) -{ - if (wait_page->folio != key->folio) - return false; - key->page_match = 1; - - if (wait_page->bit_nr != key->bit_nr) - return false; - - return true; -} - -void __folio_lock(struct folio *folio); -int __folio_lock_killable(struct folio *folio); -vm_fault_t __folio_lock_or_retry(struct folio *folio, struct vm_fault *vmf); -void unlock_page(struct page *page); -void folio_unlock(struct folio *folio); - -/** - * folio_trylock() - Attempt to lock a folio. - * @folio: The folio to attempt to lock. - * - * Sometimes it is undesirable to wait for a folio to be unlocked (eg - * when the locks are being taken in the wrong order, or if making - * progress through a batch of folios is more important than processing - * them in order). Usually folio_lock() is the correct function to call. - * - * Context: Any context. - * Return: Whether the lock was successfully acquired. - */ -static inline bool folio_trylock(struct folio *folio) -{ - return likely(!test_and_set_bit_lock(PG_locked, folio_flags(folio, 0))); -} - -/* - * Return true if the page was successfully locked - */ -static inline bool trylock_page(struct page *page) -{ - return folio_trylock(page_folio(page)); -} - -/** - * folio_lock() - Lock this folio. - * @folio: The folio to lock. - * - * The folio lock protects against many things, probably more than it - * should. It is primarily held while a folio is being brought uptodate, - * either from its backing file or from swap. It is also held while a - * folio is being truncated from its address_space, so holding the lock - * is sufficient to keep folio->mapping stable. - * - * The folio lock is also held while write() is modifying the page to - * provide POSIX atomicity guarantees (as long as the write does not - * cross a page boundary). Other modifications to the data in the folio - * do not hold the folio lock and can race with writes, eg DMA and stores - * to mapped pages. - * - * Context: May sleep. If you need to acquire the locks of two or - * more folios, they must be in order of ascending index, if they are - * in the same address_space. If they are in different address_spaces, - * acquire the lock of the folio which belongs to the address_space which - * has the lowest address in memory first. - */ -static inline void folio_lock(struct folio *folio) -{ - might_sleep(); - if (!folio_trylock(folio)) - __folio_lock(folio); -} - -/** - * lock_page() - Lock the folio containing this page. - * @page: The page to lock. - * - * See folio_lock() for a description of what the lock protects. - * This is a legacy function and new code should probably use folio_lock() - * instead. - * - * Context: May sleep. Pages in the same folio share a lock, so do not - * attempt to lock two pages which share a folio. - */ -static inline void lock_page(struct page *page) -{ - struct folio *folio; - might_sleep(); - - folio = page_folio(page); - if (!folio_trylock(folio)) - __folio_lock(folio); -} - -/** - * folio_lock_killable() - Lock this folio, interruptible by a fatal signal. - * @folio: The folio to lock. - * - * Attempts to lock the folio, like folio_lock(), except that the sleep - * to acquire the lock is interruptible by a fatal signal. - * - * Context: May sleep; see folio_lock(). - * Return: 0 if the lock was acquired; -EINTR if a fatal signal was received. - */ -static inline int folio_lock_killable(struct folio *folio) -{ - might_sleep(); - if (!folio_trylock(folio)) - return __folio_lock_killable(folio); - return 0; -} - -/* - * folio_lock_or_retry - Lock the folio, unless this would block and the - * caller indicated that it can handle a retry. - * - * Return value and mmap_lock implications depend on flags; see - * __folio_lock_or_retry(). - */ -static inline vm_fault_t folio_lock_or_retry(struct folio *folio, - struct vm_fault *vmf) -{ - might_sleep(); - if (!folio_trylock(folio)) - return __folio_lock_or_retry(folio, vmf); - return 0; -} - -/* - * This is exported only for folio_wait_locked/folio_wait_writeback, etc., - * and should not be used directly. - */ -void folio_wait_bit(struct folio *folio, int bit_nr); -int folio_wait_bit_killable(struct folio *folio, int bit_nr); - -/* - * Wait for a folio to be unlocked. - * - * This must be called with the caller "holding" the folio, - * ie with increased folio reference count so that the folio won't - * go away during the wait. - */ -static inline void folio_wait_locked(struct folio *folio) -{ - if (folio_test_locked(folio)) - folio_wait_bit(folio, PG_locked); -} - -static inline int folio_wait_locked_killable(struct folio *folio) -{ - if (!folio_test_locked(folio)) - return 0; - return folio_wait_bit_killable(folio, PG_locked); -} - -void folio_end_read(struct folio *folio, bool success); void wait_on_page_writeback(struct page *page); void folio_wait_writeback(struct folio *folio); int folio_wait_writeback_killable(struct folio *folio); @@ -1268,9 +1101,6 @@ int filemap_migrate_folio(struct address_space *mapping, struct folio *dst, #else #define filemap_migrate_folio NULL #endif -void folio_end_private_2(struct folio *folio); -void folio_wait_private_2(struct folio *folio); -int folio_wait_private_2_killable(struct folio *folio); /* * Fault in userspace address range. diff --git a/mm/folio_wait.c b/mm/folio_wait.c index 18b42488ce37..06156e138c09 100644 --- a/mm/folio_wait.c +++ b/mm/folio_wait.c @@ -8,7 +8,7 @@ #include #include #include -#include +#include #include #include #include -- 2.39.5