hugetlb_reserve_pages / hugetlb_unreserve_pages have two callers and one of them is outside hugetlb. Make both functions to take PAGE granularity index to be consistent with the rest of MM. Signed-off-by: Jane Chu --- Documentation/mm/hugetlbfs_reserv.rst | 19 ++++++++++--------- fs/hugetlbfs/inode.c | 25 +++++++++++-------------- mm/hugetlb.c | 23 ++++++++++++++++++----- mm/memfd.c | 20 ++++++-------------- 4 files changed, 45 insertions(+), 42 deletions(-) diff --git a/Documentation/mm/hugetlbfs_reserv.rst b/Documentation/mm/hugetlbfs_reserv.rst index a49115db18c7..880e9ccd5b57 100644 --- a/Documentation/mm/hugetlbfs_reserv.rst +++ b/Documentation/mm/hugetlbfs_reserv.rst @@ -112,11 +112,12 @@ flag was specified in either the shmget() or mmap() call. If NORESERVE was specified, then this routine returns immediately as no reservations are desired. -The arguments 'from' and 'to' are huge page indices into the mapping or -underlying file. For shmget(), 'from' is always 0 and 'to' corresponds to -the length of the segment/mapping. For mmap(), the offset argument could -be used to specify the offset into the underlying file. In such a case, -the 'from' and 'to' arguments have been adjusted by this offset. +The arguments 'from' and 'to' are base page indices into the mapping or +underlying file that must be huge page aligned. For shmget(), +'from' is always 0 and 'to' corresponds to the length of the segment/mapping. +For mmap(), the offset argument could be used to specify the offset into +the underlying file. In such a case, the 'from' and 'to' arguments have been +adjusted by this offset. One of the big differences between PRIVATE and SHARED mappings is the way in which reservations are represented in the reservation map. @@ -136,10 +137,10 @@ to indicate this VMA owns the reservations. The reservation map is consulted to determine how many huge page reservations are needed for the current mapping/segment. For private mappings, this is -always the value (to - from). However, for shared mappings it is possible that -some reservations may already exist within the range (to - from). See the -section :ref:`Reservation Map Modifications ` -for details on how this is accomplished. +always the number of huge pages covered by the range [from, to). +However, for shared mappings it is possible that some reservations may already +exist within the range [from, to). See the section :ref:`Reservation Map +Modifications ` for details on how this is accomplished. The mapping may be associated with a subpool. If so, the subpool is consulted to ensure there is sufficient space for the mapping. It is possible that the diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c index 0b49a79efb08..fe1ebfd604dc 100644 --- a/fs/hugetlbfs/inode.c +++ b/fs/hugetlbfs/inode.c @@ -150,10 +150,8 @@ static int hugetlbfs_file_mmap(struct file *file, struct vm_area_struct *vma) if (inode->i_flags & S_PRIVATE) vma_flags_set(&vma_flags, VMA_NORESERVE_BIT); - if (hugetlb_reserve_pages(inode, - vma->vm_pgoff >> huge_page_order(h), - len >> huge_page_shift(h), vma, - vma_flags) < 0) + if (hugetlb_reserve_pages(inode, vma->vm_pgoff, len >> PAGE_SHIFT, + vma, vma_flags) < 0) goto out; ret = 0; @@ -389,7 +387,7 @@ hugetlb_vmdelete_list(struct rb_root_cached *root, pgoff_t start, pgoff_t end, */ static void remove_inode_single_folio(struct hstate *h, struct inode *inode, struct address_space *mapping, struct folio *folio, - pgoff_t index, bool truncate_op) + pgoff_t idx, bool truncate_op) { /* * If folio is mapped, it was faulted in after being @@ -401,7 +399,7 @@ static void remove_inode_single_folio(struct hstate *h, struct inode *inode, */ folio_lock(folio); if (unlikely(folio_mapped(folio))) - hugetlb_unmap_file_folio(h, mapping, folio, index); + hugetlb_unmap_file_folio(h, mapping, folio, idx); /* * We must remove the folio from page cache before removing @@ -413,8 +411,10 @@ static void remove_inode_single_folio(struct hstate *h, struct inode *inode, VM_BUG_ON_FOLIO(folio_test_hugetlb_restore_reserve(folio), folio); hugetlb_delete_from_page_cache(folio); if (!truncate_op) { - if (unlikely(hugetlb_unreserve_pages(inode, index, - index + 1, 1))) + pgoff_t index = idx << huge_page_order(h); + pgoff_t next = index + pages_per_huge_page(h); + + if (unlikely(hugetlb_unreserve_pages(inode, index, next, 1))) hugetlb_fix_reserve_counts(inode); } @@ -476,9 +476,8 @@ static void remove_inode_hugepages(struct inode *inode, loff_t lstart, } if (truncate_op) - (void)hugetlb_unreserve_pages(inode, - lstart >> huge_page_shift(h), - LONG_MAX, freed); + (void)hugetlb_unreserve_pages(inode, lstart >> PAGE_SHIFT, + LONG_MAX, freed); } static void hugetlbfs_evict_inode(struct inode *inode) @@ -1429,9 +1428,7 @@ struct file *hugetlb_file_setup(const char *name, size_t size, inode->i_size = size; clear_nlink(inode); - if (hugetlb_reserve_pages(inode, 0, - size >> huge_page_shift(hstate_inode(inode)), NULL, - acctflag) < 0) + if (hugetlb_reserve_pages(inode, 0, size >> PAGE_SHIFT, NULL, acctflag) < 0) file = ERR_PTR(-ENOMEM); else file = alloc_file_pseudo(inode, mnt, name, O_RDWR, diff --git a/mm/hugetlb.c b/mm/hugetlb.c index a677ea774143..302f9cf9ef6b 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -6528,7 +6528,7 @@ long hugetlb_change_protection(struct vm_area_struct *vma, */ long hugetlb_reserve_pages(struct inode *inode, - long from, long to, + long from_idx, long to_idx, struct vm_area_struct *vma, vma_flags_t vma_flags) { @@ -6538,14 +6538,21 @@ long hugetlb_reserve_pages(struct inode *inode, struct resv_map *resv_map; struct hugetlb_cgroup *h_cg = NULL; long gbl_reserve, regions_needed = 0; + long from, to; int err; + VM_WARN_ON(!IS_ALIGNED(from_idx, 1UL << huge_page_order(h))); + VM_WARN_ON(!IS_ALIGNED(to_idx, 1UL << huge_page_order(h))); + /* This should never happen */ - if (from > to) { + if (from_idx > to_idx) { VM_WARN(1, "%s called with a negative range\n", __func__); return -EINVAL; } + from = from_idx >> huge_page_order(h); + to = to_idx >> huge_page_order(h); + /* * vma specific semaphore used for pmd sharing and fault/truncation * synchronization @@ -6715,14 +6722,20 @@ long hugetlb_reserve_pages(struct inode *inode, return err; } -long hugetlb_unreserve_pages(struct inode *inode, long start, long end, - long freed) +long hugetlb_unreserve_pages(struct inode *inode, long start_idx, + long end_idx, long freed) { struct hstate *h = hstate_inode(inode); struct resv_map *resv_map = inode_resv_map(inode); long chg = 0; struct hugepage_subpool *spool = subpool_inode(inode); - long gbl_reserve; + long gbl_reserve, start, end; + + VM_WARN_ON(!IS_ALIGNED(start_idx, 1UL << huge_page_order(h))); + VM_WARN_ON(!IS_ALIGNED(end_idx, 1UL << huge_page_order(h))); + + start = start_idx >> huge_page_order(h); + end = end_idx >> huge_page_order(h); /* * Since this routine can be called in the evict inode path for all diff --git a/mm/memfd.c b/mm/memfd.c index 0b5e8f111b39..24fefb1d2761 100644 --- a/mm/memfd.c +++ b/mm/memfd.c @@ -79,22 +79,19 @@ struct folio *memfd_alloc_folio(struct file *memfd, pgoff_t index) */ struct inode *inode = file_inode(memfd); struct hstate *h = hstate_file(memfd); - pgoff_t idx; + pgoff_t next; int err = -ENOMEM; long nr_resv; gfp_mask = htlb_alloc_mask(h); gfp_mask &= ~(__GFP_HIGHMEM | __GFP_MOVABLE); - idx = index >> huge_page_order(h); + next = index + pages_per_huge_page(h); - nr_resv = hugetlb_reserve_pages(inode, idx, idx + 1, NULL, EMPTY_VMA_FLAGS); + nr_resv = hugetlb_reserve_pages(inode, index, next, NULL, EMPTY_VMA_FLAGS); if (nr_resv < 0) return ERR_PTR(nr_resv); - folio = alloc_hugetlb_folio_reserve(h, - numa_node_id(), - NULL, - gfp_mask); + folio = alloc_hugetlb_folio_reserve(h, numa_node_id(), NULL, gfp_mask); if (folio) { u32 hash; @@ -119,13 +116,8 @@ struct folio *memfd_alloc_folio(struct file *memfd, pgoff_t index) */ hash = hugetlb_fault_mutex_hash(memfd->f_mapping, index); mutex_lock(&hugetlb_fault_mutex_table[hash]); - - err = hugetlb_add_to_page_cache(folio, - memfd->f_mapping, - index); - + err = hugetlb_add_to_page_cache(folio, memfd->f_mapping, index); mutex_unlock(&hugetlb_fault_mutex_table[hash]); - if (err) { folio_put(folio); goto err_unresv; @@ -137,7 +129,7 @@ struct folio *memfd_alloc_folio(struct file *memfd, pgoff_t index) } err_unresv: if (nr_resv > 0) - hugetlb_unreserve_pages(inode, idx, idx + 1, 0); + hugetlb_unreserve_pages(inode, index, next, 0); return ERR_PTR(err); } #endif -- 2.43.5