The compound_head() function is a hot path. For example, the zap path calls it for every leaf page table entry. Rewrite the helper function in a branchless manner to eliminate the risk of CPU branch misprediction. Signed-off-by: Kiryl Shutsemau --- include/linux/page-flags.h | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h index 02a851ab7f5e..01d9893c4bd8 100644 --- a/include/linux/page-flags.h +++ b/include/linux/page-flags.h @@ -201,17 +201,15 @@ enum pageflags { static __always_inline unsigned long _compound_head(const struct page *page) { unsigned long info = READ_ONCE(page->compound_info); + unsigned long mask; + + if (!is_power_of_2(sizeof(struct page))) { + /* Bit 0 encodes PageTail() */ + if (info & 1) + return info - 1; - /* Bit 0 encodes PageTail() */ - if (!(info & 1)) return (unsigned long)page; - - /* - * If the size of struct page is not power-of-2, the rest if - * compound_info is the pointer to the head page. - */ - if (!is_power_of_2(sizeof(struct page))) - return info - 1; + } /* * If the size of struct page is power-of-2 it is set the rest of @@ -219,8 +217,17 @@ static __always_inline unsigned long _compound_head(const struct page *page) * page to the head page. * * No need to clear bit 0 in the mask as 'page' always has it clear. + * + * Let's do it in a branchless manner. */ - return (unsigned long)page & info; + + /* Non-tail: -1UL, Tail: 0 */ + mask = (info & 1) - 1; + + /* Non-tail: -1UL, Tail: info */ + mask |= info; + + return (unsigned long)page & mask; } #define compound_head(page) ((typeof(page))_compound_head(page)) -- 2.51.2