Transmit socket buffers use memory coming from skb_page_frag_refill(). Sockets whose leaf device has enabled pisa_tx_bufs should use PISA memory. Add a dedicated per-task page_frag, current->task_frag_pisa, populated with read-only PISA blocks via __GFP_PISA, and select it on TX when the socket's (or cork's) route goes out in the clear through a PISA-enabled device. Special paths that hold plaintext before encryption (IPsec, kTLS) or do not use a DMA device keep using regular pages; ESP output uses pisa_tx_bufs_gfp() to opt in when the egress device has pisa_tx_bufs enabled. Signed-off-by: Luigi Rizzo --- include/linux/pisa.h | 15 +++++++++++++++ include/linux/sched.h | 4 ++++ include/net/pisa.h | 33 +++++++++++++++++++++++++++++++++ kernel/exit.c | 2 ++ kernel/fork.c | 2 ++ net/core/sock.c | 10 ++++++++++ net/ipv4/esp4.c | 7 +++++-- net/ipv4/ip_output.c | 12 ++++++++---- net/ipv4/tcp.c | 3 ++- net/ipv6/esp6.c | 7 +++++-- net/ipv6/ip6_output.c | 12 ++++++++---- 11 files changed, 94 insertions(+), 13 deletions(-) diff --git a/include/linux/pisa.h b/include/linux/pisa.h index 6d184e393e297..84692852299f1 100644 --- a/include/linux/pisa.h +++ b/include/linux/pisa.h @@ -202,6 +202,18 @@ static inline bool pisa_page_can_recycle(const struct page *page) __pisa_page_can_recycle(page); } +/* Hooks for dup_task_struct() and do_exit() (guarded by @pisa_key on release). */ +static inline void pisa_task_frag_init(struct task_struct *tsk) +{ + tsk->task_frag_pisa.page = NULL; +} + +static inline void pisa_task_frag_release(struct task_struct *tsk) +{ + if (pisa_enabled() && tsk->task_frag_pisa.page) + put_page(tsk->task_frag_pisa.page); +} + #else /* !CONFIG_PISA */ static inline int pisa_meta_init(void) { return -EOPNOTSUPP; } @@ -240,6 +252,9 @@ static inline bool pisa_page_can_recycle(const struct page *page) return true; } +static inline void pisa_task_frag_init(struct task_struct *tsk) {} +static inline void pisa_task_frag_release(struct task_struct *tsk) {} + static inline bool pisa_upgrade_user_pages(struct iov_iter *iter, struct page **pages, size_t size, size_t offset, diff --git a/include/linux/sched.h b/include/linux/sched.h index e4f6323d78555..a8edeeb2661f4 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -1455,6 +1455,10 @@ struct task_struct { struct pipe_inode_info *splice_pipe; struct page_frag task_frag; +#ifdef CONFIG_PISA + /* TX frag that may hold PISA pages, see sk_page_frag_dst(). */ + struct page_frag task_frag_pisa; +#endif #ifdef CONFIG_ARCH_HAS_LAZY_MMU_MODE struct lazy_mmu_state lazy_mmu_state; diff --git a/include/net/pisa.h b/include/net/pisa.h index 1f95e67e99bf8..8e23d8b01e73d 100644 --- a/include/net/pisa.h +++ b/include/net/pisa.h @@ -4,6 +4,7 @@ #define _NET_PISA_H #include +#include #include /* @@ -72,10 +73,42 @@ static inline gfp_t pisa_tx_bufs_gfp(const struct sk_buff *skb) __pisa_tx_gfp(skb_dst(skb), PISA_TX_FLAG_BUFS | PISA_TX_XFRM_OK) : 0; } +/* Hook for skb_page_frag_refill() and __ip*_append_data() (@net_pisa_tx_bufs_key). */ +static inline bool is_pisa_pfrag(const struct page_frag *pfrag) +{ + return static_branch_unlikely(&net_pisa_tx_bufs_key) && + pfrag == ¤t->task_frag_pisa; +} + +/* Hook for task/socket TX page_frag selection (@net_pisa_tx_bufs_key). */ +static inline struct page_frag *task_page_frag(const struct dst_entry *dst) +{ + return use_pisa_tx_bufs(dst) ? ¤t->task_frag_pisa : ¤t->task_frag; +} + +static inline struct page_frag *sk_page_frag_dst(struct sock *sk, + const struct dst_entry *dst) +{ + return static_branch_unlikely(&net_pisa_tx_bufs_key) && + sk->sk_use_task_frag ? task_page_frag(dst) : sk_page_frag(sk); +} + #else /* !CONFIG_PISA */ static inline bool use_pisa_tx_bufs(const struct dst_entry *dst) { return false; } static inline gfp_t pisa_tx_bufs_gfp(const struct sk_buff *skb) { return 0; } +static inline bool is_pisa_pfrag(const struct page_frag *pfrag) { return false; } + +static inline struct page_frag *task_page_frag(const struct dst_entry *dst) +{ + return ¤t->task_frag; +} + +static inline struct page_frag *sk_page_frag_dst(struct sock *sk, + const struct dst_entry *dst) +{ + return sk_page_frag(sk); +} #endif /* CONFIG_PISA */ #endif /* _NET_PISA_H */ diff --git a/kernel/exit.c b/kernel/exit.c index d5e7bc2af804a..ac7ca1fb180c3 100644 --- a/kernel/exit.c +++ b/kernel/exit.c @@ -6,6 +6,7 @@ */ #include +#include #include #include #include @@ -1009,6 +1010,7 @@ void __noreturn do_exit(long code) if (tsk->task_frag.page) put_page(tsk->task_frag.page); + pisa_task_frag_release(tsk); exit_task_stack_account(tsk); diff --git a/kernel/fork.c b/kernel/fork.c index 5efceb8d58a78..d235d7ab04687 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -44,6 +44,7 @@ #include #include #include +#include #include #include #include @@ -982,6 +983,7 @@ static struct task_struct *dup_task_struct(struct task_struct *orig, int node) #endif tsk->splice_pipe = NULL; tsk->task_frag.page = NULL; + pisa_task_frag_init(tsk); tsk->wake_q.next = NULL; tsk->worker_private = NULL; diff --git a/net/core/sock.c b/net/core/sock.c index ad414878dc5ba..56dea49baaec2 100644 --- a/net/core/sock.c +++ b/net/core/sock.c @@ -115,6 +115,7 @@ #include #include #include +#include #include #include #include @@ -147,6 +148,7 @@ #include #include #include +#include #include @@ -3257,13 +3259,21 @@ DEFINE_STATIC_KEY_FALSE(net_high_order_alloc_disable_key); */ bool skb_page_frag_refill(unsigned int sz, struct page_frag *pfrag, gfp_t gfp) { + /* PISA pages only go into current->task_frag_pisa, see sk_page_frag_dst(). */ + if (is_pisa_pfrag(pfrag)) + gfp |= __GFP_PISA; + if (pfrag->page) { if (page_ref_count(pfrag->page) == 1) { + if (!!(gfp & __GFP_PISA) != is_pisa_page(pfrag->page) || + ((gfp & __GFP_PISA) && !pisa_page_can_recycle(pfrag->page))) + goto drop_page; pfrag->offset = 0; return true; } if (pfrag->offset + sz <= pfrag->size) return true; +drop_page: put_page(pfrag->page); } diff --git a/net/ipv4/esp4.c b/net/ipv4/esp4.c index ae701561d9f12..8ed713772e4ee 100644 --- a/net/ipv4/esp4.c +++ b/net/ipv4/esp4.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -451,7 +452,8 @@ int esp_output_head(struct xfrm_state *x, struct sk_buff *skb, struct esp_info * spin_lock_bh(&x->lock); - if (unlikely(!skb_page_frag_refill(allocsize, pfrag, GFP_ATOMIC))) { + if (unlikely(!skb_page_frag_refill(allocsize, pfrag, + GFP_ATOMIC | pisa_tx_bufs_gfp(skb)))) { spin_unlock_bh(&x->lock); goto cow; } @@ -560,7 +562,8 @@ int esp_output_tail(struct xfrm_state *x, struct sk_buff *skb, struct esp_info * allocsize = ALIGN(skb->data_len, L1_CACHE_BYTES); spin_lock_bh(&x->lock); - if (unlikely(!skb_page_frag_refill(allocsize, pfrag, GFP_ATOMIC))) { + if (unlikely(!skb_page_frag_refill(allocsize, pfrag, + GFP_ATOMIC | pisa_tx_bufs_gfp(skb)))) { spin_unlock_bh(&x->lock); goto error_free; } diff --git a/net/ipv4/ip_output.c b/net/ipv4/ip_output.c index d42d776af273e..50b8c92c3cd5e 100644 --- a/net/ipv4/ip_output.c +++ b/net/ipv4/ip_output.c @@ -71,6 +71,7 @@ #include #include #include +#include #include #include #include @@ -986,7 +987,10 @@ static int __ip_append_data(struct sock *sk, exthdrlen = !skb ? rt->dst.header_len : 0; mtu = cork->gso_size ? IP_MAX_MTU : cork->fragsize; - paged = !!cork->gso_size; + paged = !!cork->gso_size || + (getfrag == ip_generic_getfrag && + (rt->dst.dev->features & NETIF_F_SG) && + is_pisa_pfrag(pfrag)); hh_len = LL_RESERVED_SPACE(rt->dst.dev); @@ -1383,8 +1387,8 @@ int ip_append_data(struct sock *sk, struct flowi4 *fl4, } return __ip_append_data(sk, fl4, &sk->sk_write_queue, &inet->cork.base, - sk_page_frag(sk), getfrag, - from, length, transhdrlen, flags); + sk_page_frag_dst(sk, inet->cork.base.dst), + getfrag, from, length, transhdrlen, flags); } static void ip_cork_release(struct inet_cork *cork) @@ -1592,7 +1596,7 @@ struct sk_buff *ip_make_skb(struct sock *sk, return ERR_PTR(err); err = __ip_append_data(sk, fl4, &queue, cork, - ¤t->task_frag, getfrag, + sk_page_frag_dst(sk, cork->dst), getfrag, from, length, transhdrlen, flags); if (err) { __ip_flush_pending_frames(sk, &queue, cork); diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c index 832b25ddd3c2e..3c3edc48705d7 100644 --- a/net/ipv4/tcp.c +++ b/net/ipv4/tcp.c @@ -273,6 +273,7 @@ #include #include #include +#include #include #include #include @@ -1295,7 +1296,7 @@ int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size) if (zc == 0) { bool merge = true; int i = skb_shinfo(skb)->nr_frags; - struct page_frag *pfrag = sk_page_frag(sk); + struct page_frag *pfrag = sk_page_frag_dst(sk, __sk_dst_get(sk)); if (!sk_page_frag_refill(sk, pfrag)) goto wait_for_space; diff --git a/net/ipv6/esp6.c b/net/ipv6/esp6.c index d654f0f442388..98ed9120ae7b4 100644 --- a/net/ipv6/esp6.c +++ b/net/ipv6/esp6.c @@ -36,6 +36,7 @@ #include #include #include +#include #include #include @@ -481,7 +482,8 @@ int esp6_output_head(struct xfrm_state *x, struct sk_buff *skb, struct esp_info spin_lock_bh(&x->lock); - if (unlikely(!skb_page_frag_refill(allocsize, pfrag, GFP_ATOMIC))) { + if (unlikely(!skb_page_frag_refill(allocsize, pfrag, + GFP_ATOMIC | pisa_tx_bufs_gfp(skb)))) { spin_unlock_bh(&x->lock); goto cow; } @@ -592,7 +594,8 @@ int esp6_output_tail(struct xfrm_state *x, struct sk_buff *skb, struct esp_info allocsize = ALIGN(skb->data_len, L1_CACHE_BYTES); spin_lock_bh(&x->lock); - if (unlikely(!skb_page_frag_refill(allocsize, pfrag, GFP_ATOMIC))) { + if (unlikely(!skb_page_frag_refill(allocsize, pfrag, + GFP_ATOMIC | pisa_tx_bufs_gfp(skb)))) { spin_unlock_bh(&x->lock); goto error_free; } diff --git a/net/ipv6/ip6_output.c b/net/ipv6/ip6_output.c index cb648c8394310..8f6f0bed15499 100644 --- a/net/ipv6/ip6_output.c +++ b/net/ipv6/ip6_output.c @@ -40,6 +40,7 @@ #include #include +#include #include #include @@ -1489,7 +1490,10 @@ static int __ip6_append_data(struct sock *sk, dst_exthdrlen = rt->dst.header_len - rt->rt6i_nfheader_len; } - paged = !!cork->gso_size; + paged = !!cork->gso_size || + (getfrag == ip_generic_getfrag && + (rt->dst.dev->features & NETIF_F_SG) && + is_pisa_pfrag(pfrag)); mtu = cork->gso_size ? IP6_MAX_MTU : cork->fragsize; orig_mtu = mtu; @@ -1892,8 +1896,8 @@ int ip6_append_data(struct sock *sk, } return __ip6_append_data(sk, &sk->sk_write_queue, &inet->cork, - sk_page_frag(sk), getfrag, - from, length, transhdrlen, flags); + sk_page_frag_dst(sk, inet->cork.base.dst), + getfrag, from, length, transhdrlen, flags); } EXPORT_SYMBOL_GPL(ip6_append_data); @@ -2095,7 +2099,7 @@ struct sk_buff *ip6_make_skb(struct sock *sk, } err = __ip6_append_data(sk, &queue, cork, - ¤t->task_frag, getfrag, from, + sk_page_frag_dst(sk, cork->base.dst), getfrag, from, length + exthdrlen, transhdrlen + exthdrlen, flags); if (err) { -- 2.56.1.809.ga0b61035b5-goog