XDP and memory providers are currently rejected on the same device. Header-split RX keeps headers in normal memory, so a program can filter packets without reading the provider payload. Keep this restriction unless the driver declares xdp_unreadable support. Admit native XDP in either attach order on these drivers. Existing header-split checks require frags-aware programs. The driver must mark provider fragments and reject XDP_TX while those fragments remain. The core rejects helper access and frame redirects. Keep generic and hardware-offloaded XDP rejected. Generic XDP builds an xdp_buff from an skb without carrying the unreadable flag. Its transmit and redirect paths also bypass the native driver guard. Signed-off-by: Björn Töpel --- Documentation/networking/netmem.rst | 8 ++++++++ include/net/netdev_queues.h | 6 ++++++ include/net/xdp.h | 5 ++--- net/core/dev.c | 28 ++++++++++++++++++++++++++-- net/core/dev.h | 1 + net/core/netdev_rx_queue.c | 2 +- 6 files changed, 44 insertions(+), 6 deletions(-) diff --git a/Documentation/networking/netmem.rst b/Documentation/networking/netmem.rst index 6871dcdd700c..1ac94fc9d667 100644 --- a/Documentation/networking/netmem.rst +++ b/Documentation/networking/netmem.rst @@ -131,6 +131,14 @@ Driver RX Requirements circulation times, such as when userspace holds a reference in zerocopy scenarios. +8. XDP programs run next to unreadable memory providers only if the driver + sets netdev_queue_mgmt_ops.xdp_unreadable, and only in native mode. The + programs see the headers in the linear part, and helpers that reach + unreadable fragments fail. Attach payload fragments with + xdp_buff_add_frag(), which marks unreadable ones, and drop XDP_TX of a + buffer for which xdp_buff_is_frag_unreadable() is true: the payload is + mapped for receive only. Redirects of such a buffer fail in the core. + Driver TX Requirements ====================== diff --git a/include/net/netdev_queues.h b/include/net/netdev_queues.h index c5335e935b7d..4f6702b284b4 100644 --- a/include/net/netdev_queues.h +++ b/include/net/netdev_queues.h @@ -199,6 +199,11 @@ enum { * * @supported_params: Bitmask of supported parameters, see QCFG_*. * + * @xdp_unreadable: Native XDP programs may run on queues with unreadable + * memory providers. The driver builds XDP buffers with + * xdp_buff_add_frag(), and drops XDP_TX of a buffer for + * which xdp_buff_is_frag_unreadable() is true. + * * Note that @ndo_queue_mem_alloc and @ndo_queue_mem_free may be called while * the interface is closed. @ndo_queue_start and @ndo_queue_stop will only * be called for an interface which is open. @@ -229,6 +234,7 @@ struct netdev_queue_mgmt_ops { struct netlink_ext_ack *extack); unsigned int supported_params; + bool xdp_unreadable; }; void netdev_queue_config(struct net_device *dev, int rxq, diff --git a/include/net/xdp.h b/include/net/xdp.h index 20d173fe163c..167f733a0bb0 100644 --- a/include/net/xdp.h +++ b/include/net/xdp.h @@ -76,9 +76,8 @@ enum xdp_buff_flags { XDP_FLAGS_FRAGS_PF_MEMALLOC = BIT(1), /* xdp paged memory is under * pressure */ - /* frags have unreadable mem, this can't be true for real XDP packets, - * but drivers may use XDP helpers to construct Rx pkt state even when - * XDP program is not attached. + /* frags have unreadable mem: XDP programs reach only the linear part, + * and the buffer cannot become an xdp_frame */ XDP_FLAGS_FRAGS_UNREADABLE = BIT(2), }; diff --git a/net/core/dev.c b/net/core/dev.c index a230ecce9327..f0c3d56d5b83 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -10575,6 +10575,27 @@ u8 dev_xdp_sb_prog_count(struct net_device *dev) return count; } +/* Only native programs run next to unreadable memory providers. Drivers must + * mark unreadable fragments and reject XDP_TX while they remain attached. + * Tcp-data-split, which these providers need, also needs frags support. + */ +static bool netif_xdp_mp_mode_ok(const struct net_device *dev, + enum bpf_xdp_mode mode) +{ + return mode == XDP_MODE_DRV && dev->queue_mgmt_ops->xdp_unreadable; +} + +/* Can the XDP programs of @dev run next to a new memory provider? */ +bool netif_xdp_mp_ok(struct net_device *dev) +{ + int i; + + for (i = 0; i < __MAX_XDP_MODE; i++) + if (dev_xdp_prog(dev, i) && !netif_xdp_mp_mode_ok(dev, i)) + return false; + return true; +} + int netif_xdp_propagate(struct net_device *dev, struct netdev_bpf *bpf) { if (!dev->netdev_ops->ndo_bpf) @@ -10588,7 +10609,9 @@ int netif_xdp_propagate(struct net_device *dev, struct netdev_bpf *bpf) return -EBUSY; } - if (dev_get_min_mp_channel_count(dev)) { + if (dev_get_min_mp_channel_count(dev) && + (bpf->command != XDP_SETUP_PROG || + !netif_xdp_mp_mode_ok(dev, XDP_MODE_DRV))) { NL_SET_ERR_MSG(bpf->extack, "unable to propagate XDP to device using memory provider"); return -EBUSY; } @@ -10664,7 +10687,8 @@ static int dev_xdp_install(struct net_device *dev, enum bpf_xdp_mode mode, return -EBUSY; } - if (dev_get_min_mp_channel_count(dev)) { + if (dev_get_min_mp_channel_count(dev) && + !netif_xdp_mp_mode_ok(dev, mode)) { NL_SET_ERR_MSG(extack, "unable to install XDP to device using memory provider"); return -EBUSY; } diff --git a/net/core/dev.h b/net/core/dev.h index 4e9e8cd15697..a7036302f120 100644 --- a/net/core/dev.h +++ b/net/core/dev.h @@ -118,6 +118,7 @@ int netdev_queue_config_revalidate(struct net_device *dev, bool netif_rxq_has_mp(struct net_device *dev, unsigned int rxq_idx); bool netif_rxq_is_leased(struct net_device *dev, unsigned int rxq_idx); bool netif_is_queue_leasee(const struct net_device *dev); +bool netif_xdp_mp_ok(struct net_device *dev); void __netif_mp_uninstall_rxq(struct netdev_rx_queue *rxq, const struct pp_memory_provider_params *p); diff --git a/net/core/netdev_rx_queue.c b/net/core/netdev_rx_queue.c index a7261fedd5e7..dab5107e31f8 100644 --- a/net/core/netdev_rx_queue.c +++ b/net/core/netdev_rx_queue.c @@ -204,7 +204,7 @@ static int __netif_mp_open_rxq(struct net_device *dev, unsigned int rxq_idx, NL_SET_ERR_MSG(extack, "hds-thresh is not zero"); return -EINVAL; } - if (dev_xdp_prog_count(dev)) { + if (!netif_xdp_mp_ok(dev)) { NL_SET_ERR_MSG(extack, "unable to custom memory provider to device with XDP program attached"); return -EEXIST; } -- 2.56.1