Add XDP support for the PCIe FDMA path. The implementation operates on contiguous ATU-mapped buffers with memcpy-based XDP_TX, unlike the platform path which uses page_pool. XDP sees the frame with IFH and FCS stripped. These are removed in lan966x_fdma_pci_rx_check_frame() before the BPF program runs, because after the program returns the driver cannot tell whether the tail region was modified. The skb_pull/skb_trim previously done in lan966x_fdma_pci_rx_get_frame() are removed for the same reason; the frame pointer and length are pre-computed by rx_check_frame() and passed through rx_get_frame() and lan966x_xdp_pci_run() to the caller. lan966x_fdma_pci_xmit_xdpf() handles XDP_TX: it rebuilds a fresh IFH in the TX slot, copies the post-XDP frame after it, and lets HW insert a new FCS. lan966x_xdp_setup() is extended so the PCIe path skips the page_pool reload that the platform path needs. Only XDP_ACT_BASIC is supported. Tested-by: Herve Codina Signed-off-by: Daniel Machon --- .../ethernet/microchip/lan966x/lan966x_fdma_pci.c | 162 ++++++++++++++++++--- .../net/ethernet/microchip/lan966x/lan966x_main.c | 11 +- .../net/ethernet/microchip/lan966x/lan966x_main.h | 10 ++ .../net/ethernet/microchip/lan966x/lan966x_xdp.c | 10 ++ 4 files changed, 169 insertions(+), 24 deletions(-) diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_fdma_pci.c b/drivers/net/ethernet/microchip/lan966x/lan966x_fdma_pci.c index 0568251a95d9..cf3d3afbcc8a 100644 --- a/drivers/net/ethernet/microchip/lan966x/lan966x_fdma_pci.c +++ b/drivers/net/ethernet/microchip/lan966x/lan966x_fdma_pci.c @@ -1,5 +1,7 @@ // SPDX-License-Identifier: GPL-2.0+ +#include + #include "fdma_api.h" #include "lan966x_main.h" @@ -114,7 +116,118 @@ static bool lan966x_fdma_pci_rx_size_fits(struct fdma *fdma, u32 blockl) blockl <= fdma->db_size - XDP_PACKET_HEADROOM; } -static int lan966x_fdma_pci_rx_check_frame(struct lan966x_rx *rx, u64 *src_port) +static int lan966x_fdma_pci_xmit_xdpf(struct lan966x_port *port, + void *ptr, u32 len) +{ + struct lan966x *lan966x = port->lan966x; + struct lan966x_tx *tx = &lan966x->tx; + struct fdma *fdma = &tx->fdma; + int next_to_use, ret = 0; + void *virt_addr; + + spin_lock(&lan966x->tx_lock); + + next_to_use = lan966x_fdma_pci_get_next_dcb(fdma); + + if (next_to_use < 0) { + netif_stop_queue(port->dev); + ret = NETDEV_TX_BUSY; + goto out; + } + + if (!lan966x_fdma_pci_tx_size_fits(fdma, len)) { + port->dev->stats.tx_dropped++; + ret = -EINVAL; + goto out; + } + + /* virt_addr points to the IFH. */ + virt_addr = fdma_dataptr_virt_addr_contiguous(fdma, next_to_use, 0); + + /* Construct a fresh IFH. */ + memset(virt_addr, 0, IFH_LEN_BYTES); + lan966x_ifh_set_bypass(virt_addr, 1); + lan966x_ifh_set_port(virt_addr, BIT_ULL(port->chip_port)); + + /* Copy the (post-XDP) frame after the IFH. */ + memcpy(virt_addr + IFH_LEN_BYTES, ptr, len); + + /* Order frame write before DCB status write below. */ + dma_wmb(); + + /* Reserve ETH_FCS_LEN for the HW-inserted FCS (len is FCS-stripped). */ + fdma_dcb_add(fdma, + next_to_use, + 0, + FDMA_DCB_STATUS_INTR | + FDMA_DCB_STATUS_SOF | + FDMA_DCB_STATUS_EOF | + FDMA_DCB_STATUS_BLOCKO(0) | + FDMA_DCB_STATUS_BLOCKL(IFH_LEN_BYTES + len + ETH_FCS_LEN)); + + /* Start the transmission. */ + lan966x_fdma_tx_start(tx); + + port->dev->stats.tx_bytes += len; + port->dev->stats.tx_packets++; + +out: + spin_unlock(&lan966x->tx_lock); + + return ret; +} + +static int lan966x_xdp_pci_run(struct lan966x_port *port, void *data, + u32 data_len, void **xdp_data, u32 *xdp_len) +{ + /* Read once so the NULL check and bpf_prog_run_xdp() see the same + * pointer. + */ + struct bpf_prog *xdp_prog = READ_ONCE(port->xdp_prog); + struct lan966x *lan966x = port->lan966x; + struct fdma *fdma = &lan966x->rx.fdma; + struct xdp_buff xdp; + u32 act; + + if (!xdp_prog) + return FDMA_PASS; + + xdp_init_buff(&xdp, fdma->db_size, &port->xdp_rxq); + + /* hard_start is set to slot start (virt_addr is XDP_PACKET_HEADROOM + * into the slot). Headroom includes the IFH; BPF may grow into it + * via adjust_head. IFH is rebuilt on XDP_TX and unread on XDP_PASS. + */ + xdp_prepare_buff(&xdp, + data - XDP_PACKET_HEADROOM, + XDP_PACKET_HEADROOM + IFH_LEN_BYTES, + data_len, + false); + + act = bpf_prog_run_xdp(xdp_prog, &xdp); + + *xdp_data = xdp.data; + *xdp_len = xdp.data_end - xdp.data; + + switch (act) { + case XDP_PASS: + return FDMA_PASS; + case XDP_TX: + return lan966x_fdma_pci_xmit_xdpf(port, *xdp_data, *xdp_len) ? + FDMA_DROP : FDMA_TX; + default: + bpf_warn_invalid_xdp_action(port->dev, xdp_prog, act); + fallthrough; + case XDP_ABORTED: + trace_xdp_exception(port->dev, xdp_prog, act); + fallthrough; + case XDP_DROP: + return FDMA_DROP; + } +} + +static int lan966x_fdma_pci_rx_check_frame(struct lan966x_rx *rx, u64 *src_port, + void **data, u32 *data_len) { struct lan966x *lan966x = rx->lan966x; struct fdma *fdma = &rx->fdma; @@ -146,38 +259,33 @@ static int lan966x_fdma_pci_rx_check_frame(struct lan966x_rx *rx, u64 *src_port) if (!lan966x_fdma_pci_rx_size_fits(fdma, blockl)) return FDMA_ERROR; - return FDMA_PASS; + /* Present the Ethernet frame (no IFH, no FCS). HW re-inserts the + * FCS on TX; see lan966x_fdma_pci_xmit_xdpf(). May be overridden + * by XDP. The FCS strip is unconditional because NETIF_F_RXFCS + * is not advertised in hw_features. + */ + *data = virt_addr + IFH_LEN_BYTES; + *data_len = blockl - IFH_LEN_BYTES - ETH_FCS_LEN; + + return lan966x_xdp_pci_run(port, virt_addr, *data_len, data, data_len); } static struct sk_buff *lan966x_fdma_pci_rx_get_frame(struct lan966x_rx *rx, - u64 src_port) + u64 src_port, void *data, + u32 data_len) { struct lan966x *lan966x = rx->lan966x; - struct fdma *fdma = &rx->fdma; struct sk_buff *skb; - struct fdma_db *db; - u32 data_len; - - /* Get the received frame and create an SKB for it. */ - db = fdma_db_next_get(fdma); - data_len = FDMA_DCB_STATUS_BLOCKL(db->status); skb = napi_alloc_skb(&lan966x->napi, data_len); if (unlikely(!skb)) return NULL; - memcpy(skb->data, - fdma_dataptr_virt_addr_contiguous(fdma, - fdma->dcb_index, - fdma->db_index), - data_len); + memcpy(skb->data, data, data_len); skb_put(skb, data_len); skb->dev = lan966x->ports[src_port]->dev; - skb_pull(skb, IFH_LEN_BYTES); - - skb_trim(skb, skb->len - ETH_FCS_LEN); skb->protocol = eth_type_trans(skb, skb->dev); @@ -266,6 +374,8 @@ static int lan966x_fdma_pci_napi_poll(struct napi_struct *napi, int weight) struct sk_buff *skb; int counter = 0; u64 src_port; + u32 data_len; + void *data; /* Wake any stopped TX queues if a TX DCB is available. */ spin_lock(&lan966x->tx_lock); @@ -282,14 +392,26 @@ static int lan966x_fdma_pci_napi_poll(struct napi_struct *napi, int weight) /* Order DONE read before DCB/frame reads below. */ dma_rmb(); counter++; - switch (lan966x_fdma_pci_rx_check_frame(rx, &src_port)) { + switch (lan966x_fdma_pci_rx_check_frame(rx, + &src_port, + &data, + &data_len)) { case FDMA_PASS: break; case FDMA_ERROR: fdma_dcb_advance(fdma); goto allocate_new; + case FDMA_TX: + fdma_dcb_advance(fdma); + continue; + case FDMA_DROP: + fdma_dcb_advance(fdma); + continue; } - skb = lan966x_fdma_pci_rx_get_frame(rx, src_port); + skb = lan966x_fdma_pci_rx_get_frame(rx, + src_port, + data, + data_len); fdma_dcb_advance(fdma); if (!skb) goto allocate_new; diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_main.c b/drivers/net/ethernet/microchip/lan966x/lan966x_main.c index 0bbc9d40b69b..adbd16bab46d 100644 --- a/drivers/net/ethernet/microchip/lan966x/lan966x_main.c +++ b/drivers/net/ethernet/microchip/lan966x/lan966x_main.c @@ -877,10 +877,13 @@ static int lan966x_probe_port(struct lan966x *lan966x, u32 p, port->phylink = phylink; - if (lan966x->fdma) - dev->xdp_features = NETDEV_XDP_ACT_BASIC | - NETDEV_XDP_ACT_REDIRECT | - NETDEV_XDP_ACT_NDO_XMIT; + if (lan966x->fdma) { + dev->xdp_features = NETDEV_XDP_ACT_BASIC; + + if (!lan966x_is_pci(lan966x)) + dev->xdp_features |= NETDEV_XDP_ACT_REDIRECT | + NETDEV_XDP_ACT_NDO_XMIT; + } err = register_netdev(dev); if (err) { diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_main.h b/drivers/net/ethernet/microchip/lan966x/lan966x_main.h index e7fdd4447fb6..8911825eab77 100644 --- a/drivers/net/ethernet/microchip/lan966x/lan966x_main.h +++ b/drivers/net/ethernet/microchip/lan966x/lan966x_main.h @@ -595,6 +595,16 @@ int lan966x_qsys_sw_status(struct lan966x *lan966x); #if IS_ENABLED(CONFIG_MCHP_LAN966X_PCI) extern const struct lan966x_fdma_ops lan966x_fdma_pci_ops; + +static inline bool lan966x_is_pci(struct lan966x *lan966x) +{ + return lan966x->ops == &lan966x_fdma_pci_ops; +} +#else +static inline bool lan966x_is_pci(struct lan966x *lan966x) +{ + return false; +} #endif int lan966x_lag_port_join(struct lan966x_port *port, diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_xdp.c b/drivers/net/ethernet/microchip/lan966x/lan966x_xdp.c index 9ee61db8690b..b470f731e25c 100644 --- a/drivers/net/ethernet/microchip/lan966x/lan966x_xdp.c +++ b/drivers/net/ethernet/microchip/lan966x/lan966x_xdp.c @@ -24,6 +24,16 @@ static int lan966x_xdp_setup(struct net_device *dev, struct netdev_bpf *xdp) old_prog = xchg(&port->xdp_prog, xdp->prog); new_xdp = lan966x_xdp_present(lan966x); + /* PCIe FDMA uses contiguous buffers, so no page_pool reload + * is needed. Drain NAPI before freeing the old program so + * no in-flight poll holds a stale pointer. + */ + if (lan966x_is_pci(lan966x)) { + if (old_prog) + napi_synchronize(&lan966x->napi); + goto out; + } + if (old_xdp == new_xdp) goto out; -- 2.34.1