Add XDP program attachment via ndo_bpf and execute XDP programs in the RX path. XDP_PASS builds an SKB from the xdp_buff (handling xdp_adjust_head/tail), XDP_DROP returns the page to page_pool without SKB allocation. XDP_TX and XDP_REDIRECT are not yet supported and return XDP_ABORTED. Advertise NETDEV_XDP_ACT_BASIC in xdp_features. Signed-off-by: Nicolai Buchwitz --- .../net/ethernet/broadcom/genet/bcmgenet.c | 129 +++++++++++++++--- .../net/ethernet/broadcom/genet/bcmgenet.h | 4 + 2 files changed, 116 insertions(+), 17 deletions(-) diff --git a/drivers/net/ethernet/broadcom/genet/bcmgenet.c b/drivers/net/ethernet/broadcom/genet/bcmgenet.c index 5bedc18685b0..ee1d4ecc2b87 100644 --- a/drivers/net/ethernet/broadcom/genet/bcmgenet.c +++ b/drivers/net/ethernet/broadcom/genet/bcmgenet.c @@ -36,6 +36,8 @@ #include #include #include +#include +#include #include @@ -2276,6 +2278,56 @@ static int bcmgenet_rx_refill(struct bcmgenet_rx_ring *ring, return 0; } +static struct sk_buff *bcmgenet_xdp_build_skb(struct bcmgenet_rx_ring *ring, + struct xdp_buff *xdp) +{ + unsigned int metasize; + struct sk_buff *skb; + + skb = napi_build_skb(xdp->data_hard_start, PAGE_SIZE); + if (unlikely(!skb)) + return NULL; + + skb_mark_for_recycle(skb); + + metasize = xdp->data - xdp->data_meta; + skb_reserve(skb, xdp->data - xdp->data_hard_start); + __skb_put(skb, xdp->data_end - xdp->data); + + if (metasize) + skb_metadata_set(skb, metasize); + + return skb; +} + +static unsigned int bcmgenet_run_xdp(struct bcmgenet_rx_ring *ring, + struct bpf_prog *prog, + struct xdp_buff *xdp, + struct page *rx_page) +{ + unsigned int act; + + if (!prog) + return XDP_PASS; + + act = bpf_prog_run_xdp(prog, xdp); + + switch (act) { + case XDP_PASS: + return XDP_PASS; + case XDP_DROP: + page_pool_put_full_page(ring->page_pool, rx_page, true); + return XDP_DROP; + default: + bpf_warn_invalid_xdp_action(ring->priv->dev, prog, act); + fallthrough; + case XDP_ABORTED: + trace_xdp_exception(ring->priv->dev, prog, act); + page_pool_put_full_page(ring->page_pool, rx_page, true); + return XDP_ABORTED; + } +} + /* bcmgenet_desc_rx - descriptor based rx process. * this could be called from bottom half, or from NAPI polling method. */ @@ -2285,6 +2337,7 @@ static unsigned int bcmgenet_desc_rx(struct bcmgenet_rx_ring *ring, struct bcmgenet_rx_stats64 *stats = &ring->stats64; struct bcmgenet_priv *priv = ring->priv; struct net_device *dev = priv->dev; + struct bpf_prog *xdp_prog; struct enet_cb *cb; struct sk_buff *skb; u32 dma_length_status; @@ -2295,6 +2348,8 @@ static unsigned int bcmgenet_desc_rx(struct bcmgenet_rx_ring *ring, unsigned int p_index, mask; unsigned int discards; + xdp_prog = READ_ONCE(priv->xdp_prog); + /* Clear status before servicing to reduce spurious interrupts */ mask = 1 << (UMAC_IRQ1_RX_INTR_SHIFT + ring->index); bcmgenet_intrl2_1_writel(priv, mask, INTRL2_CPU_CLEAR); @@ -2326,9 +2381,12 @@ static unsigned int bcmgenet_desc_rx(struct bcmgenet_rx_ring *ring, (rxpktprocessed < budget)) { struct status_64 *status; struct page *rx_page; + unsigned int xdp_act; unsigned int rx_off; - __be16 rx_csum; + struct xdp_buff xdp; + __be16 rx_csum = 0; void *hard_start; + int pkt_len; cb = &priv->rx_cbs[ring->read_ptr]; @@ -2413,30 +2471,34 @@ static unsigned int bcmgenet_desc_rx(struct bcmgenet_rx_ring *ring, goto next; } /* error packet */ - /* Build SKB from the page - data starts at hard_start, - * frame begins after RSB(64) + pad(2) = 66 bytes. + pkt_len = len - GENET_RSB_PAD; + if (priv->crc_fwd_en) + pkt_len -= ETH_FCS_LEN; + + /* Save rx_csum before XDP runs - an XDP program + * could overwrite the RSB via bpf_xdp_adjust_head. */ - skb = napi_build_skb(hard_start, PAGE_SIZE - XDP_PACKET_HEADROOM); - if (unlikely(!skb)) { - BCMGENET_STATS64_INC(stats, dropped); - page_pool_put_full_page(ring->page_pool, rx_page, - true); - goto next; - } + if (dev->features & NETIF_F_RXCSUM) + rx_csum = (__force __be16)(status->rx_csum & 0xffff); - skb_mark_for_recycle(skb); + xdp_init_buff(&xdp, PAGE_SIZE, &ring->xdp_rxq); + xdp_prepare_buff(&xdp, page_address(rx_page), + GENET_RX_HEADROOM, pkt_len, true); - /* Reserve the RSB + pad, then set the data length */ - skb_reserve(skb, GENET_RSB_PAD); - __skb_put(skb, len - GENET_RSB_PAD); + xdp_act = bcmgenet_run_xdp(ring, xdp_prog, &xdp, rx_page); + if (xdp_act != XDP_PASS) + goto next; - if (priv->crc_fwd_en) { - skb_trim(skb, skb->len - ETH_FCS_LEN); + skb = bcmgenet_xdp_build_skb(ring, &xdp); + if (unlikely(!skb)) { + BCMGENET_STATS64_INC(stats, dropped); + page_pool_put_full_page(ring->page_pool, + rx_page, true); + goto next; } /* Set up checksum offload */ if (dev->features & NETIF_F_RXCSUM) { - rx_csum = (__force __be16)(status->rx_csum & 0xffff); if (rx_csum) { skb->csum = (__force __wsum)ntohs(rx_csum); skb->ip_summed = CHECKSUM_COMPLETE; @@ -3750,6 +3812,37 @@ static int bcmgenet_change_carrier(struct net_device *dev, bool new_carrier) return 0; } +static int bcmgenet_xdp_setup(struct net_device *dev, + struct netdev_bpf *xdp) +{ + struct bcmgenet_priv *priv = netdev_priv(dev); + struct bpf_prog *old_prog; + struct bpf_prog *prog = xdp->prog; + + if (prog && dev->mtu > PAGE_SIZE - GENET_RX_HEADROOM - + SKB_DATA_ALIGN(sizeof(struct skb_shared_info))) { + NL_SET_ERR_MSG_MOD(xdp->extack, + "MTU too large for single-page XDP buffer"); + return -EOPNOTSUPP; + } + + old_prog = xchg(&priv->xdp_prog, prog); + if (old_prog) + bpf_prog_put(old_prog); + + return 0; +} + +static int bcmgenet_xdp(struct net_device *dev, struct netdev_bpf *xdp) +{ + switch (xdp->command) { + case XDP_SETUP_PROG: + return bcmgenet_xdp_setup(dev, xdp); + default: + return -EOPNOTSUPP; + } +} + static const struct net_device_ops bcmgenet_netdev_ops = { .ndo_open = bcmgenet_open, .ndo_stop = bcmgenet_close, @@ -3761,6 +3854,7 @@ static const struct net_device_ops bcmgenet_netdev_ops = { .ndo_set_features = bcmgenet_set_features, .ndo_get_stats64 = bcmgenet_get_stats64, .ndo_change_carrier = bcmgenet_change_carrier, + .ndo_bpf = bcmgenet_xdp, }; /* GENET hardware parameters/characteristics */ @@ -4063,6 +4157,7 @@ static int bcmgenet_probe(struct platform_device *pdev) NETIF_F_RXCSUM; dev->hw_features |= dev->features; dev->vlan_features |= dev->features; + dev->xdp_features = NETDEV_XDP_ACT_BASIC; netdev_sw_irq_coalesce_default_on(dev); diff --git a/drivers/net/ethernet/broadcom/genet/bcmgenet.h b/drivers/net/ethernet/broadcom/genet/bcmgenet.h index da7b7fee896f..3d65f0e4b4b4 100644 --- a/drivers/net/ethernet/broadcom/genet/bcmgenet.h +++ b/drivers/net/ethernet/broadcom/genet/bcmgenet.h @@ -16,6 +16,7 @@ #include #include #include +#include #include #include "../unimac.h" @@ -670,6 +671,9 @@ struct bcmgenet_priv { u8 sopass[SOPASS_MAX]; struct bcmgenet_mib_counters mib; + + /* XDP */ + struct bpf_prog *xdp_prog; }; static inline bool bcmgenet_has_40bits(struct bcmgenet_priv *priv) -- 2.51.0