Add XDP program attachment via ndo_bpf and execute XDP programs in the RX path. XDP_PASS builds an SKB from the xdp_buff (handling xdp_adjust_head/tail), XDP_DROP returns the page to page_pool without SKB allocation. XDP_TX and XDP_REDIRECT are not yet supported and return XDP_ABORTED. Advertise NETDEV_XDP_ACT_BASIC in xdp_features. Signed-off-by: Nicolai Buchwitz --- .../net/ethernet/broadcom/genet/bcmgenet.c | 150 +++++++++++++++--- .../net/ethernet/broadcom/genet/bcmgenet.h | 4 + 2 files changed, 136 insertions(+), 18 deletions(-) diff --git a/drivers/net/ethernet/broadcom/genet/bcmgenet.c b/drivers/net/ethernet/broadcom/genet/bcmgenet.c index 9f81192d9e31..b45ba2c2857e 100644 --- a/drivers/net/ethernet/broadcom/genet/bcmgenet.c +++ b/drivers/net/ethernet/broadcom/genet/bcmgenet.c @@ -35,6 +35,8 @@ #include #include #include +#include +#include #include @@ -2274,6 +2276,54 @@ static int bcmgenet_rx_refill(struct bcmgenet_rx_ring *ring, return 0; } +static struct sk_buff *bcmgenet_xdp_build_skb(struct bcmgenet_rx_ring *ring, + struct xdp_buff *xdp, + struct page *rx_page) +{ + unsigned int metasize; + struct sk_buff *skb; + + skb = napi_build_skb(xdp->data_hard_start, PAGE_SIZE); + if (unlikely(!skb)) + return NULL; + + skb_mark_for_recycle(skb); + + metasize = xdp->data - xdp->data_meta; + skb_reserve(skb, xdp->data - xdp->data_hard_start); + __skb_put(skb, xdp->data_end - xdp->data); + + if (metasize) + skb_metadata_set(skb, metasize); + + return skb; +} + +static unsigned int bcmgenet_run_xdp(struct bcmgenet_rx_ring *ring, + struct bpf_prog *prog, + struct xdp_buff *xdp, + struct page *rx_page) +{ + unsigned int act; + + act = bpf_prog_run_xdp(prog, xdp); + + switch (act) { + case XDP_PASS: + return XDP_PASS; + case XDP_DROP: + page_pool_put_full_page(ring->page_pool, rx_page, true); + return XDP_DROP; + default: + bpf_warn_invalid_xdp_action(ring->priv->dev, prog, act); + fallthrough; + case XDP_ABORTED: + trace_xdp_exception(ring->priv->dev, prog, act); + page_pool_put_full_page(ring->page_pool, rx_page, true); + return XDP_ABORTED; + } +} + /* bcmgenet_desc_rx - descriptor based rx process. * this could be called from bottom half, or from NAPI polling method. */ @@ -2283,6 +2333,7 @@ static unsigned int bcmgenet_desc_rx(struct bcmgenet_rx_ring *ring, struct bcmgenet_rx_stats64 *stats = &ring->stats64; struct bcmgenet_priv *priv = ring->priv; struct net_device *dev = priv->dev; + struct bpf_prog *xdp_prog; struct enet_cb *cb; struct sk_buff *skb; u32 dma_length_status; @@ -2293,6 +2344,8 @@ static unsigned int bcmgenet_desc_rx(struct bcmgenet_rx_ring *ring, unsigned int p_index, mask; unsigned int discards; + xdp_prog = READ_ONCE(priv->xdp_prog); + /* Clear status before servicing to reduce spurious interrupts */ mask = 1 << (UMAC_IRQ1_RX_INTR_SHIFT + ring->index); bcmgenet_intrl2_1_writel(priv, mask, INTRL2_CPU_CLEAR); @@ -2403,26 +2456,52 @@ static unsigned int bcmgenet_desc_rx(struct bcmgenet_rx_ring *ring, goto next; } /* error packet */ - /* Build SKB from the page - data starts at hard_start, - * frame begins after RSB(64) + pad(2) = 66 bytes. - */ - skb = napi_build_skb(hard_start, PAGE_SIZE - GENET_XDP_HEADROOM); - if (unlikely(!skb)) { - BCMGENET_STATS64_INC(stats, dropped); - page_pool_put_full_page(ring->page_pool, rx_page, - true); - goto next; - } - - skb_mark_for_recycle(skb); + /* XDP: frame data starts after RSB + pad */ + if (xdp_prog) { + struct xdp_buff xdp; + unsigned int xdp_act; + int pkt_len; + + pkt_len = len - GENET_RSB_PAD; + if (priv->crc_fwd_en) + pkt_len -= ETH_FCS_LEN; + + xdp_init_buff(&xdp, PAGE_SIZE, &ring->xdp_rxq); + xdp_prepare_buff(&xdp, page_address(rx_page), + GENET_RX_HEADROOM, pkt_len, true); + + xdp_act = bcmgenet_run_xdp(ring, xdp_prog, &xdp, + rx_page); + if (xdp_act != XDP_PASS) + goto next; + + /* XDP_PASS: build SKB from (possibly modified) xdp */ + skb = bcmgenet_xdp_build_skb(ring, &xdp, rx_page); + if (unlikely(!skb)) { + BCMGENET_STATS64_INC(stats, dropped); + page_pool_put_full_page(ring->page_pool, + rx_page, true); + goto next; + } + } else { + /* Build SKB from the page - data starts at + * hard_start, frame begins after RSB(64) + pad(2). + */ + skb = napi_build_skb(hard_start, + PAGE_SIZE - GENET_XDP_HEADROOM); + if (unlikely(!skb)) { + BCMGENET_STATS64_INC(stats, dropped); + page_pool_put_full_page(ring->page_pool, + rx_page, true); + goto next; + } - /* Reserve the RSB + pad, then set the data length */ - skb_reserve(skb, GENET_RSB_PAD); - __skb_put(skb, len - GENET_RSB_PAD); + skb_mark_for_recycle(skb); + skb_reserve(skb, GENET_RSB_PAD); + __skb_put(skb, len - GENET_RSB_PAD); - if (priv->crc_fwd_en) { - skb_trim(skb, skb->len - ETH_FCS_LEN); - len -= ETH_FCS_LEN; + if (priv->crc_fwd_en) + skb_trim(skb, skb->len - ETH_FCS_LEN); } /* Set up checksum offload */ @@ -3745,6 +3824,39 @@ static int bcmgenet_change_carrier(struct net_device *dev, bool new_carrier) return 0; } +static int bcmgenet_xdp_setup(struct net_device *dev, + struct netdev_bpf *xdp) +{ + struct bcmgenet_priv *priv = netdev_priv(dev); + struct bpf_prog *old_prog; + struct bpf_prog *prog = xdp->prog; + + if (prog && dev->mtu > PAGE_SIZE - GENET_RX_HEADROOM - + SKB_DATA_ALIGN(sizeof(struct skb_shared_info))) { + NL_SET_ERR_MSG_MOD(xdp->extack, + "MTU too large for single-page XDP buffer"); + return -EOPNOTSUPP; + } + + old_prog = xchg(&priv->xdp_prog, prog); + if (old_prog) { + synchronize_net(); + bpf_prog_put(old_prog); + } + + return 0; +} + +static int bcmgenet_xdp(struct net_device *dev, struct netdev_bpf *xdp) +{ + switch (xdp->command) { + case XDP_SETUP_PROG: + return bcmgenet_xdp_setup(dev, xdp); + default: + return -EOPNOTSUPP; + } +} + static const struct net_device_ops bcmgenet_netdev_ops = { .ndo_open = bcmgenet_open, .ndo_stop = bcmgenet_close, @@ -3756,6 +3868,7 @@ static const struct net_device_ops bcmgenet_netdev_ops = { .ndo_set_features = bcmgenet_set_features, .ndo_get_stats64 = bcmgenet_get_stats64, .ndo_change_carrier = bcmgenet_change_carrier, + .ndo_bpf = bcmgenet_xdp, }; /* GENET hardware parameters/characteristics */ @@ -4058,6 +4171,7 @@ static int bcmgenet_probe(struct platform_device *pdev) NETIF_F_RXCSUM; dev->hw_features |= dev->features; dev->vlan_features |= dev->features; + dev->xdp_features = NETDEV_XDP_ACT_BASIC; netdev_sw_irq_coalesce_default_on(dev); diff --git a/drivers/net/ethernet/broadcom/genet/bcmgenet.h b/drivers/net/ethernet/broadcom/genet/bcmgenet.h index 82a6d29f481d..1459473ac1b0 100644 --- a/drivers/net/ethernet/broadcom/genet/bcmgenet.h +++ b/drivers/net/ethernet/broadcom/genet/bcmgenet.h @@ -16,6 +16,7 @@ #include #include #include +#include #include #include "../unimac.h" @@ -671,6 +672,9 @@ struct bcmgenet_priv { u8 sopass[SOPASS_MAX]; struct bcmgenet_mib_counters mib; + + /* XDP */ + struct bpf_prog *xdp_prog; }; static inline bool bcmgenet_has_40bits(struct bcmgenet_priv *priv) -- 2.51.0