Use the page pool allocator for the data buffers and enable skb recycling support, instead of relying on netdev_alloc_skb allocating the entire skb during the refill. The change replaces the rx_skbuff array with rx_buff array to store page pool allocated buffers instead of pre-allocated skbs. DMA mapping, rx and refill path were updated accordingly. Signed-off-by: Paolo Valerio --- drivers/net/ethernet/cadence/Kconfig | 1 + drivers/net/ethernet/cadence/macb.h | 10 +- drivers/net/ethernet/cadence/macb_main.c | 169 +++++++++++++++-------- 3 files changed, 121 insertions(+), 59 deletions(-) diff --git a/drivers/net/ethernet/cadence/Kconfig b/drivers/net/ethernet/cadence/Kconfig index 5b2a461dfd28..ae500f717433 100644 --- a/drivers/net/ethernet/cadence/Kconfig +++ b/drivers/net/ethernet/cadence/Kconfig @@ -25,6 +25,7 @@ config MACB depends on PTP_1588_CLOCK_OPTIONAL select PHYLINK select CRC32 + select PAGE_POOL help The Cadence MACB ethernet interface is found on many Atmel AT32 and AT91 parts. This driver also supports the Cadence GEM (Gigabit diff --git a/drivers/net/ethernet/cadence/macb.h b/drivers/net/ethernet/cadence/macb.h index 87414a2ddf6e..dcf768bd1bc1 100644 --- a/drivers/net/ethernet/cadence/macb.h +++ b/drivers/net/ethernet/cadence/macb.h @@ -14,6 +14,8 @@ #include #include #include +#include +#include #define MACB_GREGS_NBR 16 #define MACB_GREGS_VERSION 2 @@ -957,6 +959,10 @@ struct macb_dma_desc_ptp { /* Scaled PPM fraction */ #define PPM_FRACTION 16 +/* The buf includes headroom compatible with both skb and xdpf */ +#define MACB_PP_HEADROOM XDP_PACKET_HEADROOM +#define MACB_PP_MAX_BUF_SIZE(num) (((num) * PAGE_SIZE) - MACB_PP_HEADROOM) + /* struct macb_tx_skb - data about an skb which is being transmitted * @skb: skb currently being transmitted, only set for the last buffer * of the frame @@ -1262,10 +1268,11 @@ struct macb_queue { unsigned int rx_tail; unsigned int rx_prepared_head; struct macb_dma_desc *rx_ring; - struct sk_buff **rx_skbuff; + void **rx_buff; void *rx_buffers; struct napi_struct napi_rx; struct queue_stats stats; + struct page_pool *page_pool; }; struct ethtool_rx_fs_item { @@ -1289,6 +1296,7 @@ struct macb { struct macb_dma_desc *rx_ring_tieoff; dma_addr_t rx_ring_tieoff_dma; size_t rx_buffer_size; + u16 rx_offset; unsigned int rx_ring_size; unsigned int tx_ring_size; diff --git a/drivers/net/ethernet/cadence/macb_main.c b/drivers/net/ethernet/cadence/macb_main.c index e461f5072884..985c81913ba6 100644 --- a/drivers/net/ethernet/cadence/macb_main.c +++ b/drivers/net/ethernet/cadence/macb_main.c @@ -1250,11 +1250,28 @@ static int macb_tx_complete(struct macb_queue *queue, int budget) return packets; } -static void gem_rx_refill(struct macb_queue *queue) +static void *gem_page_pool_get_buff(struct page_pool *pool, + dma_addr_t *dma_addr, gfp_t gfp_mask) +{ + struct page *page; + + if (!pool) + return NULL; + + page = page_pool_alloc_pages(pool, gfp_mask | __GFP_NOWARN); + if (!page) + return NULL; + + *dma_addr = page_pool_get_dma_addr(page) + MACB_PP_HEADROOM; + + return page_address(page); +} + +static void gem_rx_refill(struct macb_queue *queue, bool napi) { unsigned int entry; - struct sk_buff *skb; dma_addr_t paddr; + void *data; struct macb *bp = queue->bp; struct macb_dma_desc *desc; @@ -1267,25 +1284,17 @@ static void gem_rx_refill(struct macb_queue *queue) desc = macb_rx_desc(queue, entry); - if (!queue->rx_skbuff[entry]) { + if (!queue->rx_buff[entry]) { /* allocate sk_buff for this free entry in ring */ - skb = netdev_alloc_skb(bp->dev, bp->rx_buffer_size); - if (unlikely(!skb)) { + data = gem_page_pool_get_buff(queue->page_pool, &paddr, + napi ? GFP_ATOMIC : GFP_KERNEL); + if (unlikely(!data)) { netdev_err(bp->dev, - "Unable to allocate sk_buff\n"); + "Unable to allocate page\n"); break; } - /* now fill corresponding descriptor entry */ - paddr = dma_map_single(&bp->pdev->dev, skb->data, - bp->rx_buffer_size, - DMA_FROM_DEVICE); - if (dma_mapping_error(&bp->pdev->dev, paddr)) { - dev_kfree_skb(skb); - break; - } - - queue->rx_skbuff[entry] = skb; + queue->rx_buff[entry] = data; if (entry == bp->rx_ring_size - 1) paddr |= MACB_BIT(RX_WRAP); @@ -1295,20 +1304,6 @@ static void gem_rx_refill(struct macb_queue *queue) */ dma_wmb(); macb_set_addr(bp, desc, paddr); - - /* Properly align Ethernet header. - * - * Hardware can add dummy bytes if asked using the RBOF - * field inside the NCFGR register. That feature isn't - * available if hardware is RSC capable. - * - * We cannot fallback to doing the 2-byte shift before - * DMA mapping because the address field does not allow - * setting the low 2/3 bits. - * It is 3 bits if HW_DMA_CAP_PTP, else 2 bits. - */ - if (!(bp->caps & MACB_CAPS_RSC)) - skb_reserve(skb, NET_IP_ALIGN); } else { desc->ctrl = 0; dma_wmb(); @@ -1349,12 +1344,16 @@ static int gem_rx(struct macb_queue *queue, struct napi_struct *napi, int budget) { struct macb *bp = queue->bp; + int buffer_size; unsigned int len; unsigned int entry; + void *data; struct sk_buff *skb; struct macb_dma_desc *desc; int count = 0; + buffer_size = DIV_ROUND_UP(bp->rx_buffer_size, PAGE_SIZE) * PAGE_SIZE; + while (count < budget) { u32 ctrl; dma_addr_t addr; @@ -1387,24 +1386,49 @@ static int gem_rx(struct macb_queue *queue, struct napi_struct *napi, queue->stats.rx_dropped++; break; } - skb = queue->rx_skbuff[entry]; - if (unlikely(!skb)) { + data = queue->rx_buff[entry]; + if (unlikely(!data)) { netdev_err(bp->dev, "inconsistent Rx descriptor chain\n"); bp->dev->stats.rx_dropped++; queue->stats.rx_dropped++; break; } + + skb = napi_build_skb(data, buffer_size); + if (unlikely(!skb)) { + netdev_err(bp->dev, + "Unable to allocate sk_buff\n"); + page_pool_put_full_page(queue->page_pool, + virt_to_head_page(data), + false); + break; + } + + /* Properly align Ethernet header. + * + * Hardware can add dummy bytes if asked using the RBOF + * field inside the NCFGR register. That feature isn't + * available if hardware is RSC capable. + * + * We cannot fallback to doing the 2-byte shift before + * DMA mapping because the address field does not allow + * setting the low 2/3 bits. + * It is 3 bits if HW_DMA_CAP_PTP, else 2 bits. + */ + skb_reserve(skb, bp->rx_offset); + skb_mark_for_recycle(skb); + /* now everything is ready for receiving packet */ - queue->rx_skbuff[entry] = NULL; + queue->rx_buff[entry] = NULL; len = ctrl & bp->rx_frm_len_mask; netdev_vdbg(bp->dev, "gem_rx %u (len %u)\n", entry, len); + dma_sync_single_for_cpu(&bp->pdev->dev, + addr, len, + page_pool_get_dma_dir(queue->page_pool)); skb_put(skb, len); - dma_unmap_single(&bp->pdev->dev, addr, - bp->rx_buffer_size, DMA_FROM_DEVICE); - skb->protocol = eth_type_trans(skb, bp->dev); skb_checksum_none_assert(skb); if (bp->dev->features & NETIF_F_RXCSUM && @@ -1431,7 +1455,7 @@ static int gem_rx(struct macb_queue *queue, struct napi_struct *napi, napi_gro_receive(napi, skb); } - gem_rx_refill(queue); + gem_rx_refill(queue, true); return count; } @@ -2387,34 +2411,30 @@ static void macb_init_rx_buffer_size(struct macb *bp, size_t size) static void gem_free_rx_buffers(struct macb *bp) { - struct sk_buff *skb; - struct macb_dma_desc *desc; struct macb_queue *queue; - dma_addr_t addr; unsigned int q; + void *data; int i; for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { - if (!queue->rx_skbuff) + if (!queue->rx_buff) continue; for (i = 0; i < bp->rx_ring_size; i++) { - skb = queue->rx_skbuff[i]; - - if (!skb) + data = queue->rx_buff[i]; + if (!data) continue; - desc = macb_rx_desc(queue, i); - addr = macb_get_addr(bp, desc); - - dma_unmap_single(&bp->pdev->dev, addr, bp->rx_buffer_size, - DMA_FROM_DEVICE); - dev_kfree_skb_any(skb); - skb = NULL; + page_pool_put_full_page(queue->page_pool, + virt_to_head_page(data), + false); + queue->rx_buff[i] = NULL; } - kfree(queue->rx_skbuff); - queue->rx_skbuff = NULL; + kfree(queue->rx_buff); + queue->rx_buff = NULL; + page_pool_destroy(queue->page_pool); + queue->page_pool = NULL; } } @@ -2477,13 +2497,13 @@ static int gem_alloc_rx_buffers(struct macb *bp) for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { size = bp->rx_ring_size * sizeof(struct sk_buff *); - queue->rx_skbuff = kzalloc(size, GFP_KERNEL); - if (!queue->rx_skbuff) + queue->rx_buff = kzalloc(size, GFP_KERNEL); + if (!queue->rx_buff) return -ENOMEM; else netdev_dbg(bp->dev, - "Allocated %d RX struct sk_buff entries at %p\n", - bp->rx_ring_size, queue->rx_skbuff); + "Allocated %d RX buff entries at %p\n", + bp->rx_ring_size, queue->rx_buff); } return 0; } @@ -2567,6 +2587,32 @@ static int macb_alloc_consistent(struct macb *bp) return -ENOMEM; } +static void gem_create_page_pool(struct macb_queue *queue) +{ + unsigned int num_pages = DIV_ROUND_UP(queue->bp->rx_buffer_size, PAGE_SIZE); + struct macb *bp = queue->bp; + struct page_pool_params pp_params = { + .order = order_base_2(num_pages), + .flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV, + .pool_size = queue->bp->rx_ring_size, + .nid = NUMA_NO_NODE, + .dma_dir = DMA_FROM_DEVICE, + .dev = &queue->bp->pdev->dev, + .napi = &queue->napi_rx, + .offset = bp->rx_offset, + .max_len = MACB_PP_MAX_BUF_SIZE(num_pages), + }; + struct page_pool *pool; + + pool = page_pool_create(&pp_params); + if (IS_ERR(pool)) { + netdev_err(queue->bp->dev, "cannot create rx page pool\n"); + pool = NULL; + } + + queue->page_pool = pool; +} + static void macb_init_tieoff(struct macb *bp) { struct macb_dma_desc *desc = bp->rx_ring_tieoff; @@ -2600,7 +2646,8 @@ static void gem_init_rings(struct macb *bp) queue->rx_tail = 0; queue->rx_prepared_head = 0; - gem_rx_refill(queue); + gem_create_page_pool(queue); + gem_rx_refill(queue, false); } macb_init_tieoff(bp); @@ -5604,6 +5651,12 @@ static int macb_probe(struct platform_device *pdev) if (err) goto err_out_phy_exit; + if (macb_is_gem(bp)) { + bp->rx_offset = MACB_PP_HEADROOM; + if (!(bp->caps & MACB_CAPS_RSC)) + bp->rx_offset += NET_IP_ALIGN; + } + netif_carrier_off(dev); err = register_netdev(dev); -- 2.51.1 Add support for receiving network frames that span multiple DMA descriptors in the Cadence MACB/GEM Ethernet driver. The patch removes the requirement that limited frame reception to a single descriptor (RX_SOF && RX_EOF), also avoiding potential contiguous multi-page allocation for large frames. It also uses page pool fragments allocation instead of full page for saving memory. Signed-off-by: Paolo Valerio --- drivers/net/ethernet/cadence/macb.h | 4 +- drivers/net/ethernet/cadence/macb_main.c | 180 ++++++++++++++--------- 2 files changed, 111 insertions(+), 73 deletions(-) diff --git a/drivers/net/ethernet/cadence/macb.h b/drivers/net/ethernet/cadence/macb.h index dcf768bd1bc1..e2f397b7a27f 100644 --- a/drivers/net/ethernet/cadence/macb.h +++ b/drivers/net/ethernet/cadence/macb.h @@ -960,8 +960,7 @@ struct macb_dma_desc_ptp { #define PPM_FRACTION 16 /* The buf includes headroom compatible with both skb and xdpf */ -#define MACB_PP_HEADROOM XDP_PACKET_HEADROOM -#define MACB_PP_MAX_BUF_SIZE(num) (((num) * PAGE_SIZE) - MACB_PP_HEADROOM) +#define MACB_PP_HEADROOM XDP_PACKET_HEADROOM /* struct macb_tx_skb - data about an skb which is being transmitted * @skb: skb currently being transmitted, only set for the last buffer @@ -1273,6 +1272,7 @@ struct macb_queue { struct napi_struct napi_rx; struct queue_stats stats; struct page_pool *page_pool; + struct sk_buff *skb; }; struct ethtool_rx_fs_item { diff --git a/drivers/net/ethernet/cadence/macb_main.c b/drivers/net/ethernet/cadence/macb_main.c index 985c81913ba6..be0c8e101639 100644 --- a/drivers/net/ethernet/cadence/macb_main.c +++ b/drivers/net/ethernet/cadence/macb_main.c @@ -1250,21 +1250,25 @@ static int macb_tx_complete(struct macb_queue *queue, int budget) return packets; } -static void *gem_page_pool_get_buff(struct page_pool *pool, +static void *gem_page_pool_get_buff(struct macb_queue *queue, dma_addr_t *dma_addr, gfp_t gfp_mask) { + struct macb *bp = queue->bp; struct page *page; + int offset; - if (!pool) + if (!queue->page_pool) return NULL; - page = page_pool_alloc_pages(pool, gfp_mask | __GFP_NOWARN); + page = page_pool_alloc_frag(queue->page_pool, &offset, + bp->rx_buffer_size, + gfp_mask | __GFP_NOWARN); if (!page) return NULL; - *dma_addr = page_pool_get_dma_addr(page) + MACB_PP_HEADROOM; + *dma_addr = page_pool_get_dma_addr(page) + MACB_PP_HEADROOM + offset; - return page_address(page); + return page_address(page) + offset; } static void gem_rx_refill(struct macb_queue *queue, bool napi) @@ -1286,7 +1290,7 @@ static void gem_rx_refill(struct macb_queue *queue, bool napi) if (!queue->rx_buff[entry]) { /* allocate sk_buff for this free entry in ring */ - data = gem_page_pool_get_buff(queue->page_pool, &paddr, + data = gem_page_pool_get_buff(queue, &paddr, napi ? GFP_ATOMIC : GFP_KERNEL); if (unlikely(!data)) { netdev_err(bp->dev, @@ -1344,20 +1348,17 @@ static int gem_rx(struct macb_queue *queue, struct napi_struct *napi, int budget) { struct macb *bp = queue->bp; - int buffer_size; unsigned int len; unsigned int entry; void *data; - struct sk_buff *skb; struct macb_dma_desc *desc; + int data_len; int count = 0; - buffer_size = DIV_ROUND_UP(bp->rx_buffer_size, PAGE_SIZE) * PAGE_SIZE; - while (count < budget) { u32 ctrl; dma_addr_t addr; - bool rxused; + bool rxused, first_frame; entry = macb_rx_ring_wrap(bp, queue->rx_tail); desc = macb_rx_desc(queue, entry); @@ -1368,6 +1369,9 @@ static int gem_rx(struct macb_queue *queue, struct napi_struct *napi, rxused = (desc->addr & MACB_BIT(RX_USED)) ? true : false; addr = macb_get_addr(bp, desc); + dma_sync_single_for_cpu(&bp->pdev->dev, + addr, bp->rx_buffer_size - bp->rx_offset, + page_pool_get_dma_dir(queue->page_pool)); if (!rxused) break; @@ -1379,13 +1383,6 @@ static int gem_rx(struct macb_queue *queue, struct napi_struct *napi, queue->rx_tail++; count++; - if (!(ctrl & MACB_BIT(RX_SOF) && ctrl & MACB_BIT(RX_EOF))) { - netdev_err(bp->dev, - "not whole frame pointed by descriptor\n"); - bp->dev->stats.rx_dropped++; - queue->stats.rx_dropped++; - break; - } data = queue->rx_buff[entry]; if (unlikely(!data)) { netdev_err(bp->dev, @@ -1395,64 +1392,102 @@ static int gem_rx(struct macb_queue *queue, struct napi_struct *napi, break; } - skb = napi_build_skb(data, buffer_size); - if (unlikely(!skb)) { - netdev_err(bp->dev, - "Unable to allocate sk_buff\n"); - page_pool_put_full_page(queue->page_pool, - virt_to_head_page(data), - false); - break; + first_frame = ctrl & MACB_BIT(RX_SOF); + len = ctrl & bp->rx_frm_len_mask; + + if (len) { + data_len = len; + if (!first_frame) + data_len -= queue->skb->len; + } else { + data_len = SKB_WITH_OVERHEAD(bp->rx_buffer_size) - bp->rx_offset; } - /* Properly align Ethernet header. - * - * Hardware can add dummy bytes if asked using the RBOF - * field inside the NCFGR register. That feature isn't - * available if hardware is RSC capable. - * - * We cannot fallback to doing the 2-byte shift before - * DMA mapping because the address field does not allow - * setting the low 2/3 bits. - * It is 3 bits if HW_DMA_CAP_PTP, else 2 bits. - */ - skb_reserve(skb, bp->rx_offset); - skb_mark_for_recycle(skb); + if (first_frame) { + queue->skb = napi_build_skb(data, bp->rx_buffer_size); + if (unlikely(!queue->skb)) { + netdev_err(bp->dev, + "Unable to allocate sk_buff\n"); + goto free_frags; + } + + /* Properly align Ethernet header. + * + * Hardware can add dummy bytes if asked using the RBOF + * field inside the NCFGR register. That feature isn't + * available if hardware is RSC capable. + * + * We cannot fallback to doing the 2-byte shift before + * DMA mapping because the address field does not allow + * setting the low 2/3 bits. + * It is 3 bits if HW_DMA_CAP_PTP, else 2 bits. + */ + skb_reserve(queue->skb, bp->rx_offset); + skb_mark_for_recycle(queue->skb); + skb_put(queue->skb, data_len); + queue->skb->protocol = eth_type_trans(queue->skb, bp->dev); + + skb_checksum_none_assert(queue->skb); + if (bp->dev->features & NETIF_F_RXCSUM && + !(bp->dev->flags & IFF_PROMISC) && + GEM_BFEXT(RX_CSUM, ctrl) & GEM_RX_CSUM_CHECKED_MASK) + queue->skb->ip_summed = CHECKSUM_UNNECESSARY; + } else { + if (!queue->skb) { + netdev_err(bp->dev, + "Received non-starting frame while expecting it\n"); + goto free_frags; + } + + struct skb_shared_info *shinfo = skb_shinfo(queue->skb); + struct page *page = virt_to_head_page(data); + int nr_frags = shinfo->nr_frags; + + if (nr_frags >= ARRAY_SIZE(shinfo->frags)) + goto free_frags; + + skb_add_rx_frag(queue->skb, nr_frags, page, + data - page_address(page) + bp->rx_offset, + data_len, bp->rx_buffer_size); + } /* now everything is ready for receiving packet */ queue->rx_buff[entry] = NULL; - len = ctrl & bp->rx_frm_len_mask; - - netdev_vdbg(bp->dev, "gem_rx %u (len %u)\n", entry, len); - dma_sync_single_for_cpu(&bp->pdev->dev, - addr, len, - page_pool_get_dma_dir(queue->page_pool)); - skb_put(skb, len); - skb->protocol = eth_type_trans(skb, bp->dev); - skb_checksum_none_assert(skb); - if (bp->dev->features & NETIF_F_RXCSUM && - !(bp->dev->flags & IFF_PROMISC) && - GEM_BFEXT(RX_CSUM, ctrl) & GEM_RX_CSUM_CHECKED_MASK) - skb->ip_summed = CHECKSUM_UNNECESSARY; + netdev_vdbg(bp->dev, "%s %u (len %u)\n", __func__, entry, data_len); - bp->dev->stats.rx_packets++; - queue->stats.rx_packets++; - bp->dev->stats.rx_bytes += skb->len; - queue->stats.rx_bytes += skb->len; + if (ctrl & MACB_BIT(RX_EOF)) { + bp->dev->stats.rx_packets++; + queue->stats.rx_packets++; + bp->dev->stats.rx_bytes += queue->skb->len; + queue->stats.rx_bytes += queue->skb->len; - gem_ptp_do_rxstamp(bp, skb, desc); + gem_ptp_do_rxstamp(bp, queue->skb, desc); #if defined(DEBUG) && defined(VERBOSE_DEBUG) - netdev_vdbg(bp->dev, "received skb of length %u, csum: %08x\n", - skb->len, skb->csum); - print_hex_dump(KERN_DEBUG, " mac: ", DUMP_PREFIX_ADDRESS, 16, 1, - skb_mac_header(skb), 16, true); - print_hex_dump(KERN_DEBUG, "data: ", DUMP_PREFIX_ADDRESS, 16, 1, - skb->data, 32, true); + netdev_vdbg(bp->dev, "received skb of length %u, csum: %08x\n", + queue->skb->len, queue->skb->csum); + print_hex_dump(KERN_DEBUG, " mac: ", DUMP_PREFIX_ADDRESS, 16, 1, + skb_mac_header(queue->skb), 16, true); + print_hex_dump(KERN_DEBUG, "data: ", DUMP_PREFIX_ADDRESS, 16, 1, + queue->skb->data, 32, true); #endif - napi_gro_receive(napi, skb); + napi_gro_receive(napi, queue->skb); + queue->skb = NULL; + } + + continue; + +free_frags: + if (queue->skb) { + dev_kfree_skb(queue->skb); + queue->skb = NULL; + } else { + page_pool_put_full_page(queue->page_pool, + virt_to_head_page(data), + false); + } } gem_rx_refill(queue, true); @@ -2394,7 +2429,10 @@ static void macb_init_rx_buffer_size(struct macb *bp, size_t size) if (!macb_is_gem(bp)) { bp->rx_buffer_size = MACB_RX_BUFFER_SIZE; } else { - bp->rx_buffer_size = size; + bp->rx_buffer_size = size + SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) + + MACB_PP_HEADROOM; + if (bp->rx_buffer_size > PAGE_SIZE) + bp->rx_buffer_size = PAGE_SIZE; if (bp->rx_buffer_size % RX_BUFFER_MULTIPLE) { netdev_dbg(bp->dev, @@ -2589,18 +2627,15 @@ static int macb_alloc_consistent(struct macb *bp) static void gem_create_page_pool(struct macb_queue *queue) { - unsigned int num_pages = DIV_ROUND_UP(queue->bp->rx_buffer_size, PAGE_SIZE); - struct macb *bp = queue->bp; struct page_pool_params pp_params = { - .order = order_base_2(num_pages), + .order = 0, .flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV, .pool_size = queue->bp->rx_ring_size, .nid = NUMA_NO_NODE, .dma_dir = DMA_FROM_DEVICE, .dev = &queue->bp->pdev->dev, .napi = &queue->napi_rx, - .offset = bp->rx_offset, - .max_len = MACB_PP_MAX_BUF_SIZE(num_pages), + .max_len = PAGE_SIZE, }; struct page_pool *pool; @@ -2784,8 +2819,9 @@ static void macb_configure_dma(struct macb *bp) unsigned int q; u32 dmacfg; - buffer_size = bp->rx_buffer_size / RX_BUFFER_MULTIPLE; if (macb_is_gem(bp)) { + buffer_size = SKB_WITH_OVERHEAD(bp->rx_buffer_size) - bp->rx_offset; + buffer_size /= RX_BUFFER_MULTIPLE; dmacfg = gem_readl(bp, DMACFG) & ~GEM_BF(RXBS, -1L); for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { if (q) @@ -2816,6 +2852,8 @@ static void macb_configure_dma(struct macb *bp) netdev_dbg(bp->dev, "Cadence configure DMA with 0x%08x\n", dmacfg); gem_writel(bp, DMACFG, dmacfg); + } else { + buffer_size = bp->rx_buffer_size / RX_BUFFER_MULTIPLE; } } -- 2.51.1 gem_get_ethtool_stats calculates the size of the statistics data to copy always considering maximum number of queues. The patch makes sure the statistics are copied only for the active queues as returned in the string set count op. Signed-off-by: Paolo Valerio --- This is not related to XDP, but an issue related to this was spotted while introducing ethtool stats support resulting in page pool stats pollution. Page pool stats support patch was later dropped from the series once realized its deprecation. --- drivers/net/ethernet/cadence/macb_main.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/net/ethernet/cadence/macb_main.c b/drivers/net/ethernet/cadence/macb_main.c index be0c8e101639..5829c1f773dd 100644 --- a/drivers/net/ethernet/cadence/macb_main.c +++ b/drivers/net/ethernet/cadence/macb_main.c @@ -3212,7 +3212,7 @@ static void gem_get_ethtool_stats(struct net_device *dev, spin_lock_irq(&bp->stats_lock); gem_update_stats(bp); memcpy(data, &bp->ethtool_stats, sizeof(u64) - * (GEM_STATS_LEN + QUEUE_STATS_LEN * MACB_MAX_QUEUES)); + * (GEM_STATS_LEN + QUEUE_STATS_LEN * bp->num_queues)); spin_unlock_irq(&bp->stats_lock); } -- 2.51.1 Introduce basic XDP support for macb/gem with the XDP_PASS, XDP_DROP, XDP_REDIRECT verdict support. Signed-off-by: Paolo Valerio --- drivers/net/ethernet/cadence/macb.h | 4 + drivers/net/ethernet/cadence/macb_main.c | 161 +++++++++++++++++++++-- 2 files changed, 156 insertions(+), 9 deletions(-) diff --git a/drivers/net/ethernet/cadence/macb.h b/drivers/net/ethernet/cadence/macb.h index e2f397b7a27f..2f665260a84d 100644 --- a/drivers/net/ethernet/cadence/macb.h +++ b/drivers/net/ethernet/cadence/macb.h @@ -16,6 +16,7 @@ #include #include #include +#include #define MACB_GREGS_NBR 16 #define MACB_GREGS_VERSION 2 @@ -961,6 +962,7 @@ struct macb_dma_desc_ptp { /* The buf includes headroom compatible with both skb and xdpf */ #define MACB_PP_HEADROOM XDP_PACKET_HEADROOM +#define MACB_MAX_PAD (MACB_PP_HEADROOM + SKB_DATA_ALIGN(sizeof(struct skb_shared_info))) /* struct macb_tx_skb - data about an skb which is being transmitted * @skb: skb currently being transmitted, only set for the last buffer @@ -1273,6 +1275,7 @@ struct macb_queue { struct queue_stats stats; struct page_pool *page_pool; struct sk_buff *skb; + struct xdp_rxq_info xdp_q; }; struct ethtool_rx_fs_item { @@ -1372,6 +1375,7 @@ struct macb { struct macb_pm_data pm_data; const struct macb_usrio_config *usrio; + struct bpf_prog __rcu *prog; }; #ifdef CONFIG_MACB_USE_HWSTAMP diff --git a/drivers/net/ethernet/cadence/macb_main.c b/drivers/net/ethernet/cadence/macb_main.c index 5829c1f773dd..53ea1958b8e4 100644 --- a/drivers/net/ethernet/cadence/macb_main.c +++ b/drivers/net/ethernet/cadence/macb_main.c @@ -1344,10 +1344,51 @@ static void discard_partial_frame(struct macb_queue *queue, unsigned int begin, */ } +static u32 gem_xdp_run(struct macb_queue *queue, struct xdp_buff *xdp, + struct net_device *dev) +{ + struct bpf_prog *prog; + u32 act = XDP_PASS; + + rcu_read_lock(); + + prog = rcu_dereference(queue->bp->prog); + if (!prog) + goto out; + + act = bpf_prog_run_xdp(prog, xdp); + switch (act) { + case XDP_PASS: + goto out; + case XDP_REDIRECT: + if (unlikely(xdp_do_redirect(dev, xdp, prog))) { + act = XDP_DROP; + break; + } + goto out; + default: + bpf_warn_invalid_xdp_action(dev, prog, act); + fallthrough; + case XDP_ABORTED: + trace_xdp_exception(dev, prog, act); + fallthrough; + case XDP_DROP: + break; + } + + page_pool_put_full_page(queue->page_pool, + virt_to_head_page(xdp->data), true); +out: + rcu_read_unlock(); + + return act; +} + static int gem_rx(struct macb_queue *queue, struct napi_struct *napi, int budget) { struct macb *bp = queue->bp; + bool xdp_flush = false; unsigned int len; unsigned int entry; void *data; @@ -1356,9 +1397,11 @@ static int gem_rx(struct macb_queue *queue, struct napi_struct *napi, int count = 0; while (count < budget) { - u32 ctrl; - dma_addr_t addr; bool rxused, first_frame; + struct xdp_buff xdp; + dma_addr_t addr; + u32 ctrl; + u32 ret; entry = macb_rx_ring_wrap(bp, queue->rx_tail); desc = macb_rx_desc(queue, entry); @@ -1403,6 +1446,22 @@ static int gem_rx(struct macb_queue *queue, struct napi_struct *napi, data_len = SKB_WITH_OVERHEAD(bp->rx_buffer_size) - bp->rx_offset; } + if (!(ctrl & MACB_BIT(RX_SOF) && ctrl & MACB_BIT(RX_EOF))) + goto skip_xdp; + + xdp_init_buff(&xdp, bp->rx_buffer_size, &queue->xdp_q); + xdp_prepare_buff(&xdp, data, bp->rx_offset, len, + false); + xdp_buff_clear_frags_flag(&xdp); + + ret = gem_xdp_run(queue, &xdp, bp->dev); + if (ret == XDP_REDIRECT) + xdp_flush = true; + + if (ret != XDP_PASS) + goto next_frame; + +skip_xdp: if (first_frame) { queue->skb = napi_build_skb(data, bp->rx_buffer_size); if (unlikely(!queue->skb)) { @@ -1452,10 +1511,6 @@ static int gem_rx(struct macb_queue *queue, struct napi_struct *napi, } /* now everything is ready for receiving packet */ - queue->rx_buff[entry] = NULL; - - netdev_vdbg(bp->dev, "%s %u (len %u)\n", __func__, entry, data_len); - if (ctrl & MACB_BIT(RX_EOF)) { bp->dev->stats.rx_packets++; queue->stats.rx_packets++; @@ -1477,6 +1532,8 @@ static int gem_rx(struct macb_queue *queue, struct napi_struct *napi, queue->skb = NULL; } +next_frame: + queue->rx_buff[entry] = NULL; continue; free_frags: @@ -1490,6 +1547,9 @@ static int gem_rx(struct macb_queue *queue, struct napi_struct *napi, } } + if (xdp_flush) + xdp_do_flush(); + gem_rx_refill(queue, true); return count; @@ -2471,6 +2531,8 @@ static void gem_free_rx_buffers(struct macb *bp) kfree(queue->rx_buff); queue->rx_buff = NULL; + if (xdp_rxq_info_is_reg(&queue->xdp_q)) + xdp_rxq_info_unreg(&queue->xdp_q); page_pool_destroy(queue->page_pool); queue->page_pool = NULL; } @@ -2625,19 +2687,22 @@ static int macb_alloc_consistent(struct macb *bp) return -ENOMEM; } -static void gem_create_page_pool(struct macb_queue *queue) +static void gem_create_page_pool(struct macb_queue *queue, int qid) { struct page_pool_params pp_params = { .order = 0, .flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV, .pool_size = queue->bp->rx_ring_size, .nid = NUMA_NO_NODE, - .dma_dir = DMA_FROM_DEVICE, + .dma_dir = rcu_access_pointer(queue->bp->prog) + ? DMA_BIDIRECTIONAL + : DMA_FROM_DEVICE, .dev = &queue->bp->pdev->dev, .napi = &queue->napi_rx, .max_len = PAGE_SIZE, }; struct page_pool *pool; + int err; pool = page_pool_create(&pp_params); if (IS_ERR(pool)) { @@ -2646,6 +2711,28 @@ static void gem_create_page_pool(struct macb_queue *queue) } queue->page_pool = pool; + + err = xdp_rxq_info_reg(&queue->xdp_q, queue->bp->dev, qid, + queue->napi_rx.napi_id); + if (err < 0) { + netdev_err(queue->bp->dev, "xdp: failed to register rxq info\n"); + goto destroy_pool; + } + + err = xdp_rxq_info_reg_mem_model(&queue->xdp_q, MEM_TYPE_PAGE_POOL, + queue->page_pool); + if (err) { + netdev_err(queue->bp->dev, "xdp: failed to register rxq memory model\n"); + goto unreg_info; + } + + return; + +unreg_info: + xdp_rxq_info_unreg(&queue->xdp_q); +destroy_pool: + page_pool_destroy(pool); + queue->page_pool = NULL; } static void macb_init_tieoff(struct macb *bp) @@ -2681,7 +2768,7 @@ static void gem_init_rings(struct macb *bp) queue->rx_tail = 0; queue->rx_prepared_head = 0; - gem_create_page_pool(queue); + gem_create_page_pool(queue, q); gem_rx_refill(queue, false); } @@ -3117,9 +3204,18 @@ static int macb_close(struct net_device *dev) static int macb_change_mtu(struct net_device *dev, int new_mtu) { + int frame_size = new_mtu + ETH_HLEN + ETH_FCS_LEN + MACB_MAX_PAD; + struct macb *bp = netdev_priv(dev); + struct bpf_prog *prog = bp->prog; + if (netif_running(dev)) return -EBUSY; + if (prog && frame_size > PAGE_SIZE) { + netdev_err(dev, "MTU %d too large for XDP", new_mtu); + return -EINVAL; + } + WRITE_ONCE(dev->mtu, new_mtu); return 0; @@ -3137,6 +3233,49 @@ static int macb_set_mac_addr(struct net_device *dev, void *addr) return 0; } +static int gem_xdp_setup(struct net_device *dev, struct bpf_prog *prog, + struct netlink_ext_ack *extack) +{ + int frame = ETH_HLEN + ETH_FCS_LEN + MACB_MAX_PAD; + struct macb *bp = netdev_priv(dev); + struct bpf_prog *old_prog; + bool need_update, running; + + if (prog && dev->mtu + frame > bp->rx_buffer_size) { + NL_SET_ERR_MSG_MOD(extack, "MTU too large for XDP"); + return -EOPNOTSUPP; + } + + running = netif_running(dev); + need_update = !!bp->prog != !!prog; + if (running && need_update) + macb_close(dev); + + old_prog = rcu_replace_pointer(bp->prog, prog, lockdep_rtnl_is_held()); + if (old_prog) + bpf_prog_put(old_prog); + + if (running && need_update) + return macb_open(dev); + + return 0; +} + +static int macb_xdp(struct net_device *dev, struct netdev_bpf *xdp) +{ + struct macb *bp = netdev_priv(dev); + + if (!macb_is_gem(bp)) + return 0; + + switch (xdp->command) { + case XDP_SETUP_PROG: + return gem_xdp_setup(dev, xdp->prog, xdp->extack); + default: + return -EINVAL; + } +} + static void gem_update_stats(struct macb *bp) { struct macb_queue *queue; @@ -4390,6 +4529,7 @@ static const struct net_device_ops macb_netdev_ops = { .ndo_hwtstamp_set = macb_hwtstamp_set, .ndo_hwtstamp_get = macb_hwtstamp_get, .ndo_setup_tc = macb_setup_tc, + .ndo_bpf = macb_xdp, }; /* Configure peripheral capabilities according to device tree @@ -5693,6 +5833,9 @@ static int macb_probe(struct platform_device *pdev) bp->rx_offset = MACB_PP_HEADROOM; if (!(bp->caps & MACB_CAPS_RSC)) bp->rx_offset += NET_IP_ALIGN; + + dev->xdp_features = NETDEV_XDP_ACT_BASIC | + NETDEV_XDP_ACT_REDIRECT; } netif_carrier_off(dev); -- 2.51.1 The macb_tx_skb structure is renamed to macb_tx_buff. Also, the skb member is now called data as it can be an sk_buff or an xdp_frame depending on the tx path, along with an enum to identify the buffer type. This is a preparatory step for adding xdp xmit support. Signed-off-by: Paolo Valerio --- drivers/net/ethernet/cadence/macb.h | 28 ++++-- drivers/net/ethernet/cadence/macb_main.c | 120 +++++++++++++---------- 2 files changed, 86 insertions(+), 62 deletions(-) diff --git a/drivers/net/ethernet/cadence/macb.h b/drivers/net/ethernet/cadence/macb.h index 2f665260a84d..67bb98d3cb00 100644 --- a/drivers/net/ethernet/cadence/macb.h +++ b/drivers/net/ethernet/cadence/macb.h @@ -964,19 +964,27 @@ struct macb_dma_desc_ptp { #define MACB_PP_HEADROOM XDP_PACKET_HEADROOM #define MACB_MAX_PAD (MACB_PP_HEADROOM + SKB_DATA_ALIGN(sizeof(struct skb_shared_info))) -/* struct macb_tx_skb - data about an skb which is being transmitted - * @skb: skb currently being transmitted, only set for the last buffer - * of the frame +enum macb_tx_buff_type { + MACB_TYPE_SKB, + MACB_TYPE_XDP_TX, + MACB_TYPE_XDP_NDO, +}; + +/* struct macb_tx_buff - data about an skb or xdp frame which is being transmitted + * @data: pointer to skb or xdp frame being transmitted, only set + * for the last buffer for sk_buff * @mapping: DMA address of the skb's fragment buffer * @size: size of the DMA mapped buffer * @mapped_as_page: true when buffer was mapped with skb_frag_dma_map(), * false when buffer was mapped with dma_map_single() + * @type: type of buffer (MACB_TYPE_SKB, MACB_TYPE_XDP_TX, MACB_TYPE_XDP_NDO) */ -struct macb_tx_skb { - struct sk_buff *skb; - dma_addr_t mapping; - size_t size; - bool mapped_as_page; +struct macb_tx_buff { + void *data; + dma_addr_t mapping; + size_t size; + bool mapped_as_page; + enum macb_tx_buff_type type; }; /* Hardware-collected statistics. Used when updating the network @@ -1258,7 +1266,7 @@ struct macb_queue { spinlock_t tx_ptr_lock; unsigned int tx_head, tx_tail; struct macb_dma_desc *tx_ring; - struct macb_tx_skb *tx_skb; + struct macb_tx_buff *tx_buff; dma_addr_t tx_ring_dma; struct work_struct tx_error_task; bool txubr_pending; @@ -1336,7 +1344,7 @@ struct macb { phy_interface_t phy_interface; /* AT91RM9200 transmit queue (1 on wire + 1 queued) */ - struct macb_tx_skb rm9200_txq[2]; + struct macb_tx_buff rm9200_txq[2]; unsigned int max_tx_length; u64 ethtool_stats[GEM_STATS_LEN + QUEUE_STATS_LEN * MACB_MAX_QUEUES]; diff --git a/drivers/net/ethernet/cadence/macb_main.c b/drivers/net/ethernet/cadence/macb_main.c index 53ea1958b8e4..eeda1a3871a6 100644 --- a/drivers/net/ethernet/cadence/macb_main.c +++ b/drivers/net/ethernet/cadence/macb_main.c @@ -156,10 +156,10 @@ static struct macb_dma_desc *macb_tx_desc(struct macb_queue *queue, return &queue->tx_ring[index]; } -static struct macb_tx_skb *macb_tx_skb(struct macb_queue *queue, - unsigned int index) +static struct macb_tx_buff *macb_tx_buff(struct macb_queue *queue, + unsigned int index) { - return &queue->tx_skb[macb_tx_ring_wrap(queue->bp, index)]; + return &queue->tx_buff[macb_tx_ring_wrap(queue->bp, index)]; } static dma_addr_t macb_tx_dma(struct macb_queue *queue, unsigned int index) @@ -969,21 +969,25 @@ static int macb_halt_tx(struct macb *bp) bp, TSR); } -static void macb_tx_unmap(struct macb *bp, struct macb_tx_skb *tx_skb, int budget) +static void macb_tx_unmap(struct macb *bp, struct macb_tx_buff *tx_buff, + int budget) { - if (tx_skb->mapping) { - if (tx_skb->mapped_as_page) - dma_unmap_page(&bp->pdev->dev, tx_skb->mapping, - tx_skb->size, DMA_TO_DEVICE); + if (tx_buff->mapping) { + if (tx_buff->mapped_as_page) + dma_unmap_page(&bp->pdev->dev, tx_buff->mapping, + tx_buff->size, DMA_TO_DEVICE); else - dma_unmap_single(&bp->pdev->dev, tx_skb->mapping, - tx_skb->size, DMA_TO_DEVICE); - tx_skb->mapping = 0; + dma_unmap_single(&bp->pdev->dev, tx_buff->mapping, + tx_buff->size, DMA_TO_DEVICE); + tx_buff->mapping = 0; } - if (tx_skb->skb) { - napi_consume_skb(tx_skb->skb, budget); - tx_skb->skb = NULL; + if (tx_buff->data) { + if (tx_buff->type != MACB_TYPE_SKB) + netdev_err(bp->dev, "BUG: Unexpected tx buffer type while unmapping (%d)", + tx_buff->type); + napi_consume_skb(tx_buff->data, budget); + tx_buff->data = NULL; } } @@ -1029,7 +1033,7 @@ static void macb_tx_error_task(struct work_struct *work) u32 queue_index; u32 packets = 0; u32 bytes = 0; - struct macb_tx_skb *tx_skb; + struct macb_tx_buff *tx_buff; struct macb_dma_desc *desc; struct sk_buff *skb; unsigned int tail; @@ -1069,16 +1073,23 @@ static void macb_tx_error_task(struct work_struct *work) desc = macb_tx_desc(queue, tail); ctrl = desc->ctrl; - tx_skb = macb_tx_skb(queue, tail); - skb = tx_skb->skb; + tx_buff = macb_tx_buff(queue, tail); + + if (tx_buff->type != MACB_TYPE_SKB) + netdev_err(bp->dev, "BUG: Unexpected tx buffer type (%d)", + tx_buff->type); + skb = tx_buff->data; if (ctrl & MACB_BIT(TX_USED)) { /* skb is set for the last buffer of the frame */ while (!skb) { - macb_tx_unmap(bp, tx_skb, 0); + macb_tx_unmap(bp, tx_buff, 0); tail++; - tx_skb = macb_tx_skb(queue, tail); - skb = tx_skb->skb; + tx_buff = macb_tx_buff(queue, tail); + if (tx_buff->type != MACB_TYPE_SKB) + netdev_err(bp->dev, "BUG: Unexpected tx buffer type (%d)", + tx_buff->type); + skb = tx_buff->data; } /* ctrl still refers to the first buffer descriptor @@ -1107,7 +1118,7 @@ static void macb_tx_error_task(struct work_struct *work) desc->ctrl = ctrl | MACB_BIT(TX_USED); } - macb_tx_unmap(bp, tx_skb, 0); + macb_tx_unmap(bp, tx_buff, 0); } netdev_tx_completed_queue(netdev_get_tx_queue(bp->dev, queue_index), @@ -1185,7 +1196,7 @@ static int macb_tx_complete(struct macb_queue *queue, int budget) spin_lock_irqsave(&queue->tx_ptr_lock, flags); head = queue->tx_head; for (tail = queue->tx_tail; tail != head && packets < budget; tail++) { - struct macb_tx_skb *tx_skb; + struct macb_tx_buff *tx_buff; struct sk_buff *skb; struct macb_dma_desc *desc; u32 ctrl; @@ -1205,8 +1216,10 @@ static int macb_tx_complete(struct macb_queue *queue, int budget) /* Process all buffers of the current transmitted frame */ for (;; tail++) { - tx_skb = macb_tx_skb(queue, tail); - skb = tx_skb->skb; + tx_buff = macb_tx_buff(queue, tail); + + if (tx_buff->type == MACB_TYPE_SKB) + skb = tx_buff->data; /* First, update TX stats if needed */ if (skb) { @@ -1226,7 +1239,7 @@ static int macb_tx_complete(struct macb_queue *queue, int budget) } /* Now we can safely release resources */ - macb_tx_unmap(bp, tx_skb, budget); + macb_tx_unmap(bp, tx_buff, budget); /* skb is set only for the last buffer of the frame. * WARNING: at this point skb has been freed by @@ -2117,8 +2130,8 @@ static unsigned int macb_tx_map(struct macb *bp, unsigned int f, nr_frags = skb_shinfo(skb)->nr_frags; unsigned int len, i, tx_head = queue->tx_head; u32 ctrl, lso_ctrl = 0, seq_ctrl = 0; + struct macb_tx_buff *tx_buff = NULL; unsigned int eof = 1, mss_mfs = 0; - struct macb_tx_skb *tx_skb = NULL; struct macb_dma_desc *desc; unsigned int offset, size; dma_addr_t mapping; @@ -2141,7 +2154,7 @@ static unsigned int macb_tx_map(struct macb *bp, offset = 0; while (len) { - tx_skb = macb_tx_skb(queue, tx_head); + tx_buff = macb_tx_buff(queue, tx_head); mapping = dma_map_single(&bp->pdev->dev, skb->data + offset, @@ -2150,10 +2163,11 @@ static unsigned int macb_tx_map(struct macb *bp, goto dma_error; /* Save info to properly release resources */ - tx_skb->skb = NULL; - tx_skb->mapping = mapping; - tx_skb->size = size; - tx_skb->mapped_as_page = false; + tx_buff->data = NULL; + tx_buff->type = MACB_TYPE_SKB; + tx_buff->mapping = mapping; + tx_buff->size = size; + tx_buff->mapped_as_page = false; len -= size; offset += size; @@ -2170,7 +2184,7 @@ static unsigned int macb_tx_map(struct macb *bp, offset = 0; while (len) { size = umin(len, bp->max_tx_length); - tx_skb = macb_tx_skb(queue, tx_head); + tx_buff = macb_tx_buff(queue, tx_head); mapping = skb_frag_dma_map(&bp->pdev->dev, frag, offset, size, DMA_TO_DEVICE); @@ -2178,10 +2192,11 @@ static unsigned int macb_tx_map(struct macb *bp, goto dma_error; /* Save info to properly release resources */ - tx_skb->skb = NULL; - tx_skb->mapping = mapping; - tx_skb->size = size; - tx_skb->mapped_as_page = true; + tx_buff->data = NULL; + tx_buff->type = MACB_TYPE_SKB; + tx_buff->mapping = mapping; + tx_buff->size = size; + tx_buff->mapped_as_page = true; len -= size; offset += size; @@ -2190,13 +2205,14 @@ static unsigned int macb_tx_map(struct macb *bp, } /* Should never happen */ - if (unlikely(!tx_skb)) { + if (unlikely(!tx_buff)) { netdev_err(bp->dev, "BUG! empty skb!\n"); return 0; } /* This is the last buffer of the frame: save socket buffer */ - tx_skb->skb = skb; + tx_buff->data = skb; + tx_buff->type = MACB_TYPE_SKB; /* Update TX ring: update buffer descriptors in reverse order * to avoid race condition @@ -2227,10 +2243,10 @@ static unsigned int macb_tx_map(struct macb *bp, do { i--; - tx_skb = macb_tx_skb(queue, i); + tx_buff = macb_tx_buff(queue, i); desc = macb_tx_desc(queue, i); - ctrl = (u32)tx_skb->size; + ctrl = (u32)tx_buff->size; if (eof) { ctrl |= MACB_BIT(TX_LAST); eof = 0; @@ -2253,7 +2269,7 @@ static unsigned int macb_tx_map(struct macb *bp, ctrl |= MACB_BF(MSS_MFS, mss_mfs); /* Set TX buffer descriptor */ - macb_set_addr(bp, desc, tx_skb->mapping); + macb_set_addr(bp, desc, tx_buff->mapping); /* desc->addr must be visible to hardware before clearing * 'TX_USED' bit in desc->ctrl. */ @@ -2269,9 +2285,9 @@ static unsigned int macb_tx_map(struct macb *bp, netdev_err(bp->dev, "TX DMA map failed\n"); for (i = queue->tx_head; i != tx_head; i++) { - tx_skb = macb_tx_skb(queue, i); + tx_buff = macb_tx_buff(queue, i); - macb_tx_unmap(bp, tx_skb, 0); + macb_tx_unmap(bp, tx_buff, 0); } return -ENOMEM; @@ -2582,8 +2598,8 @@ static void macb_free_consistent(struct macb *bp) dma_free_coherent(dev, size, bp->queues[0].rx_ring, bp->queues[0].rx_ring_dma); for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { - kfree(queue->tx_skb); - queue->tx_skb = NULL; + kfree(queue->tx_buff); + queue->tx_buff = NULL; queue->tx_ring = NULL; queue->rx_ring = NULL; } @@ -2662,9 +2678,9 @@ static int macb_alloc_consistent(struct macb *bp) queue->rx_ring = rx + macb_rx_ring_size_per_queue(bp) * q; queue->rx_ring_dma = rx_dma + macb_rx_ring_size_per_queue(bp) * q; - size = bp->tx_ring_size * sizeof(struct macb_tx_skb); - queue->tx_skb = kmalloc(size, GFP_KERNEL); - if (!queue->tx_skb) + size = bp->tx_ring_size * sizeof(struct macb_tx_buff); + queue->tx_buff = kmalloc(size, GFP_KERNEL); + if (!queue->tx_buff) goto out_err; } if (bp->macbgem_ops.mog_alloc_rx_buffers(bp)) @@ -5050,7 +5066,7 @@ static netdev_tx_t at91ether_start_xmit(struct sk_buff *skb, netif_stop_queue(dev); /* Store packet information (to free when Tx completed) */ - lp->rm9200_txq[desc].skb = skb; + lp->rm9200_txq[desc].data = skb; lp->rm9200_txq[desc].size = skb->len; lp->rm9200_txq[desc].mapping = dma_map_single(&lp->pdev->dev, skb->data, skb->len, DMA_TO_DEVICE); @@ -5143,9 +5159,9 @@ static irqreturn_t at91ether_interrupt(int irq, void *dev_id) dev->stats.tx_errors++; desc = 0; - if (lp->rm9200_txq[desc].skb) { - dev_consume_skb_irq(lp->rm9200_txq[desc].skb); - lp->rm9200_txq[desc].skb = NULL; + if (lp->rm9200_txq[desc].data) { + dev_consume_skb_irq(lp->rm9200_txq[desc].data); + lp->rm9200_txq[desc].data = NULL; dma_unmap_single(&lp->pdev->dev, lp->rm9200_txq[desc].mapping, lp->rm9200_txq[desc].size, DMA_TO_DEVICE); dev->stats.tx_packets++; -- 2.51.1 Add XDP_TX verdict support, also introduce ndo_xdp_xmit function for redirection, and update macb_tx_unmap() to handle both skbs and xdp frames advertising NETDEV_XDP_ACT_NDO_XMIT capability and the ability to process XDP_TX verdicts. Signed-off-by: Paolo Valerio --- drivers/net/ethernet/cadence/macb_main.c | 166 +++++++++++++++++++++-- 1 file changed, 153 insertions(+), 13 deletions(-) diff --git a/drivers/net/ethernet/cadence/macb_main.c b/drivers/net/ethernet/cadence/macb_main.c index eeda1a3871a6..bd62d3febeb1 100644 --- a/drivers/net/ethernet/cadence/macb_main.c +++ b/drivers/net/ethernet/cadence/macb_main.c @@ -969,6 +969,17 @@ static int macb_halt_tx(struct macb *bp) bp, TSR); } +static void release_buff(void *buff, enum macb_tx_buff_type type, int budget) +{ + if (type == MACB_TYPE_SKB) { + napi_consume_skb(buff, budget); + } else if (type == MACB_TYPE_XDP_TX) { + xdp_return_frame_rx_napi(buff); + } else { + xdp_return_frame(buff); + } +} + static void macb_tx_unmap(struct macb *bp, struct macb_tx_buff *tx_buff, int budget) { @@ -983,10 +994,7 @@ static void macb_tx_unmap(struct macb *bp, struct macb_tx_buff *tx_buff, } if (tx_buff->data) { - if (tx_buff->type != MACB_TYPE_SKB) - netdev_err(bp->dev, "BUG: Unexpected tx buffer type while unmapping (%d)", - tx_buff->type); - napi_consume_skb(tx_buff->data, budget); + release_buff(tx_buff->data, tx_buff->type, budget); tx_buff->data = NULL; } } @@ -1076,8 +1084,8 @@ static void macb_tx_error_task(struct work_struct *work) tx_buff = macb_tx_buff(queue, tail); if (tx_buff->type != MACB_TYPE_SKB) - netdev_err(bp->dev, "BUG: Unexpected tx buffer type (%d)", - tx_buff->type); + goto unmap; + skb = tx_buff->data; if (ctrl & MACB_BIT(TX_USED)) { @@ -1118,6 +1126,7 @@ static void macb_tx_error_task(struct work_struct *work) desc->ctrl = ctrl | MACB_BIT(TX_USED); } +unmap: macb_tx_unmap(bp, tx_buff, 0); } @@ -1196,6 +1205,7 @@ static int macb_tx_complete(struct macb_queue *queue, int budget) spin_lock_irqsave(&queue->tx_ptr_lock, flags); head = queue->tx_head; for (tail = queue->tx_tail; tail != head && packets < budget; tail++) { + void *data = NULL; struct macb_tx_buff *tx_buff; struct sk_buff *skb; struct macb_dma_desc *desc; @@ -1218,11 +1228,16 @@ static int macb_tx_complete(struct macb_queue *queue, int budget) for (;; tail++) { tx_buff = macb_tx_buff(queue, tail); - if (tx_buff->type == MACB_TYPE_SKB) - skb = tx_buff->data; + if (tx_buff->type != MACB_TYPE_SKB) { + data = tx_buff->data; + goto unmap; + } /* First, update TX stats if needed */ - if (skb) { + if (tx_buff->type == MACB_TYPE_SKB && tx_buff->data) { + data = tx_buff->data; + skb = tx_buff->data; + if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && !ptp_one_step_sync(skb)) gem_ptp_do_txstamp(bp, skb, desc); @@ -1238,6 +1253,7 @@ static int macb_tx_complete(struct macb_queue *queue, int budget) bytes += skb->len; } +unmap: /* Now we can safely release resources */ macb_tx_unmap(bp, tx_buff, budget); @@ -1245,7 +1261,7 @@ static int macb_tx_complete(struct macb_queue *queue, int budget) * WARNING: at this point skb has been freed by * macb_tx_unmap(). */ - if (skb) + if (data) break; } } @@ -1357,8 +1373,124 @@ static void discard_partial_frame(struct macb_queue *queue, unsigned int begin, */ } +static int macb_xdp_submit_frame(struct macb *bp, struct xdp_frame *xdpf, + struct net_device *dev, dma_addr_t addr) +{ + enum macb_tx_buff_type buff_type; + struct macb_tx_buff *tx_buff; + int cpu = smp_processor_id(); + struct macb_dma_desc *desc; + struct macb_queue *queue; + unsigned long flags; + dma_addr_t mapping; + u16 queue_index; + int err = 0; + u32 ctrl; + + queue_index = cpu % bp->num_queues; + queue = &bp->queues[queue_index]; + buff_type = !addr ? MACB_TYPE_XDP_NDO : MACB_TYPE_XDP_TX; + + spin_lock_irqsave(&queue->tx_ptr_lock, flags); + + /* This is a hard error, log it. */ + if (CIRC_SPACE(queue->tx_head, queue->tx_tail, + bp->tx_ring_size) < 1) { + netif_stop_subqueue(dev, queue_index); + netdev_dbg(bp->dev, "tx_head = %u, tx_tail = %u\n", + queue->tx_head, queue->tx_tail); + err = -ENOMEM; + goto unlock; + } + + if (!addr) { + mapping = dma_map_single(&bp->pdev->dev, + xdpf->data, + xdpf->len, DMA_TO_DEVICE); + if (unlikely(dma_mapping_error(&bp->pdev->dev, mapping))) { + err = -ENOMEM; + goto unlock; + } + } else { + mapping = addr; + dma_sync_single_for_device(&bp->pdev->dev, mapping, + xdpf->len, DMA_BIDIRECTIONAL); + } + + unsigned int tx_head = queue->tx_head + 1; + + ctrl = MACB_BIT(TX_USED); + desc = macb_tx_desc(queue, tx_head); + desc->ctrl = ctrl; + + desc = macb_tx_desc(queue, queue->tx_head); + tx_buff = macb_tx_buff(queue, queue->tx_head); + tx_buff->data = xdpf; + tx_buff->type = buff_type; + tx_buff->mapping = mapping; + tx_buff->size = xdpf->len; + tx_buff->mapped_as_page = false; + + ctrl = (u32)tx_buff->size; + ctrl |= MACB_BIT(TX_LAST); + + if (unlikely(macb_tx_ring_wrap(bp, queue->tx_head) == (bp->tx_ring_size - 1))) + ctrl |= MACB_BIT(TX_WRAP); + + /* Set TX buffer descriptor */ + macb_set_addr(bp, desc, tx_buff->mapping); + /* desc->addr must be visible to hardware before clearing + * 'TX_USED' bit in desc->ctrl. + */ + wmb(); + desc->ctrl = ctrl; + queue->tx_head = tx_head; + + /* Make newly initialized descriptor visible to hardware */ + wmb(); + + spin_lock(&bp->lock); + macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART)); + spin_unlock(&bp->lock); + + if (CIRC_SPACE(queue->tx_head, queue->tx_tail, bp->tx_ring_size) < 1) + netif_stop_subqueue(dev, queue_index); + +unlock: + spin_unlock_irqrestore(&queue->tx_ptr_lock, flags); + + if (err) + release_buff(xdpf, buff_type, 0); + + return err; +} + +static int +macb_xdp_xmit(struct net_device *dev, int num_frame, + struct xdp_frame **frames, u32 flags) +{ + struct macb *bp = netdev_priv(dev); + u32 xmitted = 0; + int i; + + if (!macb_is_gem(bp)) + return -EOPNOTSUPP; + + if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) + return -EINVAL; + + for (i = 0; i < num_frame; i++) { + if (macb_xdp_submit_frame(bp, frames[i], dev, 0)) + break; + + xmitted++; + } + + return xmitted; +} + static u32 gem_xdp_run(struct macb_queue *queue, struct xdp_buff *xdp, - struct net_device *dev) + struct net_device *dev, dma_addr_t addr) { struct bpf_prog *prog; u32 act = XDP_PASS; @@ -1379,6 +1511,12 @@ static u32 gem_xdp_run(struct macb_queue *queue, struct xdp_buff *xdp, break; } goto out; + case XDP_TX: + struct xdp_frame *xdpf = xdp_convert_buff_to_frame(xdp); + + if (!xdpf || macb_xdp_submit_frame(queue->bp, xdpf, dev, addr)) + act = XDP_DROP; + goto out; default: bpf_warn_invalid_xdp_action(dev, prog, act); fallthrough; @@ -1467,7 +1605,7 @@ static int gem_rx(struct macb_queue *queue, struct napi_struct *napi, false); xdp_buff_clear_frags_flag(&xdp); - ret = gem_xdp_run(queue, &xdp, bp->dev); + ret = gem_xdp_run(queue, &xdp, bp->dev, addr); if (ret == XDP_REDIRECT) xdp_flush = true; @@ -4546,6 +4684,7 @@ static const struct net_device_ops macb_netdev_ops = { .ndo_hwtstamp_get = macb_hwtstamp_get, .ndo_setup_tc = macb_setup_tc, .ndo_bpf = macb_xdp, + .ndo_xdp_xmit = macb_xdp_xmit, }; /* Configure peripheral capabilities according to device tree @@ -5851,7 +5990,8 @@ static int macb_probe(struct platform_device *pdev) bp->rx_offset += NET_IP_ALIGN; dev->xdp_features = NETDEV_XDP_ACT_BASIC | - NETDEV_XDP_ACT_REDIRECT; + NETDEV_XDP_ACT_REDIRECT | + NETDEV_XDP_ACT_NDO_XMIT; } netif_carrier_off(dev); -- 2.51.1