From: Nazim Amirul Use page pool buffers instead of streaming skb buffers in stmmac_xmit when snps,pagepool-tx-buf-quirk is enabled. As page pool buffers are recycled, the overhead of dma map/unmap for each transmit can be avoided. There is a trade-off of copying from the skb into the page pool buffer, but this helps where the copy is cheaper than dma map/unmap (for example with IOMMU invalidate cost). Currently page pool buffers are only used for single-fragment transmit in the non-TSO path. Signed-off-by: Nazim Amirul --- drivers/net/ethernet/stmicro/stmmac/stmmac.h | 3 + .../net/ethernet/stmicro/stmmac/stmmac_main.c | 75 ++++++++++++++++--- .../ethernet/stmicro/stmmac/stmmac_platform.c | 2 + include/linux/stmmac.h | 1 + 4 files changed, 70 insertions(+), 11 deletions(-) diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac.h b/drivers/net/ethernet/stmicro/stmmac/stmmac.h index 8ba8f03e1ce0..093c052b4908 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac.h +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac.h @@ -41,6 +41,7 @@ struct stmmac_resources { enum stmmac_txbuf_type { STMMAC_TXBUF_T_SKB, + STMMAC_TXBUF_T_DMA, STMMAC_TXBUF_T_XDP_TX, STMMAC_TXBUF_T_XDP_NDO, STMMAC_TXBUF_T_XSK_TX, @@ -49,6 +50,7 @@ enum stmmac_txbuf_type { struct stmmac_tx_info { struct xsk_tx_metadata_compl xsk_meta; dma_addr_t buf; + struct page *page; unsigned len; enum stmmac_txbuf_type buf_type; bool map_as_page; @@ -65,6 +67,7 @@ struct stmmac_tx_queue { int tbs; struct hrtimer txtimer; u32 queue_index; + struct page_pool *page_pool; struct stmmac_priv *priv_data; struct dma_extended_desc *dma_etx ____cacheline_aligned_in_smp; struct dma_edesc *dma_entx; diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c index ee44bd6f4d48..4aa6b1d1507b 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c @@ -1712,7 +1712,10 @@ static void stmmac_free_tx_buffer(struct stmmac_priv *priv, if (tx_q->tx_skbuff_dma[i].buf && tx_q->tx_skbuff_dma[i].buf_type != STMMAC_TXBUF_T_XDP_TX) { - if (tx_q->tx_skbuff_dma[i].map_as_page) + if (tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_DMA) + page_pool_recycle_direct(tx_q->page_pool, + tx_q->tx_skbuff_dma[i].page); + else if (tx_q->tx_skbuff_dma[i].map_as_page) dma_unmap_page(priv->device, tx_q->tx_skbuff_dma[i].buf, tx_q->tx_skbuff_dma[i].len, @@ -1735,7 +1738,8 @@ static void stmmac_free_tx_buffer(struct stmmac_priv *priv, tx_q->xsk_frames_done++; if (tx_q->tx_skbuff[i] && - tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_SKB) { + (tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_SKB || + tx_q->tx_skbuff_dma[i].buf_type == STMMAC_TXBUF_T_DMA)) { dev_kfree_skb_any(tx_q->tx_skbuff[i]); tx_q->tx_skbuff[i] = NULL; } @@ -2201,6 +2205,8 @@ static void __free_dma_tx_desc_resources(struct stmmac_priv *priv, kfree(tx_q->tx_skbuff_dma); kfree(tx_q->tx_skbuff); + if (tx_q->page_pool) + page_pool_destroy(tx_q->page_pool); } static void free_dma_tx_desc_resources(struct stmmac_priv *priv, @@ -2335,12 +2341,32 @@ static int __alloc_dma_tx_desc_resources(struct stmmac_priv *priv, u32 queue) { struct stmmac_tx_queue *tx_q = &dma_conf->tx_queue[queue]; + struct page_pool_params pp_params = { 0 }; + unsigned int num_pages; size_t size; void *addr; + int ret; tx_q->queue_index = queue; tx_q->priv_data = priv; + pp_params.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV; + pp_params.pool_size = dma_conf->dma_tx_size; + num_pages = DIV_ROUND_UP(dma_conf->dma_buf_sz, PAGE_SIZE); + pp_params.order = ilog2(num_pages); + pp_params.nid = dev_to_node(priv->device); + pp_params.dev = priv->device; + pp_params.dma_dir = DMA_BIDIRECTIONAL; + pp_params.offset = 0; + pp_params.max_len = num_pages * PAGE_SIZE; + + tx_q->page_pool = page_pool_create(&pp_params); + if (IS_ERR(tx_q->page_pool)) { + ret = PTR_ERR(tx_q->page_pool); + tx_q->page_pool = NULL; + return ret; + } + tx_q->tx_skbuff_dma = kzalloc_objs(*tx_q->tx_skbuff_dma, dma_conf->dma_tx_size); if (!tx_q->tx_skbuff_dma) @@ -2836,7 +2862,8 @@ static int stmmac_tx_clean(struct stmmac_priv *priv, int budget, u32 queue, tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XDP_NDO) { xdpf = tx_q->xdpf[entry]; skb = NULL; - } else if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_SKB) { + } else if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_SKB || + tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_DMA) { xdpf = NULL; skb = tx_q->tx_skbuff[entry]; } else { @@ -2884,7 +2911,10 @@ static int stmmac_tx_clean(struct stmmac_priv *priv, int budget, u32 queue, if (likely(tx_q->tx_skbuff_dma[entry].buf && tx_q->tx_skbuff_dma[entry].buf_type != STMMAC_TXBUF_T_XDP_TX)) { - if (tx_q->tx_skbuff_dma[entry].map_as_page) + if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_DMA) + page_pool_recycle_direct(tx_q->page_pool, + tx_q->tx_skbuff_dma[entry].page); + else if (tx_q->tx_skbuff_dma[entry].map_as_page) dma_unmap_page(priv->device, tx_q->tx_skbuff_dma[entry].buf, tx_q->tx_skbuff_dma[entry].len, @@ -2922,7 +2952,8 @@ static int stmmac_tx_clean(struct stmmac_priv *priv, int budget, u32 queue, if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_XSK_TX) tx_q->xsk_frames_done++; - if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_SKB) { + if (tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_SKB || + tx_q->tx_skbuff_dma[entry].buf_type == STMMAC_TXBUF_T_DMA) { if (likely(skb)) { pkts_compl++; bytes_compl += skb->len; @@ -4742,12 +4773,14 @@ static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev) bool enh_desc, has_vlan, set_ic, is_jumbo = false; struct stmmac_priv *priv = netdev_priv(dev); unsigned int nopaged_len = skb_headlen(skb); + gfp_t gfp = GFP_ATOMIC | __GFP_NOWARN; u32 queue = skb_get_queue_mapping(skb); int nfrags = skb_shinfo(skb)->nr_frags; unsigned int first_entry, tx_packets; struct stmmac_txq_stats *txq_stats; struct dma_desc *desc, *first_desc; struct stmmac_tx_queue *tx_q; + struct page *tx_buf_page; int i, csum_insertion = 0; int entry, first_tx; dma_addr_t dma_addr; @@ -4814,13 +4847,33 @@ static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev) } else { bool last_segment = (nfrags == 0); - dma_addr = dma_map_single(priv->device, skb->data, - nopaged_len, DMA_TO_DEVICE); - if (dma_mapping_error(priv->device, dma_addr)) - goto dma_map_err; + if (!(last_segment && priv->plat->tx_buf_quirk)) { + dma_addr = dma_map_single(priv->device, skb->data, + nopaged_len, DMA_TO_DEVICE); + if (dma_mapping_error(priv->device, dma_addr)) + goto dma_map_err; - stmmac_set_tx_skb_dma_entry(tx_q, first_entry, dma_addr, - nopaged_len, false); + stmmac_set_tx_skb_dma_entry(tx_q, first_entry, dma_addr, + nopaged_len, false); + } else { + tx_buf_page = page_pool_alloc_pages(tx_q->page_pool, gfp); + if (!tx_buf_page) + goto dma_map_err; + + dma_addr = page_pool_get_dma_addr(tx_buf_page); + stmmac_set_tx_dma_entry(tx_q, first_entry, + STMMAC_TXBUF_T_DMA, dma_addr, + nopaged_len, false); + tx_q->tx_skbuff_dma[first_entry].page = tx_buf_page; + + dma_sync_single_for_cpu(priv->device, dma_addr, + nopaged_len, DMA_BIDIRECTIONAL); + skb_copy_from_linear_data(skb, page_address(tx_buf_page), + nopaged_len); + dma_sync_single_for_device(priv->device, dma_addr, + nopaged_len, + DMA_BIDIRECTIONAL); + } stmmac_set_desc_addr(priv, first_desc, dma_addr); diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c index dc5f951a311d..0f8d2fc61ce4 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c @@ -575,6 +575,8 @@ stmmac_probe_config_dt(struct platform_device *pdev, u8 *mac) of_property_read_u32(np, "snps,ps-speed", &plat->mac_port_sel_speed); + plat->tx_buf_quirk = of_property_read_bool(np, "snps,pagepool-tx-buf-quirk"); + plat->axi = stmmac_axi_setup(pdev); rc = stmmac_mtl_setup(pdev, plat); diff --git a/include/linux/stmmac.h b/include/linux/stmmac.h index 4430b967abde..d4c9d77a6f81 100644 --- a/include/linux/stmmac.h +++ b/include/linux/stmmac.h @@ -268,6 +268,7 @@ struct plat_stmmacenet_data { u8 tx_queues_to_use; u8 rx_sched_algorithm; u8 tx_sched_algorithm; + bool tx_buf_quirk; struct stmmac_rxq_cfg rx_queues_cfg[MTL_MAX_RX_QUEUES]; struct stmmac_txq_cfg tx_queues_cfg[MTL_MAX_TX_QUEUES]; void (*get_interfaces)(struct stmmac_priv *priv, void *bsp_priv, -- 2.43.7