Extract the code which converts an skb VLAN tag into WQE fields into a small helper. This helper will be used by a follow-up series implementing software USO, to avoid code duplication. Signed-off-by: Joe Damato Reviewed-by: Daniel Machon --- drivers/net/ethernet/mellanox/mlx4/en_tx.c | 33 ++++++++++++++-------- 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx4/en_tx.c b/drivers/net/ethernet/mellanox/mlx4/en_tx.c index c5d564e5a581..00210dbae1c3 100644 --- a/drivers/net/ethernet/mellanox/mlx4/en_tx.c +++ b/drivers/net/ethernet/mellanox/mlx4/en_tx.c @@ -861,6 +861,26 @@ static bool mlx4_en_build_dma_wqe(struct mlx4_en_priv *priv, return false; } +static bool mlx4_en_prep_vlan(const struct sk_buff *skb, + union mlx4_wqe_qpn_vlan *qpn_vlan) +{ + u16 vlan_proto; + + if (!skb_vlan_tag_present(skb)) + return false; + + qpn_vlan->vlan_tag = cpu_to_be16(skb_vlan_tag_get(skb)); + vlan_proto = be16_to_cpu(skb->vlan_proto); + if (vlan_proto == ETH_P_8021AD) + qpn_vlan->ins_vlan = MLX4_WQE_CTRL_INS_SVLAN; + else if (vlan_proto == ETH_P_8021Q) + qpn_vlan->ins_vlan = MLX4_WQE_CTRL_INS_CVLAN; + else + qpn_vlan->ins_vlan = 0; + + return true; +} + netdev_tx_t mlx4_en_xmit(struct sk_buff *skb, struct net_device *dev) { struct skb_shared_info *shinfo = skb_shinfo(skb); @@ -902,19 +922,8 @@ netdev_tx_t mlx4_en_xmit(struct sk_buff *skb, struct net_device *dev) nr_txbb = desc_size >> LOG_TXBB_SIZE; bf_ok = ring->bf_enabled; - if (skb_vlan_tag_present(skb)) { - u16 vlan_proto; - - qpn_vlan.vlan_tag = cpu_to_be16(skb_vlan_tag_get(skb)); - vlan_proto = be16_to_cpu(skb->vlan_proto); - if (vlan_proto == ETH_P_8021AD) - qpn_vlan.ins_vlan = MLX4_WQE_CTRL_INS_SVLAN; - else if (vlan_proto == ETH_P_8021Q) - qpn_vlan.ins_vlan = MLX4_WQE_CTRL_INS_CVLAN; - else - qpn_vlan.ins_vlan = 0; + if (mlx4_en_prep_vlan(skb, &qpn_vlan)) bf_ok = false; - } netdev_txq_bql_enqueue_prefetchw(ring->tx_queue); -- 2.53.1-Meta Extract the code which converts skb csum fields to WQE fields into a helper. This helper will be used by a follow-up series implementing software USO, to reduce code duplication. Signed-off-by: Joe Damato Reviewed-by: Daniel Machon --- drivers/net/ethernet/mellanox/mlx4/en_tx.c | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx4/en_tx.c b/drivers/net/ethernet/mellanox/mlx4/en_tx.c index 00210dbae1c3..f6059b533eac 100644 --- a/drivers/net/ethernet/mellanox/mlx4/en_tx.c +++ b/drivers/net/ethernet/mellanox/mlx4/en_tx.c @@ -861,6 +861,16 @@ static bool mlx4_en_build_dma_wqe(struct mlx4_en_priv *priv, return false; } +static __be32 mlx4_en_csum_flags(const struct sk_buff *skb) +{ + if (unlikely(skb->ip_summed != CHECKSUM_PARTIAL)) + return 0; + if (skb->encapsulation) + return cpu_to_be32(MLX4_WQE_CTRL_IP_CSUM); + + return cpu_to_be32(MLX4_WQE_CTRL_IP_CSUM | MLX4_WQE_CTRL_TCP_UDP_CSUM); +} + static bool mlx4_en_prep_vlan(const struct sk_buff *skb, union mlx4_wqe_qpn_vlan *qpn_vlan) { @@ -891,12 +901,12 @@ netdev_tx_t mlx4_en_xmit(struct sk_buff *skb, struct net_device *dev) struct mlx4_wqe_data_seg *data; struct mlx4_en_tx_info *tx_info; u32 __maybe_unused ring_cons; + __be32 op_own, csum_flags; int tx_ind; int nr_txbb; int desc_size; int real_size; u32 index, bf_index; - __be32 op_own; int lso_header_size; void *fragptr = NULL; bool bounce = false; @@ -991,12 +1001,10 @@ netdev_tx_t mlx4_en_xmit(struct sk_buff *skb, struct net_device *dev) /* Prepare ctrl segment apart opcode+ownership, which depends on * whether LSO is used */ tx_desc->ctrl.srcrb_flags = priv->ctrl_flags; - if (likely(skb->ip_summed == CHECKSUM_PARTIAL)) { - if (!skb->encapsulation) - tx_desc->ctrl.srcrb_flags |= cpu_to_be32(MLX4_WQE_CTRL_IP_CSUM | - MLX4_WQE_CTRL_TCP_UDP_CSUM); - else - tx_desc->ctrl.srcrb_flags |= cpu_to_be32(MLX4_WQE_CTRL_IP_CSUM); + + csum_flags = mlx4_en_csum_flags(skb); + if (csum_flags) { + tx_desc->ctrl.srcrb_flags |= csum_flags; ring->tx_csum++; } -- 2.53.1-Meta Extract the code which copies the destination MAC into the WQE for eSwitch loopback into a helper. This helper will be used by a follow-up series implementing software USO, so USO packets between functions on the same port are looped back too. Signed-off-by: Joe Damato Reviewed-by: Daniel Machon --- drivers/net/ethernet/mellanox/mlx4/en_tx.c | 28 ++++++++++++++-------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx4/en_tx.c b/drivers/net/ethernet/mellanox/mlx4/en_tx.c index f6059b533eac..97f27e005810 100644 --- a/drivers/net/ethernet/mellanox/mlx4/en_tx.c +++ b/drivers/net/ethernet/mellanox/mlx4/en_tx.c @@ -871,6 +871,23 @@ static __be32 mlx4_en_csum_flags(const struct sk_buff *skb) return cpu_to_be32(MLX4_WQE_CTRL_IP_CSUM | MLX4_WQE_CTRL_TCP_UDP_CSUM); } +static void mlx4_en_set_loopback_dmac(const struct mlx4_en_priv *priv, + struct mlx4_en_tx_desc *tx_desc, + const struct sk_buff *skb) +{ + struct ethhdr *ethh; + + if (!(priv->flags & MLX4_EN_FLAG_ENABLE_HW_LOOPBACK)) + return; + + /* Copy dst mac address to wqe. This allows loopback in eSwitch, + * so that VFs and PF can communicate with each other + */ + ethh = (struct ethhdr *)skb->data; + tx_desc->ctrl.srcrb_flags16[0] = get_unaligned((__be16 *)ethh->h_dest); + tx_desc->ctrl.imm = get_unaligned((__be32 *)(ethh->h_dest + 2)); +} + static bool mlx4_en_prep_vlan(const struct sk_buff *skb, union mlx4_wqe_qpn_vlan *qpn_vlan) { @@ -1008,16 +1025,7 @@ netdev_tx_t mlx4_en_xmit(struct sk_buff *skb, struct net_device *dev) ring->tx_csum++; } - if (priv->flags & MLX4_EN_FLAG_ENABLE_HW_LOOPBACK) { - struct ethhdr *ethh; - - /* Copy dst mac address to wqe. This allows loopback in eSwitch, - * so that VFs and PF can communicate with each other - */ - ethh = (struct ethhdr *)skb->data; - tx_desc->ctrl.srcrb_flags16[0] = get_unaligned((__be16 *)ethh->h_dest); - tx_desc->ctrl.imm = get_unaligned((__be32 *)(ethh->h_dest + 2)); - } + mlx4_en_set_loopback_dmac(priv, tx_desc, skb); /* Handle LSO (TSO) packets */ if (lso_header_size) { -- 2.53.1-Meta When the stack sets xmit_more, mlx4_en_xmit() skips the doorbell and relies on a later call in the same batch to write it. Nothing records that a doorbell is owed, so a path that ends the batch without writing the doorbell leaves the WQEs posted before it unannounced to HW. Add ring->doorbell_pending, set when the doorbell is deferred and cleared whenever a doorbell or a BlueFlame write is issued, so such a path can flush the deferred doorbell. Place the flag in the cache line dirtied by mlx4_en_xmit(), next to prod. That line has no room left, so move tx_dropped, which is only written when an skb is dropped, to the end of the struct among the fields not used in the fast path. Signed-off-by: Joe Damato Reviewed-by: Daniel Machon --- v2: - doorbell_pending moved to the area of the struct marked as cache line used and dirtied in mlx4_en_xmit. In exchange, tx_dropped is moved among fields not written in the fast path. It is only written for drops and is read for stats. - Update the commit message. drivers/net/ethernet/mellanox/mlx4/en_tx.c | 9 +++++++-- drivers/net/ethernet/mellanox/mlx4/mlx4_en.h | 3 ++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx4/en_tx.c b/drivers/net/ethernet/mellanox/mlx4/en_tx.c index 97f27e005810..123071a47bed 100644 --- a/drivers/net/ethernet/mellanox/mlx4/en_tx.c +++ b/drivers/net/ethernet/mellanox/mlx4/en_tx.c @@ -194,6 +194,7 @@ int mlx4_en_activate_tx_ring(struct mlx4_en_priv *priv, ring->prod = 0; ring->cons = 0xffffffff; ring->last_nr_txbb = 1; + ring->doorbell_pending = false; memset(ring->tx_info, 0, ring->size * sizeof(struct mlx4_en_tx_info)); memset(ring->buf, 0, ring->buf_size); ring->free_tx_desc = mlx4_en_free_tx_desc; @@ -757,6 +758,7 @@ void mlx4_en_xmit_doorbell(struct mlx4_en_tx_ring *ring) iowrite32be( #endif (__force u32)ring->doorbell_qpn, ring->doorbell_address); + ring->doorbell_pending = false; } static void mlx4_en_tx_write_desc(struct mlx4_en_tx_ring *ring, @@ -784,16 +786,19 @@ static void mlx4_en_tx_write_desc(struct mlx4_en_tx_ring *ring, wmb(); ring->bf.offset ^= ring->bf.buf_size; + ring->doorbell_pending = false; } else { /* Ensure new descriptor hits memory * before setting ownership of this descriptor to HW */ dma_wmb(); tx_desc->ctrl.owner_opcode = op_own; - if (send_doorbell) + if (send_doorbell) { mlx4_en_xmit_doorbell(ring); - else + } else { ring->xmit_more++; + ring->doorbell_pending = true; + } } } diff --git a/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h b/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h index aab97694f86b..c28d5213cccd 100644 --- a/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h +++ b/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h @@ -274,7 +274,7 @@ struct mlx4_en_tx_ring { /* cache line used and dirtied in mlx4_en_xmit() */ u32 prod ____cacheline_aligned_in_smp; - unsigned int tx_dropped; + bool doorbell_pending; unsigned long bytes; unsigned long packets; unsigned long tx_csum; @@ -311,6 +311,7 @@ struct mlx4_en_tx_ring { enum mlx4_qp_state sp_qp_state; u16 sp_stride; u16 sp_cqn; /* index of port CQ associated with this ring */ + unsigned int tx_dropped; } ____cacheline_aligned_in_smp; struct mlx4_en_rx_desc { -- 2.53.1-Meta If mlx4_en_xmit drops an skb (for example because DMA mapping fails) and that skb happens to be the last one in the batch, the doorbell for the WQEs posted earlier in the batch isn't written. The queue stays awake, though, so those WQEs are eventually announced to the HW after the next TX, but that depends on traffic and could induce latency. Avoid all of that by writing the pending doorbell in the drop path. Signed-off-by: Joe Damato Reviewed-by: Daniel Machon --- drivers/net/ethernet/mellanox/mlx4/en_tx.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/net/ethernet/mellanox/mlx4/en_tx.c b/drivers/net/ethernet/mellanox/mlx4/en_tx.c index 123071a47bed..3166837711be 100644 --- a/drivers/net/ethernet/mellanox/mlx4/en_tx.c +++ b/drivers/net/ethernet/mellanox/mlx4/en_tx.c @@ -1135,6 +1135,9 @@ netdev_tx_t mlx4_en_xmit(struct sk_buff *skb, struct net_device *dev) tx_drop_count: ring->tx_dropped++; + /* an earlier skb in this batch may have deferred the doorbell */ + if (ring->doorbell_pending) + mlx4_en_xmit_doorbell(ring); tx_drop: dev_kfree_skb_any(skb); return NETDEV_TX_OK; -- 2.53.1-Meta