With TX pause enabled, if a device is unable to pass packets up to the stack (e.g., CPU is hanged), the device can cause pause storm. Given that devices can have native support to protect the neighbor from such flooding, such events need some tracking. This support is to track TX pause storm events for better observability. Signed-off-by: Jakub Kicinski Signed-off-by: Mohsin Bashir --- Documentation/netlink/specs/ethtool.yaml | 13 +++++++++++++ include/linux/ethtool.h | 2 ++ include/uapi/linux/ethtool_netlink_generated.h | 1 + net/ethtool/pause.c | 4 +++- 4 files changed, 19 insertions(+), 1 deletion(-) diff --git a/Documentation/netlink/specs/ethtool.yaml b/Documentation/netlink/specs/ethtool.yaml index 0a2d2343f79a..4707063af3b4 100644 --- a/Documentation/netlink/specs/ethtool.yaml +++ b/Documentation/netlink/specs/ethtool.yaml @@ -879,6 +879,19 @@ attribute-sets: - name: rx-frames type: u64 + - + name: tx-pause-storm-events + type: u64 + doc: >- + TX pause storm event count. Increments each time device + detects that its pause assertion condition has been true + for too long for normal operation. As a result, the device + has temporarily disabled its own Pause TX function to + protect the network from itself. + This counter should never increment under normal overload + conditions; it indicates catastrophic failure like an OS + crash. The rate of incrementing is implementation specific. + - name: pause attr-cnt-name: __ethtool-a-pause-cnt diff --git a/include/linux/ethtool.h b/include/linux/ethtool.h index 798abec67a1b..83c375840835 100644 --- a/include/linux/ethtool.h +++ b/include/linux/ethtool.h @@ -512,12 +512,14 @@ struct ethtool_eth_ctrl_stats { * * Equivalent to `30.3.4.3 aPAUSEMACCtrlFramesReceived` * from the standard. + * @tx_pause_storm_events: TX pause storm event count (see ethtool.yaml). */ struct ethtool_pause_stats { enum ethtool_mac_stats_src src; struct_group(stats, u64 tx_pause_frames; u64 rx_pause_frames; + u64 tx_pause_storm_events; ); }; diff --git a/include/uapi/linux/ethtool_netlink_generated.h b/include/uapi/linux/ethtool_netlink_generated.h index 556a0c834df5..114b83017297 100644 --- a/include/uapi/linux/ethtool_netlink_generated.h +++ b/include/uapi/linux/ethtool_netlink_generated.h @@ -381,6 +381,7 @@ enum { ETHTOOL_A_PAUSE_STAT_PAD, ETHTOOL_A_PAUSE_STAT_TX_FRAMES, ETHTOOL_A_PAUSE_STAT_RX_FRAMES, + ETHTOOL_A_PAUSE_STAT_TX_PAUSE_STORM_EVENTS, __ETHTOOL_A_PAUSE_STAT_CNT, ETHTOOL_A_PAUSE_STAT_MAX = (__ETHTOOL_A_PAUSE_STAT_CNT - 1) diff --git a/net/ethtool/pause.c b/net/ethtool/pause.c index 0f9af1e66548..5d28f642764c 100644 --- a/net/ethtool/pause.c +++ b/net/ethtool/pause.c @@ -130,7 +130,9 @@ static int pause_put_stats(struct sk_buff *skb, if (ethtool_put_stat(skb, pause_stats->tx_pause_frames, ETHTOOL_A_PAUSE_STAT_TX_FRAMES, pad) || ethtool_put_stat(skb, pause_stats->rx_pause_frames, - ETHTOOL_A_PAUSE_STAT_RX_FRAMES, pad)) + ETHTOOL_A_PAUSE_STAT_RX_FRAMES, pad) || + ethtool_put_stat(skb, pause_stats->tx_pause_storm_events, + ETHTOOL_A_PAUSE_STAT_TX_PAUSE_STORM_EVENTS, pad)) goto err_cancel; nla_nest_end(skb, nest); -- 2.47.3 Enable protection against TX pause storm. When detected, the device tolerates the storm for 500ms before it stops sending pause frames. Use service task to bring the device back to the normal operation, preparing it for any subsequent episode of pause storm. Signed-off-by: Jakub Kicinski Signed-off-by: Mohsin Bashir --- drivers/net/ethernet/meta/fbnic/fbnic_csr.h | 10 ++++ drivers/net/ethernet/meta/fbnic/fbnic_irq.c | 2 + drivers/net/ethernet/meta/fbnic/fbnic_mac.c | 62 +++++++++++++++++++-- drivers/net/ethernet/meta/fbnic/fbnic_mac.h | 15 +++++ drivers/net/ethernet/meta/fbnic/fbnic_pci.c | 4 ++ 5 files changed, 89 insertions(+), 4 deletions(-) diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_csr.h b/drivers/net/ethernet/meta/fbnic/fbnic_csr.h index 422265dc7abd..39eb7b94a310 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_csr.h +++ b/drivers/net/ethernet/meta/fbnic/fbnic_csr.h @@ -230,6 +230,7 @@ enum { #define FBNIC_INTR_MSIX_CTRL_VECTOR_MASK CSR_GENMASK(7, 0) #define FBNIC_INTR_MSIX_CTRL_ENABLE CSR_BIT(31) enum { + FBNIC_INTR_MSIX_CTRL_RXB_IDX = 7, FBNIC_INTR_MSIX_CTRL_PCS_IDX = 34, }; @@ -560,6 +561,11 @@ enum { #define FBNIC_RXB_DROP_THLD_CNT 8 #define FBNIC_RXB_DROP_THLD_ON CSR_GENMASK(12, 0) #define FBNIC_RXB_DROP_THLD_OFF CSR_GENMASK(25, 13) +#define FBNIC_RXB_PAUSE_STORM(n) (0x08019 + (n)) /* 0x20064 + 4*n */ +#define FBNIC_RXB_PAUSE_STORM_CNT 4 +#define FBNIC_RXB_PAUSE_STORM_FORCE_NORMAL CSR_BIT(20) +#define FBNIC_RXB_PAUSE_STORM_THLD_TIME CSR_GENMASK(19, 0) +#define FBNIC_RXB_PAUSE_STORM_UNIT_WR 0x0801d /* 0x20074 */ #define FBNIC_RXB_ECN_THLD(n) (0x0801e + (n)) /* 0x20078 + 4*n */ #define FBNIC_RXB_ECN_THLD_CNT 8 #define FBNIC_RXB_ECN_THLD_ON CSR_GENMASK(12, 0) @@ -596,6 +602,9 @@ enum { #define FBNIC_RXB_INTF_CREDIT_MASK2 CSR_GENMASK(11, 8) #define FBNIC_RXB_INTF_CREDIT_MASK3 CSR_GENMASK(15, 12) +#define FBNIC_RXB_ERR_INTR_STS 0x08050 /* 0x20140 */ +#define FBNIC_RXB_ERR_INTR_STS_PS CSR_GENMASK(15, 12) +#define FBNIC_RXB_ERR_INTR_MASK 0x08052 /* 0x20148 */ #define FBNIC_RXB_PAUSE_EVENT_CNT(n) (0x08053 + (n)) /* 0x2014c + 4*n */ #define FBNIC_RXB_DROP_FRMS_STS(n) (0x08057 + (n)) /* 0x2015c + 4*n */ #define FBNIC_RXB_DROP_BYTES_STS_L(n) \ @@ -636,6 +645,7 @@ enum { #define FBNIC_RXB_PBUF_FIFO_LEVEL(n) (0x0811d + (n)) /* 0x20474 + 4*n */ +#define FBNIC_RXB_PAUSE_STORM_UNIT_RD 0x08125 /* 0x20494 */ #define FBNIC_RXB_INTEGRITY_ERR(n) (0x0812f + (n)) /* 0x204bc + 4*n */ #define FBNIC_RXB_MAC_ERR(n) (0x08133 + (n)) /* 0x204cc + 4*n */ #define FBNIC_RXB_PARSER_ERR(n) (0x08137 + (n)) /* 0x204dc + 4*n */ diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_irq.c b/drivers/net/ethernet/meta/fbnic/fbnic_irq.c index 02e8b0b257fe..1e6a8fd6f702 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_irq.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_irq.c @@ -170,6 +170,8 @@ int fbnic_mac_request_irq(struct fbnic_dev *fbd) fbnic_wr32(fbd, FBNIC_INTR_MSIX_CTRL(FBNIC_INTR_MSIX_CTRL_PCS_IDX), FBNIC_PCS_MSIX_ENTRY | FBNIC_INTR_MSIX_CTRL_ENABLE); + fbnic_wr32(fbd, FBNIC_INTR_MSIX_CTRL(FBNIC_INTR_MSIX_CTRL_RXB_IDX), 0); + fbd->mac_msix_vector = vector; return 0; diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_mac.c b/drivers/net/ethernet/meta/fbnic/fbnic_mac.c index 9d0e4b2cc9ac..9cdf21192f98 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_mac.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_mac.c @@ -143,6 +143,7 @@ static void fbnic_mac_init_qm(struct fbnic_dev *fbd) #define FBNIC_DROP_EN_MASK 0x7d #define FBNIC_PAUSE_EN_MASK 0x14 #define FBNIC_ECN_EN_MASK 0x10 +#define FBNIC_PS_EN_MASK 0x01 struct fbnic_fifo_config { unsigned int addr; @@ -424,13 +425,17 @@ static void fbnic_mac_tx_pause_config(struct fbnic_dev *fbd, bool tx_pause) { u32 rxb_pause_ctrl; - /* Enable generation of pause frames if enabled */ + /* Enable generation of pause frames based on tx_pause setting */ rxb_pause_ctrl = rd32(fbd, FBNIC_RXB_PAUSE_DROP_CTRL); - rxb_pause_ctrl &= ~FBNIC_RXB_PAUSE_DROP_CTRL_PAUSE_ENABLE; - if (tx_pause) + rxb_pause_ctrl &= ~(FBNIC_RXB_PAUSE_DROP_CTRL_PAUSE_ENABLE | + FBNIC_RXB_PAUSE_DROP_CTRL_PS_ENABLE); + if (tx_pause) { rxb_pause_ctrl |= FIELD_PREP(FBNIC_RXB_PAUSE_DROP_CTRL_PAUSE_ENABLE, - FBNIC_PAUSE_EN_MASK); + FBNIC_PAUSE_EN_MASK) | + FIELD_PREP(FBNIC_RXB_PAUSE_DROP_CTRL_PS_ENABLE, + FBNIC_PS_EN_MASK); + } wr32(fbd, FBNIC_RXB_PAUSE_DROP_CTRL, rxb_pause_ctrl); } @@ -445,6 +450,33 @@ static int fbnic_mac_get_link_event(struct fbnic_dev *fbd) FBNIC_LINK_EVENT_UP : FBNIC_LINK_EVENT_NONE; } +static void fbnic_mac_pause_storm_config(struct fbnic_dev *fbd, bool tx_pause) +{ + u32 reg; + + wr32(fbd, FBNIC_RXB_PAUSE_STORM_UNIT_WR, + FBNIC_RXB_PAUSE_STORM_CLK_DIV); + + wr32(fbd, FBNIC_RXB_PAUSE_STORM(0), + FIELD_PREP(FBNIC_RXB_PAUSE_STORM_THLD_TIME, + FBNIC_MAC_PAUSE_STORM_INTERVAL)); + + /* There is a possibility that in transition from tx_pause enable to + * disable, the interrupt status was set and we did not clear it. To + * handle this case, we will clear the interrupt status here. + */ + wr32(fbd, FBNIC_RXB_ERR_INTR_STS, + FIELD_PREP(FBNIC_RXB_ERR_INTR_STS_PS, FBNIC_PS_EN_MASK)); + + /* Unmask the Network to Host PS interrupt if tx_pause is on */ + reg = rd32(fbd, FBNIC_RXB_ERR_INTR_MASK); + reg |= FBNIC_RXB_ERR_INTR_STS_PS; + if (tx_pause) + reg &= ~FIELD_PREP(FBNIC_RXB_ERR_INTR_STS_PS, + FBNIC_PS_EN_MASK); + wr32(fbd, FBNIC_RXB_ERR_INTR_MASK, reg); +} + static u32 __fbnic_mac_cmd_config_asic(struct fbnic_dev *fbd, bool tx_pause, bool rx_pause) { @@ -658,6 +690,7 @@ static void fbnic_mac_link_up_asic(struct fbnic_dev *fbd, u32 cmd_cfg, mac_ctrl; fbnic_mac_tx_pause_config(fbd, tx_pause); + fbnic_mac_pause_storm_config(fbd, tx_pause); cmd_cfg = __fbnic_mac_cmd_config_asic(fbd, tx_pause, rx_pause); mac_ctrl = rd32(fbd, FBNIC_SIG_MAC_IN0); @@ -918,3 +951,24 @@ int fbnic_mac_init(struct fbnic_dev *fbd) return 0; } + +void fbnic_mac_rxb_pause_storm_handler(struct fbnic_dev *fbd) +{ + u32 rxb_err_sts = rd32(fbd, FBNIC_RXB_ERR_INTR_STS); + + /* Check if pause storm interrupt for network was triggered */ + if (rxb_err_sts & FBNIC_RXB_ERR_INTR_STS_PS) { + wr32(fbd, FBNIC_RXB_PAUSE_STORM(0), + FIELD_PREP(FBNIC_RXB_PAUSE_STORM_THLD_TIME, + FBNIC_MAC_PAUSE_STORM_INTERVAL) | + FBNIC_RXB_PAUSE_STORM_FORCE_NORMAL); + wrfl(fbd); + wr32(fbd, FBNIC_RXB_PAUSE_STORM(0), + FIELD_PREP(FBNIC_RXB_PAUSE_STORM_THLD_TIME, + FBNIC_MAC_PAUSE_STORM_INTERVAL)); + + /* Write 1 to clear the interrupt status */ + wr32(fbd, FBNIC_RXB_ERR_INTR_STS, + FIELD_PREP(FBNIC_RXB_ERR_INTR_STS_PS, FBNIC_PS_EN_MASK)); + } +} diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_mac.h b/drivers/net/ethernet/meta/fbnic/fbnic_mac.h index f08fe8b7c497..812023a33d5c 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_mac.h +++ b/drivers/net/ethernet/meta/fbnic/fbnic_mac.h @@ -8,6 +8,20 @@ struct fbnic_dev; +/* The RXB clock runs at 600 MHZ in the ASIC and the PAUSE_STORM_UNIT_WR + * is 10us granularity, so set the clock to 6000 (0x1770) + */ +#define FBNIC_RXB_PAUSE_STORM_CLK_DIV 0x1770 + +/* Configure the timer to 500msec which should be longer than any + * reasonable period of continuous pausing. The service task, which runs + * once per second, periodically resets the pause storm trigger. + + * As a result on a functioning system we enforce a 50% duty cycle should + * continuous pause ever occur. A crashed system will not have the service + * task and therefore pause will remain disabled until reboot recovery. + */ +#define FBNIC_MAC_PAUSE_STORM_INTERVAL 50000 #define FBNIC_MAX_JUMBO_FRAME_SIZE 9742 /* States loosely based on section 136.8.11.7.5 of IEEE 802.3-2022 Ethernet @@ -119,4 +133,5 @@ struct fbnic_mac { int fbnic_mac_init(struct fbnic_dev *fbd); void fbnic_mac_get_fw_settings(struct fbnic_dev *fbd, u8 *aui, u8 *fec); +void fbnic_mac_rxb_pause_storm_handler(struct fbnic_dev *fbd); #endif /* _FBNIC_MAC_H_ */ diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_pci.c b/drivers/net/ethernet/meta/fbnic/fbnic_pci.c index 9240673c7533..a8c9dc2d0d6c 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_pci.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_pci.c @@ -216,6 +216,10 @@ static void fbnic_service_task(struct work_struct *work) fbnic_get_hw_stats32(fbd); + if (!FIELD_GET(FBNIC_MAC_COMMAND_CONFIG_TX_PAUSE_DIS, + rd32(fbd, FBNIC_MAC_COMMAND_CONFIG))) + fbnic_mac_rxb_pause_storm_handler(fbd); + fbnic_fw_check_heartbeat(fbd); fbnic_health_check(fbd); -- 2.47.3 With pause storm protection in place, track the occurrence of pause storm events. Since there is a one-to-one mapping between pause storm interrupts and events, use the interrupt count to track this metric. ./ethtool -I -a eth0 Pause parameters for eth0: Autonegotiate: off RX: off TX: on Statistics: tx_pause_frames: 759657 rx_pause_frames: 0 tx_pause_storm_events: 219 Signed-off-by: Jakub Kicinski Signed-off-by: Mohsin Bashir --- drivers/net/ethernet/meta/fbnic/fbnic_csr.h | 1 + drivers/net/ethernet/meta/fbnic/fbnic_ethtool.c | 3 +++ drivers/net/ethernet/meta/fbnic/fbnic_hw_stats.h | 1 + drivers/net/ethernet/meta/fbnic/fbnic_mac.c | 15 +++++++++++++++ 4 files changed, 20 insertions(+) diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_csr.h b/drivers/net/ethernet/meta/fbnic/fbnic_csr.h index 39eb7b94a310..53c04293083c 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_csr.h +++ b/drivers/net/ethernet/meta/fbnic/fbnic_csr.h @@ -627,6 +627,7 @@ enum { FBNIC_RXB_ENQUEUE_INDICES = 4 }; +#define FBNIC_RXB_INTR_PS_COUNT(n) (0x080e9 + (n)) /* 0x203a4 + 4*n */ #define FBNIC_RXB_DRBO_FRM_CNT_SRC(n) (0x080f9 + (n)) /* 0x203e4 + 4*n */ #define FBNIC_RXB_DRBO_BYTE_CNT_SRC_L(n) \ (0x080fd + (n)) /* 0x203f4 + 4*n */ diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_ethtool.c b/drivers/net/ethernet/meta/fbnic/fbnic_ethtool.c index 61b8005a0db5..ad7bf876a044 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_ethtool.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_ethtool.c @@ -1707,6 +1707,7 @@ fbnic_get_pause_stats(struct net_device *netdev, struct fbnic_net *fbn = netdev_priv(netdev); struct fbnic_mac_stats *mac_stats; struct fbnic_dev *fbd = fbn->fbd; + u64 tx_ps_events; mac_stats = &fbd->hw_stats.mac; @@ -1714,6 +1715,8 @@ fbnic_get_pause_stats(struct net_device *netdev, pause_stats->tx_pause_frames = mac_stats->pause.tx_pause_frames.value; pause_stats->rx_pause_frames = mac_stats->pause.rx_pause_frames.value; + tx_ps_events = mac_stats->pause.tx_pause_storm_events.value; + pause_stats->tx_pause_storm_events = tx_ps_events; } static void diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_hw_stats.h b/drivers/net/ethernet/meta/fbnic/fbnic_hw_stats.h index aa3f429a9aed..caea4be46762 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_hw_stats.h +++ b/drivers/net/ethernet/meta/fbnic/fbnic_hw_stats.h @@ -54,6 +54,7 @@ struct fbnic_rmon_stats { struct fbnic_pause_stats { struct fbnic_stat_counter tx_pause_frames; struct fbnic_stat_counter rx_pause_frames; + struct fbnic_stat_counter tx_pause_storm_events; }; struct fbnic_eth_mac_stats { diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_mac.c b/drivers/net/ethernet/meta/fbnic/fbnic_mac.c index 9cdf21192f98..97ae7086d7bd 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_mac.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_mac.c @@ -418,6 +418,18 @@ static void __fbnic_mac_stat_rd64(struct fbnic_dev *fbd, bool reset, u32 reg, stat->reported = true; } +static void fbnic_mac_stat_rd32(struct fbnic_dev *fbd, bool reset, u32 reg, + struct fbnic_stat_counter *stat) +{ + u32 new_reg_value; + + new_reg_value = rd32(fbd, reg); + if (!reset) + stat->value += new_reg_value - stat->u.old_reg_value_32; + stat->u.old_reg_value_32 = new_reg_value; + stat->reported = true; +} + #define fbnic_mac_stat_rd64(fbd, reset, __stat, __CSR) \ __fbnic_mac_stat_rd64(fbd, reset, FBNIC_##__CSR##_L, &(__stat)) @@ -792,6 +804,9 @@ fbnic_mac_get_pause_stats(struct fbnic_dev *fbd, bool reset, MAC_STAT_TX_XOFF_STB); fbnic_mac_stat_rd64(fbd, reset, pause_stats->rx_pause_frames, MAC_STAT_RX_XOFF_STB); + fbnic_mac_stat_rd32(fbd, reset, + FBNIC_RXB_INTR_PS_COUNT(FBNIC_RXB_INTF_NET), + &pause_stats->tx_pause_storm_events); } static void -- 2.47.3