AI Trajectory

Trajectory: (Expand/Collapse All)
Seq	Timestamp	Type	Name	Duration
0/0	2026/11/02 14:59	flow	patch-triage	1m Args: null Results: { "EnableConfigs": null, "FocusSymbols": [], "KMSANReasoning": "", "NeedsKMSAN": false, "Reasoning": "The patch series only touches the Nebula Matrix (nbl) Ethernet driver (drivers/net/ethernet/nebula-matrix/nbl/), which is a PCIe driver for physical Nebula Matrix Leonis ASIC family NICs (m18110/m18000). These devices require dedicated physical PCIe hardware and are not emulated in standard QEMU/GCE virtualized test environments. Consequently, the modified code is structurally unreachable for fuzzing.", "WorthFuzzing": false }
1/1	2026/11/02 14:59	action	read-patch-diff	0m Args: null Results: { "PatchDiff": "commit eb6ce827881d54724a7b175422cd39cff38773db\nAuthor: syz-cluster \u003ctriage@syzkaller.com\u003e\nDate: Mon Nov 2 14:59:56 2026 +0000\n\n syz-cluster: applied patch under review\n\ndiff --git a/drivers/net/ethernet/nebula-matrix/nbl/Makefile b/drivers/net/ethernet/nebula-matrix/nbl/Makefile\nindex cc060cf8bf7596..71fbe3ee7e6204 100644\n--- a/drivers/net/ethernet/nebula-matrix/nbl/Makefile\n+++ b/drivers/net/ethernet/nebula-matrix/nbl/Makefile\n@@ -3,5 +3,13 @@\n \n obj-$(CONFIG_NBL) := nbl.o\n \n-nbl-objs +=\tnbl_hw/nbl_hw_leonis/nbl_hw_leonis.o \\\n+nbl-objs +=\tnbl_common/nbl_common.o \\\n+\t\tnbl_channel/nbl_channel.o \\\n+\t\tnbl_hw/nbl_hw_leonis/nbl_hw_leonis.o \\\n+\t\tnbl_hw/nbl_hw_leonis/nbl_resource_leonis.o \\\n+\t\tnbl_hw/nbl_resource.o \\\n+\t\tnbl_hw/nbl_interrupt.o \\\n+\t\tnbl_hw/nbl_chip.o \\\n+\t\tnbl_core/nbl_dispatch.o \\\n+\t\tnbl_core/nbl_dev.o \\\n \t\tnbl_main.o\ndiff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_channel/nbl_channel.c b/drivers/net/ethernet/nebula-matrix/nbl/nbl_channel/nbl_channel.c\nnew file mode 100644\nindex 00000000000000..5b528cc6efc452\n--- /dev/null\n+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_channel/nbl_channel.c\n@@ -0,0 +1,1528 @@\n+// SPDX-License-Identifier: GPL-2.0\n+/\n+ Copyright (c) 2026 Nebula Matrix Limited.\n+ /\n+#include \u003clinux/delay.h\u003e\n+#include \u003clinux/device.h\u003e\n+#include \u003clinux/mutex.h\u003e\n+#include \u003clinux/bitfield.h\u003e\n+#include \u003clinux/pci.h\u003e\n+#include \u003clinux/bits.h\u003e\n+#include \u003clinux/dma-mapping.h\u003e\n+#include \u003clinux/atomic.h\u003e\n+#include \u003clinux/wait.h\u003e\n+#include \"nbl_channel.h\"\n+\n+static int nbl_chan_add_msg_handler(struct nbl_channel_mgt chan_mgt,\n+\t\t\t\t u16 msg_type, nbl_chan_resp func,\n+\t\t\t\t void priv)\n+{\n+\tstruct nbl_chan_msg_node_data handler;\n+\tint ret;\n+\n+\thandler = kzalloc_obj(handler, GFP_KERNEL);\n+\tif (!handler)\n+\t\treturn -ENOMEM;\n+\n+\thandler-\u003efunc = func;\n+\thandler-\u003epriv = priv;\n+\n+\t/ Each msg_type is registered at most once; reject duplicates. /\n+\tmutex_lock(\u0026chan_mgt-\u003ehandler_lock);\n+\tret = xa_insert(\u0026chan_mgt-\u003ehandler_xa, msg_type, handler, GFP_KERNEL);\n+\tmutex_unlock(\u0026chan_mgt-\u003ehandler_lock);\n+\tif (ret)\n+\t\tkfree(handler);\n+\n+\treturn ret;\n+}\n+\n+static int nbl_chan_init_msg_handler(struct nbl_channel_mgt chan_mgt)\n+{\n+\tint ret;\n+\n+\tret = devm_mutex_init(chan_mgt-\u003ecommon-\u003edev, \u0026chan_mgt-\u003ehandler_lock);\n+\tif (ret)\n+\t\treturn ret;\n+\n+\txa_init(\u0026chan_mgt-\u003ehandler_xa);\n+\n+\treturn 0;\n+}\n+\n+static void nbl_chan_remove_msg_handler(struct nbl_channel_mgt chan_mgt)\n+{\n+\tstruct nbl_chan_msg_node_data handler;\n+\tunsigned long msg_type;\n+\n+\tmutex_lock(\u0026chan_mgt-\u003ehandler_lock);\n+\txa_for_each(\u0026chan_mgt-\u003ehandler_xa, msg_type, handler) {\n+\t\txa_erase(\u0026chan_mgt-\u003ehandler_xa, msg_type);\n+\t\tkfree(handler);\n+\t}\n+\tmutex_unlock(\u0026chan_mgt-\u003ehandler_lock);\n+\txa_destroy(\u0026chan_mgt-\u003ehandler_xa);\n+}\n+\n+static void nbl_chan_init_queue_param(struct nbl_chan_info chan_info,\n+\t\t\t\t u16 num_txq_entries, u16 num_rxq_entries,\n+\t\t\t\t u16 txq_buf_size, u16 rxq_buf_size)\n+{\n+\tchan_info-\u003enum_txq_entries = num_txq_entries;\n+\tchan_info-\u003enum_rxq_entries = num_rxq_entries;\n+\tchan_info-\u003etxq_buf_size = txq_buf_size;\n+\tchan_info-\u003erxq_buf_size = rxq_buf_size;\n+\tatomic_set(\u0026chan_info-\u003einflight_tx_cnt, 0);\n+\tWRITE_ONCE(chan_info-\u003eshutdn, false);\n+\tWRITE_ONCE(chan_info-\u003eactive, false);\n+\tWRITE_ONCE(chan_info-\u003ewait_head_index, 0);\n+\tmemset(chan_info-\u003estate, 0, sizeof(chan_info-\u003estate));\n+\tinit_waitqueue_head(\u0026chan_info-\u003einflight_wait);\n+}\n+\n+static int nbl_chan_init_tx_queue(struct nbl_common_info common,\n+\t\t\t\t struct nbl_chan_info chan_info)\n+{\n+\tstruct nbl_chan_ring txq = \u0026chan_info-\u003etxq;\n+\tstruct device dev = common-\u003edev;\n+\tdma_addr_t base_dma;\n+\tsize_t size =\n+\t\tchan_info-\u003enum_txq_entries sizeof(struct nbl_chan_tx_desc);\n+\tvoid base;\n+\tu16 i;\n+\n+\ttxq-\u003edesc.tx_desc = dmam_alloc_coherent(dev, size, \u0026txq-\u003edma,\n+\t\t\t\t\t\tGFP_KERNEL);\n+\tif (!txq-\u003edesc.tx_desc)\n+\t\treturn -ENOMEM;\n+\n+\t/\n+\t * The per-entry data buffers all come from one contiguous coherent\n+\t * block, so those buffers cost exactly one coherent allocation, one\n+\t * devres node and one IOMMU mapping for the whole ring whatever the\n+\t * page size is. Per-entry dmam_alloc_coherent() instead rounds every\n+\t * NBL_CHAN_BUF_LEN buffer up to a whole page (8 MiB per PF on 16 KiB\n+\t * pages, 32 MiB on 64 KiB pages), and a dma_pool only fixes the\n+\t * large-page case: with NBL_CHAN_BUF_LEN equal to PAGE_SIZE it still\n+\t * degenerates to one allocation per buffer on 4 KiB pages.\n+\t * NBL_CHAN_BUF_LEN is a power of two, so the block always divides\n+\t * evenly into per-entry slots.\n+\t \n+\t That is the only part of the ring that is collapsed: the descriptor\n+\t * ring above is its own dmam_alloc_coherent(), and txq-\u003ebuf[] and\n+\t * chan_info-\u003ewait[] below are separate devm_kcalloc() nodes, so a\n+\t * ring is two coherent mappings plus those arrays, not one.\n+\t /\n+\tbase = dmam_alloc_coherent(dev,\n+\t\t\t\t chan_info-\u003enum_txq_entries \n+\t\t\t\t chan_info-\u003etxq_buf_size,\n+\t\t\t\t \u0026base_dma, GFP_KERNEL);\n+\tif (!base)\n+\t\treturn -ENOMEM;\n+\n+\tchan_info-\u003ewait = devm_kcalloc(dev, chan_info-\u003enum_txq_entries,\n+\t\t\t\t sizeof(chan_info-\u003ewait), GFP_KERNEL);\n+\tif (!chan_info-\u003ewait)\n+\t\treturn -ENOMEM;\n+\tfor (i = 0; i \u003c chan_info-\u003enum_txq_entries; i++) {\n+\t\tinit_waitqueue_head(\u0026chan_info-\u003ewait[i].wait_queue);\n+\t\tWRITE_ONCE(chan_info-\u003ewait[i].status, NBL_MBX_STATUS_IDLE);\n+\t\tWRITE_ONCE(chan_info-\u003ewait[i].acked, 0);\n+\t\tWRITE_ONCE(chan_info-\u003ewait[i].ack_data, NULL);\n+\t\tWRITE_ONCE(chan_info-\u003ewait[i].ack_data_len, 0);\n+\t\tWRITE_ONCE(chan_info-\u003ewait[i].ack_err, 0);\n+\t\tWRITE_ONCE(chan_info-\u003ewait[i].msg_type, 0);\n+\t\tWRITE_ONCE(chan_info-\u003ewait[i].msg_index, 0);\n+\t\tWRITE_ONCE(chan_info-\u003ewait[i].dstid, 0);\n+\t}\n+\n+\ttxq-\u003ebuf = devm_kcalloc(dev, chan_info-\u003enum_txq_entries,\n+\t\t\t\tsizeof(txq-\u003ebuf), GFP_KERNEL);\n+\tif (!txq-\u003ebuf)\n+\t\treturn -ENOMEM;\n+\n+\t/* Carve the contiguous block into fixed-size per-entry slots /\n+\tfor (i = 0; i \u003c chan_info-\u003enum_txq_entries; i++) {\n+\t\ttxq-\u003ebuf[i].va = (u8 )base +\n+\t\t\t\t (size_t)i * chan_info-\u003etxq_buf_size;\n+\t\ttxq-\u003ebuf[i].pa = base_dma +\n+\t\t\t\t (dma_addr_t)i * chan_info-\u003etxq_buf_size;\n+\t\ttxq-\u003ebuf[i].size = chan_info-\u003etxq_buf_size;\n+\t}\n+\n+\ttxq-\u003enext_to_clean = 0;\n+\ttxq-\u003enext_to_use = 0;\n+\ttxq-\u003etail_ptr = 0;\n+\n+\treturn 0;\n+}\n+\n+static int nbl_chan_init_rx_queue(struct nbl_common_info common,\n+\t\t\t\t struct nbl_chan_info chan_info)\n+{\n+\tstruct nbl_chan_ring rxq = \u0026chan_info-\u003erxq;\n+\tstruct device dev = common-\u003edev;\n+\tstruct nbl_chan_rx_desc desc;\n+\tdma_addr_t base_dma;\n+\tsize_t size =\n+\t\tchan_info-\u003enum_rxq_entries sizeof(struct nbl_chan_rx_desc);\n+\tvoid base;\n+\tu16 i;\n+\n+\trxq-\u003edesc.rx_desc = dmam_alloc_coherent(dev, size, \u0026rxq-\u003edma,\n+\t\t\t\t\t\tGFP_KERNEL);\n+\tif (!rxq-\u003edesc.rx_desc) {\n+\t\tdev_err_ratelimited(dev,\n+\t\t\t\t \"Allocate DMA for chan rx descriptor ring failed\\n\");\n+\t\treturn -ENOMEM;\n+\t}\n+\n+\t/ See nbl_chan_init_tx_queue() for why one contiguous block is used /\n+\tbase = dmam_alloc_coherent(dev,\n+\t\t\t\t chan_info-\u003enum_rxq_entries \n+\t\t\t\t chan_info-\u003erxq_buf_size,\n+\t\t\t\t \u0026base_dma, GFP_KERNEL);\n+\tif (!base)\n+\t\treturn -ENOMEM;\n+\n+\trxq-\u003ebuf = devm_kcalloc(dev, chan_info-\u003enum_rxq_entries,\n+\t\t\t\tsizeof(rxq-\u003ebuf), GFP_KERNEL);\n+\tif (!rxq-\u003ebuf)\n+\t\treturn -ENOMEM;\n+\n+\t/ Carve the contiguous block into fixed-size per-entry slots /\n+\tfor (i = 0; i \u003c chan_info-\u003enum_rxq_entries; i++) {\n+\t\trxq-\u003ebuf[i].va = (u8 )base +\n+\t\t\t\t (size_t)i * chan_info-\u003erxq_buf_size;\n+\t\trxq-\u003ebuf[i].pa = base_dma +\n+\t\t\t\t (dma_addr_t)i * chan_info-\u003erxq_buf_size;\n+\t\trxq-\u003ebuf[i].size = chan_info-\u003erxq_buf_size;\n+\t}\n+\n+\tdesc = rxq-\u003edesc.rx_desc;\n+\t/\n+\t Initially leave one RX descriptor unused so that\n+\t * next_to_clean and next_to_use can distinguish an empty\n+\t * ring from a full ring.\n+\t \n+\t The unused slot is replenished as RX descriptors are\n+\t * consumed and recycled.\n+\t /\n+\tfor (i = 0; i \u003c chan_info-\u003enum_rxq_entries - 1; i++) {\n+\t\tdesc[i].buf_addr = cpu_to_le64(rxq-\u003ebuf[i].pa);\n+\t\tdesc[i].buf_len = cpu_to_le32(chan_info-\u003erxq_buf_size);\n+\t\tdesc[i].flags = cpu_to_le16(BIT(NBL_CHAN_RX_DESC_AVAIL));\n+\t}\n+\n+\trxq-\u003enext_to_clean = 0;\n+\trxq-\u003enext_to_use = chan_info-\u003enum_rxq_entries - 1;\n+\trxq-\u003etail_ptr = chan_info-\u003enum_rxq_entries - 1;\n+\n+\treturn 0;\n+}\n+\n+static int nbl_chan_init_queue(struct nbl_common_info common,\n+\t\t\t struct nbl_chan_info chan_info)\n+{\n+\tint err;\n+\n+\terr = nbl_chan_init_tx_queue(common, chan_info);\n+\tif (err)\n+\t\treturn err;\n+\n+\terr = nbl_chan_init_rx_queue(common, chan_info);\n+\n+\treturn err;\n+}\n+\n+static void nbl_chan_config_queue(struct nbl_channel_mgt chan_mgt,\n+\t\t\t\t struct nbl_chan_info chan_info, bool tx)\n+{\n+\tstruct nbl_hw_ops hw_ops = chan_mgt-\u003ehw_ops_tbl-\u003eops;\n+\tstruct nbl_hw_mgt p = chan_mgt-\u003ehw_ops_tbl-\u003epriv;\n+\tstruct nbl_chan_ring ring;\n+\tdma_addr_t addr;\n+\tint size_bwid;\n+\n+\tif (tx)\n+\t\tring = \u0026chan_info-\u003etxq;\n+\telse\n+\t\tring = \u0026chan_info-\u003erxq;\n+\taddr = ring-\u003edma;\n+\tif (tx) {\n+\t\tsize_bwid = ilog2(chan_info-\u003enum_txq_entries);\n+\t\thw_ops-\u003econfig_mailbox_txq(p, addr, size_bwid);\n+\t} else {\n+\t\tsize_bwid = ilog2(chan_info-\u003enum_rxq_entries);\n+\t\thw_ops-\u003econfig_mailbox_rxq(p, addr, size_bwid);\n+\t}\n+}\n+\n+static void nbl_chan_cfg_qinfo_map_table(struct nbl_channel_mgt chan_mgt,\n+\t\t\t\t\t u8 bus, u8 devid)\n+{\n+\tstruct nbl_hw_ops hw_ops = chan_mgt-\u003ehw_ops_tbl-\u003eops;\n+\tstruct nbl_hw_mgt p = chan_mgt-\u003ehw_ops_tbl-\u003epriv;\n+\tu32 pf_mask = 0;\n+\tu8 func_id;\n+\n+\t/\n+\t * k_pf_mask rule: bit N == 0 means PF#N enabled, bit N == 1 masked out.\n+\t * Program mailbox QINFO entry for each hardware-active PF func_id.\n+\t \n+\t Note: This loop iterates over the raw hardware PF func_id, so it\n+\t * assumes product constraints limit supported PF counts to the\n+\t * contiguous sets PF0, PF0~1 and PF0~3, i.e. pf_mask is one of\n+\t * 0xfe/0xfc/0xf0. Nothing enforces that here: the resource layer\n+\t * that validates the mask, and every caller of this op, arrive in\n+\t * later patches of this series.\n+\t /\n+\thw_ops-\u003eget_host_pf_mask(p, \u0026pf_mask);\n+\tfor (func_id = 0; func_id \u003c NBL_MAX_PF; func_id++) {\n+\t\tif (!(pf_mask \u0026 (1 \u003c\u003c func_id)))\n+\t\t\thw_ops-\u003ecfg_mailbox_qinfo(p, func_id, bus,\n+\t\t\t\t\t\t devid, func_id);\n+\t}\n+}\n+\n+/\n+ * The RX QINFO region is written only by config/stop paths, never by\n+ * senders (which only touch the TX QINFO region and the doorbell), so\n+ * stopping RX does not require txq_lock.\n+ /\n+static void nbl_chan_stop_rx_queue(struct nbl_channel_mgt chan_mgt)\n+{\n+\tstruct nbl_hw_ops hw_ops = chan_mgt-\u003ehw_ops_tbl-\u003eops;\n+\n+\thw_ops-\u003estop_mailbox_rxq(chan_mgt-\u003ehw_ops_tbl-\u003epriv);\n+}\n+\n+/\n+ * The TX QINFO region is also written by nbl_chan_quiesce_and_reclaim_tx()\n+ * under txq_lock, so TX stop must serialize against it via txq_lock.\n+ /\n+static void nbl_chan_stop_tx_queue(struct nbl_channel_mgt chan_mgt)\n+{\n+\tstruct nbl_hw_ops hw_ops = chan_mgt-\u003ehw_ops_tbl-\u003eops;\n+\n+\thw_ops-\u003estop_mailbox_txq(chan_mgt-\u003ehw_ops_tbl-\u003epriv);\n+}\n+\n+static void nbl_chan_reset_wait_head(struct nbl_chan_info chan_info,\n+\t\t\t\t struct nbl_chan_waitqueue_head wait_head)\n+{\n+\tlockdep_assert_held(\u0026chan_info-\u003epending_lock);\n+\n+\tWRITE_ONCE(wait_head-\u003eacked, 0);\n+\tWRITE_ONCE(wait_head-\u003estatus, NBL_MBX_STATUS_IDLE);\n+\tWRITE_ONCE(wait_head-\u003eack_data, NULL);\n+\tWRITE_ONCE(wait_head-\u003eack_data_len, 0);\n+\tWRITE_ONCE(wait_head-\u003eack_err, 0);\n+\tWRITE_ONCE(wait_head-\u003emsg_type, 0);\n+\tWRITE_ONCE(wait_head-\u003edstid, 0);\n+}\n+\n+static int nbl_chan_teardown_queue(struct nbl_channel_mgt chan_mgt,\n+\t\t\t\t u8 chan_type)\n+{\n+\tstruct nbl_chan_info chan_info = chan_mgt-\u003echan_info[chan_type];\n+\tstruct nbl_chan_waitqueue_head wait_head;\n+\tstruct work_struct task;\n+\tu16 i;\n+\n+\tif (!READ_ONCE(chan_info-\u003eactive)) {\n+\t\tdev_warn(chan_mgt-\u003ecommon-\u003edev, \"channel not active, skip duplicate teardown\\n\");\n+\t\treturn 0;\n+\t}\n+\t/\n+\t * Step1:\n+\t * block new sender\n+\t /\n+\tmutex_lock(\u0026chan_info-\u003estate_lock);\n+\tWRITE_ONCE(chan_info-\u003eshutdn, true);\n+\ttask = READ_ONCE(chan_info-\u003eclean_task);\n+\tWRITE_ONCE(chan_info-\u003eclean_task, NULL);\n+\tmutex_unlock(\u0026chan_info-\u003estate_lock);\n+\t/\n+\t * Step2:\n+\t * abort pending ACK waiters\n+\t \n+\t Poke only slots with a live waiter (status == WAITING): publish a\n+\t * synthetic (-EIO, len 0) completion and wake the owner. The slot is\n+\t * deliberately NOT reset to IDLE here - ownership stays with the\n+\t * sender, which consumes the completion and resets its own slot in\n+\t * nbl_chan_send_msg():out_clear_wait_slot under pending_lock.\n+\t \n+\t Resetting an owned slot here would corrupt two things:\n+\t * - an ACKD slot whose sender is consuming the completion\n+\t * locklessly (smp_rmb()-ordered reads) could observe torn\n+\t * ack_data_len/ack_err pairs;\n+\t * - nbl_chan_get_msg_id() hands out IDLE/TIMEOUT slots, so a slot\n+\t * reset to IDLE under its live owner could be reallocated to a\n+\t * sender admitted before shutdn was set.\n+\t \n+\t IDLE/TIMEOUT slots have no waiter and need no action. A late ACK\n+\t * racing this poke is serialized by pending_lock: either the real\n+\t * completion lands first (status becomes ACKD and is skipped) or\n+\t * this synthetic failure does, both are valid outcomes.\n+\t /\n+\tmutex_lock(\u0026chan_info-\u003epending_lock);\n+\tfor (i = 0; i \u003c chan_info-\u003enum_txq_entries; i++) {\n+\t\twait_head = \u0026chan_info-\u003ewait[i];\n+\t\tif (READ_ONCE(wait_head-\u003estatus) != NBL_MBX_STATUS_WAITING)\n+\t\t\tcontinue;\n+\t\tWRITE_ONCE(wait_head-\u003estatus, NBL_MBX_STATUS_ACKD);\n+\t\tWRITE_ONCE(wait_head-\u003eack_data_len, 0);\n+\t\tWRITE_ONCE(wait_head-\u003eack_err, (s32)-EIO);\n+\t\t/ Publish completion fields before the acked flag /\n+\t\tsmp_wmb();\n+\t\tWRITE_ONCE(wait_head-\u003eacked, 1);\n+\t\twake_up(\u0026wait_head-\u003ewait_queue);\n+\t}\n+\tmutex_unlock(\u0026chan_info-\u003epending_lock);\n+\n+\t/\n+\t * Step3:\n+\t * stop the RX queue unconditionally. The RX QINFO region is written\n+\t * only by config/stop paths, never by senders (which only touch the\n+\t * TX QINFO region and the doorbell), so no txq_lock is needed. The\n+\t * RX engine must be quiesced before the rings are freed, regardless\n+\t * of TX-side state - skipping it would leave the device DMAing peer\n+\t * messages and descriptor writebacks into released coherent memory.\n+\t /\n+\tnbl_chan_stop_rx_queue(chan_mgt);\n+\n+\t/\n+\t * Drain strategy mirrors mlx5 command interface teardown:\n+\t * set shutdown flag first, abort all pending waiters, then\n+\t * block until inflight_tx_cnt reaches zero.\n+\t \n+\t After shutdn is set every sender exits promptly at its next\n+\t * checkpoint:\n+\t * - interrupt-driven senders wake on shutdn immediately\n+\t * (it is part of the wait_event condition);\n+\t * - ACK polling senders check shutdn each iteration before\n+\t * sleeping 1000-1200us. This per-iteration check avoids\n+\t * waiting the full 5.0-6.0s worst-case ACK timeout.\n+\t \n+\t The wait is deliberately unbounded: a surviving sender still\n+\t * holds txq_lock or touches chan_info (state_lock, wait[]) on its\n+\t * exit path, so returning early would let devres free the rings\n+\t * and chan_info underneath it. In practice the TX polling loop is\n+\t * bounded (NBL_CHAN_TX_WAIT_TIMES) and wedged-link MMIO reads\n+\t * eventually complete with all-1s data, so this loop terminates;\n+\t * warn periodically to keep a genuinely stuck sender diagnosable.\n+\t /\n+\twhile (wait_event_timeout(chan_info-\u003einflight_wait,\n+\t\t\t\t atomic_read(\u0026chan_info-\u003einflight_tx_cnt) == 0,\n+\t\t\t\t msecs_to_jiffies(5000)) == 0)\n+\t\tdev_warn(chan_mgt-\u003ecommon-\u003edev,\n+\t\t\t \"teardown: still waiting for %d inflight sender(s)\\n\",\n+\t\t\t atomic_read(\u0026chan_info-\u003einflight_tx_cnt));\n+\n+\t/\n+\t * Join the last exiting sender through state_lock. A sender drops\n+\t * inflight_tx_cnt and wakes this drain while still holding\n+\t * state_lock, so observing the counter reach zero only proves the\n+\t * decrement happened; the acquire below is what proves that no\n+\t * sender still holds the lock. Once it returns, no sender can be\n+\t * using chan_info either: the decrement is the last field access a\n+\t * sender makes, and the only thing that follows it is the\n+\t * mutex_unlock() of state_lock itself.\n+\t \n+\t That trailing unlock is why this is one acquire/release pair and\n+\t * not a handshake that ends the sender: mutex_unlock() may still be\n+\t * touching the mutex structure after it has released it, so this does\n+\t * not prove the sender has returned from it (see\n+\t * Documentation/locking/mutex-design.rst). It is safe only because\n+\t * chan_info - mutexes included - is devm-allocated and released at\n+\t * detach, long after this teardown, and because nothing frees it\n+\t * here. Freeing chan_info or calling mutex_destroy() at this point\n+\t * would need a sender-exit completion the channel does not have.\n+\t \n+\t shutdn blocks new senders, so one acquire/release pair is\n+\t * sufficient.\n+\t /\n+\tmutex_lock(\u0026chan_info-\u003estate_lock);\n+\tmutex_unlock(\u0026chan_info-\u003estate_lock);\n+\n+\t/\n+\t * All senders have exited, so txq_lock is uncontended: plain lock.\n+\t * Take it so this is the sole writer of the TX QINFO block\n+\t * (sender-side stop/config/doorbell sequences all run under\n+\t * txq_lock).\n+\t /\n+\tmutex_lock(\u0026chan_info-\u003etxq_lock);\n+\tnbl_chan_stop_tx_queue(chan_mgt);\n+\tmutex_unlock(\u0026chan_info-\u003etxq_lock);\n+\n+\t/\n+\t * Join the RX clean work before active=false and before devres\n+\t * releases the rings. IRQ is already freed and clean_task is\n+\t * cleared under state_lock, so no new instance can be queued.\n+\t /\n+\tif (task)\n+\t\tcancel_work_sync(task);\n+\n+\tWRITE_ONCE(chan_info-\u003eactive, false);\n+\treturn 0;\n+}\n+\n+static int nbl_chan_setup_queue(struct nbl_channel_mgt chan_mgt, u8 chan_type)\n+{\n+\tstruct nbl_chan_info chan_info = chan_mgt-\u003echan_info[chan_type];\n+\tstruct nbl_hw_ops hw_ops = chan_mgt-\u003ehw_ops_tbl-\u003eops;\n+\tstruct nbl_common_info common = chan_mgt-\u003ecommon;\n+\tstruct nbl_chan_ring rxq = \u0026chan_info-\u003erxq;\n+\tint err;\n+\n+\tif (READ_ONCE(chan_info-\u003eactive)) {\n+\t\tdev_warn(common-\u003edev, \"channel already active, reject duplicate setup\\n\");\n+\t\treturn -EBUSY;\n+\t}\n+\t/\n+\t The descriptor rings and the per-ring contiguous buffer blocks are\n+\t * allocated once and released only by devres at device detach: there\n+\t * is no free path, so a channel is set up exactly once per device\n+\t * lifetime and teardown is terminal for it. Re-setup would allocate\n+\t * a second set - about 2 MiB of coherent DMA plus the descriptor\n+\t * rings and wait[] - and orphan the first one, and it would revive a\n+\t * queue whose state teardown has already retired, so it is rejected.\n+\t /\n+\tif (chan_info-\u003edma_allocated) {\n+\t\tdev_warn(common-\u003edev,\n+\t\t\t \"channel DMA already allocated, re-setup not supported\\n\");\n+\t\treturn -EBUSY;\n+\t}\n+\n+\tnbl_chan_init_queue_param(chan_info, NBL_CHAN_QUEUE_LEN,\n+\t\t\t\t NBL_CHAN_QUEUE_LEN, NBL_CHAN_BUF_LEN,\n+\t\t\t\t NBL_CHAN_BUF_LEN);\n+\t/\n+\t * Set the flag before the first allocation and leave it set even\n+\t * when setup fails: whatever nbl_chan_init_queue() managed to\n+\t * allocate is devres-owned and stays live until detach, so the retry\n+\t * that clearing the flag would allow would allocate a second set on\n+\t * top of the orphans.\n+\t /\n+\tchan_info-\u003edma_allocated = true;\n+\terr = nbl_chan_init_queue(common, chan_info);\n+\tif (err)\n+\t\treturn err;\n+\n+\tnbl_chan_config_queue(chan_mgt, chan_info, true); / tx /\n+\tnbl_chan_config_queue(chan_mgt, chan_info, false); / rx /\n+\tnbl_chan_update_tail_ptr(hw_ops, chan_mgt-\u003ehw_ops_tbl-\u003epriv,\n+\t\t\t\t rxq-\u003etail_ptr, NBL_MB_RX_QID);\n+\tWRITE_ONCE(chan_info-\u003eactive, true);\n+\treturn 0;\n+}\n+\n+static bool nbl_chan_txq_full(struct nbl_chan_ring txq,\n+\t\t\t u16 num_entries)\n+{\n+\treturn NBL_NEXT_ID(txq-\u003enext_to_use, num_entries - 1) ==\n+\t txq-\u003enext_to_clean;\n+}\n+\n+static int nbl_chan_update_txqueue(struct nbl_channel_mgt chan_mgt,\n+\t\t\t\t struct nbl_chan_info chan_info,\n+\t\t\t\t struct nbl_chan_tx_param param)\n+{\n+\tstruct nbl_chan_ring txq = \u0026chan_info-\u003etxq;\n+\tstruct nbl_chan_tx_desc tx_desc;\n+\tstruct nbl_chan_buf tx_buf;\n+\n+\t/\n+\t param-\u003earg is non-NULL whenever param-\u003earg_len is non-zero:\n+\t * nbl_chan_send_msg() validates that before calling here, because a\n+\t * length with no buffer would publish buf_addr/buf_len describing\n+\t * memory that was never filled.\n+\t /\n+\tif (nbl_chan_txq_full(txq, chan_info-\u003enum_txq_entries))\n+\t\treturn -EBUSY;\n+\tif (param-\u003earg_len \u003e NBL_CHAN_BUF_LEN - sizeof(tx_desc))\n+\t\treturn -EINVAL;\n+\ttx_desc =\n+\t\tNBL_CHAN_TX_RING_TO_DESC(txq, txq-\u003enext_to_use);\n+\ttx_buf =\n+\t\tNBL_CHAN_TX_RING_TO_BUF(txq, txq-\u003enext_to_use);\n+\ttx_desc-\u003edstid = cpu_to_le16(param-\u003edstid);\n+\ttx_desc-\u003emsg_type = cpu_to_le16(param-\u003emsg_type);\n+\ttx_desc-\u003emsgid = cpu_to_le16(param-\u003emsgid);\n+\n+\t/\n+\t srcid field is filled by mailbox hardware after peer receives this\n+\t * packet, driver producer never writes srcid; reused descriptor slots\n+\t * will contain stale srcid value temporarily until hardware overwrites\n+\t * it.\n+\t /\n+\tif (param-\u003earg_len \u003e NBL_CHAN_TX_DESC_EMBEDDED_DATA_LEN) {\n+\t\tmemcpy(tx_buf-\u003eva, param-\u003earg, param-\u003earg_len);\n+\t\ttx_desc-\u003ebuf_addr = cpu_to_le64(tx_buf-\u003epa);\n+\t\ttx_desc-\u003ebuf_len = cpu_to_le16(param-\u003earg_len);\n+\t\ttx_desc-\u003edata_len = 0;\n+\t\tmemset(tx_desc-\u003edata, 0, sizeof(tx_desc-\u003edata));\n+\t} else {\n+\t\tmemset(tx_desc-\u003edata, 0, sizeof(tx_desc-\u003edata));\n+\t\tmemset(\u0026tx_desc-\u003ebuf_addr, 0, sizeof(tx_desc-\u003ebuf_addr));\n+\t\tif (param-\u003earg_len)\n+\t\t\tmemcpy(tx_desc-\u003edata, param-\u003earg, param-\u003earg_len);\n+\t\ttx_desc-\u003ebuf_len = 0;\n+\t\ttx_desc-\u003edata_len = cpu_to_le16(param-\u003earg_len);\n+\t}\n+\t/ Ensure descriptor data visible to device before AVAIL flag /\n+\tdma_wmb();\n+\ttx_desc-\u003eflags = cpu_to_le16(BIT(NBL_CHAN_TX_DESC_AVAIL));\n+\n+\ttxq-\u003enext_to_use =\n+\t\tNBL_NEXT_ID(txq-\u003enext_to_use, chan_info-\u003enum_txq_entries - 1);\n+\ttxq-\u003etail_ptr++;\n+\n+\treturn 0;\n+}\n+\n+/\n+ * Quiesce the TX mailbox queue and reclaim all outstanding\n+ * descriptors. Called from the timeout path of nbl_chan_kick_tx_ring()\n+ * with txq_lock held.\n+ \n+ The device failed to fetch/complete the current descriptor within the\n+ * polling window. We assert QUEUE_RST to stop further DMA fetches,\n+ * reclaim every descriptor between next_to_clean and next_to_use,\n+ * reset the software tail_ptr counter to match the hardware reset state,\n+ * and re-enable the queue so subsequent sends can proceed.\n+ /\n+static void nbl_chan_quiesce_and_reclaim_tx(struct nbl_channel_mgt chan_mgt,\n+\t\t\t\t\t struct nbl_chan_info chan_info)\n+{\n+\tstruct nbl_hw_ops hw_ops = chan_mgt-\u003ehw_ops_tbl-\u003eops;\n+\tstruct nbl_hw_mgt hw_priv = chan_mgt-\u003ehw_ops_tbl-\u003epriv;\n+\tstruct nbl_chan_ring txq = \u0026chan_info-\u003etxq;\n+\tstruct nbl_chan_tx_desc tx_desc;\n+\n+\t/\n+\t * Assert QUEUE_RST to stop hardware fetching new descriptors.\n+\t * stop_mailbox_txq() flushes through the mailbox BAR itself, so\n+\t * the reset has reached the device when it returns.\n+\t /\n+\thw_ops-\u003estop_mailbox_txq(hw_priv);\n+\n+\t/\n+\t * Reclaim all outstanding descriptors between next_to_clean and\n+\t * next_to_use. Under txq_lock there is at most one in-flight\n+\t * descriptor, but iterate the full range for robustness.\n+\t /\n+\twhile (txq-\u003enext_to_clean != txq-\u003enext_to_use) {\n+\t\ttx_desc = NBL_CHAN_TX_RING_TO_DESC(txq,\n+\t\t\t\t\t\t txq-\u003enext_to_clean);\n+\t\tWRITE_ONCE(tx_desc-\u003eflags, 0);\n+\t\ttxq-\u003enext_to_clean =\n+\t\t\tNBL_NEXT_ID(txq-\u003enext_to_clean,\n+\t\t\t\t chan_info-\u003enum_txq_entries - 1);\n+\t}\n+\n+\t/\n+\t * Hardware tail_ptr counter is cleared by QUEUE_RST. Reset\n+\t * software counter to match so the next doorbell update does\n+\t * not produce a false 16-bit wrap delta.\n+\t /\n+\ttxq-\u003etail_ptr = 0;\n+\ttxq-\u003enext_to_use = 0;\n+\ttxq-\u003enext_to_clean = 0;\n+\n+\t/\n+\t * Authoritative re-arm gate, evaluated while txq_lock is held:\n+\t * teardown sets shutdn before draining senders and then owns\n+\t * hardware state. If teardown started while we were stopping/\n+\t * reclaiming, leave the queue in QUEUE_RST and never program\n+\t * QUEUE_EN, otherwise the queue could be armed again pointing\n+\t * at rings that are about to be freed.\n+\t /\n+\tif (READ_ONCE(chan_info-\u003eshutdn))\n+\t\treturn;\n+\n+\t/ Re-enable queue with current ring base and size /\n+\tnbl_chan_config_queue(chan_mgt, chan_info, true);\n+}\n+\n+static int nbl_chan_kick_tx_ring(struct nbl_channel_mgt chan_mgt,\n+\t\t\t\t struct nbl_chan_info chan_info)\n+{\n+\tstruct nbl_hw_ops hw_ops = chan_mgt-\u003ehw_ops_tbl-\u003eops;\n+\tstruct nbl_chan_ring txq = \u0026chan_info-\u003etxq;\n+\tstruct device dev = chan_mgt-\u003ecommon-\u003edev;\n+\tint max_retries = NBL_CHAN_TX_WAIT_TIMES;\n+\tstruct nbl_chan_tx_desc tx_desc;\n+\tint retry_count = 0;\n+\tu16 msg_type;\n+\n+\tnbl_chan_update_tail_ptr(hw_ops, chan_mgt-\u003ehw_ops_tbl-\u003epriv,\n+\t\t\t\t txq-\u003etail_ptr, NBL_MB_TX_QID);\n+\n+\ttx_desc = NBL_CHAN_TX_RING_TO_DESC(txq, txq-\u003enext_to_clean);\n+\t/\n+\t * Poll for HW to mark descriptor as USED.\n+\t * Mailbox is a low-speed control channel for management commands.\n+\t * We avoid enabling dedicated per-TX interrupt for single control\n+\t * message to reduce interrupt overhead, so use bounded polling\n+\t * with small delay instead.\n+\t /\n+\twhile (retry_count \u003c max_retries) {\n+\t\tif (READ_ONCE(chan_info-\u003eshutdn))\n+\t\t\treturn -ESHUTDOWN;\n+\n+\t\tif (le16_to_cpu(READ_ONCE(tx_desc-\u003eflags)) \u0026\n+\t\t BIT(NBL_CHAN_TX_DESC_USED)) {\n+\t\t\t /\n+\t\t\t * Order reads of other device-written descriptor\n+\t\t\t * fields after observing USED. Matches the RX side\n+\t\t\t * pattern in nbl_chan_clean_queue().\n+\t\t\t /\n+\t\t\tdma_rmb();\n+\t\t\tbreak;\n+\t\t}\n+\n+\t\tretry_count++;\n+\t\tif (retry_count == max_retries) {\n+\t\t\tmsg_type = le16_to_cpu(READ_ONCE(tx_desc-\u003emsg_type));\n+\t\t\tdev_err_ratelimited(dev, \"chan send msg type: %d timeout\\n\",\n+\t\t\t\t\t msg_type);\n+\t\t\t/\n+\t\t\t * Teardown may have started after the loop-top\n+\t\t\t * check on this final iteration. Do not run lock\n+\t\t\t * recovery then: it would stop/reclaim the queue\n+\t\t\t * and, without the in-quiesce gate, re-arm it.\n+\t\t\t * nbl_chan_teardown_queue() owns hardware state.\n+\t\t\t /\n+\t\t\tif (READ_ONCE(chan_info-\u003eshutdn))\n+\t\t\t\treturn -ESHUTDOWN;\n+\t\t\t/\n+\t\t\t * Device failed to complete this descriptor.\n+\t\t\t * Quiesce the queue, reclaim the timed-out\n+\t\t\t * descriptor, and re-enable so future sends can\n+\t\t\t * proceed instead of stalling the ring full.\n+\t\t\t /\n+\t\t\tnbl_chan_quiesce_and_reclaim_tx(chan_mgt,\n+\t\t\t\t\t\t\tchan_info);\n+\t\t\treturn -ETIMEDOUT;\n+\t\t}\n+\t\tusleep_range(NBL_CHAN_TX_WAIT_US, NBL_CHAN_TX_WAIT_US_MAX);\n+\t}\n+\n+\ttxq-\u003enext_to_clean = txq-\u003enext_to_use;\n+\n+\treturn 0;\n+}\n+\n+static void nbl_chan_recv_ack_msg(void priv, u16 srcid, u16 msgid, void data,\n+\t\t\t\t u32 data_len)\n+{\n+\tstruct nbl_channel_mgt chan_mgt = (struct nbl_channel_mgt )priv;\n+\tstruct nbl_chan_waitqueue_head wait_head = NULL;\n+\tstruct device dev = chan_mgt-\u003ecommon-\u003edev;\n+\tstruct nbl_chan_info chan_info =\n+\t\tchan_mgt-\u003echan_info[NBL_CHAN_TYPE_MAILBOX];\n+\tu16 w_dstid, w_msgtype, w_msgidx;\n+\tu32 payload = data;\n+\tu16 ack_msgtype = 0;\n+\tu16 ack_msgid = 0;\n+\tu32 ack_datalen;\n+\tvoid ack_data;\n+\tu32 copy_len;\n+\tint w_status;\n+\ts32 raw_err;\n+\n+\tif (READ_ONCE(chan_info-\u003eshutdn))\n+\t\treturn;\n+\tif (data_len \u003e NBL_CHAN_BUF_LEN \|\|\n+\t data_len \u003c NBL_CHAN_ACK_HEAD_LEN * sizeof(u32)) {\n+\t\tdev_err_ratelimited(dev, \"Invalid ACK data_len: %u\\n\",\n+\t\t\t\t data_len);\n+\t\treturn;\n+\t}\n+\tack_datalen = data_len - NBL_CHAN_ACK_HEAD_LEN * sizeof(u32);\n+\tack_msgtype = le16_to_cpu((__le16 )(payload + NBL_CHAN_MSG_TYPE_POS));\n+\tack_msgid = le16_to_cpu((__le16 )(payload + NBL_CHAN_MSG_ID_POS));\n+\tif (FIELD_GET(NBL_CHAN_MSGID_LOC_MASK, ack_msgid) \u003e=\n+\t chan_info-\u003enum_txq_entries) {\n+\t\tdev_err_ratelimited(dev, \"chan recv msg id: %u err\\n\",\n+\t\t\t\t ack_msgid);\n+\t\treturn;\n+\t}\n+\twait_head =\n+\t\t\u0026chan_info-\u003ewait[FIELD_GET(NBL_CHAN_MSGID_LOC_MASK, ack_msgid)];\n+\n+\tmutex_lock(\u0026chan_info-\u003epending_lock);\n+\n+\t/* Cache repeated READ_ONCE values /\n+\tw_dstid = READ_ONCE(wait_head-\u003edstid);\n+\tw_status = READ_ONCE(wait_head-\u003estatus);\n+\tw_msgtype = READ_ONCE(wait_head-\u003emsg_type);\n+\tw_msgidx = READ_ONCE(wait_head-\u003emsg_index);\n+\n+\tif (srcid != w_dstid) {\n+\t\tmutex_unlock(\u0026chan_info-\u003epending_lock);\n+\t\tdev_err_ratelimited(dev, \"ACK srcid=%u != dstid=%u, rejecting\\n\",\n+\t\t\t\t srcid, w_dstid);\n+\t\treturn;\n+\t}\n+\tif (w_status != NBL_MBX_STATUS_WAITING) {\n+\t\tmutex_unlock(\u0026chan_info-\u003epending_lock);\n+\t\tdev_err_ratelimited(dev,\n+\t\t\t\t \"Skip ack invalid status, wait msgtype:%u idx:%u status:%d ack msgtype:%u msgid:%u datalen:%u\\n\",\n+\t\t\t\t w_msgtype, w_msgidx, w_status,\n+\t\t\t\t ack_msgtype, ack_msgid, ack_datalen);\n+\t\treturn;\n+\t}\n+\n+\tif (w_msgtype != ack_msgtype) {\n+\t\tmutex_unlock(\u0026chan_info-\u003epending_lock);\n+\t\tdev_err_ratelimited(dev,\n+\t\t\t\t \"Skip ack msgtype mismatch, wait msgtype:%u idx:%u ack msgtype:%u msgid:%u\\n\",\n+\t\t\t\t w_msgtype, w_msgidx, ack_msgtype,\n+\t\t\t\t ack_msgid);\n+\t\treturn;\n+\t}\n+\tif (FIELD_GET(NBL_CHAN_MSGID_INDEX_MASK, ack_msgid) != w_msgidx) {\n+\t\tmutex_unlock(\u0026chan_info-\u003epending_lock);\n+\t\tdev_err_ratelimited(dev,\n+\t\t\t\t \"Stale ACK: expected index=%u, got msgid=%u\\n\",\n+\t\t\t\t w_msgidx, ack_msgid);\n+\t\treturn;\n+\t}\n+\n+\traw_err = (s32)le32_to_cpu((__le32 )\u0026payload[NBL_CHAN_ACK_RET_POS]);\n+\tif (raw_err \u003e 0 \|\| raw_err \u003c -MAX_ERRNO)\n+\t\traw_err = -EREMOTEIO;\n+\n+\tWRITE_ONCE(wait_head-\u003eack_err, raw_err);\n+\n+\tcopy_len = min_t(u32, READ_ONCE(wait_head-\u003eack_data_len), ack_datalen);\n+\tif (READ_ONCE(wait_head-\u003eack_err) \u003e= 0 \u0026\u0026 copy_len \u003e 0) {\n+\t\tack_data = READ_ONCE(wait_head-\u003eack_data);\n+\t\tif (!ack_data) {\n+\t\t\tdev_err_ratelimited(dev, \"ACK payload dropped: ack_data is NULL\\n\");\n+\t\t\tWRITE_ONCE(wait_head-\u003eack_data_len, 0);\n+\t\t\tgoto ack_done;\n+\t\t}\n+\t\tmemcpy((char )ack_data,\n+\t\t payload + NBL_CHAN_ACK_HEAD_LEN, copy_len);\n+\t\tWRITE_ONCE(wait_head-\u003eack_data_len, (u16)copy_len);\n+\t} else {\n+\t\tWRITE_ONCE(wait_head-\u003eack_data_len, 0);\n+\t}\n+ack_done:\n+\t/* Guarantee payload data finished before acked flag visible /\n+\tsmp_wmb();\n+\tWRITE_ONCE(wait_head-\u003eacked, 1);\n+\tWRITE_ONCE(wait_head-\u003estatus, NBL_MBX_STATUS_ACKD);\n+\tmutex_unlock(\u0026chan_info-\u003epending_lock);\n+\twake_up(\u0026wait_head-\u003ewait_queue);\n+}\n+\n+static void nbl_chan_recv_msg(struct nbl_channel_mgt chan_mgt, void data)\n+{\n+\tstruct device dev = chan_mgt-\u003ecommon-\u003edev;\n+\tstruct nbl_chan_msg_node_data msg_handler;\n+\tu16 msg_type, payload_len, srcid, msgid;\n+\tstruct nbl_chan_info chan_info =\n+\t\tchan_mgt-\u003echan_info[NBL_CHAN_TYPE_MAILBOX];\n+\tstruct nbl_chan_tx_desc tx_desc;\n+\tvoid payload;\n+\tsize_t avail_space;\n+\tu16 data_len_fw;\n+\n+\tif (READ_ONCE(chan_info-\u003eshutdn))\n+\t\treturn;\n+\n+\ttx_desc = data;\n+\tmsg_type = le16_to_cpu(READ_ONCE(tx_desc-\u003emsg_type));\n+\tdev_dbg(dev, \"recv msg_type: %d\\n\", msg_type);\n+\n+\tsrcid = le16_to_cpu(READ_ONCE(tx_desc-\u003esrcid));\n+\tmsgid = le16_to_cpu(READ_ONCE(tx_desc-\u003emsgid));\n+\n+\tif (msg_type \u003e= NBL_CHAN_MSG_MAILBOX_MAX)\n+\t\treturn;\n+\n+\tdata_len_fw = le16_to_cpu(READ_ONCE(tx_desc-\u003edata_len));\n+\tif (data_len_fw) {\n+\t\tpayload_len = data_len_fw;\n+\n+\t\tif (payload_len \u003e NBL_CHAN_TX_DESC_EMBEDDED_DATA_LEN) {\n+\t\t\tdev_err_ratelimited(dev,\n+\t\t\t\t\t \"data_len=%u exceeds embedded buffer size=%u\\n\",\n+\t\t\t\t\t payload_len,\n+\t\t\t\t\t NBL_CHAN_TX_DESC_EMBEDDED_DATA_LEN);\n+\t\t\treturn;\n+\t\t}\n+\t\t/* Small pkt: payload stored inside descriptor data[] array /\n+\t\tpayload = tx_desc-\u003edata;\n+\t} else {\n+\t\tpayload_len = le16_to_cpu(READ_ONCE(tx_desc-\u003ebuf_len));\n+\n+\t\tavail_space = NBL_CHAN_BUF_LEN - sizeof(tx_desc);\n+\t\tif (payload_len \u003e avail_space) {\n+\t\t\tdev_err_ratelimited(dev,\n+\t\t\t\t\t \"buf_len=%u exceeds external buffer size=%zu\\n\",\n+\t\t\t\t\t payload_len, avail_space);\n+\t\t\treturn;\n+\t\t}\n+\t\t/* Large pkt: payload follows immediately after tx_desc /\n+\t\tpayload = tx_desc + 1;\n+\t}\n+\n+\tmsg_handler = xa_load(\u0026chan_mgt-\u003ehandler_xa, msg_type);\n+\tif (!msg_handler \|\| !msg_handler-\u003efunc) {\n+\t\tdev_err_ratelimited(dev,\n+\t\t\t\t \"No handler for msg_type: %u (srcid=%u, msgid=%u)\\n\",\n+\t\t\t\t msg_type, srcid, msgid);\n+\t\treturn;\n+\t}\n+\n+\tmsg_handler-\u003efunc(msg_handler-\u003epriv, srcid, msgid, payload,\n+\t\t\t payload_len);\n+}\n+\n+static void nbl_chan_advance_rx_ring(struct nbl_channel_mgt chan_mgt,\n+\t\t\t\t struct nbl_chan_info chan_info,\n+\t\t\t\t struct nbl_chan_ring rxq)\n+{\n+\tstruct nbl_hw_ops hw_ops = chan_mgt-\u003ehw_ops_tbl-\u003eops;\n+\tstruct nbl_chan_rx_desc rx_desc;\n+\tstruct nbl_chan_buf rx_buf;\n+\tu16 next_to_use;\n+\n+\tnext_to_use = rxq-\u003enext_to_use;\n+\trx_desc = NBL_CHAN_RX_RING_TO_DESC(rxq, next_to_use);\n+\trx_buf = NBL_CHAN_RX_RING_TO_BUF(rxq, next_to_use);\n+\n+\t/\n+\t * Recycle the RX descriptor at next_to_use. The initial\n+\t * unused slot is intentionally recycled after the first\n+\t * RX descriptor is consumed, allowing the ring to become\n+\t * fully populated while next_to_clean tracks the consumer.\n+\t /\n+\trx_desc-\u003ebuf_addr = cpu_to_le64(rx_buf-\u003epa);\n+\trx_desc-\u003ebuf_len = cpu_to_le32(chan_info-\u003erxq_buf_size);\n+\n+\t/\n+\t * DMA Write Memory Barrier:\n+\t * Ensures all previous DMA-mapped writes (buffer address/length)\n+\t * are completed before the descriptor flags are updated.\n+\t * This prevents hardware from seeing a partially updated descriptor\n+\t * where flags are set but buffer info isn't ready yet.\n+\t /\n+\tdma_wmb();\n+\n+\trx_desc-\u003eflags = cpu_to_le16(BIT(NBL_CHAN_RX_DESC_AVAIL));\n+\n+\trxq-\u003enext_to_use++;\n+\tif (rxq-\u003enext_to_use == chan_info-\u003enum_rxq_entries)\n+\t\trxq-\u003enext_to_use = 0;\n+\trxq-\u003etail_ptr++;\n+\n+\tnbl_chan_update_tail_ptr(hw_ops, chan_mgt-\u003ehw_ops_tbl-\u003epriv,\n+\t\t\t\t rxq-\u003etail_ptr, NBL_MB_RX_QID);\n+}\n+\n+static void nbl_chan_clean_queue(struct nbl_channel_mgt chan_mgt,\n+\t\t\t\t struct nbl_chan_info chan_info)\n+{\n+\tstruct nbl_common_info common = chan_mgt-\u003ecommon;\n+\tstruct nbl_chan_ring rxq = \u0026chan_info-\u003erxq;\n+\tstruct device dev = chan_mgt-\u003ecommon-\u003edev;\n+\tu32 budget = NBL_CHAN_RX_CLEAN_BUDGET;\n+\tstruct nbl_chan_rx_desc rx_desc;\n+\tstruct nbl_chan_buf rx_buf;\n+\tstruct work_struct task;\n+\tbool more_work = false;\n+\tu16 next_to_clean;\n+\tu16 flags;\n+\n+\t/ Nothing to clean before setup: the RX ring is still NULL /\n+\tif (!READ_ONCE(chan_info-\u003eactive))\n+\t\treturn;\n+\n+\tnext_to_clean = rxq-\u003enext_to_clean;\n+\trx_desc = NBL_CHAN_RX_RING_TO_DESC(rxq, next_to_clean);\n+\trx_buf = NBL_CHAN_RX_RING_TO_BUF(rxq, next_to_clean);\n+\twhile (le16_to_cpu(READ_ONCE(rx_desc-\u003eflags)) \u0026\n+\t BIT(NBL_CHAN_RX_DESC_USED)) {\n+\t\tflags = le16_to_cpu(READ_ONCE(rx_desc-\u003eflags));\n+\n+\t\tif (READ_ONCE(chan_info-\u003eshutdn))\n+\t\t\tbreak;\n+\t\tif (!(flags \u0026 BIT(NBL_CHAN_RX_DESC_WRITE)))\n+\t\t\tdev_dbg(dev,\n+\t\t\t\t\"mailbox rx flag 0x%x missing NBL_CHAN_RX_DESC_WRITE\\n\",\n+\t\t\t\tflags);\n+\n+\t\t/ Make sure hardware written descriptor visible to CPU /\n+\t\tdma_rmb();\n+\t\tnbl_chan_recv_msg(chan_mgt, rx_buf-\u003eva);\n+\t\tnbl_chan_advance_rx_ring(chan_mgt, chan_info, rxq);\n+\t\tnext_to_clean++;\n+\t\tif (next_to_clean == chan_info-\u003enum_rxq_entries)\n+\t\t\tnext_to_clean = 0;\n+\t\trx_desc = NBL_CHAN_RX_RING_TO_DESC(rxq, next_to_clean);\n+\t\trx_buf = NBL_CHAN_RX_RING_TO_BUF(rxq, next_to_clean);\n+\t\tif (--budget == 0) {\n+\t\t\tmore_work = true;\n+\t\t\tbreak;\n+\t\t}\n+\t\tcond_resched();\n+\t}\n+\trxq-\u003enext_to_clean = next_to_clean;\n+\n+\tmutex_lock(\u0026chan_info-\u003estate_lock);\n+\t/ Prevent queue_work after teardown clears clean_task /\n+\tif (READ_ONCE(chan_info-\u003eshutdn)) {\n+\t\tmutex_unlock(\u0026chan_info-\u003estate_lock);\n+\t\treturn;\n+\t}\n+\tif (common-\u003ewq \u0026\u0026 more_work) {\n+\t\ttask = READ_ONCE(chan_info-\u003eclean_task);\n+\t\tif (task)\n+\t\t\tqueue_work(common-\u003ewq, task);\n+\t}\n+\tmutex_unlock(\u0026chan_info-\u003estate_lock);\n+}\n+\n+static void nbl_chan_clean_queue_subtask(struct nbl_channel_mgt chan_mgt,\n+\t\t\t\t\t u8 chan_type)\n+{\n+\tstruct nbl_chan_info chan_info = chan_mgt-\u003echan_info[chan_type];\n+\n+\tnbl_chan_clean_queue(chan_mgt, chan_info);\n+}\n+\n+static int nbl_chan_get_msg_id(struct nbl_chan_info chan_info,\n+\t\t\t u16 msgid)\n+{\n+\tint search_loc = READ_ONCE(chan_info-\u003ewait_head_index), i;\n+\tstruct nbl_chan_waitqueue_head wait = NULL;\n+\tint status;\n+\tint next;\n+\n+\tlockdep_assert_held(\u0026chan_info-\u003epending_lock);\n+\tfor (i = 0; i \u003c chan_info-\u003enum_txq_entries; i++) {\n+\t\twait = \u0026chan_info-\u003ewait[search_loc];\n+\t\tstatus = READ_ONCE(wait-\u003estatus);\n+\t\tif (status == NBL_MBX_STATUS_IDLE \|\|\n+\t\t status == NBL_MBX_STATUS_TIMEOUT) {\n+\t\t\tWRITE_ONCE(wait-\u003emsg_index,\n+\t\t\t\t NBL_NEXT_ID(wait-\u003emsg_index,\n+\t\t\t\t\t NBL_CHAN_MSG_INDEX_MAX));\n+\n+\t\t\tmsgid = FIELD_PREP(NBL_CHAN_MSGID_INDEX_MASK,\n+\t\t\t\t\t wait-\u003emsg_index) \|\n+\t\t\t\t FIELD_PREP(NBL_CHAN_MSGID_LOC_MASK,\n+\t\t\t\t\t search_loc);\n+\n+\t\t\t/ Advance starting search position for next caller /\n+\t\t\tnext = NBL_NEXT_ID(search_loc,\n+\t\t\t\t\t chan_info-\u003enum_txq_entries - 1);\n+\t\t\tWRITE_ONCE(chan_info-\u003ewait_head_index, next);\n+\t\t\treturn 0;\n+\t\t}\n+\n+\t\tsearch_loc = NBL_NEXT_ID(search_loc,\n+\t\t\t\t\t chan_info-\u003enum_txq_entries - 1);\n+\t}\n+\n+\t/\n+\t * All tx slots are occupied. May happen under high transmit load\n+\t * or delayed remote ACK responses. Caller should retry later.\n+\t /\n+\treturn -EAGAIN;\n+}\n+\n+static int nbl_chan_send_msg(struct nbl_channel_mgt chan_mgt,\n+\t\t\t struct nbl_chan_send_info chan_send)\n+{\n+\tstruct nbl_common_info common = chan_mgt-\u003ecommon;\n+\tstruct nbl_chan_waitqueue_head wait_head = NULL;\n+\tstruct nbl_chan_tx_param tx_param = { 0 };\n+\tint i = NBL_CHAN_TX_WAIT_ACK_TIMES;\n+\tstruct nbl_chan_info chan_info =\n+\t\tchan_mgt-\u003echan_info[NBL_CHAN_TYPE_MAILBOX];\n+\tstruct device dev = common-\u003edev;\n+\tstruct work_struct task;\n+\tu16 msgid = 0;\n+\tint ret;\n+\n+\tif (chan_send-\u003eresp_len \u003e NBL_CHAN_BUF_LEN) {\n+\t\tdev_err_ratelimited(dev, \"resp_len %zu exceeds max %d\\n\",\n+\t\t\t\t chan_send-\u003eresp_len, NBL_CHAN_BUF_LEN);\n+\t\treturn -EINVAL;\n+\t}\n+\n+\t/\n+\t A length without a payload would leave the descriptor pointing at\n+\t * whatever the reused slot still holds - nbl_chan_update_txqueue()\n+\t * skips the copy - and the peer would receive those stale bytes.\n+\t * Reject the combination instead of transmitting them.\n+\t /\n+\tif (chan_send-\u003earg_len \u0026\u0026 !chan_send-\u003earg) {\n+\t\tdev_err_ratelimited(dev, \"arg_len %zu without arg\\n\",\n+\t\t\t\t chan_send-\u003earg_len);\n+\t\treturn -EINVAL;\n+\t}\n+\n+\t/\n+\t * Entry points are only reached after a successful setup_queue()\n+\t * (probe path), but the channel is a dynamic object: fail cleanly\n+\t * instead of dereferencing rings/wait[] that were never allocated\n+\t * (num_txq_entries == 0). active is published last in setup and\n+\t * cleared last in teardown, so a false value here means the channel\n+\t * never came up - or has already been torn down, and shutdn tells\n+\t * the two apart: it is set before active is cleared, so a\n+\t * post-teardown sender gets the same -ESHUTDOWN as the check below\n+\t * rather than the -ENODEV reserved for a channel setup_queue()\n+\t * never completed.\n+\t /\n+\tif (!READ_ONCE(chan_info-\u003eactive)) {\n+\t\tif (READ_ONCE(chan_info-\u003eshutdn))\n+\t\t\treturn -ESHUTDOWN;\n+\t\treturn -ENODEV;\n+\t}\n+\n+\tmutex_lock(\u0026chan_info-\u003estate_lock);\n+\tif (READ_ONCE(chan_info-\u003eshutdn)) {\n+\t\tmutex_unlock(\u0026chan_info-\u003estate_lock);\n+\t\treturn -ESHUTDOWN;\n+\t}\n+\tatomic_inc(\u0026chan_info-\u003einflight_tx_cnt);\n+\tmutex_unlock(\u0026chan_info-\u003estate_lock);\n+\n+\ttx_param.msg_type = chan_send-\u003emsg_type;\n+\ttx_param.arg = chan_send-\u003earg;\n+\ttx_param.arg_len = chan_send-\u003earg_len;\n+\ttx_param.dstid = chan_send-\u003edstid;\n+\ttx_param.msgid = msgid;\n+\tif (chan_send-\u003eack) {\n+\t\tmutex_lock(\u0026chan_info-\u003epending_lock);\n+\n+\t\tret = nbl_chan_get_msg_id(chan_info, \u0026msgid);\n+\t\tif (ret) {\n+\t\t\tmutex_unlock(\u0026chan_info-\u003epending_lock);\n+\t\t\tdev_err_ratelimited(dev,\n+\t\t\t\t\t \"Channel tx wait head full, send msgtype:%u to dstid:%u failed\\n\",\n+\t\t\t\t\t chan_send-\u003emsg_type,\n+\t\t\t\t\t chan_send-\u003edstid);\n+\t\t\tgoto out_clean_inflight;\n+\t\t}\n+\t\twait_head =\n+\t\t\t\u0026chan_info-\u003ewait[FIELD_GET(NBL_CHAN_MSGID_LOC_MASK,\n+\t\t\t\t\t\t msgid)];\n+\t\tWRITE_ONCE(wait_head-\u003eacked, 0);\n+\t\tWRITE_ONCE(wait_head-\u003eack_data, chan_send-\u003eresp);\n+\t\tWRITE_ONCE(wait_head-\u003eack_data_len, chan_send-\u003eresp_len);\n+\t\tWRITE_ONCE(wait_head-\u003emsg_type, chan_send-\u003emsg_type);\n+\t\tWRITE_ONCE(wait_head-\u003emsg_index,\n+\t\t\t FIELD_GET(NBL_CHAN_MSGID_INDEX_MASK, msgid));\n+\t\tWRITE_ONCE(wait_head-\u003edstid, chan_send-\u003edstid);\n+\n+\t\tWRITE_ONCE(wait_head-\u003estatus, NBL_MBX_STATUS_WAITING);\n+\t\tmutex_unlock(\u0026chan_info-\u003epending_lock);\n+\n+\t\ttx_param.msgid = msgid;\n+\t}\n+\n+\tmutex_lock(\u0026chan_info-\u003etxq_lock);\n+\tret = nbl_chan_update_txqueue(chan_mgt, chan_info, \u0026tx_param);\n+\tif (ret) {\n+\t\tmutex_unlock(\u0026chan_info-\u003etxq_lock);\n+\t\tdev_err_ratelimited(dev,\n+\t\t\t\t \"Channel tx queue full, send msgtype:%u to dstid:%u failed\\n\",\n+\t\t\t\t chan_send-\u003emsg_type, chan_send-\u003edstid);\n+\t\tif (wait_head)\n+\t\t\tgoto out_clear_wait_slot;\n+\t\tgoto out_clean_inflight;\n+\t}\n+\n+\tret = nbl_chan_kick_tx_ring(chan_mgt, chan_info);\n+\tmutex_unlock(\u0026chan_info-\u003etxq_lock);\n+\tif (ret) {\n+\t\tif (wait_head)\n+\t\t\tgoto out_clear_wait_slot;\n+\t\tgoto out_clean_inflight;\n+\t}\n+\n+\tif (!chan_send-\u003eack) {\n+\t\tret = 0;\n+\t\tgoto out_clean_inflight;\n+\t}\n+\n+\tif (test_bit(NBL_CHAN_IRQ_RDY, chan_info-\u003estate)) {\n+\t\twhile (!READ_ONCE(wait_head-\u003eacked)) {\n+\t\t\t/\n+\t\t\t * avoids long task blocking when interrupt mode is\n+\t\t\t * disabled mid-wait. Cannot guarantee subsequent ACK\n+\t\t\t * delivery after interrupt mask off, only prevents\n+\t\t\t * infinite blocking. Spurious timeout is possible.\n+\t\t\t /\n+\t\t\tret = wait_event_timeout(wait_head-\u003ewait_queue,\n+\t\t\t\t\t\t READ_ONCE(wait_head-\u003eacked) \|\|\n+\t\t\t\t\t\t READ_ONCE(chan_info-\u003eshutdn) \|\|\n+\t\t\t\t\t\t !test_bit(NBL_CHAN_IRQ_RDY,\n+\t\t\t\t\t\t\t chan_info-\u003estate),\n+\t\t\t\t\t\t NBL_CHAN_ACK_WAIT_TIME);\n+\n+\t\t\tif (READ_ONCE(chan_info-\u003eshutdn)) {\n+\t\t\t\tret = -ESHUTDOWN;\n+\t\t\t\tgoto out_clear_wait_slot;\n+\t\t\t}\n+\t\t\tif (!test_bit(NBL_CHAN_IRQ_RDY, chan_info-\u003estate)) {\n+\t\t\t\tret = -EIO;\n+\t\t\t\tgoto out_clear_wait_slot;\n+\t\t\t}\n+\t\t\tif (ret == 0) {\n+\t\t\t\tmutex_lock(\u0026chan_info-\u003epending_lock);\n+\t\t\t\tif (READ_ONCE(wait_head-\u003estatus) ==\n+\t\t\t\t NBL_MBX_STATUS_WAITING) {\n+\t\t\t\t\tWRITE_ONCE(wait_head-\u003estatus,\n+\t\t\t\t\t\t NBL_MBX_STATUS_TIMEOUT);\n+\t\t\t\t\tWRITE_ONCE(wait_head-\u003eacked, 0);\n+\t\t\t\t\tWRITE_ONCE(wait_head-\u003eack_data, NULL);\n+\t\t\t\t\tWRITE_ONCE(wait_head-\u003eack_data_len, 0);\n+\t\t\t\t\t/\n+\t\t\t\t\t * Ensure all status/ack slot\n+\t\t\t\t\t * updates are visible before subsequent\n+\t\t\t\t\t * readers observe acked == 0\n+\t\t\t\t\t /\n+\t\t\t\t\tsmp_wmb();\n+\t\t\t\t\tmutex_unlock(\u0026chan_info-\u003epending_lock);\n+\t\t\t\t\tdev_err_ratelimited(dev,\n+\t\t\t\t\t\t\t \"Channel waiting ack failed, message type: %d, msg id: %u\\n\",\n+\t\t\t\t\t\t\t chan_send-\u003emsg_type,\n+\t\t\t\t\t\t\t msgid);\n+\t\t\t\t\tret = -ETIMEDOUT;\n+\t\t\t\t\t/\n+\t\t\t\t\t * TIMEOUT slots can be reused by\n+\t\t\t\t\t * another sender. A late ACK is\n+\t\t\t\t\t * rejected by the receive path,\n+\t\t\t\t\t * which only accepts WAITING slots\n+\t\t\t\t\t * (and by the msg_index generation\n+\t\t\t\t\t * check after reallocation). The\n+\t\t\t\t\t * current sender no longer owns the\n+\t\t\t\t\t * slot, so skip the IDLE reset.\n+\t\t\t\t\t /\n+\t\t\t\t\tgoto out_clean_inflight;\n+\t\t\t\t}\n+\t\t\t\t/\n+\t\t\t\t * ACK won the timeout race: between the\n+\t\t\t\t * final condition check and this lock the\n+\t\t\t\t * receiver published acked=1/ACKD and\n+\t\t\t\t * copied the payload into our response\n+\t\t\t\t * buffer. The slot is still exclusively\n+\t\t\t\t * ours (only IDLE/TIMEOUT slots are ever\n+\t\t\t\t * reallocated), so consume the completion:\n+\t\t\t\t * unlock and continue to the loop-bottom\n+\t\t\t\t * acked check, which enters the normal\n+\t\t\t\t * success readout and IDLE reset. Leaving\n+\t\t\t\t * here via the timeout path would strand\n+\t\t\t\t * the slot in ACKD forever, permanently\n+\t\t\t\t * shrinking the 256-entry pool, and would\n+\t\t\t\t * report a false -ETIMEDOUT for a request\n+\t\t\t\t * whose response already arrived.\n+\t\t\t\t /\n+\t\t\t\tmutex_unlock(\u0026chan_info-\u003epending_lock);\n+\t\t\t}\n+\n+\t\t\tif (READ_ONCE(wait_head-\u003eacked))\n+\t\t\t\tbreak;\n+\t\t}\n+\t\tif (READ_ONCE(wait_head-\u003eacked)) {\n+\t\t\t/\n+\t\t\t * Load ordering: observe acked flag before\n+\t\t\t * reading ACK payload metadata.\n+\t\t\t /\n+\t\t\tsmp_rmb();\n+\t\t\tchan_send-\u003eack_len = READ_ONCE(wait_head-\u003eack_data_len);\n+\t\t\tret = READ_ONCE(wait_head-\u003eack_err);\n+\t\t}\n+\t} else {\n+\t\t/ Polling path for synchronous ACK /\n+\t\twhile (i--) {\n+\t\t\tif (READ_ONCE(chan_info-\u003eshutdn)) {\n+\t\t\t\tret = -ESHUTDOWN;\n+\t\t\t\tgoto out_clear_wait_slot;\n+\t\t\t}\n+\n+\t\t\tmutex_lock(\u0026chan_info-\u003estate_lock);\n+\t\t\ttask = READ_ONCE(chan_info-\u003eclean_task);\n+\t\t\tif (common-\u003ewq \u0026\u0026 task \u0026\u0026\n+\t\t\t !READ_ONCE(chan_info-\u003eshutdn) \u0026\u0026\n+\t\t\t !work_pending(task))\n+\t\t\t\tqueue_work(common-\u003ewq, task);\n+\t\t\tmutex_unlock(\u0026chan_info-\u003estate_lock);\n+\t\t\tif (READ_ONCE(wait_head-\u003eacked)) {\n+\t\t\t\t/\n+\t\t\t\t * Guarantee load order: observe acked\n+\t\t\t\t * flag before reading ack payload metadata.\n+\t\t\t\t /\n+\t\t\t\tsmp_rmb();\n+\t\t\t\tchan_send-\u003eack_len =\n+\t\t\t\t\tREAD_ONCE(wait_head-\u003eack_data_len);\n+\t\t\t\tret = READ_ONCE(wait_head-\u003eack_err);\n+\t\t\t\tgoto out_clear_wait_slot;\n+\t\t\t}\n+\n+\t\t\tusleep_range(NBL_CHAN_TX_WAIT_ACK_US_MIN,\n+\t\t\t\t NBL_CHAN_TX_WAIT_ACK_US_MAX);\n+\t\t\tcond_resched();\n+\t\t}\n+\t\tmutex_lock(\u0026chan_info-\u003epending_lock);\n+\t\tif (READ_ONCE(wait_head-\u003estatus) == NBL_MBX_STATUS_ACKD) {\n+\t\t\tchan_send-\u003eack_len = READ_ONCE(wait_head-\u003eack_data_len);\n+\t\t\tret = READ_ONCE(wait_head-\u003eack_err);\n+\t\t\tmutex_unlock(\u0026chan_info-\u003epending_lock);\n+\t\t\tgoto out_clear_wait_slot;\n+\t\t}\n+\t\t/\n+\t\t * Polling timed out without receiving an ACK. Transition\n+\t\t * the slot to TIMEOUT so nbl_chan_get_msg_id() can reuse\n+\t\t * it. Must not proceed to out_clean_inflight with\n+\t\t * the slot still in WAITING state — that would leak the\n+\t\t * slot permanently and eventually exhaust the 256-entry\n+\t\t * channel queue.\n+\t\t /\n+\t\tif (READ_ONCE(wait_head-\u003estatus) == NBL_MBX_STATUS_WAITING) {\n+\t\t\tWRITE_ONCE(wait_head-\u003estatus, NBL_MBX_STATUS_TIMEOUT);\n+\t\t\tWRITE_ONCE(wait_head-\u003eacked, 0);\n+\t\t\tWRITE_ONCE(wait_head-\u003eack_data, NULL);\n+\t\t\tWRITE_ONCE(wait_head-\u003eack_data_len, 0);\n+\t\t}\n+\t\tmutex_unlock(\u0026chan_info-\u003epending_lock);\n+\t\tdev_err_ratelimited(dev,\n+\t\t\t\t \"Channel polling ack failed, message type: %d msg id: %u\\n\",\n+\t\t\t\t chan_send-\u003emsg_type, msgid);\n+\t\tret = -ETIMEDOUT;\n+\t\tgoto out_clean_inflight;\n+\t}\n+\n+out_clear_wait_slot:\n+\tmutex_lock(\u0026chan_info-\u003epending_lock);\n+\tnbl_chan_reset_wait_head(chan_info, wait_head);\n+\tmutex_unlock(\u0026chan_info-\u003epending_lock);\n+\n+out_clean_inflight:\n+\tmutex_lock(\u0026chan_info-\u003estate_lock);\n+\tif (atomic_dec_and_test(\u0026chan_info-\u003einflight_tx_cnt))\n+\t\twake_up(\u0026chan_info-\u003einflight_wait);\n+\tmutex_unlock(\u0026chan_info-\u003estate_lock);\n+\treturn ret;\n+}\n+\n+static int nbl_chan_send_ack(struct nbl_channel_mgt chan_mgt,\n+\t\t\t struct nbl_chan_ack_info chan_ack)\n+{\n+\tsize_t head_len = NBL_CHAN_ACK_HEAD_LEN sizeof(u32);\n+\tsize_t data_len = chan_ack-\u003edata_len;\n+\tstruct nbl_chan_send_info chan_send;\n+\t__le32 tmp;\n+\tsize_t len;\n+\tint ret;\n+\n+\tif (data_len \u003e\n+\t NBL_CHAN_BUF_LEN - sizeof(struct nbl_chan_tx_desc) - head_len)\n+\t\treturn -EINVAL;\n+\n+\tlen = head_len + data_len;\n+\ttmp = kzalloc(len, GFP_KERNEL);\n+\tif (!tmp)\n+\t\treturn -ENOMEM;\n+\n+\t(__le16 )\u0026tmp[NBL_CHAN_MSG_TYPE_POS] =\n+\t\tcpu_to_le16(chan_ack-\u003emsg_type);\n+\t(__le16 )\u0026tmp[NBL_CHAN_MSG_ID_POS] = cpu_to_le16(chan_ack-\u003emsgid);\n+\ttmp[NBL_CHAN_ACK_RET_POS] = cpu_to_le32(chan_ack-\u003eerr);\n+\tif (chan_ack-\u003edata \u0026\u0026 chan_ack-\u003edata_len)\n+\t\tmemcpy(\u0026tmp[NBL_CHAN_ACK_HEAD_LEN], chan_ack-\u003edata,\n+\t\t chan_ack-\u003edata_len);\n+\n+\tnbl_chan_fill_send_info(\u0026chan_send, chan_ack-\u003edstid, NBL_CHAN_MSG_ACK,\n+\t\t\t\ttmp, len, NULL, 0, 0);\n+\tret = nbl_chan_send_msg(chan_mgt, \u0026chan_send);\n+\tkfree(tmp);\n+\n+\treturn ret;\n+}\n+\n+static int nbl_chan_register_msg(struct nbl_channel_mgt chan_mgt, u16 msg_type,\n+\t\t\t\t nbl_chan_resp func, void callback)\n+{\n+\treturn nbl_chan_add_msg_handler(chan_mgt, msg_type, func, callback);\n+}\n+\n+static bool nbl_chan_check_queue_exist(struct nbl_channel_mgt chan_mgt,\n+\t\t\t\t u8 chan_type)\n+{\n+\tstruct nbl_chan_info chan_info = chan_mgt-\u003echan_info[chan_type];\n+\n+\treturn chan_info ? true : false;\n+}\n+\n+static void nbl_chan_register_chan_task(struct nbl_channel_mgt chan_mgt,\n+\t\t\t\t\tu8 chan_type, struct work_struct task)\n+{\n+\tstruct nbl_chan_info chan_info = chan_mgt-\u003echan_info[chan_type];\n+\n+\tmutex_lock(\u0026chan_info-\u003estate_lock);\n+\tif (!READ_ONCE(chan_info-\u003eshutdn))\n+\t\tWRITE_ONCE(chan_info-\u003eclean_task, task);\n+\tmutex_unlock(\u0026chan_info-\u003estate_lock);\n+}\n+\n+static void nbl_chan_set_queue_state(struct nbl_channel_mgt chan_mgt,\n+\t\t\t\t enum nbl_chan_state state, u8 chan_type,\n+\t\t\t\t u8 set)\n+{\n+\tstruct nbl_chan_info chan_info = chan_mgt-\u003echan_info[chan_type];\n+\tint i;\n+\n+\tif (set)\n+\t\tset_bit(state, chan_info-\u003estate);\n+\telse\n+\t\tclear_bit(state, chan_info-\u003estate);\n+\t/\n+\t When clearing IRQ_RDY, wake all per-slot wait queues so\n+\t * sleeping senders observe the condition immediately and\n+\t * return -EIO instead of waiting out the 3s timeout and\n+\t * reporting -ETIMEDOUT.\n+\t \n+\t wait[] is only allocated by a successful nbl_chan_init_tx_queue(),\n+\t * while num_txq_entries is already set by nbl_chan_init_queue_param();\n+\t * skip the walk if that allocation never happened.\n+\t /\n+\tif (!set \u0026\u0026 state == NBL_CHAN_IRQ_RDY \u0026\u0026 chan_info-\u003ewait) {\n+\t\tfor (i = 0; i \u003c chan_info-\u003enum_txq_entries; i++)\n+\t\t\twake_up_all(\u0026chan_info-\u003ewait[i].wait_queue);\n+\t}\n+}\n+\n+static struct nbl_channel_ops chan_ops = {\n+\t.send_msg\t\t\t= nbl_chan_send_msg,\n+\t.send_ack\t\t\t= nbl_chan_send_ack,\n+\t.register_msg\t\t\t= nbl_chan_register_msg,\n+\t.cfg_chan_qinfo_map_table\t= nbl_chan_cfg_qinfo_map_table,\n+\t.check_queue_exist\t\t= nbl_chan_check_queue_exist,\n+\t.setup_queue\t\t\t= nbl_chan_setup_queue,\n+\t.teardown_queue\t\t\t= nbl_chan_teardown_queue,\n+\t.clean_queue_subtask\t\t= nbl_chan_clean_queue_subtask,\n+\t.register_chan_task\t\t= nbl_chan_register_chan_task,\n+\t.set_queue_state\t\t= nbl_chan_set_queue_state,\n+};\n+\n+static struct nbl_channel_mgt \n+nbl_chan_setup_chan_mgt(struct nbl_adapter adapter)\n+{\n+\tstruct nbl_hw_ops_tbl hw_ops_tbl = adapter-\u003eintf.hw_ops_tbl;\n+\tstruct nbl_common_info common = \u0026adapter-\u003ecommon;\n+\tstruct device dev = \u0026adapter-\u003epdev-\u003edev;\n+\tstruct nbl_channel_mgt chan_mgt;\n+\tstruct nbl_chan_info mailbox;\n+\tint ret;\n+\n+\tchan_mgt = devm_kzalloc(dev, sizeof(chan_mgt), GFP_KERNEL);\n+\tif (!chan_mgt)\n+\t\treturn ERR_PTR(-ENOMEM);\n+\n+\tchan_mgt-\u003ecommon = common;\n+\tchan_mgt-\u003ehw_ops_tbl = hw_ops_tbl;\n+\n+\tmailbox = devm_kzalloc(dev, sizeof(mailbox), GFP_KERNEL);\n+\tif (!mailbox)\n+\t\treturn ERR_PTR(-ENOMEM);\n+\tmailbox-\u003echan_type = NBL_CHAN_TYPE_MAILBOX;\n+\tchan_mgt-\u003echan_info[NBL_CHAN_TYPE_MAILBOX] = mailbox;\n+\n+\tret = nbl_chan_init_msg_handler(chan_mgt);\n+\tif (ret)\n+\t\treturn ERR_PTR(ret);\n+\tret = devm_mutex_init(common-\u003edev, \u0026mailbox-\u003etxq_lock);\n+\tif (ret)\n+\t\treturn ERR_PTR(ret);\n+\tret = devm_mutex_init(common-\u003edev, \u0026mailbox-\u003estate_lock);\n+\tif (ret)\n+\t\treturn ERR_PTR(ret);\n+\tret = devm_mutex_init(common-\u003edev, \u0026mailbox-\u003epending_lock);\n+\tif (ret)\n+\t\treturn ERR_PTR(ret);\n+\treturn chan_mgt;\n+}\n+\n+static struct nbl_channel_ops_tbl \n+nbl_chan_setup_ops(struct device dev, struct nbl_channel_mgt chan_mgt)\n+{\n+\tstruct nbl_channel_ops_tbl chan_ops_tbl;\n+\tint ret;\n+\n+\tchan_ops_tbl = devm_kzalloc(dev, sizeof(chan_ops_tbl), GFP_KERNEL);\n+\tif (!chan_ops_tbl)\n+\t\treturn ERR_PTR(-ENOMEM);\n+\tif (!chan_ops.send_msg \|\| !chan_ops.send_ack \|\|\n+\t !chan_ops.register_msg \|\| !chan_ops.cfg_chan_qinfo_map_table \|\|\n+\t !chan_ops.check_queue_exist \|\| !chan_ops.setup_queue \|\|\n+\t !chan_ops.teardown_queue \|\| !chan_ops.clean_queue_subtask \|\|\n+\t !chan_ops.register_chan_task \|\| !chan_ops.set_queue_state)\n+\t\treturn ERR_PTR(-EINVAL);\n+\n+\tchan_ops_tbl-\u003eops = \u0026chan_ops;\n+\tchan_ops_tbl-\u003epriv = chan_mgt;\n+\n+\tret = nbl_chan_register_msg(chan_mgt, NBL_CHAN_MSG_ACK,\n+\t\t\t\t nbl_chan_recv_ack_msg, chan_mgt);\n+\tif (ret)\n+\t\treturn ERR_PTR(ret);\n+\n+\treturn chan_ops_tbl;\n+}\n+\n+int nbl_chan_init_common(struct nbl_adapter adap)\n+{\n+\tstruct nbl_channel_ops_tbl chan_ops_tbl;\n+\tstruct device dev = \u0026adap-\u003epdev-\u003edev;\n+\tstruct nbl_channel_mgt chan_mgt;\n+\tint ret;\n+\n+\tchan_mgt = nbl_chan_setup_chan_mgt(adap);\n+\tif (IS_ERR(chan_mgt))\n+\t\treturn PTR_ERR(chan_mgt);\n+\n+\tchan_ops_tbl = nbl_chan_setup_ops(dev, chan_mgt);\n+\tif (IS_ERR(chan_ops_tbl)) {\n+\t\tret = PTR_ERR(chan_ops_tbl);\n+\t\tgoto cleanup_mgt;\n+\t}\n+\n+\tadap-\u003eintf.channel_ops_tbl = chan_ops_tbl;\n+\tadap-\u003ecore.chan_mgt = chan_mgt;\n+\tret = nbl_common_create_wq(\u0026adap-\u003ecommon);\n+\tif (ret)\n+\t\tgoto cleanup_mgt;\n+\treturn 0;\n+\n+cleanup_mgt:\n+\tnbl_chan_remove_msg_handler(chan_mgt);\n+\treturn ret;\n+}\n+\n+void nbl_chan_remove_common(struct nbl_adapter adap)\n+{\n+\tstruct nbl_channel_mgt chan_mgt = adap-\u003ecore.chan_mgt;\n+\n+\tif (!chan_mgt)\n+\t\treturn;\n+\tnbl_common_destroy_wq(\u0026adap-\u003ecommon);\n+\t/\n+\t * All channel queues shall be torn down earlier in remove path\n+\t * to drain inflight tx workers and stop hardware before destroying\n+\t * message handler xarray.\n+\t /\n+\tnbl_chan_remove_msg_handler(chan_mgt);\n+\tadap-\u003ecore.chan_mgt = NULL;\n+}\ndiff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_channel/nbl_channel.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_channel/nbl_channel.h\nnew file mode 100644\nindex 00000000000000..168f5de221bb78\n--- /dev/null\n+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_channel/nbl_channel.h\n@@ -0,0 +1,196 @@\n+/ SPDX-License-Identifier: GPL-2.0 /\n+/\n+ * Copyright (c) 2026 Nebula Matrix Limited.\n+ /\n+\n+#ifndef _NBL_CHANNEL_H_\n+#define _NBL_CHANNEL_H_\n+\n+#include \u003clinux/types.h\u003e\n+#include \u003clinux/xarray.h\u003e\n+\n+#include \"../nbl_include/nbl_include.h\"\n+#include \"../nbl_include/nbl_def_channel.h\"\n+#include \"../nbl_include/nbl_def_hw.h\"\n+#include \"../nbl_include/nbl_def_common.h\"\n+#include \"../nbl_core.h\"\n+\n+#define NBL_CHAN_TX_RING_TO_DESC(tx_ring, i) \\\n+\t(\u0026((((tx_ring)-\u003edesc.tx_desc))[i]))\n+#define NBL_CHAN_RX_RING_TO_DESC(rx_ring, i) \\\n+\t(\u0026((((rx_ring)-\u003edesc.rx_desc))[i]))\n+#define NBL_CHAN_TX_RING_TO_BUF(tx_ring, i) (\u0026(((tx_ring)-\u003ebuf)[i]))\n+#define NBL_CHAN_RX_RING_TO_BUF(rx_ring, i) (\u0026(((rx_ring)-\u003ebuf)[i]))\n+\n+#define NBL_CHAN_TX_WAIT_US\t\t\t100\n+#define NBL_CHAN_TX_WAIT_US_MAX\t\t\t120\n+#define NBL_CHAN_TX_WAIT_TIMES\t\t\t100\n+#define NBL_CHAN_TX_WAIT_ACK_US_MIN\t\t1000\n+#define NBL_CHAN_TX_WAIT_ACK_US_MAX\t\t1200\n+#define NBL_CHAN_TX_WAIT_ACK_TIMES\t\t5000\n+#define NBL_CHAN_QUEUE_LEN\t\t\t256\n+#define NBL_CHAN_BUF_LEN\t\t\t4096\n+#define NBL_CHAN_TX_DESC_EMBEDDED_DATA_LEN\t16\n+\n+#define NBL_CHAN_TX_DESC_AVAIL\t\t\t0\n+#define NBL_CHAN_TX_DESC_USED\t\t\t1\n+#define NBL_CHAN_RX_DESC_WRITE\t\t\t1\n+#define NBL_CHAN_RX_DESC_AVAIL\t\t\t3\n+#define NBL_CHAN_RX_DESC_USED\t\t\t4\n+\n+#define NBL_CHAN_ACK_HEAD_LEN\t\t\t3\n+#define NBL_CHAN_ACK_RET_POS\t\t\t2\n+#define NBL_CHAN_MSG_ID_POS\t\t\t1\n+#define NBL_CHAN_MSG_TYPE_POS\t\t\t0\n+\n+#define NBL_CHAN_ACK_WAIT_TIME\t\t\t(3 HZ)\n+#define NBL_CHAN_RX_CLEAN_BUDGET\t\t64\n+\n+enum {\n+\tNBL_MB_RX_QID = 0,\n+\tNBL_MB_TX_QID = 1,\n+};\n+\n+enum {\n+\tNBL_MBX_STATUS_IDLE = 0,\n+\tNBL_MBX_STATUS_WAITING,\n+\tNBL_MBX_STATUS_ACKD,\n+\tNBL_MBX_STATUS_TIMEOUT,\n+};\n+\n+struct nbl_chan_tx_param {\n+\tenum nbl_chan_msg_type msg_type;\n+\tvoid arg;\n+\tsize_t arg_len;\n+\tu16 dstid;\n+\tu16 msgid;\n+};\n+\n+struct nbl_chan_buf {\n+\tvoid va;\n+\tdma_addr_t pa;\n+\tsize_t size;\n+};\n+\n+struct nbl_chan_tx_desc {\n+\t__le16 flags;\n+\t__le16 srcid;\n+\t__le16 dstid;\n+\t__le16 data_len;\n+\t__le16 buf_len;\n+\t__le64 buf_addr;\n+\t__le16 msg_type;\n+\tu8 data[NBL_CHAN_TX_DESC_EMBEDDED_DATA_LEN];\n+\t__le16 msgid;\n+\tu8 rsv[26];\n+} __packed;\n+\n+struct nbl_chan_rx_desc {\n+\t__le16 flags;\n+\t__le32 buf_len;\n+\t__le16 buf_id;\n+\t__le64 buf_addr;\n+} __packed;\n+\n+union nbl_chan_desc_ptr {\n+\tstruct nbl_chan_tx_desc tx_desc;\n+\tstruct nbl_chan_rx_desc rx_desc;\n+};\n+\n+struct nbl_chan_ring {\n+\tunion nbl_chan_desc_ptr desc;\n+\t/* Per-entry slots carved out of one contiguous coherent block /\n+\tstruct nbl_chan_buf buf;\n+\tu16 next_to_use;\n+\tu16 tail_ptr; /* hardware does modulo ring size internally /\n+\tu16 next_to_clean;\n+\tdma_addr_t dma;\n+};\n+\n+#define NBL_CHAN_MSG_INDEX_MAX 63\n+\n+#define NBL_CHAN_MSGID_INDEX_MASK GENMASK(5, 0)\n+#define NBL_CHAN_MSGID_LOC_MASK GENMASK(13, 6)\n+\n+static inline void nbl_chan_update_tail_ptr(struct nbl_hw_ops hw_ops,\n+\t\t\t\t\t void hw_priv, u32 tail_ptr, u8 qid)\n+{\n+\thw_ops-\u003eupdate_mailbox_queue_tail_ptr(hw_priv, tail_ptr, qid);\n+}\n+\n+struct nbl_chan_waitqueue_head {\n+\tstruct wait_queue_head wait_queue;\n+\tchar ack_data;\n+\tint acked;\n+\ts32 ack_err;\n+\tu16 ack_data_len;\n+\tu16 msg_type;\n+\tint status;\n+\tu8 msg_index;\n+\tu16 dstid;\n+};\n+\n+struct nbl_chan_info {\n+\twait_queue_head_t inflight_wait;\n+\tstruct nbl_chan_ring txq;\n+\tstruct nbl_chan_ring rxq;\n+\tstruct nbl_chan_waitqueue_head wait;\n+\t/\n+\t * Serializes TX ring producer state (next_to_use/next_to_clean/\n+\t * tail_ptr), descriptor publication and the doorbell, and the TX\n+\t * queue config/stop sequences against\n+\t * nbl_chan_quiesce_and_reclaim_tx().\n+\t /\n+\tstruct mutex txq_lock;\n+\t/\n+\t * Serializes the channel lifecycle: shutdn publication, clean_task\n+\t * registration/clearing, inflight_tx_cnt transitions and the\n+\t * inflight_wait wakeup. The state bitmap is not covered by this\n+\t * lock - it is only touched with atomic bit ops (set_bit(),\n+\t * clear_bit(), test_bit()).\n+\t /\n+\tstruct mutex state_lock;\n+\t/\n+\t * Serializes the ACK wait slot array (wait[]): slot allocation in\n+\t * nbl_chan_get_msg_id(), slot field publication in\n+\t * nbl_chan_send_msg(), ACK completion in nbl_chan_recv_ack_msg()\n+\t * and slot reset in nbl_chan_reset_wait_head().\n+\t /\n+\tstruct mutex pending_lock;\n+\tstruct work_struct clean_task;\n+\tu16 wait_head_index;\n+\tu16 num_txq_entries;\n+\tu16 num_rxq_entries;\n+\tu16 txq_buf_size;\n+\tu16 rxq_buf_size;\n+\tDECLARE_BITMAP(state, NBL_CHAN_STATE_NBITS);\n+\tu8 chan_type;\n+\tatomic_t inflight_tx_cnt;\n+\tbool shutdn;\n+\tbool active;\n+\t/\n+\t One-shot lifecycle flag: DMA allocations use devm (descriptor/buf\n+\t * arrays) and dmam (coherent descriptor ring, per-ring contiguous\n+\t * buffer block) and are only released at device detach. Teardown\n+\t * only stops the hardware and clears active; it does not free\n+\t * memory. setup must never run twice on the same chan_info or the\n+\t * first allocation set would be orphaned.\n+\t /\n+\tbool dma_allocated;\n+};\n+\n+struct nbl_chan_msg_node_data {\n+\tnbl_chan_resp func;\n+\tvoid priv;\n+};\n+\n+struct nbl_channel_mgt {\n+\tstruct nbl_common_info common;\n+\tstruct nbl_hw_ops_tbl hw_ops_tbl;\n+\tstruct nbl_chan_info chan_info[NBL_CHAN_TYPE_MAX];\n+\t/ Serializes handler_xa registration and teardown /\n+\tstruct mutex handler_lock;\n+\tstruct xarray handler_xa;\n+};\n+\n+#endif\ndiff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_common/nbl_common.c b/drivers/net/ethernet/nebula-matrix/nbl/nbl_common/nbl_common.c\nnew file mode 100644\nindex 00000000000000..302c3c48dc4d59\n--- /dev/null\n+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_common/nbl_common.c\n@@ -0,0 +1,52 @@\n+// SPDX-License-Identifier: GPL-2.0\n+/\n+ * Copyright (c) 2026 Nebula Matrix Limited.\n+ /\n+\n+#include \u003clinux/device.h\u003e\n+#include \"nbl_common.h\"\n+\n+void nbl_common_destroy_wq(struct nbl_common_info common)\n+{\n+\tif (!common \|\| !common-\u003ewq)\n+\t\treturn;\n+\n+\tdestroy_workqueue(common-\u003ewq);\n+\tcommon-\u003ewq = NULL;\n+}\n+\n+int nbl_common_create_wq(struct nbl_common_info common)\n+{\n+\tchar wq_name[32];\n+\n+\tsnprintf(wq_name, sizeof(wq_name), \"nbl_wq_%s\", pci_name(common-\u003epdev));\n+\tcommon-\u003ewq = alloc_workqueue(wq_name, WQ_UNBOUND, 0);\n+\tif (!common-\u003ewq) {\n+\t\tdev_err(common-\u003edev, \"Failed to alloc workqueue %s\\n\", wq_name);\n+\t\treturn -ENOMEM;\n+\t}\n+\n+\treturn 0;\n+}\n+\n+/\n+ nbl_common_func_id_to_rel_pf_id - convert absolute PF id to relative PF id\n+ * @common: common device info\n+ * @pf_id: absolute PF identifier\n+ * @rel_pf_id: output relative pf id\n+ \n+ Leonis uses fixed mgt_pf = 0. Support future non-zero management PF.\n+ \n+ Return: 0 on success, -EINVAL on invalid arguments.\n+ /\n+int nbl_common_func_id_to_rel_pf_id(struct nbl_common_info common, u32 pf_id,\n+\t\t\t\t u32 rel_pf_id)\n+{\n+\tif (!rel_pf_id)\n+\t\treturn -EINVAL;\n+\n+\tif (pf_id \u003c common-\u003emgt_pf)\n+\t\treturn -EINVAL;\n+\trel_pf_id = pf_id - common-\u003emgt_pf;\n+\treturn 0;\n+}\ndiff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_common/nbl_common.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_common/nbl_common.h\nnew file mode 100644\nindex 00000000000000..129073b08eb903\n--- /dev/null\n+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_common/nbl_common.h\n@@ -0,0 +1,14 @@\n+/* SPDX-License-Identifier: GPL-2.0 /\n+/\n+ * Copyright (c) 2026 Nebula Matrix Limited.\n+ /\n+\n+#ifndef _NBL_COMMON_H_\n+#define _NBL_COMMON_H_\n+\n+#include \u003clinux/types.h\u003e\n+\n+#include \"../nbl_include/nbl_include.h\"\n+#include \"../nbl_include/nbl_def_common.h\"\n+\n+#endif\ndiff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_core.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_core.h\nindex 1cd6587a8fcb66..655dfb43e3653d 100644\n--- a/drivers/net/ethernet/nebula-matrix/nbl/nbl_core.h\n+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_core.h\n@@ -14,13 +14,25 @@ enum {\n \tNBL_CAP_HAS_NET_BIT,\n };\n \n+struct nbl_interface {\n+\tstruct nbl_hw_ops_tbl hw_ops_tbl;\n+\tstruct nbl_resource_ops_tbl resource_ops_tbl;\n+\tstruct nbl_dispatch_ops_tbl dispatch_ops_tbl;\n+\tstruct nbl_channel_ops_tbl channel_ops_tbl;\n+};\n+\n struct nbl_core {\n \tstruct nbl_hw_mgt hw_mgt;\n+\tstruct nbl_resource_mgt res_mgt;\n+\tstruct nbl_dispatch_mgt disp_mgt;\n+\tstruct nbl_dev_mgt dev_mgt;\n+\tstruct nbl_channel_mgt chan_mgt;\n };\n \n struct nbl_adapter {\n \tstruct pci_dev pdev;\n \tstruct nbl_core core;\n+\tstruct nbl_interface intf;\n \tstruct nbl_common_info common;\n };\n \n@@ -28,4 +40,12 @@ struct nbl_adapter nbl_core_init(struct pci_dev pdev,\n \t\t\t\t struct nbl_init_param param);\n void nbl_core_remove(struct nbl_adapter adapter);\n \n+/\n+ * Single-shot start/stop pair, called once each from PCI probe/remove.\n+ * Not repeatable: MSI-X vectors stay allocated until device detach, so\n+ * a second start on a bound device is rejected by the MSI-X core.\n+ /\n+int nbl_core_start(struct nbl_adapter adapter);\n+void nbl_core_stop(struct nbl_adapter adapter);\n+\n #endif\ndiff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_core/nbl_dev.c b/drivers/net/ethernet/nebula-matrix/nbl/nbl_core/nbl_dev.c\nnew file mode 100644\nindex 00000000000000..c25def18a9e684\n--- /dev/null\n+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_core/nbl_dev.c\n@@ -0,0 +1,727 @@\n+// SPDX-License-Identifier: GPL-2.0\n+/\n+ * Copyright (c) 2026 Nebula Matrix Limited.\n+ /\n+#include \u003clinux/device.h\u003e\n+#include \u003clinux/pci.h\u003e\n+#include \"nbl_dev.h\"\n+\n+static void nbl_dev_clean_mailbox_schedule(struct nbl_dev_mgt dev_mgt);\n+\n+/* ---------- Interrupt config ---------- /\n+static irqreturn_t nbl_dev_clean_mailbox(int irq __always_unused, void data)\n+{\n+\tstruct nbl_dev_mgt dev_mgt = (struct nbl_dev_mgt )data;\n+\n+\tnbl_dev_clean_mailbox_schedule(dev_mgt);\n+\treturn IRQ_HANDLED;\n+}\n+\n+static void nbl_dev_init_msix_cnt(struct nbl_dev_mgt dev_mgt)\n+{\n+\tstruct nbl_dev_common dev_common = dev_mgt-\u003ecommon_dev;\n+\tstruct nbl_msix_info msix_info = \u0026dev_common-\u003emsix_info;\n+\n+\t/ Reserve one MSI-X vector for the mailbox. Nothing here\n+\t * allocates or requests it: vectors are allocated and the handler\n+\t * attached when the device is started, after probe, from this\n+\t * count.\n+\t /\n+\tmsix_info-\u003eserv_info[NBL_MSIX_MAILBOX_TYPE].num = 1;\n+}\n+\n+static int nbl_dev_request_mailbox_irq(struct nbl_dev_mgt dev_mgt)\n+{\n+\tstruct nbl_dev_common dev_common = dev_mgt-\u003ecommon_dev;\n+\tstruct nbl_msix_info msix_info = \u0026dev_common-\u003emsix_info;\n+\tstruct nbl_common_info common = dev_mgt-\u003ecommon;\n+\tu16 lvec;\n+\tint irq_num;\n+\tint err;\n+\n+\tif (!msix_info-\u003eserv_info[NBL_MSIX_MAILBOX_TYPE].num)\n+\t\treturn 0;\n+\n+\tlvec = msix_info-\u003eserv_info[NBL_MSIX_MAILBOX_TYPE].base_vector_id;\n+\tirq_num = pci_irq_vector(common-\u003epdev, lvec);\n+\tif (irq_num \u003c 0) {\n+\t\tdev_err(common-\u003edev, \"Failed to get mailbox IRQ vector: %d\\n\",\n+\t\t\tirq_num);\n+\t\treturn irq_num;\n+\t}\n+\n+\tsnprintf(dev_common-\u003emailbox_name, sizeof(dev_common-\u003emailbox_name),\n+\t\t \"nbl_mailbox@pci:%s\", pci_name(common-\u003epdev));\n+\terr = request_irq(irq_num, nbl_dev_clean_mailbox, 0,\n+\t\t\t dev_common-\u003emailbox_name, dev_mgt);\n+\tif (err)\n+\t\treturn err;\n+\n+\treturn 0;\n+}\n+\n+static void nbl_dev_free_mailbox_irq(struct nbl_dev_mgt dev_mgt)\n+{\n+\tstruct nbl_dev_common dev_common = dev_mgt-\u003ecommon_dev;\n+\tstruct nbl_msix_info msix_info = \u0026dev_common-\u003emsix_info;\n+\tstruct nbl_common_info common = dev_mgt-\u003ecommon;\n+\tu16 lvec;\n+\tint irq_num;\n+\n+\tif (!msix_info-\u003eserv_info[NBL_MSIX_MAILBOX_TYPE].num)\n+\t\treturn;\n+\n+\tlvec = msix_info-\u003eserv_info[NBL_MSIX_MAILBOX_TYPE].base_vector_id;\n+\tirq_num = pci_irq_vector(common-\u003epdev, lvec);\n+\tif (irq_num \u003e= 0)\n+\t\tfree_irq(irq_num, dev_mgt);\n+}\n+\n+static int nbl_dev_enable_mailbox_irq(struct nbl_dev_mgt dev_mgt)\n+{\n+\tstruct nbl_dispatch_ops disp_ops = dev_mgt-\u003edisp_ops_tbl-\u003eops;\n+\tstruct nbl_channel_ops chan_ops = dev_mgt-\u003echan_ops_tbl-\u003eops;\n+\tstruct nbl_dev_common dev_common = dev_mgt-\u003ecommon_dev;\n+\tstruct nbl_msix_info msix_info = \u0026dev_common-\u003emsix_info;\n+\tu16 lvec;\n+\tint ret;\n+\n+\tif (!msix_info-\u003eserv_info[NBL_MSIX_MAILBOX_TYPE].num)\n+\t\treturn 0;\n+\n+\tlvec = msix_info-\u003eserv_info[NBL_MSIX_MAILBOX_TYPE].base_vector_id;\n+\t/\n+\t Enable sequence: perform set_mailbox_irq RPC in polling mode first.\n+\t * Only set NBL_CHAN_IRQ_RDY after RPC succeeds, mirroring disable path.\n+\t * This avoids waiting for an interrupt which has not been armed yet.\n+\t /\n+\tret = disp_ops-\u003eset_mailbox_irq(dev_mgt-\u003edisp_ops_tbl-\u003epriv,\n+\t\t\t\t\tlvec, true);\n+\tif (ret)\n+\t\treturn ret;\n+\tchan_ops-\u003eset_queue_state(dev_mgt-\u003echan_ops_tbl-\u003epriv,\n+\t\t\t\t NBL_CHAN_IRQ_RDY,\n+\t\t\t\t NBL_CHAN_TYPE_MAILBOX, true);\n+\treturn 0;\n+}\n+\n+static int nbl_dev_disable_mailbox_irq(struct nbl_dev_mgt dev_mgt)\n+{\n+\tstruct nbl_dispatch_ops disp_ops = dev_mgt-\u003edisp_ops_tbl-\u003eops;\n+\tstruct nbl_channel_ops chan_ops = dev_mgt-\u003echan_ops_tbl-\u003eops;\n+\tstruct nbl_dev_common dev_common = dev_mgt-\u003ecommon_dev;\n+\tstruct nbl_msix_info msix_info = \u0026dev_common-\u003emsix_info;\n+\tu16 lvec;\n+\n+\tif (!msix_info-\u003eserv_info[NBL_MSIX_MAILBOX_TYPE].num)\n+\t\treturn 0;\n+\n+\tlvec = msix_info-\u003eserv_info[NBL_MSIX_MAILBOX_TYPE].base_vector_id;\n+\t/\n+\t Disable sequence invariant: clear the software READY flag first,\n+\t * then mask the hardware interrupt. Must not reverse the order.\n+\t \n+\t Clearing NBL_CHAN_IRQ_RDY is what makes an interrupt-mode sender\n+\t * fail fast: nbl_chan_set_queue_state() wakes every per-slot wait\n+\t * queue on the transition, and nbl_chan_send_msg() tests the flag\n+\t * ahead of its timeout branch, so the sender returns -EIO right\n+\t * away instead of waiting out NBL_CHAN_ACK_WAIT_TIME (3s) and\n+\t * reporting a bogus -ETIMEDOUT / \"Channel waiting ack failed\".\n+\t * Masking first would leave the sleeper blocked on an interrupt\n+\t * that is already masked, i.e. guaranteed to time out.\n+\t \n+\t This does NOT preserve in-flight ACKs, and must not be described\n+\t * as doing so: the woken sender gives up and resets its slot to\n+\t * IDLE, so an ACK arriving afterwards finds a non-WAITING slot and\n+\t * is dropped by the receive path (\"Skip ack invalid status\").\n+\t * Interrupt mode is being torn down here; the guarantee is a\n+\t * bounded, deterministic failure, not delivery. In this driver\n+\t * every ACK-requesting sender runs from probe/remove and the clean\n+\t * task only sends ACK-less replies, so nothing depends on the\n+\t * stronger claim today.\n+\t \n+\t The only caller is nbl_dev_stop(): free_irq() blocks until\n+\t * any in-flight hardirq handler completes and prevents new interrupts.\n+\t * cancel_work_sync() then waits for any already running mailbox cleanup\n+\t * work to finish, or cancels queued but unstarted work items before\n+\t * final channel destruction. No stuck descriptors linger.\n+\t \n+\t nbl_dev_start()'s unwind path does not reach here: err_free_irq runs\n+\t * before IRQ_RDY is set, so only nbl_dev_free_mailbox_irq() applies\n+\t * there and no set_mailbox_irq(false) is sent - see the comment on\n+\t * that label.\n+\t /\n+\tchan_ops-\u003eset_queue_state(dev_mgt-\u003echan_ops_tbl-\u003epriv,\n+\t\t\t\t NBL_CHAN_IRQ_RDY,\n+\t\t\t\t NBL_CHAN_TYPE_MAILBOX, false);\n+\n+\treturn disp_ops-\u003eset_mailbox_irq(dev_mgt-\u003edisp_ops_tbl-\u003epriv,\n+\t\t\t\t\t lvec, false);\n+}\n+\n+static int nbl_dev_cfg_msix_map(struct nbl_dev_mgt dev_mgt)\n+{\n+\tstruct nbl_dispatch_ops disp_ops = dev_mgt-\u003edisp_ops_tbl-\u003eops;\n+\tstruct nbl_dev_common dev_common = dev_mgt-\u003ecommon_dev;\n+\tstruct nbl_msix_info msix_info = \u0026dev_common-\u003emsix_info;\n+\tbool mask_en = msix_info-\u003eserv_info[NBL_MSIX_NET_TYPE].hw_self_mask_en;\n+\tu16 msix_net_num = msix_info-\u003eserv_info[NBL_MSIX_NET_TYPE].num;\n+\tu16 msix_not_net_num = 0;\n+\tint err, i;\n+\n+\tmsix_info-\u003eserv_info[NBL_MSIX_NET_TYPE].base_vector_id = 0;\n+\t/\n+\t * Calculate base_vector_id for each MSIX service type.\n+\t * This relies on NBL_MSIX_TYPE enum being ordered sequentially,\n+\t * starting from NBL_MSIX_NET_TYPE.\n+\t /\n+\tfor (i = NBL_MSIX_NET_TYPE + 1; i \u003c NBL_MSIX_TYPE_MAX; i++)\n+\t\tmsix_info-\u003eserv_info[i].base_vector_id =\n+\t\t\tmsix_info-\u003eserv_info[i - 1].base_vector_id +\n+\t\t\tmsix_info-\u003eserv_info[i - 1].num;\n+\n+\tfor (i = 0; i \u003c NBL_MSIX_TYPE_MAX; i++) {\n+\t\tif (i == NBL_MSIX_NET_TYPE)\n+\t\t\tcontinue;\n+\t\tmsix_not_net_num += msix_info-\u003eserv_info[i].num;\n+\t}\n+\n+\terr = disp_ops-\u003ecfg_msix_map(dev_mgt-\u003edisp_ops_tbl-\u003epriv,\n+\t\t\t\t msix_net_num, msix_not_net_num,\n+\t\t\t\t mask_en);\n+\n+\treturn err;\n+}\n+\n+static int nbl_dev_destroy_msix_map(struct nbl_dev_mgt dev_mgt)\n+{\n+\tstruct nbl_dispatch_ops disp_ops = dev_mgt-\u003edisp_ops_tbl-\u003eops;\n+\n+\treturn disp_ops-\u003edestroy_msix_map(dev_mgt-\u003edisp_ops_tbl-\u003epriv);\n+}\n+\n+static int nbl_dev_init_interrupt_scheme(struct nbl_dev_mgt dev_mgt)\n+{\n+\tstruct nbl_dev_common dev_common = dev_mgt-\u003ecommon_dev;\n+\tstruct nbl_msix_info msix_info = \u0026dev_common-\u003emsix_info;\n+\tstruct nbl_common_info common = dev_mgt-\u003ecommon;\n+\tstruct irq_affinity affd = { 0 };\n+\tint needed = 0;\n+\tint err;\n+\tint i;\n+\n+\tfor (i = 0; i \u003c NBL_MSIX_TYPE_MAX; i++)\n+\t\tneeded += msix_info-\u003eserv_info[i].num;\n+\n+\t/\n+\t * The mailbox vector is the trailing administrative vector;\n+\t * reserve it via post_vectors so it is excluded from managed\n+\t * affinity spreading. A managed vector can be shut down when\n+\t * its last assigned CPU goes offline, which would stall every\n+\t * mailbox RPC while NBL_CHAN_IRQ_RDY stays set.\n+\t /\n+\taffd.post_vectors = msix_info-\u003eserv_info[NBL_MSIX_MAILBOX_TYPE].num;\n+\n+\terr = pci_alloc_irq_vectors_affinity(common-\u003epdev, needed, needed,\n+\t\t\t\t\t PCI_IRQ_MSIX \| PCI_IRQ_AFFINITY,\n+\t\t\t\t\t \u0026affd);\n+\tif (err \u003c 0) {\n+\t\tdev_err(common-\u003edev,\n+\t\t\t\"pci_alloc_irq_vectors failed, err = %d\\n\", err);\n+\t\treturn err;\n+\t}\n+\tif (err != needed) {\n+\t\tdev_err(common-\u003edev, \"pci_alloc_irq_vectors got %d vecs, need %d\\n\",\n+\t\t\terr, needed);\n+\t\treturn -ENOSPC;\n+\t}\n+\treturn 0;\n+}\n+\n+/\n+ * Release the kernel-side MSI-X vectors here, before the hardware\n+ * pcompleter map is destroyed. pci_free_irq_vectors() ends in\n+ * pci_msix_shutdown(), which masks every descriptor by writing the\n+ * vector control word in the MSI-X table and then reads mask_base\n+ * back to flush that write; those MMIO accesses should reach the\n+ * function while FUNCTION_MSIX_MAP is still valid. This mirrors\n+ * nbl_dev_start(), where cfg_msix_map runs before\n+ * pci_alloc_irq_vectors_affinity(): the opposite order leaves the\n+ * core poking the MSI-X table after destroy_msix_map() has cleared\n+ * the valid bit, invalidated the info entries and freed the map\n+ * table, and how the device reacts to that is hardware-specific.\n+ \n+ This is not a double free of the devres action. probe uses\n+ * pcim_enable_device(), so pci_alloc_irq_vectors() also registers\n+ * pcim_msi_release, but that only calls pci_free_irq_vectors()\n+ * again, and pci_disable_msix() returns early once msix_enabled is\n+ * clear.\n+ \n+ The handler is expected to be gone by now: both callers run\n+ * nbl_dev_free_mailbox_irq() first.\n+ /\n+static void nbl_dev_clear_interrupt_scheme(struct nbl_dev_mgt dev_mgt)\n+{\n+\tstruct nbl_common_info common = dev_mgt-\u003ecommon;\n+\n+\tpci_free_irq_vectors(common-\u003epdev);\n+}\n+\n+/ ---------- Channel config ---------- /\n+static void nbl_dev_setup_chan_qinfo(struct nbl_dev_mgt dev_mgt, u8 chan_type)\n+{\n+\tstruct nbl_channel_ops chan_ops = dev_mgt-\u003echan_ops_tbl-\u003eops;\n+\tstruct nbl_channel_mgt priv = dev_mgt-\u003echan_ops_tbl-\u003epriv;\n+\tstruct nbl_common_info common = dev_mgt-\u003ecommon;\n+\n+\tif (!chan_ops-\u003echeck_queue_exist(priv, chan_type))\n+\t\treturn;\n+\n+\t/\n+\t * common-\u003ehw_bus is the control PF's real bus number, captured in\n+\t * nbl_res_ctrl_dev_sriov_info_init() during nbl_res_init_leonis().\n+\t * nbl_core_init() runs resource init before nbl_dev_init(), so the\n+\t * value is always initialized when this control-PF-only path runs;\n+\t * nbl_res_intr_cfg_msix_map() consumes it for cfg_msix_map() the\n+\t * same way.\n+\t /\n+\tchan_ops-\u003ecfg_chan_qinfo_map_table(priv, common-\u003ehw_bus, common-\u003edevid);\n+}\n+\n+static int nbl_dev_setup_chan_queue(struct nbl_dev_mgt dev_mgt, u8 chan_type)\n+{\n+\tstruct nbl_channel_ops chan_ops = dev_mgt-\u003echan_ops_tbl-\u003eops;\n+\tstruct nbl_channel_mgt priv = dev_mgt-\u003echan_ops_tbl-\u003epriv;\n+\tint ret = 0;\n+\n+\tif (chan_ops-\u003echeck_queue_exist(priv, chan_type))\n+\t\tret = chan_ops-\u003esetup_queue(priv, chan_type);\n+\n+\treturn ret;\n+}\n+\n+static int nbl_dev_remove_chan_queue(struct nbl_dev_mgt dev_mgt, u8 chan_type)\n+{\n+\tstruct nbl_channel_ops chan_ops = dev_mgt-\u003echan_ops_tbl-\u003eops;\n+\tstruct nbl_channel_mgt priv = dev_mgt-\u003echan_ops_tbl-\u003epriv;\n+\tint ret = 0;\n+\n+\tif (chan_ops-\u003echeck_queue_exist(priv, chan_type))\n+\t\tret = chan_ops-\u003eteardown_queue(priv, chan_type);\n+\n+\treturn ret;\n+}\n+\n+static void nbl_dev_register_chan_task(struct nbl_dev_mgt dev_mgt,\n+\t\t\t\t u8 chan_type, struct work_struct task)\n+{\n+\tstruct nbl_channel_ops chan_ops = dev_mgt-\u003echan_ops_tbl-\u003eops;\n+\n+\tif (chan_ops-\u003echeck_queue_exist(dev_mgt-\u003echan_ops_tbl-\u003epriv, chan_type))\n+\t\tchan_ops-\u003eregister_chan_task(dev_mgt-\u003echan_ops_tbl-\u003epriv,\n+\t\t\t\t\t chan_type, task);\n+}\n+\n+/* ---------- Tasks config ---------- /\n+static void nbl_dev_clean_mailbox_task(struct work_struct work)\n+{\n+\tstruct nbl_dev_common common_dev =\n+\t\tcontainer_of(work, struct nbl_dev_common, clean_mbx_task);\n+\tstruct nbl_dev_mgt dev_mgt = common_dev-\u003edev_mgt;\n+\tstruct nbl_channel_ops chan_ops = dev_mgt-\u003echan_ops_tbl-\u003eops;\n+\n+\tchan_ops-\u003eclean_queue_subtask(dev_mgt-\u003echan_ops_tbl-\u003epriv,\n+\t\t\t\t NBL_CHAN_TYPE_MAILBOX);\n+}\n+\n+static void nbl_dev_clean_mailbox_schedule(struct nbl_dev_mgt dev_mgt)\n+{\n+\tstruct nbl_dev_common common_dev = dev_mgt-\u003ecommon_dev;\n+\tstruct nbl_common_info common = dev_mgt-\u003ecommon;\n+\n+\tqueue_work(common-\u003ewq, \u0026common_dev-\u003eclean_mbx_task);\n+}\n+\n+/* ---------- Dev init process ---------- /\n+static int nbl_dev_setup_common_dev(struct nbl_adapter adapter)\n+{\n+\tstruct nbl_dev_mgt dev_mgt = adapter-\u003ecore.dev_mgt;\n+\tstruct nbl_dev_common common_dev;\n+\tint ret;\n+\n+\tcommon_dev = devm_kzalloc(\u0026adapter-\u003epdev-\u003edev, sizeof(common_dev),\n+\t\t\t\t GFP_KERNEL);\n+\tif (!common_dev)\n+\t\treturn -ENOMEM;\n+\tcommon_dev-\u003edev_mgt = dev_mgt;\n+\n+\t/\n+\t * INIT_WORK must precede nbl_dev_register_chan_task() below:\n+\t * registration publishes the work item to the channel, and the\n+\t * channel may queue it from its send and receive paths as soon\n+\t * as setup_queue() completes.\n+\t /\n+\tINIT_WORK(\u0026common_dev-\u003eclean_mbx_task, nbl_dev_clean_mailbox_task);\n+\n+\tret = nbl_dev_setup_chan_queue(dev_mgt, NBL_CHAN_TYPE_MAILBOX);\n+\tif (ret) {\n+\t\t/\n+\t\t * Nothing to unwind: chan_info-\u003eactive is only set on a\n+\t\t * successful setup_queue(), so a failed one left no\n+\t\t * sender, no registered work item and no mailbox\n+\t\t * traffic. Calling teardown_queue() here would only\n+\t\t * take its \"channel not active\" early exit and emit a\n+\t\t * misleading duplicate-teardown warning. The partially\n+\t\t * allocated DMA rings are intentionally kept until devres\n+\t\t * releases them at detach (chan_info-\u003edma_allocated\n+\t\t * blocks a re-setup).\n+\t\t /\n+\t\treturn ret;\n+\t}\n+\n+\tnbl_dev_register_chan_task(dev_mgt, NBL_CHAN_TYPE_MAILBOX,\n+\t\t\t\t \u0026common_dev-\u003eclean_mbx_task);\n+\t/\n+\t * VSI/ETH identity is deliberately not fetched here: it is only\n+\t * known to the control PF, so it has to come from a mailbox RPC,\n+\t * and the control PF's responder may still be probing while this\n+\t * function runs. The fetch is therefore left to the start path.\n+\t /\n+\tdev_mgt-\u003ecommon_dev = common_dev;\n+\tnbl_dev_init_msix_cnt(dev_mgt);\n+\n+\treturn 0;\n+}\n+\n+static void nbl_dev_remove_common_dev(struct nbl_adapter adapter)\n+{\n+\tstruct nbl_dev_mgt dev_mgt = adapter-\u003ecore.dev_mgt;\n+\tstruct nbl_dev_common common_dev = dev_mgt-\u003ecommon_dev;\n+\n+\tif (!common_dev)\n+\t\treturn;\n+\tcancel_work_sync(\u0026common_dev-\u003eclean_mbx_task);\n+\t/\n+\t nbl_chan_teardown_queue() never fails: it returns 0 both on the\n+\t * early exit for a channel that never became active and on the\n+\t * normal path, which drains every inflight sender before\n+\t * returning. There is therefore no partial-failure state that\n+\t * could leave mailbox DMA in flight and nothing to report here.\n+\t \n+\t Unregistering the clean task is the channel layer's job too:\n+\t * teardown_queue() clears clean_task under state_lock and then\n+\t * cancel_work_sync()es it, after shutdn has blocked new queues.\n+\t * No explicit unregistration is needed here precisely because\n+\t * teardown_queue() does that clearing itself. Passing NULL from\n+\t * here would not be a no-op either: shutdn is only set inside\n+\t * teardown_queue(), so nbl_chan_register_chan_task() would still\n+\t * accept the write - it would just be clearing the same pointer a\n+\t * second time.\n+\t /\n+\tnbl_dev_remove_chan_queue(dev_mgt, NBL_CHAN_TYPE_MAILBOX);\n+}\n+\n+static int nbl_dev_setup_ctrl_dev(struct nbl_adapter adapter)\n+{\n+\tstruct nbl_dev_mgt dev_mgt = adapter-\u003ecore.dev_mgt;\n+\tstruct nbl_dispatch_ops disp_ops = dev_mgt-\u003edisp_ops_tbl-\u003eops;\n+\n+\treturn disp_ops-\u003einit_module(dev_mgt-\u003edisp_ops_tbl-\u003epriv);\n+}\n+\n+/\n+ Tear down control device: deinit_module sets driver_status=false\n+ * to notify firmware to clean all per-PF hardware state (including\n+ * qinfo registers). The qinfo map programmed in nbl_dev_init() via\n+ * nbl_dev_setup_chan_qinfo() is not explicitly cleared; firmware\n+ * handles it on driver_status change.\n+ \n+ Teardown ordering guarantee: every non-management PF creates a\n+ * consumer-\u003econtrol PF device link in its probe path, so the driver\n+ * core always unbinds all siblings before allowing the control PF to\n+ * be detached (sysfs unbind, driver unregister and hot-unplug alike).\n+ * nbl_probe_chip_deps() defers a sibling's probe until the control PF\n+ * is fully bound (DL_DEV_DRIVER_BOUND), and holds the management PF's\n+ * device lock across both that check and device_link_add(), so a\n+ * supplier cannot slip to DL_DEV_NO_DRIVER or DL_DEV_PROBING in\n+ * between. A sibling therefore cannot\n+ * reach its mailbox setup before the control PF is fully probed, nor\n+ * race this deinit afterwards - the link forces it to be unbound\n+ * first. The link is a persistent managed link with\n+ * DL_FLAG_AUTOPROBE_CONSUMER, so binding the control PF again makes\n+ * the core re-probe the siblings; without it they would stay unbound\n+ * until an admin rebound each one by hand.\n+ \n+ Direct control-PF FLR (which never runs driver teardown) cannot be\n+ * guarded here.\n+ /\n+static void nbl_dev_remove_ctrl_dev(struct nbl_adapter adapter)\n+{\n+\tstruct nbl_dev_mgt dev_mgt = adapter-\u003ecore.dev_mgt;\n+\tstruct nbl_dispatch_ops disp_ops = dev_mgt-\u003edisp_ops_tbl-\u003eops;\n+\n+\tdisp_ops-\u003edeinit_module(dev_mgt-\u003edisp_ops_tbl-\u003epriv);\n+}\n+\n+static struct nbl_dev_mgt nbl_dev_setup_dev_mgt(struct nbl_common_info common)\n+{\n+\tstruct nbl_dev_mgt dev_mgt;\n+\n+\tdev_mgt = devm_kzalloc(common-\u003edev, sizeof(dev_mgt), GFP_KERNEL);\n+\tif (!dev_mgt)\n+\t\treturn ERR_PTR(-ENOMEM);\n+\n+\tdev_mgt-\u003ecommon = common;\n+\treturn dev_mgt;\n+}\n+\n+int nbl_dev_init(struct nbl_adapter adapter)\n+{\n+\tstruct nbl_common_info common = \u0026adapter-\u003ecommon;\n+\tstruct nbl_dispatch_ops_tbl disp_ops_tbl =\n+\t\tadapter-\u003eintf.dispatch_ops_tbl;\n+\tstruct nbl_channel_ops_tbl chan_ops_tbl =\n+\t\tadapter-\u003eintf.channel_ops_tbl;\n+\tstruct nbl_dev_mgt dev_mgt;\n+\tint ret;\n+\n+\tdev_mgt = nbl_dev_setup_dev_mgt(common);\n+\tif (IS_ERR(dev_mgt)) {\n+\t\tret = PTR_ERR(dev_mgt);\n+\t\treturn ret;\n+\t}\n+\n+\tdev_mgt-\u003edisp_ops_tbl = disp_ops_tbl;\n+\tdev_mgt-\u003echan_ops_tbl = chan_ops_tbl;\n+\tadapter-\u003ecore.dev_mgt = dev_mgt;\n+\tif (common-\u003ehas_ctrl)\n+\t\tnbl_dev_setup_chan_qinfo(dev_mgt, NBL_CHAN_TYPE_MAILBOX);\n+\t/\n+\t * Chip hardware initialization is completed by firmware at power-up.\n+\t * Only driver functional table/register config follows here, safe to\n+\t * access hardware registers before ctrl dev setup.\n+\t \n+\t The QINFO routing has to be programmed before the mailbox queue is\n+\t * enabled: nbl_dev_setup_common_dev() sets QUEUE_EN, and a queue that\n+\t * is live before its routing is in place would drop or misroute\n+\t * traffic.\n+\t /\n+\tret = nbl_dev_setup_common_dev(adapter);\n+\tif (ret)\n+\t\treturn ret;\n+\n+\tif (common-\u003ehas_ctrl) {\n+\t\tret = nbl_dev_setup_ctrl_dev(adapter);\n+\t\tif (ret)\n+\t\t\tgoto out_remove_common_dev;\n+\t}\n+\n+\treturn 0;\n+\n+/\n+ * A failure after the QINFO routing was programmed leaves those entries\n+ * in hardware: driver_status never goes 1 -\u003e 0 on these paths, so\n+ * firmware does not reclaim them. This is harmless - firmware only\n+ * routes once driver_status announces the driver, and no sibling PF can\n+ * bind until this probe succeeds. The next successful probe overwrites\n+ * the entries, and a chip reset clears them.\n+ /\n+out_remove_common_dev:\n+\tnbl_dev_remove_common_dev(adapter);\n+\treturn ret;\n+}\n+\n+/\n+ * Teardown order: Stop mailbox channel and drain all inflight DMA first,\n+ * then invoke deinit_module to notify firmware.\n+ \n+ This intentionally breaks strict init/teardown mirror symmetry due to\n+ * a hardware constraint: firmware performs asynchronous per-PF cleanup\n+ * (mailbox QINFO routing, qinfo registers and the like) once\n+ * driver_status=false is set. It does not revert the chip-wide\n+ * datapath registers configured by init_module - those stay valid until\n+ * chip reset (see nbl_hw_deinit_module()). We must guarantee no ongoing\n+ * mailbox DMA before deinit_module so firmware cannot reclaim the\n+ * routing under a live transfer and leave the device DMAing into\n+ * released memory.\n+ \n+ Init order: program mailbox QINFO routing (ctrl PF only) -\u003e\n+ * create mailbox queue/common_dev -\u003e ctrl dev init_module\n+ * Teardown order: destroy mailbox queue/common_dev -\u003e ctrl dev deinit_module\n+ /\n+void nbl_dev_remove(struct nbl_adapter adapter)\n+{\n+\tstruct nbl_common_info common = \u0026adapter-\u003ecommon;\n+\n+\tnbl_dev_remove_common_dev(adapter);\n+\tif (common-\u003ehas_ctrl)\n+\t\tnbl_dev_remove_ctrl_dev(adapter);\n+}\n+\n+/ ---------- Dev start process ---------- /\n+int nbl_dev_start(struct nbl_adapter adapter)\n+{\n+\tstruct nbl_dev_mgt dev_mgt = adapter-\u003ecore.dev_mgt;\n+\tstruct nbl_dispatch_ops disp_ops = dev_mgt-\u003edisp_ops_tbl-\u003eops;\n+\tstruct nbl_dispatch_mgt priv = dev_mgt-\u003edisp_ops_tbl-\u003epriv;\n+\tstruct nbl_dev_common common_dev = dev_mgt-\u003ecommon_dev;\n+\tstruct nbl_common_info common = dev_mgt-\u003ecommon;\n+\tint cleanup_ret;\n+\tint ret;\n+\n+\t/\n+\t * Startup RPC failures are reported as they are, including the\n+\t * -ETIMEDOUT of a 5-6s polling wait against an unresponsive peer.\n+\t * nbl_probe() already deferred this PF in nbl_probe_chip_deps()\n+\t * until the control PF reached DL_DEV_DRIVER_BOUND, and the core\n+\t * only marks a device bound once its whole probe - including the\n+\t * control PF's own nbl_core_start() - has returned. That state,\n+\t * not nbl_dev_init(), is what makes a timeout here a real failure\n+\t * instead of a \"supplier not ready yet\" condition: on the control\n+\t * PF inbound mailbox work only runs from clean_mbx_task, queued\n+\t * by the mailbox hardirq, by the task's own self-requeue, or by\n+\t * this function's polling sends - and the control PF uses the\n+\t * local dispatch ops, so it sends none. The responder is armed\n+\t * by set_mailbox_irq(true) at the very end of the peer's\n+\t * nbl_dev_start(); relaxing the dependency to \"func 0 finished\n+\t * nbl_dev_init()\" would let every sibling RPC sit out the full\n+\t * 5-6s polling wait. Turning -ETIMEDOUT into -EPROBE_DEFER would\n+\t * hide it behind dev_dbg() and leave the retry to the next\n+\t * unrelated device bind, paying that wait again on every retry.\n+\t /\n+\tret = nbl_dev_cfg_msix_map(dev_mgt);\n+\tif (ret)\n+\t\tgoto err_destroy_map;\n+\n+\t/ Fetch VSI/ETH identity after cfg_msix_map /\n+\tret = disp_ops-\u003eget_vsi_id(priv, NBL_VSI_DATA, \u0026common-\u003evsi_id);\n+\tif (ret)\n+\t\tgoto err_destroy_map;\n+\tret = disp_ops-\u003eget_eth_id(priv, common-\u003evsi_id, \u0026common-\u003eeth_num,\n+\t\t\t\t \u0026common-\u003eeth_id, \u0026common-\u003elogic_eth_id);\n+\tif (ret)\n+\t\tgoto err_destroy_map;\n+\n+\tret = nbl_dev_init_interrupt_scheme(dev_mgt);\n+\tif (ret)\n+\t\tgoto err_destroy_map;\n+\n+\tret = nbl_dev_request_mailbox_irq(dev_mgt);\n+\tif (ret)\n+\t\tgoto err_destroy_map;\n+\n+\tret = nbl_dev_enable_mailbox_irq(dev_mgt);\n+\tif (ret)\n+\t\tgoto err_free_irq;\n+\n+\treturn 0;\n+\n+err_free_irq:\n+\t/\n+\t * nbl_dev_enable_mailbox_irq() only fails before it sets\n+\t * NBL_CHAN_IRQ_RDY - the flag is set after its RPC returns - so\n+\t * there is no interrupt-mode sender to release and no software\n+\t * state to revoke. The peer-side mailbox routing, if the failed\n+\t * RPC did land, is switched off by the destroy_msix_map() below: its\n+\t * prepare phase, __nbl_res_intr_prepare_destroy_msix_map(), disables\n+\t * mailbox IRQ routing for any CONFIGURED function before the map is\n+\t * torn down. That is what makes skipping set_mailbox_irq(false)\n+\t * correct here, whatever way the enable failed - a local\n+\t * -ESHUTDOWN/-ENODEV/-EINVAL on the control PF as much as a\n+\t * polling-mode -ETIMEDOUT against an unresponsive peer.\n+\t /\n+\tnbl_dev_free_mailbox_irq(dev_mgt);\n+err_destroy_map:\n+\t/\n+\t * Always ask the peer to drop this function's MSI-X map, even when\n+\t * the cfg RPC above failed: the teardown prepare phase returns 0\n+\t * for a function that is not NBL_INTR_FUNC_CONFIGURED, so it is a\n+\t * no-op when nothing landed, but a -ETIMEDOUT only means the ACK\n+\t * never arrived - the responder runs to completion before it ACKs.\n+\t * In that case the map entry, the intr_net/other_bmap bits, the\n+\t * kcalloc()ed vector array and the coherent map table are all\n+\t * still owned by the control PF, i.e. host resources of its\n+\t * driver, released only when this function is configured again or\n+\t * when nbl_intr_mgt_stop() tears the control PF down. They are not\n+\t * firmware state, so skipping the destroy would strand them for a\n+\t * PF that has no driver to ever ask for them again.\n+\t \n+\t Kernel vectors are released first, by the\n+\t * nbl_dev_clear_interrupt_scheme() below: the PCI core's MSI-X\n+\t * shutdown writes the MSI-X table and reads it back, and those\n+\t * accesses should land while this function's pcompleter map entry\n+\t * is still valid. The destroy then masks all hardware vectors\n+\t * and clears that entry.\n+\t \n+\t For non-control PFs this is a polling-mode mailbox RPC:\n+\t * NBL_CHAN_IRQ_RDY is never set on any rollback path, since\n+\t * enabling it is the last step of nbl_dev_start().\n+\t \n+\t Note: pci_clear_master() runs in nbl_probe() core_start_err path;\n+\t * on a non-control PF it cannot stop DMA using the management PF's\n+\t * BDF, this is a known limitation.\n+\t /\n+\tnbl_dev_clear_interrupt_scheme(dev_mgt);\n+\tcleanup_ret = nbl_dev_destroy_msix_map(dev_mgt);\n+\tif (cleanup_ret)\n+\t\tdev_err(dev_mgt-\u003ecommon-\u003edev,\n+\t\t\t\"rollback: destroy MSI-X map failed: %d\\n\",\n+\t\t\tcleanup_ret);\n+\n+\tcancel_work_sync(\u0026common_dev-\u003eclean_mbx_task);\n+\treturn ret;\n+}\n+\n+void nbl_dev_stop(struct nbl_adapter adapter)\n+{\n+\tstruct nbl_dev_mgt dev_mgt = adapter-\u003ecore.dev_mgt;\n+\tstruct nbl_dev_common common_dev = dev_mgt-\u003ecommon_dev;\n+\tint ret;\n+\n+\tret = nbl_dev_disable_mailbox_irq(dev_mgt);\n+\tif (ret)\n+\t\tdev_err(dev_mgt-\u003ecommon-\u003edev,\n+\t\t\t\"Failed to disable mailbox IRQ: %d\\n\", ret);\n+\tnbl_dev_free_mailbox_irq(dev_mgt);\n+\n+\t/\n+\t Release the kernel vectors before destroying the hardware map:\n+\t * the PCI core's MSI-X shutdown writes the MSI-X table and reads\n+\t * it back, and those accesses should land while this function's\n+\t * pcompleter map entry is still valid (see\n+\t * nbl_dev_clear_interrupt_scheme).\n+\t /\n+\tnbl_dev_clear_interrupt_scheme(dev_mgt);\n+\n+\t/\n+\t * Destroy the hardware MSI-X map: masks all device vectors and\n+\t * clears the pcompleter map entry.\n+\t \n+\t Best-effort: if the destroy RPC fails, the control PF still owns\n+\t * this function's map entry, its intr_net/other_bmap bits and its\n+\t * coherent map table. Those are host resources of that driver,\n+\t * released either by a later successful configuration of this\n+\t * function - nbl_res_intr_cfg_msix_map() invalidates and rewrites\n+\t * the entry on its had_config path - or by nbl_intr_mgt_stop()\n+\t * when the control PF is removed, which walks every\n+\t * NBL_INTR_FUNC_CONFIGURED function through\n+\t * __nbl_res_intr_prepare_destroy_msix_map() and clears the\n+\t * FUNCTION_MSIX_MAP valid bit. Both rewrite the hardware entry\n+\t * without a chip reset; nothing else collects it, so a stranded\n+\t * one just stays as it is, and nothing requests one.\n+\t \n+\t pci_clear_master() on a non-control PF cannot stop DMA using the\n+\t * management PF's BDF.\n+\t /\n+\tret = nbl_dev_destroy_msix_map(dev_mgt);\n+\tif (ret)\n+\t\tdev_err(dev_mgt-\u003ecommon-\u003edev,\n+\t\t\t\"Failed to destroy MSI-X map: %d\\n\", ret);\n+\n+\t/\n+\t * destroy_msix_map() sends ack-requested messages which may\n+\t * requeue clean_mbx_task via polling send path. Drain work\n+\t * after the operation.\n+\t /\n+\tcancel_work_sync(\u0026common_dev-\u003eclean_mbx_task);\n+}\ndiff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_core/nbl_dev.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_core/nbl_dev.h\nnew file mode 100644\nindex 00000000000000..24e890fd898745\n--- /dev/null\n+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_core/nbl_dev.h\n@@ -0,0 +1,55 @@\n+/ SPDX-License-Identifier: GPL-2.0 /\n+/\n+ * Copyright (c) 2026 Nebula Matrix Limited.\n+ /\n+\n+#ifndef _NBL_DEV_H_\n+#define _NBL_DEV_H_\n+\n+#include \u003clinux/types.h\u003e\n+\n+#include \"../nbl_include/nbl_include.h\"\n+#include \"../nbl_include/nbl_def_channel.h\"\n+#include \"../nbl_include/nbl_def_hw.h\"\n+#include \"../nbl_include/nbl_def_resource.h\"\n+#include \"../nbl_include/nbl_def_dispatch.h\"\n+#include \"../nbl_include/nbl_def_dev.h\"\n+#include \"../nbl_include/nbl_def_common.h\"\n+#include \"../nbl_core.h\"\n+\n+#define NBL_STRING_NAME_LEN\t\t\t32\n+\n+enum nbl_msix_serv_type {\n+\tNBL_MSIX_NET_TYPE,\n+\tNBL_MSIX_MAILBOX_TYPE,\n+\tNBL_MSIX_TYPE_MAX\n+};\n+\n+struct nbl_msix_serv_info {\n+\tchar irq_name[NBL_STRING_NAME_LEN];\n+\tu16 num;\n+\tu16 base_vector_id;\n+\t/ true: hw report msix, hw need to mask actively /\n+\tbool hw_self_mask_en;\n+};\n+\n+struct nbl_msix_info {\n+\tstruct nbl_msix_serv_info serv_info[NBL_MSIX_TYPE_MAX];\n+};\n+\n+struct nbl_dev_common {\n+\tstruct nbl_dev_mgt dev_mgt;\n+\tstruct nbl_msix_info msix_info;\n+\tchar mailbox_name[NBL_STRING_NAME_LEN];\n+\t/* for ctrl-dev/net-dev mailbox recv msg /\n+\tstruct work_struct clean_mbx_task;\n+};\n+\n+struct nbl_dev_mgt {\n+\tstruct nbl_common_info common;\n+\tstruct nbl_dispatch_ops_tbl disp_ops_tbl;\n+\tstruct nbl_channel_ops_tbl chan_ops_tbl;\n+\tstruct nbl_dev_common common_dev;\n+};\n+\n+#endif\ndiff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_core/nbl_dispatch.c b/drivers/net/ethernet/nebula-matrix/nbl/nbl_core/nbl_dispatch.c\nnew file mode 100644\nindex 00000000000000..59d80066f88b76\n--- /dev/null\n+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_core/nbl_dispatch.c\n@@ -0,0 +1,671 @@\n+// SPDX-License-Identifier: GPL-2.0\n+/\n+ * Copyright (c) 2026 Nebula Matrix Limited.\n+ /\n+#include \u003clinux/device.h\u003e\n+#include \u003clinux/pci.h\u003e\n+#include \"nbl_dispatch.h\"\n+\n+static int nbl_disp_chan_get_vsi_id_req(struct nbl_dispatch_mgt disp_mgt,\n+\t\t\t\t\tu16 type, u16 vsi_id)\n+{\n+\tstruct nbl_channel_ops chan_ops = disp_mgt-\u003echan_ops_tbl-\u003eops;\n+\tstruct nbl_common_info common = disp_mgt-\u003ecommon;\n+\tstruct nbl_chan_param_get_vsi_id result = { 0 };\n+\tstruct nbl_chan_param_get_vsi_id param = { 0 };\n+\tstruct nbl_chan_send_info chan_send = {0};\n+\tint ret;\n+\n+\tparam.type = cpu_to_le16(type);\n+\n+\tnbl_chan_fill_send_info(\u0026chan_send, common-\u003emgt_pf,\n+\t\t\t\tNBL_CHAN_MSG_GET_VSI_ID,\n+\t\t\t\t\u0026param, sizeof(param), \u0026result,\n+\t\t\t\tsizeof(result), 1);\n+\tret = chan_ops-\u003esend_msg(disp_mgt-\u003echan_ops_tbl-\u003epriv, \u0026chan_send);\n+\tif (ret)\n+\t\treturn ret;\n+\tif (chan_send.ack_len != sizeof(result)) {\n+\t\tdev_err(disp_mgt-\u003ecommon-\u003edev,\n+\t\t\t\"get_vsi_id: short ACK, ack_len=%u expected %zu\\n\",\n+\t\t\tchan_send.ack_len, sizeof(result));\n+\t\treturn -EBADMSG;\n+\t}\n+\tvsi_id = le16_to_cpu(result.vsi_id);\n+\treturn 0;\n+}\n+\n+static void nbl_disp_chan_get_vsi_id_resp(void priv, u16 src_id, u16 msg_id,\n+\t\t\t\t\t void data, u32 data_len)\n+{\n+\tstruct nbl_dispatch_mgt disp_mgt = (struct nbl_dispatch_mgt )priv;\n+\tstruct nbl_channel_ops chan_ops = disp_mgt-\u003echan_ops_tbl-\u003eops;\n+\tstruct nbl_resource_ops res_ops = disp_mgt-\u003eres_ops_tbl-\u003eops;\n+\tstruct nbl_resource_mgt p = disp_mgt-\u003eres_ops_tbl-\u003epriv;\n+\tstruct device dev = disp_mgt-\u003ecommon-\u003edev;\n+\tstruct nbl_chan_param_get_vsi_id result = { 0 };\n+\tstruct nbl_chan_param_get_vsi_id param = { 0 };\n+\tstruct nbl_chan_ack_info chan_ack;\n+\tint err = 0;\n+\tu16 vsi_id = 0;\n+\tu32 rel_pf_id;\n+\tint ret;\n+\n+\tret = nbl_common_func_id_to_rel_pf_id(disp_mgt-\u003ecommon, src_id,\n+\t\t\t\t\t \u0026rel_pf_id);\n+\tif (ret) {\n+\t\terr = -EPERM;\n+\t\tgoto ack_out;\n+\t}\n+\tif (rel_pf_id \u003e= disp_mgt-\u003ecommon-\u003emax_pf) {\n+\t\terr = -EPERM;\n+\t\tgoto ack_out;\n+\t}\n+\tif (data_len \u003c sizeof(param)) {\n+\t\terr = -EBADMSG;\n+\t\tgoto ack_out;\n+\t}\n+\tmemcpy(\u0026param, data, sizeof(param));\n+\n+\tif (res_ops-\u003eget_vsi_id) {\n+\t\tret = res_ops-\u003eget_vsi_id(p, src_id, le16_to_cpu(param.type),\n+\t\t\t\t\t \u0026vsi_id);\n+\t\t/* Forward the resource errno verbatim to the requester /\n+\t\tif (ret)\n+\t\t\terr = ret;\n+\t} else {\n+\t\terr = -EOPNOTSUPP;\n+\t}\n+\n+\tresult.vsi_id = cpu_to_le16(vsi_id);\n+ack_out:\n+\tnbl_chan_fill_ack_info(\u0026chan_ack, src_id,\n+\t\t\t NBL_CHAN_MSG_GET_VSI_ID, msg_id, err,\n+\t\t\t \u0026result, sizeof(result));\n+\tret = chan_ops-\u003esend_ack(disp_mgt-\u003echan_ops_tbl-\u003epriv, \u0026chan_ack);\n+\tif (ret)\n+\t\tdev_err(dev,\n+\t\t\t\"channel send ack failed with ret: %d, msg_type: %d\\n\",\n+\t\t\tret, NBL_CHAN_MSG_GET_VSI_ID);\n+}\n+\n+static int nbl_disp_chan_get_eth_id_req(struct nbl_dispatch_mgt disp_mgt,\n+\t\t\t\t\tu16 vsi_id, u8 eth_num, u8 eth_id,\n+\t\t\t\t\tu8 logic_eth_id)\n+{\n+\tstruct nbl_channel_ops chan_ops = disp_mgt-\u003echan_ops_tbl-\u003eops;\n+\tstruct nbl_common_info common = disp_mgt-\u003ecommon;\n+\tstruct nbl_chan_param_get_eth_id result = { 0 };\n+\tstruct nbl_chan_param_get_eth_id param = { 0 };\n+\tstruct nbl_chan_send_info chan_send = {0};\n+\tint ret;\n+\n+\tparam.vsi_id = cpu_to_le16(vsi_id);\n+\n+\tnbl_chan_fill_send_info(\u0026chan_send, common-\u003emgt_pf,\n+\t\t\t\tNBL_CHAN_MSG_GET_ETH_ID,\n+\t\t\t\t\u0026param, sizeof(param), \u0026result,\n+\t\t\t\tsizeof(result), 1);\n+\tret = chan_ops-\u003esend_msg(disp_mgt-\u003echan_ops_tbl-\u003epriv, \u0026chan_send);\n+\tif (ret)\n+\t\treturn ret;\n+\tif (chan_send.ack_len != sizeof(result)) {\n+\t\tdev_err(disp_mgt-\u003ecommon-\u003edev,\n+\t\t\t\"get_eth_id: short ACK, ack_len=%u expected %zu\\n\",\n+\t\t\tchan_send.ack_len, sizeof(result));\n+\t\treturn -EBADMSG;\n+\t}\n+\teth_num = result.eth_num;\n+\teth_id = result.eth_id;\n+\tlogic_eth_id = result.logic_eth_id;\n+\n+\treturn 0;\n+}\n+\n+static void nbl_disp_chan_get_eth_id_resp(void priv, u16 src_id, u16 msg_id,\n+\t\t\t\t\t void data, u32 data_len)\n+{\n+\tstruct nbl_dispatch_mgt disp_mgt = (struct nbl_dispatch_mgt )priv;\n+\tstruct nbl_channel_ops chan_ops = disp_mgt-\u003echan_ops_tbl-\u003eops;\n+\tstruct nbl_resource_ops res_ops = disp_mgt-\u003eres_ops_tbl-\u003eops;\n+\tstruct nbl_resource_mgt p = disp_mgt-\u003eres_ops_tbl-\u003epriv;\n+\tstruct nbl_chan_param_get_eth_id result = { 0 };\n+\tstruct nbl_chan_param_get_eth_id param = { 0 };\n+\tstruct device dev = disp_mgt-\u003ecommon-\u003edev;\n+\tstruct nbl_chan_ack_info chan_ack;\n+\tint err = 0;\n+\tu32 rel_pf_id;\n+\tint ret;\n+\n+\tret = nbl_common_func_id_to_rel_pf_id(disp_mgt-\u003ecommon, src_id,\n+\t\t\t\t\t \u0026rel_pf_id);\n+\tif (ret) {\n+\t\terr = -EPERM;\n+\t\tgoto ack_out;\n+\t}\n+\tif (rel_pf_id \u003e= disp_mgt-\u003ecommon-\u003emax_pf) {\n+\t\terr = -EPERM;\n+\t\tgoto ack_out;\n+\t}\n+\tif (data_len \u003c sizeof(param)) {\n+\t\terr = -EBADMSG;\n+\t\tgoto ack_out;\n+\t}\n+\tmemcpy(\u0026param, data, sizeof(param));\n+\n+\tif (res_ops-\u003eget_eth_id) {\n+\t\tret = res_ops-\u003eget_eth_id(p, src_id, le16_to_cpu(param.vsi_id),\n+\t\t\t\t\t \u0026result.eth_num, \u0026result.eth_id,\n+\t\t\t\t\t \u0026result.logic_eth_id);\n+\t\t/* Forward the resource errno verbatim to the requester /\n+\t\tif (ret)\n+\t\t\terr = ret;\n+\t} else {\n+\t\terr = -EOPNOTSUPP;\n+\t}\n+ack_out:\n+\tnbl_chan_fill_ack_info(\u0026chan_ack, src_id,\n+\t\t\t NBL_CHAN_MSG_GET_ETH_ID, msg_id, err,\n+\t\t\t \u0026result, sizeof(result));\n+\tret = chan_ops-\u003esend_ack(disp_mgt-\u003echan_ops_tbl-\u003epriv, \u0026chan_ack);\n+\tif (ret)\n+\t\tdev_err(dev,\n+\t\t\t\"channel send ack failed with ret: %d, msg_type: %d\\n\",\n+\t\t\tret, NBL_CHAN_MSG_GET_ETH_ID);\n+}\n+\n+static void nbl_disp_deinit_module(struct nbl_dispatch_mgt disp_mgt)\n+{\n+\tstruct nbl_resource_ops res_ops = disp_mgt-\u003eres_ops_tbl-\u003eops;\n+\tstruct nbl_resource_mgt p = disp_mgt-\u003eres_ops_tbl-\u003epriv;\n+\n+\tif (res_ops-\u003edeinit_module)\n+\t\tres_ops-\u003edeinit_module(p);\n+}\n+\n+static int nbl_disp_init_module(struct nbl_dispatch_mgt disp_mgt)\n+{\n+\tstruct nbl_resource_ops res_ops = disp_mgt-\u003eres_ops_tbl-\u003eops;\n+\tstruct nbl_resource_mgt p = disp_mgt-\u003eres_ops_tbl-\u003epriv;\n+\n+\tif (res_ops-\u003einit_module)\n+\t\treturn res_ops-\u003einit_module(p);\n+\treturn -EOPNOTSUPP;\n+}\n+\n+static int nbl_disp_cfg_msix_map(struct nbl_dispatch_mgt disp_mgt,\n+\t\t\t\t u16 num_net_msix, u16 num_others_msix,\n+\t\t\t\t bool net_msix_mask_en)\n+{\n+\tstruct nbl_resource_ops res_ops = disp_mgt-\u003eres_ops_tbl-\u003eops;\n+\tstruct nbl_resource_mgt p = disp_mgt-\u003eres_ops_tbl-\u003epriv;\n+\tstruct nbl_common_info common = disp_mgt-\u003ecommon;\n+\n+\tif (!res_ops-\u003ecfg_msix_map)\n+\t\treturn -EOPNOTSUPP;\n+\t/\n+\t * No core-layer lock is taken here: the resource layer serializes\n+\t * concurrent callers itself (intr_mgt-\u003elock), and the dispatch\n+\t * layer has no mutable state of its own for this op to protect.\n+\t /\n+\treturn res_ops-\u003ecfg_msix_map(p, common-\u003emgt_pf, num_net_msix,\n+\t\t\t\t num_others_msix, net_msix_mask_en);\n+}\n+\n+/\n+ * Precondition: caller must disable mailbox IRQ_RDY and switch send_msg\n+ * to the polling path before issuing this RPC. The responder rewrites\n+ * the requester's mailbox MSI-X routing during the resource op, so the\n+ * ACK cannot rely on interrupt wakeup while routing is in flux.\n+ /\n+static int\n+nbl_disp_chan_cfg_msix_map_req(struct nbl_dispatch_mgt disp_mgt,\n+\t\t\t u16 num_net_msix, u16 num_others_msix,\n+\t\t\t bool net_msix_mask_en)\n+{\n+\tstruct nbl_channel_ops chan_ops = disp_mgt-\u003echan_ops_tbl-\u003eops;\n+\tstruct nbl_common_info common = disp_mgt-\u003ecommon;\n+\tstruct nbl_chan_param_cfg_msix_map param = { 0 };\n+\tstruct nbl_chan_send_info chan_send = {0};\n+\tint ret;\n+\n+\tparam.num_net_msix = cpu_to_le16(num_net_msix);\n+\tparam.num_others_msix = cpu_to_le16(num_others_msix);\n+\tparam.msix_mask_en = cpu_to_le16(!!net_msix_mask_en);\n+\n+\tnbl_chan_fill_send_info(\u0026chan_send, common-\u003emgt_pf,\n+\t\t\t\tNBL_CHAN_MSG_CONFIGURE_MSIX_MAP,\n+\t\t\t\t\u0026param, sizeof(param),\n+\t\t\t\tNULL, 0, 1);\n+\tret = chan_ops-\u003esend_msg(disp_mgt-\u003echan_ops_tbl-\u003epriv, \u0026chan_send);\n+\tif (ret)\n+\t\treturn ret;\n+\treturn 0;\n+}\n+\n+static void nbl_disp_chan_cfg_msix_map_resp(void priv, u16 src_id, u16 msg_id,\n+\t\t\t\t\t void data, u32 data_len)\n+{\n+\tstruct nbl_dispatch_mgt disp_mgt = (struct nbl_dispatch_mgt )priv;\n+\tstruct nbl_channel_ops chan_ops = disp_mgt-\u003echan_ops_tbl-\u003eops;\n+\tstruct nbl_resource_ops res_ops = disp_mgt-\u003eres_ops_tbl-\u003eops;\n+\tstruct nbl_resource_mgt p = disp_mgt-\u003eres_ops_tbl-\u003epriv;\n+\tstruct device dev = disp_mgt-\u003ecommon-\u003edev;\n+\tstruct nbl_chan_param_cfg_msix_map param = { 0 };\n+\tstruct nbl_chan_ack_info chan_ack;\n+\tint err = 0;\n+\tu32 rel_pf_id;\n+\tint ret;\n+\n+\tret = nbl_common_func_id_to_rel_pf_id(disp_mgt-\u003ecommon, src_id,\n+\t\t\t\t\t \u0026rel_pf_id);\n+\tif (ret) {\n+\t\terr = -EPERM;\n+\t\tgoto ack_out;\n+\t}\n+\tif (rel_pf_id \u003e= disp_mgt-\u003ecommon-\u003emax_pf) {\n+\t\terr = -EPERM;\n+\t\tgoto ack_out;\n+\t}\n+\tif (data_len \u003c sizeof(param)) {\n+\t\terr = -EBADMSG;\n+\t\tgoto ack_out;\n+\t}\n+\tmemcpy(\u0026param, data, sizeof(param));\n+\n+\tif (res_ops-\u003ecfg_msix_map) {\n+\t\t/\n+\t\t The vector counts come from the peer PF. A peer is\n+\t\t * trusted: hardware and firmware forbid PF passthrough to\n+\t\t * guests (see nbl_get_func_param()), so every peer runs this\n+\t\t * same host driver, and the resource layer bounds the counts\n+\t\t * by the global vector pool. A peer asking for the whole\n+\t\t * pool can starve the others with -EAGAIN, but it cannot\n+\t\t * corrupt state.\n+\t\t /\n+\t\tret = res_ops-\u003ecfg_msix_map(p, src_id,\n+\t\t\t\t\t le16_to_cpu(param.num_net_msix),\n+\t\t\t\t\t le16_to_cpu(param.num_others_msix),\n+\t\t\t\t\t !!le16_to_cpu(param.msix_mask_en));\n+\t\t/ Forward the resource errno verbatim to the requester /\n+\t\tif (ret)\n+\t\t\terr = ret;\n+\t} else {\n+\t\terr = -EOPNOTSUPP;\n+\t}\n+ack_out:\n+\tnbl_chan_fill_ack_info(\u0026chan_ack, src_id,\n+\t\t\t NBL_CHAN_MSG_CONFIGURE_MSIX_MAP, msg_id,\n+\t\t\t err, NULL, 0);\n+\tret = chan_ops-\u003esend_ack(disp_mgt-\u003echan_ops_tbl-\u003epriv, \u0026chan_ack);\n+\tif (ret)\n+\t\tdev_err(dev,\n+\t\t\t\"channel send ack failed with ret: %d, msg_type: %d\\n\",\n+\t\t\tret, NBL_CHAN_MSG_CONFIGURE_MSIX_MAP);\n+}\n+\n+/\n+ * Precondition: caller must disable mailbox IRQ_RDY and switch send_msg\n+ * to the polling path before issuing this RPC. The responder retargets\n+ * the requester's mailbox MSI-X routing during the resource op, so the\n+ * ACK cannot rely on interrupt wakeup while routing is in flux.\n+ /\n+static int nbl_disp_chan_destroy_msix_map_req(struct nbl_dispatch_mgt disp_mgt)\n+{\n+\tstruct nbl_channel_ops chan_ops = disp_mgt-\u003echan_ops_tbl-\u003eops;\n+\tstruct nbl_common_info common = disp_mgt-\u003ecommon;\n+\tstruct nbl_chan_send_info chan_send = {0};\n+\tint ret;\n+\n+\tnbl_chan_fill_send_info(\u0026chan_send, common-\u003emgt_pf,\n+\t\t\t\tNBL_CHAN_MSG_DESTROY_MSIX_MAP,\n+\t\t\t\tNULL, 0, NULL, 0, 1);\n+\tret = chan_ops-\u003esend_msg(disp_mgt-\u003echan_ops_tbl-\u003epriv, \u0026chan_send);\n+\tif (ret)\n+\t\treturn ret;\n+\treturn 0;\n+}\n+\n+static void nbl_disp_chan_destroy_msix_map_resp(void priv, u16 src_id,\n+\t\t\t\t\t\tu16 msg_id, void data,\n+\t\t\t\t\t\tu32 data_len)\n+{\n+\tstruct nbl_dispatch_mgt disp_mgt = (struct nbl_dispatch_mgt )priv;\n+\tstruct nbl_channel_ops chan_ops = disp_mgt-\u003echan_ops_tbl-\u003eops;\n+\tstruct nbl_resource_ops res_ops = disp_mgt-\u003eres_ops_tbl-\u003eops;\n+\tstruct nbl_resource_mgt p = disp_mgt-\u003eres_ops_tbl-\u003epriv;\n+\tstruct device dev = disp_mgt-\u003ecommon-\u003edev;\n+\tstruct nbl_chan_ack_info chan_ack;\n+\tint err = 0;\n+\tu32 rel_pf_id;\n+\tint ret;\n+\n+\tret = nbl_common_func_id_to_rel_pf_id(disp_mgt-\u003ecommon, src_id,\n+\t\t\t\t\t \u0026rel_pf_id);\n+\tif (ret) {\n+\t\terr = -EPERM;\n+\t\tgoto ack_out;\n+\t}\n+\tif (rel_pf_id \u003e= disp_mgt-\u003ecommon-\u003emax_pf) {\n+\t\terr = -EPERM;\n+\t\tgoto ack_out;\n+\t}\n+\tif (res_ops-\u003edestroy_msix_map) {\n+\t\tret = res_ops-\u003edestroy_msix_map(p, src_id);\n+\t\t/* Forward the resource errno verbatim to the requester /\n+\t\tif (ret)\n+\t\t\terr = ret;\n+\t} else {\n+\t\terr = -EOPNOTSUPP;\n+\t}\n+ack_out:\n+\tnbl_chan_fill_ack_info(\u0026chan_ack, src_id,\n+\t\t\t NBL_CHAN_MSG_DESTROY_MSIX_MAP, msg_id,\n+\t\t\t err, NULL, 0);\n+\tret = chan_ops-\u003esend_ack(disp_mgt-\u003echan_ops_tbl-\u003epriv, \u0026chan_ack);\n+\tif (ret)\n+\t\tdev_err(dev,\n+\t\t\t\"channel send ack failed with ret: %d, msg_type: %d\\n\",\n+\t\t\tret, NBL_CHAN_MSG_DESTROY_MSIX_MAP);\n+}\n+\n+/\n+ * Precondition: caller must disable mailbox IRQ_RDY and switch send_msg\n+ * to the polling path before issuing this RPC. The responder rewrites\n+ * the requester's own mailbox MSI-X routing (MSIX_IDX / MSIX_IDX_VALID)\n+ * before the ACK is sent, so the ACK cannot rely on interrupt wakeup\n+ * while routing is in flux.\n+ /\n+static int nbl_disp_chan_set_mailbox_irq_req(struct nbl_dispatch_mgt disp_mgt,\n+\t\t\t\t\t u16 vector_id, bool en_msix)\n+{\n+\tstruct nbl_channel_ops chan_ops = disp_mgt-\u003echan_ops_tbl-\u003eops;\n+\tstruct nbl_chan_param_set_mailbox_irq param = { 0 };\n+\tstruct nbl_common_info common = disp_mgt-\u003ecommon;\n+\tstruct nbl_chan_send_info chan_send = {0};\n+\tint ret;\n+\n+\tparam.vector_id = cpu_to_le16(vector_id);\n+\tparam.en_msix = !!en_msix;\n+\n+\tnbl_chan_fill_send_info(\u0026chan_send, common-\u003emgt_pf,\n+\t\t\t\tNBL_CHAN_MSG_MAILBOX_SET_IRQ,\n+\t\t\t\t\u0026param, sizeof(param), NULL, 0, 1);\n+\tret = chan_ops-\u003esend_msg(disp_mgt-\u003echan_ops_tbl-\u003epriv, \u0026chan_send);\n+\tif (ret)\n+\t\treturn ret;\n+\treturn 0;\n+}\n+\n+static void nbl_disp_chan_set_mailbox_irq_resp(void priv, u16 src_id,\n+\t\t\t\t\t u16 msg_id, void data,\n+\t\t\t\t\t u32 data_len)\n+{\n+\tstruct nbl_dispatch_mgt disp_mgt = (struct nbl_dispatch_mgt )priv;\n+\tstruct nbl_channel_ops chan_ops = disp_mgt-\u003echan_ops_tbl-\u003eops;\n+\tstruct nbl_resource_ops res_ops = disp_mgt-\u003eres_ops_tbl-\u003eops;\n+\tstruct nbl_resource_mgt p = disp_mgt-\u003eres_ops_tbl-\u003epriv;\n+\tstruct nbl_chan_param_set_mailbox_irq param = { 0 };\n+\tstruct device dev = disp_mgt-\u003ecommon-\u003edev;\n+\tstruct nbl_chan_ack_info chan_ack;\n+\tint err = 0;\n+\tu16 vector_id;\n+\tu32 rel_pf_id;\n+\tbool en_msix;\n+\tint ret;\n+\n+\tret = nbl_common_func_id_to_rel_pf_id(disp_mgt-\u003ecommon, src_id,\n+\t\t\t\t\t \u0026rel_pf_id);\n+\tif (ret) {\n+\t\terr = -EPERM;\n+\t\tgoto ack_out;\n+\t}\n+\tif (rel_pf_id \u003e= disp_mgt-\u003ecommon-\u003emax_pf) {\n+\t\terr = -EPERM;\n+\t\tgoto ack_out;\n+\t}\n+\tif (data_len \u003c sizeof(param)) {\n+\t\terr = -EBADMSG;\n+\t\tgoto ack_out;\n+\t}\n+\tmemcpy(\u0026param, data, sizeof(param));\n+\tvector_id = le16_to_cpu(param.vector_id);\n+\ten_msix = !!param.en_msix;\n+\n+\tif (res_ops-\u003eset_mailbox_irq) {\n+\t\tret = res_ops-\u003eset_mailbox_irq(p, src_id, vector_id, en_msix);\n+\t\t/* Forward the resource errno verbatim to the requester /\n+\t\tif (ret)\n+\t\t\terr = ret;\n+\t} else {\n+\t\terr = -EOPNOTSUPP;\n+\t}\n+\n+ack_out:\n+\tnbl_chan_fill_ack_info(\u0026chan_ack, src_id,\n+\t\t\t NBL_CHAN_MSG_MAILBOX_SET_IRQ, msg_id,\n+\t\t\t err, NULL, 0);\n+\tret = chan_ops-\u003esend_ack(disp_mgt-\u003echan_ops_tbl-\u003epriv, \u0026chan_ack);\n+\tif (ret)\n+\t\tdev_err(dev,\n+\t\t\t\"channel send ack failed with ret: %d, msg_type: %d\\n\",\n+\t\t\tret, NBL_CHAN_MSG_MAILBOX_SET_IRQ);\n+}\n+\n+static int nbl_disp_destroy_msix_map(struct nbl_dispatch_mgt disp_mgt)\n+{\n+\tstruct nbl_resource_ops res_ops = disp_mgt-\u003eres_ops_tbl-\u003eops;\n+\tstruct nbl_resource_mgt p = disp_mgt-\u003eres_ops_tbl-\u003epriv;\n+\tstruct nbl_common_info common = disp_mgt-\u003ecommon;\n+\n+\tif (!res_ops-\u003edestroy_msix_map)\n+\t\treturn -EOPNOTSUPP;\n+\treturn res_ops-\u003edestroy_msix_map(p, common-\u003emgt_pf);\n+}\n+\n+static int nbl_disp_set_mailbox_irq(struct nbl_dispatch_mgt disp_mgt,\n+\t\t\t\t u16 vector_id, bool en_msix)\n+{\n+\tstruct nbl_resource_ops res_ops = disp_mgt-\u003eres_ops_tbl-\u003eops;\n+\tstruct nbl_resource_mgt p = disp_mgt-\u003eres_ops_tbl-\u003epriv;\n+\tstruct nbl_common_info common = disp_mgt-\u003ecommon;\n+\n+\tif (!res_ops-\u003eset_mailbox_irq)\n+\t\treturn -EOPNOTSUPP;\n+\treturn res_ops-\u003eset_mailbox_irq(p, common-\u003emgt_pf, vector_id, en_msix);\n+}\n+\n+static int nbl_disp_get_vsi_id(struct nbl_dispatch_mgt disp_mgt, u16 type,\n+\t\t\t u16 vsi_id)\n+{\n+\tstruct nbl_resource_ops res_ops = disp_mgt-\u003eres_ops_tbl-\u003eops;\n+\tstruct nbl_resource_mgt p = disp_mgt-\u003eres_ops_tbl-\u003epriv;\n+\tstruct nbl_common_info common = disp_mgt-\u003ecommon;\n+\n+\tif (res_ops-\u003eget_vsi_id)\n+\t\treturn res_ops-\u003eget_vsi_id(p, common-\u003emgt_pf, type, vsi_id);\n+\treturn -EOPNOTSUPP;\n+}\n+\n+static int nbl_disp_get_eth_id(struct nbl_dispatch_mgt disp_mgt, u16 vsi_id,\n+\t\t\t u8 eth_num, u8 eth_id, u8 logic_eth_id)\n+{\n+\tstruct nbl_resource_ops res_ops = disp_mgt-\u003eres_ops_tbl-\u003eops;\n+\tstruct nbl_resource_mgt p = disp_mgt-\u003eres_ops_tbl-\u003epriv;\n+\tstruct nbl_common_info common = disp_mgt-\u003ecommon;\n+\n+\tif (res_ops-\u003eget_eth_id)\n+\t\treturn res_ops-\u003eget_eth_id(p, common-\u003emgt_pf, vsi_id,\n+\t\t\t\t\t eth_num, eth_id, logic_eth_id);\n+\treturn -EOPNOTSUPP;\n+}\n+\n+static int nbl_disp_setup_msg(struct nbl_dispatch_mgt disp_mgt)\n+{\n+\tstruct nbl_channel_ops chan_ops = disp_mgt-\u003echan_ops_tbl-\u003eops;\n+\tstruct nbl_channel_mgt p = disp_mgt-\u003echan_ops_tbl-\u003epriv;\n+\tint ret = 0;\n+\tint _ret;\n+\n+\t_ret = chan_ops-\u003eregister_msg(p, NBL_CHAN_MSG_CONFIGURE_MSIX_MAP,\n+\t\t\t\t nbl_disp_chan_cfg_msix_map_resp,\n+\t\t\t\t disp_mgt);\n+\tif (_ret \u003c 0 \u0026\u0026 !ret)\n+\t\tret = _ret;\n+\n+\t_ret = chan_ops-\u003eregister_msg(p, NBL_CHAN_MSG_DESTROY_MSIX_MAP,\n+\t\t\t\t nbl_disp_chan_destroy_msix_map_resp,\n+\t\t\t\t disp_mgt);\n+\tif (_ret \u003c 0 \u0026\u0026 !ret)\n+\t\tret = _ret;\n+\n+\t_ret = chan_ops-\u003eregister_msg(p, NBL_CHAN_MSG_MAILBOX_SET_IRQ,\n+\t\t\t\t nbl_disp_chan_set_mailbox_irq_resp,\n+\t\t\t\t disp_mgt);\n+\tif (_ret \u003c 0 \u0026\u0026 !ret)\n+\t\tret = _ret;\n+\n+\t_ret = chan_ops-\u003eregister_msg(p, NBL_CHAN_MSG_GET_VSI_ID,\n+\t\t\t\t nbl_disp_chan_get_vsi_id_resp,\n+\t\t\t\t disp_mgt);\n+\tif (_ret \u003c 0 \u0026\u0026 !ret)\n+\t\tret = _ret;\n+\n+\t_ret = chan_ops-\u003eregister_msg(p, NBL_CHAN_MSG_GET_ETH_ID,\n+\t\t\t\t nbl_disp_chan_get_eth_id_resp,\n+\t\t\t\t disp_mgt);\n+\tif (_ret \u003c 0 \u0026\u0026 !ret)\n+\t\tret = _ret;\n+\n+\treturn ret;\n+}\n+\n+static void nbl_disp_set_ctrl_bit(struct nbl_dispatch_mgt disp_mgt, u32 lvl)\n+{\n+\tset_bit(lvl, disp_mgt-\u003ectrl_lvl);\n+}\n+\n+static void nbl_disp_refresh_ctrl_ops(struct nbl_dispatch_mgt disp_mgt)\n+{\n+\tstruct nbl_dispatch_ops disp_ops = disp_mgt-\u003edisp_ops_tbl-\u003eops;\n+\n+\tmemset(disp_ops, 0, sizeof(disp_ops));\n+\tif (test_bit(NBL_DISP_CTRL_LVL_MGT, disp_mgt-\u003ectrl_lvl)) {\n+\t\tdisp_ops-\u003einit_module = nbl_disp_init_module;\n+\t\tdisp_ops-\u003edeinit_module = nbl_disp_deinit_module;\n+\t\tdisp_ops-\u003ecfg_msix_map = nbl_disp_cfg_msix_map;\n+\t\tdisp_ops-\u003edestroy_msix_map = nbl_disp_destroy_msix_map;\n+\t\tdisp_ops-\u003eset_mailbox_irq = nbl_disp_set_mailbox_irq;\n+\t\tdisp_ops-\u003eget_vsi_id = nbl_disp_get_vsi_id;\n+\t\tdisp_ops-\u003eget_eth_id = nbl_disp_get_eth_id;\n+\t} else if (test_bit(NBL_DISP_CTRL_LVL_NET, disp_mgt-\u003ectrl_lvl)) {\n+\t\tdisp_ops-\u003ecfg_msix_map =\n+\t\t\tnbl_disp_chan_cfg_msix_map_req;\n+\t\tdisp_ops-\u003edestroy_msix_map = nbl_disp_chan_destroy_msix_map_req;\n+\t\tdisp_ops-\u003eset_mailbox_irq = nbl_disp_chan_set_mailbox_irq_req;\n+\t\tdisp_ops-\u003eget_vsi_id = nbl_disp_chan_get_vsi_id_req;\n+\t\tdisp_ops-\u003eget_eth_id = nbl_disp_chan_get_eth_id_req;\n+\t}\n+}\n+\n+static struct nbl_dispatch_mgt \n+nbl_disp_setup_disp_mgt(struct nbl_common_info common)\n+{\n+\tstruct nbl_dispatch_mgt disp_mgt;\n+\tstruct device dev = common-\u003edev;\n+\n+\tdisp_mgt = devm_kzalloc(dev, sizeof(disp_mgt), GFP_KERNEL);\n+\tif (!disp_mgt)\n+\t\treturn ERR_PTR(-ENOMEM);\n+\n+\tdisp_mgt-\u003ecommon = common;\n+\treturn disp_mgt;\n+}\n+\n+static struct nbl_dispatch_ops_tbl \n+nbl_disp_setup_ops(struct device dev, struct nbl_dispatch_mgt disp_mgt)\n+{\n+\tstruct nbl_dispatch_ops_tbl disp_ops_tbl;\n+\tstruct nbl_dispatch_ops disp_ops;\n+\n+\tdisp_ops_tbl = devm_kzalloc(dev, sizeof(disp_ops_tbl), GFP_KERNEL);\n+\tif (!disp_ops_tbl)\n+\t\treturn ERR_PTR(-ENOMEM);\n+\n+\tdisp_ops = devm_kzalloc(dev, sizeof(disp_ops), GFP_KERNEL);\n+\tif (!disp_ops)\n+\t\treturn ERR_PTR(-ENOMEM);\n+\n+\tdisp_ops_tbl-\u003eops = disp_ops;\n+\tdisp_ops_tbl-\u003epriv = disp_mgt;\n+\n+\treturn disp_ops_tbl;\n+}\n+\n+int nbl_disp_init(struct nbl_adapter adapter)\n+{\n+\tstruct nbl_common_info common = \u0026adapter-\u003ecommon;\n+\tstruct nbl_dispatch_ops_tbl disp_ops_tbl;\n+\tstruct nbl_resource_ops_tbl res_ops_tbl =\n+\t\tadapter-\u003eintf.resource_ops_tbl;\n+\tstruct nbl_channel_ops_tbl chan_ops_tbl =\n+\t\tadapter-\u003eintf.channel_ops_tbl;\n+\tstruct device dev = \u0026adapter-\u003epdev-\u003edev;\n+\tstruct nbl_dispatch_mgt disp_mgt;\n+\tint ret;\n+\n+\tdisp_mgt = nbl_disp_setup_disp_mgt(common);\n+\tif (IS_ERR(disp_mgt)) {\n+\t\tret = PTR_ERR(disp_mgt);\n+\t\treturn ret;\n+\t}\n+\n+\tdisp_ops_tbl = nbl_disp_setup_ops(dev, disp_mgt);\n+\tif (IS_ERR(disp_ops_tbl)) {\n+\t\tret = PTR_ERR(disp_ops_tbl);\n+\t\treturn ret;\n+\t}\n+\n+\tdisp_mgt-\u003eres_ops_tbl = res_ops_tbl;\n+\tdisp_mgt-\u003echan_ops_tbl = chan_ops_tbl;\n+\tdisp_mgt-\u003edisp_ops_tbl = disp_ops_tbl;\n+\tadapter-\u003ecore.disp_mgt = disp_mgt;\n+\tadapter-\u003eintf.dispatch_ops_tbl = disp_ops_tbl;\n+\n+\tret = nbl_disp_setup_msg(disp_mgt);\n+\tif (ret)\n+\t\treturn ret;\n+\n+\tif (common-\u003ehas_ctrl)\n+\t\tnbl_disp_set_ctrl_bit(disp_mgt, NBL_DISP_CTRL_LVL_MGT);\n+\n+\tif (common-\u003ehas_net)\n+\t\tnbl_disp_set_ctrl_bit(disp_mgt, NBL_DISP_CTRL_LVL_NET);\n+\tnbl_disp_refresh_ctrl_ops(disp_mgt);\n+\treturn 0;\n+}\n+\n+void nbl_disp_remove(struct nbl_adapter adapter)\n+{\n+\t/\n+\t Dispatch structures are devm-allocated and freed at detach.\n+\t \n+\t The five responders registered by nbl_disp_setup_msg() are\n+\t * owned by the channel layer (xarray of handlers) and are never\n+\t * unregistered here. This is safe because the teardown order\n+\t * guarantees no responder can run after this point:\n+\t \n+\t mailbox teardown\n+\t * -\u003e cancel_work_sync(clean_mbx_task) // drain RX work\n+\t * -\u003e nbl_chan_teardown_queue() // stop HW queue,\n+\t * // join clean task,\n+\t * // active=false\n+\t * -\u003e nbl_chan_remove_common()\n+\t * -\u003e nbl_common_destroy_wq() // no new work\n+\t * -\u003e nbl_chan_remove_msg_handler() // free handler nodes\n+\t \n+\t By the time devres frees disp_mgt, the mailbox queue is stopped\n+\t * and the handler xarray is empty, so no responder can touch\n+\t * res_mgt-\u003eintr_mgt after nbl_intr_mgt_stop() has cleared it.\n+\t /\n+}\ndiff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_core/nbl_dispatch.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_core/nbl_dispatch.h\nnew file mode 100644\nindex 00000000000000..a7e5802344b414\n--- /dev/null\n+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_core/nbl_dispatch.h\n@@ -0,0 +1,23 @@\n+/ SPDX-License-Identifier: GPL-2.0 /\n+/\n+ * Copyright (c) 2026 Nebula Matrix Limited.\n+ /\n+\n+#ifndef _NBL_DISPATCH_H_\n+#define _NBL_DISPATCH_H_\n+#include \"../nbl_include/nbl_include.h\"\n+#include \"../nbl_include/nbl_def_channel.h\"\n+#include \"../nbl_include/nbl_def_resource.h\"\n+#include \"../nbl_include/nbl_def_dispatch.h\"\n+#include \"../nbl_include/nbl_def_common.h\"\n+#include \"../nbl_core.h\"\n+\n+struct nbl_dispatch_mgt {\n+\tstruct nbl_common_info common;\n+\tstruct nbl_resource_ops_tbl res_ops_tbl;\n+\tstruct nbl_channel_ops_tbl chan_ops_tbl;\n+\tstruct nbl_dispatch_ops_tbl disp_ops_tbl;\n+\tDECLARE_BITMAP(ctrl_lvl, NBL_DISP_CTRL_LVL_MAX);\n+};\n+\n+#endif\ndiff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_chip.c b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_chip.c\nnew file mode 100644\nindex 00000000000000..419eb6392ada19\n--- /dev/null\n+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_chip.c\n@@ -0,0 +1,32 @@\n+// SPDX-License-Identifier: GPL-2.0\n+/\n+ * Copyright (c) 2026 Nebula Matrix Limited.\n+ /\n+#include \u003clinux/device.h\u003e\n+#include \"nbl_chip.h\"\n+\n+void nbl_res_chip_deinit_module(struct nbl_resource_mgt res_mgt)\n+{\n+\tstruct nbl_hw_ops hw_ops = res_mgt-\u003ehw_ops_tbl-\u003eops;\n+\tstruct nbl_common_info common = res_mgt-\u003ecommon;\n+\n+\tif (!common-\u003ehas_ctrl)\n+\t\treturn;\n+\thw_ops-\u003edeinit_module(res_mgt-\u003ehw_ops_tbl-\u003epriv);\n+}\n+\n+int nbl_res_chip_init_module(struct nbl_resource_mgt res_mgt)\n+{\n+\tstruct nbl_common_info common = res_mgt-\u003ecommon;\n+\tstruct nbl_hw_ops hw_ops;\n+\tu8 eth_speed, eth_num;\n+\tstruct nbl_hw_mgt p;\n+\n+\tif (!common-\u003ehas_ctrl)\n+\t\treturn -EINVAL;\n+\teth_speed = res_mgt-\u003eresource_info-\u003eboard_info.eth_speed;\n+\teth_num = res_mgt-\u003eresource_info-\u003eboard_info.eth_num;\n+\thw_ops = res_mgt-\u003ehw_ops_tbl-\u003eops;\n+\tp = res_mgt-\u003ehw_ops_tbl-\u003epriv;\n+\treturn hw_ops-\u003einit_module(p, eth_speed, eth_num);\n+}\ndiff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_chip.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_chip.h\nnew file mode 100644\nindex 00000000000000..d14093ba916c95\n--- /dev/null\n+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_chip.h\n@@ -0,0 +1,12 @@\n+/* SPDX-License-Identifier: GPL-2.0 /\n+/\n+ * Copyright (c) 2026 Nebula Matrix Limited.\n+ /\n+\n+#ifndef _NBL_CHIP_H_\n+#define _NBL_CHIP_H_\n+\n+#include \"nbl_resource.h\"\n+int nbl_res_chip_init_module(struct nbl_resource_mgt res_mgt);\n+void nbl_res_chip_deinit_module(struct nbl_resource_mgt res_mgt);\n+#endif\ndiff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_leonis/nbl_hw_leonis.c b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_leonis/nbl_hw_leonis.c\nindex cf40ddc4519211..2b55f2c2518b4f 100644\n--- a/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_leonis/nbl_hw_leonis.c\n+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_leonis/nbl_hw_leonis.c\n@@ -5,9 +5,1232 @@\n #include \u003clinux/device.h\u003e\n #include \u003clinux/pci.h\u003e\n #include \u003clinux/bits.h\u003e\n+#include \u003clinux/delay.h\u003e\n #include \u003clinux/io.h\u003e\n+#include \u003clinux/spinlock.h\u003e\n+#include \u003clinux/bitfield.h\u003e\n #include \"nbl_hw_leonis.h\"\n \n+/\n+ * Firmware cleanup after driver_status=false is asynchronous and the\n+ * current hardware revision exposes no cleanup-complete status bit, so\n+ * this delay is best-effort only: it gives the firmware pass time to\n+ * settle before the caller returns, but cannot prove that it has.\n+ * Bounding the wait keeps a later probe from re-programming the per-PF\n+ * registers that pass reclaims (mailbox QINFO routing, qinfo registers)\n+ * while firmware is realistically still wiping them. The chip-wide\n+ * datapath registers init_module() writes are not part of that pass and\n+ * stay programmed until chip reset. The hard teardown ordering comes\n+ * from the device-link relationship between the sibling PFs and the\n+ * control PF.\n+ \n+ These bounds are empirical, not derived from a documented firmware\n+ * latency. There is nothing to poll for instead: the hardware exposes\n+ * no cleanup-complete bit, and the reclaimed entries read back the same\n+ * before and after the pass, so there is no observable side effect to\n+ * wait on and nothing to validate the delay against. A firmware pass\n+ * that takes longer than this window - an ECPU still busy right after\n+ * kexec, say - can therefore still race a later probe.\n+ /\n+#define NBL_FW_CLEANUP_SYNC_MIN_US\t2000\n+#define NBL_FW_CLEANUP_SYNC_MAX_US\t3000\n+\n+static void nbl_hw_read_mbx_regs(struct nbl_hw_mgt hw_mgt, u64 reg, u32 data,\n+\t\t\t\t u32 len)\n+{\n+\tu32 i;\n+\n+\tif (len % 4)\n+\t\treturn;\n+\tif (reg \u003e= (u64)hw_mgt-\u003emailbox_bar_size \|\|\n+\t reg + len \u003e (u64)hw_mgt-\u003emailbox_bar_size) {\n+\t\tdev_err_once(hw_mgt-\u003ecommon-\u003edev,\n+\t\t\t \"mbx read out of range: reg=0x%llx len=%u bar_size=%pa\\n\",\n+\t\t\t reg, len, \u0026hw_mgt-\u003emailbox_bar_size);\n+\t\treturn;\n+\t}\n+\tfor (i = 0; i \u003c len / 4; i++)\n+\t\tdata[i] = nbl_mbx_rd32(hw_mgt, reg + i sizeof(u32));\n+}\n+\n+static void nbl_hw_write_mbx_regs(struct nbl_hw_mgt hw_mgt, u64 reg,\n+\t\t\t\t const u32 data, u32 len)\n+{\n+\tu32 i;\n+\n+\tif (len % 4)\n+\t\treturn;\n+\tif (reg \u003e= (u64)hw_mgt-\u003emailbox_bar_size \|\|\n+\t reg + len \u003e (u64)hw_mgt-\u003emailbox_bar_size) {\n+\t\tdev_err_once(hw_mgt-\u003ecommon-\u003edev,\n+\t\t\t \"mbx write out of range: reg=0x%llx len=%u bar_size=%pa\\n\",\n+\t\t\t reg, len, \u0026hw_mgt-\u003emailbox_bar_size);\n+\t\treturn;\n+\t}\n+\tfor (i = 0; i \u003c len / 4; i++)\n+\t\tnbl_mbx_wr32(hw_mgt, reg + i * sizeof(u32), data[i]);\n+}\n+\n+/\n+ Flush posted mailbox-BAR writes by reading back through the same\n+ * BAR.\n+ /\n+static void nbl_hw_flush_mbx_write(struct nbl_hw_mgt hw_mgt, u64 reg)\n+{\n+\tu32 data;\n+\n+\tnbl_hw_read_mbx_regs(hw_mgt, reg, \u0026data, sizeof(data));\n+}\n+\n+static void nbl_hw_rd_regs(struct nbl_hw_mgt hw_mgt, u64 reg, u32 data,\n+\t\t\t u32 len)\n+{\n+\tu32 size = len / 4;\n+\tu32 i;\n+\n+\tif (len % 4)\n+\t\treturn;\n+\tfor (i = 0; i \u003c size; i++)\n+\t\tdata[i] = rd32(hw_mgt-\u003ehw_addr, reg + i * sizeof(u32));\n+}\n+\n+static void nbl_hw_wr_regs(struct nbl_hw_mgt hw_mgt, u64 reg, const u32 data,\n+\t\t\t u32 len)\n+{\n+\tu32 size = len / 4;\n+\tu32 i;\n+\n+\tif (len % 4)\n+\t\treturn;\n+\tfor (i = 0; i \u003c size; i++)\n+\t\twr32(hw_mgt-\u003ehw_addr, reg + i * sizeof(u32), data[i]);\n+}\n+\n+static void nbl_hw_rd_regs_lock(struct nbl_hw_mgt hw_mgt, u64 reg, u32 data,\n+\t\t\t\tu32 len)\n+{\n+\tu32 size = len / 4;\n+\tu32 i;\n+\n+\tif (len % 4)\n+\t\treturn;\n+\n+\tspin_lock(\u0026hw_mgt-\u003ereg_lock);\n+\n+\tfor (i = 0; i \u003c size; i++)\n+\t\tdata[i] = rd32(hw_mgt-\u003ehw_addr, reg + i * sizeof(u32));\n+\tspin_unlock(\u0026hw_mgt-\u003ereg_lock);\n+}\n+\n+static void nbl_hw_wr_regs_lock(struct nbl_hw_mgt hw_mgt, u64 reg,\n+\t\t\t\tconst u32 data, u32 len)\n+{\n+\tu32 size = len / 4;\n+\tu32 i;\n+\n+\tif (len % 4)\n+\t\treturn;\n+\tspin_lock(\u0026hw_mgt-\u003ereg_lock);\n+\tfor (i = 0; i \u003c size; i++)\n+\t\twr32(hw_mgt-\u003ehw_addr, reg + i * sizeof(u32), data[i]);\n+\tspin_unlock(\u0026hw_mgt-\u003ereg_lock);\n+}\n+\n+/\n+ Flush posted MEMORY-BAR writes by reading back a register that only\n+ * exists in the control-PF mapping.\n+ \n+ The mapped size does not derive from bar_len: nbl_hw_init_leonis()\n+ * maps PAGE_ALIGN_DOWN(NBL_MEM_BAR_TOTAL_SIZE - NBL_RDMA_NOTIFY_LEN)\n+ * (~64 MiB) when has_ctrl is set, and NBL_REG_NET_ONLY_LEN (8 KiB) when\n+ * it is not. NBL_HW_DUMMY_REG (0x1300904) is only covered by the\n+ * control-PF mapping, so this must not be called on a net-only PF.\n+ \n+ Every current caller runs on the control PF: init_module()/\n+ * deinit_module() are has_ctrl-guarded in the resource layer, and the\n+ * MSI-X paths are either has_ctrl-guarded as well or dispatched to the\n+ * management PF. The check below is therefore a defensive guard against\n+ * a future caller, not a reachable failure path.\n+ /\n+static void nbl_flush_writes(struct nbl_hw_mgt hw_mgt)\n+{\n+\tif (WARN_ON_ONCE(!hw_mgt-\u003ecommon-\u003ehas_ctrl))\n+\t\treturn;\n+\n+\tnbl_hw_rd32(hw_mgt, NBL_HW_DUMMY_REG);\n+}\n+\n+/\n+ Registers reset to zero after cold boot / FLR / bus reset. Firmware\n+ * programs valid values before driver probe, so zero is only seen on\n+ * hardware fault or register read failure. Initialize data=0 to guard\n+ * against nbl_hw_read_mbx_regs() early-return on bounds-check failure.\n+ /\n+static void nbl_hw_get_fw_eth_map(struct nbl_hw_mgt hw_mgt, u32 eth_map)\n+{\n+\tu32 data = 0;\n+\n+\tnbl_hw_read_mbx_regs(hw_mgt, NBL_FW_BOARD_DW6_OFFSET, \u0026data,\n+\t\t\t sizeof(data));\n+\teth_map = FIELD_GET(NBL_FW_BOARD_DW6_ETH_BITMAP_MASK, data);\n+}\n+\n+static u32 nbl_hw_get_quirks(struct nbl_hw_mgt hw_mgt)\n+{\n+\tu32 quirks = 0;\n+\n+\t/\n+\t * Read quirk bits from mailbox register.\n+\t * All supported firmware implement the quirk ABI,\n+\t * firmware always populates NBL_LEONIS_QUIRKS_OFFSET.\n+\t * Value ~0U indicates no active quirks.\n+\t /\n+\tnbl_hw_read_mbx_regs(hw_mgt, NBL_LEONIS_QUIRKS_OFFSET, \u0026quirks,\n+\t\t\t sizeof(u32));\n+\n+\tif (quirks == ~0u)\n+\t\treturn 0;\n+\n+\treturn quirks;\n+}\n+\n+static void nbl_configure_dped_checksum(struct nbl_hw_mgt hw_mgt)\n+{\n+\tu32 data = 0;\n+\n+\t/* DPED dped_l4_ck_cmd_40 for sctp /\n+\tspin_lock(\u0026hw_mgt-\u003ereg_lock);\n+\tnbl_hw_rd_regs(hw_mgt, NBL_DPED_L4_CK_CMD_40_ADDR, \u0026data, sizeof(data));\n+\tdata \|= FIELD_PREP(NBL_DPED_L4_CK_CMD_40_EN_MASK, 1);\n+\tnbl_hw_wr_regs(hw_mgt, NBL_DPED_L4_CK_CMD_40_ADDR, \u0026data, sizeof(data));\n+\tspin_unlock(\u0026hw_mgt-\u003ereg_lock);\n+}\n+\n+static void nbl_dped_init(struct nbl_hw_mgt hw_mgt)\n+{\n+\tspin_lock(\u0026hw_mgt-\u003ereg_lock);\n+\tnbl_hw_wr32(hw_mgt, NBL_DPED_VLAN_OFFSET, 0xC);\n+\tnbl_hw_wr32(hw_mgt, NBL_DPED_DSCP_OFFSET_0, 0x8);\n+\tnbl_hw_wr32(hw_mgt, NBL_DPED_DSCP_OFFSET_1, 0x4);\n+\tspin_unlock(\u0026hw_mgt-\u003ereg_lock);\n+\t/* dped checksum offload /\n+\tnbl_configure_dped_checksum(hw_mgt);\n+}\n+\n+static void nbl_uped_init(struct nbl_hw_mgt hw_mgt)\n+{\n+\tu32 hw_edit = 0;\n+\n+\t/* V4 TCP: l3_len = 0 /\n+\tspin_lock(\u0026hw_mgt-\u003ereg_lock);\n+\tnbl_hw_rd_regs(hw_mgt, NBL_UPED_HW_EDT_PROF_TABLE(NBL_UPED_V4_TCP_IDX),\n+\t\t \u0026hw_edit, sizeof(hw_edit));\n+\thw_edit \u0026= ~NBL_PED_HW_EDIT_PROFILE_L3_LEN_MASK;\n+\tnbl_hw_wr_regs(hw_mgt, NBL_UPED_HW_EDT_PROF_TABLE(NBL_UPED_V4_TCP_IDX),\n+\t\t \u0026hw_edit, sizeof(hw_edit));\n+\n+\t/ V6 TCP: l3_len = 1 /\n+\tnbl_hw_rd_regs(hw_mgt, NBL_UPED_HW_EDT_PROF_TABLE(NBL_UPED_V6_TCP_IDX),\n+\t\t \u0026hw_edit, sizeof(hw_edit));\n+\thw_edit = (hw_edit \u0026 ~NBL_PED_HW_EDIT_PROFILE_L3_LEN_MASK) \|\n+\t\t FIELD_PREP(NBL_PED_HW_EDIT_PROFILE_L3_LEN_MASK, 1);\n+\tnbl_hw_wr_regs(hw_mgt, NBL_UPED_HW_EDT_PROF_TABLE(NBL_UPED_V6_TCP_IDX),\n+\t\t \u0026hw_edit, sizeof(hw_edit));\n+\tspin_unlock(\u0026hw_mgt-\u003ereg_lock);\n+}\n+\n+static int nbl_shaping_eth_init(struct nbl_hw_mgt hw_mgt, u8 eth_id, u8 speed)\n+{\n+\tstruct nbl_shaping_dvn_dport_u dvn_dport = { 0 };\n+\tstruct nbl_shaping_dport_u dport = { 0 };\n+\tu32 rate, half_rate;\n+\tu32 depth;\n+\tu64 low_val, high_val;\n+\n+\tswitch (speed) {\n+\tcase NBL_FW_PORT_SPEED_100G:\n+\t\trate = 100000;\n+\t\tbreak;\n+\tcase NBL_FW_PORT_SPEED_50G:\n+\t\trate = 50000;\n+\t\tbreak;\n+\tcase NBL_FW_PORT_SPEED_25G:\n+\t\trate = 25000;\n+\t\tbreak;\n+\tcase NBL_FW_PORT_SPEED_10G:\n+\t\trate = 10000;\n+\t\tbreak;\n+\tdefault:\n+\t\tdev_err(hw_mgt-\u003ecommon-\u003edev,\n+\t\t\t\"Unsupported port speed %u for eth%u\\n\", speed, eth_id);\n+\t\treturn -EINVAL;\n+\t}\n+\n+\thalf_rate = rate / 2;\n+\tdepth = max_t(u32, rate * 2, NBL_LR_LEONIS_NET_BUCKET_DEPTH);\n+\n+\t/* 1. clear valid first\n+\t * dport and dvn_dport are zero-initialised above, so VALID=0 already\n+\t /\n+\tspin_lock(\u0026hw_mgt-\u003ereg_lock);\n+\tnbl_hw_wr_regs(hw_mgt, NBL_SHAPING_DPORT_REG(eth_id), dport.data,\n+\t\t sizeof(dport));\n+\tnbl_hw_wr_regs(hw_mgt, NBL_SHAPING_DVN_DPORT_REG(eth_id),\n+\t\t dvn_dport.data, sizeof(dvn_dport));\n+\n+\t/ 2. write config words (valid=0, safe) /\n+\tlow_val = FIELD_PREP(NBL_DPORT_CIR_MASK, rate) \|\n+\t\t FIELD_PREP(NBL_DPORT_PIR_MASK, rate) \|\n+\t\t FIELD_PREP(NBL_DPORT_DEPTH_MASK, depth) \|\n+\t\t FIELD_PREP(NBL_DPORT_CBS_MASK_LOW, depth \u0026 0x3F);\n+\thigh_val = FIELD_PREP(NBL_DPORT_CBS_MASK_HIGH, depth \u003e\u003e 6) \|\n+\t\t FIELD_PREP(NBL_DPORT_PBS_MASK, depth);\n+\t/ Fixed split, independent of host endian /\n+\tdport.data[0] = lower_32_bits(low_val);\n+\tdport.data[1] = upper_32_bits(low_val);\n+\tdport.data[2] = lower_32_bits(high_val);\n+\tdport.data[3] = upper_32_bits(high_val);\n+\n+\tlow_val = FIELD_PREP(NBL_DPORT_CIR_MASK, half_rate) \|\n+\t\t FIELD_PREP(NBL_DPORT_PIR_MASK, rate) \|\n+\t\t FIELD_PREP(NBL_DPORT_DEPTH_MASK, depth) \|\n+\t\t FIELD_PREP(NBL_DPORT_CBS_MASK_LOW, depth \u0026 0x3F);\n+\thigh_val = FIELD_PREP(NBL_DPORT_CBS_MASK_HIGH, depth \u003e\u003e 6) \|\n+\t\t FIELD_PREP(NBL_DPORT_PBS_MASK, depth);\n+\tdvn_dport.data[0] = lower_32_bits(low_val);\n+\tdvn_dport.data[1] = upper_32_bits(low_val);\n+\tdvn_dport.data[2] = lower_32_bits(high_val);\n+\tdvn_dport.data[3] = upper_32_bits(high_val);\n+\n+\tnbl_hw_wr_regs(hw_mgt, NBL_SHAPING_DPORT_REG(eth_id), dport.data,\n+\t\t sizeof(dport));\n+\tnbl_hw_wr_regs(hw_mgt, NBL_SHAPING_DVN_DPORT_REG(eth_id),\n+\t\t dvn_dport.data, sizeof(dvn_dport));\n+\n+\t/ 3. commit: set valid last /\n+\tlow_val = FIELD_PREP(NBL_DPORT_VALID_MASK, 1);\n+\tdport.data[0] \|= lower_32_bits(low_val);\n+\n+\tlow_val = FIELD_PREP(NBL_DPORT_VALID_MASK, 1);\n+\tdvn_dport.data[0] \|= lower_32_bits(low_val);\n+\n+\tnbl_hw_wr_regs(hw_mgt, NBL_SHAPING_DPORT_REG(eth_id), dport.data,\n+\t\t sizeof(dport));\n+\tnbl_hw_wr_regs(hw_mgt, NBL_SHAPING_DVN_DPORT_REG(eth_id),\n+\t\t dvn_dport.data, sizeof(dvn_dport));\n+\tspin_unlock(\u0026hw_mgt-\u003ereg_lock);\n+\treturn 0;\n+}\n+\n+static int nbl_shaping_init(struct nbl_hw_mgt hw_mgt, u8 speed)\n+{\n+\tstruct nbl_shaping_net_u net_shaping = { 0 };\n+\tu32 eth_bitmap = 0;\n+\tu32 reg_val;\n+\tint ret;\n+\tint i;\n+\n+\tnbl_hw_get_fw_eth_map(hw_mgt, \u0026eth_bitmap);\n+\tfor (i = 0; i \u003c NBL_MAX_ETHERNET; i++) {\n+\t\tif (!(eth_bitmap \u0026 BIT(i)))\n+\t\t\tcontinue;\n+\t\tret = nbl_shaping_eth_init(hw_mgt, i, speed);\n+\t\tif (ret)\n+\t\t\treturn ret;\n+\t}\n+\t/\n+\t Read-modify-write as one critical section: reg_lock is dropped\n+\t * between the two _lock() helpers, which would let another writer\n+\t * of this register land in the middle of the update. Nothing\n+\t * else touches NBL_DSCH_PSHA_EN_ADDR today, but the RMW has to be\n+\t * atomic for that to stay true.\n+\t /\n+\tspin_lock(\u0026hw_mgt-\u003ereg_lock);\n+\tnbl_hw_rd_regs(hw_mgt, NBL_DSCH_PSHA_EN_ADDR, \u0026reg_val,\n+\t\t sizeof(reg_val));\n+\treg_val \u0026= ~NBL_DSCH_PSHA_EN_MASK;\n+\treg_val \|= FIELD_PREP(NBL_DSCH_PSHA_EN_MASK,\n+\t\t\t eth_bitmap \u0026 GENMASK(3, 0));\n+\tnbl_hw_wr_regs(hw_mgt, NBL_DSCH_PSHA_EN_ADDR, \u0026reg_val,\n+\t\t sizeof(reg_val));\n+\tspin_unlock(\u0026hw_mgt-\u003ereg_lock);\n+\n+\tfor (i = 0; i \u003c NBL_MAX_PF; i++) {\n+\t\tnbl_hw_wr_regs_lock(hw_mgt, NBL_SHAPING_NET_REG(i),\n+\t\t\t\t net_shaping.data,\n+\t\t\t\t sizeof(net_shaping));\n+\t}\n+\tnbl_flush_writes(hw_mgt);\n+\treturn 0;\n+}\n+\n+static void nbl_dsch_qid_max_init(struct nbl_hw_mgt hw_mgt)\n+{\n+\tu32 quanta = 0;\n+\tu32 qid_max = 0;\n+\n+\tquanta = FIELD_PREP(NBL_DSCH_VN_QUANTA_H_QUA_MASK, NBL_HOST_QUANTA) \|\n+\t\t FIELD_PREP(NBL_DSCH_VN_QUANTA_E_QUA_MASK, NBL_ECPU_QUANTA);\n+\tspin_lock(\u0026hw_mgt-\u003ereg_lock);\n+\tnbl_hw_wr_regs(hw_mgt, NBL_DSCH_VN_QUANTA_ADDR, \u0026quanta,\n+\t\t sizeof(quanta));\n+\tnbl_hw_rd_regs(hw_mgt, NBL_DSCH_HOST_QID_MAX, \u0026qid_max,\n+\t\t sizeof(qid_max));\n+\tqid_max \u0026= ~NBL_DSCH_HOST_QID_MAX_MASK;\n+\tqid_max \|= FIELD_PREP(NBL_DSCH_HOST_QID_MAX_MASK, NBL_MAX_QUEUE_ID);\n+\tnbl_hw_wr_regs(hw_mgt, NBL_DSCH_HOST_QID_MAX, \u0026qid_max,\n+\t\t sizeof(qid_max));\n+\tspin_unlock(\u0026hw_mgt-\u003ereg_lock);\n+}\n+\n+static int nbl_ustore_init(struct nbl_hw_mgt hw_mgt, u8 eth_num)\n+{\n+\tu32 eth_bitmap = 0;\n+\tu32 drop_th = 0;\n+\tu32 pkt_len = 0;\n+\tu32 reg_val = 0;\n+\tint i;\n+\n+\t/\n+\t * eth_num is validated in the resource layer:\n+\t * nbl_res_init_pf_num() requires 1/2/4 PFs, and\n+\t * nbl_res_ctrl_dev_setup_eth_info() requires max_pf == eth_num.\n+\t * This is a defensive check only; if it triggers, the resource\n+\t * layer validation was bypassed, which is a bug.\n+\t /\n+\tif (WARN_ON(eth_num != 1 \u0026\u0026 eth_num != 2 \u0026\u0026 eth_num != 4))\n+\t\treturn -EINVAL;\n+\t/ Read current packet length config\n+\t (to preserve other fields while updating 'min')\n+\t /\n+\tspin_lock(\u0026hw_mgt-\u003ereg_lock);\n+\tnbl_hw_rd_regs(hw_mgt, NBL_USTORE_PKT_LEN_ADDR, \u0026pkt_len,\n+\t\t sizeof(pkt_len));\n+\t/* min arp packet length 42 (14 + 28) /\n+\tpkt_len \u0026= ~NBL_USTORE_PKT_LEN_MIN_MASK;\n+\tpkt_len \|= FIELD_PREP(NBL_USTORE_PKT_LEN_MIN_MASK, 42);\n+\tnbl_hw_wr_regs(hw_mgt, NBL_USTORE_PKT_LEN_ADDR, \u0026pkt_len,\n+\t\t sizeof(pkt_len));\n+\n+\tdrop_th \|= FIELD_PREP(NBL_USTORE_PORT_DROP_TH_EN_MASK, 1);\n+\tif (eth_num == 1)\n+\t\tdrop_th \|= FIELD_PREP(NBL_USTORE_PORT_DROP_TH_DISC_TH_MASK,\n+\t\t\t\t NBL_USTORE_SINGLE_ETH_DROP_TH);\n+\telse if (eth_num == 2)\n+\t\tdrop_th \|= FIELD_PREP(NBL_USTORE_PORT_DROP_TH_DISC_TH_MASK,\n+\t\t\t\t NBL_USTORE_DUAL_ETH_DROP_TH);\n+\telse\n+\t\tdrop_th \|= FIELD_PREP(NBL_USTORE_PORT_DROP_TH_DISC_TH_MASK,\n+\t\t\t\t NBL_USTORE_QUAD_ETH_DROP_TH);\n+\tnbl_hw_get_fw_eth_map(hw_mgt, \u0026eth_bitmap);\n+\tfor (i = 0; i \u003c NBL_MAX_ETHERNET; i++) {\n+\t\tif (!(eth_bitmap \u0026 BIT(i)))\n+\t\t\tcontinue;\n+\t\tnbl_hw_rd_regs(hw_mgt, NBL_USTORE_PORT_DROP_TH_REG_ARR(i),\n+\t\t\t \u0026reg_val, sizeof(reg_val));\n+\t\treg_val \u0026= ~(NBL_USTORE_PORT_DROP_TH_EN_MASK \|\n+\t\t\t NBL_USTORE_PORT_DROP_TH_DISC_TH_MASK);\n+\t\treg_val \|= drop_th;\n+\t\tnbl_hw_wr_regs(hw_mgt, NBL_USTORE_PORT_DROP_TH_REG_ARR(i),\n+\t\t\t \u0026reg_val, sizeof(reg_val));\n+\t}\n+\n+\t/ Clear port drop/truncate counters by reading them\n+\t * (hardware has read-to-clear behavior for these registers)\n+\t /\n+\tfor (i = 0; i \u003c NBL_MAX_ETHERNET; i++) {\n+\t\tif (!(eth_bitmap \u0026 BIT(i)))\n+\t\t\tcontinue;\n+\t\tnbl_hw_rd32(hw_mgt, NBL_USTORE_BUF_PORT_DROP_PKT(i));\n+\t\tnbl_hw_rd32(hw_mgt, NBL_USTORE_BUF_PORT_TRUN_PKT(i));\n+\t}\n+\tspin_unlock(\u0026hw_mgt-\u003ereg_lock);\n+\treturn 0;\n+}\n+\n+static void nbl_dstore_init(struct nbl_hw_mgt hw_mgt, u8 speed)\n+{\n+\tu32 eth_bitmap = 0;\n+\tu32 drop_th = 0;\n+\tu32 fc_th = 0;\n+\tu32 bp_th = 0;\n+\tint i;\n+\n+\tfor (i = 0; i \u003c NBL_DSTORE_PORT_DROP_TH_DEPTH; i++) {\n+\t\tspin_lock(\u0026hw_mgt-\u003ereg_lock);\n+\t\tnbl_hw_rd_regs(hw_mgt, NBL_DSTORE_PORT_DROP_TH_REG(i), \u0026drop_th,\n+\t\t\t sizeof(drop_th));\n+\t\tdrop_th \u0026= ~NBL_DSTORE_PORT_DROP_EN_MASK;\n+\t\tnbl_hw_wr_regs(hw_mgt, NBL_DSTORE_PORT_DROP_TH_REG(i), \u0026drop_th,\n+\t\t\t sizeof(drop_th));\n+\t\tspin_unlock(\u0026hw_mgt-\u003ereg_lock);\n+\t}\n+\n+\tspin_lock(\u0026hw_mgt-\u003ereg_lock);\n+\tnbl_hw_rd_regs(hw_mgt, NBL_DSTORE_DISC_BP_TH, \u0026bp_th, sizeof(bp_th));\n+\tbp_th \|= FIELD_PREP(NBL_DSTORE_DISC_BP_TH_EN_MASK, 1);\n+\tnbl_hw_wr_regs(hw_mgt, NBL_DSTORE_DISC_BP_TH, \u0026bp_th, sizeof(bp_th));\n+\tspin_unlock(\u0026hw_mgt-\u003ereg_lock);\n+\n+\tnbl_hw_get_fw_eth_map(hw_mgt, \u0026eth_bitmap);\n+\tfor (i = 0; i \u003c NBL_MAX_ETHERNET; i++) {\n+\t\tif (!(eth_bitmap \u0026 BIT(i)))\n+\t\t\tcontinue;\n+\t\tspin_lock(\u0026hw_mgt-\u003ereg_lock);\n+\t\tnbl_hw_rd_regs(hw_mgt, NBL_DSTORE_D_DPORT_FC_TH_REG(i), \u0026fc_th,\n+\t\t\t sizeof(fc_th));\n+\t\tfc_th \u0026= ~(NBL_DSTORE_D_DPORT_FC_XOFF_TH_MASK \|\n+\t\t\t NBL_DSTORE_D_DPORT_FC_XON_TH_MASK);\n+\t\tif (speed == NBL_FW_PORT_SPEED_100G) {\n+\t\t\tfc_th \|=\n+\t\t\t\tFIELD_PREP(NBL_DSTORE_D_DPORT_FC_XOFF_TH_MASK,\n+\t\t\t\t\t NBL_DSTORE_DROP_XOFF_TH_100G) \|\n+\t\t\t\tFIELD_PREP(NBL_DSTORE_D_DPORT_FC_XON_TH_MASK,\n+\t\t\t\t\t NBL_DSTORE_DROP_XON_TH_100G);\n+\t\t} else {\n+\t\t\tfc_th \|=\n+\t\t\t\tFIELD_PREP(NBL_DSTORE_D_DPORT_FC_XOFF_TH_MASK,\n+\t\t\t\t\t NBL_DSTORE_DROP_XOFF_TH) \|\n+\t\t\t\tFIELD_PREP(NBL_DSTORE_D_DPORT_FC_XON_TH_MASK,\n+\t\t\t\t\t NBL_DSTORE_DROP_XON_TH);\n+\t\t}\n+\n+\t\tfc_th \|= FIELD_PREP(NBL_DSTORE_D_DPORT_FC_FC_EN_MASK, 1);\n+\t\tnbl_hw_wr_regs(hw_mgt, NBL_DSTORE_D_DPORT_FC_TH_REG(i), \u0026fc_th,\n+\t\t\t sizeof(fc_th));\n+\t\tspin_unlock(\u0026hw_mgt-\u003ereg_lock);\n+\t}\n+}\n+\n+static void nbl_dvn_descreq_num_cfg(struct nbl_hw_mgt hw_mgt, u8 descreq_num)\n+{\n+\tu8 split_ring_num = (descreq_num \u003e\u003e 3) \u0026 0x1;\n+\tu8 ring_num = descreq_num \u0026 0x7;\n+\tu32 num_cfg;\n+\tu32 reg_val;\n+\n+\tspin_lock(\u0026hw_mgt-\u003ereg_lock);\n+\tnbl_hw_rd_regs(hw_mgt, NBL_DVN_DESCREQ_NUM_CFG, \u0026reg_val,\n+\t\t sizeof(reg_val));\n+\n+\tnum_cfg = FIELD_PREP(NBL_DVN_DESCREQ_NUM_CFG_AVRING_DESREQ_NUM_CFG_MASK,\n+\t\t\t split_ring_num) \|\n+\t\t FIELD_PREP(NBL_DVN_DESCREQ_NUM_CFG_PACKED_L1_NUM_MASK,\n+\t\t\t ring_num);\n+\treg_val \u0026= ~(NBL_DVN_DESCREQ_NUM_CFG_AVRING_DESREQ_NUM_CFG_MASK \|\n+\t\tNBL_DVN_DESCREQ_NUM_CFG_PACKED_L1_NUM_MASK);\n+\treg_val \|= num_cfg;\n+\tnbl_hw_wr_regs(hw_mgt, NBL_DVN_DESCREQ_NUM_CFG, \u0026reg_val,\n+\t\t sizeof(reg_val));\n+\tspin_unlock(\u0026hw_mgt-\u003ereg_lock);\n+}\n+\n+static void nbl_dvn_init(struct nbl_hw_mgt hw_mgt, u8 speed)\n+{\n+\tu32 timeout = 0;\n+\tu32 ro_flag = 0;\n+\n+\tspin_lock(\u0026hw_mgt-\u003ereg_lock);\n+\tnbl_hw_wr32(hw_mgt, NBL_DVN_ECPU_QUEUE_NUM, 0);\n+\tspin_unlock(\u0026hw_mgt-\u003ereg_lock);\n+\n+\ttimeout = FIELD_PREP(NBL_DVN_DESC_WR_MERGE_TIMEOUT_CFG_CYCLE_MASK,\n+\t\t\t DEFAULT_DVN_DESC_WR_MERGE_TIMEOUT_MAX);\n+\tnbl_hw_wr_regs_lock(hw_mgt, NBL_DVN_DESC_WR_MERGE_TIMEOUT, \u0026timeout,\n+\t\t\t sizeof(timeout));\n+\tspin_lock(\u0026hw_mgt-\u003ereg_lock);\n+\tnbl_hw_rd_regs(hw_mgt, NBL_DVN_DIF_REQ_RD_RO_FLAG, \u0026ro_flag,\n+\t\t sizeof(ro_flag));\n+\tif (pcie_relaxed_ordering_enabled(hw_mgt-\u003ecommon-\u003epdev)) {\n+\t\tro_flag \|=\n+\t\t\tFIELD_PREP(NBL_DVN_DIF_REQ_RD_RO_FLAG_DESC_RO_EN_MASK,\n+\t\t\t\t 1) \|\n+\t\t\tFIELD_PREP(NBL_DVN_DIF_REQ_RD_RO_FLAG_DATA_RO_EN_MASK,\n+\t\t\t\t 1) \|\n+\t\t\tFIELD_PREP(NBL_DVN_DIF_REQ_RD_RO_FLAG_AVRING_RO_EN_MASK,\n+\t\t\t\t 1);\n+\t} else {\n+\t\tro_flag \u0026=\n+\t\t\t~(FIELD_PREP(NBL_DVN_DIF_REQ_RD_RO_FLAG_DESC_RO_EN_MASK,\n+\t\t\t\t 1) \|\n+\t\t\tFIELD_PREP(NBL_DVN_DIF_REQ_RD_RO_FLAG_DATA_RO_EN_MASK,\n+\t\t\t\t 1) \|\n+\t\t\tFIELD_PREP(NBL_DVN_DIF_REQ_RD_RO_FLAG_AVRING_RO_EN_MASK,\n+\t\t\t\t 1));\n+\t}\n+\tnbl_hw_wr_regs(hw_mgt, NBL_DVN_DIF_REQ_RD_RO_FLAG, \u0026ro_flag,\n+\t\t sizeof(ro_flag));\n+\tspin_unlock(\u0026hw_mgt-\u003ereg_lock);\n+\tif (speed == NBL_FW_PORT_SPEED_100G)\n+\t\tnbl_dvn_descreq_num_cfg(hw_mgt,\n+\t\t\t\t\tDEFAULT_DVN_100G_DESCREQ_NUMCFG);\n+\telse\n+\t\tnbl_dvn_descreq_num_cfg(hw_mgt, DEFAULT_DVN_DESCREQ_NUMCFG);\n+}\n+\n+static void nbl_uvn_init(struct nbl_hw_mgt hw_mgt)\n+{\n+\tu16 wr_timeout = NBL_UVN_DESC_WR_TIMEOUT_VAL;\n+\tu32 timeout = NBL_UVN_DESC_RD_WAIT_TICKS;\n+\tu32 prefetch_init = 0;\n+\tbool ro_enabled;\n+\tu32 flag = 0;\n+\tu32 mask = 0;\n+\tu32 quirks;\n+\tu32 reg_val;\n+\n+\tspin_lock(\u0026hw_mgt-\u003ereg_lock);\n+\tnbl_hw_wr32(hw_mgt, NBL_UVN_ECPU_QUEUE_NUM, 0);\n+\tnbl_hw_wr32(hw_mgt, NBL_UVN_DESC_RD_WAIT, timeout);\n+\tnbl_hw_rd_regs(hw_mgt, NBL_UVN_DESC_WR_TIMEOUT,\n+\t\t \u0026reg_val, sizeof(reg_val));\n+\treg_val \u0026= ~NBL_UVN_DESC_WR_TIMEOUT_NUM_MASK;\n+\treg_val \|= FIELD_PREP(NBL_UVN_DESC_WR_TIMEOUT_NUM_MASK, wr_timeout);\n+\tnbl_hw_wr_regs(hw_mgt, NBL_UVN_DESC_WR_TIMEOUT, \u0026reg_val,\n+\t\t sizeof(reg_val));\n+\tro_enabled = pcie_relaxed_ordering_enabled(hw_mgt-\u003ecommon-\u003epdev);\n+\n+\tnbl_hw_rd_regs(hw_mgt, NBL_UVN_DIF_REQ_RO_FLAG, \u0026flag, sizeof(flag));\n+\tif (ro_enabled) {\n+\t\tflag \|= FIELD_PREP(NBL_UVN_DIF_REQ_RO_FLAG_AVAIL_RD_MASK, 1) \|\n+\t\tFIELD_PREP(NBL_UVN_DIF_REQ_RO_FLAG_DESC_RD_MASK, 1) \|\n+\t\tFIELD_PREP(NBL_UVN_DIF_REQ_RO_FLAG_PKT_WR_MASK, 1);\n+\t\tflag \u0026= ~FIELD_PREP(NBL_UVN_DIF_REQ_RO_FLAG_DESC_WR_MASK, 1);\n+\t} else {\n+\t\tflag \u0026= ~(FIELD_PREP(NBL_UVN_DIF_REQ_RO_FLAG_AVAIL_RD_MASK, 1) \|\n+\t\t\t FIELD_PREP(NBL_UVN_DIF_REQ_RO_FLAG_DESC_RD_MASK, 1) \|\n+\t\t\t FIELD_PREP(NBL_UVN_DIF_REQ_RO_FLAG_PKT_WR_MASK, 1) \|\n+\t\t\t FIELD_PREP(NBL_UVN_DIF_REQ_RO_FLAG_DESC_WR_MASK, 1));\n+\t}\n+\tnbl_hw_wr_regs(hw_mgt, NBL_UVN_DIF_REQ_RO_FLAG, \u0026flag, sizeof(flag));\n+\n+\tnbl_hw_rd_regs(hw_mgt, NBL_UVN_QUEUE_ERR_MASK, \u0026mask, sizeof(mask));\n+\tmask \|= FIELD_PREP(NBL_UVN_QUEUE_ERR_MASK_DIF_ERR_MASK, 1);\n+\n+\tnbl_hw_wr_regs(hw_mgt, NBL_UVN_QUEUE_ERR_MASK, \u0026mask, sizeof(mask));\n+\n+\tspin_unlock(\u0026hw_mgt-\u003ereg_lock);\n+\tquirks = nbl_hw_get_quirks(hw_mgt);\n+\t/\n+\t * sel=0: use configured num; sel=1: use internal calc (max 32)\n+\t * Default is sel=1, unless NBL_QUIRK_UVN_PREFETCH_ALIGN is set,\n+\t * in which case override to sel=0.\n+\t /\n+\tspin_lock(\u0026hw_mgt-\u003ereg_lock);\n+\tnbl_hw_rd_regs(hw_mgt, NBL_UVN_DESC_PREFETCH_INIT, \u0026reg_val,\n+\t\t sizeof(reg_val));\n+\tprefetch_init =\n+\t\tFIELD_PREP(NBL_UVN_DESC_PREFETCH_INIT_NUM_MASK,\n+\t\t\t NBL_UVN_DESC_PREFETCH_NUM) \|\n+\t\tFIELD_PREP(NBL_UVN_DESC_PREFETCH_INIT_SEL_MASK,\n+\t\t\t (quirks \u0026 NBL_QUIRK_UVN_PREFETCH_ALIGN) ? 0 : 1);\n+\treg_val \u0026= ~(NBL_UVN_DESC_PREFETCH_INIT_NUM_MASK \|\n+\t\tNBL_UVN_DESC_PREFETCH_INIT_SEL_MASK);\n+\treg_val \|= prefetch_init;\n+\tnbl_hw_wr_regs(hw_mgt, NBL_UVN_DESC_PREFETCH_INIT, \u0026reg_val,\n+\t\t sizeof(reg_val));\n+\tspin_unlock(\u0026hw_mgt-\u003ereg_lock);\n+}\n+\n+static void nbl_uqm_init(struct nbl_hw_mgt hw_mgt)\n+{\n+\tu32 que_type = 0;\n+\tu32 cnt = 0;\n+\tint i;\n+\n+\tspin_lock(\u0026hw_mgt-\u003ereg_lock);\n+\tnbl_hw_wr_regs(hw_mgt, NBL_UQM_FWD_DROP_CNT, \u0026cnt, sizeof(cnt));\n+\n+\tnbl_hw_wr_regs(hw_mgt, NBL_UQM_DROP_PKT_CNT, \u0026cnt, sizeof(cnt));\n+\tnbl_hw_wr_regs(hw_mgt, NBL_UQM_DROP_PKT_SLICE_CNT, \u0026cnt, sizeof(cnt));\n+\tnbl_hw_wr_regs(hw_mgt, NBL_UQM_DROP_PKT_LEN_ADD_CNT, \u0026cnt, sizeof(cnt));\n+\tnbl_hw_wr_regs(hw_mgt, NBL_UQM_DROP_HEAD_PNTR_ADD_CNT, \u0026cnt,\n+\t\t sizeof(cnt));\n+\tnbl_hw_wr_regs(hw_mgt, NBL_UQM_DROP_WEIGHT_ADD_CNT, \u0026cnt, sizeof(cnt));\n+\n+\tfor (i = 0; i \u003c NBL_UQM_PORT_DROP_DEPTH; i++) {\n+\t\tnbl_hw_wr_regs(hw_mgt,\n+\t\t\t NBL_UQM_PORT_DROP_PKT_CNT + (sizeof(cnt) * i),\n+\t\t\t \u0026cnt, sizeof(cnt));\n+\t\tnbl_hw_wr_regs(hw_mgt,\n+\t\t\t NBL_UQM_PORT_DROP_PKT_SLICE_CNT +\n+\t\t\t\t (sizeof(cnt) * i),\n+\t\t\t \u0026cnt, sizeof(cnt));\n+\t\tnbl_hw_wr_regs(hw_mgt,\n+\t\t\t NBL_UQM_PORT_DROP_PKT_LEN_ADD_CNT +\n+\t\t\t\t (sizeof(cnt) * i),\n+\t\t\t \u0026cnt, sizeof(cnt));\n+\t\tnbl_hw_wr_regs(hw_mgt,\n+\t\t\t NBL_UQM_PORT_DROP_HEAD_PNTR_ADD_CNT +\n+\t\t\t\t (sizeof(cnt) * i),\n+\t\t\t \u0026cnt, sizeof(cnt));\n+\t\tnbl_hw_wr_regs(hw_mgt,\n+\t\t\t NBL_UQM_PORT_DROP_WEIGHT_ADD_CNT +\n+\t\t\t\t (sizeof(cnt) * i),\n+\t\t\t \u0026cnt, sizeof(cnt));\n+\t}\n+\n+\tfor (i = 0; i \u003c NBL_UQM_DPORT_DROP_DEPTH; i++)\n+\t\tnbl_hw_wr_regs(hw_mgt,\n+\t\t\t NBL_UQM_DPORT_DROP_CNT + (sizeof(cnt) * i), \u0026cnt,\n+\t\t\t sizeof(cnt));\n+\t/* bit0: 0=bp mode, 1=drop mode, resv bit1-31 /\n+\tnbl_hw_wr_regs(hw_mgt, NBL_UQM_QUE_TYPE, \u0026que_type, sizeof(que_type));\n+\tspin_unlock(\u0026hw_mgt-\u003ereg_lock);\n+}\n+\n+static int nbl_dp_init(struct nbl_hw_mgt hw_mgt, u8 speed, u8 eth_num)\n+{\n+\tint ret;\n+\n+\tnbl_dped_init(hw_mgt);\n+\tnbl_uped_init(hw_mgt);\n+\tret = nbl_shaping_init(hw_mgt, speed);\n+\tif (ret)\n+\t\treturn ret;\n+\tnbl_dsch_qid_max_init(hw_mgt);\n+\tret = nbl_ustore_init(hw_mgt, eth_num);\n+\tif (ret)\n+\t\treturn ret;\n+\tnbl_dstore_init(hw_mgt, speed);\n+\tnbl_dvn_init(hw_mgt, speed);\n+\tnbl_uvn_init(hw_mgt);\n+\tnbl_uqm_init(hw_mgt);\n+\treturn 0;\n+}\n+\n+static void nbl_host_padpt_init(struct nbl_hw_mgt hw_mgt)\n+{\n+\t/ padpt flow control register /\n+\tspin_lock(\u0026hw_mgt-\u003ereg_lock);\n+\tnbl_hw_wr32(hw_mgt, NBL_HOST_PADPT_HOST_CFG_FC_CPLH_UP,\n+\t\t NBL_HOST_PADPT_CFG_FC_CPLH_UP_VAL);\n+\tnbl_hw_wr32(hw_mgt, NBL_HOST_PADPT_HOST_CFG_FC_PD_DN,\n+\t\t NBL_HOST_PADPT_CFG_FC_PD_DN_VAL);\n+\tnbl_hw_wr32(hw_mgt, NBL_HOST_PADPT_HOST_CFG_FC_PH_DN,\n+\t\t NBL_HOST_PADPT_CFG_FC_PH_DN_VAL);\n+\tnbl_hw_wr32(hw_mgt, NBL_HOST_PADPT_HOST_CFG_FC_NPH_DN,\n+\t\t NBL_HOST_PADPT_CFG_FC_NPH_DN_VAL);\n+\tspin_unlock(\u0026hw_mgt-\u003ereg_lock);\n+}\n+\n+static void nbl_intf_init(struct nbl_hw_mgt hw_mgt)\n+{\n+\tnbl_host_padpt_init(hw_mgt);\n+}\n+\n+/\n+ Read-modify-write under reg_lock: NBL_DRIVER_STATUS_BIT is the only\n+ * bit the driver owns, and the RMW writes back every other bit exactly\n+ * as it read it, so firmware state in those bits survives.\n+ \n+ That only holds against concurrent CPU writers. reg_lock cannot\n+ * serialise against firmware, and the register has no documented\n+ * W1S/W1C form that would let the driver flip bit 16 without the\n+ * read-modify-write; see NBL_DRIVER_STATUS_REG.\n+ /\n+static void nbl_hw_set_driver_status(struct nbl_hw_mgt hw_mgt, bool active)\n+{\n+\tu32 status;\n+\n+\tspin_lock(\u0026hw_mgt-\u003ereg_lock);\n+\tstatus = nbl_hw_rd32(hw_mgt, NBL_DRIVER_STATUS_REG);\n+\n+\tstatus \u0026= ~BIT(NBL_DRIVER_STATUS_BIT);\n+\tstatus \|= FIELD_PREP(BIT(NBL_DRIVER_STATUS_BIT), active);\n+\n+\tnbl_hw_wr32(hw_mgt, NBL_DRIVER_STATUS_REG, status);\n+\tspin_unlock(\u0026hw_mgt-\u003ereg_lock);\n+}\n+\n+/\n+ Setting driver status to false notifies firmware to clean up per-PF\n+ * hardware state such as qinfo registers.\n+ \n+ Note: firmware does NOT automatically revert chip-wide registers\n+ * configured in this init flow. Those chip-wide settings remain valid\n+ * until chip reset or explicitly overwritten by driver.\n+ \n+ This deinit_module only clears driver active status and flush writes.\n+ * It does NOT reset or restore chip-wide datapath registers.\n+ \n+ Firmware cleanup is asynchronous with no completion status register.\n+ * The flush only guarantees the status write reached the device; the\n+ * bounded delay after it (NBL_FW_CLEANUP_SYNC_MIN_US) is best-effort and\n+ * does not guarantee that the firmware pass has finished. The ordering\n+ * this relies on is the device-link one: the siblings are consumers of\n+ * the control PF, so they are unbound first, and that link is created\n+ * with DL_FLAG_AUTOPROBE_CONSUMER. AUTOPROBE only re-probes consumers,\n+ * never the control PF itself, so a later init_module() needs an admin\n+ * (or a driver rebind) to bind this PF again, and only then does the\n+ * core re-probe the siblings. Both steps are far slower than the\n+ * firmware pass, but there is no hard firmware handshake.\n+ \n+ Caller must ensure no new DMA is initiated after this point.\n+ * The mailbox channel is stopped by nbl_chan_teardown_queue()\n+ * before this function is called, so no in-flight mailbox DMA\n+ * remains.\n+ /\n+static void nbl_hw_deinit_module(struct nbl_hw_mgt hw_mgt)\n+{\n+\tnbl_hw_set_driver_status(hw_mgt, false);\n+\t/* ensure driver_status reaches the chip /\n+\tnbl_flush_writes(hw_mgt);\n+\t/\n+\t * Give the asynchronous firmware cleanup pass time to settle\n+\t * before returning; best-effort, see NBL_FW_CLEANUP_SYNC_MIN_US.\n+\t /\n+\tusleep_range(NBL_FW_CLEANUP_SYNC_MIN_US, NBL_FW_CLEANUP_SYNC_MAX_US);\n+}\n+\n+static bool nbl_hw_eth_speed_valid(u8 speed)\n+{\n+\tswitch (speed) {\n+\tcase NBL_FW_PORT_SPEED_10G:\n+\tcase NBL_FW_PORT_SPEED_25G:\n+\tcase NBL_FW_PORT_SPEED_50G:\n+\tcase NBL_FW_PORT_SPEED_100G:\n+\t\treturn true;\n+\tdefault:\n+\t\treturn false;\n+\t}\n+}\n+\n+static bool nbl_hw_eth_num_valid(u8 eth_num)\n+{\n+\treturn eth_num == 1 \|\| eth_num == 2 \|\| eth_num == 4;\n+}\n+\n+/\n+ * Clear a stale driver_status active bit before this probe programs any\n+ * per-PF state.\n+ \n+ A previous kernel instance can leave the bit set (kexec, or an unload\n+ * that never reached deinit_module()) and firmware then never ran the\n+ * per-PF cleanup it drives off the 1 -\u003e 0 transition. That cleanup\n+ * reclaims the mailbox QINFO routing and the qinfo registers, which is\n+ * exactly what nbl_dev_setup_chan_qinfo() and nbl_chan_setup_queue()\n+ * write later in this probe.\n+ \n+ So the ordering this has to guarantee is: clear the bit before any of\n+ * that per-PF state is programmed, and before the mailbox queue is live\n+ * so the cleanup cannot reclaim routing under an in-flight transfer.\n+ * Clearing it later - once nbl_dev_init() has programmed the routing and\n+ * enabled the queue - wipes those registers and leaves the inter-PF\n+ * mailbox dead, so sibling PFs' RPCs time out in nbl_dev_start() and\n+ * their probes fail. That is why init_module() must not do it.\n+ \n+ Only the control PF owns this chip-wide handshake. The bounded wait\n+ * after the clear is best-effort, see NBL_FW_CLEANUP_SYNC_MIN_US.\n+ /\n+static void nbl_hw_clear_stale_status(struct nbl_hw_mgt hw_mgt)\n+{\n+\tu32 status;\n+\n+\tif (!hw_mgt-\u003ecommon-\u003ehas_ctrl)\n+\t\treturn;\n+\n+\t/\n+\t Unlocked read: this runs only from the serialized control-PF probe,\n+\t * and the driver is the only writer of NBL_DRIVER_STATUS_BIT. A stale\n+\t * sample at worst causes one redundant clear plus sleep; the\n+\t * authoritative update happens under reg_lock in\n+\t * nbl_hw_set_driver_status().\n+\t /\n+\tstatus = nbl_hw_rd32(hw_mgt, NBL_DRIVER_STATUS_REG);\n+\tif (!(status \u0026 BIT(NBL_DRIVER_STATUS_BIT)))\n+\t\treturn;\n+\n+\tdev_warn(hw_mgt-\u003ecommon-\u003edev,\n+\t\t \"driver_status already set, stale cleanup forced\\n\");\n+\tnbl_hw_set_driver_status(hw_mgt, false);\n+\tnbl_flush_writes(hw_mgt);\n+\tusleep_range(NBL_FW_CLEANUP_SYNC_MIN_US, NBL_FW_CLEANUP_SYNC_MAX_US);\n+}\n+\n+/\n+ * Firmware brings the chip up at power-up, but the chip-wide datapath\n+ * registers this programs (SHAPING, DSCH, USTORE/DSTORE, DVN/UVN, UQM,\n+ * PADPT) are not part of that: the driver owns them from here until chip\n+ * reset, and firmware does not revert them when driver_status goes low -\n+ * see nbl_hw_deinit_module().\n+ \n+ The driver_status handshake is entered from a known state: a stale\n+ * active bit is cleared earlier, in nbl_hw_init_leonis(), before this\n+ * probe programs any per-PF state. It must not be cleared here, because\n+ * the mailbox QINFO routing and the mailbox queue registers are already\n+ * programmed by then and the cleanup the 1 -\u003e 0 transition starts would\n+ * reclaim them; see nbl_hw_clear_stale_status().\n+ /\n+static int nbl_hw_init_module(struct nbl_hw_mgt hw_mgt, u8 eth_speed,\n+\t\t\t u8 eth_num)\n+{\n+\tint ret;\n+\n+\tif (!nbl_hw_eth_speed_valid(eth_speed)) {\n+\t\tdev_err(hw_mgt-\u003ecommon-\u003edev, \"Invalid eth_speed %u\\n\",\n+\t\t\teth_speed);\n+\t\treturn -EINVAL;\n+\t}\n+\tif (!nbl_hw_eth_num_valid(eth_num)) {\n+\t\tdev_err(hw_mgt-\u003ecommon-\u003edev, \"Invalid eth_num %u\\n\", eth_num);\n+\t\treturn -EINVAL;\n+\t}\n+\n+\tret = nbl_dp_init(hw_mgt, eth_speed, eth_num);\n+\tif (ret)\n+\t\treturn ret;\n+\tnbl_intf_init(hw_mgt);\n+\tnbl_hw_set_driver_status(hw_mgt, true);\n+\t/* ensure registers written /\n+\tnbl_flush_writes(hw_mgt);\n+\n+\treturn 0;\n+}\n+\n+/\n+ * nbl_hw_set_mailbox_irq - read-modify-write NBL_MAILBOX_QINFO_MAP_REG_ARR\n+ \n+ The full RMW sequence is wrapped by reg_lock, so concurrent register\n+ * access from different CPUs is already serialized safely.\n+ * nbl_hw_cfg_mailbox_qinfo() programs the BDF fields during control-PF\n+ * init and clears MSIX_IDX/MSIX_IDX_VALID at the same time (they survive\n+ * kexec/forced unload without FLR), so mailbox MSIX routing for a PF\n+ * starts disarmed at init and is armed only by an explicit en_msix=true\n+ * call here.\n+ /\n+static void nbl_hw_set_mailbox_irq(struct nbl_hw_mgt hw_mgt, u16 func_id,\n+\t\t\t\t bool en_msix, u16 gvec)\n+{\n+\tu32 data = 0;\n+\n+\tspin_lock(\u0026hw_mgt-\u003ereg_lock);\n+\tnbl_hw_rd_regs(hw_mgt, NBL_MAILBOX_QINFO_MAP_REG_ARR(func_id), \u0026data,\n+\t\t sizeof(data));\n+\tdata \u0026= ~(NBL_MAILBOX_QINFO_MAP_MSIX_IDX_MASK \|\n+\t\t NBL_MAILBOX_QINFO_MAP_MSIX_IDX_VALID_MASK);\n+\tif (en_msix)\n+\t\tdata \|= FIELD_PREP(NBL_MAILBOX_QINFO_MAP_MSIX_IDX_MASK,\n+\t\t\t\t gvec) \|\n+\t\t\tFIELD_PREP(NBL_MAILBOX_QINFO_MAP_MSIX_IDX_VALID_MASK,\n+\t\t\t\t 1);\n+\n+\tnbl_hw_wr_regs(hw_mgt, NBL_MAILBOX_QINFO_MAP_REG_ARR(func_id), \u0026data,\n+\t\t sizeof(data));\n+\tspin_unlock(\u0026hw_mgt-\u003ereg_lock);\n+\tnbl_flush_writes(hw_mgt);\n+}\n+\n+static void nbl_hw_cfg_msix_map(struct nbl_hw_mgt hw_mgt, u16 func_id,\n+\t\t\t\tbool valid, dma_addr_t dma_addr, u8 bus,\n+\t\t\t\tu8 devid, u8 function)\n+{\n+\tstruct nbl_function_msix_map function_msix_map;\n+\n+\tmemset(\u0026function_msix_map, 0, sizeof(function_msix_map));\n+\n+\t/\n+\t * Clear VALID on its own first. nbl_hw_wr_regs_lock() writes an\n+\t * entry in ascending dword order, so a single whole-entry write\n+\t * would publish the new (or zeroed) address in data[0]/data[1]\n+\t * while the old VALID bit in data[2] was still set, exposing a\n+\t * valid entry with a half-written address to the pcompleter's\n+\t * table fetch. With VALID already zero only the address words\n+\t * change, and no fetch can be started from a torn entry.\n+\t /\n+\tnbl_hw_wr_regs_lock(hw_mgt,\n+\t\t\t NBL_PCOMPLETER_FUNCTION_MSIX_MAP(func_id) +\n+\t\t\t NBL_FUNC_MSIX_MAP_VALID_DW sizeof(u32),\n+\t\t\t \u0026function_msix_map.data[NBL_FUNC_MSIX_MAP_VALID_DW],\n+\t\t\t sizeof(u32));\n+\n+\tif (valid) {\n+\t\t/* program full entry and set VALID /\n+\t\tfunction_msix_map.data[0] = lower_32_bits(dma_addr);\n+\t\tfunction_msix_map.data[1] = upper_32_bits(dma_addr);\n+\t\tfunction_msix_map.data[2] =\n+\t\t\tFIELD_PREP(NBL_FUNCTION_MSIX_MAP_FUNCTION_MASK,\n+\t\t\t\t function) \|\n+\t\t\tFIELD_PREP(NBL_FUNCTION_MSIX_MAP_DEVID_MASK, devid) \|\n+\t\t\tFIELD_PREP(NBL_FUNCTION_MSIX_MAP_BUS_MASK, bus) \|\n+\t\t\tFIELD_PREP(NBL_FUNCTION_MSIX_MAP_VALID_MASK, 1);\n+\n+\t\tnbl_hw_wr_regs_lock(hw_mgt,\n+\t\t\t\t NBL_PCOMPLETER_FUNCTION_MSIX_MAP(func_id),\n+\t\t\t\t function_msix_map.data,\n+\t\t\t\t sizeof(function_msix_map));\n+\t} else {\n+\t\t/\n+\t\t * reg_lock prevents concurrent CPU writes to the same\n+\t\t * function's MSIX entry, but cannot synchronize hardware DMA\n+\t\t * reads. Upper layer uses two-stage destruction + sync sleep\n+\t\t * to avoid torn hardware read of partial MSIX entry.\n+\t\t * VALID is already cleared above, so the address written\n+\t\t * here is either the still-live table (prepare) or zero\n+\t\t * (complete); data[2] stays zero, leaving the BDF fields\n+\t\t * unused because the entry cannot be fetched.\n+\t\t /\n+\t\tfunction_msix_map.data[0] = lower_32_bits(dma_addr);\n+\t\tfunction_msix_map.data[1] = upper_32_bits(dma_addr);\n+\n+\t\tnbl_hw_wr_regs_lock(hw_mgt,\n+\t\t\t\t NBL_PCOMPLETER_FUNCTION_MSIX_MAP(func_id),\n+\t\t\t\t function_msix_map.data,\n+\t\t\t\t sizeof(function_msix_map));\n+\t}\n+}\n+\n+static void nbl_hw_cfg_msix_info(struct nbl_hw_mgt hw_mgt, u16 func_id,\n+\t\t\t\t bool valid, u16 interrupt_id, u8 bus,\n+\t\t\t\t u8 devid, u8 function, bool msix_mask_en)\n+{\n+\tu32 host_msix_fid = 0;\n+\tstruct nbl_host_msix_info msix_info;\n+\n+\tmemset(\u0026msix_info, 0, sizeof(msix_info));\n+\tif (valid) {\n+\t\thost_msix_fid =\n+\t\t\tFIELD_PREP(NBL_PCOMPLETER_HOST_MSIX_FID_TABLE_FID_MASK,\n+\t\t\t\t func_id) \|\n+\t\t\tFIELD_PREP(NBL_PCOMPLETER_HOST_MSIX_FID_TABLE_VLD_MASK,\n+\t\t\t\t 1);\n+\n+\t\tmsix_info.data[1] =\n+\t\t\tFIELD_PREP(NBL_HOST_MSIX_INFO_FUNCTION_MASK, function) \|\n+\t\t\tFIELD_PREP(NBL_HOST_MSIX_INFO_DEVID_MASK, devid) \|\n+\t\t\tFIELD_PREP(NBL_HOST_MSIX_INFO_BUS_MASK, bus) \|\n+\t\t\tFIELD_PREP(NBL_HOST_MSIX_INFO_VALID_MASK, 1);\n+\n+\t\tif (msix_mask_en)\n+\t\t\tmsix_info.data[1] \|=\n+\t\t\tFIELD_PREP(NBL_HOST_MSIX_INFO_MSIX_MASK_EN_MASK, 1);\n+\t}\n+\tspin_lock(\u0026hw_mgt-\u003ereg_lock);\n+\t/\n+\t Programming order rule:\n+\t * Enable: PADPT_HOST_MSIX_INFO -\u003e PCOMPLETER_HOST_MSIX_FID_TABLE\n+\t * Teardown: reverse order, clear FID VLD first to avoid inconsistent\n+\t * state\n+\t /\n+\tif (valid) {\n+\t\tnbl_hw_wr_regs(hw_mgt,\n+\t\t\t NBL_PADPT_HOST_MSIX_INFO_REG_ARR(interrupt_id),\n+\t\t\t msix_info.data, sizeof(msix_info));\n+\t\tnbl_hw_wr_regs(hw_mgt,\n+\t\t\t NBL_PCOMPLETER_HOST_MSIX_FID_TABLE(interrupt_id),\n+\t\t\t \u0026host_msix_fid, sizeof(host_msix_fid));\n+\t} else {\n+\t\tnbl_hw_wr_regs(hw_mgt,\n+\t\t\t NBL_PCOMPLETER_HOST_MSIX_FID_TABLE(interrupt_id),\n+\t\t\t \u0026host_msix_fid, sizeof(host_msix_fid));\n+\t\tnbl_hw_wr_regs(hw_mgt,\n+\t\t\t NBL_PADPT_HOST_MSIX_INFO_REG_ARR(interrupt_id),\n+\t\t\t msix_info.data, sizeof(msix_info));\n+\t}\n+\tspin_unlock(\u0026hw_mgt-\u003ereg_lock);\n+}\n+\n+static void nbl_hw_update_mailbox_queue_tail_ptr(struct nbl_hw_mgt hw_mgt,\n+\t\t\t\t\t\t u16 tail_ptr, u8 txrx)\n+{\n+\t/* local_qid 0 and 1 denote rx and tx queue respectively /\n+\tu32 local_qid = txrx;\n+\tu32 value = ((u32)tail_ptr \u003c\u003c 16) \| local_qid;\n+\n+\t/ wmb for doorbell /\n+\twmb();\n+\tnbl_mbx_wr32(hw_mgt, NBL_MAILBOX_NOTIFY_ADDR, value);\n+}\n+\n+/\n+ * Assert QUEUE_RST on a QINFO cfg entry, in a write of its own.\n+ \n+ The entry is four dwords and the device may act on each one as it\n+ * lands, while data[3] carries QUEUE_RST/QUEUE_EN and data[0..2] carry\n+ * the ring base and size. Writing a new base while QUEUE_EN is still\n+ * set from the previous configuration would have the queue fetch at a\n+ * half-updated address - IOVA 0, once data[0..2] have been cleared -\n+ * so the reset always goes in first, on its own, and the remaining\n+ * dwords are written afterwards. This also covers a device that only\n+ * commits the entry when data[3] lands.\n+ /\n+static void nbl_hw_reset_mailbox_qinfo(struct nbl_hw_mgt hw_mgt, u64 reg)\n+{\n+\tu32 rst = FIELD_PREP(NBL_MAILBOX_QINFO_CFG_QUEUE_RST_MASK, 1);\n+\n+\tnbl_hw_write_mbx_regs(hw_mgt,\n+\t\t\t reg + (MAILBOX_QINFO_CFG_TABLE_DWLEN - 1) \n+\t\t\t\t sizeof(u32),\n+\t\t\t \u0026rst, sizeof(rst));\n+}\n+\n+static void nbl_hw_config_mailbox_rxq(struct nbl_hw_mgt hw_mgt,\n+\t\t\t\t dma_addr_t dma_addr, int size_bwid)\n+{\n+\tstruct nbl_mailbox_qinfo_cfg_table cfg_tbl;\n+\n+\t/* Hold the queue in reset before its base/size are reprogrammed /\n+\tnbl_hw_reset_mailbox_qinfo(hw_mgt, NBL_MAILBOX_QINFO_CFG_RX_TABLE_ADDR);\n+\n+\tcfg_tbl.data[0] = lower_32_bits(dma_addr);\n+\tcfg_tbl.data[1] = upper_32_bits(dma_addr);\n+\tcfg_tbl.data[2] = FIELD_PREP(NBL_MAILBOX_QINFO_CFG_QUEUE_SIZE_BWID_MASK,\n+\t\t\t\t size_bwid);\n+\tcfg_tbl.data[3] = FIELD_PREP(NBL_MAILBOX_QINFO_CFG_QUEUE_RST_MASK, 0) \|\n+\t\t\t FIELD_PREP(NBL_MAILBOX_QINFO_CFG_QUEUE_EN_MASK, 1);\n+\t/\n+\t * data[0..2] land while QUEUE_RST is still set from the write above;\n+\t * data[3] is written last and starts the queue on the new ring.\n+\t /\n+\tnbl_hw_write_mbx_regs(hw_mgt, NBL_MAILBOX_QINFO_CFG_RX_TABLE_ADDR,\n+\t\t\t cfg_tbl.data, sizeof(cfg_tbl));\n+}\n+\n+static void nbl_hw_config_mailbox_txq(struct nbl_hw_mgt hw_mgt,\n+\t\t\t\t dma_addr_t dma_addr, int size_bwid)\n+{\n+\tstruct nbl_mailbox_qinfo_cfg_table cfg_tbl;\n+\n+\t/* Hold the queue in reset before its base/size are reprogrammed /\n+\tnbl_hw_reset_mailbox_qinfo(hw_mgt, NBL_MAILBOX_QINFO_CFG_TX_TABLE_ADDR);\n+\n+\tcfg_tbl.data[0] = lower_32_bits(dma_addr);\n+\tcfg_tbl.data[1] = upper_32_bits(dma_addr);\n+\tcfg_tbl.data[2] = FIELD_PREP(NBL_MAILBOX_QINFO_CFG_QUEUE_SIZE_BWID_MASK,\n+\t\t\t\t size_bwid);\n+\tcfg_tbl.data[3] = FIELD_PREP(NBL_MAILBOX_QINFO_CFG_QUEUE_RST_MASK, 0) \|\n+\t\t\t FIELD_PREP(NBL_MAILBOX_QINFO_CFG_QUEUE_EN_MASK, 1);\n+\t/\n+\t * data[0..2] land while QUEUE_RST is still set from the write above;\n+\t * data[3] is written last and starts the queue on the new ring.\n+\t /\n+\tnbl_hw_write_mbx_regs(hw_mgt, NBL_MAILBOX_QINFO_CFG_TX_TABLE_ADDR,\n+\t\t\t cfg_tbl.data, sizeof(cfg_tbl));\n+}\n+\n+static void nbl_hw_stop_mailbox_rxq(struct nbl_hw_mgt hw_mgt)\n+{\n+\tstruct nbl_mailbox_qinfo_cfg_table cfg_tbl;\n+\n+\t/\n+\t Assert QUEUE_RST before data[0..2] are cleared: writing the\n+\t * zeroed entry in one go would clear the ring base while QUEUE_EN\n+\t * is still set. The reset also clears QUEUE_EN, so what follows\n+\t * lands on an already stopped queue.\n+\t /\n+\tnbl_hw_reset_mailbox_qinfo(hw_mgt, NBL_MAILBOX_QINFO_CFG_RX_TABLE_ADDR);\n+\n+\t/ Now drop the stale base and size, keeping QUEUE_RST asserted /\n+\tmemset(\u0026cfg_tbl, 0, sizeof(cfg_tbl));\n+\tcfg_tbl.data[3] = FIELD_PREP(NBL_MAILBOX_QINFO_CFG_QUEUE_RST_MASK, 1);\n+\tnbl_hw_write_mbx_regs(hw_mgt, NBL_MAILBOX_QINFO_CFG_RX_TABLE_ADDR,\n+\t\t\t cfg_tbl.data, sizeof(cfg_tbl));\n+\t/ Ensure QUEUE_RST has reached the device before caller proceeds /\n+\tnbl_hw_flush_mbx_write(hw_mgt,\n+\t\t\t NBL_MAILBOX_QINFO_CFG_RX_TABLE_ADDR);\n+}\n+\n+static void nbl_hw_stop_mailbox_txq(struct nbl_hw_mgt hw_mgt)\n+{\n+\tstruct nbl_mailbox_qinfo_cfg_table cfg_tbl;\n+\n+\t/\n+\t Assert QUEUE_RST before data[0..2] are cleared: writing the\n+\t * zeroed entry in one go would clear the ring base while QUEUE_EN\n+\t * is still set. The reset also clears QUEUE_EN, so what follows\n+\t * lands on an already stopped queue.\n+\t /\n+\tnbl_hw_reset_mailbox_qinfo(hw_mgt, NBL_MAILBOX_QINFO_CFG_TX_TABLE_ADDR);\n+\n+\t/ Now drop the stale base and size, keeping QUEUE_RST asserted /\n+\tmemset(\u0026cfg_tbl, 0, sizeof(cfg_tbl));\n+\tcfg_tbl.data[3] = FIELD_PREP(NBL_MAILBOX_QINFO_CFG_QUEUE_RST_MASK, 1);\n+\tnbl_hw_write_mbx_regs(hw_mgt, NBL_MAILBOX_QINFO_CFG_TX_TABLE_ADDR,\n+\t\t\t cfg_tbl.data, sizeof(cfg_tbl));\n+\t/ Ensure QUEUE_RST has reached the device before caller proceeds /\n+\tnbl_hw_flush_mbx_write(hw_mgt,\n+\t\t\t NBL_MAILBOX_QINFO_CFG_TX_TABLE_ADDR);\n+}\n+\n+static void nbl_hw_get_host_pf_mask(struct nbl_hw_mgt hw_mgt, u32 pf_mask)\n+{\n+\tnbl_hw_rd_regs_lock(hw_mgt, NBL_PCIE_HOST_K_PF_MASK_REG, pf_mask,\n+\t\t\t sizeof(pf_mask));\n+}\n+\n+static void nbl_hw_get_real_bus(struct nbl_hw_mgt hw_mgt, u8 bus)\n+{\n+\tu32 data = 0;\n+\n+\tnbl_hw_rd_regs_lock(hw_mgt, NBL_PCIE_HOST_TL_CFG_BUSDEV, \u0026data,\n+\t\t\t sizeof(data));\n+\tbus = FIELD_GET(NBL_PCIE_BUS_MASK, data);\n+}\n+\n+static void nbl_hw_cfg_mailbox_qinfo(struct nbl_hw_mgt hw_mgt, u16 func_id,\n+\t\t\t\t u8 bus, u8 devid, u8 function)\n+{\n+\tu32 data = 0;\n+\n+\tspin_lock(\u0026hw_mgt-\u003ereg_lock);\n+\tnbl_hw_rd_regs(hw_mgt, NBL_MAILBOX_QINFO_MAP_REG_ARR(func_id),\n+\t\t \u0026data, sizeof(data));\n+\tdata \u0026= ~(NBL_MAILBOX_QINFO_MAP_FUNCTION_MASK \|\n+\t\t NBL_MAILBOX_QINFO_MAP_DEVID_MASK \|\n+\t\t NBL_MAILBOX_QINFO_MAP_BUS_MASK \|\n+\t\t NBL_MAILBOX_QINFO_MAP_MSIX_IDX_MASK \|\n+\t\t NBL_MAILBOX_QINFO_MAP_MSIX_IDX_VALID_MASK);\n+\tdata \|= FIELD_PREP(NBL_MAILBOX_QINFO_MAP_FUNCTION_MASK, function) \|\n+\t FIELD_PREP(NBL_MAILBOX_QINFO_MAP_DEVID_MASK, devid) \|\n+\t FIELD_PREP(NBL_MAILBOX_QINFO_MAP_BUS_MASK, bus);\n+\tnbl_hw_wr_regs(hw_mgt, NBL_MAILBOX_QINFO_MAP_REG_ARR(func_id),\n+\t\t \u0026data, sizeof(data));\n+\tspin_unlock(\u0026hw_mgt-\u003ereg_lock);\n+}\n+\n+/\n+ Registers reset to zero after cold boot / FLR / bus reset. Firmware\n+ * programs valid values before driver probe, so zero is only seen on\n+ * hardware fault or register read failure. Initialize data=0 to guard\n+ * against nbl_hw_read_mbx_regs() early-return on bounds-check failure.\n+ /\n+static void nbl_hw_get_board_info(struct nbl_hw_mgt hw_mgt,\n+\t\t\t\t struct nbl_board_port_info board_info)\n+{\n+\tu32 data = 0;\n+\n+\tnbl_hw_read_mbx_regs(hw_mgt, NBL_FW_BOARD_DW3_OFFSET, \u0026data,\n+\t\t\t sizeof(data));\n+\tboard_info-\u003eeth_num = FIELD_GET(NBL_FW_BOARD_DW3_PORT_NUM_MASK, data);\n+\tboard_info-\u003eeth_speed =\n+\t\tFIELD_GET(NBL_FW_BOARD_DW3_PORT_SPEED_MASK, data);\n+\tboard_info-\u003ep4_version =\n+\t\tFIELD_GET(NBL_FW_BOARD_DW3_P4_VERSION_MASK, data);\n+}\n+\n+static struct nbl_hw_ops hw_ops = {\n+\t.init_module = nbl_hw_init_module,\n+\t.deinit_module = nbl_hw_deinit_module,\n+\n+\t.cfg_msix_map = nbl_hw_cfg_msix_map,\n+\t.cfg_msix_info = nbl_hw_cfg_msix_info,\n+\t.flush_write = nbl_flush_writes,\n+\n+\t.update_mailbox_queue_tail_ptr = nbl_hw_update_mailbox_queue_tail_ptr,\n+\t.config_mailbox_rxq = nbl_hw_config_mailbox_rxq,\n+\t.config_mailbox_txq = nbl_hw_config_mailbox_txq,\n+\t.stop_mailbox_rxq = nbl_hw_stop_mailbox_rxq,\n+\t.stop_mailbox_txq = nbl_hw_stop_mailbox_txq,\n+\t.get_host_pf_mask = nbl_hw_get_host_pf_mask,\n+\t.get_real_bus = nbl_hw_get_real_bus,\n+\n+\t.cfg_mailbox_qinfo = nbl_hw_cfg_mailbox_qinfo,\n+\t.set_mailbox_irq = nbl_hw_set_mailbox_irq,\n+\n+\t.get_fw_eth_map = nbl_hw_get_fw_eth_map,\n+\t.get_board_info = nbl_hw_get_board_info,\n+};\n+\n / Structure starts here, adding an op should not modify anything below /\n static struct nbl_hw_mgt nbl_hw_setup_hw_mgt(struct nbl_common_info common)\n {\n@@ -23,6 +1246,31 @@ static struct nbl_hw_mgt nbl_hw_setup_hw_mgt(struct nbl_common_info common)\n \treturn hw_mgt;\n }\n \n+static struct nbl_hw_ops_tbl nbl_hw_setup_ops(struct nbl_common_info common,\n+\t\t\t\t\t struct nbl_hw_mgt hw_mgt)\n+{\n+\tstruct nbl_hw_ops_tbl hw_ops_tbl;\n+\tstruct device dev;\n+\n+\tdev = common-\u003edev;\n+\thw_ops_tbl = devm_kzalloc(dev, sizeof(hw_ops_tbl), GFP_KERNEL);\n+\tif (!hw_ops_tbl)\n+\t\treturn ERR_PTR(-ENOMEM);\n+\tif (!hw_ops.cfg_msix_map \|\| !hw_ops.cfg_msix_info \|\|\n+\t !hw_ops.flush_write \|\| !hw_ops.update_mailbox_queue_tail_ptr \|\|\n+\t !hw_ops.config_mailbox_rxq \|\| !hw_ops.config_mailbox_txq \|\|\n+\t !hw_ops.stop_mailbox_rxq \|\| !hw_ops.stop_mailbox_txq \|\|\n+\t !hw_ops.get_host_pf_mask \|\| !hw_ops.get_real_bus \|\|\n+\t !hw_ops.cfg_mailbox_qinfo \|\| !hw_ops.set_mailbox_irq \|\|\n+\t !hw_ops.get_fw_eth_map \|\| !hw_ops.get_board_info \|\|\n+\t !hw_ops.init_module \|\| !hw_ops.deinit_module)\n+\t\treturn ERR_PTR(-EINVAL);\n+\thw_ops_tbl-\u003eops = \u0026hw_ops;\n+\thw_ops_tbl-\u003epriv = hw_mgt;\n+\n+\treturn hw_ops_tbl;\n+}\n+\n static int nbl_pcim_request_selected_bars(struct pci_dev pdev, u32 mask,\n \t\t\t\t\t const char name)\n {\n@@ -43,6 +1291,7 @@ int nbl_hw_init_leonis(struct nbl_adapter adapter)\n {\n \tresource_size_t expect_sz = NBL_MEM_BAR_TOTAL_SIZE;\n \tstruct nbl_common_info common = \u0026adapter-\u003ecommon;\n+\tstruct nbl_hw_ops_tbl hw_ops_tbl = NULL;\n \tstruct pci_dev pdev = common-\u003epdev;\n \tstruct nbl_hw_mgt hw_mgt = NULL;\n \tresource_size_t bar_len;\n@@ -53,7 +1302,7 @@ int nbl_hw_init_leonis(struct nbl_adapter adapter)\n \thw_mgt = nbl_hw_setup_hw_mgt(common);\n \tif (IS_ERR(hw_mgt)) {\n \t\tret = PTR_ERR(hw_mgt);\n-\t\tgoto setup_mgt_fail;\n+\t\treturn ret;\n \t}\n \tbar_mask = BIT(NBL_MEMORY_BAR) \| BIT(NBL_MAILBOX_BAR);\n \tret = nbl_pcim_request_selected_bars(pdev, bar_mask, NBL_DRIVER_NAME);\n@@ -61,14 +1310,14 @@ int nbl_hw_init_leonis(struct nbl_adapter adapter)\n \t\tdev_err(\u0026pdev-\u003edev,\n \t\t\t\"Request memory bar failed, err = %d\\n\",\n \t\t\tret);\n-\t\tgoto setup_mgt_fail;\n+\t\treturn ret;\n \t}\n \n \tbar_len = pci_resource_len(pdev, NBL_MEMORY_BAR);\n \tif (!(pci_resource_flags(pdev, NBL_MEMORY_BAR) \u0026 IORESOURCE_MEM)) {\n \t\tdev_err(\u0026pdev-\u003edev, \"MEMORY BAR is not memory resource\\n\");\n \t\tret = -EINVAL;\n-\t\tgoto setup_mgt_fail;\n+\t\treturn ret;\n \t}\n \tif (common-\u003ehas_ctrl) {\n \t\t/\n@@ -90,7 +1339,7 @@ int nbl_hw_init_leonis(struct nbl_adapter adapter)\n \t\t\t\t\"MEMORY BAR len %pa smaller than expected %pa\\n\",\n \t\t\t\t\u0026bar_len, \u0026expect_sz);\n \t\t\tret = -EINVAL;\n-\t\t\tgoto setup_mgt_fail;\n+\t\t\treturn ret;\n \t\t}\n \t\thw_size = PAGE_ALIGN_DOWN(NBL_MEM_BAR_TOTAL_SIZE -\n \t\t\t\t\t NBL_RDMA_NOTIFY_LEN);\n@@ -103,7 +1352,7 @@ int nbl_hw_init_leonis(struct nbl_adapter adapter)\n \t\t\t\t\"MEMORY BAR len %pa too small for net only reg space\\n\",\n \t\t\t\t\u0026bar_len);\n \t\t\tret = -EINVAL;\n-\t\t\tgoto setup_mgt_fail;\n+\t\t\treturn ret;\n \t\t}\n \t\thw_size = NBL_REG_NET_ONLY_LEN;\n \t\thw_mgt-\u003ehw_addr = pcim_iomap(pdev, NBL_MEMORY_BAR,\n@@ -112,37 +1361,50 @@ int nbl_hw_init_leonis(struct nbl_adapter adapter)\n \tif (!hw_mgt-\u003ehw_addr) {\n \t\tdev_err(\u0026pdev-\u003edev, \"MEMORY BAR pcim_iomap failed\\n\");\n \t\tret = -EIO;\n-\t\tgoto setup_mgt_fail;\n+\t\treturn ret;\n \t}\n \n \tbar_len = pci_resource_len(pdev, NBL_MAILBOX_BAR);\n \tif (!(pci_resource_flags(pdev, NBL_MAILBOX_BAR) \u0026 IORESOURCE_MEM)) {\n \t\tdev_err(\u0026pdev-\u003edev, \"MAILBOX BAR is not memory resource\\n\");\n \t\tret = -EINVAL;\n-\t\tgoto setup_mgt_fail;\n+\t\treturn ret;\n \t}\n \tif (bar_len \u003c NBL_BAR2_MAX_LEN) {\n \t\tdev_err(\u0026pdev-\u003edev, \"MAILBOX BAR length %pa too small\\n\",\n \t\t\t\u0026bar_len);\n \t\tret = -EINVAL;\n-\t\tgoto setup_mgt_fail;\n+\t\treturn ret;\n \t}\n \thw_mgt-\u003emailbox_bar_hw_addr = pcim_iomap(pdev, NBL_MAILBOX_BAR,\n \t\t\t\t\t\t bar_len);\n \tif (!hw_mgt-\u003emailbox_bar_hw_addr) {\n \t\tdev_err(\u0026pdev-\u003edev, \"MAILBOX BAR pcim_iomap failed\\n\");\n \t\tret = -EIO;\n-\t\tgoto setup_mgt_fail;\n+\t\treturn ret;\n \t}\n \n \thw_mgt-\u003emailbox_bar_size = bar_len;\n+\tspin_lock_init(\u0026hw_mgt-\u003ereg_lock);\n \n+\t/\n+\t Clear a stale driver_status now, while this is the only register\n+\t * access this probe has done: the firmware cleanup the 1 -\u003e 0\n+\t * transition starts reclaims the mailbox QINFO routing and qinfo\n+\t * registers that nbl_dev_init() programs later, so it must not be\n+\t * deferred to init_module(); see nbl_hw_clear_stale_status().\n+\t /\n+\tnbl_hw_clear_stale_status(hw_mgt);\n+\n+\thw_ops_tbl = nbl_hw_setup_ops(common, hw_mgt);\n+\tif (IS_ERR(hw_ops_tbl)) {\n+\t\tret = PTR_ERR(hw_ops_tbl);\n+\t\treturn ret;\n+\t}\n+\tadapter-\u003eintf.hw_ops_tbl = hw_ops_tbl;\n \tadapter-\u003ecore.hw_mgt = hw_mgt;\n \n \treturn 0;\n-\n-setup_mgt_fail:\n-\treturn ret;\n }\n \n void nbl_hw_remove_leonis(struct nbl_adapter adapter)\ndiff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_leonis/nbl_hw_leonis.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_leonis/nbl_hw_leonis.h\nindex 1f9e509dc6312f..7d27ab19838e21 100644\n--- a/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_leonis/nbl_hw_leonis.h\n+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_leonis/nbl_hw_leonis.h\n@@ -11,5 +11,347 @@\n #include \"../../nbl_include/nbl_include.h\"\n #include \"../nbl_hw_reg.h\"\n \n+/\n+ Chip-wide driver-present handshake with firmware.\n+ \n+ NBL_DRIVER_STATUS_BIT is the only bit the driver owns: setting it tells\n+ * firmware that a driver is bound, and the 1 -\u003e 0 transition starts the\n+ * firmware pass that reclaims per-PF state (mailbox QINFO routing, qinfo\n+ * registers). Every other bit of this register belongs to firmware and\n+ * the driver must leave it as it read it.\n+ \n+ Because the register is shared, bit 16 can only be changed with a\n+ * read-modify-write: it has no documented W1S/W1C semantics that would\n+ * let the driver set or clear that one bit atomically. reg_lock makes\n+ * the read-modify-write atomic with respect to other CPU writers, but it\n+ * cannot serialise against firmware, so a firmware update to another bit\n+ * landing between the read and the write would be written back stale.\n+ * No such firmware update is known to race the three driver callers\n+ * (the probe-time clear, init_module() and deinit_module()), but that is\n+ * an observation about this hardware revision, not a guarantee the\n+ * driver can enforce.\n+ /\n+#define NBL_DRIVER_STATUS_REG\t\t\t0x1300444\n+#define NBL_DRIVER_STATUS_BIT\t\t\t16\n+\n+/ ---------- REG BASE ADDR ---------- /\n+/ Interface modules base addr /\n+#define NBL_INTF_HOST_PCOMPLETER_BASE\t\t0x00f08000\n+#define NBL_INTF_HOST_PADPT_BASE\t\t0x00f4c000\n+#define NBL_INTF_HOST_MAILBOX_BASE\t\t0x00fb0000\n+#define NBL_INTF_HOST_PCIE_BASE\t\t\t0X01504000\n+/ DP modules base addr /\n+#define NBL_DP_USTORE_BASE\t\t\t0x00104000\n+#define NBL_DP_UQM_BASE\t\t\t\t0x00114000\n+#define NBL_DP_UPED_BASE\t\t\t0x0015c000\n+#define NBL_DP_UVN_BASE\t\t\t\t0x00244000\n+#define NBL_DP_DSCH_BASE\t\t\t0x00404000\n+#define NBL_DP_SHAPING_BASE\t\t\t0x00504000\n+#define NBL_DP_DVN_BASE\t\t\t\t0x00514000\n+#define NBL_DP_DSTORE_BASE\t\t\t0x00704000\n+#define NBL_DP_DPED_BASE\t\t\t0x0075c000\n+/ -------- MAILBOX BAR2 ----- /\n+#define NBL_MAILBOX_NOTIFY_ADDR\t\t\t0x00000000\n+#define NBL_MAILBOX_QINFO_CFG_RX_TABLE_ADDR\t0x10\n+#define NBL_MAILBOX_QINFO_CFG_TX_TABLE_ADDR\t0x20\n+\n+/ -------- MAILBOX -------- /\n+\n+/ mailbox BAR qinfo_cfg_table /\n+#define MAILBOX_QINFO_CFG_TABLE_DWLEN\t4\n+/ data[2] /\n+#define NBL_MAILBOX_QINFO_CFG_QUEUE_SIZE_BWID_MASK\tGENMASK(3, 0)\n+/ data[3] /\n+#define NBL_MAILBOX_QINFO_CFG_QUEUE_RST_MASK\t\tBIT(0)\n+#define NBL_MAILBOX_QINFO_CFG_QUEUE_EN_MASK\t\tBIT(1)\n+#define NBL_MAILBOX_QINFO_CFG_DIF_ERR_MASK\t\tBIT(2)\n+#define NBL_MAILBOX_QINFO_CFG_PTR_ERR_MASK\t\tBIT(3)\n+struct nbl_mailbox_qinfo_cfg_table {\n+\tu32 data[MAILBOX_QINFO_CFG_TABLE_DWLEN];\n+};\n+\n+/ -------- MAILBOX BAR0 ----- /\n+/ mailbox qinfo_map_table /\n+#define NBL_MAILBOX_QINFO_MAP_REG_ARR(func_id) \\\n+\t(NBL_INTF_HOST_MAILBOX_BASE + 0x00001000 + (func_id) sizeof(u32))\n+\n+/* MAILBOX qinfo_map_table /\n+#define NBL_MAILBOX_QINFO_MAP_FUNCTION_MASK\t\tGENMASK(2, 0)\n+#define NBL_MAILBOX_QINFO_MAP_DEVID_MASK\t\tGENMASK(7, 3)\n+#define NBL_MAILBOX_QINFO_MAP_BUS_MASK\t\t\tGENMASK(15, 8)\n+#define NBL_MAILBOX_QINFO_MAP_MSIX_IDX_MASK\t\tGENMASK(28, 16)\n+#define NBL_MAILBOX_QINFO_MAP_MSIX_IDX_VALID_MASK\tBIT(29)\n+\n+/ -------- HOST_PCIE -------- /\n+#define NBL_PCIE_HOST_K_PF_MASK_REG (NBL_INTF_HOST_PCIE_BASE + 0x00001004)\n+#define NBL_PCIE_HOST_TL_CFG_BUSDEV (NBL_INTF_HOST_PCIE_BASE + 0x11040)\n+\n+#define NBL_PCIE_BUS_MASK\tGENMASK(12, 5)\n+\n+/ -------- HOST_PADPT -------- /\n+#define NBL_HOST_PADPT_HOST_CFG_FC_PD_DN (NBL_INTF_HOST_PADPT_BASE + 0x00000160)\n+#define NBL_HOST_PADPT_HOST_CFG_FC_PH_DN (NBL_INTF_HOST_PADPT_BASE + 0x00000164)\n+#define NBL_HOST_PADPT_HOST_CFG_FC_NPH_DN \\\n+\t(NBL_INTF_HOST_PADPT_BASE + 0x0000016C)\n+#define NBL_HOST_PADPT_HOST_CFG_FC_CPLH_UP \\\n+\t(NBL_INTF_HOST_PADPT_BASE + 0x00000170)\n+\n+#define NBL_HOST_PADPT_CFG_FC_CPLH_UP_VAL 0x10400\n+#define NBL_HOST_PADPT_CFG_FC_PD_DN_VAL 0x10080\n+#define NBL_HOST_PADPT_CFG_FC_PH_DN_VAL 0x10010\n+#define NBL_HOST_PADPT_CFG_FC_NPH_DN_VAL 0x10010\n+/ host_padpt host_msix_info /\n+#define NBL_PADPT_HOST_MSIX_INFO_REG_ARR(vector_id) \\\n+\t(NBL_INTF_HOST_PADPT_BASE + 0x00010000 + \\\n+\t (vector_id) sizeof(struct nbl_host_msix_info))\n+\n+#define NBL_HOST_MSIX_INFO_DWLEN\t2\n+/* data[0] /\n+#define NBL_HOST_MSIX_INFO_INTRL_PNUM_MASK GENMASK(15, 0)\n+#define NBL_HOST_MSIX_INFO_INTRL_RATE_MASK GENMASK(31, 16)\n+/ data[1] /\n+#define NBL_HOST_MSIX_INFO_FUNCTION_MASK GENMASK(2, 0)\n+#define NBL_HOST_MSIX_INFO_DEVID_MASK GENMASK(7, 3)\n+#define NBL_HOST_MSIX_INFO_BUS_MASK GENMASK(15, 8)\n+#define NBL_HOST_MSIX_INFO_VALID_MASK BIT(16)\n+#define NBL_HOST_MSIX_INFO_MSIX_MASK_EN_MASK BIT(17)\n+struct nbl_host_msix_info {\n+\tu32 data[NBL_HOST_MSIX_INFO_DWLEN];\n+};\n+\n+/ -------- HOST_PCOMPLETER -------- /\n+/ pcompleter_host function_msix_map_table /\n+#define NBL_PCOMPLETER_FUNCTION_MSIX_MAP(i) \\\n+\t(NBL_INTF_HOST_PCOMPLETER_BASE + 0x00004000 + \\\n+\t (i) sizeof(struct nbl_function_msix_map))\n+#define NBL_PCOMPLETER_HOST_MSIX_FID_TABLE(i) \\\n+\t(NBL_INTF_HOST_PCOMPLETER_BASE + 0x0003a000 + (i) * sizeof(u32))\n+\n+#define NBL_PCOMPLETER_HOST_MSIX_FID_TABLE_FID_MASK GENMASK(9, 0)\n+#define NBL_PCOMPLETER_HOST_MSIX_FID_TABLE_VLD_MASK BIT(10)\n+\n+#define NBL_FUNC_MSIX_MAP_DWLEN\t\t4\n+/* The VLD/BDF fields live in data[2]; it can be cleared on its own /\n+#define NBL_FUNC_MSIX_MAP_VALID_DW\t2\n+/ data[2] /\n+#define NBL_FUNCTION_MSIX_MAP_FUNCTION_MASK GENMASK(2, 0)\n+#define NBL_FUNCTION_MSIX_MAP_DEVID_MASK GENMASK(7, 3)\n+#define NBL_FUNCTION_MSIX_MAP_BUS_MASK GENMASK(15, 8)\n+#define NBL_FUNCTION_MSIX_MAP_VALID_MASK BIT(16)\n+struct nbl_function_msix_map {\n+\tu32 data[NBL_FUNC_MSIX_MAP_DWLEN];\n+};\n+\n+/ ---------- DPED ---------- /\n+#define NBL_DPED_VLAN_OFFSET\t\t(NBL_DP_DPED_BASE + 0x000003F4)\n+#define NBL_DPED_DSCP_OFFSET_0\t\t(NBL_DP_DPED_BASE + 0x000003F8)\n+#define NBL_DPED_DSCP_OFFSET_1\t\t(NBL_DP_DPED_BASE + 0x000003FC)\n+/ DPED hw_edt_prof/ UPED hw_edt_prof /\n+\n+#define NBL_DPED_L4_CK_CMD_40_ADDR\t(NBL_DP_DPED_BASE + 0x00000338)\n+\n+#define NBL_DPED_L4_CK_CMD_40_EN_MASK\t\tBIT(31)\n+\n+/ ---------- UPED ---------- /\n+/ UPED uped_hw_edt_prof /\n+#define NBL_UPED_HW_EDT_PROF_TABLE(i) \\\n+\t(NBL_DP_UPED_BASE + 0x00001000 + (i) sizeof(u32))\n+#define NBL_UPED_V4_TCP_IDX\t\t5\n+#define NBL_UPED_V6_TCP_IDX\t\t6\n+#define NBL_PED_HW_EDIT_PROFILE_L3_LEN_MASK GENMASK(3, 2)\n+\n+/* ---------- DSCH ---------- /\n+#define NBL_DSCH_PSHA_EN_MASK GENMASK(3, 0)\n+/ DSCH dsch maxqid /\n+#define NBL_DSCH_HOST_QID_MAX (NBL_DP_DSCH_BASE + 0x00000118)\n+#define NBL_DSCH_HOST_QID_MAX_MASK GENMASK(10, 0)\n+#define NBL_DSCH_VN_QUANTA_ADDR (NBL_DP_DSCH_BASE + 0x00000134)\n+\n+#define NBL_MAX_QUEUE_ID\t0x7ff\n+#define NBL_HOST_QUANTA\t\t0x8000\n+#define NBL_ECPU_QUANTA\t\t0x1000\n+\n+#define NBL_DSCH_VN_QUANTA_H_QUA_MASK GENMASK(15, 0)\n+#define NBL_DSCH_VN_QUANTA_E_QUA_MASK GENMASK(31, 16)\n+\n+/ ---------- DVN ---------- /\n+/ DVN dvn_queue_table /\n+#define NBL_DVN_ECPU_QUEUE_NUM\t\t\t(NBL_DP_DVN_BASE + 0x0000041C)\n+#define NBL_DVN_DESCREQ_NUM_CFG\t\t\t(NBL_DP_DVN_BASE + 0x00000430)\n+#define NBL_DVN_DESC_WR_MERGE_TIMEOUT\t\t(NBL_DP_DVN_BASE + 0x00000480)\n+#define NBL_DVN_DIF_REQ_RD_RO_FLAG\t\t(NBL_DP_DVN_BASE + 0x0000045C)\n+\n+#define DEFAULT_DVN_DESCREQ_NUMCFG\t\t0x03\n+#define DEFAULT_DVN_100G_DESCREQ_NUMCFG\t\t0x07\n+\n+#define DEFAULT_DVN_DESC_WR_MERGE_TIMEOUT_MAX\t0x3FF\n+\n+/ spilit ring descreq_num 0:8,1:16 /\n+#define NBL_DVN_DESCREQ_NUM_CFG_AVRING_DESREQ_NUM_CFG_MASK BIT(0)\n+/ packet ring descreq_num\n+ * 0:8,1:12,2:16;3:20,4:24,5:26;6:32,7:32\n+ /\n+#define NBL_DVN_DESCREQ_NUM_CFG_PACKED_L1_NUM_MASK GENMASK(6, 4)\n+\n+#define NBL_DVN_DESC_WR_MERGE_TIMEOUT_CFG_CYCLE_MASK GENMASK(9, 0)\n+\n+#define NBL_DVN_DIF_REQ_RD_RO_FLAG_DESC_RO_EN_MASK BIT(0)\n+#define NBL_DVN_DIF_REQ_RD_RO_FLAG_DATA_RO_EN_MASK BIT(1)\n+#define NBL_DVN_DIF_REQ_RD_RO_FLAG_AVRING_RO_EN_MASK BIT(2)\n+\n+/ ---------- UVN ---------- /\n+/ UVN uvn_queue_table /\n+\n+#define NBL_UVN_DESC_RD_WAIT\t\t\t(NBL_DP_UVN_BASE + 0x0000020C)\n+#define NBL_UVN_QUEUE_ERR_MASK\t\t\t(NBL_DP_UVN_BASE + 0x00000224)\n+#define NBL_UVN_ECPU_QUEUE_NUM\t\t\t(NBL_DP_UVN_BASE + 0x0000023C)\n+#define NBL_UVN_DESC_WR_TIMEOUT\t\t\t(NBL_DP_UVN_BASE + 0x00000214)\n+#define NBL_UVN_DIF_REQ_RO_FLAG\t\t\t(NBL_DP_UVN_BASE + 0x00000250)\n+#define NBL_UVN_DESC_PREFETCH_INIT\t\t(NBL_DP_UVN_BASE + 0x00000204)\n+#define NBL_UVN_DESC_PREFETCH_NUM\t\t4\n+\n+#define NBL_UVN_DIF_REQ_RO_FLAG_AVAIL_RD_MASK BIT(0)\n+#define NBL_UVN_DIF_REQ_RO_FLAG_DESC_RD_MASK BIT(1)\n+#define NBL_UVN_DIF_REQ_RO_FLAG_PKT_WR_MASK BIT(2)\n+#define NBL_UVN_DIF_REQ_RO_FLAG_DESC_WR_MASK BIT(3)\n+\n+#define NBL_UVN_DESC_WR_TIMEOUT_NUM_MASK GENMASK(14, 0)\n+\n+#define NBL_UVN_QUEUE_ERR_MASK_DIF_ERR_MASK BIT(5)\n+\n+#define NBL_UVN_DESC_PREFETCH_INIT_NUM_MASK GENMASK(7, 0)\n+#define NBL_UVN_DESC_PREFETCH_INIT_SEL_MASK BIT(16)\n+\n+#define NBL_UVN_DESC_WR_TIMEOUT_VAL 0x12c\n+/ 200us = 200000ns / 1.67ns per tick = 119760 ticks /\n+#define NBL_UVN_DESC_RD_WAIT_TICKS 119760\n+\n+/ -------- USTORE -------- /\n+#define NBL_USTORE_PKT_LEN_ADDR (NBL_DP_USTORE_BASE + 0x00000108)\n+#define NBL_USTORE_PORT_DROP_TH_REG_ARR(port_id) \\\n+\t(NBL_DP_USTORE_BASE + 0x00000150 + (port_id) sizeof(u32))\n+#define NBL_USTORE_BUF_PORT_DROP_PKT(eth_id) \\\n+\t(NBL_DP_USTORE_BASE + 0x00002500 + (eth_id) * sizeof(u32))\n+#define NBL_USTORE_BUF_PORT_TRUN_PKT(eth_id) \\\n+\t(NBL_DP_USTORE_BASE + 0x00002540 + (eth_id) * sizeof(u32))\n+\n+#define NBL_USTORE_SINGLE_ETH_DROP_TH\t\t0xC80\n+#define NBL_USTORE_DUAL_ETH_DROP_TH\t\t0x640\n+#define NBL_USTORE_QUAD_ETH_DROP_TH\t\t0x320\n+\n+/* USTORE pkt_len /\n+#define NBL_USTORE_PKT_LEN_MIN_MASK GENMASK(6, 0)\n+\n+/ USTORE port_drop_th /\n+#define NBL_USTORE_PORT_DROP_TH_DISC_TH_MASK GENMASK(11, 0)\n+#define NBL_USTORE_PORT_DROP_TH_EN_MASK BIT(31)\n+\n+/ UQM/\n+#define NBL_UQM_QUE_TYPE\t\t\t(NBL_DP_UQM_BASE + 0x0000013c)\n+#define NBL_UQM_DROP_PKT_CNT\t\t\t(NBL_DP_UQM_BASE + 0x000009C0)\n+#define NBL_UQM_DROP_PKT_SLICE_CNT\t\t(NBL_DP_UQM_BASE + 0x000009C4)\n+#define NBL_UQM_DROP_PKT_LEN_ADD_CNT\t\t(NBL_DP_UQM_BASE + 0x000009C8)\n+#define NBL_UQM_DROP_HEAD_PNTR_ADD_CNT\t\t(NBL_DP_UQM_BASE + 0x000009CC)\n+#define NBL_UQM_DROP_WEIGHT_ADD_CNT\t\t(NBL_DP_UQM_BASE + 0x000009D0)\n+#define NBL_UQM_PORT_DROP_PKT_CNT\t\t(NBL_DP_UQM_BASE + 0x000009D4)\n+#define NBL_UQM_PORT_DROP_PKT_SLICE_CNT\t\t(NBL_DP_UQM_BASE + 0x000009F4)\n+#define NBL_UQM_PORT_DROP_PKT_LEN_ADD_CNT\t(NBL_DP_UQM_BASE + 0x00000A14)\n+#define NBL_UQM_PORT_DROP_HEAD_PNTR_ADD_CNT\t(NBL_DP_UQM_BASE + 0x00000A34)\n+#define NBL_UQM_PORT_DROP_WEIGHT_ADD_CNT\t(NBL_DP_UQM_BASE + 0x00000A54)\n+#define NBL_UQM_FWD_DROP_CNT\t\t\t(NBL_DP_UQM_BASE + 0x00000A80)\n+#define NBL_UQM_DPORT_DROP_CNT\t\t\t(NBL_DP_UQM_BASE + 0x00000B74)\n+\n+#define NBL_UQM_PORT_DROP_DEPTH\t\t\t6\n+#define NBL_UQM_DPORT_DROP_DEPTH\t\t16\n+\n+/ --------- SHAPING --------- /\n+\n+/ Shaping rate unit: 1 = 1 Mbit/s.\n+ * e.g. 100000 = 100 Gbit/s, 25000 = 25 Gbit/s.\n+ /\n+#define NBL_LR_LEONIS_NET_BUCKET_DEPTH\t\t9600\n+#define NBL_SHAPING_DPORT_ADDR (NBL_DP_SHAPING_BASE + 0x700)\n+#define NBL_SHAPING_DPORT_DWLEN 4\n+#define NBL_SHAPING_DPORT_REG(r) \\\n+\t(NBL_SHAPING_DPORT_ADDR + (NBL_SHAPING_DPORT_DWLEN 4) * (r))\n+#define NBL_SHAPING_DVN_DPORT_ADDR (NBL_DP_SHAPING_BASE + 0x750)\n+#define NBL_SHAPING_DVN_DPORT_DWLEN 4\n+#define NBL_SHAPING_DVN_DPORT_REG(r) \\\n+\t(NBL_SHAPING_DVN_DPORT_ADDR + (NBL_SHAPING_DVN_DPORT_DWLEN * 4) * (r))\n+#define NBL_DSCH_PSHA_EN_ADDR (NBL_DP_DSCH_BASE + 0x00000314)\n+#define NBL_SHAPING_NET_ADDR (NBL_DP_SHAPING_BASE + 0x1800)\n+#define NBL_SHAPING_NET_DWLEN 4\n+#define NBL_SHAPING_NET_REG(r) \\\n+\t(NBL_SHAPING_NET_ADDR + (NBL_SHAPING_NET_DWLEN * 4) * (r))\n+\n+#define NBL_DPORT_VALID_MASK\t\tGENMASK_ULL(0, 0)\n+#define NBL_DPORT_DEPTH_MASK\t\tGENMASK_ULL(19, 1)\n+#define NBL_DPORT_CIR_MASK\t\tGENMASK_ULL(38, 20)\n+#define NBL_DPORT_PIR_MASK\t\tGENMASK_ULL(57, 39)\n+#define NBL_DPORT_CBS_MASK_LOW\t\tGENMASK_ULL(63, 58)\n+#define NBL_DPORT_CBS_MASK_HIGH\t\tGENMASK_ULL(14, 0)\n+#define NBL_DPORT_PBS_MASK\t\tGENMASK_ULL(35, 15)\n+\n+/* SHAPING shaping_net /\n+struct nbl_shaping_net_u {\n+\tu32 data[NBL_SHAPING_NET_DWLEN];\n+};\n+\n+struct nbl_shaping_dport_u {\n+\tu32 data[NBL_SHAPING_DPORT_DWLEN];\n+};\n+\n+struct nbl_shaping_dvn_dport_u {\n+\tu32 data[NBL_SHAPING_DVN_DPORT_DWLEN];\n+};\n+\n+/ -------- DSTORE -------- /\n+#define NBL_DSTORE_D_DPORT_FC_TH_ADDR (NBL_DP_DSTORE_BASE + 0x00000600)\n+#define NBL_DSTORE_D_DPORT_FC_TH_DEPTH 5\n+#define NBL_DSTORE_D_DPORT_FC_TH_WIDTH 32\n+#define NBL_DSTORE_D_DPORT_FC_TH_DWLEN 1\n+\n+#define NBL_DSTORE_D_DPORT_FC_XOFF_TH_MASK GENMASK(10, 0)\n+#define NBL_DSTORE_D_DPORT_FC_XON_TH_MASK GENMASK(26, 16)\n+#define NBL_DSTORE_D_DPORT_FC_FC_EN_MASK BIT(31)\n+\n+#define NBL_DSTORE_D_DPORT_FC_TH_REG(r) \\\n+\t(NBL_DSTORE_D_DPORT_FC_TH_ADDR + \\\n+\t (NBL_DSTORE_D_DPORT_FC_TH_DWLEN 4) * (r))\n+#define NBL_DSTORE_PORT_DROP_TH_ADDR (NBL_DP_DSTORE_BASE + 0x00000150)\n+#define NBL_DSTORE_PORT_DROP_TH_DEPTH 6\n+#define NBL_DSTORE_PORT_DROP_TH_WIDTH 32\n+#define NBL_DSTORE_PORT_DROP_TH_DWLEN 1\n+\n+#define NBL_DSTORE_PORT_DROP_EN_MASK BIT(31)\n+\n+#define NBL_DSTORE_DROP_XOFF_TH\t\t\t0xC8\n+#define NBL_DSTORE_DROP_XON_TH\t\t\t0x64\n+\n+#define NBL_DSTORE_DROP_XOFF_TH_100G\t\t0x1F4\n+#define NBL_DSTORE_DROP_XON_TH_100G\t\t0x12C\n+\n+#define NBL_DSTORE_DISC_BP_TH (NBL_DP_DSTORE_BASE + 0x00000630)\n+\n+#define NBL_DSTORE_DISC_BP_TH_EN_MASK BIT(31)\n+\n+#define NBL_DSTORE_PORT_DROP_TH_REG(r) \\\n+\t(NBL_DSTORE_PORT_DROP_TH_ADDR + \\\n+\t (NBL_DSTORE_PORT_DROP_TH_DWLEN * 4) * (r))\n+\n+#define NBL_FW_BOARD_CONFIG\t\t\t0x200\n+#define NBL_FW_BOARD_DW3_OFFSET\t\t\t(NBL_FW_BOARD_CONFIG + 12)\n+#define NBL_FW_BOARD_DW6_OFFSET\t\t\t(NBL_FW_BOARD_CONFIG + 24)\n+\n+#define NBL_FW_BOARD_DW3_PORT_TYPE_MASK BIT(0)\n+#define NBL_FW_BOARD_DW3_PORT_NUM_MASK GENMASK(7, 1)\n+#define NBL_FW_BOARD_DW3_PORT_SPEED_MASK GENMASK(9, 8)\n+#define NBL_FW_BOARD_DW3_GPIO_TYPE_MASK GENMASK(12, 10)\n+#define NBL_FW_BOARD_DW3_P4_VERSION_MASK GENMASK(13, 13)\n+\n+#define NBL_FW_BOARD_DW6_LANE_BITMAP_MASK GENMASK(7, 0)\n+#define NBL_FW_BOARD_DW6_ETH_BITMAP_MASK GENMASK(15, 8)\n+\n+#define NBL_LEONIS_QUIRKS_OFFSET\t0x00000140\n #define NBL_BAR2_MAX_LEN\t\t0x300\n #endif\ndiff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_leonis/nbl_resource_leonis.c b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_leonis/nbl_resource_leonis.c\nnew file mode 100644\nindex 00000000000000..ba589af55374b7\n--- /dev/null\n+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_leonis/nbl_resource_leonis.c\n@@ -0,0 +1,370 @@\n+// SPDX-License-Identifier: GPL-2.0\n+/\n+ Copyright (c) 2026 Nebula Matrix Limited.\n+ /\n+#include \u003clinux/device.h\u003e\n+#include \u003clinux/pci.h\u003e\n+#include \u003clinux/bits.h\u003e\n+#include \"nbl_resource_leonis.h\"\n+\n+static struct nbl_resource_ops res_ops = {\n+\t.get_vsi_id = nbl_res_func_id_to_vsi_id,\n+\t.get_eth_id = nbl_res_get_eth_id,\n+\t.cfg_msix_map = nbl_res_intr_cfg_msix_map,\n+\t.destroy_msix_map = nbl_res_intr_destroy_msix_map,\n+\t.set_mailbox_irq = nbl_res_intr_set_mailbox_irq,\n+\t.init_module = nbl_res_chip_init_module,\n+\t.deinit_module = nbl_res_chip_deinit_module,\n+};\n+\n+static struct nbl_resource_mgt \n+nbl_res_setup_res_mgt(struct nbl_common_info common)\n+{\n+\tstruct nbl_resource_info resource_info;\n+\tstruct nbl_resource_mgt res_mgt;\n+\tstruct device dev = common-\u003edev;\n+\n+\tres_mgt = devm_kzalloc(dev, sizeof(res_mgt), GFP_KERNEL);\n+\tif (!res_mgt)\n+\t\treturn ERR_PTR(-ENOMEM);\n+\tres_mgt-\u003ecommon = common;\n+\n+\tresource_info =\n+\t\tdevm_kzalloc(dev, sizeof(resource_info), GFP_KERNEL);\n+\tif (!resource_info)\n+\t\treturn ERR_PTR(-ENOMEM);\n+\tres_mgt-\u003eresource_info = resource_info;\n+\n+\treturn res_mgt;\n+}\n+\n+static struct nbl_resource_ops_tbl \n+nbl_res_setup_ops(struct device dev, struct nbl_resource_mgt res_mgt)\n+{\n+\tstruct nbl_resource_ops_tbl res_ops_tbl;\n+\n+\tres_ops_tbl = devm_kzalloc(dev, sizeof(res_ops_tbl), GFP_KERNEL);\n+\tif (!res_ops_tbl)\n+\t\treturn ERR_PTR(-ENOMEM);\n+\tif (!res_ops.get_vsi_id \|\| !res_ops.get_eth_id \|\|\n+\t !res_ops.cfg_msix_map \|\| !res_ops.destroy_msix_map \|\|\n+\t !res_ops.set_mailbox_irq \|\| !res_ops.init_module \|\|\n+\t !res_ops.deinit_module)\n+\t\treturn ERR_PTR(-EINVAL);\n+\tres_ops_tbl-\u003eops = \u0026res_ops;\n+\tres_ops_tbl-\u003epriv = res_mgt;\n+\n+\treturn res_ops_tbl;\n+}\n+\n+static int nbl_res_ctrl_dev_setup_eth_info(struct nbl_resource_mgt res_mgt)\n+{\n+\tstruct nbl_hw_ops hw_ops = res_mgt-\u003ehw_ops_tbl-\u003eops;\n+\tstruct device dev = res_mgt-\u003ecommon-\u003edev;\n+\tstruct nbl_eth_info eth_info;\n+\tu32 eth_bitmap = 0;\n+\tu32 eth_num = 0;\n+\tu32 fw_port_num;\n+\tint i;\n+\n+\teth_info = devm_kzalloc(dev, sizeof(eth_info), GFP_KERNEL);\n+\tif (!eth_info)\n+\t\treturn -ENOMEM;\n+\n+\tres_mgt-\u003eresource_info-\u003eeth_info = eth_info;\n+\n+\tfw_port_num = res_mgt-\u003eresource_info-\u003eboard_info.eth_num;\n+\n+\thw_ops-\u003eget_fw_eth_map(res_mgt-\u003ehw_ops_tbl-\u003epriv, \u0026eth_bitmap);\n+\tif (eth_bitmap \u0026 ~((1 \u003c\u003c NBL_MAX_ETHERNET) - 1)) {\n+\t\tdev_err(dev, \"FW reported invalid eth_bitmap 0x%x\\n\",\n+\t\t\teth_bitmap);\n+\t\treturn -EINVAL;\n+\t}\n+\tif (fw_port_num != hweight32(eth_bitmap)) {\n+\t\tdev_err(dev, \"FW inconsistency: port_num=%u, bitmap=0x%x\\n\",\n+\t\t\tfw_port_num, eth_bitmap);\n+\t\treturn -EINVAL;\n+\t}\n+\t/\n+\t Firmware is ready before probe. Valid port counts are 1/2/4;\n+\t * 0 (invalid config), 3 (unsupported topology), and \u003e4 (exceeds\n+\t * hardware max) are all rejected with -EINVAL.\n+\t /\n+\tif (fw_port_num == 0 \|\| fw_port_num == 3 \|\|\n+\t fw_port_num \u003e NBL_MAX_ETHERNET) {\n+\t\tdev_err(dev, \"FW reports %u Ethernet ports, unsupported (valid: 1/2/4)\\n\",\n+\t\t\tfw_port_num);\n+\t\treturn -EINVAL;\n+\t}\n+\teth_info-\u003eeth_num = fw_port_num;\n+\t/ Intentional design constraint: each PF maps to exactly one\n+\t * Ethernet port. This couples PF identity to port identity\n+\t * and is required by nbl_res_get_eth_id() which indexes\n+\t * eth_info-\u003eeth_id[] by relative PF id.\n+\t /\n+\tif (res_mgt-\u003ecommon-\u003emax_pf != eth_info-\u003eeth_num) {\n+\t\tdev_err(dev, \"Invalid PF-to-port topology: max_pf=%u, eth_num=%u\\n\",\n+\t\t\tres_mgt-\u003ecommon-\u003emax_pf, eth_info-\u003eeth_num);\n+\t\treturn -EINVAL;\n+\t}\n+\n+\t/\n+\t * Any subset of valid bitmap bits is accepted (e.g. 0/1, 0/2,\n+\t * 1/3, etc.). Firmware only needs to report the correct count\n+\t * of active ports; no hard-coded fixed bit positions required.\n+\t * eth_id[] is filled in ascending bitmap-bit order, so the Nth\n+\t * relative PF owns the Nth logical port (max_pf == eth_num is\n+\t * enforced above).\n+\t /\n+\tfor (i = 0; i \u003c NBL_MAX_ETHERNET; i++) {\n+\t\tif ((1 \u003c\u003c i) \u0026 eth_bitmap) {\n+\t\t\teth_info-\u003eeth_id[eth_num] = i;\n+\t\t\teth_num++;\n+\t\t}\n+\t}\n+\n+\treturn 0;\n+}\n+\n+static int nbl_res_ctrl_dev_sriov_info_init(struct nbl_resource_mgt res_mgt)\n+{\n+\tstruct nbl_hw_ops hw_ops = res_mgt-\u003ehw_ops_tbl-\u003eops;\n+\tstruct nbl_hw_mgt p = res_mgt-\u003ehw_ops_tbl-\u003epriv;\n+\tstruct nbl_common_info common = res_mgt-\u003ecommon;\n+\tstruct nbl_sriov_info sriov_info;\n+\tstruct device dev = common-\u003edev;\n+\tu8 hw_bus = 0;\n+\tu16 function;\n+\tu16 func_id;\n+\n+\thw_ops-\u003eget_real_bus(p, \u0026hw_bus);\n+\tif (common-\u003efunction + common-\u003emax_pf \u003e NBL_MAX_PF) {\n+\t\tdev_err(dev, \"PF count exceeds available function space\\n\");\n+\t\treturn -EINVAL;\n+\t}\n+\tsriov_info = devm_kcalloc(dev, common-\u003emax_pf,\n+\t\t\t\t sizeof(sriov_info), GFP_KERNEL);\n+\tif (!sriov_info)\n+\t\treturn -ENOMEM;\n+\n+\tres_mgt-\u003eresource_info-\u003esriov_info = sriov_info;\n+\t/\n+\t Real bus number of the control PF;\n+\t * the BDF table below is built from it.\n+\t /\n+\tcommon-\u003ehw_bus = hw_bus;\n+\n+\tfor (func_id = 0; func_id \u003c common-\u003emax_pf; func_id++) {\n+\t\tsriov_info = res_mgt-\u003eresource_info-\u003esriov_info + func_id;\n+\t\tfunction = common-\u003efunction + func_id;\n+\t\tsriov_info-\u003ebdf = PCI_DEVID(common-\u003ehw_bus,\n+\t\t\t\t\t PCI_DEVFN(common-\u003edevid, function));\n+\t}\n+\n+\treturn 0;\n+}\n+\n+static int nbl_res_ctrl_dev_vsi_info_init(struct nbl_resource_mgt res_mgt)\n+{\n+\tstruct nbl_eth_info eth_info = res_mgt-\u003eresource_info-\u003eeth_info;\n+\tstruct nbl_common_info common = res_mgt-\u003ecommon;\n+\tstruct device dev = common-\u003edev;\n+\tstruct nbl_vsi_info vsi_info;\n+\tint i;\n+\n+\tvsi_info = devm_kzalloc(dev, sizeof(vsi_info), GFP_KERNEL);\n+\tif (!vsi_info)\n+\t\treturn -ENOMEM;\n+\n+\tres_mgt-\u003eresource_info-\u003evsi_info = vsi_info;\n+\t/\n+\t * case 1 one port(1pf)\n+\t * pf0 (NBL_VSI_SERV_PF_DATA_TYPE) vsi is 0\n+\t * case 2 two port(2pf)\n+\t * pf0,pf1(NBL_VSI_SERV_PF_DATA_TYPE) vsi is 0,512\n+\t * case 3 four port(4pf)\n+\t * pf0,pf1,pf2,pf3(NBL_VSI_SERV_PF_DATA_TYPE) vsi is 0,256,512,768\n+\t /\n+\n+\tvsi_info-\u003enum = eth_info-\u003eeth_num;\n+\t/\n+\t * eth_num can be 1/2/4:\n+\t * - 2/4 ports use dedicated gap constants;\n+\t * - 1 port falls back to NBL_DEFAULT_VSI_ID_GAP (1024).\n+\t * All three values produce valid base_id offsets.\n+\t /\n+\tfor (i = 0; i \u003c vsi_info-\u003enum; i++) {\n+\t\tvsi_info-\u003eserv_info[i][NBL_VSI_SERV_PF_DATA_TYPE].base_id =\n+\t\t\ti nbl_vsi_id_gap(vsi_info-\u003enum);\n+\t\tvsi_info-\u003eserv_info[i][NBL_VSI_SERV_PF_DATA_TYPE].num = 1;\n+\t}\n+\n+\treturn 0;\n+}\n+\n+static int nbl_res_init_pf_num(struct nbl_resource_mgt res_mgt)\n+{\n+\tstruct nbl_hw_ops hw_ops = res_mgt-\u003ehw_ops_tbl-\u003eops;\n+\tu32 exp_contiguous_mask = 0;\n+\tu32 pf_mask = 0;\n+\tu32 pf_num = 0;\n+\tint i;\n+\n+\thw_ops-\u003eget_host_pf_mask(res_mgt-\u003ehw_ops_tbl-\u003epriv, \u0026pf_mask);\n+\n+\t/\n+\t k_pf_mask register rule:\n+\t * bit N == 0 -\u003e PF#N enabled; bit N == 1 -\u003e PF#N masked out.\n+\t * Hardware constraint: bit0 is PF0's mask bit; driver requires\n+\t * PF0 enabled as management PF, so bit0 must be clear.\n+\t * All-zero pf_mask means all PF0~PF7 are enabled, which is unsupported\n+\t * by the driver\n+\t \n+\t Product firmware constraint: only 3 valid configurations supported:\n+\t * 1 PF (PF0 only): pf_num = 1, mask = 0xfe\n+\t * 2 PFs (PF0,PF1): pf_num = 2, mask = 0xfc\n+\t * 4 PFs (PF0~PF3): pf_num = 4, mask = 0xf0\n+\t * No other PF count or sparse/non-contiguous PF layout is allowed.\n+\t \n+\t This is the only validation of the mask in the driver. The channel\n+\t * layer reads the same raw register value to program the mailbox\n+\t * QINFO table and assumes the layout checked here, so it depends on\n+\t * this running first: resource init completes before the channel\n+\t * layer is set up.\n+\t /\n+\tfor (i = 0; i \u003c NBL_MAX_PF; i++) {\n+\t\tif (!(pf_mask \u0026 (1 \u003c\u003c i)))\n+\t\t\tpf_num++;\n+\t}\n+\n+\t/\n+\t * Sanity check: enabled PFs must be contiguous starting from PF0.\n+\t * Current resource framework uses relative PF id, sparse PF layout\n+\t * will cause mismatch between resource layer and hardware func_id.\n+\t /\n+\tfor (i = 0; i \u003c pf_num; i++)\n+\t\texp_contiguous_mask \|= BIT(i);\n+\tif ((pf_mask \u0026 exp_contiguous_mask) != 0) {\n+\t\tdev_err(res_mgt-\u003ecommon-\u003edev,\n+\t\t\t\"pf_mask 0x%08x: non-contiguous enabled PF, unsupported\\n\",\n+\t\t\tpf_mask);\n+\t\treturn -EINVAL;\n+\t}\n+\n+\t/ Only allow product-specified PF count: 1 / 2 / 4 /\n+\tif (pf_num != 1 \u0026\u0026 pf_num != 2 \u0026\u0026 pf_num != 4) {\n+\t\tdev_err(res_mgt-\u003ecommon-\u003edev,\n+\t\t\t\"Invalid pf_num=%u (mask=0x%08x), only 1/2/4 PFs supported\\n\",\n+\t\t\tpf_num, pf_mask);\n+\t\treturn -EINVAL;\n+\t}\n+\n+\tres_mgt-\u003ecommon-\u003emax_pf = pf_num;\n+\n+\treturn 0;\n+}\n+\n+static void nbl_res_init_board_info(struct nbl_resource_mgt res_mgt)\n+{\n+\tstruct nbl_hw_ops hw_ops = res_mgt-\u003ehw_ops_tbl-\u003eops;\n+\n+\thw_ops-\u003eget_board_info(res_mgt-\u003ehw_ops_tbl-\u003epriv,\n+\t\t\t \u0026res_mgt-\u003eresource_info-\u003eboard_info);\n+}\n+\n+static int nbl_res_start(struct nbl_resource_mgt res_mgt)\n+{\n+\tstruct nbl_common_info common = res_mgt-\u003ecommon;\n+\tint ret = 0;\n+\n+\tif (common-\u003ehas_ctrl) {\n+\t\tnbl_res_init_board_info(res_mgt);\n+\n+\t\tret = nbl_res_init_pf_num(res_mgt);\n+\t\tif (ret)\n+\t\t\treturn ret;\n+\n+\t\tret = nbl_res_ctrl_dev_sriov_info_init(res_mgt);\n+\t\tif (ret)\n+\t\t\treturn ret;\n+\n+\t\tret = nbl_res_ctrl_dev_setup_eth_info(res_mgt);\n+\t\tif (ret)\n+\t\t\treturn ret;\n+\n+\t\tret = nbl_res_ctrl_dev_vsi_info_init(res_mgt);\n+\t\tif (ret)\n+\t\t\treturn ret;\n+\n+\t\tret = nbl_intr_mgt_start(res_mgt);\n+\t\tif (ret)\n+\t\t\treturn ret;\n+\t}\n+\n+\treturn 0;\n+}\n+\n+int nbl_res_init_leonis(struct nbl_adapter adap)\n+{\n+\tstruct nbl_channel_ops_tbl chan_ops_tbl = adap-\u003eintf.channel_ops_tbl;\n+\tstruct nbl_hw_ops_tbl hw_ops_tbl = adap-\u003eintf.hw_ops_tbl;\n+\tstruct nbl_common_info common = \u0026adap-\u003ecommon;\n+\tstruct nbl_resource_ops_tbl res_ops_tbl;\n+\tstruct device dev = \u0026adap-\u003epdev-\u003edev;\n+\tstruct nbl_resource_mgt res_mgt;\n+\tint ret;\n+\n+\tres_mgt = nbl_res_setup_res_mgt(common);\n+\tif (IS_ERR(res_mgt)) {\n+\t\tret = PTR_ERR(res_mgt);\n+\t\treturn ret;\n+\t}\n+\tres_mgt-\u003echan_ops_tbl = chan_ops_tbl;\n+\tres_mgt-\u003ehw_ops_tbl = hw_ops_tbl;\n+\n+\tret = nbl_res_start(res_mgt);\n+\tif (ret)\n+\t\treturn ret;\n+\n+\tres_ops_tbl = nbl_res_setup_ops(dev, res_mgt);\n+\tif (IS_ERR(res_ops_tbl)) {\n+\t\tret = PTR_ERR(res_ops_tbl);\n+\t\treturn ret;\n+\t}\n+\tadap-\u003eintf.resource_ops_tbl = res_ops_tbl;\n+\tadap-\u003ecore.res_mgt = res_mgt;\n+\n+\treturn 0;\n+}\n+\n+void nbl_res_remove_leonis(struct nbl_adapter adap)\n+{\n+\tstruct nbl_resource_mgt res_mgt = adap-\u003ecore.res_mgt;\n+\tstruct nbl_common_info common = \u0026adap-\u003ecommon;\n+\n+\tif (!res_mgt)\n+\t\treturn;\n+\n+\t/\n+\t * Tear down all MSI-X maps before destroying coherent tables.\n+\t * This is critical on the control PF, whose table carries the map\n+\t * entries installed for every configured function: they must be\n+\t * invalidated in hardware before the tables backing them are\n+\t * freed, otherwise the device keeps DMA-reading released pages.\n+\t * Sibling PFs are already unbound by the device-link ordering\n+\t * so no remote PF still has a live map entry or a running\n+\t * mailbox RPC at this point.\n+\t /\n+\tif (common-\u003ehas_ctrl \u0026\u0026 res_mgt-\u003eintr_mgt)\n+\t\tnbl_intr_mgt_stop(res_mgt);\n+\n+\t/ Note:\n+\t * per-function interrupts arrays (kcalloc) are freed by\n+\t * nbl_intr_mgt_stop().\n+\t * MSIX coherent tables are explicitly freed by dma_free_coherent()\n+\t * inside the intr destroy path, before nbl_intr_mgt_stop() returns.\n+\t * intr_mgt itself (devm_kzalloc) is released by devres after this\n+\t * function returns\n+\t /\n+}\ndiff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_leonis/nbl_resource_leonis.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_leonis/nbl_resource_leonis.h\nnew file mode 100644\nindex 00000000000000..f1cb0f23240b0c\n--- /dev/null\n+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_leonis/nbl_resource_leonis.h\n@@ -0,0 +1,12 @@\n+/ SPDX-License-Identifier: GPL-2.0 /\n+/\n+ * Copyright (c) 2026 Nebula Matrix Limited.\n+ /\n+\n+#ifndef _NBL_RESOURCE_LEONIS_H_\n+#define _NBL_RESOURCE_LEONIS_H_\n+\n+#include \"../nbl_resource.h\"\n+#include \"../nbl_interrupt.h\"\n+#include \"../nbl_chip.h\"\n+#endif\ndiff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_reg.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_reg.h\nindex e281109f502e67..00fa33eeabaab9 100644\n--- a/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_reg.h\n+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_reg.h\n@@ -8,6 +8,7 @@\n \n #include \u003clinux/types.h\u003e\n \n+#include \"../nbl_include/nbl_def_channel.h\"\n #include \"../nbl_include/nbl_def_hw.h\"\n #include \"../nbl_include/nbl_def_common.h\"\n #include \"../nbl_core.h\"\n@@ -16,6 +17,7 @@\n #define NBL_MAILBOX_BAR\t\t\t\t2\n #define NBL_RDMA_NOTIFY_LEN\t\t\t(8ULL \u003c\u003c 10)\n #define NBL_REG_NET_ONLY_LEN\t\t\t(8ULL \u003c\u003c 10)\n+#define NBL_HW_DUMMY_REG\t\t\t0x1300904\n /\n * PCI MEMORY BAR total size: 64MiB.\n /\n@@ -26,6 +28,37 @@ struct nbl_hw_mgt {\n \tu8 __iomem hw_addr;\n \tu8 __iomem mailbox_bar_hw_addr;\n \tresource_size_t mailbox_bar_size;\n+\tspinlock_t reg_lock; / Protect reg access /\n };\n \n+static inline u32 rd32(u8 __iomem addr, u64 reg)\n+{\n+\treturn readl(addr + reg);\n+}\n+\n+static inline void wr32(u8 __iomem addr, u64 reg, u32 value)\n+{\n+\twritel(value, addr + reg);\n+}\n+\n+static inline void nbl_hw_wr32(struct nbl_hw_mgt hw_mgt, u64 reg, u32 value)\n+{\n+\twr32(hw_mgt-\u003ehw_addr, reg, value);\n+}\n+\n+static inline u32 nbl_hw_rd32(struct nbl_hw_mgt hw_mgt, u64 reg)\n+{\n+\treturn rd32(hw_mgt-\u003ehw_addr, reg);\n+}\n+\n+static inline void nbl_mbx_wr32(struct nbl_hw_mgt hw_mgt, u64 reg, u32 value)\n+{\n+\twritel(value, hw_mgt-\u003emailbox_bar_hw_addr + reg);\n+}\n+\n+static inline u32 nbl_mbx_rd32(struct nbl_hw_mgt hw_mgt, u64 reg)\n+{\n+\treturn readl(hw_mgt-\u003emailbox_bar_hw_addr + reg);\n+}\n+\n #endif\ndiff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_interrupt.c b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_interrupt.c\nnew file mode 100644\nindex 00000000000000..3969392abdc64a\n--- /dev/null\n+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_interrupt.c\n@@ -0,0 +1,851 @@\n+// SPDX-License-Identifier: GPL-2.0\n+/\n+ * Copyright (c) 2026 Nebula Matrix Limited.\n+ /\n+#include \u003clinux/device.h\u003e\n+#include \u003clinux/delay.h\u003e\n+#include \u003clinux/dma-mapping.h\u003e\n+#include \u003clinux/bitfield.h\u003e\n+#include \"nbl_interrupt.h\"\n+\n+#define NBL_MSIX_DMA_SYNC_MIN_US\t1000 / us /\n+#define NBL_MSIX_DMA_SYNC_MAX_US\t1200 / us /\n+\n+/\n+ * Bounded best-effort wait for in-flight pcompleter fetches of a MSI-X\n+ * map table after that entry's VALID bit has been cleared and flushed.\n+ \n+ The chip exposes no idle/completion status for those fetches, so the\n+ * only ordering available is: clear VALID -\u003e flush_write() -\u003e wait. The\n+ * window is sized well above the tables' worst-case fetch/response\n+ * latency; it is not a guarantee, and it protects two things:\n+ * - the coherent table that __nbl_res_intr_complete_destroy_msix_map()\n+ * is about to dma_free_coherent();\n+ * - the vector set that a reconfiguration is about to hand to another\n+ * function.\n+ * A future caller that cannot tolerate the residual risk must keep the\n+ * table allocated for the lifetime of the function instead of relying on\n+ * this delay.\n+ \n+ Both are bus-master reads tagged with the control PF's BDF, so\n+ * pci_clear_master() is the only other lever, and nbl_remove() pulls it\n+ * before any teardown runs: that keeps the device from starting a new\n+ * fetch inside this window. It cannot help a sibling PF's map destroyed\n+ * at runtime, because those fetches carry the control PF's BDF and so\n+ * are unaffected by clearing bus mastering on the sibling. Runtime\n+ * destroys of sibling maps only become reachable with the mailbox RPC\n+ * added later in this series.\n+ /\n+static void nbl_intr_quiesce_wait(void)\n+{\n+\tusleep_range(NBL_MSIX_DMA_SYNC_MIN_US, NBL_MSIX_DMA_SYNC_MAX_US);\n+}\n+\n+/\n+ * Release global vector IDs back to intr_net_bmap / intr_other_bmap.\n+ * Caller must hold intr_mgt-\u003elock, passing the intr_mgt it locked:\n+ * nbl_intr_mgt_stop() clears res_mgt-\u003eintr_mgt while a caller may\n+ * already be blocked on that mutex, so it must not be re-read here.\n+ /\n+static void nbl_intr_release_bitmap(struct nbl_interrupt_mgt intr_mgt,\n+\t\t\t\t struct nbl_resource_mgt res_mgt,\n+\t\t\t\t u16 vec_buf, u16 cnt)\n+{\n+\tu16 bit;\n+\tu16 i;\n+\n+\tlockdep_assert_held(\u0026intr_mgt-\u003elock);\n+\n+\tif (!vec_buf \|\| cnt == 0)\n+\t\treturn;\n+\n+\tfor (i = 0; i \u003c cnt; i++) {\n+\t\tu16 intr_index = vec_buf[i];\n+\n+\t\tif (intr_index \u003e= NBL_NET_INTR_BASE) {\n+\t\t\tbit = intr_index - NBL_NET_INTR_BASE;\n+\t\t\tif (bit \u003c NBL_MAX_NET_INTERRUPT)\n+\t\t\t\tclear_bit(bit, intr_mgt-\u003eintr_net_bmap);\n+\t\t\telse\n+\t\t\t\tdev_warn(res_mgt-\u003ecommon-\u003edev,\n+\t\t\t\t\t \"invalid net intr index %u\\n\",\n+\t\t\t\t\t intr_index);\n+\t\t} else {\n+\t\t\tif (intr_index \u003c NBL_MAX_OTHER_INTERRUPT)\n+\t\t\t\tclear_bit(intr_index,\n+\t\t\t\t\t intr_mgt-\u003eintr_other_bmap);\n+\t\t\telse\n+\t\t\t\tdev_warn(res_mgt-\u003ecommon-\u003edev,\n+\t\t\t\t\t \"invalid other intr index %u\\n\",\n+\t\t\t\t\t intr_index);\n+\t\t}\n+\t}\n+}\n+\n+/\n+ Internal (unlocked) mailbox IRQ bind. Caller must hold\n+ * intr_mgt-\u003elock, passing the intr_mgt it locked (see\n+ * nbl_intr_release_bitmap()).\n+ \n+ The disable path deliberately carries no state check: the teardown\n+ * sequence uses it to disarm routing while func_res-\u003estate is still\n+ * CONFIGURED, and it stays harmless after nbl_intr_mgt_stop() because\n+ * the hardware op ignores gvec when en_msix=false. Only the enable\n+ * path is gated by the stopping latch and the per-function state.\n+ /\n+static int __nbl_res_intr_set_mailbox_irq(struct nbl_interrupt_mgt intr_mgt,\n+\t\t\t\t\t struct nbl_resource_mgt res_mgt,\n+\t\t\t\t\t u16 func_id, u16 vector_id,\n+\t\t\t\t\t bool en_msix)\n+{\n+\tstruct nbl_hw_ops hw_ops = res_mgt-\u003ehw_ops_tbl-\u003eops;\n+\tstruct nbl_common_info common = res_mgt-\u003ecommon;\n+\tstruct device dev = common-\u003edev;\n+\tstruct nbl_func_interrupt_resource_mng func_res;\n+\tu16 gvec;\n+\n+\tlockdep_assert_held(\u0026intr_mgt-\u003elock);\n+\n+\t/\n+\t * func_intr_res[] is indexed by the raw hardware function id; this\n+\t * bound check is the only guard on this path, so it is what keeps\n+\t * VF ids out. Whether the PF is present on this board is not\n+\t * checked here: callers pass either common-\u003emgt_pf or an id that\n+\t * nbl_res_intr_cfg_msix_map() has validated.\n+\t /\n+\tif (func_id \u003e= NBL_MAX_PF) {\n+\t\tdev_err(dev, \"func_id %u out of range\\n\", func_id);\n+\t\treturn -EINVAL;\n+\t}\n+\n+\tif (!en_msix) {\n+\t\thw_ops-\u003eset_mailbox_irq(res_mgt-\u003ehw_ops_tbl-\u003epriv,\n+\t\t\t\t\tfunc_id, false, 0);\n+\t\thw_ops-\u003eflush_write(res_mgt-\u003ehw_ops_tbl-\u003epriv);\n+\t\treturn 0;\n+\t}\n+\n+\t/\n+\t * Enable path: the map must be live and not under teardown,\n+\t * otherwise routing would point at a vector that the complete\n+\t * phase is about to release and never re-disables.\n+\t /\n+\tif (intr_mgt-\u003estopping)\n+\t\treturn -ESHUTDOWN;\n+\n+\tfunc_res = \u0026intr_mgt-\u003efunc_intr_res[func_id];\n+\tif (func_res-\u003estate != NBL_INTR_FUNC_CONFIGURED) {\n+\t\tdev_err(dev, \"func %u MSIX map not configured (state %u)\\n\",\n+\t\t\tfunc_id, func_res-\u003estate);\n+\t\treturn -ENODEV;\n+\t}\n+\n+\tif (vector_id \u003e= func_res-\u003enum_interrupts) {\n+\t\tdev_err(dev, \"vector_id %u out of range (max %u)\\n\",\n+\t\t\tvector_id, func_res-\u003enum_interrupts - 1);\n+\t\treturn -EINVAL;\n+\t}\n+\n+\tgvec = func_res-\u003einterrupts[vector_id];\n+\thw_ops-\u003eset_mailbox_irq(res_mgt-\u003ehw_ops_tbl-\u003epriv, func_id,\n+\t\t\t\ten_msix, gvec);\n+\thw_ops-\u003eflush_write(res_mgt-\u003ehw_ops_tbl-\u003epriv);\n+\n+\treturn 0;\n+}\n+\n+/\n+ * Internal (unlocked) MSI-X map teardown prepare phase: only hardware\n+ * register operations. The DMA address is retained and only the VALID\n+ * bit is cleared; zeroing the address (Stage 2) is deferred to the\n+ * complete phase after the hardware-DMA quiesce window.\n+ \n+ Caller must hold intr_mgt-\u003elock.\n+ /\n+static int\n+__nbl_res_intr_prepare_destroy_msix_map(struct nbl_interrupt_mgt intr_mgt,\n+\t\t\t\t\tstruct nbl_resource_mgt res_mgt,\n+\t\t\t\t\tu16 func)\n+{\n+\tstruct nbl_hw_ops hw_ops = res_mgt-\u003ehw_ops_tbl-\u003eops;\n+\tstruct nbl_func_interrupt_resource_mng func_res;\n+\tu16 interrupts;\n+\tu16 intr_num, i;\n+\tint ret;\n+\n+\tlockdep_assert_held(\u0026intr_mgt-\u003elock);\n+\n+\tif (func \u003e= NBL_MAX_PF) {\n+\t\tdev_err(res_mgt-\u003ecommon-\u003edev, \"Invalid func_id %u\\n\", func);\n+\t\treturn -EINVAL;\n+\t}\n+\n+\tfunc_res = \u0026intr_mgt-\u003efunc_intr_res[func];\n+\tif (func_res-\u003estate != NBL_INTR_FUNC_CONFIGURED)\n+\t\treturn 0;\n+\n+\tinterrupts = func_res-\u003einterrupts;\n+\tintr_num = func_res-\u003enum_interrupts;\n+\n+\t/* Step 0: disable mailbox IRQ routing before tearing down map /\n+\tret = __nbl_res_intr_set_mailbox_irq(intr_mgt, res_mgt, func, 0, false);\n+\tif (ret) {\n+\t\tdev_err(res_mgt-\u003ecommon-\u003edev,\n+\t\t\t\"disable mailbox irq failed, func=%u ret=%d\\n\",\n+\t\t\tfunc, ret);\n+\t\treturn ret;\n+\t}\n+\n+\t/ Step 1: invalidate each MSIX info entry in hardware first /\n+\tfor (i = 0; i \u003c intr_num; i++) {\n+\t\thw_ops-\u003ecfg_msix_info(res_mgt-\u003ehw_ops_tbl-\u003epriv,\n+\t\t\t\t func, false, interrupts[i],\n+\t\t\t\t 0, 0, 0, false);\n+\t}\n+\thw_ops-\u003eflush_write(res_mgt-\u003ehw_ops_tbl-\u003epriv);\n+\n+\t/\n+\t * Stage 1: retain the DMA address, only clear the VALID bit.\n+\t * Stage 2 runs after the quiesce window in the complete phase.\n+\t /\n+\thw_ops-\u003ecfg_msix_map(res_mgt-\u003ehw_ops_tbl-\u003epriv, func,\n+\t\t\t false, func_res-\u003emsix_map_table.dma,\n+\t\t\t 0, 0, 0);\n+\thw_ops-\u003eflush_write(res_mgt-\u003ehw_ops_tbl-\u003epriv);\n+\n+\tfunc_res-\u003estate = NBL_INTR_FUNC_DESTROYING;\n+\n+\treturn 0;\n+}\n+\n+/\n+ * __nbl_res_intr_complete_destroy_msix_map - finish hardware teardown and\n+ * release vector bitmap, DMA memory and interrupt buffer after the\n+ * hardware quiesce window has elapsed.\n+ \n+ Caller must hold intr_mgt-\u003elock.\n+ /\n+static int\n+__nbl_res_intr_complete_destroy_msix_map(struct nbl_interrupt_mgt intr_mgt,\n+\t\t\t\t\t struct nbl_resource_mgt res_mgt,\n+\t\t\t\t\t u16 func_id)\n+{\n+\tstruct nbl_hw_ops hw_ops = res_mgt-\u003ehw_ops_tbl-\u003eops;\n+\tstruct nbl_func_interrupt_resource_mng func_res;\n+\tstruct nbl_msix_map_table msix_map_table;\n+\tstruct device dev = res_mgt-\u003ecommon-\u003edev;\n+\tu16 interrupts;\n+\tu16 intr_num;\n+\n+\tlockdep_assert_held(\u0026intr_mgt-\u003elock);\n+\n+\tif (func_id \u003e= NBL_MAX_PF) {\n+\t\tdev_err(dev, \"Invalid func_id %u\\n\", func_id);\n+\t\treturn -EINVAL;\n+\t}\n+\n+\tfunc_res = \u0026intr_mgt-\u003efunc_intr_res[func_id];\n+\tif (func_res-\u003estate != NBL_INTR_FUNC_DESTROYING)\n+\t\treturn 0;\n+\n+\t/\n+\t Stage 2: the quiesce window has elapsed, it is now safe to\n+\t * zero the DMA base address in the hardware map register.\n+\t /\n+\thw_ops-\u003ecfg_msix_map(res_mgt-\u003ehw_ops_tbl-\u003epriv, func_id,\n+\t\t\t false, 0, 0, 0, 0);\n+\thw_ops-\u003eflush_write(res_mgt-\u003ehw_ops_tbl-\u003epriv);\n+\n+\tinterrupts = func_res-\u003einterrupts;\n+\tintr_num = func_res-\u003enum_interrupts;\n+\tmsix_map_table = \u0026func_res-\u003emsix_map_table;\n+\n+\tif (interrupts) {\n+\t\tnbl_intr_release_bitmap(intr_mgt, res_mgt, interrupts,\n+\t\t\t\t\tintr_num);\n+\t\tkfree(interrupts);\n+\t}\n+\n+\t/\n+\t * Release the coherent table independently of interrupts so a\n+\t * partially built config (table allocated, vectors never\n+\t * published) cannot leak coherent DMA memory.\n+\t /\n+\tif (msix_map_table-\u003ebase_addr) {\n+\t\tdma_free_coherent(dev, msix_map_table-\u003esize,\n+\t\t\t\t msix_map_table-\u003ebase_addr,\n+\t\t\t\t msix_map_table-\u003edma);\n+\t\tmsix_map_table-\u003ebase_addr = NULL;\n+\t\tmsix_map_table-\u003edma = 0;\n+\t\tmsix_map_table-\u003esize = 0;\n+\t}\n+\n+\tfunc_res-\u003einterrupts = NULL;\n+\tfunc_res-\u003enum_interrupts = 0;\n+\tfunc_res-\u003enum_net_interrupts = 0;\n+\tfunc_res-\u003estate = NBL_INTR_FUNC_IDLE;\n+\n+\treturn 0;\n+}\n+\n+/\n+ * Internal (unlocked) MSI-X map teardown. Caller must hold\n+ * intr_mgt-\u003elock for the whole sequence, including the hardware-DMA\n+ * quiesce window: dropping the lock would let a concurrent caller (or\n+ * nbl_intr_mgt_stop()) install/free state against this teardown.\n+ \n+ This is used for the single function synchronous destroy path.\n+ /\n+static int __nbl_res_intr_destroy_msix_map(struct nbl_interrupt_mgt intr_mgt,\n+\t\t\t\t\t struct nbl_resource_mgt res_mgt,\n+\t\t\t\t\t u16 func_id)\n+{\n+\tint ret;\n+\n+\tlockdep_assert_held(\u0026intr_mgt-\u003elock);\n+\n+\tif (intr_mgt-\u003estopping)\n+\t\treturn -ESHUTDOWN;\n+\n+\tret = __nbl_res_intr_prepare_destroy_msix_map(intr_mgt, res_mgt,\n+\t\t\t\t\t\t func_id);\n+\tif (ret)\n+\t\treturn ret;\n+\t/\n+\t * prepare() only transitions CONFIGURED functions; an IDLE func\n+\t * has nothing to wait for or complete.\n+\t /\n+\tif (intr_mgt-\u003efunc_intr_res[func_id].state !=\n+\t NBL_INTR_FUNC_DESTROYING)\n+\t\treturn 0;\n+\n+\tnbl_intr_quiesce_wait();\n+\n+\treturn __nbl_res_intr_complete_destroy_msix_map(intr_mgt, res_mgt,\n+\t\t\t\t\t\t\tfunc_id);\n+}\n+\n+int nbl_res_intr_destroy_msix_map(struct nbl_resource_mgt res_mgt,\n+\t\t\t\t u16 func_id)\n+{\n+\tstruct nbl_interrupt_mgt intr_mgt = res_mgt-\u003eintr_mgt;\n+\tint ret;\n+\n+\tif (!intr_mgt)\n+\t\treturn -EINVAL;\n+\n+\tmutex_lock(\u0026intr_mgt-\u003elock);\n+\tret = __nbl_res_intr_destroy_msix_map(intr_mgt, res_mgt, func_id);\n+\tmutex_unlock(\u0026intr_mgt-\u003elock);\n+\n+\treturn ret;\n+}\n+\n+/\n+ nbl_res_intr_cfg_msix_map - allocate \u0026 program MSI-X mapping table\n+ * @res_mgt: resource management instance\n+ * @func_id: target function identifier\n+ * @num_net_msix: required net data interrupt vectors\n+ * @num_others_msix: required control interrupt vectors\n+ * @net_msix_mask_en: enable mask for net interrupt entries\n+ \n+ Allocate interrupt vectors; MSIX coherent DMA table is allocated once\n+ * per function on first configuration, entries are rewritten while the\n+ * map is invalidated on subsequent reconfigurations. No free/realloc of\n+ * DMA table on vector count changes. This removes the DMA table\n+ * free/realloc cycle. On reconfiguration the map VALID bit is cleared\n+ * and the hardware-DMA quiesce window is observed (lock held) before old\n+ * vectors are recycled and the table is rewritten.\n+ \n+ Reconfiguration transiently needs room for the whole new vector set on\n+ * top of the old one, because the old set is only released after the new\n+ * allocation has succeeded (a failed reconfiguration must leave the old\n+ * configuration intact). The bitmap pools are therefore sized for peak,\n+ * not maximum, concurrent use: a reconfigure of the same or a smaller\n+ * size still fails with -EAGAIN when the pool is nearly exhausted. The\n+ * in-tree caller (nbl_dev_init_msix_cnt()) asks for one non-net vector\n+ * per PF, far below NBL_MAX_OTHER_INTERRUPT.\n+ \n+ Serialization: this function takes intr_mgt-\u003elock internally to\n+ * protect the global vector bitmaps and per-function state against\n+ * concurrent callers.\n+ \n+ Return: 0 on success, negative errno on failure\n+ /\n+int nbl_res_intr_cfg_msix_map(struct nbl_resource_mgt res_mgt,\n+\t\t\t u16 func_id, u16 num_net_msix,\n+\t\t\t u16 num_others_msix,\n+\t\t\t bool net_msix_mask_en)\n+{\n+\tstruct nbl_interrupt_mgt intr_mgt = res_mgt-\u003eintr_mgt;\n+\tstruct nbl_hw_ops hw_ops = res_mgt-\u003ehw_ops_tbl-\u003eops;\n+\tstruct nbl_common_info common = res_mgt-\u003ecommon;\n+\tstruct nbl_msix_map_table official_tbl;\n+\tstruct nbl_msix_map msix_map_entries;\n+\tstruct device dev = common-\u003edev;\n+\tu16 requested, intr_index;\n+\tu8 bus, devid, function;\n+\tbool entry_masked = false;\n+\tu16 tmp_interrupts = NULL;\n+\tu16 allocated_cnt = 0;\n+\tu16 old_interrupts;\n+\tu16 old_num;\n+\tbool had_config;\n+\tint ret = 0;\n+\tu16 gvec;\n+\tu16 i, j;\n+\n+\tif (!intr_mgt)\n+\t\treturn -EINVAL;\n+\n+\tif (!common-\u003ehas_ctrl)\n+\t\treturn -EINVAL;\n+\n+\tif (func_id \u003e= NBL_MAX_PF) {\n+\t\tdev_err(dev, \"Invalid func_id %u\\n\", func_id);\n+\t\treturn -EINVAL;\n+\t}\n+\n+\tif (num_net_msix == 0 \u0026\u0026 num_others_msix == 0) {\n+\t\tdev_err(dev, \"MSI-X vector count cannot both be zero\\n\");\n+\t\treturn -EINVAL;\n+\t}\n+\n+\tif (num_net_msix \u003e NBL_MSIX_MAP_TABLE_MAX_ENTRIES \|\|\n+\t num_others_msix \u003e NBL_MSIX_MAP_TABLE_MAX_ENTRIES) {\n+\t\tdev_err(dev, \"MSI-X count out of limit: net=%u, others=%u\\n\",\n+\t\t\tnum_net_msix, num_others_msix);\n+\t\treturn -EINVAL;\n+\t}\n+\n+\tif (check_add_overflow(num_net_msix, num_others_msix, \u0026requested) \|\|\n+\t requested \u003e NBL_MSIX_MAP_TABLE_MAX_ENTRIES) {\n+\t\tdev_err(dev, \"Total MSI-X vectors %u exceeds maximum %u\\n\",\n+\t\t\trequested, NBL_MSIX_MAP_TABLE_MAX_ENTRIES);\n+\t\treturn -EINVAL;\n+\t}\n+\n+\tret = nbl_res_func_id_to_bdf(res_mgt, func_id, \u0026bus, \u0026devid, \u0026function);\n+\tif (ret) {\n+\t\tif (ret == -EOPNOTSUPP)\n+\t\t\tdev_err(dev,\n+\t\t\t\t\"MSI-X mapping: func_id=%u is not a PF present on this board\\n\",\n+\t\t\t\tfunc_id);\n+\t\treturn ret;\n+\t}\n+\n+\tmutex_lock(\u0026intr_mgt-\u003elock);\n+\tofficial_tbl = \u0026intr_mgt-\u003efunc_intr_res[func_id].msix_map_table;\n+\n+\t/* Reject new configs once global teardown has started /\n+\tif (intr_mgt-\u003estopping) {\n+\t\tret = -ESHUTDOWN;\n+\t\tgoto out_unlock;\n+\t}\n+\n+\thad_config = intr_mgt-\u003efunc_intr_res[func_id].state ==\n+\t\t NBL_INTR_FUNC_CONFIGURED;\n+\n+\t/\n+\t * Phase1: allocate global vector array first.\n+\t * Allocate the fixed-size MSIX DMA table only ONCE for this function.\n+\t /\n+\ttmp_interrupts = kcalloc(requested, sizeof(tmp_interrupts),\n+\t\t\t\t GFP_KERNEL);\n+\tif (!tmp_interrupts) {\n+\t\tret = -ENOMEM;\n+\t\tgoto out_unlock;\n+\t}\n+\t/* Allocate MSIX DMA table once per function /\n+\tif (!official_tbl-\u003ebase_addr) {\n+\t\tofficial_tbl-\u003esize =\n+\t\t\tsizeof(struct nbl_msix_map) \n+\t\t\tNBL_MSIX_MAP_TABLE_MAX_ENTRIES;\n+\t\tofficial_tbl-\u003ebase_addr = dma_alloc_coherent(dev,\n+\t\t\t\t\t\t\t official_tbl-\u003esize,\n+\t\t\t\t\t\t\t \u0026official_tbl-\u003edma,\n+\t\t\t\t\t\t\t GFP_KERNEL);\n+\t\tif (!official_tbl-\u003ebase_addr) {\n+\t\t\tdev_err(dev, \"Failed to allocate DMA memory for MSIX table\\n\");\n+\t\t\tret = -ENOMEM;\n+\t\t\tgoto release_vecs_unlock;\n+\t\t}\n+\t}\n+\n+\t/* Allocate net interrupt vectors /\n+\tfor (i = 0; i \u003c num_net_msix; i++) {\n+\t\tintr_index = find_first_zero_bit(intr_mgt-\u003eintr_net_bmap,\n+\t\t\t\t\t\t NBL_MAX_NET_INTERRUPT);\n+\t\tif (intr_index == NBL_MAX_NET_INTERRUPT) {\n+\t\t\tdev_err(dev, \"No free net interrupt vectors left\\n\");\n+\t\t\tret = -EAGAIN;\n+\t\t\tgoto release_vecs_unlock;\n+\t\t}\n+\t\ttmp_interrupts[i] = intr_index + NBL_NET_INTR_BASE;\n+\t\tset_bit(intr_index, intr_mgt-\u003eintr_net_bmap);\n+\t\tallocated_cnt++;\n+\t}\n+\n+\t/ Allocate other interrupt vectors /\n+\tfor (; i \u003c requested; i++) {\n+\t\tintr_index =\n+\t\t\tfind_first_zero_bit(intr_mgt-\u003eintr_other_bmap,\n+\t\t\t\t\t NBL_MAX_OTHER_INTERRUPT);\n+\t\tif (intr_index == NBL_MAX_OTHER_INTERRUPT) {\n+\t\t\tdev_err(dev, \"No free control interrupt vectors left\\n\");\n+\t\t\tret = -EAGAIN;\n+\t\t\tgoto release_vecs_unlock;\n+\t\t}\n+\t\ttmp_interrupts[i] = intr_index;\n+\t\tset_bit(intr_index, intr_mgt-\u003eintr_other_bmap);\n+\t\tallocated_cnt++;\n+\t}\n+\n+\t/\n+\t * Phase2: quiesce the old hardware MSIX config before touching\n+\t * the live DMA table. Same sequence as destroy:\n+\t * disable mailbox routing -\u003e invalidate per-vector INFO -\u003e\n+\t * clear map VALID -\u003e flush -\u003e wait for in-flight table fetches.\n+\t * The lock stays held across the wait, so no concurrent caller\n+\t * can program the quiesced function. Only then are old vectors\n+\t * recycled.\n+\t * NOTE: NO DMA table free here.\n+\t /\n+\tif (had_config) {\n+\t\told_interrupts =\n+\t\t\tintr_mgt-\u003efunc_intr_res[func_id].interrupts;\n+\t\told_num = intr_mgt-\u003efunc_intr_res[func_id].num_interrupts;\n+\n+\t\tret = __nbl_res_intr_set_mailbox_irq(intr_mgt, res_mgt, func_id,\n+\t\t\t\t\t\t 0, false);\n+\t\tif (ret) {\n+\t\t\tdev_err(dev, \"%s: disable old mailbox irq failed, keep old config\\n\",\n+\t\t\t\t__func__);\n+\t\t\tgoto release_vecs_unlock;\n+\t\t}\n+\t\tfor (j = 0; j \u003c old_num; j++) {\n+\t\t\thw_ops-\u003ecfg_msix_info(res_mgt-\u003ehw_ops_tbl-\u003epriv,\n+\t\t\t\t\t func_id, false,\n+\t\t\t\t\t old_interrupts[j],\n+\t\t\t\t\t 0, 0, 0, false);\n+\t\t}\n+\t\thw_ops-\u003ecfg_msix_map(res_mgt-\u003ehw_ops_tbl-\u003epriv, func_id,\n+\t\t\t\t false, official_tbl-\u003edma,\n+\t\t\t\t 0, 0, 0);\n+\t\thw_ops-\u003eflush_write(res_mgt-\u003ehw_ops_tbl-\u003epriv);\n+\n+\t\tnbl_intr_quiesce_wait();\n+\n+\t\tnbl_intr_release_bitmap(intr_mgt, res_mgt, old_interrupts,\n+\t\t\t\t\told_num);\n+\t\tkfree(old_interrupts);\n+\t\tintr_mgt-\u003efunc_intr_res[func_id].interrupts = NULL;\n+\t\tintr_mgt-\u003efunc_intr_res[func_id].num_interrupts = 0;\n+\t\tintr_mgt-\u003efunc_intr_res[func_id].num_net_interrupts = 0;\n+\t}\n+\n+\t/ Swap new vector array into func state /\n+\tintr_mgt-\u003efunc_intr_res[func_id].interrupts = tmp_interrupts;\n+\tintr_mgt-\u003efunc_intr_res[func_id].num_interrupts = requested;\n+\tintr_mgt-\u003efunc_intr_res[func_id].num_net_interrupts = num_net_msix;\n+\ttmp_interrupts = NULL;\n+\n+\t/\n+\t * First configuration of this function. Re-invalidate the entry\n+\t * and take a quiesce window before the table below is rewritten:\n+\t * while the entry is VALID the pcompleter could fetch a table\n+\t * that is being written and route stale gvecs.\n+\t \n+\t This is a second, real quiesce window rather than part of the\n+\t * reconfiguration path, and it costs one delay per function on the\n+\t * first configuration only. Reusing the table removes the\n+\t * free/realloc cycle and the destroy-path quiesce window from\n+\t * every reconfiguration; it does not remove this one.\n+\t /\n+\tif (!had_config) {\n+\t\thw_ops-\u003ecfg_msix_map(res_mgt-\u003ehw_ops_tbl-\u003epriv, func_id,\n+\t\t\t\t false, 0, 0, 0, 0);\n+\t\thw_ops-\u003eflush_write(res_mgt-\u003ehw_ops_tbl-\u003epriv);\n+\t\tnbl_intr_quiesce_wait();\n+\t}\n+\n+\t/\n+\t * Rewrite the table in the pre-allocated DMA buffer while the\n+\t * map is invalid, so the device cannot observe a torn old/new\n+\t * mix. Only entries beyond requested count need explicit zeroing.\n+\t /\n+\tmsix_map_entries = official_tbl-\u003ebase_addr;\n+\tmemset(msix_map_entries + requested, 0,\n+\t (NBL_MSIX_MAP_TABLE_MAX_ENTRIES - requested) \n+\t sizeof(msix_map_entries));\n+\n+\tfor (i = 0; i \u003c requested; i++) {\n+\t\tgvec = intr_mgt-\u003efunc_intr_res[func_id].interrupts[i];\n+\t\tmsix_map_entries[i].data =\n+\t\t\tcpu_to_le16(FIELD_PREP(NBL_MSIX_MAP_VALID_MASK, 1) \|\n+\t\t\t\t FIELD_PREP(NBL_MSIX_MAP_INDEX_MASK,\n+\t\t\t\t\t gvec));\n+\t}\n+\n+\t/ Ensure coherent table writes are visible before HW fetch/enable /\n+\tdma_wmb();\n+\n+\t/ Enable per-vector INFO entries after the table is published /\n+\tfor (i = 0; i \u003c requested; i++) {\n+\t\tgvec = intr_mgt-\u003efunc_intr_res[func_id].interrupts[i];\n+\t\tentry_masked = (i \u003c num_net_msix \u0026\u0026 net_msix_mask_en);\n+\t\thw_ops-\u003ecfg_msix_info(res_mgt-\u003ehw_ops_tbl-\u003epriv,\n+\t\t\t\t func_id, true, gvec,\n+\t\t\t\t bus, devid, function,\n+\t\t\t\t entry_masked);\n+\t}\n+\thw_ops-\u003eflush_write(res_mgt-\u003ehw_ops_tbl-\u003epriv);\n+\n+\t/\n+\t * Point the map at the table last and set VALID.\n+\t \n+\t cfg_msix_map uses the control PF's own BDF (common-\u003ehw_bus etc.),\n+\t * not the target function's BDF. This BDF tags the pcompler DMA\n+\t * read of the MSI-X map table as originating from the control PF.\n+\t * The target function's BDF (bus/devid/function from\n+\t * nbl_res_func_id_to_bdf) is used only in cfg_msix_info for the\n+\t * host_msix_ctrl table entry BDF filtering.\n+\t /\n+\thw_ops-\u003ecfg_msix_map(res_mgt-\u003ehw_ops_tbl-\u003epriv, func_id,\n+\t\t\t true, official_tbl-\u003edma, common-\u003ehw_bus,\n+\t\t\t common-\u003edevid, common-\u003efunction);\n+\thw_ops-\u003eflush_write(res_mgt-\u003ehw_ops_tbl-\u003epriv);\n+\n+\tintr_mgt-\u003efunc_intr_res[func_id].state = NBL_INTR_FUNC_CONFIGURED;\n+\tmutex_unlock(\u0026intr_mgt-\u003elock);\n+\treturn 0;\n+\n+release_vecs_unlock:\n+\tnbl_intr_release_bitmap(intr_mgt, res_mgt, tmp_interrupts,\n+\t\t\t\tallocated_cnt);\n+\tkfree(tmp_interrupts);\n+\t/\n+\t * On a failed fresh configuration, release the DMA table\n+\t * allocated during this call. On a failed reconfiguration the\n+\t * old configuration is still intact (it is only torn down\n+\t * after all vector allocations succeed) and owns the table.\n+\t /\n+\tif (!had_config \u0026\u0026 official_tbl-\u003ebase_addr) {\n+\t\tdma_free_coherent(dev, official_tbl-\u003esize,\n+\t\t\t\t official_tbl-\u003ebase_addr,\n+\t\t\t\t official_tbl-\u003edma);\n+\t\tofficial_tbl-\u003ebase_addr = NULL;\n+\t\tofficial_tbl-\u003edma = 0;\n+\t\tofficial_tbl-\u003esize = 0;\n+\t}\n+out_unlock:\n+\tmutex_unlock(\u0026intr_mgt-\u003elock);\n+\treturn ret;\n+}\n+\n+/\n+ nbl_res_intr_set_mailbox_irq - bind mailbox IRQ to specified vector\n+ * @res_mgt: resource management instance\n+ * @func_id: target function identifier\n+ * @vector_id: index inside local interrupt array\n+ * @en_msix: enable/disable mailbox interrupt\n+ \n+ Serialization: takes intr_mgt-\u003elock internally and passes the\n+ * pointer it read (before taking the lock) down to the helpers, which\n+ * must not re-read res_mgt-\u003eintr_mgt: nbl_intr_mgt_stop() clears that\n+ * pointer while a caller may already be blocked on this mutex.\n+ \n+ Return: 0 on success, negative errno on parameter or state check\n+ * failure. The hardware op is void and cannot report failure.\n+ /\n+int nbl_res_intr_set_mailbox_irq(struct nbl_resource_mgt res_mgt,\n+\t\t\t\t u16 func_id, u16 vector_id,\n+\t\t\t\t bool en_msix)\n+{\n+\tstruct nbl_interrupt_mgt intr_mgt = res_mgt-\u003eintr_mgt;\n+\tstruct nbl_common_info common = res_mgt-\u003ecommon;\n+\tint ret;\n+\n+\tif (!intr_mgt)\n+\t\treturn -EINVAL;\n+\n+\tif (!common-\u003ehas_ctrl)\n+\t\treturn -EINVAL;\n+\n+\tmutex_lock(\u0026intr_mgt-\u003elock);\n+\tret = __nbl_res_intr_set_mailbox_irq(intr_mgt, res_mgt, func_id,\n+\t\t\t\t\t vector_id, en_msix);\n+\tmutex_unlock(\u0026intr_mgt-\u003elock);\n+\n+\treturn ret;\n+}\n+\n+/\n+ Reset the software state and invalidate every function's hardware MSI-X\n+ * map entry once. The chip-internal tables (FUNCTION_MSIX_MAP,\n+ * PADPT_HOST_MSIX_INFO, HOST_MSIX_FID_TABLE) can survive a kexec or a\n+ * forced unload without FLR and there is no .shutdown callback to scrub\n+ * them. Only FUNCTION_MSIX_MAP is scrubbed here: it is indexed by\n+ * func_id, so covering all NBL_MAX_PF entries costs one register write\n+ * each, after which the pcompleter can no longer fetch a stale table that\n+ * points into memory this kernel may have reallocated. The INFO/FID\n+ * tables are indexed by global vector id and are not cleared; instead a\n+ * map entry is always invalidated by nbl_res_intr_cfg_msix_map() and by\n+ * the teardown paths before it is armed for a new configuration.\n+ \n+ No sibling PF is live at this point: the device links make them probe\n+ * only after this PF and remove before it, so clearing all NBL_MAX_PF\n+ * entries cannot disturb a running function.\n+ /\n+static struct nbl_interrupt_mgt \n+nbl_intr_setup_mgt(struct nbl_resource_mgt res_mgt)\n+{\n+\tstruct device dev = res_mgt-\u003ecommon-\u003edev;\n+\tstruct nbl_hw_ops hw_ops = res_mgt-\u003ehw_ops_tbl-\u003eops;\n+\tstruct nbl_interrupt_mgt intr_mgt;\n+\tu16 func_id;\n+\tint err;\n+\n+\tintr_mgt = devm_kzalloc(dev, sizeof(intr_mgt), GFP_KERNEL);\n+\tif (!intr_mgt)\n+\t\treturn ERR_PTR(-ENOMEM);\n+\n+\terr = devm_mutex_init(dev, \u0026intr_mgt-\u003elock);\n+\tif (err)\n+\t\treturn ERR_PTR(err);\n+\n+\tintr_mgt-\u003estopping = false;\n+\tbitmap_zero(intr_mgt-\u003eintr_net_bmap, NBL_MAX_NET_INTERRUPT);\n+\tbitmap_zero(intr_mgt-\u003eintr_other_bmap, NBL_MAX_OTHER_INTERRUPT);\n+\n+\t/\n+\t * Invalidate any map entry left VALID by a previous kernel. One\n+\t * flush plus one quiesce window covers the outstanding table\n+\t * fetches, same as the teardown paths do.\n+\t \n+\t This is the scrub that makes it safe to enable bus mastering:\n+\t * nbl_probe() calls it with bus mastering still disabled, so no\n+\t * new fetch can start while these entries are still VALID.\n+\t /\n+\tfor (func_id = 0; func_id \u003c NBL_MAX_PF; func_id++)\n+\t\thw_ops-\u003ecfg_msix_map(res_mgt-\u003ehw_ops_tbl-\u003epriv, func_id,\n+\t\t\t\t false, 0, 0, 0, 0);\n+\thw_ops-\u003eflush_write(res_mgt-\u003ehw_ops_tbl-\u003epriv);\n+\tnbl_intr_quiesce_wait();\n+\n+\treturn intr_mgt;\n+}\n+\n+int nbl_intr_mgt_start(struct nbl_resource_mgt res_mgt)\n+{\n+\tstruct nbl_interrupt_mgt intr_mgt;\n+\tint ret;\n+\n+\tintr_mgt = nbl_intr_setup_mgt(res_mgt);\n+\tif (IS_ERR(intr_mgt)) {\n+\t\tret = PTR_ERR(intr_mgt);\n+\t\treturn ret;\n+\t}\n+\n+\tres_mgt-\u003eintr_mgt = intr_mgt;\n+\treturn 0;\n+}\n+\n+/\n+ * nbl_intr_mgt_stop - global control-PF interrupt teardown\n+ \n+ Phase 1 sets the stopping latch and invalidates every configured\n+ * function's hardware map entry while holding the lock. The lock is\n+ * dropped for the global quiesce window, so a caller that races the\n+ * window is rejected by the latch (checked under the lock), not by the\n+ * lock being held.\n+ \n+ The latch is what rejects a caller that raced the window; it is not\n+ * what makes the manager safe to touch. No caller is in flight here by\n+ * construction: nbl_remove() clears bus mastering and stops the device\n+ * before reaching this point, and every sibling PF - the only remote\n+ * source of these calls - is unbound first through its\n+ * DL_FLAG_AUTOPROBE_CONSUMER device link. A caller that did latch the\n+ * pointer earlier and then gets in under the lock hits the latch\n+ * instead:\n+ * - the config path, destroy_msix_map and the enable path of\n+ * nbl_res_intr_set_mailbox_irq() report -ESHUTDOWN. The latch is\n+ * checked ahead of that path's per-function state check, so its\n+ * -ENODEV is only reachable while the manager is not stopping.\n+ * - the en_msix=false disable path consults neither and returns 0,\n+ * which is what lets teardown disarm routing after the latch is set.\n+ * Once this returns res_mgt-\u003eintr_mgt is NULL and the public entry\n+ * points report -EINVAL.\n+ /\n+void nbl_intr_mgt_stop(struct nbl_resource_mgt res_mgt)\n+{\n+\tstruct nbl_interrupt_mgt intr_mgt = res_mgt-\u003eintr_mgt;\n+\tu16 func_id;\n+\tint ret;\n+\n+\tif (!intr_mgt)\n+\t\treturn;\n+\n+\t/\n+\t * Phase 1: set the stopping latch and invalidate all hardware\n+\t * MSIX map entries. A caller racing the quiesce window below is\n+\t * either still waiting for the lock, in which case it observes\n+\t * stopping == true at its next checkpoint once it is let in, or\n+\t * it already holds the lock, in which case this phase waits for\n+\t * it to finish. The latch is never cleared.\n+\t \n+\t The en_msix=false mailbox path is exempt from the latch on\n+\t * purpose: this phase and the per-function teardown use it to\n+\t * disarm routing, and it only rewrites bits that are already\n+\t * clear by then.\n+\t /\n+\tmutex_lock(\u0026intr_mgt-\u003elock);\n+\tintr_mgt-\u003estopping = true;\n+\tfor (func_id = 0; func_id \u003c NBL_MAX_PF; func_id++) {\n+\t\tif (intr_mgt-\u003efunc_intr_res[func_id].state ==\n+\t\t NBL_INTR_FUNC_CONFIGURED) {\n+\t\t\tdev_info(res_mgt-\u003ecommon-\u003edev,\n+\t\t\t\t \"intr_mgt_stop: preparing destroy map for func %u\\n\",\n+\t\t\t\t func_id);\n+\t\t\tret = __nbl_res_intr_prepare_destroy_msix_map(intr_mgt,\n+\t\t\t\t\t\t\t\t res_mgt,\n+\t\t\t\t\t\t\t\t func_id);\n+\t\t\tif (ret)\n+\t\t\t\tdev_warn(res_mgt-\u003ecommon-\u003edev,\n+\t\t\t\t\t \"intr_mgt_stop: prepare destroy map for func %u failed: %d\\n\",\n+\t\t\t\t\t func_id, ret);\n+\t\t}\n+\t}\n+\tmutex_unlock(\u0026intr_mgt-\u003elock);\n+\n+\t/\n+\t * Global quiesce: wait for straggler DMA table reads after all\n+\t * MSIX map entries have been invalidated in hardware, before\n+\t * freeing coherent memory. Best-effort only, see\n+\t * nbl_intr_quiesce_wait().\n+\t /\n+\tnbl_intr_quiesce_wait();\n+\n+\t/ Phase2: safely release MSIX coherent memory and intr resources /\n+\tmutex_lock(\u0026intr_mgt-\u003elock);\n+\tfor (func_id = 0; func_id \u003c NBL_MAX_PF; func_id++) {\n+\t\tif (intr_mgt-\u003efunc_intr_res[func_id].state ==\n+\t\t NBL_INTR_FUNC_DESTROYING) {\n+\t\t\tret = __nbl_res_intr_complete_destroy_msix_map(intr_mgt,\n+\t\t\t\t\t\t\t\t res_mgt,\n+\t\t\t\t\t\t\t\t func_id);\n+\t\t\tif (ret)\n+\t\t\t\tdev_warn(res_mgt-\u003ecommon-\u003edev,\n+\t\t\t\t\t \"intr_mgt_stop: complete destroy map for func %u failed: %d\\n\",\n+\t\t\t\t\t func_id, ret);\n+\t\t}\n+\t}\n+\t/\n+\t * Unpublish last. Readers do not synchronize on intr_mgt-\u003elock,\n+\t * so this only has to happen after every hardware access and\n+\t * every free above; a caller that latched the pointer earlier is\n+\t * rejected by the stopping latch.\n+\t /\n+\tres_mgt-\u003eintr_mgt = NULL;\n+\tmutex_unlock(\u0026intr_mgt-\u003elock);\n+}\ndiff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_interrupt.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_interrupt.h\nnew file mode 100644\nindex 00000000000000..9f66f5e19c984b\n--- /dev/null\n+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_interrupt.h\n@@ -0,0 +1,21 @@\n+/ SPDX-License-Identifier: GPL-2.0 /\n+/\n+ * Copyright (c) 2026 Nebula Matrix Limited.\n+ /\n+\n+#ifndef _NBL_INTERRUPT_H_\n+#define _NBL_INTERRUPT_H_\n+\n+#include \"nbl_resource.h\"\n+\n+#define NBL_MSIX_MAP_TABLE_MAX_ENTRIES\t1024\n+int nbl_res_intr_destroy_msix_map(struct nbl_resource_mgt res_mgt,\n+\t\t\t\t u16 func_id);\n+int nbl_res_intr_cfg_msix_map(struct nbl_resource_mgt res_mgt,\n+\t\t\t u16 func_id, u16 num_net_msix,\n+\t\t\t u16 num_others_msix,\n+\t\t\t bool net_msix_mask_en);\n+int nbl_res_intr_set_mailbox_irq(struct nbl_resource_mgt res_mgt,\n+\t\t\t\t u16 func_id, u16 vector_id,\n+\t\t\t\t bool en_msix);\n+#endif\ndiff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_resource.c b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_resource.c\nnew file mode 100644\nindex 00000000000000..49c7d4412e38ad\n--- /dev/null\n+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_resource.c\n@@ -0,0 +1,157 @@\n+// SPDX-License-Identifier: GPL-2.0\n+/\n+ Copyright (c) 2026 Nebula Matrix Limited.\n+ /\n+\n+#include \u003clinux/pci.h\u003e\n+#include \"nbl_resource.h\"\n+\n+int nbl_res_func_id_to_vsi_id(struct nbl_resource_mgt res_mgt, u16 func_id,\n+\t\t\t u16 type, u16 vsi_id)\n+{\n+\tstruct nbl_vsi_info vsi_info = res_mgt-\u003eresource_info-\u003evsi_info;\n+\tenum nbl_vsi_serv_type dst_type = NBL_VSI_SERV_PF_DATA_TYPE;\n+\tstruct nbl_common_info common = res_mgt-\u003ecommon;\n+\tstruct device dev = res_mgt-\u003ecommon-\u003edev;\n+\tint pfid = func_id;\n+\tu32 rel_pf_id;\n+\tint ret;\n+\n+\tif (!common-\u003ehas_ctrl \|\| !vsi_id) {\n+\t\tdev_dbg(dev, \"No control plane or null vsi output ptr\\n\");\n+\t\treturn -EINVAL;\n+\t}\n+\tret = nbl_common_func_id_to_rel_pf_id(common, pfid, \u0026rel_pf_id);\n+\tif (ret)\n+\t\treturn ret;\n+\tif (rel_pf_id \u003e= vsi_info-\u003enum) {\n+\t\tdev_err(dev, \"PF %d (diff=%u) exceeds vsi_info-\u003enum (%u)\\n\",\n+\t\t\tpfid, rel_pf_id, vsi_info-\u003enum);\n+\t\treturn -EINVAL;\n+\t}\n+\n+\tret = nbl_res_pf_dev_vsi_type_to_hw_vsi_type(res_mgt, type, \u0026dst_type);\n+\tif (ret) {\n+\t\tdev_err(dev, \"Invalid vsi type %u func_id %u\\n\", type, func_id);\n+\t\treturn ret;\n+\t}\n+\tvsi_id = vsi_info-\u003eserv_info[rel_pf_id][dst_type].base_id;\n+\treturn 0;\n+}\n+\n+int nbl_res_vsi_id_to_pf_id(struct nbl_resource_mgt res_mgt, u16 vsi_id)\n+{\n+\tstruct nbl_vsi_info vsi_info = res_mgt-\u003eresource_info-\u003evsi_info;\n+\tstruct nbl_common_info common = res_mgt-\u003ecommon;\n+\tstruct device dev = res_mgt-\u003ecommon-\u003edev;\n+\tint j = NBL_VSI_SERV_PF_DATA_TYPE;\n+\tint pf_id, i;\n+\n+\tif (!common-\u003ehas_ctrl) {\n+\t\tdev_dbg(dev, \"No control plane available\\n\");\n+\t\treturn -EINVAL;\n+\t}\n+\tfor (i = 0; i \u003c vsi_info-\u003enum; i++) {\n+\t\tif (vsi_id \u003e= vsi_info-\u003eserv_info[i][j].base_id \u0026\u0026\n+\t\t (vsi_id \u003c vsi_info-\u003eserv_info[i][j].base_id +\n+\t\t\t\t\tvsi_info-\u003eserv_info[i][j].num)) {\n+\t\t\tpf_id = i + common-\u003emgt_pf;\n+\t\t\tif (pf_id \u003e= NBL_MAX_PF) {\n+\t\t\t\tdev_err(dev, \"PF ID overflow\\n\");\n+\t\t\t\treturn -ERANGE;\n+\t\t\t}\n+\t\t\treturn pf_id;\n+\t\t}\n+\t}\n+\n+\tdev_dbg(dev, \"VSI ID %u not found\\n\", vsi_id);\n+\treturn -ENOENT;\n+}\n+\n+int nbl_res_func_id_to_bdf(struct nbl_resource_mgt res_mgt, u16 func_id,\n+\t\t\t u8 bus, u8 dev, u8 function)\n+{\n+\tstruct nbl_common_info common = res_mgt-\u003ecommon;\n+\tstruct nbl_sriov_info sriov_info;\n+\tint pfid = func_id;\n+\tu8 pf_bus, devfn;\n+\tu32 rel_pf_id;\n+\tint ret;\n+\n+\tif (!common-\u003ehas_ctrl \|\| !bus \|\| !dev \|\| !function)\n+\t\treturn -EINVAL;\n+\tret = nbl_common_func_id_to_rel_pf_id(common, pfid, \u0026rel_pf_id);\n+\tif (ret)\n+\t\treturn ret;\n+\tif (rel_pf_id \u003e= common-\u003emax_pf) {\n+\t\tdev_err(common-\u003edev,\n+\t\t\t\"func_id=%u rel_pf_id=%u exceeds max_pf=%u, no such PF on this board\\n\",\n+\t\t\tpfid, rel_pf_id,\n+\t\t\tcommon-\u003emax_pf);\n+\t\treturn -EOPNOTSUPP;\n+\t}\n+\tsriov_info = res_mgt-\u003eresource_info-\u003esriov_info + rel_pf_id;\n+\tpf_bus = PCI_BUS_NUM(sriov_info-\u003ebdf);\n+\tdevfn = sriov_info-\u003ebdf \u0026 0xff;\n+\tbus = pf_bus;\n+\tdev = PCI_SLOT(devfn);\n+\tfunction = PCI_FUNC(devfn);\n+\n+\treturn 0;\n+}\n+\n+int nbl_res_get_eth_id(struct nbl_resource_mgt res_mgt, u16 func_id,\n+\t\t u16 vsi_id, u8 eth_num, u8 eth_id, u8 logic_eth_id)\n+{\n+\tstruct nbl_eth_info eth_info = res_mgt-\u003eresource_info-\u003eeth_info;\n+\tstruct nbl_common_info common = res_mgt-\u003ecommon;\n+\tstruct device dev = res_mgt-\u003ecommon-\u003edev;\n+\tint pfid = func_id;\n+\tint rel_pf_id;\n+\tint abs_pf_id;\n+\n+\tif (!common-\u003ehas_ctrl \|\| !eth_num \|\| !eth_id \|\| !logic_eth_id)\n+\t\treturn -EINVAL;\n+\tabs_pf_id = nbl_res_vsi_id_to_pf_id(res_mgt, vsi_id);\n+\tif (abs_pf_id \u003c 0) {\n+\t\tdev_err(dev, \"Failed to get PF ID from VSI ID %u\\n\", vsi_id);\n+\t\treturn -EINVAL;\n+\t}\n+\tif (abs_pf_id != pfid) {\n+\t\tdev_err(dev, \"func_id %u does not match pf derived from vsi_id %u\\n\",\n+\t\t\tpfid, vsi_id);\n+\t\treturn -EINVAL;\n+\t}\n+\trel_pf_id = abs_pf_id - common-\u003emgt_pf;\n+\n+\tif (rel_pf_id \u003e= eth_info-\u003eeth_num) {\n+\t\tdev_err(dev, \"rel_pf_id %d out of range [0, %u)\\n\",\n+\t\t\trel_pf_id, eth_info-\u003eeth_num);\n+\t\treturn -ERANGE;\n+\t}\n+\n+\teth_num = eth_info-\u003eeth_num;\n+\teth_id = eth_info-\u003eeth_id[rel_pf_id];\n+\t/\n+\t * Logical eth id equals the relative PF id: eth_info setup enforces\n+\t * max_pf == eth_num and fills eth_id[] in ascending bitmap-bit\n+\t * order, so the Nth PF owns the Nth logical port.\n+\t /\n+\tlogic_eth_id = rel_pf_id;\n+\treturn 0;\n+}\n+\n+int nbl_res_pf_dev_vsi_type_to_hw_vsi_type(struct nbl_resource_mgt res_mgt,\n+\t\t\t\t\t u16 src_type,\n+\t\t\t\t\t enum nbl_vsi_serv_type dst_type)\n+{\n+\tswitch (src_type) {\n+\tcase NBL_VSI_DATA:\n+\t\tdst_type = NBL_VSI_SERV_PF_DATA_TYPE;\n+\t\treturn 0;\n+\tdefault:\n+\t\tdev_err_once(res_mgt-\u003ecommon-\u003edev,\n+\t\t\t \"Unsupported vsi src_type %u\\n\", src_type);\n+\t\treturn -EINVAL;\n+\t}\n+}\ndiff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_resource.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_resource.h\nnew file mode 100644\nindex 00000000000000..24cf49475bbd90\n--- /dev/null\n+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_resource.h\n@@ -0,0 +1,159 @@\n+/ SPDX-License-Identifier: GPL-2.0 /\n+/\n+ * Copyright (c) 2026 Nebula Matrix Limited.\n+ /\n+\n+#ifndef _NBL_RESOURCE_H_\n+#define _NBL_RESOURCE_H_\n+\n+#include \u003clinux/types.h\u003e\n+\n+#include \"../nbl_include/nbl_include.h\"\n+#include \"../nbl_include/nbl_def_channel.h\"\n+#include \"../nbl_include/nbl_def_hw.h\"\n+#include \"../nbl_include/nbl_def_resource.h\"\n+#include \"../nbl_include/nbl_def_common.h\"\n+#include \"../nbl_core.h\"\n+\n+struct nbl_resource_mgt;\n+\n+/ --------- INTERRUPT ---------- /\n+#define NBL_MAX_OTHER_INTERRUPT\t\t\t1024\n+#define NBL_MAX_NET_INTERRUPT\t\t\t4096\n+#define NBL_NET_INTR_BASE\t\tNBL_MAX_OTHER_INTERRUPT\n+\n+#define NBL_MSIX_MAP_VALID_MASK\t\tBIT(0)\n+#define NBL_MSIX_MAP_INDEX_MASK\t\tGENMASK(13, 1)\n+#define NBL_MSIX_MAP_RSV_MASK\t\tGENMASK(15, 14)\n+\n+struct nbl_msix_map {\n+\t__le16 data;\n+};\n+\n+struct nbl_msix_map_table {\n+\tstruct nbl_msix_map base_addr;\n+\tdma_addr_t dma;\n+\tsize_t size;\n+};\n+\n+/\n+ Per-function MSI-X resource state.\n+ \n+ IDLE - nothing configured for this function.\n+ * CONFIGURED - map valid in hardware, vectors owned by this function.\n+ * DESTROYING - map invalidated, teardown in flight: neither the vector\n+ * bitmaps nor the coherent table may be released until the\n+ * hardware-DMA quiesce window has elapsed.\n+ \n+ The single-function destroy path holds intr_mgt-\u003elock across that\n+ * window. nbl_intr_mgt_stop() cannot, because it quiesces every\n+ * function at once: it drops the lock and relies on the stopping latch\n+ * to keep a concurrent configuration from installing a map that the\n+ * in-flight teardown would free.\n+ /\n+enum nbl_intr_func_state {\n+\tNBL_INTR_FUNC_IDLE = 0,\n+\tNBL_INTR_FUNC_CONFIGURED,\n+\tNBL_INTR_FUNC_DESTROYING,\n+};\n+\n+struct nbl_func_interrupt_resource_mng {\n+\tu16 num_interrupts;\n+\tu16 num_net_interrupts;\n+\tu16 interrupts;\n+\tstruct nbl_msix_map_table msix_map_table;\n+\tu8 state; /* enum nbl_intr_func_state /\n+};\n+\n+struct nbl_interrupt_mgt {\n+\tstruct mutex lock; / Protects the bitmaps + func_intr_res[] /\n+\tDECLARE_BITMAP(intr_net_bmap, NBL_MAX_NET_INTERRUPT);\n+\tDECLARE_BITMAP(intr_other_bmap, NBL_MAX_OTHER_INTERRUPT);\n+\t/ One-way latch: set on teardown, never cleared /\n+\tbool stopping;\n+\t/\n+\t * Indexed by the raw hardware function id. Only PF entries are ever\n+\t * used: the public entry points reject ids outside the PF range (VFs\n+\t * included) with -EINVAL, and nbl_res_intr_cfg_msix_map() also\n+\t * rejects a PF that is not present on this board. The destroy and\n+\t * mailbox paths only bound-check NBL_MAX_PF; they rely on their\n+\t * callers passing either common-\u003emgt_pf or an id that the config\n+\t * path has already validated.\n+\t /\n+\tstruct nbl_func_interrupt_resource_mng func_intr_res[NBL_MAX_PF];\n+};\n+\n+/ --------- INFO ---------- /\n+struct nbl_sriov_info {\n+\tunsigned int bdf;\n+};\n+\n+struct nbl_eth_info {\n+\tu8 eth_num;\n+\tu8 resv[3];\n+\tu8 eth_id[NBL_MAX_ETHERNET];\n+};\n+\n+enum nbl_vsi_serv_type {\n+\tNBL_VSI_SERV_PF_DATA_TYPE,\n+\tNBL_VSI_SERV_MAX_TYPE,\n+};\n+\n+struct nbl_vsi_serv_info {\n+\tu16 base_id;\n+\tu16 num;\n+};\n+\n+struct nbl_vsi_info {\n+\tu16 num;\n+\tstruct nbl_vsi_serv_info serv_info[NBL_MAX_ETHERNET]\n+\t\t\t\t\t [NBL_VSI_SERV_MAX_TYPE];\n+};\n+\n+struct nbl_resource_info {\n+\tstruct nbl_sriov_info sriov_info;\n+\tstruct nbl_eth_info eth_info;\n+\tstruct nbl_vsi_info vsi_info;\n+\tstruct nbl_board_port_info board_info;\n+};\n+\n+struct nbl_resource_mgt {\n+\tstruct nbl_common_info common;\n+\tstruct nbl_resource_info resource_info;\n+\tstruct nbl_channel_ops_tbl chan_ops_tbl;\n+\tstruct nbl_hw_ops_tbl hw_ops_tbl;\n+\t/\n+\t Published interrupt manager, control PF only. The public entry\n+\t * points read this pointer without the manager lock and then pass\n+\t * the value they read down to the internal helpers;\n+\t * nbl_intr_mgt_stop() clears it under the manager lock as its last\n+\t * step, after every hardware access and every free, so a caller\n+\t * that reads it after that gets -EINVAL.\n+\t \n+\t The object is devres-owned and is released together with the\n+\t * device rather than by nbl_intr_mgt_stop(), so a latched pointer\n+\t * is never dereferenced into freed memory. That is not what keeps\n+\t * a concurrent caller safe, though: see nbl_intr_mgt_stop() for\n+\t * the ordering that makes sure there is none. A caller that\n+\t * latched the pointer before it was cleared and then gets in under\n+\t * the lock hits the stopping latch: the config path,\n+\t * destroy_msix_map and the mailbox-enable path report -ESHUTDOWN,\n+\t * while the mailbox-disable path (en_msix == false) returns 0\n+\t * without consulting the latch.\n+\t /\n+\tstruct nbl_interrupt_mgt intr_mgt;\n+};\n+\n+int nbl_res_vsi_id_to_pf_id(struct nbl_resource_mgt res_mgt, u16 vsi_id);\n+int nbl_res_func_id_to_vsi_id(struct nbl_resource_mgt res_mgt, u16 func_id,\n+\t\t\t u16 type, u16 vsi_id);\n+int nbl_res_func_id_to_bdf(struct nbl_resource_mgt res_mgt, u16 func_id,\n+\t\t\t u8 bus, u8 dev, u8 function);\n+int nbl_res_get_eth_id(struct nbl_resource_mgt res_mgt, u16 func_id,\n+\t\t u16 vsi_id, u8 eth_num, u8 eth_id, u8 logic_eth_id);\n+int nbl_intr_mgt_start(struct nbl_resource_mgt res_mgt);\n+int nbl_res_pf_dev_vsi_type_to_hw_vsi_type(struct nbl_resource_mgt res_mgt,\n+\t\t\t\t\t u16 src_type,\n+\t\t\t\t\t enum nbl_vsi_serv_type dst_type);\n+void nbl_intr_mgt_stop(struct nbl_resource_mgt res_mgt);\n+#endif\ndiff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_channel.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_channel.h\nnew file mode 100644\nindex 00000000000000..5aa16d4cd4321f\n--- /dev/null\n+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_channel.h\n@@ -0,0 +1,175 @@\n+/ SPDX-License-Identifier: GPL-2.0 /\n+/\n+ * Copyright (c) 2026 Nebula Matrix Limited.\n+ /\n+\n+#ifndef _NBL_DEF_CHANNEL_H_\n+#define _NBL_DEF_CHANNEL_H_\n+\n+#include \u003clinux/types.h\u003e\n+#include \u003clinux/build_bug.h\u003e\n+\n+struct nbl_channel_mgt;\n+struct nbl_adapter;\n+\n+typedef void (nbl_chan_resp)(void , u16, u16, void , u32);\n+\n+/\n+ Mailbox wire opcodes, stable wire ABI shared between driver and firmware.\n+ * Each opcode has a fixed assigned number to preserve compatibility.\n+ * ABI compatibility rules:\n+ * 1. New opcodes shall only be appended before NBL_CHAN_MSG_MAILBOX_MAX;\n+ * 2. Reordering, inserting or deleting existing enumerators breaks driver-\n+ * firmware interoperability and must be avoided;\n+ * 3. Modifications to existing opcodes require synchronized firmware ABI\n+ * updates.\n+ \n+ Only opcodes currently used by in-tree driver logic are defined here.\n+ * Unimplemented feature opcodes (KTLS, IPsec, vDPA, mirror etc.) will be\n+ * added incrementally together with their corresponding driver\n+ * implementation patches.\n+ /\n+enum nbl_chan_msg_type {\n+\tNBL_CHAN_MSG_ACK = 0,\n+\tNBL_CHAN_MSG_CONFIGURE_MSIX_MAP = 17,\n+\tNBL_CHAN_MSG_DESTROY_MSIX_MAP = 18,\n+\tNBL_CHAN_MSG_MAILBOX_SET_IRQ = 19,\n+\tNBL_CHAN_MSG_GET_VSI_ID = 21,\n+\tNBL_CHAN_MSG_GET_ETH_ID = 67,\n+\t/ mailbox msg end /\n+\tNBL_CHAN_MSG_MAILBOX_MAX,\n+};\n+\n+enum nbl_chan_state {\n+\tNBL_CHAN_IRQ_RDY,\n+\tNBL_CHAN_STATE_NBITS\n+};\n+\n+struct nbl_chan_param_cfg_msix_map {\n+\t__le16 num_net_msix;\n+\t__le16 num_others_msix;\n+\t__le16 msix_mask_en;\n+\t__le16 rsvd;\n+};\n+\n+struct nbl_chan_param_set_mailbox_irq {\n+\t__le16 vector_id;\n+\tu8 en_msix;\n+\tu8 rsvd;\n+};\n+\n+struct nbl_chan_param_get_vsi_id {\n+\t__le16 vsi_id;\n+\t__le16 type;\n+};\n+\n+struct nbl_chan_param_get_eth_id {\n+\t__le16 vsi_id;\n+\tu8 eth_num;\n+\tu8 eth_id;\n+\tu8 logic_eth_id;\n+\tu8 rsvd[3];\n+};\n+\n+struct nbl_board_port_info {\n+\tu8 eth_num;\n+\tu8 eth_speed;\n+\tu8 p4_version;\n+\tu8 rsv[5];\n+};\n+\n+static_assert(sizeof(struct nbl_chan_param_cfg_msix_map) == 8,\n+\t \"nbl_chan_param_cfg_msix_map size must be 8 bytes\");\n+static_assert(sizeof(struct nbl_chan_param_set_mailbox_irq) == 4,\n+\t \"nbl_chan_param_set_mailbox_irq size must be 4 bytes\");\n+static_assert(sizeof(struct nbl_chan_param_get_vsi_id) == 4,\n+\t \"nbl_chan_param_get_vsi_id size must be 4 bytes\");\n+static_assert(sizeof(struct nbl_chan_param_get_eth_id) == 8,\n+\t \"nbl_chan_param_get_eth_id size must be 8 bytes\");\n+static_assert(sizeof(struct nbl_board_port_info) == 8,\n+\t \"nbl_board_port_info size must be 8 bytes\");\n+\n+struct nbl_chan_send_info {\n+\tvoid arg;\n+\tsize_t arg_len;\n+\tvoid resp;\n+\tsize_t resp_len;\n+\tu16 dstid;\n+\tu16 msg_type;\n+\tu16 ack;\n+\tu16 ack_len;\n+};\n+\n+struct nbl_chan_ack_info {\n+\tvoid data;\n+\tint err;\n+\tu32 data_len;\n+\tu16 dstid;\n+\tu16 msg_type;\n+\tu16 msgid;\n+};\n+\n+enum nbl_channel_type {\n+\tNBL_CHAN_TYPE_MAILBOX,\n+\tNBL_CHAN_TYPE_MAX\n+};\n+\n+static inline void\n+nbl_chan_fill_send_info(struct nbl_chan_send_info info,\n+\t\t\tu16 dst_id, u16 msg_type,\n+\t\t\tvoid argument, u32 arg_length,\n+\t\t\tvoid response, u32 resp_length,\n+\t\t\tbool need_ack)\n+{\n+\tinfo-\u003edstid = dst_id;\n+\tinfo-\u003emsg_type = msg_type;\n+\tinfo-\u003earg = argument;\n+\tinfo-\u003earg_len = arg_length;\n+\tinfo-\u003eresp = response;\n+\tinfo-\u003eresp_len = resp_length;\n+\tinfo-\u003eack = need_ack;\n+}\n+\n+static inline void\n+nbl_chan_fill_ack_info(struct nbl_chan_ack_info info,\n+\t\t u16 dst_id, u16 msg_type, u16 msg_id,\n+\t\t int err_code, void ack_data, u32 data_length)\n+{\n+\tinfo-\u003edstid = dst_id;\n+\tinfo-\u003emsg_type = msg_type;\n+\tinfo-\u003emsgid = msg_id;\n+\tinfo-\u003eerr = err_code;\n+\tinfo-\u003edata = ack_data;\n+\tinfo-\u003edata_len = data_length;\n+}\n+\n+struct nbl_channel_ops {\n+\tint (send_msg)(struct nbl_channel_mgt chan_mgt,\n+\t\t\tstruct nbl_chan_send_info chan_send);\n+\tint (send_ack)(struct nbl_channel_mgt chan_mgt,\n+\t\t\tstruct nbl_chan_ack_info chan_ack);\n+\tint (register_msg)(struct nbl_channel_mgt chan_mgt, u16 msg_type,\n+\t\t\t nbl_chan_resp func, void callback_priv);\n+\tvoid (cfg_chan_qinfo_map_table)(struct nbl_channel_mgt chan_mgt,\n+\t\t\t\t\t u8 bus, u8 devid);\n+\tbool (check_queue_exist)(struct nbl_channel_mgt chan_mgt,\n+\t\t\t\t u8 chan_type);\n+\tint (setup_queue)(struct nbl_channel_mgt chan_mgt, u8 chan_type);\n+\tint (teardown_queue)(struct nbl_channel_mgt chan_mgt, u8 chan_type);\n+\tvoid (clean_queue_subtask)(struct nbl_channel_mgt chan_mgt,\n+\t\t\t\t u8 chan_type);\n+\tvoid (register_chan_task)(struct nbl_channel_mgt chan_mgt,\n+\t\t\t\t u8 chan_type, struct work_struct task);\n+\tvoid (set_queue_state)(struct nbl_channel_mgt chan_mgt,\n+\t\t\t\tenum nbl_chan_state state, u8 chan_type,\n+\t\t\t\tu8 set);\n+};\n+\n+struct nbl_channel_ops_tbl {\n+\tstruct nbl_channel_ops ops;\n+\tstruct nbl_channel_mgt priv;\n+};\n+\n+int nbl_chan_init_common(struct nbl_adapter adapter);\n+void nbl_chan_remove_common(struct nbl_adapter adapter);\n+#endif\ndiff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_common.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_common.h\nindex ea646d1efc2dfb..3e18a2a97850e6 100644\n--- a/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_common.h\n+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_common.h\n@@ -11,7 +11,24 @@\n #include \u003clinux/device.h\u003e\n #include \"nbl_include.h\"\n \n+#define NBL_TWO_ETHERNET_PORT\t\t\t2\n+#define NBL_FOUR_ETHERNET_PORT\t\t\t4\n+#define NBL_DEFAULT_VSI_ID_GAP\t\t\t1024\n+#define NBL_TWO_ETHERNET_VSI_ID_GAP\t\t512\n+#define NBL_FOUR_ETHERNET_VSI_ID_GAP\t\t256\n+\n+static inline u32 nbl_vsi_id_gap(u32 m)\n+{\n+\tif (m == NBL_FOUR_ETHERNET_PORT)\n+\t\treturn NBL_FOUR_ETHERNET_VSI_ID_GAP;\n+\telse if (m == NBL_TWO_ETHERNET_PORT)\n+\t\treturn NBL_TWO_ETHERNET_VSI_ID_GAP;\n+\n+\treturn NBL_DEFAULT_VSI_ID_GAP;\n+}\n+\n struct nbl_common_info {\n+\tstruct workqueue_struct wq;\n \tstruct pci_dev pdev;\n \tstruct device dev;\n \tu16 vsi_id;\n@@ -23,9 +40,15 @@ struct nbl_common_info {\n \tu8 devid;\n \tu8 bus;\n \tu8 hw_bus;\n+\tu16 mgt_pf;\n \n \tu8 has_ctrl;\n \tu8 has_net;\n+\tu8 max_pf;\n };\n \n+void nbl_common_destroy_wq(struct nbl_common_info common);\n+int nbl_common_create_wq(struct nbl_common_info common);\n+int nbl_common_func_id_to_rel_pf_id(struct nbl_common_info common, u32 pf_id,\n+\t\t\t\t u32 rel_pf_id);\n #endif\ndiff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_dev.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_dev.h\nnew file mode 100644\nindex 00000000000000..a66c633a0e7af7\n--- /dev/null\n+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_dev.h\n@@ -0,0 +1,16 @@\n+/* SPDX-License-Identifier: GPL-2.0 /\n+/\n+ * Copyright (c) 2026 Nebula Matrix Limited.\n+ /\n+\n+#ifndef _NBL_DEF_DEV_H_\n+#define _NBL_DEF_DEV_H_\n+\n+struct nbl_adapter;\n+\n+int nbl_dev_init(struct nbl_adapter adapter);\n+void nbl_dev_remove(struct nbl_adapter adapter);\n+int nbl_dev_start(struct nbl_adapter adapter);\n+void nbl_dev_stop(struct nbl_adapter adapter);\n+\n+#endif\ndiff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_dispatch.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_dispatch.h\nnew file mode 100644\nindex 00000000000000..7da9e242c621e4\n--- /dev/null\n+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_dispatch.h\n@@ -0,0 +1,83 @@\n+/ SPDX-License-Identifier: GPL-2.0 /\n+/\n+ * Copyright (c) 2026 Nebula Matrix Limited.\n+ /\n+\n+#ifndef _NBL_DEF_DISPATCH_H_\n+#define _NBL_DEF_DISPATCH_H_\n+\n+#include \u003clinux/types.h\u003e\n+\n+struct nbl_dispatch_mgt;\n+struct nbl_adapter;\n+enum {\n+\tNBL_DISP_CTRL_LVL_MGT,\n+\tNBL_DISP_CTRL_LVL_NET,\n+\tNBL_DISP_CTRL_LVL_MAX,\n+};\n+\n+/\n+ struct nbl_dispatch_ops - dispatch control plane operation callbacks\n+ * @init_module: chip-wide datapath init, forwarded to the resource layer\n+ * (res_ops-\u003einit_module() and then hw_ops-\u003einit_module());\n+ * on leonis it programs the chip-wide datapath registers and\n+ * sets driver_status=true. Returns -EOPNOTSUPP if the\n+ * resource layer installs no init_module op. Control-PF\n+ * exclusive: the pointer is NULL on a function without\n+ * has_ctrl, so the caller must check has_ctrl first.\n+ * @deinit_module: chip-wide teardown, forwarded the same way; on leonis\n+ * it clears driver_status, which starts an asynchronous\n+ * firmware per-PF cleanup pass with no completion status\n+ * bit, waited for best-effort only. The caller must have\n+ * stopped the mailbox channel and drained all in-flight\n+ * mailbox DMA before calling it, see nbl_dev_remove().\n+ * Returns void. Control-PF exclusive, same NULL rule as\n+ * @init_module.\n+ * @cfg_msix_map: configure function msix mapping table\n+ * @destroy_msix_map: tear down msix mapping resource\n+ * @set_mailbox_irq: bind mailbox interrupt to specified msix vector\n+ * @get_vsi_id: resolve VSI ID by type\n+ * @get_eth_id: resolve eth port info from VSI ID\n+ \n+ These callbacks are not dispatch-layer setup/teardown: the dispatch\n+ * structures are devm-allocated in nbl_disp_init(), so nbl_disp_remove()\n+ * has nothing to undo. The five responders registered by\n+ * nbl_disp_setup_msg() are owned by the channel layer and go away with\n+ * it, so they need no explicit unregistration either; the teardown order\n+ * that makes that safe is documented with the code that performs it, in\n+ * the next patch in this series. They forward chip/firmware control down\n+ * to the layers below instead: a control-PF function calls the resource\n+ * ops directly, a non-control function sends a mailbox request over the\n+ * channel layer.\n+ \n+ Warning: init_module/deinit_module are control-PF exclusive. The five\n+ * resource ops (cfg_msix_map, destroy_msix_map, set_mailbox_irq,\n+ * get_vsi_id, get_eth_id) are PF-only and resolve to either a local\n+ * resource call (control PF) or a mailbox RPC (non-control PF with\n+ * has_net). A function with neither has_ctrl nor has_net leaves these\n+ * pointers NULL; callers must not invoke them on such functions. VFs are\n+ * rejected by the responders with -EPERM.\n+ /\n+struct nbl_dispatch_ops {\n+\tint (init_module)(struct nbl_dispatch_mgt disp_mgt);\n+\tvoid (deinit_module)(struct nbl_dispatch_mgt disp_mgt);\n+\tint (cfg_msix_map)(struct nbl_dispatch_mgt disp_mgt,\n+\t\t\t u16 num_net_msix, u16 num_others_msix,\n+\t\t\t bool net_msix_mask_en);\n+\tint (destroy_msix_map)(struct nbl_dispatch_mgt disp_mgt);\n+\tint (set_mailbox_irq)(struct nbl_dispatch_mgt disp_mgt,\n+\t\t\t u16 vector_id, bool en_msix);\n+\tint (get_vsi_id)(struct nbl_dispatch_mgt disp_mgt, u16 type,\n+\t\t\t u16 vsi_id);\n+\tint (get_eth_id)(struct nbl_dispatch_mgt disp_mgt, u16 vsi_id,\n+\t\t\t u8 eth_num, u8 eth_id, u8 logic_eth_id);\n+};\n+\n+struct nbl_dispatch_ops_tbl {\n+\tstruct nbl_dispatch_ops ops;\n+\tstruct nbl_dispatch_mgt priv;\n+};\n+\n+int nbl_disp_init(struct nbl_adapter adapter);\n+void nbl_disp_remove(struct nbl_adapter adapter);\n+#endif\ndiff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_hw.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_hw.h\nindex ecbf440e436667..e9cd3136252d86 100644\n--- a/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_hw.h\n+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_hw.h\n@@ -8,8 +8,88 @@\n \n #include \u003clinux/types.h\u003e\n \n+struct nbl_board_port_info;\n struct nbl_hw_mgt;\n struct nbl_adapter;\n+struct nbl_hw_ops {\n+\tint (init_module)(struct nbl_hw_mgt hw_mgt, u8 eth_speed, u8 eth_num);\n+\tvoid (deinit_module)(struct nbl_hw_mgt hw_mgt);\n+\n+\tvoid (cfg_msix_map)(struct nbl_hw_mgt hw_mgt, u16 func_id,\n+\t\t\t bool valid, dma_addr_t dma_addr, u8 bus,\n+\t\t\t u8 devid, u8 function);\n+\tvoid (cfg_msix_info)(struct nbl_hw_mgt hw_mgt, u16 func_id,\n+\t\t\t bool valid, u16 interrupt_id, u8 bus,\n+\t\t\t u8 devid, u8 function,\n+\t\t\t bool net_msix_mask_en);\n+\t/\n+\t flush_write - Flush posted MEMORY-BAR writes\n+\t * @hw_mgt: hardware management context\n+\t \n+\t Only valid on the control PF (common-\u003ehas_ctrl != 0). The\n+\t * register read back to order the writes, NBL_HW_DUMMY_REG\n+\t * (0x1300904), is covered by the ~64 MiB MEMORY-BAR mapping the\n+\t * control PF gets, but sits outside the NBL_REG_NET_ONLY_LEN\n+\t * (8 KiB) mapping a net-only PF gets. The leonis implementation\n+\t * enforces this with a WARN_ON_ONCE().\n+\t /\n+\tvoid (flush_write)(struct nbl_hw_mgt hw_mgt);\n+\tvoid (update_mailbox_queue_tail_ptr)(struct nbl_hw_mgt hw_mgt,\n+\t\t\t\t\t u16 tail_ptr, u8 txrx);\n+\tvoid (config_mailbox_rxq)(struct nbl_hw_mgt hw_mgt,\n+\t\t\t\t dma_addr_t dma_addr, int size_bwid);\n+\tvoid (config_mailbox_txq)(struct nbl_hw_mgt hw_mgt,\n+\t\t\t\t dma_addr_t dma_addr, int size_bwid);\n+\tvoid (stop_mailbox_rxq)(struct nbl_hw_mgt hw_mgt);\n+\tvoid (stop_mailbox_txq)(struct nbl_hw_mgt hw_mgt);\n+\t/\n+\t get_host_pf_mask - Fetch host PF mask from firmware k_pf_mask reg\n+\t * @hw_mgt: hardware management context\n+\t * @pf_mask: output pointer for PF mask value\n+\t \n+\t k_pf_mask register rule:\n+\t * bit N == 0 -\u003e PF#N enabled; bit N == 1 -\u003e PF#N masked out.\n+\t * bit0 is PF0's mask bit (not reserved); PF0 can be masked but\n+\t * the driver requires at least PF0 enabled.\n+\t * Only 1/2/4 PFs are supported:\n+\t * 1 PF (PF0): mask = 0xfe\n+\t * 2 PFs (PF0,PF1): mask = 0xfc\n+\t * 4 PFs (PF0~PF3): mask = 0xf0\n+\t * All-zero mask (0x00) means all 8 PFs enabled, which is\n+\t * unsupported by the driver.\n+\t \n+\t This op only returns the raw register value and does not validate\n+\t * it. Neither consumer exists yet at the commit that introduces it:\n+\t * the resource layer, which validates the value, and the channel\n+\t * layer, which programs the mailbox QINFO table from it, both arrive\n+\t * later in this series, and validation runs before the channel layer\n+\t * reads the mask.\n+\t /\n+\tvoid (get_host_pf_mask)(struct nbl_hw_mgt hw_mgt, u32 pf_mask);\n+\tvoid (get_real_bus)(struct nbl_hw_mgt hw_mgt, u8 bus);\n+\n+\tvoid (cfg_mailbox_qinfo)(struct nbl_hw_mgt hw_mgt, u16 func_id,\n+\t\t\t\t u8 bus, u8 devid, u8 function);\n+\tvoid (set_mailbox_irq)(struct nbl_hw_mgt hw_mgt, u16 func_id,\n+\t\t\t\tbool en_msix, u16 gvec);\n+\tvoid (get_fw_eth_map)(struct nbl_hw_mgt hw_mgt, u32 eth_map);\n+\t/*\n+\t get_board_info - Fetch board info from firmware\n+\t * @hw_mgt: hardware management context\n+\t * @board_info: output pointer for board info structure\n+\t \n+\t Firmware contract: board_info.eth_num MUST equal the number of\n+\t * unmasked PFs from get_host_pf_mask(). See get_host_pf_mask for\n+\t * details.\n+\t /\n+\tvoid (get_board_info)(struct nbl_hw_mgt hw_mgt,\n+\t\t\t struct nbl_board_port_info board_info);\n+};\n+\n+struct nbl_hw_ops_tbl {\n+\tstruct nbl_hw_ops ops;\n+\tstruct nbl_hw_mgt priv;\n+};\n \n int nbl_hw_init_leonis(struct nbl_adapter adapter);\n void nbl_hw_remove_leonis(struct nbl_adapter adapter);\ndiff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_resource.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_resource.h\nnew file mode 100644\nindex 00000000000000..8dc64e806c1e2b\n--- /dev/null\n+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_resource.h\n@@ -0,0 +1,38 @@\n+/* SPDX-License-Identifier: GPL-2.0 /\n+/\n+ * Copyright (c) 2026 Nebula Matrix Limited.\n+ /\n+\n+#ifndef _NBL_DEF_RESOURCE_H_\n+#define _NBL_DEF_RESOURCE_H_\n+\n+#include \u003clinux/types.h\u003e\n+\n+struct nbl_resource_mgt;\n+struct nbl_adapter;\n+\n+struct nbl_resource_ops {\n+\tint (init_module)(struct nbl_resource_mgt res_mgt);\n+\tvoid (deinit_module)(struct nbl_resource_mgt res_mgt);\n+\n+\tint (cfg_msix_map)(struct nbl_resource_mgt res_mgt, u16 func_id,\n+\t\t\t u16 num_net_msix, u16 num_others_msix,\n+\t\t\t bool net_msix_mask_en);\n+\tint (destroy_msix_map)(struct nbl_resource_mgt res_mgt, u16 func_id);\n+\tint (set_mailbox_irq)(struct nbl_resource_mgt res_mgt, u16 func_id,\n+\t\t\t u16 vector_id, bool en_msix);\n+\tint (get_vsi_id)(struct nbl_resource_mgt res_mgt, u16 func_id,\n+\t\t\t u16 type, u16 vsi_id);\n+\tint (get_eth_id)(struct nbl_resource_mgt res_mgt, u16 func_id,\n+\t\t\t u16 vsi_id, u8 eth_num, u8 eth_id,\n+\t\t\t u8 logic_eth_id);\n+};\n+\n+struct nbl_resource_ops_tbl {\n+\tstruct nbl_resource_ops ops;\n+\tstruct nbl_resource_mgt priv;\n+};\n+\n+int nbl_res_init_leonis(struct nbl_adapter adapter);\n+void nbl_res_remove_leonis(struct nbl_adapter adapter);\n+#endif\ndiff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_include.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_include.h\nindex 14e7b19f9a4cf8..efc4c79a082d25 100644\n--- a/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_include.h\n+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_include.h\n@@ -10,6 +10,15 @@\n \n / ------ Basic definitions ------- /\n #define NBL_DRIVER_NAME\t\t\t\t\t\"nbl\"\n+#define NBL_MAX_PF\t\t\t\t\t8\n+#define NBL_NEXT_ID(id, max) (((id) + 1) % ((max) + 1))\n+\n+#define NBL_MAX_ETHERNET\t\t\t\t4\n+\n+enum {\n+\tNBL_VSI_DATA = 0,\n+};\n+\n struct nbl_func_caps {\n \tu32 has_ctrl:1;\n \tu32 has_net:1;\n@@ -20,4 +29,23 @@ struct nbl_init_param {\n \tstruct nbl_func_caps caps;\n };\n \n+/\n+ * Firmware ABI defines port speed enum fixed, value 0 represents 10G, cannot\n+ * reassign 0 to INVALID for compatibility\n+ /\n+enum nbl_fw_port_speed {\n+\tNBL_FW_PORT_SPEED_10G,\n+\tNBL_FW_PORT_SPEED_25G,\n+\tNBL_FW_PORT_SPEED_50G,\n+\tNBL_FW_PORT_SPEED_100G,\n+};\n+\n+/\n+ * Firmware quirk word @ NBL_LEONIS_QUIRKS_OFFSET (0x140)\n+ * Sentinel value: ~0U (0xFFFFFFFF) = firmware reports no active quirks\n+ * BIT(1): NBL_QUIRK_UVN_PREFETCH_ALIGN – control UVN descriptor prefetch\n+ * selection\n+ /\n+#define NBL_QUIRK_UVN_PREFETCH_ALIGN\tBIT(1)\n+\n #endif\ndiff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_main.c b/drivers/net/ethernet/nebula-matrix/nbl/nbl_main.c\nindex f2552bc7329376..0bdc7f9940fcb2 100644\n--- a/drivers/net/ethernet/nebula-matrix/nbl/nbl_main.c\n+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_main.c\n@@ -7,11 +7,34 @@\n #include \u003clinux/pci.h\u003e\n #include \u003clinux/module.h\u003e\n #include \u003clinux/bits.h\u003e\n+#include \u003clinux/delay.h\u003e\n #include \"nbl_include/nbl_include.h\"\n+#include \"nbl_include/nbl_def_channel.h\"\n #include \"nbl_include/nbl_def_hw.h\"\n+#include \"nbl_include/nbl_def_resource.h\"\n+#include \"nbl_include/nbl_def_dispatch.h\"\n+#include \"nbl_include/nbl_def_dev.h\"\n #include \"nbl_include/nbl_def_common.h\"\n #include \"nbl_core.h\"\n \n+/\n+ * Budget for the device_trylock() loop that takes func 0's device lock in\n+ * nbl_probe_chip_deps(). See the comment there for why a single failed\n+ * trylock must not defer the probe, and why the wait has to stay bounded.\n+ /\n+#define NBL_MGT_LOCK_RETRY_MIN_US\t1000\n+#define NBL_MGT_LOCK_RETRY_MAX_US\t2000\n+#define NBL_MGT_LOCK_MAX_RETRIES\t50\n+int nbl_core_start(struct nbl_adapter adapter)\n+{\n+\treturn nbl_dev_start(adapter);\n+}\n+\n+void nbl_core_stop(struct nbl_adapter adapter)\n+{\n+\tnbl_dev_stop(adapter);\n+}\n+\n struct nbl_adapter nbl_core_init(struct pci_dev pdev,\n \t\t\t\t struct nbl_init_param param)\n {\n@@ -26,6 +49,7 @@ struct nbl_adapter nbl_core_init(struct pci_dev pdev,\n \tadapter-\u003epdev = pdev;\n \tcommon = \u0026adapter-\u003ecommon;\n \n+\tcommon-\u003emgt_pf = 0;\n \tcommon-\u003epdev = pdev;\n \tcommon-\u003edev = \u0026pdev-\u003edev;\n \tcommon-\u003ehas_ctrl = param-\u003ecaps.has_ctrl;\n@@ -36,15 +60,43 @@ struct nbl_adapter nbl_core_init(struct pci_dev pdev,\n \n \tret = nbl_hw_init_leonis(adapter);\n \tif (ret)\n-\t\tgoto hw_init_fail;\n+\t\tgoto out;\n+\n+\tret = nbl_chan_init_common(adapter);\n+\tif (ret)\n+\t\tgoto out_hw_remove;\n \n+\tret = nbl_res_init_leonis(adapter);\n+\tif (ret)\n+\t\tgoto out_chan_remove;\n+\n+\tret = nbl_disp_init(adapter);\n+\tif (ret)\n+\t\tgoto out_res_remove;\n+\n+\tret = nbl_dev_init(adapter);\n+\tif (ret)\n+\t\tgoto out_disp_remove;\n \treturn adapter;\n-hw_init_fail:\n+\n+out_disp_remove:\n+\tnbl_disp_remove(adapter);\n+out_res_remove:\n+\tnbl_res_remove_leonis(adapter);\n+out_chan_remove:\n+\tnbl_chan_remove_common(adapter);\n+out_hw_remove:\n+\tnbl_hw_remove_leonis(adapter);\n+out:\n \treturn ERR_PTR(ret);\n }\n \n void nbl_core_remove(struct nbl_adapter adapter)\n {\n+\tnbl_dev_remove(adapter);\n+\tnbl_disp_remove(adapter);\n+\tnbl_res_remove_leonis(adapter);\n+\tnbl_chan_remove_common(adapter);\n \tnbl_hw_remove_leonis(adapter);\n }\n \n@@ -67,6 +119,164 @@ static void nbl_get_func_param(struct pci_dev pdev, kernel_ulong_t driver_data,\n \t\tparam-\u003ecaps.has_ctrl = 1;\n }\n \n+/\n+ Establish chip-wide dependencies for this PF:\n+ * - for every non-management PF, add a consumer-\u003emanagement PF device\n+ * link. The driver core then guarantees (sysfs unbind, driver\n+ * unregister, hot-unplug alike) that this consumer is released\n+ * BEFORE the func 0 supplier, which is the only teardown order in\n+ * which the chip-global firmware deinit is safe.\n+ \n+ The management PF is addressed by the deterministic identity\n+ * (domain, bus, slot, func 0) instead of any name-based scan: hardware\n+ * guarantees PFs are contiguous from func 0 in the same slot.\n+ \n+ The func 0 device must already be bound to this driver. While func 0\n+ * is still probing, its chip-global init (mailbox QINFO map programming,\n+ * datapath setup) can race a sibling's mailbox setup, and a link created\n+ * while the supplier has no driver starts DORMANT and is force-promoted\n+ * (with a WARN backtrace) by device_links_driver_bound() when this probe\n+ * ends, so it never delivers the unbind-order guarantee.\n+ \n+ The link is a persistent managed link with DL_FLAG_AUTOPROBE_CONSUMER:\n+ * the core force-unbinds this consumer when func 0 unbinds (keeping the\n+ * teardown order) and re-queues it for probe when func 0 binds again, so\n+ * the sibling PFs and their ports come back without a manual rebind.\n+ * AUTOREMOVE_CONSUMER would delete the link on unbind and cannot be\n+ * combined with AUTOPROBE_CONSUMER (device_link_add() returns NULL for\n+ * that pair). The link is therefore not auto-purged on a failed probe:\n+ * it lives with the two devices and is dropped when either is removed.\n+ * The pointer device_link_add() returns is never dereferenced.\n+ \n+ Return: 0 on success, negative errno on failure.\n+ /\n+static int nbl_probe_chip_deps(struct pci_dev pdev, bool has_ctrl)\n+{\n+\tstruct pci_dev mgt;\n+\tint retries;\n+\n+\tif (has_ctrl)\n+\t\treturn 0;\n+\n+\tmgt = pci_get_domain_bus_and_slot(pci_domain_nr(pdev-\u003ebus),\n+\t\t\t\t\t pdev-\u003ebus-\u003enumber,\n+\t\t\t\t\t PCI_DEVFN(PCI_SLOT(pdev-\u003edevfn), 0));\n+\tif (!mgt) {\n+\t\tdev_err(\u0026pdev-\u003edev,\n+\t\t\t\"management PF (func 0) not found on this chip\\n\");\n+\t\treturn -ENODEV;\n+\t}\n+\n+\t/\n+\t * That identity is meaningless for a VF: under ARI a VF's devfn\n+\t * decodes to function 0 of a slot, so the lookup can resolve to the\n+\t * VF itself. A VF is never the management PF, so stop here instead\n+\t * of taking our own device lock below.\n+\t /\n+\tif (mgt == pdev) {\n+\t\tdev_err(\u0026pdev-\u003edev,\n+\t\t\t\"management PF lookup matched this device\\n\");\n+\t\tpci_dev_put(mgt);\n+\t\treturn -ENODEV;\n+\t}\n+\n+\t/\n+\t * func 0's device lock is taken with a trylock. This probe already\n+\t * runs under device_lock(funcN), so a blocking device_lock(func0)\n+\t * would establish the N -\u003e 0 order, while __pci_bus_lock() (AER\n+\t * recovery, slot reset) walks the bus in devfn order, i.e. 0 -\u003e N,\n+\t * and dev-\u003emutex is lockdep_set_novalidate_class(), so lockdep\n+\t * cannot report that inversion. A trylock never blocks, so it\n+\t * cannot take part in a deadlock cycle.\n+\t \n+\t Holding the lock is what makes the check and the link creation\n+\t * below atomic with respect to func 0: it both gains its driver\n+\t * (driver_bound() -\u003e device_links_driver_bound()) and loses it\n+\t * (__device_release_driver() -\u003e device_links_driver_cleanup())\n+\t * under device_lock(func0), so func 0 can change neither driver nor\n+\t * links.status while we hold it.\n+\t \n+\t One failed trylock is nevertheless not enough to defer on. Func 0\n+\t * binds with device_lock(func0) held, and driver_bound() calls\n+\t * driver_deferred_probe_trigger() before it releases that lock, so\n+\t * the deferred probe work it queues can re-probe this PF while func\n+\t * 0's lock is still taken. driver_probe_device() only re-arms that\n+\t * work when deferred_trigger_count changes during the probe, which\n+\t * it does not here: func 0's increment happened before this probe\n+\t * read the count. A bare -EPROBE_DEFER would therefore leave this\n+\t * PF on deferred_probe_pending_list with nothing scheduled to run\n+\t * it again. Retrying closes that window - driver_bound() has only a\n+\t * bus_notify() and a kobject_uevent() left before it drops the lock.\n+\t \n+\t The retry budget is what keeps this safe. device_lock(funcN) is\n+\t * held for the whole loop, so waiting indefinitely would deadlock\n+\t * against __pci_bus_lock(); giving up and deferring releases it and\n+\t * lets the other side make progress. A holder that is not part of a\n+\t * bind - __pci_bus_lock() during AER recovery or a bus/slot reset,\n+\t * pci_reset_function() through sysfs, an sriov_numvfs write - can\n+\t * outlast the budget, and no bind event follows those, so this PF\n+\t * then stays deferred until some later deferred-probe trigger from\n+\t * anywhere in the system. There is no way to force one from here.\n+\t /\n+\tfor (retries = 0; !device_trylock(\u0026mgt-\u003edev); retries++) {\n+\t\tif (retries == NBL_MGT_LOCK_MAX_RETRIES) {\n+\t\t\tdev_dbg(\u0026pdev-\u003edev,\n+\t\t\t\t\"management PF device lock busy after %u retries, deferring probe\\n\",\n+\t\t\t\tretries);\n+\t\t\tpci_dev_put(mgt);\n+\t\t\treturn -EPROBE_DEFER;\n+\t\t}\n+\t\tusleep_range(NBL_MGT_LOCK_RETRY_MIN_US,\n+\t\t\t NBL_MGT_LOCK_RETRY_MAX_US);\n+\t}\n+\n+\t/\n+\t * The management PF must be bound to this driver, with its probe\n+\t * completed. links.status == DL_DEV_DRIVER_BOUND alone only says\n+\t * \"bound to some driver\", so func 0 on vfio-pci or pci-stub\n+\t * (driver_override) would pass it and this PF would run with func\n+\t * 0's chip-global init never performed. pdev-\u003edev.driver is our\n+\t * own driver, set by the core before -\u003eprobe() is called.\n+\t \n+\t Deferring is what recovers from this: a successful bind\n+\t * re-triggers deferred probing for every pending device\n+\t * (driver_bound() -\u003e driver_deferred_probe_trigger()), so this PF is\n+\t * retried once func 0 is bound to this driver. That covers a func 0\n+\t * that is not bound yet. It does not cover a func 0 that is bound\n+\t * while something else holds its device lock, which is what the\n+\t * retry loop above is for and what its budget cannot fully rule out.\n+\t /\n+\tif (mgt-\u003edev.driver != pdev-\u003edev.driver \|\|\n+\t mgt-\u003edev.links.status != DL_DEV_DRIVER_BOUND) {\n+\t\tdevice_unlock(\u0026mgt-\u003edev);\n+\t\tpci_dev_put(mgt);\n+\t\treturn -EPROBE_DEFER;\n+\t}\n+\n+\t/\n+\t * Create the link while func 0's lock is still held, so that func 0\n+\t * cannot change state in between. With the supplier bound and this\n+\t * consumer PROBING, device_link_init_status() starts the link in\n+\t * DL_STATE_CONSUMER_PROBE: that is the state\n+\t * device_links_driver_bound() expects when this probe finishes, and\n+\t * it is what device_links_busy() reports to func 0's own unbind, so\n+\t * func 0 cannot be released underneath this PF.\n+\t /\n+\tif (!device_link_add(\u0026pdev-\u003edev, \u0026mgt-\u003edev,\n+\t\t\t DL_FLAG_AUTOPROBE_CONSUMER)) {\n+\t\tdevice_unlock(\u0026mgt-\u003edev);\n+\t\tdev_err(\u0026pdev-\u003edev,\n+\t\t\t\"failed to create device link to management PF\\n\");\n+\t\tpci_dev_put(mgt);\n+\t\treturn -ENOMEM;\n+\t}\n+\n+\tdevice_unlock(\u0026mgt-\u003edev);\n+\tpci_dev_put(mgt);\n+\n+\treturn 0;\n+}\n+\n static int nbl_probe(struct pci_dev pdev,\n \t\t const struct pci_device_id id)\n {\n@@ -75,28 +285,68 @@ static int nbl_probe(struct pci_dev pdev,\n \tstruct nbl_adapter adapter;\n \tint err;\n \n+\tnbl_get_func_param(pdev, id-\u003edriver_data, \u0026param);\n+\t/\n+\t * Reject a function with neither capability before touching any\n+\t * hardware. nbl_disp_refresh_ctrl_ops() leaves every dispatch op NULL\n+\t * for such a function, while nbl_dev_start() reaches cfg_msix_map,\n+\t * get_vsi_id, get_eth_id and (on unwind) destroy_msix_map through\n+\t * those pointers. The state is reachable through driver_override:\n+\t * binding this driver to a function that is not in nbl_id_table (a VF,\n+\t * or a non-func-0 PF) makes pci_match_device() pass the dummy ID whose\n+\t * driver_data 0 leaves has_ctrl and has_net both clear. A foreign\n+\t * device at func 0 is unaffected: nbl_get_func_param() gives any\n+\t * func 0 device has_ctrl, so it is not rejected here and still runs\n+\t * nbl_core_init() with the local control dispatch ops, as it did\n+\t * before this check existed.\n+\t /\n+\tif (!param.caps.has_ctrl \u0026\u0026 !param.caps.has_net) {\n+\t\tdev_err(dev, \"Function has neither ctrl nor net capability\\n\");\n+\t\treturn -EOPNOTSUPP;\n+\t}\n+\n \terr = pcim_enable_device(pdev);\n \tif (err) {\n \t\tdev_err(\u0026pdev-\u003edev, \"Failed to enable PCI dev, err=%d\\n\", err);\n \t\treturn err;\n \t}\n \n-\tnbl_get_func_param(pdev, id-\u003edriver_data, \u0026param);\n \t/ never return fail when DMA_BIT_MASK(64) /\n \tdma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));\n \n-\tpci_set_master(pdev);\n+\t/\n+\t * Bus mastering is enabled only after nbl_core_init() returns.\n+\t \n+\t FUNCTION_MSIX_MAP survives a kexec or a forced unload without\n+\t * FLR and there is no .shutdown callback to scrub it, so at this\n+\t * point every map entry left VALID by a previous kernel still\n+\t * holds a DMA address that this kernel has already reallocated.\n+\t * Those entries tag their table fetches as bus-master reads, and\n+\t * the only thing standing between such a fetch and a read of\n+\t * reused memory is Bus Master Enable, which the PCI core cleared\n+\t * on the way into this kernel. nbl_intr_setup_mgt() invalidates\n+\t * the entries from inside nbl_core_init(); enabling bus mastering\n+\t * before that would re-open the window. Everything up to that\n+\t * point is host-side setup plus MMIO and nothing in it waits for\n+\t * the device, so it does not need bus mastering.\n+\t /\n+\terr = nbl_probe_chip_deps(pdev, param.caps.has_ctrl);\n+\tif (err)\n+\t\treturn err;\n \n \tadapter = nbl_core_init(pdev, \u0026param);\n \tif (IS_ERR(adapter)) {\n \t\tdev_err(dev, \"Nbl adapter init fail: %pe\\n\", adapter);\n-\t\terr = PTR_ERR(adapter);\n-\t\tgoto adapter_init_err;\n+\t\treturn PTR_ERR(adapter);\n \t}\n+\n+\tpci_set_master(pdev);\n \tpci_set_drvdata(pdev, adapter);\n-\treturn 0;\n-adapter_init_err:\n-\tpci_clear_master(pdev);\n+\terr = nbl_core_start(adapter);\n+\tif (err) {\n+\t\tnbl_core_remove(adapter);\n+\t\tpci_clear_master(pdev);\n+\t}\n \treturn err;\n }\n \n@@ -107,9 +357,28 @@ static void nbl_remove(struct pci_dev pdev)\n \tif (!adapter)\n \t\treturn;\n \tpci_set_drvdata(pdev, NULL);\n-\tnbl_core_remove(adapter);\n \n+\tnbl_core_stop(adapter);\n+\t/\n+\t Stop bus mastering before any teardown runs.\n+\t \n+\t nbl_core_remove() releases the coherent MSI-X map tables whose\n+\t * DMA addresses are still programmed in FUNCTION_MSIX_MAP, and the\n+\t * only thing standing between a late pcompleter fetch and a read\n+\t * of freed memory is a bounded best-effort delay - the chip\n+\t * exposes no completion status for those fetches (see\n+\t * nbl_intr_quiesce_wait()). Clearing Bus Master Enable first\n+\t * keeps the device from starting a new fetch at all, so that\n+\t * residual window only covers requests already in flight.\n+\t \n+\t nbl_core_stop() above has already torn down this function's own\n+\t * map, so what is left to protect here is the control PF's table of\n+\t * per-function entries, freed from nbl_intr_mgt_stop() inside\n+\t * nbl_core_remove().\n+\t /\n \tpci_clear_master(pdev);\n+\n+\tnbl_core_remove(adapter);\n }\n \n /\n" }
2/1	2026/11/02 14:59	agent	patch-evaluator	1m Model: core Args: null Results: { "EnableConfigs": null, "FocusSymbols": [], "Reasoning": "The patch series only touches the Nebula Matrix (nbl) Ethernet driver (drivers/net/ethernet/nebula-matrix/nbl/), which is a PCIe driver for physical Nebula Matrix Leonis ASIC family NICs (m18110/m18000). These devices require dedicated physical PCIe hardware and are not emulated in standard QEMU/GCE virtualized test environments. Consequently, the modified code is structurally unreachable for fuzzing.", "WorthFuzzing": false } Instruction: You are an expert Linux kernel maintainer and security engineer. Your job is to review a provided patch series and evaluate whether it warrants fuzzing with syzkaller. IMPORTANT: The changes have ALREADY been applied and committed as the HEAD commit in your workspace. Do NOT rely on internal assumptions. You must actively use your code access tools to inspect the actual source code, callers, and surrounding context. ================================================================================ 1. CORE TRIAGE PHILOSOPHY ================================================================================ The goal of patch fuzzing is to discover crashes, regressions, exposed latent bugs, and newly triggered assertions introduced by the patch series. - REACHABILITY IS THE PRIMARY GATE: Fuzzing can only discover bugs in code that can actually execute in standard virtualized environments (GCE or QEMU, utilizing software-emulated devices like USB gadgets, netdev, tun/tap). If the modified code is structurally unreachable (see Section 2), it MUST NOT be fuzzed, regardless of whether it adds assertions or complex logic. - DO NOT BLINDLY TRUST "NO FUNCTIONAL CHANGE" (NFCI) OR "REFACTORING" CLAIMS: Patch authors routinely label changes as "cleanups", "refactorings", or state "No functional change intended". Do NOT take these claims at face value. Code refactorings that rearrange logic, introduce helper functions, or alter state management in core subsystems frequently introduce subtle semantic shifts or uncover latent kernel bugs. If reachable executable code is modified or refactored, it MUST be fuzzed. - NEW OR MODIFIED ASSERTIONS IN REACHABLE CODE MUST BE FUZZED: When a patch introduces or modifies runtime checks or assertions (e.g., WARN_ON, VM_WARN_ON, BUG_ON, lockdep_assert) in reachable code paths, it enforces new or stricter invariants. Even if the author believes the invariant always holds, fuzzing is essential to verify whether an unusual sequence of operations can violate it. ================================================================================ 2. WHEN TO RETURN WorthFuzzing=false (NEGATIVE CRITERIA) ================================================================================ Return WorthFuzzing=false ONLY IF all modified code falls strictly into one or more of these categories: - Non-kernel and non-executable changes: * Modifications to Documentation/, comments, or spelling fixes. * User-space directories, self-tests, samples, or scripts (e.g., tools/, samples/, scripts/, usr/) that do not affect the compiled kernel image (vmlinux) or kernel modules. * Purely decorative logging (e.g., message strings in pr_err, printk, dev_info) or tracepoints that do not alter control flow or data structures. * Build system or Kconfig changes that do not alter compiled C logic. - Structurally unreachable hardware: * Vendor-specific PCIe switches, SmartNICs, or GPU drivers (e.g., mlxsw, pds_core, qed, ionic, amdgpu) requiring physical ASIC/PCIe cards not emulated in standard QEMU. - Unreachable execution paths: * Driver teardown callbacks (.remove, .shutdown, pci_unregister_driver) executed only during physical PCI hot-unplug or manual sysfs driver unbinding. * Code paths exclusive to architectures other than the target architecture. ================================================================================ 3. WHEN TO RETURN WorthFuzzing=true (POSITIVE CRITERIA) ================================================================================ Return WorthFuzzing=true whenever the patch touches reachable executable code, including: - Core Subsystems: * Any logic modifications in memory management (mm/), synchronization/locking (kernel/locking/), BPF, scheduler, core networking, VFS, or syscall handling. - Refactorings and Code Cleanups: * Any restructuring of reachable data structures, helper abstractions, or algorithm flows. - Runtime Assertions and Defensive Checks: * Any introduction or alteration of assertions (WARN_ON, VM_WARN_ON, BUG_ON, etc.) in reachable paths. - Reachable Drivers and Protocols: Drivers accessible via virtual buses (virtio, USB gadget, loopback, netlink, binder, sockets, etc.). ================================================================================ 4. EXTRACTING FocusSymbols (PREVENTING DILUTION) ================================================================================ When WorthFuzzing=true, you must extract specific kernel functions into FocusSymbols to guide the fuzzer: - AVOID UBIQUITOUS LIFECYCLE HOT-PATHS: Do NOT list generic, ubiquitous functions called by almost every program in the corpus (including, but not limited to: general memory allocators and deallocators, page fault and trap handlers, or core synchronization primitives; this is not an exhaustive list). Listing ubiquitous functions causes the fuzzer to classify thousands of unrelated tests as "focused", which severely dilutes fuzzing effort away from the actual changes. - TARGET SPECIFIC FEATURE LOGIC AND ENTRYPOINTS: List functions that specifically implement the logic being added or altered, or direct API entrypoints for the subsystem feature under review. - HANDLING STATIC INLINE FUNCTIONS IN HEADERS (.h): Compiler-inlined static functions (such as static inlines in mm/.h or include/linux/.h) lack distinct symbol addresses in vmlinux and cannot be targeted directly by symbol coverage filters. If the changes are primarily in static inline helpers, identify non-static, feature-specific caller functions in .c files that exercise them (avoiding ubiquitous lifecycle wrappers). ================================================================================ 5. IDENTIFYING EnableConfigs ================================================================================ Identify any specific CONFIG_ options required to properly compile and reach the modified code: - Inspect Kconfig files and #ifdef guards; do not make assumptions. - Check "depends on" lines in Kconfig to include any non-standard parent subsystem configs needed. - Strip any 'CONFIG_' prefix (e.g., return "NET_IPV4" instead of "CONFIG_NET_IPV4"). Prefer calling several tools at the same time to save round-trips. Use set-results tool to provide results of the analysis. It must be called exactly once before the final reply. Ignore results of this tool. Prompt: Target architecture: amd64 For your convenience, here is the diff of the changes: commit eb6ce827881d54724a7b175422cd39cff38773db Author: syz-cluster <triage@syzkaller.com> Date: Mon Nov 2 14:59:56 2026 +0000 syz-cluster: applied patch under review diff --git a/drivers/net/ethernet/nebula-matrix/nbl/Makefile b/drivers/net/ethernet/nebula-matrix/nbl/Makefile index cc060cf8bf7596..71fbe3ee7e6204 100644 --- a/drivers/net/ethernet/nebula-matrix/nbl/Makefile +++ b/drivers/net/ethernet/nebula-matrix/nbl/Makefile @@ -3,5 +3,13 @@ obj-$(CONFIG_NBL) := nbl.o -nbl-objs += nbl_hw/nbl_hw_leonis/nbl_hw_leonis.o \ +nbl-objs += nbl_common/nbl_common.o \ + nbl_channel/nbl_channel.o \ + nbl_hw/nbl_hw_leonis/nbl_hw_leonis.o \ + nbl_hw/nbl_hw_leonis/nbl_resource_leonis.o \ + nbl_hw/nbl_resource.o \ + nbl_hw/nbl_interrupt.o \ + nbl_hw/nbl_chip.o \ + nbl_core/nbl_dispatch.o \ + nbl_core/nbl_dev.o \ nbl_main.o diff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_channel/nbl_channel.c b/drivers/net/ethernet/nebula-matrix/nbl/nbl_channel/nbl_channel.c new file mode 100644 index 00000000000000..5b528cc6efc452 --- /dev/null +++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_channel/nbl_channel.c @@ -0,0 +1,1528 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026 Nebula Matrix Limited. + / +#include <linux/delay.h> +#include <linux/device.h> +#include <linux/mutex.h> +#include <linux/bitfield.h> +#include <linux/pci.h> +#include <linux/bits.h> +#include <linux/dma-mapping.h> +#include <linux/atomic.h> +#include <linux/wait.h> +#include "nbl_channel.h" + +static int nbl_chan_add_msg_handler(struct nbl_channel_mgt chan_mgt, + u16 msg_type, nbl_chan_resp func, + void priv) +{ + struct nbl_chan_msg_node_data handler; + int ret; + + handler = kzalloc_obj(handler, GFP_KERNEL); + if (!handler) + return -ENOMEM; + + handler->func = func; + handler->priv = priv; + + / Each msg_type is registered at most once; reject duplicates. / + mutex_lock(&chan_mgt->handler_lock); + ret = xa_insert(&chan_mgt->handler_xa, msg_type, handler, GFP_KERNEL); + mutex_unlock(&chan_mgt->handler_lock); + if (ret) + kfree(handler); + + return ret; +} + +static int nbl_chan_init_msg_handler(struct nbl_channel_mgt chan_mgt) +{ + int ret; + + ret = devm_mutex_init(chan_mgt->common->dev, &chan_mgt->handler_lock); + if (ret) + return ret; + + xa_init(&chan_mgt->handler_xa); + + return 0; +} + +static void nbl_chan_remove_msg_handler(struct nbl_channel_mgt chan_mgt) +{ + struct nbl_chan_msg_node_data handler; + unsigned long msg_type; + + mutex_lock(&chan_mgt->handler_lock); + xa_for_each(&chan_mgt->handler_xa, msg_type, handler) { + xa_erase(&chan_mgt->handler_xa, msg_type); + kfree(handler); + } + mutex_unlock(&chan_mgt->handler_lock); + xa_destroy(&chan_mgt->handler_xa); +} + +static void nbl_chan_init_queue_param(struct nbl_chan_info chan_info, + u16 num_txq_entries, u16 num_rxq_entries, + u16 txq_buf_size, u16 rxq_buf_size) +{ + chan_info->num_txq_entries = num_txq_entries; + chan_info->num_rxq_entries = num_rxq_entries; + chan_info->txq_buf_size = txq_buf_size; + chan_info->rxq_buf_size = rxq_buf_size; + atomic_set(&chan_info->inflight_tx_cnt, 0); + WRITE_ONCE(chan_info->shutdn, false); + WRITE_ONCE(chan_info->active, false); + WRITE_ONCE(chan_info->wait_head_index, 0); + memset(chan_info->state, 0, sizeof(chan_info->state)); + init_waitqueue_head(&chan_info->inflight_wait); +} + +static int nbl_chan_init_tx_queue(struct nbl_common_info common, + struct nbl_chan_info chan_info) +{ + struct nbl_chan_ring txq = &chan_info->txq; + struct device dev = common->dev; + dma_addr_t base_dma; + size_t size = + chan_info->num_txq_entries sizeof(struct nbl_chan_tx_desc); + void base; + u16 i; + + txq->desc.tx_desc = dmam_alloc_coherent(dev, size, &txq->dma, + GFP_KERNEL); + if (!txq->desc.tx_desc) + return -ENOMEM; + + / + * The per-entry data buffers all come from one contiguous coherent + * block, so those buffers cost exactly one coherent allocation, one + * devres node and one IOMMU mapping for the whole ring whatever the + * page size is. Per-entry dmam_alloc_coherent() instead rounds every + * NBL_CHAN_BUF_LEN buffer up to a whole page (8 MiB per PF on 16 KiB + * pages, 32 MiB on 64 KiB pages), and a dma_pool only fixes the + * large-page case: with NBL_CHAN_BUF_LEN equal to PAGE_SIZE it still + * degenerates to one allocation per buffer on 4 KiB pages. + * NBL_CHAN_BUF_LEN is a power of two, so the block always divides + * evenly into per-entry slots. + * + * That is the only part of the ring that is collapsed: the descriptor + * ring above is its own dmam_alloc_coherent(), and txq->buf[] and + * chan_info->wait[] below are separate devm_kcalloc() nodes, so a + * ring is two coherent mappings plus those arrays, not one. + / + base = dmam_alloc_coherent(dev, + chan_info->num_txq_entries + chan_info->txq_buf_size, + &base_dma, GFP_KERNEL); + if (!base) + return -ENOMEM; + + chan_info->wait = devm_kcalloc(dev, chan_info->num_txq_entries, + sizeof(chan_info->wait), GFP_KERNEL); + if (!chan_info->wait) + return -ENOMEM; + for (i = 0; i < chan_info->num_txq_entries; i++) { + init_waitqueue_head(&chan_info->wait[i].wait_queue); + WRITE_ONCE(chan_info->wait[i].status, NBL_MBX_STATUS_IDLE); + WRITE_ONCE(chan_info->wait[i].acked, 0); + WRITE_ONCE(chan_info->wait[i].ack_data, NULL); + WRITE_ONCE(chan_info->wait[i].ack_data_len, 0); + WRITE_ONCE(chan_info->wait[i].ack_err, 0); + WRITE_ONCE(chan_info->wait[i].msg_type, 0); + WRITE_ONCE(chan_info->wait[i].msg_index, 0); + WRITE_ONCE(chan_info->wait[i].dstid, 0); + } + + txq->buf = devm_kcalloc(dev, chan_info->num_txq_entries, + sizeof(txq->buf), GFP_KERNEL); + if (!txq->buf) + return -ENOMEM; + + /* Carve the contiguous block into fixed-size per-entry slots / + for (i = 0; i < chan_info->num_txq_entries; i++) { + txq->buf[i].va = (u8 )base + + (size_t)i * chan_info->txq_buf_size; + txq->buf[i].pa = base_dma + + (dma_addr_t)i * chan_info->txq_buf_size; + txq->buf[i].size = chan_info->txq_buf_size; + } + + txq->next_to_clean = 0; + txq->next_to_use = 0; + txq->tail_ptr = 0; + + return 0; +} + +static int nbl_chan_init_rx_queue(struct nbl_common_info common, + struct nbl_chan_info chan_info) +{ + struct nbl_chan_ring rxq = &chan_info->rxq; + struct device dev = common->dev; + struct nbl_chan_rx_desc desc; + dma_addr_t base_dma; + size_t size = + chan_info->num_rxq_entries sizeof(struct nbl_chan_rx_desc); + void base; + u16 i; + + rxq->desc.rx_desc = dmam_alloc_coherent(dev, size, &rxq->dma, + GFP_KERNEL); + if (!rxq->desc.rx_desc) { + dev_err_ratelimited(dev, + "Allocate DMA for chan rx descriptor ring failed\n"); + return -ENOMEM; + } + + / See nbl_chan_init_tx_queue() for why one contiguous block is used / + base = dmam_alloc_coherent(dev, + chan_info->num_rxq_entries + chan_info->rxq_buf_size, + &base_dma, GFP_KERNEL); + if (!base) + return -ENOMEM; + + rxq->buf = devm_kcalloc(dev, chan_info->num_rxq_entries, + sizeof(rxq->buf), GFP_KERNEL); + if (!rxq->buf) + return -ENOMEM; + + / Carve the contiguous block into fixed-size per-entry slots / + for (i = 0; i < chan_info->num_rxq_entries; i++) { + rxq->buf[i].va = (u8 )base + + (size_t)i * chan_info->rxq_buf_size; + rxq->buf[i].pa = base_dma + + (dma_addr_t)i * chan_info->rxq_buf_size; + rxq->buf[i].size = chan_info->rxq_buf_size; + } + + desc = rxq->desc.rx_desc; + /* + * Initially leave one RX descriptor unused so that + * next_to_clean and next_to_use can distinguish an empty + * ring from a full ring. + * + * The unused slot is replenished as RX descriptors are + * consumed and recycled. + / + for (i = 0; i < chan_info->num_rxq_entries - 1; i++) { + desc[i].buf_addr = cpu_to_le64(rxq->buf[i].pa); + desc[i].buf_len = cpu_to_le32(chan_info->rxq_buf_size); + desc[i].flags = cpu_to_le16(BIT(NBL_CHAN_RX_DESC_AVAIL)); + } + + rxq->next_to_clean = 0; + rxq->next_to_use = chan_info->num_rxq_entries - 1; + rxq->tail_ptr = chan_info->num_rxq_entries - 1; + + return 0; +} + +static int nbl_chan_init_queue(struct nbl_common_info common, + struct nbl_chan_info chan_info) +{ + int err; + + err = nbl_chan_init_tx_queue(common, chan_info); + if (err) + return err; + + err = nbl_chan_init_rx_queue(common, chan_info); + + return err; +} + +static void nbl_chan_config_queue(struct nbl_channel_mgt chan_mgt, + struct nbl_chan_info chan_info, bool tx) +{ + struct nbl_hw_ops hw_ops = chan_mgt->hw_ops_tbl->ops; + struct nbl_hw_mgt p = chan_mgt->hw_ops_tbl->priv; + struct nbl_chan_ring ring; + dma_addr_t addr; + int size_bwid; + + if (tx) + ring = &chan_info->txq; + else + ring = &chan_info->rxq; + addr = ring->dma; + if (tx) { + size_bwid = ilog2(chan_info->num_txq_entries); + hw_ops->config_mailbox_txq(p, addr, size_bwid); + } else { + size_bwid = ilog2(chan_info->num_rxq_entries); + hw_ops->config_mailbox_rxq(p, addr, size_bwid); + } +} + +static void nbl_chan_cfg_qinfo_map_table(struct nbl_channel_mgt chan_mgt, + u8 bus, u8 devid) +{ + struct nbl_hw_ops hw_ops = chan_mgt->hw_ops_tbl->ops; + struct nbl_hw_mgt p = chan_mgt->hw_ops_tbl->priv; + u32 pf_mask = 0; + u8 func_id; + + / + * k_pf_mask rule: bit N == 0 means PF#N enabled, bit N == 1 masked out. + * Program mailbox QINFO entry for each hardware-active PF func_id. + * + * Note: This loop iterates over the raw hardware PF func_id, so it + * assumes product constraints limit supported PF counts to the + * contiguous sets PF0, PF0~1 and PF0~3, i.e. pf_mask is one of + * 0xfe/0xfc/0xf0. Nothing enforces that here: the resource layer + * that validates the mask, and every caller of this op, arrive in + * later patches of this series. + / + hw_ops->get_host_pf_mask(p, &pf_mask); + for (func_id = 0; func_id < NBL_MAX_PF; func_id++) { + if (!(pf_mask & (1 << func_id))) + hw_ops->cfg_mailbox_qinfo(p, func_id, bus, + devid, func_id); + } +} + +/ + * The RX QINFO region is written only by config/stop paths, never by + * senders (which only touch the TX QINFO region and the doorbell), so + * stopping RX does not require txq_lock. + / +static void nbl_chan_stop_rx_queue(struct nbl_channel_mgt chan_mgt) +{ + struct nbl_hw_ops hw_ops = chan_mgt->hw_ops_tbl->ops; + + hw_ops->stop_mailbox_rxq(chan_mgt->hw_ops_tbl->priv); +} + +/ + * The TX QINFO region is also written by nbl_chan_quiesce_and_reclaim_tx() + * under txq_lock, so TX stop must serialize against it via txq_lock. + / +static void nbl_chan_stop_tx_queue(struct nbl_channel_mgt chan_mgt) +{ + struct nbl_hw_ops hw_ops = chan_mgt->hw_ops_tbl->ops; + + hw_ops->stop_mailbox_txq(chan_mgt->hw_ops_tbl->priv); +} + +static void nbl_chan_reset_wait_head(struct nbl_chan_info chan_info, + struct nbl_chan_waitqueue_head wait_head) +{ + lockdep_assert_held(&chan_info->pending_lock); + + WRITE_ONCE(wait_head->acked, 0); + WRITE_ONCE(wait_head->status, NBL_MBX_STATUS_IDLE); + WRITE_ONCE(wait_head->ack_data, NULL); + WRITE_ONCE(wait_head->ack_data_len, 0); + WRITE_ONCE(wait_head->ack_err, 0); + WRITE_ONCE(wait_head->msg_type, 0); + WRITE_ONCE(wait_head->dstid, 0); +} + +static int nbl_chan_teardown_queue(struct nbl_channel_mgt chan_mgt, + u8 chan_type) +{ + struct nbl_chan_info chan_info = chan_mgt->chan_info[chan_type]; + struct nbl_chan_waitqueue_head wait_head; + struct work_struct task; + u16 i; + + if (!READ_ONCE(chan_info->active)) { + dev_warn(chan_mgt->common->dev, "channel not active, skip duplicate teardown\n"); + return 0; + } + / + * Step1: + * block new sender + / + mutex_lock(&chan_info->state_lock); + WRITE_ONCE(chan_info->shutdn, true); + task = READ_ONCE(chan_info->clean_task); + WRITE_ONCE(chan_info->clean_task, NULL); + mutex_unlock(&chan_info->state_lock); + / + * Step2: + * abort pending ACK waiters + * + * Poke only slots with a live waiter (status == WAITING): publish a + * synthetic (-EIO, len 0) completion and wake the owner. The slot is + * deliberately NOT reset to IDLE here - ownership stays with the + * sender, which consumes the completion and resets its own slot in + * nbl_chan_send_msg():out_clear_wait_slot under pending_lock. + * + * Resetting an owned slot here would corrupt two things: + * - an ACKD slot whose sender is consuming the completion + * locklessly (smp_rmb()-ordered reads) could observe torn + * ack_data_len/ack_err pairs; + * - nbl_chan_get_msg_id() hands out IDLE/TIMEOUT slots, so a slot + * reset to IDLE under its live owner could be reallocated to a + * sender admitted before shutdn was set. + * + * IDLE/TIMEOUT slots have no waiter and need no action. A late ACK + * racing this poke is serialized by pending_lock: either the real + * completion lands first (status becomes ACKD and is skipped) or + * this synthetic failure does, both are valid outcomes. + / + mutex_lock(&chan_info->pending_lock); + for (i = 0; i < chan_info->num_txq_entries; i++) { + wait_head = &chan_info->wait[i]; + if (READ_ONCE(wait_head->status) != NBL_MBX_STATUS_WAITING) + continue; + WRITE_ONCE(wait_head->status, NBL_MBX_STATUS_ACKD); + WRITE_ONCE(wait_head->ack_data_len, 0); + WRITE_ONCE(wait_head->ack_err, (s32)-EIO); + / Publish completion fields before the acked flag / + smp_wmb(); + WRITE_ONCE(wait_head->acked, 1); + wake_up(&wait_head->wait_queue); + } + mutex_unlock(&chan_info->pending_lock); + + / + * Step3: + * stop the RX queue unconditionally. The RX QINFO region is written + * only by config/stop paths, never by senders (which only touch the + * TX QINFO region and the doorbell), so no txq_lock is needed. The + * RX engine must be quiesced before the rings are freed, regardless + * of TX-side state - skipping it would leave the device DMAing peer + * messages and descriptor writebacks into released coherent memory. + / + nbl_chan_stop_rx_queue(chan_mgt); + + / + * Drain strategy mirrors mlx5 command interface teardown: + * set shutdown flag first, abort all pending waiters, then + * block until inflight_tx_cnt reaches zero. + * + * After shutdn is set every sender exits promptly at its next + * checkpoint: + * - interrupt-driven senders wake on shutdn immediately + * (it is part of the wait_event condition); + * - ACK polling senders check shutdn each iteration before + * sleeping 1000-1200us. This per-iteration check avoids + * waiting the full 5.0-6.0s worst-case ACK timeout. + * + * The wait is deliberately unbounded: a surviving sender still + * holds txq_lock or touches chan_info (state_lock, wait[]) on its + * exit path, so returning early would let devres free the rings + * and chan_info underneath it. In practice the TX polling loop is + * bounded (NBL_CHAN_TX_WAIT_TIMES) and wedged-link MMIO reads + * eventually complete with all-1s data, so this loop terminates; + * warn periodically to keep a genuinely stuck sender diagnosable. + / + while (wait_event_timeout(chan_info->inflight_wait, + atomic_read(&chan_info->inflight_tx_cnt) == 0, + msecs_to_jiffies(5000)) == 0) + dev_warn(chan_mgt->common->dev, + "teardown: still waiting for %d inflight sender(s)\n", + atomic_read(&chan_info->inflight_tx_cnt)); + + / + * Join the last exiting sender through state_lock. A sender drops + * inflight_tx_cnt and wakes this drain while still holding + * state_lock, so observing the counter reach zero only proves the + * decrement happened; the acquire below is what proves that no + * sender still holds the lock. Once it returns, no sender can be + * using chan_info either: the decrement is the last field access a + * sender makes, and the only thing that follows it is the + * mutex_unlock() of state_lock itself. + * + * That trailing unlock is why this is one acquire/release pair and + * not a handshake that ends the sender: mutex_unlock() may still be + * touching the mutex structure after it has released it, so this does + * not prove the sender has returned from it (see + * Documentation/locking/mutex-design.rst). It is safe only because + * chan_info - mutexes included - is devm-allocated and released at + * detach, long after this teardown, and because nothing frees it + * here. Freeing chan_info or calling mutex_destroy() at this point + * would need a sender-exit completion the channel does not have. + * + * shutdn blocks new senders, so one acquire/release pair is + * sufficient. + / + mutex_lock(&chan_info->state_lock); + mutex_unlock(&chan_info->state_lock); + + / + * All senders have exited, so txq_lock is uncontended: plain lock. + * Take it so this is the sole writer of the TX QINFO block + * (sender-side stop/config/doorbell sequences all run under + * txq_lock). + / + mutex_lock(&chan_info->txq_lock); + nbl_chan_stop_tx_queue(chan_mgt); + mutex_unlock(&chan_info->txq_lock); + + / + * Join the RX clean work before active=false and before devres + * releases the rings. IRQ is already freed and clean_task is + * cleared under state_lock, so no new instance can be queued. + / + if (task) + cancel_work_sync(task); + + WRITE_ONCE(chan_info->active, false); + return 0; +} + +static int nbl_chan_setup_queue(struct nbl_channel_mgt chan_mgt, u8 chan_type) +{ + struct nbl_chan_info chan_info = chan_mgt->chan_info[chan_type]; + struct nbl_hw_ops hw_ops = chan_mgt->hw_ops_tbl->ops; + struct nbl_common_info common = chan_mgt->common; + struct nbl_chan_ring rxq = &chan_info->rxq; + int err; + + if (READ_ONCE(chan_info->active)) { + dev_warn(common->dev, "channel already active, reject duplicate setup\n"); + return -EBUSY; + } + /* + * The descriptor rings and the per-ring contiguous buffer blocks are + * allocated once and released only by devres at device detach: there + * is no free path, so a channel is set up exactly once per device + * lifetime and teardown is terminal for it. Re-setup would allocate + * a second set - about 2 MiB of coherent DMA plus the descriptor + * rings and wait[] - and orphan the first one, and it would revive a + * queue whose state teardown has already retired, so it is rejected. + / + if (chan_info->dma_allocated) { + dev_warn(common->dev, + "channel DMA already allocated, re-setup not supported\n"); + return -EBUSY; + } + + nbl_chan_init_queue_param(chan_info, NBL_CHAN_QUEUE_LEN, + NBL_CHAN_QUEUE_LEN, NBL_CHAN_BUF_LEN, + NBL_CHAN_BUF_LEN); + / + * Set the flag before the first allocation and leave it set even + * when setup fails: whatever nbl_chan_init_queue() managed to + * allocate is devres-owned and stays live until detach, so the retry + * that clearing the flag would allow would allocate a second set on + * top of the orphans. + / + chan_info->dma_allocated = true; + err = nbl_chan_init_queue(common, chan_info); + if (err) + return err; + + nbl_chan_config_queue(chan_mgt, chan_info, true); / tx / + nbl_chan_config_queue(chan_mgt, chan_info, false); / rx / + nbl_chan_update_tail_ptr(hw_ops, chan_mgt->hw_ops_tbl->priv, + rxq->tail_ptr, NBL_MB_RX_QID); + WRITE_ONCE(chan_info->active, true); + return 0; +} + +static bool nbl_chan_txq_full(struct nbl_chan_ring txq, + u16 num_entries) +{ + return NBL_NEXT_ID(txq->next_to_use, num_entries - 1) == + txq->next_to_clean; +} + +static int nbl_chan_update_txqueue(struct nbl_channel_mgt chan_mgt, + struct nbl_chan_info chan_info, + struct nbl_chan_tx_param param) +{ + struct nbl_chan_ring txq = &chan_info->txq; + struct nbl_chan_tx_desc tx_desc; + struct nbl_chan_buf tx_buf; + + /* + * param->arg is non-NULL whenever param->arg_len is non-zero: + * nbl_chan_send_msg() validates that before calling here, because a + * length with no buffer would publish buf_addr/buf_len describing + * memory that was never filled. + / + if (nbl_chan_txq_full(txq, chan_info->num_txq_entries)) + return -EBUSY; + if (param->arg_len > NBL_CHAN_BUF_LEN - sizeof(tx_desc)) + return -EINVAL; + tx_desc = + NBL_CHAN_TX_RING_TO_DESC(txq, txq->next_to_use); + tx_buf = + NBL_CHAN_TX_RING_TO_BUF(txq, txq->next_to_use); + tx_desc->dstid = cpu_to_le16(param->dstid); + tx_desc->msg_type = cpu_to_le16(param->msg_type); + tx_desc->msgid = cpu_to_le16(param->msgid); + + /* + * srcid field is filled by mailbox hardware after peer receives this + * packet, driver producer never writes srcid; reused descriptor slots + * will contain stale srcid value temporarily until hardware overwrites + * it. + / + if (param->arg_len > NBL_CHAN_TX_DESC_EMBEDDED_DATA_LEN) { + memcpy(tx_buf->va, param->arg, param->arg_len); + tx_desc->buf_addr = cpu_to_le64(tx_buf->pa); + tx_desc->buf_len = cpu_to_le16(param->arg_len); + tx_desc->data_len = 0; + memset(tx_desc->data, 0, sizeof(tx_desc->data)); + } else { + memset(tx_desc->data, 0, sizeof(tx_desc->data)); + memset(&tx_desc->buf_addr, 0, sizeof(tx_desc->buf_addr)); + if (param->arg_len) + memcpy(tx_desc->data, param->arg, param->arg_len); + tx_desc->buf_len = 0; + tx_desc->data_len = cpu_to_le16(param->arg_len); + } + / Ensure descriptor data visible to device before AVAIL flag / + dma_wmb(); + tx_desc->flags = cpu_to_le16(BIT(NBL_CHAN_TX_DESC_AVAIL)); + + txq->next_to_use = + NBL_NEXT_ID(txq->next_to_use, chan_info->num_txq_entries - 1); + txq->tail_ptr++; + + return 0; +} + +/ + * Quiesce the TX mailbox queue and reclaim all outstanding + * descriptors. Called from the timeout path of nbl_chan_kick_tx_ring() + * with txq_lock held. + * + * The device failed to fetch/complete the current descriptor within the + * polling window. We assert QUEUE_RST to stop further DMA fetches, + * reclaim every descriptor between next_to_clean and next_to_use, + * reset the software tail_ptr counter to match the hardware reset state, + * and re-enable the queue so subsequent sends can proceed. + / +static void nbl_chan_quiesce_and_reclaim_tx(struct nbl_channel_mgt chan_mgt, + struct nbl_chan_info chan_info) +{ + struct nbl_hw_ops hw_ops = chan_mgt->hw_ops_tbl->ops; + struct nbl_hw_mgt hw_priv = chan_mgt->hw_ops_tbl->priv; + struct nbl_chan_ring txq = &chan_info->txq; + struct nbl_chan_tx_desc tx_desc; + + / + * Assert QUEUE_RST to stop hardware fetching new descriptors. + * stop_mailbox_txq() flushes through the mailbox BAR itself, so + * the reset has reached the device when it returns. + / + hw_ops->stop_mailbox_txq(hw_priv); + + / + * Reclaim all outstanding descriptors between next_to_clean and + * next_to_use. Under txq_lock there is at most one in-flight + * descriptor, but iterate the full range for robustness. + / + while (txq->next_to_clean != txq->next_to_use) { + tx_desc = NBL_CHAN_TX_RING_TO_DESC(txq, + txq->next_to_clean); + WRITE_ONCE(tx_desc->flags, 0); + txq->next_to_clean = + NBL_NEXT_ID(txq->next_to_clean, + chan_info->num_txq_entries - 1); + } + + / + * Hardware tail_ptr counter is cleared by QUEUE_RST. Reset + * software counter to match so the next doorbell update does + * not produce a false 16-bit wrap delta. + / + txq->tail_ptr = 0; + txq->next_to_use = 0; + txq->next_to_clean = 0; + + / + * Authoritative re-arm gate, evaluated while txq_lock is held: + * teardown sets shutdn before draining senders and then owns + * hardware state. If teardown started while we were stopping/ + * reclaiming, leave the queue in QUEUE_RST and never program + * QUEUE_EN, otherwise the queue could be armed again pointing + * at rings that are about to be freed. + / + if (READ_ONCE(chan_info->shutdn)) + return; + + / Re-enable queue with current ring base and size / + nbl_chan_config_queue(chan_mgt, chan_info, true); +} + +static int nbl_chan_kick_tx_ring(struct nbl_channel_mgt chan_mgt, + struct nbl_chan_info chan_info) +{ + struct nbl_hw_ops hw_ops = chan_mgt->hw_ops_tbl->ops; + struct nbl_chan_ring txq = &chan_info->txq; + struct device dev = chan_mgt->common->dev; + int max_retries = NBL_CHAN_TX_WAIT_TIMES; + struct nbl_chan_tx_desc tx_desc; + int retry_count = 0; + u16 msg_type; + + nbl_chan_update_tail_ptr(hw_ops, chan_mgt->hw_ops_tbl->priv, + txq->tail_ptr, NBL_MB_TX_QID); + + tx_desc = NBL_CHAN_TX_RING_TO_DESC(txq, txq->next_to_clean); + / + * Poll for HW to mark descriptor as USED. + * Mailbox is a low-speed control channel for management commands. + * We avoid enabling dedicated per-TX interrupt for single control + * message to reduce interrupt overhead, so use bounded polling + * with small delay instead. + / + while (retry_count < max_retries) { + if (READ_ONCE(chan_info->shutdn)) + return -ESHUTDOWN; + + if (le16_to_cpu(READ_ONCE(tx_desc->flags)) & + BIT(NBL_CHAN_TX_DESC_USED)) { + / + * Order reads of other device-written descriptor + * fields after observing USED. Matches the RX side + * pattern in nbl_chan_clean_queue(). + / + dma_rmb(); + break; + } + + retry_count++; + if (retry_count == max_retries) { + msg_type = le16_to_cpu(READ_ONCE(tx_desc->msg_type)); + dev_err_ratelimited(dev, "chan send msg type: %d timeout\n", + msg_type); + / + * Teardown may have started after the loop-top + * check on this final iteration. Do not run lock + * recovery then: it would stop/reclaim the queue + * and, without the in-quiesce gate, re-arm it. + * nbl_chan_teardown_queue() owns hardware state. + / + if (READ_ONCE(chan_info->shutdn)) + return -ESHUTDOWN; + / + * Device failed to complete this descriptor. + * Quiesce the queue, reclaim the timed-out + * descriptor, and re-enable so future sends can + * proceed instead of stalling the ring full. + / + nbl_chan_quiesce_and_reclaim_tx(chan_mgt, + chan_info); + return -ETIMEDOUT; + } + usleep_range(NBL_CHAN_TX_WAIT_US, NBL_CHAN_TX_WAIT_US_MAX); + } + + txq->next_to_clean = txq->next_to_use; + + return 0; +} + +static void nbl_chan_recv_ack_msg(void priv, u16 srcid, u16 msgid, void data, + u32 data_len) +{ + struct nbl_channel_mgt chan_mgt = (struct nbl_channel_mgt )priv; + struct nbl_chan_waitqueue_head wait_head = NULL; + struct device dev = chan_mgt->common->dev; + struct nbl_chan_info chan_info = + chan_mgt->chan_info[NBL_CHAN_TYPE_MAILBOX]; + u16 w_dstid, w_msgtype, w_msgidx; + u32 payload = data; + u16 ack_msgtype = 0; + u16 ack_msgid = 0; + u32 ack_datalen; + void ack_data; + u32 copy_len; + int w_status; + s32 raw_err; + + if (READ_ONCE(chan_info->shutdn)) + return; + if (data_len > NBL_CHAN_BUF_LEN \|\| + data_len < NBL_CHAN_ACK_HEAD_LEN * sizeof(u32)) { + dev_err_ratelimited(dev, "Invalid ACK data_len: %u\n", + data_len); + return; + } + ack_datalen = data_len - NBL_CHAN_ACK_HEAD_LEN * sizeof(u32); + ack_msgtype = le16_to_cpu((__le16 )(payload + NBL_CHAN_MSG_TYPE_POS)); + ack_msgid = le16_to_cpu((__le16 )(payload + NBL_CHAN_MSG_ID_POS)); + if (FIELD_GET(NBL_CHAN_MSGID_LOC_MASK, ack_msgid) >= + chan_info->num_txq_entries) { + dev_err_ratelimited(dev, "chan recv msg id: %u err\n", + ack_msgid); + return; + } + wait_head = + &chan_info->wait[FIELD_GET(NBL_CHAN_MSGID_LOC_MASK, ack_msgid)]; + + mutex_lock(&chan_info->pending_lock); + + /* Cache repeated READ_ONCE values / + w_dstid = READ_ONCE(wait_head->dstid); + w_status = READ_ONCE(wait_head->status); + w_msgtype = READ_ONCE(wait_head->msg_type); + w_msgidx = READ_ONCE(wait_head->msg_index); + + if (srcid != w_dstid) { + mutex_unlock(&chan_info->pending_lock); + dev_err_ratelimited(dev, "ACK srcid=%u != dstid=%u, rejecting\n", + srcid, w_dstid); + return; + } + if (w_status != NBL_MBX_STATUS_WAITING) { + mutex_unlock(&chan_info->pending_lock); + dev_err_ratelimited(dev, + "Skip ack invalid status, wait msgtype:%u idx:%u status:%d ack msgtype:%u msgid:%u datalen:%u\n", + w_msgtype, w_msgidx, w_status, + ack_msgtype, ack_msgid, ack_datalen); + return; + } + + if (w_msgtype != ack_msgtype) { + mutex_unlock(&chan_info->pending_lock); + dev_err_ratelimited(dev, + "Skip ack msgtype mismatch, wait msgtype:%u idx:%u ack msgtype:%u msgid:%u\n", + w_msgtype, w_msgidx, ack_msgtype, + ack_msgid); + return; + } + if (FIELD_GET(NBL_CHAN_MSGID_INDEX_MASK, ack_msgid) != w_msgidx) { + mutex_unlock(&chan_info->pending_lock); + dev_err_ratelimited(dev, + "Stale ACK: expected index=%u, got msgid=%u\n", + w_msgidx, ack_msgid); + return; + } + + raw_err = (s32)le32_to_cpu((__le32 )&payload[NBL_CHAN_ACK_RET_POS]); + if (raw_err > 0 \|\| raw_err < -MAX_ERRNO) + raw_err = -EREMOTEIO; + + WRITE_ONCE(wait_head->ack_err, raw_err); + + copy_len = min_t(u32, READ_ONCE(wait_head->ack_data_len), ack_datalen); + if (READ_ONCE(wait_head->ack_err) >= 0 && copy_len > 0) { + ack_data = READ_ONCE(wait_head->ack_data); + if (!ack_data) { + dev_err_ratelimited(dev, "ACK payload dropped: ack_data is NULL\n"); + WRITE_ONCE(wait_head->ack_data_len, 0); + goto ack_done; + } + memcpy((char )ack_data, + payload + NBL_CHAN_ACK_HEAD_LEN, copy_len); + WRITE_ONCE(wait_head->ack_data_len, (u16)copy_len); + } else { + WRITE_ONCE(wait_head->ack_data_len, 0); + } +ack_done: + /* Guarantee payload data finished before acked flag visible / + smp_wmb(); + WRITE_ONCE(wait_head->acked, 1); + WRITE_ONCE(wait_head->status, NBL_MBX_STATUS_ACKD); + mutex_unlock(&chan_info->pending_lock); + wake_up(&wait_head->wait_queue); +} + +static void nbl_chan_recv_msg(struct nbl_channel_mgt chan_mgt, void data) +{ + struct device dev = chan_mgt->common->dev; + struct nbl_chan_msg_node_data msg_handler; + u16 msg_type, payload_len, srcid, msgid; + struct nbl_chan_info chan_info = + chan_mgt->chan_info[NBL_CHAN_TYPE_MAILBOX]; + struct nbl_chan_tx_desc tx_desc; + void payload; + size_t avail_space; + u16 data_len_fw; + + if (READ_ONCE(chan_info->shutdn)) + return; + + tx_desc = data; + msg_type = le16_to_cpu(READ_ONCE(tx_desc->msg_type)); + dev_dbg(dev, "recv msg_type: %d\n", msg_type); + + srcid = le16_to_cpu(READ_ONCE(tx_desc->srcid)); + msgid = le16_to_cpu(READ_ONCE(tx_desc->msgid)); + + if (msg_type >= NBL_CHAN_MSG_MAILBOX_MAX) + return; + + data_len_fw = le16_to_cpu(READ_ONCE(tx_desc->data_len)); + if (data_len_fw) { + payload_len = data_len_fw; + + if (payload_len > NBL_CHAN_TX_DESC_EMBEDDED_DATA_LEN) { + dev_err_ratelimited(dev, + "data_len=%u exceeds embedded buffer size=%u\n", + payload_len, + NBL_CHAN_TX_DESC_EMBEDDED_DATA_LEN); + return; + } + /* Small pkt: payload stored inside descriptor data[] array / + payload = tx_desc->data; + } else { + payload_len = le16_to_cpu(READ_ONCE(tx_desc->buf_len)); + + avail_space = NBL_CHAN_BUF_LEN - sizeof(tx_desc); + if (payload_len > avail_space) { + dev_err_ratelimited(dev, + "buf_len=%u exceeds external buffer size=%zu\n", + payload_len, avail_space); + return; + } + /* Large pkt: payload follows immediately after tx_desc / + payload = tx_desc + 1; + } + + msg_handler = xa_load(&chan_mgt->handler_xa, msg_type); + if (!msg_handler \|\| !msg_handler->func) { + dev_err_ratelimited(dev, + "No handler for msg_type: %u (srcid=%u, msgid=%u)\n", + msg_type, srcid, msgid); + return; + } + + msg_handler->func(msg_handler->priv, srcid, msgid, payload, + payload_len); +} + +static void nbl_chan_advance_rx_ring(struct nbl_channel_mgt chan_mgt, + struct nbl_chan_info chan_info, + struct nbl_chan_ring rxq) +{ + struct nbl_hw_ops hw_ops = chan_mgt->hw_ops_tbl->ops; + struct nbl_chan_rx_desc rx_desc; + struct nbl_chan_buf rx_buf; + u16 next_to_use; + + next_to_use = rxq->next_to_use; + rx_desc = NBL_CHAN_RX_RING_TO_DESC(rxq, next_to_use); + rx_buf = NBL_CHAN_RX_RING_TO_BUF(rxq, next_to_use); + + / + * Recycle the RX descriptor at next_to_use. The initial + * unused slot is intentionally recycled after the first + * RX descriptor is consumed, allowing the ring to become + * fully populated while next_to_clean tracks the consumer. + / + rx_desc->buf_addr = cpu_to_le64(rx_buf->pa); + rx_desc->buf_len = cpu_to_le32(chan_info->rxq_buf_size); + + / + * DMA Write Memory Barrier: + * Ensures all previous DMA-mapped writes (buffer address/length) + * are completed before the descriptor flags are updated. + * This prevents hardware from seeing a partially updated descriptor + * where flags are set but buffer info isn't ready yet. + / + dma_wmb(); + + rx_desc->flags = cpu_to_le16(BIT(NBL_CHAN_RX_DESC_AVAIL)); + + rxq->next_to_use++; + if (rxq->next_to_use == chan_info->num_rxq_entries) + rxq->next_to_use = 0; + rxq->tail_ptr++; + + nbl_chan_update_tail_ptr(hw_ops, chan_mgt->hw_ops_tbl->priv, + rxq->tail_ptr, NBL_MB_RX_QID); +} + +static void nbl_chan_clean_queue(struct nbl_channel_mgt chan_mgt, + struct nbl_chan_info chan_info) +{ + struct nbl_common_info common = chan_mgt->common; + struct nbl_chan_ring rxq = &chan_info->rxq; + struct device dev = chan_mgt->common->dev; + u32 budget = NBL_CHAN_RX_CLEAN_BUDGET; + struct nbl_chan_rx_desc rx_desc; + struct nbl_chan_buf rx_buf; + struct work_struct task; + bool more_work = false; + u16 next_to_clean; + u16 flags; + + / Nothing to clean before setup: the RX ring is still NULL / + if (!READ_ONCE(chan_info->active)) + return; + + next_to_clean = rxq->next_to_clean; + rx_desc = NBL_CHAN_RX_RING_TO_DESC(rxq, next_to_clean); + rx_buf = NBL_CHAN_RX_RING_TO_BUF(rxq, next_to_clean); + while (le16_to_cpu(READ_ONCE(rx_desc->flags)) & + BIT(NBL_CHAN_RX_DESC_USED)) { + flags = le16_to_cpu(READ_ONCE(rx_desc->flags)); + + if (READ_ONCE(chan_info->shutdn)) + break; + if (!(flags & BIT(NBL_CHAN_RX_DESC_WRITE))) + dev_dbg(dev, + "mailbox rx flag 0x%x missing NBL_CHAN_RX_DESC_WRITE\n", + flags); + + / Make sure hardware written descriptor visible to CPU / + dma_rmb(); + nbl_chan_recv_msg(chan_mgt, rx_buf->va); + nbl_chan_advance_rx_ring(chan_mgt, chan_info, rxq); + next_to_clean++; + if (next_to_clean == chan_info->num_rxq_entries) + next_to_clean = 0; + rx_desc = NBL_CHAN_RX_RING_TO_DESC(rxq, next_to_clean); + rx_buf = NBL_CHAN_RX_RING_TO_BUF(rxq, next_to_clean); + if (--budget == 0) { + more_work = true; + break; + } + cond_resched(); + } + rxq->next_to_clean = next_to_clean; + + mutex_lock(&chan_info->state_lock); + / Prevent queue_work after teardown clears clean_task / + if (READ_ONCE(chan_info->shutdn)) { + mutex_unlock(&chan_info->state_lock); + return; + } + if (common->wq && more_work) { + task = READ_ONCE(chan_info->clean_task); + if (task) + queue_work(common->wq, task); + } + mutex_unlock(&chan_info->state_lock); +} + +static void nbl_chan_clean_queue_subtask(struct nbl_channel_mgt chan_mgt, + u8 chan_type) +{ + struct nbl_chan_info chan_info = chan_mgt->chan_info[chan_type]; + + nbl_chan_clean_queue(chan_mgt, chan_info); +} + +static int nbl_chan_get_msg_id(struct nbl_chan_info chan_info, + u16 msgid) +{ + int search_loc = READ_ONCE(chan_info->wait_head_index), i; + struct nbl_chan_waitqueue_head wait = NULL; + int status; + int next; + + lockdep_assert_held(&chan_info->pending_lock); + for (i = 0; i < chan_info->num_txq_entries; i++) { + wait = &chan_info->wait[search_loc]; + status = READ_ONCE(wait->status); + if (status == NBL_MBX_STATUS_IDLE \|\| + status == NBL_MBX_STATUS_TIMEOUT) { + WRITE_ONCE(wait->msg_index, + NBL_NEXT_ID(wait->msg_index, + NBL_CHAN_MSG_INDEX_MAX)); + + msgid = FIELD_PREP(NBL_CHAN_MSGID_INDEX_MASK, + wait->msg_index) \| + FIELD_PREP(NBL_CHAN_MSGID_LOC_MASK, + search_loc); + + / Advance starting search position for next caller / + next = NBL_NEXT_ID(search_loc, + chan_info->num_txq_entries - 1); + WRITE_ONCE(chan_info->wait_head_index, next); + return 0; + } + + search_loc = NBL_NEXT_ID(search_loc, + chan_info->num_txq_entries - 1); + } + + / + * All tx slots are occupied. May happen under high transmit load + * or delayed remote ACK responses. Caller should retry later. + / + return -EAGAIN; +} + +static int nbl_chan_send_msg(struct nbl_channel_mgt chan_mgt, + struct nbl_chan_send_info chan_send) +{ + struct nbl_common_info common = chan_mgt->common; + struct nbl_chan_waitqueue_head wait_head = NULL; + struct nbl_chan_tx_param tx_param = { 0 }; + int i = NBL_CHAN_TX_WAIT_ACK_TIMES; + struct nbl_chan_info chan_info = + chan_mgt->chan_info[NBL_CHAN_TYPE_MAILBOX]; + struct device dev = common->dev; + struct work_struct task; + u16 msgid = 0; + int ret; + + if (chan_send->resp_len > NBL_CHAN_BUF_LEN) { + dev_err_ratelimited(dev, "resp_len %zu exceeds max %d\n", + chan_send->resp_len, NBL_CHAN_BUF_LEN); + return -EINVAL; + } + + /* + * A length without a payload would leave the descriptor pointing at + * whatever the reused slot still holds - nbl_chan_update_txqueue() + * skips the copy - and the peer would receive those stale bytes. + * Reject the combination instead of transmitting them. + / + if (chan_send->arg_len && !chan_send->arg) { + dev_err_ratelimited(dev, "arg_len %zu without arg\n", + chan_send->arg_len); + return -EINVAL; + } + + / + * Entry points are only reached after a successful setup_queue() + * (probe path), but the channel is a dynamic object: fail cleanly + * instead of dereferencing rings/wait[] that were never allocated + * (num_txq_entries == 0). active is published last in setup and + * cleared last in teardown, so a false value here means the channel + * never came up - or has already been torn down, and shutdn tells + * the two apart: it is set before active is cleared, so a + * post-teardown sender gets the same -ESHUTDOWN as the check below + * rather than the -ENODEV reserved for a channel setup_queue() + * never completed. + / + if (!READ_ONCE(chan_info->active)) { + if (READ_ONCE(chan_info->shutdn)) + return -ESHUTDOWN; + return -ENODEV; + } + + mutex_lock(&chan_info->state_lock); + if (READ_ONCE(chan_info->shutdn)) { + mutex_unlock(&chan_info->state_lock); + return -ESHUTDOWN; + } + atomic_inc(&chan_info->inflight_tx_cnt); + mutex_unlock(&chan_info->state_lock); + + tx_param.msg_type = chan_send->msg_type; + tx_param.arg = chan_send->arg; + tx_param.arg_len = chan_send->arg_len; + tx_param.dstid = chan_send->dstid; + tx_param.msgid = msgid; + if (chan_send->ack) { + mutex_lock(&chan_info->pending_lock); + + ret = nbl_chan_get_msg_id(chan_info, &msgid); + if (ret) { + mutex_unlock(&chan_info->pending_lock); + dev_err_ratelimited(dev, + "Channel tx wait head full, send msgtype:%u to dstid:%u failed\n", + chan_send->msg_type, + chan_send->dstid); + goto out_clean_inflight; + } + wait_head = + &chan_info->wait[FIELD_GET(NBL_CHAN_MSGID_LOC_MASK, + msgid)]; + WRITE_ONCE(wait_head->acked, 0); + WRITE_ONCE(wait_head->ack_data, chan_send->resp); + WRITE_ONCE(wait_head->ack_data_len, chan_send->resp_len); + WRITE_ONCE(wait_head->msg_type, chan_send->msg_type); + WRITE_ONCE(wait_head->msg_index, + FIELD_GET(NBL_CHAN_MSGID_INDEX_MASK, msgid)); + WRITE_ONCE(wait_head->dstid, chan_send->dstid); + + WRITE_ONCE(wait_head->status, NBL_MBX_STATUS_WAITING); + mutex_unlock(&chan_info->pending_lock); + + tx_param.msgid = msgid; + } + + mutex_lock(&chan_info->txq_lock); + ret = nbl_chan_update_txqueue(chan_mgt, chan_info, &tx_param); + if (ret) { + mutex_unlock(&chan_info->txq_lock); + dev_err_ratelimited(dev, + "Channel tx queue full, send msgtype:%u to dstid:%u failed\n", + chan_send->msg_type, chan_send->dstid); + if (wait_head) + goto out_clear_wait_slot; + goto out_clean_inflight; + } + + ret = nbl_chan_kick_tx_ring(chan_mgt, chan_info); + mutex_unlock(&chan_info->txq_lock); + if (ret) { + if (wait_head) + goto out_clear_wait_slot; + goto out_clean_inflight; + } + + if (!chan_send->ack) { + ret = 0; + goto out_clean_inflight; + } + + if (test_bit(NBL_CHAN_IRQ_RDY, chan_info->state)) { + while (!READ_ONCE(wait_head->acked)) { + / + * avoids long task blocking when interrupt mode is + * disabled mid-wait. Cannot guarantee subsequent ACK + * delivery after interrupt mask off, only prevents + * infinite blocking. Spurious timeout is possible. + / + ret = wait_event_timeout(wait_head->wait_queue, + READ_ONCE(wait_head->acked) \|\| + READ_ONCE(chan_info->shutdn) \|\| + !test_bit(NBL_CHAN_IRQ_RDY, + chan_info->state), + NBL_CHAN_ACK_WAIT_TIME); + + if (READ_ONCE(chan_info->shutdn)) { + ret = -ESHUTDOWN; + goto out_clear_wait_slot; + } + if (!test_bit(NBL_CHAN_IRQ_RDY, chan_info->state)) { + ret = -EIO; + goto out_clear_wait_slot; + } + if (ret == 0) { + mutex_lock(&chan_info->pending_lock); + if (READ_ONCE(wait_head->status) == + NBL_MBX_STATUS_WAITING) { + WRITE_ONCE(wait_head->status, + NBL_MBX_STATUS_TIMEOUT); + WRITE_ONCE(wait_head->acked, 0); + WRITE_ONCE(wait_head->ack_data, NULL); + WRITE_ONCE(wait_head->ack_data_len, 0); + / + * Ensure all status/ack slot + * updates are visible before subsequent + * readers observe acked == 0 + / + smp_wmb(); + mutex_unlock(&chan_info->pending_lock); + dev_err_ratelimited(dev, + "Channel waiting ack failed, message type: %d, msg id: %u\n", + chan_send->msg_type, + msgid); + ret = -ETIMEDOUT; + / + * TIMEOUT slots can be reused by + * another sender. A late ACK is + * rejected by the receive path, + * which only accepts WAITING slots + * (and by the msg_index generation + * check after reallocation). The + * current sender no longer owns the + * slot, so skip the IDLE reset. + / + goto out_clean_inflight; + } + / + * ACK won the timeout race: between the + * final condition check and this lock the + * receiver published acked=1/ACKD and + * copied the payload into our response + * buffer. The slot is still exclusively + * ours (only IDLE/TIMEOUT slots are ever + * reallocated), so consume the completion: + * unlock and continue to the loop-bottom + * acked check, which enters the normal + * success readout and IDLE reset. Leaving + * here via the timeout path would strand + * the slot in ACKD forever, permanently + * shrinking the 256-entry pool, and would + * report a false -ETIMEDOUT for a request + * whose response already arrived. + / + mutex_unlock(&chan_info->pending_lock); + } + + if (READ_ONCE(wait_head->acked)) + break; + } + if (READ_ONCE(wait_head->acked)) { + / + * Load ordering: observe acked flag before + * reading ACK payload metadata. + / + smp_rmb(); + chan_send->ack_len = READ_ONCE(wait_head->ack_data_len); + ret = READ_ONCE(wait_head->ack_err); + } + } else { + / Polling path for synchronous ACK / + while (i--) { + if (READ_ONCE(chan_info->shutdn)) { + ret = -ESHUTDOWN; + goto out_clear_wait_slot; + } + + mutex_lock(&chan_info->state_lock); + task = READ_ONCE(chan_info->clean_task); + if (common->wq && task && + !READ_ONCE(chan_info->shutdn) && + !work_pending(task)) + queue_work(common->wq, task); + mutex_unlock(&chan_info->state_lock); + if (READ_ONCE(wait_head->acked)) { + / + * Guarantee load order: observe acked + * flag before reading ack payload metadata. + / + smp_rmb(); + chan_send->ack_len = + READ_ONCE(wait_head->ack_data_len); + ret = READ_ONCE(wait_head->ack_err); + goto out_clear_wait_slot; + } + + usleep_range(NBL_CHAN_TX_WAIT_ACK_US_MIN, + NBL_CHAN_TX_WAIT_ACK_US_MAX); + cond_resched(); + } + mutex_lock(&chan_info->pending_lock); + if (READ_ONCE(wait_head->status) == NBL_MBX_STATUS_ACKD) { + chan_send->ack_len = READ_ONCE(wait_head->ack_data_len); + ret = READ_ONCE(wait_head->ack_err); + mutex_unlock(&chan_info->pending_lock); + goto out_clear_wait_slot; + } + / + * Polling timed out without receiving an ACK. Transition + * the slot to TIMEOUT so nbl_chan_get_msg_id() can reuse + * it. Must not proceed to out_clean_inflight with + * the slot still in WAITING state — that would leak the + * slot permanently and eventually exhaust the 256-entry + * channel queue. + / + if (READ_ONCE(wait_head->status) == NBL_MBX_STATUS_WAITING) { + WRITE_ONCE(wait_head->status, NBL_MBX_STATUS_TIMEOUT); + WRITE_ONCE(wait_head->acked, 0); + WRITE_ONCE(wait_head->ack_data, NULL); + WRITE_ONCE(wait_head->ack_data_len, 0); + } + mutex_unlock(&chan_info->pending_lock); + dev_err_ratelimited(dev, + "Channel polling ack failed, message type: %d msg id: %u\n", + chan_send->msg_type, msgid); + ret = -ETIMEDOUT; + goto out_clean_inflight; + } + +out_clear_wait_slot: + mutex_lock(&chan_info->pending_lock); + nbl_chan_reset_wait_head(chan_info, wait_head); + mutex_unlock(&chan_info->pending_lock); + +out_clean_inflight: + mutex_lock(&chan_info->state_lock); + if (atomic_dec_and_test(&chan_info->inflight_tx_cnt)) + wake_up(&chan_info->inflight_wait); + mutex_unlock(&chan_info->state_lock); + return ret; +} + +static int nbl_chan_send_ack(struct nbl_channel_mgt chan_mgt, + struct nbl_chan_ack_info chan_ack) +{ + size_t head_len = NBL_CHAN_ACK_HEAD_LEN sizeof(u32); + size_t data_len = chan_ack->data_len; + struct nbl_chan_send_info chan_send; + __le32 tmp; + size_t len; + int ret; + + if (data_len > + NBL_CHAN_BUF_LEN - sizeof(struct nbl_chan_tx_desc) - head_len) + return -EINVAL; + + len = head_len + data_len; + tmp = kzalloc(len, GFP_KERNEL); + if (!tmp) + return -ENOMEM; + + (__le16 )&tmp[NBL_CHAN_MSG_TYPE_POS] = + cpu_to_le16(chan_ack->msg_type); + (__le16 )&tmp[NBL_CHAN_MSG_ID_POS] = cpu_to_le16(chan_ack->msgid); + tmp[NBL_CHAN_ACK_RET_POS] = cpu_to_le32(chan_ack->err); + if (chan_ack->data && chan_ack->data_len) + memcpy(&tmp[NBL_CHAN_ACK_HEAD_LEN], chan_ack->data, + chan_ack->data_len); + + nbl_chan_fill_send_info(&chan_send, chan_ack->dstid, NBL_CHAN_MSG_ACK, + tmp, len, NULL, 0, 0); + ret = nbl_chan_send_msg(chan_mgt, &chan_send); + kfree(tmp); + + return ret; +} + +static int nbl_chan_register_msg(struct nbl_channel_mgt chan_mgt, u16 msg_type, + nbl_chan_resp func, void callback) +{ + return nbl_chan_add_msg_handler(chan_mgt, msg_type, func, callback); +} + +static bool nbl_chan_check_queue_exist(struct nbl_channel_mgt chan_mgt, + u8 chan_type) +{ + struct nbl_chan_info chan_info = chan_mgt->chan_info[chan_type]; + + return chan_info ? true : false; +} + +static void nbl_chan_register_chan_task(struct nbl_channel_mgt chan_mgt, + u8 chan_type, struct work_struct task) +{ + struct nbl_chan_info chan_info = chan_mgt->chan_info[chan_type]; + + mutex_lock(&chan_info->state_lock); + if (!READ_ONCE(chan_info->shutdn)) + WRITE_ONCE(chan_info->clean_task, task); + mutex_unlock(&chan_info->state_lock); +} + +static void nbl_chan_set_queue_state(struct nbl_channel_mgt chan_mgt, + enum nbl_chan_state state, u8 chan_type, + u8 set) +{ + struct nbl_chan_info chan_info = chan_mgt->chan_info[chan_type]; + int i; + + if (set) + set_bit(state, chan_info->state); + else + clear_bit(state, chan_info->state); + /* + * When clearing IRQ_RDY, wake all per-slot wait queues so + * sleeping senders observe the condition immediately and + * return -EIO instead of waiting out the 3s timeout and + * reporting -ETIMEDOUT. + * + * wait[] is only allocated by a successful nbl_chan_init_tx_queue(), + * while num_txq_entries is already set by nbl_chan_init_queue_param(); + * skip the walk if that allocation never happened. + / + if (!set && state == NBL_CHAN_IRQ_RDY && chan_info->wait) { + for (i = 0; i < chan_info->num_txq_entries; i++) + wake_up_all(&chan_info->wait[i].wait_queue); + } +} + +static struct nbl_channel_ops chan_ops = { + .send_msg = nbl_chan_send_msg, + .send_ack = nbl_chan_send_ack, + .register_msg = nbl_chan_register_msg, + .cfg_chan_qinfo_map_table = nbl_chan_cfg_qinfo_map_table, + .check_queue_exist = nbl_chan_check_queue_exist, + .setup_queue = nbl_chan_setup_queue, + .teardown_queue = nbl_chan_teardown_queue, + .clean_queue_subtask = nbl_chan_clean_queue_subtask, + .register_chan_task = nbl_chan_register_chan_task, + .set_queue_state = nbl_chan_set_queue_state, +}; + +static struct nbl_channel_mgt +nbl_chan_setup_chan_mgt(struct nbl_adapter adapter) +{ + struct nbl_hw_ops_tbl hw_ops_tbl = adapter->intf.hw_ops_tbl; + struct nbl_common_info common = &adapter->common; + struct device dev = &adapter->pdev->dev; + struct nbl_channel_mgt chan_mgt; + struct nbl_chan_info mailbox; + int ret; + + chan_mgt = devm_kzalloc(dev, sizeof(chan_mgt), GFP_KERNEL); + if (!chan_mgt) + return ERR_PTR(-ENOMEM); + + chan_mgt->common = common; + chan_mgt->hw_ops_tbl = hw_ops_tbl; + + mailbox = devm_kzalloc(dev, sizeof(mailbox), GFP_KERNEL); + if (!mailbox) + return ERR_PTR(-ENOMEM); + mailbox->chan_type = NBL_CHAN_TYPE_MAILBOX; + chan_mgt->chan_info[NBL_CHAN_TYPE_MAILBOX] = mailbox; + + ret = nbl_chan_init_msg_handler(chan_mgt); + if (ret) + return ERR_PTR(ret); + ret = devm_mutex_init(common->dev, &mailbox->txq_lock); + if (ret) + return ERR_PTR(ret); + ret = devm_mutex_init(common->dev, &mailbox->state_lock); + if (ret) + return ERR_PTR(ret); + ret = devm_mutex_init(common->dev, &mailbox->pending_lock); + if (ret) + return ERR_PTR(ret); + return chan_mgt; +} + +static struct nbl_channel_ops_tbl * +nbl_chan_setup_ops(struct device dev, struct nbl_channel_mgt chan_mgt) +{ + struct nbl_channel_ops_tbl chan_ops_tbl; + int ret; + + chan_ops_tbl = devm_kzalloc(dev, sizeof(chan_ops_tbl), GFP_KERNEL); + if (!chan_ops_tbl) + return ERR_PTR(-ENOMEM); + if (!chan_ops.send_msg \|\| !chan_ops.send_ack \|\| + !chan_ops.register_msg \|\| !chan_ops.cfg_chan_qinfo_map_table \|\| + !chan_ops.check_queue_exist \|\| !chan_ops.setup_queue \|\| + !chan_ops.teardown_queue \|\| !chan_ops.clean_queue_subtask \|\| + !chan_ops.register_chan_task \|\| !chan_ops.set_queue_state) + return ERR_PTR(-EINVAL); + + chan_ops_tbl->ops = &chan_ops; + chan_ops_tbl->priv = chan_mgt; + + ret = nbl_chan_register_msg(chan_mgt, NBL_CHAN_MSG_ACK, + nbl_chan_recv_ack_msg, chan_mgt); + if (ret) + return ERR_PTR(ret); + + return chan_ops_tbl; +} + +int nbl_chan_init_common(struct nbl_adapter adap) +{ + struct nbl_channel_ops_tbl chan_ops_tbl; + struct device dev = &adap->pdev->dev; + struct nbl_channel_mgt chan_mgt; + int ret; + + chan_mgt = nbl_chan_setup_chan_mgt(adap); + if (IS_ERR(chan_mgt)) + return PTR_ERR(chan_mgt); + + chan_ops_tbl = nbl_chan_setup_ops(dev, chan_mgt); + if (IS_ERR(chan_ops_tbl)) { + ret = PTR_ERR(chan_ops_tbl); + goto cleanup_mgt; + } + + adap->intf.channel_ops_tbl = chan_ops_tbl; + adap->core.chan_mgt = chan_mgt; + ret = nbl_common_create_wq(&adap->common); + if (ret) + goto cleanup_mgt; + return 0; + +cleanup_mgt: + nbl_chan_remove_msg_handler(chan_mgt); + return ret; +} + +void nbl_chan_remove_common(struct nbl_adapter adap) +{ + struct nbl_channel_mgt chan_mgt = adap->core.chan_mgt; + + if (!chan_mgt) + return; + nbl_common_destroy_wq(&adap->common); + /* + * All channel queues shall be torn down earlier in remove path + * to drain inflight tx workers and stop hardware before destroying + * message handler xarray. + / + nbl_chan_remove_msg_handler(chan_mgt); + adap->core.chan_mgt = NULL; +} diff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_channel/nbl_channel.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_channel/nbl_channel.h new file mode 100644 index 00000000000000..168f5de221bb78 --- /dev/null +++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_channel/nbl_channel.h @@ -0,0 +1,196 @@ +/ SPDX-License-Identifier: GPL-2.0 / +/ + * Copyright (c) 2026 Nebula Matrix Limited. + / + +#ifndef _NBL_CHANNEL_H_ +#define _NBL_CHANNEL_H_ + +#include <linux/types.h> +#include <linux/xarray.h> + +#include "../nbl_include/nbl_include.h" +#include "../nbl_include/nbl_def_channel.h" +#include "../nbl_include/nbl_def_hw.h" +#include "../nbl_include/nbl_def_common.h" +#include "../nbl_core.h" + +#define NBL_CHAN_TX_RING_TO_DESC(tx_ring, i) \ + (&((((tx_ring)->desc.tx_desc))[i])) +#define NBL_CHAN_RX_RING_TO_DESC(rx_ring, i) \ + (&((((rx_ring)->desc.rx_desc))[i])) +#define NBL_CHAN_TX_RING_TO_BUF(tx_ring, i) (&(((tx_ring)->buf)[i])) +#define NBL_CHAN_RX_RING_TO_BUF(rx_ring, i) (&(((rx_ring)->buf)[i])) + +#define NBL_CHAN_TX_WAIT_US 100 +#define NBL_CHAN_TX_WAIT_US_MAX 120 +#define NBL_CHAN_TX_WAIT_TIMES 100 +#define NBL_CHAN_TX_WAIT_ACK_US_MIN 1000 +#define NBL_CHAN_TX_WAIT_ACK_US_MAX 1200 +#define NBL_CHAN_TX_WAIT_ACK_TIMES 5000 +#define NBL_CHAN_QUEUE_LEN 256 +#define NBL_CHAN_BUF_LEN 4096 +#define NBL_CHAN_TX_DESC_EMBEDDED_DATA_LEN 16 + +#define NBL_CHAN_TX_DESC_AVAIL 0 +#define NBL_CHAN_TX_DESC_USED 1 +#define NBL_CHAN_RX_DESC_WRITE 1 +#define NBL_CHAN_RX_DESC_AVAIL 3 +#define NBL_CHAN_RX_DESC_USED 4 + +#define NBL_CHAN_ACK_HEAD_LEN 3 +#define NBL_CHAN_ACK_RET_POS 2 +#define NBL_CHAN_MSG_ID_POS 1 +#define NBL_CHAN_MSG_TYPE_POS 0 + +#define NBL_CHAN_ACK_WAIT_TIME (3 HZ) +#define NBL_CHAN_RX_CLEAN_BUDGET 64 + +enum { + NBL_MB_RX_QID = 0, + NBL_MB_TX_QID = 1, +}; + +enum { + NBL_MBX_STATUS_IDLE = 0, + NBL_MBX_STATUS_WAITING, + NBL_MBX_STATUS_ACKD, + NBL_MBX_STATUS_TIMEOUT, +}; + +struct nbl_chan_tx_param { + enum nbl_chan_msg_type msg_type; + void arg; + size_t arg_len; + u16 dstid; + u16 msgid; +}; + +struct nbl_chan_buf { + void va; + dma_addr_t pa; + size_t size; +}; + +struct nbl_chan_tx_desc { + __le16 flags; + __le16 srcid; + __le16 dstid; + __le16 data_len; + __le16 buf_len; + __le64 buf_addr; + __le16 msg_type; + u8 data[NBL_CHAN_TX_DESC_EMBEDDED_DATA_LEN]; + __le16 msgid; + u8 rsv[26]; +} __packed; + +struct nbl_chan_rx_desc { + __le16 flags; + __le32 buf_len; + __le16 buf_id; + __le64 buf_addr; +} __packed; + +union nbl_chan_desc_ptr { + struct nbl_chan_tx_desc tx_desc; + struct nbl_chan_rx_desc rx_desc; +}; + +struct nbl_chan_ring { + union nbl_chan_desc_ptr desc; + /* Per-entry slots carved out of one contiguous coherent block / + struct nbl_chan_buf buf; + u16 next_to_use; + u16 tail_ptr; /* hardware does modulo ring size internally / + u16 next_to_clean; + dma_addr_t dma; +}; + +#define NBL_CHAN_MSG_INDEX_MAX 63 + +#define NBL_CHAN_MSGID_INDEX_MASK GENMASK(5, 0) +#define NBL_CHAN_MSGID_LOC_MASK GENMASK(13, 6) + +static inline void nbl_chan_update_tail_ptr(struct nbl_hw_ops hw_ops, + void hw_priv, u32 tail_ptr, u8 qid) +{ + hw_ops->update_mailbox_queue_tail_ptr(hw_priv, tail_ptr, qid); +} + +struct nbl_chan_waitqueue_head { + struct wait_queue_head wait_queue; + char ack_data; + int acked; + s32 ack_err; + u16 ack_data_len; + u16 msg_type; + int status; + u8 msg_index; + u16 dstid; +}; + +struct nbl_chan_info { + wait_queue_head_t inflight_wait; + struct nbl_chan_ring txq; + struct nbl_chan_ring rxq; + struct nbl_chan_waitqueue_head wait; + / + * Serializes TX ring producer state (next_to_use/next_to_clean/ + * tail_ptr), descriptor publication and the doorbell, and the TX + * queue config/stop sequences against + * nbl_chan_quiesce_and_reclaim_tx(). + / + struct mutex txq_lock; + / + * Serializes the channel lifecycle: shutdn publication, clean_task + * registration/clearing, inflight_tx_cnt transitions and the + * inflight_wait wakeup. The state bitmap is not covered by this + * lock - it is only touched with atomic bit ops (set_bit(), + * clear_bit(), test_bit()). + / + struct mutex state_lock; + / + * Serializes the ACK wait slot array (wait[]): slot allocation in + * nbl_chan_get_msg_id(), slot field publication in + * nbl_chan_send_msg(), ACK completion in nbl_chan_recv_ack_msg() + * and slot reset in nbl_chan_reset_wait_head(). + / + struct mutex pending_lock; + struct work_struct clean_task; + u16 wait_head_index; + u16 num_txq_entries; + u16 num_rxq_entries; + u16 txq_buf_size; + u16 rxq_buf_size; + DECLARE_BITMAP(state, NBL_CHAN_STATE_NBITS); + u8 chan_type; + atomic_t inflight_tx_cnt; + bool shutdn; + bool active; + /* + * One-shot lifecycle flag: DMA allocations use devm (descriptor/buf + * arrays) and dmam (coherent descriptor ring, per-ring contiguous + * buffer block) and are only released at device detach. Teardown + * only stops the hardware and clears active; it does not free + * memory. setup must never run twice on the same chan_info or the + * first allocation set would be orphaned. + / + bool dma_allocated; +}; + +struct nbl_chan_msg_node_data { + nbl_chan_resp func; + void priv; +}; + +struct nbl_channel_mgt { + struct nbl_common_info common; + struct nbl_hw_ops_tbl hw_ops_tbl; + struct nbl_chan_info chan_info[NBL_CHAN_TYPE_MAX]; + / Serializes handler_xa registration and teardown / + struct mutex handler_lock; + struct xarray handler_xa; +}; + +#endif diff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_common/nbl_common.c b/drivers/net/ethernet/nebula-matrix/nbl/nbl_common/nbl_common.c new file mode 100644 index 00000000000000..302c3c48dc4d59 --- /dev/null +++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_common/nbl_common.c @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: GPL-2.0 +/ + * Copyright (c) 2026 Nebula Matrix Limited. + / + +#include <linux/device.h> +#include "nbl_common.h" + +void nbl_common_destroy_wq(struct nbl_common_info common) +{ + if (!common \|\| !common->wq) + return; + + destroy_workqueue(common->wq); + common->wq = NULL; +} + +int nbl_common_create_wq(struct nbl_common_info common) +{ + char wq_name[32]; + + snprintf(wq_name, sizeof(wq_name), "nbl_wq_%s", pci_name(common->pdev)); + common->wq = alloc_workqueue(wq_name, WQ_UNBOUND, 0); + if (!common->wq) { + dev_err(common->dev, "Failed to alloc workqueue %s\n", wq_name); + return -ENOMEM; + } + + return 0; +} + +/* + * nbl_common_func_id_to_rel_pf_id - convert absolute PF id to relative PF id + * @common: common device info + * @pf_id: absolute PF identifier + * @rel_pf_id: output relative pf id + * + * Leonis uses fixed mgt_pf = 0. Support future non-zero management PF. + * + * Return: 0 on success, -EINVAL on invalid arguments. + / +int nbl_common_func_id_to_rel_pf_id(struct nbl_common_info common, u32 pf_id, + u32 rel_pf_id) +{ + if (!rel_pf_id) + return -EINVAL; + + if (pf_id < common->mgt_pf) + return -EINVAL; + rel_pf_id = pf_id - common->mgt_pf; + return 0; +} diff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_common/nbl_common.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_common/nbl_common.h new file mode 100644 index 00000000000000..129073b08eb903 --- /dev/null +++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_common/nbl_common.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0 / +/ + * Copyright (c) 2026 Nebula Matrix Limited. + / + +#ifndef _NBL_COMMON_H_ +#define _NBL_COMMON_H_ + +#include <linux/types.h> + +#include "../nbl_include/nbl_include.h" +#include "../nbl_include/nbl_def_common.h" + +#endif diff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_core.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_core.h index 1cd6587a8fcb66..655dfb43e3653d 100644 --- a/drivers/net/ethernet/nebula-matrix/nbl/nbl_core.h +++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_core.h @@ -14,13 +14,25 @@ enum { NBL_CAP_HAS_NET_BIT, }; +struct nbl_interface { + struct nbl_hw_ops_tbl hw_ops_tbl; + struct nbl_resource_ops_tbl resource_ops_tbl; + struct nbl_dispatch_ops_tbl dispatch_ops_tbl; + struct nbl_channel_ops_tbl channel_ops_tbl; +}; + struct nbl_core { struct nbl_hw_mgt hw_mgt; + struct nbl_resource_mgt res_mgt; + struct nbl_dispatch_mgt disp_mgt; + struct nbl_dev_mgt dev_mgt; + struct nbl_channel_mgt chan_mgt; }; struct nbl_adapter { struct pci_dev pdev; struct nbl_core core; + struct nbl_interface intf; struct nbl_common_info common; }; @@ -28,4 +40,12 @@ struct nbl_adapter nbl_core_init(struct pci_dev pdev, struct nbl_init_param param); void nbl_core_remove(struct nbl_adapter adapter); +/ + * Single-shot start/stop pair, called once each from PCI probe/remove. + * Not repeatable: MSI-X vectors stay allocated until device detach, so + * a second start on a bound device is rejected by the MSI-X core. + / +int nbl_core_start(struct nbl_adapter adapter); +void nbl_core_stop(struct nbl_adapter adapter); + #endif diff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_core/nbl_dev.c b/drivers/net/ethernet/nebula-matrix/nbl/nbl_core/nbl_dev.c new file mode 100644 index 00000000000000..c25def18a9e684 --- /dev/null +++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_core/nbl_dev.c @@ -0,0 +1,727 @@ +// SPDX-License-Identifier: GPL-2.0 +/ + * Copyright (c) 2026 Nebula Matrix Limited. + / +#include <linux/device.h> +#include <linux/pci.h> +#include "nbl_dev.h" + +static void nbl_dev_clean_mailbox_schedule(struct nbl_dev_mgt dev_mgt); + +/* ---------- Interrupt config ---------- / +static irqreturn_t nbl_dev_clean_mailbox(int irq __always_unused, void data) +{ + struct nbl_dev_mgt dev_mgt = (struct nbl_dev_mgt )data; + + nbl_dev_clean_mailbox_schedule(dev_mgt); + return IRQ_HANDLED; +} + +static void nbl_dev_init_msix_cnt(struct nbl_dev_mgt dev_mgt) +{ + struct nbl_dev_common dev_common = dev_mgt->common_dev; + struct nbl_msix_info msix_info = &dev_common->msix_info; + + / Reserve one MSI-X vector for the mailbox. Nothing here + * allocates or requests it: vectors are allocated and the handler + * attached when the device is started, after probe, from this + * count. + / + msix_info->serv_info[NBL_MSIX_MAILBOX_TYPE].num = 1; +} + +static int nbl_dev_request_mailbox_irq(struct nbl_dev_mgt dev_mgt) +{ + struct nbl_dev_common dev_common = dev_mgt->common_dev; + struct nbl_msix_info msix_info = &dev_common->msix_info; + struct nbl_common_info common = dev_mgt->common; + u16 lvec; + int irq_num; + int err; + + if (!msix_info->serv_info[NBL_MSIX_MAILBOX_TYPE].num) + return 0; + + lvec = msix_info->serv_info[NBL_MSIX_MAILBOX_TYPE].base_vector_id; + irq_num = pci_irq_vector(common->pdev, lvec); + if (irq_num < 0) { + dev_err(common->dev, "Failed to get mailbox IRQ vector: %d\n", + irq_num); + return irq_num; + } + + snprintf(dev_common->mailbox_name, sizeof(dev_common->mailbox_name), + "nbl_mailbox@pci:%s", pci_name(common->pdev)); + err = request_irq(irq_num, nbl_dev_clean_mailbox, 0, + dev_common->mailbox_name, dev_mgt); + if (err) + return err; + + return 0; +} + +static void nbl_dev_free_mailbox_irq(struct nbl_dev_mgt dev_mgt) +{ + struct nbl_dev_common dev_common = dev_mgt->common_dev; + struct nbl_msix_info msix_info = &dev_common->msix_info; + struct nbl_common_info common = dev_mgt->common; + u16 lvec; + int irq_num; + + if (!msix_info->serv_info[NBL_MSIX_MAILBOX_TYPE].num) + return; + + lvec = msix_info->serv_info[NBL_MSIX_MAILBOX_TYPE].base_vector_id; + irq_num = pci_irq_vector(common->pdev, lvec); + if (irq_num >= 0) + free_irq(irq_num, dev_mgt); +} + +static int nbl_dev_enable_mailbox_irq(struct nbl_dev_mgt dev_mgt) +{ + struct nbl_dispatch_ops disp_ops = dev_mgt->disp_ops_tbl->ops; + struct nbl_channel_ops chan_ops = dev_mgt->chan_ops_tbl->ops; + struct nbl_dev_common dev_common = dev_mgt->common_dev; + struct nbl_msix_info msix_info = &dev_common->msix_info; + u16 lvec; + int ret; + + if (!msix_info->serv_info[NBL_MSIX_MAILBOX_TYPE].num) + return 0; + + lvec = msix_info->serv_info[NBL_MSIX_MAILBOX_TYPE].base_vector_id; + /* + * Enable sequence: perform set_mailbox_irq RPC in polling mode first. + * Only set NBL_CHAN_IRQ_RDY after RPC succeeds, mirroring disable path. + * This avoids waiting for an interrupt which has not been armed yet. + / + ret = disp_ops->set_mailbox_irq(dev_mgt->disp_ops_tbl->priv, + lvec, true); + if (ret) + return ret; + chan_ops->set_queue_state(dev_mgt->chan_ops_tbl->priv, + NBL_CHAN_IRQ_RDY, + NBL_CHAN_TYPE_MAILBOX, true); + return 0; +} + +static int nbl_dev_disable_mailbox_irq(struct nbl_dev_mgt dev_mgt) +{ + struct nbl_dispatch_ops disp_ops = dev_mgt->disp_ops_tbl->ops; + struct nbl_channel_ops chan_ops = dev_mgt->chan_ops_tbl->ops; + struct nbl_dev_common dev_common = dev_mgt->common_dev; + struct nbl_msix_info msix_info = &dev_common->msix_info; + u16 lvec; + + if (!msix_info->serv_info[NBL_MSIX_MAILBOX_TYPE].num) + return 0; + + lvec = msix_info->serv_info[NBL_MSIX_MAILBOX_TYPE].base_vector_id; + /* + * Disable sequence invariant: clear the software READY flag first, + * then mask the hardware interrupt. Must not reverse the order. + * + * Clearing NBL_CHAN_IRQ_RDY is what makes an interrupt-mode sender + * fail fast: nbl_chan_set_queue_state() wakes every per-slot wait + * queue on the transition, and nbl_chan_send_msg() tests the flag + * ahead of its timeout branch, so the sender returns -EIO right + * away instead of waiting out NBL_CHAN_ACK_WAIT_TIME (3s) and + * reporting a bogus -ETIMEDOUT / "Channel waiting ack failed". + * Masking first would leave the sleeper blocked on an interrupt + * that is already masked, i.e. guaranteed to time out. + * + * This does NOT preserve in-flight ACKs, and must not be described + * as doing so: the woken sender gives up and resets its slot to + * IDLE, so an ACK arriving afterwards finds a non-WAITING slot and + * is dropped by the receive path ("Skip ack invalid status"). + * Interrupt mode is being torn down here; the guarantee is a + * bounded, deterministic failure, not delivery. In this driver + * every ACK-requesting sender runs from probe/remove and the clean + * task only sends ACK-less replies, so nothing depends on the + * stronger claim today. + * + * The only caller is nbl_dev_stop(): free_irq() blocks until + * any in-flight hardirq handler completes and prevents new interrupts. + * cancel_work_sync() then waits for any already running mailbox cleanup + * work to finish, or cancels queued but unstarted work items before + * final channel destruction. No stuck descriptors linger. + * + * nbl_dev_start()'s unwind path does not reach here: err_free_irq runs + * before IRQ_RDY is set, so only nbl_dev_free_mailbox_irq() applies + * there and no set_mailbox_irq(false) is sent - see the comment on + * that label. + / + chan_ops->set_queue_state(dev_mgt->chan_ops_tbl->priv, + NBL_CHAN_IRQ_RDY, + NBL_CHAN_TYPE_MAILBOX, false); + + return disp_ops->set_mailbox_irq(dev_mgt->disp_ops_tbl->priv, + lvec, false); +} + +static int nbl_dev_cfg_msix_map(struct nbl_dev_mgt dev_mgt) +{ + struct nbl_dispatch_ops disp_ops = dev_mgt->disp_ops_tbl->ops; + struct nbl_dev_common dev_common = dev_mgt->common_dev; + struct nbl_msix_info msix_info = &dev_common->msix_info; + bool mask_en = msix_info->serv_info[NBL_MSIX_NET_TYPE].hw_self_mask_en; + u16 msix_net_num = msix_info->serv_info[NBL_MSIX_NET_TYPE].num; + u16 msix_not_net_num = 0; + int err, i; + + msix_info->serv_info[NBL_MSIX_NET_TYPE].base_vector_id = 0; + / + * Calculate base_vector_id for each MSIX service type. + * This relies on NBL_MSIX_TYPE enum being ordered sequentially, + * starting from NBL_MSIX_NET_TYPE. + / + for (i = NBL_MSIX_NET_TYPE + 1; i < NBL_MSIX_TYPE_MAX; i++) + msix_info->serv_info[i].base_vector_id = + msix_info->serv_info[i - 1].base_vector_id + + msix_info->serv_info[i - 1].num; + + for (i = 0; i < NBL_MSIX_TYPE_MAX; i++) { + if (i == NBL_MSIX_NET_TYPE) + continue; + msix_not_net_num += msix_info->serv_info[i].num; + } + + err = disp_ops->cfg_msix_map(dev_mgt->disp_ops_tbl->priv, + msix_net_num, msix_not_net_num, + mask_en); + + return err; +} + +static int nbl_dev_destroy_msix_map(struct nbl_dev_mgt dev_mgt) +{ + struct nbl_dispatch_ops disp_ops = dev_mgt->disp_ops_tbl->ops; + + return disp_ops->destroy_msix_map(dev_mgt->disp_ops_tbl->priv); +} + +static int nbl_dev_init_interrupt_scheme(struct nbl_dev_mgt dev_mgt) +{ + struct nbl_dev_common dev_common = dev_mgt->common_dev; + struct nbl_msix_info msix_info = &dev_common->msix_info; + struct nbl_common_info common = dev_mgt->common; + struct irq_affinity affd = { 0 }; + int needed = 0; + int err; + int i; + + for (i = 0; i < NBL_MSIX_TYPE_MAX; i++) + needed += msix_info->serv_info[i].num; + + / + * The mailbox vector is the trailing administrative vector; + * reserve it via post_vectors so it is excluded from managed + * affinity spreading. A managed vector can be shut down when + * its last assigned CPU goes offline, which would stall every + * mailbox RPC while NBL_CHAN_IRQ_RDY stays set. + / + affd.post_vectors = msix_info->serv_info[NBL_MSIX_MAILBOX_TYPE].num; + + err = pci_alloc_irq_vectors_affinity(common->pdev, needed, needed, + PCI_IRQ_MSIX \| PCI_IRQ_AFFINITY, + &affd); + if (err < 0) { + dev_err(common->dev, + "pci_alloc_irq_vectors failed, err = %d\n", err); + return err; + } + if (err != needed) { + dev_err(common->dev, "pci_alloc_irq_vectors got %d vecs, need %d\n", + err, needed); + return -ENOSPC; + } + return 0; +} + +/ + * Release the kernel-side MSI-X vectors here, before the hardware + * pcompleter map is destroyed. pci_free_irq_vectors() ends in + * pci_msix_shutdown(), which masks every descriptor by writing the + * vector control word in the MSI-X table and then reads mask_base + * back to flush that write; those MMIO accesses should reach the + * function while FUNCTION_MSIX_MAP is still valid. This mirrors + * nbl_dev_start(), where cfg_msix_map runs before + * pci_alloc_irq_vectors_affinity(): the opposite order leaves the + * core poking the MSI-X table after destroy_msix_map() has cleared + * the valid bit, invalidated the info entries and freed the map + * table, and how the device reacts to that is hardware-specific. + * + * This is not a double free of the devres action. probe uses + * pcim_enable_device(), so pci_alloc_irq_vectors() also registers + * pcim_msi_release, but that only calls pci_free_irq_vectors() + * again, and pci_disable_msix() returns early once msix_enabled is + * clear. + * + * The handler is expected to be gone by now: both callers run + * nbl_dev_free_mailbox_irq() first. + / +static void nbl_dev_clear_interrupt_scheme(struct nbl_dev_mgt dev_mgt) +{ + struct nbl_common_info common = dev_mgt->common; + + pci_free_irq_vectors(common->pdev); +} + +/ ---------- Channel config ---------- / +static void nbl_dev_setup_chan_qinfo(struct nbl_dev_mgt dev_mgt, u8 chan_type) +{ + struct nbl_channel_ops chan_ops = dev_mgt->chan_ops_tbl->ops; + struct nbl_channel_mgt priv = dev_mgt->chan_ops_tbl->priv; + struct nbl_common_info common = dev_mgt->common; + + if (!chan_ops->check_queue_exist(priv, chan_type)) + return; + + / + * common->hw_bus is the control PF's real bus number, captured in + * nbl_res_ctrl_dev_sriov_info_init() during nbl_res_init_leonis(). + * nbl_core_init() runs resource init before nbl_dev_init(), so the + * value is always initialized when this control-PF-only path runs; + * nbl_res_intr_cfg_msix_map() consumes it for cfg_msix_map() the + * same way. + / + chan_ops->cfg_chan_qinfo_map_table(priv, common->hw_bus, common->devid); +} + +static int nbl_dev_setup_chan_queue(struct nbl_dev_mgt dev_mgt, u8 chan_type) +{ + struct nbl_channel_ops chan_ops = dev_mgt->chan_ops_tbl->ops; + struct nbl_channel_mgt priv = dev_mgt->chan_ops_tbl->priv; + int ret = 0; + + if (chan_ops->check_queue_exist(priv, chan_type)) + ret = chan_ops->setup_queue(priv, chan_type); + + return ret; +} + +static int nbl_dev_remove_chan_queue(struct nbl_dev_mgt dev_mgt, u8 chan_type) +{ + struct nbl_channel_ops chan_ops = dev_mgt->chan_ops_tbl->ops; + struct nbl_channel_mgt priv = dev_mgt->chan_ops_tbl->priv; + int ret = 0; + + if (chan_ops->check_queue_exist(priv, chan_type)) + ret = chan_ops->teardown_queue(priv, chan_type); + + return ret; +} + +static void nbl_dev_register_chan_task(struct nbl_dev_mgt dev_mgt, + u8 chan_type, struct work_struct task) +{ + struct nbl_channel_ops chan_ops = dev_mgt->chan_ops_tbl->ops; + + if (chan_ops->check_queue_exist(dev_mgt->chan_ops_tbl->priv, chan_type)) + chan_ops->register_chan_task(dev_mgt->chan_ops_tbl->priv, + chan_type, task); +} + +/* ---------- Tasks config ---------- / +static void nbl_dev_clean_mailbox_task(struct work_struct work) +{ + struct nbl_dev_common common_dev = + container_of(work, struct nbl_dev_common, clean_mbx_task); + struct nbl_dev_mgt dev_mgt = common_dev->dev_mgt; + struct nbl_channel_ops chan_ops = dev_mgt->chan_ops_tbl->ops; + + chan_ops->clean_queue_subtask(dev_mgt->chan_ops_tbl->priv, + NBL_CHAN_TYPE_MAILBOX); +} + +static void nbl_dev_clean_mailbox_schedule(struct nbl_dev_mgt dev_mgt) +{ + struct nbl_dev_common common_dev = dev_mgt->common_dev; + struct nbl_common_info common = dev_mgt->common; + + queue_work(common->wq, &common_dev->clean_mbx_task); +} + +/* ---------- Dev init process ---------- / +static int nbl_dev_setup_common_dev(struct nbl_adapter adapter) +{ + struct nbl_dev_mgt dev_mgt = adapter->core.dev_mgt; + struct nbl_dev_common common_dev; + int ret; + + common_dev = devm_kzalloc(&adapter->pdev->dev, sizeof(common_dev), + GFP_KERNEL); + if (!common_dev) + return -ENOMEM; + common_dev->dev_mgt = dev_mgt; + + / + * INIT_WORK must precede nbl_dev_register_chan_task() below: + * registration publishes the work item to the channel, and the + * channel may queue it from its send and receive paths as soon + * as setup_queue() completes. + / + INIT_WORK(&common_dev->clean_mbx_task, nbl_dev_clean_mailbox_task); + + ret = nbl_dev_setup_chan_queue(dev_mgt, NBL_CHAN_TYPE_MAILBOX); + if (ret) { + / + * Nothing to unwind: chan_info->active is only set on a + * successful setup_queue(), so a failed one left no + * sender, no registered work item and no mailbox + * traffic. Calling teardown_queue() here would only + * take its "channel not active" early exit and emit a + * misleading duplicate-teardown warning. The partially + * allocated DMA rings are intentionally kept until devres + * releases them at detach (chan_info->dma_allocated + * blocks a re-setup). + / + return ret; + } + + nbl_dev_register_chan_task(dev_mgt, NBL_CHAN_TYPE_MAILBOX, + &common_dev->clean_mbx_task); + / + * VSI/ETH identity is deliberately not fetched here: it is only + * known to the control PF, so it has to come from a mailbox RPC, + * and the control PF's responder may still be probing while this + * function runs. The fetch is therefore left to the start path. + / + dev_mgt->common_dev = common_dev; + nbl_dev_init_msix_cnt(dev_mgt); + + return 0; +} + +static void nbl_dev_remove_common_dev(struct nbl_adapter adapter) +{ + struct nbl_dev_mgt dev_mgt = adapter->core.dev_mgt; + struct nbl_dev_common common_dev = dev_mgt->common_dev; + + if (!common_dev) + return; + cancel_work_sync(&common_dev->clean_mbx_task); + /* + * nbl_chan_teardown_queue() never fails: it returns 0 both on the + * early exit for a channel that never became active and on the + * normal path, which drains every inflight sender before + * returning. There is therefore no partial-failure state that + * could leave mailbox DMA in flight and nothing to report here. + * + * Unregistering the clean task is the channel layer's job too: + * teardown_queue() clears clean_task under state_lock and then + * cancel_work_sync()es it, after shutdn has blocked new queues. + * No explicit unregistration is needed here precisely because + * teardown_queue() does that clearing itself. Passing NULL from + * here would not be a no-op either: shutdn is only set inside + * teardown_queue(), so nbl_chan_register_chan_task() would still + * accept the write - it would just be clearing the same pointer a + * second time. + / + nbl_dev_remove_chan_queue(dev_mgt, NBL_CHAN_TYPE_MAILBOX); +} + +static int nbl_dev_setup_ctrl_dev(struct nbl_adapter adapter) +{ + struct nbl_dev_mgt dev_mgt = adapter->core.dev_mgt; + struct nbl_dispatch_ops disp_ops = dev_mgt->disp_ops_tbl->ops; + + return disp_ops->init_module(dev_mgt->disp_ops_tbl->priv); +} + +/* + * Tear down control device: deinit_module sets driver_status=false + * to notify firmware to clean all per-PF hardware state (including + * qinfo registers). The qinfo map programmed in nbl_dev_init() via + * nbl_dev_setup_chan_qinfo() is not explicitly cleared; firmware + * handles it on driver_status change. + * + * Teardown ordering guarantee: every non-management PF creates a + * consumer->control PF device link in its probe path, so the driver + * core always unbinds all siblings before allowing the control PF to + * be detached (sysfs unbind, driver unregister and hot-unplug alike). + * nbl_probe_chip_deps() defers a sibling's probe until the control PF + * is fully bound (DL_DEV_DRIVER_BOUND), and holds the management PF's + * device lock across both that check and device_link_add(), so a + * supplier cannot slip to DL_DEV_NO_DRIVER or DL_DEV_PROBING in + * between. A sibling therefore cannot + * reach its mailbox setup before the control PF is fully probed, nor + * race this deinit afterwards - the link forces it to be unbound + * first. The link is a persistent managed link with + * DL_FLAG_AUTOPROBE_CONSUMER, so binding the control PF again makes + * the core re-probe the siblings; without it they would stay unbound + * until an admin rebound each one by hand. + * + * Direct control-PF FLR (which never runs driver teardown) cannot be + * guarded here. + / +static void nbl_dev_remove_ctrl_dev(struct nbl_adapter adapter) +{ + struct nbl_dev_mgt dev_mgt = adapter->core.dev_mgt; + struct nbl_dispatch_ops disp_ops = dev_mgt->disp_ops_tbl->ops; + + disp_ops->deinit_module(dev_mgt->disp_ops_tbl->priv); +} + +static struct nbl_dev_mgt nbl_dev_setup_dev_mgt(struct nbl_common_info common) +{ + struct nbl_dev_mgt dev_mgt; + + dev_mgt = devm_kzalloc(common->dev, sizeof(dev_mgt), GFP_KERNEL); + if (!dev_mgt) + return ERR_PTR(-ENOMEM); + + dev_mgt->common = common; + return dev_mgt; +} + +int nbl_dev_init(struct nbl_adapter adapter) +{ + struct nbl_common_info common = &adapter->common; + struct nbl_dispatch_ops_tbl disp_ops_tbl = + adapter->intf.dispatch_ops_tbl; + struct nbl_channel_ops_tbl chan_ops_tbl = + adapter->intf.channel_ops_tbl; + struct nbl_dev_mgt dev_mgt; + int ret; + + dev_mgt = nbl_dev_setup_dev_mgt(common); + if (IS_ERR(dev_mgt)) { + ret = PTR_ERR(dev_mgt); + return ret; + } + + dev_mgt->disp_ops_tbl = disp_ops_tbl; + dev_mgt->chan_ops_tbl = chan_ops_tbl; + adapter->core.dev_mgt = dev_mgt; + if (common->has_ctrl) + nbl_dev_setup_chan_qinfo(dev_mgt, NBL_CHAN_TYPE_MAILBOX); + / + * Chip hardware initialization is completed by firmware at power-up. + * Only driver functional table/register config follows here, safe to + * access hardware registers before ctrl dev setup. + * + * The QINFO routing has to be programmed before the mailbox queue is + * enabled: nbl_dev_setup_common_dev() sets QUEUE_EN, and a queue that + * is live before its routing is in place would drop or misroute + * traffic. + / + ret = nbl_dev_setup_common_dev(adapter); + if (ret) + return ret; + + if (common->has_ctrl) { + ret = nbl_dev_setup_ctrl_dev(adapter); + if (ret) + goto out_remove_common_dev; + } + + return 0; + +/ + * A failure after the QINFO routing was programmed leaves those entries + * in hardware: driver_status never goes 1 -> 0 on these paths, so + * firmware does not reclaim them. This is harmless - firmware only + * routes once driver_status announces the driver, and no sibling PF can + * bind until this probe succeeds. The next successful probe overwrites + * the entries, and a chip reset clears them. + / +out_remove_common_dev: + nbl_dev_remove_common_dev(adapter); + return ret; +} + +/ + * Teardown order: Stop mailbox channel and drain all inflight DMA first, + * then invoke deinit_module to notify firmware. + * + * This intentionally breaks strict init/teardown mirror symmetry due to + * a hardware constraint: firmware performs asynchronous per-PF cleanup + * (mailbox QINFO routing, qinfo registers and the like) once + * driver_status=false is set. It does not revert the chip-wide + * datapath registers configured by init_module - those stay valid until + * chip reset (see nbl_hw_deinit_module()). We must guarantee no ongoing + * mailbox DMA before deinit_module so firmware cannot reclaim the + * routing under a live transfer and leave the device DMAing into + * released memory. + * + * Init order: program mailbox QINFO routing (ctrl PF only) -> + * create mailbox queue/common_dev -> ctrl dev init_module + * Teardown order: destroy mailbox queue/common_dev -> ctrl dev deinit_module + / +void nbl_dev_remove(struct nbl_adapter adapter) +{ + struct nbl_common_info common = &adapter->common; + + nbl_dev_remove_common_dev(adapter); + if (common->has_ctrl) + nbl_dev_remove_ctrl_dev(adapter); +} + +/ ---------- Dev start process ---------- / +int nbl_dev_start(struct nbl_adapter adapter) +{ + struct nbl_dev_mgt dev_mgt = adapter->core.dev_mgt; + struct nbl_dispatch_ops disp_ops = dev_mgt->disp_ops_tbl->ops; + struct nbl_dispatch_mgt priv = dev_mgt->disp_ops_tbl->priv; + struct nbl_dev_common common_dev = dev_mgt->common_dev; + struct nbl_common_info common = dev_mgt->common; + int cleanup_ret; + int ret; + + / + * Startup RPC failures are reported as they are, including the + * -ETIMEDOUT of a 5-6s polling wait against an unresponsive peer. + * nbl_probe() already deferred this PF in nbl_probe_chip_deps() + * until the control PF reached DL_DEV_DRIVER_BOUND, and the core + * only marks a device bound once its whole probe - including the + * control PF's own nbl_core_start() - has returned. That state, + * not nbl_dev_init(), is what makes a timeout here a real failure + * instead of a "supplier not ready yet" condition: on the control + * PF inbound mailbox work only runs from clean_mbx_task, queued + * by the mailbox hardirq, by the task's own self-requeue, or by + * this function's polling sends - and the control PF uses the + * local dispatch ops, so it sends none. The responder is armed + * by set_mailbox_irq(true) at the very end of the peer's + * nbl_dev_start(); relaxing the dependency to "func 0 finished + * nbl_dev_init()" would let every sibling RPC sit out the full + * 5-6s polling wait. Turning -ETIMEDOUT into -EPROBE_DEFER would + * hide it behind dev_dbg() and leave the retry to the next + * unrelated device bind, paying that wait again on every retry. + / + ret = nbl_dev_cfg_msix_map(dev_mgt); + if (ret) + goto err_destroy_map; + + / Fetch VSI/ETH identity after cfg_msix_map / + ret = disp_ops->get_vsi_id(priv, NBL_VSI_DATA, &common->vsi_id); + if (ret) + goto err_destroy_map; + ret = disp_ops->get_eth_id(priv, common->vsi_id, &common->eth_num, + &common->eth_id, &common->logic_eth_id); + if (ret) + goto err_destroy_map; + + ret = nbl_dev_init_interrupt_scheme(dev_mgt); + if (ret) + goto err_destroy_map; + + ret = nbl_dev_request_mailbox_irq(dev_mgt); + if (ret) + goto err_destroy_map; + + ret = nbl_dev_enable_mailbox_irq(dev_mgt); + if (ret) + goto err_free_irq; + + return 0; + +err_free_irq: + / + * nbl_dev_enable_mailbox_irq() only fails before it sets + * NBL_CHAN_IRQ_RDY - the flag is set after its RPC returns - so + * there is no interrupt-mode sender to release and no software + * state to revoke. The peer-side mailbox routing, if the failed + * RPC did land, is switched off by the destroy_msix_map() below: its + * prepare phase, __nbl_res_intr_prepare_destroy_msix_map(), disables + * mailbox IRQ routing for any CONFIGURED function before the map is + * torn down. That is what makes skipping set_mailbox_irq(false) + * correct here, whatever way the enable failed - a local + * -ESHUTDOWN/-ENODEV/-EINVAL on the control PF as much as a + * polling-mode -ETIMEDOUT against an unresponsive peer. + / + nbl_dev_free_mailbox_irq(dev_mgt); +err_destroy_map: + / + * Always ask the peer to drop this function's MSI-X map, even when + * the cfg RPC above failed: the teardown prepare phase returns 0 + * for a function that is not NBL_INTR_FUNC_CONFIGURED, so it is a + * no-op when nothing landed, but a -ETIMEDOUT only means the ACK + * never arrived - the responder runs to completion before it ACKs. + * In that case the map entry, the intr_net/other_bmap bits, the + * kcalloc()ed vector array and the coherent map table are all + * still owned by the control PF, i.e. host resources of its + * driver, released only when this function is configured again or + * when nbl_intr_mgt_stop() tears the control PF down. They are not + * firmware state, so skipping the destroy would strand them for a + * PF that has no driver to ever ask for them again. + * + * Kernel vectors are released first, by the + * nbl_dev_clear_interrupt_scheme() below: the PCI core's MSI-X + * shutdown writes the MSI-X table and reads it back, and those + * accesses should land while this function's pcompleter map entry + * is still valid. The destroy then masks all hardware vectors + * and clears that entry. + * + * For non-control PFs this is a polling-mode mailbox RPC: + * NBL_CHAN_IRQ_RDY is never set on any rollback path, since + * enabling it is the last step of nbl_dev_start(). + * + * Note: pci_clear_master() runs in nbl_probe() core_start_err path; + * on a non-control PF it cannot stop DMA using the management PF's + * BDF, this is a known limitation. + / + nbl_dev_clear_interrupt_scheme(dev_mgt); + cleanup_ret = nbl_dev_destroy_msix_map(dev_mgt); + if (cleanup_ret) + dev_err(dev_mgt->common->dev, + "rollback: destroy MSI-X map failed: %d\n", + cleanup_ret); + + cancel_work_sync(&common_dev->clean_mbx_task); + return ret; +} + +void nbl_dev_stop(struct nbl_adapter adapter) +{ + struct nbl_dev_mgt dev_mgt = adapter->core.dev_mgt; + struct nbl_dev_common common_dev = dev_mgt->common_dev; + int ret; + + ret = nbl_dev_disable_mailbox_irq(dev_mgt); + if (ret) + dev_err(dev_mgt->common->dev, + "Failed to disable mailbox IRQ: %d\n", ret); + nbl_dev_free_mailbox_irq(dev_mgt); + + /* + * Release the kernel vectors before destroying the hardware map: + * the PCI core's MSI-X shutdown writes the MSI-X table and reads + * it back, and those accesses should land while this function's + * pcompleter map entry is still valid (see + * nbl_dev_clear_interrupt_scheme). + / + nbl_dev_clear_interrupt_scheme(dev_mgt); + + / + * Destroy the hardware MSI-X map: masks all device vectors and + * clears the pcompleter map entry. + * + * Best-effort: if the destroy RPC fails, the control PF still owns + * this function's map entry, its intr_net/other_bmap bits and its + * coherent map table. Those are host resources of that driver, + * released either by a later successful configuration of this + * function - nbl_res_intr_cfg_msix_map() invalidates and rewrites + * the entry on its had_config path - or by nbl_intr_mgt_stop() + * when the control PF is removed, which walks every + * NBL_INTR_FUNC_CONFIGURED function through + * __nbl_res_intr_prepare_destroy_msix_map() and clears the + * FUNCTION_MSIX_MAP valid bit. Both rewrite the hardware entry + * without a chip reset; nothing else collects it, so a stranded + * one just stays as it is, and nothing requests one. + * + * pci_clear_master() on a non-control PF cannot stop DMA using the + * management PF's BDF. + / + ret = nbl_dev_destroy_msix_map(dev_mgt); + if (ret) + dev_err(dev_mgt->common->dev, + "Failed to destroy MSI-X map: %d\n", ret); + + / + * destroy_msix_map() sends ack-requested messages which may + * requeue clean_mbx_task via polling send path. Drain work + * after the operation. + / + cancel_work_sync(&common_dev->clean_mbx_task); +} diff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_core/nbl_dev.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_core/nbl_dev.h new file mode 100644 index 00000000000000..24e890fd898745 --- /dev/null +++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_core/nbl_dev.h @@ -0,0 +1,55 @@ +/ SPDX-License-Identifier: GPL-2.0 / +/ + * Copyright (c) 2026 Nebula Matrix Limited. + / + +#ifndef _NBL_DEV_H_ +#define _NBL_DEV_H_ + +#include <linux/types.h> + +#include "../nbl_include/nbl_include.h" +#include "../nbl_include/nbl_def_channel.h" +#include "../nbl_include/nbl_def_hw.h" +#include "../nbl_include/nbl_def_resource.h" +#include "../nbl_include/nbl_def_dispatch.h" +#include "../nbl_include/nbl_def_dev.h" +#include "../nbl_include/nbl_def_common.h" +#include "../nbl_core.h" + +#define NBL_STRING_NAME_LEN 32 + +enum nbl_msix_serv_type { + NBL_MSIX_NET_TYPE, + NBL_MSIX_MAILBOX_TYPE, + NBL_MSIX_TYPE_MAX +}; + +struct nbl_msix_serv_info { + char irq_name[NBL_STRING_NAME_LEN]; + u16 num; + u16 base_vector_id; + / true: hw report msix, hw need to mask actively / + bool hw_self_mask_en; +}; + +struct nbl_msix_info { + struct nbl_msix_serv_info serv_info[NBL_MSIX_TYPE_MAX]; +}; + +struct nbl_dev_common { + struct nbl_dev_mgt dev_mgt; + struct nbl_msix_info msix_info; + char mailbox_name[NBL_STRING_NAME_LEN]; + /* for ctrl-dev/net-dev mailbox recv msg / + struct work_struct clean_mbx_task; +}; + +struct nbl_dev_mgt { + struct nbl_common_info common; + struct nbl_dispatch_ops_tbl disp_ops_tbl; + struct nbl_channel_ops_tbl chan_ops_tbl; + struct nbl_dev_common common_dev; +}; + +#endif diff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_core/nbl_dispatch.c b/drivers/net/ethernet/nebula-matrix/nbl/nbl_core/nbl_dispatch.c new file mode 100644 index 00000000000000..59d80066f88b76 --- /dev/null +++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_core/nbl_dispatch.c @@ -0,0 +1,671 @@ +// SPDX-License-Identifier: GPL-2.0 +/ + * Copyright (c) 2026 Nebula Matrix Limited. + / +#include <linux/device.h> +#include <linux/pci.h> +#include "nbl_dispatch.h" + +static int nbl_disp_chan_get_vsi_id_req(struct nbl_dispatch_mgt disp_mgt, + u16 type, u16 vsi_id) +{ + struct nbl_channel_ops chan_ops = disp_mgt->chan_ops_tbl->ops; + struct nbl_common_info common = disp_mgt->common; + struct nbl_chan_param_get_vsi_id result = { 0 }; + struct nbl_chan_param_get_vsi_id param = { 0 }; + struct nbl_chan_send_info chan_send = {0}; + int ret; + + param.type = cpu_to_le16(type); + + nbl_chan_fill_send_info(&chan_send, common->mgt_pf, + NBL_CHAN_MSG_GET_VSI_ID, + &param, sizeof(param), &result, + sizeof(result), 1); + ret = chan_ops->send_msg(disp_mgt->chan_ops_tbl->priv, &chan_send); + if (ret) + return ret; + if (chan_send.ack_len != sizeof(result)) { + dev_err(disp_mgt->common->dev, + "get_vsi_id: short ACK, ack_len=%u expected %zu\n", + chan_send.ack_len, sizeof(result)); + return -EBADMSG; + } + vsi_id = le16_to_cpu(result.vsi_id); + return 0; +} + +static void nbl_disp_chan_get_vsi_id_resp(void priv, u16 src_id, u16 msg_id, + void data, u32 data_len) +{ + struct nbl_dispatch_mgt disp_mgt = (struct nbl_dispatch_mgt )priv; + struct nbl_channel_ops chan_ops = disp_mgt->chan_ops_tbl->ops; + struct nbl_resource_ops res_ops = disp_mgt->res_ops_tbl->ops; + struct nbl_resource_mgt p = disp_mgt->res_ops_tbl->priv; + struct device dev = disp_mgt->common->dev; + struct nbl_chan_param_get_vsi_id result = { 0 }; + struct nbl_chan_param_get_vsi_id param = { 0 }; + struct nbl_chan_ack_info chan_ack; + int err = 0; + u16 vsi_id = 0; + u32 rel_pf_id; + int ret; + + ret = nbl_common_func_id_to_rel_pf_id(disp_mgt->common, src_id, + &rel_pf_id); + if (ret) { + err = -EPERM; + goto ack_out; + } + if (rel_pf_id >= disp_mgt->common->max_pf) { + err = -EPERM; + goto ack_out; + } + if (data_len < sizeof(param)) { + err = -EBADMSG; + goto ack_out; + } + memcpy(&param, data, sizeof(param)); + + if (res_ops->get_vsi_id) { + ret = res_ops->get_vsi_id(p, src_id, le16_to_cpu(param.type), + &vsi_id); + /* Forward the resource errno verbatim to the requester / + if (ret) + err = ret; + } else { + err = -EOPNOTSUPP; + } + + result.vsi_id = cpu_to_le16(vsi_id); +ack_out: + nbl_chan_fill_ack_info(&chan_ack, src_id, + NBL_CHAN_MSG_GET_VSI_ID, msg_id, err, + &result, sizeof(result)); + ret = chan_ops->send_ack(disp_mgt->chan_ops_tbl->priv, &chan_ack); + if (ret) + dev_err(dev, + "channel send ack failed with ret: %d, msg_type: %d\n", + ret, NBL_CHAN_MSG_GET_VSI_ID); +} + +static int nbl_disp_chan_get_eth_id_req(struct nbl_dispatch_mgt disp_mgt, + u16 vsi_id, u8 eth_num, u8 eth_id, + u8 logic_eth_id) +{ + struct nbl_channel_ops chan_ops = disp_mgt->chan_ops_tbl->ops; + struct nbl_common_info common = disp_mgt->common; + struct nbl_chan_param_get_eth_id result = { 0 }; + struct nbl_chan_param_get_eth_id param = { 0 }; + struct nbl_chan_send_info chan_send = {0}; + int ret; + + param.vsi_id = cpu_to_le16(vsi_id); + + nbl_chan_fill_send_info(&chan_send, common->mgt_pf, + NBL_CHAN_MSG_GET_ETH_ID, + &param, sizeof(param), &result, + sizeof(result), 1); + ret = chan_ops->send_msg(disp_mgt->chan_ops_tbl->priv, &chan_send); + if (ret) + return ret; + if (chan_send.ack_len != sizeof(result)) { + dev_err(disp_mgt->common->dev, + "get_eth_id: short ACK, ack_len=%u expected %zu\n", + chan_send.ack_len, sizeof(result)); + return -EBADMSG; + } + eth_num = result.eth_num; + eth_id = result.eth_id; + logic_eth_id = result.logic_eth_id; + + return 0; +} + +static void nbl_disp_chan_get_eth_id_resp(void priv, u16 src_id, u16 msg_id, + void data, u32 data_len) +{ + struct nbl_dispatch_mgt disp_mgt = (struct nbl_dispatch_mgt )priv; + struct nbl_channel_ops chan_ops = disp_mgt->chan_ops_tbl->ops; + struct nbl_resource_ops res_ops = disp_mgt->res_ops_tbl->ops; + struct nbl_resource_mgt p = disp_mgt->res_ops_tbl->priv; + struct nbl_chan_param_get_eth_id result = { 0 }; + struct nbl_chan_param_get_eth_id param = { 0 }; + struct device dev = disp_mgt->common->dev; + struct nbl_chan_ack_info chan_ack; + int err = 0; + u32 rel_pf_id; + int ret; + + ret = nbl_common_func_id_to_rel_pf_id(disp_mgt->common, src_id, + &rel_pf_id); + if (ret) { + err = -EPERM; + goto ack_out; + } + if (rel_pf_id >= disp_mgt->common->max_pf) { + err = -EPERM; + goto ack_out; + } + if (data_len < sizeof(param)) { + err = -EBADMSG; + goto ack_out; + } + memcpy(&param, data, sizeof(param)); + + if (res_ops->get_eth_id) { + ret = res_ops->get_eth_id(p, src_id, le16_to_cpu(param.vsi_id), + &result.eth_num, &result.eth_id, + &result.logic_eth_id); + /* Forward the resource errno verbatim to the requester / + if (ret) + err = ret; + } else { + err = -EOPNOTSUPP; + } +ack_out: + nbl_chan_fill_ack_info(&chan_ack, src_id, + NBL_CHAN_MSG_GET_ETH_ID, msg_id, err, + &result, sizeof(result)); + ret = chan_ops->send_ack(disp_mgt->chan_ops_tbl->priv, &chan_ack); + if (ret) + dev_err(dev, + "channel send ack failed with ret: %d, msg_type: %d\n", + ret, NBL_CHAN_MSG_GET_ETH_ID); +} + +static void nbl_disp_deinit_module(struct nbl_dispatch_mgt disp_mgt) +{ + struct nbl_resource_ops res_ops = disp_mgt->res_ops_tbl->ops; + struct nbl_resource_mgt p = disp_mgt->res_ops_tbl->priv; + + if (res_ops->deinit_module) + res_ops->deinit_module(p); +} + +static int nbl_disp_init_module(struct nbl_dispatch_mgt disp_mgt) +{ + struct nbl_resource_ops res_ops = disp_mgt->res_ops_tbl->ops; + struct nbl_resource_mgt p = disp_mgt->res_ops_tbl->priv; + + if (res_ops->init_module) + return res_ops->init_module(p); + return -EOPNOTSUPP; +} + +static int nbl_disp_cfg_msix_map(struct nbl_dispatch_mgt disp_mgt, + u16 num_net_msix, u16 num_others_msix, + bool net_msix_mask_en) +{ + struct nbl_resource_ops res_ops = disp_mgt->res_ops_tbl->ops; + struct nbl_resource_mgt p = disp_mgt->res_ops_tbl->priv; + struct nbl_common_info common = disp_mgt->common; + + if (!res_ops->cfg_msix_map) + return -EOPNOTSUPP; + / + * No core-layer lock is taken here: the resource layer serializes + * concurrent callers itself (intr_mgt->lock), and the dispatch + * layer has no mutable state of its own for this op to protect. + / + return res_ops->cfg_msix_map(p, common->mgt_pf, num_net_msix, + num_others_msix, net_msix_mask_en); +} + +/ + * Precondition: caller must disable mailbox IRQ_RDY and switch send_msg + * to the polling path before issuing this RPC. The responder rewrites + * the requester's mailbox MSI-X routing during the resource op, so the + * ACK cannot rely on interrupt wakeup while routing is in flux. + / +static int +nbl_disp_chan_cfg_msix_map_req(struct nbl_dispatch_mgt disp_mgt, + u16 num_net_msix, u16 num_others_msix, + bool net_msix_mask_en) +{ + struct nbl_channel_ops chan_ops = disp_mgt->chan_ops_tbl->ops; + struct nbl_common_info common = disp_mgt->common; + struct nbl_chan_param_cfg_msix_map param = { 0 }; + struct nbl_chan_send_info chan_send = {0}; + int ret; + + param.num_net_msix = cpu_to_le16(num_net_msix); + param.num_others_msix = cpu_to_le16(num_others_msix); + param.msix_mask_en = cpu_to_le16(!!net_msix_mask_en); + + nbl_chan_fill_send_info(&chan_send, common->mgt_pf, + NBL_CHAN_MSG_CONFIGURE_MSIX_MAP, + &param, sizeof(param), + NULL, 0, 1); + ret = chan_ops->send_msg(disp_mgt->chan_ops_tbl->priv, &chan_send); + if (ret) + return ret; + return 0; +} + +static void nbl_disp_chan_cfg_msix_map_resp(void priv, u16 src_id, u16 msg_id, + void data, u32 data_len) +{ + struct nbl_dispatch_mgt disp_mgt = (struct nbl_dispatch_mgt )priv; + struct nbl_channel_ops chan_ops = disp_mgt->chan_ops_tbl->ops; + struct nbl_resource_ops res_ops = disp_mgt->res_ops_tbl->ops; + struct nbl_resource_mgt p = disp_mgt->res_ops_tbl->priv; + struct device dev = disp_mgt->common->dev; + struct nbl_chan_param_cfg_msix_map param = { 0 }; + struct nbl_chan_ack_info chan_ack; + int err = 0; + u32 rel_pf_id; + int ret; + + ret = nbl_common_func_id_to_rel_pf_id(disp_mgt->common, src_id, + &rel_pf_id); + if (ret) { + err = -EPERM; + goto ack_out; + } + if (rel_pf_id >= disp_mgt->common->max_pf) { + err = -EPERM; + goto ack_out; + } + if (data_len < sizeof(param)) { + err = -EBADMSG; + goto ack_out; + } + memcpy(&param, data, sizeof(param)); + + if (res_ops->cfg_msix_map) { + /* + * The vector counts come from the peer PF. A peer is + * trusted: hardware and firmware forbid PF passthrough to + * guests (see nbl_get_func_param()), so every peer runs this + * same host driver, and the resource layer bounds the counts + * by the global vector pool. A peer asking for the whole + * pool can starve the others with -EAGAIN, but it cannot + * corrupt state. + / + ret = res_ops->cfg_msix_map(p, src_id, + le16_to_cpu(param.num_net_msix), + le16_to_cpu(param.num_others_msix), + !!le16_to_cpu(param.msix_mask_en)); + / Forward the resource errno verbatim to the requester / + if (ret) + err = ret; + } else { + err = -EOPNOTSUPP; + } +ack_out: + nbl_chan_fill_ack_info(&chan_ack, src_id, + NBL_CHAN_MSG_CONFIGURE_MSIX_MAP, msg_id, + err, NULL, 0); + ret = chan_ops->send_ack(disp_mgt->chan_ops_tbl->priv, &chan_ack); + if (ret) + dev_err(dev, + "channel send ack failed with ret: %d, msg_type: %d\n", + ret, NBL_CHAN_MSG_CONFIGURE_MSIX_MAP); +} + +/ + * Precondition: caller must disable mailbox IRQ_RDY and switch send_msg + * to the polling path before issuing this RPC. The responder retargets + * the requester's mailbox MSI-X routing during the resource op, so the + * ACK cannot rely on interrupt wakeup while routing is in flux. + / +static int nbl_disp_chan_destroy_msix_map_req(struct nbl_dispatch_mgt disp_mgt) +{ + struct nbl_channel_ops chan_ops = disp_mgt->chan_ops_tbl->ops; + struct nbl_common_info common = disp_mgt->common; + struct nbl_chan_send_info chan_send = {0}; + int ret; + + nbl_chan_fill_send_info(&chan_send, common->mgt_pf, + NBL_CHAN_MSG_DESTROY_MSIX_MAP, + NULL, 0, NULL, 0, 1); + ret = chan_ops->send_msg(disp_mgt->chan_ops_tbl->priv, &chan_send); + if (ret) + return ret; + return 0; +} + +static void nbl_disp_chan_destroy_msix_map_resp(void priv, u16 src_id, + u16 msg_id, void data, + u32 data_len) +{ + struct nbl_dispatch_mgt disp_mgt = (struct nbl_dispatch_mgt )priv; + struct nbl_channel_ops chan_ops = disp_mgt->chan_ops_tbl->ops; + struct nbl_resource_ops res_ops = disp_mgt->res_ops_tbl->ops; + struct nbl_resource_mgt p = disp_mgt->res_ops_tbl->priv; + struct device dev = disp_mgt->common->dev; + struct nbl_chan_ack_info chan_ack; + int err = 0; + u32 rel_pf_id; + int ret; + + ret = nbl_common_func_id_to_rel_pf_id(disp_mgt->common, src_id, + &rel_pf_id); + if (ret) { + err = -EPERM; + goto ack_out; + } + if (rel_pf_id >= disp_mgt->common->max_pf) { + err = -EPERM; + goto ack_out; + } + if (res_ops->destroy_msix_map) { + ret = res_ops->destroy_msix_map(p, src_id); + /* Forward the resource errno verbatim to the requester / + if (ret) + err = ret; + } else { + err = -EOPNOTSUPP; + } +ack_out: + nbl_chan_fill_ack_info(&chan_ack, src_id, + NBL_CHAN_MSG_DESTROY_MSIX_MAP, msg_id, + err, NULL, 0); + ret = chan_ops->send_ack(disp_mgt->chan_ops_tbl->priv, &chan_ack); + if (ret) + dev_err(dev, + "channel send ack failed with ret: %d, msg_type: %d\n", + ret, NBL_CHAN_MSG_DESTROY_MSIX_MAP); +} + +/ + * Precondition: caller must disable mailbox IRQ_RDY and switch send_msg + * to the polling path before issuing this RPC. The responder rewrites + * the requester's own mailbox MSI-X routing (MSIX_IDX / MSIX_IDX_VALID) + * before the ACK is sent, so the ACK cannot rely on interrupt wakeup + * while routing is in flux. + / +static int nbl_disp_chan_set_mailbox_irq_req(struct nbl_dispatch_mgt disp_mgt, + u16 vector_id, bool en_msix) +{ + struct nbl_channel_ops chan_ops = disp_mgt->chan_ops_tbl->ops; + struct nbl_chan_param_set_mailbox_irq param = { 0 }; + struct nbl_common_info common = disp_mgt->common; + struct nbl_chan_send_info chan_send = {0}; + int ret; + + param.vector_id = cpu_to_le16(vector_id); + param.en_msix = !!en_msix; + + nbl_chan_fill_send_info(&chan_send, common->mgt_pf, + NBL_CHAN_MSG_MAILBOX_SET_IRQ, + &param, sizeof(param), NULL, 0, 1); + ret = chan_ops->send_msg(disp_mgt->chan_ops_tbl->priv, &chan_send); + if (ret) + return ret; + return 0; +} + +static void nbl_disp_chan_set_mailbox_irq_resp(void priv, u16 src_id, + u16 msg_id, void data, + u32 data_len) +{ + struct nbl_dispatch_mgt disp_mgt = (struct nbl_dispatch_mgt )priv; + struct nbl_channel_ops chan_ops = disp_mgt->chan_ops_tbl->ops; + struct nbl_resource_ops res_ops = disp_mgt->res_ops_tbl->ops; + struct nbl_resource_mgt p = disp_mgt->res_ops_tbl->priv; + struct nbl_chan_param_set_mailbox_irq param = { 0 }; + struct device dev = disp_mgt->common->dev; + struct nbl_chan_ack_info chan_ack; + int err = 0; + u16 vector_id; + u32 rel_pf_id; + bool en_msix; + int ret; + + ret = nbl_common_func_id_to_rel_pf_id(disp_mgt->common, src_id, + &rel_pf_id); + if (ret) { + err = -EPERM; + goto ack_out; + } + if (rel_pf_id >= disp_mgt->common->max_pf) { + err = -EPERM; + goto ack_out; + } + if (data_len < sizeof(param)) { + err = -EBADMSG; + goto ack_out; + } + memcpy(&param, data, sizeof(param)); + vector_id = le16_to_cpu(param.vector_id); + en_msix = !!param.en_msix; + + if (res_ops->set_mailbox_irq) { + ret = res_ops->set_mailbox_irq(p, src_id, vector_id, en_msix); + /* Forward the resource errno verbatim to the requester / + if (ret) + err = ret; + } else { + err = -EOPNOTSUPP; + } + +ack_out: + nbl_chan_fill_ack_info(&chan_ack, src_id, + NBL_CHAN_MSG_MAILBOX_SET_IRQ, msg_id, + err, NULL, 0); + ret = chan_ops->send_ack(disp_mgt->chan_ops_tbl->priv, &chan_ack); + if (ret) + dev_err(dev, + "channel send ack failed with ret: %d, msg_type: %d\n", + ret, NBL_CHAN_MSG_MAILBOX_SET_IRQ); +} + +static int nbl_disp_destroy_msix_map(struct nbl_dispatch_mgt disp_mgt) +{ + struct nbl_resource_ops res_ops = disp_mgt->res_ops_tbl->ops; + struct nbl_resource_mgt p = disp_mgt->res_ops_tbl->priv; + struct nbl_common_info common = disp_mgt->common; + + if (!res_ops->destroy_msix_map) + return -EOPNOTSUPP; + return res_ops->destroy_msix_map(p, common->mgt_pf); +} + +static int nbl_disp_set_mailbox_irq(struct nbl_dispatch_mgt disp_mgt, + u16 vector_id, bool en_msix) +{ + struct nbl_resource_ops res_ops = disp_mgt->res_ops_tbl->ops; + struct nbl_resource_mgt p = disp_mgt->res_ops_tbl->priv; + struct nbl_common_info common = disp_mgt->common; + + if (!res_ops->set_mailbox_irq) + return -EOPNOTSUPP; + return res_ops->set_mailbox_irq(p, common->mgt_pf, vector_id, en_msix); +} + +static int nbl_disp_get_vsi_id(struct nbl_dispatch_mgt disp_mgt, u16 type, + u16 vsi_id) +{ + struct nbl_resource_ops res_ops = disp_mgt->res_ops_tbl->ops; + struct nbl_resource_mgt p = disp_mgt->res_ops_tbl->priv; + struct nbl_common_info common = disp_mgt->common; + + if (res_ops->get_vsi_id) + return res_ops->get_vsi_id(p, common->mgt_pf, type, vsi_id); + return -EOPNOTSUPP; +} + +static int nbl_disp_get_eth_id(struct nbl_dispatch_mgt disp_mgt, u16 vsi_id, + u8 eth_num, u8 eth_id, u8 logic_eth_id) +{ + struct nbl_resource_ops res_ops = disp_mgt->res_ops_tbl->ops; + struct nbl_resource_mgt p = disp_mgt->res_ops_tbl->priv; + struct nbl_common_info common = disp_mgt->common; + + if (res_ops->get_eth_id) + return res_ops->get_eth_id(p, common->mgt_pf, vsi_id, + eth_num, eth_id, logic_eth_id); + return -EOPNOTSUPP; +} + +static int nbl_disp_setup_msg(struct nbl_dispatch_mgt disp_mgt) +{ + struct nbl_channel_ops chan_ops = disp_mgt->chan_ops_tbl->ops; + struct nbl_channel_mgt p = disp_mgt->chan_ops_tbl->priv; + int ret = 0; + int _ret; + + _ret = chan_ops->register_msg(p, NBL_CHAN_MSG_CONFIGURE_MSIX_MAP, + nbl_disp_chan_cfg_msix_map_resp, + disp_mgt); + if (_ret < 0 && !ret) + ret = _ret; + + _ret = chan_ops->register_msg(p, NBL_CHAN_MSG_DESTROY_MSIX_MAP, + nbl_disp_chan_destroy_msix_map_resp, + disp_mgt); + if (_ret < 0 && !ret) + ret = _ret; + + _ret = chan_ops->register_msg(p, NBL_CHAN_MSG_MAILBOX_SET_IRQ, + nbl_disp_chan_set_mailbox_irq_resp, + disp_mgt); + if (_ret < 0 && !ret) + ret = _ret; + + _ret = chan_ops->register_msg(p, NBL_CHAN_MSG_GET_VSI_ID, + nbl_disp_chan_get_vsi_id_resp, + disp_mgt); + if (_ret < 0 && !ret) + ret = _ret; + + _ret = chan_ops->register_msg(p, NBL_CHAN_MSG_GET_ETH_ID, + nbl_disp_chan_get_eth_id_resp, + disp_mgt); + if (_ret < 0 && !ret) + ret = _ret; + + return ret; +} + +static void nbl_disp_set_ctrl_bit(struct nbl_dispatch_mgt disp_mgt, u32 lvl) +{ + set_bit(lvl, disp_mgt->ctrl_lvl); +} + +static void nbl_disp_refresh_ctrl_ops(struct nbl_dispatch_mgt disp_mgt) +{ + struct nbl_dispatch_ops disp_ops = disp_mgt->disp_ops_tbl->ops; + + memset(disp_ops, 0, sizeof(disp_ops)); + if (test_bit(NBL_DISP_CTRL_LVL_MGT, disp_mgt->ctrl_lvl)) { + disp_ops->init_module = nbl_disp_init_module; + disp_ops->deinit_module = nbl_disp_deinit_module; + disp_ops->cfg_msix_map = nbl_disp_cfg_msix_map; + disp_ops->destroy_msix_map = nbl_disp_destroy_msix_map; + disp_ops->set_mailbox_irq = nbl_disp_set_mailbox_irq; + disp_ops->get_vsi_id = nbl_disp_get_vsi_id; + disp_ops->get_eth_id = nbl_disp_get_eth_id; + } else if (test_bit(NBL_DISP_CTRL_LVL_NET, disp_mgt->ctrl_lvl)) { + disp_ops->cfg_msix_map = + nbl_disp_chan_cfg_msix_map_req; + disp_ops->destroy_msix_map = nbl_disp_chan_destroy_msix_map_req; + disp_ops->set_mailbox_irq = nbl_disp_chan_set_mailbox_irq_req; + disp_ops->get_vsi_id = nbl_disp_chan_get_vsi_id_req; + disp_ops->get_eth_id = nbl_disp_chan_get_eth_id_req; + } +} + +static struct nbl_dispatch_mgt * +nbl_disp_setup_disp_mgt(struct nbl_common_info common) +{ + struct nbl_dispatch_mgt disp_mgt; + struct device dev = common->dev; + + disp_mgt = devm_kzalloc(dev, sizeof(disp_mgt), GFP_KERNEL); + if (!disp_mgt) + return ERR_PTR(-ENOMEM); + + disp_mgt->common = common; + return disp_mgt; +} + +static struct nbl_dispatch_ops_tbl * +nbl_disp_setup_ops(struct device dev, struct nbl_dispatch_mgt disp_mgt) +{ + struct nbl_dispatch_ops_tbl disp_ops_tbl; + struct nbl_dispatch_ops disp_ops; + + disp_ops_tbl = devm_kzalloc(dev, sizeof(disp_ops_tbl), GFP_KERNEL); + if (!disp_ops_tbl) + return ERR_PTR(-ENOMEM); + + disp_ops = devm_kzalloc(dev, sizeof(disp_ops), GFP_KERNEL); + if (!disp_ops) + return ERR_PTR(-ENOMEM); + + disp_ops_tbl->ops = disp_ops; + disp_ops_tbl->priv = disp_mgt; + + return disp_ops_tbl; +} + +int nbl_disp_init(struct nbl_adapter adapter) +{ + struct nbl_common_info common = &adapter->common; + struct nbl_dispatch_ops_tbl disp_ops_tbl; + struct nbl_resource_ops_tbl res_ops_tbl = + adapter->intf.resource_ops_tbl; + struct nbl_channel_ops_tbl chan_ops_tbl = + adapter->intf.channel_ops_tbl; + struct device dev = &adapter->pdev->dev; + struct nbl_dispatch_mgt disp_mgt; + int ret; + + disp_mgt = nbl_disp_setup_disp_mgt(common); + if (IS_ERR(disp_mgt)) { + ret = PTR_ERR(disp_mgt); + return ret; + } + + disp_ops_tbl = nbl_disp_setup_ops(dev, disp_mgt); + if (IS_ERR(disp_ops_tbl)) { + ret = PTR_ERR(disp_ops_tbl); + return ret; + } + + disp_mgt->res_ops_tbl = res_ops_tbl; + disp_mgt->chan_ops_tbl = chan_ops_tbl; + disp_mgt->disp_ops_tbl = disp_ops_tbl; + adapter->core.disp_mgt = disp_mgt; + adapter->intf.dispatch_ops_tbl = disp_ops_tbl; + + ret = nbl_disp_setup_msg(disp_mgt); + if (ret) + return ret; + + if (common->has_ctrl) + nbl_disp_set_ctrl_bit(disp_mgt, NBL_DISP_CTRL_LVL_MGT); + + if (common->has_net) + nbl_disp_set_ctrl_bit(disp_mgt, NBL_DISP_CTRL_LVL_NET); + nbl_disp_refresh_ctrl_ops(disp_mgt); + return 0; +} + +void nbl_disp_remove(struct nbl_adapter adapter) +{ + /* + * Dispatch structures are devm-allocated and freed at detach. + * + * The five responders registered by nbl_disp_setup_msg() are + * owned by the channel layer (xarray of handlers) and are never + * unregistered here. This is safe because the teardown order + * guarantees no responder can run after this point: + * + * mailbox teardown + * -> cancel_work_sync(clean_mbx_task) // drain RX work + * -> nbl_chan_teardown_queue() // stop HW queue, + * // join clean task, + * // active=false + * -> nbl_chan_remove_common() + * -> nbl_common_destroy_wq() // no new work + * -> nbl_chan_remove_msg_handler() // free handler nodes + * + * By the time devres frees disp_mgt, the mailbox queue is stopped + * and the handler xarray is empty, so no responder can touch + * res_mgt->intr_mgt after nbl_intr_mgt_stop() has cleared it. + / +} diff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_core/nbl_dispatch.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_core/nbl_dispatch.h new file mode 100644 index 00000000000000..a7e5802344b414 --- /dev/null +++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_core/nbl_dispatch.h @@ -0,0 +1,23 @@ +/ SPDX-License-Identifier: GPL-2.0 / +/ + * Copyright (c) 2026 Nebula Matrix Limited. + / + +#ifndef _NBL_DISPATCH_H_ +#define _NBL_DISPATCH_H_ +#include "../nbl_include/nbl_include.h" +#include "../nbl_include/nbl_def_channel.h" +#include "../nbl_include/nbl_def_resource.h" +#include "../nbl_include/nbl_def_dispatch.h" +#include "../nbl_include/nbl_def_common.h" +#include "../nbl_core.h" + +struct nbl_dispatch_mgt { + struct nbl_common_info common; + struct nbl_resource_ops_tbl res_ops_tbl; + struct nbl_channel_ops_tbl chan_ops_tbl; + struct nbl_dispatch_ops_tbl disp_ops_tbl; + DECLARE_BITMAP(ctrl_lvl, NBL_DISP_CTRL_LVL_MAX); +}; + +#endif diff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_chip.c b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_chip.c new file mode 100644 index 00000000000000..419eb6392ada19 --- /dev/null +++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_chip.c @@ -0,0 +1,32 @@ +// SPDX-License-Identifier: GPL-2.0 +/ + * Copyright (c) 2026 Nebula Matrix Limited. + / +#include <linux/device.h> +#include "nbl_chip.h" + +void nbl_res_chip_deinit_module(struct nbl_resource_mgt res_mgt) +{ + struct nbl_hw_ops hw_ops = res_mgt->hw_ops_tbl->ops; + struct nbl_common_info common = res_mgt->common; + + if (!common->has_ctrl) + return; + hw_ops->deinit_module(res_mgt->hw_ops_tbl->priv); +} + +int nbl_res_chip_init_module(struct nbl_resource_mgt res_mgt) +{ + struct nbl_common_info common = res_mgt->common; + struct nbl_hw_ops hw_ops; + u8 eth_speed, eth_num; + struct nbl_hw_mgt p; + + if (!common->has_ctrl) + return -EINVAL; + eth_speed = res_mgt->resource_info->board_info.eth_speed; + eth_num = res_mgt->resource_info->board_info.eth_num; + hw_ops = res_mgt->hw_ops_tbl->ops; + p = res_mgt->hw_ops_tbl->priv; + return hw_ops->init_module(p, eth_speed, eth_num); +} diff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_chip.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_chip.h new file mode 100644 index 00000000000000..d14093ba916c95 --- /dev/null +++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_chip.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0 / +/ + * Copyright (c) 2026 Nebula Matrix Limited. + / + +#ifndef _NBL_CHIP_H_ +#define _NBL_CHIP_H_ + +#include "nbl_resource.h" +int nbl_res_chip_init_module(struct nbl_resource_mgt res_mgt); +void nbl_res_chip_deinit_module(struct nbl_resource_mgt res_mgt); +#endif diff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_leonis/nbl_hw_leonis.c b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_leonis/nbl_hw_leonis.c index cf40ddc4519211..2b55f2c2518b4f 100644 --- a/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_leonis/nbl_hw_leonis.c +++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_leonis/nbl_hw_leonis.c @@ -5,9 +5,1232 @@ #include <linux/device.h> #include <linux/pci.h> #include <linux/bits.h> +#include <linux/delay.h> #include <linux/io.h> +#include <linux/spinlock.h> +#include <linux/bitfield.h> #include "nbl_hw_leonis.h" +/ + * Firmware cleanup after driver_status=false is asynchronous and the + * current hardware revision exposes no cleanup-complete status bit, so + * this delay is best-effort only: it gives the firmware pass time to + * settle before the caller returns, but cannot prove that it has. + * Bounding the wait keeps a later probe from re-programming the per-PF + * registers that pass reclaims (mailbox QINFO routing, qinfo registers) + * while firmware is realistically still wiping them. The chip-wide + * datapath registers init_module() writes are not part of that pass and + * stay programmed until chip reset. The hard teardown ordering comes + * from the device-link relationship between the sibling PFs and the + * control PF. + * + * These bounds are empirical, not derived from a documented firmware + * latency. There is nothing to poll for instead: the hardware exposes + * no cleanup-complete bit, and the reclaimed entries read back the same + * before and after the pass, so there is no observable side effect to + * wait on and nothing to validate the delay against. A firmware pass + * that takes longer than this window - an ECPU still busy right after + * kexec, say - can therefore still race a later probe. + / +#define NBL_FW_CLEANUP_SYNC_MIN_US 2000 +#define NBL_FW_CLEANUP_SYNC_MAX_US 3000 + +static void nbl_hw_read_mbx_regs(struct nbl_hw_mgt hw_mgt, u64 reg, u32 data, + u32 len) +{ + u32 i; + + if (len % 4) + return; + if (reg >= (u64)hw_mgt->mailbox_bar_size \|\| + reg + len > (u64)hw_mgt->mailbox_bar_size) { + dev_err_once(hw_mgt->common->dev, + "mbx read out of range: reg=0x%llx len=%u bar_size=%pa\n", + reg, len, &hw_mgt->mailbox_bar_size); + return; + } + for (i = 0; i < len / 4; i++) + data[i] = nbl_mbx_rd32(hw_mgt, reg + i sizeof(u32)); +} + +static void nbl_hw_write_mbx_regs(struct nbl_hw_mgt hw_mgt, u64 reg, + const u32 data, u32 len) +{ + u32 i; + + if (len % 4) + return; + if (reg >= (u64)hw_mgt->mailbox_bar_size \|\| + reg + len > (u64)hw_mgt->mailbox_bar_size) { + dev_err_once(hw_mgt->common->dev, + "mbx write out of range: reg=0x%llx len=%u bar_size=%pa\n", + reg, len, &hw_mgt->mailbox_bar_size); + return; + } + for (i = 0; i < len / 4; i++) + nbl_mbx_wr32(hw_mgt, reg + i * sizeof(u32), data[i]); +} + +/* + * Flush posted mailbox-BAR writes by reading back through the same + * BAR. + / +static void nbl_hw_flush_mbx_write(struct nbl_hw_mgt hw_mgt, u64 reg) +{ + u32 data; + + nbl_hw_read_mbx_regs(hw_mgt, reg, &data, sizeof(data)); +} + +static void nbl_hw_rd_regs(struct nbl_hw_mgt hw_mgt, u64 reg, u32 data, + u32 len) +{ + u32 size = len / 4; + u32 i; + + if (len % 4) + return; + for (i = 0; i < size; i++) + data[i] = rd32(hw_mgt->hw_addr, reg + i * sizeof(u32)); +} + +static void nbl_hw_wr_regs(struct nbl_hw_mgt hw_mgt, u64 reg, const u32 data, + u32 len) +{ + u32 size = len / 4; + u32 i; + + if (len % 4) + return; + for (i = 0; i < size; i++) + wr32(hw_mgt->hw_addr, reg + i * sizeof(u32), data[i]); +} + +static void nbl_hw_rd_regs_lock(struct nbl_hw_mgt hw_mgt, u64 reg, u32 data, + u32 len) +{ + u32 size = len / 4; + u32 i; + + if (len % 4) + return; + + spin_lock(&hw_mgt->reg_lock); + + for (i = 0; i < size; i++) + data[i] = rd32(hw_mgt->hw_addr, reg + i * sizeof(u32)); + spin_unlock(&hw_mgt->reg_lock); +} + +static void nbl_hw_wr_regs_lock(struct nbl_hw_mgt hw_mgt, u64 reg, + const u32 data, u32 len) +{ + u32 size = len / 4; + u32 i; + + if (len % 4) + return; + spin_lock(&hw_mgt->reg_lock); + for (i = 0; i < size; i++) + wr32(hw_mgt->hw_addr, reg + i * sizeof(u32), data[i]); + spin_unlock(&hw_mgt->reg_lock); +} + +/* + * Flush posted MEMORY-BAR writes by reading back a register that only + * exists in the control-PF mapping. + * + * The mapped size does not derive from bar_len: nbl_hw_init_leonis() + * maps PAGE_ALIGN_DOWN(NBL_MEM_BAR_TOTAL_SIZE - NBL_RDMA_NOTIFY_LEN) + * (~64 MiB) when has_ctrl is set, and NBL_REG_NET_ONLY_LEN (8 KiB) when + * it is not. NBL_HW_DUMMY_REG (0x1300904) is only covered by the + * control-PF mapping, so this must not be called on a net-only PF. + * + * Every current caller runs on the control PF: init_module()/ + * deinit_module() are has_ctrl-guarded in the resource layer, and the + * MSI-X paths are either has_ctrl-guarded as well or dispatched to the + * management PF. The check below is therefore a defensive guard against + * a future caller, not a reachable failure path. + / +static void nbl_flush_writes(struct nbl_hw_mgt hw_mgt) +{ + if (WARN_ON_ONCE(!hw_mgt->common->has_ctrl)) + return; + + nbl_hw_rd32(hw_mgt, NBL_HW_DUMMY_REG); +} + +/* + * Registers reset to zero after cold boot / FLR / bus reset. Firmware + * programs valid values before driver probe, so zero is only seen on + * hardware fault or register read failure. Initialize data=0 to guard + * against nbl_hw_read_mbx_regs() early-return on bounds-check failure. + / +static void nbl_hw_get_fw_eth_map(struct nbl_hw_mgt hw_mgt, u32 eth_map) +{ + u32 data = 0; + + nbl_hw_read_mbx_regs(hw_mgt, NBL_FW_BOARD_DW6_OFFSET, &data, + sizeof(data)); + eth_map = FIELD_GET(NBL_FW_BOARD_DW6_ETH_BITMAP_MASK, data); +} + +static u32 nbl_hw_get_quirks(struct nbl_hw_mgt hw_mgt) +{ + u32 quirks = 0; + + / + * Read quirk bits from mailbox register. + * All supported firmware implement the quirk ABI, + * firmware always populates NBL_LEONIS_QUIRKS_OFFSET. + * Value ~0U indicates no active quirks. + / + nbl_hw_read_mbx_regs(hw_mgt, NBL_LEONIS_QUIRKS_OFFSET, &quirks, + sizeof(u32)); + + if (quirks == ~0u) + return 0; + + return quirks; +} + +static void nbl_configure_dped_checksum(struct nbl_hw_mgt hw_mgt) +{ + u32 data = 0; + + /* DPED dped_l4_ck_cmd_40 for sctp / + spin_lock(&hw_mgt->reg_lock); + nbl_hw_rd_regs(hw_mgt, NBL_DPED_L4_CK_CMD_40_ADDR, &data, sizeof(data)); + data \|= FIELD_PREP(NBL_DPED_L4_CK_CMD_40_EN_MASK, 1); + nbl_hw_wr_regs(hw_mgt, NBL_DPED_L4_CK_CMD_40_ADDR, &data, sizeof(data)); + spin_unlock(&hw_mgt->reg_lock); +} + +static void nbl_dped_init(struct nbl_hw_mgt hw_mgt) +{ + spin_lock(&hw_mgt->reg_lock); + nbl_hw_wr32(hw_mgt, NBL_DPED_VLAN_OFFSET, 0xC); + nbl_hw_wr32(hw_mgt, NBL_DPED_DSCP_OFFSET_0, 0x8); + nbl_hw_wr32(hw_mgt, NBL_DPED_DSCP_OFFSET_1, 0x4); + spin_unlock(&hw_mgt->reg_lock); + /* dped checksum offload / + nbl_configure_dped_checksum(hw_mgt); +} + +static void nbl_uped_init(struct nbl_hw_mgt hw_mgt) +{ + u32 hw_edit = 0; + + /* V4 TCP: l3_len = 0 / + spin_lock(&hw_mgt->reg_lock); + nbl_hw_rd_regs(hw_mgt, NBL_UPED_HW_EDT_PROF_TABLE(NBL_UPED_V4_TCP_IDX), + &hw_edit, sizeof(hw_edit)); + hw_edit &= ~NBL_PED_HW_EDIT_PROFILE_L3_LEN_MASK; + nbl_hw_wr_regs(hw_mgt, NBL_UPED_HW_EDT_PROF_TABLE(NBL_UPED_V4_TCP_IDX), + &hw_edit, sizeof(hw_edit)); + + / V6 TCP: l3_len = 1 / + nbl_hw_rd_regs(hw_mgt, NBL_UPED_HW_EDT_PROF_TABLE(NBL_UPED_V6_TCP_IDX), + &hw_edit, sizeof(hw_edit)); + hw_edit = (hw_edit & ~NBL_PED_HW_EDIT_PROFILE_L3_LEN_MASK) \| + FIELD_PREP(NBL_PED_HW_EDIT_PROFILE_L3_LEN_MASK, 1); + nbl_hw_wr_regs(hw_mgt, NBL_UPED_HW_EDT_PROF_TABLE(NBL_UPED_V6_TCP_IDX), + &hw_edit, sizeof(hw_edit)); + spin_unlock(&hw_mgt->reg_lock); +} + +static int nbl_shaping_eth_init(struct nbl_hw_mgt hw_mgt, u8 eth_id, u8 speed) +{ + struct nbl_shaping_dvn_dport_u dvn_dport = { 0 }; + struct nbl_shaping_dport_u dport = { 0 }; + u32 rate, half_rate; + u32 depth; + u64 low_val, high_val; + + switch (speed) { + case NBL_FW_PORT_SPEED_100G: + rate = 100000; + break; + case NBL_FW_PORT_SPEED_50G: + rate = 50000; + break; + case NBL_FW_PORT_SPEED_25G: + rate = 25000; + break; + case NBL_FW_PORT_SPEED_10G: + rate = 10000; + break; + default: + dev_err(hw_mgt->common->dev, + "Unsupported port speed %u for eth%u\n", speed, eth_id); + return -EINVAL; + } + + half_rate = rate / 2; + depth = max_t(u32, rate * 2, NBL_LR_LEONIS_NET_BUCKET_DEPTH); + + /* 1. clear valid first + * dport and dvn_dport are zero-initialised above, so VALID=0 already + / + spin_lock(&hw_mgt->reg_lock); + nbl_hw_wr_regs(hw_mgt, NBL_SHAPING_DPORT_REG(eth_id), dport.data, + sizeof(dport)); + nbl_hw_wr_regs(hw_mgt, NBL_SHAPING_DVN_DPORT_REG(eth_id), + dvn_dport.data, sizeof(dvn_dport)); + + / 2. write config words (valid=0, safe) / + low_val = FIELD_PREP(NBL_DPORT_CIR_MASK, rate) \| + FIELD_PREP(NBL_DPORT_PIR_MASK, rate) \| + FIELD_PREP(NBL_DPORT_DEPTH_MASK, depth) \| + FIELD_PREP(NBL_DPORT_CBS_MASK_LOW, depth & 0x3F); + high_val = FIELD_PREP(NBL_DPORT_CBS_MASK_HIGH, depth >> 6) \| + FIELD_PREP(NBL_DPORT_PBS_MASK, depth); + / Fixed split, independent of host endian / + dport.data[0] = lower_32_bits(low_val); + dport.data[1] = upper_32_bits(low_val); + dport.data[2] = lower_32_bits(high_val); + dport.data[3] = upper_32_bits(high_val); + + low_val = FIELD_PREP(NBL_DPORT_CIR_MASK, half_rate) \| + FIELD_PREP(NBL_DPORT_PIR_MASK, rate) \| + FIELD_PREP(NBL_DPORT_DEPTH_MASK, depth) \| + FIELD_PREP(NBL_DPORT_CBS_MASK_LOW, depth & 0x3F); + high_val = FIELD_PREP(NBL_DPORT_CBS_MASK_HIGH, depth >> 6) \| + FIELD_PREP(NBL_DPORT_PBS_MASK, depth); + dvn_dport.data[0] = lower_32_bits(low_val); + dvn_dport.data[1] = upper_32_bits(low_val); + dvn_dport.data[2] = lower_32_bits(high_val); + dvn_dport.data[3] = upper_32_bits(high_val); + + nbl_hw_wr_regs(hw_mgt, NBL_SHAPING_DPORT_REG(eth_id), dport.data, + sizeof(dport)); + nbl_hw_wr_regs(hw_mgt, NBL_SHAPING_DVN_DPORT_REG(eth_id), + dvn_dport.data, sizeof(dvn_dport)); + + / 3. commit: set valid last / + low_val = FIELD_PREP(NBL_DPORT_VALID_MASK, 1); + dport.data[0] \|= lower_32_bits(low_val); + + low_val = FIELD_PREP(NBL_DPORT_VALID_MASK, 1); + dvn_dport.data[0] \|= lower_32_bits(low_val); + + nbl_hw_wr_regs(hw_mgt, NBL_SHAPING_DPORT_REG(eth_id), dport.data, + sizeof(dport)); + nbl_hw_wr_regs(hw_mgt, NBL_SHAPING_DVN_DPORT_REG(eth_id), + dvn_dport.data, sizeof(dvn_dport)); + spin_unlock(&hw_mgt->reg_lock); + return 0; +} + +static int nbl_shaping_init(struct nbl_hw_mgt hw_mgt, u8 speed) +{ + struct nbl_shaping_net_u net_shaping = { 0 }; + u32 eth_bitmap = 0; + u32 reg_val; + int ret; + int i; + + nbl_hw_get_fw_eth_map(hw_mgt, &eth_bitmap); + for (i = 0; i < NBL_MAX_ETHERNET; i++) { + if (!(eth_bitmap & BIT(i))) + continue; + ret = nbl_shaping_eth_init(hw_mgt, i, speed); + if (ret) + return ret; + } + /* + * Read-modify-write as one critical section: reg_lock is dropped + * between the two _lock() helpers, which would let another writer + * of this register land in the middle of the update. Nothing + * else touches NBL_DSCH_PSHA_EN_ADDR today, but the RMW has to be + * atomic for that to stay true. + / + spin_lock(&hw_mgt->reg_lock); + nbl_hw_rd_regs(hw_mgt, NBL_DSCH_PSHA_EN_ADDR, &reg_val, + sizeof(reg_val)); + reg_val &= ~NBL_DSCH_PSHA_EN_MASK; + reg_val \|= FIELD_PREP(NBL_DSCH_PSHA_EN_MASK, + eth_bitmap & GENMASK(3, 0)); + nbl_hw_wr_regs(hw_mgt, NBL_DSCH_PSHA_EN_ADDR, &reg_val, + sizeof(reg_val)); + spin_unlock(&hw_mgt->reg_lock); + + for (i = 0; i < NBL_MAX_PF; i++) { + nbl_hw_wr_regs_lock(hw_mgt, NBL_SHAPING_NET_REG(i), + net_shaping.data, + sizeof(net_shaping)); + } + nbl_flush_writes(hw_mgt); + return 0; +} + +static void nbl_dsch_qid_max_init(struct nbl_hw_mgt hw_mgt) +{ + u32 quanta = 0; + u32 qid_max = 0; + + quanta = FIELD_PREP(NBL_DSCH_VN_QUANTA_H_QUA_MASK, NBL_HOST_QUANTA) \| + FIELD_PREP(NBL_DSCH_VN_QUANTA_E_QUA_MASK, NBL_ECPU_QUANTA); + spin_lock(&hw_mgt->reg_lock); + nbl_hw_wr_regs(hw_mgt, NBL_DSCH_VN_QUANTA_ADDR, &quanta, + sizeof(quanta)); + nbl_hw_rd_regs(hw_mgt, NBL_DSCH_HOST_QID_MAX, &qid_max, + sizeof(qid_max)); + qid_max &= ~NBL_DSCH_HOST_QID_MAX_MASK; + qid_max \|= FIELD_PREP(NBL_DSCH_HOST_QID_MAX_MASK, NBL_MAX_QUEUE_ID); + nbl_hw_wr_regs(hw_mgt, NBL_DSCH_HOST_QID_MAX, &qid_max, + sizeof(qid_max)); + spin_unlock(&hw_mgt->reg_lock); +} + +static int nbl_ustore_init(struct nbl_hw_mgt hw_mgt, u8 eth_num) +{ + u32 eth_bitmap = 0; + u32 drop_th = 0; + u32 pkt_len = 0; + u32 reg_val = 0; + int i; + + / + * eth_num is validated in the resource layer: + * nbl_res_init_pf_num() requires 1/2/4 PFs, and + * nbl_res_ctrl_dev_setup_eth_info() requires max_pf == eth_num. + * This is a defensive check only; if it triggers, the resource + * layer validation was bypassed, which is a bug. + / + if (WARN_ON(eth_num != 1 && eth_num != 2 && eth_num != 4)) + return -EINVAL; + / Read current packet length config + (to preserve other fields while updating 'min') + / + spin_lock(&hw_mgt->reg_lock); + nbl_hw_rd_regs(hw_mgt, NBL_USTORE_PKT_LEN_ADDR, &pkt_len, + sizeof(pkt_len)); + /* min arp packet length 42 (14 + 28) / + pkt_len &= ~NBL_USTORE_PKT_LEN_MIN_MASK; + pkt_len \|= FIELD_PREP(NBL_USTORE_PKT_LEN_MIN_MASK, 42); + nbl_hw_wr_regs(hw_mgt, NBL_USTORE_PKT_LEN_ADDR, &pkt_len, + sizeof(pkt_len)); + + drop_th \|= FIELD_PREP(NBL_USTORE_PORT_DROP_TH_EN_MASK, 1); + if (eth_num == 1) + drop_th \|= FIELD_PREP(NBL_USTORE_PORT_DROP_TH_DISC_TH_MASK, + NBL_USTORE_SINGLE_ETH_DROP_TH); + else if (eth_num == 2) + drop_th \|= FIELD_PREP(NBL_USTORE_PORT_DROP_TH_DISC_TH_MASK, + NBL_USTORE_DUAL_ETH_DROP_TH); + else + drop_th \|= FIELD_PREP(NBL_USTORE_PORT_DROP_TH_DISC_TH_MASK, + NBL_USTORE_QUAD_ETH_DROP_TH); + nbl_hw_get_fw_eth_map(hw_mgt, &eth_bitmap); + for (i = 0; i < NBL_MAX_ETHERNET; i++) { + if (!(eth_bitmap & BIT(i))) + continue; + nbl_hw_rd_regs(hw_mgt, NBL_USTORE_PORT_DROP_TH_REG_ARR(i), + &reg_val, sizeof(reg_val)); + reg_val &= ~(NBL_USTORE_PORT_DROP_TH_EN_MASK \| + NBL_USTORE_PORT_DROP_TH_DISC_TH_MASK); + reg_val \|= drop_th; + nbl_hw_wr_regs(hw_mgt, NBL_USTORE_PORT_DROP_TH_REG_ARR(i), + &reg_val, sizeof(reg_val)); + } + + / Clear port drop/truncate counters by reading them + * (hardware has read-to-clear behavior for these registers) + / + for (i = 0; i < NBL_MAX_ETHERNET; i++) { + if (!(eth_bitmap & BIT(i))) + continue; + nbl_hw_rd32(hw_mgt, NBL_USTORE_BUF_PORT_DROP_PKT(i)); + nbl_hw_rd32(hw_mgt, NBL_USTORE_BUF_PORT_TRUN_PKT(i)); + } + spin_unlock(&hw_mgt->reg_lock); + return 0; +} + +static void nbl_dstore_init(struct nbl_hw_mgt hw_mgt, u8 speed) +{ + u32 eth_bitmap = 0; + u32 drop_th = 0; + u32 fc_th = 0; + u32 bp_th = 0; + int i; + + for (i = 0; i < NBL_DSTORE_PORT_DROP_TH_DEPTH; i++) { + spin_lock(&hw_mgt->reg_lock); + nbl_hw_rd_regs(hw_mgt, NBL_DSTORE_PORT_DROP_TH_REG(i), &drop_th, + sizeof(drop_th)); + drop_th &= ~NBL_DSTORE_PORT_DROP_EN_MASK; + nbl_hw_wr_regs(hw_mgt, NBL_DSTORE_PORT_DROP_TH_REG(i), &drop_th, + sizeof(drop_th)); + spin_unlock(&hw_mgt->reg_lock); + } + + spin_lock(&hw_mgt->reg_lock); + nbl_hw_rd_regs(hw_mgt, NBL_DSTORE_DISC_BP_TH, &bp_th, sizeof(bp_th)); + bp_th \|= FIELD_PREP(NBL_DSTORE_DISC_BP_TH_EN_MASK, 1); + nbl_hw_wr_regs(hw_mgt, NBL_DSTORE_DISC_BP_TH, &bp_th, sizeof(bp_th)); + spin_unlock(&hw_mgt->reg_lock); + + nbl_hw_get_fw_eth_map(hw_mgt, &eth_bitmap); + for (i = 0; i < NBL_MAX_ETHERNET; i++) { + if (!(eth_bitmap & BIT(i))) + continue; + spin_lock(&hw_mgt->reg_lock); + nbl_hw_rd_regs(hw_mgt, NBL_DSTORE_D_DPORT_FC_TH_REG(i), &fc_th, + sizeof(fc_th)); + fc_th &= ~(NBL_DSTORE_D_DPORT_FC_XOFF_TH_MASK \| + NBL_DSTORE_D_DPORT_FC_XON_TH_MASK); + if (speed == NBL_FW_PORT_SPEED_100G) { + fc_th \|= + FIELD_PREP(NBL_DSTORE_D_DPORT_FC_XOFF_TH_MASK, + NBL_DSTORE_DROP_XOFF_TH_100G) \| + FIELD_PREP(NBL_DSTORE_D_DPORT_FC_XON_TH_MASK, + NBL_DSTORE_DROP_XON_TH_100G); + } else { + fc_th \|= + FIELD_PREP(NBL_DSTORE_D_DPORT_FC_XOFF_TH_MASK, + NBL_DSTORE_DROP_XOFF_TH) \| + FIELD_PREP(NBL_DSTORE_D_DPORT_FC_XON_TH_MASK, + NBL_DSTORE_DROP_XON_TH); + } + + fc_th \|= FIELD_PREP(NBL_DSTORE_D_DPORT_FC_FC_EN_MASK, 1); + nbl_hw_wr_regs(hw_mgt, NBL_DSTORE_D_DPORT_FC_TH_REG(i), &fc_th, + sizeof(fc_th)); + spin_unlock(&hw_mgt->reg_lock); + } +} + +static void nbl_dvn_descreq_num_cfg(struct nbl_hw_mgt hw_mgt, u8 descreq_num) +{ + u8 split_ring_num = (descreq_num >> 3) & 0x1; + u8 ring_num = descreq_num & 0x7; + u32 num_cfg; + u32 reg_val; + + spin_lock(&hw_mgt->reg_lock); + nbl_hw_rd_regs(hw_mgt, NBL_DVN_DESCREQ_NUM_CFG, &reg_val, + sizeof(reg_val)); + + num_cfg = FIELD_PREP(NBL_DVN_DESCREQ_NUM_CFG_AVRING_DESREQ_NUM_CFG_MASK, + split_ring_num) \| + FIELD_PREP(NBL_DVN_DESCREQ_NUM_CFG_PACKED_L1_NUM_MASK, + ring_num); + reg_val &= ~(NBL_DVN_DESCREQ_NUM_CFG_AVRING_DESREQ_NUM_CFG_MASK \| + NBL_DVN_DESCREQ_NUM_CFG_PACKED_L1_NUM_MASK); + reg_val \|= num_cfg; + nbl_hw_wr_regs(hw_mgt, NBL_DVN_DESCREQ_NUM_CFG, &reg_val, + sizeof(reg_val)); + spin_unlock(&hw_mgt->reg_lock); +} + +static void nbl_dvn_init(struct nbl_hw_mgt hw_mgt, u8 speed) +{ + u32 timeout = 0; + u32 ro_flag = 0; + + spin_lock(&hw_mgt->reg_lock); + nbl_hw_wr32(hw_mgt, NBL_DVN_ECPU_QUEUE_NUM, 0); + spin_unlock(&hw_mgt->reg_lock); + + timeout = FIELD_PREP(NBL_DVN_DESC_WR_MERGE_TIMEOUT_CFG_CYCLE_MASK, + DEFAULT_DVN_DESC_WR_MERGE_TIMEOUT_MAX); + nbl_hw_wr_regs_lock(hw_mgt, NBL_DVN_DESC_WR_MERGE_TIMEOUT, &timeout, + sizeof(timeout)); + spin_lock(&hw_mgt->reg_lock); + nbl_hw_rd_regs(hw_mgt, NBL_DVN_DIF_REQ_RD_RO_FLAG, &ro_flag, + sizeof(ro_flag)); + if (pcie_relaxed_ordering_enabled(hw_mgt->common->pdev)) { + ro_flag \|= + FIELD_PREP(NBL_DVN_DIF_REQ_RD_RO_FLAG_DESC_RO_EN_MASK, + 1) \| + FIELD_PREP(NBL_DVN_DIF_REQ_RD_RO_FLAG_DATA_RO_EN_MASK, + 1) \| + FIELD_PREP(NBL_DVN_DIF_REQ_RD_RO_FLAG_AVRING_RO_EN_MASK, + 1); + } else { + ro_flag &= + ~(FIELD_PREP(NBL_DVN_DIF_REQ_RD_RO_FLAG_DESC_RO_EN_MASK, + 1) \| + FIELD_PREP(NBL_DVN_DIF_REQ_RD_RO_FLAG_DATA_RO_EN_MASK, + 1) \| + FIELD_PREP(NBL_DVN_DIF_REQ_RD_RO_FLAG_AVRING_RO_EN_MASK, + 1)); + } + nbl_hw_wr_regs(hw_mgt, NBL_DVN_DIF_REQ_RD_RO_FLAG, &ro_flag, + sizeof(ro_flag)); + spin_unlock(&hw_mgt->reg_lock); + if (speed == NBL_FW_PORT_SPEED_100G) + nbl_dvn_descreq_num_cfg(hw_mgt, + DEFAULT_DVN_100G_DESCREQ_NUMCFG); + else + nbl_dvn_descreq_num_cfg(hw_mgt, DEFAULT_DVN_DESCREQ_NUMCFG); +} + +static void nbl_uvn_init(struct nbl_hw_mgt hw_mgt) +{ + u16 wr_timeout = NBL_UVN_DESC_WR_TIMEOUT_VAL; + u32 timeout = NBL_UVN_DESC_RD_WAIT_TICKS; + u32 prefetch_init = 0; + bool ro_enabled; + u32 flag = 0; + u32 mask = 0; + u32 quirks; + u32 reg_val; + + spin_lock(&hw_mgt->reg_lock); + nbl_hw_wr32(hw_mgt, NBL_UVN_ECPU_QUEUE_NUM, 0); + nbl_hw_wr32(hw_mgt, NBL_UVN_DESC_RD_WAIT, timeout); + nbl_hw_rd_regs(hw_mgt, NBL_UVN_DESC_WR_TIMEOUT, + &reg_val, sizeof(reg_val)); + reg_val &= ~NBL_UVN_DESC_WR_TIMEOUT_NUM_MASK; + reg_val \|= FIELD_PREP(NBL_UVN_DESC_WR_TIMEOUT_NUM_MASK, wr_timeout); + nbl_hw_wr_regs(hw_mgt, NBL_UVN_DESC_WR_TIMEOUT, &reg_val, + sizeof(reg_val)); + ro_enabled = pcie_relaxed_ordering_enabled(hw_mgt->common->pdev); + + nbl_hw_rd_regs(hw_mgt, NBL_UVN_DIF_REQ_RO_FLAG, &flag, sizeof(flag)); + if (ro_enabled) { + flag \|= FIELD_PREP(NBL_UVN_DIF_REQ_RO_FLAG_AVAIL_RD_MASK, 1) \| + FIELD_PREP(NBL_UVN_DIF_REQ_RO_FLAG_DESC_RD_MASK, 1) \| + FIELD_PREP(NBL_UVN_DIF_REQ_RO_FLAG_PKT_WR_MASK, 1); + flag &= ~FIELD_PREP(NBL_UVN_DIF_REQ_RO_FLAG_DESC_WR_MASK, 1); + } else { + flag &= ~(FIELD_PREP(NBL_UVN_DIF_REQ_RO_FLAG_AVAIL_RD_MASK, 1) \| + FIELD_PREP(NBL_UVN_DIF_REQ_RO_FLAG_DESC_RD_MASK, 1) \| + FIELD_PREP(NBL_UVN_DIF_REQ_RO_FLAG_PKT_WR_MASK, 1) \| + FIELD_PREP(NBL_UVN_DIF_REQ_RO_FLAG_DESC_WR_MASK, 1)); + } + nbl_hw_wr_regs(hw_mgt, NBL_UVN_DIF_REQ_RO_FLAG, &flag, sizeof(flag)); + + nbl_hw_rd_regs(hw_mgt, NBL_UVN_QUEUE_ERR_MASK, &mask, sizeof(mask)); + mask \|= FIELD_PREP(NBL_UVN_QUEUE_ERR_MASK_DIF_ERR_MASK, 1); + + nbl_hw_wr_regs(hw_mgt, NBL_UVN_QUEUE_ERR_MASK, &mask, sizeof(mask)); + + spin_unlock(&hw_mgt->reg_lock); + quirks = nbl_hw_get_quirks(hw_mgt); + / + * sel=0: use configured num; sel=1: use internal calc (max 32) + * Default is sel=1, unless NBL_QUIRK_UVN_PREFETCH_ALIGN is set, + * in which case override to sel=0. + / + spin_lock(&hw_mgt->reg_lock); + nbl_hw_rd_regs(hw_mgt, NBL_UVN_DESC_PREFETCH_INIT, &reg_val, + sizeof(reg_val)); + prefetch_init = + FIELD_PREP(NBL_UVN_DESC_PREFETCH_INIT_NUM_MASK, + NBL_UVN_DESC_PREFETCH_NUM) \| + FIELD_PREP(NBL_UVN_DESC_PREFETCH_INIT_SEL_MASK, + (quirks & NBL_QUIRK_UVN_PREFETCH_ALIGN) ? 0 : 1); + reg_val &= ~(NBL_UVN_DESC_PREFETCH_INIT_NUM_MASK \| + NBL_UVN_DESC_PREFETCH_INIT_SEL_MASK); + reg_val \|= prefetch_init; + nbl_hw_wr_regs(hw_mgt, NBL_UVN_DESC_PREFETCH_INIT, &reg_val, + sizeof(reg_val)); + spin_unlock(&hw_mgt->reg_lock); +} + +static void nbl_uqm_init(struct nbl_hw_mgt hw_mgt) +{ + u32 que_type = 0; + u32 cnt = 0; + int i; + + spin_lock(&hw_mgt->reg_lock); + nbl_hw_wr_regs(hw_mgt, NBL_UQM_FWD_DROP_CNT, &cnt, sizeof(cnt)); + + nbl_hw_wr_regs(hw_mgt, NBL_UQM_DROP_PKT_CNT, &cnt, sizeof(cnt)); + nbl_hw_wr_regs(hw_mgt, NBL_UQM_DROP_PKT_SLICE_CNT, &cnt, sizeof(cnt)); + nbl_hw_wr_regs(hw_mgt, NBL_UQM_DROP_PKT_LEN_ADD_CNT, &cnt, sizeof(cnt)); + nbl_hw_wr_regs(hw_mgt, NBL_UQM_DROP_HEAD_PNTR_ADD_CNT, &cnt, + sizeof(cnt)); + nbl_hw_wr_regs(hw_mgt, NBL_UQM_DROP_WEIGHT_ADD_CNT, &cnt, sizeof(cnt)); + + for (i = 0; i < NBL_UQM_PORT_DROP_DEPTH; i++) { + nbl_hw_wr_regs(hw_mgt, + NBL_UQM_PORT_DROP_PKT_CNT + (sizeof(cnt) * i), + &cnt, sizeof(cnt)); + nbl_hw_wr_regs(hw_mgt, + NBL_UQM_PORT_DROP_PKT_SLICE_CNT + + (sizeof(cnt) * i), + &cnt, sizeof(cnt)); + nbl_hw_wr_regs(hw_mgt, + NBL_UQM_PORT_DROP_PKT_LEN_ADD_CNT + + (sizeof(cnt) * i), + &cnt, sizeof(cnt)); + nbl_hw_wr_regs(hw_mgt, + NBL_UQM_PORT_DROP_HEAD_PNTR_ADD_CNT + + (sizeof(cnt) * i), + &cnt, sizeof(cnt)); + nbl_hw_wr_regs(hw_mgt, + NBL_UQM_PORT_DROP_WEIGHT_ADD_CNT + + (sizeof(cnt) * i), + &cnt, sizeof(cnt)); + } + + for (i = 0; i < NBL_UQM_DPORT_DROP_DEPTH; i++) + nbl_hw_wr_regs(hw_mgt, + NBL_UQM_DPORT_DROP_CNT + (sizeof(cnt) * i), &cnt, + sizeof(cnt)); + /* bit0: 0=bp mode, 1=drop mode, resv bit1-31 / + nbl_hw_wr_regs(hw_mgt, NBL_UQM_QUE_TYPE, &que_type, sizeof(que_type)); + spin_unlock(&hw_mgt->reg_lock); +} + +static int nbl_dp_init(struct nbl_hw_mgt hw_mgt, u8 speed, u8 eth_num) +{ + int ret; + + nbl_dped_init(hw_mgt); + nbl_uped_init(hw_mgt); + ret = nbl_shaping_init(hw_mgt, speed); + if (ret) + return ret; + nbl_dsch_qid_max_init(hw_mgt); + ret = nbl_ustore_init(hw_mgt, eth_num); + if (ret) + return ret; + nbl_dstore_init(hw_mgt, speed); + nbl_dvn_init(hw_mgt, speed); + nbl_uvn_init(hw_mgt); + nbl_uqm_init(hw_mgt); + return 0; +} + +static void nbl_host_padpt_init(struct nbl_hw_mgt hw_mgt) +{ + / padpt flow control register / + spin_lock(&hw_mgt->reg_lock); + nbl_hw_wr32(hw_mgt, NBL_HOST_PADPT_HOST_CFG_FC_CPLH_UP, + NBL_HOST_PADPT_CFG_FC_CPLH_UP_VAL); + nbl_hw_wr32(hw_mgt, NBL_HOST_PADPT_HOST_CFG_FC_PD_DN, + NBL_HOST_PADPT_CFG_FC_PD_DN_VAL); + nbl_hw_wr32(hw_mgt, NBL_HOST_PADPT_HOST_CFG_FC_PH_DN, + NBL_HOST_PADPT_CFG_FC_PH_DN_VAL); + nbl_hw_wr32(hw_mgt, NBL_HOST_PADPT_HOST_CFG_FC_NPH_DN, + NBL_HOST_PADPT_CFG_FC_NPH_DN_VAL); + spin_unlock(&hw_mgt->reg_lock); +} + +static void nbl_intf_init(struct nbl_hw_mgt hw_mgt) +{ + nbl_host_padpt_init(hw_mgt); +} + +/* + * Read-modify-write under reg_lock: NBL_DRIVER_STATUS_BIT is the only + * bit the driver owns, and the RMW writes back every other bit exactly + * as it read it, so firmware state in those bits survives. + * + * That only holds against concurrent CPU writers. reg_lock cannot + * serialise against firmware, and the register has no documented + * W1S/W1C form that would let the driver flip bit 16 without the + * read-modify-write; see NBL_DRIVER_STATUS_REG. + / +static void nbl_hw_set_driver_status(struct nbl_hw_mgt hw_mgt, bool active) +{ + u32 status; + + spin_lock(&hw_mgt->reg_lock); + status = nbl_hw_rd32(hw_mgt, NBL_DRIVER_STATUS_REG); + + status &= ~BIT(NBL_DRIVER_STATUS_BIT); + status \|= FIELD_PREP(BIT(NBL_DRIVER_STATUS_BIT), active); + + nbl_hw_wr32(hw_mgt, NBL_DRIVER_STATUS_REG, status); + spin_unlock(&hw_mgt->reg_lock); +} + +/* + * Setting driver status to false notifies firmware to clean up per-PF + * hardware state such as qinfo registers. + * + * Note: firmware does NOT automatically revert chip-wide registers + * configured in this init flow. Those chip-wide settings remain valid + * until chip reset or explicitly overwritten by driver. + * + * This deinit_module only clears driver active status and flush writes. + * It does NOT reset or restore chip-wide datapath registers. + * + * Firmware cleanup is asynchronous with no completion status register. + * The flush only guarantees the status write reached the device; the + * bounded delay after it (NBL_FW_CLEANUP_SYNC_MIN_US) is best-effort and + * does not guarantee that the firmware pass has finished. The ordering + * this relies on is the device-link one: the siblings are consumers of + * the control PF, so they are unbound first, and that link is created + * with DL_FLAG_AUTOPROBE_CONSUMER. AUTOPROBE only re-probes consumers, + * never the control PF itself, so a later init_module() needs an admin + * (or a driver rebind) to bind this PF again, and only then does the + * core re-probe the siblings. Both steps are far slower than the + * firmware pass, but there is no hard firmware handshake. + * + * Caller must ensure no new DMA is initiated after this point. + * The mailbox channel is stopped by nbl_chan_teardown_queue() + * before this function is called, so no in-flight mailbox DMA + * remains. + / +static void nbl_hw_deinit_module(struct nbl_hw_mgt hw_mgt) +{ + nbl_hw_set_driver_status(hw_mgt, false); + /* ensure driver_status reaches the chip / + nbl_flush_writes(hw_mgt); + / + * Give the asynchronous firmware cleanup pass time to settle + * before returning; best-effort, see NBL_FW_CLEANUP_SYNC_MIN_US. + / + usleep_range(NBL_FW_CLEANUP_SYNC_MIN_US, NBL_FW_CLEANUP_SYNC_MAX_US); +} + +static bool nbl_hw_eth_speed_valid(u8 speed) +{ + switch (speed) { + case NBL_FW_PORT_SPEED_10G: + case NBL_FW_PORT_SPEED_25G: + case NBL_FW_PORT_SPEED_50G: + case NBL_FW_PORT_SPEED_100G: + return true; + default: + return false; + } +} + +static bool nbl_hw_eth_num_valid(u8 eth_num) +{ + return eth_num == 1 \|\| eth_num == 2 \|\| eth_num == 4; +} + +/ + * Clear a stale driver_status active bit before this probe programs any + * per-PF state. + * + * A previous kernel instance can leave the bit set (kexec, or an unload + * that never reached deinit_module()) and firmware then never ran the + * per-PF cleanup it drives off the 1 -> 0 transition. That cleanup + * reclaims the mailbox QINFO routing and the qinfo registers, which is + * exactly what nbl_dev_setup_chan_qinfo() and nbl_chan_setup_queue() + * write later in this probe. + * + * So the ordering this has to guarantee is: clear the bit before any of + * that per-PF state is programmed, and before the mailbox queue is live + * so the cleanup cannot reclaim routing under an in-flight transfer. + * Clearing it later - once nbl_dev_init() has programmed the routing and + * enabled the queue - wipes those registers and leaves the inter-PF + * mailbox dead, so sibling PFs' RPCs time out in nbl_dev_start() and + * their probes fail. That is why init_module() must not do it. + * + * Only the control PF owns this chip-wide handshake. The bounded wait + * after the clear is best-effort, see NBL_FW_CLEANUP_SYNC_MIN_US. + / +static void nbl_hw_clear_stale_status(struct nbl_hw_mgt hw_mgt) +{ + u32 status; + + if (!hw_mgt->common->has_ctrl) + return; + + /* + * Unlocked read: this runs only from the serialized control-PF probe, + * and the driver is the only writer of NBL_DRIVER_STATUS_BIT. A stale + * sample at worst causes one redundant clear plus sleep; the + * authoritative update happens under reg_lock in + * nbl_hw_set_driver_status(). + / + status = nbl_hw_rd32(hw_mgt, NBL_DRIVER_STATUS_REG); + if (!(status & BIT(NBL_DRIVER_STATUS_BIT))) + return; + + dev_warn(hw_mgt->common->dev, + "driver_status already set, stale cleanup forced\n"); + nbl_hw_set_driver_status(hw_mgt, false); + nbl_flush_writes(hw_mgt); + usleep_range(NBL_FW_CLEANUP_SYNC_MIN_US, NBL_FW_CLEANUP_SYNC_MAX_US); +} + +/ + * Firmware brings the chip up at power-up, but the chip-wide datapath + * registers this programs (SHAPING, DSCH, USTORE/DSTORE, DVN/UVN, UQM, + * PADPT) are not part of that: the driver owns them from here until chip + * reset, and firmware does not revert them when driver_status goes low - + * see nbl_hw_deinit_module(). + * + * The driver_status handshake is entered from a known state: a stale + * active bit is cleared earlier, in nbl_hw_init_leonis(), before this + * probe programs any per-PF state. It must not be cleared here, because + * the mailbox QINFO routing and the mailbox queue registers are already + * programmed by then and the cleanup the 1 -> 0 transition starts would + * reclaim them; see nbl_hw_clear_stale_status(). + / +static int nbl_hw_init_module(struct nbl_hw_mgt hw_mgt, u8 eth_speed, + u8 eth_num) +{ + int ret; + + if (!nbl_hw_eth_speed_valid(eth_speed)) { + dev_err(hw_mgt->common->dev, "Invalid eth_speed %u\n", + eth_speed); + return -EINVAL; + } + if (!nbl_hw_eth_num_valid(eth_num)) { + dev_err(hw_mgt->common->dev, "Invalid eth_num %u\n", eth_num); + return -EINVAL; + } + + ret = nbl_dp_init(hw_mgt, eth_speed, eth_num); + if (ret) + return ret; + nbl_intf_init(hw_mgt); + nbl_hw_set_driver_status(hw_mgt, true); + /* ensure registers written / + nbl_flush_writes(hw_mgt); + + return 0; +} + +/ + * nbl_hw_set_mailbox_irq - read-modify-write NBL_MAILBOX_QINFO_MAP_REG_ARR + * + * The full RMW sequence is wrapped by reg_lock, so concurrent register + * access from different CPUs is already serialized safely. + * nbl_hw_cfg_mailbox_qinfo() programs the BDF fields during control-PF + * init and clears MSIX_IDX/MSIX_IDX_VALID at the same time (they survive + * kexec/forced unload without FLR), so mailbox MSIX routing for a PF + * starts disarmed at init and is armed only by an explicit en_msix=true + * call here. + / +static void nbl_hw_set_mailbox_irq(struct nbl_hw_mgt hw_mgt, u16 func_id, + bool en_msix, u16 gvec) +{ + u32 data = 0; + + spin_lock(&hw_mgt->reg_lock); + nbl_hw_rd_regs(hw_mgt, NBL_MAILBOX_QINFO_MAP_REG_ARR(func_id), &data, + sizeof(data)); + data &= ~(NBL_MAILBOX_QINFO_MAP_MSIX_IDX_MASK \| + NBL_MAILBOX_QINFO_MAP_MSIX_IDX_VALID_MASK); + if (en_msix) + data \|= FIELD_PREP(NBL_MAILBOX_QINFO_MAP_MSIX_IDX_MASK, + gvec) \| + FIELD_PREP(NBL_MAILBOX_QINFO_MAP_MSIX_IDX_VALID_MASK, + 1); + + nbl_hw_wr_regs(hw_mgt, NBL_MAILBOX_QINFO_MAP_REG_ARR(func_id), &data, + sizeof(data)); + spin_unlock(&hw_mgt->reg_lock); + nbl_flush_writes(hw_mgt); +} + +static void nbl_hw_cfg_msix_map(struct nbl_hw_mgt hw_mgt, u16 func_id, + bool valid, dma_addr_t dma_addr, u8 bus, + u8 devid, u8 function) +{ + struct nbl_function_msix_map function_msix_map; + + memset(&function_msix_map, 0, sizeof(function_msix_map)); + + / + * Clear VALID on its own first. nbl_hw_wr_regs_lock() writes an + * entry in ascending dword order, so a single whole-entry write + * would publish the new (or zeroed) address in data[0]/data[1] + * while the old VALID bit in data[2] was still set, exposing a + * valid entry with a half-written address to the pcompleter's + * table fetch. With VALID already zero only the address words + * change, and no fetch can be started from a torn entry. + / + nbl_hw_wr_regs_lock(hw_mgt, + NBL_PCOMPLETER_FUNCTION_MSIX_MAP(func_id) + + NBL_FUNC_MSIX_MAP_VALID_DW sizeof(u32), + &function_msix_map.data[NBL_FUNC_MSIX_MAP_VALID_DW], + sizeof(u32)); + + if (valid) { + /* program full entry and set VALID / + function_msix_map.data[0] = lower_32_bits(dma_addr); + function_msix_map.data[1] = upper_32_bits(dma_addr); + function_msix_map.data[2] = + FIELD_PREP(NBL_FUNCTION_MSIX_MAP_FUNCTION_MASK, + function) \| + FIELD_PREP(NBL_FUNCTION_MSIX_MAP_DEVID_MASK, devid) \| + FIELD_PREP(NBL_FUNCTION_MSIX_MAP_BUS_MASK, bus) \| + FIELD_PREP(NBL_FUNCTION_MSIX_MAP_VALID_MASK, 1); + + nbl_hw_wr_regs_lock(hw_mgt, + NBL_PCOMPLETER_FUNCTION_MSIX_MAP(func_id), + function_msix_map.data, + sizeof(function_msix_map)); + } else { + / + * reg_lock prevents concurrent CPU writes to the same + * function's MSIX entry, but cannot synchronize hardware DMA + * reads. Upper layer uses two-stage destruction + sync sleep + * to avoid torn hardware read of partial MSIX entry. + * VALID is already cleared above, so the address written + * here is either the still-live table (prepare) or zero + * (complete); data[2] stays zero, leaving the BDF fields + * unused because the entry cannot be fetched. + / + function_msix_map.data[0] = lower_32_bits(dma_addr); + function_msix_map.data[1] = upper_32_bits(dma_addr); + + nbl_hw_wr_regs_lock(hw_mgt, + NBL_PCOMPLETER_FUNCTION_MSIX_MAP(func_id), + function_msix_map.data, + sizeof(function_msix_map)); + } +} + +static void nbl_hw_cfg_msix_info(struct nbl_hw_mgt hw_mgt, u16 func_id, + bool valid, u16 interrupt_id, u8 bus, + u8 devid, u8 function, bool msix_mask_en) +{ + u32 host_msix_fid = 0; + struct nbl_host_msix_info msix_info; + + memset(&msix_info, 0, sizeof(msix_info)); + if (valid) { + host_msix_fid = + FIELD_PREP(NBL_PCOMPLETER_HOST_MSIX_FID_TABLE_FID_MASK, + func_id) \| + FIELD_PREP(NBL_PCOMPLETER_HOST_MSIX_FID_TABLE_VLD_MASK, + 1); + + msix_info.data[1] = + FIELD_PREP(NBL_HOST_MSIX_INFO_FUNCTION_MASK, function) \| + FIELD_PREP(NBL_HOST_MSIX_INFO_DEVID_MASK, devid) \| + FIELD_PREP(NBL_HOST_MSIX_INFO_BUS_MASK, bus) \| + FIELD_PREP(NBL_HOST_MSIX_INFO_VALID_MASK, 1); + + if (msix_mask_en) + msix_info.data[1] \|= + FIELD_PREP(NBL_HOST_MSIX_INFO_MSIX_MASK_EN_MASK, 1); + } + spin_lock(&hw_mgt->reg_lock); + /* + * Programming order rule: + * Enable: PADPT_HOST_MSIX_INFO -> PCOMPLETER_HOST_MSIX_FID_TABLE + * Teardown: reverse order, clear FID VLD first to avoid inconsistent + * state + / + if (valid) { + nbl_hw_wr_regs(hw_mgt, + NBL_PADPT_HOST_MSIX_INFO_REG_ARR(interrupt_id), + msix_info.data, sizeof(msix_info)); + nbl_hw_wr_regs(hw_mgt, + NBL_PCOMPLETER_HOST_MSIX_FID_TABLE(interrupt_id), + &host_msix_fid, sizeof(host_msix_fid)); + } else { + nbl_hw_wr_regs(hw_mgt, + NBL_PCOMPLETER_HOST_MSIX_FID_TABLE(interrupt_id), + &host_msix_fid, sizeof(host_msix_fid)); + nbl_hw_wr_regs(hw_mgt, + NBL_PADPT_HOST_MSIX_INFO_REG_ARR(interrupt_id), + msix_info.data, sizeof(msix_info)); + } + spin_unlock(&hw_mgt->reg_lock); +} + +static void nbl_hw_update_mailbox_queue_tail_ptr(struct nbl_hw_mgt hw_mgt, + u16 tail_ptr, u8 txrx) +{ + /* local_qid 0 and 1 denote rx and tx queue respectively / + u32 local_qid = txrx; + u32 value = ((u32)tail_ptr << 16) \| local_qid; + + / wmb for doorbell / + wmb(); + nbl_mbx_wr32(hw_mgt, NBL_MAILBOX_NOTIFY_ADDR, value); +} + +/ + * Assert QUEUE_RST on a QINFO cfg entry, in a write of its own. + * + * The entry is four dwords and the device may act on each one as it + * lands, while data[3] carries QUEUE_RST/QUEUE_EN and data[0..2] carry + * the ring base and size. Writing a new base while QUEUE_EN is still + * set from the previous configuration would have the queue fetch at a + * half-updated address - IOVA 0, once data[0..2] have been cleared - + * so the reset always goes in first, on its own, and the remaining + * dwords are written afterwards. This also covers a device that only + * commits the entry when data[3] lands. + / +static void nbl_hw_reset_mailbox_qinfo(struct nbl_hw_mgt hw_mgt, u64 reg) +{ + u32 rst = FIELD_PREP(NBL_MAILBOX_QINFO_CFG_QUEUE_RST_MASK, 1); + + nbl_hw_write_mbx_regs(hw_mgt, + reg + (MAILBOX_QINFO_CFG_TABLE_DWLEN - 1) * + sizeof(u32), + &rst, sizeof(rst)); +} + +static void nbl_hw_config_mailbox_rxq(struct nbl_hw_mgt hw_mgt, + dma_addr_t dma_addr, int size_bwid) +{ + struct nbl_mailbox_qinfo_cfg_table cfg_tbl; + + / Hold the queue in reset before its base/size are reprogrammed / + nbl_hw_reset_mailbox_qinfo(hw_mgt, NBL_MAILBOX_QINFO_CFG_RX_TABLE_ADDR); + + cfg_tbl.data[0] = lower_32_bits(dma_addr); + cfg_tbl.data[1] = upper_32_bits(dma_addr); + cfg_tbl.data[2] = FIELD_PREP(NBL_MAILBOX_QINFO_CFG_QUEUE_SIZE_BWID_MASK, + size_bwid); + cfg_tbl.data[3] = FIELD_PREP(NBL_MAILBOX_QINFO_CFG_QUEUE_RST_MASK, 0) \| + FIELD_PREP(NBL_MAILBOX_QINFO_CFG_QUEUE_EN_MASK, 1); + / + * data[0..2] land while QUEUE_RST is still set from the write above; + * data[3] is written last and starts the queue on the new ring. + / + nbl_hw_write_mbx_regs(hw_mgt, NBL_MAILBOX_QINFO_CFG_RX_TABLE_ADDR, + cfg_tbl.data, sizeof(cfg_tbl)); +} + +static void nbl_hw_config_mailbox_txq(struct nbl_hw_mgt hw_mgt, + dma_addr_t dma_addr, int size_bwid) +{ + struct nbl_mailbox_qinfo_cfg_table cfg_tbl; + + /* Hold the queue in reset before its base/size are reprogrammed / + nbl_hw_reset_mailbox_qinfo(hw_mgt, NBL_MAILBOX_QINFO_CFG_TX_TABLE_ADDR); + + cfg_tbl.data[0] = lower_32_bits(dma_addr); + cfg_tbl.data[1] = upper_32_bits(dma_addr); + cfg_tbl.data[2] = FIELD_PREP(NBL_MAILBOX_QINFO_CFG_QUEUE_SIZE_BWID_MASK, + size_bwid); + cfg_tbl.data[3] = FIELD_PREP(NBL_MAILBOX_QINFO_CFG_QUEUE_RST_MASK, 0) \| + FIELD_PREP(NBL_MAILBOX_QINFO_CFG_QUEUE_EN_MASK, 1); + / + * data[0..2] land while QUEUE_RST is still set from the write above; + * data[3] is written last and starts the queue on the new ring. + / + nbl_hw_write_mbx_regs(hw_mgt, NBL_MAILBOX_QINFO_CFG_TX_TABLE_ADDR, + cfg_tbl.data, sizeof(cfg_tbl)); +} + +static void nbl_hw_stop_mailbox_rxq(struct nbl_hw_mgt hw_mgt) +{ + struct nbl_mailbox_qinfo_cfg_table cfg_tbl; + + /* + * Assert QUEUE_RST before data[0..2] are cleared: writing the + * zeroed entry in one go would clear the ring base while QUEUE_EN + * is still set. The reset also clears QUEUE_EN, so what follows + * lands on an already stopped queue. + / + nbl_hw_reset_mailbox_qinfo(hw_mgt, NBL_MAILBOX_QINFO_CFG_RX_TABLE_ADDR); + + / Now drop the stale base and size, keeping QUEUE_RST asserted / + memset(&cfg_tbl, 0, sizeof(cfg_tbl)); + cfg_tbl.data[3] = FIELD_PREP(NBL_MAILBOX_QINFO_CFG_QUEUE_RST_MASK, 1); + nbl_hw_write_mbx_regs(hw_mgt, NBL_MAILBOX_QINFO_CFG_RX_TABLE_ADDR, + cfg_tbl.data, sizeof(cfg_tbl)); + / Ensure QUEUE_RST has reached the device before caller proceeds / + nbl_hw_flush_mbx_write(hw_mgt, + NBL_MAILBOX_QINFO_CFG_RX_TABLE_ADDR); +} + +static void nbl_hw_stop_mailbox_txq(struct nbl_hw_mgt hw_mgt) +{ + struct nbl_mailbox_qinfo_cfg_table cfg_tbl; + + /* + * Assert QUEUE_RST before data[0..2] are cleared: writing the + * zeroed entry in one go would clear the ring base while QUEUE_EN + * is still set. The reset also clears QUEUE_EN, so what follows + * lands on an already stopped queue. + / + nbl_hw_reset_mailbox_qinfo(hw_mgt, NBL_MAILBOX_QINFO_CFG_TX_TABLE_ADDR); + + / Now drop the stale base and size, keeping QUEUE_RST asserted / + memset(&cfg_tbl, 0, sizeof(cfg_tbl)); + cfg_tbl.data[3] = FIELD_PREP(NBL_MAILBOX_QINFO_CFG_QUEUE_RST_MASK, 1); + nbl_hw_write_mbx_regs(hw_mgt, NBL_MAILBOX_QINFO_CFG_TX_TABLE_ADDR, + cfg_tbl.data, sizeof(cfg_tbl)); + / Ensure QUEUE_RST has reached the device before caller proceeds / + nbl_hw_flush_mbx_write(hw_mgt, + NBL_MAILBOX_QINFO_CFG_TX_TABLE_ADDR); +} + +static void nbl_hw_get_host_pf_mask(struct nbl_hw_mgt hw_mgt, u32 pf_mask) +{ + nbl_hw_rd_regs_lock(hw_mgt, NBL_PCIE_HOST_K_PF_MASK_REG, pf_mask, + sizeof(pf_mask)); +} + +static void nbl_hw_get_real_bus(struct nbl_hw_mgt hw_mgt, u8 bus) +{ + u32 data = 0; + + nbl_hw_rd_regs_lock(hw_mgt, NBL_PCIE_HOST_TL_CFG_BUSDEV, &data, + sizeof(data)); + bus = FIELD_GET(NBL_PCIE_BUS_MASK, data); +} + +static void nbl_hw_cfg_mailbox_qinfo(struct nbl_hw_mgt hw_mgt, u16 func_id, + u8 bus, u8 devid, u8 function) +{ + u32 data = 0; + + spin_lock(&hw_mgt->reg_lock); + nbl_hw_rd_regs(hw_mgt, NBL_MAILBOX_QINFO_MAP_REG_ARR(func_id), + &data, sizeof(data)); + data &= ~(NBL_MAILBOX_QINFO_MAP_FUNCTION_MASK \| + NBL_MAILBOX_QINFO_MAP_DEVID_MASK \| + NBL_MAILBOX_QINFO_MAP_BUS_MASK \| + NBL_MAILBOX_QINFO_MAP_MSIX_IDX_MASK \| + NBL_MAILBOX_QINFO_MAP_MSIX_IDX_VALID_MASK); + data \|= FIELD_PREP(NBL_MAILBOX_QINFO_MAP_FUNCTION_MASK, function) \| + FIELD_PREP(NBL_MAILBOX_QINFO_MAP_DEVID_MASK, devid) \| + FIELD_PREP(NBL_MAILBOX_QINFO_MAP_BUS_MASK, bus); + nbl_hw_wr_regs(hw_mgt, NBL_MAILBOX_QINFO_MAP_REG_ARR(func_id), + &data, sizeof(data)); + spin_unlock(&hw_mgt->reg_lock); +} + +/* + * Registers reset to zero after cold boot / FLR / bus reset. Firmware + * programs valid values before driver probe, so zero is only seen on + * hardware fault or register read failure. Initialize data=0 to guard + * against nbl_hw_read_mbx_regs() early-return on bounds-check failure. + / +static void nbl_hw_get_board_info(struct nbl_hw_mgt hw_mgt, + struct nbl_board_port_info board_info) +{ + u32 data = 0; + + nbl_hw_read_mbx_regs(hw_mgt, NBL_FW_BOARD_DW3_OFFSET, &data, + sizeof(data)); + board_info->eth_num = FIELD_GET(NBL_FW_BOARD_DW3_PORT_NUM_MASK, data); + board_info->eth_speed = + FIELD_GET(NBL_FW_BOARD_DW3_PORT_SPEED_MASK, data); + board_info->p4_version = + FIELD_GET(NBL_FW_BOARD_DW3_P4_VERSION_MASK, data); +} + +static struct nbl_hw_ops hw_ops = { + .init_module = nbl_hw_init_module, + .deinit_module = nbl_hw_deinit_module, + + .cfg_msix_map = nbl_hw_cfg_msix_map, + .cfg_msix_info = nbl_hw_cfg_msix_info, + .flush_write = nbl_flush_writes, + + .update_mailbox_queue_tail_ptr = nbl_hw_update_mailbox_queue_tail_ptr, + .config_mailbox_rxq = nbl_hw_config_mailbox_rxq, + .config_mailbox_txq = nbl_hw_config_mailbox_txq, + .stop_mailbox_rxq = nbl_hw_stop_mailbox_rxq, + .stop_mailbox_txq = nbl_hw_stop_mailbox_txq, + .get_host_pf_mask = nbl_hw_get_host_pf_mask, + .get_real_bus = nbl_hw_get_real_bus, + + .cfg_mailbox_qinfo = nbl_hw_cfg_mailbox_qinfo, + .set_mailbox_irq = nbl_hw_set_mailbox_irq, + + .get_fw_eth_map = nbl_hw_get_fw_eth_map, + .get_board_info = nbl_hw_get_board_info, +}; + / Structure starts here, adding an op should not modify anything below / static struct nbl_hw_mgt nbl_hw_setup_hw_mgt(struct nbl_common_info common) { @@ -23,6 +1246,31 @@ static struct nbl_hw_mgt nbl_hw_setup_hw_mgt(struct nbl_common_info common) return hw_mgt; } +static struct nbl_hw_ops_tbl nbl_hw_setup_ops(struct nbl_common_info common, + struct nbl_hw_mgt hw_mgt) +{ + struct nbl_hw_ops_tbl hw_ops_tbl; + struct device dev; + + dev = common->dev; + hw_ops_tbl = devm_kzalloc(dev, sizeof(hw_ops_tbl), GFP_KERNEL); + if (!hw_ops_tbl) + return ERR_PTR(-ENOMEM); + if (!hw_ops.cfg_msix_map \|\| !hw_ops.cfg_msix_info \|\| + !hw_ops.flush_write \|\| !hw_ops.update_mailbox_queue_tail_ptr \|\| + !hw_ops.config_mailbox_rxq \|\| !hw_ops.config_mailbox_txq \|\| + !hw_ops.stop_mailbox_rxq \|\| !hw_ops.stop_mailbox_txq \|\| + !hw_ops.get_host_pf_mask \|\| !hw_ops.get_real_bus \|\| + !hw_ops.cfg_mailbox_qinfo \|\| !hw_ops.set_mailbox_irq \|\| + !hw_ops.get_fw_eth_map \|\| !hw_ops.get_board_info \|\| + !hw_ops.init_module \|\| !hw_ops.deinit_module) + return ERR_PTR(-EINVAL); + hw_ops_tbl->ops = &hw_ops; + hw_ops_tbl->priv = hw_mgt; + + return hw_ops_tbl; +} + static int nbl_pcim_request_selected_bars(struct pci_dev pdev, u32 mask, const char name) { @@ -43,6 +1291,7 @@ int nbl_hw_init_leonis(struct nbl_adapter adapter) { resource_size_t expect_sz = NBL_MEM_BAR_TOTAL_SIZE; struct nbl_common_info common = &adapter->common; + struct nbl_hw_ops_tbl hw_ops_tbl = NULL; struct pci_dev pdev = common->pdev; struct nbl_hw_mgt hw_mgt = NULL; resource_size_t bar_len; @@ -53,7 +1302,7 @@ int nbl_hw_init_leonis(struct nbl_adapter adapter) hw_mgt = nbl_hw_setup_hw_mgt(common); if (IS_ERR(hw_mgt)) { ret = PTR_ERR(hw_mgt); - goto setup_mgt_fail; + return ret; } bar_mask = BIT(NBL_MEMORY_BAR) \| BIT(NBL_MAILBOX_BAR); ret = nbl_pcim_request_selected_bars(pdev, bar_mask, NBL_DRIVER_NAME); @@ -61,14 +1310,14 @@ int nbl_hw_init_leonis(struct nbl_adapter adapter) dev_err(&pdev->dev, "Request memory bar failed, err = %d\n", ret); - goto setup_mgt_fail; + return ret; } bar_len = pci_resource_len(pdev, NBL_MEMORY_BAR); if (!(pci_resource_flags(pdev, NBL_MEMORY_BAR) & IORESOURCE_MEM)) { dev_err(&pdev->dev, "MEMORY BAR is not memory resource\n"); ret = -EINVAL; - goto setup_mgt_fail; + return ret; } if (common->has_ctrl) { /* @@ -90,7 +1339,7 @@ int nbl_hw_init_leonis(struct nbl_adapter adapter) "MEMORY BAR len %pa smaller than expected %pa\n", &bar_len, &expect_sz); ret = -EINVAL; - goto setup_mgt_fail; + return ret; } hw_size = PAGE_ALIGN_DOWN(NBL_MEM_BAR_TOTAL_SIZE - NBL_RDMA_NOTIFY_LEN); @@ -103,7 +1352,7 @@ int nbl_hw_init_leonis(struct nbl_adapter adapter) "MEMORY BAR len %pa too small for net only reg space\n", &bar_len); ret = -EINVAL; - goto setup_mgt_fail; + return ret; } hw_size = NBL_REG_NET_ONLY_LEN; hw_mgt->hw_addr = pcim_iomap(pdev, NBL_MEMORY_BAR, @@ -112,37 +1361,50 @@ int nbl_hw_init_leonis(struct nbl_adapter adapter) if (!hw_mgt->hw_addr) { dev_err(&pdev->dev, "MEMORY BAR pcim_iomap failed\n"); ret = -EIO; - goto setup_mgt_fail; + return ret; } bar_len = pci_resource_len(pdev, NBL_MAILBOX_BAR); if (!(pci_resource_flags(pdev, NBL_MAILBOX_BAR) & IORESOURCE_MEM)) { dev_err(&pdev->dev, "MAILBOX BAR is not memory resource\n"); ret = -EINVAL; - goto setup_mgt_fail; + return ret; } if (bar_len < NBL_BAR2_MAX_LEN) { dev_err(&pdev->dev, "MAILBOX BAR length %pa too small\n", &bar_len); ret = -EINVAL; - goto setup_mgt_fail; + return ret; } hw_mgt->mailbox_bar_hw_addr = pcim_iomap(pdev, NBL_MAILBOX_BAR, bar_len); if (!hw_mgt->mailbox_bar_hw_addr) { dev_err(&pdev->dev, "MAILBOX BAR pcim_iomap failed\n"); ret = -EIO; - goto setup_mgt_fail; + return ret; } hw_mgt->mailbox_bar_size = bar_len; + spin_lock_init(&hw_mgt->reg_lock); + / + * Clear a stale driver_status now, while this is the only register + * access this probe has done: the firmware cleanup the 1 -> 0 + * transition starts reclaims the mailbox QINFO routing and qinfo + * registers that nbl_dev_init() programs later, so it must not be + * deferred to init_module(); see nbl_hw_clear_stale_status(). + / + nbl_hw_clear_stale_status(hw_mgt); + + hw_ops_tbl = nbl_hw_setup_ops(common, hw_mgt); + if (IS_ERR(hw_ops_tbl)) { + ret = PTR_ERR(hw_ops_tbl); + return ret; + } + adapter->intf.hw_ops_tbl = hw_ops_tbl; adapter->core.hw_mgt = hw_mgt; return 0; - -setup_mgt_fail: - return ret; } void nbl_hw_remove_leonis(struct nbl_adapter adapter) diff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_leonis/nbl_hw_leonis.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_leonis/nbl_hw_leonis.h index 1f9e509dc6312f..7d27ab19838e21 100644 --- a/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_leonis/nbl_hw_leonis.h +++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_leonis/nbl_hw_leonis.h @@ -11,5 +11,347 @@ #include "../../nbl_include/nbl_include.h" #include "../nbl_hw_reg.h" +/* + * Chip-wide driver-present handshake with firmware. + * + * NBL_DRIVER_STATUS_BIT is the only bit the driver owns: setting it tells + * firmware that a driver is bound, and the 1 -> 0 transition starts the + * firmware pass that reclaims per-PF state (mailbox QINFO routing, qinfo + * registers). Every other bit of this register belongs to firmware and + * the driver must leave it as it read it. + * + * Because the register is shared, bit 16 can only be changed with a + * read-modify-write: it has no documented W1S/W1C semantics that would + * let the driver set or clear that one bit atomically. reg_lock makes + * the read-modify-write atomic with respect to other CPU writers, but it + * cannot serialise against firmware, so a firmware update to another bit + * landing between the read and the write would be written back stale. + * No such firmware update is known to race the three driver callers + * (the probe-time clear, init_module() and deinit_module()), but that is + * an observation about this hardware revision, not a guarantee the + * driver can enforce. + / +#define NBL_DRIVER_STATUS_REG 0x1300444 +#define NBL_DRIVER_STATUS_BIT 16 + +/ ---------- REG BASE ADDR ---------- / +/ Interface modules base addr / +#define NBL_INTF_HOST_PCOMPLETER_BASE 0x00f08000 +#define NBL_INTF_HOST_PADPT_BASE 0x00f4c000 +#define NBL_INTF_HOST_MAILBOX_BASE 0x00fb0000 +#define NBL_INTF_HOST_PCIE_BASE 0X01504000 +/ DP modules base addr / +#define NBL_DP_USTORE_BASE 0x00104000 +#define NBL_DP_UQM_BASE 0x00114000 +#define NBL_DP_UPED_BASE 0x0015c000 +#define NBL_DP_UVN_BASE 0x00244000 +#define NBL_DP_DSCH_BASE 0x00404000 +#define NBL_DP_SHAPING_BASE 0x00504000 +#define NBL_DP_DVN_BASE 0x00514000 +#define NBL_DP_DSTORE_BASE 0x00704000 +#define NBL_DP_DPED_BASE 0x0075c000 +/ -------- MAILBOX BAR2 ----- / +#define NBL_MAILBOX_NOTIFY_ADDR 0x00000000 +#define NBL_MAILBOX_QINFO_CFG_RX_TABLE_ADDR 0x10 +#define NBL_MAILBOX_QINFO_CFG_TX_TABLE_ADDR 0x20 + +/ -------- MAILBOX -------- / + +/ mailbox BAR qinfo_cfg_table / +#define MAILBOX_QINFO_CFG_TABLE_DWLEN 4 +/ data[2] / +#define NBL_MAILBOX_QINFO_CFG_QUEUE_SIZE_BWID_MASK GENMASK(3, 0) +/ data[3] / +#define NBL_MAILBOX_QINFO_CFG_QUEUE_RST_MASK BIT(0) +#define NBL_MAILBOX_QINFO_CFG_QUEUE_EN_MASK BIT(1) +#define NBL_MAILBOX_QINFO_CFG_DIF_ERR_MASK BIT(2) +#define NBL_MAILBOX_QINFO_CFG_PTR_ERR_MASK BIT(3) +struct nbl_mailbox_qinfo_cfg_table { + u32 data[MAILBOX_QINFO_CFG_TABLE_DWLEN]; +}; + +/ -------- MAILBOX BAR0 ----- / +/ mailbox qinfo_map_table / +#define NBL_MAILBOX_QINFO_MAP_REG_ARR(func_id) \ + (NBL_INTF_HOST_MAILBOX_BASE + 0x00001000 + (func_id) sizeof(u32)) + +/* MAILBOX qinfo_map_table / +#define NBL_MAILBOX_QINFO_MAP_FUNCTION_MASK GENMASK(2, 0) +#define NBL_MAILBOX_QINFO_MAP_DEVID_MASK GENMASK(7, 3) +#define NBL_MAILBOX_QINFO_MAP_BUS_MASK GENMASK(15, 8) +#define NBL_MAILBOX_QINFO_MAP_MSIX_IDX_MASK GENMASK(28, 16) +#define NBL_MAILBOX_QINFO_MAP_MSIX_IDX_VALID_MASK BIT(29) + +/ -------- HOST_PCIE -------- / +#define NBL_PCIE_HOST_K_PF_MASK_REG (NBL_INTF_HOST_PCIE_BASE + 0x00001004) +#define NBL_PCIE_HOST_TL_CFG_BUSDEV (NBL_INTF_HOST_PCIE_BASE + 0x11040) + +#define NBL_PCIE_BUS_MASK GENMASK(12, 5) + +/ -------- HOST_PADPT -------- / +#define NBL_HOST_PADPT_HOST_CFG_FC_PD_DN (NBL_INTF_HOST_PADPT_BASE + 0x00000160) +#define NBL_HOST_PADPT_HOST_CFG_FC_PH_DN (NBL_INTF_HOST_PADPT_BASE + 0x00000164) +#define NBL_HOST_PADPT_HOST_CFG_FC_NPH_DN \ + (NBL_INTF_HOST_PADPT_BASE + 0x0000016C) +#define NBL_HOST_PADPT_HOST_CFG_FC_CPLH_UP \ + (NBL_INTF_HOST_PADPT_BASE + 0x00000170) + +#define NBL_HOST_PADPT_CFG_FC_CPLH_UP_VAL 0x10400 +#define NBL_HOST_PADPT_CFG_FC_PD_DN_VAL 0x10080 +#define NBL_HOST_PADPT_CFG_FC_PH_DN_VAL 0x10010 +#define NBL_HOST_PADPT_CFG_FC_NPH_DN_VAL 0x10010 +/ host_padpt host_msix_info / +#define NBL_PADPT_HOST_MSIX_INFO_REG_ARR(vector_id) \ + (NBL_INTF_HOST_PADPT_BASE + 0x00010000 + \ + (vector_id) sizeof(struct nbl_host_msix_info)) + +#define NBL_HOST_MSIX_INFO_DWLEN 2 +/* data[0] / +#define NBL_HOST_MSIX_INFO_INTRL_PNUM_MASK GENMASK(15, 0) +#define NBL_HOST_MSIX_INFO_INTRL_RATE_MASK GENMASK(31, 16) +/ data[1] / +#define NBL_HOST_MSIX_INFO_FUNCTION_MASK GENMASK(2, 0) +#define NBL_HOST_MSIX_INFO_DEVID_MASK GENMASK(7, 3) +#define NBL_HOST_MSIX_INFO_BUS_MASK GENMASK(15, 8) +#define NBL_HOST_MSIX_INFO_VALID_MASK BIT(16) +#define NBL_HOST_MSIX_INFO_MSIX_MASK_EN_MASK BIT(17) +struct nbl_host_msix_info { + u32 data[NBL_HOST_MSIX_INFO_DWLEN]; +}; + +/ -------- HOST_PCOMPLETER -------- / +/ pcompleter_host function_msix_map_table / +#define NBL_PCOMPLETER_FUNCTION_MSIX_MAP(i) \ + (NBL_INTF_HOST_PCOMPLETER_BASE + 0x00004000 + \ + (i) sizeof(struct nbl_function_msix_map)) +#define NBL_PCOMPLETER_HOST_MSIX_FID_TABLE(i) \ + (NBL_INTF_HOST_PCOMPLETER_BASE + 0x0003a000 + (i) * sizeof(u32)) + +#define NBL_PCOMPLETER_HOST_MSIX_FID_TABLE_FID_MASK GENMASK(9, 0) +#define NBL_PCOMPLETER_HOST_MSIX_FID_TABLE_VLD_MASK BIT(10) + +#define NBL_FUNC_MSIX_MAP_DWLEN 4 +/* The VLD/BDF fields live in data[2]; it can be cleared on its own / +#define NBL_FUNC_MSIX_MAP_VALID_DW 2 +/ data[2] / +#define NBL_FUNCTION_MSIX_MAP_FUNCTION_MASK GENMASK(2, 0) +#define NBL_FUNCTION_MSIX_MAP_DEVID_MASK GENMASK(7, 3) +#define NBL_FUNCTION_MSIX_MAP_BUS_MASK GENMASK(15, 8) +#define NBL_FUNCTION_MSIX_MAP_VALID_MASK BIT(16) +struct nbl_function_msix_map { + u32 data[NBL_FUNC_MSIX_MAP_DWLEN]; +}; + +/ ---------- DPED ---------- / +#define NBL_DPED_VLAN_OFFSET (NBL_DP_DPED_BASE + 0x000003F4) +#define NBL_DPED_DSCP_OFFSET_0 (NBL_DP_DPED_BASE + 0x000003F8) +#define NBL_DPED_DSCP_OFFSET_1 (NBL_DP_DPED_BASE + 0x000003FC) +/ DPED hw_edt_prof/ UPED hw_edt_prof / + +#define NBL_DPED_L4_CK_CMD_40_ADDR (NBL_DP_DPED_BASE + 0x00000338) + +#define NBL_DPED_L4_CK_CMD_40_EN_MASK BIT(31) + +/ ---------- UPED ---------- / +/ UPED uped_hw_edt_prof / +#define NBL_UPED_HW_EDT_PROF_TABLE(i) \ + (NBL_DP_UPED_BASE + 0x00001000 + (i) sizeof(u32)) +#define NBL_UPED_V4_TCP_IDX 5 +#define NBL_UPED_V6_TCP_IDX 6 +#define NBL_PED_HW_EDIT_PROFILE_L3_LEN_MASK GENMASK(3, 2) + +/* ---------- DSCH ---------- / +#define NBL_DSCH_PSHA_EN_MASK GENMASK(3, 0) +/ DSCH dsch maxqid / +#define NBL_DSCH_HOST_QID_MAX (NBL_DP_DSCH_BASE + 0x00000118) +#define NBL_DSCH_HOST_QID_MAX_MASK GENMASK(10, 0) +#define NBL_DSCH_VN_QUANTA_ADDR (NBL_DP_DSCH_BASE + 0x00000134) + +#define NBL_MAX_QUEUE_ID 0x7ff +#define NBL_HOST_QUANTA 0x8000 +#define NBL_ECPU_QUANTA 0x1000 + +#define NBL_DSCH_VN_QUANTA_H_QUA_MASK GENMASK(15, 0) +#define NBL_DSCH_VN_QUANTA_E_QUA_MASK GENMASK(31, 16) + +/ ---------- DVN ---------- / +/ DVN dvn_queue_table / +#define NBL_DVN_ECPU_QUEUE_NUM (NBL_DP_DVN_BASE + 0x0000041C) +#define NBL_DVN_DESCREQ_NUM_CFG (NBL_DP_DVN_BASE + 0x00000430) +#define NBL_DVN_DESC_WR_MERGE_TIMEOUT (NBL_DP_DVN_BASE + 0x00000480) +#define NBL_DVN_DIF_REQ_RD_RO_FLAG (NBL_DP_DVN_BASE + 0x0000045C) + +#define DEFAULT_DVN_DESCREQ_NUMCFG 0x03 +#define DEFAULT_DVN_100G_DESCREQ_NUMCFG 0x07 + +#define DEFAULT_DVN_DESC_WR_MERGE_TIMEOUT_MAX 0x3FF + +/ spilit ring descreq_num 0:8,1:16 / +#define NBL_DVN_DESCREQ_NUM_CFG_AVRING_DESREQ_NUM_CFG_MASK BIT(0) +/ packet ring descreq_num + * 0:8,1:12,2:16;3:20,4:24,5:26;6:32,7:32 + / +#define NBL_DVN_DESCREQ_NUM_CFG_PACKED_L1_NUM_MASK GENMASK(6, 4) + +#define NBL_DVN_DESC_WR_MERGE_TIMEOUT_CFG_CYCLE_MASK GENMASK(9, 0) + +#define NBL_DVN_DIF_REQ_RD_RO_FLAG_DESC_RO_EN_MASK BIT(0) +#define NBL_DVN_DIF_REQ_RD_RO_FLAG_DATA_RO_EN_MASK BIT(1) +#define NBL_DVN_DIF_REQ_RD_RO_FLAG_AVRING_RO_EN_MASK BIT(2) + +/ ---------- UVN ---------- / +/ UVN uvn_queue_table / + +#define NBL_UVN_DESC_RD_WAIT (NBL_DP_UVN_BASE + 0x0000020C) +#define NBL_UVN_QUEUE_ERR_MASK (NBL_DP_UVN_BASE + 0x00000224) +#define NBL_UVN_ECPU_QUEUE_NUM (NBL_DP_UVN_BASE + 0x0000023C) +#define NBL_UVN_DESC_WR_TIMEOUT (NBL_DP_UVN_BASE + 0x00000214) +#define NBL_UVN_DIF_REQ_RO_FLAG (NBL_DP_UVN_BASE + 0x00000250) +#define NBL_UVN_DESC_PREFETCH_INIT (NBL_DP_UVN_BASE + 0x00000204) +#define NBL_UVN_DESC_PREFETCH_NUM 4 + +#define NBL_UVN_DIF_REQ_RO_FLAG_AVAIL_RD_MASK BIT(0) +#define NBL_UVN_DIF_REQ_RO_FLAG_DESC_RD_MASK BIT(1) +#define NBL_UVN_DIF_REQ_RO_FLAG_PKT_WR_MASK BIT(2) +#define NBL_UVN_DIF_REQ_RO_FLAG_DESC_WR_MASK BIT(3) + +#define NBL_UVN_DESC_WR_TIMEOUT_NUM_MASK GENMASK(14, 0) + +#define NBL_UVN_QUEUE_ERR_MASK_DIF_ERR_MASK BIT(5) + +#define NBL_UVN_DESC_PREFETCH_INIT_NUM_MASK GENMASK(7, 0) +#define NBL_UVN_DESC_PREFETCH_INIT_SEL_MASK BIT(16) + +#define NBL_UVN_DESC_WR_TIMEOUT_VAL 0x12c +/ 200us = 200000ns / 1.67ns per tick = 119760 ticks / +#define NBL_UVN_DESC_RD_WAIT_TICKS 119760 + +/ -------- USTORE -------- / +#define NBL_USTORE_PKT_LEN_ADDR (NBL_DP_USTORE_BASE + 0x00000108) +#define NBL_USTORE_PORT_DROP_TH_REG_ARR(port_id) \ + (NBL_DP_USTORE_BASE + 0x00000150 + (port_id) sizeof(u32)) +#define NBL_USTORE_BUF_PORT_DROP_PKT(eth_id) \ + (NBL_DP_USTORE_BASE + 0x00002500 + (eth_id) * sizeof(u32)) +#define NBL_USTORE_BUF_PORT_TRUN_PKT(eth_id) \ + (NBL_DP_USTORE_BASE + 0x00002540 + (eth_id) * sizeof(u32)) + +#define NBL_USTORE_SINGLE_ETH_DROP_TH 0xC80 +#define NBL_USTORE_DUAL_ETH_DROP_TH 0x640 +#define NBL_USTORE_QUAD_ETH_DROP_TH 0x320 + +/* USTORE pkt_len / +#define NBL_USTORE_PKT_LEN_MIN_MASK GENMASK(6, 0) + +/ USTORE port_drop_th / +#define NBL_USTORE_PORT_DROP_TH_DISC_TH_MASK GENMASK(11, 0) +#define NBL_USTORE_PORT_DROP_TH_EN_MASK BIT(31) + +/ UQM/ +#define NBL_UQM_QUE_TYPE (NBL_DP_UQM_BASE + 0x0000013c) +#define NBL_UQM_DROP_PKT_CNT (NBL_DP_UQM_BASE + 0x000009C0) +#define NBL_UQM_DROP_PKT_SLICE_CNT (NBL_DP_UQM_BASE + 0x000009C4) +#define NBL_UQM_DROP_PKT_LEN_ADD_CNT (NBL_DP_UQM_BASE + 0x000009C8) +#define NBL_UQM_DROP_HEAD_PNTR_ADD_CNT (NBL_DP_UQM_BASE + 0x000009CC) +#define NBL_UQM_DROP_WEIGHT_ADD_CNT (NBL_DP_UQM_BASE + 0x000009D0) +#define NBL_UQM_PORT_DROP_PKT_CNT (NBL_DP_UQM_BASE + 0x000009D4) +#define NBL_UQM_PORT_DROP_PKT_SLICE_CNT (NBL_DP_UQM_BASE + 0x000009F4) +#define NBL_UQM_PORT_DROP_PKT_LEN_ADD_CNT (NBL_DP_UQM_BASE + 0x00000A14) +#define NBL_UQM_PORT_DROP_HEAD_PNTR_ADD_CNT (NBL_DP_UQM_BASE + 0x00000A34) +#define NBL_UQM_PORT_DROP_WEIGHT_ADD_CNT (NBL_DP_UQM_BASE + 0x00000A54) +#define NBL_UQM_FWD_DROP_CNT (NBL_DP_UQM_BASE + 0x00000A80) +#define NBL_UQM_DPORT_DROP_CNT (NBL_DP_UQM_BASE + 0x00000B74) + +#define NBL_UQM_PORT_DROP_DEPTH 6 +#define NBL_UQM_DPORT_DROP_DEPTH 16 + +/ --------- SHAPING --------- / + +/ Shaping rate unit: 1 = 1 Mbit/s. + * e.g. 100000 = 100 Gbit/s, 25000 = 25 Gbit/s. + / +#define NBL_LR_LEONIS_NET_BUCKET_DEPTH 9600 +#define NBL_SHAPING_DPORT_ADDR (NBL_DP_SHAPING_BASE + 0x700) +#define NBL_SHAPING_DPORT_DWLEN 4 +#define NBL_SHAPING_DPORT_REG(r) \ + (NBL_SHAPING_DPORT_ADDR + (NBL_SHAPING_DPORT_DWLEN 4) * (r)) +#define NBL_SHAPING_DVN_DPORT_ADDR (NBL_DP_SHAPING_BASE + 0x750) +#define NBL_SHAPING_DVN_DPORT_DWLEN 4 +#define NBL_SHAPING_DVN_DPORT_REG(r) \ + (NBL_SHAPING_DVN_DPORT_ADDR + (NBL_SHAPING_DVN_DPORT_DWLEN * 4) * (r)) +#define NBL_DSCH_PSHA_EN_ADDR (NBL_DP_DSCH_BASE + 0x00000314) +#define NBL_SHAPING_NET_ADDR (NBL_DP_SHAPING_BASE + 0x1800) +#define NBL_SHAPING_NET_DWLEN 4 +#define NBL_SHAPING_NET_REG(r) \ + (NBL_SHAPING_NET_ADDR + (NBL_SHAPING_NET_DWLEN * 4) * (r)) + +#define NBL_DPORT_VALID_MASK GENMASK_ULL(0, 0) +#define NBL_DPORT_DEPTH_MASK GENMASK_ULL(19, 1) +#define NBL_DPORT_CIR_MASK GENMASK_ULL(38, 20) +#define NBL_DPORT_PIR_MASK GENMASK_ULL(57, 39) +#define NBL_DPORT_CBS_MASK_LOW GENMASK_ULL(63, 58) +#define NBL_DPORT_CBS_MASK_HIGH GENMASK_ULL(14, 0) +#define NBL_DPORT_PBS_MASK GENMASK_ULL(35, 15) + +/* SHAPING shaping_net / +struct nbl_shaping_net_u { + u32 data[NBL_SHAPING_NET_DWLEN]; +}; + +struct nbl_shaping_dport_u { + u32 data[NBL_SHAPING_DPORT_DWLEN]; +}; + +struct nbl_shaping_dvn_dport_u { + u32 data[NBL_SHAPING_DVN_DPORT_DWLEN]; +}; + +/ -------- DSTORE -------- / +#define NBL_DSTORE_D_DPORT_FC_TH_ADDR (NBL_DP_DSTORE_BASE + 0x00000600) +#define NBL_DSTORE_D_DPORT_FC_TH_DEPTH 5 +#define NBL_DSTORE_D_DPORT_FC_TH_WIDTH 32 +#define NBL_DSTORE_D_DPORT_FC_TH_DWLEN 1 + +#define NBL_DSTORE_D_DPORT_FC_XOFF_TH_MASK GENMASK(10, 0) +#define NBL_DSTORE_D_DPORT_FC_XON_TH_MASK GENMASK(26, 16) +#define NBL_DSTORE_D_DPORT_FC_FC_EN_MASK BIT(31) + +#define NBL_DSTORE_D_DPORT_FC_TH_REG(r) \ + (NBL_DSTORE_D_DPORT_FC_TH_ADDR + \ + (NBL_DSTORE_D_DPORT_FC_TH_DWLEN 4) * (r)) +#define NBL_DSTORE_PORT_DROP_TH_ADDR (NBL_DP_DSTORE_BASE + 0x00000150) +#define NBL_DSTORE_PORT_DROP_TH_DEPTH 6 +#define NBL_DSTORE_PORT_DROP_TH_WIDTH 32 +#define NBL_DSTORE_PORT_DROP_TH_DWLEN 1 + +#define NBL_DSTORE_PORT_DROP_EN_MASK BIT(31) + +#define NBL_DSTORE_DROP_XOFF_TH 0xC8 +#define NBL_DSTORE_DROP_XON_TH 0x64 + +#define NBL_DSTORE_DROP_XOFF_TH_100G 0x1F4 +#define NBL_DSTORE_DROP_XON_TH_100G 0x12C + +#define NBL_DSTORE_DISC_BP_TH (NBL_DP_DSTORE_BASE + 0x00000630) + +#define NBL_DSTORE_DISC_BP_TH_EN_MASK BIT(31) + +#define NBL_DSTORE_PORT_DROP_TH_REG(r) \ + (NBL_DSTORE_PORT_DROP_TH_ADDR + \ + (NBL_DSTORE_PORT_DROP_TH_DWLEN * 4) * (r)) + +#define NBL_FW_BOARD_CONFIG 0x200 +#define NBL_FW_BOARD_DW3_OFFSET (NBL_FW_BOARD_CONFIG + 12) +#define NBL_FW_BOARD_DW6_OFFSET (NBL_FW_BOARD_CONFIG + 24) + +#define NBL_FW_BOARD_DW3_PORT_TYPE_MASK BIT(0) +#define NBL_FW_BOARD_DW3_PORT_NUM_MASK GENMASK(7, 1) +#define NBL_FW_BOARD_DW3_PORT_SPEED_MASK GENMASK(9, 8) +#define NBL_FW_BOARD_DW3_GPIO_TYPE_MASK GENMASK(12, 10) +#define NBL_FW_BOARD_DW3_P4_VERSION_MASK GENMASK(13, 13) + +#define NBL_FW_BOARD_DW6_LANE_BITMAP_MASK GENMASK(7, 0) +#define NBL_FW_BOARD_DW6_ETH_BITMAP_MASK GENMASK(15, 8) + +#define NBL_LEONIS_QUIRKS_OFFSET 0x00000140 #define NBL_BAR2_MAX_LEN 0x300 #endif diff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_leonis/nbl_resource_leonis.c b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_leonis/nbl_resource_leonis.c new file mode 100644 index 00000000000000..ba589af55374b7 --- /dev/null +++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_leonis/nbl_resource_leonis.c @@ -0,0 +1,370 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026 Nebula Matrix Limited. + / +#include <linux/device.h> +#include <linux/pci.h> +#include <linux/bits.h> +#include "nbl_resource_leonis.h" + +static struct nbl_resource_ops res_ops = { + .get_vsi_id = nbl_res_func_id_to_vsi_id, + .get_eth_id = nbl_res_get_eth_id, + .cfg_msix_map = nbl_res_intr_cfg_msix_map, + .destroy_msix_map = nbl_res_intr_destroy_msix_map, + .set_mailbox_irq = nbl_res_intr_set_mailbox_irq, + .init_module = nbl_res_chip_init_module, + .deinit_module = nbl_res_chip_deinit_module, +}; + +static struct nbl_resource_mgt +nbl_res_setup_res_mgt(struct nbl_common_info common) +{ + struct nbl_resource_info resource_info; + struct nbl_resource_mgt res_mgt; + struct device dev = common->dev; + + res_mgt = devm_kzalloc(dev, sizeof(res_mgt), GFP_KERNEL); + if (!res_mgt) + return ERR_PTR(-ENOMEM); + res_mgt->common = common; + + resource_info = + devm_kzalloc(dev, sizeof(resource_info), GFP_KERNEL); + if (!resource_info) + return ERR_PTR(-ENOMEM); + res_mgt->resource_info = resource_info; + + return res_mgt; +} + +static struct nbl_resource_ops_tbl * +nbl_res_setup_ops(struct device dev, struct nbl_resource_mgt res_mgt) +{ + struct nbl_resource_ops_tbl res_ops_tbl; + + res_ops_tbl = devm_kzalloc(dev, sizeof(res_ops_tbl), GFP_KERNEL); + if (!res_ops_tbl) + return ERR_PTR(-ENOMEM); + if (!res_ops.get_vsi_id \|\| !res_ops.get_eth_id \|\| + !res_ops.cfg_msix_map \|\| !res_ops.destroy_msix_map \|\| + !res_ops.set_mailbox_irq \|\| !res_ops.init_module \|\| + !res_ops.deinit_module) + return ERR_PTR(-EINVAL); + res_ops_tbl->ops = &res_ops; + res_ops_tbl->priv = res_mgt; + + return res_ops_tbl; +} + +static int nbl_res_ctrl_dev_setup_eth_info(struct nbl_resource_mgt res_mgt) +{ + struct nbl_hw_ops hw_ops = res_mgt->hw_ops_tbl->ops; + struct device dev = res_mgt->common->dev; + struct nbl_eth_info eth_info; + u32 eth_bitmap = 0; + u32 eth_num = 0; + u32 fw_port_num; + int i; + + eth_info = devm_kzalloc(dev, sizeof(eth_info), GFP_KERNEL); + if (!eth_info) + return -ENOMEM; + + res_mgt->resource_info->eth_info = eth_info; + + fw_port_num = res_mgt->resource_info->board_info.eth_num; + + hw_ops->get_fw_eth_map(res_mgt->hw_ops_tbl->priv, &eth_bitmap); + if (eth_bitmap & ~((1 << NBL_MAX_ETHERNET) - 1)) { + dev_err(dev, "FW reported invalid eth_bitmap 0x%x\n", + eth_bitmap); + return -EINVAL; + } + if (fw_port_num != hweight32(eth_bitmap)) { + dev_err(dev, "FW inconsistency: port_num=%u, bitmap=0x%x\n", + fw_port_num, eth_bitmap); + return -EINVAL; + } + / + * Firmware is ready before probe. Valid port counts are 1/2/4; + * 0 (invalid config), 3 (unsupported topology), and >4 (exceeds + * hardware max) are all rejected with -EINVAL. + / + if (fw_port_num == 0 \|\| fw_port_num == 3 \|\| + fw_port_num > NBL_MAX_ETHERNET) { + dev_err(dev, "FW reports %u Ethernet ports, unsupported (valid: 1/2/4)\n", + fw_port_num); + return -EINVAL; + } + eth_info->eth_num = fw_port_num; + / Intentional design constraint: each PF maps to exactly one + * Ethernet port. This couples PF identity to port identity + * and is required by nbl_res_get_eth_id() which indexes + * eth_info->eth_id[] by relative PF id. + / + if (res_mgt->common->max_pf != eth_info->eth_num) { + dev_err(dev, "Invalid PF-to-port topology: max_pf=%u, eth_num=%u\n", + res_mgt->common->max_pf, eth_info->eth_num); + return -EINVAL; + } + + / + * Any subset of valid bitmap bits is accepted (e.g. 0/1, 0/2, + * 1/3, etc.). Firmware only needs to report the correct count + * of active ports; no hard-coded fixed bit positions required. + * eth_id[] is filled in ascending bitmap-bit order, so the Nth + * relative PF owns the Nth logical port (max_pf == eth_num is + * enforced above). + / + for (i = 0; i < NBL_MAX_ETHERNET; i++) { + if ((1 << i) & eth_bitmap) { + eth_info->eth_id[eth_num] = i; + eth_num++; + } + } + + return 0; +} + +static int nbl_res_ctrl_dev_sriov_info_init(struct nbl_resource_mgt res_mgt) +{ + struct nbl_hw_ops hw_ops = res_mgt->hw_ops_tbl->ops; + struct nbl_hw_mgt p = res_mgt->hw_ops_tbl->priv; + struct nbl_common_info common = res_mgt->common; + struct nbl_sriov_info sriov_info; + struct device dev = common->dev; + u8 hw_bus = 0; + u16 function; + u16 func_id; + + hw_ops->get_real_bus(p, &hw_bus); + if (common->function + common->max_pf > NBL_MAX_PF) { + dev_err(dev, "PF count exceeds available function space\n"); + return -EINVAL; + } + sriov_info = devm_kcalloc(dev, common->max_pf, + sizeof(sriov_info), GFP_KERNEL); + if (!sriov_info) + return -ENOMEM; + + res_mgt->resource_info->sriov_info = sriov_info; + /* + * Real bus number of the control PF; + * the BDF table below is built from it. + / + common->hw_bus = hw_bus; + + for (func_id = 0; func_id < common->max_pf; func_id++) { + sriov_info = res_mgt->resource_info->sriov_info + func_id; + function = common->function + func_id; + sriov_info->bdf = PCI_DEVID(common->hw_bus, + PCI_DEVFN(common->devid, function)); + } + + return 0; +} + +static int nbl_res_ctrl_dev_vsi_info_init(struct nbl_resource_mgt res_mgt) +{ + struct nbl_eth_info eth_info = res_mgt->resource_info->eth_info; + struct nbl_common_info common = res_mgt->common; + struct device dev = common->dev; + struct nbl_vsi_info vsi_info; + int i; + + vsi_info = devm_kzalloc(dev, sizeof(vsi_info), GFP_KERNEL); + if (!vsi_info) + return -ENOMEM; + + res_mgt->resource_info->vsi_info = vsi_info; + / + * case 1 one port(1pf) + * pf0 (NBL_VSI_SERV_PF_DATA_TYPE) vsi is 0 + * case 2 two port(2pf) + * pf0,pf1(NBL_VSI_SERV_PF_DATA_TYPE) vsi is 0,512 + * case 3 four port(4pf) + * pf0,pf1,pf2,pf3(NBL_VSI_SERV_PF_DATA_TYPE) vsi is 0,256,512,768 + / + + vsi_info->num = eth_info->eth_num; + / + * eth_num can be 1/2/4: + * - 2/4 ports use dedicated gap constants; + * - 1 port falls back to NBL_DEFAULT_VSI_ID_GAP (1024). + * All three values produce valid base_id offsets. + / + for (i = 0; i < vsi_info->num; i++) { + vsi_info->serv_info[i][NBL_VSI_SERV_PF_DATA_TYPE].base_id = + i nbl_vsi_id_gap(vsi_info->num); + vsi_info->serv_info[i][NBL_VSI_SERV_PF_DATA_TYPE].num = 1; + } + + return 0; +} + +static int nbl_res_init_pf_num(struct nbl_resource_mgt res_mgt) +{ + struct nbl_hw_ops hw_ops = res_mgt->hw_ops_tbl->ops; + u32 exp_contiguous_mask = 0; + u32 pf_mask = 0; + u32 pf_num = 0; + int i; + + hw_ops->get_host_pf_mask(res_mgt->hw_ops_tbl->priv, &pf_mask); + + /* + * k_pf_mask register rule: + * bit N == 0 -> PF#N enabled; bit N == 1 -> PF#N masked out. + * Hardware constraint: bit0 is PF0's mask bit; driver requires + * PF0 enabled as management PF, so bit0 must be clear. + * All-zero pf_mask means all PF0~PF7 are enabled, which is unsupported + * by the driver + * + * Product firmware constraint: only 3 valid configurations supported: + * 1 PF (PF0 only): pf_num = 1, mask = 0xfe + * 2 PFs (PF0,PF1): pf_num = 2, mask = 0xfc + * 4 PFs (PF0~PF3): pf_num = 4, mask = 0xf0 + * No other PF count or sparse/non-contiguous PF layout is allowed. + * + * This is the only validation of the mask in the driver. The channel + * layer reads the same raw register value to program the mailbox + * QINFO table and assumes the layout checked here, so it depends on + * this running first: resource init completes before the channel + * layer is set up. + / + for (i = 0; i < NBL_MAX_PF; i++) { + if (!(pf_mask & (1 << i))) + pf_num++; + } + + / + * Sanity check: enabled PFs must be contiguous starting from PF0. + * Current resource framework uses relative PF id, sparse PF layout + * will cause mismatch between resource layer and hardware func_id. + / + for (i = 0; i < pf_num; i++) + exp_contiguous_mask \|= BIT(i); + if ((pf_mask & exp_contiguous_mask) != 0) { + dev_err(res_mgt->common->dev, + "pf_mask 0x%08x: non-contiguous enabled PF, unsupported\n", + pf_mask); + return -EINVAL; + } + + / Only allow product-specified PF count: 1 / 2 / 4 / + if (pf_num != 1 && pf_num != 2 && pf_num != 4) { + dev_err(res_mgt->common->dev, + "Invalid pf_num=%u (mask=0x%08x), only 1/2/4 PFs supported\n", + pf_num, pf_mask); + return -EINVAL; + } + + res_mgt->common->max_pf = pf_num; + + return 0; +} + +static void nbl_res_init_board_info(struct nbl_resource_mgt res_mgt) +{ + struct nbl_hw_ops hw_ops = res_mgt->hw_ops_tbl->ops; + + hw_ops->get_board_info(res_mgt->hw_ops_tbl->priv, + &res_mgt->resource_info->board_info); +} + +static int nbl_res_start(struct nbl_resource_mgt res_mgt) +{ + struct nbl_common_info common = res_mgt->common; + int ret = 0; + + if (common->has_ctrl) { + nbl_res_init_board_info(res_mgt); + + ret = nbl_res_init_pf_num(res_mgt); + if (ret) + return ret; + + ret = nbl_res_ctrl_dev_sriov_info_init(res_mgt); + if (ret) + return ret; + + ret = nbl_res_ctrl_dev_setup_eth_info(res_mgt); + if (ret) + return ret; + + ret = nbl_res_ctrl_dev_vsi_info_init(res_mgt); + if (ret) + return ret; + + ret = nbl_intr_mgt_start(res_mgt); + if (ret) + return ret; + } + + return 0; +} + +int nbl_res_init_leonis(struct nbl_adapter adap) +{ + struct nbl_channel_ops_tbl chan_ops_tbl = adap->intf.channel_ops_tbl; + struct nbl_hw_ops_tbl hw_ops_tbl = adap->intf.hw_ops_tbl; + struct nbl_common_info common = &adap->common; + struct nbl_resource_ops_tbl res_ops_tbl; + struct device dev = &adap->pdev->dev; + struct nbl_resource_mgt res_mgt; + int ret; + + res_mgt = nbl_res_setup_res_mgt(common); + if (IS_ERR(res_mgt)) { + ret = PTR_ERR(res_mgt); + return ret; + } + res_mgt->chan_ops_tbl = chan_ops_tbl; + res_mgt->hw_ops_tbl = hw_ops_tbl; + + ret = nbl_res_start(res_mgt); + if (ret) + return ret; + + res_ops_tbl = nbl_res_setup_ops(dev, res_mgt); + if (IS_ERR(res_ops_tbl)) { + ret = PTR_ERR(res_ops_tbl); + return ret; + } + adap->intf.resource_ops_tbl = res_ops_tbl; + adap->core.res_mgt = res_mgt; + + return 0; +} + +void nbl_res_remove_leonis(struct nbl_adapter adap) +{ + struct nbl_resource_mgt res_mgt = adap->core.res_mgt; + struct nbl_common_info common = &adap->common; + + if (!res_mgt) + return; + + / + * Tear down all MSI-X maps before destroying coherent tables. + * This is critical on the control PF, whose table carries the map + * entries installed for every configured function: they must be + * invalidated in hardware before the tables backing them are + * freed, otherwise the device keeps DMA-reading released pages. + * Sibling PFs are already unbound by the device-link ordering + * so no remote PF still has a live map entry or a running + * mailbox RPC at this point. + / + if (common->has_ctrl && res_mgt->intr_mgt) + nbl_intr_mgt_stop(res_mgt); + + / Note: + * per-function interrupts arrays (kcalloc) are freed by + * nbl_intr_mgt_stop(). + * MSIX coherent tables are explicitly freed by dma_free_coherent() + * inside the intr destroy path, before nbl_intr_mgt_stop() returns. + * intr_mgt itself (devm_kzalloc) is released by devres after this + * function returns + / +} diff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_leonis/nbl_resource_leonis.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_leonis/nbl_resource_leonis.h new file mode 100644 index 00000000000000..f1cb0f23240b0c --- /dev/null +++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_leonis/nbl_resource_leonis.h @@ -0,0 +1,12 @@ +/ SPDX-License-Identifier: GPL-2.0 / +/ + * Copyright (c) 2026 Nebula Matrix Limited. + / + +#ifndef _NBL_RESOURCE_LEONIS_H_ +#define _NBL_RESOURCE_LEONIS_H_ + +#include "../nbl_resource.h" +#include "../nbl_interrupt.h" +#include "../nbl_chip.h" +#endif diff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_reg.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_reg.h index e281109f502e67..00fa33eeabaab9 100644 --- a/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_reg.h +++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_reg.h @@ -8,6 +8,7 @@ #include <linux/types.h> +#include "../nbl_include/nbl_def_channel.h" #include "../nbl_include/nbl_def_hw.h" #include "../nbl_include/nbl_def_common.h" #include "../nbl_core.h" @@ -16,6 +17,7 @@ #define NBL_MAILBOX_BAR 2 #define NBL_RDMA_NOTIFY_LEN (8ULL << 10) #define NBL_REG_NET_ONLY_LEN (8ULL << 10) +#define NBL_HW_DUMMY_REG 0x1300904 / * PCI MEMORY BAR total size: 64MiB. / @@ -26,6 +28,37 @@ struct nbl_hw_mgt { u8 __iomem hw_addr; u8 __iomem mailbox_bar_hw_addr; resource_size_t mailbox_bar_size; + spinlock_t reg_lock; / Protect reg access / }; +static inline u32 rd32(u8 __iomem addr, u64 reg) +{ + return readl(addr + reg); +} + +static inline void wr32(u8 __iomem addr, u64 reg, u32 value) +{ + writel(value, addr + reg); +} + +static inline void nbl_hw_wr32(struct nbl_hw_mgt hw_mgt, u64 reg, u32 value) +{ + wr32(hw_mgt->hw_addr, reg, value); +} + +static inline u32 nbl_hw_rd32(struct nbl_hw_mgt hw_mgt, u64 reg) +{ + return rd32(hw_mgt->hw_addr, reg); +} + +static inline void nbl_mbx_wr32(struct nbl_hw_mgt hw_mgt, u64 reg, u32 value) +{ + writel(value, hw_mgt->mailbox_bar_hw_addr + reg); +} + +static inline u32 nbl_mbx_rd32(struct nbl_hw_mgt hw_mgt, u64 reg) +{ + return readl(hw_mgt->mailbox_bar_hw_addr + reg); +} + #endif diff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_interrupt.c b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_interrupt.c new file mode 100644 index 00000000000000..3969392abdc64a --- /dev/null +++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_interrupt.c @@ -0,0 +1,851 @@ +// SPDX-License-Identifier: GPL-2.0 +/ + * Copyright (c) 2026 Nebula Matrix Limited. + / +#include <linux/device.h> +#include <linux/delay.h> +#include <linux/dma-mapping.h> +#include <linux/bitfield.h> +#include "nbl_interrupt.h" + +#define NBL_MSIX_DMA_SYNC_MIN_US 1000 / us / +#define NBL_MSIX_DMA_SYNC_MAX_US 1200 / us / + +/ + * Bounded best-effort wait for in-flight pcompleter fetches of a MSI-X + * map table after that entry's VALID bit has been cleared and flushed. + * + * The chip exposes no idle/completion status for those fetches, so the + * only ordering available is: clear VALID -> flush_write() -> wait. The + * window is sized well above the tables' worst-case fetch/response + * latency; it is not a guarantee, and it protects two things: + * - the coherent table that __nbl_res_intr_complete_destroy_msix_map() + * is about to dma_free_coherent(); + * - the vector set that a reconfiguration is about to hand to another + * function. + * A future caller that cannot tolerate the residual risk must keep the + * table allocated for the lifetime of the function instead of relying on + * this delay. + * + * Both are bus-master reads tagged with the control PF's BDF, so + * pci_clear_master() is the only other lever, and nbl_remove() pulls it + * before any teardown runs: that keeps the device from starting a new + * fetch inside this window. It cannot help a sibling PF's map destroyed + * at runtime, because those fetches carry the control PF's BDF and so + * are unaffected by clearing bus mastering on the sibling. Runtime + * destroys of sibling maps only become reachable with the mailbox RPC + * added later in this series. + / +static void nbl_intr_quiesce_wait(void) +{ + usleep_range(NBL_MSIX_DMA_SYNC_MIN_US, NBL_MSIX_DMA_SYNC_MAX_US); +} + +/ + * Release global vector IDs back to intr_net_bmap / intr_other_bmap. + * Caller must hold intr_mgt->lock, passing the intr_mgt it locked: + * nbl_intr_mgt_stop() clears res_mgt->intr_mgt while a caller may + * already be blocked on that mutex, so it must not be re-read here. + / +static void nbl_intr_release_bitmap(struct nbl_interrupt_mgt intr_mgt, + struct nbl_resource_mgt res_mgt, + u16 vec_buf, u16 cnt) +{ + u16 bit; + u16 i; + + lockdep_assert_held(&intr_mgt->lock); + + if (!vec_buf \|\| cnt == 0) + return; + + for (i = 0; i < cnt; i++) { + u16 intr_index = vec_buf[i]; + + if (intr_index >= NBL_NET_INTR_BASE) { + bit = intr_index - NBL_NET_INTR_BASE; + if (bit < NBL_MAX_NET_INTERRUPT) + clear_bit(bit, intr_mgt->intr_net_bmap); + else + dev_warn(res_mgt->common->dev, + "invalid net intr index %u\n", + intr_index); + } else { + if (intr_index < NBL_MAX_OTHER_INTERRUPT) + clear_bit(intr_index, + intr_mgt->intr_other_bmap); + else + dev_warn(res_mgt->common->dev, + "invalid other intr index %u\n", + intr_index); + } + } +} + +/* + * Internal (unlocked) mailbox IRQ bind. Caller must hold + * intr_mgt->lock, passing the intr_mgt it locked (see + * nbl_intr_release_bitmap()). + * + * The disable path deliberately carries no state check: the teardown + * sequence uses it to disarm routing while func_res->state is still + * CONFIGURED, and it stays harmless after nbl_intr_mgt_stop() because + * the hardware op ignores gvec when en_msix=false. Only the enable + * path is gated by the stopping latch and the per-function state. + / +static int __nbl_res_intr_set_mailbox_irq(struct nbl_interrupt_mgt intr_mgt, + struct nbl_resource_mgt res_mgt, + u16 func_id, u16 vector_id, + bool en_msix) +{ + struct nbl_hw_ops hw_ops = res_mgt->hw_ops_tbl->ops; + struct nbl_common_info common = res_mgt->common; + struct device dev = common->dev; + struct nbl_func_interrupt_resource_mng func_res; + u16 gvec; + + lockdep_assert_held(&intr_mgt->lock); + + / + * func_intr_res[] is indexed by the raw hardware function id; this + * bound check is the only guard on this path, so it is what keeps + * VF ids out. Whether the PF is present on this board is not + * checked here: callers pass either common->mgt_pf or an id that + * nbl_res_intr_cfg_msix_map() has validated. + / + if (func_id >= NBL_MAX_PF) { + dev_err(dev, "func_id %u out of range\n", func_id); + return -EINVAL; + } + + if (!en_msix) { + hw_ops->set_mailbox_irq(res_mgt->hw_ops_tbl->priv, + func_id, false, 0); + hw_ops->flush_write(res_mgt->hw_ops_tbl->priv); + return 0; + } + + / + * Enable path: the map must be live and not under teardown, + * otherwise routing would point at a vector that the complete + * phase is about to release and never re-disables. + / + if (intr_mgt->stopping) + return -ESHUTDOWN; + + func_res = &intr_mgt->func_intr_res[func_id]; + if (func_res->state != NBL_INTR_FUNC_CONFIGURED) { + dev_err(dev, "func %u MSIX map not configured (state %u)\n", + func_id, func_res->state); + return -ENODEV; + } + + if (vector_id >= func_res->num_interrupts) { + dev_err(dev, "vector_id %u out of range (max %u)\n", + vector_id, func_res->num_interrupts - 1); + return -EINVAL; + } + + gvec = func_res->interrupts[vector_id]; + hw_ops->set_mailbox_irq(res_mgt->hw_ops_tbl->priv, func_id, + en_msix, gvec); + hw_ops->flush_write(res_mgt->hw_ops_tbl->priv); + + return 0; +} + +/ + * Internal (unlocked) MSI-X map teardown prepare phase: only hardware + * register operations. The DMA address is retained and only the VALID + * bit is cleared; zeroing the address (Stage 2) is deferred to the + * complete phase after the hardware-DMA quiesce window. + * + * Caller must hold intr_mgt->lock. + / +static int +__nbl_res_intr_prepare_destroy_msix_map(struct nbl_interrupt_mgt intr_mgt, + struct nbl_resource_mgt res_mgt, + u16 func) +{ + struct nbl_hw_ops hw_ops = res_mgt->hw_ops_tbl->ops; + struct nbl_func_interrupt_resource_mng func_res; + u16 interrupts; + u16 intr_num, i; + int ret; + + lockdep_assert_held(&intr_mgt->lock); + + if (func >= NBL_MAX_PF) { + dev_err(res_mgt->common->dev, "Invalid func_id %u\n", func); + return -EINVAL; + } + + func_res = &intr_mgt->func_intr_res[func]; + if (func_res->state != NBL_INTR_FUNC_CONFIGURED) + return 0; + + interrupts = func_res->interrupts; + intr_num = func_res->num_interrupts; + + /* Step 0: disable mailbox IRQ routing before tearing down map / + ret = __nbl_res_intr_set_mailbox_irq(intr_mgt, res_mgt, func, 0, false); + if (ret) { + dev_err(res_mgt->common->dev, + "disable mailbox irq failed, func=%u ret=%d\n", + func, ret); + return ret; + } + + / Step 1: invalidate each MSIX info entry in hardware first / + for (i = 0; i < intr_num; i++) { + hw_ops->cfg_msix_info(res_mgt->hw_ops_tbl->priv, + func, false, interrupts[i], + 0, 0, 0, false); + } + hw_ops->flush_write(res_mgt->hw_ops_tbl->priv); + + / + * Stage 1: retain the DMA address, only clear the VALID bit. + * Stage 2 runs after the quiesce window in the complete phase. + / + hw_ops->cfg_msix_map(res_mgt->hw_ops_tbl->priv, func, + false, func_res->msix_map_table.dma, + 0, 0, 0); + hw_ops->flush_write(res_mgt->hw_ops_tbl->priv); + + func_res->state = NBL_INTR_FUNC_DESTROYING; + + return 0; +} + +/ + * __nbl_res_intr_complete_destroy_msix_map - finish hardware teardown and + * release vector bitmap, DMA memory and interrupt buffer after the + * hardware quiesce window has elapsed. + * + * Caller must hold intr_mgt->lock. + / +static int +__nbl_res_intr_complete_destroy_msix_map(struct nbl_interrupt_mgt intr_mgt, + struct nbl_resource_mgt res_mgt, + u16 func_id) +{ + struct nbl_hw_ops hw_ops = res_mgt->hw_ops_tbl->ops; + struct nbl_func_interrupt_resource_mng func_res; + struct nbl_msix_map_table msix_map_table; + struct device dev = res_mgt->common->dev; + u16 interrupts; + u16 intr_num; + + lockdep_assert_held(&intr_mgt->lock); + + if (func_id >= NBL_MAX_PF) { + dev_err(dev, "Invalid func_id %u\n", func_id); + return -EINVAL; + } + + func_res = &intr_mgt->func_intr_res[func_id]; + if (func_res->state != NBL_INTR_FUNC_DESTROYING) + return 0; + + /* + * Stage 2: the quiesce window has elapsed, it is now safe to + * zero the DMA base address in the hardware map register. + / + hw_ops->cfg_msix_map(res_mgt->hw_ops_tbl->priv, func_id, + false, 0, 0, 0, 0); + hw_ops->flush_write(res_mgt->hw_ops_tbl->priv); + + interrupts = func_res->interrupts; + intr_num = func_res->num_interrupts; + msix_map_table = &func_res->msix_map_table; + + if (interrupts) { + nbl_intr_release_bitmap(intr_mgt, res_mgt, interrupts, + intr_num); + kfree(interrupts); + } + + / + * Release the coherent table independently of interrupts so a + * partially built config (table allocated, vectors never + * published) cannot leak coherent DMA memory. + / + if (msix_map_table->base_addr) { + dma_free_coherent(dev, msix_map_table->size, + msix_map_table->base_addr, + msix_map_table->dma); + msix_map_table->base_addr = NULL; + msix_map_table->dma = 0; + msix_map_table->size = 0; + } + + func_res->interrupts = NULL; + func_res->num_interrupts = 0; + func_res->num_net_interrupts = 0; + func_res->state = NBL_INTR_FUNC_IDLE; + + return 0; +} + +/ + * Internal (unlocked) MSI-X map teardown. Caller must hold + * intr_mgt->lock for the whole sequence, including the hardware-DMA + * quiesce window: dropping the lock would let a concurrent caller (or + * nbl_intr_mgt_stop()) install/free state against this teardown. + * + * This is used for the single function synchronous destroy path. + / +static int __nbl_res_intr_destroy_msix_map(struct nbl_interrupt_mgt intr_mgt, + struct nbl_resource_mgt res_mgt, + u16 func_id) +{ + int ret; + + lockdep_assert_held(&intr_mgt->lock); + + if (intr_mgt->stopping) + return -ESHUTDOWN; + + ret = __nbl_res_intr_prepare_destroy_msix_map(intr_mgt, res_mgt, + func_id); + if (ret) + return ret; + / + * prepare() only transitions CONFIGURED functions; an IDLE func + * has nothing to wait for or complete. + / + if (intr_mgt->func_intr_res[func_id].state != + NBL_INTR_FUNC_DESTROYING) + return 0; + + nbl_intr_quiesce_wait(); + + return __nbl_res_intr_complete_destroy_msix_map(intr_mgt, res_mgt, + func_id); +} + +int nbl_res_intr_destroy_msix_map(struct nbl_resource_mgt res_mgt, + u16 func_id) +{ + struct nbl_interrupt_mgt intr_mgt = res_mgt->intr_mgt; + int ret; + + if (!intr_mgt) + return -EINVAL; + + mutex_lock(&intr_mgt->lock); + ret = __nbl_res_intr_destroy_msix_map(intr_mgt, res_mgt, func_id); + mutex_unlock(&intr_mgt->lock); + + return ret; +} + +/* + * nbl_res_intr_cfg_msix_map - allocate & program MSI-X mapping table + * @res_mgt: resource management instance + * @func_id: target function identifier + * @num_net_msix: required net data interrupt vectors + * @num_others_msix: required control interrupt vectors + * @net_msix_mask_en: enable mask for net interrupt entries + * + * Allocate interrupt vectors; MSIX coherent DMA table is allocated once + * per function on first configuration, entries are rewritten while the + * map is invalidated on subsequent reconfigurations. No free/realloc of + * DMA table on vector count changes. This removes the DMA table + * free/realloc cycle. On reconfiguration the map VALID bit is cleared + * and the hardware-DMA quiesce window is observed (lock held) before old + * vectors are recycled and the table is rewritten. + * + * Reconfiguration transiently needs room for the whole new vector set on + * top of the old one, because the old set is only released after the new + * allocation has succeeded (a failed reconfiguration must leave the old + * configuration intact). The bitmap pools are therefore sized for peak, + * not maximum, concurrent use: a reconfigure of the same or a smaller + * size still fails with -EAGAIN when the pool is nearly exhausted. The + * in-tree caller (nbl_dev_init_msix_cnt()) asks for one non-net vector + * per PF, far below NBL_MAX_OTHER_INTERRUPT. + * + * Serialization: this function takes intr_mgt->lock internally to + * protect the global vector bitmaps and per-function state against + * concurrent callers. + * + * Return: 0 on success, negative errno on failure + / +int nbl_res_intr_cfg_msix_map(struct nbl_resource_mgt res_mgt, + u16 func_id, u16 num_net_msix, + u16 num_others_msix, + bool net_msix_mask_en) +{ + struct nbl_interrupt_mgt intr_mgt = res_mgt->intr_mgt; + struct nbl_hw_ops hw_ops = res_mgt->hw_ops_tbl->ops; + struct nbl_common_info common = res_mgt->common; + struct nbl_msix_map_table official_tbl; + struct nbl_msix_map msix_map_entries; + struct device dev = common->dev; + u16 requested, intr_index; + u8 bus, devid, function; + bool entry_masked = false; + u16 tmp_interrupts = NULL; + u16 allocated_cnt = 0; + u16 old_interrupts; + u16 old_num; + bool had_config; + int ret = 0; + u16 gvec; + u16 i, j; + + if (!intr_mgt) + return -EINVAL; + + if (!common->has_ctrl) + return -EINVAL; + + if (func_id >= NBL_MAX_PF) { + dev_err(dev, "Invalid func_id %u\n", func_id); + return -EINVAL; + } + + if (num_net_msix == 0 && num_others_msix == 0) { + dev_err(dev, "MSI-X vector count cannot both be zero\n"); + return -EINVAL; + } + + if (num_net_msix > NBL_MSIX_MAP_TABLE_MAX_ENTRIES \|\| + num_others_msix > NBL_MSIX_MAP_TABLE_MAX_ENTRIES) { + dev_err(dev, "MSI-X count out of limit: net=%u, others=%u\n", + num_net_msix, num_others_msix); + return -EINVAL; + } + + if (check_add_overflow(num_net_msix, num_others_msix, &requested) \|\| + requested > NBL_MSIX_MAP_TABLE_MAX_ENTRIES) { + dev_err(dev, "Total MSI-X vectors %u exceeds maximum %u\n", + requested, NBL_MSIX_MAP_TABLE_MAX_ENTRIES); + return -EINVAL; + } + + ret = nbl_res_func_id_to_bdf(res_mgt, func_id, &bus, &devid, &function); + if (ret) { + if (ret == -EOPNOTSUPP) + dev_err(dev, + "MSI-X mapping: func_id=%u is not a PF present on this board\n", + func_id); + return ret; + } + + mutex_lock(&intr_mgt->lock); + official_tbl = &intr_mgt->func_intr_res[func_id].msix_map_table; + + /* Reject new configs once global teardown has started / + if (intr_mgt->stopping) { + ret = -ESHUTDOWN; + goto out_unlock; + } + + had_config = intr_mgt->func_intr_res[func_id].state == + NBL_INTR_FUNC_CONFIGURED; + + / + * Phase1: allocate global vector array first. + * Allocate the fixed-size MSIX DMA table only ONCE for this function. + / + tmp_interrupts = kcalloc(requested, sizeof(tmp_interrupts), + GFP_KERNEL); + if (!tmp_interrupts) { + ret = -ENOMEM; + goto out_unlock; + } + /* Allocate MSIX DMA table once per function / + if (!official_tbl->base_addr) { + official_tbl->size = + sizeof(struct nbl_msix_map) + NBL_MSIX_MAP_TABLE_MAX_ENTRIES; + official_tbl->base_addr = dma_alloc_coherent(dev, + official_tbl->size, + &official_tbl->dma, + GFP_KERNEL); + if (!official_tbl->base_addr) { + dev_err(dev, "Failed to allocate DMA memory for MSIX table\n"); + ret = -ENOMEM; + goto release_vecs_unlock; + } + } + + /* Allocate net interrupt vectors / + for (i = 0; i < num_net_msix; i++) { + intr_index = find_first_zero_bit(intr_mgt->intr_net_bmap, + NBL_MAX_NET_INTERRUPT); + if (intr_index == NBL_MAX_NET_INTERRUPT) { + dev_err(dev, "No free net interrupt vectors left\n"); + ret = -EAGAIN; + goto release_vecs_unlock; + } + tmp_interrupts[i] = intr_index + NBL_NET_INTR_BASE; + set_bit(intr_index, intr_mgt->intr_net_bmap); + allocated_cnt++; + } + + / Allocate other interrupt vectors / + for (; i < requested; i++) { + intr_index = + find_first_zero_bit(intr_mgt->intr_other_bmap, + NBL_MAX_OTHER_INTERRUPT); + if (intr_index == NBL_MAX_OTHER_INTERRUPT) { + dev_err(dev, "No free control interrupt vectors left\n"); + ret = -EAGAIN; + goto release_vecs_unlock; + } + tmp_interrupts[i] = intr_index; + set_bit(intr_index, intr_mgt->intr_other_bmap); + allocated_cnt++; + } + + / + * Phase2: quiesce the old hardware MSIX config before touching + * the live DMA table. Same sequence as destroy: + * disable mailbox routing -> invalidate per-vector INFO -> + * clear map VALID -> flush -> wait for in-flight table fetches. + * The lock stays held across the wait, so no concurrent caller + * can program the quiesced function. Only then are old vectors + * recycled. + * NOTE: NO DMA table free here. + / + if (had_config) { + old_interrupts = + intr_mgt->func_intr_res[func_id].interrupts; + old_num = intr_mgt->func_intr_res[func_id].num_interrupts; + + ret = __nbl_res_intr_set_mailbox_irq(intr_mgt, res_mgt, func_id, + 0, false); + if (ret) { + dev_err(dev, "%s: disable old mailbox irq failed, keep old config\n", + __func__); + goto release_vecs_unlock; + } + for (j = 0; j < old_num; j++) { + hw_ops->cfg_msix_info(res_mgt->hw_ops_tbl->priv, + func_id, false, + old_interrupts[j], + 0, 0, 0, false); + } + hw_ops->cfg_msix_map(res_mgt->hw_ops_tbl->priv, func_id, + false, official_tbl->dma, + 0, 0, 0); + hw_ops->flush_write(res_mgt->hw_ops_tbl->priv); + + nbl_intr_quiesce_wait(); + + nbl_intr_release_bitmap(intr_mgt, res_mgt, old_interrupts, + old_num); + kfree(old_interrupts); + intr_mgt->func_intr_res[func_id].interrupts = NULL; + intr_mgt->func_intr_res[func_id].num_interrupts = 0; + intr_mgt->func_intr_res[func_id].num_net_interrupts = 0; + } + + / Swap new vector array into func state / + intr_mgt->func_intr_res[func_id].interrupts = tmp_interrupts; + intr_mgt->func_intr_res[func_id].num_interrupts = requested; + intr_mgt->func_intr_res[func_id].num_net_interrupts = num_net_msix; + tmp_interrupts = NULL; + + / + * First configuration of this function. Re-invalidate the entry + * and take a quiesce window before the table below is rewritten: + * while the entry is VALID the pcompleter could fetch a table + * that is being written and route stale gvecs. + * + * This is a second, real quiesce window rather than part of the + * reconfiguration path, and it costs one delay per function on the + * first configuration only. Reusing the table removes the + * free/realloc cycle and the destroy-path quiesce window from + * every reconfiguration; it does not remove this one. + / + if (!had_config) { + hw_ops->cfg_msix_map(res_mgt->hw_ops_tbl->priv, func_id, + false, 0, 0, 0, 0); + hw_ops->flush_write(res_mgt->hw_ops_tbl->priv); + nbl_intr_quiesce_wait(); + } + + / + * Rewrite the table in the pre-allocated DMA buffer while the + * map is invalid, so the device cannot observe a torn old/new + * mix. Only entries beyond requested count need explicit zeroing. + / + msix_map_entries = official_tbl->base_addr; + memset(msix_map_entries + requested, 0, + (NBL_MSIX_MAP_TABLE_MAX_ENTRIES - requested) + sizeof(msix_map_entries)); + + for (i = 0; i < requested; i++) { + gvec = intr_mgt->func_intr_res[func_id].interrupts[i]; + msix_map_entries[i].data = + cpu_to_le16(FIELD_PREP(NBL_MSIX_MAP_VALID_MASK, 1) \| + FIELD_PREP(NBL_MSIX_MAP_INDEX_MASK, + gvec)); + } + + / Ensure coherent table writes are visible before HW fetch/enable / + dma_wmb(); + + / Enable per-vector INFO entries after the table is published / + for (i = 0; i < requested; i++) { + gvec = intr_mgt->func_intr_res[func_id].interrupts[i]; + entry_masked = (i < num_net_msix && net_msix_mask_en); + hw_ops->cfg_msix_info(res_mgt->hw_ops_tbl->priv, + func_id, true, gvec, + bus, devid, function, + entry_masked); + } + hw_ops->flush_write(res_mgt->hw_ops_tbl->priv); + + / + * Point the map at the table last and set VALID. + * + * cfg_msix_map uses the control PF's own BDF (common->hw_bus etc.), + * not the target function's BDF. This BDF tags the pcompler DMA + * read of the MSI-X map table as originating from the control PF. + * The target function's BDF (bus/devid/function from + * nbl_res_func_id_to_bdf) is used only in cfg_msix_info for the + * host_msix_ctrl table entry BDF filtering. + / + hw_ops->cfg_msix_map(res_mgt->hw_ops_tbl->priv, func_id, + true, official_tbl->dma, common->hw_bus, + common->devid, common->function); + hw_ops->flush_write(res_mgt->hw_ops_tbl->priv); + + intr_mgt->func_intr_res[func_id].state = NBL_INTR_FUNC_CONFIGURED; + mutex_unlock(&intr_mgt->lock); + return 0; + +release_vecs_unlock: + nbl_intr_release_bitmap(intr_mgt, res_mgt, tmp_interrupts, + allocated_cnt); + kfree(tmp_interrupts); + / + * On a failed fresh configuration, release the DMA table + * allocated during this call. On a failed reconfiguration the + * old configuration is still intact (it is only torn down + * after all vector allocations succeed) and owns the table. + / + if (!had_config && official_tbl->base_addr) { + dma_free_coherent(dev, official_tbl->size, + official_tbl->base_addr, + official_tbl->dma); + official_tbl->base_addr = NULL; + official_tbl->dma = 0; + official_tbl->size = 0; + } +out_unlock: + mutex_unlock(&intr_mgt->lock); + return ret; +} + +/* + * nbl_res_intr_set_mailbox_irq - bind mailbox IRQ to specified vector + * @res_mgt: resource management instance + * @func_id: target function identifier + * @vector_id: index inside local interrupt array + * @en_msix: enable/disable mailbox interrupt + * + * Serialization: takes intr_mgt->lock internally and passes the + * pointer it read (before taking the lock) down to the helpers, which + * must not re-read res_mgt->intr_mgt: nbl_intr_mgt_stop() clears that + * pointer while a caller may already be blocked on this mutex. + * + * Return: 0 on success, negative errno on parameter or state check + * failure. The hardware op is void and cannot report failure. + / +int nbl_res_intr_set_mailbox_irq(struct nbl_resource_mgt res_mgt, + u16 func_id, u16 vector_id, + bool en_msix) +{ + struct nbl_interrupt_mgt intr_mgt = res_mgt->intr_mgt; + struct nbl_common_info common = res_mgt->common; + int ret; + + if (!intr_mgt) + return -EINVAL; + + if (!common->has_ctrl) + return -EINVAL; + + mutex_lock(&intr_mgt->lock); + ret = __nbl_res_intr_set_mailbox_irq(intr_mgt, res_mgt, func_id, + vector_id, en_msix); + mutex_unlock(&intr_mgt->lock); + + return ret; +} + +/* + * Reset the software state and invalidate every function's hardware MSI-X + * map entry once. The chip-internal tables (FUNCTION_MSIX_MAP, + * PADPT_HOST_MSIX_INFO, HOST_MSIX_FID_TABLE) can survive a kexec or a + * forced unload without FLR and there is no .shutdown callback to scrub + * them. Only FUNCTION_MSIX_MAP is scrubbed here: it is indexed by + * func_id, so covering all NBL_MAX_PF entries costs one register write + * each, after which the pcompleter can no longer fetch a stale table that + * points into memory this kernel may have reallocated. The INFO/FID + * tables are indexed by global vector id and are not cleared; instead a + * map entry is always invalidated by nbl_res_intr_cfg_msix_map() and by + * the teardown paths before it is armed for a new configuration. + * + * No sibling PF is live at this point: the device links make them probe + * only after this PF and remove before it, so clearing all NBL_MAX_PF + * entries cannot disturb a running function. + / +static struct nbl_interrupt_mgt +nbl_intr_setup_mgt(struct nbl_resource_mgt res_mgt) +{ + struct device dev = res_mgt->common->dev; + struct nbl_hw_ops hw_ops = res_mgt->hw_ops_tbl->ops; + struct nbl_interrupt_mgt intr_mgt; + u16 func_id; + int err; + + intr_mgt = devm_kzalloc(dev, sizeof(intr_mgt), GFP_KERNEL); + if (!intr_mgt) + return ERR_PTR(-ENOMEM); + + err = devm_mutex_init(dev, &intr_mgt->lock); + if (err) + return ERR_PTR(err); + + intr_mgt->stopping = false; + bitmap_zero(intr_mgt->intr_net_bmap, NBL_MAX_NET_INTERRUPT); + bitmap_zero(intr_mgt->intr_other_bmap, NBL_MAX_OTHER_INTERRUPT); + + / + * Invalidate any map entry left VALID by a previous kernel. One + * flush plus one quiesce window covers the outstanding table + * fetches, same as the teardown paths do. + * + * This is the scrub that makes it safe to enable bus mastering: + * nbl_probe() calls it with bus mastering still disabled, so no + * new fetch can start while these entries are still VALID. + / + for (func_id = 0; func_id < NBL_MAX_PF; func_id++) + hw_ops->cfg_msix_map(res_mgt->hw_ops_tbl->priv, func_id, + false, 0, 0, 0, 0); + hw_ops->flush_write(res_mgt->hw_ops_tbl->priv); + nbl_intr_quiesce_wait(); + + return intr_mgt; +} + +int nbl_intr_mgt_start(struct nbl_resource_mgt res_mgt) +{ + struct nbl_interrupt_mgt intr_mgt; + int ret; + + intr_mgt = nbl_intr_setup_mgt(res_mgt); + if (IS_ERR(intr_mgt)) { + ret = PTR_ERR(intr_mgt); + return ret; + } + + res_mgt->intr_mgt = intr_mgt; + return 0; +} + +/ + * nbl_intr_mgt_stop - global control-PF interrupt teardown + * + * Phase 1 sets the stopping latch and invalidates every configured + * function's hardware map entry while holding the lock. The lock is + * dropped for the global quiesce window, so a caller that races the + * window is rejected by the latch (checked under the lock), not by the + * lock being held. + * + * The latch is what rejects a caller that raced the window; it is not + * what makes the manager safe to touch. No caller is in flight here by + * construction: nbl_remove() clears bus mastering and stops the device + * before reaching this point, and every sibling PF - the only remote + * source of these calls - is unbound first through its + * DL_FLAG_AUTOPROBE_CONSUMER device link. A caller that did latch the + * pointer earlier and then gets in under the lock hits the latch + * instead: + * - the config path, destroy_msix_map and the enable path of + * nbl_res_intr_set_mailbox_irq() report -ESHUTDOWN. The latch is + * checked ahead of that path's per-function state check, so its + * -ENODEV is only reachable while the manager is not stopping. + * - the en_msix=false disable path consults neither and returns 0, + * which is what lets teardown disarm routing after the latch is set. + * Once this returns res_mgt->intr_mgt is NULL and the public entry + * points report -EINVAL. + / +void nbl_intr_mgt_stop(struct nbl_resource_mgt res_mgt) +{ + struct nbl_interrupt_mgt intr_mgt = res_mgt->intr_mgt; + u16 func_id; + int ret; + + if (!intr_mgt) + return; + + / + * Phase 1: set the stopping latch and invalidate all hardware + * MSIX map entries. A caller racing the quiesce window below is + * either still waiting for the lock, in which case it observes + * stopping == true at its next checkpoint once it is let in, or + * it already holds the lock, in which case this phase waits for + * it to finish. The latch is never cleared. + * + * The en_msix=false mailbox path is exempt from the latch on + * purpose: this phase and the per-function teardown use it to + * disarm routing, and it only rewrites bits that are already + * clear by then. + / + mutex_lock(&intr_mgt->lock); + intr_mgt->stopping = true; + for (func_id = 0; func_id < NBL_MAX_PF; func_id++) { + if (intr_mgt->func_intr_res[func_id].state == + NBL_INTR_FUNC_CONFIGURED) { + dev_info(res_mgt->common->dev, + "intr_mgt_stop: preparing destroy map for func %u\n", + func_id); + ret = __nbl_res_intr_prepare_destroy_msix_map(intr_mgt, + res_mgt, + func_id); + if (ret) + dev_warn(res_mgt->common->dev, + "intr_mgt_stop: prepare destroy map for func %u failed: %d\n", + func_id, ret); + } + } + mutex_unlock(&intr_mgt->lock); + + / + * Global quiesce: wait for straggler DMA table reads after all + * MSIX map entries have been invalidated in hardware, before + * freeing coherent memory. Best-effort only, see + * nbl_intr_quiesce_wait(). + / + nbl_intr_quiesce_wait(); + + / Phase2: safely release MSIX coherent memory and intr resources / + mutex_lock(&intr_mgt->lock); + for (func_id = 0; func_id < NBL_MAX_PF; func_id++) { + if (intr_mgt->func_intr_res[func_id].state == + NBL_INTR_FUNC_DESTROYING) { + ret = __nbl_res_intr_complete_destroy_msix_map(intr_mgt, + res_mgt, + func_id); + if (ret) + dev_warn(res_mgt->common->dev, + "intr_mgt_stop: complete destroy map for func %u failed: %d\n", + func_id, ret); + } + } + / + * Unpublish last. Readers do not synchronize on intr_mgt->lock, + * so this only has to happen after every hardware access and + * every free above; a caller that latched the pointer earlier is + * rejected by the stopping latch. + / + res_mgt->intr_mgt = NULL; + mutex_unlock(&intr_mgt->lock); +} diff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_interrupt.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_interrupt.h new file mode 100644 index 00000000000000..9f66f5e19c984b --- /dev/null +++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_interrupt.h @@ -0,0 +1,21 @@ +/ SPDX-License-Identifier: GPL-2.0 / +/ + * Copyright (c) 2026 Nebula Matrix Limited. + / + +#ifndef _NBL_INTERRUPT_H_ +#define _NBL_INTERRUPT_H_ + +#include "nbl_resource.h" + +#define NBL_MSIX_MAP_TABLE_MAX_ENTRIES 1024 +int nbl_res_intr_destroy_msix_map(struct nbl_resource_mgt res_mgt, + u16 func_id); +int nbl_res_intr_cfg_msix_map(struct nbl_resource_mgt res_mgt, + u16 func_id, u16 num_net_msix, + u16 num_others_msix, + bool net_msix_mask_en); +int nbl_res_intr_set_mailbox_irq(struct nbl_resource_mgt res_mgt, + u16 func_id, u16 vector_id, + bool en_msix); +#endif diff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_resource.c b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_resource.c new file mode 100644 index 00000000000000..49c7d4412e38ad --- /dev/null +++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_resource.c @@ -0,0 +1,157 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026 Nebula Matrix Limited. + / + +#include <linux/pci.h> +#include "nbl_resource.h" + +int nbl_res_func_id_to_vsi_id(struct nbl_resource_mgt res_mgt, u16 func_id, + u16 type, u16 vsi_id) +{ + struct nbl_vsi_info vsi_info = res_mgt->resource_info->vsi_info; + enum nbl_vsi_serv_type dst_type = NBL_VSI_SERV_PF_DATA_TYPE; + struct nbl_common_info common = res_mgt->common; + struct device dev = res_mgt->common->dev; + int pfid = func_id; + u32 rel_pf_id; + int ret; + + if (!common->has_ctrl \|\| !vsi_id) { + dev_dbg(dev, "No control plane or null vsi output ptr\n"); + return -EINVAL; + } + ret = nbl_common_func_id_to_rel_pf_id(common, pfid, &rel_pf_id); + if (ret) + return ret; + if (rel_pf_id >= vsi_info->num) { + dev_err(dev, "PF %d (diff=%u) exceeds vsi_info->num (%u)\n", + pfid, rel_pf_id, vsi_info->num); + return -EINVAL; + } + + ret = nbl_res_pf_dev_vsi_type_to_hw_vsi_type(res_mgt, type, &dst_type); + if (ret) { + dev_err(dev, "Invalid vsi type %u func_id %u\n", type, func_id); + return ret; + } + vsi_id = vsi_info->serv_info[rel_pf_id][dst_type].base_id; + return 0; +} + +int nbl_res_vsi_id_to_pf_id(struct nbl_resource_mgt res_mgt, u16 vsi_id) +{ + struct nbl_vsi_info vsi_info = res_mgt->resource_info->vsi_info; + struct nbl_common_info common = res_mgt->common; + struct device dev = res_mgt->common->dev; + int j = NBL_VSI_SERV_PF_DATA_TYPE; + int pf_id, i; + + if (!common->has_ctrl) { + dev_dbg(dev, "No control plane available\n"); + return -EINVAL; + } + for (i = 0; i < vsi_info->num; i++) { + if (vsi_id >= vsi_info->serv_info[i][j].base_id && + (vsi_id < vsi_info->serv_info[i][j].base_id + + vsi_info->serv_info[i][j].num)) { + pf_id = i + common->mgt_pf; + if (pf_id >= NBL_MAX_PF) { + dev_err(dev, "PF ID overflow\n"); + return -ERANGE; + } + return pf_id; + } + } + + dev_dbg(dev, "VSI ID %u not found\n", vsi_id); + return -ENOENT; +} + +int nbl_res_func_id_to_bdf(struct nbl_resource_mgt res_mgt, u16 func_id, + u8 bus, u8 dev, u8 function) +{ + struct nbl_common_info common = res_mgt->common; + struct nbl_sriov_info sriov_info; + int pfid = func_id; + u8 pf_bus, devfn; + u32 rel_pf_id; + int ret; + + if (!common->has_ctrl \|\| !bus \|\| !dev \|\| !function) + return -EINVAL; + ret = nbl_common_func_id_to_rel_pf_id(common, pfid, &rel_pf_id); + if (ret) + return ret; + if (rel_pf_id >= common->max_pf) { + dev_err(common->dev, + "func_id=%u rel_pf_id=%u exceeds max_pf=%u, no such PF on this board\n", + pfid, rel_pf_id, + common->max_pf); + return -EOPNOTSUPP; + } + sriov_info = res_mgt->resource_info->sriov_info + rel_pf_id; + pf_bus = PCI_BUS_NUM(sriov_info->bdf); + devfn = sriov_info->bdf & 0xff; + bus = pf_bus; + dev = PCI_SLOT(devfn); + function = PCI_FUNC(devfn); + + return 0; +} + +int nbl_res_get_eth_id(struct nbl_resource_mgt res_mgt, u16 func_id, + u16 vsi_id, u8 eth_num, u8 eth_id, u8 logic_eth_id) +{ + struct nbl_eth_info eth_info = res_mgt->resource_info->eth_info; + struct nbl_common_info common = res_mgt->common; + struct device dev = res_mgt->common->dev; + int pfid = func_id; + int rel_pf_id; + int abs_pf_id; + + if (!common->has_ctrl \|\| !eth_num \|\| !eth_id \|\| !logic_eth_id) + return -EINVAL; + abs_pf_id = nbl_res_vsi_id_to_pf_id(res_mgt, vsi_id); + if (abs_pf_id < 0) { + dev_err(dev, "Failed to get PF ID from VSI ID %u\n", vsi_id); + return -EINVAL; + } + if (abs_pf_id != pfid) { + dev_err(dev, "func_id %u does not match pf derived from vsi_id %u\n", + pfid, vsi_id); + return -EINVAL; + } + rel_pf_id = abs_pf_id - common->mgt_pf; + + if (rel_pf_id >= eth_info->eth_num) { + dev_err(dev, "rel_pf_id %d out of range [0, %u)\n", + rel_pf_id, eth_info->eth_num); + return -ERANGE; + } + + eth_num = eth_info->eth_num; + eth_id = eth_info->eth_id[rel_pf_id]; + / + * Logical eth id equals the relative PF id: eth_info setup enforces + * max_pf == eth_num and fills eth_id[] in ascending bitmap-bit + * order, so the Nth PF owns the Nth logical port. + / + logic_eth_id = rel_pf_id; + return 0; +} + +int nbl_res_pf_dev_vsi_type_to_hw_vsi_type(struct nbl_resource_mgt res_mgt, + u16 src_type, + enum nbl_vsi_serv_type dst_type) +{ + switch (src_type) { + case NBL_VSI_DATA: + dst_type = NBL_VSI_SERV_PF_DATA_TYPE; + return 0; + default: + dev_err_once(res_mgt->common->dev, + "Unsupported vsi src_type %u\n", src_type); + return -EINVAL; + } +} diff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_resource.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_resource.h new file mode 100644 index 00000000000000..24cf49475bbd90 --- /dev/null +++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_resource.h @@ -0,0 +1,159 @@ +/ SPDX-License-Identifier: GPL-2.0 / +/ + * Copyright (c) 2026 Nebula Matrix Limited. + / + +#ifndef _NBL_RESOURCE_H_ +#define _NBL_RESOURCE_H_ + +#include <linux/types.h> + +#include "../nbl_include/nbl_include.h" +#include "../nbl_include/nbl_def_channel.h" +#include "../nbl_include/nbl_def_hw.h" +#include "../nbl_include/nbl_def_resource.h" +#include "../nbl_include/nbl_def_common.h" +#include "../nbl_core.h" + +struct nbl_resource_mgt; + +/ --------- INTERRUPT ---------- / +#define NBL_MAX_OTHER_INTERRUPT 1024 +#define NBL_MAX_NET_INTERRUPT 4096 +#define NBL_NET_INTR_BASE NBL_MAX_OTHER_INTERRUPT + +#define NBL_MSIX_MAP_VALID_MASK BIT(0) +#define NBL_MSIX_MAP_INDEX_MASK GENMASK(13, 1) +#define NBL_MSIX_MAP_RSV_MASK GENMASK(15, 14) + +struct nbl_msix_map { + __le16 data; +}; + +struct nbl_msix_map_table { + struct nbl_msix_map base_addr; + dma_addr_t dma; + size_t size; +}; + +/* + * Per-function MSI-X resource state. + * + * IDLE - nothing configured for this function. + * CONFIGURED - map valid in hardware, vectors owned by this function. + * DESTROYING - map invalidated, teardown in flight: neither the vector + * bitmaps nor the coherent table may be released until the + * hardware-DMA quiesce window has elapsed. + * + * The single-function destroy path holds intr_mgt->lock across that + * window. nbl_intr_mgt_stop() cannot, because it quiesces every + * function at once: it drops the lock and relies on the stopping latch + * to keep a concurrent configuration from installing a map that the + * in-flight teardown would free. + / +enum nbl_intr_func_state { + NBL_INTR_FUNC_IDLE = 0, + NBL_INTR_FUNC_CONFIGURED, + NBL_INTR_FUNC_DESTROYING, +}; + +struct nbl_func_interrupt_resource_mng { + u16 num_interrupts; + u16 num_net_interrupts; + u16 interrupts; + struct nbl_msix_map_table msix_map_table; + u8 state; /* enum nbl_intr_func_state / +}; + +struct nbl_interrupt_mgt { + struct mutex lock; / Protects the bitmaps + func_intr_res[] / + DECLARE_BITMAP(intr_net_bmap, NBL_MAX_NET_INTERRUPT); + DECLARE_BITMAP(intr_other_bmap, NBL_MAX_OTHER_INTERRUPT); + / One-way latch: set on teardown, never cleared / + bool stopping; + / + * Indexed by the raw hardware function id. Only PF entries are ever + * used: the public entry points reject ids outside the PF range (VFs + * included) with -EINVAL, and nbl_res_intr_cfg_msix_map() also + * rejects a PF that is not present on this board. The destroy and + * mailbox paths only bound-check NBL_MAX_PF; they rely on their + * callers passing either common->mgt_pf or an id that the config + * path has already validated. + / + struct nbl_func_interrupt_resource_mng func_intr_res[NBL_MAX_PF]; +}; + +/ --------- INFO ---------- / +struct nbl_sriov_info { + unsigned int bdf; +}; + +struct nbl_eth_info { + u8 eth_num; + u8 resv[3]; + u8 eth_id[NBL_MAX_ETHERNET]; +}; + +enum nbl_vsi_serv_type { + NBL_VSI_SERV_PF_DATA_TYPE, + NBL_VSI_SERV_MAX_TYPE, +}; + +struct nbl_vsi_serv_info { + u16 base_id; + u16 num; +}; + +struct nbl_vsi_info { + u16 num; + struct nbl_vsi_serv_info serv_info[NBL_MAX_ETHERNET] + [NBL_VSI_SERV_MAX_TYPE]; +}; + +struct nbl_resource_info { + struct nbl_sriov_info sriov_info; + struct nbl_eth_info eth_info; + struct nbl_vsi_info vsi_info; + struct nbl_board_port_info board_info; +}; + +struct nbl_resource_mgt { + struct nbl_common_info common; + struct nbl_resource_info resource_info; + struct nbl_channel_ops_tbl chan_ops_tbl; + struct nbl_hw_ops_tbl hw_ops_tbl; + /* + * Published interrupt manager, control PF only. The public entry + * points read this pointer without the manager lock and then pass + * the value they read down to the internal helpers; + * nbl_intr_mgt_stop() clears it under the manager lock as its last + * step, after every hardware access and every free, so a caller + * that reads it after that gets -EINVAL. + * + * The object is devres-owned and is released together with the + * device rather than by nbl_intr_mgt_stop(), so a latched pointer + * is never dereferenced into freed memory. That is not what keeps + * a concurrent caller safe, though: see nbl_intr_mgt_stop() for + * the ordering that makes sure there is none. A caller that + * latched the pointer before it was cleared and then gets in under + * the lock hits the stopping latch: the config path, + * destroy_msix_map and the mailbox-enable path report -ESHUTDOWN, + * while the mailbox-disable path (en_msix == false) returns 0 + * without consulting the latch. + / + struct nbl_interrupt_mgt intr_mgt; +}; + +int nbl_res_vsi_id_to_pf_id(struct nbl_resource_mgt res_mgt, u16 vsi_id); +int nbl_res_func_id_to_vsi_id(struct nbl_resource_mgt res_mgt, u16 func_id, + u16 type, u16 vsi_id); +int nbl_res_func_id_to_bdf(struct nbl_resource_mgt res_mgt, u16 func_id, + u8 bus, u8 dev, u8 function); +int nbl_res_get_eth_id(struct nbl_resource_mgt res_mgt, u16 func_id, + u16 vsi_id, u8 eth_num, u8 eth_id, u8 logic_eth_id); +int nbl_intr_mgt_start(struct nbl_resource_mgt res_mgt); +int nbl_res_pf_dev_vsi_type_to_hw_vsi_type(struct nbl_resource_mgt res_mgt, + u16 src_type, + enum nbl_vsi_serv_type dst_type); +void nbl_intr_mgt_stop(struct nbl_resource_mgt res_mgt); +#endif diff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_channel.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_channel.h new file mode 100644 index 00000000000000..5aa16d4cd4321f --- /dev/null +++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_channel.h @@ -0,0 +1,175 @@ +/ SPDX-License-Identifier: GPL-2.0 / +/ + * Copyright (c) 2026 Nebula Matrix Limited. + / + +#ifndef _NBL_DEF_CHANNEL_H_ +#define _NBL_DEF_CHANNEL_H_ + +#include <linux/types.h> +#include <linux/build_bug.h> + +struct nbl_channel_mgt; +struct nbl_adapter; + +typedef void (nbl_chan_resp)(void , u16, u16, void , u32); + +/* + * Mailbox wire opcodes, stable wire ABI shared between driver and firmware. + * Each opcode has a fixed assigned number to preserve compatibility. + * ABI compatibility rules: + * 1. New opcodes shall only be appended before NBL_CHAN_MSG_MAILBOX_MAX; + * 2. Reordering, inserting or deleting existing enumerators breaks driver- + * firmware interoperability and must be avoided; + * 3. Modifications to existing opcodes require synchronized firmware ABI + * updates. + * + * Only opcodes currently used by in-tree driver logic are defined here. + * Unimplemented feature opcodes (KTLS, IPsec, vDPA, mirror etc.) will be + * added incrementally together with their corresponding driver + * implementation patches. + / +enum nbl_chan_msg_type { + NBL_CHAN_MSG_ACK = 0, + NBL_CHAN_MSG_CONFIGURE_MSIX_MAP = 17, + NBL_CHAN_MSG_DESTROY_MSIX_MAP = 18, + NBL_CHAN_MSG_MAILBOX_SET_IRQ = 19, + NBL_CHAN_MSG_GET_VSI_ID = 21, + NBL_CHAN_MSG_GET_ETH_ID = 67, + / mailbox msg end / + NBL_CHAN_MSG_MAILBOX_MAX, +}; + +enum nbl_chan_state { + NBL_CHAN_IRQ_RDY, + NBL_CHAN_STATE_NBITS +}; + +struct nbl_chan_param_cfg_msix_map { + __le16 num_net_msix; + __le16 num_others_msix; + __le16 msix_mask_en; + __le16 rsvd; +}; + +struct nbl_chan_param_set_mailbox_irq { + __le16 vector_id; + u8 en_msix; + u8 rsvd; +}; + +struct nbl_chan_param_get_vsi_id { + __le16 vsi_id; + __le16 type; +}; + +struct nbl_chan_param_get_eth_id { + __le16 vsi_id; + u8 eth_num; + u8 eth_id; + u8 logic_eth_id; + u8 rsvd[3]; +}; + +struct nbl_board_port_info { + u8 eth_num; + u8 eth_speed; + u8 p4_version; + u8 rsv[5]; +}; + +static_assert(sizeof(struct nbl_chan_param_cfg_msix_map) == 8, + "nbl_chan_param_cfg_msix_map size must be 8 bytes"); +static_assert(sizeof(struct nbl_chan_param_set_mailbox_irq) == 4, + "nbl_chan_param_set_mailbox_irq size must be 4 bytes"); +static_assert(sizeof(struct nbl_chan_param_get_vsi_id) == 4, + "nbl_chan_param_get_vsi_id size must be 4 bytes"); +static_assert(sizeof(struct nbl_chan_param_get_eth_id) == 8, + "nbl_chan_param_get_eth_id size must be 8 bytes"); +static_assert(sizeof(struct nbl_board_port_info) == 8, + "nbl_board_port_info size must be 8 bytes"); + +struct nbl_chan_send_info { + void arg; + size_t arg_len; + void resp; + size_t resp_len; + u16 dstid; + u16 msg_type; + u16 ack; + u16 ack_len; +}; + +struct nbl_chan_ack_info { + void data; + int err; + u32 data_len; + u16 dstid; + u16 msg_type; + u16 msgid; +}; + +enum nbl_channel_type { + NBL_CHAN_TYPE_MAILBOX, + NBL_CHAN_TYPE_MAX +}; + +static inline void +nbl_chan_fill_send_info(struct nbl_chan_send_info info, + u16 dst_id, u16 msg_type, + void argument, u32 arg_length, + void response, u32 resp_length, + bool need_ack) +{ + info->dstid = dst_id; + info->msg_type = msg_type; + info->arg = argument; + info->arg_len = arg_length; + info->resp = response; + info->resp_len = resp_length; + info->ack = need_ack; +} + +static inline void +nbl_chan_fill_ack_info(struct nbl_chan_ack_info info, + u16 dst_id, u16 msg_type, u16 msg_id, + int err_code, void ack_data, u32 data_length) +{ + info->dstid = dst_id; + info->msg_type = msg_type; + info->msgid = msg_id; + info->err = err_code; + info->data = ack_data; + info->data_len = data_length; +} + +struct nbl_channel_ops { + int (send_msg)(struct nbl_channel_mgt chan_mgt, + struct nbl_chan_send_info chan_send); + int (send_ack)(struct nbl_channel_mgt chan_mgt, + struct nbl_chan_ack_info chan_ack); + int (register_msg)(struct nbl_channel_mgt chan_mgt, u16 msg_type, + nbl_chan_resp func, void callback_priv); + void (cfg_chan_qinfo_map_table)(struct nbl_channel_mgt chan_mgt, + u8 bus, u8 devid); + bool (check_queue_exist)(struct nbl_channel_mgt chan_mgt, + u8 chan_type); + int (setup_queue)(struct nbl_channel_mgt chan_mgt, u8 chan_type); + int (teardown_queue)(struct nbl_channel_mgt chan_mgt, u8 chan_type); + void (clean_queue_subtask)(struct nbl_channel_mgt chan_mgt, + u8 chan_type); + void (register_chan_task)(struct nbl_channel_mgt chan_mgt, + u8 chan_type, struct work_struct task); + void (set_queue_state)(struct nbl_channel_mgt chan_mgt, + enum nbl_chan_state state, u8 chan_type, + u8 set); +}; + +struct nbl_channel_ops_tbl { + struct nbl_channel_ops ops; + struct nbl_channel_mgt priv; +}; + +int nbl_chan_init_common(struct nbl_adapter adapter); +void nbl_chan_remove_common(struct nbl_adapter adapter); +#endif diff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_common.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_common.h index ea646d1efc2dfb..3e18a2a97850e6 100644 --- a/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_common.h +++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_common.h @@ -11,7 +11,24 @@ #include <linux/device.h> #include "nbl_include.h" +#define NBL_TWO_ETHERNET_PORT 2 +#define NBL_FOUR_ETHERNET_PORT 4 +#define NBL_DEFAULT_VSI_ID_GAP 1024 +#define NBL_TWO_ETHERNET_VSI_ID_GAP 512 +#define NBL_FOUR_ETHERNET_VSI_ID_GAP 256 + +static inline u32 nbl_vsi_id_gap(u32 m) +{ + if (m == NBL_FOUR_ETHERNET_PORT) + return NBL_FOUR_ETHERNET_VSI_ID_GAP; + else if (m == NBL_TWO_ETHERNET_PORT) + return NBL_TWO_ETHERNET_VSI_ID_GAP; + + return NBL_DEFAULT_VSI_ID_GAP; +} + struct nbl_common_info { + struct workqueue_struct wq; struct pci_dev pdev; struct device dev; u16 vsi_id; @@ -23,9 +40,15 @@ struct nbl_common_info { u8 devid; u8 bus; u8 hw_bus; + u16 mgt_pf; u8 has_ctrl; u8 has_net; + u8 max_pf; }; +void nbl_common_destroy_wq(struct nbl_common_info common); +int nbl_common_create_wq(struct nbl_common_info common); +int nbl_common_func_id_to_rel_pf_id(struct nbl_common_info common, u32 pf_id, + u32 rel_pf_id); #endif diff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_dev.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_dev.h new file mode 100644 index 00000000000000..a66c633a0e7af7 --- /dev/null +++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_dev.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0 / +/ + * Copyright (c) 2026 Nebula Matrix Limited. + / + +#ifndef _NBL_DEF_DEV_H_ +#define _NBL_DEF_DEV_H_ + +struct nbl_adapter; + +int nbl_dev_init(struct nbl_adapter adapter); +void nbl_dev_remove(struct nbl_adapter adapter); +int nbl_dev_start(struct nbl_adapter adapter); +void nbl_dev_stop(struct nbl_adapter adapter); + +#endif diff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_dispatch.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_dispatch.h new file mode 100644 index 00000000000000..7da9e242c621e4 --- /dev/null +++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_dispatch.h @@ -0,0 +1,83 @@ +/ SPDX-License-Identifier: GPL-2.0 / +/ + * Copyright (c) 2026 Nebula Matrix Limited. + / + +#ifndef _NBL_DEF_DISPATCH_H_ +#define _NBL_DEF_DISPATCH_H_ + +#include <linux/types.h> + +struct nbl_dispatch_mgt; +struct nbl_adapter; +enum { + NBL_DISP_CTRL_LVL_MGT, + NBL_DISP_CTRL_LVL_NET, + NBL_DISP_CTRL_LVL_MAX, +}; + +/* + * struct nbl_dispatch_ops - dispatch control plane operation callbacks + * @init_module: chip-wide datapath init, forwarded to the resource layer + * (res_ops->init_module() and then hw_ops->init_module()); + * on leonis it programs the chip-wide datapath registers and + * sets driver_status=true. Returns -EOPNOTSUPP if the + * resource layer installs no init_module op. Control-PF + * exclusive: the pointer is NULL on a function without + * has_ctrl, so the caller must check has_ctrl first. + * @deinit_module: chip-wide teardown, forwarded the same way; on leonis + * it clears driver_status, which starts an asynchronous + * firmware per-PF cleanup pass with no completion status + * bit, waited for best-effort only. The caller must have + * stopped the mailbox channel and drained all in-flight + * mailbox DMA before calling it, see nbl_dev_remove(). + * Returns void. Control-PF exclusive, same NULL rule as + * @init_module. + * @cfg_msix_map: configure function msix mapping table + * @destroy_msix_map: tear down msix mapping resource + * @set_mailbox_irq: bind mailbox interrupt to specified msix vector + * @get_vsi_id: resolve VSI ID by type + * @get_eth_id: resolve eth port info from VSI ID + * + * These callbacks are not dispatch-layer setup/teardown: the dispatch + * structures are devm-allocated in nbl_disp_init(), so nbl_disp_remove() + * has nothing to undo. The five responders registered by + * nbl_disp_setup_msg() are owned by the channel layer and go away with + * it, so they need no explicit unregistration either; the teardown order + * that makes that safe is documented with the code that performs it, in + * the next patch in this series. They forward chip/firmware control down + * to the layers below instead: a control-PF function calls the resource + * ops directly, a non-control function sends a mailbox request over the + * channel layer. + * + * Warning: init_module/deinit_module are control-PF exclusive. The five + * resource ops (cfg_msix_map, destroy_msix_map, set_mailbox_irq, + * get_vsi_id, get_eth_id) are PF-only and resolve to either a local + * resource call (control PF) or a mailbox RPC (non-control PF with + * has_net). A function with neither has_ctrl nor has_net leaves these + * pointers NULL; callers must not invoke them on such functions. VFs are + * rejected by the responders with -EPERM. + / +struct nbl_dispatch_ops { + int (init_module)(struct nbl_dispatch_mgt disp_mgt); + void (deinit_module)(struct nbl_dispatch_mgt disp_mgt); + int (cfg_msix_map)(struct nbl_dispatch_mgt disp_mgt, + u16 num_net_msix, u16 num_others_msix, + bool net_msix_mask_en); + int (destroy_msix_map)(struct nbl_dispatch_mgt disp_mgt); + int (set_mailbox_irq)(struct nbl_dispatch_mgt disp_mgt, + u16 vector_id, bool en_msix); + int (get_vsi_id)(struct nbl_dispatch_mgt disp_mgt, u16 type, + u16 vsi_id); + int (get_eth_id)(struct nbl_dispatch_mgt disp_mgt, u16 vsi_id, + u8 eth_num, u8 eth_id, u8 logic_eth_id); +}; + +struct nbl_dispatch_ops_tbl { + struct nbl_dispatch_ops ops; + struct nbl_dispatch_mgt priv; +}; + +int nbl_disp_init(struct nbl_adapter adapter); +void nbl_disp_remove(struct nbl_adapter adapter); +#endif diff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_hw.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_hw.h index ecbf440e436667..e9cd3136252d86 100644 --- a/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_hw.h +++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_hw.h @@ -8,8 +8,88 @@ #include <linux/types.h> +struct nbl_board_port_info; struct nbl_hw_mgt; struct nbl_adapter; +struct nbl_hw_ops { + int (init_module)(struct nbl_hw_mgt hw_mgt, u8 eth_speed, u8 eth_num); + void (deinit_module)(struct nbl_hw_mgt hw_mgt); + + void (cfg_msix_map)(struct nbl_hw_mgt hw_mgt, u16 func_id, + bool valid, dma_addr_t dma_addr, u8 bus, + u8 devid, u8 function); + void (cfg_msix_info)(struct nbl_hw_mgt hw_mgt, u16 func_id, + bool valid, u16 interrupt_id, u8 bus, + u8 devid, u8 function, + bool net_msix_mask_en); + /* + * flush_write - Flush posted MEMORY-BAR writes + * @hw_mgt: hardware management context + * + * Only valid on the control PF (common->has_ctrl != 0). The + * register read back to order the writes, NBL_HW_DUMMY_REG + * (0x1300904), is covered by the ~64 MiB MEMORY-BAR mapping the + * control PF gets, but sits outside the NBL_REG_NET_ONLY_LEN + * (8 KiB) mapping a net-only PF gets. The leonis implementation + * enforces this with a WARN_ON_ONCE(). + / + void (flush_write)(struct nbl_hw_mgt hw_mgt); + void (update_mailbox_queue_tail_ptr)(struct nbl_hw_mgt hw_mgt, + u16 tail_ptr, u8 txrx); + void (config_mailbox_rxq)(struct nbl_hw_mgt hw_mgt, + dma_addr_t dma_addr, int size_bwid); + void (config_mailbox_txq)(struct nbl_hw_mgt hw_mgt, + dma_addr_t dma_addr, int size_bwid); + void (stop_mailbox_rxq)(struct nbl_hw_mgt hw_mgt); + void (stop_mailbox_txq)(struct nbl_hw_mgt hw_mgt); + /* + * get_host_pf_mask - Fetch host PF mask from firmware k_pf_mask reg + * @hw_mgt: hardware management context + * @pf_mask: output pointer for PF mask value + * + * k_pf_mask register rule: + * bit N == 0 -> PF#N enabled; bit N == 1 -> PF#N masked out. + * bit0 is PF0's mask bit (not reserved); PF0 can be masked but + * the driver requires at least PF0 enabled. + * Only 1/2/4 PFs are supported: + * 1 PF (PF0): mask = 0xfe + * 2 PFs (PF0,PF1): mask = 0xfc + * 4 PFs (PF0~PF3): mask = 0xf0 + * All-zero mask (0x00) means all 8 PFs enabled, which is + * unsupported by the driver. + * + * This op only returns the raw register value and does not validate + * it. Neither consumer exists yet at the commit that introduces it: + * the resource layer, which validates the value, and the channel + * layer, which programs the mailbox QINFO table from it, both arrive + * later in this series, and validation runs before the channel layer + * reads the mask. + / + void (get_host_pf_mask)(struct nbl_hw_mgt hw_mgt, u32 pf_mask); + void (get_real_bus)(struct nbl_hw_mgt hw_mgt, u8 bus); + + void (cfg_mailbox_qinfo)(struct nbl_hw_mgt hw_mgt, u16 func_id, + u8 bus, u8 devid, u8 function); + void (set_mailbox_irq)(struct nbl_hw_mgt hw_mgt, u16 func_id, + bool en_msix, u16 gvec); + void (get_fw_eth_map)(struct nbl_hw_mgt hw_mgt, u32 eth_map); + /** + * get_board_info - Fetch board info from firmware + * @hw_mgt: hardware management context + * @board_info: output pointer for board info structure + * + * Firmware contract: board_info.eth_num MUST equal the number of + * unmasked PFs from get_host_pf_mask(). See get_host_pf_mask for + * details. + / + void (get_board_info)(struct nbl_hw_mgt hw_mgt, + struct nbl_board_port_info board_info); +}; + +struct nbl_hw_ops_tbl { + struct nbl_hw_ops ops; + struct nbl_hw_mgt priv; +}; int nbl_hw_init_leonis(struct nbl_adapter adapter); void nbl_hw_remove_leonis(struct nbl_adapter adapter); diff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_resource.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_resource.h new file mode 100644 index 00000000000000..8dc64e806c1e2b --- /dev/null +++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_resource.h @@ -0,0 +1,38 @@ +/* SPDX-License-Identifier: GPL-2.0 / +/ + * Copyright (c) 2026 Nebula Matrix Limited. + / + +#ifndef _NBL_DEF_RESOURCE_H_ +#define _NBL_DEF_RESOURCE_H_ + +#include <linux/types.h> + +struct nbl_resource_mgt; +struct nbl_adapter; + +struct nbl_resource_ops { + int (init_module)(struct nbl_resource_mgt res_mgt); + void (deinit_module)(struct nbl_resource_mgt res_mgt); + + int (cfg_msix_map)(struct nbl_resource_mgt res_mgt, u16 func_id, + u16 num_net_msix, u16 num_others_msix, + bool net_msix_mask_en); + int (destroy_msix_map)(struct nbl_resource_mgt res_mgt, u16 func_id); + int (set_mailbox_irq)(struct nbl_resource_mgt res_mgt, u16 func_id, + u16 vector_id, bool en_msix); + int (get_vsi_id)(struct nbl_resource_mgt res_mgt, u16 func_id, + u16 type, u16 vsi_id); + int (get_eth_id)(struct nbl_resource_mgt res_mgt, u16 func_id, + u16 vsi_id, u8 eth_num, u8 eth_id, + u8 logic_eth_id); +}; + +struct nbl_resource_ops_tbl { + struct nbl_resource_ops ops; + struct nbl_resource_mgt priv; +}; + +int nbl_res_init_leonis(struct nbl_adapter adapter); +void nbl_res_remove_leonis(struct nbl_adapter adapter); +#endif diff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_include.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_include.h index 14e7b19f9a4cf8..efc4c79a082d25 100644 --- a/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_include.h +++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_include.h @@ -10,6 +10,15 @@ / ------ Basic definitions ------- / #define NBL_DRIVER_NAME "nbl" +#define NBL_MAX_PF 8 +#define NBL_NEXT_ID(id, max) (((id) + 1) % ((max) + 1)) + +#define NBL_MAX_ETHERNET 4 + +enum { + NBL_VSI_DATA = 0, +}; + struct nbl_func_caps { u32 has_ctrl:1; u32 has_net:1; @@ -20,4 +29,23 @@ struct nbl_init_param { struct nbl_func_caps caps; }; +/ + * Firmware ABI defines port speed enum fixed, value 0 represents 10G, cannot + * reassign 0 to INVALID for compatibility + / +enum nbl_fw_port_speed { + NBL_FW_PORT_SPEED_10G, + NBL_FW_PORT_SPEED_25G, + NBL_FW_PORT_SPEED_50G, + NBL_FW_PORT_SPEED_100G, +}; + +/ + * Firmware quirk word @ NBL_LEONIS_QUIRKS_OFFSET (0x140) + * Sentinel value: ~0U (0xFFFFFFFF) = firmware reports no active quirks + * BIT(1): NBL_QUIRK_UVN_PREFETCH_ALIGN – control UVN descriptor prefetch + * selection + / +#define NBL_QUIRK_UVN_PREFETCH_ALIGN BIT(1) + #endif diff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_main.c b/drivers/net/ethernet/nebula-matrix/nbl/nbl_main.c index f2552bc7329376..0bdc7f9940fcb2 100644 --- a/drivers/net/ethernet/nebula-matrix/nbl/nbl_main.c +++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_main.c @@ -7,11 +7,34 @@ #include <linux/pci.h> #include <linux/module.h> #include <linux/bits.h> +#include <linux/delay.h> #include "nbl_include/nbl_include.h" +#include "nbl_include/nbl_def_channel.h" #include "nbl_include/nbl_def_hw.h" +#include "nbl_include/nbl_def_resource.h" +#include "nbl_include/nbl_def_dispatch.h" +#include "nbl_include/nbl_def_dev.h" #include "nbl_include/nbl_def_common.h" #include "nbl_core.h" +/ + * Budget for the device_trylock() loop that takes func 0's device lock in + * nbl_probe_chip_deps(). See the comment there for why a single failed + * trylock must not defer the probe, and why the wait has to stay bounded. + / +#define NBL_MGT_LOCK_RETRY_MIN_US 1000 +#define NBL_MGT_LOCK_RETRY_MAX_US 2000 +#define NBL_MGT_LOCK_MAX_RETRIES 50 +int nbl_core_start(struct nbl_adapter adapter) +{ + return nbl_dev_start(adapter); +} + +void nbl_core_stop(struct nbl_adapter adapter) +{ + nbl_dev_stop(adapter); +} + struct nbl_adapter nbl_core_init(struct pci_dev pdev, struct nbl_init_param param) { @@ -26,6 +49,7 @@ struct nbl_adapter nbl_core_init(struct pci_dev pdev, adapter->pdev = pdev; common = &adapter->common; + common->mgt_pf = 0; common->pdev = pdev; common->dev = &pdev->dev; common->has_ctrl = param->caps.has_ctrl; @@ -36,15 +60,43 @@ struct nbl_adapter nbl_core_init(struct pci_dev pdev, ret = nbl_hw_init_leonis(adapter); if (ret) - goto hw_init_fail; + goto out; + + ret = nbl_chan_init_common(adapter); + if (ret) + goto out_hw_remove; + ret = nbl_res_init_leonis(adapter); + if (ret) + goto out_chan_remove; + + ret = nbl_disp_init(adapter); + if (ret) + goto out_res_remove; + + ret = nbl_dev_init(adapter); + if (ret) + goto out_disp_remove; return adapter; -hw_init_fail: + +out_disp_remove: + nbl_disp_remove(adapter); +out_res_remove: + nbl_res_remove_leonis(adapter); +out_chan_remove: + nbl_chan_remove_common(adapter); +out_hw_remove: + nbl_hw_remove_leonis(adapter); +out: return ERR_PTR(ret); } void nbl_core_remove(struct nbl_adapter adapter) { + nbl_dev_remove(adapter); + nbl_disp_remove(adapter); + nbl_res_remove_leonis(adapter); + nbl_chan_remove_common(adapter); nbl_hw_remove_leonis(adapter); } @@ -67,6 +119,164 @@ static void nbl_get_func_param(struct pci_dev pdev, kernel_ulong_t driver_data, param->caps.has_ctrl = 1; } +/* + * Establish chip-wide dependencies for this PF: + * - for every non-management PF, add a consumer->management PF device + * link. The driver core then guarantees (sysfs unbind, driver + * unregister, hot-unplug alike) that this consumer is released + * BEFORE the func 0 supplier, which is the only teardown order in + * which the chip-global firmware deinit is safe. + * + * The management PF is addressed by the deterministic identity + * (domain, bus, slot, func 0) instead of any name-based scan: hardware + * guarantees PFs are contiguous from func 0 in the same slot. + * + * The func 0 device must already be bound to this driver. While func 0 + * is still probing, its chip-global init (mailbox QINFO map programming, + * datapath setup) can race a sibling's mailbox setup, and a link created + * while the supplier has no driver starts DORMANT and is force-promoted + * (with a WARN backtrace) by device_links_driver_bound() when this probe + * ends, so it never delivers the unbind-order guarantee. + * + * The link is a persistent managed link with DL_FLAG_AUTOPROBE_CONSUMER: + * the core force-unbinds this consumer when func 0 unbinds (keeping the + * teardown order) and re-queues it for probe when func 0 binds again, so + * the sibling PFs and their ports come back without a manual rebind. + * AUTOREMOVE_CONSUMER would delete the link on unbind and cannot be + * combined with AUTOPROBE_CONSUMER (device_link_add() returns NULL for + * that pair). The link is therefore not auto-purged on a failed probe: + * it lives with the two devices and is dropped when either is removed. + * The pointer device_link_add() returns is never dereferenced. + * + * Return: 0 on success, negative errno on failure. + / +static int nbl_probe_chip_deps(struct pci_dev pdev, bool has_ctrl) +{ + struct pci_dev mgt; + int retries; + + if (has_ctrl) + return 0; + + mgt = pci_get_domain_bus_and_slot(pci_domain_nr(pdev->bus), + pdev->bus->number, + PCI_DEVFN(PCI_SLOT(pdev->devfn), 0)); + if (!mgt) { + dev_err(&pdev->dev, + "management PF (func 0) not found on this chip\n"); + return -ENODEV; + } + + / + * That identity is meaningless for a VF: under ARI a VF's devfn + * decodes to function 0 of a slot, so the lookup can resolve to the + * VF itself. A VF is never the management PF, so stop here instead + * of taking our own device lock below. + / + if (mgt == pdev) { + dev_err(&pdev->dev, + "management PF lookup matched this device\n"); + pci_dev_put(mgt); + return -ENODEV; + } + + / + * func 0's device lock is taken with a trylock. This probe already + * runs under device_lock(funcN), so a blocking device_lock(func0) + * would establish the N -> 0 order, while __pci_bus_lock() (AER + * recovery, slot reset) walks the bus in devfn order, i.e. 0 -> N, + * and dev->mutex is lockdep_set_novalidate_class(), so lockdep + * cannot report that inversion. A trylock never blocks, so it + * cannot take part in a deadlock cycle. + * + * Holding the lock is what makes the check and the link creation + * below atomic with respect to func 0: it both gains its driver + * (driver_bound() -> device_links_driver_bound()) and loses it + * (__device_release_driver() -> device_links_driver_cleanup()) + * under device_lock(func0), so func 0 can change neither driver nor + * links.status while we hold it. + * + * One failed trylock is nevertheless not enough to defer on. Func 0 + * binds with device_lock(func0) held, and driver_bound() calls + * driver_deferred_probe_trigger() before it releases that lock, so + * the deferred probe work it queues can re-probe this PF while func + * 0's lock is still taken. driver_probe_device() only re-arms that + * work when deferred_trigger_count changes during the probe, which + * it does not here: func 0's increment happened before this probe + * read the count. A bare -EPROBE_DEFER would therefore leave this + * PF on deferred_probe_pending_list with nothing scheduled to run + * it again. Retrying closes that window - driver_bound() has only a + * bus_notify() and a kobject_uevent() left before it drops the lock. + * + * The retry budget is what keeps this safe. device_lock(funcN) is + * held for the whole loop, so waiting indefinitely would deadlock + * against __pci_bus_lock(); giving up and deferring releases it and + * lets the other side make progress. A holder that is not part of a + * bind - __pci_bus_lock() during AER recovery or a bus/slot reset, + * pci_reset_function() through sysfs, an sriov_numvfs write - can + * outlast the budget, and no bind event follows those, so this PF + * then stays deferred until some later deferred-probe trigger from + * anywhere in the system. There is no way to force one from here. + / + for (retries = 0; !device_trylock(&mgt->dev); retries++) { + if (retries == NBL_MGT_LOCK_MAX_RETRIES) { + dev_dbg(&pdev->dev, + "management PF device lock busy after %u retries, deferring probe\n", + retries); + pci_dev_put(mgt); + return -EPROBE_DEFER; + } + usleep_range(NBL_MGT_LOCK_RETRY_MIN_US, + NBL_MGT_LOCK_RETRY_MAX_US); + } + + / + * The management PF must be bound to this driver, with its probe + * completed. links.status == DL_DEV_DRIVER_BOUND alone only says + * "bound to some driver", so func 0 on vfio-pci or pci-stub + * (driver_override) would pass it and this PF would run with func + * 0's chip-global init never performed. pdev->dev.driver is our + * own driver, set by the core before ->probe() is called. + * + * Deferring is what recovers from this: a successful bind + * re-triggers deferred probing for every pending device + * (driver_bound() -> driver_deferred_probe_trigger()), so this PF is + * retried once func 0 is bound to this driver. That covers a func 0 + * that is not bound yet. It does not cover a func 0 that is bound + * while something else holds its device lock, which is what the + * retry loop above is for and what its budget cannot fully rule out. + / + if (mgt->dev.driver != pdev->dev.driver \|\| + mgt->dev.links.status != DL_DEV_DRIVER_BOUND) { + device_unlock(&mgt->dev); + pci_dev_put(mgt); + return -EPROBE_DEFER; + } + + / + * Create the link while func 0's lock is still held, so that func 0 + * cannot change state in between. With the supplier bound and this + * consumer PROBING, device_link_init_status() starts the link in + * DL_STATE_CONSUMER_PROBE: that is the state + * device_links_driver_bound() expects when this probe finishes, and + * it is what device_links_busy() reports to func 0's own unbind, so + * func 0 cannot be released underneath this PF. + / + if (!device_link_add(&pdev->dev, &mgt->dev, + DL_FLAG_AUTOPROBE_CONSUMER)) { + device_unlock(&mgt->dev); + dev_err(&pdev->dev, + "failed to create device link to management PF\n"); + pci_dev_put(mgt); + return -ENOMEM; + } + + device_unlock(&mgt->dev); + pci_dev_put(mgt); + + return 0; +} + static int nbl_probe(struct pci_dev pdev, const struct pci_device_id id) { @@ -75,28 +285,68 @@ static int nbl_probe(struct pci_dev pdev, struct nbl_adapter adapter; int err; + nbl_get_func_param(pdev, id->driver_data, &param); + / + * Reject a function with neither capability before touching any + * hardware. nbl_disp_refresh_ctrl_ops() leaves every dispatch op NULL + * for such a function, while nbl_dev_start() reaches cfg_msix_map, + * get_vsi_id, get_eth_id and (on unwind) destroy_msix_map through + * those pointers. The state is reachable through driver_override: + * binding this driver to a function that is not in nbl_id_table (a VF, + * or a non-func-0 PF) makes pci_match_device() pass the dummy ID whose + * driver_data 0 leaves has_ctrl and has_net both clear. A foreign + * device at func 0 is unaffected: nbl_get_func_param() gives any + * func 0 device has_ctrl, so it is not rejected here and still runs + * nbl_core_init() with the local control dispatch ops, as it did + * before this check existed. + / + if (!param.caps.has_ctrl && !param.caps.has_net) { + dev_err(dev, "Function has neither ctrl nor net capability\n"); + return -EOPNOTSUPP; + } + err = pcim_enable_device(pdev); if (err) { dev_err(&pdev->dev, "Failed to enable PCI dev, err=%d\n", err); return err; } - nbl_get_func_param(pdev, id->driver_data, &param); / never return fail when DMA_BIT_MASK(64) / dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64)); - pci_set_master(pdev); + / + * Bus mastering is enabled only after nbl_core_init() returns. + * + * FUNCTION_MSIX_MAP survives a kexec or a forced unload without + * FLR and there is no .shutdown callback to scrub it, so at this + * point every map entry left VALID by a previous kernel still + * holds a DMA address that this kernel has already reallocated. + * Those entries tag their table fetches as bus-master reads, and + * the only thing standing between such a fetch and a read of + * reused memory is Bus Master Enable, which the PCI core cleared + * on the way into this kernel. nbl_intr_setup_mgt() invalidates + * the entries from inside nbl_core_init(); enabling bus mastering + * before that would re-open the window. Everything up to that + * point is host-side setup plus MMIO and nothing in it waits for + * the device, so it does not need bus mastering. + / + err = nbl_probe_chip_deps(pdev, param.caps.has_ctrl); + if (err) + return err; adapter = nbl_core_init(pdev, &param); if (IS_ERR(adapter)) { dev_err(dev, "Nbl adapter init fail: %pe\n", adapter); - err = PTR_ERR(adapter); - goto adapter_init_err; + return PTR_ERR(adapter); } + + pci_set_master(pdev); pci_set_drvdata(pdev, adapter); - return 0; -adapter_init_err: - pci_clear_master(pdev); + err = nbl_core_start(adapter); + if (err) { + nbl_core_remove(adapter); + pci_clear_master(pdev); + } return err; } @@ -107,9 +357,28 @@ static void nbl_remove(struct pci_dev pdev) if (!adapter) return; pci_set_drvdata(pdev, NULL); - nbl_core_remove(adapter); + nbl_core_stop(adapter); + /* + * Stop bus mastering before any teardown runs. + * + * nbl_core_remove() releases the coherent MSI-X map tables whose + * DMA addresses are still programmed in FUNCTION_MSIX_MAP, and the + * only thing standing between a late pcompleter fetch and a read + * of freed memory is a bounded best-effort delay - the chip + * exposes no completion status for those fetches (see + * nbl_intr_quiesce_wait()). Clearing Bus Master Enable first + * keeps the device from starting a new fetch at all, so that + * residual window only covers requests already in flight. + * + * nbl_core_stop() above has already torn down this function's own + * map, so what is left to protect here is the control PF's table of + * per-function entries, freed from nbl_intr_mgt_stop() inside + * nbl_core_remove(). + / pci_clear_master(pdev); + + nbl_core_remove(adapter); } /
3/2	2026/11/02 14:59	llm	patch-evaluator	0m Model: gemini-3.8-flash Args: null Results: null Tokens: input: 112611 output: 37 thoughts: 963 Thoughts: Initial Assessment of the Nebula Matrix Patch Series Okay, so I've got a patch series here targeting the amd64 architecture, and it seems to be centered around the `drivers/net/ethernet/nebula-matrix/nbl` directory. My immediate thought is, "What the heck is Nebula Matrix?". I need to understand what this driver is actually doing before I can even begin a proper review. The diff lists a substantial number of files being added or modified within that directory: Makefiles, core functionality, channel management, hardware-specific code (including `leonis`), interrupt handling, resource management, and some include files. This looks like a fairly comprehensive set of changes, potentially a new driver implementation or a significant overhaul of an existing one. It's a bit overwhelming at first glance, but nothing I can't handle. My first port of call is to determine the nature of the hardware this driver supports. Is this a physical PCI ethernet NIC? Based on the path, I'm leaning toward a specific hardware vendor ("Nebula Matrix Limited"). I need to check if this falls under "structurally unreachable hardware," as this would mean it's unlikely to be able to be reasonably tested. I need to be sure before I dive into the code. I need to quickly figure out whether this is a PCI device, and who makes it. I'm thinking I'll use `grepper` or `read-file` to dig into the kernel's source and check the Kconfig files and any PCI device IDs associated with this driver. I'll search for things like device identifiers, vendor IDs, and any mentions of specific hardware models. This is my priority now, before I spend time on code. Once I know the nature of the hardware, I can accurately assess its testability and whether the patch falls under any negative criteria, before I even start reviewing the code.
4/2	2026/11/02 15:00	tool	grepper	0m Args: { "Expression": "CONFIG_NBL\|nebula-matrix", "PathPrefixes": [ "drivers/net/ethernet/" ] } Results: { "Output": "drivers/net/ethernet/Kconfig=132=source \"drivers/net/ethernet/natsemi/Kconfig\"\ndrivers/net/ethernet/Kconfig:133:source \"drivers/net/ethernet/nebula-matrix/Kconfig\"\ndrivers/net/ethernet/Kconfig-134-source \"drivers/net/ethernet/netronome/Kconfig\"\n--\ndrivers/net/ethernet/Makefile=69=obj-$(CONFIG_NET_VENDOR_NATSEMI) += natsemi/\ndrivers/net/ethernet/Makefile:70:obj-$(CONFIG_NET_VENDOR_NEBULA_MATRIX) += nebula-matrix/\ndrivers/net/ethernet/Makefile-71-obj-$(CONFIG_NET_VENDOR_NETRONOME) += netronome/\n--\ndrivers/net/ethernet/nebula-matrix/Kconfig=18=config NBL\n--\ndrivers/net/ethernet/nebula-matrix/Kconfig-26-\t More specific information on configuring the driver is in\ndrivers/net/ethernet/nebula-matrix/Kconfig:27:\t \u003cfile:Documentation/networking/device_drivers/ethernet/nebula-matrix/nbl.rst\u003e.\ndrivers/net/ethernet/nebula-matrix/Kconfig-28-\n--\ndrivers/net/ethernet/nebula-matrix/Makefile-5-\ndrivers/net/ethernet/nebula-matrix/Makefile:6:obj-$(CONFIG_NBL) += nbl/\n--\ndrivers/net/ethernet/nebula-matrix/nbl/Makefile-3-\ndrivers/net/ethernet/nebula-matrix/nbl/Makefile:4:obj-$(CONFIG_NBL) := nbl.o\ndrivers/net/ethernet/nebula-matrix/nbl/Makefile-5-\n--\ndrivers/net/ethernet/nebula-matrix/nbl/nbl_main.c=460=MODULE_DESCRIPTION(\"Nebula Matrix Network Driver\");\ndrivers/net/ethernet/nebula-matrix/nbl/nbl_main.c:461:MODULE_AUTHOR(\"Illusion Wang \u003cillusion.wang@nebula-matrix.com\u003e\");\n" }
5/2	2026/11/02 15:00	llm	patch-evaluator	0m Model: gemini-3.8-flash Args: null Results: null Tokens: input: 114532 output: 40 thoughts: 44 Thoughts: Okay, let's dive into this. Here's my thought process on the matter: Analyzing the Nebula-Matrix Ethernet Driver Right, so I need to get a handle on the Nebula-Matrix Ethernet driver, specifically. I'm going to start by getting a handle on its configuration options, so let's check out `drivers/net/ethernet/nebula-matrix/Kconfig`. That should give me the big picture – the different features that can be enabled or disabled, any dependencies, and the overall structure of how this driver is configured. Understanding the Kconfig is key to navigating the rest of the code effectively. Then, I need to understand the core logic of the driver. I will look into the main driver file `drivers/net/ethernet/nebula-matrix/nbl/nbl_main.c` because it appears to be central to the network buffer management (nbl). That's likely where the interface initialization, packet processing, and interrupt handling will be centered. I'm expecting to find the standard stuff here – registration with the network stack, probe and remove functions, interrupt handlers, and possibly some device-specific initialization sequences. From there, I can begin to unravel its functionality and how it fits into the broader networking ecosystem.
6/2	2026/11/02 15:00	tool	read-file	0m Args: { "File": "drivers/net/ethernet/nebula-matrix/Kconfig", "FirstLine": 1, "LineCount": 40 } Results: { "Contents": " 1:\t# SPDX-License-Identifier: GPL-2.0\n 2:\t#\n 3:\t# Nebula-matrix network device configuration\n 4:\t#\n 5:\t\n 6:\tconfig NET_VENDOR_NEBULA_MATRIX\n 7:\t\tbool \"Nebula-matrix devices\"\n 8:\t\tdefault y\n 9:\t\thelp\n 10:\t\t If you have a network (Ethernet) card belonging to this class, say Y.\n 11:\t\t Note that the answer to this question doesn't directly affect the\n 12:\t\t kernel: saying N will just cause the configurator to skip all\n 13:\t\t the questions about Nebula-matrix cards. If you say Y, you will be asked\n 14:\t\t for your specific card in the following questions.\n 15:\t\n 16:\tif NET_VENDOR_NEBULA_MATRIX\n 17:\t\n 18:\tconfig NBL\n 19:\t\ttristate \"Nebula-matrix Ethernet Controller m18110/m18000 support\"\n 20:\t\tdepends on PCI \u0026\u0026 (64BIT \|\| COMPILE_TEST) \u0026\u0026 !CPU_BIG_ENDIAN\n 21:\t\thelp\n 22:\t\t This driver provides the core infrastructure for Nebula-matrix\n 23:\t\t Ethernet Controller m18110/m18000 Family of devices, including\n 24:\t\t PCI enumeration, firmware mailbox and channel communication.\n 25:\t\n 26:\t\t More specific information on configuring the driver is in\n 27:\t\t \u003cfile:Documentation/networking/device_drivers/ethernet/nebula-matrix/nbl.rst\u003e.\n 28:\t\n 29:\t\t To compile this driver as a module, choose M here. The module\n 30:\t\t will be called nbl.\n 31:\t\n 32:\tendif # NET_VENDOR_NEBULA_MATRIX\n" }
7/2	2026/11/02 15:00	llm	patch-evaluator	0m Model: gemini-3.8-flash Args: null Results: null Tokens: input: 115373 output: 39 thoughts: 21 Thoughts: Here's my take on those thoughts, tailored for an expert audience: Diving into `pci_device_id` in `nbl_main.c` Okay, I'm thinking, let's zero in on `pci_device_id` and the context within `nbl_main.c`. My initial assumption is that we're likely dealing with some device initialization or driver binding, and this identifier is going to be crucial. Given the naming convention and my experience, this could be a table or structure defining the supported PCI devices. The goal here is probably to understand how devices are identified and matched against the driver. I need to see how the ID is structured, how it's used in comparisons, and where the appropriate driver module gets loaded based on a match. Are we using a simple vendor/device ID comparison, or are there more complex matching criteria involved, like class codes or subsystem IDs? I need to check for any potential for conflicts or ambiguities in the ID handling. Is there any handling for device remapping or driver overrides? Time to see if the implementation makes sense.
8/2	2026/11/02 15:00	tool	grepper	0m Args: { "Expression": "pci_device_id", "PathPrefixes": [ "drivers/net/ethernet/nebula-matrix/" ] } Results: { "Output": "drivers/net/ethernet/nebula-matrix/nbl/nbl_main.c=280=static int nbl_probe(struct pci_dev pdev,\ndrivers/net/ethernet/nebula-matrix/nbl/nbl_main.c:281:\t\t const struct pci_device_id id)\ndrivers/net/ethernet/nebula-matrix/nbl/nbl_main.c-282-{\n--\ndrivers/net/ethernet/nebula-matrix/nbl/nbl_main.c=353=static void nbl_remove(struct pci_dev pdev)\n--\ndrivers/net/ethernet/nebula-matrix/nbl/nbl_main.c-411- /\ndrivers/net/ethernet/nebula-matrix/nbl/nbl_main.c:412:static const struct pci_device_id nbl_id_table[] = {\ndrivers/net/ethernet/nebula-matrix/nbl/nbl_main.c-413-\t{ PCI_DEVICE(NBL_VENDOR_ID, NBL_DEVICE_ID_M18110),\n" }
9/2	2026/11/02 15:00	llm	patch-evaluator	0m Model: gemini-3.8-flash Args: null Results: null Tokens: input: 115962 output: 49 thoughts: 19 Thoughts: Here's my attempt at summarizing the thought process, tailored for an expert audience: Diving into `nbl_main.c` Okay, let's zero in on `nbl_main.c` – specifically, around line 410. I know this code base pretty well, and I have a nagging feeling about something related to network buffer lists that's probably being addressed in this area. My intuition tells me there's likely an interesting data structure manipulation going on around that line. Perhaps some sort of buffer allocation, deallocation, or list traversal within the context of the network subsystem? I'm anticipating some critical section synchronization that I need to mentally trace. Let's see if this hunch holds up, and what the specific logic surrounding line 410 entails. Time to get my mental compiler going!
10/2	2026/11/02 15:00	tool	read-file	0m Args: { "File": "drivers/net/ethernet/nebula-matrix/nbl/nbl_main.c", "FirstLine": 400, "LineCount": 40 } Results: { "Contents": " 400:\t#define NBL_DEVICE_ID_M18000\t\t\t0x340b\n 401:\t#define NBL_DEVICE_ID_M18000_LX\t\t\t0x340c\n 402:\t#define NBL_DEVICE_ID_M18000_BASE_T\t\t0x340d\n 403:\t#define NBL_DEVICE_ID_M18000_LX_BASE_T\t\t0x340e\n 404:\t#define NBL_DEVICE_ID_M18000_OCP\t\t0x340f\n 405:\t#define NBL_DEVICE_ID_M18000_LX_OCP\t\t0x3410\n 406:\t#define NBL_DEVICE_ID_M18000_BASE_T_OCP\t\t0x3411\n 407:\t#define NBL_DEVICE_ID_M18000_LX_BASE_T_OCP\t0x3412\n 408:\t\n 409:\t/* All below IDs belong to Leonis ASIC family, different form-factor variants,\n 410:\t * share the same hardware initialization flow without differentiated ops.\n 411:\t */\n 412:\tstatic const struct pci_device_id nbl_id_table[] = {\n 413:\t\t{ PCI_DEVICE(NBL_VENDOR_ID, NBL_DEVICE_ID_M18110),\n 414:\t\t .driver_data = BIT(NBL_CAP_HAS_NET_BIT) },\n 415:\t\t{ PCI_DEVICE(NBL_VENDOR_ID, NBL_DEVICE_ID_M18110_LX),\n 416:\t\t .driver_data = BIT(NBL_CAP_HAS_NET_BIT) },\n 417:\t\t{ PCI_DEVICE(NBL_VENDOR_ID, NBL_DEVICE_ID_M18110_BASE_T),\n 418:\t\t .driver_data = BIT(NBL_CAP_HAS_NET_BIT) },\n 419:\t\t{ PCI_DEVICE(NBL_VENDOR_ID, NBL_DEVICE_ID_M18110_LX_BASE_T),\n 420:\t\t .driver_data = BIT(NBL_CAP_HAS_NET_BIT) },\n 421:\t\t{ PCI_DEVICE(NBL_VENDOR_ID, NBL_DEVICE_ID_M18110_OCP),\n 422:\t\t .driver_data = BIT(NBL_CAP_HAS_NET_BIT) },\n 423:\t\t{ PCI_DEVICE(NBL_VENDOR_ID, NBL_DEVICE_ID_M18110_LX_OCP),\n 424:\t\t .driver_data = BIT(NBL_CAP_HAS_NET_BIT) },\n 425:\t\t{ PCI_DEVICE(NBL_VENDOR_ID, NBL_DEVICE_ID_M18110_BASE_T_OCP),\n 426:\t\t .driver_data = BIT(NBL_CAP_HAS_NET_BIT) },\n 427:\t\t{ PCI_DEVICE(NBL_VENDOR_ID, NBL_DEVICE_ID_M18110_LX_BASE_T_OCP),\n 428:\t\t .driver_data = BIT(NBL_CAP_HAS_NET_BIT) },\n 429:\t\t{ PCI_DEVICE(NBL_VENDOR_ID, NBL_DEVICE_ID_M18000),\n 430:\t\t .driver_data = BIT(NBL_CAP_HAS_NET_BIT) },\n 431:\t\t{ PCI_DEVICE(NBL_VENDOR_ID, NBL_DEVICE_ID_M18000_LX),\n 432:\t\t .driver_data = BIT(NBL_CAP_HAS_NET_BIT) },\n 433:\t\t{ PCI_DEVICE(NBL_VENDOR_ID, NBL_DEVICE_ID_M18000_BASE_T),\n 434:\t\t .driver_data = BIT(NBL_CAP_HAS_NET_BIT) },\n 435:\t\t{ PCI_DEVICE(NBL_VENDOR_ID, NBL_DEVICE_ID_M18000_LX_BASE_T),\n 436:\t\t .driver_data = BIT(NBL_CAP_HAS_NET_BIT) },\n 437:\t\t{ PCI_DEVICE(NBL_VENDOR_ID, NBL_DEVICE_ID_M18000_OCP),\n 438:\t\t .driver_data = BIT(NBL_CAP_HAS_NET_BIT) },\n 439:\t\t{ PCI_DEVICE(NBL_VENDOR_ID, NBL_DEVICE_ID_M18000_LX_OCP),\n" }
11/2	2026/11/02 15:00	llm	patch-evaluator	0m Model: gemini-3.8-flash Args: null Results: null Tokens: input: 117438 output: 128 thoughts: 1434 Thoughts: My Assessment: `nbl` Driver Not Fuzzable Okay, let's break this down. My initial assessment is that this `nbl` driver, being a vendor-specific PCIe Ethernet NIC driver for the Nebula Matrix Leonis ASIC family (m18110/m18000), falls squarely into the "structurally unreachable hardware" category, as per the negative criteria for fuzzing. The prompt explicitly states that we should set `WorthFuzzing=false` if the modified code is exclusively within drivers for hardware not emulated in standard QEMU. First, I need to confirm my initial assumption. I need to make absolutely sure that `nbl` doesn't have some software emulation path or virtual bus integration. No. This is a PCIe driver, and it's for physical hardware. QEMU and GCE definitely don't emulate these specific Nebula Matrix Leonis devices. QEMU supports the usual suspects – e1000, virtio-net, etc. – but not this. Now, I'm going to double-check the files to leave no doubt. I can list the files to be sure. Alright, `Makefile`, `nbl_channel`, `nbl_core`, `nbl_hw`, `nbl_include` and `nbl_main.c`. Everything, and I mean everything, is inside `drivers/net/ethernet/nebula-matrix/nbl/`. No escape. No hooks. It's all tightly coupled to that physical Leonis ASIC. This doesn't integrate with core networking or expose a virtual interface in a way that would be accessible without the physical PCIe card. Therefore, because this driver requires a physical card that isn't emulated and only has code that is for the Nebula Matrix Leonis PCIe Ethernet NIC, it is structurally unreachable in standard fuzzing environments. There's no way to execute any of this code without the physical hardware. `WorthFuzzing` is unequivocally `false`. The `FocusSymbols` and `EnableConfigs` are empty. We're done here. Let's send `set-results`.
12/2	2026/11/02 15:01	tool	set-results	0m Args: { "EnableConfigs": [], "FocusSymbols": [], "Reasoning": "The patch series only touches the Nebula Matrix (nbl) Ethernet driver (drivers/net/ethernet/nebula-matrix/nbl/), which is a PCIe driver for physical Nebula Matrix Leonis ASIC family NICs (m18110/m18000). These devices require dedicated physical PCIe hardware and are not emulated in standard QEMU/GCE virtualized test environments. Consequently, the modified code is structurally unreachable for fuzzing.", "WorthFuzzing": false } Results: { "EnableConfigs": null, "FocusSymbols": [], "Reasoning": "The patch series only touches the Nebula Matrix (nbl) Ethernet driver (drivers/net/ethernet/nebula-matrix/nbl/), which is a PCIe driver for physical Nebula Matrix Leonis ASIC family NICs (m18110/m18000). These devices require dedicated physical PCIe hardware and are not emulated in standard QEMU/GCE virtualized test environments. Consequently, the modified code is structurally unreachable for fuzzing.", "WorthFuzzing": false }
13/1	2026/11/02 15:01	action	If	0m Args: { "WorthFuzzing": false } Results: null

{
  "PatchDiff": "commit eb6ce827881d54724a7b175422cd39cff38773db\nAuthor: syz-cluster \u003ctriage@syzkaller.com\u003e\nDate:   Mon Nov 2 14:59:56 2026 +0000\n\n    syz-cluster: applied patch under review\n\ndiff --git a/drivers/net/ethernet/nebula-matrix/nbl/Makefile b/drivers/net/ethernet/nebula-matrix/nbl/Makefile\nindex cc060cf8bf7596..71fbe3ee7e6204 100644\n--- a/drivers/net/ethernet/nebula-matrix/nbl/Makefile\n+++ b/drivers/net/ethernet/nebula-matrix/nbl/Makefile\n@@ -3,5 +3,13 @@\n \n obj-$(CONFIG_NBL) := nbl.o\n \n-nbl-objs +=\tnbl_hw/nbl_hw_leonis/nbl_hw_leonis.o \\\n+nbl-objs +=\tnbl_common/nbl_common.o \\\n+\t\tnbl_channel/nbl_channel.o \\\n+\t\tnbl_hw/nbl_hw_leonis/nbl_hw_leonis.o \\\n+\t\tnbl_hw/nbl_hw_leonis/nbl_resource_leonis.o \\\n+\t\tnbl_hw/nbl_resource.o \\\n+\t\tnbl_hw/nbl_interrupt.o \\\n+\t\tnbl_hw/nbl_chip.o \\\n+\t\tnbl_core/nbl_dispatch.o \\\n+\t\tnbl_core/nbl_dev.o \\\n \t\tnbl_main.o\ndiff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_channel/nbl_channel.c b/drivers/net/ethernet/nebula-matrix/nbl/nbl_channel/nbl_channel.c\nnew file mode 100644\nindex 00000000000000..5b528cc6efc452\n--- /dev/null\n+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_channel/nbl_channel.c\n@@ -0,0 +1,1528 @@\n+// SPDX-License-Identifier: GPL-2.0\n+/*\n+ * Copyright (c) 2026 Nebula Matrix Limited.\n+ */\n+#include \u003clinux/delay.h\u003e\n+#include \u003clinux/device.h\u003e\n+#include \u003clinux/mutex.h\u003e\n+#include \u003clinux/bitfield.h\u003e\n+#include \u003clinux/pci.h\u003e\n+#include \u003clinux/bits.h\u003e\n+#include \u003clinux/dma-mapping.h\u003e\n+#include \u003clinux/atomic.h\u003e\n+#include \u003clinux/wait.h\u003e\n+#include \"nbl_channel.h\"\n+\n+static int nbl_chan_add_msg_handler(struct nbl_channel_mgt *chan_mgt,\n+\t\t\t\t    u16 msg_type, nbl_chan_resp func,\n+\t\t\t\t    void *priv)\n+{\n+\tstruct nbl_chan_msg_node_data *handler;\n+\tint ret;\n+\n+\thandler = kzalloc_obj(*handler, GFP_KERNEL);\n+\tif (!handler)\n+\t\treturn -ENOMEM;\n+\n+\thandler-\u003efunc = func;\n+\thandler-\u003epriv = priv;\n+\n+\t/* Each msg_type is registered at most once; reject duplicates. */\n+\tmutex_lock(\u0026chan_mgt-\u003ehandler_lock);\n+\tret = xa_insert(\u0026chan_mgt-\u003ehandler_xa, msg_type, handler, GFP_KERNEL);\n+\tmutex_unlock(\u0026chan_mgt-\u003ehandler_lock);\n+\tif (ret)\n+\t\tkfree(handler);\n+\n+\treturn ret;\n+}\n+\n+static int nbl_chan_init_msg_handler(struct nbl_channel_mgt *chan_mgt)\n+{\n+\tint ret;\n+\n+\tret = devm_mutex_init(chan_mgt-\u003ecommon-\u003edev, \u0026chan_mgt-\u003ehandler_lock);\n+\tif (ret)\n+\t\treturn ret;\n+\n+\txa_init(\u0026chan_mgt-\u003ehandler_xa);\n+\n+\treturn 0;\n+}\n+\n+static void nbl_chan_remove_msg_handler(struct nbl_channel_mgt *chan_mgt)\n+{\n+\tstruct nbl_chan_msg_node_data *handler;\n+\tunsigned long msg_type;\n+\n+\tmutex_lock(\u0026chan_mgt-\u003ehandler_lock);\n+\txa_for_each(\u0026chan_mgt-\u003ehandler_xa, msg_type, handler) {\n+\t\txa_erase(\u0026chan_mgt-\u003ehandler_xa, msg_type);\n+\t\tkfree(handler);\n+\t}\n+\tmutex_unlock(\u0026chan_mgt-\u003ehandler_lock);\n+\txa_destroy(\u0026chan_mgt-\u003ehandler_xa);\n+}\n+\n+static void nbl_chan_init_queue_param(struct nbl_chan_info *chan_info,\n+\t\t\t\t      u16 num_txq_entries, u16 num_rxq_entries,\n+\t\t\t\t      u16 txq_buf_size, u16 rxq_buf_size)\n+{\n+\tchan_info-\u003enum_txq_entries = num_txq_entries;\n+\tchan_info-\u003enum_rxq_entries = num_rxq_entries;\n+\tchan_info-\u003etxq_buf_size = txq_buf_size;\n+\tchan_info-\u003erxq_buf_size = rxq_buf_size;\n+\tatomic_set(\u0026chan_info-\u003einflight_tx_cnt, 0);\n+\tWRITE_ONCE(chan_info-\u003eshutdn, false);\n+\tWRITE_ONCE(chan_info-\u003eactive, false);\n+\tWRITE_ONCE(chan_info-\u003ewait_head_index, 0);\n+\tmemset(chan_info-\u003estate, 0, sizeof(chan_info-\u003estate));\n+\tinit_waitqueue_head(\u0026chan_info-\u003einflight_wait);\n+}\n+\n+static int nbl_chan_init_tx_queue(struct nbl_common_info *common,\n+\t\t\t\t  struct nbl_chan_info *chan_info)\n+{\n+\tstruct nbl_chan_ring *txq = \u0026chan_info-\u003etxq;\n+\tstruct device *dev = common-\u003edev;\n+\tdma_addr_t base_dma;\n+\tsize_t size =\n+\t\tchan_info-\u003enum_txq_entries * sizeof(struct nbl_chan_tx_desc);\n+\tvoid *base;\n+\tu16 i;\n+\n+\ttxq-\u003edesc.tx_desc = dmam_alloc_coherent(dev, size, \u0026txq-\u003edma,\n+\t\t\t\t\t\tGFP_KERNEL);\n+\tif (!txq-\u003edesc.tx_desc)\n+\t\treturn -ENOMEM;\n+\n+\t/*\n+\t * The per-entry *data buffers* all come from one contiguous coherent\n+\t * block, so those buffers cost exactly one coherent allocation, one\n+\t * devres node and one IOMMU mapping for the whole ring whatever the\n+\t * page size is.  Per-entry dmam_alloc_coherent() instead rounds every\n+\t * NBL_CHAN_BUF_LEN buffer up to a whole page (8 MiB per PF on 16 KiB\n+\t * pages, 32 MiB on 64 KiB pages), and a dma_pool only fixes the\n+\t * large-page case: with NBL_CHAN_BUF_LEN equal to PAGE_SIZE it still\n+\t * degenerates to one allocation per buffer on 4 KiB pages.\n+\t * NBL_CHAN_BUF_LEN is a power of two, so the block always divides\n+\t * evenly into per-entry slots.\n+\t *\n+\t * That is the only part of the ring that is collapsed: the descriptor\n+\t * ring above is its own dmam_alloc_coherent(), and txq-\u003ebuf[] and\n+\t * chan_info-\u003ewait[] below are separate devm_kcalloc() nodes, so a\n+\t * ring is two coherent mappings plus those arrays, not one.\n+\t */\n+\tbase = dmam_alloc_coherent(dev,\n+\t\t\t\t   chan_info-\u003enum_txq_entries *\n+\t\t\t\t   chan_info-\u003etxq_buf_size,\n+\t\t\t\t   \u0026base_dma, GFP_KERNEL);\n+\tif (!base)\n+\t\treturn -ENOMEM;\n+\n+\tchan_info-\u003ewait = devm_kcalloc(dev, chan_info-\u003enum_txq_entries,\n+\t\t\t\t       sizeof(*chan_info-\u003ewait), GFP_KERNEL);\n+\tif (!chan_info-\u003ewait)\n+\t\treturn -ENOMEM;\n+\tfor (i = 0; i \u003c chan_info-\u003enum_txq_entries; i++) {\n+\t\tinit_waitqueue_head(\u0026chan_info-\u003ewait[i].wait_queue);\n+\t\tWRITE_ONCE(chan_info-\u003ewait[i].status, NBL_MBX_STATUS_IDLE);\n+\t\tWRITE_ONCE(chan_info-\u003ewait[i].acked, 0);\n+\t\tWRITE_ONCE(chan_info-\u003ewait[i].ack_data, NULL);\n+\t\tWRITE_ONCE(chan_info-\u003ewait[i].ack_data_len, 0);\n+\t\tWRITE_ONCE(chan_info-\u003ewait[i].ack_err, 0);\n+\t\tWRITE_ONCE(chan_info-\u003ewait[i].msg_type, 0);\n+\t\tWRITE_ONCE(chan_info-\u003ewait[i].msg_index, 0);\n+\t\tWRITE_ONCE(chan_info-\u003ewait[i].dstid, 0);\n+\t}\n+\n+\ttxq-\u003ebuf = devm_kcalloc(dev, chan_info-\u003enum_txq_entries,\n+\t\t\t\tsizeof(*txq-\u003ebuf), GFP_KERNEL);\n+\tif (!txq-\u003ebuf)\n+\t\treturn -ENOMEM;\n+\n+\t/* Carve the contiguous block into fixed-size per-entry slots */\n+\tfor (i = 0; i \u003c chan_info-\u003enum_txq_entries; i++) {\n+\t\ttxq-\u003ebuf[i].va = (u8 *)base +\n+\t\t\t\t (size_t)i * chan_info-\u003etxq_buf_size;\n+\t\ttxq-\u003ebuf[i].pa = base_dma +\n+\t\t\t\t (dma_addr_t)i * chan_info-\u003etxq_buf_size;\n+\t\ttxq-\u003ebuf[i].size = chan_info-\u003etxq_buf_size;\n+\t}\n+\n+\ttxq-\u003enext_to_clean = 0;\n+\ttxq-\u003enext_to_use = 0;\n+\ttxq-\u003etail_ptr = 0;\n+\n+\treturn 0;\n+}\n+\n+static int nbl_chan_init_rx_queue(struct nbl_common_info *common,\n+\t\t\t\t  struct nbl_chan_info *chan_info)\n+{\n+\tstruct nbl_chan_ring *rxq = \u0026chan_info-\u003erxq;\n+\tstruct device *dev = common-\u003edev;\n+\tstruct nbl_chan_rx_desc *desc;\n+\tdma_addr_t base_dma;\n+\tsize_t size =\n+\t\tchan_info-\u003enum_rxq_entries * sizeof(struct nbl_chan_rx_desc);\n+\tvoid *base;\n+\tu16 i;\n+\n+\trxq-\u003edesc.rx_desc = dmam_alloc_coherent(dev, size, \u0026rxq-\u003edma,\n+\t\t\t\t\t\tGFP_KERNEL);\n+\tif (!rxq-\u003edesc.rx_desc) {\n+\t\tdev_err_ratelimited(dev,\n+\t\t\t\t    \"Allocate DMA for chan rx descriptor ring failed\\n\");\n+\t\treturn -ENOMEM;\n+\t}\n+\n+\t/* See nbl_chan_init_tx_queue() for why one contiguous block is used */\n+\tbase = dmam_alloc_coherent(dev,\n+\t\t\t\t   chan_info-\u003enum_rxq_entries *\n+\t\t\t\t   chan_info-\u003erxq_buf_size,\n+\t\t\t\t   \u0026base_dma, GFP_KERNEL);\n+\tif (!base)\n+\t\treturn -ENOMEM;\n+\n+\trxq-\u003ebuf = devm_kcalloc(dev, chan_info-\u003enum_rxq_entries,\n+\t\t\t\tsizeof(*rxq-\u003ebuf), GFP_KERNEL);\n+\tif (!rxq-\u003ebuf)\n+\t\treturn -ENOMEM;\n+\n+\t/* Carve the contiguous block into fixed-size per-entry slots */\n+\tfor (i = 0; i \u003c chan_info-\u003enum_rxq_entries; i++) {\n+\t\trxq-\u003ebuf[i].va = (u8 *)base +\n+\t\t\t\t (size_t)i * chan_info-\u003erxq_buf_size;\n+\t\trxq-\u003ebuf[i].pa = base_dma +\n+\t\t\t\t (dma_addr_t)i * chan_info-\u003erxq_buf_size;\n+\t\trxq-\u003ebuf[i].size = chan_info-\u003erxq_buf_size;\n+\t}\n+\n+\tdesc = rxq-\u003edesc.rx_desc;\n+\t/*\n+\t * Initially leave one RX descriptor unused so that\n+\t * next_to_clean and next_to_use can distinguish an empty\n+\t * ring from a full ring.\n+\t *\n+\t * The unused slot is replenished as RX descriptors are\n+\t * consumed and recycled.\n+\t */\n+\tfor (i = 0; i \u003c chan_info-\u003enum_rxq_entries - 1; i++) {\n+\t\tdesc[i].buf_addr = cpu_to_le64(rxq-\u003ebuf[i].pa);\n+\t\tdesc[i].buf_len = cpu_to_le32(chan_info-\u003erxq_buf_size);\n+\t\tdesc[i].flags = cpu_to_le16(BIT(NBL_CHAN_RX_DESC_AVAIL));\n+\t}\n+\n+\trxq-\u003enext_to_clean = 0;\n+\trxq-\u003enext_to_use = chan_info-\u003enum_rxq_entries - 1;\n+\trxq-\u003etail_ptr = chan_info-\u003enum_rxq_entries - 1;\n+\n+\treturn 0;\n+}\n+\n+static int nbl_chan_init_queue(struct nbl_common_info *common,\n+\t\t\t       struct nbl_chan_info *chan_info)\n+{\n+\tint err;\n+\n+\terr = nbl_chan_init_tx_queue(common, chan_info);\n+\tif (err)\n+\t\treturn err;\n+\n+\terr = nbl_chan_init_rx_queue(common, chan_info);\n+\n+\treturn err;\n+}\n+\n+static void nbl_chan_config_queue(struct nbl_channel_mgt *chan_mgt,\n+\t\t\t\t  struct nbl_chan_info *chan_info, bool tx)\n+{\n+\tstruct nbl_hw_ops *hw_ops = chan_mgt-\u003ehw_ops_tbl-\u003eops;\n+\tstruct nbl_hw_mgt *p = chan_mgt-\u003ehw_ops_tbl-\u003epriv;\n+\tstruct nbl_chan_ring *ring;\n+\tdma_addr_t addr;\n+\tint size_bwid;\n+\n+\tif (tx)\n+\t\tring = \u0026chan_info-\u003etxq;\n+\telse\n+\t\tring = \u0026chan_info-\u003erxq;\n+\taddr = ring-\u003edma;\n+\tif (tx) {\n+\t\tsize_bwid = ilog2(chan_info-\u003enum_txq_entries);\n+\t\thw_ops-\u003econfig_mailbox_txq(p, addr, size_bwid);\n+\t} else {\n+\t\tsize_bwid = ilog2(chan_info-\u003enum_rxq_entries);\n+\t\thw_ops-\u003econfig_mailbox_rxq(p, addr, size_bwid);\n+\t}\n+}\n+\n+static void nbl_chan_cfg_qinfo_map_table(struct nbl_channel_mgt *chan_mgt,\n+\t\t\t\t\t u8 bus, u8 devid)\n+{\n+\tstruct nbl_hw_ops *hw_ops = chan_mgt-\u003ehw_ops_tbl-\u003eops;\n+\tstruct nbl_hw_mgt *p = chan_mgt-\u003ehw_ops_tbl-\u003epriv;\n+\tu32 pf_mask = 0;\n+\tu8 func_id;\n+\n+\t/*\n+\t * k_pf_mask rule: bit N == 0 means PF#N enabled, bit N == 1 masked out.\n+\t * Program mailbox QINFO entry for each hardware-active PF func_id.\n+\t *\n+\t * Note: This loop iterates over the raw hardware PF func_id, so it\n+\t * assumes product constraints limit supported PF counts to the\n+\t * contiguous sets PF0, PF0~1 and PF0~3, i.e. pf_mask is one of\n+\t * 0xfe/0xfc/0xf0.  Nothing enforces that here: the resource layer\n+\t * that validates the mask, and every caller of this op, arrive in\n+\t * later patches of this series.\n+\t */\n+\thw_ops-\u003eget_host_pf_mask(p, \u0026pf_mask);\n+\tfor (func_id = 0; func_id \u003c NBL_MAX_PF; func_id++) {\n+\t\tif (!(pf_mask \u0026 (1 \u003c\u003c func_id)))\n+\t\t\thw_ops-\u003ecfg_mailbox_qinfo(p, func_id, bus,\n+\t\t\t\t\t\t  devid, func_id);\n+\t}\n+}\n+\n+/*\n+ * The RX QINFO region is written only by config/stop paths, never by\n+ * senders (which only touch the TX QINFO region and the doorbell), so\n+ * stopping RX does not require txq_lock.\n+ */\n+static void nbl_chan_stop_rx_queue(struct nbl_channel_mgt *chan_mgt)\n+{\n+\tstruct nbl_hw_ops *hw_ops = chan_mgt-\u003ehw_ops_tbl-\u003eops;\n+\n+\thw_ops-\u003estop_mailbox_rxq(chan_mgt-\u003ehw_ops_tbl-\u003epriv);\n+}\n+\n+/*\n+ * The TX QINFO region is also written by nbl_chan_quiesce_and_reclaim_tx()\n+ * under txq_lock, so TX stop must serialize against it via txq_lock.\n+ */\n+static void nbl_chan_stop_tx_queue(struct nbl_channel_mgt *chan_mgt)\n+{\n+\tstruct nbl_hw_ops *hw_ops = chan_mgt-\u003ehw_ops_tbl-\u003eops;\n+\n+\thw_ops-\u003estop_mailbox_txq(chan_mgt-\u003ehw_ops_tbl-\u003epriv);\n+}\n+\n+static void nbl_chan_reset_wait_head(struct nbl_chan_info *chan_info,\n+\t\t\t\t     struct nbl_chan_waitqueue_head *wait_head)\n+{\n+\tlockdep_assert_held(\u0026chan_info-\u003epending_lock);\n+\n+\tWRITE_ONCE(wait_head-\u003eacked, 0);\n+\tWRITE_ONCE(wait_head-\u003estatus, NBL_MBX_STATUS_IDLE);\n+\tWRITE_ONCE(wait_head-\u003eack_data, NULL);\n+\tWRITE_ONCE(wait_head-\u003eack_data_len, 0);\n+\tWRITE_ONCE(wait_head-\u003eack_err, 0);\n+\tWRITE_ONCE(wait_head-\u003emsg_type, 0);\n+\tWRITE_ONCE(wait_head-\u003edstid, 0);\n+}\n+\n+static int nbl_chan_teardown_queue(struct nbl_channel_mgt *chan_mgt,\n+\t\t\t\t   u8 chan_type)\n+{\n+\tstruct nbl_chan_info *chan_info = chan_mgt-\u003echan_info[chan_type];\n+\tstruct nbl_chan_waitqueue_head *wait_head;\n+\tstruct work_struct *task;\n+\tu16 i;\n+\n+\tif (!READ_ONCE(chan_info-\u003eactive)) {\n+\t\tdev_warn(chan_mgt-\u003ecommon-\u003edev, \"channel not active, skip duplicate teardown\\n\");\n+\t\treturn 0;\n+\t}\n+\t/*\n+\t * Step1:\n+\t * block new sender\n+\t */\n+\tmutex_lock(\u0026chan_info-\u003estate_lock);\n+\tWRITE_ONCE(chan_info-\u003eshutdn, true);\n+\ttask = READ_ONCE(chan_info-\u003eclean_task);\n+\tWRITE_ONCE(chan_info-\u003eclean_task, NULL);\n+\tmutex_unlock(\u0026chan_info-\u003estate_lock);\n+\t/*\n+\t * Step2:\n+\t * abort pending ACK waiters\n+\t *\n+\t * Poke only slots with a live waiter (status == WAITING): publish a\n+\t * synthetic (-EIO, len 0) completion and wake the owner. The slot is\n+\t * deliberately NOT reset to IDLE here - ownership stays with the\n+\t * sender, which consumes the completion and resets its own slot in\n+\t * nbl_chan_send_msg():out_clear_wait_slot under pending_lock.\n+\t *\n+\t * Resetting an owned slot here would corrupt two things:\n+\t *   - an ACKD slot whose sender is consuming the completion\n+\t *     locklessly (smp_rmb()-ordered reads) could observe torn\n+\t *     ack_data_len/ack_err pairs;\n+\t *   - nbl_chan_get_msg_id() hands out IDLE/TIMEOUT slots, so a slot\n+\t *     reset to IDLE under its live owner could be reallocated to a\n+\t *     sender admitted before shutdn was set.\n+\t *\n+\t * IDLE/TIMEOUT slots have no waiter and need no action. A late ACK\n+\t * racing this poke is serialized by pending_lock: either the real\n+\t * completion lands first (status becomes ACKD and is skipped) or\n+\t * this synthetic failure does, both are valid outcomes.\n+\t */\n+\tmutex_lock(\u0026chan_info-\u003epending_lock);\n+\tfor (i = 0; i \u003c chan_info-\u003enum_txq_entries; i++) {\n+\t\twait_head = \u0026chan_info-\u003ewait[i];\n+\t\tif (READ_ONCE(wait_head-\u003estatus) != NBL_MBX_STATUS_WAITING)\n+\t\t\tcontinue;\n+\t\tWRITE_ONCE(wait_head-\u003estatus, NBL_MBX_STATUS_ACKD);\n+\t\tWRITE_ONCE(wait_head-\u003eack_data_len, 0);\n+\t\tWRITE_ONCE(wait_head-\u003eack_err, (s32)-EIO);\n+\t\t/* Publish completion fields before the acked flag */\n+\t\tsmp_wmb();\n+\t\tWRITE_ONCE(wait_head-\u003eacked, 1);\n+\t\twake_up(\u0026wait_head-\u003ewait_queue);\n+\t}\n+\tmutex_unlock(\u0026chan_info-\u003epending_lock);\n+\n+\t/*\n+\t * Step3:\n+\t * stop the RX queue unconditionally. The RX QINFO region is written\n+\t * only by config/stop paths, never by senders (which only touch the\n+\t * TX QINFO region and the doorbell), so no txq_lock is needed. The\n+\t * RX engine must be quiesced before the rings are freed, regardless\n+\t * of TX-side state - skipping it would leave the device DMAing peer\n+\t * messages and descriptor writebacks into released coherent memory.\n+\t */\n+\tnbl_chan_stop_rx_queue(chan_mgt);\n+\n+\t/*\n+\t * Drain strategy mirrors mlx5 command interface teardown:\n+\t * set shutdown flag first, abort all pending waiters, then\n+\t * block until inflight_tx_cnt reaches zero.\n+\t *\n+\t * After shutdn is set every sender exits promptly at its next\n+\t * checkpoint:\n+\t *   - interrupt-driven senders wake on shutdn immediately\n+\t *     (it is part of the wait_event condition);\n+\t *   - ACK polling senders check shutdn each iteration before\n+\t *     sleeping 1000-1200us. This per-iteration check avoids\n+\t *     waiting the full 5.0-6.0s worst-case ACK timeout.\n+\t *\n+\t * The wait is deliberately unbounded: a surviving sender still\n+\t * holds txq_lock or touches chan_info (state_lock, wait[]) on its\n+\t * exit path, so returning early would let devres free the rings\n+\t * and chan_info underneath it. In practice the TX polling loop is\n+\t * bounded (NBL_CHAN_TX_WAIT_TIMES) and wedged-link MMIO reads\n+\t * eventually complete with all-1s data, so this loop terminates;\n+\t * warn periodically to keep a genuinely stuck sender diagnosable.\n+\t */\n+\twhile (wait_event_timeout(chan_info-\u003einflight_wait,\n+\t\t\t\t  atomic_read(\u0026chan_info-\u003einflight_tx_cnt) == 0,\n+\t\t\t\t  msecs_to_jiffies(5000)) == 0)\n+\t\tdev_warn(chan_mgt-\u003ecommon-\u003edev,\n+\t\t\t \"teardown: still waiting for %d inflight sender(s)\\n\",\n+\t\t\t atomic_read(\u0026chan_info-\u003einflight_tx_cnt));\n+\n+\t/*\n+\t * Join the last exiting sender through state_lock.  A sender drops\n+\t * inflight_tx_cnt and wakes this drain while still holding\n+\t * state_lock, so observing the counter reach zero only proves the\n+\t * decrement happened; the acquire below is what proves that no\n+\t * sender still holds the lock.  Once it returns, no sender can be\n+\t * using chan_info either: the decrement is the last field access a\n+\t * sender makes, and the only thing that follows it is the\n+\t * mutex_unlock() of state_lock itself.\n+\t *\n+\t * That trailing unlock is why this is one acquire/release pair and\n+\t * not a handshake that ends the sender: mutex_unlock() may still be\n+\t * touching the mutex structure after it has released it, so this does\n+\t * not prove the sender has returned from it (see\n+\t * Documentation/locking/mutex-design.rst).  It is safe only because\n+\t * chan_info - mutexes included - is devm-allocated and released at\n+\t * detach, long after this teardown, and because nothing frees it\n+\t * here.  Freeing chan_info or calling mutex_destroy() at this point\n+\t * would need a sender-exit completion the channel does not have.\n+\t *\n+\t * shutdn blocks new senders, so one acquire/release pair is\n+\t * sufficient.\n+\t */\n+\tmutex_lock(\u0026chan_info-\u003estate_lock);\n+\tmutex_unlock(\u0026chan_info-\u003estate_lock);\n+\n+\t/*\n+\t * All senders have exited, so txq_lock is uncontended: plain lock.\n+\t * Take it so this is the sole writer of the TX QINFO block\n+\t * (sender-side stop/config/doorbell sequences all run under\n+\t * txq_lock).\n+\t */\n+\tmutex_lock(\u0026chan_info-\u003etxq_lock);\n+\tnbl_chan_stop_tx_queue(chan_mgt);\n+\tmutex_unlock(\u0026chan_info-\u003etxq_lock);\n+\n+\t/*\n+\t * Join the RX clean work before active=false and before devres\n+\t * releases the rings. IRQ is already freed and clean_task is\n+\t * cleared under state_lock, so no new instance can be queued.\n+\t */\n+\tif (task)\n+\t\tcancel_work_sync(task);\n+\n+\tWRITE_ONCE(chan_info-\u003eactive, false);\n+\treturn 0;\n+}\n+\n+static int nbl_chan_setup_queue(struct nbl_channel_mgt *chan_mgt, u8 chan_type)\n+{\n+\tstruct nbl_chan_info *chan_info = chan_mgt-\u003echan_info[chan_type];\n+\tstruct nbl_hw_ops *hw_ops = chan_mgt-\u003ehw_ops_tbl-\u003eops;\n+\tstruct nbl_common_info *common = chan_mgt-\u003ecommon;\n+\tstruct nbl_chan_ring *rxq = \u0026chan_info-\u003erxq;\n+\tint err;\n+\n+\tif (READ_ONCE(chan_info-\u003eactive)) {\n+\t\tdev_warn(common-\u003edev, \"channel already active, reject duplicate setup\\n\");\n+\t\treturn -EBUSY;\n+\t}\n+\t/*\n+\t * The descriptor rings and the per-ring contiguous buffer blocks are\n+\t * allocated once and released only by devres at device detach: there\n+\t * is no free path, so a channel is set up exactly once per device\n+\t * lifetime and teardown is terminal for it.  Re-setup would allocate\n+\t * a second set - about 2 MiB of coherent DMA plus the descriptor\n+\t * rings and wait[] - and orphan the first one, and it would revive a\n+\t * queue whose state teardown has already retired, so it is rejected.\n+\t */\n+\tif (chan_info-\u003edma_allocated) {\n+\t\tdev_warn(common-\u003edev,\n+\t\t\t \"channel DMA already allocated, re-setup not supported\\n\");\n+\t\treturn -EBUSY;\n+\t}\n+\n+\tnbl_chan_init_queue_param(chan_info, NBL_CHAN_QUEUE_LEN,\n+\t\t\t\t  NBL_CHAN_QUEUE_LEN, NBL_CHAN_BUF_LEN,\n+\t\t\t\t  NBL_CHAN_BUF_LEN);\n+\t/*\n+\t * Set the flag before the first allocation and leave it set even\n+\t * when setup fails: whatever nbl_chan_init_queue() managed to\n+\t * allocate is devres-owned and stays live until detach, so the retry\n+\t * that clearing the flag would allow would allocate a second set on\n+\t * top of the orphans.\n+\t */\n+\tchan_info-\u003edma_allocated = true;\n+\terr = nbl_chan_init_queue(common, chan_info);\n+\tif (err)\n+\t\treturn err;\n+\n+\tnbl_chan_config_queue(chan_mgt, chan_info, true); /* tx */\n+\tnbl_chan_config_queue(chan_mgt, chan_info, false); /* rx */\n+\tnbl_chan_update_tail_ptr(hw_ops, chan_mgt-\u003ehw_ops_tbl-\u003epriv,\n+\t\t\t\t rxq-\u003etail_ptr, NBL_MB_RX_QID);\n+\tWRITE_ONCE(chan_info-\u003eactive, true);\n+\treturn 0;\n+}\n+\n+static bool nbl_chan_txq_full(struct nbl_chan_ring *txq,\n+\t\t\t      u16 num_entries)\n+{\n+\treturn NBL_NEXT_ID(txq-\u003enext_to_use, num_entries - 1) ==\n+\t       txq-\u003enext_to_clean;\n+}\n+\n+static int nbl_chan_update_txqueue(struct nbl_channel_mgt *chan_mgt,\n+\t\t\t\t   struct nbl_chan_info *chan_info,\n+\t\t\t\t   struct nbl_chan_tx_param *param)\n+{\n+\tstruct nbl_chan_ring *txq = \u0026chan_info-\u003etxq;\n+\tstruct nbl_chan_tx_desc *tx_desc;\n+\tstruct nbl_chan_buf *tx_buf;\n+\n+\t/*\n+\t * param-\u003earg is non-NULL whenever param-\u003earg_len is non-zero:\n+\t * nbl_chan_send_msg() validates that before calling here, because a\n+\t * length with no buffer would publish buf_addr/buf_len describing\n+\t * memory that was never filled.\n+\t */\n+\tif (nbl_chan_txq_full(txq, chan_info-\u003enum_txq_entries))\n+\t\treturn -EBUSY;\n+\tif (param-\u003earg_len \u003e NBL_CHAN_BUF_LEN - sizeof(*tx_desc))\n+\t\treturn -EINVAL;\n+\ttx_desc =\n+\t\tNBL_CHAN_TX_RING_TO_DESC(txq, txq-\u003enext_to_use);\n+\ttx_buf =\n+\t\tNBL_CHAN_TX_RING_TO_BUF(txq, txq-\u003enext_to_use);\n+\ttx_desc-\u003edstid = cpu_to_le16(param-\u003edstid);\n+\ttx_desc-\u003emsg_type = cpu_to_le16(param-\u003emsg_type);\n+\ttx_desc-\u003emsgid = cpu_to_le16(param-\u003emsgid);\n+\n+\t/*\n+\t * srcid field is filled by mailbox hardware after peer receives this\n+\t * packet, driver producer never writes srcid; reused descriptor slots\n+\t * will contain stale srcid value temporarily until hardware overwrites\n+\t * it.\n+\t */\n+\tif (param-\u003earg_len \u003e NBL_CHAN_TX_DESC_EMBEDDED_DATA_LEN) {\n+\t\tmemcpy(tx_buf-\u003eva, param-\u003earg, param-\u003earg_len);\n+\t\ttx_desc-\u003ebuf_addr = cpu_to_le64(tx_buf-\u003epa);\n+\t\ttx_desc-\u003ebuf_len = cpu_to_le16(param-\u003earg_len);\n+\t\ttx_desc-\u003edata_len = 0;\n+\t\tmemset(tx_desc-\u003edata, 0, sizeof(tx_desc-\u003edata));\n+\t} else {\n+\t\tmemset(tx_desc-\u003edata, 0, sizeof(tx_desc-\u003edata));\n+\t\tmemset(\u0026tx_desc-\u003ebuf_addr, 0, sizeof(tx_desc-\u003ebuf_addr));\n+\t\tif (param-\u003earg_len)\n+\t\t\tmemcpy(tx_desc-\u003edata, param-\u003earg, param-\u003earg_len);\n+\t\ttx_desc-\u003ebuf_len = 0;\n+\t\ttx_desc-\u003edata_len = cpu_to_le16(param-\u003earg_len);\n+\t}\n+\t/* Ensure descriptor data visible to device before AVAIL flag */\n+\tdma_wmb();\n+\ttx_desc-\u003eflags = cpu_to_le16(BIT(NBL_CHAN_TX_DESC_AVAIL));\n+\n+\ttxq-\u003enext_to_use =\n+\t\tNBL_NEXT_ID(txq-\u003enext_to_use, chan_info-\u003enum_txq_entries - 1);\n+\ttxq-\u003etail_ptr++;\n+\n+\treturn 0;\n+}\n+\n+/*\n+ * Quiesce the TX mailbox queue and reclaim all outstanding\n+ * descriptors.  Called from the timeout path of nbl_chan_kick_tx_ring()\n+ * with txq_lock held.\n+ *\n+ * The device failed to fetch/complete the current descriptor within the\n+ * polling window.  We assert QUEUE_RST to stop further DMA fetches,\n+ * reclaim every descriptor between next_to_clean and next_to_use,\n+ * reset the software tail_ptr counter to match the hardware reset state,\n+ * and re-enable the queue so subsequent sends can proceed.\n+ */\n+static void nbl_chan_quiesce_and_reclaim_tx(struct nbl_channel_mgt *chan_mgt,\n+\t\t\t\t\t    struct nbl_chan_info *chan_info)\n+{\n+\tstruct nbl_hw_ops *hw_ops = chan_mgt-\u003ehw_ops_tbl-\u003eops;\n+\tstruct nbl_hw_mgt *hw_priv = chan_mgt-\u003ehw_ops_tbl-\u003epriv;\n+\tstruct nbl_chan_ring *txq = \u0026chan_info-\u003etxq;\n+\tstruct nbl_chan_tx_desc *tx_desc;\n+\n+\t/*\n+\t * Assert QUEUE_RST to stop hardware fetching new descriptors.\n+\t * stop_mailbox_txq() flushes through the mailbox BAR itself, so\n+\t * the reset has reached the device when it returns.\n+\t */\n+\thw_ops-\u003estop_mailbox_txq(hw_priv);\n+\n+\t/*\n+\t * Reclaim all outstanding descriptors between next_to_clean and\n+\t * next_to_use.  Under txq_lock there is at most one in-flight\n+\t * descriptor, but iterate the full range for robustness.\n+\t */\n+\twhile (txq-\u003enext_to_clean != txq-\u003enext_to_use) {\n+\t\ttx_desc = NBL_CHAN_TX_RING_TO_DESC(txq,\n+\t\t\t\t\t\t   txq-\u003enext_to_clean);\n+\t\tWRITE_ONCE(tx_desc-\u003eflags, 0);\n+\t\ttxq-\u003enext_to_clean =\n+\t\t\tNBL_NEXT_ID(txq-\u003enext_to_clean,\n+\t\t\t\t    chan_info-\u003enum_txq_entries - 1);\n+\t}\n+\n+\t/*\n+\t * Hardware tail_ptr counter is cleared by QUEUE_RST.  Reset\n+\t * software counter to match so the next doorbell update does\n+\t * not produce a false 16-bit wrap delta.\n+\t */\n+\ttxq-\u003etail_ptr = 0;\n+\ttxq-\u003enext_to_use = 0;\n+\ttxq-\u003enext_to_clean = 0;\n+\n+\t/*\n+\t * Authoritative re-arm gate, evaluated while txq_lock is held:\n+\t * teardown sets shutdn before draining senders and then owns\n+\t * hardware state. If teardown started while we were stopping/\n+\t * reclaiming, leave the queue in QUEUE_RST and never program\n+\t * QUEUE_EN, otherwise the queue could be armed again pointing\n+\t * at rings that are about to be freed.\n+\t */\n+\tif (READ_ONCE(chan_info-\u003eshutdn))\n+\t\treturn;\n+\n+\t/* Re-enable queue with current ring base and size */\n+\tnbl_chan_config_queue(chan_mgt, chan_info, true);\n+}\n+\n+static int nbl_chan_kick_tx_ring(struct nbl_channel_mgt *chan_mgt,\n+\t\t\t\t struct nbl_chan_info *chan_info)\n+{\n+\tstruct nbl_hw_ops *hw_ops = chan_mgt-\u003ehw_ops_tbl-\u003eops;\n+\tstruct nbl_chan_ring *txq = \u0026chan_info-\u003etxq;\n+\tstruct device *dev = chan_mgt-\u003ecommon-\u003edev;\n+\tint max_retries = NBL_CHAN_TX_WAIT_TIMES;\n+\tstruct nbl_chan_tx_desc *tx_desc;\n+\tint retry_count = 0;\n+\tu16 msg_type;\n+\n+\tnbl_chan_update_tail_ptr(hw_ops, chan_mgt-\u003ehw_ops_tbl-\u003epriv,\n+\t\t\t\t txq-\u003etail_ptr, NBL_MB_TX_QID);\n+\n+\ttx_desc = NBL_CHAN_TX_RING_TO_DESC(txq, txq-\u003enext_to_clean);\n+\t/*\n+\t * Poll for HW to mark descriptor as USED.\n+\t * Mailbox is a low-speed control channel for management commands.\n+\t * We avoid enabling dedicated per-TX interrupt for single control\n+\t * message to reduce interrupt overhead, so use bounded polling\n+\t * with small delay instead.\n+\t */\n+\twhile (retry_count \u003c max_retries) {\n+\t\tif (READ_ONCE(chan_info-\u003eshutdn))\n+\t\t\treturn -ESHUTDOWN;\n+\n+\t\tif (le16_to_cpu(READ_ONCE(tx_desc-\u003eflags)) \u0026\n+\t\t    BIT(NBL_CHAN_TX_DESC_USED)) {\n+\t\t\t /*\n+\t\t\t  * Order reads of other device-written descriptor\n+\t\t\t  * fields after observing USED.  Matches the RX side\n+\t\t\t  * pattern in nbl_chan_clean_queue().\n+\t\t\t  */\n+\t\t\tdma_rmb();\n+\t\t\tbreak;\n+\t\t}\n+\n+\t\tretry_count++;\n+\t\tif (retry_count == max_retries) {\n+\t\t\tmsg_type = le16_to_cpu(READ_ONCE(tx_desc-\u003emsg_type));\n+\t\t\tdev_err_ratelimited(dev, \"chan send msg type: %d timeout\\n\",\n+\t\t\t\t\t    msg_type);\n+\t\t\t/*\n+\t\t\t * Teardown may have started after the loop-top\n+\t\t\t * check on this final iteration. Do not run lock\n+\t\t\t * recovery then: it would stop/reclaim the queue\n+\t\t\t * and, without the in-quiesce gate, re-arm it.\n+\t\t\t * nbl_chan_teardown_queue() owns hardware state.\n+\t\t\t */\n+\t\t\tif (READ_ONCE(chan_info-\u003eshutdn))\n+\t\t\t\treturn -ESHUTDOWN;\n+\t\t\t/*\n+\t\t\t * Device failed to complete this descriptor.\n+\t\t\t * Quiesce the queue, reclaim the timed-out\n+\t\t\t * descriptor, and re-enable so future sends can\n+\t\t\t * proceed instead of stalling the ring full.\n+\t\t\t */\n+\t\t\tnbl_chan_quiesce_and_reclaim_tx(chan_mgt,\n+\t\t\t\t\t\t\tchan_info);\n+\t\t\treturn -ETIMEDOUT;\n+\t\t}\n+\t\tusleep_range(NBL_CHAN_TX_WAIT_US, NBL_CHAN_TX_WAIT_US_MAX);\n+\t}\n+\n+\ttxq-\u003enext_to_clean = txq-\u003enext_to_use;\n+\n+\treturn 0;\n+}\n+\n+static void nbl_chan_recv_ack_msg(void *priv, u16 srcid, u16 msgid, void *data,\n+\t\t\t\t  u32 data_len)\n+{\n+\tstruct nbl_channel_mgt *chan_mgt = (struct nbl_channel_mgt *)priv;\n+\tstruct nbl_chan_waitqueue_head *wait_head = NULL;\n+\tstruct device *dev = chan_mgt-\u003ecommon-\u003edev;\n+\tstruct nbl_chan_info *chan_info =\n+\t\tchan_mgt-\u003echan_info[NBL_CHAN_TYPE_MAILBOX];\n+\tu16 w_dstid, w_msgtype, w_msgidx;\n+\tu32 *payload = data;\n+\tu16 ack_msgtype = 0;\n+\tu16 ack_msgid = 0;\n+\tu32 ack_datalen;\n+\tvoid *ack_data;\n+\tu32 copy_len;\n+\tint w_status;\n+\ts32 raw_err;\n+\n+\tif (READ_ONCE(chan_info-\u003eshutdn))\n+\t\treturn;\n+\tif (data_len \u003e NBL_CHAN_BUF_LEN ||\n+\t    data_len \u003c NBL_CHAN_ACK_HEAD_LEN * sizeof(u32)) {\n+\t\tdev_err_ratelimited(dev, \"Invalid ACK data_len: %u\\n\",\n+\t\t\t\t    data_len);\n+\t\treturn;\n+\t}\n+\tack_datalen = data_len - NBL_CHAN_ACK_HEAD_LEN * sizeof(u32);\n+\tack_msgtype = le16_to_cpu(*(__le16 *)(payload + NBL_CHAN_MSG_TYPE_POS));\n+\tack_msgid = le16_to_cpu(*(__le16 *)(payload + NBL_CHAN_MSG_ID_POS));\n+\tif (FIELD_GET(NBL_CHAN_MSGID_LOC_MASK, ack_msgid) \u003e=\n+\t    chan_info-\u003enum_txq_entries) {\n+\t\tdev_err_ratelimited(dev, \"chan recv msg id: %u err\\n\",\n+\t\t\t\t    ack_msgid);\n+\t\treturn;\n+\t}\n+\twait_head =\n+\t\t\u0026chan_info-\u003ewait[FIELD_GET(NBL_CHAN_MSGID_LOC_MASK, ack_msgid)];\n+\n+\tmutex_lock(\u0026chan_info-\u003epending_lock);\n+\n+\t/* Cache repeated READ_ONCE values */\n+\tw_dstid = READ_ONCE(wait_head-\u003edstid);\n+\tw_status = READ_ONCE(wait_head-\u003estatus);\n+\tw_msgtype = READ_ONCE(wait_head-\u003emsg_type);\n+\tw_msgidx = READ_ONCE(wait_head-\u003emsg_index);\n+\n+\tif (srcid != w_dstid) {\n+\t\tmutex_unlock(\u0026chan_info-\u003epending_lock);\n+\t\tdev_err_ratelimited(dev, \"ACK srcid=%u != dstid=%u, rejecting\\n\",\n+\t\t\t\t    srcid, w_dstid);\n+\t\treturn;\n+\t}\n+\tif (w_status != NBL_MBX_STATUS_WAITING) {\n+\t\tmutex_unlock(\u0026chan_info-\u003epending_lock);\n+\t\tdev_err_ratelimited(dev,\n+\t\t\t\t    \"Skip ack invalid status, wait msgtype:%u idx:%u status:%d ack msgtype:%u msgid:%u datalen:%u\\n\",\n+\t\t\t\t    w_msgtype, w_msgidx, w_status,\n+\t\t\t\t    ack_msgtype, ack_msgid, ack_datalen);\n+\t\treturn;\n+\t}\n+\n+\tif (w_msgtype != ack_msgtype) {\n+\t\tmutex_unlock(\u0026chan_info-\u003epending_lock);\n+\t\tdev_err_ratelimited(dev,\n+\t\t\t\t    \"Skip ack msgtype mismatch, wait msgtype:%u idx:%u ack msgtype:%u msgid:%u\\n\",\n+\t\t\t\t    w_msgtype, w_msgidx, ack_msgtype,\n+\t\t\t\t    ack_msgid);\n+\t\treturn;\n+\t}\n+\tif (FIELD_GET(NBL_CHAN_MSGID_INDEX_MASK, ack_msgid) != w_msgidx) {\n+\t\tmutex_unlock(\u0026chan_info-\u003epending_lock);\n+\t\tdev_err_ratelimited(dev,\n+\t\t\t\t    \"Stale ACK: expected index=%u, got msgid=%u\\n\",\n+\t\t\t\t    w_msgidx, ack_msgid);\n+\t\treturn;\n+\t}\n+\n+\traw_err = (s32)le32_to_cpu(*(__le32 *)\u0026payload[NBL_CHAN_ACK_RET_POS]);\n+\tif (raw_err \u003e 0 || raw_err \u003c -MAX_ERRNO)\n+\t\traw_err = -EREMOTEIO;\n+\n+\tWRITE_ONCE(wait_head-\u003eack_err, raw_err);\n+\n+\tcopy_len = min_t(u32, READ_ONCE(wait_head-\u003eack_data_len), ack_datalen);\n+\tif (READ_ONCE(wait_head-\u003eack_err) \u003e= 0 \u0026\u0026 copy_len \u003e 0) {\n+\t\tack_data = READ_ONCE(wait_head-\u003eack_data);\n+\t\tif (!ack_data) {\n+\t\t\tdev_err_ratelimited(dev, \"ACK payload dropped: ack_data is NULL\\n\");\n+\t\t\tWRITE_ONCE(wait_head-\u003eack_data_len, 0);\n+\t\t\tgoto ack_done;\n+\t\t}\n+\t\tmemcpy((char *)ack_data,\n+\t\t       payload + NBL_CHAN_ACK_HEAD_LEN, copy_len);\n+\t\tWRITE_ONCE(wait_head-\u003eack_data_len, (u16)copy_len);\n+\t} else {\n+\t\tWRITE_ONCE(wait_head-\u003eack_data_len, 0);\n+\t}\n+ack_done:\n+\t/* Guarantee payload data finished before acked flag visible */\n+\tsmp_wmb();\n+\tWRITE_ONCE(wait_head-\u003eacked, 1);\n+\tWRITE_ONCE(wait_head-\u003estatus, NBL_MBX_STATUS_ACKD);\n+\tmutex_unlock(\u0026chan_info-\u003epending_lock);\n+\twake_up(\u0026wait_head-\u003ewait_queue);\n+}\n+\n+static void nbl_chan_recv_msg(struct nbl_channel_mgt *chan_mgt, void *data)\n+{\n+\tstruct device *dev = chan_mgt-\u003ecommon-\u003edev;\n+\tstruct nbl_chan_msg_node_data *msg_handler;\n+\tu16 msg_type, payload_len, srcid, msgid;\n+\tstruct nbl_chan_info *chan_info =\n+\t\tchan_mgt-\u003echan_info[NBL_CHAN_TYPE_MAILBOX];\n+\tstruct nbl_chan_tx_desc *tx_desc;\n+\tvoid *payload;\n+\tsize_t avail_space;\n+\tu16 data_len_fw;\n+\n+\tif (READ_ONCE(chan_info-\u003eshutdn))\n+\t\treturn;\n+\n+\ttx_desc = data;\n+\tmsg_type = le16_to_cpu(READ_ONCE(tx_desc-\u003emsg_type));\n+\tdev_dbg(dev, \"recv msg_type: %d\\n\", msg_type);\n+\n+\tsrcid = le16_to_cpu(READ_ONCE(tx_desc-\u003esrcid));\n+\tmsgid = le16_to_cpu(READ_ONCE(tx_desc-\u003emsgid));\n+\n+\tif (msg_type \u003e= NBL_CHAN_MSG_MAILBOX_MAX)\n+\t\treturn;\n+\n+\tdata_len_fw = le16_to_cpu(READ_ONCE(tx_desc-\u003edata_len));\n+\tif (data_len_fw) {\n+\t\tpayload_len = data_len_fw;\n+\n+\t\tif (payload_len \u003e NBL_CHAN_TX_DESC_EMBEDDED_DATA_LEN) {\n+\t\t\tdev_err_ratelimited(dev,\n+\t\t\t\t\t    \"data_len=%u exceeds embedded buffer size=%u\\n\",\n+\t\t\t\t\t    payload_len,\n+\t\t\t\t\t    NBL_CHAN_TX_DESC_EMBEDDED_DATA_LEN);\n+\t\t\treturn;\n+\t\t}\n+\t\t/* Small pkt: payload stored inside descriptor data[] array */\n+\t\tpayload = tx_desc-\u003edata;\n+\t} else {\n+\t\tpayload_len = le16_to_cpu(READ_ONCE(tx_desc-\u003ebuf_len));\n+\n+\t\tavail_space = NBL_CHAN_BUF_LEN - sizeof(*tx_desc);\n+\t\tif (payload_len \u003e avail_space) {\n+\t\t\tdev_err_ratelimited(dev,\n+\t\t\t\t\t    \"buf_len=%u exceeds external buffer size=%zu\\n\",\n+\t\t\t\t\t    payload_len, avail_space);\n+\t\t\treturn;\n+\t\t}\n+\t\t/* Large pkt: payload follows immediately after tx_desc */\n+\t\tpayload = tx_desc + 1;\n+\t}\n+\n+\tmsg_handler = xa_load(\u0026chan_mgt-\u003ehandler_xa, msg_type);\n+\tif (!msg_handler || !msg_handler-\u003efunc) {\n+\t\tdev_err_ratelimited(dev,\n+\t\t\t\t    \"No handler for msg_type: %u (srcid=%u, msgid=%u)\\n\",\n+\t\t\t\t    msg_type, srcid, msgid);\n+\t\treturn;\n+\t}\n+\n+\tmsg_handler-\u003efunc(msg_handler-\u003epriv, srcid, msgid, payload,\n+\t\t\t  payload_len);\n+}\n+\n+static void nbl_chan_advance_rx_ring(struct nbl_channel_mgt *chan_mgt,\n+\t\t\t\t     struct nbl_chan_info *chan_info,\n+\t\t\t\t     struct nbl_chan_ring *rxq)\n+{\n+\tstruct nbl_hw_ops *hw_ops = chan_mgt-\u003ehw_ops_tbl-\u003eops;\n+\tstruct nbl_chan_rx_desc *rx_desc;\n+\tstruct nbl_chan_buf *rx_buf;\n+\tu16 next_to_use;\n+\n+\tnext_to_use = rxq-\u003enext_to_use;\n+\trx_desc = NBL_CHAN_RX_RING_TO_DESC(rxq, next_to_use);\n+\trx_buf = NBL_CHAN_RX_RING_TO_BUF(rxq, next_to_use);\n+\n+\t/*\n+\t * Recycle the RX descriptor at next_to_use. The initial\n+\t * unused slot is intentionally recycled after the first\n+\t * RX descriptor is consumed, allowing the ring to become\n+\t * fully populated while next_to_clean tracks the consumer.\n+\t */\n+\trx_desc-\u003ebuf_addr = cpu_to_le64(rx_buf-\u003epa);\n+\trx_desc-\u003ebuf_len = cpu_to_le32(chan_info-\u003erxq_buf_size);\n+\n+\t/*\n+\t * DMA Write Memory Barrier:\n+\t * Ensures all previous DMA-mapped writes (buffer address/length)\n+\t * are completed before the descriptor flags are updated.\n+\t * This prevents hardware from seeing a partially updated descriptor\n+\t * where flags are set but buffer info isn't ready yet.\n+\t */\n+\tdma_wmb();\n+\n+\trx_desc-\u003eflags = cpu_to_le16(BIT(NBL_CHAN_RX_DESC_AVAIL));\n+\n+\trxq-\u003enext_to_use++;\n+\tif (rxq-\u003enext_to_use == chan_info-\u003enum_rxq_entries)\n+\t\trxq-\u003enext_to_use = 0;\n+\trxq-\u003etail_ptr++;\n+\n+\tnbl_chan_update_tail_ptr(hw_ops, chan_mgt-\u003ehw_ops_tbl-\u003epriv,\n+\t\t\t\t rxq-\u003etail_ptr, NBL_MB_RX_QID);\n+}\n+\n+static void nbl_chan_clean_queue(struct nbl_channel_mgt *chan_mgt,\n+\t\t\t\t struct nbl_chan_info *chan_info)\n+{\n+\tstruct nbl_common_info *common = chan_mgt-\u003ecommon;\n+\tstruct nbl_chan_ring *rxq = \u0026chan_info-\u003erxq;\n+\tstruct device *dev = chan_mgt-\u003ecommon-\u003edev;\n+\tu32 budget = NBL_CHAN_RX_CLEAN_BUDGET;\n+\tstruct nbl_chan_rx_desc *rx_desc;\n+\tstruct nbl_chan_buf *rx_buf;\n+\tstruct work_struct *task;\n+\tbool more_work = false;\n+\tu16 next_to_clean;\n+\tu16 flags;\n+\n+\t/* Nothing to clean before setup: the RX ring is still NULL */\n+\tif (!READ_ONCE(chan_info-\u003eactive))\n+\t\treturn;\n+\n+\tnext_to_clean = rxq-\u003enext_to_clean;\n+\trx_desc = NBL_CHAN_RX_RING_TO_DESC(rxq, next_to_clean);\n+\trx_buf = NBL_CHAN_RX_RING_TO_BUF(rxq, next_to_clean);\n+\twhile (le16_to_cpu(READ_ONCE(rx_desc-\u003eflags)) \u0026\n+\t       BIT(NBL_CHAN_RX_DESC_USED)) {\n+\t\tflags = le16_to_cpu(READ_ONCE(rx_desc-\u003eflags));\n+\n+\t\tif (READ_ONCE(chan_info-\u003eshutdn))\n+\t\t\tbreak;\n+\t\tif (!(flags \u0026 BIT(NBL_CHAN_RX_DESC_WRITE)))\n+\t\t\tdev_dbg(dev,\n+\t\t\t\t\"mailbox rx flag 0x%x missing NBL_CHAN_RX_DESC_WRITE\\n\",\n+\t\t\t\tflags);\n+\n+\t\t/* Make sure hardware written descriptor visible to CPU */\n+\t\tdma_rmb();\n+\t\tnbl_chan_recv_msg(chan_mgt, rx_buf-\u003eva);\n+\t\tnbl_chan_advance_rx_ring(chan_mgt, chan_info, rxq);\n+\t\tnext_to_clean++;\n+\t\tif (next_to_clean == chan_info-\u003enum_rxq_entries)\n+\t\t\tnext_to_clean = 0;\n+\t\trx_desc = NBL_CHAN_RX_RING_TO_DESC(rxq, next_to_clean);\n+\t\trx_buf = NBL_CHAN_RX_RING_TO_BUF(rxq, next_to_clean);\n+\t\tif (--budget == 0) {\n+\t\t\tmore_work = true;\n+\t\t\tbreak;\n+\t\t}\n+\t\tcond_resched();\n+\t}\n+\trxq-\u003enext_to_clean = next_to_clean;\n+\n+\tmutex_lock(\u0026chan_info-\u003estate_lock);\n+\t/* Prevent queue_work after teardown clears clean_task */\n+\tif (READ_ONCE(chan_info-\u003eshutdn)) {\n+\t\tmutex_unlock(\u0026chan_info-\u003estate_lock);\n+\t\treturn;\n+\t}\n+\tif (common-\u003ewq \u0026\u0026 more_work) {\n+\t\ttask = READ_ONCE(chan_info-\u003eclean_task);\n+\t\tif (task)\n+\t\t\tqueue_work(common-\u003ewq, task);\n+\t}\n+\tmutex_unlock(\u0026chan_info-\u003estate_lock);\n+}\n+\n+static void nbl_chan_clean_queue_subtask(struct nbl_channel_mgt *chan_mgt,\n+\t\t\t\t\t u8 chan_type)\n+{\n+\tstruct nbl_chan_info *chan_info = chan_mgt-\u003echan_info[chan_type];\n+\n+\tnbl_chan_clean_queue(chan_mgt, chan_info);\n+}\n+\n+static int nbl_chan_get_msg_id(struct nbl_chan_info *chan_info,\n+\t\t\t       u16 *msgid)\n+{\n+\tint search_loc = READ_ONCE(chan_info-\u003ewait_head_index), i;\n+\tstruct nbl_chan_waitqueue_head *wait = NULL;\n+\tint status;\n+\tint next;\n+\n+\tlockdep_assert_held(\u0026chan_info-\u003epending_lock);\n+\tfor (i = 0; i \u003c chan_info-\u003enum_txq_entries; i++) {\n+\t\twait = \u0026chan_info-\u003ewait[search_loc];\n+\t\tstatus = READ_ONCE(wait-\u003estatus);\n+\t\tif (status == NBL_MBX_STATUS_IDLE ||\n+\t\t    status == NBL_MBX_STATUS_TIMEOUT) {\n+\t\t\tWRITE_ONCE(wait-\u003emsg_index,\n+\t\t\t\t   NBL_NEXT_ID(wait-\u003emsg_index,\n+\t\t\t\t\t       NBL_CHAN_MSG_INDEX_MAX));\n+\n+\t\t\t*msgid = FIELD_PREP(NBL_CHAN_MSGID_INDEX_MASK,\n+\t\t\t\t\t    wait-\u003emsg_index) |\n+\t\t\t\t FIELD_PREP(NBL_CHAN_MSGID_LOC_MASK,\n+\t\t\t\t\t    search_loc);\n+\n+\t\t\t/* Advance starting search position for next caller */\n+\t\t\tnext = NBL_NEXT_ID(search_loc,\n+\t\t\t\t\t   chan_info-\u003enum_txq_entries - 1);\n+\t\t\tWRITE_ONCE(chan_info-\u003ewait_head_index, next);\n+\t\t\treturn 0;\n+\t\t}\n+\n+\t\tsearch_loc = NBL_NEXT_ID(search_loc,\n+\t\t\t\t\t chan_info-\u003enum_txq_entries - 1);\n+\t}\n+\n+\t/*\n+\t * All tx slots are occupied. May happen under high transmit load\n+\t * or delayed remote ACK responses. Caller should retry later.\n+\t */\n+\treturn -EAGAIN;\n+}\n+\n+static int nbl_chan_send_msg(struct nbl_channel_mgt *chan_mgt,\n+\t\t\t     struct nbl_chan_send_info *chan_send)\n+{\n+\tstruct nbl_common_info *common = chan_mgt-\u003ecommon;\n+\tstruct nbl_chan_waitqueue_head *wait_head = NULL;\n+\tstruct nbl_chan_tx_param tx_param = { 0 };\n+\tint i = NBL_CHAN_TX_WAIT_ACK_TIMES;\n+\tstruct nbl_chan_info *chan_info =\n+\t\tchan_mgt-\u003echan_info[NBL_CHAN_TYPE_MAILBOX];\n+\tstruct device *dev = common-\u003edev;\n+\tstruct work_struct *task;\n+\tu16 msgid = 0;\n+\tint ret;\n+\n+\tif (chan_send-\u003eresp_len \u003e NBL_CHAN_BUF_LEN) {\n+\t\tdev_err_ratelimited(dev, \"resp_len %zu exceeds max %d\\n\",\n+\t\t\t\t    chan_send-\u003eresp_len, NBL_CHAN_BUF_LEN);\n+\t\treturn -EINVAL;\n+\t}\n+\n+\t/*\n+\t * A length without a payload would leave the descriptor pointing at\n+\t * whatever the reused slot still holds - nbl_chan_update_txqueue()\n+\t * skips the copy - and the peer would receive those stale bytes.\n+\t * Reject the combination instead of transmitting them.\n+\t */\n+\tif (chan_send-\u003earg_len \u0026\u0026 !chan_send-\u003earg) {\n+\t\tdev_err_ratelimited(dev, \"arg_len %zu without arg\\n\",\n+\t\t\t\t    chan_send-\u003earg_len);\n+\t\treturn -EINVAL;\n+\t}\n+\n+\t/*\n+\t * Entry points are only reached after a successful setup_queue()\n+\t * (probe path), but the channel is a dynamic object: fail cleanly\n+\t * instead of dereferencing rings/wait[] that were never allocated\n+\t * (num_txq_entries == 0).  active is published last in setup and\n+\t * cleared last in teardown, so a false value here means the channel\n+\t * never came up - or has already been torn down, and shutdn tells\n+\t * the two apart: it is set before active is cleared, so a\n+\t * post-teardown sender gets the same -ESHUTDOWN as the check below\n+\t * rather than the -ENODEV reserved for a channel setup_queue()\n+\t * never completed.\n+\t */\n+\tif (!READ_ONCE(chan_info-\u003eactive)) {\n+\t\tif (READ_ONCE(chan_info-\u003eshutdn))\n+\t\t\treturn -ESHUTDOWN;\n+\t\treturn -ENODEV;\n+\t}\n+\n+\tmutex_lock(\u0026chan_info-\u003estate_lock);\n+\tif (READ_ONCE(chan_info-\u003eshutdn)) {\n+\t\tmutex_unlock(\u0026chan_info-\u003estate_lock);\n+\t\treturn -ESHUTDOWN;\n+\t}\n+\tatomic_inc(\u0026chan_info-\u003einflight_tx_cnt);\n+\tmutex_unlock(\u0026chan_info-\u003estate_lock);\n+\n+\ttx_param.msg_type = chan_send-\u003emsg_type;\n+\ttx_param.arg = chan_send-\u003earg;\n+\ttx_param.arg_len = chan_send-\u003earg_len;\n+\ttx_param.dstid = chan_send-\u003edstid;\n+\ttx_param.msgid = msgid;\n+\tif (chan_send-\u003eack) {\n+\t\tmutex_lock(\u0026chan_info-\u003epending_lock);\n+\n+\t\tret = nbl_chan_get_msg_id(chan_info, \u0026msgid);\n+\t\tif (ret) {\n+\t\t\tmutex_unlock(\u0026chan_info-\u003epending_lock);\n+\t\t\tdev_err_ratelimited(dev,\n+\t\t\t\t\t    \"Channel tx wait head full, send msgtype:%u to dstid:%u failed\\n\",\n+\t\t\t\t\t    chan_send-\u003emsg_type,\n+\t\t\t\t\t    chan_send-\u003edstid);\n+\t\t\tgoto out_clean_inflight;\n+\t\t}\n+\t\twait_head =\n+\t\t\t\u0026chan_info-\u003ewait[FIELD_GET(NBL_CHAN_MSGID_LOC_MASK,\n+\t\t\t\t\t\t   msgid)];\n+\t\tWRITE_ONCE(wait_head-\u003eacked, 0);\n+\t\tWRITE_ONCE(wait_head-\u003eack_data, chan_send-\u003eresp);\n+\t\tWRITE_ONCE(wait_head-\u003eack_data_len, chan_send-\u003eresp_len);\n+\t\tWRITE_ONCE(wait_head-\u003emsg_type, chan_send-\u003emsg_type);\n+\t\tWRITE_ONCE(wait_head-\u003emsg_index,\n+\t\t\t   FIELD_GET(NBL_CHAN_MSGID_INDEX_MASK, msgid));\n+\t\tWRITE_ONCE(wait_head-\u003edstid, chan_send-\u003edstid);\n+\n+\t\tWRITE_ONCE(wait_head-\u003estatus, NBL_MBX_STATUS_WAITING);\n+\t\tmutex_unlock(\u0026chan_info-\u003epending_lock);\n+\n+\t\ttx_param.msgid = msgid;\n+\t}\n+\n+\tmutex_lock(\u0026chan_info-\u003etxq_lock);\n+\tret = nbl_chan_update_txqueue(chan_mgt, chan_info, \u0026tx_param);\n+\tif (ret) {\n+\t\tmutex_unlock(\u0026chan_info-\u003etxq_lock);\n+\t\tdev_err_ratelimited(dev,\n+\t\t\t\t    \"Channel tx queue full, send msgtype:%u to dstid:%u failed\\n\",\n+\t\t\t\t    chan_send-\u003emsg_type, chan_send-\u003edstid);\n+\t\tif (wait_head)\n+\t\t\tgoto out_clear_wait_slot;\n+\t\tgoto out_clean_inflight;\n+\t}\n+\n+\tret = nbl_chan_kick_tx_ring(chan_mgt, chan_info);\n+\tmutex_unlock(\u0026chan_info-\u003etxq_lock);\n+\tif (ret) {\n+\t\tif (wait_head)\n+\t\t\tgoto out_clear_wait_slot;\n+\t\tgoto out_clean_inflight;\n+\t}\n+\n+\tif (!chan_send-\u003eack) {\n+\t\tret = 0;\n+\t\tgoto out_clean_inflight;\n+\t}\n+\n+\tif (test_bit(NBL_CHAN_IRQ_RDY, chan_info-\u003estate)) {\n+\t\twhile (!READ_ONCE(wait_head-\u003eacked)) {\n+\t\t\t/*\n+\t\t\t * avoids long task blocking when interrupt mode is\n+\t\t\t * disabled mid-wait. Cannot guarantee subsequent ACK\n+\t\t\t * delivery after interrupt mask off, only prevents\n+\t\t\t * infinite blocking. Spurious timeout is possible.\n+\t\t\t */\n+\t\t\tret = wait_event_timeout(wait_head-\u003ewait_queue,\n+\t\t\t\t\t\t READ_ONCE(wait_head-\u003eacked) ||\n+\t\t\t\t\t\t READ_ONCE(chan_info-\u003eshutdn) ||\n+\t\t\t\t\t\t !test_bit(NBL_CHAN_IRQ_RDY,\n+\t\t\t\t\t\t\t   chan_info-\u003estate),\n+\t\t\t\t\t\t NBL_CHAN_ACK_WAIT_TIME);\n+\n+\t\t\tif (READ_ONCE(chan_info-\u003eshutdn)) {\n+\t\t\t\tret = -ESHUTDOWN;\n+\t\t\t\tgoto out_clear_wait_slot;\n+\t\t\t}\n+\t\t\tif (!test_bit(NBL_CHAN_IRQ_RDY, chan_info-\u003estate)) {\n+\t\t\t\tret = -EIO;\n+\t\t\t\tgoto out_clear_wait_slot;\n+\t\t\t}\n+\t\t\tif (ret == 0) {\n+\t\t\t\tmutex_lock(\u0026chan_info-\u003epending_lock);\n+\t\t\t\tif (READ_ONCE(wait_head-\u003estatus) ==\n+\t\t\t\t    NBL_MBX_STATUS_WAITING) {\n+\t\t\t\t\tWRITE_ONCE(wait_head-\u003estatus,\n+\t\t\t\t\t\t   NBL_MBX_STATUS_TIMEOUT);\n+\t\t\t\t\tWRITE_ONCE(wait_head-\u003eacked, 0);\n+\t\t\t\t\tWRITE_ONCE(wait_head-\u003eack_data, NULL);\n+\t\t\t\t\tWRITE_ONCE(wait_head-\u003eack_data_len, 0);\n+\t\t\t\t\t/*\n+\t\t\t\t\t * Ensure all status/ack slot\n+\t\t\t\t\t * updates are visible before subsequent\n+\t\t\t\t\t * readers observe acked == 0\n+\t\t\t\t\t */\n+\t\t\t\t\tsmp_wmb();\n+\t\t\t\t\tmutex_unlock(\u0026chan_info-\u003epending_lock);\n+\t\t\t\t\tdev_err_ratelimited(dev,\n+\t\t\t\t\t\t\t    \"Channel waiting ack failed, message type: %d, msg id: %u\\n\",\n+\t\t\t\t\t\t\t    chan_send-\u003emsg_type,\n+\t\t\t\t\t\t\t    msgid);\n+\t\t\t\t\tret = -ETIMEDOUT;\n+\t\t\t\t\t/*\n+\t\t\t\t\t * TIMEOUT slots can be reused by\n+\t\t\t\t\t * another sender. A late ACK is\n+\t\t\t\t\t * rejected by the receive path,\n+\t\t\t\t\t * which only accepts WAITING slots\n+\t\t\t\t\t * (and by the msg_index generation\n+\t\t\t\t\t * check after reallocation). The\n+\t\t\t\t\t * current sender no longer owns the\n+\t\t\t\t\t * slot, so skip the IDLE reset.\n+\t\t\t\t\t */\n+\t\t\t\t\tgoto out_clean_inflight;\n+\t\t\t\t}\n+\t\t\t\t/*\n+\t\t\t\t * ACK won the timeout race: between the\n+\t\t\t\t * final condition check and this lock the\n+\t\t\t\t * receiver published acked=1/ACKD and\n+\t\t\t\t * copied the payload into our response\n+\t\t\t\t * buffer. The slot is still exclusively\n+\t\t\t\t * ours (only IDLE/TIMEOUT slots are ever\n+\t\t\t\t * reallocated), so consume the completion:\n+\t\t\t\t * unlock and continue to the loop-bottom\n+\t\t\t\t * acked check, which enters the normal\n+\t\t\t\t * success readout and IDLE reset. Leaving\n+\t\t\t\t * here via the timeout path would strand\n+\t\t\t\t * the slot in ACKD forever, permanently\n+\t\t\t\t * shrinking the 256-entry pool, and would\n+\t\t\t\t * report a false -ETIMEDOUT for a request\n+\t\t\t\t * whose response already arrived.\n+\t\t\t\t */\n+\t\t\t\tmutex_unlock(\u0026chan_info-\u003epending_lock);\n+\t\t\t}\n+\n+\t\t\tif (READ_ONCE(wait_head-\u003eacked))\n+\t\t\t\tbreak;\n+\t\t}\n+\t\tif (READ_ONCE(wait_head-\u003eacked)) {\n+\t\t\t/*\n+\t\t\t * Load ordering: observe acked flag before\n+\t\t\t * reading ACK payload metadata.\n+\t\t\t */\n+\t\t\tsmp_rmb();\n+\t\t\tchan_send-\u003eack_len = READ_ONCE(wait_head-\u003eack_data_len);\n+\t\t\tret = READ_ONCE(wait_head-\u003eack_err);\n+\t\t}\n+\t} else {\n+\t\t/* Polling path for synchronous ACK */\n+\t\twhile (i--) {\n+\t\t\tif (READ_ONCE(chan_info-\u003eshutdn)) {\n+\t\t\t\tret = -ESHUTDOWN;\n+\t\t\t\tgoto out_clear_wait_slot;\n+\t\t\t}\n+\n+\t\t\tmutex_lock(\u0026chan_info-\u003estate_lock);\n+\t\t\ttask = READ_ONCE(chan_info-\u003eclean_task);\n+\t\t\tif (common-\u003ewq \u0026\u0026 task \u0026\u0026\n+\t\t\t    !READ_ONCE(chan_info-\u003eshutdn) \u0026\u0026\n+\t\t\t    !work_pending(task))\n+\t\t\t\tqueue_work(common-\u003ewq, task);\n+\t\t\tmutex_unlock(\u0026chan_info-\u003estate_lock);\n+\t\t\tif (READ_ONCE(wait_head-\u003eacked)) {\n+\t\t\t\t/*\n+\t\t\t\t * Guarantee load order: observe acked\n+\t\t\t\t * flag before reading ack payload metadata.\n+\t\t\t\t */\n+\t\t\t\tsmp_rmb();\n+\t\t\t\tchan_send-\u003eack_len =\n+\t\t\t\t\tREAD_ONCE(wait_head-\u003eack_data_len);\n+\t\t\t\tret = READ_ONCE(wait_head-\u003eack_err);\n+\t\t\t\tgoto out_clear_wait_slot;\n+\t\t\t}\n+\n+\t\t\tusleep_range(NBL_CHAN_TX_WAIT_ACK_US_MIN,\n+\t\t\t\t     NBL_CHAN_TX_WAIT_ACK_US_MAX);\n+\t\t\tcond_resched();\n+\t\t}\n+\t\tmutex_lock(\u0026chan_info-\u003epending_lock);\n+\t\tif (READ_ONCE(wait_head-\u003estatus) == NBL_MBX_STATUS_ACKD) {\n+\t\t\tchan_send-\u003eack_len = READ_ONCE(wait_head-\u003eack_data_len);\n+\t\t\tret = READ_ONCE(wait_head-\u003eack_err);\n+\t\t\tmutex_unlock(\u0026chan_info-\u003epending_lock);\n+\t\t\tgoto out_clear_wait_slot;\n+\t\t}\n+\t\t/*\n+\t\t * Polling timed out without receiving an ACK.  Transition\n+\t\t * the slot to TIMEOUT so nbl_chan_get_msg_id() can reuse\n+\t\t * it.  Must not proceed to out_clean_inflight with\n+\t\t * the slot still in WAITING state — that would leak the\n+\t\t * slot permanently and eventually exhaust the 256-entry\n+\t\t * channel queue.\n+\t\t */\n+\t\tif (READ_ONCE(wait_head-\u003estatus) == NBL_MBX_STATUS_WAITING) {\n+\t\t\tWRITE_ONCE(wait_head-\u003estatus, NBL_MBX_STATUS_TIMEOUT);\n+\t\t\tWRITE_ONCE(wait_head-\u003eacked, 0);\n+\t\t\tWRITE_ONCE(wait_head-\u003eack_data, NULL);\n+\t\t\tWRITE_ONCE(wait_head-\u003eack_data_len, 0);\n+\t\t}\n+\t\tmutex_unlock(\u0026chan_info-\u003epending_lock);\n+\t\tdev_err_ratelimited(dev,\n+\t\t\t\t    \"Channel polling ack failed, message type: %d msg id: %u\\n\",\n+\t\t\t\t    chan_send-\u003emsg_type, msgid);\n+\t\tret = -ETIMEDOUT;\n+\t\tgoto out_clean_inflight;\n+\t}\n+\n+out_clear_wait_slot:\n+\tmutex_lock(\u0026chan_info-\u003epending_lock);\n+\tnbl_chan_reset_wait_head(chan_info, wait_head);\n+\tmutex_unlock(\u0026chan_info-\u003epending_lock);\n+\n+out_clean_inflight:\n+\tmutex_lock(\u0026chan_info-\u003estate_lock);\n+\tif (atomic_dec_and_test(\u0026chan_info-\u003einflight_tx_cnt))\n+\t\twake_up(\u0026chan_info-\u003einflight_wait);\n+\tmutex_unlock(\u0026chan_info-\u003estate_lock);\n+\treturn ret;\n+}\n+\n+static int nbl_chan_send_ack(struct nbl_channel_mgt *chan_mgt,\n+\t\t\t     struct nbl_chan_ack_info *chan_ack)\n+{\n+\tsize_t head_len = NBL_CHAN_ACK_HEAD_LEN * sizeof(u32);\n+\tsize_t data_len = chan_ack-\u003edata_len;\n+\tstruct nbl_chan_send_info chan_send;\n+\t__le32 *tmp;\n+\tsize_t len;\n+\tint ret;\n+\n+\tif (data_len \u003e\n+\t    NBL_CHAN_BUF_LEN - sizeof(struct nbl_chan_tx_desc) - head_len)\n+\t\treturn -EINVAL;\n+\n+\tlen = head_len + data_len;\n+\ttmp = kzalloc(len, GFP_KERNEL);\n+\tif (!tmp)\n+\t\treturn -ENOMEM;\n+\n+\t*(__le16 *)\u0026tmp[NBL_CHAN_MSG_TYPE_POS] =\n+\t\tcpu_to_le16(chan_ack-\u003emsg_type);\n+\t*(__le16 *)\u0026tmp[NBL_CHAN_MSG_ID_POS] = cpu_to_le16(chan_ack-\u003emsgid);\n+\ttmp[NBL_CHAN_ACK_RET_POS] = cpu_to_le32(chan_ack-\u003eerr);\n+\tif (chan_ack-\u003edata \u0026\u0026 chan_ack-\u003edata_len)\n+\t\tmemcpy(\u0026tmp[NBL_CHAN_ACK_HEAD_LEN], chan_ack-\u003edata,\n+\t\t       chan_ack-\u003edata_len);\n+\n+\tnbl_chan_fill_send_info(\u0026chan_send, chan_ack-\u003edstid, NBL_CHAN_MSG_ACK,\n+\t\t\t\ttmp, len, NULL, 0, 0);\n+\tret = nbl_chan_send_msg(chan_mgt, \u0026chan_send);\n+\tkfree(tmp);\n+\n+\treturn ret;\n+}\n+\n+static int nbl_chan_register_msg(struct nbl_channel_mgt *chan_mgt, u16 msg_type,\n+\t\t\t\t nbl_chan_resp func, void *callback)\n+{\n+\treturn nbl_chan_add_msg_handler(chan_mgt, msg_type, func, callback);\n+}\n+\n+static bool nbl_chan_check_queue_exist(struct nbl_channel_mgt *chan_mgt,\n+\t\t\t\t       u8 chan_type)\n+{\n+\tstruct nbl_chan_info *chan_info = chan_mgt-\u003echan_info[chan_type];\n+\n+\treturn chan_info ? true : false;\n+}\n+\n+static void nbl_chan_register_chan_task(struct nbl_channel_mgt *chan_mgt,\n+\t\t\t\t\tu8 chan_type, struct work_struct *task)\n+{\n+\tstruct nbl_chan_info *chan_info = chan_mgt-\u003echan_info[chan_type];\n+\n+\tmutex_lock(\u0026chan_info-\u003estate_lock);\n+\tif (!READ_ONCE(chan_info-\u003eshutdn))\n+\t\tWRITE_ONCE(chan_info-\u003eclean_task, task);\n+\tmutex_unlock(\u0026chan_info-\u003estate_lock);\n+}\n+\n+static void nbl_chan_set_queue_state(struct nbl_channel_mgt *chan_mgt,\n+\t\t\t\t     enum nbl_chan_state state, u8 chan_type,\n+\t\t\t\t     u8 set)\n+{\n+\tstruct nbl_chan_info *chan_info = chan_mgt-\u003echan_info[chan_type];\n+\tint i;\n+\n+\tif (set)\n+\t\tset_bit(state, chan_info-\u003estate);\n+\telse\n+\t\tclear_bit(state, chan_info-\u003estate);\n+\t/*\n+\t * When clearing IRQ_RDY, wake all per-slot wait queues so\n+\t * sleeping senders observe the condition immediately and\n+\t * return -EIO instead of waiting out the 3s timeout and\n+\t * reporting -ETIMEDOUT.\n+\t *\n+\t * wait[] is only allocated by a successful nbl_chan_init_tx_queue(),\n+\t * while num_txq_entries is already set by nbl_chan_init_queue_param();\n+\t * skip the walk if that allocation never happened.\n+\t */\n+\tif (!set \u0026\u0026 state == NBL_CHAN_IRQ_RDY \u0026\u0026 chan_info-\u003ewait) {\n+\t\tfor (i = 0; i \u003c chan_info-\u003enum_txq_entries; i++)\n+\t\t\twake_up_all(\u0026chan_info-\u003ewait[i].wait_queue);\n+\t}\n+}\n+\n+static struct nbl_channel_ops chan_ops = {\n+\t.send_msg\t\t\t= nbl_chan_send_msg,\n+\t.send_ack\t\t\t= nbl_chan_send_ack,\n+\t.register_msg\t\t\t= nbl_chan_register_msg,\n+\t.cfg_chan_qinfo_map_table\t= nbl_chan_cfg_qinfo_map_table,\n+\t.check_queue_exist\t\t= nbl_chan_check_queue_exist,\n+\t.setup_queue\t\t\t= nbl_chan_setup_queue,\n+\t.teardown_queue\t\t\t= nbl_chan_teardown_queue,\n+\t.clean_queue_subtask\t\t= nbl_chan_clean_queue_subtask,\n+\t.register_chan_task\t\t= nbl_chan_register_chan_task,\n+\t.set_queue_state\t\t= nbl_chan_set_queue_state,\n+};\n+\n+static struct nbl_channel_mgt *\n+nbl_chan_setup_chan_mgt(struct nbl_adapter *adapter)\n+{\n+\tstruct nbl_hw_ops_tbl *hw_ops_tbl = adapter-\u003eintf.hw_ops_tbl;\n+\tstruct nbl_common_info *common = \u0026adapter-\u003ecommon;\n+\tstruct device *dev = \u0026adapter-\u003epdev-\u003edev;\n+\tstruct nbl_channel_mgt *chan_mgt;\n+\tstruct nbl_chan_info *mailbox;\n+\tint ret;\n+\n+\tchan_mgt = devm_kzalloc(dev, sizeof(*chan_mgt), GFP_KERNEL);\n+\tif (!chan_mgt)\n+\t\treturn ERR_PTR(-ENOMEM);\n+\n+\tchan_mgt-\u003ecommon = common;\n+\tchan_mgt-\u003ehw_ops_tbl = hw_ops_tbl;\n+\n+\tmailbox = devm_kzalloc(dev, sizeof(*mailbox), GFP_KERNEL);\n+\tif (!mailbox)\n+\t\treturn ERR_PTR(-ENOMEM);\n+\tmailbox-\u003echan_type = NBL_CHAN_TYPE_MAILBOX;\n+\tchan_mgt-\u003echan_info[NBL_CHAN_TYPE_MAILBOX] = mailbox;\n+\n+\tret = nbl_chan_init_msg_handler(chan_mgt);\n+\tif (ret)\n+\t\treturn ERR_PTR(ret);\n+\tret = devm_mutex_init(common-\u003edev, \u0026mailbox-\u003etxq_lock);\n+\tif (ret)\n+\t\treturn ERR_PTR(ret);\n+\tret = devm_mutex_init(common-\u003edev, \u0026mailbox-\u003estate_lock);\n+\tif (ret)\n+\t\treturn ERR_PTR(ret);\n+\tret = devm_mutex_init(common-\u003edev, \u0026mailbox-\u003epending_lock);\n+\tif (ret)\n+\t\treturn ERR_PTR(ret);\n+\treturn chan_mgt;\n+}\n+\n+static struct nbl_channel_ops_tbl *\n+nbl_chan_setup_ops(struct device *dev, struct nbl_channel_mgt *chan_mgt)\n+{\n+\tstruct nbl_channel_ops_tbl *chan_ops_tbl;\n+\tint ret;\n+\n+\tchan_ops_tbl = devm_kzalloc(dev, sizeof(*chan_ops_tbl), GFP_KERNEL);\n+\tif (!chan_ops_tbl)\n+\t\treturn ERR_PTR(-ENOMEM);\n+\tif (!chan_ops.send_msg || !chan_ops.send_ack ||\n+\t    !chan_ops.register_msg || !chan_ops.cfg_chan_qinfo_map_table ||\n+\t    !chan_ops.check_queue_exist || !chan_ops.setup_queue ||\n+\t    !chan_ops.teardown_queue || !chan_ops.clean_queue_subtask ||\n+\t    !chan_ops.register_chan_task || !chan_ops.set_queue_state)\n+\t\treturn ERR_PTR(-EINVAL);\n+\n+\tchan_ops_tbl-\u003eops = \u0026chan_ops;\n+\tchan_ops_tbl-\u003epriv = chan_mgt;\n+\n+\tret = nbl_chan_register_msg(chan_mgt, NBL_CHAN_MSG_ACK,\n+\t\t\t\t    nbl_chan_recv_ack_msg, chan_mgt);\n+\tif (ret)\n+\t\treturn ERR_PTR(ret);\n+\n+\treturn chan_ops_tbl;\n+}\n+\n+int nbl_chan_init_common(struct nbl_adapter *adap)\n+{\n+\tstruct nbl_channel_ops_tbl *chan_ops_tbl;\n+\tstruct device *dev = \u0026adap-\u003epdev-\u003edev;\n+\tstruct nbl_channel_mgt *chan_mgt;\n+\tint ret;\n+\n+\tchan_mgt = nbl_chan_setup_chan_mgt(adap);\n+\tif (IS_ERR(chan_mgt))\n+\t\treturn PTR_ERR(chan_mgt);\n+\n+\tchan_ops_tbl = nbl_chan_setup_ops(dev, chan_mgt);\n+\tif (IS_ERR(chan_ops_tbl)) {\n+\t\tret = PTR_ERR(chan_ops_tbl);\n+\t\tgoto cleanup_mgt;\n+\t}\n+\n+\tadap-\u003eintf.channel_ops_tbl = chan_ops_tbl;\n+\tadap-\u003ecore.chan_mgt = chan_mgt;\n+\tret = nbl_common_create_wq(\u0026adap-\u003ecommon);\n+\tif (ret)\n+\t\tgoto cleanup_mgt;\n+\treturn 0;\n+\n+cleanup_mgt:\n+\tnbl_chan_remove_msg_handler(chan_mgt);\n+\treturn ret;\n+}\n+\n+void nbl_chan_remove_common(struct nbl_adapter *adap)\n+{\n+\tstruct nbl_channel_mgt *chan_mgt = adap-\u003ecore.chan_mgt;\n+\n+\tif (!chan_mgt)\n+\t\treturn;\n+\tnbl_common_destroy_wq(\u0026adap-\u003ecommon);\n+\t/*\n+\t * All channel queues shall be torn down earlier in remove path\n+\t * to drain inflight tx workers and stop hardware before destroying\n+\t * message handler xarray.\n+\t */\n+\tnbl_chan_remove_msg_handler(chan_mgt);\n+\tadap-\u003ecore.chan_mgt = NULL;\n+}\ndiff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_channel/nbl_channel.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_channel/nbl_channel.h\nnew file mode 100644\nindex 00000000000000..168f5de221bb78\n--- /dev/null\n+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_channel/nbl_channel.h\n@@ -0,0 +1,196 @@\n+/* SPDX-License-Identifier: GPL-2.0 */\n+/*\n+ * Copyright (c) 2026 Nebula Matrix Limited.\n+ */\n+\n+#ifndef _NBL_CHANNEL_H_\n+#define _NBL_CHANNEL_H_\n+\n+#include \u003clinux/types.h\u003e\n+#include \u003clinux/xarray.h\u003e\n+\n+#include \"../nbl_include/nbl_include.h\"\n+#include \"../nbl_include/nbl_def_channel.h\"\n+#include \"../nbl_include/nbl_def_hw.h\"\n+#include \"../nbl_include/nbl_def_common.h\"\n+#include \"../nbl_core.h\"\n+\n+#define NBL_CHAN_TX_RING_TO_DESC(tx_ring, i) \\\n+\t(\u0026((((tx_ring)-\u003edesc.tx_desc))[i]))\n+#define NBL_CHAN_RX_RING_TO_DESC(rx_ring, i) \\\n+\t(\u0026((((rx_ring)-\u003edesc.rx_desc))[i]))\n+#define NBL_CHAN_TX_RING_TO_BUF(tx_ring, i) (\u0026(((tx_ring)-\u003ebuf)[i]))\n+#define NBL_CHAN_RX_RING_TO_BUF(rx_ring, i) (\u0026(((rx_ring)-\u003ebuf)[i]))\n+\n+#define NBL_CHAN_TX_WAIT_US\t\t\t100\n+#define NBL_CHAN_TX_WAIT_US_MAX\t\t\t120\n+#define NBL_CHAN_TX_WAIT_TIMES\t\t\t100\n+#define NBL_CHAN_TX_WAIT_ACK_US_MIN\t\t1000\n+#define NBL_CHAN_TX_WAIT_ACK_US_MAX\t\t1200\n+#define NBL_CHAN_TX_WAIT_ACK_TIMES\t\t5000\n+#define NBL_CHAN_QUEUE_LEN\t\t\t256\n+#define NBL_CHAN_BUF_LEN\t\t\t4096\n+#define NBL_CHAN_TX_DESC_EMBEDDED_DATA_LEN\t16\n+\n+#define NBL_CHAN_TX_DESC_AVAIL\t\t\t0\n+#define NBL_CHAN_TX_DESC_USED\t\t\t1\n+#define NBL_CHAN_RX_DESC_WRITE\t\t\t1\n+#define NBL_CHAN_RX_DESC_AVAIL\t\t\t3\n+#define NBL_CHAN_RX_DESC_USED\t\t\t4\n+\n+#define NBL_CHAN_ACK_HEAD_LEN\t\t\t3\n+#define NBL_CHAN_ACK_RET_POS\t\t\t2\n+#define NBL_CHAN_MSG_ID_POS\t\t\t1\n+#define NBL_CHAN_MSG_TYPE_POS\t\t\t0\n+\n+#define NBL_CHAN_ACK_WAIT_TIME\t\t\t(3 * HZ)\n+#define NBL_CHAN_RX_CLEAN_BUDGET\t\t64\n+\n+enum {\n+\tNBL_MB_RX_QID = 0,\n+\tNBL_MB_TX_QID = 1,\n+};\n+\n+enum {\n+\tNBL_MBX_STATUS_IDLE = 0,\n+\tNBL_MBX_STATUS_WAITING,\n+\tNBL_MBX_STATUS_ACKD,\n+\tNBL_MBX_STATUS_TIMEOUT,\n+};\n+\n+struct nbl_chan_tx_param {\n+\tenum nbl_chan_msg_type msg_type;\n+\tvoid *arg;\n+\tsize_t arg_len;\n+\tu16 dstid;\n+\tu16 msgid;\n+};\n+\n+struct nbl_chan_buf {\n+\tvoid *va;\n+\tdma_addr_t pa;\n+\tsize_t size;\n+};\n+\n+struct nbl_chan_tx_desc {\n+\t__le16 flags;\n+\t__le16 srcid;\n+\t__le16 dstid;\n+\t__le16 data_len;\n+\t__le16 buf_len;\n+\t__le64 buf_addr;\n+\t__le16 msg_type;\n+\tu8 data[NBL_CHAN_TX_DESC_EMBEDDED_DATA_LEN];\n+\t__le16 msgid;\n+\tu8 rsv[26];\n+} __packed;\n+\n+struct nbl_chan_rx_desc {\n+\t__le16 flags;\n+\t__le32 buf_len;\n+\t__le16 buf_id;\n+\t__le64 buf_addr;\n+} __packed;\n+\n+union nbl_chan_desc_ptr {\n+\tstruct nbl_chan_tx_desc *tx_desc;\n+\tstruct nbl_chan_rx_desc *rx_desc;\n+};\n+\n+struct nbl_chan_ring {\n+\tunion nbl_chan_desc_ptr desc;\n+\t/* Per-entry slots carved out of one contiguous coherent block */\n+\tstruct nbl_chan_buf *buf;\n+\tu16 next_to_use;\n+\tu16 tail_ptr; /* hardware does modulo ring size internally */\n+\tu16 next_to_clean;\n+\tdma_addr_t dma;\n+};\n+\n+#define NBL_CHAN_MSG_INDEX_MAX 63\n+\n+#define NBL_CHAN_MSGID_INDEX_MASK GENMASK(5, 0)\n+#define NBL_CHAN_MSGID_LOC_MASK GENMASK(13, 6)\n+\n+static inline void nbl_chan_update_tail_ptr(struct nbl_hw_ops *hw_ops,\n+\t\t\t\t\t    void *hw_priv, u32 tail_ptr, u8 qid)\n+{\n+\thw_ops-\u003eupdate_mailbox_queue_tail_ptr(hw_priv, tail_ptr, qid);\n+}\n+\n+struct nbl_chan_waitqueue_head {\n+\tstruct wait_queue_head wait_queue;\n+\tchar *ack_data;\n+\tint acked;\n+\ts32 ack_err;\n+\tu16 ack_data_len;\n+\tu16 msg_type;\n+\tint status;\n+\tu8 msg_index;\n+\tu16 dstid;\n+};\n+\n+struct nbl_chan_info {\n+\twait_queue_head_t inflight_wait;\n+\tstruct nbl_chan_ring txq;\n+\tstruct nbl_chan_ring rxq;\n+\tstruct nbl_chan_waitqueue_head *wait;\n+\t/*\n+\t * Serializes TX ring producer state (next_to_use/next_to_clean/\n+\t * tail_ptr), descriptor publication and the doorbell, and the TX\n+\t * queue config/stop sequences against\n+\t * nbl_chan_quiesce_and_reclaim_tx().\n+\t */\n+\tstruct mutex txq_lock;\n+\t/*\n+\t * Serializes the channel lifecycle: shutdn publication, clean_task\n+\t * registration/clearing, inflight_tx_cnt transitions and the\n+\t * inflight_wait wakeup.  The state bitmap is not covered by this\n+\t * lock - it is only touched with atomic bit ops (set_bit(),\n+\t * clear_bit(), test_bit()).\n+\t */\n+\tstruct mutex state_lock;\n+\t/*\n+\t * Serializes the ACK wait slot array (wait[]): slot allocation in\n+\t * nbl_chan_get_msg_id(), slot field publication in\n+\t * nbl_chan_send_msg(), ACK completion in nbl_chan_recv_ack_msg()\n+\t * and slot reset in nbl_chan_reset_wait_head().\n+\t */\n+\tstruct mutex pending_lock;\n+\tstruct work_struct *clean_task;\n+\tu16 wait_head_index;\n+\tu16 num_txq_entries;\n+\tu16 num_rxq_entries;\n+\tu16 txq_buf_size;\n+\tu16 rxq_buf_size;\n+\tDECLARE_BITMAP(state, NBL_CHAN_STATE_NBITS);\n+\tu8 chan_type;\n+\tatomic_t inflight_tx_cnt;\n+\tbool shutdn;\n+\tbool active;\n+\t/*\n+\t * One-shot lifecycle flag: DMA allocations use devm (descriptor/buf\n+\t * arrays) and dmam (coherent descriptor ring, per-ring contiguous\n+\t * buffer block) and are only released at device detach.  Teardown\n+\t * only stops the hardware and clears active; it does not free\n+\t * memory.  setup must never run twice on the same chan_info or the\n+\t * first allocation set would be orphaned.\n+\t */\n+\tbool dma_allocated;\n+};\n+\n+struct nbl_chan_msg_node_data {\n+\tnbl_chan_resp func;\n+\tvoid *priv;\n+};\n+\n+struct nbl_channel_mgt {\n+\tstruct nbl_common_info *common;\n+\tstruct nbl_hw_ops_tbl *hw_ops_tbl;\n+\tstruct nbl_chan_info *chan_info[NBL_CHAN_TYPE_MAX];\n+\t/* Serializes handler_xa registration and teardown */\n+\tstruct mutex handler_lock;\n+\tstruct xarray handler_xa;\n+};\n+\n+#endif\ndiff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_common/nbl_common.c b/drivers/net/ethernet/nebula-matrix/nbl/nbl_common/nbl_common.c\nnew file mode 100644\nindex 00000000000000..302c3c48dc4d59\n--- /dev/null\n+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_common/nbl_common.c\n@@ -0,0 +1,52 @@\n+// SPDX-License-Identifier: GPL-2.0\n+/*\n+ * Copyright (c) 2026 Nebula Matrix Limited.\n+ */\n+\n+#include \u003clinux/device.h\u003e\n+#include \"nbl_common.h\"\n+\n+void nbl_common_destroy_wq(struct nbl_common_info *common)\n+{\n+\tif (!common || !common-\u003ewq)\n+\t\treturn;\n+\n+\tdestroy_workqueue(common-\u003ewq);\n+\tcommon-\u003ewq = NULL;\n+}\n+\n+int nbl_common_create_wq(struct nbl_common_info *common)\n+{\n+\tchar wq_name[32];\n+\n+\tsnprintf(wq_name, sizeof(wq_name), \"nbl_wq_%s\", pci_name(common-\u003epdev));\n+\tcommon-\u003ewq = alloc_workqueue(wq_name, WQ_UNBOUND, 0);\n+\tif (!common-\u003ewq) {\n+\t\tdev_err(common-\u003edev, \"Failed to alloc workqueue %s\\n\", wq_name);\n+\t\treturn -ENOMEM;\n+\t}\n+\n+\treturn 0;\n+}\n+\n+/**\n+ * nbl_common_func_id_to_rel_pf_id - convert absolute PF id to relative PF id\n+ * @common: common device info\n+ * @pf_id: absolute PF identifier\n+ * @rel_pf_id: output relative pf id\n+ *\n+ * Leonis uses fixed mgt_pf = 0. Support future non-zero management PF.\n+ *\n+ * Return: 0 on success, -EINVAL on invalid arguments.\n+ */\n+int nbl_common_func_id_to_rel_pf_id(struct nbl_common_info *common, u32 pf_id,\n+\t\t\t\t    u32 *rel_pf_id)\n+{\n+\tif (!rel_pf_id)\n+\t\treturn -EINVAL;\n+\n+\tif (pf_id \u003c common-\u003emgt_pf)\n+\t\treturn -EINVAL;\n+\t*rel_pf_id = pf_id - common-\u003emgt_pf;\n+\treturn 0;\n+}\ndiff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_common/nbl_common.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_common/nbl_common.h\nnew file mode 100644\nindex 00000000000000..129073b08eb903\n--- /dev/null\n+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_common/nbl_common.h\n@@ -0,0 +1,14 @@\n+/* SPDX-License-Identifier: GPL-2.0 */\n+/*\n+ * Copyright (c) 2026 Nebula Matrix Limited.\n+ */\n+\n+#ifndef _NBL_COMMON_H_\n+#define _NBL_COMMON_H_\n+\n+#include \u003clinux/types.h\u003e\n+\n+#include \"../nbl_include/nbl_include.h\"\n+#include \"../nbl_include/nbl_def_common.h\"\n+\n+#endif\ndiff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_core.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_core.h\nindex 1cd6587a8fcb66..655dfb43e3653d 100644\n--- a/drivers/net/ethernet/nebula-matrix/nbl/nbl_core.h\n+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_core.h\n@@ -14,13 +14,25 @@ enum {\n \tNBL_CAP_HAS_NET_BIT,\n };\n \n+struct nbl_interface {\n+\tstruct nbl_hw_ops_tbl *hw_ops_tbl;\n+\tstruct nbl_resource_ops_tbl *resource_ops_tbl;\n+\tstruct nbl_dispatch_ops_tbl *dispatch_ops_tbl;\n+\tstruct nbl_channel_ops_tbl *channel_ops_tbl;\n+};\n+\n struct nbl_core {\n \tstruct nbl_hw_mgt *hw_mgt;\n+\tstruct nbl_resource_mgt *res_mgt;\n+\tstruct nbl_dispatch_mgt *disp_mgt;\n+\tstruct nbl_dev_mgt *dev_mgt;\n+\tstruct nbl_channel_mgt *chan_mgt;\n };\n \n struct nbl_adapter {\n \tstruct pci_dev *pdev;\n \tstruct nbl_core core;\n+\tstruct nbl_interface intf;\n \tstruct nbl_common_info common;\n };\n \n@@ -28,4 +40,12 @@ struct nbl_adapter *nbl_core_init(struct pci_dev *pdev,\n \t\t\t\t  struct nbl_init_param *param);\n void nbl_core_remove(struct nbl_adapter *adapter);\n \n+/*\n+ * Single-shot start/stop pair, called once each from PCI probe/remove.\n+ * Not repeatable: MSI-X vectors stay allocated until device detach, so\n+ * a second start on a bound device is rejected by the MSI-X core.\n+ */\n+int nbl_core_start(struct nbl_adapter *adapter);\n+void nbl_core_stop(struct nbl_adapter *adapter);\n+\n #endif\ndiff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_core/nbl_dev.c b/drivers/net/ethernet/nebula-matrix/nbl/nbl_core/nbl_dev.c\nnew file mode 100644\nindex 00000000000000..c25def18a9e684\n--- /dev/null\n+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_core/nbl_dev.c\n@@ -0,0 +1,727 @@\n+// SPDX-License-Identifier: GPL-2.0\n+/*\n+ * Copyright (c) 2026 Nebula Matrix Limited.\n+ */\n+#include \u003clinux/device.h\u003e\n+#include \u003clinux/pci.h\u003e\n+#include \"nbl_dev.h\"\n+\n+static void nbl_dev_clean_mailbox_schedule(struct nbl_dev_mgt *dev_mgt);\n+\n+/* ----------  Interrupt config  ---------- */\n+static irqreturn_t nbl_dev_clean_mailbox(int irq __always_unused, void *data)\n+{\n+\tstruct nbl_dev_mgt *dev_mgt = (struct nbl_dev_mgt *)data;\n+\n+\tnbl_dev_clean_mailbox_schedule(dev_mgt);\n+\treturn IRQ_HANDLED;\n+}\n+\n+static void nbl_dev_init_msix_cnt(struct nbl_dev_mgt *dev_mgt)\n+{\n+\tstruct nbl_dev_common *dev_common = dev_mgt-\u003ecommon_dev;\n+\tstruct nbl_msix_info *msix_info = \u0026dev_common-\u003emsix_info;\n+\n+\t/* Reserve one MSI-X vector for the mailbox.  Nothing here\n+\t * allocates or requests it: vectors are allocated and the handler\n+\t * attached when the device is started, after probe, from this\n+\t * count.\n+\t */\n+\tmsix_info-\u003eserv_info[NBL_MSIX_MAILBOX_TYPE].num = 1;\n+}\n+\n+static int nbl_dev_request_mailbox_irq(struct nbl_dev_mgt *dev_mgt)\n+{\n+\tstruct nbl_dev_common *dev_common = dev_mgt-\u003ecommon_dev;\n+\tstruct nbl_msix_info *msix_info = \u0026dev_common-\u003emsix_info;\n+\tstruct nbl_common_info *common = dev_mgt-\u003ecommon;\n+\tu16 lvec;\n+\tint irq_num;\n+\tint err;\n+\n+\tif (!msix_info-\u003eserv_info[NBL_MSIX_MAILBOX_TYPE].num)\n+\t\treturn 0;\n+\n+\tlvec = msix_info-\u003eserv_info[NBL_MSIX_MAILBOX_TYPE].base_vector_id;\n+\tirq_num = pci_irq_vector(common-\u003epdev, lvec);\n+\tif (irq_num \u003c 0) {\n+\t\tdev_err(common-\u003edev, \"Failed to get mailbox IRQ vector: %d\\n\",\n+\t\t\tirq_num);\n+\t\treturn irq_num;\n+\t}\n+\n+\tsnprintf(dev_common-\u003emailbox_name, sizeof(dev_common-\u003emailbox_name),\n+\t\t \"nbl_mailbox@pci:%s\", pci_name(common-\u003epdev));\n+\terr = request_irq(irq_num, nbl_dev_clean_mailbox, 0,\n+\t\t\t  dev_common-\u003emailbox_name, dev_mgt);\n+\tif (err)\n+\t\treturn err;\n+\n+\treturn 0;\n+}\n+\n+static void nbl_dev_free_mailbox_irq(struct nbl_dev_mgt *dev_mgt)\n+{\n+\tstruct nbl_dev_common *dev_common = dev_mgt-\u003ecommon_dev;\n+\tstruct nbl_msix_info *msix_info = \u0026dev_common-\u003emsix_info;\n+\tstruct nbl_common_info *common = dev_mgt-\u003ecommon;\n+\tu16 lvec;\n+\tint irq_num;\n+\n+\tif (!msix_info-\u003eserv_info[NBL_MSIX_MAILBOX_TYPE].num)\n+\t\treturn;\n+\n+\tlvec = msix_info-\u003eserv_info[NBL_MSIX_MAILBOX_TYPE].base_vector_id;\n+\tirq_num = pci_irq_vector(common-\u003epdev, lvec);\n+\tif (irq_num \u003e= 0)\n+\t\tfree_irq(irq_num, dev_mgt);\n+}\n+\n+static int nbl_dev_enable_mailbox_irq(struct nbl_dev_mgt *dev_mgt)\n+{\n+\tstruct nbl_dispatch_ops *disp_ops = dev_mgt-\u003edisp_ops_tbl-\u003eops;\n+\tstruct nbl_channel_ops *chan_ops = dev_mgt-\u003echan_ops_tbl-\u003eops;\n+\tstruct nbl_dev_common *dev_common = dev_mgt-\u003ecommon_dev;\n+\tstruct nbl_msix_info *msix_info = \u0026dev_common-\u003emsix_info;\n+\tu16 lvec;\n+\tint ret;\n+\n+\tif (!msix_info-\u003eserv_info[NBL_MSIX_MAILBOX_TYPE].num)\n+\t\treturn 0;\n+\n+\tlvec = msix_info-\u003eserv_info[NBL_MSIX_MAILBOX_TYPE].base_vector_id;\n+\t/*\n+\t * Enable sequence: perform set_mailbox_irq RPC in polling mode first.\n+\t * Only set NBL_CHAN_IRQ_RDY after RPC succeeds, mirroring disable path.\n+\t * This avoids waiting for an interrupt which has not been armed yet.\n+\t */\n+\tret = disp_ops-\u003eset_mailbox_irq(dev_mgt-\u003edisp_ops_tbl-\u003epriv,\n+\t\t\t\t\tlvec, true);\n+\tif (ret)\n+\t\treturn ret;\n+\tchan_ops-\u003eset_queue_state(dev_mgt-\u003echan_ops_tbl-\u003epriv,\n+\t\t\t\t  NBL_CHAN_IRQ_RDY,\n+\t\t\t\t  NBL_CHAN_TYPE_MAILBOX, true);\n+\treturn 0;\n+}\n+\n+static int nbl_dev_disable_mailbox_irq(struct nbl_dev_mgt *dev_mgt)\n+{\n+\tstruct nbl_dispatch_ops *disp_ops = dev_mgt-\u003edisp_ops_tbl-\u003eops;\n+\tstruct nbl_channel_ops *chan_ops = dev_mgt-\u003echan_ops_tbl-\u003eops;\n+\tstruct nbl_dev_common *dev_common = dev_mgt-\u003ecommon_dev;\n+\tstruct nbl_msix_info *msix_info = \u0026dev_common-\u003emsix_info;\n+\tu16 lvec;\n+\n+\tif (!msix_info-\u003eserv_info[NBL_MSIX_MAILBOX_TYPE].num)\n+\t\treturn 0;\n+\n+\tlvec = msix_info-\u003eserv_info[NBL_MSIX_MAILBOX_TYPE].base_vector_id;\n+\t/*\n+\t * Disable sequence invariant: clear the software READY flag first,\n+\t * then mask the hardware interrupt. Must not reverse the order.\n+\t *\n+\t * Clearing NBL_CHAN_IRQ_RDY is what makes an interrupt-mode sender\n+\t * fail fast: nbl_chan_set_queue_state() wakes every per-slot wait\n+\t * queue on the transition, and nbl_chan_send_msg() tests the flag\n+\t * ahead of its timeout branch, so the sender returns -EIO right\n+\t * away instead of waiting out NBL_CHAN_ACK_WAIT_TIME (3s) and\n+\t * reporting a bogus -ETIMEDOUT / \"Channel waiting ack failed\".\n+\t * Masking first would leave the sleeper blocked on an interrupt\n+\t * that is already masked, i.e. guaranteed to time out.\n+\t *\n+\t * This does NOT preserve in-flight ACKs, and must not be described\n+\t * as doing so: the woken sender gives up and resets its slot to\n+\t * IDLE, so an ACK arriving afterwards finds a non-WAITING slot and\n+\t * is dropped by the receive path (\"Skip ack invalid status\").\n+\t * Interrupt mode is being torn down here; the guarantee is a\n+\t * bounded, deterministic failure, not delivery.  In this driver\n+\t * every ACK-requesting sender runs from probe/remove and the clean\n+\t * task only sends ACK-less replies, so nothing depends on the\n+\t * stronger claim today.\n+\t *\n+\t * The only caller is nbl_dev_stop(): free_irq() blocks until\n+\t * any in-flight hardirq handler completes and prevents new interrupts.\n+\t * cancel_work_sync() then waits for any already running mailbox cleanup\n+\t * work to finish, or cancels queued but unstarted work items before\n+\t * final channel destruction. No stuck descriptors linger.\n+\t *\n+\t * nbl_dev_start()'s unwind path does not reach here: err_free_irq runs\n+\t * before IRQ_RDY is set, so only nbl_dev_free_mailbox_irq() applies\n+\t * there and no set_mailbox_irq(false) is sent - see the comment on\n+\t * that label.\n+\t */\n+\tchan_ops-\u003eset_queue_state(dev_mgt-\u003echan_ops_tbl-\u003epriv,\n+\t\t\t\t  NBL_CHAN_IRQ_RDY,\n+\t\t\t\t  NBL_CHAN_TYPE_MAILBOX, false);\n+\n+\treturn disp_ops-\u003eset_mailbox_irq(dev_mgt-\u003edisp_ops_tbl-\u003epriv,\n+\t\t\t\t\t lvec, false);\n+}\n+\n+static int nbl_dev_cfg_msix_map(struct nbl_dev_mgt *dev_mgt)\n+{\n+\tstruct nbl_dispatch_ops *disp_ops = dev_mgt-\u003edisp_ops_tbl-\u003eops;\n+\tstruct nbl_dev_common *dev_common = dev_mgt-\u003ecommon_dev;\n+\tstruct nbl_msix_info *msix_info = \u0026dev_common-\u003emsix_info;\n+\tbool mask_en = msix_info-\u003eserv_info[NBL_MSIX_NET_TYPE].hw_self_mask_en;\n+\tu16 msix_net_num = msix_info-\u003eserv_info[NBL_MSIX_NET_TYPE].num;\n+\tu16 msix_not_net_num = 0;\n+\tint err, i;\n+\n+\tmsix_info-\u003eserv_info[NBL_MSIX_NET_TYPE].base_vector_id = 0;\n+\t/*\n+\t * Calculate base_vector_id for each MSIX service type.\n+\t * This relies on NBL_MSIX_TYPE enum being ordered sequentially,\n+\t * starting from NBL_MSIX_NET_TYPE.\n+\t */\n+\tfor (i = NBL_MSIX_NET_TYPE + 1; i \u003c NBL_MSIX_TYPE_MAX; i++)\n+\t\tmsix_info-\u003eserv_info[i].base_vector_id =\n+\t\t\tmsix_info-\u003eserv_info[i - 1].base_vector_id +\n+\t\t\tmsix_info-\u003eserv_info[i - 1].num;\n+\n+\tfor (i = 0; i \u003c NBL_MSIX_TYPE_MAX; i++) {\n+\t\tif (i == NBL_MSIX_NET_TYPE)\n+\t\t\tcontinue;\n+\t\tmsix_not_net_num += msix_info-\u003eserv_info[i].num;\n+\t}\n+\n+\terr = disp_ops-\u003ecfg_msix_map(dev_mgt-\u003edisp_ops_tbl-\u003epriv,\n+\t\t\t\t     msix_net_num, msix_not_net_num,\n+\t\t\t\t     mask_en);\n+\n+\treturn err;\n+}\n+\n+static int nbl_dev_destroy_msix_map(struct nbl_dev_mgt *dev_mgt)\n+{\n+\tstruct nbl_dispatch_ops *disp_ops = dev_mgt-\u003edisp_ops_tbl-\u003eops;\n+\n+\treturn disp_ops-\u003edestroy_msix_map(dev_mgt-\u003edisp_ops_tbl-\u003epriv);\n+}\n+\n+static int nbl_dev_init_interrupt_scheme(struct nbl_dev_mgt *dev_mgt)\n+{\n+\tstruct nbl_dev_common *dev_common = dev_mgt-\u003ecommon_dev;\n+\tstruct nbl_msix_info *msix_info = \u0026dev_common-\u003emsix_info;\n+\tstruct nbl_common_info *common = dev_mgt-\u003ecommon;\n+\tstruct irq_affinity affd = { 0 };\n+\tint needed = 0;\n+\tint err;\n+\tint i;\n+\n+\tfor (i = 0; i \u003c NBL_MSIX_TYPE_MAX; i++)\n+\t\tneeded += msix_info-\u003eserv_info[i].num;\n+\n+\t/*\n+\t * The mailbox vector is the trailing administrative vector;\n+\t * reserve it via post_vectors so it is excluded from managed\n+\t * affinity spreading. A managed vector can be shut down when\n+\t * its last assigned CPU goes offline, which would stall every\n+\t * mailbox RPC while NBL_CHAN_IRQ_RDY stays set.\n+\t */\n+\taffd.post_vectors = msix_info-\u003eserv_info[NBL_MSIX_MAILBOX_TYPE].num;\n+\n+\terr = pci_alloc_irq_vectors_affinity(common-\u003epdev, needed, needed,\n+\t\t\t\t\t     PCI_IRQ_MSIX | PCI_IRQ_AFFINITY,\n+\t\t\t\t\t     \u0026affd);\n+\tif (err \u003c 0) {\n+\t\tdev_err(common-\u003edev,\n+\t\t\t\"pci_alloc_irq_vectors failed, err = %d\\n\", err);\n+\t\treturn err;\n+\t}\n+\tif (err != needed) {\n+\t\tdev_err(common-\u003edev, \"pci_alloc_irq_vectors got %d vecs, need %d\\n\",\n+\t\t\terr, needed);\n+\t\treturn -ENOSPC;\n+\t}\n+\treturn 0;\n+}\n+\n+/*\n+ * Release the kernel-side MSI-X vectors here, before the hardware\n+ * pcompleter map is destroyed.  pci_free_irq_vectors() ends in\n+ * pci_msix_shutdown(), which masks every descriptor by writing the\n+ * vector control word in the MSI-X table and then reads mask_base\n+ * back to flush that write; those MMIO accesses should reach the\n+ * function while FUNCTION_MSIX_MAP is still valid.  This mirrors\n+ * nbl_dev_start(), where cfg_msix_map runs before\n+ * pci_alloc_irq_vectors_affinity(): the opposite order leaves the\n+ * core poking the MSI-X table after destroy_msix_map() has cleared\n+ * the valid bit, invalidated the info entries and freed the map\n+ * table, and how the device reacts to that is hardware-specific.\n+ *\n+ * This is not a double free of the devres action.  probe uses\n+ * pcim_enable_device(), so pci_alloc_irq_vectors() also registers\n+ * pcim_msi_release, but that only calls pci_free_irq_vectors()\n+ * again, and pci_disable_msix() returns early once msix_enabled is\n+ * clear.\n+ *\n+ * The handler is expected to be gone by now: both callers run\n+ * nbl_dev_free_mailbox_irq() first.\n+ */\n+static void nbl_dev_clear_interrupt_scheme(struct nbl_dev_mgt *dev_mgt)\n+{\n+\tstruct nbl_common_info *common = dev_mgt-\u003ecommon;\n+\n+\tpci_free_irq_vectors(common-\u003epdev);\n+}\n+\n+/* ----------  Channel config  ---------- */\n+static void nbl_dev_setup_chan_qinfo(struct nbl_dev_mgt *dev_mgt, u8 chan_type)\n+{\n+\tstruct nbl_channel_ops *chan_ops = dev_mgt-\u003echan_ops_tbl-\u003eops;\n+\tstruct nbl_channel_mgt *priv = dev_mgt-\u003echan_ops_tbl-\u003epriv;\n+\tstruct nbl_common_info *common = dev_mgt-\u003ecommon;\n+\n+\tif (!chan_ops-\u003echeck_queue_exist(priv, chan_type))\n+\t\treturn;\n+\n+\t/*\n+\t * common-\u003ehw_bus is the control PF's real bus number, captured in\n+\t * nbl_res_ctrl_dev_sriov_info_init() during nbl_res_init_leonis().\n+\t * nbl_core_init() runs resource init before nbl_dev_init(), so the\n+\t * value is always initialized when this control-PF-only path runs;\n+\t * nbl_res_intr_cfg_msix_map() consumes it for cfg_msix_map() the\n+\t * same way.\n+\t */\n+\tchan_ops-\u003ecfg_chan_qinfo_map_table(priv, common-\u003ehw_bus, common-\u003edevid);\n+}\n+\n+static int nbl_dev_setup_chan_queue(struct nbl_dev_mgt *dev_mgt, u8 chan_type)\n+{\n+\tstruct nbl_channel_ops *chan_ops = dev_mgt-\u003echan_ops_tbl-\u003eops;\n+\tstruct nbl_channel_mgt *priv = dev_mgt-\u003echan_ops_tbl-\u003epriv;\n+\tint ret = 0;\n+\n+\tif (chan_ops-\u003echeck_queue_exist(priv, chan_type))\n+\t\tret = chan_ops-\u003esetup_queue(priv, chan_type);\n+\n+\treturn ret;\n+}\n+\n+static int nbl_dev_remove_chan_queue(struct nbl_dev_mgt *dev_mgt, u8 chan_type)\n+{\n+\tstruct nbl_channel_ops *chan_ops = dev_mgt-\u003echan_ops_tbl-\u003eops;\n+\tstruct nbl_channel_mgt *priv = dev_mgt-\u003echan_ops_tbl-\u003epriv;\n+\tint ret = 0;\n+\n+\tif (chan_ops-\u003echeck_queue_exist(priv, chan_type))\n+\t\tret = chan_ops-\u003eteardown_queue(priv, chan_type);\n+\n+\treturn ret;\n+}\n+\n+static void nbl_dev_register_chan_task(struct nbl_dev_mgt *dev_mgt,\n+\t\t\t\t       u8 chan_type, struct work_struct *task)\n+{\n+\tstruct nbl_channel_ops *chan_ops = dev_mgt-\u003echan_ops_tbl-\u003eops;\n+\n+\tif (chan_ops-\u003echeck_queue_exist(dev_mgt-\u003echan_ops_tbl-\u003epriv, chan_type))\n+\t\tchan_ops-\u003eregister_chan_task(dev_mgt-\u003echan_ops_tbl-\u003epriv,\n+\t\t\t\t\t     chan_type, task);\n+}\n+\n+/* ----------  Tasks config  ---------- */\n+static void nbl_dev_clean_mailbox_task(struct work_struct *work)\n+{\n+\tstruct nbl_dev_common *common_dev =\n+\t\tcontainer_of(work, struct nbl_dev_common, clean_mbx_task);\n+\tstruct nbl_dev_mgt *dev_mgt = common_dev-\u003edev_mgt;\n+\tstruct nbl_channel_ops *chan_ops = dev_mgt-\u003echan_ops_tbl-\u003eops;\n+\n+\tchan_ops-\u003eclean_queue_subtask(dev_mgt-\u003echan_ops_tbl-\u003epriv,\n+\t\t\t\t      NBL_CHAN_TYPE_MAILBOX);\n+}\n+\n+static void nbl_dev_clean_mailbox_schedule(struct nbl_dev_mgt *dev_mgt)\n+{\n+\tstruct nbl_dev_common *common_dev = dev_mgt-\u003ecommon_dev;\n+\tstruct nbl_common_info *common = dev_mgt-\u003ecommon;\n+\n+\tqueue_work(common-\u003ewq, \u0026common_dev-\u003eclean_mbx_task);\n+}\n+\n+/* ----------  Dev init process  ---------- */\n+static int nbl_dev_setup_common_dev(struct nbl_adapter *adapter)\n+{\n+\tstruct nbl_dev_mgt *dev_mgt = adapter-\u003ecore.dev_mgt;\n+\tstruct nbl_dev_common *common_dev;\n+\tint ret;\n+\n+\tcommon_dev = devm_kzalloc(\u0026adapter-\u003epdev-\u003edev, sizeof(*common_dev),\n+\t\t\t\t  GFP_KERNEL);\n+\tif (!common_dev)\n+\t\treturn -ENOMEM;\n+\tcommon_dev-\u003edev_mgt = dev_mgt;\n+\n+\t/*\n+\t * INIT_WORK must precede nbl_dev_register_chan_task() below:\n+\t * registration publishes the work item to the channel, and the\n+\t * channel may queue it from its send and receive paths as soon\n+\t * as setup_queue() completes.\n+\t */\n+\tINIT_WORK(\u0026common_dev-\u003eclean_mbx_task, nbl_dev_clean_mailbox_task);\n+\n+\tret = nbl_dev_setup_chan_queue(dev_mgt, NBL_CHAN_TYPE_MAILBOX);\n+\tif (ret) {\n+\t\t/*\n+\t\t * Nothing to unwind: chan_info-\u003eactive is only set on a\n+\t\t * successful setup_queue(), so a failed one left no\n+\t\t * sender, no registered work item and no mailbox\n+\t\t * traffic.  Calling teardown_queue() here would only\n+\t\t * take its \"channel not active\" early exit and emit a\n+\t\t * misleading duplicate-teardown warning.  The partially\n+\t\t * allocated DMA rings are intentionally kept until devres\n+\t\t * releases them at detach (chan_info-\u003edma_allocated\n+\t\t * blocks a re-setup).\n+\t\t */\n+\t\treturn ret;\n+\t}\n+\n+\tnbl_dev_register_chan_task(dev_mgt, NBL_CHAN_TYPE_MAILBOX,\n+\t\t\t\t   \u0026common_dev-\u003eclean_mbx_task);\n+\t/*\n+\t * VSI/ETH identity is deliberately not fetched here: it is only\n+\t * known to the control PF, so it has to come from a mailbox RPC,\n+\t * and the control PF's responder may still be probing while this\n+\t * function runs.  The fetch is therefore left to the start path.\n+\t */\n+\tdev_mgt-\u003ecommon_dev = common_dev;\n+\tnbl_dev_init_msix_cnt(dev_mgt);\n+\n+\treturn 0;\n+}\n+\n+static void nbl_dev_remove_common_dev(struct nbl_adapter *adapter)\n+{\n+\tstruct nbl_dev_mgt *dev_mgt = adapter-\u003ecore.dev_mgt;\n+\tstruct nbl_dev_common *common_dev = dev_mgt-\u003ecommon_dev;\n+\n+\tif (!common_dev)\n+\t\treturn;\n+\tcancel_work_sync(\u0026common_dev-\u003eclean_mbx_task);\n+\t/*\n+\t * nbl_chan_teardown_queue() never fails: it returns 0 both on the\n+\t * early exit for a channel that never became active and on the\n+\t * normal path, which drains every inflight sender before\n+\t * returning.  There is therefore no partial-failure state that\n+\t * could leave mailbox DMA in flight and nothing to report here.\n+\t *\n+\t * Unregistering the clean task is the channel layer's job too:\n+\t * teardown_queue() clears clean_task under state_lock and then\n+\t * cancel_work_sync()es it, after shutdn has blocked new queues.\n+\t * No explicit unregistration is needed here precisely because\n+\t * teardown_queue() does that clearing itself.  Passing NULL from\n+\t * here would not be a no-op either: shutdn is only set inside\n+\t * teardown_queue(), so nbl_chan_register_chan_task() would still\n+\t * accept the write - it would just be clearing the same pointer a\n+\t * second time.\n+\t */\n+\tnbl_dev_remove_chan_queue(dev_mgt, NBL_CHAN_TYPE_MAILBOX);\n+}\n+\n+static int nbl_dev_setup_ctrl_dev(struct nbl_adapter *adapter)\n+{\n+\tstruct nbl_dev_mgt *dev_mgt = adapter-\u003ecore.dev_mgt;\n+\tstruct nbl_dispatch_ops *disp_ops = dev_mgt-\u003edisp_ops_tbl-\u003eops;\n+\n+\treturn disp_ops-\u003einit_module(dev_mgt-\u003edisp_ops_tbl-\u003epriv);\n+}\n+\n+/*\n+ * Tear down control device: deinit_module sets driver_status=false\n+ * to notify firmware to clean all per-PF hardware state (including\n+ * qinfo registers).  The qinfo map programmed in nbl_dev_init() via\n+ * nbl_dev_setup_chan_qinfo() is not explicitly cleared; firmware\n+ * handles it on driver_status change.\n+ *\n+ * Teardown ordering guarantee: every non-management PF creates a\n+ * consumer-\u003econtrol PF device link in its probe path, so the driver\n+ * core always unbinds all siblings before allowing the control PF to\n+ * be detached (sysfs unbind, driver unregister and hot-unplug alike).\n+ * nbl_probe_chip_deps() defers a sibling's probe until the control PF\n+ * is fully bound (DL_DEV_DRIVER_BOUND), and holds the management PF's\n+ * device lock across both that check and device_link_add(), so a\n+ * supplier cannot slip to DL_DEV_NO_DRIVER or DL_DEV_PROBING in\n+ * between.  A sibling therefore cannot\n+ * reach its mailbox setup before the control PF is fully probed, nor\n+ * race this deinit afterwards - the link forces it to be unbound\n+ * first.  The link is a persistent managed link with\n+ * DL_FLAG_AUTOPROBE_CONSUMER, so binding the control PF again makes\n+ * the core re-probe the siblings; without it they would stay unbound\n+ * until an admin rebound each one by hand.\n+ *\n+ * Direct control-PF FLR (which never runs driver teardown) cannot be\n+ * guarded here.\n+ */\n+static void nbl_dev_remove_ctrl_dev(struct nbl_adapter *adapter)\n+{\n+\tstruct nbl_dev_mgt *dev_mgt = adapter-\u003ecore.dev_mgt;\n+\tstruct nbl_dispatch_ops *disp_ops = dev_mgt-\u003edisp_ops_tbl-\u003eops;\n+\n+\tdisp_ops-\u003edeinit_module(dev_mgt-\u003edisp_ops_tbl-\u003epriv);\n+}\n+\n+static struct nbl_dev_mgt *nbl_dev_setup_dev_mgt(struct nbl_common_info *common)\n+{\n+\tstruct nbl_dev_mgt *dev_mgt;\n+\n+\tdev_mgt = devm_kzalloc(common-\u003edev, sizeof(*dev_mgt), GFP_KERNEL);\n+\tif (!dev_mgt)\n+\t\treturn ERR_PTR(-ENOMEM);\n+\n+\tdev_mgt-\u003ecommon = common;\n+\treturn dev_mgt;\n+}\n+\n+int nbl_dev_init(struct nbl_adapter *adapter)\n+{\n+\tstruct nbl_common_info *common = \u0026adapter-\u003ecommon;\n+\tstruct nbl_dispatch_ops_tbl *disp_ops_tbl =\n+\t\tadapter-\u003eintf.dispatch_ops_tbl;\n+\tstruct nbl_channel_ops_tbl *chan_ops_tbl =\n+\t\tadapter-\u003eintf.channel_ops_tbl;\n+\tstruct nbl_dev_mgt *dev_mgt;\n+\tint ret;\n+\n+\tdev_mgt = nbl_dev_setup_dev_mgt(common);\n+\tif (IS_ERR(dev_mgt)) {\n+\t\tret = PTR_ERR(dev_mgt);\n+\t\treturn ret;\n+\t}\n+\n+\tdev_mgt-\u003edisp_ops_tbl = disp_ops_tbl;\n+\tdev_mgt-\u003echan_ops_tbl = chan_ops_tbl;\n+\tadapter-\u003ecore.dev_mgt = dev_mgt;\n+\tif (common-\u003ehas_ctrl)\n+\t\tnbl_dev_setup_chan_qinfo(dev_mgt, NBL_CHAN_TYPE_MAILBOX);\n+\t/*\n+\t * Chip hardware initialization is completed by firmware at power-up.\n+\t * Only driver functional table/register config follows here, safe to\n+\t * access hardware registers before ctrl dev setup.\n+\t *\n+\t * The QINFO routing has to be programmed before the mailbox queue is\n+\t * enabled: nbl_dev_setup_common_dev() sets QUEUE_EN, and a queue that\n+\t * is live before its routing is in place would drop or misroute\n+\t * traffic.\n+\t */\n+\tret = nbl_dev_setup_common_dev(adapter);\n+\tif (ret)\n+\t\treturn ret;\n+\n+\tif (common-\u003ehas_ctrl) {\n+\t\tret = nbl_dev_setup_ctrl_dev(adapter);\n+\t\tif (ret)\n+\t\t\tgoto out_remove_common_dev;\n+\t}\n+\n+\treturn 0;\n+\n+/*\n+ * A failure after the QINFO routing was programmed leaves those entries\n+ * in hardware: driver_status never goes 1 -\u003e 0 on these paths, so\n+ * firmware does not reclaim them.  This is harmless - firmware only\n+ * routes once driver_status announces the driver, and no sibling PF can\n+ * bind until this probe succeeds.  The next successful probe overwrites\n+ * the entries, and a chip reset clears them.\n+ */\n+out_remove_common_dev:\n+\tnbl_dev_remove_common_dev(adapter);\n+\treturn ret;\n+}\n+\n+/*\n+ * Teardown order: Stop mailbox channel and drain all inflight DMA first,\n+ * then invoke deinit_module to notify firmware.\n+ *\n+ * This intentionally breaks strict init/teardown mirror symmetry due to\n+ * a hardware constraint: firmware performs asynchronous *per-PF* cleanup\n+ * (mailbox QINFO routing, qinfo registers and the like) once\n+ * driver_status=false is set.  It does not revert the chip-wide\n+ * datapath registers configured by init_module - those stay valid until\n+ * chip reset (see nbl_hw_deinit_module()).  We must guarantee no ongoing\n+ * mailbox DMA before deinit_module so firmware cannot reclaim the\n+ * routing under a live transfer and leave the device DMAing into\n+ * released memory.\n+ *\n+ * Init order: program mailbox QINFO routing (ctrl PF only) -\u003e\n+ *            create mailbox queue/common_dev -\u003e ctrl dev init_module\n+ * Teardown order: destroy mailbox queue/common_dev -\u003e ctrl dev deinit_module\n+ */\n+void nbl_dev_remove(struct nbl_adapter *adapter)\n+{\n+\tstruct nbl_common_info *common = \u0026adapter-\u003ecommon;\n+\n+\tnbl_dev_remove_common_dev(adapter);\n+\tif (common-\u003ehas_ctrl)\n+\t\tnbl_dev_remove_ctrl_dev(adapter);\n+}\n+\n+/* ----------  Dev start process  ---------- */\n+int nbl_dev_start(struct nbl_adapter *adapter)\n+{\n+\tstruct nbl_dev_mgt *dev_mgt = adapter-\u003ecore.dev_mgt;\n+\tstruct nbl_dispatch_ops *disp_ops = dev_mgt-\u003edisp_ops_tbl-\u003eops;\n+\tstruct nbl_dispatch_mgt *priv = dev_mgt-\u003edisp_ops_tbl-\u003epriv;\n+\tstruct nbl_dev_common *common_dev = dev_mgt-\u003ecommon_dev;\n+\tstruct nbl_common_info *common = dev_mgt-\u003ecommon;\n+\tint cleanup_ret;\n+\tint ret;\n+\n+\t/*\n+\t * Startup RPC failures are reported as they are, including the\n+\t * -ETIMEDOUT of a 5-6s polling wait against an unresponsive peer.\n+\t * nbl_probe() already deferred this PF in nbl_probe_chip_deps()\n+\t * until the control PF reached DL_DEV_DRIVER_BOUND, and the core\n+\t * only marks a device bound once its whole probe - including the\n+\t * control PF's own nbl_core_start() - has returned.  That state,\n+\t * not nbl_dev_init(), is what makes a timeout here a real failure\n+\t * instead of a \"supplier not ready yet\" condition: on the control\n+\t * PF inbound mailbox work only runs from clean_mbx_task, queued\n+\t * by the mailbox hardirq, by the task's own self-requeue, or by\n+\t * this function's polling sends - and the control PF uses the\n+\t * local dispatch ops, so it sends none.  The responder is armed\n+\t * by set_mailbox_irq(true) at the very end of the peer's\n+\t * nbl_dev_start(); relaxing the dependency to \"func 0 finished\n+\t * nbl_dev_init()\" would let every sibling RPC sit out the full\n+\t * 5-6s polling wait.  Turning -ETIMEDOUT into -EPROBE_DEFER would\n+\t * hide it behind dev_dbg() and leave the retry to the next\n+\t * unrelated device bind, paying that wait again on every retry.\n+\t */\n+\tret = nbl_dev_cfg_msix_map(dev_mgt);\n+\tif (ret)\n+\t\tgoto err_destroy_map;\n+\n+\t/* Fetch VSI/ETH identity after cfg_msix_map */\n+\tret = disp_ops-\u003eget_vsi_id(priv, NBL_VSI_DATA, \u0026common-\u003evsi_id);\n+\tif (ret)\n+\t\tgoto err_destroy_map;\n+\tret = disp_ops-\u003eget_eth_id(priv, common-\u003evsi_id, \u0026common-\u003eeth_num,\n+\t\t\t\t   \u0026common-\u003eeth_id, \u0026common-\u003elogic_eth_id);\n+\tif (ret)\n+\t\tgoto err_destroy_map;\n+\n+\tret = nbl_dev_init_interrupt_scheme(dev_mgt);\n+\tif (ret)\n+\t\tgoto err_destroy_map;\n+\n+\tret = nbl_dev_request_mailbox_irq(dev_mgt);\n+\tif (ret)\n+\t\tgoto err_destroy_map;\n+\n+\tret = nbl_dev_enable_mailbox_irq(dev_mgt);\n+\tif (ret)\n+\t\tgoto err_free_irq;\n+\n+\treturn 0;\n+\n+err_free_irq:\n+\t/*\n+\t * nbl_dev_enable_mailbox_irq() only fails before it sets\n+\t * NBL_CHAN_IRQ_RDY - the flag is set after its RPC returns - so\n+\t * there is no interrupt-mode sender to release and no software\n+\t * state to revoke.  The peer-side mailbox routing, if the failed\n+\t * RPC did land, is switched off by the destroy_msix_map() below: its\n+\t * prepare phase, __nbl_res_intr_prepare_destroy_msix_map(), disables\n+\t * mailbox IRQ routing for any CONFIGURED function before the map is\n+\t * torn down.  That is what makes skipping set_mailbox_irq(false)\n+\t * correct here, whatever way the enable failed - a local\n+\t * -ESHUTDOWN/-ENODEV/-EINVAL on the control PF as much as a\n+\t * polling-mode -ETIMEDOUT against an unresponsive peer.\n+\t */\n+\tnbl_dev_free_mailbox_irq(dev_mgt);\n+err_destroy_map:\n+\t/*\n+\t * Always ask the peer to drop this function's MSI-X map, even when\n+\t * the cfg RPC above failed: the teardown prepare phase returns 0\n+\t * for a function that is not NBL_INTR_FUNC_CONFIGURED, so it is a\n+\t * no-op when nothing landed, but a -ETIMEDOUT only means the ACK\n+\t * never arrived - the responder runs to completion before it ACKs.\n+\t * In that case the map entry, the intr_net/other_bmap bits, the\n+\t * kcalloc()ed vector array and the coherent map table are all\n+\t * still owned by the control PF, i.e. host resources of its\n+\t * driver, released only when this function is configured again or\n+\t * when nbl_intr_mgt_stop() tears the control PF down.  They are not\n+\t * firmware state, so skipping the destroy would strand them for a\n+\t * PF that has no driver to ever ask for them again.\n+\t *\n+\t * Kernel vectors are released first, by the\n+\t * nbl_dev_clear_interrupt_scheme() below: the PCI core's MSI-X\n+\t * shutdown writes the MSI-X table and reads it back, and those\n+\t * accesses should land while this function's pcompleter map entry\n+\t * is still valid.  The destroy then masks all hardware vectors\n+\t * and clears that entry.\n+\t *\n+\t * For non-control PFs this is a polling-mode mailbox RPC:\n+\t * NBL_CHAN_IRQ_RDY is never set on any rollback path, since\n+\t * enabling it is the last step of nbl_dev_start().\n+\t *\n+\t * Note: pci_clear_master() runs in nbl_probe() core_start_err path;\n+\t * on a non-control PF it cannot stop DMA using the management PF's\n+\t * BDF, this is a known limitation.\n+\t */\n+\tnbl_dev_clear_interrupt_scheme(dev_mgt);\n+\tcleanup_ret = nbl_dev_destroy_msix_map(dev_mgt);\n+\tif (cleanup_ret)\n+\t\tdev_err(dev_mgt-\u003ecommon-\u003edev,\n+\t\t\t\"rollback: destroy MSI-X map failed: %d\\n\",\n+\t\t\tcleanup_ret);\n+\n+\tcancel_work_sync(\u0026common_dev-\u003eclean_mbx_task);\n+\treturn ret;\n+}\n+\n+void nbl_dev_stop(struct nbl_adapter *adapter)\n+{\n+\tstruct nbl_dev_mgt *dev_mgt = adapter-\u003ecore.dev_mgt;\n+\tstruct nbl_dev_common *common_dev = dev_mgt-\u003ecommon_dev;\n+\tint ret;\n+\n+\tret = nbl_dev_disable_mailbox_irq(dev_mgt);\n+\tif (ret)\n+\t\tdev_err(dev_mgt-\u003ecommon-\u003edev,\n+\t\t\t\"Failed to disable mailbox IRQ: %d\\n\", ret);\n+\tnbl_dev_free_mailbox_irq(dev_mgt);\n+\n+\t/*\n+\t * Release the kernel vectors before destroying the hardware map:\n+\t * the PCI core's MSI-X shutdown writes the MSI-X table and reads\n+\t * it back, and those accesses should land while this function's\n+\t * pcompleter map entry is still valid (see\n+\t * nbl_dev_clear_interrupt_scheme).\n+\t */\n+\tnbl_dev_clear_interrupt_scheme(dev_mgt);\n+\n+\t/*\n+\t * Destroy the hardware MSI-X map: masks all device vectors and\n+\t * clears the pcompleter map entry.\n+\t *\n+\t * Best-effort: if the destroy RPC fails, the control PF still owns\n+\t * this function's map entry, its intr_net/other_bmap bits and its\n+\t * coherent map table.  Those are host resources of that driver,\n+\t * released either by a later successful configuration of this\n+\t * function - nbl_res_intr_cfg_msix_map() invalidates and rewrites\n+\t * the entry on its had_config path - or by nbl_intr_mgt_stop()\n+\t * when the control PF is removed, which walks every\n+\t * NBL_INTR_FUNC_CONFIGURED function through\n+\t * __nbl_res_intr_prepare_destroy_msix_map() and clears the\n+\t * FUNCTION_MSIX_MAP valid bit.  Both rewrite the hardware entry\n+\t * without a chip reset; nothing else collects it, so a stranded\n+\t * one just stays as it is, and nothing requests one.\n+\t *\n+\t * pci_clear_master() on a non-control PF cannot stop DMA using the\n+\t * management PF's BDF.\n+\t */\n+\tret = nbl_dev_destroy_msix_map(dev_mgt);\n+\tif (ret)\n+\t\tdev_err(dev_mgt-\u003ecommon-\u003edev,\n+\t\t\t\"Failed to destroy MSI-X map: %d\\n\", ret);\n+\n+\t/*\n+\t * destroy_msix_map() sends ack-requested messages which may\n+\t * requeue clean_mbx_task via polling send path.  Drain work\n+\t * after the operation.\n+\t */\n+\tcancel_work_sync(\u0026common_dev-\u003eclean_mbx_task);\n+}\ndiff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_core/nbl_dev.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_core/nbl_dev.h\nnew file mode 100644\nindex 00000000000000..24e890fd898745\n--- /dev/null\n+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_core/nbl_dev.h\n@@ -0,0 +1,55 @@\n+/* SPDX-License-Identifier: GPL-2.0 */\n+/*\n+ * Copyright (c) 2026 Nebula Matrix Limited.\n+ */\n+\n+#ifndef _NBL_DEV_H_\n+#define _NBL_DEV_H_\n+\n+#include \u003clinux/types.h\u003e\n+\n+#include \"../nbl_include/nbl_include.h\"\n+#include \"../nbl_include/nbl_def_channel.h\"\n+#include \"../nbl_include/nbl_def_hw.h\"\n+#include \"../nbl_include/nbl_def_resource.h\"\n+#include \"../nbl_include/nbl_def_dispatch.h\"\n+#include \"../nbl_include/nbl_def_dev.h\"\n+#include \"../nbl_include/nbl_def_common.h\"\n+#include \"../nbl_core.h\"\n+\n+#define NBL_STRING_NAME_LEN\t\t\t32\n+\n+enum nbl_msix_serv_type {\n+\tNBL_MSIX_NET_TYPE,\n+\tNBL_MSIX_MAILBOX_TYPE,\n+\tNBL_MSIX_TYPE_MAX\n+};\n+\n+struct nbl_msix_serv_info {\n+\tchar irq_name[NBL_STRING_NAME_LEN];\n+\tu16 num;\n+\tu16 base_vector_id;\n+\t/* true: hw report msix, hw need to mask actively */\n+\tbool hw_self_mask_en;\n+};\n+\n+struct nbl_msix_info {\n+\tstruct nbl_msix_serv_info serv_info[NBL_MSIX_TYPE_MAX];\n+};\n+\n+struct nbl_dev_common {\n+\tstruct nbl_dev_mgt *dev_mgt;\n+\tstruct nbl_msix_info msix_info;\n+\tchar mailbox_name[NBL_STRING_NAME_LEN];\n+\t/* for ctrl-dev/net-dev mailbox recv msg */\n+\tstruct work_struct clean_mbx_task;\n+};\n+\n+struct nbl_dev_mgt {\n+\tstruct nbl_common_info *common;\n+\tstruct nbl_dispatch_ops_tbl *disp_ops_tbl;\n+\tstruct nbl_channel_ops_tbl *chan_ops_tbl;\n+\tstruct nbl_dev_common *common_dev;\n+};\n+\n+#endif\ndiff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_core/nbl_dispatch.c b/drivers/net/ethernet/nebula-matrix/nbl/nbl_core/nbl_dispatch.c\nnew file mode 100644\nindex 00000000000000..59d80066f88b76\n--- /dev/null\n+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_core/nbl_dispatch.c\n@@ -0,0 +1,671 @@\n+// SPDX-License-Identifier: GPL-2.0\n+/*\n+ * Copyright (c) 2026 Nebula Matrix Limited.\n+ */\n+#include \u003clinux/device.h\u003e\n+#include \u003clinux/pci.h\u003e\n+#include \"nbl_dispatch.h\"\n+\n+static int nbl_disp_chan_get_vsi_id_req(struct nbl_dispatch_mgt *disp_mgt,\n+\t\t\t\t\tu16 type, u16 *vsi_id)\n+{\n+\tstruct nbl_channel_ops *chan_ops = disp_mgt-\u003echan_ops_tbl-\u003eops;\n+\tstruct nbl_common_info *common = disp_mgt-\u003ecommon;\n+\tstruct nbl_chan_param_get_vsi_id result = { 0 };\n+\tstruct nbl_chan_param_get_vsi_id param = { 0 };\n+\tstruct nbl_chan_send_info chan_send = {0};\n+\tint ret;\n+\n+\tparam.type = cpu_to_le16(type);\n+\n+\tnbl_chan_fill_send_info(\u0026chan_send, common-\u003emgt_pf,\n+\t\t\t\tNBL_CHAN_MSG_GET_VSI_ID,\n+\t\t\t\t\u0026param, sizeof(param), \u0026result,\n+\t\t\t\tsizeof(result), 1);\n+\tret = chan_ops-\u003esend_msg(disp_mgt-\u003echan_ops_tbl-\u003epriv, \u0026chan_send);\n+\tif (ret)\n+\t\treturn ret;\n+\tif (chan_send.ack_len != sizeof(result)) {\n+\t\tdev_err(disp_mgt-\u003ecommon-\u003edev,\n+\t\t\t\"get_vsi_id: short ACK, ack_len=%u expected %zu\\n\",\n+\t\t\tchan_send.ack_len, sizeof(result));\n+\t\treturn -EBADMSG;\n+\t}\n+\t*vsi_id = le16_to_cpu(result.vsi_id);\n+\treturn 0;\n+}\n+\n+static void nbl_disp_chan_get_vsi_id_resp(void *priv, u16 src_id, u16 msg_id,\n+\t\t\t\t\t  void *data, u32 data_len)\n+{\n+\tstruct nbl_dispatch_mgt *disp_mgt = (struct nbl_dispatch_mgt *)priv;\n+\tstruct nbl_channel_ops *chan_ops = disp_mgt-\u003echan_ops_tbl-\u003eops;\n+\tstruct nbl_resource_ops *res_ops = disp_mgt-\u003eres_ops_tbl-\u003eops;\n+\tstruct nbl_resource_mgt *p = disp_mgt-\u003eres_ops_tbl-\u003epriv;\n+\tstruct device *dev = disp_mgt-\u003ecommon-\u003edev;\n+\tstruct nbl_chan_param_get_vsi_id result = { 0 };\n+\tstruct nbl_chan_param_get_vsi_id param = { 0 };\n+\tstruct nbl_chan_ack_info chan_ack;\n+\tint err = 0;\n+\tu16 vsi_id = 0;\n+\tu32 rel_pf_id;\n+\tint ret;\n+\n+\tret = nbl_common_func_id_to_rel_pf_id(disp_mgt-\u003ecommon, src_id,\n+\t\t\t\t\t      \u0026rel_pf_id);\n+\tif (ret) {\n+\t\terr = -EPERM;\n+\t\tgoto ack_out;\n+\t}\n+\tif (rel_pf_id \u003e= disp_mgt-\u003ecommon-\u003emax_pf) {\n+\t\terr = -EPERM;\n+\t\tgoto ack_out;\n+\t}\n+\tif (data_len \u003c sizeof(param)) {\n+\t\terr = -EBADMSG;\n+\t\tgoto ack_out;\n+\t}\n+\tmemcpy(\u0026param, data, sizeof(param));\n+\n+\tif (res_ops-\u003eget_vsi_id) {\n+\t\tret = res_ops-\u003eget_vsi_id(p, src_id, le16_to_cpu(param.type),\n+\t\t\t\t\t  \u0026vsi_id);\n+\t\t/* Forward the resource errno verbatim to the requester */\n+\t\tif (ret)\n+\t\t\terr = ret;\n+\t} else {\n+\t\terr = -EOPNOTSUPP;\n+\t}\n+\n+\tresult.vsi_id = cpu_to_le16(vsi_id);\n+ack_out:\n+\tnbl_chan_fill_ack_info(\u0026chan_ack, src_id,\n+\t\t\t       NBL_CHAN_MSG_GET_VSI_ID, msg_id, err,\n+\t\t\t       \u0026result, sizeof(result));\n+\tret = chan_ops-\u003esend_ack(disp_mgt-\u003echan_ops_tbl-\u003epriv, \u0026chan_ack);\n+\tif (ret)\n+\t\tdev_err(dev,\n+\t\t\t\"channel send ack failed with ret: %d, msg_type: %d\\n\",\n+\t\t\tret, NBL_CHAN_MSG_GET_VSI_ID);\n+}\n+\n+static int nbl_disp_chan_get_eth_id_req(struct nbl_dispatch_mgt *disp_mgt,\n+\t\t\t\t\tu16 vsi_id, u8 *eth_num, u8 *eth_id,\n+\t\t\t\t\tu8 *logic_eth_id)\n+{\n+\tstruct nbl_channel_ops *chan_ops = disp_mgt-\u003echan_ops_tbl-\u003eops;\n+\tstruct nbl_common_info *common = disp_mgt-\u003ecommon;\n+\tstruct nbl_chan_param_get_eth_id result = { 0 };\n+\tstruct nbl_chan_param_get_eth_id param = { 0 };\n+\tstruct nbl_chan_send_info chan_send = {0};\n+\tint ret;\n+\n+\tparam.vsi_id = cpu_to_le16(vsi_id);\n+\n+\tnbl_chan_fill_send_info(\u0026chan_send, common-\u003emgt_pf,\n+\t\t\t\tNBL_CHAN_MSG_GET_ETH_ID,\n+\t\t\t\t\u0026param, sizeof(param), \u0026result,\n+\t\t\t\tsizeof(result), 1);\n+\tret = chan_ops-\u003esend_msg(disp_mgt-\u003echan_ops_tbl-\u003epriv, \u0026chan_send);\n+\tif (ret)\n+\t\treturn ret;\n+\tif (chan_send.ack_len != sizeof(result)) {\n+\t\tdev_err(disp_mgt-\u003ecommon-\u003edev,\n+\t\t\t\"get_eth_id: short ACK, ack_len=%u expected %zu\\n\",\n+\t\t\tchan_send.ack_len, sizeof(result));\n+\t\treturn -EBADMSG;\n+\t}\n+\t*eth_num = result.eth_num;\n+\t*eth_id = result.eth_id;\n+\t*logic_eth_id = result.logic_eth_id;\n+\n+\treturn 0;\n+}\n+\n+static void nbl_disp_chan_get_eth_id_resp(void *priv, u16 src_id, u16 msg_id,\n+\t\t\t\t\t  void *data, u32 data_len)\n+{\n+\tstruct nbl_dispatch_mgt *disp_mgt = (struct nbl_dispatch_mgt *)priv;\n+\tstruct nbl_channel_ops *chan_ops = disp_mgt-\u003echan_ops_tbl-\u003eops;\n+\tstruct nbl_resource_ops *res_ops = disp_mgt-\u003eres_ops_tbl-\u003eops;\n+\tstruct nbl_resource_mgt *p = disp_mgt-\u003eres_ops_tbl-\u003epriv;\n+\tstruct nbl_chan_param_get_eth_id result = { 0 };\n+\tstruct nbl_chan_param_get_eth_id param = { 0 };\n+\tstruct device *dev = disp_mgt-\u003ecommon-\u003edev;\n+\tstruct nbl_chan_ack_info chan_ack;\n+\tint err = 0;\n+\tu32 rel_pf_id;\n+\tint ret;\n+\n+\tret = nbl_common_func_id_to_rel_pf_id(disp_mgt-\u003ecommon, src_id,\n+\t\t\t\t\t      \u0026rel_pf_id);\n+\tif (ret) {\n+\t\terr = -EPERM;\n+\t\tgoto ack_out;\n+\t}\n+\tif (rel_pf_id \u003e= disp_mgt-\u003ecommon-\u003emax_pf) {\n+\t\terr = -EPERM;\n+\t\tgoto ack_out;\n+\t}\n+\tif (data_len \u003c sizeof(param)) {\n+\t\terr = -EBADMSG;\n+\t\tgoto ack_out;\n+\t}\n+\tmemcpy(\u0026param, data, sizeof(param));\n+\n+\tif (res_ops-\u003eget_eth_id) {\n+\t\tret = res_ops-\u003eget_eth_id(p, src_id, le16_to_cpu(param.vsi_id),\n+\t\t\t\t\t  \u0026result.eth_num, \u0026result.eth_id,\n+\t\t\t\t\t  \u0026result.logic_eth_id);\n+\t\t/* Forward the resource errno verbatim to the requester */\n+\t\tif (ret)\n+\t\t\terr = ret;\n+\t} else {\n+\t\terr = -EOPNOTSUPP;\n+\t}\n+ack_out:\n+\tnbl_chan_fill_ack_info(\u0026chan_ack, src_id,\n+\t\t\t       NBL_CHAN_MSG_GET_ETH_ID, msg_id, err,\n+\t\t\t       \u0026result, sizeof(result));\n+\tret = chan_ops-\u003esend_ack(disp_mgt-\u003echan_ops_tbl-\u003epriv, \u0026chan_ack);\n+\tif (ret)\n+\t\tdev_err(dev,\n+\t\t\t\"channel send ack failed with ret: %d, msg_type: %d\\n\",\n+\t\t\tret, NBL_CHAN_MSG_GET_ETH_ID);\n+}\n+\n+static void nbl_disp_deinit_module(struct nbl_dispatch_mgt *disp_mgt)\n+{\n+\tstruct nbl_resource_ops *res_ops = disp_mgt-\u003eres_ops_tbl-\u003eops;\n+\tstruct nbl_resource_mgt *p = disp_mgt-\u003eres_ops_tbl-\u003epriv;\n+\n+\tif (res_ops-\u003edeinit_module)\n+\t\tres_ops-\u003edeinit_module(p);\n+}\n+\n+static int nbl_disp_init_module(struct nbl_dispatch_mgt *disp_mgt)\n+{\n+\tstruct nbl_resource_ops *res_ops = disp_mgt-\u003eres_ops_tbl-\u003eops;\n+\tstruct nbl_resource_mgt *p = disp_mgt-\u003eres_ops_tbl-\u003epriv;\n+\n+\tif (res_ops-\u003einit_module)\n+\t\treturn res_ops-\u003einit_module(p);\n+\treturn -EOPNOTSUPP;\n+}\n+\n+static int nbl_disp_cfg_msix_map(struct nbl_dispatch_mgt *disp_mgt,\n+\t\t\t\t u16 num_net_msix, u16 num_others_msix,\n+\t\t\t\t bool net_msix_mask_en)\n+{\n+\tstruct nbl_resource_ops *res_ops = disp_mgt-\u003eres_ops_tbl-\u003eops;\n+\tstruct nbl_resource_mgt *p = disp_mgt-\u003eres_ops_tbl-\u003epriv;\n+\tstruct nbl_common_info *common = disp_mgt-\u003ecommon;\n+\n+\tif (!res_ops-\u003ecfg_msix_map)\n+\t\treturn -EOPNOTSUPP;\n+\t/*\n+\t * No core-layer lock is taken here: the resource layer serializes\n+\t * concurrent callers itself (intr_mgt-\u003elock), and the dispatch\n+\t * layer has no mutable state of its own for this op to protect.\n+\t */\n+\treturn res_ops-\u003ecfg_msix_map(p, common-\u003emgt_pf, num_net_msix,\n+\t\t\t\t     num_others_msix, net_msix_mask_en);\n+}\n+\n+/*\n+ * Precondition: caller must disable mailbox IRQ_RDY and switch send_msg\n+ * to the polling path before issuing this RPC. The responder rewrites\n+ * the requester's mailbox MSI-X routing during the resource op, so the\n+ * ACK cannot rely on interrupt wakeup while routing is in flux.\n+ */\n+static int\n+nbl_disp_chan_cfg_msix_map_req(struct nbl_dispatch_mgt *disp_mgt,\n+\t\t\t       u16 num_net_msix, u16 num_others_msix,\n+\t\t\t       bool net_msix_mask_en)\n+{\n+\tstruct nbl_channel_ops *chan_ops = disp_mgt-\u003echan_ops_tbl-\u003eops;\n+\tstruct nbl_common_info *common = disp_mgt-\u003ecommon;\n+\tstruct nbl_chan_param_cfg_msix_map param = { 0 };\n+\tstruct nbl_chan_send_info chan_send = {0};\n+\tint ret;\n+\n+\tparam.num_net_msix = cpu_to_le16(num_net_msix);\n+\tparam.num_others_msix = cpu_to_le16(num_others_msix);\n+\tparam.msix_mask_en = cpu_to_le16(!!net_msix_mask_en);\n+\n+\tnbl_chan_fill_send_info(\u0026chan_send, common-\u003emgt_pf,\n+\t\t\t\tNBL_CHAN_MSG_CONFIGURE_MSIX_MAP,\n+\t\t\t\t\u0026param, sizeof(param),\n+\t\t\t\tNULL, 0, 1);\n+\tret = chan_ops-\u003esend_msg(disp_mgt-\u003echan_ops_tbl-\u003epriv, \u0026chan_send);\n+\tif (ret)\n+\t\treturn ret;\n+\treturn 0;\n+}\n+\n+static void nbl_disp_chan_cfg_msix_map_resp(void *priv, u16 src_id, u16 msg_id,\n+\t\t\t\t\t    void *data, u32 data_len)\n+{\n+\tstruct nbl_dispatch_mgt *disp_mgt = (struct nbl_dispatch_mgt *)priv;\n+\tstruct nbl_channel_ops *chan_ops = disp_mgt-\u003echan_ops_tbl-\u003eops;\n+\tstruct nbl_resource_ops *res_ops = disp_mgt-\u003eres_ops_tbl-\u003eops;\n+\tstruct nbl_resource_mgt *p = disp_mgt-\u003eres_ops_tbl-\u003epriv;\n+\tstruct device *dev = disp_mgt-\u003ecommon-\u003edev;\n+\tstruct nbl_chan_param_cfg_msix_map param = { 0 };\n+\tstruct nbl_chan_ack_info chan_ack;\n+\tint err = 0;\n+\tu32 rel_pf_id;\n+\tint ret;\n+\n+\tret = nbl_common_func_id_to_rel_pf_id(disp_mgt-\u003ecommon, src_id,\n+\t\t\t\t\t      \u0026rel_pf_id);\n+\tif (ret) {\n+\t\terr = -EPERM;\n+\t\tgoto ack_out;\n+\t}\n+\tif (rel_pf_id \u003e= disp_mgt-\u003ecommon-\u003emax_pf) {\n+\t\terr = -EPERM;\n+\t\tgoto ack_out;\n+\t}\n+\tif (data_len \u003c sizeof(param)) {\n+\t\terr = -EBADMSG;\n+\t\tgoto ack_out;\n+\t}\n+\tmemcpy(\u0026param, data, sizeof(param));\n+\n+\tif (res_ops-\u003ecfg_msix_map) {\n+\t\t/*\n+\t\t * The vector counts come from the peer PF.  A peer is\n+\t\t * trusted: hardware and firmware forbid PF passthrough to\n+\t\t * guests (see nbl_get_func_param()), so every peer runs this\n+\t\t * same host driver, and the resource layer bounds the counts\n+\t\t * by the global vector pool.  A peer asking for the whole\n+\t\t * pool can starve the others with -EAGAIN, but it cannot\n+\t\t * corrupt state.\n+\t\t */\n+\t\tret = res_ops-\u003ecfg_msix_map(p, src_id,\n+\t\t\t\t\t    le16_to_cpu(param.num_net_msix),\n+\t\t\t\t\t    le16_to_cpu(param.num_others_msix),\n+\t\t\t\t\t    !!le16_to_cpu(param.msix_mask_en));\n+\t\t/* Forward the resource errno verbatim to the requester */\n+\t\tif (ret)\n+\t\t\terr = ret;\n+\t} else {\n+\t\terr = -EOPNOTSUPP;\n+\t}\n+ack_out:\n+\tnbl_chan_fill_ack_info(\u0026chan_ack, src_id,\n+\t\t\t       NBL_CHAN_MSG_CONFIGURE_MSIX_MAP, msg_id,\n+\t\t\t       err, NULL, 0);\n+\tret = chan_ops-\u003esend_ack(disp_mgt-\u003echan_ops_tbl-\u003epriv, \u0026chan_ack);\n+\tif (ret)\n+\t\tdev_err(dev,\n+\t\t\t\"channel send ack failed with ret: %d, msg_type: %d\\n\",\n+\t\t\tret, NBL_CHAN_MSG_CONFIGURE_MSIX_MAP);\n+}\n+\n+/*\n+ * Precondition: caller must disable mailbox IRQ_RDY and switch send_msg\n+ * to the polling path before issuing this RPC. The responder retargets\n+ * the requester's mailbox MSI-X routing during the resource op, so the\n+ * ACK cannot rely on interrupt wakeup while routing is in flux.\n+ */\n+static int nbl_disp_chan_destroy_msix_map_req(struct nbl_dispatch_mgt *disp_mgt)\n+{\n+\tstruct nbl_channel_ops *chan_ops = disp_mgt-\u003echan_ops_tbl-\u003eops;\n+\tstruct nbl_common_info *common = disp_mgt-\u003ecommon;\n+\tstruct nbl_chan_send_info chan_send = {0};\n+\tint ret;\n+\n+\tnbl_chan_fill_send_info(\u0026chan_send, common-\u003emgt_pf,\n+\t\t\t\tNBL_CHAN_MSG_DESTROY_MSIX_MAP,\n+\t\t\t\tNULL, 0, NULL, 0, 1);\n+\tret = chan_ops-\u003esend_msg(disp_mgt-\u003echan_ops_tbl-\u003epriv, \u0026chan_send);\n+\tif (ret)\n+\t\treturn ret;\n+\treturn 0;\n+}\n+\n+static void nbl_disp_chan_destroy_msix_map_resp(void *priv, u16 src_id,\n+\t\t\t\t\t\tu16 msg_id, void *data,\n+\t\t\t\t\t\tu32 data_len)\n+{\n+\tstruct nbl_dispatch_mgt *disp_mgt = (struct nbl_dispatch_mgt *)priv;\n+\tstruct nbl_channel_ops *chan_ops = disp_mgt-\u003echan_ops_tbl-\u003eops;\n+\tstruct nbl_resource_ops *res_ops = disp_mgt-\u003eres_ops_tbl-\u003eops;\n+\tstruct nbl_resource_mgt *p = disp_mgt-\u003eres_ops_tbl-\u003epriv;\n+\tstruct device *dev = disp_mgt-\u003ecommon-\u003edev;\n+\tstruct nbl_chan_ack_info chan_ack;\n+\tint err = 0;\n+\tu32 rel_pf_id;\n+\tint ret;\n+\n+\tret = nbl_common_func_id_to_rel_pf_id(disp_mgt-\u003ecommon, src_id,\n+\t\t\t\t\t      \u0026rel_pf_id);\n+\tif (ret) {\n+\t\terr = -EPERM;\n+\t\tgoto ack_out;\n+\t}\n+\tif (rel_pf_id \u003e= disp_mgt-\u003ecommon-\u003emax_pf) {\n+\t\terr = -EPERM;\n+\t\tgoto ack_out;\n+\t}\n+\tif (res_ops-\u003edestroy_msix_map) {\n+\t\tret = res_ops-\u003edestroy_msix_map(p, src_id);\n+\t\t/* Forward the resource errno verbatim to the requester */\n+\t\tif (ret)\n+\t\t\terr = ret;\n+\t} else {\n+\t\terr = -EOPNOTSUPP;\n+\t}\n+ack_out:\n+\tnbl_chan_fill_ack_info(\u0026chan_ack, src_id,\n+\t\t\t       NBL_CHAN_MSG_DESTROY_MSIX_MAP, msg_id,\n+\t\t\t       err, NULL, 0);\n+\tret = chan_ops-\u003esend_ack(disp_mgt-\u003echan_ops_tbl-\u003epriv, \u0026chan_ack);\n+\tif (ret)\n+\t\tdev_err(dev,\n+\t\t\t\"channel send ack failed with ret: %d, msg_type: %d\\n\",\n+\t\t\tret, NBL_CHAN_MSG_DESTROY_MSIX_MAP);\n+}\n+\n+/*\n+ * Precondition: caller must disable mailbox IRQ_RDY and switch send_msg\n+ * to the polling path before issuing this RPC. The responder rewrites\n+ * the requester's own mailbox MSI-X routing (MSIX_IDX / MSIX_IDX_VALID)\n+ * before the ACK is sent, so the ACK cannot rely on interrupt wakeup\n+ * while routing is in flux.\n+ */\n+static int nbl_disp_chan_set_mailbox_irq_req(struct nbl_dispatch_mgt *disp_mgt,\n+\t\t\t\t\t     u16 vector_id, bool en_msix)\n+{\n+\tstruct nbl_channel_ops *chan_ops = disp_mgt-\u003echan_ops_tbl-\u003eops;\n+\tstruct nbl_chan_param_set_mailbox_irq param = { 0 };\n+\tstruct nbl_common_info *common = disp_mgt-\u003ecommon;\n+\tstruct nbl_chan_send_info chan_send = {0};\n+\tint ret;\n+\n+\tparam.vector_id = cpu_to_le16(vector_id);\n+\tparam.en_msix = !!en_msix;\n+\n+\tnbl_chan_fill_send_info(\u0026chan_send, common-\u003emgt_pf,\n+\t\t\t\tNBL_CHAN_MSG_MAILBOX_SET_IRQ,\n+\t\t\t\t\u0026param, sizeof(param), NULL, 0, 1);\n+\tret = chan_ops-\u003esend_msg(disp_mgt-\u003echan_ops_tbl-\u003epriv, \u0026chan_send);\n+\tif (ret)\n+\t\treturn ret;\n+\treturn 0;\n+}\n+\n+static void nbl_disp_chan_set_mailbox_irq_resp(void *priv, u16 src_id,\n+\t\t\t\t\t       u16 msg_id, void *data,\n+\t\t\t\t\t       u32 data_len)\n+{\n+\tstruct nbl_dispatch_mgt *disp_mgt = (struct nbl_dispatch_mgt *)priv;\n+\tstruct nbl_channel_ops *chan_ops = disp_mgt-\u003echan_ops_tbl-\u003eops;\n+\tstruct nbl_resource_ops *res_ops = disp_mgt-\u003eres_ops_tbl-\u003eops;\n+\tstruct nbl_resource_mgt *p = disp_mgt-\u003eres_ops_tbl-\u003epriv;\n+\tstruct nbl_chan_param_set_mailbox_irq param = { 0 };\n+\tstruct device *dev = disp_mgt-\u003ecommon-\u003edev;\n+\tstruct nbl_chan_ack_info chan_ack;\n+\tint err = 0;\n+\tu16 vector_id;\n+\tu32 rel_pf_id;\n+\tbool en_msix;\n+\tint ret;\n+\n+\tret = nbl_common_func_id_to_rel_pf_id(disp_mgt-\u003ecommon, src_id,\n+\t\t\t\t\t      \u0026rel_pf_id);\n+\tif (ret) {\n+\t\terr = -EPERM;\n+\t\tgoto ack_out;\n+\t}\n+\tif (rel_pf_id \u003e= disp_mgt-\u003ecommon-\u003emax_pf) {\n+\t\terr = -EPERM;\n+\t\tgoto ack_out;\n+\t}\n+\tif (data_len \u003c sizeof(param)) {\n+\t\terr = -EBADMSG;\n+\t\tgoto ack_out;\n+\t}\n+\tmemcpy(\u0026param, data, sizeof(param));\n+\tvector_id = le16_to_cpu(param.vector_id);\n+\ten_msix = !!param.en_msix;\n+\n+\tif (res_ops-\u003eset_mailbox_irq) {\n+\t\tret = res_ops-\u003eset_mailbox_irq(p, src_id, vector_id, en_msix);\n+\t\t/* Forward the resource errno verbatim to the requester */\n+\t\tif (ret)\n+\t\t\terr = ret;\n+\t} else {\n+\t\terr = -EOPNOTSUPP;\n+\t}\n+\n+ack_out:\n+\tnbl_chan_fill_ack_info(\u0026chan_ack, src_id,\n+\t\t\t       NBL_CHAN_MSG_MAILBOX_SET_IRQ, msg_id,\n+\t\t\t       err, NULL, 0);\n+\tret = chan_ops-\u003esend_ack(disp_mgt-\u003echan_ops_tbl-\u003epriv, \u0026chan_ack);\n+\tif (ret)\n+\t\tdev_err(dev,\n+\t\t\t\"channel send ack failed with ret: %d, msg_type: %d\\n\",\n+\t\t\tret, NBL_CHAN_MSG_MAILBOX_SET_IRQ);\n+}\n+\n+static int nbl_disp_destroy_msix_map(struct nbl_dispatch_mgt *disp_mgt)\n+{\n+\tstruct nbl_resource_ops *res_ops = disp_mgt-\u003eres_ops_tbl-\u003eops;\n+\tstruct nbl_resource_mgt *p = disp_mgt-\u003eres_ops_tbl-\u003epriv;\n+\tstruct nbl_common_info *common = disp_mgt-\u003ecommon;\n+\n+\tif (!res_ops-\u003edestroy_msix_map)\n+\t\treturn -EOPNOTSUPP;\n+\treturn res_ops-\u003edestroy_msix_map(p, common-\u003emgt_pf);\n+}\n+\n+static int nbl_disp_set_mailbox_irq(struct nbl_dispatch_mgt *disp_mgt,\n+\t\t\t\t    u16 vector_id, bool en_msix)\n+{\n+\tstruct nbl_resource_ops *res_ops = disp_mgt-\u003eres_ops_tbl-\u003eops;\n+\tstruct nbl_resource_mgt *p = disp_mgt-\u003eres_ops_tbl-\u003epriv;\n+\tstruct nbl_common_info *common = disp_mgt-\u003ecommon;\n+\n+\tif (!res_ops-\u003eset_mailbox_irq)\n+\t\treturn -EOPNOTSUPP;\n+\treturn res_ops-\u003eset_mailbox_irq(p, common-\u003emgt_pf, vector_id, en_msix);\n+}\n+\n+static int nbl_disp_get_vsi_id(struct nbl_dispatch_mgt *disp_mgt, u16 type,\n+\t\t\t       u16 *vsi_id)\n+{\n+\tstruct nbl_resource_ops *res_ops = disp_mgt-\u003eres_ops_tbl-\u003eops;\n+\tstruct nbl_resource_mgt *p = disp_mgt-\u003eres_ops_tbl-\u003epriv;\n+\tstruct nbl_common_info *common = disp_mgt-\u003ecommon;\n+\n+\tif (res_ops-\u003eget_vsi_id)\n+\t\treturn res_ops-\u003eget_vsi_id(p, common-\u003emgt_pf, type, vsi_id);\n+\treturn -EOPNOTSUPP;\n+}\n+\n+static int nbl_disp_get_eth_id(struct nbl_dispatch_mgt *disp_mgt, u16 vsi_id,\n+\t\t\t       u8 *eth_num, u8 *eth_id, u8 *logic_eth_id)\n+{\n+\tstruct nbl_resource_ops *res_ops = disp_mgt-\u003eres_ops_tbl-\u003eops;\n+\tstruct nbl_resource_mgt *p = disp_mgt-\u003eres_ops_tbl-\u003epriv;\n+\tstruct nbl_common_info *common = disp_mgt-\u003ecommon;\n+\n+\tif (res_ops-\u003eget_eth_id)\n+\t\treturn res_ops-\u003eget_eth_id(p, common-\u003emgt_pf, vsi_id,\n+\t\t\t\t\t   eth_num, eth_id, logic_eth_id);\n+\treturn -EOPNOTSUPP;\n+}\n+\n+static int nbl_disp_setup_msg(struct nbl_dispatch_mgt *disp_mgt)\n+{\n+\tstruct nbl_channel_ops *chan_ops = disp_mgt-\u003echan_ops_tbl-\u003eops;\n+\tstruct nbl_channel_mgt *p = disp_mgt-\u003echan_ops_tbl-\u003epriv;\n+\tint ret = 0;\n+\tint _ret;\n+\n+\t_ret = chan_ops-\u003eregister_msg(p, NBL_CHAN_MSG_CONFIGURE_MSIX_MAP,\n+\t\t\t\t      nbl_disp_chan_cfg_msix_map_resp,\n+\t\t\t\t      disp_mgt);\n+\tif (_ret \u003c 0 \u0026\u0026 !ret)\n+\t\tret = _ret;\n+\n+\t_ret = chan_ops-\u003eregister_msg(p, NBL_CHAN_MSG_DESTROY_MSIX_MAP,\n+\t\t\t\t      nbl_disp_chan_destroy_msix_map_resp,\n+\t\t\t\t      disp_mgt);\n+\tif (_ret \u003c 0 \u0026\u0026 !ret)\n+\t\tret = _ret;\n+\n+\t_ret = chan_ops-\u003eregister_msg(p, NBL_CHAN_MSG_MAILBOX_SET_IRQ,\n+\t\t\t\t      nbl_disp_chan_set_mailbox_irq_resp,\n+\t\t\t\t      disp_mgt);\n+\tif (_ret \u003c 0 \u0026\u0026 !ret)\n+\t\tret = _ret;\n+\n+\t_ret = chan_ops-\u003eregister_msg(p, NBL_CHAN_MSG_GET_VSI_ID,\n+\t\t\t\t      nbl_disp_chan_get_vsi_id_resp,\n+\t\t\t\t      disp_mgt);\n+\tif (_ret \u003c 0 \u0026\u0026 !ret)\n+\t\tret = _ret;\n+\n+\t_ret = chan_ops-\u003eregister_msg(p, NBL_CHAN_MSG_GET_ETH_ID,\n+\t\t\t\t      nbl_disp_chan_get_eth_id_resp,\n+\t\t\t\t      disp_mgt);\n+\tif (_ret \u003c 0 \u0026\u0026 !ret)\n+\t\tret = _ret;\n+\n+\treturn ret;\n+}\n+\n+static void nbl_disp_set_ctrl_bit(struct nbl_dispatch_mgt *disp_mgt, u32 lvl)\n+{\n+\tset_bit(lvl, disp_mgt-\u003ectrl_lvl);\n+}\n+\n+static void nbl_disp_refresh_ctrl_ops(struct nbl_dispatch_mgt *disp_mgt)\n+{\n+\tstruct nbl_dispatch_ops *disp_ops = disp_mgt-\u003edisp_ops_tbl-\u003eops;\n+\n+\tmemset(disp_ops, 0, sizeof(*disp_ops));\n+\tif (test_bit(NBL_DISP_CTRL_LVL_MGT, disp_mgt-\u003ectrl_lvl)) {\n+\t\tdisp_ops-\u003einit_module = nbl_disp_init_module;\n+\t\tdisp_ops-\u003edeinit_module = nbl_disp_deinit_module;\n+\t\tdisp_ops-\u003ecfg_msix_map = nbl_disp_cfg_msix_map;\n+\t\tdisp_ops-\u003edestroy_msix_map = nbl_disp_destroy_msix_map;\n+\t\tdisp_ops-\u003eset_mailbox_irq = nbl_disp_set_mailbox_irq;\n+\t\tdisp_ops-\u003eget_vsi_id = nbl_disp_get_vsi_id;\n+\t\tdisp_ops-\u003eget_eth_id = nbl_disp_get_eth_id;\n+\t} else if (test_bit(NBL_DISP_CTRL_LVL_NET, disp_mgt-\u003ectrl_lvl)) {\n+\t\tdisp_ops-\u003ecfg_msix_map =\n+\t\t\tnbl_disp_chan_cfg_msix_map_req;\n+\t\tdisp_ops-\u003edestroy_msix_map = nbl_disp_chan_destroy_msix_map_req;\n+\t\tdisp_ops-\u003eset_mailbox_irq = nbl_disp_chan_set_mailbox_irq_req;\n+\t\tdisp_ops-\u003eget_vsi_id = nbl_disp_chan_get_vsi_id_req;\n+\t\tdisp_ops-\u003eget_eth_id = nbl_disp_chan_get_eth_id_req;\n+\t}\n+}\n+\n+static struct nbl_dispatch_mgt *\n+nbl_disp_setup_disp_mgt(struct nbl_common_info *common)\n+{\n+\tstruct nbl_dispatch_mgt *disp_mgt;\n+\tstruct device *dev = common-\u003edev;\n+\n+\tdisp_mgt = devm_kzalloc(dev, sizeof(*disp_mgt), GFP_KERNEL);\n+\tif (!disp_mgt)\n+\t\treturn ERR_PTR(-ENOMEM);\n+\n+\tdisp_mgt-\u003ecommon = common;\n+\treturn disp_mgt;\n+}\n+\n+static struct nbl_dispatch_ops_tbl *\n+nbl_disp_setup_ops(struct device *dev, struct nbl_dispatch_mgt *disp_mgt)\n+{\n+\tstruct nbl_dispatch_ops_tbl *disp_ops_tbl;\n+\tstruct nbl_dispatch_ops *disp_ops;\n+\n+\tdisp_ops_tbl = devm_kzalloc(dev, sizeof(*disp_ops_tbl), GFP_KERNEL);\n+\tif (!disp_ops_tbl)\n+\t\treturn ERR_PTR(-ENOMEM);\n+\n+\tdisp_ops = devm_kzalloc(dev, sizeof(*disp_ops), GFP_KERNEL);\n+\tif (!disp_ops)\n+\t\treturn ERR_PTR(-ENOMEM);\n+\n+\tdisp_ops_tbl-\u003eops = disp_ops;\n+\tdisp_ops_tbl-\u003epriv = disp_mgt;\n+\n+\treturn disp_ops_tbl;\n+}\n+\n+int nbl_disp_init(struct nbl_adapter *adapter)\n+{\n+\tstruct nbl_common_info *common = \u0026adapter-\u003ecommon;\n+\tstruct nbl_dispatch_ops_tbl *disp_ops_tbl;\n+\tstruct nbl_resource_ops_tbl *res_ops_tbl =\n+\t\tadapter-\u003eintf.resource_ops_tbl;\n+\tstruct nbl_channel_ops_tbl *chan_ops_tbl =\n+\t\tadapter-\u003eintf.channel_ops_tbl;\n+\tstruct device *dev = \u0026adapter-\u003epdev-\u003edev;\n+\tstruct nbl_dispatch_mgt *disp_mgt;\n+\tint ret;\n+\n+\tdisp_mgt = nbl_disp_setup_disp_mgt(common);\n+\tif (IS_ERR(disp_mgt)) {\n+\t\tret = PTR_ERR(disp_mgt);\n+\t\treturn ret;\n+\t}\n+\n+\tdisp_ops_tbl = nbl_disp_setup_ops(dev, disp_mgt);\n+\tif (IS_ERR(disp_ops_tbl)) {\n+\t\tret = PTR_ERR(disp_ops_tbl);\n+\t\treturn ret;\n+\t}\n+\n+\tdisp_mgt-\u003eres_ops_tbl = res_ops_tbl;\n+\tdisp_mgt-\u003echan_ops_tbl = chan_ops_tbl;\n+\tdisp_mgt-\u003edisp_ops_tbl = disp_ops_tbl;\n+\tadapter-\u003ecore.disp_mgt = disp_mgt;\n+\tadapter-\u003eintf.dispatch_ops_tbl = disp_ops_tbl;\n+\n+\tret = nbl_disp_setup_msg(disp_mgt);\n+\tif (ret)\n+\t\treturn ret;\n+\n+\tif (common-\u003ehas_ctrl)\n+\t\tnbl_disp_set_ctrl_bit(disp_mgt, NBL_DISP_CTRL_LVL_MGT);\n+\n+\tif (common-\u003ehas_net)\n+\t\tnbl_disp_set_ctrl_bit(disp_mgt, NBL_DISP_CTRL_LVL_NET);\n+\tnbl_disp_refresh_ctrl_ops(disp_mgt);\n+\treturn 0;\n+}\n+\n+void nbl_disp_remove(struct nbl_adapter *adapter)\n+{\n+\t/*\n+\t * Dispatch structures are devm-allocated and freed at detach.\n+\t *\n+\t * The five responders registered by nbl_disp_setup_msg() are\n+\t * owned by the channel layer (xarray of handlers) and are never\n+\t * unregistered here. This is safe because the teardown order\n+\t * guarantees no responder can run after this point:\n+\t *\n+\t *   mailbox teardown\n+\t *     -\u003e cancel_work_sync(clean_mbx_task)   // drain RX work\n+\t *     -\u003e nbl_chan_teardown_queue()          // stop HW queue,\n+\t *                                                 // join clean task,\n+\t *                                                 // active=false\n+\t *     -\u003e nbl_chan_remove_common()\n+\t *          -\u003e nbl_common_destroy_wq()            // no new work\n+\t *          -\u003e nbl_chan_remove_msg_handler()      // free handler nodes\n+\t *\n+\t * By the time devres frees disp_mgt, the mailbox queue is stopped\n+\t * and the handler xarray is empty, so no responder can touch\n+\t * res_mgt-\u003eintr_mgt after nbl_intr_mgt_stop() has cleared it.\n+\t */\n+}\ndiff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_core/nbl_dispatch.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_core/nbl_dispatch.h\nnew file mode 100644\nindex 00000000000000..a7e5802344b414\n--- /dev/null\n+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_core/nbl_dispatch.h\n@@ -0,0 +1,23 @@\n+/* SPDX-License-Identifier: GPL-2.0 */\n+/*\n+ * Copyright (c) 2026 Nebula Matrix Limited.\n+ */\n+\n+#ifndef _NBL_DISPATCH_H_\n+#define _NBL_DISPATCH_H_\n+#include \"../nbl_include/nbl_include.h\"\n+#include \"../nbl_include/nbl_def_channel.h\"\n+#include \"../nbl_include/nbl_def_resource.h\"\n+#include \"../nbl_include/nbl_def_dispatch.h\"\n+#include \"../nbl_include/nbl_def_common.h\"\n+#include \"../nbl_core.h\"\n+\n+struct nbl_dispatch_mgt {\n+\tstruct nbl_common_info *common;\n+\tstruct nbl_resource_ops_tbl *res_ops_tbl;\n+\tstruct nbl_channel_ops_tbl *chan_ops_tbl;\n+\tstruct nbl_dispatch_ops_tbl *disp_ops_tbl;\n+\tDECLARE_BITMAP(ctrl_lvl, NBL_DISP_CTRL_LVL_MAX);\n+};\n+\n+#endif\ndiff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_chip.c b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_chip.c\nnew file mode 100644\nindex 00000000000000..419eb6392ada19\n--- /dev/null\n+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_chip.c\n@@ -0,0 +1,32 @@\n+// SPDX-License-Identifier: GPL-2.0\n+/*\n+ * Copyright (c) 2026 Nebula Matrix Limited.\n+ */\n+#include \u003clinux/device.h\u003e\n+#include \"nbl_chip.h\"\n+\n+void nbl_res_chip_deinit_module(struct nbl_resource_mgt *res_mgt)\n+{\n+\tstruct nbl_hw_ops *hw_ops = res_mgt-\u003ehw_ops_tbl-\u003eops;\n+\tstruct nbl_common_info *common = res_mgt-\u003ecommon;\n+\n+\tif (!common-\u003ehas_ctrl)\n+\t\treturn;\n+\thw_ops-\u003edeinit_module(res_mgt-\u003ehw_ops_tbl-\u003epriv);\n+}\n+\n+int nbl_res_chip_init_module(struct nbl_resource_mgt *res_mgt)\n+{\n+\tstruct nbl_common_info *common = res_mgt-\u003ecommon;\n+\tstruct nbl_hw_ops *hw_ops;\n+\tu8 eth_speed, eth_num;\n+\tstruct nbl_hw_mgt *p;\n+\n+\tif (!common-\u003ehas_ctrl)\n+\t\treturn -EINVAL;\n+\teth_speed = res_mgt-\u003eresource_info-\u003eboard_info.eth_speed;\n+\teth_num = res_mgt-\u003eresource_info-\u003eboard_info.eth_num;\n+\thw_ops = res_mgt-\u003ehw_ops_tbl-\u003eops;\n+\tp = res_mgt-\u003ehw_ops_tbl-\u003epriv;\n+\treturn hw_ops-\u003einit_module(p, eth_speed, eth_num);\n+}\ndiff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_chip.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_chip.h\nnew file mode 100644\nindex 00000000000000..d14093ba916c95\n--- /dev/null\n+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_chip.h\n@@ -0,0 +1,12 @@\n+/* SPDX-License-Identifier: GPL-2.0 */\n+/*\n+ * Copyright (c) 2026 Nebula Matrix Limited.\n+ */\n+\n+#ifndef _NBL_CHIP_H_\n+#define _NBL_CHIP_H_\n+\n+#include \"nbl_resource.h\"\n+int nbl_res_chip_init_module(struct nbl_resource_mgt *res_mgt);\n+void nbl_res_chip_deinit_module(struct nbl_resource_mgt *res_mgt);\n+#endif\ndiff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_leonis/nbl_hw_leonis.c b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_leonis/nbl_hw_leonis.c\nindex cf40ddc4519211..2b55f2c2518b4f 100644\n--- a/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_leonis/nbl_hw_leonis.c\n+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_leonis/nbl_hw_leonis.c\n@@ -5,9 +5,1232 @@\n #include \u003clinux/device.h\u003e\n #include \u003clinux/pci.h\u003e\n #include \u003clinux/bits.h\u003e\n+#include \u003clinux/delay.h\u003e\n #include \u003clinux/io.h\u003e\n+#include \u003clinux/spinlock.h\u003e\n+#include \u003clinux/bitfield.h\u003e\n #include \"nbl_hw_leonis.h\"\n \n+/*\n+ * Firmware cleanup after driver_status=false is asynchronous and the\n+ * current hardware revision exposes no cleanup-complete status bit, so\n+ * this delay is best-effort only: it gives the firmware pass time to\n+ * settle before the caller returns, but cannot prove that it has.\n+ * Bounding the wait keeps a later probe from re-programming the per-PF\n+ * registers that pass reclaims (mailbox QINFO routing, qinfo registers)\n+ * while firmware is realistically still wiping them.  The chip-wide\n+ * datapath registers init_module() writes are not part of that pass and\n+ * stay programmed until chip reset.  The hard teardown ordering comes\n+ * from the device-link relationship between the sibling PFs and the\n+ * control PF.\n+ *\n+ * These bounds are empirical, not derived from a documented firmware\n+ * latency.  There is nothing to poll for instead: the hardware exposes\n+ * no cleanup-complete bit, and the reclaimed entries read back the same\n+ * before and after the pass, so there is no observable side effect to\n+ * wait on and nothing to validate the delay against.  A firmware pass\n+ * that takes longer than this window - an ECPU still busy right after\n+ * kexec, say - can therefore still race a later probe.\n+ */\n+#define NBL_FW_CLEANUP_SYNC_MIN_US\t2000\n+#define NBL_FW_CLEANUP_SYNC_MAX_US\t3000\n+\n+static void nbl_hw_read_mbx_regs(struct nbl_hw_mgt *hw_mgt, u64 reg, u32 *data,\n+\t\t\t\t u32 len)\n+{\n+\tu32 i;\n+\n+\tif (len % 4)\n+\t\treturn;\n+\tif (reg \u003e= (u64)hw_mgt-\u003emailbox_bar_size ||\n+\t    reg + len \u003e (u64)hw_mgt-\u003emailbox_bar_size) {\n+\t\tdev_err_once(hw_mgt-\u003ecommon-\u003edev,\n+\t\t\t     \"mbx read out of range: reg=0x%llx len=%u bar_size=%pa\\n\",\n+\t\t\t     reg, len, \u0026hw_mgt-\u003emailbox_bar_size);\n+\t\treturn;\n+\t}\n+\tfor (i = 0; i \u003c len / 4; i++)\n+\t\tdata[i] = nbl_mbx_rd32(hw_mgt, reg + i * sizeof(u32));\n+}\n+\n+static void nbl_hw_write_mbx_regs(struct nbl_hw_mgt *hw_mgt, u64 reg,\n+\t\t\t\t  const u32 *data, u32 len)\n+{\n+\tu32 i;\n+\n+\tif (len % 4)\n+\t\treturn;\n+\tif (reg \u003e= (u64)hw_mgt-\u003emailbox_bar_size ||\n+\t    reg + len \u003e (u64)hw_mgt-\u003emailbox_bar_size) {\n+\t\tdev_err_once(hw_mgt-\u003ecommon-\u003edev,\n+\t\t\t     \"mbx write out of range: reg=0x%llx len=%u bar_size=%pa\\n\",\n+\t\t\t     reg, len, \u0026hw_mgt-\u003emailbox_bar_size);\n+\t\treturn;\n+\t}\n+\tfor (i = 0; i \u003c len / 4; i++)\n+\t\tnbl_mbx_wr32(hw_mgt, reg + i * sizeof(u32), data[i]);\n+}\n+\n+/*\n+ * Flush posted mailbox-BAR writes by reading back through the same\n+ * BAR.\n+ */\n+static void nbl_hw_flush_mbx_write(struct nbl_hw_mgt *hw_mgt, u64 reg)\n+{\n+\tu32 data;\n+\n+\tnbl_hw_read_mbx_regs(hw_mgt, reg, \u0026data, sizeof(data));\n+}\n+\n+static void nbl_hw_rd_regs(struct nbl_hw_mgt *hw_mgt, u64 reg, u32 *data,\n+\t\t\t   u32 len)\n+{\n+\tu32 size = len / 4;\n+\tu32 i;\n+\n+\tif (len % 4)\n+\t\treturn;\n+\tfor (i = 0; i \u003c size; i++)\n+\t\tdata[i] = rd32(hw_mgt-\u003ehw_addr, reg + i * sizeof(u32));\n+}\n+\n+static void nbl_hw_wr_regs(struct nbl_hw_mgt *hw_mgt, u64 reg, const u32 *data,\n+\t\t\t   u32 len)\n+{\n+\tu32 size = len / 4;\n+\tu32 i;\n+\n+\tif (len % 4)\n+\t\treturn;\n+\tfor (i = 0; i \u003c size; i++)\n+\t\twr32(hw_mgt-\u003ehw_addr, reg + i * sizeof(u32), data[i]);\n+}\n+\n+static void nbl_hw_rd_regs_lock(struct nbl_hw_mgt *hw_mgt, u64 reg, u32 *data,\n+\t\t\t\tu32 len)\n+{\n+\tu32 size = len / 4;\n+\tu32 i;\n+\n+\tif (len % 4)\n+\t\treturn;\n+\n+\tspin_lock(\u0026hw_mgt-\u003ereg_lock);\n+\n+\tfor (i = 0; i \u003c size; i++)\n+\t\tdata[i] = rd32(hw_mgt-\u003ehw_addr, reg + i * sizeof(u32));\n+\tspin_unlock(\u0026hw_mgt-\u003ereg_lock);\n+}\n+\n+static void nbl_hw_wr_regs_lock(struct nbl_hw_mgt *hw_mgt, u64 reg,\n+\t\t\t\tconst u32 *data, u32 len)\n+{\n+\tu32 size = len / 4;\n+\tu32 i;\n+\n+\tif (len % 4)\n+\t\treturn;\n+\tspin_lock(\u0026hw_mgt-\u003ereg_lock);\n+\tfor (i = 0; i \u003c size; i++)\n+\t\twr32(hw_mgt-\u003ehw_addr, reg + i * sizeof(u32), data[i]);\n+\tspin_unlock(\u0026hw_mgt-\u003ereg_lock);\n+}\n+\n+/*\n+ * Flush posted MEMORY-BAR writes by reading back a register that only\n+ * exists in the control-PF mapping.\n+ *\n+ * The mapped size does not derive from bar_len: nbl_hw_init_leonis()\n+ * maps PAGE_ALIGN_DOWN(NBL_MEM_BAR_TOTAL_SIZE - NBL_RDMA_NOTIFY_LEN)\n+ * (~64 MiB) when has_ctrl is set, and NBL_REG_NET_ONLY_LEN (8 KiB) when\n+ * it is not.  NBL_HW_DUMMY_REG (0x1300904) is only covered by the\n+ * control-PF mapping, so this must not be called on a net-only PF.\n+ *\n+ * Every current caller runs on the control PF: init_module()/\n+ * deinit_module() are has_ctrl-guarded in the resource layer, and the\n+ * MSI-X paths are either has_ctrl-guarded as well or dispatched to the\n+ * management PF.  The check below is therefore a defensive guard against\n+ * a future caller, not a reachable failure path.\n+ */\n+static void nbl_flush_writes(struct nbl_hw_mgt *hw_mgt)\n+{\n+\tif (WARN_ON_ONCE(!hw_mgt-\u003ecommon-\u003ehas_ctrl))\n+\t\treturn;\n+\n+\tnbl_hw_rd32(hw_mgt, NBL_HW_DUMMY_REG);\n+}\n+\n+/*\n+ * Registers reset to zero after cold boot / FLR / bus reset. Firmware\n+ * programs valid values before driver probe, so zero is only seen on\n+ * hardware fault or register read failure. Initialize data=0 to guard\n+ * against nbl_hw_read_mbx_regs() early-return on bounds-check failure.\n+ */\n+static void nbl_hw_get_fw_eth_map(struct nbl_hw_mgt *hw_mgt, u32 *eth_map)\n+{\n+\tu32 data = 0;\n+\n+\tnbl_hw_read_mbx_regs(hw_mgt, NBL_FW_BOARD_DW6_OFFSET, \u0026data,\n+\t\t\t     sizeof(data));\n+\t*eth_map = FIELD_GET(NBL_FW_BOARD_DW6_ETH_BITMAP_MASK, data);\n+}\n+\n+static u32 nbl_hw_get_quirks(struct nbl_hw_mgt *hw_mgt)\n+{\n+\tu32 quirks = 0;\n+\n+\t/*\n+\t * Read quirk bits from mailbox register.\n+\t * All supported firmware implement the quirk ABI,\n+\t * firmware always populates NBL_LEONIS_QUIRKS_OFFSET.\n+\t * Value ~0U indicates no active quirks.\n+\t */\n+\tnbl_hw_read_mbx_regs(hw_mgt, NBL_LEONIS_QUIRKS_OFFSET, \u0026quirks,\n+\t\t\t     sizeof(u32));\n+\n+\tif (quirks == ~0u)\n+\t\treturn 0;\n+\n+\treturn quirks;\n+}\n+\n+static void nbl_configure_dped_checksum(struct nbl_hw_mgt *hw_mgt)\n+{\n+\tu32 data = 0;\n+\n+\t/* DPED dped_l4_ck_cmd_40 for sctp */\n+\tspin_lock(\u0026hw_mgt-\u003ereg_lock);\n+\tnbl_hw_rd_regs(hw_mgt, NBL_DPED_L4_CK_CMD_40_ADDR, \u0026data, sizeof(data));\n+\tdata |= FIELD_PREP(NBL_DPED_L4_CK_CMD_40_EN_MASK, 1);\n+\tnbl_hw_wr_regs(hw_mgt, NBL_DPED_L4_CK_CMD_40_ADDR, \u0026data, sizeof(data));\n+\tspin_unlock(\u0026hw_mgt-\u003ereg_lock);\n+}\n+\n+static void nbl_dped_init(struct nbl_hw_mgt *hw_mgt)\n+{\n+\tspin_lock(\u0026hw_mgt-\u003ereg_lock);\n+\tnbl_hw_wr32(hw_mgt, NBL_DPED_VLAN_OFFSET, 0xC);\n+\tnbl_hw_wr32(hw_mgt, NBL_DPED_DSCP_OFFSET_0, 0x8);\n+\tnbl_hw_wr32(hw_mgt, NBL_DPED_DSCP_OFFSET_1, 0x4);\n+\tspin_unlock(\u0026hw_mgt-\u003ereg_lock);\n+\t/* dped checksum offload */\n+\tnbl_configure_dped_checksum(hw_mgt);\n+}\n+\n+static void nbl_uped_init(struct nbl_hw_mgt *hw_mgt)\n+{\n+\tu32 hw_edit = 0;\n+\n+\t/* V4 TCP: l3_len = 0 */\n+\tspin_lock(\u0026hw_mgt-\u003ereg_lock);\n+\tnbl_hw_rd_regs(hw_mgt, NBL_UPED_HW_EDT_PROF_TABLE(NBL_UPED_V4_TCP_IDX),\n+\t\t       \u0026hw_edit, sizeof(hw_edit));\n+\thw_edit \u0026= ~NBL_PED_HW_EDIT_PROFILE_L3_LEN_MASK;\n+\tnbl_hw_wr_regs(hw_mgt, NBL_UPED_HW_EDT_PROF_TABLE(NBL_UPED_V4_TCP_IDX),\n+\t\t       \u0026hw_edit, sizeof(hw_edit));\n+\n+\t/* V6 TCP: l3_len = 1 */\n+\tnbl_hw_rd_regs(hw_mgt, NBL_UPED_HW_EDT_PROF_TABLE(NBL_UPED_V6_TCP_IDX),\n+\t\t       \u0026hw_edit, sizeof(hw_edit));\n+\thw_edit = (hw_edit \u0026 ~NBL_PED_HW_EDIT_PROFILE_L3_LEN_MASK) |\n+\t\t  FIELD_PREP(NBL_PED_HW_EDIT_PROFILE_L3_LEN_MASK, 1);\n+\tnbl_hw_wr_regs(hw_mgt, NBL_UPED_HW_EDT_PROF_TABLE(NBL_UPED_V6_TCP_IDX),\n+\t\t       \u0026hw_edit, sizeof(hw_edit));\n+\tspin_unlock(\u0026hw_mgt-\u003ereg_lock);\n+}\n+\n+static int nbl_shaping_eth_init(struct nbl_hw_mgt *hw_mgt, u8 eth_id, u8 speed)\n+{\n+\tstruct nbl_shaping_dvn_dport_u dvn_dport = { 0 };\n+\tstruct nbl_shaping_dport_u dport = { 0 };\n+\tu32 rate, half_rate;\n+\tu32 depth;\n+\tu64 low_val, high_val;\n+\n+\tswitch (speed) {\n+\tcase NBL_FW_PORT_SPEED_100G:\n+\t\trate = 100000;\n+\t\tbreak;\n+\tcase NBL_FW_PORT_SPEED_50G:\n+\t\trate = 50000;\n+\t\tbreak;\n+\tcase NBL_FW_PORT_SPEED_25G:\n+\t\trate = 25000;\n+\t\tbreak;\n+\tcase NBL_FW_PORT_SPEED_10G:\n+\t\trate = 10000;\n+\t\tbreak;\n+\tdefault:\n+\t\tdev_err(hw_mgt-\u003ecommon-\u003edev,\n+\t\t\t\"Unsupported port speed %u for eth%u\\n\", speed, eth_id);\n+\t\treturn -EINVAL;\n+\t}\n+\n+\thalf_rate = rate / 2;\n+\tdepth = max_t(u32, rate * 2, NBL_LR_LEONIS_NET_BUCKET_DEPTH);\n+\n+\t/* 1. clear valid first\n+\t * dport and dvn_dport are zero-initialised above, so VALID=0 already\n+\t */\n+\tspin_lock(\u0026hw_mgt-\u003ereg_lock);\n+\tnbl_hw_wr_regs(hw_mgt, NBL_SHAPING_DPORT_REG(eth_id), dport.data,\n+\t\t       sizeof(dport));\n+\tnbl_hw_wr_regs(hw_mgt, NBL_SHAPING_DVN_DPORT_REG(eth_id),\n+\t\t       dvn_dport.data, sizeof(dvn_dport));\n+\n+\t/* 2. write config words (valid=0, safe) */\n+\tlow_val = FIELD_PREP(NBL_DPORT_CIR_MASK, rate) |\n+\t\t  FIELD_PREP(NBL_DPORT_PIR_MASK, rate) |\n+\t\t  FIELD_PREP(NBL_DPORT_DEPTH_MASK, depth) |\n+\t\t  FIELD_PREP(NBL_DPORT_CBS_MASK_LOW, depth \u0026 0x3F);\n+\thigh_val = FIELD_PREP(NBL_DPORT_CBS_MASK_HIGH, depth \u003e\u003e 6) |\n+\t\t   FIELD_PREP(NBL_DPORT_PBS_MASK, depth);\n+\t/* Fixed split, independent of host endian */\n+\tdport.data[0] = lower_32_bits(low_val);\n+\tdport.data[1] = upper_32_bits(low_val);\n+\tdport.data[2] = lower_32_bits(high_val);\n+\tdport.data[3] = upper_32_bits(high_val);\n+\n+\tlow_val = FIELD_PREP(NBL_DPORT_CIR_MASK, half_rate) |\n+\t\t  FIELD_PREP(NBL_DPORT_PIR_MASK, rate) |\n+\t\t  FIELD_PREP(NBL_DPORT_DEPTH_MASK, depth) |\n+\t\t  FIELD_PREP(NBL_DPORT_CBS_MASK_LOW, depth \u0026 0x3F);\n+\thigh_val = FIELD_PREP(NBL_DPORT_CBS_MASK_HIGH, depth \u003e\u003e 6) |\n+\t\t   FIELD_PREP(NBL_DPORT_PBS_MASK, depth);\n+\tdvn_dport.data[0] = lower_32_bits(low_val);\n+\tdvn_dport.data[1] = upper_32_bits(low_val);\n+\tdvn_dport.data[2] = lower_32_bits(high_val);\n+\tdvn_dport.data[3] = upper_32_bits(high_val);\n+\n+\tnbl_hw_wr_regs(hw_mgt, NBL_SHAPING_DPORT_REG(eth_id), dport.data,\n+\t\t       sizeof(dport));\n+\tnbl_hw_wr_regs(hw_mgt, NBL_SHAPING_DVN_DPORT_REG(eth_id),\n+\t\t       dvn_dport.data, sizeof(dvn_dport));\n+\n+\t/* 3. commit: set valid last */\n+\tlow_val = FIELD_PREP(NBL_DPORT_VALID_MASK, 1);\n+\tdport.data[0] |= lower_32_bits(low_val);\n+\n+\tlow_val = FIELD_PREP(NBL_DPORT_VALID_MASK, 1);\n+\tdvn_dport.data[0] |= lower_32_bits(low_val);\n+\n+\tnbl_hw_wr_regs(hw_mgt, NBL_SHAPING_DPORT_REG(eth_id), dport.data,\n+\t\t       sizeof(dport));\n+\tnbl_hw_wr_regs(hw_mgt, NBL_SHAPING_DVN_DPORT_REG(eth_id),\n+\t\t       dvn_dport.data, sizeof(dvn_dport));\n+\tspin_unlock(\u0026hw_mgt-\u003ereg_lock);\n+\treturn 0;\n+}\n+\n+static int nbl_shaping_init(struct nbl_hw_mgt *hw_mgt, u8 speed)\n+{\n+\tstruct nbl_shaping_net_u net_shaping = { 0 };\n+\tu32 eth_bitmap = 0;\n+\tu32 reg_val;\n+\tint ret;\n+\tint i;\n+\n+\tnbl_hw_get_fw_eth_map(hw_mgt, \u0026eth_bitmap);\n+\tfor (i = 0; i \u003c NBL_MAX_ETHERNET; i++) {\n+\t\tif (!(eth_bitmap \u0026 BIT(i)))\n+\t\t\tcontinue;\n+\t\tret = nbl_shaping_eth_init(hw_mgt, i, speed);\n+\t\tif (ret)\n+\t\t\treturn ret;\n+\t}\n+\t/*\n+\t * Read-modify-write as one critical section: reg_lock is dropped\n+\t * between the two _lock() helpers, which would let another writer\n+\t * of this register land in the middle of the update.  Nothing\n+\t * else touches NBL_DSCH_PSHA_EN_ADDR today, but the RMW has to be\n+\t * atomic for that to stay true.\n+\t */\n+\tspin_lock(\u0026hw_mgt-\u003ereg_lock);\n+\tnbl_hw_rd_regs(hw_mgt, NBL_DSCH_PSHA_EN_ADDR, \u0026reg_val,\n+\t\t       sizeof(reg_val));\n+\treg_val \u0026= ~NBL_DSCH_PSHA_EN_MASK;\n+\treg_val |= FIELD_PREP(NBL_DSCH_PSHA_EN_MASK,\n+\t\t\t      eth_bitmap \u0026 GENMASK(3, 0));\n+\tnbl_hw_wr_regs(hw_mgt, NBL_DSCH_PSHA_EN_ADDR, \u0026reg_val,\n+\t\t       sizeof(reg_val));\n+\tspin_unlock(\u0026hw_mgt-\u003ereg_lock);\n+\n+\tfor (i = 0; i \u003c NBL_MAX_PF; i++) {\n+\t\tnbl_hw_wr_regs_lock(hw_mgt, NBL_SHAPING_NET_REG(i),\n+\t\t\t\t    net_shaping.data,\n+\t\t\t\t    sizeof(net_shaping));\n+\t}\n+\tnbl_flush_writes(hw_mgt);\n+\treturn 0;\n+}\n+\n+static void nbl_dsch_qid_max_init(struct nbl_hw_mgt *hw_mgt)\n+{\n+\tu32 quanta = 0;\n+\tu32 qid_max = 0;\n+\n+\tquanta = FIELD_PREP(NBL_DSCH_VN_QUANTA_H_QUA_MASK, NBL_HOST_QUANTA) |\n+\t\t FIELD_PREP(NBL_DSCH_VN_QUANTA_E_QUA_MASK, NBL_ECPU_QUANTA);\n+\tspin_lock(\u0026hw_mgt-\u003ereg_lock);\n+\tnbl_hw_wr_regs(hw_mgt, NBL_DSCH_VN_QUANTA_ADDR, \u0026quanta,\n+\t\t       sizeof(quanta));\n+\tnbl_hw_rd_regs(hw_mgt, NBL_DSCH_HOST_QID_MAX, \u0026qid_max,\n+\t\t       sizeof(qid_max));\n+\tqid_max \u0026= ~NBL_DSCH_HOST_QID_MAX_MASK;\n+\tqid_max |= FIELD_PREP(NBL_DSCH_HOST_QID_MAX_MASK, NBL_MAX_QUEUE_ID);\n+\tnbl_hw_wr_regs(hw_mgt, NBL_DSCH_HOST_QID_MAX, \u0026qid_max,\n+\t\t       sizeof(qid_max));\n+\tspin_unlock(\u0026hw_mgt-\u003ereg_lock);\n+}\n+\n+static int nbl_ustore_init(struct nbl_hw_mgt *hw_mgt, u8 eth_num)\n+{\n+\tu32 eth_bitmap = 0;\n+\tu32 drop_th = 0;\n+\tu32 pkt_len = 0;\n+\tu32 reg_val = 0;\n+\tint i;\n+\n+\t/*\n+\t * eth_num is validated in the resource layer:\n+\t * nbl_res_init_pf_num() requires 1/2/4 PFs, and\n+\t * nbl_res_ctrl_dev_setup_eth_info() requires max_pf == eth_num.\n+\t * This is a defensive check only; if it triggers, the resource\n+\t * layer validation was bypassed, which is a bug.\n+\t */\n+\tif (WARN_ON(eth_num != 1 \u0026\u0026 eth_num != 2 \u0026\u0026 eth_num != 4))\n+\t\treturn -EINVAL;\n+\t/* Read current packet length config\n+\t *(to preserve other fields while updating 'min')\n+\t */\n+\tspin_lock(\u0026hw_mgt-\u003ereg_lock);\n+\tnbl_hw_rd_regs(hw_mgt, NBL_USTORE_PKT_LEN_ADDR, \u0026pkt_len,\n+\t\t       sizeof(pkt_len));\n+\t/* min arp packet length 42 (14 + 28) */\n+\tpkt_len \u0026= ~NBL_USTORE_PKT_LEN_MIN_MASK;\n+\tpkt_len |= FIELD_PREP(NBL_USTORE_PKT_LEN_MIN_MASK, 42);\n+\tnbl_hw_wr_regs(hw_mgt, NBL_USTORE_PKT_LEN_ADDR, \u0026pkt_len,\n+\t\t       sizeof(pkt_len));\n+\n+\tdrop_th |= FIELD_PREP(NBL_USTORE_PORT_DROP_TH_EN_MASK, 1);\n+\tif (eth_num == 1)\n+\t\tdrop_th |= FIELD_PREP(NBL_USTORE_PORT_DROP_TH_DISC_TH_MASK,\n+\t\t\t\t      NBL_USTORE_SINGLE_ETH_DROP_TH);\n+\telse if (eth_num == 2)\n+\t\tdrop_th |= FIELD_PREP(NBL_USTORE_PORT_DROP_TH_DISC_TH_MASK,\n+\t\t\t\t      NBL_USTORE_DUAL_ETH_DROP_TH);\n+\telse\n+\t\tdrop_th |= FIELD_PREP(NBL_USTORE_PORT_DROP_TH_DISC_TH_MASK,\n+\t\t\t\t      NBL_USTORE_QUAD_ETH_DROP_TH);\n+\tnbl_hw_get_fw_eth_map(hw_mgt, \u0026eth_bitmap);\n+\tfor (i = 0; i \u003c NBL_MAX_ETHERNET; i++) {\n+\t\tif (!(eth_bitmap \u0026 BIT(i)))\n+\t\t\tcontinue;\n+\t\tnbl_hw_rd_regs(hw_mgt, NBL_USTORE_PORT_DROP_TH_REG_ARR(i),\n+\t\t\t       \u0026reg_val, sizeof(reg_val));\n+\t\treg_val \u0026= ~(NBL_USTORE_PORT_DROP_TH_EN_MASK |\n+\t\t\t    NBL_USTORE_PORT_DROP_TH_DISC_TH_MASK);\n+\t\treg_val |= drop_th;\n+\t\tnbl_hw_wr_regs(hw_mgt, NBL_USTORE_PORT_DROP_TH_REG_ARR(i),\n+\t\t\t       \u0026reg_val, sizeof(reg_val));\n+\t}\n+\n+\t/* Clear port drop/truncate counters by reading them\n+\t * (hardware has read-to-clear behavior for these registers)\n+\t */\n+\tfor (i = 0; i \u003c NBL_MAX_ETHERNET; i++) {\n+\t\tif (!(eth_bitmap \u0026 BIT(i)))\n+\t\t\tcontinue;\n+\t\tnbl_hw_rd32(hw_mgt, NBL_USTORE_BUF_PORT_DROP_PKT(i));\n+\t\tnbl_hw_rd32(hw_mgt, NBL_USTORE_BUF_PORT_TRUN_PKT(i));\n+\t}\n+\tspin_unlock(\u0026hw_mgt-\u003ereg_lock);\n+\treturn 0;\n+}\n+\n+static void nbl_dstore_init(struct nbl_hw_mgt *hw_mgt, u8 speed)\n+{\n+\tu32 eth_bitmap = 0;\n+\tu32 drop_th = 0;\n+\tu32 fc_th = 0;\n+\tu32 bp_th = 0;\n+\tint i;\n+\n+\tfor (i = 0; i \u003c NBL_DSTORE_PORT_DROP_TH_DEPTH; i++) {\n+\t\tspin_lock(\u0026hw_mgt-\u003ereg_lock);\n+\t\tnbl_hw_rd_regs(hw_mgt, NBL_DSTORE_PORT_DROP_TH_REG(i), \u0026drop_th,\n+\t\t\t       sizeof(drop_th));\n+\t\tdrop_th \u0026= ~NBL_DSTORE_PORT_DROP_EN_MASK;\n+\t\tnbl_hw_wr_regs(hw_mgt, NBL_DSTORE_PORT_DROP_TH_REG(i), \u0026drop_th,\n+\t\t\t       sizeof(drop_th));\n+\t\tspin_unlock(\u0026hw_mgt-\u003ereg_lock);\n+\t}\n+\n+\tspin_lock(\u0026hw_mgt-\u003ereg_lock);\n+\tnbl_hw_rd_regs(hw_mgt, NBL_DSTORE_DISC_BP_TH, \u0026bp_th, sizeof(bp_th));\n+\tbp_th |= FIELD_PREP(NBL_DSTORE_DISC_BP_TH_EN_MASK, 1);\n+\tnbl_hw_wr_regs(hw_mgt, NBL_DSTORE_DISC_BP_TH, \u0026bp_th, sizeof(bp_th));\n+\tspin_unlock(\u0026hw_mgt-\u003ereg_lock);\n+\n+\tnbl_hw_get_fw_eth_map(hw_mgt, \u0026eth_bitmap);\n+\tfor (i = 0; i \u003c NBL_MAX_ETHERNET; i++) {\n+\t\tif (!(eth_bitmap \u0026 BIT(i)))\n+\t\t\tcontinue;\n+\t\tspin_lock(\u0026hw_mgt-\u003ereg_lock);\n+\t\tnbl_hw_rd_regs(hw_mgt, NBL_DSTORE_D_DPORT_FC_TH_REG(i), \u0026fc_th,\n+\t\t\t       sizeof(fc_th));\n+\t\tfc_th \u0026= ~(NBL_DSTORE_D_DPORT_FC_XOFF_TH_MASK |\n+\t\t\t   NBL_DSTORE_D_DPORT_FC_XON_TH_MASK);\n+\t\tif (speed == NBL_FW_PORT_SPEED_100G) {\n+\t\t\tfc_th |=\n+\t\t\t\tFIELD_PREP(NBL_DSTORE_D_DPORT_FC_XOFF_TH_MASK,\n+\t\t\t\t\t   NBL_DSTORE_DROP_XOFF_TH_100G) |\n+\t\t\t\tFIELD_PREP(NBL_DSTORE_D_DPORT_FC_XON_TH_MASK,\n+\t\t\t\t\t   NBL_DSTORE_DROP_XON_TH_100G);\n+\t\t} else {\n+\t\t\tfc_th |=\n+\t\t\t\tFIELD_PREP(NBL_DSTORE_D_DPORT_FC_XOFF_TH_MASK,\n+\t\t\t\t\t   NBL_DSTORE_DROP_XOFF_TH) |\n+\t\t\t\tFIELD_PREP(NBL_DSTORE_D_DPORT_FC_XON_TH_MASK,\n+\t\t\t\t\t   NBL_DSTORE_DROP_XON_TH);\n+\t\t}\n+\n+\t\tfc_th |= FIELD_PREP(NBL_DSTORE_D_DPORT_FC_FC_EN_MASK, 1);\n+\t\tnbl_hw_wr_regs(hw_mgt, NBL_DSTORE_D_DPORT_FC_TH_REG(i), \u0026fc_th,\n+\t\t\t       sizeof(fc_th));\n+\t\tspin_unlock(\u0026hw_mgt-\u003ereg_lock);\n+\t}\n+}\n+\n+static void nbl_dvn_descreq_num_cfg(struct nbl_hw_mgt *hw_mgt, u8 descreq_num)\n+{\n+\tu8 split_ring_num = (descreq_num \u003e\u003e 3) \u0026 0x1;\n+\tu8 ring_num = descreq_num \u0026 0x7;\n+\tu32 num_cfg;\n+\tu32 reg_val;\n+\n+\tspin_lock(\u0026hw_mgt-\u003ereg_lock);\n+\tnbl_hw_rd_regs(hw_mgt, NBL_DVN_DESCREQ_NUM_CFG, \u0026reg_val,\n+\t\t       sizeof(reg_val));\n+\n+\tnum_cfg = FIELD_PREP(NBL_DVN_DESCREQ_NUM_CFG_AVRING_DESREQ_NUM_CFG_MASK,\n+\t\t\t     split_ring_num) |\n+\t\t  FIELD_PREP(NBL_DVN_DESCREQ_NUM_CFG_PACKED_L1_NUM_MASK,\n+\t\t\t     ring_num);\n+\treg_val \u0026= ~(NBL_DVN_DESCREQ_NUM_CFG_AVRING_DESREQ_NUM_CFG_MASK |\n+\t\tNBL_DVN_DESCREQ_NUM_CFG_PACKED_L1_NUM_MASK);\n+\treg_val |= num_cfg;\n+\tnbl_hw_wr_regs(hw_mgt, NBL_DVN_DESCREQ_NUM_CFG, \u0026reg_val,\n+\t\t       sizeof(reg_val));\n+\tspin_unlock(\u0026hw_mgt-\u003ereg_lock);\n+}\n+\n+static void nbl_dvn_init(struct nbl_hw_mgt *hw_mgt, u8 speed)\n+{\n+\tu32 timeout = 0;\n+\tu32 ro_flag = 0;\n+\n+\tspin_lock(\u0026hw_mgt-\u003ereg_lock);\n+\tnbl_hw_wr32(hw_mgt, NBL_DVN_ECPU_QUEUE_NUM, 0);\n+\tspin_unlock(\u0026hw_mgt-\u003ereg_lock);\n+\n+\ttimeout = FIELD_PREP(NBL_DVN_DESC_WR_MERGE_TIMEOUT_CFG_CYCLE_MASK,\n+\t\t\t     DEFAULT_DVN_DESC_WR_MERGE_TIMEOUT_MAX);\n+\tnbl_hw_wr_regs_lock(hw_mgt, NBL_DVN_DESC_WR_MERGE_TIMEOUT, \u0026timeout,\n+\t\t\t    sizeof(timeout));\n+\tspin_lock(\u0026hw_mgt-\u003ereg_lock);\n+\tnbl_hw_rd_regs(hw_mgt, NBL_DVN_DIF_REQ_RD_RO_FLAG, \u0026ro_flag,\n+\t\t       sizeof(ro_flag));\n+\tif (pcie_relaxed_ordering_enabled(hw_mgt-\u003ecommon-\u003epdev)) {\n+\t\tro_flag |=\n+\t\t\tFIELD_PREP(NBL_DVN_DIF_REQ_RD_RO_FLAG_DESC_RO_EN_MASK,\n+\t\t\t\t   1) |\n+\t\t\tFIELD_PREP(NBL_DVN_DIF_REQ_RD_RO_FLAG_DATA_RO_EN_MASK,\n+\t\t\t\t   1) |\n+\t\t\tFIELD_PREP(NBL_DVN_DIF_REQ_RD_RO_FLAG_AVRING_RO_EN_MASK,\n+\t\t\t\t   1);\n+\t} else {\n+\t\tro_flag \u0026=\n+\t\t\t~(FIELD_PREP(NBL_DVN_DIF_REQ_RD_RO_FLAG_DESC_RO_EN_MASK,\n+\t\t\t\t     1) |\n+\t\t\tFIELD_PREP(NBL_DVN_DIF_REQ_RD_RO_FLAG_DATA_RO_EN_MASK,\n+\t\t\t\t   1) |\n+\t\t\tFIELD_PREP(NBL_DVN_DIF_REQ_RD_RO_FLAG_AVRING_RO_EN_MASK,\n+\t\t\t\t   1));\n+\t}\n+\tnbl_hw_wr_regs(hw_mgt, NBL_DVN_DIF_REQ_RD_RO_FLAG, \u0026ro_flag,\n+\t\t       sizeof(ro_flag));\n+\tspin_unlock(\u0026hw_mgt-\u003ereg_lock);\n+\tif (speed == NBL_FW_PORT_SPEED_100G)\n+\t\tnbl_dvn_descreq_num_cfg(hw_mgt,\n+\t\t\t\t\tDEFAULT_DVN_100G_DESCREQ_NUMCFG);\n+\telse\n+\t\tnbl_dvn_descreq_num_cfg(hw_mgt, DEFAULT_DVN_DESCREQ_NUMCFG);\n+}\n+\n+static void nbl_uvn_init(struct nbl_hw_mgt *hw_mgt)\n+{\n+\tu16 wr_timeout = NBL_UVN_DESC_WR_TIMEOUT_VAL;\n+\tu32 timeout = NBL_UVN_DESC_RD_WAIT_TICKS;\n+\tu32 prefetch_init = 0;\n+\tbool ro_enabled;\n+\tu32 flag = 0;\n+\tu32 mask = 0;\n+\tu32 quirks;\n+\tu32 reg_val;\n+\n+\tspin_lock(\u0026hw_mgt-\u003ereg_lock);\n+\tnbl_hw_wr32(hw_mgt, NBL_UVN_ECPU_QUEUE_NUM, 0);\n+\tnbl_hw_wr32(hw_mgt, NBL_UVN_DESC_RD_WAIT, timeout);\n+\tnbl_hw_rd_regs(hw_mgt, NBL_UVN_DESC_WR_TIMEOUT,\n+\t\t       \u0026reg_val, sizeof(reg_val));\n+\treg_val \u0026= ~NBL_UVN_DESC_WR_TIMEOUT_NUM_MASK;\n+\treg_val |= FIELD_PREP(NBL_UVN_DESC_WR_TIMEOUT_NUM_MASK, wr_timeout);\n+\tnbl_hw_wr_regs(hw_mgt, NBL_UVN_DESC_WR_TIMEOUT, \u0026reg_val,\n+\t\t       sizeof(reg_val));\n+\tro_enabled = pcie_relaxed_ordering_enabled(hw_mgt-\u003ecommon-\u003epdev);\n+\n+\tnbl_hw_rd_regs(hw_mgt, NBL_UVN_DIF_REQ_RO_FLAG, \u0026flag, sizeof(flag));\n+\tif (ro_enabled) {\n+\t\tflag |= FIELD_PREP(NBL_UVN_DIF_REQ_RO_FLAG_AVAIL_RD_MASK, 1) |\n+\t\tFIELD_PREP(NBL_UVN_DIF_REQ_RO_FLAG_DESC_RD_MASK, 1) |\n+\t\tFIELD_PREP(NBL_UVN_DIF_REQ_RO_FLAG_PKT_WR_MASK, 1);\n+\t\tflag \u0026= ~FIELD_PREP(NBL_UVN_DIF_REQ_RO_FLAG_DESC_WR_MASK, 1);\n+\t} else {\n+\t\tflag \u0026= ~(FIELD_PREP(NBL_UVN_DIF_REQ_RO_FLAG_AVAIL_RD_MASK, 1) |\n+\t\t\t  FIELD_PREP(NBL_UVN_DIF_REQ_RO_FLAG_DESC_RD_MASK, 1) |\n+\t\t\t  FIELD_PREP(NBL_UVN_DIF_REQ_RO_FLAG_PKT_WR_MASK, 1) |\n+\t\t\t  FIELD_PREP(NBL_UVN_DIF_REQ_RO_FLAG_DESC_WR_MASK, 1));\n+\t}\n+\tnbl_hw_wr_regs(hw_mgt, NBL_UVN_DIF_REQ_RO_FLAG, \u0026flag, sizeof(flag));\n+\n+\tnbl_hw_rd_regs(hw_mgt, NBL_UVN_QUEUE_ERR_MASK, \u0026mask, sizeof(mask));\n+\tmask |= FIELD_PREP(NBL_UVN_QUEUE_ERR_MASK_DIF_ERR_MASK, 1);\n+\n+\tnbl_hw_wr_regs(hw_mgt, NBL_UVN_QUEUE_ERR_MASK, \u0026mask, sizeof(mask));\n+\n+\tspin_unlock(\u0026hw_mgt-\u003ereg_lock);\n+\tquirks = nbl_hw_get_quirks(hw_mgt);\n+\t/*\n+\t * sel=0: use configured num; sel=1: use internal calc (max 32)\n+\t * Default is sel=1, unless NBL_QUIRK_UVN_PREFETCH_ALIGN is set,\n+\t * in which case override to sel=0.\n+\t */\n+\tspin_lock(\u0026hw_mgt-\u003ereg_lock);\n+\tnbl_hw_rd_regs(hw_mgt, NBL_UVN_DESC_PREFETCH_INIT, \u0026reg_val,\n+\t\t       sizeof(reg_val));\n+\tprefetch_init =\n+\t\tFIELD_PREP(NBL_UVN_DESC_PREFETCH_INIT_NUM_MASK,\n+\t\t\t   NBL_UVN_DESC_PREFETCH_NUM) |\n+\t\tFIELD_PREP(NBL_UVN_DESC_PREFETCH_INIT_SEL_MASK,\n+\t\t\t   (quirks \u0026 NBL_QUIRK_UVN_PREFETCH_ALIGN) ? 0 : 1);\n+\treg_val \u0026= ~(NBL_UVN_DESC_PREFETCH_INIT_NUM_MASK |\n+\t\tNBL_UVN_DESC_PREFETCH_INIT_SEL_MASK);\n+\treg_val |= prefetch_init;\n+\tnbl_hw_wr_regs(hw_mgt, NBL_UVN_DESC_PREFETCH_INIT, \u0026reg_val,\n+\t\t       sizeof(reg_val));\n+\tspin_unlock(\u0026hw_mgt-\u003ereg_lock);\n+}\n+\n+static void nbl_uqm_init(struct nbl_hw_mgt *hw_mgt)\n+{\n+\tu32 que_type = 0;\n+\tu32 cnt = 0;\n+\tint i;\n+\n+\tspin_lock(\u0026hw_mgt-\u003ereg_lock);\n+\tnbl_hw_wr_regs(hw_mgt, NBL_UQM_FWD_DROP_CNT, \u0026cnt, sizeof(cnt));\n+\n+\tnbl_hw_wr_regs(hw_mgt, NBL_UQM_DROP_PKT_CNT, \u0026cnt, sizeof(cnt));\n+\tnbl_hw_wr_regs(hw_mgt, NBL_UQM_DROP_PKT_SLICE_CNT, \u0026cnt, sizeof(cnt));\n+\tnbl_hw_wr_regs(hw_mgt, NBL_UQM_DROP_PKT_LEN_ADD_CNT, \u0026cnt, sizeof(cnt));\n+\tnbl_hw_wr_regs(hw_mgt, NBL_UQM_DROP_HEAD_PNTR_ADD_CNT, \u0026cnt,\n+\t\t       sizeof(cnt));\n+\tnbl_hw_wr_regs(hw_mgt, NBL_UQM_DROP_WEIGHT_ADD_CNT, \u0026cnt, sizeof(cnt));\n+\n+\tfor (i = 0; i \u003c NBL_UQM_PORT_DROP_DEPTH; i++) {\n+\t\tnbl_hw_wr_regs(hw_mgt,\n+\t\t\t       NBL_UQM_PORT_DROP_PKT_CNT + (sizeof(cnt) * i),\n+\t\t\t       \u0026cnt, sizeof(cnt));\n+\t\tnbl_hw_wr_regs(hw_mgt,\n+\t\t\t       NBL_UQM_PORT_DROP_PKT_SLICE_CNT +\n+\t\t\t\t       (sizeof(cnt) * i),\n+\t\t\t       \u0026cnt, sizeof(cnt));\n+\t\tnbl_hw_wr_regs(hw_mgt,\n+\t\t\t       NBL_UQM_PORT_DROP_PKT_LEN_ADD_CNT +\n+\t\t\t\t       (sizeof(cnt) * i),\n+\t\t\t       \u0026cnt, sizeof(cnt));\n+\t\tnbl_hw_wr_regs(hw_mgt,\n+\t\t\t       NBL_UQM_PORT_DROP_HEAD_PNTR_ADD_CNT +\n+\t\t\t\t       (sizeof(cnt) * i),\n+\t\t\t       \u0026cnt, sizeof(cnt));\n+\t\tnbl_hw_wr_regs(hw_mgt,\n+\t\t\t       NBL_UQM_PORT_DROP_WEIGHT_ADD_CNT +\n+\t\t\t\t       (sizeof(cnt) * i),\n+\t\t\t       \u0026cnt, sizeof(cnt));\n+\t}\n+\n+\tfor (i = 0; i \u003c NBL_UQM_DPORT_DROP_DEPTH; i++)\n+\t\tnbl_hw_wr_regs(hw_mgt,\n+\t\t\t       NBL_UQM_DPORT_DROP_CNT + (sizeof(cnt) * i), \u0026cnt,\n+\t\t\t       sizeof(cnt));\n+\t/* bit0: 0=bp mode, 1=drop mode, resv bit1-31 */\n+\tnbl_hw_wr_regs(hw_mgt, NBL_UQM_QUE_TYPE, \u0026que_type, sizeof(que_type));\n+\tspin_unlock(\u0026hw_mgt-\u003ereg_lock);\n+}\n+\n+static int nbl_dp_init(struct nbl_hw_mgt *hw_mgt, u8 speed, u8 eth_num)\n+{\n+\tint ret;\n+\n+\tnbl_dped_init(hw_mgt);\n+\tnbl_uped_init(hw_mgt);\n+\tret = nbl_shaping_init(hw_mgt, speed);\n+\tif (ret)\n+\t\treturn ret;\n+\tnbl_dsch_qid_max_init(hw_mgt);\n+\tret = nbl_ustore_init(hw_mgt, eth_num);\n+\tif (ret)\n+\t\treturn ret;\n+\tnbl_dstore_init(hw_mgt, speed);\n+\tnbl_dvn_init(hw_mgt, speed);\n+\tnbl_uvn_init(hw_mgt);\n+\tnbl_uqm_init(hw_mgt);\n+\treturn 0;\n+}\n+\n+static void nbl_host_padpt_init(struct nbl_hw_mgt *hw_mgt)\n+{\n+\t/* padpt flow  control register */\n+\tspin_lock(\u0026hw_mgt-\u003ereg_lock);\n+\tnbl_hw_wr32(hw_mgt, NBL_HOST_PADPT_HOST_CFG_FC_CPLH_UP,\n+\t\t    NBL_HOST_PADPT_CFG_FC_CPLH_UP_VAL);\n+\tnbl_hw_wr32(hw_mgt, NBL_HOST_PADPT_HOST_CFG_FC_PD_DN,\n+\t\t    NBL_HOST_PADPT_CFG_FC_PD_DN_VAL);\n+\tnbl_hw_wr32(hw_mgt, NBL_HOST_PADPT_HOST_CFG_FC_PH_DN,\n+\t\t    NBL_HOST_PADPT_CFG_FC_PH_DN_VAL);\n+\tnbl_hw_wr32(hw_mgt, NBL_HOST_PADPT_HOST_CFG_FC_NPH_DN,\n+\t\t    NBL_HOST_PADPT_CFG_FC_NPH_DN_VAL);\n+\tspin_unlock(\u0026hw_mgt-\u003ereg_lock);\n+}\n+\n+static void nbl_intf_init(struct nbl_hw_mgt *hw_mgt)\n+{\n+\tnbl_host_padpt_init(hw_mgt);\n+}\n+\n+/*\n+ * Read-modify-write under reg_lock: NBL_DRIVER_STATUS_BIT is the only\n+ * bit the driver owns, and the RMW writes back every other bit exactly\n+ * as it read it, so firmware state in those bits survives.\n+ *\n+ * That only holds against concurrent CPU writers.  reg_lock cannot\n+ * serialise against firmware, and the register has no documented\n+ * W1S/W1C form that would let the driver flip bit 16 without the\n+ * read-modify-write; see NBL_DRIVER_STATUS_REG.\n+ */\n+static void nbl_hw_set_driver_status(struct nbl_hw_mgt *hw_mgt, bool active)\n+{\n+\tu32 status;\n+\n+\tspin_lock(\u0026hw_mgt-\u003ereg_lock);\n+\tstatus = nbl_hw_rd32(hw_mgt, NBL_DRIVER_STATUS_REG);\n+\n+\tstatus \u0026= ~BIT(NBL_DRIVER_STATUS_BIT);\n+\tstatus |= FIELD_PREP(BIT(NBL_DRIVER_STATUS_BIT), active);\n+\n+\tnbl_hw_wr32(hw_mgt, NBL_DRIVER_STATUS_REG, status);\n+\tspin_unlock(\u0026hw_mgt-\u003ereg_lock);\n+}\n+\n+/*\n+ * Setting driver status to false notifies firmware to clean up per-PF\n+ * hardware state such as qinfo registers.\n+ *\n+ * Note: firmware does NOT automatically revert chip-wide registers\n+ * configured in this init flow. Those chip-wide settings remain valid\n+ * until chip reset or explicitly overwritten by driver.\n+ *\n+ * This deinit_module only clears driver active status and flush writes.\n+ * It does NOT reset or restore chip-wide datapath registers.\n+ *\n+ * Firmware cleanup is asynchronous with no completion status register.\n+ * The flush only guarantees the status write reached the device; the\n+ * bounded delay after it (NBL_FW_CLEANUP_SYNC_MIN_US) is best-effort and\n+ * does not guarantee that the firmware pass has finished.  The ordering\n+ * this relies on is the device-link one: the siblings are consumers of\n+ * the control PF, so they are unbound first, and that link is created\n+ * with DL_FLAG_AUTOPROBE_CONSUMER.  AUTOPROBE only re-probes consumers,\n+ * never the control PF itself, so a later init_module() needs an admin\n+ * (or a driver rebind) to bind this PF again, and only then does the\n+ * core re-probe the siblings.  Both steps are far slower than the\n+ * firmware pass, but there is no hard firmware handshake.\n+ *\n+ * Caller must ensure no new DMA is initiated after this point.\n+ * The mailbox channel is stopped by nbl_chan_teardown_queue()\n+ * before this function is called, so no in-flight mailbox DMA\n+ * remains.\n+ */\n+static void nbl_hw_deinit_module(struct nbl_hw_mgt *hw_mgt)\n+{\n+\tnbl_hw_set_driver_status(hw_mgt, false);\n+\t/* ensure driver_status reaches the chip */\n+\tnbl_flush_writes(hw_mgt);\n+\t/*\n+\t * Give the asynchronous firmware cleanup pass time to settle\n+\t * before returning; best-effort, see NBL_FW_CLEANUP_SYNC_MIN_US.\n+\t */\n+\tusleep_range(NBL_FW_CLEANUP_SYNC_MIN_US, NBL_FW_CLEANUP_SYNC_MAX_US);\n+}\n+\n+static bool nbl_hw_eth_speed_valid(u8 speed)\n+{\n+\tswitch (speed) {\n+\tcase NBL_FW_PORT_SPEED_10G:\n+\tcase NBL_FW_PORT_SPEED_25G:\n+\tcase NBL_FW_PORT_SPEED_50G:\n+\tcase NBL_FW_PORT_SPEED_100G:\n+\t\treturn true;\n+\tdefault:\n+\t\treturn false;\n+\t}\n+}\n+\n+static bool nbl_hw_eth_num_valid(u8 eth_num)\n+{\n+\treturn eth_num == 1 || eth_num == 2 || eth_num == 4;\n+}\n+\n+/*\n+ * Clear a stale driver_status active bit before this probe programs any\n+ * per-PF state.\n+ *\n+ * A previous kernel instance can leave the bit set (kexec, or an unload\n+ * that never reached deinit_module()) and firmware then never ran the\n+ * per-PF cleanup it drives off the 1 -\u003e 0 transition.  That cleanup\n+ * reclaims the mailbox QINFO routing and the qinfo registers, which is\n+ * exactly what nbl_dev_setup_chan_qinfo() and nbl_chan_setup_queue()\n+ * write later in this probe.\n+ *\n+ * So the ordering this has to guarantee is: clear the bit before any of\n+ * that per-PF state is programmed, and before the mailbox queue is live\n+ * so the cleanup cannot reclaim routing under an in-flight transfer.\n+ * Clearing it later - once nbl_dev_init() has programmed the routing and\n+ * enabled the queue - wipes those registers and leaves the inter-PF\n+ * mailbox dead, so sibling PFs' RPCs time out in nbl_dev_start() and\n+ * their probes fail.  That is why init_module() must not do it.\n+ *\n+ * Only the control PF owns this chip-wide handshake.  The bounded wait\n+ * after the clear is best-effort, see NBL_FW_CLEANUP_SYNC_MIN_US.\n+ */\n+static void nbl_hw_clear_stale_status(struct nbl_hw_mgt *hw_mgt)\n+{\n+\tu32 status;\n+\n+\tif (!hw_mgt-\u003ecommon-\u003ehas_ctrl)\n+\t\treturn;\n+\n+\t/*\n+\t * Unlocked read: this runs only from the serialized control-PF probe,\n+\t * and the driver is the only writer of NBL_DRIVER_STATUS_BIT.  A stale\n+\t * sample at worst causes one redundant clear plus sleep; the\n+\t * authoritative update happens under reg_lock in\n+\t * nbl_hw_set_driver_status().\n+\t */\n+\tstatus = nbl_hw_rd32(hw_mgt, NBL_DRIVER_STATUS_REG);\n+\tif (!(status \u0026 BIT(NBL_DRIVER_STATUS_BIT)))\n+\t\treturn;\n+\n+\tdev_warn(hw_mgt-\u003ecommon-\u003edev,\n+\t\t \"driver_status already set, stale cleanup forced\\n\");\n+\tnbl_hw_set_driver_status(hw_mgt, false);\n+\tnbl_flush_writes(hw_mgt);\n+\tusleep_range(NBL_FW_CLEANUP_SYNC_MIN_US, NBL_FW_CLEANUP_SYNC_MAX_US);\n+}\n+\n+/*\n+ * Firmware brings the chip up at power-up, but the chip-wide datapath\n+ * registers this programs (SHAPING, DSCH, USTORE/DSTORE, DVN/UVN, UQM,\n+ * PADPT) are not part of that: the driver owns them from here until chip\n+ * reset, and firmware does not revert them when driver_status goes low -\n+ * see nbl_hw_deinit_module().\n+ *\n+ * The driver_status handshake is entered from a known state: a stale\n+ * active bit is cleared earlier, in nbl_hw_init_leonis(), before this\n+ * probe programs any per-PF state.  It must not be cleared here, because\n+ * the mailbox QINFO routing and the mailbox queue registers are already\n+ * programmed by then and the cleanup the 1 -\u003e 0 transition starts would\n+ * reclaim them; see nbl_hw_clear_stale_status().\n+ */\n+static int nbl_hw_init_module(struct nbl_hw_mgt *hw_mgt, u8 eth_speed,\n+\t\t\t      u8 eth_num)\n+{\n+\tint ret;\n+\n+\tif (!nbl_hw_eth_speed_valid(eth_speed)) {\n+\t\tdev_err(hw_mgt-\u003ecommon-\u003edev, \"Invalid eth_speed %u\\n\",\n+\t\t\teth_speed);\n+\t\treturn -EINVAL;\n+\t}\n+\tif (!nbl_hw_eth_num_valid(eth_num)) {\n+\t\tdev_err(hw_mgt-\u003ecommon-\u003edev, \"Invalid eth_num %u\\n\", eth_num);\n+\t\treturn -EINVAL;\n+\t}\n+\n+\tret = nbl_dp_init(hw_mgt, eth_speed, eth_num);\n+\tif (ret)\n+\t\treturn ret;\n+\tnbl_intf_init(hw_mgt);\n+\tnbl_hw_set_driver_status(hw_mgt, true);\n+\t/* ensure registers written */\n+\tnbl_flush_writes(hw_mgt);\n+\n+\treturn 0;\n+}\n+\n+/*\n+ * nbl_hw_set_mailbox_irq - read-modify-write NBL_MAILBOX_QINFO_MAP_REG_ARR\n+ *\n+ * The full RMW sequence is wrapped by reg_lock, so concurrent register\n+ * access from different CPUs is already serialized safely.\n+ * nbl_hw_cfg_mailbox_qinfo() programs the BDF fields during control-PF\n+ * init and clears MSIX_IDX/MSIX_IDX_VALID at the same time (they survive\n+ * kexec/forced unload without FLR), so mailbox MSIX routing for a PF\n+ * starts disarmed at init and is armed only by an explicit en_msix=true\n+ * call here.\n+ */\n+static void nbl_hw_set_mailbox_irq(struct nbl_hw_mgt *hw_mgt, u16 func_id,\n+\t\t\t\t   bool en_msix, u16 gvec)\n+{\n+\tu32 data = 0;\n+\n+\tspin_lock(\u0026hw_mgt-\u003ereg_lock);\n+\tnbl_hw_rd_regs(hw_mgt, NBL_MAILBOX_QINFO_MAP_REG_ARR(func_id), \u0026data,\n+\t\t       sizeof(data));\n+\tdata \u0026= ~(NBL_MAILBOX_QINFO_MAP_MSIX_IDX_MASK |\n+\t\t  NBL_MAILBOX_QINFO_MAP_MSIX_IDX_VALID_MASK);\n+\tif (en_msix)\n+\t\tdata |= FIELD_PREP(NBL_MAILBOX_QINFO_MAP_MSIX_IDX_MASK,\n+\t\t\t\t   gvec) |\n+\t\t\tFIELD_PREP(NBL_MAILBOX_QINFO_MAP_MSIX_IDX_VALID_MASK,\n+\t\t\t\t   1);\n+\n+\tnbl_hw_wr_regs(hw_mgt, NBL_MAILBOX_QINFO_MAP_REG_ARR(func_id), \u0026data,\n+\t\t       sizeof(data));\n+\tspin_unlock(\u0026hw_mgt-\u003ereg_lock);\n+\tnbl_flush_writes(hw_mgt);\n+}\n+\n+static void nbl_hw_cfg_msix_map(struct nbl_hw_mgt *hw_mgt, u16 func_id,\n+\t\t\t\tbool valid, dma_addr_t dma_addr, u8 bus,\n+\t\t\t\tu8 devid, u8 function)\n+{\n+\tstruct nbl_function_msix_map function_msix_map;\n+\n+\tmemset(\u0026function_msix_map, 0, sizeof(function_msix_map));\n+\n+\t/*\n+\t * Clear VALID on its own first.  nbl_hw_wr_regs_lock() writes an\n+\t * entry in ascending dword order, so a single whole-entry write\n+\t * would publish the new (or zeroed) address in data[0]/data[1]\n+\t * while the old VALID bit in data[2] was still set, exposing a\n+\t * valid entry with a half-written address to the pcompleter's\n+\t * table fetch.  With VALID already zero only the address words\n+\t * change, and no fetch can be started from a torn entry.\n+\t */\n+\tnbl_hw_wr_regs_lock(hw_mgt,\n+\t\t\t    NBL_PCOMPLETER_FUNCTION_MSIX_MAP(func_id) +\n+\t\t\t    NBL_FUNC_MSIX_MAP_VALID_DW * sizeof(u32),\n+\t\t\t    \u0026function_msix_map.data[NBL_FUNC_MSIX_MAP_VALID_DW],\n+\t\t\t    sizeof(u32));\n+\n+\tif (valid) {\n+\t\t/* program full entry and set VALID */\n+\t\tfunction_msix_map.data[0] = lower_32_bits(dma_addr);\n+\t\tfunction_msix_map.data[1] = upper_32_bits(dma_addr);\n+\t\tfunction_msix_map.data[2] =\n+\t\t\tFIELD_PREP(NBL_FUNCTION_MSIX_MAP_FUNCTION_MASK,\n+\t\t\t\t   function) |\n+\t\t\tFIELD_PREP(NBL_FUNCTION_MSIX_MAP_DEVID_MASK, devid) |\n+\t\t\tFIELD_PREP(NBL_FUNCTION_MSIX_MAP_BUS_MASK, bus) |\n+\t\t\tFIELD_PREP(NBL_FUNCTION_MSIX_MAP_VALID_MASK, 1);\n+\n+\t\tnbl_hw_wr_regs_lock(hw_mgt,\n+\t\t\t\t    NBL_PCOMPLETER_FUNCTION_MSIX_MAP(func_id),\n+\t\t\t\t    function_msix_map.data,\n+\t\t\t\t    sizeof(function_msix_map));\n+\t} else {\n+\t\t/*\n+\t\t * reg_lock prevents concurrent CPU writes to the same\n+\t\t * function's MSIX entry, but cannot synchronize hardware DMA\n+\t\t * reads. Upper layer uses two-stage destruction + sync sleep\n+\t\t * to avoid torn hardware read of partial MSIX entry.\n+\t\t * VALID is already cleared above, so the address written\n+\t\t * here is either the still-live table (prepare) or zero\n+\t\t * (complete); data[2] stays zero, leaving the BDF fields\n+\t\t * unused because the entry cannot be fetched.\n+\t\t */\n+\t\tfunction_msix_map.data[0] = lower_32_bits(dma_addr);\n+\t\tfunction_msix_map.data[1] = upper_32_bits(dma_addr);\n+\n+\t\tnbl_hw_wr_regs_lock(hw_mgt,\n+\t\t\t\t    NBL_PCOMPLETER_FUNCTION_MSIX_MAP(func_id),\n+\t\t\t\t    function_msix_map.data,\n+\t\t\t\t    sizeof(function_msix_map));\n+\t}\n+}\n+\n+static void nbl_hw_cfg_msix_info(struct nbl_hw_mgt *hw_mgt, u16 func_id,\n+\t\t\t\t bool valid, u16 interrupt_id, u8 bus,\n+\t\t\t\t u8 devid, u8 function, bool msix_mask_en)\n+{\n+\tu32 host_msix_fid = 0;\n+\tstruct nbl_host_msix_info msix_info;\n+\n+\tmemset(\u0026msix_info, 0, sizeof(msix_info));\n+\tif (valid) {\n+\t\thost_msix_fid =\n+\t\t\tFIELD_PREP(NBL_PCOMPLETER_HOST_MSIX_FID_TABLE_FID_MASK,\n+\t\t\t\t   func_id) |\n+\t\t\tFIELD_PREP(NBL_PCOMPLETER_HOST_MSIX_FID_TABLE_VLD_MASK,\n+\t\t\t\t   1);\n+\n+\t\tmsix_info.data[1] =\n+\t\t\tFIELD_PREP(NBL_HOST_MSIX_INFO_FUNCTION_MASK, function) |\n+\t\t\tFIELD_PREP(NBL_HOST_MSIX_INFO_DEVID_MASK, devid) |\n+\t\t\tFIELD_PREP(NBL_HOST_MSIX_INFO_BUS_MASK, bus) |\n+\t\t\tFIELD_PREP(NBL_HOST_MSIX_INFO_VALID_MASK, 1);\n+\n+\t\tif (msix_mask_en)\n+\t\t\tmsix_info.data[1] |=\n+\t\t\tFIELD_PREP(NBL_HOST_MSIX_INFO_MSIX_MASK_EN_MASK, 1);\n+\t}\n+\tspin_lock(\u0026hw_mgt-\u003ereg_lock);\n+\t/*\n+\t * Programming order rule:\n+\t * Enable: PADPT_HOST_MSIX_INFO -\u003e PCOMPLETER_HOST_MSIX_FID_TABLE\n+\t * Teardown: reverse order, clear FID VLD first to avoid inconsistent\n+\t * state\n+\t */\n+\tif (valid) {\n+\t\tnbl_hw_wr_regs(hw_mgt,\n+\t\t\t       NBL_PADPT_HOST_MSIX_INFO_REG_ARR(interrupt_id),\n+\t\t\t       msix_info.data, sizeof(msix_info));\n+\t\tnbl_hw_wr_regs(hw_mgt,\n+\t\t\t       NBL_PCOMPLETER_HOST_MSIX_FID_TABLE(interrupt_id),\n+\t\t\t       \u0026host_msix_fid, sizeof(host_msix_fid));\n+\t} else {\n+\t\tnbl_hw_wr_regs(hw_mgt,\n+\t\t\t       NBL_PCOMPLETER_HOST_MSIX_FID_TABLE(interrupt_id),\n+\t\t\t       \u0026host_msix_fid, sizeof(host_msix_fid));\n+\t\tnbl_hw_wr_regs(hw_mgt,\n+\t\t\t       NBL_PADPT_HOST_MSIX_INFO_REG_ARR(interrupt_id),\n+\t\t\t       msix_info.data, sizeof(msix_info));\n+\t}\n+\tspin_unlock(\u0026hw_mgt-\u003ereg_lock);\n+}\n+\n+static void nbl_hw_update_mailbox_queue_tail_ptr(struct nbl_hw_mgt *hw_mgt,\n+\t\t\t\t\t\t u16 tail_ptr, u8 txrx)\n+{\n+\t/* local_qid 0 and 1 denote rx and tx queue respectively */\n+\tu32 local_qid = txrx;\n+\tu32 value = ((u32)tail_ptr \u003c\u003c 16) | local_qid;\n+\n+\t/* wmb for doorbell */\n+\twmb();\n+\tnbl_mbx_wr32(hw_mgt, NBL_MAILBOX_NOTIFY_ADDR, value);\n+}\n+\n+/*\n+ * Assert QUEUE_RST on a QINFO cfg entry, in a write of its own.\n+ *\n+ * The entry is four dwords and the device may act on each one as it\n+ * lands, while data[3] carries QUEUE_RST/QUEUE_EN and data[0..2] carry\n+ * the ring base and size.  Writing a new base while QUEUE_EN is still\n+ * set from the previous configuration would have the queue fetch at a\n+ * half-updated address - IOVA 0, once data[0..2] have been cleared -\n+ * so the reset always goes in first, on its own, and the remaining\n+ * dwords are written afterwards.  This also covers a device that only\n+ * commits the entry when data[3] lands.\n+ */\n+static void nbl_hw_reset_mailbox_qinfo(struct nbl_hw_mgt *hw_mgt, u64 reg)\n+{\n+\tu32 rst = FIELD_PREP(NBL_MAILBOX_QINFO_CFG_QUEUE_RST_MASK, 1);\n+\n+\tnbl_hw_write_mbx_regs(hw_mgt,\n+\t\t\t      reg + (MAILBOX_QINFO_CFG_TABLE_DWLEN - 1) *\n+\t\t\t\t    sizeof(u32),\n+\t\t\t      \u0026rst, sizeof(rst));\n+}\n+\n+static void nbl_hw_config_mailbox_rxq(struct nbl_hw_mgt *hw_mgt,\n+\t\t\t\t      dma_addr_t dma_addr, int size_bwid)\n+{\n+\tstruct nbl_mailbox_qinfo_cfg_table cfg_tbl;\n+\n+\t/* Hold the queue in reset before its base/size are reprogrammed */\n+\tnbl_hw_reset_mailbox_qinfo(hw_mgt, NBL_MAILBOX_QINFO_CFG_RX_TABLE_ADDR);\n+\n+\tcfg_tbl.data[0] = lower_32_bits(dma_addr);\n+\tcfg_tbl.data[1] = upper_32_bits(dma_addr);\n+\tcfg_tbl.data[2] = FIELD_PREP(NBL_MAILBOX_QINFO_CFG_QUEUE_SIZE_BWID_MASK,\n+\t\t\t\t     size_bwid);\n+\tcfg_tbl.data[3] = FIELD_PREP(NBL_MAILBOX_QINFO_CFG_QUEUE_RST_MASK, 0) |\n+\t\t\t  FIELD_PREP(NBL_MAILBOX_QINFO_CFG_QUEUE_EN_MASK, 1);\n+\t/*\n+\t * data[0..2] land while QUEUE_RST is still set from the write above;\n+\t * data[3] is written last and starts the queue on the new ring.\n+\t */\n+\tnbl_hw_write_mbx_regs(hw_mgt, NBL_MAILBOX_QINFO_CFG_RX_TABLE_ADDR,\n+\t\t\t      cfg_tbl.data, sizeof(cfg_tbl));\n+}\n+\n+static void nbl_hw_config_mailbox_txq(struct nbl_hw_mgt *hw_mgt,\n+\t\t\t\t      dma_addr_t dma_addr, int size_bwid)\n+{\n+\tstruct nbl_mailbox_qinfo_cfg_table cfg_tbl;\n+\n+\t/* Hold the queue in reset before its base/size are reprogrammed */\n+\tnbl_hw_reset_mailbox_qinfo(hw_mgt, NBL_MAILBOX_QINFO_CFG_TX_TABLE_ADDR);\n+\n+\tcfg_tbl.data[0] = lower_32_bits(dma_addr);\n+\tcfg_tbl.data[1] = upper_32_bits(dma_addr);\n+\tcfg_tbl.data[2] = FIELD_PREP(NBL_MAILBOX_QINFO_CFG_QUEUE_SIZE_BWID_MASK,\n+\t\t\t\t     size_bwid);\n+\tcfg_tbl.data[3] = FIELD_PREP(NBL_MAILBOX_QINFO_CFG_QUEUE_RST_MASK, 0) |\n+\t\t\t  FIELD_PREP(NBL_MAILBOX_QINFO_CFG_QUEUE_EN_MASK, 1);\n+\t/*\n+\t * data[0..2] land while QUEUE_RST is still set from the write above;\n+\t * data[3] is written last and starts the queue on the new ring.\n+\t */\n+\tnbl_hw_write_mbx_regs(hw_mgt, NBL_MAILBOX_QINFO_CFG_TX_TABLE_ADDR,\n+\t\t\t      cfg_tbl.data, sizeof(cfg_tbl));\n+}\n+\n+static void nbl_hw_stop_mailbox_rxq(struct nbl_hw_mgt *hw_mgt)\n+{\n+\tstruct nbl_mailbox_qinfo_cfg_table cfg_tbl;\n+\n+\t/*\n+\t * Assert QUEUE_RST before data[0..2] are cleared: writing the\n+\t * zeroed entry in one go would clear the ring base while QUEUE_EN\n+\t * is still set.  The reset also clears QUEUE_EN, so what follows\n+\t * lands on an already stopped queue.\n+\t */\n+\tnbl_hw_reset_mailbox_qinfo(hw_mgt, NBL_MAILBOX_QINFO_CFG_RX_TABLE_ADDR);\n+\n+\t/* Now drop the stale base and size, keeping QUEUE_RST asserted */\n+\tmemset(\u0026cfg_tbl, 0, sizeof(cfg_tbl));\n+\tcfg_tbl.data[3] = FIELD_PREP(NBL_MAILBOX_QINFO_CFG_QUEUE_RST_MASK, 1);\n+\tnbl_hw_write_mbx_regs(hw_mgt, NBL_MAILBOX_QINFO_CFG_RX_TABLE_ADDR,\n+\t\t\t      cfg_tbl.data, sizeof(cfg_tbl));\n+\t/* Ensure QUEUE_RST has reached the device before caller proceeds */\n+\tnbl_hw_flush_mbx_write(hw_mgt,\n+\t\t\t       NBL_MAILBOX_QINFO_CFG_RX_TABLE_ADDR);\n+}\n+\n+static void nbl_hw_stop_mailbox_txq(struct nbl_hw_mgt *hw_mgt)\n+{\n+\tstruct nbl_mailbox_qinfo_cfg_table cfg_tbl;\n+\n+\t/*\n+\t * Assert QUEUE_RST before data[0..2] are cleared: writing the\n+\t * zeroed entry in one go would clear the ring base while QUEUE_EN\n+\t * is still set.  The reset also clears QUEUE_EN, so what follows\n+\t * lands on an already stopped queue.\n+\t */\n+\tnbl_hw_reset_mailbox_qinfo(hw_mgt, NBL_MAILBOX_QINFO_CFG_TX_TABLE_ADDR);\n+\n+\t/* Now drop the stale base and size, keeping QUEUE_RST asserted */\n+\tmemset(\u0026cfg_tbl, 0, sizeof(cfg_tbl));\n+\tcfg_tbl.data[3] = FIELD_PREP(NBL_MAILBOX_QINFO_CFG_QUEUE_RST_MASK, 1);\n+\tnbl_hw_write_mbx_regs(hw_mgt, NBL_MAILBOX_QINFO_CFG_TX_TABLE_ADDR,\n+\t\t\t      cfg_tbl.data, sizeof(cfg_tbl));\n+\t/* Ensure QUEUE_RST has reached the device before caller proceeds */\n+\tnbl_hw_flush_mbx_write(hw_mgt,\n+\t\t\t       NBL_MAILBOX_QINFO_CFG_TX_TABLE_ADDR);\n+}\n+\n+static void nbl_hw_get_host_pf_mask(struct nbl_hw_mgt *hw_mgt, u32 *pf_mask)\n+{\n+\tnbl_hw_rd_regs_lock(hw_mgt, NBL_PCIE_HOST_K_PF_MASK_REG, pf_mask,\n+\t\t\t    sizeof(*pf_mask));\n+}\n+\n+static void nbl_hw_get_real_bus(struct nbl_hw_mgt *hw_mgt, u8 *bus)\n+{\n+\tu32 data = 0;\n+\n+\tnbl_hw_rd_regs_lock(hw_mgt, NBL_PCIE_HOST_TL_CFG_BUSDEV, \u0026data,\n+\t\t\t    sizeof(data));\n+\t*bus = FIELD_GET(NBL_PCIE_BUS_MASK, data);\n+}\n+\n+static void nbl_hw_cfg_mailbox_qinfo(struct nbl_hw_mgt *hw_mgt, u16 func_id,\n+\t\t\t\t     u8 bus, u8 devid, u8 function)\n+{\n+\tu32 data = 0;\n+\n+\tspin_lock(\u0026hw_mgt-\u003ereg_lock);\n+\tnbl_hw_rd_regs(hw_mgt, NBL_MAILBOX_QINFO_MAP_REG_ARR(func_id),\n+\t\t       \u0026data, sizeof(data));\n+\tdata \u0026= ~(NBL_MAILBOX_QINFO_MAP_FUNCTION_MASK |\n+\t\t  NBL_MAILBOX_QINFO_MAP_DEVID_MASK |\n+\t\t  NBL_MAILBOX_QINFO_MAP_BUS_MASK |\n+\t\t  NBL_MAILBOX_QINFO_MAP_MSIX_IDX_MASK |\n+\t\t  NBL_MAILBOX_QINFO_MAP_MSIX_IDX_VALID_MASK);\n+\tdata |= FIELD_PREP(NBL_MAILBOX_QINFO_MAP_FUNCTION_MASK, function) |\n+\t       FIELD_PREP(NBL_MAILBOX_QINFO_MAP_DEVID_MASK, devid) |\n+\t       FIELD_PREP(NBL_MAILBOX_QINFO_MAP_BUS_MASK, bus);\n+\tnbl_hw_wr_regs(hw_mgt, NBL_MAILBOX_QINFO_MAP_REG_ARR(func_id),\n+\t\t       \u0026data, sizeof(data));\n+\tspin_unlock(\u0026hw_mgt-\u003ereg_lock);\n+}\n+\n+/*\n+ * Registers reset to zero after cold boot / FLR / bus reset. Firmware\n+ * programs valid values before driver probe, so zero is only seen on\n+ * hardware fault or register read failure. Initialize data=0 to guard\n+ * against nbl_hw_read_mbx_regs() early-return on bounds-check failure.\n+ */\n+static void nbl_hw_get_board_info(struct nbl_hw_mgt *hw_mgt,\n+\t\t\t\t  struct nbl_board_port_info *board_info)\n+{\n+\tu32 data = 0;\n+\n+\tnbl_hw_read_mbx_regs(hw_mgt, NBL_FW_BOARD_DW3_OFFSET, \u0026data,\n+\t\t\t     sizeof(data));\n+\tboard_info-\u003eeth_num = FIELD_GET(NBL_FW_BOARD_DW3_PORT_NUM_MASK, data);\n+\tboard_info-\u003eeth_speed =\n+\t\tFIELD_GET(NBL_FW_BOARD_DW3_PORT_SPEED_MASK, data);\n+\tboard_info-\u003ep4_version =\n+\t\tFIELD_GET(NBL_FW_BOARD_DW3_P4_VERSION_MASK, data);\n+}\n+\n+static struct nbl_hw_ops hw_ops = {\n+\t.init_module = nbl_hw_init_module,\n+\t.deinit_module = nbl_hw_deinit_module,\n+\n+\t.cfg_msix_map = nbl_hw_cfg_msix_map,\n+\t.cfg_msix_info = nbl_hw_cfg_msix_info,\n+\t.flush_write = nbl_flush_writes,\n+\n+\t.update_mailbox_queue_tail_ptr = nbl_hw_update_mailbox_queue_tail_ptr,\n+\t.config_mailbox_rxq = nbl_hw_config_mailbox_rxq,\n+\t.config_mailbox_txq = nbl_hw_config_mailbox_txq,\n+\t.stop_mailbox_rxq = nbl_hw_stop_mailbox_rxq,\n+\t.stop_mailbox_txq = nbl_hw_stop_mailbox_txq,\n+\t.get_host_pf_mask = nbl_hw_get_host_pf_mask,\n+\t.get_real_bus = nbl_hw_get_real_bus,\n+\n+\t.cfg_mailbox_qinfo = nbl_hw_cfg_mailbox_qinfo,\n+\t.set_mailbox_irq = nbl_hw_set_mailbox_irq,\n+\n+\t.get_fw_eth_map = nbl_hw_get_fw_eth_map,\n+\t.get_board_info = nbl_hw_get_board_info,\n+};\n+\n /* Structure starts here, adding an op should not modify anything below */\n static struct nbl_hw_mgt *nbl_hw_setup_hw_mgt(struct nbl_common_info *common)\n {\n@@ -23,6 +1246,31 @@ static struct nbl_hw_mgt *nbl_hw_setup_hw_mgt(struct nbl_common_info *common)\n \treturn hw_mgt;\n }\n \n+static struct nbl_hw_ops_tbl *nbl_hw_setup_ops(struct nbl_common_info *common,\n+\t\t\t\t\t       struct nbl_hw_mgt *hw_mgt)\n+{\n+\tstruct nbl_hw_ops_tbl *hw_ops_tbl;\n+\tstruct device *dev;\n+\n+\tdev = common-\u003edev;\n+\thw_ops_tbl = devm_kzalloc(dev, sizeof(*hw_ops_tbl), GFP_KERNEL);\n+\tif (!hw_ops_tbl)\n+\t\treturn ERR_PTR(-ENOMEM);\n+\tif (!hw_ops.cfg_msix_map || !hw_ops.cfg_msix_info ||\n+\t    !hw_ops.flush_write || !hw_ops.update_mailbox_queue_tail_ptr ||\n+\t    !hw_ops.config_mailbox_rxq || !hw_ops.config_mailbox_txq ||\n+\t    !hw_ops.stop_mailbox_rxq || !hw_ops.stop_mailbox_txq ||\n+\t    !hw_ops.get_host_pf_mask || !hw_ops.get_real_bus ||\n+\t    !hw_ops.cfg_mailbox_qinfo || !hw_ops.set_mailbox_irq ||\n+\t    !hw_ops.get_fw_eth_map || !hw_ops.get_board_info ||\n+\t    !hw_ops.init_module || !hw_ops.deinit_module)\n+\t\treturn ERR_PTR(-EINVAL);\n+\thw_ops_tbl-\u003eops = \u0026hw_ops;\n+\thw_ops_tbl-\u003epriv = hw_mgt;\n+\n+\treturn hw_ops_tbl;\n+}\n+\n static int nbl_pcim_request_selected_bars(struct pci_dev *pdev, u32 mask,\n \t\t\t\t\t  const char *name)\n {\n@@ -43,6 +1291,7 @@ int nbl_hw_init_leonis(struct nbl_adapter *adapter)\n {\n \tresource_size_t expect_sz = NBL_MEM_BAR_TOTAL_SIZE;\n \tstruct nbl_common_info *common = \u0026adapter-\u003ecommon;\n+\tstruct nbl_hw_ops_tbl *hw_ops_tbl = NULL;\n \tstruct pci_dev *pdev = common-\u003epdev;\n \tstruct nbl_hw_mgt *hw_mgt = NULL;\n \tresource_size_t bar_len;\n@@ -53,7 +1302,7 @@ int nbl_hw_init_leonis(struct nbl_adapter *adapter)\n \thw_mgt = nbl_hw_setup_hw_mgt(common);\n \tif (IS_ERR(hw_mgt)) {\n \t\tret = PTR_ERR(hw_mgt);\n-\t\tgoto setup_mgt_fail;\n+\t\treturn ret;\n \t}\n \tbar_mask = BIT(NBL_MEMORY_BAR) | BIT(NBL_MAILBOX_BAR);\n \tret = nbl_pcim_request_selected_bars(pdev, bar_mask, NBL_DRIVER_NAME);\n@@ -61,14 +1310,14 @@ int nbl_hw_init_leonis(struct nbl_adapter *adapter)\n \t\tdev_err(\u0026pdev-\u003edev,\n \t\t\t\"Request memory bar failed, err = %d\\n\",\n \t\t\tret);\n-\t\tgoto setup_mgt_fail;\n+\t\treturn ret;\n \t}\n \n \tbar_len = pci_resource_len(pdev, NBL_MEMORY_BAR);\n \tif (!(pci_resource_flags(pdev, NBL_MEMORY_BAR) \u0026 IORESOURCE_MEM)) {\n \t\tdev_err(\u0026pdev-\u003edev, \"MEMORY BAR is not memory resource\\n\");\n \t\tret = -EINVAL;\n-\t\tgoto setup_mgt_fail;\n+\t\treturn ret;\n \t}\n \tif (common-\u003ehas_ctrl) {\n \t\t/*\n@@ -90,7 +1339,7 @@ int nbl_hw_init_leonis(struct nbl_adapter *adapter)\n \t\t\t\t\"MEMORY BAR len %pa smaller than expected %pa\\n\",\n \t\t\t\t\u0026bar_len, \u0026expect_sz);\n \t\t\tret = -EINVAL;\n-\t\t\tgoto setup_mgt_fail;\n+\t\t\treturn ret;\n \t\t}\n \t\thw_size = PAGE_ALIGN_DOWN(NBL_MEM_BAR_TOTAL_SIZE -\n \t\t\t\t\t  NBL_RDMA_NOTIFY_LEN);\n@@ -103,7 +1352,7 @@ int nbl_hw_init_leonis(struct nbl_adapter *adapter)\n \t\t\t\t\"MEMORY BAR len %pa too small for net only reg space\\n\",\n \t\t\t\t\u0026bar_len);\n \t\t\tret = -EINVAL;\n-\t\t\tgoto setup_mgt_fail;\n+\t\t\treturn ret;\n \t\t}\n \t\thw_size = NBL_REG_NET_ONLY_LEN;\n \t\thw_mgt-\u003ehw_addr = pcim_iomap(pdev, NBL_MEMORY_BAR,\n@@ -112,37 +1361,50 @@ int nbl_hw_init_leonis(struct nbl_adapter *adapter)\n \tif (!hw_mgt-\u003ehw_addr) {\n \t\tdev_err(\u0026pdev-\u003edev, \"MEMORY BAR pcim_iomap failed\\n\");\n \t\tret = -EIO;\n-\t\tgoto setup_mgt_fail;\n+\t\treturn ret;\n \t}\n \n \tbar_len = pci_resource_len(pdev, NBL_MAILBOX_BAR);\n \tif (!(pci_resource_flags(pdev, NBL_MAILBOX_BAR) \u0026 IORESOURCE_MEM)) {\n \t\tdev_err(\u0026pdev-\u003edev, \"MAILBOX BAR is not memory resource\\n\");\n \t\tret = -EINVAL;\n-\t\tgoto setup_mgt_fail;\n+\t\treturn ret;\n \t}\n \tif (bar_len \u003c NBL_BAR2_MAX_LEN) {\n \t\tdev_err(\u0026pdev-\u003edev, \"MAILBOX BAR length %pa too small\\n\",\n \t\t\t\u0026bar_len);\n \t\tret = -EINVAL;\n-\t\tgoto setup_mgt_fail;\n+\t\treturn ret;\n \t}\n \thw_mgt-\u003emailbox_bar_hw_addr = pcim_iomap(pdev, NBL_MAILBOX_BAR,\n \t\t\t\t\t\t bar_len);\n \tif (!hw_mgt-\u003emailbox_bar_hw_addr) {\n \t\tdev_err(\u0026pdev-\u003edev, \"MAILBOX BAR pcim_iomap failed\\n\");\n \t\tret = -EIO;\n-\t\tgoto setup_mgt_fail;\n+\t\treturn ret;\n \t}\n \n \thw_mgt-\u003emailbox_bar_size = bar_len;\n+\tspin_lock_init(\u0026hw_mgt-\u003ereg_lock);\n \n+\t/*\n+\t * Clear a stale driver_status now, while this is the only register\n+\t * access this probe has done: the firmware cleanup the 1 -\u003e 0\n+\t * transition starts reclaims the mailbox QINFO routing and qinfo\n+\t * registers that nbl_dev_init() programs later, so it must not be\n+\t * deferred to init_module(); see nbl_hw_clear_stale_status().\n+\t */\n+\tnbl_hw_clear_stale_status(hw_mgt);\n+\n+\thw_ops_tbl = nbl_hw_setup_ops(common, hw_mgt);\n+\tif (IS_ERR(hw_ops_tbl)) {\n+\t\tret = PTR_ERR(hw_ops_tbl);\n+\t\treturn ret;\n+\t}\n+\tadapter-\u003eintf.hw_ops_tbl = hw_ops_tbl;\n \tadapter-\u003ecore.hw_mgt = hw_mgt;\n \n \treturn 0;\n-\n-setup_mgt_fail:\n-\treturn ret;\n }\n \n void nbl_hw_remove_leonis(struct nbl_adapter *adapter)\ndiff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_leonis/nbl_hw_leonis.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_leonis/nbl_hw_leonis.h\nindex 1f9e509dc6312f..7d27ab19838e21 100644\n--- a/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_leonis/nbl_hw_leonis.h\n+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_leonis/nbl_hw_leonis.h\n@@ -11,5 +11,347 @@\n #include \"../../nbl_include/nbl_include.h\"\n #include \"../nbl_hw_reg.h\"\n \n+/*\n+ * Chip-wide driver-present handshake with firmware.\n+ *\n+ * NBL_DRIVER_STATUS_BIT is the only bit the driver owns: setting it tells\n+ * firmware that a driver is bound, and the 1 -\u003e 0 transition starts the\n+ * firmware pass that reclaims per-PF state (mailbox QINFO routing, qinfo\n+ * registers).  Every other bit of this register belongs to firmware and\n+ * the driver must leave it as it read it.\n+ *\n+ * Because the register is shared, bit 16 can only be changed with a\n+ * read-modify-write: it has no documented W1S/W1C semantics that would\n+ * let the driver set or clear that one bit atomically.  reg_lock makes\n+ * the read-modify-write atomic with respect to other CPU writers, but it\n+ * cannot serialise against firmware, so a firmware update to another bit\n+ * landing between the read and the write would be written back stale.\n+ * No such firmware update is known to race the three driver callers\n+ * (the probe-time clear, init_module() and deinit_module()), but that is\n+ * an observation about this hardware revision, not a guarantee the\n+ * driver can enforce.\n+ */\n+#define NBL_DRIVER_STATUS_REG\t\t\t0x1300444\n+#define NBL_DRIVER_STATUS_BIT\t\t\t16\n+\n+/*  ----------  REG BASE ADDR  ----------  */\n+/* Interface modules base addr */\n+#define NBL_INTF_HOST_PCOMPLETER_BASE\t\t0x00f08000\n+#define NBL_INTF_HOST_PADPT_BASE\t\t0x00f4c000\n+#define NBL_INTF_HOST_MAILBOX_BASE\t\t0x00fb0000\n+#define NBL_INTF_HOST_PCIE_BASE\t\t\t0X01504000\n+/* DP modules base addr */\n+#define NBL_DP_USTORE_BASE\t\t\t0x00104000\n+#define NBL_DP_UQM_BASE\t\t\t\t0x00114000\n+#define NBL_DP_UPED_BASE\t\t\t0x0015c000\n+#define NBL_DP_UVN_BASE\t\t\t\t0x00244000\n+#define NBL_DP_DSCH_BASE\t\t\t0x00404000\n+#define NBL_DP_SHAPING_BASE\t\t\t0x00504000\n+#define NBL_DP_DVN_BASE\t\t\t\t0x00514000\n+#define NBL_DP_DSTORE_BASE\t\t\t0x00704000\n+#define NBL_DP_DPED_BASE\t\t\t0x0075c000\n+/*  --------  MAILBOX BAR2 -----  */\n+#define NBL_MAILBOX_NOTIFY_ADDR\t\t\t0x00000000\n+#define NBL_MAILBOX_QINFO_CFG_RX_TABLE_ADDR\t0x10\n+#define NBL_MAILBOX_QINFO_CFG_TX_TABLE_ADDR\t0x20\n+\n+/*  --------  MAILBOX  --------  */\n+\n+/* mailbox BAR qinfo_cfg_table */\n+#define MAILBOX_QINFO_CFG_TABLE_DWLEN\t4\n+/* data[2] */\n+#define NBL_MAILBOX_QINFO_CFG_QUEUE_SIZE_BWID_MASK\tGENMASK(3, 0)\n+/* data[3] */\n+#define NBL_MAILBOX_QINFO_CFG_QUEUE_RST_MASK\t\tBIT(0)\n+#define NBL_MAILBOX_QINFO_CFG_QUEUE_EN_MASK\t\tBIT(1)\n+#define NBL_MAILBOX_QINFO_CFG_DIF_ERR_MASK\t\tBIT(2)\n+#define NBL_MAILBOX_QINFO_CFG_PTR_ERR_MASK\t\tBIT(3)\n+struct nbl_mailbox_qinfo_cfg_table {\n+\tu32 data[MAILBOX_QINFO_CFG_TABLE_DWLEN];\n+};\n+\n+/*  --------  MAILBOX BAR0 -----  */\n+/* mailbox qinfo_map_table */\n+#define NBL_MAILBOX_QINFO_MAP_REG_ARR(func_id) \\\n+\t(NBL_INTF_HOST_MAILBOX_BASE + 0x00001000 + (func_id) * sizeof(u32))\n+\n+/* MAILBOX qinfo_map_table */\n+#define NBL_MAILBOX_QINFO_MAP_FUNCTION_MASK\t\tGENMASK(2, 0)\n+#define NBL_MAILBOX_QINFO_MAP_DEVID_MASK\t\tGENMASK(7, 3)\n+#define NBL_MAILBOX_QINFO_MAP_BUS_MASK\t\t\tGENMASK(15, 8)\n+#define NBL_MAILBOX_QINFO_MAP_MSIX_IDX_MASK\t\tGENMASK(28, 16)\n+#define NBL_MAILBOX_QINFO_MAP_MSIX_IDX_VALID_MASK\tBIT(29)\n+\n+/*  --------  HOST_PCIE  --------  */\n+#define NBL_PCIE_HOST_K_PF_MASK_REG (NBL_INTF_HOST_PCIE_BASE + 0x00001004)\n+#define NBL_PCIE_HOST_TL_CFG_BUSDEV (NBL_INTF_HOST_PCIE_BASE + 0x11040)\n+\n+#define NBL_PCIE_BUS_MASK\tGENMASK(12, 5)\n+\n+/*  --------  HOST_PADPT  --------  */\n+#define NBL_HOST_PADPT_HOST_CFG_FC_PD_DN (NBL_INTF_HOST_PADPT_BASE + 0x00000160)\n+#define NBL_HOST_PADPT_HOST_CFG_FC_PH_DN (NBL_INTF_HOST_PADPT_BASE + 0x00000164)\n+#define NBL_HOST_PADPT_HOST_CFG_FC_NPH_DN \\\n+\t(NBL_INTF_HOST_PADPT_BASE + 0x0000016C)\n+#define NBL_HOST_PADPT_HOST_CFG_FC_CPLH_UP \\\n+\t(NBL_INTF_HOST_PADPT_BASE + 0x00000170)\n+\n+#define NBL_HOST_PADPT_CFG_FC_CPLH_UP_VAL      0x10400\n+#define NBL_HOST_PADPT_CFG_FC_PD_DN_VAL        0x10080\n+#define NBL_HOST_PADPT_CFG_FC_PH_DN_VAL        0x10010\n+#define NBL_HOST_PADPT_CFG_FC_NPH_DN_VAL       0x10010\n+/* host_padpt host_msix_info */\n+#define NBL_PADPT_HOST_MSIX_INFO_REG_ARR(vector_id) \\\n+\t(NBL_INTF_HOST_PADPT_BASE + 0x00010000 +    \\\n+\t (vector_id) * sizeof(struct nbl_host_msix_info))\n+\n+#define NBL_HOST_MSIX_INFO_DWLEN\t2\n+/* data[0] */\n+#define NBL_HOST_MSIX_INFO_INTRL_PNUM_MASK GENMASK(15, 0)\n+#define NBL_HOST_MSIX_INFO_INTRL_RATE_MASK GENMASK(31, 16)\n+/* data[1] */\n+#define NBL_HOST_MSIX_INFO_FUNCTION_MASK GENMASK(2, 0)\n+#define NBL_HOST_MSIX_INFO_DEVID_MASK GENMASK(7, 3)\n+#define NBL_HOST_MSIX_INFO_BUS_MASK GENMASK(15, 8)\n+#define NBL_HOST_MSIX_INFO_VALID_MASK BIT(16)\n+#define NBL_HOST_MSIX_INFO_MSIX_MASK_EN_MASK BIT(17)\n+struct nbl_host_msix_info {\n+\tu32 data[NBL_HOST_MSIX_INFO_DWLEN];\n+};\n+\n+/*  --------  HOST_PCOMPLETER  --------  */\n+/* pcompleter_host function_msix_map_table */\n+#define NBL_PCOMPLETER_FUNCTION_MSIX_MAP(i)   \\\n+\t(NBL_INTF_HOST_PCOMPLETER_BASE + 0x00004000 + \\\n+\t (i) * sizeof(struct nbl_function_msix_map))\n+#define NBL_PCOMPLETER_HOST_MSIX_FID_TABLE(i) \\\n+\t(NBL_INTF_HOST_PCOMPLETER_BASE + 0x0003a000 + (i) * sizeof(u32))\n+\n+#define NBL_PCOMPLETER_HOST_MSIX_FID_TABLE_FID_MASK  GENMASK(9, 0)\n+#define NBL_PCOMPLETER_HOST_MSIX_FID_TABLE_VLD_MASK  BIT(10)\n+\n+#define NBL_FUNC_MSIX_MAP_DWLEN\t\t4\n+/* The VLD/BDF fields live in data[2]; it can be cleared on its own */\n+#define NBL_FUNC_MSIX_MAP_VALID_DW\t2\n+/* data[2] */\n+#define NBL_FUNCTION_MSIX_MAP_FUNCTION_MASK GENMASK(2, 0)\n+#define NBL_FUNCTION_MSIX_MAP_DEVID_MASK GENMASK(7, 3)\n+#define NBL_FUNCTION_MSIX_MAP_BUS_MASK GENMASK(15, 8)\n+#define NBL_FUNCTION_MSIX_MAP_VALID_MASK BIT(16)\n+struct nbl_function_msix_map {\n+\tu32 data[NBL_FUNC_MSIX_MAP_DWLEN];\n+};\n+\n+/*  ----------  DPED  ----------  */\n+#define NBL_DPED_VLAN_OFFSET\t\t(NBL_DP_DPED_BASE + 0x000003F4)\n+#define NBL_DPED_DSCP_OFFSET_0\t\t(NBL_DP_DPED_BASE + 0x000003F8)\n+#define NBL_DPED_DSCP_OFFSET_1\t\t(NBL_DP_DPED_BASE + 0x000003FC)\n+/* DPED hw_edt_prof/ UPED hw_edt_prof */\n+\n+#define NBL_DPED_L4_CK_CMD_40_ADDR\t(NBL_DP_DPED_BASE + 0x00000338)\n+\n+#define NBL_DPED_L4_CK_CMD_40_EN_MASK\t\tBIT(31)\n+\n+/*  ----------  UPED  ----------  */\n+/* UPED uped_hw_edt_prof */\n+#define NBL_UPED_HW_EDT_PROF_TABLE(i) \\\n+\t(NBL_DP_UPED_BASE + 0x00001000 + (i) * sizeof(u32))\n+#define NBL_UPED_V4_TCP_IDX\t\t5\n+#define NBL_UPED_V6_TCP_IDX\t\t6\n+#define NBL_PED_HW_EDIT_PROFILE_L3_LEN_MASK GENMASK(3, 2)\n+\n+/*  ----------  DSCH  ----------  */\n+#define NBL_DSCH_PSHA_EN_MASK  GENMASK(3, 0)\n+/* DSCH dsch maxqid */\n+#define NBL_DSCH_HOST_QID_MAX (NBL_DP_DSCH_BASE + 0x00000118)\n+#define NBL_DSCH_HOST_QID_MAX_MASK GENMASK(10, 0)\n+#define NBL_DSCH_VN_QUANTA_ADDR (NBL_DP_DSCH_BASE + 0x00000134)\n+\n+#define NBL_MAX_QUEUE_ID\t0x7ff\n+#define NBL_HOST_QUANTA\t\t0x8000\n+#define NBL_ECPU_QUANTA\t\t0x1000\n+\n+#define NBL_DSCH_VN_QUANTA_H_QUA_MASK GENMASK(15, 0)\n+#define NBL_DSCH_VN_QUANTA_E_QUA_MASK GENMASK(31, 16)\n+\n+/*  ----------  DVN  ----------  */\n+/* DVN dvn_queue_table */\n+#define NBL_DVN_ECPU_QUEUE_NUM\t\t\t(NBL_DP_DVN_BASE + 0x0000041C)\n+#define NBL_DVN_DESCREQ_NUM_CFG\t\t\t(NBL_DP_DVN_BASE + 0x00000430)\n+#define NBL_DVN_DESC_WR_MERGE_TIMEOUT\t\t(NBL_DP_DVN_BASE + 0x00000480)\n+#define NBL_DVN_DIF_REQ_RD_RO_FLAG\t\t(NBL_DP_DVN_BASE + 0x0000045C)\n+\n+#define DEFAULT_DVN_DESCREQ_NUMCFG\t\t0x03\n+#define DEFAULT_DVN_100G_DESCREQ_NUMCFG\t\t0x07\n+\n+#define DEFAULT_DVN_DESC_WR_MERGE_TIMEOUT_MAX\t0x3FF\n+\n+/* spilit ring descreq_num 0:8,1:16 */\n+#define NBL_DVN_DESCREQ_NUM_CFG_AVRING_DESREQ_NUM_CFG_MASK BIT(0)\n+/* packet ring descreq_num\n+ * 0:8,1:12,2:16;3:20,4:24,5:26;6:32,7:32\n+ */\n+#define NBL_DVN_DESCREQ_NUM_CFG_PACKED_L1_NUM_MASK GENMASK(6, 4)\n+\n+#define NBL_DVN_DESC_WR_MERGE_TIMEOUT_CFG_CYCLE_MASK GENMASK(9, 0)\n+\n+#define NBL_DVN_DIF_REQ_RD_RO_FLAG_DESC_RO_EN_MASK BIT(0)\n+#define NBL_DVN_DIF_REQ_RD_RO_FLAG_DATA_RO_EN_MASK BIT(1)\n+#define NBL_DVN_DIF_REQ_RD_RO_FLAG_AVRING_RO_EN_MASK BIT(2)\n+\n+/*  ----------  UVN  ----------  */\n+/* UVN uvn_queue_table */\n+\n+#define NBL_UVN_DESC_RD_WAIT\t\t\t(NBL_DP_UVN_BASE + 0x0000020C)\n+#define NBL_UVN_QUEUE_ERR_MASK\t\t\t(NBL_DP_UVN_BASE + 0x00000224)\n+#define NBL_UVN_ECPU_QUEUE_NUM\t\t\t(NBL_DP_UVN_BASE + 0x0000023C)\n+#define NBL_UVN_DESC_WR_TIMEOUT\t\t\t(NBL_DP_UVN_BASE + 0x00000214)\n+#define NBL_UVN_DIF_REQ_RO_FLAG\t\t\t(NBL_DP_UVN_BASE + 0x00000250)\n+#define NBL_UVN_DESC_PREFETCH_INIT\t\t(NBL_DP_UVN_BASE + 0x00000204)\n+#define NBL_UVN_DESC_PREFETCH_NUM\t\t4\n+\n+#define NBL_UVN_DIF_REQ_RO_FLAG_AVAIL_RD_MASK BIT(0)\n+#define NBL_UVN_DIF_REQ_RO_FLAG_DESC_RD_MASK BIT(1)\n+#define NBL_UVN_DIF_REQ_RO_FLAG_PKT_WR_MASK BIT(2)\n+#define NBL_UVN_DIF_REQ_RO_FLAG_DESC_WR_MASK BIT(3)\n+\n+#define NBL_UVN_DESC_WR_TIMEOUT_NUM_MASK GENMASK(14, 0)\n+\n+#define NBL_UVN_QUEUE_ERR_MASK_DIF_ERR_MASK BIT(5)\n+\n+#define NBL_UVN_DESC_PREFETCH_INIT_NUM_MASK GENMASK(7, 0)\n+#define NBL_UVN_DESC_PREFETCH_INIT_SEL_MASK BIT(16)\n+\n+#define NBL_UVN_DESC_WR_TIMEOUT_VAL 0x12c\n+/* 200us = 200000ns / 1.67ns per tick = 119760 ticks */\n+#define NBL_UVN_DESC_RD_WAIT_TICKS 119760\n+\n+/*  --------  USTORE  --------  */\n+#define NBL_USTORE_PKT_LEN_ADDR (NBL_DP_USTORE_BASE + 0x00000108)\n+#define NBL_USTORE_PORT_DROP_TH_REG_ARR(port_id) \\\n+\t(NBL_DP_USTORE_BASE + 0x00000150 + (port_id) * sizeof(u32))\n+#define NBL_USTORE_BUF_PORT_DROP_PKT(eth_id) \\\n+\t(NBL_DP_USTORE_BASE + 0x00002500 + (eth_id) * sizeof(u32))\n+#define NBL_USTORE_BUF_PORT_TRUN_PKT(eth_id) \\\n+\t(NBL_DP_USTORE_BASE + 0x00002540 + (eth_id) * sizeof(u32))\n+\n+#define NBL_USTORE_SINGLE_ETH_DROP_TH\t\t0xC80\n+#define NBL_USTORE_DUAL_ETH_DROP_TH\t\t0x640\n+#define NBL_USTORE_QUAD_ETH_DROP_TH\t\t0x320\n+\n+/* USTORE pkt_len */\n+#define NBL_USTORE_PKT_LEN_MIN_MASK GENMASK(6, 0)\n+\n+/* USTORE port_drop_th */\n+#define NBL_USTORE_PORT_DROP_TH_DISC_TH_MASK GENMASK(11, 0)\n+#define NBL_USTORE_PORT_DROP_TH_EN_MASK BIT(31)\n+\n+/* UQM*/\n+#define NBL_UQM_QUE_TYPE\t\t\t(NBL_DP_UQM_BASE + 0x0000013c)\n+#define NBL_UQM_DROP_PKT_CNT\t\t\t(NBL_DP_UQM_BASE + 0x000009C0)\n+#define NBL_UQM_DROP_PKT_SLICE_CNT\t\t(NBL_DP_UQM_BASE + 0x000009C4)\n+#define NBL_UQM_DROP_PKT_LEN_ADD_CNT\t\t(NBL_DP_UQM_BASE + 0x000009C8)\n+#define NBL_UQM_DROP_HEAD_PNTR_ADD_CNT\t\t(NBL_DP_UQM_BASE + 0x000009CC)\n+#define NBL_UQM_DROP_WEIGHT_ADD_CNT\t\t(NBL_DP_UQM_BASE + 0x000009D0)\n+#define NBL_UQM_PORT_DROP_PKT_CNT\t\t(NBL_DP_UQM_BASE + 0x000009D4)\n+#define NBL_UQM_PORT_DROP_PKT_SLICE_CNT\t\t(NBL_DP_UQM_BASE + 0x000009F4)\n+#define NBL_UQM_PORT_DROP_PKT_LEN_ADD_CNT\t(NBL_DP_UQM_BASE + 0x00000A14)\n+#define NBL_UQM_PORT_DROP_HEAD_PNTR_ADD_CNT\t(NBL_DP_UQM_BASE + 0x00000A34)\n+#define NBL_UQM_PORT_DROP_WEIGHT_ADD_CNT\t(NBL_DP_UQM_BASE + 0x00000A54)\n+#define NBL_UQM_FWD_DROP_CNT\t\t\t(NBL_DP_UQM_BASE + 0x00000A80)\n+#define NBL_UQM_DPORT_DROP_CNT\t\t\t(NBL_DP_UQM_BASE + 0x00000B74)\n+\n+#define NBL_UQM_PORT_DROP_DEPTH\t\t\t6\n+#define NBL_UQM_DPORT_DROP_DEPTH\t\t16\n+\n+/*  ---------  SHAPING  ---------  */\n+\n+/* Shaping rate unit: 1 = 1 Mbit/s.\n+ * e.g. 100000 = 100 Gbit/s, 25000 = 25 Gbit/s.\n+ */\n+#define NBL_LR_LEONIS_NET_BUCKET_DEPTH\t\t9600\n+#define NBL_SHAPING_DPORT_ADDR (NBL_DP_SHAPING_BASE + 0x700)\n+#define NBL_SHAPING_DPORT_DWLEN 4\n+#define NBL_SHAPING_DPORT_REG(r) \\\n+\t(NBL_SHAPING_DPORT_ADDR + (NBL_SHAPING_DPORT_DWLEN * 4) * (r))\n+#define NBL_SHAPING_DVN_DPORT_ADDR (NBL_DP_SHAPING_BASE + 0x750)\n+#define NBL_SHAPING_DVN_DPORT_DWLEN 4\n+#define NBL_SHAPING_DVN_DPORT_REG(r) \\\n+\t(NBL_SHAPING_DVN_DPORT_ADDR + (NBL_SHAPING_DVN_DPORT_DWLEN * 4) * (r))\n+#define NBL_DSCH_PSHA_EN_ADDR (NBL_DP_DSCH_BASE + 0x00000314)\n+#define NBL_SHAPING_NET_ADDR (NBL_DP_SHAPING_BASE + 0x1800)\n+#define NBL_SHAPING_NET_DWLEN 4\n+#define NBL_SHAPING_NET_REG(r) \\\n+\t(NBL_SHAPING_NET_ADDR + (NBL_SHAPING_NET_DWLEN * 4) * (r))\n+\n+#define NBL_DPORT_VALID_MASK\t\tGENMASK_ULL(0, 0)\n+#define NBL_DPORT_DEPTH_MASK\t\tGENMASK_ULL(19, 1)\n+#define NBL_DPORT_CIR_MASK\t\tGENMASK_ULL(38, 20)\n+#define NBL_DPORT_PIR_MASK\t\tGENMASK_ULL(57, 39)\n+#define NBL_DPORT_CBS_MASK_LOW\t\tGENMASK_ULL(63, 58)\n+#define NBL_DPORT_CBS_MASK_HIGH\t\tGENMASK_ULL(14, 0)\n+#define NBL_DPORT_PBS_MASK\t\tGENMASK_ULL(35, 15)\n+\n+/* SHAPING shaping_net */\n+struct nbl_shaping_net_u {\n+\tu32 data[NBL_SHAPING_NET_DWLEN];\n+};\n+\n+struct nbl_shaping_dport_u {\n+\tu32 data[NBL_SHAPING_DPORT_DWLEN];\n+};\n+\n+struct nbl_shaping_dvn_dport_u {\n+\tu32 data[NBL_SHAPING_DVN_DPORT_DWLEN];\n+};\n+\n+/*  --------  DSTORE  --------  */\n+#define NBL_DSTORE_D_DPORT_FC_TH_ADDR  (NBL_DP_DSTORE_BASE + 0x00000600)\n+#define NBL_DSTORE_D_DPORT_FC_TH_DEPTH 5\n+#define NBL_DSTORE_D_DPORT_FC_TH_WIDTH 32\n+#define NBL_DSTORE_D_DPORT_FC_TH_DWLEN 1\n+\n+#define NBL_DSTORE_D_DPORT_FC_XOFF_TH_MASK GENMASK(10, 0)\n+#define NBL_DSTORE_D_DPORT_FC_XON_TH_MASK GENMASK(26, 16)\n+#define NBL_DSTORE_D_DPORT_FC_FC_EN_MASK BIT(31)\n+\n+#define NBL_DSTORE_D_DPORT_FC_TH_REG(r)  \\\n+\t(NBL_DSTORE_D_DPORT_FC_TH_ADDR + \\\n+\t (NBL_DSTORE_D_DPORT_FC_TH_DWLEN * 4) * (r))\n+#define NBL_DSTORE_PORT_DROP_TH_ADDR (NBL_DP_DSTORE_BASE + 0x00000150)\n+#define NBL_DSTORE_PORT_DROP_TH_DEPTH 6\n+#define NBL_DSTORE_PORT_DROP_TH_WIDTH 32\n+#define NBL_DSTORE_PORT_DROP_TH_DWLEN 1\n+\n+#define NBL_DSTORE_PORT_DROP_EN_MASK BIT(31)\n+\n+#define NBL_DSTORE_DROP_XOFF_TH\t\t\t0xC8\n+#define NBL_DSTORE_DROP_XON_TH\t\t\t0x64\n+\n+#define NBL_DSTORE_DROP_XOFF_TH_100G\t\t0x1F4\n+#define NBL_DSTORE_DROP_XON_TH_100G\t\t0x12C\n+\n+#define NBL_DSTORE_DISC_BP_TH (NBL_DP_DSTORE_BASE + 0x00000630)\n+\n+#define NBL_DSTORE_DISC_BP_TH_EN_MASK BIT(31)\n+\n+#define NBL_DSTORE_PORT_DROP_TH_REG(r)  \\\n+\t(NBL_DSTORE_PORT_DROP_TH_ADDR + \\\n+\t (NBL_DSTORE_PORT_DROP_TH_DWLEN * 4) * (r))\n+\n+#define NBL_FW_BOARD_CONFIG\t\t\t0x200\n+#define NBL_FW_BOARD_DW3_OFFSET\t\t\t(NBL_FW_BOARD_CONFIG + 12)\n+#define NBL_FW_BOARD_DW6_OFFSET\t\t\t(NBL_FW_BOARD_CONFIG + 24)\n+\n+#define NBL_FW_BOARD_DW3_PORT_TYPE_MASK BIT(0)\n+#define NBL_FW_BOARD_DW3_PORT_NUM_MASK GENMASK(7, 1)\n+#define NBL_FW_BOARD_DW3_PORT_SPEED_MASK GENMASK(9, 8)\n+#define NBL_FW_BOARD_DW3_GPIO_TYPE_MASK GENMASK(12, 10)\n+#define NBL_FW_BOARD_DW3_P4_VERSION_MASK GENMASK(13, 13)\n+\n+#define NBL_FW_BOARD_DW6_LANE_BITMAP_MASK GENMASK(7, 0)\n+#define NBL_FW_BOARD_DW6_ETH_BITMAP_MASK GENMASK(15, 8)\n+\n+#define NBL_LEONIS_QUIRKS_OFFSET\t0x00000140\n #define NBL_BAR2_MAX_LEN\t\t0x300\n #endif\ndiff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_leonis/nbl_resource_leonis.c b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_leonis/nbl_resource_leonis.c\nnew file mode 100644\nindex 00000000000000..ba589af55374b7\n--- /dev/null\n+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_leonis/nbl_resource_leonis.c\n@@ -0,0 +1,370 @@\n+// SPDX-License-Identifier: GPL-2.0\n+/*\n+ * Copyright (c) 2026 Nebula Matrix Limited.\n+ */\n+#include \u003clinux/device.h\u003e\n+#include \u003clinux/pci.h\u003e\n+#include \u003clinux/bits.h\u003e\n+#include \"nbl_resource_leonis.h\"\n+\n+static struct nbl_resource_ops res_ops = {\n+\t.get_vsi_id = nbl_res_func_id_to_vsi_id,\n+\t.get_eth_id = nbl_res_get_eth_id,\n+\t.cfg_msix_map = nbl_res_intr_cfg_msix_map,\n+\t.destroy_msix_map = nbl_res_intr_destroy_msix_map,\n+\t.set_mailbox_irq = nbl_res_intr_set_mailbox_irq,\n+\t.init_module = nbl_res_chip_init_module,\n+\t.deinit_module = nbl_res_chip_deinit_module,\n+};\n+\n+static struct nbl_resource_mgt *\n+nbl_res_setup_res_mgt(struct nbl_common_info *common)\n+{\n+\tstruct nbl_resource_info *resource_info;\n+\tstruct nbl_resource_mgt *res_mgt;\n+\tstruct device *dev = common-\u003edev;\n+\n+\tres_mgt = devm_kzalloc(dev, sizeof(*res_mgt), GFP_KERNEL);\n+\tif (!res_mgt)\n+\t\treturn ERR_PTR(-ENOMEM);\n+\tres_mgt-\u003ecommon = common;\n+\n+\tresource_info =\n+\t\tdevm_kzalloc(dev, sizeof(*resource_info), GFP_KERNEL);\n+\tif (!resource_info)\n+\t\treturn ERR_PTR(-ENOMEM);\n+\tres_mgt-\u003eresource_info = resource_info;\n+\n+\treturn res_mgt;\n+}\n+\n+static struct nbl_resource_ops_tbl *\n+nbl_res_setup_ops(struct device *dev, struct nbl_resource_mgt *res_mgt)\n+{\n+\tstruct nbl_resource_ops_tbl *res_ops_tbl;\n+\n+\tres_ops_tbl = devm_kzalloc(dev, sizeof(*res_ops_tbl), GFP_KERNEL);\n+\tif (!res_ops_tbl)\n+\t\treturn ERR_PTR(-ENOMEM);\n+\tif (!res_ops.get_vsi_id || !res_ops.get_eth_id ||\n+\t    !res_ops.cfg_msix_map || !res_ops.destroy_msix_map ||\n+\t    !res_ops.set_mailbox_irq || !res_ops.init_module ||\n+\t    !res_ops.deinit_module)\n+\t\treturn ERR_PTR(-EINVAL);\n+\tres_ops_tbl-\u003eops = \u0026res_ops;\n+\tres_ops_tbl-\u003epriv = res_mgt;\n+\n+\treturn res_ops_tbl;\n+}\n+\n+static int nbl_res_ctrl_dev_setup_eth_info(struct nbl_resource_mgt *res_mgt)\n+{\n+\tstruct nbl_hw_ops *hw_ops = res_mgt-\u003ehw_ops_tbl-\u003eops;\n+\tstruct device *dev = res_mgt-\u003ecommon-\u003edev;\n+\tstruct nbl_eth_info *eth_info;\n+\tu32 eth_bitmap = 0;\n+\tu32 eth_num = 0;\n+\tu32 fw_port_num;\n+\tint i;\n+\n+\teth_info = devm_kzalloc(dev, sizeof(*eth_info), GFP_KERNEL);\n+\tif (!eth_info)\n+\t\treturn -ENOMEM;\n+\n+\tres_mgt-\u003eresource_info-\u003eeth_info = eth_info;\n+\n+\tfw_port_num = res_mgt-\u003eresource_info-\u003eboard_info.eth_num;\n+\n+\thw_ops-\u003eget_fw_eth_map(res_mgt-\u003ehw_ops_tbl-\u003epriv, \u0026eth_bitmap);\n+\tif (eth_bitmap \u0026 ~((1 \u003c\u003c NBL_MAX_ETHERNET) - 1)) {\n+\t\tdev_err(dev, \"FW reported invalid eth_bitmap 0x%x\\n\",\n+\t\t\teth_bitmap);\n+\t\treturn -EINVAL;\n+\t}\n+\tif (fw_port_num != hweight32(eth_bitmap)) {\n+\t\tdev_err(dev, \"FW inconsistency: port_num=%u, bitmap=0x%x\\n\",\n+\t\t\tfw_port_num, eth_bitmap);\n+\t\treturn -EINVAL;\n+\t}\n+\t/*\n+\t * Firmware is ready before probe. Valid port counts are 1/2/4;\n+\t * 0 (invalid config), 3 (unsupported topology), and \u003e4 (exceeds\n+\t * hardware max) are all rejected with -EINVAL.\n+\t */\n+\tif (fw_port_num == 0 || fw_port_num == 3 ||\n+\t    fw_port_num \u003e NBL_MAX_ETHERNET) {\n+\t\tdev_err(dev, \"FW reports %u Ethernet ports, unsupported (valid: 1/2/4)\\n\",\n+\t\t\tfw_port_num);\n+\t\treturn -EINVAL;\n+\t}\n+\teth_info-\u003eeth_num = fw_port_num;\n+\t/* Intentional design constraint: each PF maps to exactly one\n+\t * Ethernet port. This couples PF identity to port identity\n+\t * and is required by nbl_res_get_eth_id() which indexes\n+\t * eth_info-\u003eeth_id[] by relative PF id.\n+\t */\n+\tif (res_mgt-\u003ecommon-\u003emax_pf != eth_info-\u003eeth_num) {\n+\t\tdev_err(dev, \"Invalid PF-to-port topology: max_pf=%u, eth_num=%u\\n\",\n+\t\t\tres_mgt-\u003ecommon-\u003emax_pf, eth_info-\u003eeth_num);\n+\t\treturn -EINVAL;\n+\t}\n+\n+\t/*\n+\t * Any subset of valid bitmap bits is accepted (e.g. 0/1, 0/2,\n+\t * 1/3, etc.).  Firmware only needs to report the correct count\n+\t * of active ports; no hard-coded fixed bit positions required.\n+\t * eth_id[] is filled in ascending bitmap-bit order, so the Nth\n+\t * relative PF owns the Nth logical port (max_pf == eth_num is\n+\t * enforced above).\n+\t */\n+\tfor (i = 0; i \u003c NBL_MAX_ETHERNET; i++) {\n+\t\tif ((1 \u003c\u003c i) \u0026 eth_bitmap) {\n+\t\t\teth_info-\u003eeth_id[eth_num] = i;\n+\t\t\teth_num++;\n+\t\t}\n+\t}\n+\n+\treturn 0;\n+}\n+\n+static int nbl_res_ctrl_dev_sriov_info_init(struct nbl_resource_mgt *res_mgt)\n+{\n+\tstruct nbl_hw_ops *hw_ops = res_mgt-\u003ehw_ops_tbl-\u003eops;\n+\tstruct nbl_hw_mgt *p = res_mgt-\u003ehw_ops_tbl-\u003epriv;\n+\tstruct nbl_common_info *common = res_mgt-\u003ecommon;\n+\tstruct nbl_sriov_info *sriov_info;\n+\tstruct device *dev = common-\u003edev;\n+\tu8 hw_bus = 0;\n+\tu16 function;\n+\tu16 func_id;\n+\n+\thw_ops-\u003eget_real_bus(p, \u0026hw_bus);\n+\tif (common-\u003efunction + common-\u003emax_pf \u003e NBL_MAX_PF) {\n+\t\tdev_err(dev, \"PF count exceeds available function space\\n\");\n+\t\treturn -EINVAL;\n+\t}\n+\tsriov_info = devm_kcalloc(dev, common-\u003emax_pf,\n+\t\t\t\t  sizeof(*sriov_info), GFP_KERNEL);\n+\tif (!sriov_info)\n+\t\treturn -ENOMEM;\n+\n+\tres_mgt-\u003eresource_info-\u003esriov_info = sriov_info;\n+\t/*\n+\t * Real bus number of the control PF;\n+\t * the BDF table below is built from it.\n+\t */\n+\tcommon-\u003ehw_bus = hw_bus;\n+\n+\tfor (func_id = 0; func_id \u003c common-\u003emax_pf; func_id++) {\n+\t\tsriov_info = res_mgt-\u003eresource_info-\u003esriov_info + func_id;\n+\t\tfunction = common-\u003efunction + func_id;\n+\t\tsriov_info-\u003ebdf = PCI_DEVID(common-\u003ehw_bus,\n+\t\t\t\t\t    PCI_DEVFN(common-\u003edevid, function));\n+\t}\n+\n+\treturn 0;\n+}\n+\n+static int nbl_res_ctrl_dev_vsi_info_init(struct nbl_resource_mgt *res_mgt)\n+{\n+\tstruct nbl_eth_info *eth_info = res_mgt-\u003eresource_info-\u003eeth_info;\n+\tstruct nbl_common_info *common = res_mgt-\u003ecommon;\n+\tstruct device *dev = common-\u003edev;\n+\tstruct nbl_vsi_info *vsi_info;\n+\tint i;\n+\n+\tvsi_info = devm_kzalloc(dev, sizeof(*vsi_info), GFP_KERNEL);\n+\tif (!vsi_info)\n+\t\treturn -ENOMEM;\n+\n+\tres_mgt-\u003eresource_info-\u003evsi_info = vsi_info;\n+\t/*\n+\t * case 1 one port(1pf)\n+\t * pf0 (NBL_VSI_SERV_PF_DATA_TYPE) vsi is 0\n+\t * case 2 two port(2pf)\n+\t * pf0,pf1(NBL_VSI_SERV_PF_DATA_TYPE) vsi is 0,512\n+\t * case 3 four port(4pf)\n+\t * pf0,pf1,pf2,pf3(NBL_VSI_SERV_PF_DATA_TYPE) vsi is 0,256,512,768\n+\t */\n+\n+\tvsi_info-\u003enum = eth_info-\u003eeth_num;\n+\t/*\n+\t * eth_num can be 1/2/4:\n+\t * - 2/4 ports use dedicated gap constants;\n+\t * - 1 port falls back to NBL_DEFAULT_VSI_ID_GAP (1024).\n+\t * All three values produce valid base_id offsets.\n+\t */\n+\tfor (i = 0; i \u003c vsi_info-\u003enum; i++) {\n+\t\tvsi_info-\u003eserv_info[i][NBL_VSI_SERV_PF_DATA_TYPE].base_id =\n+\t\t\ti * nbl_vsi_id_gap(vsi_info-\u003enum);\n+\t\tvsi_info-\u003eserv_info[i][NBL_VSI_SERV_PF_DATA_TYPE].num = 1;\n+\t}\n+\n+\treturn 0;\n+}\n+\n+static int nbl_res_init_pf_num(struct nbl_resource_mgt *res_mgt)\n+{\n+\tstruct nbl_hw_ops *hw_ops = res_mgt-\u003ehw_ops_tbl-\u003eops;\n+\tu32 exp_contiguous_mask = 0;\n+\tu32 pf_mask = 0;\n+\tu32 pf_num = 0;\n+\tint i;\n+\n+\thw_ops-\u003eget_host_pf_mask(res_mgt-\u003ehw_ops_tbl-\u003epriv, \u0026pf_mask);\n+\n+\t/*\n+\t * k_pf_mask register rule:\n+\t * bit N == 0  -\u003e PF#N enabled; bit N == 1 -\u003e PF#N masked out.\n+\t * Hardware constraint: bit0 is PF0's mask bit; driver requires\n+\t * PF0 enabled as management PF, so bit0 must be clear.\n+\t * All-zero pf_mask means all PF0~PF7 are enabled, which is unsupported\n+\t * by the driver\n+\t *\n+\t * Product firmware constraint: only 3 valid configurations supported:\n+\t * 1 PF  (PF0 only): pf_num = 1, mask = 0xfe\n+\t * 2 PFs (PF0,PF1):  pf_num = 2, mask = 0xfc\n+\t * 4 PFs (PF0~PF3): pf_num = 4, mask = 0xf0\n+\t * No other PF count or sparse/non-contiguous PF layout is allowed.\n+\t *\n+\t * This is the only validation of the mask in the driver.  The channel\n+\t * layer reads the same raw register value to program the mailbox\n+\t * QINFO table and assumes the layout checked here, so it depends on\n+\t * this running first: resource init completes before the channel\n+\t * layer is set up.\n+\t */\n+\tfor (i = 0; i \u003c NBL_MAX_PF; i++) {\n+\t\tif (!(pf_mask \u0026 (1 \u003c\u003c i)))\n+\t\t\tpf_num++;\n+\t}\n+\n+\t/*\n+\t * Sanity check: enabled PFs must be contiguous starting from PF0.\n+\t * Current resource framework uses relative PF id, sparse PF layout\n+\t * will cause mismatch between resource layer and hardware func_id.\n+\t */\n+\tfor (i = 0; i \u003c pf_num; i++)\n+\t\texp_contiguous_mask |= BIT(i);\n+\tif ((pf_mask \u0026 exp_contiguous_mask) != 0) {\n+\t\tdev_err(res_mgt-\u003ecommon-\u003edev,\n+\t\t\t\"pf_mask 0x%08x: non-contiguous enabled PF, unsupported\\n\",\n+\t\t\tpf_mask);\n+\t\treturn -EINVAL;\n+\t}\n+\n+\t/* Only allow product-specified PF count: 1 / 2 / 4 */\n+\tif (pf_num != 1 \u0026\u0026 pf_num != 2 \u0026\u0026 pf_num != 4) {\n+\t\tdev_err(res_mgt-\u003ecommon-\u003edev,\n+\t\t\t\"Invalid pf_num=%u (mask=0x%08x), only 1/2/4 PFs supported\\n\",\n+\t\t\tpf_num, pf_mask);\n+\t\treturn -EINVAL;\n+\t}\n+\n+\tres_mgt-\u003ecommon-\u003emax_pf = pf_num;\n+\n+\treturn 0;\n+}\n+\n+static void nbl_res_init_board_info(struct nbl_resource_mgt *res_mgt)\n+{\n+\tstruct nbl_hw_ops *hw_ops = res_mgt-\u003ehw_ops_tbl-\u003eops;\n+\n+\thw_ops-\u003eget_board_info(res_mgt-\u003ehw_ops_tbl-\u003epriv,\n+\t\t\t       \u0026res_mgt-\u003eresource_info-\u003eboard_info);\n+}\n+\n+static int nbl_res_start(struct nbl_resource_mgt *res_mgt)\n+{\n+\tstruct nbl_common_info *common = res_mgt-\u003ecommon;\n+\tint ret = 0;\n+\n+\tif (common-\u003ehas_ctrl) {\n+\t\tnbl_res_init_board_info(res_mgt);\n+\n+\t\tret = nbl_res_init_pf_num(res_mgt);\n+\t\tif (ret)\n+\t\t\treturn ret;\n+\n+\t\tret = nbl_res_ctrl_dev_sriov_info_init(res_mgt);\n+\t\tif (ret)\n+\t\t\treturn ret;\n+\n+\t\tret = nbl_res_ctrl_dev_setup_eth_info(res_mgt);\n+\t\tif (ret)\n+\t\t\treturn ret;\n+\n+\t\tret = nbl_res_ctrl_dev_vsi_info_init(res_mgt);\n+\t\tif (ret)\n+\t\t\treturn ret;\n+\n+\t\tret = nbl_intr_mgt_start(res_mgt);\n+\t\tif (ret)\n+\t\t\treturn ret;\n+\t}\n+\n+\treturn 0;\n+}\n+\n+int nbl_res_init_leonis(struct nbl_adapter *adap)\n+{\n+\tstruct nbl_channel_ops_tbl *chan_ops_tbl = adap-\u003eintf.channel_ops_tbl;\n+\tstruct nbl_hw_ops_tbl *hw_ops_tbl = adap-\u003eintf.hw_ops_tbl;\n+\tstruct nbl_common_info *common = \u0026adap-\u003ecommon;\n+\tstruct nbl_resource_ops_tbl *res_ops_tbl;\n+\tstruct device *dev = \u0026adap-\u003epdev-\u003edev;\n+\tstruct nbl_resource_mgt *res_mgt;\n+\tint ret;\n+\n+\tres_mgt = nbl_res_setup_res_mgt(common);\n+\tif (IS_ERR(res_mgt)) {\n+\t\tret = PTR_ERR(res_mgt);\n+\t\treturn ret;\n+\t}\n+\tres_mgt-\u003echan_ops_tbl = chan_ops_tbl;\n+\tres_mgt-\u003ehw_ops_tbl = hw_ops_tbl;\n+\n+\tret = nbl_res_start(res_mgt);\n+\tif (ret)\n+\t\treturn ret;\n+\n+\tres_ops_tbl = nbl_res_setup_ops(dev, res_mgt);\n+\tif (IS_ERR(res_ops_tbl)) {\n+\t\tret = PTR_ERR(res_ops_tbl);\n+\t\treturn ret;\n+\t}\n+\tadap-\u003eintf.resource_ops_tbl = res_ops_tbl;\n+\tadap-\u003ecore.res_mgt = res_mgt;\n+\n+\treturn 0;\n+}\n+\n+void nbl_res_remove_leonis(struct nbl_adapter *adap)\n+{\n+\tstruct nbl_resource_mgt *res_mgt = adap-\u003ecore.res_mgt;\n+\tstruct nbl_common_info *common = \u0026adap-\u003ecommon;\n+\n+\tif (!res_mgt)\n+\t\treturn;\n+\n+\t/*\n+\t * Tear down all MSI-X maps before destroying coherent tables.\n+\t * This is critical on the control PF, whose table carries the map\n+\t * entries installed for every configured function: they must be\n+\t * invalidated in hardware before the tables backing them are\n+\t * freed, otherwise the device keeps DMA-reading released pages.\n+\t * Sibling PFs are already unbound by the device-link ordering\n+\t * so no remote PF still has a live map entry or a running\n+\t * mailbox RPC at this point.\n+\t */\n+\tif (common-\u003ehas_ctrl \u0026\u0026 res_mgt-\u003eintr_mgt)\n+\t\tnbl_intr_mgt_stop(res_mgt);\n+\n+\t/* Note:\n+\t * per-function interrupts arrays (kcalloc) are freed by\n+\t * nbl_intr_mgt_stop().\n+\t * MSIX coherent tables are explicitly freed by dma_free_coherent()\n+\t * inside the intr destroy path, before nbl_intr_mgt_stop() returns.\n+\t * intr_mgt itself (devm_kzalloc) is released by devres after this\n+\t * function returns\n+\t */\n+}\ndiff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_leonis/nbl_resource_leonis.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_leonis/nbl_resource_leonis.h\nnew file mode 100644\nindex 00000000000000..f1cb0f23240b0c\n--- /dev/null\n+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_leonis/nbl_resource_leonis.h\n@@ -0,0 +1,12 @@\n+/* SPDX-License-Identifier: GPL-2.0 */\n+/*\n+ * Copyright (c) 2026 Nebula Matrix Limited.\n+ */\n+\n+#ifndef _NBL_RESOURCE_LEONIS_H_\n+#define _NBL_RESOURCE_LEONIS_H_\n+\n+#include \"../nbl_resource.h\"\n+#include \"../nbl_interrupt.h\"\n+#include \"../nbl_chip.h\"\n+#endif\ndiff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_reg.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_reg.h\nindex e281109f502e67..00fa33eeabaab9 100644\n--- a/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_reg.h\n+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_reg.h\n@@ -8,6 +8,7 @@\n \n #include \u003clinux/types.h\u003e\n \n+#include \"../nbl_include/nbl_def_channel.h\"\n #include \"../nbl_include/nbl_def_hw.h\"\n #include \"../nbl_include/nbl_def_common.h\"\n #include \"../nbl_core.h\"\n@@ -16,6 +17,7 @@\n #define NBL_MAILBOX_BAR\t\t\t\t2\n #define NBL_RDMA_NOTIFY_LEN\t\t\t(8ULL \u003c\u003c 10)\n #define NBL_REG_NET_ONLY_LEN\t\t\t(8ULL \u003c\u003c 10)\n+#define NBL_HW_DUMMY_REG\t\t\t0x1300904\n /*\n  * PCI MEMORY BAR total size: 64MiB.\n  */\n@@ -26,6 +28,37 @@ struct nbl_hw_mgt {\n \tu8 __iomem *hw_addr;\n \tu8 __iomem *mailbox_bar_hw_addr;\n \tresource_size_t mailbox_bar_size;\n+\tspinlock_t reg_lock; /* Protect reg access */\n };\n \n+static inline u32 rd32(u8 __iomem *addr, u64 reg)\n+{\n+\treturn readl(addr + reg);\n+}\n+\n+static inline void wr32(u8 __iomem *addr, u64 reg, u32 value)\n+{\n+\twritel(value, addr + reg);\n+}\n+\n+static inline void nbl_hw_wr32(struct nbl_hw_mgt *hw_mgt, u64 reg, u32 value)\n+{\n+\twr32(hw_mgt-\u003ehw_addr, reg, value);\n+}\n+\n+static inline u32 nbl_hw_rd32(struct nbl_hw_mgt *hw_mgt, u64 reg)\n+{\n+\treturn rd32(hw_mgt-\u003ehw_addr, reg);\n+}\n+\n+static inline void nbl_mbx_wr32(struct nbl_hw_mgt *hw_mgt, u64 reg, u32 value)\n+{\n+\twritel(value, hw_mgt-\u003emailbox_bar_hw_addr + reg);\n+}\n+\n+static inline u32 nbl_mbx_rd32(struct nbl_hw_mgt *hw_mgt, u64 reg)\n+{\n+\treturn readl(hw_mgt-\u003emailbox_bar_hw_addr + reg);\n+}\n+\n #endif\ndiff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_interrupt.c b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_interrupt.c\nnew file mode 100644\nindex 00000000000000..3969392abdc64a\n--- /dev/null\n+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_interrupt.c\n@@ -0,0 +1,851 @@\n+// SPDX-License-Identifier: GPL-2.0\n+/*\n+ * Copyright (c) 2026 Nebula Matrix Limited.\n+ */\n+#include \u003clinux/device.h\u003e\n+#include \u003clinux/delay.h\u003e\n+#include \u003clinux/dma-mapping.h\u003e\n+#include \u003clinux/bitfield.h\u003e\n+#include \"nbl_interrupt.h\"\n+\n+#define NBL_MSIX_DMA_SYNC_MIN_US\t1000 /* us */\n+#define NBL_MSIX_DMA_SYNC_MAX_US\t1200 /* us */\n+\n+/*\n+ * Bounded best-effort wait for in-flight pcompleter fetches of a MSI-X\n+ * map table after that entry's VALID bit has been cleared and flushed.\n+ *\n+ * The chip exposes no idle/completion status for those fetches, so the\n+ * only ordering available is: clear VALID -\u003e flush_write() -\u003e wait.  The\n+ * window is sized well above the tables' worst-case fetch/response\n+ * latency; it is not a guarantee, and it protects two things:\n+ *   - the coherent table that __nbl_res_intr_complete_destroy_msix_map()\n+ *     is about to dma_free_coherent();\n+ *   - the vector set that a reconfiguration is about to hand to another\n+ *     function.\n+ * A future caller that cannot tolerate the residual risk must keep the\n+ * table allocated for the lifetime of the function instead of relying on\n+ * this delay.\n+ *\n+ * Both are bus-master reads tagged with the control PF's BDF, so\n+ * pci_clear_master() is the only other lever, and nbl_remove() pulls it\n+ * before any teardown runs: that keeps the device from starting a new\n+ * fetch inside this window.  It cannot help a sibling PF's map destroyed\n+ * at runtime, because those fetches carry the control PF's BDF and so\n+ * are unaffected by clearing bus mastering on the sibling.  Runtime\n+ * destroys of sibling maps only become reachable with the mailbox RPC\n+ * added later in this series.\n+ */\n+static void nbl_intr_quiesce_wait(void)\n+{\n+\tusleep_range(NBL_MSIX_DMA_SYNC_MIN_US, NBL_MSIX_DMA_SYNC_MAX_US);\n+}\n+\n+/*\n+ * Release global vector IDs back to intr_net_bmap / intr_other_bmap.\n+ * Caller must hold intr_mgt-\u003elock, passing the intr_mgt it locked:\n+ * nbl_intr_mgt_stop() clears res_mgt-\u003eintr_mgt while a caller may\n+ * already be blocked on that mutex, so it must not be re-read here.\n+ */\n+static void nbl_intr_release_bitmap(struct nbl_interrupt_mgt *intr_mgt,\n+\t\t\t\t    struct nbl_resource_mgt *res_mgt,\n+\t\t\t\t    u16 *vec_buf, u16 cnt)\n+{\n+\tu16 bit;\n+\tu16 i;\n+\n+\tlockdep_assert_held(\u0026intr_mgt-\u003elock);\n+\n+\tif (!vec_buf || cnt == 0)\n+\t\treturn;\n+\n+\tfor (i = 0; i \u003c cnt; i++) {\n+\t\tu16 intr_index = vec_buf[i];\n+\n+\t\tif (intr_index \u003e= NBL_NET_INTR_BASE) {\n+\t\t\tbit = intr_index - NBL_NET_INTR_BASE;\n+\t\t\tif (bit \u003c NBL_MAX_NET_INTERRUPT)\n+\t\t\t\tclear_bit(bit, intr_mgt-\u003eintr_net_bmap);\n+\t\t\telse\n+\t\t\t\tdev_warn(res_mgt-\u003ecommon-\u003edev,\n+\t\t\t\t\t \"invalid net intr index %u\\n\",\n+\t\t\t\t\t intr_index);\n+\t\t} else {\n+\t\t\tif (intr_index \u003c NBL_MAX_OTHER_INTERRUPT)\n+\t\t\t\tclear_bit(intr_index,\n+\t\t\t\t\t  intr_mgt-\u003eintr_other_bmap);\n+\t\t\telse\n+\t\t\t\tdev_warn(res_mgt-\u003ecommon-\u003edev,\n+\t\t\t\t\t \"invalid other intr index %u\\n\",\n+\t\t\t\t\t intr_index);\n+\t\t}\n+\t}\n+}\n+\n+/*\n+ * Internal (unlocked) mailbox IRQ bind.  Caller must hold\n+ * intr_mgt-\u003elock, passing the intr_mgt it locked (see\n+ * nbl_intr_release_bitmap()).\n+ *\n+ * The disable path deliberately carries no state check: the teardown\n+ * sequence uses it to disarm routing while func_res-\u003estate is still\n+ * CONFIGURED, and it stays harmless after nbl_intr_mgt_stop() because\n+ * the hardware op ignores gvec when en_msix=false.  Only the enable\n+ * path is gated by the stopping latch and the per-function state.\n+ */\n+static int __nbl_res_intr_set_mailbox_irq(struct nbl_interrupt_mgt *intr_mgt,\n+\t\t\t\t\t  struct nbl_resource_mgt *res_mgt,\n+\t\t\t\t\t  u16 func_id, u16 vector_id,\n+\t\t\t\t\t  bool en_msix)\n+{\n+\tstruct nbl_hw_ops *hw_ops = res_mgt-\u003ehw_ops_tbl-\u003eops;\n+\tstruct nbl_common_info *common = res_mgt-\u003ecommon;\n+\tstruct device *dev = common-\u003edev;\n+\tstruct nbl_func_interrupt_resource_mng *func_res;\n+\tu16 gvec;\n+\n+\tlockdep_assert_held(\u0026intr_mgt-\u003elock);\n+\n+\t/*\n+\t * func_intr_res[] is indexed by the raw hardware function id; this\n+\t * bound check is the only guard on this path, so it is what keeps\n+\t * VF ids out.  Whether the PF is present on this board is not\n+\t * checked here: callers pass either common-\u003emgt_pf or an id that\n+\t * nbl_res_intr_cfg_msix_map() has validated.\n+\t */\n+\tif (func_id \u003e= NBL_MAX_PF) {\n+\t\tdev_err(dev, \"func_id %u out of range\\n\", func_id);\n+\t\treturn -EINVAL;\n+\t}\n+\n+\tif (!en_msix) {\n+\t\thw_ops-\u003eset_mailbox_irq(res_mgt-\u003ehw_ops_tbl-\u003epriv,\n+\t\t\t\t\tfunc_id, false, 0);\n+\t\thw_ops-\u003eflush_write(res_mgt-\u003ehw_ops_tbl-\u003epriv);\n+\t\treturn 0;\n+\t}\n+\n+\t/*\n+\t * Enable path: the map must be live and not under teardown,\n+\t * otherwise routing would point at a vector that the complete\n+\t * phase is about to release and never re-disables.\n+\t */\n+\tif (intr_mgt-\u003estopping)\n+\t\treturn -ESHUTDOWN;\n+\n+\tfunc_res = \u0026intr_mgt-\u003efunc_intr_res[func_id];\n+\tif (func_res-\u003estate != NBL_INTR_FUNC_CONFIGURED) {\n+\t\tdev_err(dev, \"func %u MSIX map not configured (state %u)\\n\",\n+\t\t\tfunc_id, func_res-\u003estate);\n+\t\treturn -ENODEV;\n+\t}\n+\n+\tif (vector_id \u003e= func_res-\u003enum_interrupts) {\n+\t\tdev_err(dev, \"vector_id %u out of range (max %u)\\n\",\n+\t\t\tvector_id, func_res-\u003enum_interrupts - 1);\n+\t\treturn -EINVAL;\n+\t}\n+\n+\tgvec = func_res-\u003einterrupts[vector_id];\n+\thw_ops-\u003eset_mailbox_irq(res_mgt-\u003ehw_ops_tbl-\u003epriv, func_id,\n+\t\t\t\ten_msix, gvec);\n+\thw_ops-\u003eflush_write(res_mgt-\u003ehw_ops_tbl-\u003epriv);\n+\n+\treturn 0;\n+}\n+\n+/*\n+ * Internal (unlocked) MSI-X map teardown prepare phase: only hardware\n+ * register operations. The DMA address is retained and only the VALID\n+ * bit is cleared; zeroing the address (Stage 2) is deferred to the\n+ * complete phase after the hardware-DMA quiesce window.\n+ *\n+ * Caller must hold intr_mgt-\u003elock.\n+ */\n+static int\n+__nbl_res_intr_prepare_destroy_msix_map(struct nbl_interrupt_mgt *intr_mgt,\n+\t\t\t\t\tstruct nbl_resource_mgt *res_mgt,\n+\t\t\t\t\tu16 func)\n+{\n+\tstruct nbl_hw_ops *hw_ops = res_mgt-\u003ehw_ops_tbl-\u003eops;\n+\tstruct nbl_func_interrupt_resource_mng *func_res;\n+\tu16 *interrupts;\n+\tu16 intr_num, i;\n+\tint ret;\n+\n+\tlockdep_assert_held(\u0026intr_mgt-\u003elock);\n+\n+\tif (func \u003e= NBL_MAX_PF) {\n+\t\tdev_err(res_mgt-\u003ecommon-\u003edev, \"Invalid func_id %u\\n\", func);\n+\t\treturn -EINVAL;\n+\t}\n+\n+\tfunc_res = \u0026intr_mgt-\u003efunc_intr_res[func];\n+\tif (func_res-\u003estate != NBL_INTR_FUNC_CONFIGURED)\n+\t\treturn 0;\n+\n+\tinterrupts = func_res-\u003einterrupts;\n+\tintr_num = func_res-\u003enum_interrupts;\n+\n+\t/* Step 0: disable mailbox IRQ routing before tearing down map */\n+\tret = __nbl_res_intr_set_mailbox_irq(intr_mgt, res_mgt, func, 0, false);\n+\tif (ret) {\n+\t\tdev_err(res_mgt-\u003ecommon-\u003edev,\n+\t\t\t\"disable mailbox irq failed, func=%u ret=%d\\n\",\n+\t\t\tfunc, ret);\n+\t\treturn ret;\n+\t}\n+\n+\t/* Step 1: invalidate each MSIX info entry in hardware first */\n+\tfor (i = 0; i \u003c intr_num; i++) {\n+\t\thw_ops-\u003ecfg_msix_info(res_mgt-\u003ehw_ops_tbl-\u003epriv,\n+\t\t\t\t      func, false, interrupts[i],\n+\t\t\t\t      0, 0, 0, false);\n+\t}\n+\thw_ops-\u003eflush_write(res_mgt-\u003ehw_ops_tbl-\u003epriv);\n+\n+\t/*\n+\t * Stage 1: retain the DMA address, only clear the VALID bit.\n+\t * Stage 2 runs after the quiesce window in the complete phase.\n+\t */\n+\thw_ops-\u003ecfg_msix_map(res_mgt-\u003ehw_ops_tbl-\u003epriv, func,\n+\t\t\t     false, func_res-\u003emsix_map_table.dma,\n+\t\t\t     0, 0, 0);\n+\thw_ops-\u003eflush_write(res_mgt-\u003ehw_ops_tbl-\u003epriv);\n+\n+\tfunc_res-\u003estate = NBL_INTR_FUNC_DESTROYING;\n+\n+\treturn 0;\n+}\n+\n+/*\n+ * __nbl_res_intr_complete_destroy_msix_map - finish hardware teardown and\n+ * release vector bitmap, DMA memory and interrupt buffer after the\n+ * hardware quiesce window has elapsed.\n+ *\n+ * Caller must hold intr_mgt-\u003elock.\n+ */\n+static int\n+__nbl_res_intr_complete_destroy_msix_map(struct nbl_interrupt_mgt *intr_mgt,\n+\t\t\t\t\t struct nbl_resource_mgt *res_mgt,\n+\t\t\t\t\t u16 func_id)\n+{\n+\tstruct nbl_hw_ops *hw_ops = res_mgt-\u003ehw_ops_tbl-\u003eops;\n+\tstruct nbl_func_interrupt_resource_mng *func_res;\n+\tstruct nbl_msix_map_table *msix_map_table;\n+\tstruct device *dev = res_mgt-\u003ecommon-\u003edev;\n+\tu16 *interrupts;\n+\tu16 intr_num;\n+\n+\tlockdep_assert_held(\u0026intr_mgt-\u003elock);\n+\n+\tif (func_id \u003e= NBL_MAX_PF) {\n+\t\tdev_err(dev, \"Invalid func_id %u\\n\", func_id);\n+\t\treturn -EINVAL;\n+\t}\n+\n+\tfunc_res = \u0026intr_mgt-\u003efunc_intr_res[func_id];\n+\tif (func_res-\u003estate != NBL_INTR_FUNC_DESTROYING)\n+\t\treturn 0;\n+\n+\t/*\n+\t * Stage 2: the quiesce window has elapsed, it is now safe to\n+\t * zero the DMA base address in the hardware map register.\n+\t */\n+\thw_ops-\u003ecfg_msix_map(res_mgt-\u003ehw_ops_tbl-\u003epriv, func_id,\n+\t\t\t     false, 0, 0, 0, 0);\n+\thw_ops-\u003eflush_write(res_mgt-\u003ehw_ops_tbl-\u003epriv);\n+\n+\tinterrupts = func_res-\u003einterrupts;\n+\tintr_num = func_res-\u003enum_interrupts;\n+\tmsix_map_table = \u0026func_res-\u003emsix_map_table;\n+\n+\tif (interrupts) {\n+\t\tnbl_intr_release_bitmap(intr_mgt, res_mgt, interrupts,\n+\t\t\t\t\tintr_num);\n+\t\tkfree(interrupts);\n+\t}\n+\n+\t/*\n+\t * Release the coherent table independently of interrupts so a\n+\t * partially built config (table allocated, vectors never\n+\t * published) cannot leak coherent DMA memory.\n+\t */\n+\tif (msix_map_table-\u003ebase_addr) {\n+\t\tdma_free_coherent(dev, msix_map_table-\u003esize,\n+\t\t\t\t  msix_map_table-\u003ebase_addr,\n+\t\t\t\t  msix_map_table-\u003edma);\n+\t\tmsix_map_table-\u003ebase_addr = NULL;\n+\t\tmsix_map_table-\u003edma = 0;\n+\t\tmsix_map_table-\u003esize = 0;\n+\t}\n+\n+\tfunc_res-\u003einterrupts = NULL;\n+\tfunc_res-\u003enum_interrupts = 0;\n+\tfunc_res-\u003enum_net_interrupts = 0;\n+\tfunc_res-\u003estate = NBL_INTR_FUNC_IDLE;\n+\n+\treturn 0;\n+}\n+\n+/*\n+ * Internal (unlocked) MSI-X map teardown.  Caller must hold\n+ * intr_mgt-\u003elock for the whole sequence, including the hardware-DMA\n+ * quiesce window: dropping the lock would let a concurrent caller (or\n+ * nbl_intr_mgt_stop()) install/free state against this teardown.\n+ *\n+ * This is used for the single function synchronous destroy path.\n+ */\n+static int __nbl_res_intr_destroy_msix_map(struct nbl_interrupt_mgt *intr_mgt,\n+\t\t\t\t\t   struct nbl_resource_mgt *res_mgt,\n+\t\t\t\t\t   u16 func_id)\n+{\n+\tint ret;\n+\n+\tlockdep_assert_held(\u0026intr_mgt-\u003elock);\n+\n+\tif (intr_mgt-\u003estopping)\n+\t\treturn -ESHUTDOWN;\n+\n+\tret = __nbl_res_intr_prepare_destroy_msix_map(intr_mgt, res_mgt,\n+\t\t\t\t\t\t      func_id);\n+\tif (ret)\n+\t\treturn ret;\n+\t/*\n+\t * prepare() only transitions CONFIGURED functions; an IDLE func\n+\t * has nothing to wait for or complete.\n+\t */\n+\tif (intr_mgt-\u003efunc_intr_res[func_id].state !=\n+\t    NBL_INTR_FUNC_DESTROYING)\n+\t\treturn 0;\n+\n+\tnbl_intr_quiesce_wait();\n+\n+\treturn __nbl_res_intr_complete_destroy_msix_map(intr_mgt, res_mgt,\n+\t\t\t\t\t\t\tfunc_id);\n+}\n+\n+int nbl_res_intr_destroy_msix_map(struct nbl_resource_mgt *res_mgt,\n+\t\t\t\t  u16 func_id)\n+{\n+\tstruct nbl_interrupt_mgt *intr_mgt = res_mgt-\u003eintr_mgt;\n+\tint ret;\n+\n+\tif (!intr_mgt)\n+\t\treturn -EINVAL;\n+\n+\tmutex_lock(\u0026intr_mgt-\u003elock);\n+\tret = __nbl_res_intr_destroy_msix_map(intr_mgt, res_mgt, func_id);\n+\tmutex_unlock(\u0026intr_mgt-\u003elock);\n+\n+\treturn ret;\n+}\n+\n+/**\n+ * nbl_res_intr_cfg_msix_map - allocate \u0026 program MSI-X mapping table\n+ * @res_mgt: resource management instance\n+ * @func_id: target function identifier\n+ * @num_net_msix: required net data interrupt vectors\n+ * @num_others_msix: required control interrupt vectors\n+ * @net_msix_mask_en: enable mask for net interrupt entries\n+ *\n+ * Allocate interrupt vectors; MSIX coherent DMA table is allocated once\n+ * per function on first configuration, entries are rewritten while the\n+ * map is invalidated on subsequent reconfigurations. No free/realloc of\n+ * DMA table on vector count changes. This removes the DMA table\n+ * free/realloc cycle. On reconfiguration the map VALID bit is cleared\n+ * and the hardware-DMA quiesce window is observed (lock held) before old\n+ * vectors are recycled and the table is rewritten.\n+ *\n+ * Reconfiguration transiently needs room for the whole new vector set on\n+ * top of the old one, because the old set is only released after the new\n+ * allocation has succeeded (a failed reconfiguration must leave the old\n+ * configuration intact).  The bitmap pools are therefore sized for peak,\n+ * not maximum, concurrent use: a reconfigure of the same or a smaller\n+ * size still fails with -EAGAIN when the pool is nearly exhausted.  The\n+ * in-tree caller (nbl_dev_init_msix_cnt()) asks for one non-net vector\n+ * per PF, far below NBL_MAX_OTHER_INTERRUPT.\n+ *\n+ * Serialization: this function takes intr_mgt-\u003elock internally to\n+ * protect the global vector bitmaps and per-function state against\n+ * concurrent callers.\n+ *\n+ * Return: 0 on success, negative errno on failure\n+ */\n+int nbl_res_intr_cfg_msix_map(struct nbl_resource_mgt *res_mgt,\n+\t\t\t      u16 func_id, u16 num_net_msix,\n+\t\t\t      u16 num_others_msix,\n+\t\t\t      bool net_msix_mask_en)\n+{\n+\tstruct nbl_interrupt_mgt *intr_mgt = res_mgt-\u003eintr_mgt;\n+\tstruct nbl_hw_ops *hw_ops = res_mgt-\u003ehw_ops_tbl-\u003eops;\n+\tstruct nbl_common_info *common = res_mgt-\u003ecommon;\n+\tstruct nbl_msix_map_table *official_tbl;\n+\tstruct nbl_msix_map *msix_map_entries;\n+\tstruct device *dev = common-\u003edev;\n+\tu16 requested, intr_index;\n+\tu8 bus, devid, function;\n+\tbool entry_masked = false;\n+\tu16 *tmp_interrupts = NULL;\n+\tu16 allocated_cnt = 0;\n+\tu16 *old_interrupts;\n+\tu16 old_num;\n+\tbool had_config;\n+\tint ret = 0;\n+\tu16 gvec;\n+\tu16 i, j;\n+\n+\tif (!intr_mgt)\n+\t\treturn -EINVAL;\n+\n+\tif (!common-\u003ehas_ctrl)\n+\t\treturn -EINVAL;\n+\n+\tif (func_id \u003e= NBL_MAX_PF) {\n+\t\tdev_err(dev, \"Invalid func_id %u\\n\", func_id);\n+\t\treturn -EINVAL;\n+\t}\n+\n+\tif (num_net_msix == 0 \u0026\u0026 num_others_msix == 0) {\n+\t\tdev_err(dev, \"MSI-X vector count cannot both be zero\\n\");\n+\t\treturn -EINVAL;\n+\t}\n+\n+\tif (num_net_msix \u003e NBL_MSIX_MAP_TABLE_MAX_ENTRIES ||\n+\t    num_others_msix \u003e NBL_MSIX_MAP_TABLE_MAX_ENTRIES) {\n+\t\tdev_err(dev, \"MSI-X count out of limit: net=%u, others=%u\\n\",\n+\t\t\tnum_net_msix, num_others_msix);\n+\t\treturn -EINVAL;\n+\t}\n+\n+\tif (check_add_overflow(num_net_msix, num_others_msix, \u0026requested) ||\n+\t    requested \u003e NBL_MSIX_MAP_TABLE_MAX_ENTRIES) {\n+\t\tdev_err(dev, \"Total MSI-X vectors %u exceeds maximum %u\\n\",\n+\t\t\trequested, NBL_MSIX_MAP_TABLE_MAX_ENTRIES);\n+\t\treturn -EINVAL;\n+\t}\n+\n+\tret = nbl_res_func_id_to_bdf(res_mgt, func_id, \u0026bus, \u0026devid, \u0026function);\n+\tif (ret) {\n+\t\tif (ret == -EOPNOTSUPP)\n+\t\t\tdev_err(dev,\n+\t\t\t\t\"MSI-X mapping: func_id=%u is not a PF present on this board\\n\",\n+\t\t\t\tfunc_id);\n+\t\treturn ret;\n+\t}\n+\n+\tmutex_lock(\u0026intr_mgt-\u003elock);\n+\tofficial_tbl = \u0026intr_mgt-\u003efunc_intr_res[func_id].msix_map_table;\n+\n+\t/* Reject new configs once global teardown has started */\n+\tif (intr_mgt-\u003estopping) {\n+\t\tret = -ESHUTDOWN;\n+\t\tgoto out_unlock;\n+\t}\n+\n+\thad_config = intr_mgt-\u003efunc_intr_res[func_id].state ==\n+\t\t     NBL_INTR_FUNC_CONFIGURED;\n+\n+\t/*\n+\t * Phase1: allocate global vector array first.\n+\t * Allocate the fixed-size MSIX DMA table only ONCE for this function.\n+\t */\n+\ttmp_interrupts = kcalloc(requested, sizeof(*tmp_interrupts),\n+\t\t\t\t GFP_KERNEL);\n+\tif (!tmp_interrupts) {\n+\t\tret = -ENOMEM;\n+\t\tgoto out_unlock;\n+\t}\n+\t/* Allocate MSIX DMA table once per function */\n+\tif (!official_tbl-\u003ebase_addr) {\n+\t\tofficial_tbl-\u003esize =\n+\t\t\tsizeof(struct nbl_msix_map) *\n+\t\t\tNBL_MSIX_MAP_TABLE_MAX_ENTRIES;\n+\t\tofficial_tbl-\u003ebase_addr = dma_alloc_coherent(dev,\n+\t\t\t\t\t\t\t     official_tbl-\u003esize,\n+\t\t\t\t\t\t\t     \u0026official_tbl-\u003edma,\n+\t\t\t\t\t\t\t     GFP_KERNEL);\n+\t\tif (!official_tbl-\u003ebase_addr) {\n+\t\t\tdev_err(dev, \"Failed to allocate DMA memory for MSIX table\\n\");\n+\t\t\tret = -ENOMEM;\n+\t\t\tgoto release_vecs_unlock;\n+\t\t}\n+\t}\n+\n+\t/* Allocate net interrupt vectors */\n+\tfor (i = 0; i \u003c num_net_msix; i++) {\n+\t\tintr_index = find_first_zero_bit(intr_mgt-\u003eintr_net_bmap,\n+\t\t\t\t\t\t NBL_MAX_NET_INTERRUPT);\n+\t\tif (intr_index == NBL_MAX_NET_INTERRUPT) {\n+\t\t\tdev_err(dev, \"No free net interrupt vectors left\\n\");\n+\t\t\tret = -EAGAIN;\n+\t\t\tgoto release_vecs_unlock;\n+\t\t}\n+\t\ttmp_interrupts[i] = intr_index + NBL_NET_INTR_BASE;\n+\t\tset_bit(intr_index, intr_mgt-\u003eintr_net_bmap);\n+\t\tallocated_cnt++;\n+\t}\n+\n+\t/* Allocate other interrupt vectors */\n+\tfor (; i \u003c requested; i++) {\n+\t\tintr_index =\n+\t\t\tfind_first_zero_bit(intr_mgt-\u003eintr_other_bmap,\n+\t\t\t\t\t    NBL_MAX_OTHER_INTERRUPT);\n+\t\tif (intr_index == NBL_MAX_OTHER_INTERRUPT) {\n+\t\t\tdev_err(dev, \"No free control interrupt vectors left\\n\");\n+\t\t\tret = -EAGAIN;\n+\t\t\tgoto release_vecs_unlock;\n+\t\t}\n+\t\ttmp_interrupts[i] = intr_index;\n+\t\tset_bit(intr_index, intr_mgt-\u003eintr_other_bmap);\n+\t\tallocated_cnt++;\n+\t}\n+\n+\t/*\n+\t * Phase2: quiesce the old hardware MSIX config before touching\n+\t * the live DMA table. Same sequence as destroy:\n+\t *   disable mailbox routing -\u003e invalidate per-vector INFO -\u003e\n+\t *   clear map VALID -\u003e flush -\u003e wait for in-flight table fetches.\n+\t * The lock stays held across the wait, so no concurrent caller\n+\t * can program the quiesced function. Only then are old vectors\n+\t * recycled.\n+\t * NOTE: NO DMA table free here.\n+\t */\n+\tif (had_config) {\n+\t\told_interrupts =\n+\t\t\tintr_mgt-\u003efunc_intr_res[func_id].interrupts;\n+\t\told_num = intr_mgt-\u003efunc_intr_res[func_id].num_interrupts;\n+\n+\t\tret = __nbl_res_intr_set_mailbox_irq(intr_mgt, res_mgt, func_id,\n+\t\t\t\t\t\t     0, false);\n+\t\tif (ret) {\n+\t\t\tdev_err(dev, \"%s: disable old mailbox irq failed, keep old config\\n\",\n+\t\t\t\t__func__);\n+\t\t\tgoto release_vecs_unlock;\n+\t\t}\n+\t\tfor (j = 0; j \u003c old_num; j++) {\n+\t\t\thw_ops-\u003ecfg_msix_info(res_mgt-\u003ehw_ops_tbl-\u003epriv,\n+\t\t\t\t\t      func_id, false,\n+\t\t\t\t\t      old_interrupts[j],\n+\t\t\t\t\t      0, 0, 0, false);\n+\t\t}\n+\t\thw_ops-\u003ecfg_msix_map(res_mgt-\u003ehw_ops_tbl-\u003epriv, func_id,\n+\t\t\t\t     false, official_tbl-\u003edma,\n+\t\t\t\t     0, 0, 0);\n+\t\thw_ops-\u003eflush_write(res_mgt-\u003ehw_ops_tbl-\u003epriv);\n+\n+\t\tnbl_intr_quiesce_wait();\n+\n+\t\tnbl_intr_release_bitmap(intr_mgt, res_mgt, old_interrupts,\n+\t\t\t\t\told_num);\n+\t\tkfree(old_interrupts);\n+\t\tintr_mgt-\u003efunc_intr_res[func_id].interrupts = NULL;\n+\t\tintr_mgt-\u003efunc_intr_res[func_id].num_interrupts = 0;\n+\t\tintr_mgt-\u003efunc_intr_res[func_id].num_net_interrupts = 0;\n+\t}\n+\n+\t/* Swap new vector array into func state */\n+\tintr_mgt-\u003efunc_intr_res[func_id].interrupts = tmp_interrupts;\n+\tintr_mgt-\u003efunc_intr_res[func_id].num_interrupts = requested;\n+\tintr_mgt-\u003efunc_intr_res[func_id].num_net_interrupts = num_net_msix;\n+\ttmp_interrupts = NULL;\n+\n+\t/*\n+\t * First configuration of this function.  Re-invalidate the entry\n+\t * and take a quiesce window before the table below is rewritten:\n+\t * while the entry is VALID the pcompleter could fetch a table\n+\t * that is being written and route stale gvecs.\n+\t *\n+\t * This is a second, real quiesce window rather than part of the\n+\t * reconfiguration path, and it costs one delay per function on the\n+\t * first configuration only.  Reusing the table removes the\n+\t * free/realloc cycle and the destroy-path quiesce window from\n+\t * every *re*configuration; it does not remove this one.\n+\t */\n+\tif (!had_config) {\n+\t\thw_ops-\u003ecfg_msix_map(res_mgt-\u003ehw_ops_tbl-\u003epriv, func_id,\n+\t\t\t\t     false, 0, 0, 0, 0);\n+\t\thw_ops-\u003eflush_write(res_mgt-\u003ehw_ops_tbl-\u003epriv);\n+\t\tnbl_intr_quiesce_wait();\n+\t}\n+\n+\t/*\n+\t * Rewrite the table in the pre-allocated DMA buffer while the\n+\t * map is invalid, so the device cannot observe a torn old/new\n+\t * mix. Only entries beyond requested count need explicit zeroing.\n+\t */\n+\tmsix_map_entries = official_tbl-\u003ebase_addr;\n+\tmemset(msix_map_entries + requested, 0,\n+\t       (NBL_MSIX_MAP_TABLE_MAX_ENTRIES - requested) *\n+\t       sizeof(*msix_map_entries));\n+\n+\tfor (i = 0; i \u003c requested; i++) {\n+\t\tgvec = intr_mgt-\u003efunc_intr_res[func_id].interrupts[i];\n+\t\tmsix_map_entries[i].data =\n+\t\t\tcpu_to_le16(FIELD_PREP(NBL_MSIX_MAP_VALID_MASK, 1) |\n+\t\t\t\t    FIELD_PREP(NBL_MSIX_MAP_INDEX_MASK,\n+\t\t\t\t\t       gvec));\n+\t}\n+\n+\t/* Ensure coherent table writes are visible before HW fetch/enable */\n+\tdma_wmb();\n+\n+\t/* Enable per-vector INFO entries after the table is published */\n+\tfor (i = 0; i \u003c requested; i++) {\n+\t\tgvec = intr_mgt-\u003efunc_intr_res[func_id].interrupts[i];\n+\t\tentry_masked = (i \u003c num_net_msix \u0026\u0026 net_msix_mask_en);\n+\t\thw_ops-\u003ecfg_msix_info(res_mgt-\u003ehw_ops_tbl-\u003epriv,\n+\t\t\t\t      func_id, true, gvec,\n+\t\t\t\t      bus, devid, function,\n+\t\t\t\t      entry_masked);\n+\t}\n+\thw_ops-\u003eflush_write(res_mgt-\u003ehw_ops_tbl-\u003epriv);\n+\n+\t/*\n+\t * Point the map at the table last and set VALID.\n+\t *\n+\t * cfg_msix_map uses the control PF's own BDF (common-\u003ehw_bus etc.),\n+\t * not the target function's BDF.  This BDF tags the pcompler DMA\n+\t * read of the MSI-X map table as originating from the control PF.\n+\t * The target function's BDF (bus/devid/function from\n+\t * nbl_res_func_id_to_bdf) is used only in cfg_msix_info for the\n+\t * host_msix_ctrl table entry BDF filtering.\n+\t */\n+\thw_ops-\u003ecfg_msix_map(res_mgt-\u003ehw_ops_tbl-\u003epriv, func_id,\n+\t\t\t     true, official_tbl-\u003edma, common-\u003ehw_bus,\n+\t\t\t     common-\u003edevid, common-\u003efunction);\n+\thw_ops-\u003eflush_write(res_mgt-\u003ehw_ops_tbl-\u003epriv);\n+\n+\tintr_mgt-\u003efunc_intr_res[func_id].state = NBL_INTR_FUNC_CONFIGURED;\n+\tmutex_unlock(\u0026intr_mgt-\u003elock);\n+\treturn 0;\n+\n+release_vecs_unlock:\n+\tnbl_intr_release_bitmap(intr_mgt, res_mgt, tmp_interrupts,\n+\t\t\t\tallocated_cnt);\n+\tkfree(tmp_interrupts);\n+\t/*\n+\t * On a failed fresh configuration, release the DMA table\n+\t * allocated during this call. On a failed reconfiguration the\n+\t * old configuration is still intact (it is only torn down\n+\t * after all vector allocations succeed) and owns the table.\n+\t */\n+\tif (!had_config \u0026\u0026 official_tbl-\u003ebase_addr) {\n+\t\tdma_free_coherent(dev, official_tbl-\u003esize,\n+\t\t\t\t  official_tbl-\u003ebase_addr,\n+\t\t\t\t  official_tbl-\u003edma);\n+\t\tofficial_tbl-\u003ebase_addr = NULL;\n+\t\tofficial_tbl-\u003edma = 0;\n+\t\tofficial_tbl-\u003esize = 0;\n+\t}\n+out_unlock:\n+\tmutex_unlock(\u0026intr_mgt-\u003elock);\n+\treturn ret;\n+}\n+\n+/**\n+ * nbl_res_intr_set_mailbox_irq - bind mailbox IRQ to specified vector\n+ * @res_mgt: resource management instance\n+ * @func_id: target function identifier\n+ * @vector_id: index inside local interrupt array\n+ * @en_msix: enable/disable mailbox interrupt\n+ *\n+ * Serialization: takes intr_mgt-\u003elock internally and passes the\n+ * pointer it read (before taking the lock) down to the helpers, which\n+ * must not re-read res_mgt-\u003eintr_mgt: nbl_intr_mgt_stop() clears that\n+ * pointer while a caller may already be blocked on this mutex.\n+ *\n+ * Return: 0 on success, negative errno on parameter or state check\n+ * failure.  The hardware op is void and cannot report failure.\n+ */\n+int nbl_res_intr_set_mailbox_irq(struct nbl_resource_mgt *res_mgt,\n+\t\t\t\t u16 func_id, u16 vector_id,\n+\t\t\t\t bool en_msix)\n+{\n+\tstruct nbl_interrupt_mgt *intr_mgt = res_mgt-\u003eintr_mgt;\n+\tstruct nbl_common_info *common = res_mgt-\u003ecommon;\n+\tint ret;\n+\n+\tif (!intr_mgt)\n+\t\treturn -EINVAL;\n+\n+\tif (!common-\u003ehas_ctrl)\n+\t\treturn -EINVAL;\n+\n+\tmutex_lock(\u0026intr_mgt-\u003elock);\n+\tret = __nbl_res_intr_set_mailbox_irq(intr_mgt, res_mgt, func_id,\n+\t\t\t\t\t     vector_id, en_msix);\n+\tmutex_unlock(\u0026intr_mgt-\u003elock);\n+\n+\treturn ret;\n+}\n+\n+/*\n+ * Reset the software state and invalidate every function's hardware MSI-X\n+ * map entry once.  The chip-internal tables (FUNCTION_MSIX_MAP,\n+ * PADPT_HOST_MSIX_INFO, HOST_MSIX_FID_TABLE) can survive a kexec or a\n+ * forced unload without FLR and there is no .shutdown callback to scrub\n+ * them.  Only FUNCTION_MSIX_MAP is scrubbed here: it is indexed by\n+ * func_id, so covering all NBL_MAX_PF entries costs one register write\n+ * each, after which the pcompleter can no longer fetch a stale table that\n+ * points into memory this kernel may have reallocated.  The INFO/FID\n+ * tables are indexed by global vector id and are not cleared; instead a\n+ * map entry is always invalidated by nbl_res_intr_cfg_msix_map() and by\n+ * the teardown paths before it is armed for a new configuration.\n+ *\n+ * No sibling PF is live at this point: the device links make them probe\n+ * only after this PF and remove before it, so clearing all NBL_MAX_PF\n+ * entries cannot disturb a running function.\n+ */\n+static struct nbl_interrupt_mgt *\n+nbl_intr_setup_mgt(struct nbl_resource_mgt *res_mgt)\n+{\n+\tstruct device *dev = res_mgt-\u003ecommon-\u003edev;\n+\tstruct nbl_hw_ops *hw_ops = res_mgt-\u003ehw_ops_tbl-\u003eops;\n+\tstruct nbl_interrupt_mgt *intr_mgt;\n+\tu16 func_id;\n+\tint err;\n+\n+\tintr_mgt = devm_kzalloc(dev, sizeof(*intr_mgt), GFP_KERNEL);\n+\tif (!intr_mgt)\n+\t\treturn ERR_PTR(-ENOMEM);\n+\n+\terr = devm_mutex_init(dev, \u0026intr_mgt-\u003elock);\n+\tif (err)\n+\t\treturn ERR_PTR(err);\n+\n+\tintr_mgt-\u003estopping = false;\n+\tbitmap_zero(intr_mgt-\u003eintr_net_bmap, NBL_MAX_NET_INTERRUPT);\n+\tbitmap_zero(intr_mgt-\u003eintr_other_bmap, NBL_MAX_OTHER_INTERRUPT);\n+\n+\t/*\n+\t * Invalidate any map entry left VALID by a previous kernel.  One\n+\t * flush plus one quiesce window covers the outstanding table\n+\t * fetches, same as the teardown paths do.\n+\t *\n+\t * This is the scrub that makes it safe to enable bus mastering:\n+\t * nbl_probe() calls it with bus mastering still disabled, so no\n+\t * new fetch can start while these entries are still VALID.\n+\t */\n+\tfor (func_id = 0; func_id \u003c NBL_MAX_PF; func_id++)\n+\t\thw_ops-\u003ecfg_msix_map(res_mgt-\u003ehw_ops_tbl-\u003epriv, func_id,\n+\t\t\t\t     false, 0, 0, 0, 0);\n+\thw_ops-\u003eflush_write(res_mgt-\u003ehw_ops_tbl-\u003epriv);\n+\tnbl_intr_quiesce_wait();\n+\n+\treturn intr_mgt;\n+}\n+\n+int nbl_intr_mgt_start(struct nbl_resource_mgt *res_mgt)\n+{\n+\tstruct nbl_interrupt_mgt *intr_mgt;\n+\tint ret;\n+\n+\tintr_mgt = nbl_intr_setup_mgt(res_mgt);\n+\tif (IS_ERR(intr_mgt)) {\n+\t\tret = PTR_ERR(intr_mgt);\n+\t\treturn ret;\n+\t}\n+\n+\tres_mgt-\u003eintr_mgt = intr_mgt;\n+\treturn 0;\n+}\n+\n+/*\n+ * nbl_intr_mgt_stop - global control-PF interrupt teardown\n+ *\n+ * Phase 1 sets the stopping latch and invalidates every configured\n+ * function's hardware map entry while holding the lock.  The lock is\n+ * dropped for the global quiesce window, so a caller that races the\n+ * window is rejected by the latch (checked under the lock), not by the\n+ * lock being held.\n+ *\n+ * The latch is what rejects a caller that raced the window; it is not\n+ * what makes the manager safe to touch.  No caller is in flight here by\n+ * construction: nbl_remove() clears bus mastering and stops the device\n+ * before reaching this point, and every sibling PF - the only remote\n+ * source of these calls - is unbound first through its\n+ * DL_FLAG_AUTOPROBE_CONSUMER device link.  A caller that did latch the\n+ * pointer earlier and then gets in under the lock hits the latch\n+ * instead:\n+ *   - the config path, destroy_msix_map and the enable path of\n+ *     nbl_res_intr_set_mailbox_irq() report -ESHUTDOWN.  The latch is\n+ *     checked ahead of that path's per-function state check, so its\n+ *     -ENODEV is only reachable while the manager is not stopping.\n+ *   - the en_msix=false disable path consults neither and returns 0,\n+ *     which is what lets teardown disarm routing after the latch is set.\n+ * Once this returns res_mgt-\u003eintr_mgt is NULL and the public entry\n+ * points report -EINVAL.\n+ */\n+void nbl_intr_mgt_stop(struct nbl_resource_mgt *res_mgt)\n+{\n+\tstruct nbl_interrupt_mgt *intr_mgt = res_mgt-\u003eintr_mgt;\n+\tu16 func_id;\n+\tint ret;\n+\n+\tif (!intr_mgt)\n+\t\treturn;\n+\n+\t/*\n+\t * Phase 1: set the stopping latch and invalidate all hardware\n+\t * MSIX map entries.  A caller racing the quiesce window below is\n+\t * either still waiting for the lock, in which case it observes\n+\t * stopping == true at its next checkpoint once it is let in, or\n+\t * it already holds the lock, in which case this phase waits for\n+\t * it to finish.  The latch is never cleared.\n+\t *\n+\t * The en_msix=false mailbox path is exempt from the latch on\n+\t * purpose: this phase and the per-function teardown use it to\n+\t * disarm routing, and it only rewrites bits that are already\n+\t * clear by then.\n+\t */\n+\tmutex_lock(\u0026intr_mgt-\u003elock);\n+\tintr_mgt-\u003estopping = true;\n+\tfor (func_id = 0; func_id \u003c NBL_MAX_PF; func_id++) {\n+\t\tif (intr_mgt-\u003efunc_intr_res[func_id].state ==\n+\t\t    NBL_INTR_FUNC_CONFIGURED) {\n+\t\t\tdev_info(res_mgt-\u003ecommon-\u003edev,\n+\t\t\t\t \"intr_mgt_stop: preparing destroy map for func %u\\n\",\n+\t\t\t\t func_id);\n+\t\t\tret = __nbl_res_intr_prepare_destroy_msix_map(intr_mgt,\n+\t\t\t\t\t\t\t\t      res_mgt,\n+\t\t\t\t\t\t\t\t      func_id);\n+\t\t\tif (ret)\n+\t\t\t\tdev_warn(res_mgt-\u003ecommon-\u003edev,\n+\t\t\t\t\t \"intr_mgt_stop: prepare destroy map for func %u failed: %d\\n\",\n+\t\t\t\t\t func_id, ret);\n+\t\t}\n+\t}\n+\tmutex_unlock(\u0026intr_mgt-\u003elock);\n+\n+\t/*\n+\t * Global quiesce: wait for straggler DMA table reads after all\n+\t * MSIX map entries have been invalidated in hardware, before\n+\t * freeing coherent memory.  Best-effort only, see\n+\t * nbl_intr_quiesce_wait().\n+\t */\n+\tnbl_intr_quiesce_wait();\n+\n+\t/* Phase2: safely release MSIX coherent memory and intr resources */\n+\tmutex_lock(\u0026intr_mgt-\u003elock);\n+\tfor (func_id = 0; func_id \u003c NBL_MAX_PF; func_id++) {\n+\t\tif (intr_mgt-\u003efunc_intr_res[func_id].state ==\n+\t\t    NBL_INTR_FUNC_DESTROYING) {\n+\t\t\tret = __nbl_res_intr_complete_destroy_msix_map(intr_mgt,\n+\t\t\t\t\t\t\t\t       res_mgt,\n+\t\t\t\t\t\t\t\t       func_id);\n+\t\t\tif (ret)\n+\t\t\t\tdev_warn(res_mgt-\u003ecommon-\u003edev,\n+\t\t\t\t\t \"intr_mgt_stop: complete destroy map for func %u failed: %d\\n\",\n+\t\t\t\t\t func_id, ret);\n+\t\t}\n+\t}\n+\t/*\n+\t * Unpublish last.  Readers do not synchronize on intr_mgt-\u003elock,\n+\t * so this only has to happen after every hardware access and\n+\t * every free above; a caller that latched the pointer earlier is\n+\t * rejected by the stopping latch.\n+\t */\n+\tres_mgt-\u003eintr_mgt = NULL;\n+\tmutex_unlock(\u0026intr_mgt-\u003elock);\n+}\ndiff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_interrupt.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_interrupt.h\nnew file mode 100644\nindex 00000000000000..9f66f5e19c984b\n--- /dev/null\n+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_interrupt.h\n@@ -0,0 +1,21 @@\n+/* SPDX-License-Identifier: GPL-2.0 */\n+/*\n+ * Copyright (c) 2026 Nebula Matrix Limited.\n+ */\n+\n+#ifndef _NBL_INTERRUPT_H_\n+#define _NBL_INTERRUPT_H_\n+\n+#include \"nbl_resource.h\"\n+\n+#define NBL_MSIX_MAP_TABLE_MAX_ENTRIES\t1024\n+int nbl_res_intr_destroy_msix_map(struct nbl_resource_mgt *res_mgt,\n+\t\t\t\t  u16 func_id);\n+int nbl_res_intr_cfg_msix_map(struct nbl_resource_mgt *res_mgt,\n+\t\t\t      u16 func_id, u16 num_net_msix,\n+\t\t\t      u16 num_others_msix,\n+\t\t\t      bool net_msix_mask_en);\n+int nbl_res_intr_set_mailbox_irq(struct nbl_resource_mgt *res_mgt,\n+\t\t\t\t u16 func_id, u16 vector_id,\n+\t\t\t\t bool en_msix);\n+#endif\ndiff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_resource.c b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_resource.c\nnew file mode 100644\nindex 00000000000000..49c7d4412e38ad\n--- /dev/null\n+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_resource.c\n@@ -0,0 +1,157 @@\n+// SPDX-License-Identifier: GPL-2.0\n+/*\n+ * Copyright (c) 2026 Nebula Matrix Limited.\n+ */\n+\n+#include \u003clinux/pci.h\u003e\n+#include \"nbl_resource.h\"\n+\n+int nbl_res_func_id_to_vsi_id(struct nbl_resource_mgt *res_mgt, u16 func_id,\n+\t\t\t      u16 type, u16 *vsi_id)\n+{\n+\tstruct nbl_vsi_info *vsi_info = res_mgt-\u003eresource_info-\u003evsi_info;\n+\tenum nbl_vsi_serv_type dst_type = NBL_VSI_SERV_PF_DATA_TYPE;\n+\tstruct nbl_common_info *common = res_mgt-\u003ecommon;\n+\tstruct device *dev = res_mgt-\u003ecommon-\u003edev;\n+\tint pfid = func_id;\n+\tu32 rel_pf_id;\n+\tint ret;\n+\n+\tif (!common-\u003ehas_ctrl || !vsi_id) {\n+\t\tdev_dbg(dev, \"No control plane or null vsi output ptr\\n\");\n+\t\treturn -EINVAL;\n+\t}\n+\tret = nbl_common_func_id_to_rel_pf_id(common, pfid, \u0026rel_pf_id);\n+\tif (ret)\n+\t\treturn ret;\n+\tif (rel_pf_id \u003e= vsi_info-\u003enum) {\n+\t\tdev_err(dev, \"PF %d (diff=%u) exceeds vsi_info-\u003enum (%u)\\n\",\n+\t\t\tpfid, rel_pf_id, vsi_info-\u003enum);\n+\t\treturn -EINVAL;\n+\t}\n+\n+\tret = nbl_res_pf_dev_vsi_type_to_hw_vsi_type(res_mgt, type, \u0026dst_type);\n+\tif (ret) {\n+\t\tdev_err(dev, \"Invalid vsi type %u func_id %u\\n\", type, func_id);\n+\t\treturn ret;\n+\t}\n+\t*vsi_id = vsi_info-\u003eserv_info[rel_pf_id][dst_type].base_id;\n+\treturn 0;\n+}\n+\n+int nbl_res_vsi_id_to_pf_id(struct nbl_resource_mgt *res_mgt, u16 vsi_id)\n+{\n+\tstruct nbl_vsi_info *vsi_info = res_mgt-\u003eresource_info-\u003evsi_info;\n+\tstruct nbl_common_info *common = res_mgt-\u003ecommon;\n+\tstruct device *dev = res_mgt-\u003ecommon-\u003edev;\n+\tint j = NBL_VSI_SERV_PF_DATA_TYPE;\n+\tint pf_id, i;\n+\n+\tif (!common-\u003ehas_ctrl) {\n+\t\tdev_dbg(dev, \"No control plane available\\n\");\n+\t\treturn -EINVAL;\n+\t}\n+\tfor (i = 0; i \u003c vsi_info-\u003enum; i++) {\n+\t\tif (vsi_id \u003e= vsi_info-\u003eserv_info[i][j].base_id \u0026\u0026\n+\t\t    (vsi_id \u003c vsi_info-\u003eserv_info[i][j].base_id +\n+\t\t\t\t\tvsi_info-\u003eserv_info[i][j].num)) {\n+\t\t\tpf_id = i + common-\u003emgt_pf;\n+\t\t\tif (pf_id \u003e= NBL_MAX_PF) {\n+\t\t\t\tdev_err(dev, \"PF ID overflow\\n\");\n+\t\t\t\treturn -ERANGE;\n+\t\t\t}\n+\t\t\treturn pf_id;\n+\t\t}\n+\t}\n+\n+\tdev_dbg(dev, \"VSI ID %u not found\\n\", vsi_id);\n+\treturn -ENOENT;\n+}\n+\n+int nbl_res_func_id_to_bdf(struct nbl_resource_mgt *res_mgt, u16 func_id,\n+\t\t\t   u8 *bus, u8 *dev, u8 *function)\n+{\n+\tstruct nbl_common_info *common = res_mgt-\u003ecommon;\n+\tstruct nbl_sriov_info *sriov_info;\n+\tint pfid = func_id;\n+\tu8 pf_bus, devfn;\n+\tu32 rel_pf_id;\n+\tint ret;\n+\n+\tif (!common-\u003ehas_ctrl || !bus || !dev || !function)\n+\t\treturn -EINVAL;\n+\tret = nbl_common_func_id_to_rel_pf_id(common, pfid, \u0026rel_pf_id);\n+\tif (ret)\n+\t\treturn ret;\n+\tif (rel_pf_id \u003e= common-\u003emax_pf) {\n+\t\tdev_err(common-\u003edev,\n+\t\t\t\"func_id=%u rel_pf_id=%u exceeds max_pf=%u, no such PF on this board\\n\",\n+\t\t\tpfid, rel_pf_id,\n+\t\t\tcommon-\u003emax_pf);\n+\t\treturn -EOPNOTSUPP;\n+\t}\n+\tsriov_info = res_mgt-\u003eresource_info-\u003esriov_info + rel_pf_id;\n+\tpf_bus = PCI_BUS_NUM(sriov_info-\u003ebdf);\n+\tdevfn = sriov_info-\u003ebdf \u0026 0xff;\n+\t*bus = pf_bus;\n+\t*dev = PCI_SLOT(devfn);\n+\t*function = PCI_FUNC(devfn);\n+\n+\treturn 0;\n+}\n+\n+int nbl_res_get_eth_id(struct nbl_resource_mgt *res_mgt, u16 func_id,\n+\t\t       u16 vsi_id, u8 *eth_num, u8 *eth_id, u8 *logic_eth_id)\n+{\n+\tstruct nbl_eth_info *eth_info = res_mgt-\u003eresource_info-\u003eeth_info;\n+\tstruct nbl_common_info *common = res_mgt-\u003ecommon;\n+\tstruct device *dev = res_mgt-\u003ecommon-\u003edev;\n+\tint pfid = func_id;\n+\tint rel_pf_id;\n+\tint abs_pf_id;\n+\n+\tif (!common-\u003ehas_ctrl || !eth_num || !eth_id || !logic_eth_id)\n+\t\treturn -EINVAL;\n+\tabs_pf_id = nbl_res_vsi_id_to_pf_id(res_mgt, vsi_id);\n+\tif (abs_pf_id \u003c 0) {\n+\t\tdev_err(dev, \"Failed to get PF ID from VSI ID %u\\n\", vsi_id);\n+\t\treturn -EINVAL;\n+\t}\n+\tif (abs_pf_id != pfid) {\n+\t\tdev_err(dev, \"func_id %u does not match pf derived from vsi_id %u\\n\",\n+\t\t\tpfid, vsi_id);\n+\t\treturn -EINVAL;\n+\t}\n+\trel_pf_id = abs_pf_id - common-\u003emgt_pf;\n+\n+\tif (rel_pf_id \u003e= eth_info-\u003eeth_num) {\n+\t\tdev_err(dev, \"rel_pf_id %d out of range [0, %u)\\n\",\n+\t\t\trel_pf_id, eth_info-\u003eeth_num);\n+\t\treturn -ERANGE;\n+\t}\n+\n+\t*eth_num = eth_info-\u003eeth_num;\n+\t*eth_id = eth_info-\u003eeth_id[rel_pf_id];\n+\t/*\n+\t * Logical eth id equals the relative PF id: eth_info setup enforces\n+\t * max_pf == eth_num and fills eth_id[] in ascending bitmap-bit\n+\t * order, so the Nth PF owns the Nth logical port.\n+\t */\n+\t*logic_eth_id = rel_pf_id;\n+\treturn 0;\n+}\n+\n+int nbl_res_pf_dev_vsi_type_to_hw_vsi_type(struct nbl_resource_mgt *res_mgt,\n+\t\t\t\t\t   u16 src_type,\n+\t\t\t\t\t   enum nbl_vsi_serv_type *dst_type)\n+{\n+\tswitch (src_type) {\n+\tcase NBL_VSI_DATA:\n+\t\t*dst_type = NBL_VSI_SERV_PF_DATA_TYPE;\n+\t\treturn 0;\n+\tdefault:\n+\t\tdev_err_once(res_mgt-\u003ecommon-\u003edev,\n+\t\t\t     \"Unsupported vsi src_type %u\\n\", src_type);\n+\t\treturn -EINVAL;\n+\t}\n+}\ndiff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_resource.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_resource.h\nnew file mode 100644\nindex 00000000000000..24cf49475bbd90\n--- /dev/null\n+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_resource.h\n@@ -0,0 +1,159 @@\n+/* SPDX-License-Identifier: GPL-2.0 */\n+/*\n+ * Copyright (c) 2026 Nebula Matrix Limited.\n+ */\n+\n+#ifndef _NBL_RESOURCE_H_\n+#define _NBL_RESOURCE_H_\n+\n+#include \u003clinux/types.h\u003e\n+\n+#include \"../nbl_include/nbl_include.h\"\n+#include \"../nbl_include/nbl_def_channel.h\"\n+#include \"../nbl_include/nbl_def_hw.h\"\n+#include \"../nbl_include/nbl_def_resource.h\"\n+#include \"../nbl_include/nbl_def_common.h\"\n+#include \"../nbl_core.h\"\n+\n+struct nbl_resource_mgt;\n+\n+/* --------- INTERRUPT ---------- */\n+#define NBL_MAX_OTHER_INTERRUPT\t\t\t1024\n+#define NBL_MAX_NET_INTERRUPT\t\t\t4096\n+#define NBL_NET_INTR_BASE\t\tNBL_MAX_OTHER_INTERRUPT\n+\n+#define NBL_MSIX_MAP_VALID_MASK\t\tBIT(0)\n+#define NBL_MSIX_MAP_INDEX_MASK\t\tGENMASK(13, 1)\n+#define NBL_MSIX_MAP_RSV_MASK\t\tGENMASK(15, 14)\n+\n+struct nbl_msix_map {\n+\t__le16 data;\n+};\n+\n+struct nbl_msix_map_table {\n+\tstruct nbl_msix_map *base_addr;\n+\tdma_addr_t dma;\n+\tsize_t size;\n+};\n+\n+/*\n+ * Per-function MSI-X resource state.\n+ *\n+ * IDLE       - nothing configured for this function.\n+ * CONFIGURED - map valid in hardware, vectors owned by this function.\n+ * DESTROYING - map invalidated, teardown in flight: neither the vector\n+ *              bitmaps nor the coherent table may be released until the\n+ *              hardware-DMA quiesce window has elapsed.\n+ *\n+ * The single-function destroy path holds intr_mgt-\u003elock across that\n+ * window.  nbl_intr_mgt_stop() cannot, because it quiesces every\n+ * function at once: it drops the lock and relies on the stopping latch\n+ * to keep a concurrent configuration from installing a map that the\n+ * in-flight teardown would free.\n+ */\n+enum nbl_intr_func_state {\n+\tNBL_INTR_FUNC_IDLE = 0,\n+\tNBL_INTR_FUNC_CONFIGURED,\n+\tNBL_INTR_FUNC_DESTROYING,\n+};\n+\n+struct nbl_func_interrupt_resource_mng {\n+\tu16 num_interrupts;\n+\tu16 num_net_interrupts;\n+\tu16 *interrupts;\n+\tstruct nbl_msix_map_table msix_map_table;\n+\tu8 state; /* enum nbl_intr_func_state */\n+};\n+\n+struct nbl_interrupt_mgt {\n+\tstruct mutex lock; /* Protects the bitmaps + func_intr_res[] */\n+\tDECLARE_BITMAP(intr_net_bmap, NBL_MAX_NET_INTERRUPT);\n+\tDECLARE_BITMAP(intr_other_bmap, NBL_MAX_OTHER_INTERRUPT);\n+\t/* One-way latch: set on teardown, never cleared */\n+\tbool stopping;\n+\t/*\n+\t * Indexed by the raw hardware function id.  Only PF entries are ever\n+\t * used: the public entry points reject ids outside the PF range (VFs\n+\t * included) with -EINVAL, and nbl_res_intr_cfg_msix_map() also\n+\t * rejects a PF that is not present on this board.  The destroy and\n+\t * mailbox paths only bound-check NBL_MAX_PF; they rely on their\n+\t * callers passing either common-\u003emgt_pf or an id that the config\n+\t * path has already validated.\n+\t */\n+\tstruct nbl_func_interrupt_resource_mng func_intr_res[NBL_MAX_PF];\n+};\n+\n+/* --------- INFO ---------- */\n+struct nbl_sriov_info {\n+\tunsigned int bdf;\n+};\n+\n+struct nbl_eth_info {\n+\tu8 eth_num;\n+\tu8 resv[3];\n+\tu8 eth_id[NBL_MAX_ETHERNET];\n+};\n+\n+enum nbl_vsi_serv_type {\n+\tNBL_VSI_SERV_PF_DATA_TYPE,\n+\tNBL_VSI_SERV_MAX_TYPE,\n+};\n+\n+struct nbl_vsi_serv_info {\n+\tu16 base_id;\n+\tu16 num;\n+};\n+\n+struct nbl_vsi_info {\n+\tu16 num;\n+\tstruct nbl_vsi_serv_info serv_info[NBL_MAX_ETHERNET]\n+\t\t\t\t\t  [NBL_VSI_SERV_MAX_TYPE];\n+};\n+\n+struct nbl_resource_info {\n+\tstruct nbl_sriov_info *sriov_info;\n+\tstruct nbl_eth_info *eth_info;\n+\tstruct nbl_vsi_info *vsi_info;\n+\tstruct nbl_board_port_info board_info;\n+};\n+\n+struct nbl_resource_mgt {\n+\tstruct nbl_common_info *common;\n+\tstruct nbl_resource_info *resource_info;\n+\tstruct nbl_channel_ops_tbl *chan_ops_tbl;\n+\tstruct nbl_hw_ops_tbl *hw_ops_tbl;\n+\t/*\n+\t * Published interrupt manager, control PF only.  The public entry\n+\t * points read this pointer without the manager lock and then pass\n+\t * the value they read down to the internal helpers;\n+\t * nbl_intr_mgt_stop() clears it under the manager lock as its last\n+\t * step, after every hardware access and every free, so a caller\n+\t * that reads it after that gets -EINVAL.\n+\t *\n+\t * The object is devres-owned and is released together with the\n+\t * device rather than by nbl_intr_mgt_stop(), so a latched pointer\n+\t * is never dereferenced into freed memory.  That is not what keeps\n+\t * a concurrent caller safe, though: see nbl_intr_mgt_stop() for\n+\t * the ordering that makes sure there is none.  A caller that\n+\t * latched the pointer before it was cleared and then gets in under\n+\t * the lock hits the stopping latch: the config path,\n+\t * destroy_msix_map and the mailbox-enable path report -ESHUTDOWN,\n+\t * while the mailbox-disable path (en_msix == false) returns 0\n+\t * without consulting the latch.\n+\t */\n+\tstruct nbl_interrupt_mgt *intr_mgt;\n+};\n+\n+int nbl_res_vsi_id_to_pf_id(struct nbl_resource_mgt *res_mgt, u16 vsi_id);\n+int nbl_res_func_id_to_vsi_id(struct nbl_resource_mgt *res_mgt, u16 func_id,\n+\t\t\t      u16 type, u16 *vsi_id);\n+int nbl_res_func_id_to_bdf(struct nbl_resource_mgt *res_mgt, u16 func_id,\n+\t\t\t   u8 *bus, u8 *dev, u8 *function);\n+int nbl_res_get_eth_id(struct nbl_resource_mgt *res_mgt, u16 func_id,\n+\t\t       u16 vsi_id, u8 *eth_num, u8 *eth_id, u8 *logic_eth_id);\n+int nbl_intr_mgt_start(struct nbl_resource_mgt *res_mgt);\n+int nbl_res_pf_dev_vsi_type_to_hw_vsi_type(struct nbl_resource_mgt *res_mgt,\n+\t\t\t\t\t   u16 src_type,\n+\t\t\t\t\t   enum nbl_vsi_serv_type *dst_type);\n+void nbl_intr_mgt_stop(struct nbl_resource_mgt *res_mgt);\n+#endif\ndiff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_channel.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_channel.h\nnew file mode 100644\nindex 00000000000000..5aa16d4cd4321f\n--- /dev/null\n+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_channel.h\n@@ -0,0 +1,175 @@\n+/* SPDX-License-Identifier: GPL-2.0 */\n+/*\n+ * Copyright (c) 2026 Nebula Matrix Limited.\n+ */\n+\n+#ifndef _NBL_DEF_CHANNEL_H_\n+#define _NBL_DEF_CHANNEL_H_\n+\n+#include \u003clinux/types.h\u003e\n+#include \u003clinux/build_bug.h\u003e\n+\n+struct nbl_channel_mgt;\n+struct nbl_adapter;\n+\n+typedef void (*nbl_chan_resp)(void *, u16, u16, void *, u32);\n+\n+/*\n+ * Mailbox wire opcodes, stable wire ABI shared between driver and firmware.\n+ * Each opcode has a fixed assigned number to preserve compatibility.\n+ * ABI compatibility rules:\n+ * 1. New opcodes shall only be appended before NBL_CHAN_MSG_MAILBOX_MAX;\n+ * 2. Reordering, inserting or deleting existing enumerators breaks driver-\n+ *    firmware interoperability and must be avoided;\n+ * 3. Modifications to existing opcodes require synchronized firmware ABI\n+ *    updates.\n+ *\n+ * Only opcodes currently used by in-tree driver logic are defined here.\n+ * Unimplemented feature opcodes (KTLS, IPsec, vDPA, mirror etc.) will be\n+ * added incrementally together with their corresponding driver\n+ * implementation patches.\n+ */\n+enum nbl_chan_msg_type {\n+\tNBL_CHAN_MSG_ACK = 0,\n+\tNBL_CHAN_MSG_CONFIGURE_MSIX_MAP = 17,\n+\tNBL_CHAN_MSG_DESTROY_MSIX_MAP = 18,\n+\tNBL_CHAN_MSG_MAILBOX_SET_IRQ = 19,\n+\tNBL_CHAN_MSG_GET_VSI_ID = 21,\n+\tNBL_CHAN_MSG_GET_ETH_ID = 67,\n+\t/* mailbox msg end */\n+\tNBL_CHAN_MSG_MAILBOX_MAX,\n+};\n+\n+enum nbl_chan_state {\n+\tNBL_CHAN_IRQ_RDY,\n+\tNBL_CHAN_STATE_NBITS\n+};\n+\n+struct nbl_chan_param_cfg_msix_map {\n+\t__le16 num_net_msix;\n+\t__le16 num_others_msix;\n+\t__le16 msix_mask_en;\n+\t__le16 rsvd;\n+};\n+\n+struct nbl_chan_param_set_mailbox_irq {\n+\t__le16 vector_id;\n+\tu8 en_msix;\n+\tu8 rsvd;\n+};\n+\n+struct nbl_chan_param_get_vsi_id {\n+\t__le16 vsi_id;\n+\t__le16 type;\n+};\n+\n+struct nbl_chan_param_get_eth_id {\n+\t__le16 vsi_id;\n+\tu8 eth_num;\n+\tu8 eth_id;\n+\tu8 logic_eth_id;\n+\tu8 rsvd[3];\n+};\n+\n+struct nbl_board_port_info {\n+\tu8 eth_num;\n+\tu8 eth_speed;\n+\tu8 p4_version;\n+\tu8 rsv[5];\n+};\n+\n+static_assert(sizeof(struct nbl_chan_param_cfg_msix_map) == 8,\n+\t      \"nbl_chan_param_cfg_msix_map size must be 8 bytes\");\n+static_assert(sizeof(struct nbl_chan_param_set_mailbox_irq) == 4,\n+\t      \"nbl_chan_param_set_mailbox_irq size must be 4 bytes\");\n+static_assert(sizeof(struct nbl_chan_param_get_vsi_id) == 4,\n+\t      \"nbl_chan_param_get_vsi_id size must be 4 bytes\");\n+static_assert(sizeof(struct nbl_chan_param_get_eth_id) == 8,\n+\t      \"nbl_chan_param_get_eth_id size must be 8 bytes\");\n+static_assert(sizeof(struct nbl_board_port_info) == 8,\n+\t      \"nbl_board_port_info size must be 8 bytes\");\n+\n+struct nbl_chan_send_info {\n+\tvoid *arg;\n+\tsize_t arg_len;\n+\tvoid *resp;\n+\tsize_t resp_len;\n+\tu16 dstid;\n+\tu16 msg_type;\n+\tu16 ack;\n+\tu16 ack_len;\n+};\n+\n+struct nbl_chan_ack_info {\n+\tvoid *data;\n+\tint err;\n+\tu32 data_len;\n+\tu16 dstid;\n+\tu16 msg_type;\n+\tu16 msgid;\n+};\n+\n+enum nbl_channel_type {\n+\tNBL_CHAN_TYPE_MAILBOX,\n+\tNBL_CHAN_TYPE_MAX\n+};\n+\n+static inline void\n+nbl_chan_fill_send_info(struct nbl_chan_send_info *info,\n+\t\t\tu16 dst_id, u16 msg_type,\n+\t\t\tvoid *argument, u32 arg_length,\n+\t\t\tvoid *response, u32 resp_length,\n+\t\t\tbool need_ack)\n+{\n+\tinfo-\u003edstid = dst_id;\n+\tinfo-\u003emsg_type = msg_type;\n+\tinfo-\u003earg = argument;\n+\tinfo-\u003earg_len = arg_length;\n+\tinfo-\u003eresp = response;\n+\tinfo-\u003eresp_len = resp_length;\n+\tinfo-\u003eack = need_ack;\n+}\n+\n+static inline void\n+nbl_chan_fill_ack_info(struct nbl_chan_ack_info *info,\n+\t\t       u16 dst_id, u16 msg_type, u16 msg_id,\n+\t\t       int err_code, void *ack_data, u32 data_length)\n+{\n+\tinfo-\u003edstid = dst_id;\n+\tinfo-\u003emsg_type = msg_type;\n+\tinfo-\u003emsgid = msg_id;\n+\tinfo-\u003eerr = err_code;\n+\tinfo-\u003edata = ack_data;\n+\tinfo-\u003edata_len = data_length;\n+}\n+\n+struct nbl_channel_ops {\n+\tint (*send_msg)(struct nbl_channel_mgt *chan_mgt,\n+\t\t\tstruct nbl_chan_send_info *chan_send);\n+\tint (*send_ack)(struct nbl_channel_mgt *chan_mgt,\n+\t\t\tstruct nbl_chan_ack_info *chan_ack);\n+\tint (*register_msg)(struct nbl_channel_mgt *chan_mgt, u16 msg_type,\n+\t\t\t    nbl_chan_resp func, void *callback_priv);\n+\tvoid (*cfg_chan_qinfo_map_table)(struct nbl_channel_mgt *chan_mgt,\n+\t\t\t\t\t u8 bus, u8 devid);\n+\tbool (*check_queue_exist)(struct nbl_channel_mgt *chan_mgt,\n+\t\t\t\t  u8 chan_type);\n+\tint (*setup_queue)(struct nbl_channel_mgt *chan_mgt, u8 chan_type);\n+\tint (*teardown_queue)(struct nbl_channel_mgt *chan_mgt, u8 chan_type);\n+\tvoid (*clean_queue_subtask)(struct nbl_channel_mgt *chan_mgt,\n+\t\t\t\t    u8 chan_type);\n+\tvoid (*register_chan_task)(struct nbl_channel_mgt *chan_mgt,\n+\t\t\t\t   u8 chan_type, struct work_struct *task);\n+\tvoid (*set_queue_state)(struct nbl_channel_mgt *chan_mgt,\n+\t\t\t\tenum nbl_chan_state state, u8 chan_type,\n+\t\t\t\tu8 set);\n+};\n+\n+struct nbl_channel_ops_tbl {\n+\tstruct nbl_channel_ops *ops;\n+\tstruct nbl_channel_mgt *priv;\n+};\n+\n+int nbl_chan_init_common(struct nbl_adapter *adapter);\n+void nbl_chan_remove_common(struct nbl_adapter *adapter);\n+#endif\ndiff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_common.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_common.h\nindex ea646d1efc2dfb..3e18a2a97850e6 100644\n--- a/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_common.h\n+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_common.h\n@@ -11,7 +11,24 @@\n #include \u003clinux/device.h\u003e\n #include \"nbl_include.h\"\n \n+#define NBL_TWO_ETHERNET_PORT\t\t\t2\n+#define NBL_FOUR_ETHERNET_PORT\t\t\t4\n+#define NBL_DEFAULT_VSI_ID_GAP\t\t\t1024\n+#define NBL_TWO_ETHERNET_VSI_ID_GAP\t\t512\n+#define NBL_FOUR_ETHERNET_VSI_ID_GAP\t\t256\n+\n+static inline u32 nbl_vsi_id_gap(u32 m)\n+{\n+\tif (m == NBL_FOUR_ETHERNET_PORT)\n+\t\treturn NBL_FOUR_ETHERNET_VSI_ID_GAP;\n+\telse if (m == NBL_TWO_ETHERNET_PORT)\n+\t\treturn NBL_TWO_ETHERNET_VSI_ID_GAP;\n+\n+\treturn NBL_DEFAULT_VSI_ID_GAP;\n+}\n+\n struct nbl_common_info {\n+\tstruct workqueue_struct *wq;\n \tstruct pci_dev *pdev;\n \tstruct device *dev;\n \tu16 vsi_id;\n@@ -23,9 +40,15 @@ struct nbl_common_info {\n \tu8 devid;\n \tu8 bus;\n \tu8 hw_bus;\n+\tu16 mgt_pf;\n \n \tu8 has_ctrl;\n \tu8 has_net;\n+\tu8 max_pf;\n };\n \n+void nbl_common_destroy_wq(struct nbl_common_info *common);\n+int nbl_common_create_wq(struct nbl_common_info *common);\n+int nbl_common_func_id_to_rel_pf_id(struct nbl_common_info *common, u32 pf_id,\n+\t\t\t\t    u32 *rel_pf_id);\n #endif\ndiff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_dev.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_dev.h\nnew file mode 100644\nindex 00000000000000..a66c633a0e7af7\n--- /dev/null\n+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_dev.h\n@@ -0,0 +1,16 @@\n+/* SPDX-License-Identifier: GPL-2.0 */\n+/*\n+ * Copyright (c) 2026 Nebula Matrix Limited.\n+ */\n+\n+#ifndef _NBL_DEF_DEV_H_\n+#define _NBL_DEF_DEV_H_\n+\n+struct nbl_adapter;\n+\n+int nbl_dev_init(struct nbl_adapter *adapter);\n+void nbl_dev_remove(struct nbl_adapter *adapter);\n+int nbl_dev_start(struct nbl_adapter *adapter);\n+void nbl_dev_stop(struct nbl_adapter *adapter);\n+\n+#endif\ndiff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_dispatch.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_dispatch.h\nnew file mode 100644\nindex 00000000000000..7da9e242c621e4\n--- /dev/null\n+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_dispatch.h\n@@ -0,0 +1,83 @@\n+/* SPDX-License-Identifier: GPL-2.0 */\n+/*\n+ * Copyright (c) 2026 Nebula Matrix Limited.\n+ */\n+\n+#ifndef _NBL_DEF_DISPATCH_H_\n+#define _NBL_DEF_DISPATCH_H_\n+\n+#include \u003clinux/types.h\u003e\n+\n+struct nbl_dispatch_mgt;\n+struct nbl_adapter;\n+enum {\n+\tNBL_DISP_CTRL_LVL_MGT,\n+\tNBL_DISP_CTRL_LVL_NET,\n+\tNBL_DISP_CTRL_LVL_MAX,\n+};\n+\n+/**\n+ * struct nbl_dispatch_ops - dispatch control plane operation callbacks\n+ * @init_module: chip-wide datapath init, forwarded to the resource layer\n+ *               (res_ops-\u003einit_module() and then hw_ops-\u003einit_module());\n+ *               on leonis it programs the chip-wide datapath registers and\n+ *               sets driver_status=true.  Returns -EOPNOTSUPP if the\n+ *               resource layer installs no init_module op.  Control-PF\n+ *               exclusive: the pointer is NULL on a function without\n+ *               has_ctrl, so the caller must check has_ctrl first.\n+ * @deinit_module: chip-wide teardown, forwarded the same way; on leonis\n+ *               it clears driver_status, which starts an asynchronous\n+ *               firmware per-PF cleanup pass with no completion status\n+ *               bit, waited for best-effort only.  The caller must have\n+ *               stopped the mailbox channel and drained all in-flight\n+ *               mailbox DMA before calling it, see nbl_dev_remove().\n+ *               Returns void.  Control-PF exclusive, same NULL rule as\n+ *               @init_module.\n+ * @cfg_msix_map: configure function msix mapping table\n+ * @destroy_msix_map: tear down msix mapping resource\n+ * @set_mailbox_irq: bind mailbox interrupt to specified msix vector\n+ * @get_vsi_id: resolve VSI ID by type\n+ * @get_eth_id: resolve eth port info from VSI ID\n+ *\n+ * These callbacks are not dispatch-layer setup/teardown: the dispatch\n+ * structures are devm-allocated in nbl_disp_init(), so nbl_disp_remove()\n+ * has nothing to undo. The five responders registered by\n+ * nbl_disp_setup_msg() are owned by the channel layer and go away with\n+ * it, so they need no explicit unregistration either; the teardown order\n+ * that makes that safe is documented with the code that performs it, in\n+ * the next patch in this series. They forward chip/firmware control down\n+ * to the layers below instead: a control-PF function calls the resource\n+ * ops directly, a non-control function sends a mailbox request over the\n+ * channel layer.\n+ *\n+ * Warning: init_module/deinit_module are control-PF exclusive. The five\n+ * resource ops (cfg_msix_map, destroy_msix_map, set_mailbox_irq,\n+ * get_vsi_id, get_eth_id) are PF-only and resolve to either a local\n+ * resource call (control PF) or a mailbox RPC (non-control PF with\n+ * has_net). A function with neither has_ctrl nor has_net leaves these\n+ * pointers NULL; callers must not invoke them on such functions. VFs are\n+ * rejected by the responders with -EPERM.\n+ */\n+struct nbl_dispatch_ops {\n+\tint (*init_module)(struct nbl_dispatch_mgt *disp_mgt);\n+\tvoid (*deinit_module)(struct nbl_dispatch_mgt *disp_mgt);\n+\tint (*cfg_msix_map)(struct nbl_dispatch_mgt *disp_mgt,\n+\t\t\t    u16 num_net_msix, u16 num_others_msix,\n+\t\t\t    bool net_msix_mask_en);\n+\tint (*destroy_msix_map)(struct nbl_dispatch_mgt *disp_mgt);\n+\tint (*set_mailbox_irq)(struct nbl_dispatch_mgt *disp_mgt,\n+\t\t\t       u16 vector_id, bool en_msix);\n+\tint (*get_vsi_id)(struct nbl_dispatch_mgt *disp_mgt, u16 type,\n+\t\t\t  u16 *vsi_id);\n+\tint (*get_eth_id)(struct nbl_dispatch_mgt *disp_mgt, u16 vsi_id,\n+\t\t\t  u8 *eth_num, u8 *eth_id, u8 *logic_eth_id);\n+};\n+\n+struct nbl_dispatch_ops_tbl {\n+\tstruct nbl_dispatch_ops *ops;\n+\tstruct nbl_dispatch_mgt *priv;\n+};\n+\n+int nbl_disp_init(struct nbl_adapter *adapter);\n+void nbl_disp_remove(struct nbl_adapter *adapter);\n+#endif\ndiff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_hw.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_hw.h\nindex ecbf440e436667..e9cd3136252d86 100644\n--- a/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_hw.h\n+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_hw.h\n@@ -8,8 +8,88 @@\n \n #include \u003clinux/types.h\u003e\n \n+struct nbl_board_port_info;\n struct nbl_hw_mgt;\n struct nbl_adapter;\n+struct nbl_hw_ops {\n+\tint (*init_module)(struct nbl_hw_mgt *hw_mgt, u8 eth_speed, u8 eth_num);\n+\tvoid (*deinit_module)(struct nbl_hw_mgt *hw_mgt);\n+\n+\tvoid (*cfg_msix_map)(struct nbl_hw_mgt *hw_mgt, u16 func_id,\n+\t\t\t     bool valid, dma_addr_t dma_addr, u8 bus,\n+\t\t\t     u8 devid, u8 function);\n+\tvoid (*cfg_msix_info)(struct nbl_hw_mgt *hw_mgt, u16 func_id,\n+\t\t\t      bool valid, u16 interrupt_id, u8 bus,\n+\t\t\t      u8 devid, u8 function,\n+\t\t\t      bool net_msix_mask_en);\n+\t/**\n+\t * flush_write - Flush posted MEMORY-BAR writes\n+\t * @hw_mgt: hardware management context\n+\t *\n+\t * Only valid on the control PF (common-\u003ehas_ctrl != 0).  The\n+\t * register read back to order the writes, NBL_HW_DUMMY_REG\n+\t * (0x1300904), is covered by the ~64 MiB MEMORY-BAR mapping the\n+\t * control PF gets, but sits outside the NBL_REG_NET_ONLY_LEN\n+\t * (8 KiB) mapping a net-only PF gets.  The leonis implementation\n+\t * enforces this with a WARN_ON_ONCE().\n+\t */\n+\tvoid (*flush_write)(struct nbl_hw_mgt *hw_mgt);\n+\tvoid (*update_mailbox_queue_tail_ptr)(struct nbl_hw_mgt *hw_mgt,\n+\t\t\t\t\t      u16 tail_ptr, u8 txrx);\n+\tvoid (*config_mailbox_rxq)(struct nbl_hw_mgt *hw_mgt,\n+\t\t\t\t   dma_addr_t dma_addr, int size_bwid);\n+\tvoid (*config_mailbox_txq)(struct nbl_hw_mgt *hw_mgt,\n+\t\t\t\t   dma_addr_t dma_addr, int size_bwid);\n+\tvoid (*stop_mailbox_rxq)(struct nbl_hw_mgt *hw_mgt);\n+\tvoid (*stop_mailbox_txq)(struct nbl_hw_mgt *hw_mgt);\n+\t/**\n+\t * get_host_pf_mask - Fetch host PF mask from firmware k_pf_mask reg\n+\t * @hw_mgt: hardware management context\n+\t * @pf_mask: output pointer for PF mask value\n+\t *\n+\t * k_pf_mask register rule:\n+\t *   bit N == 0 -\u003e PF#N enabled; bit N == 1 -\u003e PF#N masked out.\n+\t *   bit0 is PF0's mask bit (not reserved); PF0 can be masked but\n+\t *   the driver requires at least PF0 enabled.\n+\t *   Only 1/2/4 PFs are supported:\n+\t *     1 PF  (PF0):     mask = 0xfe\n+\t *     2 PFs (PF0,PF1): mask = 0xfc\n+\t *     4 PFs (PF0~PF3): mask = 0xf0\n+\t *   All-zero mask (0x00) means all 8 PFs enabled, which is\n+\t *   unsupported by the driver.\n+\t *\n+\t * This op only returns the raw register value and does not validate\n+\t * it.  Neither consumer exists yet at the commit that introduces it:\n+\t * the resource layer, which validates the value, and the channel\n+\t * layer, which programs the mailbox QINFO table from it, both arrive\n+\t * later in this series, and validation runs before the channel layer\n+\t * reads the mask.\n+\t */\n+\tvoid (*get_host_pf_mask)(struct nbl_hw_mgt *hw_mgt, u32 *pf_mask);\n+\tvoid (*get_real_bus)(struct nbl_hw_mgt *hw_mgt, u8 *bus);\n+\n+\tvoid (*cfg_mailbox_qinfo)(struct nbl_hw_mgt *hw_mgt, u16 func_id,\n+\t\t\t\t  u8 bus, u8 devid, u8 function);\n+\tvoid (*set_mailbox_irq)(struct nbl_hw_mgt *hw_mgt, u16 func_id,\n+\t\t\t\tbool en_msix, u16 gvec);\n+\tvoid (*get_fw_eth_map)(struct nbl_hw_mgt *hw_mgt, u32 *eth_map);\n+\t/**\n+\t * get_board_info - Fetch board info from firmware\n+\t * @hw_mgt: hardware management context\n+\t * @board_info: output pointer for board info structure\n+\t *\n+\t * Firmware contract: board_info.eth_num MUST equal the number of\n+\t * unmasked PFs from get_host_pf_mask(). See get_host_pf_mask for\n+\t * details.\n+\t */\n+\tvoid (*get_board_info)(struct nbl_hw_mgt *hw_mgt,\n+\t\t\t       struct nbl_board_port_info *board_info);\n+};\n+\n+struct nbl_hw_ops_tbl {\n+\tstruct nbl_hw_ops *ops;\n+\tstruct nbl_hw_mgt *priv;\n+};\n \n int nbl_hw_init_leonis(struct nbl_adapter *adapter);\n void nbl_hw_remove_leonis(struct nbl_adapter *adapter);\ndiff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_resource.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_resource.h\nnew file mode 100644\nindex 00000000000000..8dc64e806c1e2b\n--- /dev/null\n+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_resource.h\n@@ -0,0 +1,38 @@\n+/* SPDX-License-Identifier: GPL-2.0 */\n+/*\n+ * Copyright (c) 2026 Nebula Matrix Limited.\n+ */\n+\n+#ifndef _NBL_DEF_RESOURCE_H_\n+#define _NBL_DEF_RESOURCE_H_\n+\n+#include \u003clinux/types.h\u003e\n+\n+struct nbl_resource_mgt;\n+struct nbl_adapter;\n+\n+struct nbl_resource_ops {\n+\tint (*init_module)(struct nbl_resource_mgt *res_mgt);\n+\tvoid (*deinit_module)(struct nbl_resource_mgt *res_mgt);\n+\n+\tint (*cfg_msix_map)(struct nbl_resource_mgt *res_mgt, u16 func_id,\n+\t\t\t    u16 num_net_msix, u16 num_others_msix,\n+\t\t\t    bool net_msix_mask_en);\n+\tint (*destroy_msix_map)(struct nbl_resource_mgt *res_mgt, u16 func_id);\n+\tint (*set_mailbox_irq)(struct nbl_resource_mgt *res_mgt, u16 func_id,\n+\t\t\t       u16 vector_id, bool en_msix);\n+\tint (*get_vsi_id)(struct nbl_resource_mgt *res_mgt, u16 func_id,\n+\t\t\t  u16 type, u16 *vsi_id);\n+\tint (*get_eth_id)(struct nbl_resource_mgt *res_mgt, u16 func_id,\n+\t\t\t  u16 vsi_id, u8 *eth_num, u8 *eth_id,\n+\t\t\t  u8 *logic_eth_id);\n+};\n+\n+struct nbl_resource_ops_tbl {\n+\tstruct nbl_resource_ops *ops;\n+\tstruct nbl_resource_mgt *priv;\n+};\n+\n+int nbl_res_init_leonis(struct nbl_adapter *adapter);\n+void nbl_res_remove_leonis(struct nbl_adapter *adapter);\n+#endif\ndiff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_include.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_include.h\nindex 14e7b19f9a4cf8..efc4c79a082d25 100644\n--- a/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_include.h\n+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_include.h\n@@ -10,6 +10,15 @@\n \n /*  ------  Basic definitions  -------  */\n #define NBL_DRIVER_NAME\t\t\t\t\t\"nbl\"\n+#define NBL_MAX_PF\t\t\t\t\t8\n+#define NBL_NEXT_ID(id, max) (((id) + 1) % ((max) + 1))\n+\n+#define NBL_MAX_ETHERNET\t\t\t\t4\n+\n+enum {\n+\tNBL_VSI_DATA = 0,\n+};\n+\n struct nbl_func_caps {\n \tu32 has_ctrl:1;\n \tu32 has_net:1;\n@@ -20,4 +29,23 @@ struct nbl_init_param {\n \tstruct nbl_func_caps caps;\n };\n \n+/*\n+ * Firmware ABI defines port speed enum fixed, value 0 represents 10G, cannot\n+ * reassign 0 to INVALID for compatibility\n+ */\n+enum nbl_fw_port_speed {\n+\tNBL_FW_PORT_SPEED_10G,\n+\tNBL_FW_PORT_SPEED_25G,\n+\tNBL_FW_PORT_SPEED_50G,\n+\tNBL_FW_PORT_SPEED_100G,\n+};\n+\n+/*\n+ * Firmware quirk word @ NBL_LEONIS_QUIRKS_OFFSET (0x140)\n+ * Sentinel value: ~0U (0xFFFFFFFF) = firmware reports no active quirks\n+ * BIT(1): NBL_QUIRK_UVN_PREFETCH_ALIGN – control UVN descriptor prefetch\n+ * selection\n+ */\n+#define NBL_QUIRK_UVN_PREFETCH_ALIGN\tBIT(1)\n+\n #endif\ndiff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_main.c b/drivers/net/ethernet/nebula-matrix/nbl/nbl_main.c\nindex f2552bc7329376..0bdc7f9940fcb2 100644\n--- a/drivers/net/ethernet/nebula-matrix/nbl/nbl_main.c\n+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_main.c\n@@ -7,11 +7,34 @@\n #include \u003clinux/pci.h\u003e\n #include \u003clinux/module.h\u003e\n #include \u003clinux/bits.h\u003e\n+#include \u003clinux/delay.h\u003e\n #include \"nbl_include/nbl_include.h\"\n+#include \"nbl_include/nbl_def_channel.h\"\n #include \"nbl_include/nbl_def_hw.h\"\n+#include \"nbl_include/nbl_def_resource.h\"\n+#include \"nbl_include/nbl_def_dispatch.h\"\n+#include \"nbl_include/nbl_def_dev.h\"\n #include \"nbl_include/nbl_def_common.h\"\n #include \"nbl_core.h\"\n \n+/*\n+ * Budget for the device_trylock() loop that takes func 0's device lock in\n+ * nbl_probe_chip_deps().  See the comment there for why a single failed\n+ * trylock must not defer the probe, and why the wait has to stay bounded.\n+ */\n+#define NBL_MGT_LOCK_RETRY_MIN_US\t1000\n+#define NBL_MGT_LOCK_RETRY_MAX_US\t2000\n+#define NBL_MGT_LOCK_MAX_RETRIES\t50\n+int nbl_core_start(struct nbl_adapter *adapter)\n+{\n+\treturn nbl_dev_start(adapter);\n+}\n+\n+void nbl_core_stop(struct nbl_adapter *adapter)\n+{\n+\tnbl_dev_stop(adapter);\n+}\n+\n struct nbl_adapter *nbl_core_init(struct pci_dev *pdev,\n \t\t\t\t  struct nbl_init_param *param)\n {\n@@ -26,6 +49,7 @@ struct nbl_adapter *nbl_core_init(struct pci_dev *pdev,\n \tadapter-\u003epdev = pdev;\n \tcommon = \u0026adapter-\u003ecommon;\n \n+\tcommon-\u003emgt_pf = 0;\n \tcommon-\u003epdev = pdev;\n \tcommon-\u003edev = \u0026pdev-\u003edev;\n \tcommon-\u003ehas_ctrl = param-\u003ecaps.has_ctrl;\n@@ -36,15 +60,43 @@ struct nbl_adapter *nbl_core_init(struct pci_dev *pdev,\n \n \tret = nbl_hw_init_leonis(adapter);\n \tif (ret)\n-\t\tgoto hw_init_fail;\n+\t\tgoto out;\n+\n+\tret = nbl_chan_init_common(adapter);\n+\tif (ret)\n+\t\tgoto out_hw_remove;\n \n+\tret = nbl_res_init_leonis(adapter);\n+\tif (ret)\n+\t\tgoto out_chan_remove;\n+\n+\tret = nbl_disp_init(adapter);\n+\tif (ret)\n+\t\tgoto out_res_remove;\n+\n+\tret = nbl_dev_init(adapter);\n+\tif (ret)\n+\t\tgoto out_disp_remove;\n \treturn adapter;\n-hw_init_fail:\n+\n+out_disp_remove:\n+\tnbl_disp_remove(adapter);\n+out_res_remove:\n+\tnbl_res_remove_leonis(adapter);\n+out_chan_remove:\n+\tnbl_chan_remove_common(adapter);\n+out_hw_remove:\n+\tnbl_hw_remove_leonis(adapter);\n+out:\n \treturn ERR_PTR(ret);\n }\n \n void nbl_core_remove(struct nbl_adapter *adapter)\n {\n+\tnbl_dev_remove(adapter);\n+\tnbl_disp_remove(adapter);\n+\tnbl_res_remove_leonis(adapter);\n+\tnbl_chan_remove_common(adapter);\n \tnbl_hw_remove_leonis(adapter);\n }\n \n@@ -67,6 +119,164 @@ static void nbl_get_func_param(struct pci_dev *pdev, kernel_ulong_t driver_data,\n \t\tparam-\u003ecaps.has_ctrl = 1;\n }\n \n+/*\n+ * Establish chip-wide dependencies for this PF:\n+ *  - for every non-management PF, add a consumer-\u003emanagement PF device\n+ *    link. The driver core then guarantees (sysfs unbind, driver\n+ *    unregister, hot-unplug alike) that this consumer is released\n+ *    BEFORE the func 0 supplier, which is the only teardown order in\n+ *    which the chip-global firmware deinit is safe.\n+ *\n+ * The management PF is addressed by the deterministic identity\n+ * (domain, bus, slot, func 0) instead of any name-based scan: hardware\n+ * guarantees PFs are contiguous from func 0 in the same slot.\n+ *\n+ * The func 0 device must already be bound to this driver.  While func 0\n+ * is still probing, its chip-global init (mailbox QINFO map programming,\n+ * datapath setup) can race a sibling's mailbox setup, and a link created\n+ * while the supplier has no driver starts DORMANT and is force-promoted\n+ * (with a WARN backtrace) by device_links_driver_bound() when this probe\n+ * ends, so it never delivers the unbind-order guarantee.\n+ *\n+ * The link is a persistent managed link with DL_FLAG_AUTOPROBE_CONSUMER:\n+ * the core force-unbinds this consumer when func 0 unbinds (keeping the\n+ * teardown order) and re-queues it for probe when func 0 binds again, so\n+ * the sibling PFs and their ports come back without a manual rebind.\n+ * AUTOREMOVE_CONSUMER would delete the link on unbind and cannot be\n+ * combined with AUTOPROBE_CONSUMER (device_link_add() returns NULL for\n+ * that pair).  The link is therefore not auto-purged on a failed probe:\n+ * it lives with the two devices and is dropped when either is removed.\n+ * The pointer device_link_add() returns is never dereferenced.\n+ *\n+ * Return: 0 on success, negative errno on failure.\n+ */\n+static int nbl_probe_chip_deps(struct pci_dev *pdev, bool has_ctrl)\n+{\n+\tstruct pci_dev *mgt;\n+\tint retries;\n+\n+\tif (has_ctrl)\n+\t\treturn 0;\n+\n+\tmgt = pci_get_domain_bus_and_slot(pci_domain_nr(pdev-\u003ebus),\n+\t\t\t\t\t  pdev-\u003ebus-\u003enumber,\n+\t\t\t\t\t  PCI_DEVFN(PCI_SLOT(pdev-\u003edevfn), 0));\n+\tif (!mgt) {\n+\t\tdev_err(\u0026pdev-\u003edev,\n+\t\t\t\"management PF (func 0) not found on this chip\\n\");\n+\t\treturn -ENODEV;\n+\t}\n+\n+\t/*\n+\t * That identity is meaningless for a VF: under ARI a VF's devfn\n+\t * decodes to function 0 of a slot, so the lookup can resolve to the\n+\t * VF itself.  A VF is never the management PF, so stop here instead\n+\t * of taking our own device lock below.\n+\t */\n+\tif (mgt == pdev) {\n+\t\tdev_err(\u0026pdev-\u003edev,\n+\t\t\t\"management PF lookup matched this device\\n\");\n+\t\tpci_dev_put(mgt);\n+\t\treturn -ENODEV;\n+\t}\n+\n+\t/*\n+\t * func 0's device lock is taken with a trylock.  This probe already\n+\t * runs under device_lock(funcN), so a blocking device_lock(func0)\n+\t * would establish the N -\u003e 0 order, while __pci_bus_lock() (AER\n+\t * recovery, slot reset) walks the bus in devfn order, i.e. 0 -\u003e N,\n+\t * and dev-\u003emutex is lockdep_set_novalidate_class(), so lockdep\n+\t * cannot report that inversion.  A trylock never blocks, so it\n+\t * cannot take part in a deadlock cycle.\n+\t *\n+\t * Holding the lock is what makes the check and the link creation\n+\t * below atomic with respect to func 0: it both gains its driver\n+\t * (driver_bound() -\u003e device_links_driver_bound()) and loses it\n+\t * (__device_release_driver() -\u003e device_links_driver_cleanup())\n+\t * under device_lock(func0), so func 0 can change neither driver nor\n+\t * links.status while we hold it.\n+\t *\n+\t * One failed trylock is nevertheless not enough to defer on.  Func 0\n+\t * binds with device_lock(func0) held, and driver_bound() calls\n+\t * driver_deferred_probe_trigger() before it releases that lock, so\n+\t * the deferred probe work it queues can re-probe this PF while func\n+\t * 0's lock is still taken.  driver_probe_device() only re-arms that\n+\t * work when deferred_trigger_count changes *during* the probe, which\n+\t * it does not here: func 0's increment happened before this probe\n+\t * read the count.  A bare -EPROBE_DEFER would therefore leave this\n+\t * PF on deferred_probe_pending_list with nothing scheduled to run\n+\t * it again.  Retrying closes that window - driver_bound() has only a\n+\t * bus_notify() and a kobject_uevent() left before it drops the lock.\n+\t *\n+\t * The retry budget is what keeps this safe.  device_lock(funcN) is\n+\t * held for the whole loop, so waiting indefinitely would deadlock\n+\t * against __pci_bus_lock(); giving up and deferring releases it and\n+\t * lets the other side make progress.  A holder that is not part of a\n+\t * bind - __pci_bus_lock() during AER recovery or a bus/slot reset,\n+\t * pci_reset_function() through sysfs, an sriov_numvfs write - can\n+\t * outlast the budget, and no bind event follows those, so this PF\n+\t * then stays deferred until some later deferred-probe trigger from\n+\t * anywhere in the system.  There is no way to force one from here.\n+\t */\n+\tfor (retries = 0; !device_trylock(\u0026mgt-\u003edev); retries++) {\n+\t\tif (retries == NBL_MGT_LOCK_MAX_RETRIES) {\n+\t\t\tdev_dbg(\u0026pdev-\u003edev,\n+\t\t\t\t\"management PF device lock busy after %u retries, deferring probe\\n\",\n+\t\t\t\tretries);\n+\t\t\tpci_dev_put(mgt);\n+\t\t\treturn -EPROBE_DEFER;\n+\t\t}\n+\t\tusleep_range(NBL_MGT_LOCK_RETRY_MIN_US,\n+\t\t\t     NBL_MGT_LOCK_RETRY_MAX_US);\n+\t}\n+\n+\t/*\n+\t * The management PF must be bound to *this* driver, with its probe\n+\t * completed.  links.status == DL_DEV_DRIVER_BOUND alone only says\n+\t * \"bound to some driver\", so func 0 on vfio-pci or pci-stub\n+\t * (driver_override) would pass it and this PF would run with func\n+\t * 0's chip-global init never performed.  pdev-\u003edev.driver is our\n+\t * own driver, set by the core before -\u003eprobe() is called.\n+\t *\n+\t * Deferring is what recovers from this: a successful bind\n+\t * re-triggers deferred probing for every pending device\n+\t * (driver_bound() -\u003e driver_deferred_probe_trigger()), so this PF is\n+\t * retried once func 0 is bound to this driver.  That covers a func 0\n+\t * that is not bound yet.  It does not cover a func 0 that is bound\n+\t * while something else holds its device lock, which is what the\n+\t * retry loop above is for and what its budget cannot fully rule out.\n+\t */\n+\tif (mgt-\u003edev.driver != pdev-\u003edev.driver ||\n+\t    mgt-\u003edev.links.status != DL_DEV_DRIVER_BOUND) {\n+\t\tdevice_unlock(\u0026mgt-\u003edev);\n+\t\tpci_dev_put(mgt);\n+\t\treturn -EPROBE_DEFER;\n+\t}\n+\n+\t/*\n+\t * Create the link while func 0's lock is still held, so that func 0\n+\t * cannot change state in between.  With the supplier bound and this\n+\t * consumer PROBING, device_link_init_status() starts the link in\n+\t * DL_STATE_CONSUMER_PROBE: that is the state\n+\t * device_links_driver_bound() expects when this probe finishes, and\n+\t * it is what device_links_busy() reports to func 0's own unbind, so\n+\t * func 0 cannot be released underneath this PF.\n+\t */\n+\tif (!device_link_add(\u0026pdev-\u003edev, \u0026mgt-\u003edev,\n+\t\t\t     DL_FLAG_AUTOPROBE_CONSUMER)) {\n+\t\tdevice_unlock(\u0026mgt-\u003edev);\n+\t\tdev_err(\u0026pdev-\u003edev,\n+\t\t\t\"failed to create device link to management PF\\n\");\n+\t\tpci_dev_put(mgt);\n+\t\treturn -ENOMEM;\n+\t}\n+\n+\tdevice_unlock(\u0026mgt-\u003edev);\n+\tpci_dev_put(mgt);\n+\n+\treturn 0;\n+}\n+\n static int nbl_probe(struct pci_dev *pdev,\n \t\t     const struct pci_device_id *id)\n {\n@@ -75,28 +285,68 @@ static int nbl_probe(struct pci_dev *pdev,\n \tstruct nbl_adapter *adapter;\n \tint err;\n \n+\tnbl_get_func_param(pdev, id-\u003edriver_data, \u0026param);\n+\t/*\n+\t * Reject a function with neither capability before touching any\n+\t * hardware.  nbl_disp_refresh_ctrl_ops() leaves every dispatch op NULL\n+\t * for such a function, while nbl_dev_start() reaches cfg_msix_map,\n+\t * get_vsi_id, get_eth_id and (on unwind) destroy_msix_map through\n+\t * those pointers.  The state is reachable through driver_override:\n+\t * binding this driver to a function that is not in nbl_id_table (a VF,\n+\t * or a non-func-0 PF) makes pci_match_device() pass the dummy ID whose\n+\t * driver_data 0 leaves has_ctrl and has_net both clear.  A foreign\n+\t * device at func 0 is unaffected: nbl_get_func_param() gives any\n+\t * func 0 device has_ctrl, so it is not rejected here and still runs\n+\t * nbl_core_init() with the local control dispatch ops, as it did\n+\t * before this check existed.\n+\t */\n+\tif (!param.caps.has_ctrl \u0026\u0026 !param.caps.has_net) {\n+\t\tdev_err(dev, \"Function has neither ctrl nor net capability\\n\");\n+\t\treturn -EOPNOTSUPP;\n+\t}\n+\n \terr = pcim_enable_device(pdev);\n \tif (err) {\n \t\tdev_err(\u0026pdev-\u003edev, \"Failed to enable PCI dev, err=%d\\n\", err);\n \t\treturn err;\n \t}\n \n-\tnbl_get_func_param(pdev, id-\u003edriver_data, \u0026param);\n \t/* never return fail when DMA_BIT_MASK(64) */\n \tdma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));\n \n-\tpci_set_master(pdev);\n+\t/*\n+\t * Bus mastering is enabled only after nbl_core_init() returns.\n+\t *\n+\t * FUNCTION_MSIX_MAP survives a kexec or a forced unload without\n+\t * FLR and there is no .shutdown callback to scrub it, so at this\n+\t * point every map entry left VALID by a previous kernel still\n+\t * holds a DMA address that this kernel has already reallocated.\n+\t * Those entries tag their table fetches as bus-master reads, and\n+\t * the only thing standing between such a fetch and a read of\n+\t * reused memory is Bus Master Enable, which the PCI core cleared\n+\t * on the way into this kernel.  nbl_intr_setup_mgt() invalidates\n+\t * the entries from inside nbl_core_init(); enabling bus mastering\n+\t * before that would re-open the window.  Everything up to that\n+\t * point is host-side setup plus MMIO and nothing in it waits for\n+\t * the device, so it does not need bus mastering.\n+\t */\n+\terr = nbl_probe_chip_deps(pdev, param.caps.has_ctrl);\n+\tif (err)\n+\t\treturn err;\n \n \tadapter = nbl_core_init(pdev, \u0026param);\n \tif (IS_ERR(adapter)) {\n \t\tdev_err(dev, \"Nbl adapter init fail: %pe\\n\", adapter);\n-\t\terr = PTR_ERR(adapter);\n-\t\tgoto adapter_init_err;\n+\t\treturn PTR_ERR(adapter);\n \t}\n+\n+\tpci_set_master(pdev);\n \tpci_set_drvdata(pdev, adapter);\n-\treturn 0;\n-adapter_init_err:\n-\tpci_clear_master(pdev);\n+\terr = nbl_core_start(adapter);\n+\tif (err) {\n+\t\tnbl_core_remove(adapter);\n+\t\tpci_clear_master(pdev);\n+\t}\n \treturn err;\n }\n \n@@ -107,9 +357,28 @@ static void nbl_remove(struct pci_dev *pdev)\n \tif (!adapter)\n \t\treturn;\n \tpci_set_drvdata(pdev, NULL);\n-\tnbl_core_remove(adapter);\n \n+\tnbl_core_stop(adapter);\n+\t/*\n+\t * Stop bus mastering before any teardown runs.\n+\t *\n+\t * nbl_core_remove() releases the coherent MSI-X map tables whose\n+\t * DMA addresses are still programmed in FUNCTION_MSIX_MAP, and the\n+\t * only thing standing between a late pcompleter fetch and a read\n+\t * of freed memory is a bounded best-effort delay - the chip\n+\t * exposes no completion status for those fetches (see\n+\t * nbl_intr_quiesce_wait()).  Clearing Bus Master Enable first\n+\t * keeps the device from starting a new fetch at all, so that\n+\t * residual window only covers requests already in flight.\n+\t *\n+\t * nbl_core_stop() above has already torn down this function's own\n+\t * map, so what is left to protect here is the control PF's table of\n+\t * per-function entries, freed from nbl_intr_mgt_stop() inside\n+\t * nbl_core_remove().\n+\t */\n \tpci_clear_master(pdev);\n+\n+\tnbl_core_remove(adapter);\n }\n \n /*\n"
}

Target architecture: amd64

For your convenience, here is the diff of the changes:
commit eb6ce827881d54724a7b175422cd39cff38773db
Author: syz-cluster <triage@syzkaller.com>
Date:   Mon Nov 2 14:59:56 2026 +0000

    syz-cluster: applied patch under review

diff --git a/drivers/net/ethernet/nebula-matrix/nbl/Makefile b/drivers/net/ethernet/nebula-matrix/nbl/Makefile
index cc060cf8bf7596..71fbe3ee7e6204 100644
--- a/drivers/net/ethernet/nebula-matrix/nbl/Makefile
+++ b/drivers/net/ethernet/nebula-matrix/nbl/Makefile
@@ -3,5 +3,13 @@
 
 obj-$(CONFIG_NBL) := nbl.o
 
-nbl-objs +=	nbl_hw/nbl_hw_leonis/nbl_hw_leonis.o \
+nbl-objs +=	nbl_common/nbl_common.o \
+		nbl_channel/nbl_channel.o \
+		nbl_hw/nbl_hw_leonis/nbl_hw_leonis.o \
+		nbl_hw/nbl_hw_leonis/nbl_resource_leonis.o \
+		nbl_hw/nbl_resource.o \
+		nbl_hw/nbl_interrupt.o \
+		nbl_hw/nbl_chip.o \
+		nbl_core/nbl_dispatch.o \
+		nbl_core/nbl_dev.o \
 		nbl_main.o
diff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_channel/nbl_channel.c b/drivers/net/ethernet/nebula-matrix/nbl/nbl_channel/nbl_channel.c
new file mode 100644
index 00000000000000..5b528cc6efc452
--- /dev/null
+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_channel/nbl_channel.c
@@ -0,0 +1,1528 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2026 Nebula Matrix Limited.
+ */
+#include <linux/delay.h>
+#include <linux/device.h>
+#include <linux/mutex.h>
+#include <linux/bitfield.h>
+#include <linux/pci.h>
+#include <linux/bits.h>
+#include <linux/dma-mapping.h>
+#include <linux/atomic.h>
+#include <linux/wait.h>
+#include "nbl_channel.h"
+
+static int nbl_chan_add_msg_handler(struct nbl_channel_mgt *chan_mgt,
+				    u16 msg_type, nbl_chan_resp func,
+				    void *priv)
+{
+	struct nbl_chan_msg_node_data *handler;
+	int ret;
+
+	handler = kzalloc_obj(*handler, GFP_KERNEL);
+	if (!handler)
+		return -ENOMEM;
+
+	handler->func = func;
+	handler->priv = priv;
+
+	/* Each msg_type is registered at most once; reject duplicates. */
+	mutex_lock(&chan_mgt->handler_lock);
+	ret = xa_insert(&chan_mgt->handler_xa, msg_type, handler, GFP_KERNEL);
+	mutex_unlock(&chan_mgt->handler_lock);
+	if (ret)
+		kfree(handler);
+
+	return ret;
+}
+
+static int nbl_chan_init_msg_handler(struct nbl_channel_mgt *chan_mgt)
+{
+	int ret;
+
+	ret = devm_mutex_init(chan_mgt->common->dev, &chan_mgt->handler_lock);
+	if (ret)
+		return ret;
+
+	xa_init(&chan_mgt->handler_xa);
+
+	return 0;
+}
+
+static void nbl_chan_remove_msg_handler(struct nbl_channel_mgt *chan_mgt)
+{
+	struct nbl_chan_msg_node_data *handler;
+	unsigned long msg_type;
+
+	mutex_lock(&chan_mgt->handler_lock);
+	xa_for_each(&chan_mgt->handler_xa, msg_type, handler) {
+		xa_erase(&chan_mgt->handler_xa, msg_type);
+		kfree(handler);
+	}
+	mutex_unlock(&chan_mgt->handler_lock);
+	xa_destroy(&chan_mgt->handler_xa);
+}
+
+static void nbl_chan_init_queue_param(struct nbl_chan_info *chan_info,
+				      u16 num_txq_entries, u16 num_rxq_entries,
+				      u16 txq_buf_size, u16 rxq_buf_size)
+{
+	chan_info->num_txq_entries = num_txq_entries;
+	chan_info->num_rxq_entries = num_rxq_entries;
+	chan_info->txq_buf_size = txq_buf_size;
+	chan_info->rxq_buf_size = rxq_buf_size;
+	atomic_set(&chan_info->inflight_tx_cnt, 0);
+	WRITE_ONCE(chan_info->shutdn, false);
+	WRITE_ONCE(chan_info->active, false);
+	WRITE_ONCE(chan_info->wait_head_index, 0);
+	memset(chan_info->state, 0, sizeof(chan_info->state));
+	init_waitqueue_head(&chan_info->inflight_wait);
+}
+
+static int nbl_chan_init_tx_queue(struct nbl_common_info *common,
+				  struct nbl_chan_info *chan_info)
+{
+	struct nbl_chan_ring *txq = &chan_info->txq;
+	struct device *dev = common->dev;
+	dma_addr_t base_dma;
+	size_t size =
+		chan_info->num_txq_entries * sizeof(struct nbl_chan_tx_desc);
+	void *base;
+	u16 i;
+
+	txq->desc.tx_desc = dmam_alloc_coherent(dev, size, &txq->dma,
+						GFP_KERNEL);
+	if (!txq->desc.tx_desc)
+		return -ENOMEM;
+
+	/*
+	 * The per-entry *data buffers* all come from one contiguous coherent
+	 * block, so those buffers cost exactly one coherent allocation, one
+	 * devres node and one IOMMU mapping for the whole ring whatever the
+	 * page size is.  Per-entry dmam_alloc_coherent() instead rounds every
+	 * NBL_CHAN_BUF_LEN buffer up to a whole page (8 MiB per PF on 16 KiB
+	 * pages, 32 MiB on 64 KiB pages), and a dma_pool only fixes the
+	 * large-page case: with NBL_CHAN_BUF_LEN equal to PAGE_SIZE it still
+	 * degenerates to one allocation per buffer on 4 KiB pages.
+	 * NBL_CHAN_BUF_LEN is a power of two, so the block always divides
+	 * evenly into per-entry slots.
+	 *
+	 * That is the only part of the ring that is collapsed: the descriptor
+	 * ring above is its own dmam_alloc_coherent(), and txq->buf[] and
+	 * chan_info->wait[] below are separate devm_kcalloc() nodes, so a
+	 * ring is two coherent mappings plus those arrays, not one.
+	 */
+	base = dmam_alloc_coherent(dev,
+				   chan_info->num_txq_entries *
+				   chan_info->txq_buf_size,
+				   &base_dma, GFP_KERNEL);
+	if (!base)
+		return -ENOMEM;
+
+	chan_info->wait = devm_kcalloc(dev, chan_info->num_txq_entries,
+				       sizeof(*chan_info->wait), GFP_KERNEL);
+	if (!chan_info->wait)
+		return -ENOMEM;
+	for (i = 0; i < chan_info->num_txq_entries; i++) {
+		init_waitqueue_head(&chan_info->wait[i].wait_queue);
+		WRITE_ONCE(chan_info->wait[i].status, NBL_MBX_STATUS_IDLE);
+		WRITE_ONCE(chan_info->wait[i].acked, 0);
+		WRITE_ONCE(chan_info->wait[i].ack_data, NULL);
+		WRITE_ONCE(chan_info->wait[i].ack_data_len, 0);
+		WRITE_ONCE(chan_info->wait[i].ack_err, 0);
+		WRITE_ONCE(chan_info->wait[i].msg_type, 0);
+		WRITE_ONCE(chan_info->wait[i].msg_index, 0);
+		WRITE_ONCE(chan_info->wait[i].dstid, 0);
+	}
+
+	txq->buf = devm_kcalloc(dev, chan_info->num_txq_entries,
+				sizeof(*txq->buf), GFP_KERNEL);
+	if (!txq->buf)
+		return -ENOMEM;
+
+	/* Carve the contiguous block into fixed-size per-entry slots */
+	for (i = 0; i < chan_info->num_txq_entries; i++) {
+		txq->buf[i].va = (u8 *)base +
+				 (size_t)i * chan_info->txq_buf_size;
+		txq->buf[i].pa = base_dma +
+				 (dma_addr_t)i * chan_info->txq_buf_size;
+		txq->buf[i].size = chan_info->txq_buf_size;
+	}
+
+	txq->next_to_clean = 0;
+	txq->next_to_use = 0;
+	txq->tail_ptr = 0;
+
+	return 0;
+}
+
+static int nbl_chan_init_rx_queue(struct nbl_common_info *common,
+				  struct nbl_chan_info *chan_info)
+{
+	struct nbl_chan_ring *rxq = &chan_info->rxq;
+	struct device *dev = common->dev;
+	struct nbl_chan_rx_desc *desc;
+	dma_addr_t base_dma;
+	size_t size =
+		chan_info->num_rxq_entries * sizeof(struct nbl_chan_rx_desc);
+	void *base;
+	u16 i;
+
+	rxq->desc.rx_desc = dmam_alloc_coherent(dev, size, &rxq->dma,
+						GFP_KERNEL);
+	if (!rxq->desc.rx_desc) {
+		dev_err_ratelimited(dev,
+				    "Allocate DMA for chan rx descriptor ring failed\n");
+		return -ENOMEM;
+	}
+
+	/* See nbl_chan_init_tx_queue() for why one contiguous block is used */
+	base = dmam_alloc_coherent(dev,
+				   chan_info->num_rxq_entries *
+				   chan_info->rxq_buf_size,
+				   &base_dma, GFP_KERNEL);
+	if (!base)
+		return -ENOMEM;
+
+	rxq->buf = devm_kcalloc(dev, chan_info->num_rxq_entries,
+				sizeof(*rxq->buf), GFP_KERNEL);
+	if (!rxq->buf)
+		return -ENOMEM;
+
+	/* Carve the contiguous block into fixed-size per-entry slots */
+	for (i = 0; i < chan_info->num_rxq_entries; i++) {
+		rxq->buf[i].va = (u8 *)base +
+				 (size_t)i * chan_info->rxq_buf_size;
+		rxq->buf[i].pa = base_dma +
+				 (dma_addr_t)i * chan_info->rxq_buf_size;
+		rxq->buf[i].size = chan_info->rxq_buf_size;
+	}
+
+	desc = rxq->desc.rx_desc;
+	/*
+	 * Initially leave one RX descriptor unused so that
+	 * next_to_clean and next_to_use can distinguish an empty
+	 * ring from a full ring.
+	 *
+	 * The unused slot is replenished as RX descriptors are
+	 * consumed and recycled.
+	 */
+	for (i = 0; i < chan_info->num_rxq_entries - 1; i++) {
+		desc[i].buf_addr = cpu_to_le64(rxq->buf[i].pa);
+		desc[i].buf_len = cpu_to_le32(chan_info->rxq_buf_size);
+		desc[i].flags = cpu_to_le16(BIT(NBL_CHAN_RX_DESC_AVAIL));
+	}
+
+	rxq->next_to_clean = 0;
+	rxq->next_to_use = chan_info->num_rxq_entries - 1;
+	rxq->tail_ptr = chan_info->num_rxq_entries - 1;
+
+	return 0;
+}
+
+static int nbl_chan_init_queue(struct nbl_common_info *common,
+			       struct nbl_chan_info *chan_info)
+{
+	int err;
+
+	err = nbl_chan_init_tx_queue(common, chan_info);
+	if (err)
+		return err;
+
+	err = nbl_chan_init_rx_queue(common, chan_info);
+
+	return err;
+}
+
+static void nbl_chan_config_queue(struct nbl_channel_mgt *chan_mgt,
+				  struct nbl_chan_info *chan_info, bool tx)
+{
+	struct nbl_hw_ops *hw_ops = chan_mgt->hw_ops_tbl->ops;
+	struct nbl_hw_mgt *p = chan_mgt->hw_ops_tbl->priv;
+	struct nbl_chan_ring *ring;
+	dma_addr_t addr;
+	int size_bwid;
+
+	if (tx)
+		ring = &chan_info->txq;
+	else
+		ring = &chan_info->rxq;
+	addr = ring->dma;
+	if (tx) {
+		size_bwid = ilog2(chan_info->num_txq_entries);
+		hw_ops->config_mailbox_txq(p, addr, size_bwid);
+	} else {
+		size_bwid = ilog2(chan_info->num_rxq_entries);
+		hw_ops->config_mailbox_rxq(p, addr, size_bwid);
+	}
+}
+
+static void nbl_chan_cfg_qinfo_map_table(struct nbl_channel_mgt *chan_mgt,
+					 u8 bus, u8 devid)
+{
+	struct nbl_hw_ops *hw_ops = chan_mgt->hw_ops_tbl->ops;
+	struct nbl_hw_mgt *p = chan_mgt->hw_ops_tbl->priv;
+	u32 pf_mask = 0;
+	u8 func_id;
+
+	/*
+	 * k_pf_mask rule: bit N == 0 means PF#N enabled, bit N == 1 masked out.
+	 * Program mailbox QINFO entry for each hardware-active PF func_id.
+	 *
+	 * Note: This loop iterates over the raw hardware PF func_id, so it
+	 * assumes product constraints limit supported PF counts to the
+	 * contiguous sets PF0, PF0~1 and PF0~3, i.e. pf_mask is one of
+	 * 0xfe/0xfc/0xf0.  Nothing enforces that here: the resource layer
+	 * that validates the mask, and every caller of this op, arrive in
+	 * later patches of this series.
+	 */
+	hw_ops->get_host_pf_mask(p, &pf_mask);
+	for (func_id = 0; func_id < NBL_MAX_PF; func_id++) {
+		if (!(pf_mask & (1 << func_id)))
+			hw_ops->cfg_mailbox_qinfo(p, func_id, bus,
+						  devid, func_id);
+	}
+}
+
+/*
+ * The RX QINFO region is written only by config/stop paths, never by
+ * senders (which only touch the TX QINFO region and the doorbell), so
+ * stopping RX does not require txq_lock.
+ */
+static void nbl_chan_stop_rx_queue(struct nbl_channel_mgt *chan_mgt)
+{
+	struct nbl_hw_ops *hw_ops = chan_mgt->hw_ops_tbl->ops;
+
+	hw_ops->stop_mailbox_rxq(chan_mgt->hw_ops_tbl->priv);
+}
+
+/*
+ * The TX QINFO region is also written by nbl_chan_quiesce_and_reclaim_tx()
+ * under txq_lock, so TX stop must serialize against it via txq_lock.
+ */
+static void nbl_chan_stop_tx_queue(struct nbl_channel_mgt *chan_mgt)
+{
+	struct nbl_hw_ops *hw_ops = chan_mgt->hw_ops_tbl->ops;
+
+	hw_ops->stop_mailbox_txq(chan_mgt->hw_ops_tbl->priv);
+}
+
+static void nbl_chan_reset_wait_head(struct nbl_chan_info *chan_info,
+				     struct nbl_chan_waitqueue_head *wait_head)
+{
+	lockdep_assert_held(&chan_info->pending_lock);
+
+	WRITE_ONCE(wait_head->acked, 0);
+	WRITE_ONCE(wait_head->status, NBL_MBX_STATUS_IDLE);
+	WRITE_ONCE(wait_head->ack_data, NULL);
+	WRITE_ONCE(wait_head->ack_data_len, 0);
+	WRITE_ONCE(wait_head->ack_err, 0);
+	WRITE_ONCE(wait_head->msg_type, 0);
+	WRITE_ONCE(wait_head->dstid, 0);
+}
+
+static int nbl_chan_teardown_queue(struct nbl_channel_mgt *chan_mgt,
+				   u8 chan_type)
+{
+	struct nbl_chan_info *chan_info = chan_mgt->chan_info[chan_type];
+	struct nbl_chan_waitqueue_head *wait_head;
+	struct work_struct *task;
+	u16 i;
+
+	if (!READ_ONCE(chan_info->active)) {
+		dev_warn(chan_mgt->common->dev, "channel not active, skip duplicate teardown\n");
+		return 0;
+	}
+	/*
+	 * Step1:
+	 * block new sender
+	 */
+	mutex_lock(&chan_info->state_lock);
+	WRITE_ONCE(chan_info->shutdn, true);
+	task = READ_ONCE(chan_info->clean_task);
+	WRITE_ONCE(chan_info->clean_task, NULL);
+	mutex_unlock(&chan_info->state_lock);
+	/*
+	 * Step2:
+	 * abort pending ACK waiters
+	 *
+	 * Poke only slots with a live waiter (status == WAITING): publish a
+	 * synthetic (-EIO, len 0) completion and wake the owner. The slot is
+	 * deliberately NOT reset to IDLE here - ownership stays with the
+	 * sender, which consumes the completion and resets its own slot in
+	 * nbl_chan_send_msg():out_clear_wait_slot under pending_lock.
+	 *
+	 * Resetting an owned slot here would corrupt two things:
+	 *   - an ACKD slot whose sender is consuming the completion
+	 *     locklessly (smp_rmb()-ordered reads) could observe torn
+	 *     ack_data_len/ack_err pairs;
+	 *   - nbl_chan_get_msg_id() hands out IDLE/TIMEOUT slots, so a slot
+	 *     reset to IDLE under its live owner could be reallocated to a
+	 *     sender admitted before shutdn was set.
+	 *
+	 * IDLE/TIMEOUT slots have no waiter and need no action. A late ACK
+	 * racing this poke is serialized by pending_lock: either the real
+	 * completion lands first (status becomes ACKD and is skipped) or
+	 * this synthetic failure does, both are valid outcomes.
+	 */
+	mutex_lock(&chan_info->pending_lock);
+	for (i = 0; i < chan_info->num_txq_entries; i++) {
+		wait_head = &chan_info->wait[i];
+		if (READ_ONCE(wait_head->status) != NBL_MBX_STATUS_WAITING)
+			continue;
+		WRITE_ONCE(wait_head->status, NBL_MBX_STATUS_ACKD);
+		WRITE_ONCE(wait_head->ack_data_len, 0);
+		WRITE_ONCE(wait_head->ack_err, (s32)-EIO);
+		/* Publish completion fields before the acked flag */
+		smp_wmb();
+		WRITE_ONCE(wait_head->acked, 1);
+		wake_up(&wait_head->wait_queue);
+	}
+	mutex_unlock(&chan_info->pending_lock);
+
+	/*
+	 * Step3:
+	 * stop the RX queue unconditionally. The RX QINFO region is written
+	 * only by config/stop paths, never by senders (which only touch the
+	 * TX QINFO region and the doorbell), so no txq_lock is needed. The
+	 * RX engine must be quiesced before the rings are freed, regardless
+	 * of TX-side state - skipping it would leave the device DMAing peer
+	 * messages and descriptor writebacks into released coherent memory.
+	 */
+	nbl_chan_stop_rx_queue(chan_mgt);
+
+	/*
+	 * Drain strategy mirrors mlx5 command interface teardown:
+	 * set shutdown flag first, abort all pending waiters, then
+	 * block until inflight_tx_cnt reaches zero.
+	 *
+	 * After shutdn is set every sender exits promptly at its next
+	 * checkpoint:
+	 *   - interrupt-driven senders wake on shutdn immediately
+	 *     (it is part of the wait_event condition);
+	 *   - ACK polling senders check shutdn each iteration before
+	 *     sleeping 1000-1200us. This per-iteration check avoids
+	 *     waiting the full 5.0-6.0s worst-case ACK timeout.
+	 *
+	 * The wait is deliberately unbounded: a surviving sender still
+	 * holds txq_lock or touches chan_info (state_lock, wait[]) on its
+	 * exit path, so returning early would let devres free the rings
+	 * and chan_info underneath it. In practice the TX polling loop is
+	 * bounded (NBL_CHAN_TX_WAIT_TIMES) and wedged-link MMIO reads
+	 * eventually complete with all-1s data, so this loop terminates;
+	 * warn periodically to keep a genuinely stuck sender diagnosable.
+	 */
+	while (wait_event_timeout(chan_info->inflight_wait,
+				  atomic_read(&chan_info->inflight_tx_cnt) == 0,
+				  msecs_to_jiffies(5000)) == 0)
+		dev_warn(chan_mgt->common->dev,
+			 "teardown: still waiting for %d inflight sender(s)\n",
+			 atomic_read(&chan_info->inflight_tx_cnt));
+
+	/*
+	 * Join the last exiting sender through state_lock.  A sender drops
+	 * inflight_tx_cnt and wakes this drain while still holding
+	 * state_lock, so observing the counter reach zero only proves the
+	 * decrement happened; the acquire below is what proves that no
+	 * sender still holds the lock.  Once it returns, no sender can be
+	 * using chan_info either: the decrement is the last field access a
+	 * sender makes, and the only thing that follows it is the
+	 * mutex_unlock() of state_lock itself.
+	 *
+	 * That trailing unlock is why this is one acquire/release pair and
+	 * not a handshake that ends the sender: mutex_unlock() may still be
+	 * touching the mutex structure after it has released it, so this does
+	 * not prove the sender has returned from it (see
+	 * Documentation/locking/mutex-design.rst).  It is safe only because
+	 * chan_info - mutexes included - is devm-allocated and released at
+	 * detach, long after this teardown, and because nothing frees it
+	 * here.  Freeing chan_info or calling mutex_destroy() at this point
+	 * would need a sender-exit completion the channel does not have.
+	 *
+	 * shutdn blocks new senders, so one acquire/release pair is
+	 * sufficient.
+	 */
+	mutex_lock(&chan_info->state_lock);
+	mutex_unlock(&chan_info->state_lock);
+
+	/*
+	 * All senders have exited, so txq_lock is uncontended: plain lock.
+	 * Take it so this is the sole writer of the TX QINFO block
+	 * (sender-side stop/config/doorbell sequences all run under
+	 * txq_lock).
+	 */
+	mutex_lock(&chan_info->txq_lock);
+	nbl_chan_stop_tx_queue(chan_mgt);
+	mutex_unlock(&chan_info->txq_lock);
+
+	/*
+	 * Join the RX clean work before active=false and before devres
+	 * releases the rings. IRQ is already freed and clean_task is
+	 * cleared under state_lock, so no new instance can be queued.
+	 */
+	if (task)
+		cancel_work_sync(task);
+
+	WRITE_ONCE(chan_info->active, false);
+	return 0;
+}
+
+static int nbl_chan_setup_queue(struct nbl_channel_mgt *chan_mgt, u8 chan_type)
+{
+	struct nbl_chan_info *chan_info = chan_mgt->chan_info[chan_type];
+	struct nbl_hw_ops *hw_ops = chan_mgt->hw_ops_tbl->ops;
+	struct nbl_common_info *common = chan_mgt->common;
+	struct nbl_chan_ring *rxq = &chan_info->rxq;
+	int err;
+
+	if (READ_ONCE(chan_info->active)) {
+		dev_warn(common->dev, "channel already active, reject duplicate setup\n");
+		return -EBUSY;
+	}
+	/*
+	 * The descriptor rings and the per-ring contiguous buffer blocks are
+	 * allocated once and released only by devres at device detach: there
+	 * is no free path, so a channel is set up exactly once per device
+	 * lifetime and teardown is terminal for it.  Re-setup would allocate
+	 * a second set - about 2 MiB of coherent DMA plus the descriptor
+	 * rings and wait[] - and orphan the first one, and it would revive a
+	 * queue whose state teardown has already retired, so it is rejected.
+	 */
+	if (chan_info->dma_allocated) {
+		dev_warn(common->dev,
+			 "channel DMA already allocated, re-setup not supported\n");
+		return -EBUSY;
+	}
+
+	nbl_chan_init_queue_param(chan_info, NBL_CHAN_QUEUE_LEN,
+				  NBL_CHAN_QUEUE_LEN, NBL_CHAN_BUF_LEN,
+				  NBL_CHAN_BUF_LEN);
+	/*
+	 * Set the flag before the first allocation and leave it set even
+	 * when setup fails: whatever nbl_chan_init_queue() managed to
+	 * allocate is devres-owned and stays live until detach, so the retry
+	 * that clearing the flag would allow would allocate a second set on
+	 * top of the orphans.
+	 */
+	chan_info->dma_allocated = true;
+	err = nbl_chan_init_queue(common, chan_info);
+	if (err)
+		return err;
+
+	nbl_chan_config_queue(chan_mgt, chan_info, true); /* tx */
+	nbl_chan_config_queue(chan_mgt, chan_info, false); /* rx */
+	nbl_chan_update_tail_ptr(hw_ops, chan_mgt->hw_ops_tbl->priv,
+				 rxq->tail_ptr, NBL_MB_RX_QID);
+	WRITE_ONCE(chan_info->active, true);
+	return 0;
+}
+
+static bool nbl_chan_txq_full(struct nbl_chan_ring *txq,
+			      u16 num_entries)
+{
+	return NBL_NEXT_ID(txq->next_to_use, num_entries - 1) ==
+	       txq->next_to_clean;
+}
+
+static int nbl_chan_update_txqueue(struct nbl_channel_mgt *chan_mgt,
+				   struct nbl_chan_info *chan_info,
+				   struct nbl_chan_tx_param *param)
+{
+	struct nbl_chan_ring *txq = &chan_info->txq;
+	struct nbl_chan_tx_desc *tx_desc;
+	struct nbl_chan_buf *tx_buf;
+
+	/*
+	 * param->arg is non-NULL whenever param->arg_len is non-zero:
+	 * nbl_chan_send_msg() validates that before calling here, because a
+	 * length with no buffer would publish buf_addr/buf_len describing
+	 * memory that was never filled.
+	 */
+	if (nbl_chan_txq_full(txq, chan_info->num_txq_entries))
+		return -EBUSY;
+	if (param->arg_len > NBL_CHAN_BUF_LEN - sizeof(*tx_desc))
+		return -EINVAL;
+	tx_desc =
+		NBL_CHAN_TX_RING_TO_DESC(txq, txq->next_to_use);
+	tx_buf =
+		NBL_CHAN_TX_RING_TO_BUF(txq, txq->next_to_use);
+	tx_desc->dstid = cpu_to_le16(param->dstid);
+	tx_desc->msg_type = cpu_to_le16(param->msg_type);
+	tx_desc->msgid = cpu_to_le16(param->msgid);
+
+	/*
+	 * srcid field is filled by mailbox hardware after peer receives this
+	 * packet, driver producer never writes srcid; reused descriptor slots
+	 * will contain stale srcid value temporarily until hardware overwrites
+	 * it.
+	 */
+	if (param->arg_len > NBL_CHAN_TX_DESC_EMBEDDED_DATA_LEN) {
+		memcpy(tx_buf->va, param->arg, param->arg_len);
+		tx_desc->buf_addr = cpu_to_le64(tx_buf->pa);
+		tx_desc->buf_len = cpu_to_le16(param->arg_len);
+		tx_desc->data_len = 0;
+		memset(tx_desc->data, 0, sizeof(tx_desc->data));
+	} else {
+		memset(tx_desc->data, 0, sizeof(tx_desc->data));
+		memset(&tx_desc->buf_addr, 0, sizeof(tx_desc->buf_addr));
+		if (param->arg_len)
+			memcpy(tx_desc->data, param->arg, param->arg_len);
+		tx_desc->buf_len = 0;
+		tx_desc->data_len = cpu_to_le16(param->arg_len);
+	}
+	/* Ensure descriptor data visible to device before AVAIL flag */
+	dma_wmb();
+	tx_desc->flags = cpu_to_le16(BIT(NBL_CHAN_TX_DESC_AVAIL));
+
+	txq->next_to_use =
+		NBL_NEXT_ID(txq->next_to_use, chan_info->num_txq_entries - 1);
+	txq->tail_ptr++;
+
+	return 0;
+}
+
+/*
+ * Quiesce the TX mailbox queue and reclaim all outstanding
+ * descriptors.  Called from the timeout path of nbl_chan_kick_tx_ring()
+ * with txq_lock held.
+ *
+ * The device failed to fetch/complete the current descriptor within the
+ * polling window.  We assert QUEUE_RST to stop further DMA fetches,
+ * reclaim every descriptor between next_to_clean and next_to_use,
+ * reset the software tail_ptr counter to match the hardware reset state,
+ * and re-enable the queue so subsequent sends can proceed.
+ */
+static void nbl_chan_quiesce_and_reclaim_tx(struct nbl_channel_mgt *chan_mgt,
+					    struct nbl_chan_info *chan_info)
+{
+	struct nbl_hw_ops *hw_ops = chan_mgt->hw_ops_tbl->ops;
+	struct nbl_hw_mgt *hw_priv = chan_mgt->hw_ops_tbl->priv;
+	struct nbl_chan_ring *txq = &chan_info->txq;
+	struct nbl_chan_tx_desc *tx_desc;
+
+	/*
+	 * Assert QUEUE_RST to stop hardware fetching new descriptors.
+	 * stop_mailbox_txq() flushes through the mailbox BAR itself, so
+	 * the reset has reached the device when it returns.
+	 */
+	hw_ops->stop_mailbox_txq(hw_priv);
+
+	/*
+	 * Reclaim all outstanding descriptors between next_to_clean and
+	 * next_to_use.  Under txq_lock there is at most one in-flight
+	 * descriptor, but iterate the full range for robustness.
+	 */
+	while (txq->next_to_clean != txq->next_to_use) {
+		tx_desc = NBL_CHAN_TX_RING_TO_DESC(txq,
+						   txq->next_to_clean);
+		WRITE_ONCE(tx_desc->flags, 0);
+		txq->next_to_clean =
+			NBL_NEXT_ID(txq->next_to_clean,
+				    chan_info->num_txq_entries - 1);
+	}
+
+	/*
+	 * Hardware tail_ptr counter is cleared by QUEUE_RST.  Reset
+	 * software counter to match so the next doorbell update does
+	 * not produce a false 16-bit wrap delta.
+	 */
+	txq->tail_ptr = 0;
+	txq->next_to_use = 0;
+	txq->next_to_clean = 0;
+
+	/*
+	 * Authoritative re-arm gate, evaluated while txq_lock is held:
+	 * teardown sets shutdn before draining senders and then owns
+	 * hardware state. If teardown started while we were stopping/
+	 * reclaiming, leave the queue in QUEUE_RST and never program
+	 * QUEUE_EN, otherwise the queue could be armed again pointing
+	 * at rings that are about to be freed.
+	 */
+	if (READ_ONCE(chan_info->shutdn))
+		return;
+
+	/* Re-enable queue with current ring base and size */
+	nbl_chan_config_queue(chan_mgt, chan_info, true);
+}
+
+static int nbl_chan_kick_tx_ring(struct nbl_channel_mgt *chan_mgt,
+				 struct nbl_chan_info *chan_info)
+{
+	struct nbl_hw_ops *hw_ops = chan_mgt->hw_ops_tbl->ops;
+	struct nbl_chan_ring *txq = &chan_info->txq;
+	struct device *dev = chan_mgt->common->dev;
+	int max_retries = NBL_CHAN_TX_WAIT_TIMES;
+	struct nbl_chan_tx_desc *tx_desc;
+	int retry_count = 0;
+	u16 msg_type;
+
+	nbl_chan_update_tail_ptr(hw_ops, chan_mgt->hw_ops_tbl->priv,
+				 txq->tail_ptr, NBL_MB_TX_QID);
+
+	tx_desc = NBL_CHAN_TX_RING_TO_DESC(txq, txq->next_to_clean);
+	/*
+	 * Poll for HW to mark descriptor as USED.
+	 * Mailbox is a low-speed control channel for management commands.
+	 * We avoid enabling dedicated per-TX interrupt for single control
+	 * message to reduce interrupt overhead, so use bounded polling
+	 * with small delay instead.
+	 */
+	while (retry_count < max_retries) {
+		if (READ_ONCE(chan_info->shutdn))
+			return -ESHUTDOWN;
+
+		if (le16_to_cpu(READ_ONCE(tx_desc->flags)) &
+		    BIT(NBL_CHAN_TX_DESC_USED)) {
+			 /*
+			  * Order reads of other device-written descriptor
+			  * fields after observing USED.  Matches the RX side
+			  * pattern in nbl_chan_clean_queue().
+			  */
+			dma_rmb();
+			break;
+		}
+
+		retry_count++;
+		if (retry_count == max_retries) {
+			msg_type = le16_to_cpu(READ_ONCE(tx_desc->msg_type));
+			dev_err_ratelimited(dev, "chan send msg type: %d timeout\n",
+					    msg_type);
+			/*
+			 * Teardown may have started after the loop-top
+			 * check on this final iteration. Do not run lock
+			 * recovery then: it would stop/reclaim the queue
+			 * and, without the in-quiesce gate, re-arm it.
+			 * nbl_chan_teardown_queue() owns hardware state.
+			 */
+			if (READ_ONCE(chan_info->shutdn))
+				return -ESHUTDOWN;
+			/*
+			 * Device failed to complete this descriptor.
+			 * Quiesce the queue, reclaim the timed-out
+			 * descriptor, and re-enable so future sends can
+			 * proceed instead of stalling the ring full.
+			 */
+			nbl_chan_quiesce_and_reclaim_tx(chan_mgt,
+							chan_info);
+			return -ETIMEDOUT;
+		}
+		usleep_range(NBL_CHAN_TX_WAIT_US, NBL_CHAN_TX_WAIT_US_MAX);
+	}
+
+	txq->next_to_clean = txq->next_to_use;
+
+	return 0;
+}
+
+static void nbl_chan_recv_ack_msg(void *priv, u16 srcid, u16 msgid, void *data,
+				  u32 data_len)
+{
+	struct nbl_channel_mgt *chan_mgt = (struct nbl_channel_mgt *)priv;
+	struct nbl_chan_waitqueue_head *wait_head = NULL;
+	struct device *dev = chan_mgt->common->dev;
+	struct nbl_chan_info *chan_info =
+		chan_mgt->chan_info[NBL_CHAN_TYPE_MAILBOX];
+	u16 w_dstid, w_msgtype, w_msgidx;
+	u32 *payload = data;
+	u16 ack_msgtype = 0;
+	u16 ack_msgid = 0;
+	u32 ack_datalen;
+	void *ack_data;
+	u32 copy_len;
+	int w_status;
+	s32 raw_err;
+
+	if (READ_ONCE(chan_info->shutdn))
+		return;
+	if (data_len > NBL_CHAN_BUF_LEN ||
+	    data_len < NBL_CHAN_ACK_HEAD_LEN * sizeof(u32)) {
+		dev_err_ratelimited(dev, "Invalid ACK data_len: %u\n",
+				    data_len);
+		return;
+	}
+	ack_datalen = data_len - NBL_CHAN_ACK_HEAD_LEN * sizeof(u32);
+	ack_msgtype = le16_to_cpu(*(__le16 *)(payload + NBL_CHAN_MSG_TYPE_POS));
+	ack_msgid = le16_to_cpu(*(__le16 *)(payload + NBL_CHAN_MSG_ID_POS));
+	if (FIELD_GET(NBL_CHAN_MSGID_LOC_MASK, ack_msgid) >=
+	    chan_info->num_txq_entries) {
+		dev_err_ratelimited(dev, "chan recv msg id: %u err\n",
+				    ack_msgid);
+		return;
+	}
+	wait_head =
+		&chan_info->wait[FIELD_GET(NBL_CHAN_MSGID_LOC_MASK, ack_msgid)];
+
+	mutex_lock(&chan_info->pending_lock);
+
+	/* Cache repeated READ_ONCE values */
+	w_dstid = READ_ONCE(wait_head->dstid);
+	w_status = READ_ONCE(wait_head->status);
+	w_msgtype = READ_ONCE(wait_head->msg_type);
+	w_msgidx = READ_ONCE(wait_head->msg_index);
+
+	if (srcid != w_dstid) {
+		mutex_unlock(&chan_info->pending_lock);
+		dev_err_ratelimited(dev, "ACK srcid=%u != dstid=%u, rejecting\n",
+				    srcid, w_dstid);
+		return;
+	}
+	if (w_status != NBL_MBX_STATUS_WAITING) {
+		mutex_unlock(&chan_info->pending_lock);
+		dev_err_ratelimited(dev,
+				    "Skip ack invalid status, wait msgtype:%u idx:%u status:%d ack msgtype:%u msgid:%u datalen:%u\n",
+				    w_msgtype, w_msgidx, w_status,
+				    ack_msgtype, ack_msgid, ack_datalen);
+		return;
+	}
+
+	if (w_msgtype != ack_msgtype) {
+		mutex_unlock(&chan_info->pending_lock);
+		dev_err_ratelimited(dev,
+				    "Skip ack msgtype mismatch, wait msgtype:%u idx:%u ack msgtype:%u msgid:%u\n",
+				    w_msgtype, w_msgidx, ack_msgtype,
+				    ack_msgid);
+		return;
+	}
+	if (FIELD_GET(NBL_CHAN_MSGID_INDEX_MASK, ack_msgid) != w_msgidx) {
+		mutex_unlock(&chan_info->pending_lock);
+		dev_err_ratelimited(dev,
+				    "Stale ACK: expected index=%u, got msgid=%u\n",
+				    w_msgidx, ack_msgid);
+		return;
+	}
+
+	raw_err = (s32)le32_to_cpu(*(__le32 *)&payload[NBL_CHAN_ACK_RET_POS]);
+	if (raw_err > 0 || raw_err < -MAX_ERRNO)
+		raw_err = -EREMOTEIO;
+
+	WRITE_ONCE(wait_head->ack_err, raw_err);
+
+	copy_len = min_t(u32, READ_ONCE(wait_head->ack_data_len), ack_datalen);
+	if (READ_ONCE(wait_head->ack_err) >= 0 && copy_len > 0) {
+		ack_data = READ_ONCE(wait_head->ack_data);
+		if (!ack_data) {
+			dev_err_ratelimited(dev, "ACK payload dropped: ack_data is NULL\n");
+			WRITE_ONCE(wait_head->ack_data_len, 0);
+			goto ack_done;
+		}
+		memcpy((char *)ack_data,
+		       payload + NBL_CHAN_ACK_HEAD_LEN, copy_len);
+		WRITE_ONCE(wait_head->ack_data_len, (u16)copy_len);
+	} else {
+		WRITE_ONCE(wait_head->ack_data_len, 0);
+	}
+ack_done:
+	/* Guarantee payload data finished before acked flag visible */
+	smp_wmb();
+	WRITE_ONCE(wait_head->acked, 1);
+	WRITE_ONCE(wait_head->status, NBL_MBX_STATUS_ACKD);
+	mutex_unlock(&chan_info->pending_lock);
+	wake_up(&wait_head->wait_queue);
+}
+
+static void nbl_chan_recv_msg(struct nbl_channel_mgt *chan_mgt, void *data)
+{
+	struct device *dev = chan_mgt->common->dev;
+	struct nbl_chan_msg_node_data *msg_handler;
+	u16 msg_type, payload_len, srcid, msgid;
+	struct nbl_chan_info *chan_info =
+		chan_mgt->chan_info[NBL_CHAN_TYPE_MAILBOX];
+	struct nbl_chan_tx_desc *tx_desc;
+	void *payload;
+	size_t avail_space;
+	u16 data_len_fw;
+
+	if (READ_ONCE(chan_info->shutdn))
+		return;
+
+	tx_desc = data;
+	msg_type = le16_to_cpu(READ_ONCE(tx_desc->msg_type));
+	dev_dbg(dev, "recv msg_type: %d\n", msg_type);
+
+	srcid = le16_to_cpu(READ_ONCE(tx_desc->srcid));
+	msgid = le16_to_cpu(READ_ONCE(tx_desc->msgid));
+
+	if (msg_type >= NBL_CHAN_MSG_MAILBOX_MAX)
+		return;
+
+	data_len_fw = le16_to_cpu(READ_ONCE(tx_desc->data_len));
+	if (data_len_fw) {
+		payload_len = data_len_fw;
+
+		if (payload_len > NBL_CHAN_TX_DESC_EMBEDDED_DATA_LEN) {
+			dev_err_ratelimited(dev,
+					    "data_len=%u exceeds embedded buffer size=%u\n",
+					    payload_len,
+					    NBL_CHAN_TX_DESC_EMBEDDED_DATA_LEN);
+			return;
+		}
+		/* Small pkt: payload stored inside descriptor data[] array */
+		payload = tx_desc->data;
+	} else {
+		payload_len = le16_to_cpu(READ_ONCE(tx_desc->buf_len));
+
+		avail_space = NBL_CHAN_BUF_LEN - sizeof(*tx_desc);
+		if (payload_len > avail_space) {
+			dev_err_ratelimited(dev,
+					    "buf_len=%u exceeds external buffer size=%zu\n",
+					    payload_len, avail_space);
+			return;
+		}
+		/* Large pkt: payload follows immediately after tx_desc */
+		payload = tx_desc + 1;
+	}
+
+	msg_handler = xa_load(&chan_mgt->handler_xa, msg_type);
+	if (!msg_handler || !msg_handler->func) {
+		dev_err_ratelimited(dev,
+				    "No handler for msg_type: %u (srcid=%u, msgid=%u)\n",
+				    msg_type, srcid, msgid);
+		return;
+	}
+
+	msg_handler->func(msg_handler->priv, srcid, msgid, payload,
+			  payload_len);
+}
+
+static void nbl_chan_advance_rx_ring(struct nbl_channel_mgt *chan_mgt,
+				     struct nbl_chan_info *chan_info,
+				     struct nbl_chan_ring *rxq)
+{
+	struct nbl_hw_ops *hw_ops = chan_mgt->hw_ops_tbl->ops;
+	struct nbl_chan_rx_desc *rx_desc;
+	struct nbl_chan_buf *rx_buf;
+	u16 next_to_use;
+
+	next_to_use = rxq->next_to_use;
+	rx_desc = NBL_CHAN_RX_RING_TO_DESC(rxq, next_to_use);
+	rx_buf = NBL_CHAN_RX_RING_TO_BUF(rxq, next_to_use);
+
+	/*
+	 * Recycle the RX descriptor at next_to_use. The initial
+	 * unused slot is intentionally recycled after the first
+	 * RX descriptor is consumed, allowing the ring to become
+	 * fully populated while next_to_clean tracks the consumer.
+	 */
+	rx_desc->buf_addr = cpu_to_le64(rx_buf->pa);
+	rx_desc->buf_len = cpu_to_le32(chan_info->rxq_buf_size);
+
+	/*
+	 * DMA Write Memory Barrier:
+	 * Ensures all previous DMA-mapped writes (buffer address/length)
+	 * are completed before the descriptor flags are updated.
+	 * This prevents hardware from seeing a partially updated descriptor
+	 * where flags are set but buffer info isn't ready yet.
+	 */
+	dma_wmb();
+
+	rx_desc->flags = cpu_to_le16(BIT(NBL_CHAN_RX_DESC_AVAIL));
+
+	rxq->next_to_use++;
+	if (rxq->next_to_use == chan_info->num_rxq_entries)
+		rxq->next_to_use = 0;
+	rxq->tail_ptr++;
+
+	nbl_chan_update_tail_ptr(hw_ops, chan_mgt->hw_ops_tbl->priv,
+				 rxq->tail_ptr, NBL_MB_RX_QID);
+}
+
+static void nbl_chan_clean_queue(struct nbl_channel_mgt *chan_mgt,
+				 struct nbl_chan_info *chan_info)
+{
+	struct nbl_common_info *common = chan_mgt->common;
+	struct nbl_chan_ring *rxq = &chan_info->rxq;
+	struct device *dev = chan_mgt->common->dev;
+	u32 budget = NBL_CHAN_RX_CLEAN_BUDGET;
+	struct nbl_chan_rx_desc *rx_desc;
+	struct nbl_chan_buf *rx_buf;
+	struct work_struct *task;
+	bool more_work = false;
+	u16 next_to_clean;
+	u16 flags;
+
+	/* Nothing to clean before setup: the RX ring is still NULL */
+	if (!READ_ONCE(chan_info->active))
+		return;
+
+	next_to_clean = rxq->next_to_clean;
+	rx_desc = NBL_CHAN_RX_RING_TO_DESC(rxq, next_to_clean);
+	rx_buf = NBL_CHAN_RX_RING_TO_BUF(rxq, next_to_clean);
+	while (le16_to_cpu(READ_ONCE(rx_desc->flags)) &
+	       BIT(NBL_CHAN_RX_DESC_USED)) {
+		flags = le16_to_cpu(READ_ONCE(rx_desc->flags));
+
+		if (READ_ONCE(chan_info->shutdn))
+			break;
+		if (!(flags & BIT(NBL_CHAN_RX_DESC_WRITE)))
+			dev_dbg(dev,
+				"mailbox rx flag 0x%x missing NBL_CHAN_RX_DESC_WRITE\n",
+				flags);
+
+		/* Make sure hardware written descriptor visible to CPU */
+		dma_rmb();
+		nbl_chan_recv_msg(chan_mgt, rx_buf->va);
+		nbl_chan_advance_rx_ring(chan_mgt, chan_info, rxq);
+		next_to_clean++;
+		if (next_to_clean == chan_info->num_rxq_entries)
+			next_to_clean = 0;
+		rx_desc = NBL_CHAN_RX_RING_TO_DESC(rxq, next_to_clean);
+		rx_buf = NBL_CHAN_RX_RING_TO_BUF(rxq, next_to_clean);
+		if (--budget == 0) {
+			more_work = true;
+			break;
+		}
+		cond_resched();
+	}
+	rxq->next_to_clean = next_to_clean;
+
+	mutex_lock(&chan_info->state_lock);
+	/* Prevent queue_work after teardown clears clean_task */
+	if (READ_ONCE(chan_info->shutdn)) {
+		mutex_unlock(&chan_info->state_lock);
+		return;
+	}
+	if (common->wq && more_work) {
+		task = READ_ONCE(chan_info->clean_task);
+		if (task)
+			queue_work(common->wq, task);
+	}
+	mutex_unlock(&chan_info->state_lock);
+}
+
+static void nbl_chan_clean_queue_subtask(struct nbl_channel_mgt *chan_mgt,
+					 u8 chan_type)
+{
+	struct nbl_chan_info *chan_info = chan_mgt->chan_info[chan_type];
+
+	nbl_chan_clean_queue(chan_mgt, chan_info);
+}
+
+static int nbl_chan_get_msg_id(struct nbl_chan_info *chan_info,
+			       u16 *msgid)
+{
+	int search_loc = READ_ONCE(chan_info->wait_head_index), i;
+	struct nbl_chan_waitqueue_head *wait = NULL;
+	int status;
+	int next;
+
+	lockdep_assert_held(&chan_info->pending_lock);
+	for (i = 0; i < chan_info->num_txq_entries; i++) {
+		wait = &chan_info->wait[search_loc];
+		status = READ_ONCE(wait->status);
+		if (status == NBL_MBX_STATUS_IDLE ||
+		    status == NBL_MBX_STATUS_TIMEOUT) {
+			WRITE_ONCE(wait->msg_index,
+				   NBL_NEXT_ID(wait->msg_index,
+					       NBL_CHAN_MSG_INDEX_MAX));
+
+			*msgid = FIELD_PREP(NBL_CHAN_MSGID_INDEX_MASK,
+					    wait->msg_index) |
+				 FIELD_PREP(NBL_CHAN_MSGID_LOC_MASK,
+					    search_loc);
+
+			/* Advance starting search position for next caller */
+			next = NBL_NEXT_ID(search_loc,
+					   chan_info->num_txq_entries - 1);
+			WRITE_ONCE(chan_info->wait_head_index, next);
+			return 0;
+		}
+
+		search_loc = NBL_NEXT_ID(search_loc,
+					 chan_info->num_txq_entries - 1);
+	}
+
+	/*
+	 * All tx slots are occupied. May happen under high transmit load
+	 * or delayed remote ACK responses. Caller should retry later.
+	 */
+	return -EAGAIN;
+}
+
+static int nbl_chan_send_msg(struct nbl_channel_mgt *chan_mgt,
+			     struct nbl_chan_send_info *chan_send)
+{
+	struct nbl_common_info *common = chan_mgt->common;
+	struct nbl_chan_waitqueue_head *wait_head = NULL;
+	struct nbl_chan_tx_param tx_param = { 0 };
+	int i = NBL_CHAN_TX_WAIT_ACK_TIMES;
+	struct nbl_chan_info *chan_info =
+		chan_mgt->chan_info[NBL_CHAN_TYPE_MAILBOX];
+	struct device *dev = common->dev;
+	struct work_struct *task;
+	u16 msgid = 0;
+	int ret;
+
+	if (chan_send->resp_len > NBL_CHAN_BUF_LEN) {
+		dev_err_ratelimited(dev, "resp_len %zu exceeds max %d\n",
+				    chan_send->resp_len, NBL_CHAN_BUF_LEN);
+		return -EINVAL;
+	}
+
+	/*
+	 * A length without a payload would leave the descriptor pointing at
+	 * whatever the reused slot still holds - nbl_chan_update_txqueue()
+	 * skips the copy - and the peer would receive those stale bytes.
+	 * Reject the combination instead of transmitting them.
+	 */
+	if (chan_send->arg_len && !chan_send->arg) {
+		dev_err_ratelimited(dev, "arg_len %zu without arg\n",
+				    chan_send->arg_len);
+		return -EINVAL;
+	}
+
+	/*
+	 * Entry points are only reached after a successful setup_queue()
+	 * (probe path), but the channel is a dynamic object: fail cleanly
+	 * instead of dereferencing rings/wait[] that were never allocated
+	 * (num_txq_entries == 0).  active is published last in setup and
+	 * cleared last in teardown, so a false value here means the channel
+	 * never came up - or has already been torn down, and shutdn tells
+	 * the two apart: it is set before active is cleared, so a
+	 * post-teardown sender gets the same -ESHUTDOWN as the check below
+	 * rather than the -ENODEV reserved for a channel setup_queue()
+	 * never completed.
+	 */
+	if (!READ_ONCE(chan_info->active)) {
+		if (READ_ONCE(chan_info->shutdn))
+			return -ESHUTDOWN;
+		return -ENODEV;
+	}
+
+	mutex_lock(&chan_info->state_lock);
+	if (READ_ONCE(chan_info->shutdn)) {
+		mutex_unlock(&chan_info->state_lock);
+		return -ESHUTDOWN;
+	}
+	atomic_inc(&chan_info->inflight_tx_cnt);
+	mutex_unlock(&chan_info->state_lock);
+
+	tx_param.msg_type = chan_send->msg_type;
+	tx_param.arg = chan_send->arg;
+	tx_param.arg_len = chan_send->arg_len;
+	tx_param.dstid = chan_send->dstid;
+	tx_param.msgid = msgid;
+	if (chan_send->ack) {
+		mutex_lock(&chan_info->pending_lock);
+
+		ret = nbl_chan_get_msg_id(chan_info, &msgid);
+		if (ret) {
+			mutex_unlock(&chan_info->pending_lock);
+			dev_err_ratelimited(dev,
+					    "Channel tx wait head full, send msgtype:%u to dstid:%u failed\n",
+					    chan_send->msg_type,
+					    chan_send->dstid);
+			goto out_clean_inflight;
+		}
+		wait_head =
+			&chan_info->wait[FIELD_GET(NBL_CHAN_MSGID_LOC_MASK,
+						   msgid)];
+		WRITE_ONCE(wait_head->acked, 0);
+		WRITE_ONCE(wait_head->ack_data, chan_send->resp);
+		WRITE_ONCE(wait_head->ack_data_len, chan_send->resp_len);
+		WRITE_ONCE(wait_head->msg_type, chan_send->msg_type);
+		WRITE_ONCE(wait_head->msg_index,
+			   FIELD_GET(NBL_CHAN_MSGID_INDEX_MASK, msgid));
+		WRITE_ONCE(wait_head->dstid, chan_send->dstid);
+
+		WRITE_ONCE(wait_head->status, NBL_MBX_STATUS_WAITING);
+		mutex_unlock(&chan_info->pending_lock);
+
+		tx_param.msgid = msgid;
+	}
+
+	mutex_lock(&chan_info->txq_lock);
+	ret = nbl_chan_update_txqueue(chan_mgt, chan_info, &tx_param);
+	if (ret) {
+		mutex_unlock(&chan_info->txq_lock);
+		dev_err_ratelimited(dev,
+				    "Channel tx queue full, send msgtype:%u to dstid:%u failed\n",
+				    chan_send->msg_type, chan_send->dstid);
+		if (wait_head)
+			goto out_clear_wait_slot;
+		goto out_clean_inflight;
+	}
+
+	ret = nbl_chan_kick_tx_ring(chan_mgt, chan_info);
+	mutex_unlock(&chan_info->txq_lock);
+	if (ret) {
+		if (wait_head)
+			goto out_clear_wait_slot;
+		goto out_clean_inflight;
+	}
+
+	if (!chan_send->ack) {
+		ret = 0;
+		goto out_clean_inflight;
+	}
+
+	if (test_bit(NBL_CHAN_IRQ_RDY, chan_info->state)) {
+		while (!READ_ONCE(wait_head->acked)) {
+			/*
+			 * avoids long task blocking when interrupt mode is
+			 * disabled mid-wait. Cannot guarantee subsequent ACK
+			 * delivery after interrupt mask off, only prevents
+			 * infinite blocking. Spurious timeout is possible.
+			 */
+			ret = wait_event_timeout(wait_head->wait_queue,
+						 READ_ONCE(wait_head->acked) ||
+						 READ_ONCE(chan_info->shutdn) ||
+						 !test_bit(NBL_CHAN_IRQ_RDY,
+							   chan_info->state),
+						 NBL_CHAN_ACK_WAIT_TIME);
+
+			if (READ_ONCE(chan_info->shutdn)) {
+				ret = -ESHUTDOWN;
+				goto out_clear_wait_slot;
+			}
+			if (!test_bit(NBL_CHAN_IRQ_RDY, chan_info->state)) {
+				ret = -EIO;
+				goto out_clear_wait_slot;
+			}
+			if (ret == 0) {
+				mutex_lock(&chan_info->pending_lock);
+				if (READ_ONCE(wait_head->status) ==
+				    NBL_MBX_STATUS_WAITING) {
+					WRITE_ONCE(wait_head->status,
+						   NBL_MBX_STATUS_TIMEOUT);
+					WRITE_ONCE(wait_head->acked, 0);
+					WRITE_ONCE(wait_head->ack_data, NULL);
+					WRITE_ONCE(wait_head->ack_data_len, 0);
+					/*
+					 * Ensure all status/ack slot
+					 * updates are visible before subsequent
+					 * readers observe acked == 0
+					 */
+					smp_wmb();
+					mutex_unlock(&chan_info->pending_lock);
+					dev_err_ratelimited(dev,
+							    "Channel waiting ack failed, message type: %d, msg id: %u\n",
+							    chan_send->msg_type,
+							    msgid);
+					ret = -ETIMEDOUT;
+					/*
+					 * TIMEOUT slots can be reused by
+					 * another sender. A late ACK is
+					 * rejected by the receive path,
+					 * which only accepts WAITING slots
+					 * (and by the msg_index generation
+					 * check after reallocation). The
+					 * current sender no longer owns the
+					 * slot, so skip the IDLE reset.
+					 */
+					goto out_clean_inflight;
+				}
+				/*
+				 * ACK won the timeout race: between the
+				 * final condition check and this lock the
+				 * receiver published acked=1/ACKD and
+				 * copied the payload into our response
+				 * buffer. The slot is still exclusively
+				 * ours (only IDLE/TIMEOUT slots are ever
+				 * reallocated), so consume the completion:
+				 * unlock and continue to the loop-bottom
+				 * acked check, which enters the normal
+				 * success readout and IDLE reset. Leaving
+				 * here via the timeout path would strand
+				 * the slot in ACKD forever, permanently
+				 * shrinking the 256-entry pool, and would
+				 * report a false -ETIMEDOUT for a request
+				 * whose response already arrived.
+				 */
+				mutex_unlock(&chan_info->pending_lock);
+			}
+
+			if (READ_ONCE(wait_head->acked))
+				break;
+		}
+		if (READ_ONCE(wait_head->acked)) {
+			/*
+			 * Load ordering: observe acked flag before
+			 * reading ACK payload metadata.
+			 */
+			smp_rmb();
+			chan_send->ack_len = READ_ONCE(wait_head->ack_data_len);
+			ret = READ_ONCE(wait_head->ack_err);
+		}
+	} else {
+		/* Polling path for synchronous ACK */
+		while (i--) {
+			if (READ_ONCE(chan_info->shutdn)) {
+				ret = -ESHUTDOWN;
+				goto out_clear_wait_slot;
+			}
+
+			mutex_lock(&chan_info->state_lock);
+			task = READ_ONCE(chan_info->clean_task);
+			if (common->wq && task &&
+			    !READ_ONCE(chan_info->shutdn) &&
+			    !work_pending(task))
+				queue_work(common->wq, task);
+			mutex_unlock(&chan_info->state_lock);
+			if (READ_ONCE(wait_head->acked)) {
+				/*
+				 * Guarantee load order: observe acked
+				 * flag before reading ack payload metadata.
+				 */
+				smp_rmb();
+				chan_send->ack_len =
+					READ_ONCE(wait_head->ack_data_len);
+				ret = READ_ONCE(wait_head->ack_err);
+				goto out_clear_wait_slot;
+			}
+
+			usleep_range(NBL_CHAN_TX_WAIT_ACK_US_MIN,
+				     NBL_CHAN_TX_WAIT_ACK_US_MAX);
+			cond_resched();
+		}
+		mutex_lock(&chan_info->pending_lock);
+		if (READ_ONCE(wait_head->status) == NBL_MBX_STATUS_ACKD) {
+			chan_send->ack_len = READ_ONCE(wait_head->ack_data_len);
+			ret = READ_ONCE(wait_head->ack_err);
+			mutex_unlock(&chan_info->pending_lock);
+			goto out_clear_wait_slot;
+		}
+		/*
+		 * Polling timed out without receiving an ACK.  Transition
+		 * the slot to TIMEOUT so nbl_chan_get_msg_id() can reuse
+		 * it.  Must not proceed to out_clean_inflight with
+		 * the slot still in WAITING state — that would leak the
+		 * slot permanently and eventually exhaust the 256-entry
+		 * channel queue.
+		 */
+		if (READ_ONCE(wait_head->status) == NBL_MBX_STATUS_WAITING) {
+			WRITE_ONCE(wait_head->status, NBL_MBX_STATUS_TIMEOUT);
+			WRITE_ONCE(wait_head->acked, 0);
+			WRITE_ONCE(wait_head->ack_data, NULL);
+			WRITE_ONCE(wait_head->ack_data_len, 0);
+		}
+		mutex_unlock(&chan_info->pending_lock);
+		dev_err_ratelimited(dev,
+				    "Channel polling ack failed, message type: %d msg id: %u\n",
+				    chan_send->msg_type, msgid);
+		ret = -ETIMEDOUT;
+		goto out_clean_inflight;
+	}
+
+out_clear_wait_slot:
+	mutex_lock(&chan_info->pending_lock);
+	nbl_chan_reset_wait_head(chan_info, wait_head);
+	mutex_unlock(&chan_info->pending_lock);
+
+out_clean_inflight:
+	mutex_lock(&chan_info->state_lock);
+	if (atomic_dec_and_test(&chan_info->inflight_tx_cnt))
+		wake_up(&chan_info->inflight_wait);
+	mutex_unlock(&chan_info->state_lock);
+	return ret;
+}
+
+static int nbl_chan_send_ack(struct nbl_channel_mgt *chan_mgt,
+			     struct nbl_chan_ack_info *chan_ack)
+{
+	size_t head_len = NBL_CHAN_ACK_HEAD_LEN * sizeof(u32);
+	size_t data_len = chan_ack->data_len;
+	struct nbl_chan_send_info chan_send;
+	__le32 *tmp;
+	size_t len;
+	int ret;
+
+	if (data_len >
+	    NBL_CHAN_BUF_LEN - sizeof(struct nbl_chan_tx_desc) - head_len)
+		return -EINVAL;
+
+	len = head_len + data_len;
+	tmp = kzalloc(len, GFP_KERNEL);
+	if (!tmp)
+		return -ENOMEM;
+
+	*(__le16 *)&tmp[NBL_CHAN_MSG_TYPE_POS] =
+		cpu_to_le16(chan_ack->msg_type);
+	*(__le16 *)&tmp[NBL_CHAN_MSG_ID_POS] = cpu_to_le16(chan_ack->msgid);
+	tmp[NBL_CHAN_ACK_RET_POS] = cpu_to_le32(chan_ack->err);
+	if (chan_ack->data && chan_ack->data_len)
+		memcpy(&tmp[NBL_CHAN_ACK_HEAD_LEN], chan_ack->data,
+		       chan_ack->data_len);
+
+	nbl_chan_fill_send_info(&chan_send, chan_ack->dstid, NBL_CHAN_MSG_ACK,
+				tmp, len, NULL, 0, 0);
+	ret = nbl_chan_send_msg(chan_mgt, &chan_send);
+	kfree(tmp);
+
+	return ret;
+}
+
+static int nbl_chan_register_msg(struct nbl_channel_mgt *chan_mgt, u16 msg_type,
+				 nbl_chan_resp func, void *callback)
+{
+	return nbl_chan_add_msg_handler(chan_mgt, msg_type, func, callback);
+}
+
+static bool nbl_chan_check_queue_exist(struct nbl_channel_mgt *chan_mgt,
+				       u8 chan_type)
+{
+	struct nbl_chan_info *chan_info = chan_mgt->chan_info[chan_type];
+
+	return chan_info ? true : false;
+}
+
+static void nbl_chan_register_chan_task(struct nbl_channel_mgt *chan_mgt,
+					u8 chan_type, struct work_struct *task)
+{
+	struct nbl_chan_info *chan_info = chan_mgt->chan_info[chan_type];
+
+	mutex_lock(&chan_info->state_lock);
+	if (!READ_ONCE(chan_info->shutdn))
+		WRITE_ONCE(chan_info->clean_task, task);
+	mutex_unlock(&chan_info->state_lock);
+}
+
+static void nbl_chan_set_queue_state(struct nbl_channel_mgt *chan_mgt,
+				     enum nbl_chan_state state, u8 chan_type,
+				     u8 set)
+{
+	struct nbl_chan_info *chan_info = chan_mgt->chan_info[chan_type];
+	int i;
+
+	if (set)
+		set_bit(state, chan_info->state);
+	else
+		clear_bit(state, chan_info->state);
+	/*
+	 * When clearing IRQ_RDY, wake all per-slot wait queues so
+	 * sleeping senders observe the condition immediately and
+	 * return -EIO instead of waiting out the 3s timeout and
+	 * reporting -ETIMEDOUT.
+	 *
+	 * wait[] is only allocated by a successful nbl_chan_init_tx_queue(),
+	 * while num_txq_entries is already set by nbl_chan_init_queue_param();
+	 * skip the walk if that allocation never happened.
+	 */
+	if (!set && state == NBL_CHAN_IRQ_RDY && chan_info->wait) {
+		for (i = 0; i < chan_info->num_txq_entries; i++)
+			wake_up_all(&chan_info->wait[i].wait_queue);
+	}
+}
+
+static struct nbl_channel_ops chan_ops = {
+	.send_msg			= nbl_chan_send_msg,
+	.send_ack			= nbl_chan_send_ack,
+	.register_msg			= nbl_chan_register_msg,
+	.cfg_chan_qinfo_map_table	= nbl_chan_cfg_qinfo_map_table,
+	.check_queue_exist		= nbl_chan_check_queue_exist,
+	.setup_queue			= nbl_chan_setup_queue,
+	.teardown_queue			= nbl_chan_teardown_queue,
+	.clean_queue_subtask		= nbl_chan_clean_queue_subtask,
+	.register_chan_task		= nbl_chan_register_chan_task,
+	.set_queue_state		= nbl_chan_set_queue_state,
+};
+
+static struct nbl_channel_mgt *
+nbl_chan_setup_chan_mgt(struct nbl_adapter *adapter)
+{
+	struct nbl_hw_ops_tbl *hw_ops_tbl = adapter->intf.hw_ops_tbl;
+	struct nbl_common_info *common = &adapter->common;
+	struct device *dev = &adapter->pdev->dev;
+	struct nbl_channel_mgt *chan_mgt;
+	struct nbl_chan_info *mailbox;
+	int ret;
+
+	chan_mgt = devm_kzalloc(dev, sizeof(*chan_mgt), GFP_KERNEL);
+	if (!chan_mgt)
+		return ERR_PTR(-ENOMEM);
+
+	chan_mgt->common = common;
+	chan_mgt->hw_ops_tbl = hw_ops_tbl;
+
+	mailbox = devm_kzalloc(dev, sizeof(*mailbox), GFP_KERNEL);
+	if (!mailbox)
+		return ERR_PTR(-ENOMEM);
+	mailbox->chan_type = NBL_CHAN_TYPE_MAILBOX;
+	chan_mgt->chan_info[NBL_CHAN_TYPE_MAILBOX] = mailbox;
+
+	ret = nbl_chan_init_msg_handler(chan_mgt);
+	if (ret)
+		return ERR_PTR(ret);
+	ret = devm_mutex_init(common->dev, &mailbox->txq_lock);
+	if (ret)
+		return ERR_PTR(ret);
+	ret = devm_mutex_init(common->dev, &mailbox->state_lock);
+	if (ret)
+		return ERR_PTR(ret);
+	ret = devm_mutex_init(common->dev, &mailbox->pending_lock);
+	if (ret)
+		return ERR_PTR(ret);
+	return chan_mgt;
+}
+
+static struct nbl_channel_ops_tbl *
+nbl_chan_setup_ops(struct device *dev, struct nbl_channel_mgt *chan_mgt)
+{
+	struct nbl_channel_ops_tbl *chan_ops_tbl;
+	int ret;
+
+	chan_ops_tbl = devm_kzalloc(dev, sizeof(*chan_ops_tbl), GFP_KERNEL);
+	if (!chan_ops_tbl)
+		return ERR_PTR(-ENOMEM);
+	if (!chan_ops.send_msg || !chan_ops.send_ack ||
+	    !chan_ops.register_msg || !chan_ops.cfg_chan_qinfo_map_table ||
+	    !chan_ops.check_queue_exist || !chan_ops.setup_queue ||
+	    !chan_ops.teardown_queue || !chan_ops.clean_queue_subtask ||
+	    !chan_ops.register_chan_task || !chan_ops.set_queue_state)
+		return ERR_PTR(-EINVAL);
+
+	chan_ops_tbl->ops = &chan_ops;
+	chan_ops_tbl->priv = chan_mgt;
+
+	ret = nbl_chan_register_msg(chan_mgt, NBL_CHAN_MSG_ACK,
+				    nbl_chan_recv_ack_msg, chan_mgt);
+	if (ret)
+		return ERR_PTR(ret);
+
+	return chan_ops_tbl;
+}
+
+int nbl_chan_init_common(struct nbl_adapter *adap)
+{
+	struct nbl_channel_ops_tbl *chan_ops_tbl;
+	struct device *dev = &adap->pdev->dev;
+	struct nbl_channel_mgt *chan_mgt;
+	int ret;
+
+	chan_mgt = nbl_chan_setup_chan_mgt(adap);
+	if (IS_ERR(chan_mgt))
+		return PTR_ERR(chan_mgt);
+
+	chan_ops_tbl = nbl_chan_setup_ops(dev, chan_mgt);
+	if (IS_ERR(chan_ops_tbl)) {
+		ret = PTR_ERR(chan_ops_tbl);
+		goto cleanup_mgt;
+	}
+
+	adap->intf.channel_ops_tbl = chan_ops_tbl;
+	adap->core.chan_mgt = chan_mgt;
+	ret = nbl_common_create_wq(&adap->common);
+	if (ret)
+		goto cleanup_mgt;
+	return 0;
+
+cleanup_mgt:
+	nbl_chan_remove_msg_handler(chan_mgt);
+	return ret;
+}
+
+void nbl_chan_remove_common(struct nbl_adapter *adap)
+{
+	struct nbl_channel_mgt *chan_mgt = adap->core.chan_mgt;
+
+	if (!chan_mgt)
+		return;
+	nbl_common_destroy_wq(&adap->common);
+	/*
+	 * All channel queues shall be torn down earlier in remove path
+	 * to drain inflight tx workers and stop hardware before destroying
+	 * message handler xarray.
+	 */
+	nbl_chan_remove_msg_handler(chan_mgt);
+	adap->core.chan_mgt = NULL;
+}
diff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_channel/nbl_channel.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_channel/nbl_channel.h
new file mode 100644
index 00000000000000..168f5de221bb78
--- /dev/null
+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_channel/nbl_channel.h
@@ -0,0 +1,196 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2026 Nebula Matrix Limited.
+ */
+
+#ifndef _NBL_CHANNEL_H_
+#define _NBL_CHANNEL_H_
+
+#include <linux/types.h>
+#include <linux/xarray.h>
+
+#include "../nbl_include/nbl_include.h"
+#include "../nbl_include/nbl_def_channel.h"
+#include "../nbl_include/nbl_def_hw.h"
+#include "../nbl_include/nbl_def_common.h"
+#include "../nbl_core.h"
+
+#define NBL_CHAN_TX_RING_TO_DESC(tx_ring, i) \
+	(&((((tx_ring)->desc.tx_desc))[i]))
+#define NBL_CHAN_RX_RING_TO_DESC(rx_ring, i) \
+	(&((((rx_ring)->desc.rx_desc))[i]))
+#define NBL_CHAN_TX_RING_TO_BUF(tx_ring, i) (&(((tx_ring)->buf)[i]))
+#define NBL_CHAN_RX_RING_TO_BUF(rx_ring, i) (&(((rx_ring)->buf)[i]))
+
+#define NBL_CHAN_TX_WAIT_US			100
+#define NBL_CHAN_TX_WAIT_US_MAX			120
+#define NBL_CHAN_TX_WAIT_TIMES			100
+#define NBL_CHAN_TX_WAIT_ACK_US_MIN		1000
+#define NBL_CHAN_TX_WAIT_ACK_US_MAX		1200
+#define NBL_CHAN_TX_WAIT_ACK_TIMES		5000
+#define NBL_CHAN_QUEUE_LEN			256
+#define NBL_CHAN_BUF_LEN			4096
+#define NBL_CHAN_TX_DESC_EMBEDDED_DATA_LEN	16
+
+#define NBL_CHAN_TX_DESC_AVAIL			0
+#define NBL_CHAN_TX_DESC_USED			1
+#define NBL_CHAN_RX_DESC_WRITE			1
+#define NBL_CHAN_RX_DESC_AVAIL			3
+#define NBL_CHAN_RX_DESC_USED			4
+
+#define NBL_CHAN_ACK_HEAD_LEN			3
+#define NBL_CHAN_ACK_RET_POS			2
+#define NBL_CHAN_MSG_ID_POS			1
+#define NBL_CHAN_MSG_TYPE_POS			0
+
+#define NBL_CHAN_ACK_WAIT_TIME			(3 * HZ)
+#define NBL_CHAN_RX_CLEAN_BUDGET		64
+
+enum {
+	NBL_MB_RX_QID = 0,
+	NBL_MB_TX_QID = 1,
+};
+
+enum {
+	NBL_MBX_STATUS_IDLE = 0,
+	NBL_MBX_STATUS_WAITING,
+	NBL_MBX_STATUS_ACKD,
+	NBL_MBX_STATUS_TIMEOUT,
+};
+
+struct nbl_chan_tx_param {
+	enum nbl_chan_msg_type msg_type;
+	void *arg;
+	size_t arg_len;
+	u16 dstid;
+	u16 msgid;
+};
+
+struct nbl_chan_buf {
+	void *va;
+	dma_addr_t pa;
+	size_t size;
+};
+
+struct nbl_chan_tx_desc {
+	__le16 flags;
+	__le16 srcid;
+	__le16 dstid;
+	__le16 data_len;
+	__le16 buf_len;
+	__le64 buf_addr;
+	__le16 msg_type;
+	u8 data[NBL_CHAN_TX_DESC_EMBEDDED_DATA_LEN];
+	__le16 msgid;
+	u8 rsv[26];
+} __packed;
+
+struct nbl_chan_rx_desc {
+	__le16 flags;
+	__le32 buf_len;
+	__le16 buf_id;
+	__le64 buf_addr;
+} __packed;
+
+union nbl_chan_desc_ptr {
+	struct nbl_chan_tx_desc *tx_desc;
+	struct nbl_chan_rx_desc *rx_desc;
+};
+
+struct nbl_chan_ring {
+	union nbl_chan_desc_ptr desc;
+	/* Per-entry slots carved out of one contiguous coherent block */
+	struct nbl_chan_buf *buf;
+	u16 next_to_use;
+	u16 tail_ptr; /* hardware does modulo ring size internally */
+	u16 next_to_clean;
+	dma_addr_t dma;
+};
+
+#define NBL_CHAN_MSG_INDEX_MAX 63
+
+#define NBL_CHAN_MSGID_INDEX_MASK GENMASK(5, 0)
+#define NBL_CHAN_MSGID_LOC_MASK GENMASK(13, 6)
+
+static inline void nbl_chan_update_tail_ptr(struct nbl_hw_ops *hw_ops,
+					    void *hw_priv, u32 tail_ptr, u8 qid)
+{
+	hw_ops->update_mailbox_queue_tail_ptr(hw_priv, tail_ptr, qid);
+}
+
+struct nbl_chan_waitqueue_head {
+	struct wait_queue_head wait_queue;
+	char *ack_data;
+	int acked;
+	s32 ack_err;
+	u16 ack_data_len;
+	u16 msg_type;
+	int status;
+	u8 msg_index;
+	u16 dstid;
+};
+
+struct nbl_chan_info {
+	wait_queue_head_t inflight_wait;
+	struct nbl_chan_ring txq;
+	struct nbl_chan_ring rxq;
+	struct nbl_chan_waitqueue_head *wait;
+	/*
+	 * Serializes TX ring producer state (next_to_use/next_to_clean/
+	 * tail_ptr), descriptor publication and the doorbell, and the TX
+	 * queue config/stop sequences against
+	 * nbl_chan_quiesce_and_reclaim_tx().
+	 */
+	struct mutex txq_lock;
+	/*
+	 * Serializes the channel lifecycle: shutdn publication, clean_task
+	 * registration/clearing, inflight_tx_cnt transitions and the
+	 * inflight_wait wakeup.  The state bitmap is not covered by this
+	 * lock - it is only touched with atomic bit ops (set_bit(),
+	 * clear_bit(), test_bit()).
+	 */
+	struct mutex state_lock;
+	/*
+	 * Serializes the ACK wait slot array (wait[]): slot allocation in
+	 * nbl_chan_get_msg_id(), slot field publication in
+	 * nbl_chan_send_msg(), ACK completion in nbl_chan_recv_ack_msg()
+	 * and slot reset in nbl_chan_reset_wait_head().
+	 */
+	struct mutex pending_lock;
+	struct work_struct *clean_task;
+	u16 wait_head_index;
+	u16 num_txq_entries;
+	u16 num_rxq_entries;
+	u16 txq_buf_size;
+	u16 rxq_buf_size;
+	DECLARE_BITMAP(state, NBL_CHAN_STATE_NBITS);
+	u8 chan_type;
+	atomic_t inflight_tx_cnt;
+	bool shutdn;
+	bool active;
+	/*
+	 * One-shot lifecycle flag: DMA allocations use devm (descriptor/buf
+	 * arrays) and dmam (coherent descriptor ring, per-ring contiguous
+	 * buffer block) and are only released at device detach.  Teardown
+	 * only stops the hardware and clears active; it does not free
+	 * memory.  setup must never run twice on the same chan_info or the
+	 * first allocation set would be orphaned.
+	 */
+	bool dma_allocated;
+};
+
+struct nbl_chan_msg_node_data {
+	nbl_chan_resp func;
+	void *priv;
+};
+
+struct nbl_channel_mgt {
+	struct nbl_common_info *common;
+	struct nbl_hw_ops_tbl *hw_ops_tbl;
+	struct nbl_chan_info *chan_info[NBL_CHAN_TYPE_MAX];
+	/* Serializes handler_xa registration and teardown */
+	struct mutex handler_lock;
+	struct xarray handler_xa;
+};
+
+#endif
diff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_common/nbl_common.c b/drivers/net/ethernet/nebula-matrix/nbl/nbl_common/nbl_common.c
new file mode 100644
index 00000000000000..302c3c48dc4d59
--- /dev/null
+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_common/nbl_common.c
@@ -0,0 +1,52 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2026 Nebula Matrix Limited.
+ */
+
+#include <linux/device.h>
+#include "nbl_common.h"
+
+void nbl_common_destroy_wq(struct nbl_common_info *common)
+{
+	if (!common || !common->wq)
+		return;
+
+	destroy_workqueue(common->wq);
+	common->wq = NULL;
+}
+
+int nbl_common_create_wq(struct nbl_common_info *common)
+{
+	char wq_name[32];
+
+	snprintf(wq_name, sizeof(wq_name), "nbl_wq_%s", pci_name(common->pdev));
+	common->wq = alloc_workqueue(wq_name, WQ_UNBOUND, 0);
+	if (!common->wq) {
+		dev_err(common->dev, "Failed to alloc workqueue %s\n", wq_name);
+		return -ENOMEM;
+	}
+
+	return 0;
+}
+
+/**
+ * nbl_common_func_id_to_rel_pf_id - convert absolute PF id to relative PF id
+ * @common: common device info
+ * @pf_id: absolute PF identifier
+ * @rel_pf_id: output relative pf id
+ *
+ * Leonis uses fixed mgt_pf = 0. Support future non-zero management PF.
+ *
+ * Return: 0 on success, -EINVAL on invalid arguments.
+ */
+int nbl_common_func_id_to_rel_pf_id(struct nbl_common_info *common, u32 pf_id,
+				    u32 *rel_pf_id)
+{
+	if (!rel_pf_id)
+		return -EINVAL;
+
+	if (pf_id < common->mgt_pf)
+		return -EINVAL;
+	*rel_pf_id = pf_id - common->mgt_pf;
+	return 0;
+}
diff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_common/nbl_common.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_common/nbl_common.h
new file mode 100644
index 00000000000000..129073b08eb903
--- /dev/null
+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_common/nbl_common.h
@@ -0,0 +1,14 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2026 Nebula Matrix Limited.
+ */
+
+#ifndef _NBL_COMMON_H_
+#define _NBL_COMMON_H_
+
+#include <linux/types.h>
+
+#include "../nbl_include/nbl_include.h"
+#include "../nbl_include/nbl_def_common.h"
+
+#endif
diff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_core.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_core.h
index 1cd6587a8fcb66..655dfb43e3653d 100644
--- a/drivers/net/ethernet/nebula-matrix/nbl/nbl_core.h
+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_core.h
@@ -14,13 +14,25 @@ enum {
 	NBL_CAP_HAS_NET_BIT,
 };
 
+struct nbl_interface {
+	struct nbl_hw_ops_tbl *hw_ops_tbl;
+	struct nbl_resource_ops_tbl *resource_ops_tbl;
+	struct nbl_dispatch_ops_tbl *dispatch_ops_tbl;
+	struct nbl_channel_ops_tbl *channel_ops_tbl;
+};
+
 struct nbl_core {
 	struct nbl_hw_mgt *hw_mgt;
+	struct nbl_resource_mgt *res_mgt;
+	struct nbl_dispatch_mgt *disp_mgt;
+	struct nbl_dev_mgt *dev_mgt;
+	struct nbl_channel_mgt *chan_mgt;
 };
 
 struct nbl_adapter {
 	struct pci_dev *pdev;
 	struct nbl_core core;
+	struct nbl_interface intf;
 	struct nbl_common_info common;
 };
 
@@ -28,4 +40,12 @@ struct nbl_adapter *nbl_core_init(struct pci_dev *pdev,
 				  struct nbl_init_param *param);
 void nbl_core_remove(struct nbl_adapter *adapter);
 
+/*
+ * Single-shot start/stop pair, called once each from PCI probe/remove.
+ * Not repeatable: MSI-X vectors stay allocated until device detach, so
+ * a second start on a bound device is rejected by the MSI-X core.
+ */
+int nbl_core_start(struct nbl_adapter *adapter);
+void nbl_core_stop(struct nbl_adapter *adapter);
+
 #endif
diff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_core/nbl_dev.c b/drivers/net/ethernet/nebula-matrix/nbl/nbl_core/nbl_dev.c
new file mode 100644
index 00000000000000..c25def18a9e684
--- /dev/null
+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_core/nbl_dev.c
@@ -0,0 +1,727 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2026 Nebula Matrix Limited.
+ */
+#include <linux/device.h>
+#include <linux/pci.h>
+#include "nbl_dev.h"
+
+static void nbl_dev_clean_mailbox_schedule(struct nbl_dev_mgt *dev_mgt);
+
+/* ----------  Interrupt config  ---------- */
+static irqreturn_t nbl_dev_clean_mailbox(int irq __always_unused, void *data)
+{
+	struct nbl_dev_mgt *dev_mgt = (struct nbl_dev_mgt *)data;
+
+	nbl_dev_clean_mailbox_schedule(dev_mgt);
+	return IRQ_HANDLED;
+}
+
+static void nbl_dev_init_msix_cnt(struct nbl_dev_mgt *dev_mgt)
+{
+	struct nbl_dev_common *dev_common = dev_mgt->common_dev;
+	struct nbl_msix_info *msix_info = &dev_common->msix_info;
+
+	/* Reserve one MSI-X vector for the mailbox.  Nothing here
+	 * allocates or requests it: vectors are allocated and the handler
+	 * attached when the device is started, after probe, from this
+	 * count.
+	 */
+	msix_info->serv_info[NBL_MSIX_MAILBOX_TYPE].num = 1;
+}
+
+static int nbl_dev_request_mailbox_irq(struct nbl_dev_mgt *dev_mgt)
+{
+	struct nbl_dev_common *dev_common = dev_mgt->common_dev;
+	struct nbl_msix_info *msix_info = &dev_common->msix_info;
+	struct nbl_common_info *common = dev_mgt->common;
+	u16 lvec;
+	int irq_num;
+	int err;
+
+	if (!msix_info->serv_info[NBL_MSIX_MAILBOX_TYPE].num)
+		return 0;
+
+	lvec = msix_info->serv_info[NBL_MSIX_MAILBOX_TYPE].base_vector_id;
+	irq_num = pci_irq_vector(common->pdev, lvec);
+	if (irq_num < 0) {
+		dev_err(common->dev, "Failed to get mailbox IRQ vector: %d\n",
+			irq_num);
+		return irq_num;
+	}
+
+	snprintf(dev_common->mailbox_name, sizeof(dev_common->mailbox_name),
+		 "nbl_mailbox@pci:%s", pci_name(common->pdev));
+	err = request_irq(irq_num, nbl_dev_clean_mailbox, 0,
+			  dev_common->mailbox_name, dev_mgt);
+	if (err)
+		return err;
+
+	return 0;
+}
+
+static void nbl_dev_free_mailbox_irq(struct nbl_dev_mgt *dev_mgt)
+{
+	struct nbl_dev_common *dev_common = dev_mgt->common_dev;
+	struct nbl_msix_info *msix_info = &dev_common->msix_info;
+	struct nbl_common_info *common = dev_mgt->common;
+	u16 lvec;
+	int irq_num;
+
+	if (!msix_info->serv_info[NBL_MSIX_MAILBOX_TYPE].num)
+		return;
+
+	lvec = msix_info->serv_info[NBL_MSIX_MAILBOX_TYPE].base_vector_id;
+	irq_num = pci_irq_vector(common->pdev, lvec);
+	if (irq_num >= 0)
+		free_irq(irq_num, dev_mgt);
+}
+
+static int nbl_dev_enable_mailbox_irq(struct nbl_dev_mgt *dev_mgt)
+{
+	struct nbl_dispatch_ops *disp_ops = dev_mgt->disp_ops_tbl->ops;
+	struct nbl_channel_ops *chan_ops = dev_mgt->chan_ops_tbl->ops;
+	struct nbl_dev_common *dev_common = dev_mgt->common_dev;
+	struct nbl_msix_info *msix_info = &dev_common->msix_info;
+	u16 lvec;
+	int ret;
+
+	if (!msix_info->serv_info[NBL_MSIX_MAILBOX_TYPE].num)
+		return 0;
+
+	lvec = msix_info->serv_info[NBL_MSIX_MAILBOX_TYPE].base_vector_id;
+	/*
+	 * Enable sequence: perform set_mailbox_irq RPC in polling mode first.
+	 * Only set NBL_CHAN_IRQ_RDY after RPC succeeds, mirroring disable path.
+	 * This avoids waiting for an interrupt which has not been armed yet.
+	 */
+	ret = disp_ops->set_mailbox_irq(dev_mgt->disp_ops_tbl->priv,
+					lvec, true);
+	if (ret)
+		return ret;
+	chan_ops->set_queue_state(dev_mgt->chan_ops_tbl->priv,
+				  NBL_CHAN_IRQ_RDY,
+				  NBL_CHAN_TYPE_MAILBOX, true);
+	return 0;
+}
+
+static int nbl_dev_disable_mailbox_irq(struct nbl_dev_mgt *dev_mgt)
+{
+	struct nbl_dispatch_ops *disp_ops = dev_mgt->disp_ops_tbl->ops;
+	struct nbl_channel_ops *chan_ops = dev_mgt->chan_ops_tbl->ops;
+	struct nbl_dev_common *dev_common = dev_mgt->common_dev;
+	struct nbl_msix_info *msix_info = &dev_common->msix_info;
+	u16 lvec;
+
+	if (!msix_info->serv_info[NBL_MSIX_MAILBOX_TYPE].num)
+		return 0;
+
+	lvec = msix_info->serv_info[NBL_MSIX_MAILBOX_TYPE].base_vector_id;
+	/*
+	 * Disable sequence invariant: clear the software READY flag first,
+	 * then mask the hardware interrupt. Must not reverse the order.
+	 *
+	 * Clearing NBL_CHAN_IRQ_RDY is what makes an interrupt-mode sender
+	 * fail fast: nbl_chan_set_queue_state() wakes every per-slot wait
+	 * queue on the transition, and nbl_chan_send_msg() tests the flag
+	 * ahead of its timeout branch, so the sender returns -EIO right
+	 * away instead of waiting out NBL_CHAN_ACK_WAIT_TIME (3s) and
+	 * reporting a bogus -ETIMEDOUT / "Channel waiting ack failed".
+	 * Masking first would leave the sleeper blocked on an interrupt
+	 * that is already masked, i.e. guaranteed to time out.
+	 *
+	 * This does NOT preserve in-flight ACKs, and must not be described
+	 * as doing so: the woken sender gives up and resets its slot to
+	 * IDLE, so an ACK arriving afterwards finds a non-WAITING slot and
+	 * is dropped by the receive path ("Skip ack invalid status").
+	 * Interrupt mode is being torn down here; the guarantee is a
+	 * bounded, deterministic failure, not delivery.  In this driver
+	 * every ACK-requesting sender runs from probe/remove and the clean
+	 * task only sends ACK-less replies, so nothing depends on the
+	 * stronger claim today.
+	 *
+	 * The only caller is nbl_dev_stop(): free_irq() blocks until
+	 * any in-flight hardirq handler completes and prevents new interrupts.
+	 * cancel_work_sync() then waits for any already running mailbox cleanup
+	 * work to finish, or cancels queued but unstarted work items before
+	 * final channel destruction. No stuck descriptors linger.
+	 *
+	 * nbl_dev_start()'s unwind path does not reach here: err_free_irq runs
+	 * before IRQ_RDY is set, so only nbl_dev_free_mailbox_irq() applies
+	 * there and no set_mailbox_irq(false) is sent - see the comment on
+	 * that label.
+	 */
+	chan_ops->set_queue_state(dev_mgt->chan_ops_tbl->priv,
+				  NBL_CHAN_IRQ_RDY,
+				  NBL_CHAN_TYPE_MAILBOX, false);
+
+	return disp_ops->set_mailbox_irq(dev_mgt->disp_ops_tbl->priv,
+					 lvec, false);
+}
+
+static int nbl_dev_cfg_msix_map(struct nbl_dev_mgt *dev_mgt)
+{
+	struct nbl_dispatch_ops *disp_ops = dev_mgt->disp_ops_tbl->ops;
+	struct nbl_dev_common *dev_common = dev_mgt->common_dev;
+	struct nbl_msix_info *msix_info = &dev_common->msix_info;
+	bool mask_en = msix_info->serv_info[NBL_MSIX_NET_TYPE].hw_self_mask_en;
+	u16 msix_net_num = msix_info->serv_info[NBL_MSIX_NET_TYPE].num;
+	u16 msix_not_net_num = 0;
+	int err, i;
+
+	msix_info->serv_info[NBL_MSIX_NET_TYPE].base_vector_id = 0;
+	/*
+	 * Calculate base_vector_id for each MSIX service type.
+	 * This relies on NBL_MSIX_TYPE enum being ordered sequentially,
+	 * starting from NBL_MSIX_NET_TYPE.
+	 */
+	for (i = NBL_MSIX_NET_TYPE + 1; i < NBL_MSIX_TYPE_MAX; i++)
+		msix_info->serv_info[i].base_vector_id =
+			msix_info->serv_info[i - 1].base_vector_id +
+			msix_info->serv_info[i - 1].num;
+
+	for (i = 0; i < NBL_MSIX_TYPE_MAX; i++) {
+		if (i == NBL_MSIX_NET_TYPE)
+			continue;
+		msix_not_net_num += msix_info->serv_info[i].num;
+	}
+
+	err = disp_ops->cfg_msix_map(dev_mgt->disp_ops_tbl->priv,
+				     msix_net_num, msix_not_net_num,
+				     mask_en);
+
+	return err;
+}
+
+static int nbl_dev_destroy_msix_map(struct nbl_dev_mgt *dev_mgt)
+{
+	struct nbl_dispatch_ops *disp_ops = dev_mgt->disp_ops_tbl->ops;
+
+	return disp_ops->destroy_msix_map(dev_mgt->disp_ops_tbl->priv);
+}
+
+static int nbl_dev_init_interrupt_scheme(struct nbl_dev_mgt *dev_mgt)
+{
+	struct nbl_dev_common *dev_common = dev_mgt->common_dev;
+	struct nbl_msix_info *msix_info = &dev_common->msix_info;
+	struct nbl_common_info *common = dev_mgt->common;
+	struct irq_affinity affd = { 0 };
+	int needed = 0;
+	int err;
+	int i;
+
+	for (i = 0; i < NBL_MSIX_TYPE_MAX; i++)
+		needed += msix_info->serv_info[i].num;
+
+	/*
+	 * The mailbox vector is the trailing administrative vector;
+	 * reserve it via post_vectors so it is excluded from managed
+	 * affinity spreading. A managed vector can be shut down when
+	 * its last assigned CPU goes offline, which would stall every
+	 * mailbox RPC while NBL_CHAN_IRQ_RDY stays set.
+	 */
+	affd.post_vectors = msix_info->serv_info[NBL_MSIX_MAILBOX_TYPE].num;
+
+	err = pci_alloc_irq_vectors_affinity(common->pdev, needed, needed,
+					     PCI_IRQ_MSIX | PCI_IRQ_AFFINITY,
+					     &affd);
+	if (err < 0) {
+		dev_err(common->dev,
+			"pci_alloc_irq_vectors failed, err = %d\n", err);
+		return err;
+	}
+	if (err != needed) {
+		dev_err(common->dev, "pci_alloc_irq_vectors got %d vecs, need %d\n",
+			err, needed);
+		return -ENOSPC;
+	}
+	return 0;
+}
+
+/*
+ * Release the kernel-side MSI-X vectors here, before the hardware
+ * pcompleter map is destroyed.  pci_free_irq_vectors() ends in
+ * pci_msix_shutdown(), which masks every descriptor by writing the
+ * vector control word in the MSI-X table and then reads mask_base
+ * back to flush that write; those MMIO accesses should reach the
+ * function while FUNCTION_MSIX_MAP is still valid.  This mirrors
+ * nbl_dev_start(), where cfg_msix_map runs before
+ * pci_alloc_irq_vectors_affinity(): the opposite order leaves the
+ * core poking the MSI-X table after destroy_msix_map() has cleared
+ * the valid bit, invalidated the info entries and freed the map
+ * table, and how the device reacts to that is hardware-specific.
+ *
+ * This is not a double free of the devres action.  probe uses
+ * pcim_enable_device(), so pci_alloc_irq_vectors() also registers
+ * pcim_msi_release, but that only calls pci_free_irq_vectors()
+ * again, and pci_disable_msix() returns early once msix_enabled is
+ * clear.
+ *
+ * The handler is expected to be gone by now: both callers run
+ * nbl_dev_free_mailbox_irq() first.
+ */
+static void nbl_dev_clear_interrupt_scheme(struct nbl_dev_mgt *dev_mgt)
+{
+	struct nbl_common_info *common = dev_mgt->common;
+
+	pci_free_irq_vectors(common->pdev);
+}
+
+/* ----------  Channel config  ---------- */
+static void nbl_dev_setup_chan_qinfo(struct nbl_dev_mgt *dev_mgt, u8 chan_type)
+{
+	struct nbl_channel_ops *chan_ops = dev_mgt->chan_ops_tbl->ops;
+	struct nbl_channel_mgt *priv = dev_mgt->chan_ops_tbl->priv;
+	struct nbl_common_info *common = dev_mgt->common;
+
+	if (!chan_ops->check_queue_exist(priv, chan_type))
+		return;
+
+	/*
+	 * common->hw_bus is the control PF's real bus number, captured in
+	 * nbl_res_ctrl_dev_sriov_info_init() during nbl_res_init_leonis().
+	 * nbl_core_init() runs resource init before nbl_dev_init(), so the
+	 * value is always initialized when this control-PF-only path runs;
+	 * nbl_res_intr_cfg_msix_map() consumes it for cfg_msix_map() the
+	 * same way.
+	 */
+	chan_ops->cfg_chan_qinfo_map_table(priv, common->hw_bus, common->devid);
+}
+
+static int nbl_dev_setup_chan_queue(struct nbl_dev_mgt *dev_mgt, u8 chan_type)
+{
+	struct nbl_channel_ops *chan_ops = dev_mgt->chan_ops_tbl->ops;
+	struct nbl_channel_mgt *priv = dev_mgt->chan_ops_tbl->priv;
+	int ret = 0;
+
+	if (chan_ops->check_queue_exist(priv, chan_type))
+		ret = chan_ops->setup_queue(priv, chan_type);
+
+	return ret;
+}
+
+static int nbl_dev_remove_chan_queue(struct nbl_dev_mgt *dev_mgt, u8 chan_type)
+{
+	struct nbl_channel_ops *chan_ops = dev_mgt->chan_ops_tbl->ops;
+	struct nbl_channel_mgt *priv = dev_mgt->chan_ops_tbl->priv;
+	int ret = 0;
+
+	if (chan_ops->check_queue_exist(priv, chan_type))
+		ret = chan_ops->teardown_queue(priv, chan_type);
+
+	return ret;
+}
+
+static void nbl_dev_register_chan_task(struct nbl_dev_mgt *dev_mgt,
+				       u8 chan_type, struct work_struct *task)
+{
+	struct nbl_channel_ops *chan_ops = dev_mgt->chan_ops_tbl->ops;
+
+	if (chan_ops->check_queue_exist(dev_mgt->chan_ops_tbl->priv, chan_type))
+		chan_ops->register_chan_task(dev_mgt->chan_ops_tbl->priv,
+					     chan_type, task);
+}
+
+/* ----------  Tasks config  ---------- */
+static void nbl_dev_clean_mailbox_task(struct work_struct *work)
+{
+	struct nbl_dev_common *common_dev =
+		container_of(work, struct nbl_dev_common, clean_mbx_task);
+	struct nbl_dev_mgt *dev_mgt = common_dev->dev_mgt;
+	struct nbl_channel_ops *chan_ops = dev_mgt->chan_ops_tbl->ops;
+
+	chan_ops->clean_queue_subtask(dev_mgt->chan_ops_tbl->priv,
+				      NBL_CHAN_TYPE_MAILBOX);
+}
+
+static void nbl_dev_clean_mailbox_schedule(struct nbl_dev_mgt *dev_mgt)
+{
+	struct nbl_dev_common *common_dev = dev_mgt->common_dev;
+	struct nbl_common_info *common = dev_mgt->common;
+
+	queue_work(common->wq, &common_dev->clean_mbx_task);
+}
+
+/* ----------  Dev init process  ---------- */
+static int nbl_dev_setup_common_dev(struct nbl_adapter *adapter)
+{
+	struct nbl_dev_mgt *dev_mgt = adapter->core.dev_mgt;
+	struct nbl_dev_common *common_dev;
+	int ret;
+
+	common_dev = devm_kzalloc(&adapter->pdev->dev, sizeof(*common_dev),
+				  GFP_KERNEL);
+	if (!common_dev)
+		return -ENOMEM;
+	common_dev->dev_mgt = dev_mgt;
+
+	/*
+	 * INIT_WORK must precede nbl_dev_register_chan_task() below:
+	 * registration publishes the work item to the channel, and the
+	 * channel may queue it from its send and receive paths as soon
+	 * as setup_queue() completes.
+	 */
+	INIT_WORK(&common_dev->clean_mbx_task, nbl_dev_clean_mailbox_task);
+
+	ret = nbl_dev_setup_chan_queue(dev_mgt, NBL_CHAN_TYPE_MAILBOX);
+	if (ret) {
+		/*
+		 * Nothing to unwind: chan_info->active is only set on a
+		 * successful setup_queue(), so a failed one left no
+		 * sender, no registered work item and no mailbox
+		 * traffic.  Calling teardown_queue() here would only
+		 * take its "channel not active" early exit and emit a
+		 * misleading duplicate-teardown warning.  The partially
+		 * allocated DMA rings are intentionally kept until devres
+		 * releases them at detach (chan_info->dma_allocated
+		 * blocks a re-setup).
+		 */
+		return ret;
+	}
+
+	nbl_dev_register_chan_task(dev_mgt, NBL_CHAN_TYPE_MAILBOX,
+				   &common_dev->clean_mbx_task);
+	/*
+	 * VSI/ETH identity is deliberately not fetched here: it is only
+	 * known to the control PF, so it has to come from a mailbox RPC,
+	 * and the control PF's responder may still be probing while this
+	 * function runs.  The fetch is therefore left to the start path.
+	 */
+	dev_mgt->common_dev = common_dev;
+	nbl_dev_init_msix_cnt(dev_mgt);
+
+	return 0;
+}
+
+static void nbl_dev_remove_common_dev(struct nbl_adapter *adapter)
+{
+	struct nbl_dev_mgt *dev_mgt = adapter->core.dev_mgt;
+	struct nbl_dev_common *common_dev = dev_mgt->common_dev;
+
+	if (!common_dev)
+		return;
+	cancel_work_sync(&common_dev->clean_mbx_task);
+	/*
+	 * nbl_chan_teardown_queue() never fails: it returns 0 both on the
+	 * early exit for a channel that never became active and on the
+	 * normal path, which drains every inflight sender before
+	 * returning.  There is therefore no partial-failure state that
+	 * could leave mailbox DMA in flight and nothing to report here.
+	 *
+	 * Unregistering the clean task is the channel layer's job too:
+	 * teardown_queue() clears clean_task under state_lock and then
+	 * cancel_work_sync()es it, after shutdn has blocked new queues.
+	 * No explicit unregistration is needed here precisely because
+	 * teardown_queue() does that clearing itself.  Passing NULL from
+	 * here would not be a no-op either: shutdn is only set inside
+	 * teardown_queue(), so nbl_chan_register_chan_task() would still
+	 * accept the write - it would just be clearing the same pointer a
+	 * second time.
+	 */
+	nbl_dev_remove_chan_queue(dev_mgt, NBL_CHAN_TYPE_MAILBOX);
+}
+
+static int nbl_dev_setup_ctrl_dev(struct nbl_adapter *adapter)
+{
+	struct nbl_dev_mgt *dev_mgt = adapter->core.dev_mgt;
+	struct nbl_dispatch_ops *disp_ops = dev_mgt->disp_ops_tbl->ops;
+
+	return disp_ops->init_module(dev_mgt->disp_ops_tbl->priv);
+}
+
+/*
+ * Tear down control device: deinit_module sets driver_status=false
+ * to notify firmware to clean all per-PF hardware state (including
+ * qinfo registers).  The qinfo map programmed in nbl_dev_init() via
+ * nbl_dev_setup_chan_qinfo() is not explicitly cleared; firmware
+ * handles it on driver_status change.
+ *
+ * Teardown ordering guarantee: every non-management PF creates a
+ * consumer->control PF device link in its probe path, so the driver
+ * core always unbinds all siblings before allowing the control PF to
+ * be detached (sysfs unbind, driver unregister and hot-unplug alike).
+ * nbl_probe_chip_deps() defers a sibling's probe until the control PF
+ * is fully bound (DL_DEV_DRIVER_BOUND), and holds the management PF's
+ * device lock across both that check and device_link_add(), so a
+ * supplier cannot slip to DL_DEV_NO_DRIVER or DL_DEV_PROBING in
+ * between.  A sibling therefore cannot
+ * reach its mailbox setup before the control PF is fully probed, nor
+ * race this deinit afterwards - the link forces it to be unbound
+ * first.  The link is a persistent managed link with
+ * DL_FLAG_AUTOPROBE_CONSUMER, so binding the control PF again makes
+ * the core re-probe the siblings; without it they would stay unbound
+ * until an admin rebound each one by hand.
+ *
+ * Direct control-PF FLR (which never runs driver teardown) cannot be
+ * guarded here.
+ */
+static void nbl_dev_remove_ctrl_dev(struct nbl_adapter *adapter)
+{
+	struct nbl_dev_mgt *dev_mgt = adapter->core.dev_mgt;
+	struct nbl_dispatch_ops *disp_ops = dev_mgt->disp_ops_tbl->ops;
+
+	disp_ops->deinit_module(dev_mgt->disp_ops_tbl->priv);
+}
+
+static struct nbl_dev_mgt *nbl_dev_setup_dev_mgt(struct nbl_common_info *common)
+{
+	struct nbl_dev_mgt *dev_mgt;
+
+	dev_mgt = devm_kzalloc(common->dev, sizeof(*dev_mgt), GFP_KERNEL);
+	if (!dev_mgt)
+		return ERR_PTR(-ENOMEM);
+
+	dev_mgt->common = common;
+	return dev_mgt;
+}
+
+int nbl_dev_init(struct nbl_adapter *adapter)
+{
+	struct nbl_common_info *common = &adapter->common;
+	struct nbl_dispatch_ops_tbl *disp_ops_tbl =
+		adapter->intf.dispatch_ops_tbl;
+	struct nbl_channel_ops_tbl *chan_ops_tbl =
+		adapter->intf.channel_ops_tbl;
+	struct nbl_dev_mgt *dev_mgt;
+	int ret;
+
+	dev_mgt = nbl_dev_setup_dev_mgt(common);
+	if (IS_ERR(dev_mgt)) {
+		ret = PTR_ERR(dev_mgt);
+		return ret;
+	}
+
+	dev_mgt->disp_ops_tbl = disp_ops_tbl;
+	dev_mgt->chan_ops_tbl = chan_ops_tbl;
+	adapter->core.dev_mgt = dev_mgt;
+	if (common->has_ctrl)
+		nbl_dev_setup_chan_qinfo(dev_mgt, NBL_CHAN_TYPE_MAILBOX);
+	/*
+	 * Chip hardware initialization is completed by firmware at power-up.
+	 * Only driver functional table/register config follows here, safe to
+	 * access hardware registers before ctrl dev setup.
+	 *
+	 * The QINFO routing has to be programmed before the mailbox queue is
+	 * enabled: nbl_dev_setup_common_dev() sets QUEUE_EN, and a queue that
+	 * is live before its routing is in place would drop or misroute
+	 * traffic.
+	 */
+	ret = nbl_dev_setup_common_dev(adapter);
+	if (ret)
+		return ret;
+
+	if (common->has_ctrl) {
+		ret = nbl_dev_setup_ctrl_dev(adapter);
+		if (ret)
+			goto out_remove_common_dev;
+	}
+
+	return 0;
+
+/*
+ * A failure after the QINFO routing was programmed leaves those entries
+ * in hardware: driver_status never goes 1 -> 0 on these paths, so
+ * firmware does not reclaim them.  This is harmless - firmware only
+ * routes once driver_status announces the driver, and no sibling PF can
+ * bind until this probe succeeds.  The next successful probe overwrites
+ * the entries, and a chip reset clears them.
+ */
+out_remove_common_dev:
+	nbl_dev_remove_common_dev(adapter);
+	return ret;
+}
+
+/*
+ * Teardown order: Stop mailbox channel and drain all inflight DMA first,
+ * then invoke deinit_module to notify firmware.
+ *
+ * This intentionally breaks strict init/teardown mirror symmetry due to
+ * a hardware constraint: firmware performs asynchronous *per-PF* cleanup
+ * (mailbox QINFO routing, qinfo registers and the like) once
+ * driver_status=false is set.  It does not revert the chip-wide
+ * datapath registers configured by init_module - those stay valid until
+ * chip reset (see nbl_hw_deinit_module()).  We must guarantee no ongoing
+ * mailbox DMA before deinit_module so firmware cannot reclaim the
+ * routing under a live transfer and leave the device DMAing into
+ * released memory.
+ *
+ * Init order: program mailbox QINFO routing (ctrl PF only) ->
+ *            create mailbox queue/common_dev -> ctrl dev init_module
+ * Teardown order: destroy mailbox queue/common_dev -> ctrl dev deinit_module
+ */
+void nbl_dev_remove(struct nbl_adapter *adapter)
+{
+	struct nbl_common_info *common = &adapter->common;
+
+	nbl_dev_remove_common_dev(adapter);
+	if (common->has_ctrl)
+		nbl_dev_remove_ctrl_dev(adapter);
+}
+
+/* ----------  Dev start process  ---------- */
+int nbl_dev_start(struct nbl_adapter *adapter)
+{
+	struct nbl_dev_mgt *dev_mgt = adapter->core.dev_mgt;
+	struct nbl_dispatch_ops *disp_ops = dev_mgt->disp_ops_tbl->ops;
+	struct nbl_dispatch_mgt *priv = dev_mgt->disp_ops_tbl->priv;
+	struct nbl_dev_common *common_dev = dev_mgt->common_dev;
+	struct nbl_common_info *common = dev_mgt->common;
+	int cleanup_ret;
+	int ret;
+
+	/*
+	 * Startup RPC failures are reported as they are, including the
+	 * -ETIMEDOUT of a 5-6s polling wait against an unresponsive peer.
+	 * nbl_probe() already deferred this PF in nbl_probe_chip_deps()
+	 * until the control PF reached DL_DEV_DRIVER_BOUND, and the core
+	 * only marks a device bound once its whole probe - including the
+	 * control PF's own nbl_core_start() - has returned.  That state,
+	 * not nbl_dev_init(), is what makes a timeout here a real failure
+	 * instead of a "supplier not ready yet" condition: on the control
+	 * PF inbound mailbox work only runs from clean_mbx_task, queued
+	 * by the mailbox hardirq, by the task's own self-requeue, or by
+	 * this function's polling sends - and the control PF uses the
+	 * local dispatch ops, so it sends none.  The responder is armed
+	 * by set_mailbox_irq(true) at the very end of the peer's
+	 * nbl_dev_start(); relaxing the dependency to "func 0 finished
+	 * nbl_dev_init()" would let every sibling RPC sit out the full
+	 * 5-6s polling wait.  Turning -ETIMEDOUT into -EPROBE_DEFER would
+	 * hide it behind dev_dbg() and leave the retry to the next
+	 * unrelated device bind, paying that wait again on every retry.
+	 */
+	ret = nbl_dev_cfg_msix_map(dev_mgt);
+	if (ret)
+		goto err_destroy_map;
+
+	/* Fetch VSI/ETH identity after cfg_msix_map */
+	ret = disp_ops->get_vsi_id(priv, NBL_VSI_DATA, &common->vsi_id);
+	if (ret)
+		goto err_destroy_map;
+	ret = disp_ops->get_eth_id(priv, common->vsi_id, &common->eth_num,
+				   &common->eth_id, &common->logic_eth_id);
+	if (ret)
+		goto err_destroy_map;
+
+	ret = nbl_dev_init_interrupt_scheme(dev_mgt);
+	if (ret)
+		goto err_destroy_map;
+
+	ret = nbl_dev_request_mailbox_irq(dev_mgt);
+	if (ret)
+		goto err_destroy_map;
+
+	ret = nbl_dev_enable_mailbox_irq(dev_mgt);
+	if (ret)
+		goto err_free_irq;
+
+	return 0;
+
+err_free_irq:
+	/*
+	 * nbl_dev_enable_mailbox_irq() only fails before it sets
+	 * NBL_CHAN_IRQ_RDY - the flag is set after its RPC returns - so
+	 * there is no interrupt-mode sender to release and no software
+	 * state to revoke.  The peer-side mailbox routing, if the failed
+	 * RPC did land, is switched off by the destroy_msix_map() below: its
+	 * prepare phase, __nbl_res_intr_prepare_destroy_msix_map(), disables
+	 * mailbox IRQ routing for any CONFIGURED function before the map is
+	 * torn down.  That is what makes skipping set_mailbox_irq(false)
+	 * correct here, whatever way the enable failed - a local
+	 * -ESHUTDOWN/-ENODEV/-EINVAL on the control PF as much as a
+	 * polling-mode -ETIMEDOUT against an unresponsive peer.
+	 */
+	nbl_dev_free_mailbox_irq(dev_mgt);
+err_destroy_map:
+	/*
+	 * Always ask the peer to drop this function's MSI-X map, even when
+	 * the cfg RPC above failed: the teardown prepare phase returns 0
+	 * for a function that is not NBL_INTR_FUNC_CONFIGURED, so it is a
+	 * no-op when nothing landed, but a -ETIMEDOUT only means the ACK
+	 * never arrived - the responder runs to completion before it ACKs.
+	 * In that case the map entry, the intr_net/other_bmap bits, the
+	 * kcalloc()ed vector array and the coherent map table are all
+	 * still owned by the control PF, i.e. host resources of its
+	 * driver, released only when this function is configured again or
+	 * when nbl_intr_mgt_stop() tears the control PF down.  They are not
+	 * firmware state, so skipping the destroy would strand them for a
+	 * PF that has no driver to ever ask for them again.
+	 *
+	 * Kernel vectors are released first, by the
+	 * nbl_dev_clear_interrupt_scheme() below: the PCI core's MSI-X
+	 * shutdown writes the MSI-X table and reads it back, and those
+	 * accesses should land while this function's pcompleter map entry
+	 * is still valid.  The destroy then masks all hardware vectors
+	 * and clears that entry.
+	 *
+	 * For non-control PFs this is a polling-mode mailbox RPC:
+	 * NBL_CHAN_IRQ_RDY is never set on any rollback path, since
+	 * enabling it is the last step of nbl_dev_start().
+	 *
+	 * Note: pci_clear_master() runs in nbl_probe() core_start_err path;
+	 * on a non-control PF it cannot stop DMA using the management PF's
+	 * BDF, this is a known limitation.
+	 */
+	nbl_dev_clear_interrupt_scheme(dev_mgt);
+	cleanup_ret = nbl_dev_destroy_msix_map(dev_mgt);
+	if (cleanup_ret)
+		dev_err(dev_mgt->common->dev,
+			"rollback: destroy MSI-X map failed: %d\n",
+			cleanup_ret);
+
+	cancel_work_sync(&common_dev->clean_mbx_task);
+	return ret;
+}
+
+void nbl_dev_stop(struct nbl_adapter *adapter)
+{
+	struct nbl_dev_mgt *dev_mgt = adapter->core.dev_mgt;
+	struct nbl_dev_common *common_dev = dev_mgt->common_dev;
+	int ret;
+
+	ret = nbl_dev_disable_mailbox_irq(dev_mgt);
+	if (ret)
+		dev_err(dev_mgt->common->dev,
+			"Failed to disable mailbox IRQ: %d\n", ret);
+	nbl_dev_free_mailbox_irq(dev_mgt);
+
+	/*
+	 * Release the kernel vectors before destroying the hardware map:
+	 * the PCI core's MSI-X shutdown writes the MSI-X table and reads
+	 * it back, and those accesses should land while this function's
+	 * pcompleter map entry is still valid (see
+	 * nbl_dev_clear_interrupt_scheme).
+	 */
+	nbl_dev_clear_interrupt_scheme(dev_mgt);
+
+	/*
+	 * Destroy the hardware MSI-X map: masks all device vectors and
+	 * clears the pcompleter map entry.
+	 *
+	 * Best-effort: if the destroy RPC fails, the control PF still owns
+	 * this function's map entry, its intr_net/other_bmap bits and its
+	 * coherent map table.  Those are host resources of that driver,
+	 * released either by a later successful configuration of this
+	 * function - nbl_res_intr_cfg_msix_map() invalidates and rewrites
+	 * the entry on its had_config path - or by nbl_intr_mgt_stop()
+	 * when the control PF is removed, which walks every
+	 * NBL_INTR_FUNC_CONFIGURED function through
+	 * __nbl_res_intr_prepare_destroy_msix_map() and clears the
+	 * FUNCTION_MSIX_MAP valid bit.  Both rewrite the hardware entry
+	 * without a chip reset; nothing else collects it, so a stranded
+	 * one just stays as it is, and nothing requests one.
+	 *
+	 * pci_clear_master() on a non-control PF cannot stop DMA using the
+	 * management PF's BDF.
+	 */
+	ret = nbl_dev_destroy_msix_map(dev_mgt);
+	if (ret)
+		dev_err(dev_mgt->common->dev,
+			"Failed to destroy MSI-X map: %d\n", ret);
+
+	/*
+	 * destroy_msix_map() sends ack-requested messages which may
+	 * requeue clean_mbx_task via polling send path.  Drain work
+	 * after the operation.
+	 */
+	cancel_work_sync(&common_dev->clean_mbx_task);
+}
diff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_core/nbl_dev.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_core/nbl_dev.h
new file mode 100644
index 00000000000000..24e890fd898745
--- /dev/null
+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_core/nbl_dev.h
@@ -0,0 +1,55 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2026 Nebula Matrix Limited.
+ */
+
+#ifndef _NBL_DEV_H_
+#define _NBL_DEV_H_
+
+#include <linux/types.h>
+
+#include "../nbl_include/nbl_include.h"
+#include "../nbl_include/nbl_def_channel.h"
+#include "../nbl_include/nbl_def_hw.h"
+#include "../nbl_include/nbl_def_resource.h"
+#include "../nbl_include/nbl_def_dispatch.h"
+#include "../nbl_include/nbl_def_dev.h"
+#include "../nbl_include/nbl_def_common.h"
+#include "../nbl_core.h"
+
+#define NBL_STRING_NAME_LEN			32
+
+enum nbl_msix_serv_type {
+	NBL_MSIX_NET_TYPE,
+	NBL_MSIX_MAILBOX_TYPE,
+	NBL_MSIX_TYPE_MAX
+};
+
+struct nbl_msix_serv_info {
+	char irq_name[NBL_STRING_NAME_LEN];
+	u16 num;
+	u16 base_vector_id;
+	/* true: hw report msix, hw need to mask actively */
+	bool hw_self_mask_en;
+};
+
+struct nbl_msix_info {
+	struct nbl_msix_serv_info serv_info[NBL_MSIX_TYPE_MAX];
+};
+
+struct nbl_dev_common {
+	struct nbl_dev_mgt *dev_mgt;
+	struct nbl_msix_info msix_info;
+	char mailbox_name[NBL_STRING_NAME_LEN];
+	/* for ctrl-dev/net-dev mailbox recv msg */
+	struct work_struct clean_mbx_task;
+};
+
+struct nbl_dev_mgt {
+	struct nbl_common_info *common;
+	struct nbl_dispatch_ops_tbl *disp_ops_tbl;
+	struct nbl_channel_ops_tbl *chan_ops_tbl;
+	struct nbl_dev_common *common_dev;
+};
+
+#endif
diff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_core/nbl_dispatch.c b/drivers/net/ethernet/nebula-matrix/nbl/nbl_core/nbl_dispatch.c
new file mode 100644
index 00000000000000..59d80066f88b76
--- /dev/null
+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_core/nbl_dispatch.c
@@ -0,0 +1,671 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2026 Nebula Matrix Limited.
+ */
+#include <linux/device.h>
+#include <linux/pci.h>
+#include "nbl_dispatch.h"
+
+static int nbl_disp_chan_get_vsi_id_req(struct nbl_dispatch_mgt *disp_mgt,
+					u16 type, u16 *vsi_id)
+{
+	struct nbl_channel_ops *chan_ops = disp_mgt->chan_ops_tbl->ops;
+	struct nbl_common_info *common = disp_mgt->common;
+	struct nbl_chan_param_get_vsi_id result = { 0 };
+	struct nbl_chan_param_get_vsi_id param = { 0 };
+	struct nbl_chan_send_info chan_send = {0};
+	int ret;
+
+	param.type = cpu_to_le16(type);
+
+	nbl_chan_fill_send_info(&chan_send, common->mgt_pf,
+				NBL_CHAN_MSG_GET_VSI_ID,
+				&param, sizeof(param), &result,
+				sizeof(result), 1);
+	ret = chan_ops->send_msg(disp_mgt->chan_ops_tbl->priv, &chan_send);
+	if (ret)
+		return ret;
+	if (chan_send.ack_len != sizeof(result)) {
+		dev_err(disp_mgt->common->dev,
+			"get_vsi_id: short ACK, ack_len=%u expected %zu\n",
+			chan_send.ack_len, sizeof(result));
+		return -EBADMSG;
+	}
+	*vsi_id = le16_to_cpu(result.vsi_id);
+	return 0;
+}
+
+static void nbl_disp_chan_get_vsi_id_resp(void *priv, u16 src_id, u16 msg_id,
+					  void *data, u32 data_len)
+{
+	struct nbl_dispatch_mgt *disp_mgt = (struct nbl_dispatch_mgt *)priv;
+	struct nbl_channel_ops *chan_ops = disp_mgt->chan_ops_tbl->ops;
+	struct nbl_resource_ops *res_ops = disp_mgt->res_ops_tbl->ops;
+	struct nbl_resource_mgt *p = disp_mgt->res_ops_tbl->priv;
+	struct device *dev = disp_mgt->common->dev;
+	struct nbl_chan_param_get_vsi_id result = { 0 };
+	struct nbl_chan_param_get_vsi_id param = { 0 };
+	struct nbl_chan_ack_info chan_ack;
+	int err = 0;
+	u16 vsi_id = 0;
+	u32 rel_pf_id;
+	int ret;
+
+	ret = nbl_common_func_id_to_rel_pf_id(disp_mgt->common, src_id,
+					      &rel_pf_id);
+	if (ret) {
+		err = -EPERM;
+		goto ack_out;
+	}
+	if (rel_pf_id >= disp_mgt->common->max_pf) {
+		err = -EPERM;
+		goto ack_out;
+	}
+	if (data_len < sizeof(param)) {
+		err = -EBADMSG;
+		goto ack_out;
+	}
+	memcpy(&param, data, sizeof(param));
+
+	if (res_ops->get_vsi_id) {
+		ret = res_ops->get_vsi_id(p, src_id, le16_to_cpu(param.type),
+					  &vsi_id);
+		/* Forward the resource errno verbatim to the requester */
+		if (ret)
+			err = ret;
+	} else {
+		err = -EOPNOTSUPP;
+	}
+
+	result.vsi_id = cpu_to_le16(vsi_id);
+ack_out:
+	nbl_chan_fill_ack_info(&chan_ack, src_id,
+			       NBL_CHAN_MSG_GET_VSI_ID, msg_id, err,
+			       &result, sizeof(result));
+	ret = chan_ops->send_ack(disp_mgt->chan_ops_tbl->priv, &chan_ack);
+	if (ret)
+		dev_err(dev,
+			"channel send ack failed with ret: %d, msg_type: %d\n",
+			ret, NBL_CHAN_MSG_GET_VSI_ID);
+}
+
+static int nbl_disp_chan_get_eth_id_req(struct nbl_dispatch_mgt *disp_mgt,
+					u16 vsi_id, u8 *eth_num, u8 *eth_id,
+					u8 *logic_eth_id)
+{
+	struct nbl_channel_ops *chan_ops = disp_mgt->chan_ops_tbl->ops;
+	struct nbl_common_info *common = disp_mgt->common;
+	struct nbl_chan_param_get_eth_id result = { 0 };
+	struct nbl_chan_param_get_eth_id param = { 0 };
+	struct nbl_chan_send_info chan_send = {0};
+	int ret;
+
+	param.vsi_id = cpu_to_le16(vsi_id);
+
+	nbl_chan_fill_send_info(&chan_send, common->mgt_pf,
+				NBL_CHAN_MSG_GET_ETH_ID,
+				&param, sizeof(param), &result,
+				sizeof(result), 1);
+	ret = chan_ops->send_msg(disp_mgt->chan_ops_tbl->priv, &chan_send);
+	if (ret)
+		return ret;
+	if (chan_send.ack_len != sizeof(result)) {
+		dev_err(disp_mgt->common->dev,
+			"get_eth_id: short ACK, ack_len=%u expected %zu\n",
+			chan_send.ack_len, sizeof(result));
+		return -EBADMSG;
+	}
+	*eth_num = result.eth_num;
+	*eth_id = result.eth_id;
+	*logic_eth_id = result.logic_eth_id;
+
+	return 0;
+}
+
+static void nbl_disp_chan_get_eth_id_resp(void *priv, u16 src_id, u16 msg_id,
+					  void *data, u32 data_len)
+{
+	struct nbl_dispatch_mgt *disp_mgt = (struct nbl_dispatch_mgt *)priv;
+	struct nbl_channel_ops *chan_ops = disp_mgt->chan_ops_tbl->ops;
+	struct nbl_resource_ops *res_ops = disp_mgt->res_ops_tbl->ops;
+	struct nbl_resource_mgt *p = disp_mgt->res_ops_tbl->priv;
+	struct nbl_chan_param_get_eth_id result = { 0 };
+	struct nbl_chan_param_get_eth_id param = { 0 };
+	struct device *dev = disp_mgt->common->dev;
+	struct nbl_chan_ack_info chan_ack;
+	int err = 0;
+	u32 rel_pf_id;
+	int ret;
+
+	ret = nbl_common_func_id_to_rel_pf_id(disp_mgt->common, src_id,
+					      &rel_pf_id);
+	if (ret) {
+		err = -EPERM;
+		goto ack_out;
+	}
+	if (rel_pf_id >= disp_mgt->common->max_pf) {
+		err = -EPERM;
+		goto ack_out;
+	}
+	if (data_len < sizeof(param)) {
+		err = -EBADMSG;
+		goto ack_out;
+	}
+	memcpy(&param, data, sizeof(param));
+
+	if (res_ops->get_eth_id) {
+		ret = res_ops->get_eth_id(p, src_id, le16_to_cpu(param.vsi_id),
+					  &result.eth_num, &result.eth_id,
+					  &result.logic_eth_id);
+		/* Forward the resource errno verbatim to the requester */
+		if (ret)
+			err = ret;
+	} else {
+		err = -EOPNOTSUPP;
+	}
+ack_out:
+	nbl_chan_fill_ack_info(&chan_ack, src_id,
+			       NBL_CHAN_MSG_GET_ETH_ID, msg_id, err,
+			       &result, sizeof(result));
+	ret = chan_ops->send_ack(disp_mgt->chan_ops_tbl->priv, &chan_ack);
+	if (ret)
+		dev_err(dev,
+			"channel send ack failed with ret: %d, msg_type: %d\n",
+			ret, NBL_CHAN_MSG_GET_ETH_ID);
+}
+
+static void nbl_disp_deinit_module(struct nbl_dispatch_mgt *disp_mgt)
+{
+	struct nbl_resource_ops *res_ops = disp_mgt->res_ops_tbl->ops;
+	struct nbl_resource_mgt *p = disp_mgt->res_ops_tbl->priv;
+
+	if (res_ops->deinit_module)
+		res_ops->deinit_module(p);
+}
+
+static int nbl_disp_init_module(struct nbl_dispatch_mgt *disp_mgt)
+{
+	struct nbl_resource_ops *res_ops = disp_mgt->res_ops_tbl->ops;
+	struct nbl_resource_mgt *p = disp_mgt->res_ops_tbl->priv;
+
+	if (res_ops->init_module)
+		return res_ops->init_module(p);
+	return -EOPNOTSUPP;
+}
+
+static int nbl_disp_cfg_msix_map(struct nbl_dispatch_mgt *disp_mgt,
+				 u16 num_net_msix, u16 num_others_msix,
+				 bool net_msix_mask_en)
+{
+	struct nbl_resource_ops *res_ops = disp_mgt->res_ops_tbl->ops;
+	struct nbl_resource_mgt *p = disp_mgt->res_ops_tbl->priv;
+	struct nbl_common_info *common = disp_mgt->common;
+
+	if (!res_ops->cfg_msix_map)
+		return -EOPNOTSUPP;
+	/*
+	 * No core-layer lock is taken here: the resource layer serializes
+	 * concurrent callers itself (intr_mgt->lock), and the dispatch
+	 * layer has no mutable state of its own for this op to protect.
+	 */
+	return res_ops->cfg_msix_map(p, common->mgt_pf, num_net_msix,
+				     num_others_msix, net_msix_mask_en);
+}
+
+/*
+ * Precondition: caller must disable mailbox IRQ_RDY and switch send_msg
+ * to the polling path before issuing this RPC. The responder rewrites
+ * the requester's mailbox MSI-X routing during the resource op, so the
+ * ACK cannot rely on interrupt wakeup while routing is in flux.
+ */
+static int
+nbl_disp_chan_cfg_msix_map_req(struct nbl_dispatch_mgt *disp_mgt,
+			       u16 num_net_msix, u16 num_others_msix,
+			       bool net_msix_mask_en)
+{
+	struct nbl_channel_ops *chan_ops = disp_mgt->chan_ops_tbl->ops;
+	struct nbl_common_info *common = disp_mgt->common;
+	struct nbl_chan_param_cfg_msix_map param = { 0 };
+	struct nbl_chan_send_info chan_send = {0};
+	int ret;
+
+	param.num_net_msix = cpu_to_le16(num_net_msix);
+	param.num_others_msix = cpu_to_le16(num_others_msix);
+	param.msix_mask_en = cpu_to_le16(!!net_msix_mask_en);
+
+	nbl_chan_fill_send_info(&chan_send, common->mgt_pf,
+				NBL_CHAN_MSG_CONFIGURE_MSIX_MAP,
+				&param, sizeof(param),
+				NULL, 0, 1);
+	ret = chan_ops->send_msg(disp_mgt->chan_ops_tbl->priv, &chan_send);
+	if (ret)
+		return ret;
+	return 0;
+}
+
+static void nbl_disp_chan_cfg_msix_map_resp(void *priv, u16 src_id, u16 msg_id,
+					    void *data, u32 data_len)
+{
+	struct nbl_dispatch_mgt *disp_mgt = (struct nbl_dispatch_mgt *)priv;
+	struct nbl_channel_ops *chan_ops = disp_mgt->chan_ops_tbl->ops;
+	struct nbl_resource_ops *res_ops = disp_mgt->res_ops_tbl->ops;
+	struct nbl_resource_mgt *p = disp_mgt->res_ops_tbl->priv;
+	struct device *dev = disp_mgt->common->dev;
+	struct nbl_chan_param_cfg_msix_map param = { 0 };
+	struct nbl_chan_ack_info chan_ack;
+	int err = 0;
+	u32 rel_pf_id;
+	int ret;
+
+	ret = nbl_common_func_id_to_rel_pf_id(disp_mgt->common, src_id,
+					      &rel_pf_id);
+	if (ret) {
+		err = -EPERM;
+		goto ack_out;
+	}
+	if (rel_pf_id >= disp_mgt->common->max_pf) {
+		err = -EPERM;
+		goto ack_out;
+	}
+	if (data_len < sizeof(param)) {
+		err = -EBADMSG;
+		goto ack_out;
+	}
+	memcpy(&param, data, sizeof(param));
+
+	if (res_ops->cfg_msix_map) {
+		/*
+		 * The vector counts come from the peer PF.  A peer is
+		 * trusted: hardware and firmware forbid PF passthrough to
+		 * guests (see nbl_get_func_param()), so every peer runs this
+		 * same host driver, and the resource layer bounds the counts
+		 * by the global vector pool.  A peer asking for the whole
+		 * pool can starve the others with -EAGAIN, but it cannot
+		 * corrupt state.
+		 */
+		ret = res_ops->cfg_msix_map(p, src_id,
+					    le16_to_cpu(param.num_net_msix),
+					    le16_to_cpu(param.num_others_msix),
+					    !!le16_to_cpu(param.msix_mask_en));
+		/* Forward the resource errno verbatim to the requester */
+		if (ret)
+			err = ret;
+	} else {
+		err = -EOPNOTSUPP;
+	}
+ack_out:
+	nbl_chan_fill_ack_info(&chan_ack, src_id,
+			       NBL_CHAN_MSG_CONFIGURE_MSIX_MAP, msg_id,
+			       err, NULL, 0);
+	ret = chan_ops->send_ack(disp_mgt->chan_ops_tbl->priv, &chan_ack);
+	if (ret)
+		dev_err(dev,
+			"channel send ack failed with ret: %d, msg_type: %d\n",
+			ret, NBL_CHAN_MSG_CONFIGURE_MSIX_MAP);
+}
+
+/*
+ * Precondition: caller must disable mailbox IRQ_RDY and switch send_msg
+ * to the polling path before issuing this RPC. The responder retargets
+ * the requester's mailbox MSI-X routing during the resource op, so the
+ * ACK cannot rely on interrupt wakeup while routing is in flux.
+ */
+static int nbl_disp_chan_destroy_msix_map_req(struct nbl_dispatch_mgt *disp_mgt)
+{
+	struct nbl_channel_ops *chan_ops = disp_mgt->chan_ops_tbl->ops;
+	struct nbl_common_info *common = disp_mgt->common;
+	struct nbl_chan_send_info chan_send = {0};
+	int ret;
+
+	nbl_chan_fill_send_info(&chan_send, common->mgt_pf,
+				NBL_CHAN_MSG_DESTROY_MSIX_MAP,
+				NULL, 0, NULL, 0, 1);
+	ret = chan_ops->send_msg(disp_mgt->chan_ops_tbl->priv, &chan_send);
+	if (ret)
+		return ret;
+	return 0;
+}
+
+static void nbl_disp_chan_destroy_msix_map_resp(void *priv, u16 src_id,
+						u16 msg_id, void *data,
+						u32 data_len)
+{
+	struct nbl_dispatch_mgt *disp_mgt = (struct nbl_dispatch_mgt *)priv;
+	struct nbl_channel_ops *chan_ops = disp_mgt->chan_ops_tbl->ops;
+	struct nbl_resource_ops *res_ops = disp_mgt->res_ops_tbl->ops;
+	struct nbl_resource_mgt *p = disp_mgt->res_ops_tbl->priv;
+	struct device *dev = disp_mgt->common->dev;
+	struct nbl_chan_ack_info chan_ack;
+	int err = 0;
+	u32 rel_pf_id;
+	int ret;
+
+	ret = nbl_common_func_id_to_rel_pf_id(disp_mgt->common, src_id,
+					      &rel_pf_id);
+	if (ret) {
+		err = -EPERM;
+		goto ack_out;
+	}
+	if (rel_pf_id >= disp_mgt->common->max_pf) {
+		err = -EPERM;
+		goto ack_out;
+	}
+	if (res_ops->destroy_msix_map) {
+		ret = res_ops->destroy_msix_map(p, src_id);
+		/* Forward the resource errno verbatim to the requester */
+		if (ret)
+			err = ret;
+	} else {
+		err = -EOPNOTSUPP;
+	}
+ack_out:
+	nbl_chan_fill_ack_info(&chan_ack, src_id,
+			       NBL_CHAN_MSG_DESTROY_MSIX_MAP, msg_id,
+			       err, NULL, 0);
+	ret = chan_ops->send_ack(disp_mgt->chan_ops_tbl->priv, &chan_ack);
+	if (ret)
+		dev_err(dev,
+			"channel send ack failed with ret: %d, msg_type: %d\n",
+			ret, NBL_CHAN_MSG_DESTROY_MSIX_MAP);
+}
+
+/*
+ * Precondition: caller must disable mailbox IRQ_RDY and switch send_msg
+ * to the polling path before issuing this RPC. The responder rewrites
+ * the requester's own mailbox MSI-X routing (MSIX_IDX / MSIX_IDX_VALID)
+ * before the ACK is sent, so the ACK cannot rely on interrupt wakeup
+ * while routing is in flux.
+ */
+static int nbl_disp_chan_set_mailbox_irq_req(struct nbl_dispatch_mgt *disp_mgt,
+					     u16 vector_id, bool en_msix)
+{
+	struct nbl_channel_ops *chan_ops = disp_mgt->chan_ops_tbl->ops;
+	struct nbl_chan_param_set_mailbox_irq param = { 0 };
+	struct nbl_common_info *common = disp_mgt->common;
+	struct nbl_chan_send_info chan_send = {0};
+	int ret;
+
+	param.vector_id = cpu_to_le16(vector_id);
+	param.en_msix = !!en_msix;
+
+	nbl_chan_fill_send_info(&chan_send, common->mgt_pf,
+				NBL_CHAN_MSG_MAILBOX_SET_IRQ,
+				&param, sizeof(param), NULL, 0, 1);
+	ret = chan_ops->send_msg(disp_mgt->chan_ops_tbl->priv, &chan_send);
+	if (ret)
+		return ret;
+	return 0;
+}
+
+static void nbl_disp_chan_set_mailbox_irq_resp(void *priv, u16 src_id,
+					       u16 msg_id, void *data,
+					       u32 data_len)
+{
+	struct nbl_dispatch_mgt *disp_mgt = (struct nbl_dispatch_mgt *)priv;
+	struct nbl_channel_ops *chan_ops = disp_mgt->chan_ops_tbl->ops;
+	struct nbl_resource_ops *res_ops = disp_mgt->res_ops_tbl->ops;
+	struct nbl_resource_mgt *p = disp_mgt->res_ops_tbl->priv;
+	struct nbl_chan_param_set_mailbox_irq param = { 0 };
+	struct device *dev = disp_mgt->common->dev;
+	struct nbl_chan_ack_info chan_ack;
+	int err = 0;
+	u16 vector_id;
+	u32 rel_pf_id;
+	bool en_msix;
+	int ret;
+
+	ret = nbl_common_func_id_to_rel_pf_id(disp_mgt->common, src_id,
+					      &rel_pf_id);
+	if (ret) {
+		err = -EPERM;
+		goto ack_out;
+	}
+	if (rel_pf_id >= disp_mgt->common->max_pf) {
+		err = -EPERM;
+		goto ack_out;
+	}
+	if (data_len < sizeof(param)) {
+		err = -EBADMSG;
+		goto ack_out;
+	}
+	memcpy(&param, data, sizeof(param));
+	vector_id = le16_to_cpu(param.vector_id);
+	en_msix = !!param.en_msix;
+
+	if (res_ops->set_mailbox_irq) {
+		ret = res_ops->set_mailbox_irq(p, src_id, vector_id, en_msix);
+		/* Forward the resource errno verbatim to the requester */
+		if (ret)
+			err = ret;
+	} else {
+		err = -EOPNOTSUPP;
+	}
+
+ack_out:
+	nbl_chan_fill_ack_info(&chan_ack, src_id,
+			       NBL_CHAN_MSG_MAILBOX_SET_IRQ, msg_id,
+			       err, NULL, 0);
+	ret = chan_ops->send_ack(disp_mgt->chan_ops_tbl->priv, &chan_ack);
+	if (ret)
+		dev_err(dev,
+			"channel send ack failed with ret: %d, msg_type: %d\n",
+			ret, NBL_CHAN_MSG_MAILBOX_SET_IRQ);
+}
+
+static int nbl_disp_destroy_msix_map(struct nbl_dispatch_mgt *disp_mgt)
+{
+	struct nbl_resource_ops *res_ops = disp_mgt->res_ops_tbl->ops;
+	struct nbl_resource_mgt *p = disp_mgt->res_ops_tbl->priv;
+	struct nbl_common_info *common = disp_mgt->common;
+
+	if (!res_ops->destroy_msix_map)
+		return -EOPNOTSUPP;
+	return res_ops->destroy_msix_map(p, common->mgt_pf);
+}
+
+static int nbl_disp_set_mailbox_irq(struct nbl_dispatch_mgt *disp_mgt,
+				    u16 vector_id, bool en_msix)
+{
+	struct nbl_resource_ops *res_ops = disp_mgt->res_ops_tbl->ops;
+	struct nbl_resource_mgt *p = disp_mgt->res_ops_tbl->priv;
+	struct nbl_common_info *common = disp_mgt->common;
+
+	if (!res_ops->set_mailbox_irq)
+		return -EOPNOTSUPP;
+	return res_ops->set_mailbox_irq(p, common->mgt_pf, vector_id, en_msix);
+}
+
+static int nbl_disp_get_vsi_id(struct nbl_dispatch_mgt *disp_mgt, u16 type,
+			       u16 *vsi_id)
+{
+	struct nbl_resource_ops *res_ops = disp_mgt->res_ops_tbl->ops;
+	struct nbl_resource_mgt *p = disp_mgt->res_ops_tbl->priv;
+	struct nbl_common_info *common = disp_mgt->common;
+
+	if (res_ops->get_vsi_id)
+		return res_ops->get_vsi_id(p, common->mgt_pf, type, vsi_id);
+	return -EOPNOTSUPP;
+}
+
+static int nbl_disp_get_eth_id(struct nbl_dispatch_mgt *disp_mgt, u16 vsi_id,
+			       u8 *eth_num, u8 *eth_id, u8 *logic_eth_id)
+{
+	struct nbl_resource_ops *res_ops = disp_mgt->res_ops_tbl->ops;
+	struct nbl_resource_mgt *p = disp_mgt->res_ops_tbl->priv;
+	struct nbl_common_info *common = disp_mgt->common;
+
+	if (res_ops->get_eth_id)
+		return res_ops->get_eth_id(p, common->mgt_pf, vsi_id,
+					   eth_num, eth_id, logic_eth_id);
+	return -EOPNOTSUPP;
+}
+
+static int nbl_disp_setup_msg(struct nbl_dispatch_mgt *disp_mgt)
+{
+	struct nbl_channel_ops *chan_ops = disp_mgt->chan_ops_tbl->ops;
+	struct nbl_channel_mgt *p = disp_mgt->chan_ops_tbl->priv;
+	int ret = 0;
+	int _ret;
+
+	_ret = chan_ops->register_msg(p, NBL_CHAN_MSG_CONFIGURE_MSIX_MAP,
+				      nbl_disp_chan_cfg_msix_map_resp,
+				      disp_mgt);
+	if (_ret < 0 && !ret)
+		ret = _ret;
+
+	_ret = chan_ops->register_msg(p, NBL_CHAN_MSG_DESTROY_MSIX_MAP,
+				      nbl_disp_chan_destroy_msix_map_resp,
+				      disp_mgt);
+	if (_ret < 0 && !ret)
+		ret = _ret;
+
+	_ret = chan_ops->register_msg(p, NBL_CHAN_MSG_MAILBOX_SET_IRQ,
+				      nbl_disp_chan_set_mailbox_irq_resp,
+				      disp_mgt);
+	if (_ret < 0 && !ret)
+		ret = _ret;
+
+	_ret = chan_ops->register_msg(p, NBL_CHAN_MSG_GET_VSI_ID,
+				      nbl_disp_chan_get_vsi_id_resp,
+				      disp_mgt);
+	if (_ret < 0 && !ret)
+		ret = _ret;
+
+	_ret = chan_ops->register_msg(p, NBL_CHAN_MSG_GET_ETH_ID,
+				      nbl_disp_chan_get_eth_id_resp,
+				      disp_mgt);
+	if (_ret < 0 && !ret)
+		ret = _ret;
+
+	return ret;
+}
+
+static void nbl_disp_set_ctrl_bit(struct nbl_dispatch_mgt *disp_mgt, u32 lvl)
+{
+	set_bit(lvl, disp_mgt->ctrl_lvl);
+}
+
+static void nbl_disp_refresh_ctrl_ops(struct nbl_dispatch_mgt *disp_mgt)
+{
+	struct nbl_dispatch_ops *disp_ops = disp_mgt->disp_ops_tbl->ops;
+
+	memset(disp_ops, 0, sizeof(*disp_ops));
+	if (test_bit(NBL_DISP_CTRL_LVL_MGT, disp_mgt->ctrl_lvl)) {
+		disp_ops->init_module = nbl_disp_init_module;
+		disp_ops->deinit_module = nbl_disp_deinit_module;
+		disp_ops->cfg_msix_map = nbl_disp_cfg_msix_map;
+		disp_ops->destroy_msix_map = nbl_disp_destroy_msix_map;
+		disp_ops->set_mailbox_irq = nbl_disp_set_mailbox_irq;
+		disp_ops->get_vsi_id = nbl_disp_get_vsi_id;
+		disp_ops->get_eth_id = nbl_disp_get_eth_id;
+	} else if (test_bit(NBL_DISP_CTRL_LVL_NET, disp_mgt->ctrl_lvl)) {
+		disp_ops->cfg_msix_map =
+			nbl_disp_chan_cfg_msix_map_req;
+		disp_ops->destroy_msix_map = nbl_disp_chan_destroy_msix_map_req;
+		disp_ops->set_mailbox_irq = nbl_disp_chan_set_mailbox_irq_req;
+		disp_ops->get_vsi_id = nbl_disp_chan_get_vsi_id_req;
+		disp_ops->get_eth_id = nbl_disp_chan_get_eth_id_req;
+	}
+}
+
+static struct nbl_dispatch_mgt *
+nbl_disp_setup_disp_mgt(struct nbl_common_info *common)
+{
+	struct nbl_dispatch_mgt *disp_mgt;
+	struct device *dev = common->dev;
+
+	disp_mgt = devm_kzalloc(dev, sizeof(*disp_mgt), GFP_KERNEL);
+	if (!disp_mgt)
+		return ERR_PTR(-ENOMEM);
+
+	disp_mgt->common = common;
+	return disp_mgt;
+}
+
+static struct nbl_dispatch_ops_tbl *
+nbl_disp_setup_ops(struct device *dev, struct nbl_dispatch_mgt *disp_mgt)
+{
+	struct nbl_dispatch_ops_tbl *disp_ops_tbl;
+	struct nbl_dispatch_ops *disp_ops;
+
+	disp_ops_tbl = devm_kzalloc(dev, sizeof(*disp_ops_tbl), GFP_KERNEL);
+	if (!disp_ops_tbl)
+		return ERR_PTR(-ENOMEM);
+
+	disp_ops = devm_kzalloc(dev, sizeof(*disp_ops), GFP_KERNEL);
+	if (!disp_ops)
+		return ERR_PTR(-ENOMEM);
+
+	disp_ops_tbl->ops = disp_ops;
+	disp_ops_tbl->priv = disp_mgt;
+
+	return disp_ops_tbl;
+}
+
+int nbl_disp_init(struct nbl_adapter *adapter)
+{
+	struct nbl_common_info *common = &adapter->common;
+	struct nbl_dispatch_ops_tbl *disp_ops_tbl;
+	struct nbl_resource_ops_tbl *res_ops_tbl =
+		adapter->intf.resource_ops_tbl;
+	struct nbl_channel_ops_tbl *chan_ops_tbl =
+		adapter->intf.channel_ops_tbl;
+	struct device *dev = &adapter->pdev->dev;
+	struct nbl_dispatch_mgt *disp_mgt;
+	int ret;
+
+	disp_mgt = nbl_disp_setup_disp_mgt(common);
+	if (IS_ERR(disp_mgt)) {
+		ret = PTR_ERR(disp_mgt);
+		return ret;
+	}
+
+	disp_ops_tbl = nbl_disp_setup_ops(dev, disp_mgt);
+	if (IS_ERR(disp_ops_tbl)) {
+		ret = PTR_ERR(disp_ops_tbl);
+		return ret;
+	}
+
+	disp_mgt->res_ops_tbl = res_ops_tbl;
+	disp_mgt->chan_ops_tbl = chan_ops_tbl;
+	disp_mgt->disp_ops_tbl = disp_ops_tbl;
+	adapter->core.disp_mgt = disp_mgt;
+	adapter->intf.dispatch_ops_tbl = disp_ops_tbl;
+
+	ret = nbl_disp_setup_msg(disp_mgt);
+	if (ret)
+		return ret;
+
+	if (common->has_ctrl)
+		nbl_disp_set_ctrl_bit(disp_mgt, NBL_DISP_CTRL_LVL_MGT);
+
+	if (common->has_net)
+		nbl_disp_set_ctrl_bit(disp_mgt, NBL_DISP_CTRL_LVL_NET);
+	nbl_disp_refresh_ctrl_ops(disp_mgt);
+	return 0;
+}
+
+void nbl_disp_remove(struct nbl_adapter *adapter)
+{
+	/*
+	 * Dispatch structures are devm-allocated and freed at detach.
+	 *
+	 * The five responders registered by nbl_disp_setup_msg() are
+	 * owned by the channel layer (xarray of handlers) and are never
+	 * unregistered here. This is safe because the teardown order
+	 * guarantees no responder can run after this point:
+	 *
+	 *   mailbox teardown
+	 *     -> cancel_work_sync(clean_mbx_task)   // drain RX work
+	 *     -> nbl_chan_teardown_queue()          // stop HW queue,
+	 *                                                 // join clean task,
+	 *                                                 // active=false
+	 *     -> nbl_chan_remove_common()
+	 *          -> nbl_common_destroy_wq()            // no new work
+	 *          -> nbl_chan_remove_msg_handler()      // free handler nodes
+	 *
+	 * By the time devres frees disp_mgt, the mailbox queue is stopped
+	 * and the handler xarray is empty, so no responder can touch
+	 * res_mgt->intr_mgt after nbl_intr_mgt_stop() has cleared it.
+	 */
+}
diff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_core/nbl_dispatch.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_core/nbl_dispatch.h
new file mode 100644
index 00000000000000..a7e5802344b414
--- /dev/null
+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_core/nbl_dispatch.h
@@ -0,0 +1,23 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2026 Nebula Matrix Limited.
+ */
+
+#ifndef _NBL_DISPATCH_H_
+#define _NBL_DISPATCH_H_
+#include "../nbl_include/nbl_include.h"
+#include "../nbl_include/nbl_def_channel.h"
+#include "../nbl_include/nbl_def_resource.h"
+#include "../nbl_include/nbl_def_dispatch.h"
+#include "../nbl_include/nbl_def_common.h"
+#include "../nbl_core.h"
+
+struct nbl_dispatch_mgt {
+	struct nbl_common_info *common;
+	struct nbl_resource_ops_tbl *res_ops_tbl;
+	struct nbl_channel_ops_tbl *chan_ops_tbl;
+	struct nbl_dispatch_ops_tbl *disp_ops_tbl;
+	DECLARE_BITMAP(ctrl_lvl, NBL_DISP_CTRL_LVL_MAX);
+};
+
+#endif
diff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_chip.c b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_chip.c
new file mode 100644
index 00000000000000..419eb6392ada19
--- /dev/null
+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_chip.c
@@ -0,0 +1,32 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2026 Nebula Matrix Limited.
+ */
+#include <linux/device.h>
+#include "nbl_chip.h"
+
+void nbl_res_chip_deinit_module(struct nbl_resource_mgt *res_mgt)
+{
+	struct nbl_hw_ops *hw_ops = res_mgt->hw_ops_tbl->ops;
+	struct nbl_common_info *common = res_mgt->common;
+
+	if (!common->has_ctrl)
+		return;
+	hw_ops->deinit_module(res_mgt->hw_ops_tbl->priv);
+}
+
+int nbl_res_chip_init_module(struct nbl_resource_mgt *res_mgt)
+{
+	struct nbl_common_info *common = res_mgt->common;
+	struct nbl_hw_ops *hw_ops;
+	u8 eth_speed, eth_num;
+	struct nbl_hw_mgt *p;
+
+	if (!common->has_ctrl)
+		return -EINVAL;
+	eth_speed = res_mgt->resource_info->board_info.eth_speed;
+	eth_num = res_mgt->resource_info->board_info.eth_num;
+	hw_ops = res_mgt->hw_ops_tbl->ops;
+	p = res_mgt->hw_ops_tbl->priv;
+	return hw_ops->init_module(p, eth_speed, eth_num);
+}
diff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_chip.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_chip.h
new file mode 100644
index 00000000000000..d14093ba916c95
--- /dev/null
+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_chip.h
@@ -0,0 +1,12 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2026 Nebula Matrix Limited.
+ */
+
+#ifndef _NBL_CHIP_H_
+#define _NBL_CHIP_H_
+
+#include "nbl_resource.h"
+int nbl_res_chip_init_module(struct nbl_resource_mgt *res_mgt);
+void nbl_res_chip_deinit_module(struct nbl_resource_mgt *res_mgt);
+#endif
diff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_leonis/nbl_hw_leonis.c b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_leonis/nbl_hw_leonis.c
index cf40ddc4519211..2b55f2c2518b4f 100644
--- a/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_leonis/nbl_hw_leonis.c
+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_leonis/nbl_hw_leonis.c
@@ -5,9 +5,1232 @@
 #include <linux/device.h>
 #include <linux/pci.h>
 #include <linux/bits.h>
+#include <linux/delay.h>
 #include <linux/io.h>
+#include <linux/spinlock.h>
+#include <linux/bitfield.h>
 #include "nbl_hw_leonis.h"
 
+/*
+ * Firmware cleanup after driver_status=false is asynchronous and the
+ * current hardware revision exposes no cleanup-complete status bit, so
+ * this delay is best-effort only: it gives the firmware pass time to
+ * settle before the caller returns, but cannot prove that it has.
+ * Bounding the wait keeps a later probe from re-programming the per-PF
+ * registers that pass reclaims (mailbox QINFO routing, qinfo registers)
+ * while firmware is realistically still wiping them.  The chip-wide
+ * datapath registers init_module() writes are not part of that pass and
+ * stay programmed until chip reset.  The hard teardown ordering comes
+ * from the device-link relationship between the sibling PFs and the
+ * control PF.
+ *
+ * These bounds are empirical, not derived from a documented firmware
+ * latency.  There is nothing to poll for instead: the hardware exposes
+ * no cleanup-complete bit, and the reclaimed entries read back the same
+ * before and after the pass, so there is no observable side effect to
+ * wait on and nothing to validate the delay against.  A firmware pass
+ * that takes longer than this window - an ECPU still busy right after
+ * kexec, say - can therefore still race a later probe.
+ */
+#define NBL_FW_CLEANUP_SYNC_MIN_US	2000
+#define NBL_FW_CLEANUP_SYNC_MAX_US	3000
+
+static void nbl_hw_read_mbx_regs(struct nbl_hw_mgt *hw_mgt, u64 reg, u32 *data,
+				 u32 len)
+{
+	u32 i;
+
+	if (len % 4)
+		return;
+	if (reg >= (u64)hw_mgt->mailbox_bar_size ||
+	    reg + len > (u64)hw_mgt->mailbox_bar_size) {
+		dev_err_once(hw_mgt->common->dev,
+			     "mbx read out of range: reg=0x%llx len=%u bar_size=%pa\n",
+			     reg, len, &hw_mgt->mailbox_bar_size);
+		return;
+	}
+	for (i = 0; i < len / 4; i++)
+		data[i] = nbl_mbx_rd32(hw_mgt, reg + i * sizeof(u32));
+}
+
+static void nbl_hw_write_mbx_regs(struct nbl_hw_mgt *hw_mgt, u64 reg,
+				  const u32 *data, u32 len)
+{
+	u32 i;
+
+	if (len % 4)
+		return;
+	if (reg >= (u64)hw_mgt->mailbox_bar_size ||
+	    reg + len > (u64)hw_mgt->mailbox_bar_size) {
+		dev_err_once(hw_mgt->common->dev,
+			     "mbx write out of range: reg=0x%llx len=%u bar_size=%pa\n",
+			     reg, len, &hw_mgt->mailbox_bar_size);
+		return;
+	}
+	for (i = 0; i < len / 4; i++)
+		nbl_mbx_wr32(hw_mgt, reg + i * sizeof(u32), data[i]);
+}
+
+/*
+ * Flush posted mailbox-BAR writes by reading back through the same
+ * BAR.
+ */
+static void nbl_hw_flush_mbx_write(struct nbl_hw_mgt *hw_mgt, u64 reg)
+{
+	u32 data;
+
+	nbl_hw_read_mbx_regs(hw_mgt, reg, &data, sizeof(data));
+}
+
+static void nbl_hw_rd_regs(struct nbl_hw_mgt *hw_mgt, u64 reg, u32 *data,
+			   u32 len)
+{
+	u32 size = len / 4;
+	u32 i;
+
+	if (len % 4)
+		return;
+	for (i = 0; i < size; i++)
+		data[i] = rd32(hw_mgt->hw_addr, reg + i * sizeof(u32));
+}
+
+static void nbl_hw_wr_regs(struct nbl_hw_mgt *hw_mgt, u64 reg, const u32 *data,
+			   u32 len)
+{
+	u32 size = len / 4;
+	u32 i;
+
+	if (len % 4)
+		return;
+	for (i = 0; i < size; i++)
+		wr32(hw_mgt->hw_addr, reg + i * sizeof(u32), data[i]);
+}
+
+static void nbl_hw_rd_regs_lock(struct nbl_hw_mgt *hw_mgt, u64 reg, u32 *data,
+				u32 len)
+{
+	u32 size = len / 4;
+	u32 i;
+
+	if (len % 4)
+		return;
+
+	spin_lock(&hw_mgt->reg_lock);
+
+	for (i = 0; i < size; i++)
+		data[i] = rd32(hw_mgt->hw_addr, reg + i * sizeof(u32));
+	spin_unlock(&hw_mgt->reg_lock);
+}
+
+static void nbl_hw_wr_regs_lock(struct nbl_hw_mgt *hw_mgt, u64 reg,
+				const u32 *data, u32 len)
+{
+	u32 size = len / 4;
+	u32 i;
+
+	if (len % 4)
+		return;
+	spin_lock(&hw_mgt->reg_lock);
+	for (i = 0; i < size; i++)
+		wr32(hw_mgt->hw_addr, reg + i * sizeof(u32), data[i]);
+	spin_unlock(&hw_mgt->reg_lock);
+}
+
+/*
+ * Flush posted MEMORY-BAR writes by reading back a register that only
+ * exists in the control-PF mapping.
+ *
+ * The mapped size does not derive from bar_len: nbl_hw_init_leonis()
+ * maps PAGE_ALIGN_DOWN(NBL_MEM_BAR_TOTAL_SIZE - NBL_RDMA_NOTIFY_LEN)
+ * (~64 MiB) when has_ctrl is set, and NBL_REG_NET_ONLY_LEN (8 KiB) when
+ * it is not.  NBL_HW_DUMMY_REG (0x1300904) is only covered by the
+ * control-PF mapping, so this must not be called on a net-only PF.
+ *
+ * Every current caller runs on the control PF: init_module()/
+ * deinit_module() are has_ctrl-guarded in the resource layer, and the
+ * MSI-X paths are either has_ctrl-guarded as well or dispatched to the
+ * management PF.  The check below is therefore a defensive guard against
+ * a future caller, not a reachable failure path.
+ */
+static void nbl_flush_writes(struct nbl_hw_mgt *hw_mgt)
+{
+	if (WARN_ON_ONCE(!hw_mgt->common->has_ctrl))
+		return;
+
+	nbl_hw_rd32(hw_mgt, NBL_HW_DUMMY_REG);
+}
+
+/*
+ * Registers reset to zero after cold boot / FLR / bus reset. Firmware
+ * programs valid values before driver probe, so zero is only seen on
+ * hardware fault or register read failure. Initialize data=0 to guard
+ * against nbl_hw_read_mbx_regs() early-return on bounds-check failure.
+ */
+static void nbl_hw_get_fw_eth_map(struct nbl_hw_mgt *hw_mgt, u32 *eth_map)
+{
+	u32 data = 0;
+
+	nbl_hw_read_mbx_regs(hw_mgt, NBL_FW_BOARD_DW6_OFFSET, &data,
+			     sizeof(data));
+	*eth_map = FIELD_GET(NBL_FW_BOARD_DW6_ETH_BITMAP_MASK, data);
+}
+
+static u32 nbl_hw_get_quirks(struct nbl_hw_mgt *hw_mgt)
+{
+	u32 quirks = 0;
+
+	/*
+	 * Read quirk bits from mailbox register.
+	 * All supported firmware implement the quirk ABI,
+	 * firmware always populates NBL_LEONIS_QUIRKS_OFFSET.
+	 * Value ~0U indicates no active quirks.
+	 */
+	nbl_hw_read_mbx_regs(hw_mgt, NBL_LEONIS_QUIRKS_OFFSET, &quirks,
+			     sizeof(u32));
+
+	if (quirks == ~0u)
+		return 0;
+
+	return quirks;
+}
+
+static void nbl_configure_dped_checksum(struct nbl_hw_mgt *hw_mgt)
+{
+	u32 data = 0;
+
+	/* DPED dped_l4_ck_cmd_40 for sctp */
+	spin_lock(&hw_mgt->reg_lock);
+	nbl_hw_rd_regs(hw_mgt, NBL_DPED_L4_CK_CMD_40_ADDR, &data, sizeof(data));
+	data |= FIELD_PREP(NBL_DPED_L4_CK_CMD_40_EN_MASK, 1);
+	nbl_hw_wr_regs(hw_mgt, NBL_DPED_L4_CK_CMD_40_ADDR, &data, sizeof(data));
+	spin_unlock(&hw_mgt->reg_lock);
+}
+
+static void nbl_dped_init(struct nbl_hw_mgt *hw_mgt)
+{
+	spin_lock(&hw_mgt->reg_lock);
+	nbl_hw_wr32(hw_mgt, NBL_DPED_VLAN_OFFSET, 0xC);
+	nbl_hw_wr32(hw_mgt, NBL_DPED_DSCP_OFFSET_0, 0x8);
+	nbl_hw_wr32(hw_mgt, NBL_DPED_DSCP_OFFSET_1, 0x4);
+	spin_unlock(&hw_mgt->reg_lock);
+	/* dped checksum offload */
+	nbl_configure_dped_checksum(hw_mgt);
+}
+
+static void nbl_uped_init(struct nbl_hw_mgt *hw_mgt)
+{
+	u32 hw_edit = 0;
+
+	/* V4 TCP: l3_len = 0 */
+	spin_lock(&hw_mgt->reg_lock);
+	nbl_hw_rd_regs(hw_mgt, NBL_UPED_HW_EDT_PROF_TABLE(NBL_UPED_V4_TCP_IDX),
+		       &hw_edit, sizeof(hw_edit));
+	hw_edit &= ~NBL_PED_HW_EDIT_PROFILE_L3_LEN_MASK;
+	nbl_hw_wr_regs(hw_mgt, NBL_UPED_HW_EDT_PROF_TABLE(NBL_UPED_V4_TCP_IDX),
+		       &hw_edit, sizeof(hw_edit));
+
+	/* V6 TCP: l3_len = 1 */
+	nbl_hw_rd_regs(hw_mgt, NBL_UPED_HW_EDT_PROF_TABLE(NBL_UPED_V6_TCP_IDX),
+		       &hw_edit, sizeof(hw_edit));
+	hw_edit = (hw_edit & ~NBL_PED_HW_EDIT_PROFILE_L3_LEN_MASK) |
+		  FIELD_PREP(NBL_PED_HW_EDIT_PROFILE_L3_LEN_MASK, 1);
+	nbl_hw_wr_regs(hw_mgt, NBL_UPED_HW_EDT_PROF_TABLE(NBL_UPED_V6_TCP_IDX),
+		       &hw_edit, sizeof(hw_edit));
+	spin_unlock(&hw_mgt->reg_lock);
+}
+
+static int nbl_shaping_eth_init(struct nbl_hw_mgt *hw_mgt, u8 eth_id, u8 speed)
+{
+	struct nbl_shaping_dvn_dport_u dvn_dport = { 0 };
+	struct nbl_shaping_dport_u dport = { 0 };
+	u32 rate, half_rate;
+	u32 depth;
+	u64 low_val, high_val;
+
+	switch (speed) {
+	case NBL_FW_PORT_SPEED_100G:
+		rate = 100000;
+		break;
+	case NBL_FW_PORT_SPEED_50G:
+		rate = 50000;
+		break;
+	case NBL_FW_PORT_SPEED_25G:
+		rate = 25000;
+		break;
+	case NBL_FW_PORT_SPEED_10G:
+		rate = 10000;
+		break;
+	default:
+		dev_err(hw_mgt->common->dev,
+			"Unsupported port speed %u for eth%u\n", speed, eth_id);
+		return -EINVAL;
+	}
+
+	half_rate = rate / 2;
+	depth = max_t(u32, rate * 2, NBL_LR_LEONIS_NET_BUCKET_DEPTH);
+
+	/* 1. clear valid first
+	 * dport and dvn_dport are zero-initialised above, so VALID=0 already
+	 */
+	spin_lock(&hw_mgt->reg_lock);
+	nbl_hw_wr_regs(hw_mgt, NBL_SHAPING_DPORT_REG(eth_id), dport.data,
+		       sizeof(dport));
+	nbl_hw_wr_regs(hw_mgt, NBL_SHAPING_DVN_DPORT_REG(eth_id),
+		       dvn_dport.data, sizeof(dvn_dport));
+
+	/* 2. write config words (valid=0, safe) */
+	low_val = FIELD_PREP(NBL_DPORT_CIR_MASK, rate) |
+		  FIELD_PREP(NBL_DPORT_PIR_MASK, rate) |
+		  FIELD_PREP(NBL_DPORT_DEPTH_MASK, depth) |
+		  FIELD_PREP(NBL_DPORT_CBS_MASK_LOW, depth & 0x3F);
+	high_val = FIELD_PREP(NBL_DPORT_CBS_MASK_HIGH, depth >> 6) |
+		   FIELD_PREP(NBL_DPORT_PBS_MASK, depth);
+	/* Fixed split, independent of host endian */
+	dport.data[0] = lower_32_bits(low_val);
+	dport.data[1] = upper_32_bits(low_val);
+	dport.data[2] = lower_32_bits(high_val);
+	dport.data[3] = upper_32_bits(high_val);
+
+	low_val = FIELD_PREP(NBL_DPORT_CIR_MASK, half_rate) |
+		  FIELD_PREP(NBL_DPORT_PIR_MASK, rate) |
+		  FIELD_PREP(NBL_DPORT_DEPTH_MASK, depth) |
+		  FIELD_PREP(NBL_DPORT_CBS_MASK_LOW, depth & 0x3F);
+	high_val = FIELD_PREP(NBL_DPORT_CBS_MASK_HIGH, depth >> 6) |
+		   FIELD_PREP(NBL_DPORT_PBS_MASK, depth);
+	dvn_dport.data[0] = lower_32_bits(low_val);
+	dvn_dport.data[1] = upper_32_bits(low_val);
+	dvn_dport.data[2] = lower_32_bits(high_val);
+	dvn_dport.data[3] = upper_32_bits(high_val);
+
+	nbl_hw_wr_regs(hw_mgt, NBL_SHAPING_DPORT_REG(eth_id), dport.data,
+		       sizeof(dport));
+	nbl_hw_wr_regs(hw_mgt, NBL_SHAPING_DVN_DPORT_REG(eth_id),
+		       dvn_dport.data, sizeof(dvn_dport));
+
+	/* 3. commit: set valid last */
+	low_val = FIELD_PREP(NBL_DPORT_VALID_MASK, 1);
+	dport.data[0] |= lower_32_bits(low_val);
+
+	low_val = FIELD_PREP(NBL_DPORT_VALID_MASK, 1);
+	dvn_dport.data[0] |= lower_32_bits(low_val);
+
+	nbl_hw_wr_regs(hw_mgt, NBL_SHAPING_DPORT_REG(eth_id), dport.data,
+		       sizeof(dport));
+	nbl_hw_wr_regs(hw_mgt, NBL_SHAPING_DVN_DPORT_REG(eth_id),
+		       dvn_dport.data, sizeof(dvn_dport));
+	spin_unlock(&hw_mgt->reg_lock);
+	return 0;
+}
+
+static int nbl_shaping_init(struct nbl_hw_mgt *hw_mgt, u8 speed)
+{
+	struct nbl_shaping_net_u net_shaping = { 0 };
+	u32 eth_bitmap = 0;
+	u32 reg_val;
+	int ret;
+	int i;
+
+	nbl_hw_get_fw_eth_map(hw_mgt, &eth_bitmap);
+	for (i = 0; i < NBL_MAX_ETHERNET; i++) {
+		if (!(eth_bitmap & BIT(i)))
+			continue;
+		ret = nbl_shaping_eth_init(hw_mgt, i, speed);
+		if (ret)
+			return ret;
+	}
+	/*
+	 * Read-modify-write as one critical section: reg_lock is dropped
+	 * between the two _lock() helpers, which would let another writer
+	 * of this register land in the middle of the update.  Nothing
+	 * else touches NBL_DSCH_PSHA_EN_ADDR today, but the RMW has to be
+	 * atomic for that to stay true.
+	 */
+	spin_lock(&hw_mgt->reg_lock);
+	nbl_hw_rd_regs(hw_mgt, NBL_DSCH_PSHA_EN_ADDR, &reg_val,
+		       sizeof(reg_val));
+	reg_val &= ~NBL_DSCH_PSHA_EN_MASK;
+	reg_val |= FIELD_PREP(NBL_DSCH_PSHA_EN_MASK,
+			      eth_bitmap & GENMASK(3, 0));
+	nbl_hw_wr_regs(hw_mgt, NBL_DSCH_PSHA_EN_ADDR, &reg_val,
+		       sizeof(reg_val));
+	spin_unlock(&hw_mgt->reg_lock);
+
+	for (i = 0; i < NBL_MAX_PF; i++) {
+		nbl_hw_wr_regs_lock(hw_mgt, NBL_SHAPING_NET_REG(i),
+				    net_shaping.data,
+				    sizeof(net_shaping));
+	}
+	nbl_flush_writes(hw_mgt);
+	return 0;
+}
+
+static void nbl_dsch_qid_max_init(struct nbl_hw_mgt *hw_mgt)
+{
+	u32 quanta = 0;
+	u32 qid_max = 0;
+
+	quanta = FIELD_PREP(NBL_DSCH_VN_QUANTA_H_QUA_MASK, NBL_HOST_QUANTA) |
+		 FIELD_PREP(NBL_DSCH_VN_QUANTA_E_QUA_MASK, NBL_ECPU_QUANTA);
+	spin_lock(&hw_mgt->reg_lock);
+	nbl_hw_wr_regs(hw_mgt, NBL_DSCH_VN_QUANTA_ADDR, &quanta,
+		       sizeof(quanta));
+	nbl_hw_rd_regs(hw_mgt, NBL_DSCH_HOST_QID_MAX, &qid_max,
+		       sizeof(qid_max));
+	qid_max &= ~NBL_DSCH_HOST_QID_MAX_MASK;
+	qid_max |= FIELD_PREP(NBL_DSCH_HOST_QID_MAX_MASK, NBL_MAX_QUEUE_ID);
+	nbl_hw_wr_regs(hw_mgt, NBL_DSCH_HOST_QID_MAX, &qid_max,
+		       sizeof(qid_max));
+	spin_unlock(&hw_mgt->reg_lock);
+}
+
+static int nbl_ustore_init(struct nbl_hw_mgt *hw_mgt, u8 eth_num)
+{
+	u32 eth_bitmap = 0;
+	u32 drop_th = 0;
+	u32 pkt_len = 0;
+	u32 reg_val = 0;
+	int i;
+
+	/*
+	 * eth_num is validated in the resource layer:
+	 * nbl_res_init_pf_num() requires 1/2/4 PFs, and
+	 * nbl_res_ctrl_dev_setup_eth_info() requires max_pf == eth_num.
+	 * This is a defensive check only; if it triggers, the resource
+	 * layer validation was bypassed, which is a bug.
+	 */
+	if (WARN_ON(eth_num != 1 && eth_num != 2 && eth_num != 4))
+		return -EINVAL;
+	/* Read current packet length config
+	 *(to preserve other fields while updating 'min')
+	 */
+	spin_lock(&hw_mgt->reg_lock);
+	nbl_hw_rd_regs(hw_mgt, NBL_USTORE_PKT_LEN_ADDR, &pkt_len,
+		       sizeof(pkt_len));
+	/* min arp packet length 42 (14 + 28) */
+	pkt_len &= ~NBL_USTORE_PKT_LEN_MIN_MASK;
+	pkt_len |= FIELD_PREP(NBL_USTORE_PKT_LEN_MIN_MASK, 42);
+	nbl_hw_wr_regs(hw_mgt, NBL_USTORE_PKT_LEN_ADDR, &pkt_len,
+		       sizeof(pkt_len));
+
+	drop_th |= FIELD_PREP(NBL_USTORE_PORT_DROP_TH_EN_MASK, 1);
+	if (eth_num == 1)
+		drop_th |= FIELD_PREP(NBL_USTORE_PORT_DROP_TH_DISC_TH_MASK,
+				      NBL_USTORE_SINGLE_ETH_DROP_TH);
+	else if (eth_num == 2)
+		drop_th |= FIELD_PREP(NBL_USTORE_PORT_DROP_TH_DISC_TH_MASK,
+				      NBL_USTORE_DUAL_ETH_DROP_TH);
+	else
+		drop_th |= FIELD_PREP(NBL_USTORE_PORT_DROP_TH_DISC_TH_MASK,
+				      NBL_USTORE_QUAD_ETH_DROP_TH);
+	nbl_hw_get_fw_eth_map(hw_mgt, &eth_bitmap);
+	for (i = 0; i < NBL_MAX_ETHERNET; i++) {
+		if (!(eth_bitmap & BIT(i)))
+			continue;
+		nbl_hw_rd_regs(hw_mgt, NBL_USTORE_PORT_DROP_TH_REG_ARR(i),
+			       &reg_val, sizeof(reg_val));
+		reg_val &= ~(NBL_USTORE_PORT_DROP_TH_EN_MASK |
+			    NBL_USTORE_PORT_DROP_TH_DISC_TH_MASK);
+		reg_val |= drop_th;
+		nbl_hw_wr_regs(hw_mgt, NBL_USTORE_PORT_DROP_TH_REG_ARR(i),
+			       &reg_val, sizeof(reg_val));
+	}
+
+	/* Clear port drop/truncate counters by reading them
+	 * (hardware has read-to-clear behavior for these registers)
+	 */
+	for (i = 0; i < NBL_MAX_ETHERNET; i++) {
+		if (!(eth_bitmap & BIT(i)))
+			continue;
+		nbl_hw_rd32(hw_mgt, NBL_USTORE_BUF_PORT_DROP_PKT(i));
+		nbl_hw_rd32(hw_mgt, NBL_USTORE_BUF_PORT_TRUN_PKT(i));
+	}
+	spin_unlock(&hw_mgt->reg_lock);
+	return 0;
+}
+
+static void nbl_dstore_init(struct nbl_hw_mgt *hw_mgt, u8 speed)
+{
+	u32 eth_bitmap = 0;
+	u32 drop_th = 0;
+	u32 fc_th = 0;
+	u32 bp_th = 0;
+	int i;
+
+	for (i = 0; i < NBL_DSTORE_PORT_DROP_TH_DEPTH; i++) {
+		spin_lock(&hw_mgt->reg_lock);
+		nbl_hw_rd_regs(hw_mgt, NBL_DSTORE_PORT_DROP_TH_REG(i), &drop_th,
+			       sizeof(drop_th));
+		drop_th &= ~NBL_DSTORE_PORT_DROP_EN_MASK;
+		nbl_hw_wr_regs(hw_mgt, NBL_DSTORE_PORT_DROP_TH_REG(i), &drop_th,
+			       sizeof(drop_th));
+		spin_unlock(&hw_mgt->reg_lock);
+	}
+
+	spin_lock(&hw_mgt->reg_lock);
+	nbl_hw_rd_regs(hw_mgt, NBL_DSTORE_DISC_BP_TH, &bp_th, sizeof(bp_th));
+	bp_th |= FIELD_PREP(NBL_DSTORE_DISC_BP_TH_EN_MASK, 1);
+	nbl_hw_wr_regs(hw_mgt, NBL_DSTORE_DISC_BP_TH, &bp_th, sizeof(bp_th));
+	spin_unlock(&hw_mgt->reg_lock);
+
+	nbl_hw_get_fw_eth_map(hw_mgt, &eth_bitmap);
+	for (i = 0; i < NBL_MAX_ETHERNET; i++) {
+		if (!(eth_bitmap & BIT(i)))
+			continue;
+		spin_lock(&hw_mgt->reg_lock);
+		nbl_hw_rd_regs(hw_mgt, NBL_DSTORE_D_DPORT_FC_TH_REG(i), &fc_th,
+			       sizeof(fc_th));
+		fc_th &= ~(NBL_DSTORE_D_DPORT_FC_XOFF_TH_MASK |
+			   NBL_DSTORE_D_DPORT_FC_XON_TH_MASK);
+		if (speed == NBL_FW_PORT_SPEED_100G) {
+			fc_th |=
+				FIELD_PREP(NBL_DSTORE_D_DPORT_FC_XOFF_TH_MASK,
+					   NBL_DSTORE_DROP_XOFF_TH_100G) |
+				FIELD_PREP(NBL_DSTORE_D_DPORT_FC_XON_TH_MASK,
+					   NBL_DSTORE_DROP_XON_TH_100G);
+		} else {
+			fc_th |=
+				FIELD_PREP(NBL_DSTORE_D_DPORT_FC_XOFF_TH_MASK,
+					   NBL_DSTORE_DROP_XOFF_TH) |
+				FIELD_PREP(NBL_DSTORE_D_DPORT_FC_XON_TH_MASK,
+					   NBL_DSTORE_DROP_XON_TH);
+		}
+
+		fc_th |= FIELD_PREP(NBL_DSTORE_D_DPORT_FC_FC_EN_MASK, 1);
+		nbl_hw_wr_regs(hw_mgt, NBL_DSTORE_D_DPORT_FC_TH_REG(i), &fc_th,
+			       sizeof(fc_th));
+		spin_unlock(&hw_mgt->reg_lock);
+	}
+}
+
+static void nbl_dvn_descreq_num_cfg(struct nbl_hw_mgt *hw_mgt, u8 descreq_num)
+{
+	u8 split_ring_num = (descreq_num >> 3) & 0x1;
+	u8 ring_num = descreq_num & 0x7;
+	u32 num_cfg;
+	u32 reg_val;
+
+	spin_lock(&hw_mgt->reg_lock);
+	nbl_hw_rd_regs(hw_mgt, NBL_DVN_DESCREQ_NUM_CFG, &reg_val,
+		       sizeof(reg_val));
+
+	num_cfg = FIELD_PREP(NBL_DVN_DESCREQ_NUM_CFG_AVRING_DESREQ_NUM_CFG_MASK,
+			     split_ring_num) |
+		  FIELD_PREP(NBL_DVN_DESCREQ_NUM_CFG_PACKED_L1_NUM_MASK,
+			     ring_num);
+	reg_val &= ~(NBL_DVN_DESCREQ_NUM_CFG_AVRING_DESREQ_NUM_CFG_MASK |
+		NBL_DVN_DESCREQ_NUM_CFG_PACKED_L1_NUM_MASK);
+	reg_val |= num_cfg;
+	nbl_hw_wr_regs(hw_mgt, NBL_DVN_DESCREQ_NUM_CFG, &reg_val,
+		       sizeof(reg_val));
+	spin_unlock(&hw_mgt->reg_lock);
+}
+
+static void nbl_dvn_init(struct nbl_hw_mgt *hw_mgt, u8 speed)
+{
+	u32 timeout = 0;
+	u32 ro_flag = 0;
+
+	spin_lock(&hw_mgt->reg_lock);
+	nbl_hw_wr32(hw_mgt, NBL_DVN_ECPU_QUEUE_NUM, 0);
+	spin_unlock(&hw_mgt->reg_lock);
+
+	timeout = FIELD_PREP(NBL_DVN_DESC_WR_MERGE_TIMEOUT_CFG_CYCLE_MASK,
+			     DEFAULT_DVN_DESC_WR_MERGE_TIMEOUT_MAX);
+	nbl_hw_wr_regs_lock(hw_mgt, NBL_DVN_DESC_WR_MERGE_TIMEOUT, &timeout,
+			    sizeof(timeout));
+	spin_lock(&hw_mgt->reg_lock);
+	nbl_hw_rd_regs(hw_mgt, NBL_DVN_DIF_REQ_RD_RO_FLAG, &ro_flag,
+		       sizeof(ro_flag));
+	if (pcie_relaxed_ordering_enabled(hw_mgt->common->pdev)) {
+		ro_flag |=
+			FIELD_PREP(NBL_DVN_DIF_REQ_RD_RO_FLAG_DESC_RO_EN_MASK,
+				   1) |
+			FIELD_PREP(NBL_DVN_DIF_REQ_RD_RO_FLAG_DATA_RO_EN_MASK,
+				   1) |
+			FIELD_PREP(NBL_DVN_DIF_REQ_RD_RO_FLAG_AVRING_RO_EN_MASK,
+				   1);
+	} else {
+		ro_flag &=
+			~(FIELD_PREP(NBL_DVN_DIF_REQ_RD_RO_FLAG_DESC_RO_EN_MASK,
+				     1) |
+			FIELD_PREP(NBL_DVN_DIF_REQ_RD_RO_FLAG_DATA_RO_EN_MASK,
+				   1) |
+			FIELD_PREP(NBL_DVN_DIF_REQ_RD_RO_FLAG_AVRING_RO_EN_MASK,
+				   1));
+	}
+	nbl_hw_wr_regs(hw_mgt, NBL_DVN_DIF_REQ_RD_RO_FLAG, &ro_flag,
+		       sizeof(ro_flag));
+	spin_unlock(&hw_mgt->reg_lock);
+	if (speed == NBL_FW_PORT_SPEED_100G)
+		nbl_dvn_descreq_num_cfg(hw_mgt,
+					DEFAULT_DVN_100G_DESCREQ_NUMCFG);
+	else
+		nbl_dvn_descreq_num_cfg(hw_mgt, DEFAULT_DVN_DESCREQ_NUMCFG);
+}
+
+static void nbl_uvn_init(struct nbl_hw_mgt *hw_mgt)
+{
+	u16 wr_timeout = NBL_UVN_DESC_WR_TIMEOUT_VAL;
+	u32 timeout = NBL_UVN_DESC_RD_WAIT_TICKS;
+	u32 prefetch_init = 0;
+	bool ro_enabled;
+	u32 flag = 0;
+	u32 mask = 0;
+	u32 quirks;
+	u32 reg_val;
+
+	spin_lock(&hw_mgt->reg_lock);
+	nbl_hw_wr32(hw_mgt, NBL_UVN_ECPU_QUEUE_NUM, 0);
+	nbl_hw_wr32(hw_mgt, NBL_UVN_DESC_RD_WAIT, timeout);
+	nbl_hw_rd_regs(hw_mgt, NBL_UVN_DESC_WR_TIMEOUT,
+		       &reg_val, sizeof(reg_val));
+	reg_val &= ~NBL_UVN_DESC_WR_TIMEOUT_NUM_MASK;
+	reg_val |= FIELD_PREP(NBL_UVN_DESC_WR_TIMEOUT_NUM_MASK, wr_timeout);
+	nbl_hw_wr_regs(hw_mgt, NBL_UVN_DESC_WR_TIMEOUT, &reg_val,
+		       sizeof(reg_val));
+	ro_enabled = pcie_relaxed_ordering_enabled(hw_mgt->common->pdev);
+
+	nbl_hw_rd_regs(hw_mgt, NBL_UVN_DIF_REQ_RO_FLAG, &flag, sizeof(flag));
+	if (ro_enabled) {
+		flag |= FIELD_PREP(NBL_UVN_DIF_REQ_RO_FLAG_AVAIL_RD_MASK, 1) |
+		FIELD_PREP(NBL_UVN_DIF_REQ_RO_FLAG_DESC_RD_MASK, 1) |
+		FIELD_PREP(NBL_UVN_DIF_REQ_RO_FLAG_PKT_WR_MASK, 1);
+		flag &= ~FIELD_PREP(NBL_UVN_DIF_REQ_RO_FLAG_DESC_WR_MASK, 1);
+	} else {
+		flag &= ~(FIELD_PREP(NBL_UVN_DIF_REQ_RO_FLAG_AVAIL_RD_MASK, 1) |
+			  FIELD_PREP(NBL_UVN_DIF_REQ_RO_FLAG_DESC_RD_MASK, 1) |
+			  FIELD_PREP(NBL_UVN_DIF_REQ_RO_FLAG_PKT_WR_MASK, 1) |
+			  FIELD_PREP(NBL_UVN_DIF_REQ_RO_FLAG_DESC_WR_MASK, 1));
+	}
+	nbl_hw_wr_regs(hw_mgt, NBL_UVN_DIF_REQ_RO_FLAG, &flag, sizeof(flag));
+
+	nbl_hw_rd_regs(hw_mgt, NBL_UVN_QUEUE_ERR_MASK, &mask, sizeof(mask));
+	mask |= FIELD_PREP(NBL_UVN_QUEUE_ERR_MASK_DIF_ERR_MASK, 1);
+
+	nbl_hw_wr_regs(hw_mgt, NBL_UVN_QUEUE_ERR_MASK, &mask, sizeof(mask));
+
+	spin_unlock(&hw_mgt->reg_lock);
+	quirks = nbl_hw_get_quirks(hw_mgt);
+	/*
+	 * sel=0: use configured num; sel=1: use internal calc (max 32)
+	 * Default is sel=1, unless NBL_QUIRK_UVN_PREFETCH_ALIGN is set,
+	 * in which case override to sel=0.
+	 */
+	spin_lock(&hw_mgt->reg_lock);
+	nbl_hw_rd_regs(hw_mgt, NBL_UVN_DESC_PREFETCH_INIT, &reg_val,
+		       sizeof(reg_val));
+	prefetch_init =
+		FIELD_PREP(NBL_UVN_DESC_PREFETCH_INIT_NUM_MASK,
+			   NBL_UVN_DESC_PREFETCH_NUM) |
+		FIELD_PREP(NBL_UVN_DESC_PREFETCH_INIT_SEL_MASK,
+			   (quirks & NBL_QUIRK_UVN_PREFETCH_ALIGN) ? 0 : 1);
+	reg_val &= ~(NBL_UVN_DESC_PREFETCH_INIT_NUM_MASK |
+		NBL_UVN_DESC_PREFETCH_INIT_SEL_MASK);
+	reg_val |= prefetch_init;
+	nbl_hw_wr_regs(hw_mgt, NBL_UVN_DESC_PREFETCH_INIT, &reg_val,
+		       sizeof(reg_val));
+	spin_unlock(&hw_mgt->reg_lock);
+}
+
+static void nbl_uqm_init(struct nbl_hw_mgt *hw_mgt)
+{
+	u32 que_type = 0;
+	u32 cnt = 0;
+	int i;
+
+	spin_lock(&hw_mgt->reg_lock);
+	nbl_hw_wr_regs(hw_mgt, NBL_UQM_FWD_DROP_CNT, &cnt, sizeof(cnt));
+
+	nbl_hw_wr_regs(hw_mgt, NBL_UQM_DROP_PKT_CNT, &cnt, sizeof(cnt));
+	nbl_hw_wr_regs(hw_mgt, NBL_UQM_DROP_PKT_SLICE_CNT, &cnt, sizeof(cnt));
+	nbl_hw_wr_regs(hw_mgt, NBL_UQM_DROP_PKT_LEN_ADD_CNT, &cnt, sizeof(cnt));
+	nbl_hw_wr_regs(hw_mgt, NBL_UQM_DROP_HEAD_PNTR_ADD_CNT, &cnt,
+		       sizeof(cnt));
+	nbl_hw_wr_regs(hw_mgt, NBL_UQM_DROP_WEIGHT_ADD_CNT, &cnt, sizeof(cnt));
+
+	for (i = 0; i < NBL_UQM_PORT_DROP_DEPTH; i++) {
+		nbl_hw_wr_regs(hw_mgt,
+			       NBL_UQM_PORT_DROP_PKT_CNT + (sizeof(cnt) * i),
+			       &cnt, sizeof(cnt));
+		nbl_hw_wr_regs(hw_mgt,
+			       NBL_UQM_PORT_DROP_PKT_SLICE_CNT +
+				       (sizeof(cnt) * i),
+			       &cnt, sizeof(cnt));
+		nbl_hw_wr_regs(hw_mgt,
+			       NBL_UQM_PORT_DROP_PKT_LEN_ADD_CNT +
+				       (sizeof(cnt) * i),
+			       &cnt, sizeof(cnt));
+		nbl_hw_wr_regs(hw_mgt,
+			       NBL_UQM_PORT_DROP_HEAD_PNTR_ADD_CNT +
+				       (sizeof(cnt) * i),
+			       &cnt, sizeof(cnt));
+		nbl_hw_wr_regs(hw_mgt,
+			       NBL_UQM_PORT_DROP_WEIGHT_ADD_CNT +
+				       (sizeof(cnt) * i),
+			       &cnt, sizeof(cnt));
+	}
+
+	for (i = 0; i < NBL_UQM_DPORT_DROP_DEPTH; i++)
+		nbl_hw_wr_regs(hw_mgt,
+			       NBL_UQM_DPORT_DROP_CNT + (sizeof(cnt) * i), &cnt,
+			       sizeof(cnt));
+	/* bit0: 0=bp mode, 1=drop mode, resv bit1-31 */
+	nbl_hw_wr_regs(hw_mgt, NBL_UQM_QUE_TYPE, &que_type, sizeof(que_type));
+	spin_unlock(&hw_mgt->reg_lock);
+}
+
+static int nbl_dp_init(struct nbl_hw_mgt *hw_mgt, u8 speed, u8 eth_num)
+{
+	int ret;
+
+	nbl_dped_init(hw_mgt);
+	nbl_uped_init(hw_mgt);
+	ret = nbl_shaping_init(hw_mgt, speed);
+	if (ret)
+		return ret;
+	nbl_dsch_qid_max_init(hw_mgt);
+	ret = nbl_ustore_init(hw_mgt, eth_num);
+	if (ret)
+		return ret;
+	nbl_dstore_init(hw_mgt, speed);
+	nbl_dvn_init(hw_mgt, speed);
+	nbl_uvn_init(hw_mgt);
+	nbl_uqm_init(hw_mgt);
+	return 0;
+}
+
+static void nbl_host_padpt_init(struct nbl_hw_mgt *hw_mgt)
+{
+	/* padpt flow  control register */
+	spin_lock(&hw_mgt->reg_lock);
+	nbl_hw_wr32(hw_mgt, NBL_HOST_PADPT_HOST_CFG_FC_CPLH_UP,
+		    NBL_HOST_PADPT_CFG_FC_CPLH_UP_VAL);
+	nbl_hw_wr32(hw_mgt, NBL_HOST_PADPT_HOST_CFG_FC_PD_DN,
+		    NBL_HOST_PADPT_CFG_FC_PD_DN_VAL);
+	nbl_hw_wr32(hw_mgt, NBL_HOST_PADPT_HOST_CFG_FC_PH_DN,
+		    NBL_HOST_PADPT_CFG_FC_PH_DN_VAL);
+	nbl_hw_wr32(hw_mgt, NBL_HOST_PADPT_HOST_CFG_FC_NPH_DN,
+		    NBL_HOST_PADPT_CFG_FC_NPH_DN_VAL);
+	spin_unlock(&hw_mgt->reg_lock);
+}
+
+static void nbl_intf_init(struct nbl_hw_mgt *hw_mgt)
+{
+	nbl_host_padpt_init(hw_mgt);
+}
+
+/*
+ * Read-modify-write under reg_lock: NBL_DRIVER_STATUS_BIT is the only
+ * bit the driver owns, and the RMW writes back every other bit exactly
+ * as it read it, so firmware state in those bits survives.
+ *
+ * That only holds against concurrent CPU writers.  reg_lock cannot
+ * serialise against firmware, and the register has no documented
+ * W1S/W1C form that would let the driver flip bit 16 without the
+ * read-modify-write; see NBL_DRIVER_STATUS_REG.
+ */
+static void nbl_hw_set_driver_status(struct nbl_hw_mgt *hw_mgt, bool active)
+{
+	u32 status;
+
+	spin_lock(&hw_mgt->reg_lock);
+	status = nbl_hw_rd32(hw_mgt, NBL_DRIVER_STATUS_REG);
+
+	status &= ~BIT(NBL_DRIVER_STATUS_BIT);
+	status |= FIELD_PREP(BIT(NBL_DRIVER_STATUS_BIT), active);
+
+	nbl_hw_wr32(hw_mgt, NBL_DRIVER_STATUS_REG, status);
+	spin_unlock(&hw_mgt->reg_lock);
+}
+
+/*
+ * Setting driver status to false notifies firmware to clean up per-PF
+ * hardware state such as qinfo registers.
+ *
+ * Note: firmware does NOT automatically revert chip-wide registers
+ * configured in this init flow. Those chip-wide settings remain valid
+ * until chip reset or explicitly overwritten by driver.
+ *
+ * This deinit_module only clears driver active status and flush writes.
+ * It does NOT reset or restore chip-wide datapath registers.
+ *
+ * Firmware cleanup is asynchronous with no completion status register.
+ * The flush only guarantees the status write reached the device; the
+ * bounded delay after it (NBL_FW_CLEANUP_SYNC_MIN_US) is best-effort and
+ * does not guarantee that the firmware pass has finished.  The ordering
+ * this relies on is the device-link one: the siblings are consumers of
+ * the control PF, so they are unbound first, and that link is created
+ * with DL_FLAG_AUTOPROBE_CONSUMER.  AUTOPROBE only re-probes consumers,
+ * never the control PF itself, so a later init_module() needs an admin
+ * (or a driver rebind) to bind this PF again, and only then does the
+ * core re-probe the siblings.  Both steps are far slower than the
+ * firmware pass, but there is no hard firmware handshake.
+ *
+ * Caller must ensure no new DMA is initiated after this point.
+ * The mailbox channel is stopped by nbl_chan_teardown_queue()
+ * before this function is called, so no in-flight mailbox DMA
+ * remains.
+ */
+static void nbl_hw_deinit_module(struct nbl_hw_mgt *hw_mgt)
+{
+	nbl_hw_set_driver_status(hw_mgt, false);
+	/* ensure driver_status reaches the chip */
+	nbl_flush_writes(hw_mgt);
+	/*
+	 * Give the asynchronous firmware cleanup pass time to settle
+	 * before returning; best-effort, see NBL_FW_CLEANUP_SYNC_MIN_US.
+	 */
+	usleep_range(NBL_FW_CLEANUP_SYNC_MIN_US, NBL_FW_CLEANUP_SYNC_MAX_US);
+}
+
+static bool nbl_hw_eth_speed_valid(u8 speed)
+{
+	switch (speed) {
+	case NBL_FW_PORT_SPEED_10G:
+	case NBL_FW_PORT_SPEED_25G:
+	case NBL_FW_PORT_SPEED_50G:
+	case NBL_FW_PORT_SPEED_100G:
+		return true;
+	default:
+		return false;
+	}
+}
+
+static bool nbl_hw_eth_num_valid(u8 eth_num)
+{
+	return eth_num == 1 || eth_num == 2 || eth_num == 4;
+}
+
+/*
+ * Clear a stale driver_status active bit before this probe programs any
+ * per-PF state.
+ *
+ * A previous kernel instance can leave the bit set (kexec, or an unload
+ * that never reached deinit_module()) and firmware then never ran the
+ * per-PF cleanup it drives off the 1 -> 0 transition.  That cleanup
+ * reclaims the mailbox QINFO routing and the qinfo registers, which is
+ * exactly what nbl_dev_setup_chan_qinfo() and nbl_chan_setup_queue()
+ * write later in this probe.
+ *
+ * So the ordering this has to guarantee is: clear the bit before any of
+ * that per-PF state is programmed, and before the mailbox queue is live
+ * so the cleanup cannot reclaim routing under an in-flight transfer.
+ * Clearing it later - once nbl_dev_init() has programmed the routing and
+ * enabled the queue - wipes those registers and leaves the inter-PF
+ * mailbox dead, so sibling PFs' RPCs time out in nbl_dev_start() and
+ * their probes fail.  That is why init_module() must not do it.
+ *
+ * Only the control PF owns this chip-wide handshake.  The bounded wait
+ * after the clear is best-effort, see NBL_FW_CLEANUP_SYNC_MIN_US.
+ */
+static void nbl_hw_clear_stale_status(struct nbl_hw_mgt *hw_mgt)
+{
+	u32 status;
+
+	if (!hw_mgt->common->has_ctrl)
+		return;
+
+	/*
+	 * Unlocked read: this runs only from the serialized control-PF probe,
+	 * and the driver is the only writer of NBL_DRIVER_STATUS_BIT.  A stale
+	 * sample at worst causes one redundant clear plus sleep; the
+	 * authoritative update happens under reg_lock in
+	 * nbl_hw_set_driver_status().
+	 */
+	status = nbl_hw_rd32(hw_mgt, NBL_DRIVER_STATUS_REG);
+	if (!(status & BIT(NBL_DRIVER_STATUS_BIT)))
+		return;
+
+	dev_warn(hw_mgt->common->dev,
+		 "driver_status already set, stale cleanup forced\n");
+	nbl_hw_set_driver_status(hw_mgt, false);
+	nbl_flush_writes(hw_mgt);
+	usleep_range(NBL_FW_CLEANUP_SYNC_MIN_US, NBL_FW_CLEANUP_SYNC_MAX_US);
+}
+
+/*
+ * Firmware brings the chip up at power-up, but the chip-wide datapath
+ * registers this programs (SHAPING, DSCH, USTORE/DSTORE, DVN/UVN, UQM,
+ * PADPT) are not part of that: the driver owns them from here until chip
+ * reset, and firmware does not revert them when driver_status goes low -
+ * see nbl_hw_deinit_module().
+ *
+ * The driver_status handshake is entered from a known state: a stale
+ * active bit is cleared earlier, in nbl_hw_init_leonis(), before this
+ * probe programs any per-PF state.  It must not be cleared here, because
+ * the mailbox QINFO routing and the mailbox queue registers are already
+ * programmed by then and the cleanup the 1 -> 0 transition starts would
+ * reclaim them; see nbl_hw_clear_stale_status().
+ */
+static int nbl_hw_init_module(struct nbl_hw_mgt *hw_mgt, u8 eth_speed,
+			      u8 eth_num)
+{
+	int ret;
+
+	if (!nbl_hw_eth_speed_valid(eth_speed)) {
+		dev_err(hw_mgt->common->dev, "Invalid eth_speed %u\n",
+			eth_speed);
+		return -EINVAL;
+	}
+	if (!nbl_hw_eth_num_valid(eth_num)) {
+		dev_err(hw_mgt->common->dev, "Invalid eth_num %u\n", eth_num);
+		return -EINVAL;
+	}
+
+	ret = nbl_dp_init(hw_mgt, eth_speed, eth_num);
+	if (ret)
+		return ret;
+	nbl_intf_init(hw_mgt);
+	nbl_hw_set_driver_status(hw_mgt, true);
+	/* ensure registers written */
+	nbl_flush_writes(hw_mgt);
+
+	return 0;
+}
+
+/*
+ * nbl_hw_set_mailbox_irq - read-modify-write NBL_MAILBOX_QINFO_MAP_REG_ARR
+ *
+ * The full RMW sequence is wrapped by reg_lock, so concurrent register
+ * access from different CPUs is already serialized safely.
+ * nbl_hw_cfg_mailbox_qinfo() programs the BDF fields during control-PF
+ * init and clears MSIX_IDX/MSIX_IDX_VALID at the same time (they survive
+ * kexec/forced unload without FLR), so mailbox MSIX routing for a PF
+ * starts disarmed at init and is armed only by an explicit en_msix=true
+ * call here.
+ */
+static void nbl_hw_set_mailbox_irq(struct nbl_hw_mgt *hw_mgt, u16 func_id,
+				   bool en_msix, u16 gvec)
+{
+	u32 data = 0;
+
+	spin_lock(&hw_mgt->reg_lock);
+	nbl_hw_rd_regs(hw_mgt, NBL_MAILBOX_QINFO_MAP_REG_ARR(func_id), &data,
+		       sizeof(data));
+	data &= ~(NBL_MAILBOX_QINFO_MAP_MSIX_IDX_MASK |
+		  NBL_MAILBOX_QINFO_MAP_MSIX_IDX_VALID_MASK);
+	if (en_msix)
+		data |= FIELD_PREP(NBL_MAILBOX_QINFO_MAP_MSIX_IDX_MASK,
+				   gvec) |
+			FIELD_PREP(NBL_MAILBOX_QINFO_MAP_MSIX_IDX_VALID_MASK,
+				   1);
+
+	nbl_hw_wr_regs(hw_mgt, NBL_MAILBOX_QINFO_MAP_REG_ARR(func_id), &data,
+		       sizeof(data));
+	spin_unlock(&hw_mgt->reg_lock);
+	nbl_flush_writes(hw_mgt);
+}
+
+static void nbl_hw_cfg_msix_map(struct nbl_hw_mgt *hw_mgt, u16 func_id,
+				bool valid, dma_addr_t dma_addr, u8 bus,
+				u8 devid, u8 function)
+{
+	struct nbl_function_msix_map function_msix_map;
+
+	memset(&function_msix_map, 0, sizeof(function_msix_map));
+
+	/*
+	 * Clear VALID on its own first.  nbl_hw_wr_regs_lock() writes an
+	 * entry in ascending dword order, so a single whole-entry write
+	 * would publish the new (or zeroed) address in data[0]/data[1]
+	 * while the old VALID bit in data[2] was still set, exposing a
+	 * valid entry with a half-written address to the pcompleter's
+	 * table fetch.  With VALID already zero only the address words
+	 * change, and no fetch can be started from a torn entry.
+	 */
+	nbl_hw_wr_regs_lock(hw_mgt,
+			    NBL_PCOMPLETER_FUNCTION_MSIX_MAP(func_id) +
+			    NBL_FUNC_MSIX_MAP_VALID_DW * sizeof(u32),
+			    &function_msix_map.data[NBL_FUNC_MSIX_MAP_VALID_DW],
+			    sizeof(u32));
+
+	if (valid) {
+		/* program full entry and set VALID */
+		function_msix_map.data[0] = lower_32_bits(dma_addr);
+		function_msix_map.data[1] = upper_32_bits(dma_addr);
+		function_msix_map.data[2] =
+			FIELD_PREP(NBL_FUNCTION_MSIX_MAP_FUNCTION_MASK,
+				   function) |
+			FIELD_PREP(NBL_FUNCTION_MSIX_MAP_DEVID_MASK, devid) |
+			FIELD_PREP(NBL_FUNCTION_MSIX_MAP_BUS_MASK, bus) |
+			FIELD_PREP(NBL_FUNCTION_MSIX_MAP_VALID_MASK, 1);
+
+		nbl_hw_wr_regs_lock(hw_mgt,
+				    NBL_PCOMPLETER_FUNCTION_MSIX_MAP(func_id),
+				    function_msix_map.data,
+				    sizeof(function_msix_map));
+	} else {
+		/*
+		 * reg_lock prevents concurrent CPU writes to the same
+		 * function's MSIX entry, but cannot synchronize hardware DMA
+		 * reads. Upper layer uses two-stage destruction + sync sleep
+		 * to avoid torn hardware read of partial MSIX entry.
+		 * VALID is already cleared above, so the address written
+		 * here is either the still-live table (prepare) or zero
+		 * (complete); data[2] stays zero, leaving the BDF fields
+		 * unused because the entry cannot be fetched.
+		 */
+		function_msix_map.data[0] = lower_32_bits(dma_addr);
+		function_msix_map.data[1] = upper_32_bits(dma_addr);
+
+		nbl_hw_wr_regs_lock(hw_mgt,
+				    NBL_PCOMPLETER_FUNCTION_MSIX_MAP(func_id),
+				    function_msix_map.data,
+				    sizeof(function_msix_map));
+	}
+}
+
+static void nbl_hw_cfg_msix_info(struct nbl_hw_mgt *hw_mgt, u16 func_id,
+				 bool valid, u16 interrupt_id, u8 bus,
+				 u8 devid, u8 function, bool msix_mask_en)
+{
+	u32 host_msix_fid = 0;
+	struct nbl_host_msix_info msix_info;
+
+	memset(&msix_info, 0, sizeof(msix_info));
+	if (valid) {
+		host_msix_fid =
+			FIELD_PREP(NBL_PCOMPLETER_HOST_MSIX_FID_TABLE_FID_MASK,
+				   func_id) |
+			FIELD_PREP(NBL_PCOMPLETER_HOST_MSIX_FID_TABLE_VLD_MASK,
+				   1);
+
+		msix_info.data[1] =
+			FIELD_PREP(NBL_HOST_MSIX_INFO_FUNCTION_MASK, function) |
+			FIELD_PREP(NBL_HOST_MSIX_INFO_DEVID_MASK, devid) |
+			FIELD_PREP(NBL_HOST_MSIX_INFO_BUS_MASK, bus) |
+			FIELD_PREP(NBL_HOST_MSIX_INFO_VALID_MASK, 1);
+
+		if (msix_mask_en)
+			msix_info.data[1] |=
+			FIELD_PREP(NBL_HOST_MSIX_INFO_MSIX_MASK_EN_MASK, 1);
+	}
+	spin_lock(&hw_mgt->reg_lock);
+	/*
+	 * Programming order rule:
+	 * Enable: PADPT_HOST_MSIX_INFO -> PCOMPLETER_HOST_MSIX_FID_TABLE
+	 * Teardown: reverse order, clear FID VLD first to avoid inconsistent
+	 * state
+	 */
+	if (valid) {
+		nbl_hw_wr_regs(hw_mgt,
+			       NBL_PADPT_HOST_MSIX_INFO_REG_ARR(interrupt_id),
+			       msix_info.data, sizeof(msix_info));
+		nbl_hw_wr_regs(hw_mgt,
+			       NBL_PCOMPLETER_HOST_MSIX_FID_TABLE(interrupt_id),
+			       &host_msix_fid, sizeof(host_msix_fid));
+	} else {
+		nbl_hw_wr_regs(hw_mgt,
+			       NBL_PCOMPLETER_HOST_MSIX_FID_TABLE(interrupt_id),
+			       &host_msix_fid, sizeof(host_msix_fid));
+		nbl_hw_wr_regs(hw_mgt,
+			       NBL_PADPT_HOST_MSIX_INFO_REG_ARR(interrupt_id),
+			       msix_info.data, sizeof(msix_info));
+	}
+	spin_unlock(&hw_mgt->reg_lock);
+}
+
+static void nbl_hw_update_mailbox_queue_tail_ptr(struct nbl_hw_mgt *hw_mgt,
+						 u16 tail_ptr, u8 txrx)
+{
+	/* local_qid 0 and 1 denote rx and tx queue respectively */
+	u32 local_qid = txrx;
+	u32 value = ((u32)tail_ptr << 16) | local_qid;
+
+	/* wmb for doorbell */
+	wmb();
+	nbl_mbx_wr32(hw_mgt, NBL_MAILBOX_NOTIFY_ADDR, value);
+}
+
+/*
+ * Assert QUEUE_RST on a QINFO cfg entry, in a write of its own.
+ *
+ * The entry is four dwords and the device may act on each one as it
+ * lands, while data[3] carries QUEUE_RST/QUEUE_EN and data[0..2] carry
+ * the ring base and size.  Writing a new base while QUEUE_EN is still
+ * set from the previous configuration would have the queue fetch at a
+ * half-updated address - IOVA 0, once data[0..2] have been cleared -
+ * so the reset always goes in first, on its own, and the remaining
+ * dwords are written afterwards.  This also covers a device that only
+ * commits the entry when data[3] lands.
+ */
+static void nbl_hw_reset_mailbox_qinfo(struct nbl_hw_mgt *hw_mgt, u64 reg)
+{
+	u32 rst = FIELD_PREP(NBL_MAILBOX_QINFO_CFG_QUEUE_RST_MASK, 1);
+
+	nbl_hw_write_mbx_regs(hw_mgt,
+			      reg + (MAILBOX_QINFO_CFG_TABLE_DWLEN - 1) *
+				    sizeof(u32),
+			      &rst, sizeof(rst));
+}
+
+static void nbl_hw_config_mailbox_rxq(struct nbl_hw_mgt *hw_mgt,
+				      dma_addr_t dma_addr, int size_bwid)
+{
+	struct nbl_mailbox_qinfo_cfg_table cfg_tbl;
+
+	/* Hold the queue in reset before its base/size are reprogrammed */
+	nbl_hw_reset_mailbox_qinfo(hw_mgt, NBL_MAILBOX_QINFO_CFG_RX_TABLE_ADDR);
+
+	cfg_tbl.data[0] = lower_32_bits(dma_addr);
+	cfg_tbl.data[1] = upper_32_bits(dma_addr);
+	cfg_tbl.data[2] = FIELD_PREP(NBL_MAILBOX_QINFO_CFG_QUEUE_SIZE_BWID_MASK,
+				     size_bwid);
+	cfg_tbl.data[3] = FIELD_PREP(NBL_MAILBOX_QINFO_CFG_QUEUE_RST_MASK, 0) |
+			  FIELD_PREP(NBL_MAILBOX_QINFO_CFG_QUEUE_EN_MASK, 1);
+	/*
+	 * data[0..2] land while QUEUE_RST is still set from the write above;
+	 * data[3] is written last and starts the queue on the new ring.
+	 */
+	nbl_hw_write_mbx_regs(hw_mgt, NBL_MAILBOX_QINFO_CFG_RX_TABLE_ADDR,
+			      cfg_tbl.data, sizeof(cfg_tbl));
+}
+
+static void nbl_hw_config_mailbox_txq(struct nbl_hw_mgt *hw_mgt,
+				      dma_addr_t dma_addr, int size_bwid)
+{
+	struct nbl_mailbox_qinfo_cfg_table cfg_tbl;
+
+	/* Hold the queue in reset before its base/size are reprogrammed */
+	nbl_hw_reset_mailbox_qinfo(hw_mgt, NBL_MAILBOX_QINFO_CFG_TX_TABLE_ADDR);
+
+	cfg_tbl.data[0] = lower_32_bits(dma_addr);
+	cfg_tbl.data[1] = upper_32_bits(dma_addr);
+	cfg_tbl.data[2] = FIELD_PREP(NBL_MAILBOX_QINFO_CFG_QUEUE_SIZE_BWID_MASK,
+				     size_bwid);
+	cfg_tbl.data[3] = FIELD_PREP(NBL_MAILBOX_QINFO_CFG_QUEUE_RST_MASK, 0) |
+			  FIELD_PREP(NBL_MAILBOX_QINFO_CFG_QUEUE_EN_MASK, 1);
+	/*
+	 * data[0..2] land while QUEUE_RST is still set from the write above;
+	 * data[3] is written last and starts the queue on the new ring.
+	 */
+	nbl_hw_write_mbx_regs(hw_mgt, NBL_MAILBOX_QINFO_CFG_TX_TABLE_ADDR,
+			      cfg_tbl.data, sizeof(cfg_tbl));
+}
+
+static void nbl_hw_stop_mailbox_rxq(struct nbl_hw_mgt *hw_mgt)
+{
+	struct nbl_mailbox_qinfo_cfg_table cfg_tbl;
+
+	/*
+	 * Assert QUEUE_RST before data[0..2] are cleared: writing the
+	 * zeroed entry in one go would clear the ring base while QUEUE_EN
+	 * is still set.  The reset also clears QUEUE_EN, so what follows
+	 * lands on an already stopped queue.
+	 */
+	nbl_hw_reset_mailbox_qinfo(hw_mgt, NBL_MAILBOX_QINFO_CFG_RX_TABLE_ADDR);
+
+	/* Now drop the stale base and size, keeping QUEUE_RST asserted */
+	memset(&cfg_tbl, 0, sizeof(cfg_tbl));
+	cfg_tbl.data[3] = FIELD_PREP(NBL_MAILBOX_QINFO_CFG_QUEUE_RST_MASK, 1);
+	nbl_hw_write_mbx_regs(hw_mgt, NBL_MAILBOX_QINFO_CFG_RX_TABLE_ADDR,
+			      cfg_tbl.data, sizeof(cfg_tbl));
+	/* Ensure QUEUE_RST has reached the device before caller proceeds */
+	nbl_hw_flush_mbx_write(hw_mgt,
+			       NBL_MAILBOX_QINFO_CFG_RX_TABLE_ADDR);
+}
+
+static void nbl_hw_stop_mailbox_txq(struct nbl_hw_mgt *hw_mgt)
+{
+	struct nbl_mailbox_qinfo_cfg_table cfg_tbl;
+
+	/*
+	 * Assert QUEUE_RST before data[0..2] are cleared: writing the
+	 * zeroed entry in one go would clear the ring base while QUEUE_EN
+	 * is still set.  The reset also clears QUEUE_EN, so what follows
+	 * lands on an already stopped queue.
+	 */
+	nbl_hw_reset_mailbox_qinfo(hw_mgt, NBL_MAILBOX_QINFO_CFG_TX_TABLE_ADDR);
+
+	/* Now drop the stale base and size, keeping QUEUE_RST asserted */
+	memset(&cfg_tbl, 0, sizeof(cfg_tbl));
+	cfg_tbl.data[3] = FIELD_PREP(NBL_MAILBOX_QINFO_CFG_QUEUE_RST_MASK, 1);
+	nbl_hw_write_mbx_regs(hw_mgt, NBL_MAILBOX_QINFO_CFG_TX_TABLE_ADDR,
+			      cfg_tbl.data, sizeof(cfg_tbl));
+	/* Ensure QUEUE_RST has reached the device before caller proceeds */
+	nbl_hw_flush_mbx_write(hw_mgt,
+			       NBL_MAILBOX_QINFO_CFG_TX_TABLE_ADDR);
+}
+
+static void nbl_hw_get_host_pf_mask(struct nbl_hw_mgt *hw_mgt, u32 *pf_mask)
+{
+	nbl_hw_rd_regs_lock(hw_mgt, NBL_PCIE_HOST_K_PF_MASK_REG, pf_mask,
+			    sizeof(*pf_mask));
+}
+
+static void nbl_hw_get_real_bus(struct nbl_hw_mgt *hw_mgt, u8 *bus)
+{
+	u32 data = 0;
+
+	nbl_hw_rd_regs_lock(hw_mgt, NBL_PCIE_HOST_TL_CFG_BUSDEV, &data,
+			    sizeof(data));
+	*bus = FIELD_GET(NBL_PCIE_BUS_MASK, data);
+}
+
+static void nbl_hw_cfg_mailbox_qinfo(struct nbl_hw_mgt *hw_mgt, u16 func_id,
+				     u8 bus, u8 devid, u8 function)
+{
+	u32 data = 0;
+
+	spin_lock(&hw_mgt->reg_lock);
+	nbl_hw_rd_regs(hw_mgt, NBL_MAILBOX_QINFO_MAP_REG_ARR(func_id),
+		       &data, sizeof(data));
+	data &= ~(NBL_MAILBOX_QINFO_MAP_FUNCTION_MASK |
+		  NBL_MAILBOX_QINFO_MAP_DEVID_MASK |
+		  NBL_MAILBOX_QINFO_MAP_BUS_MASK |
+		  NBL_MAILBOX_QINFO_MAP_MSIX_IDX_MASK |
+		  NBL_MAILBOX_QINFO_MAP_MSIX_IDX_VALID_MASK);
+	data |= FIELD_PREP(NBL_MAILBOX_QINFO_MAP_FUNCTION_MASK, function) |
+	       FIELD_PREP(NBL_MAILBOX_QINFO_MAP_DEVID_MASK, devid) |
+	       FIELD_PREP(NBL_MAILBOX_QINFO_MAP_BUS_MASK, bus);
+	nbl_hw_wr_regs(hw_mgt, NBL_MAILBOX_QINFO_MAP_REG_ARR(func_id),
+		       &data, sizeof(data));
+	spin_unlock(&hw_mgt->reg_lock);
+}
+
+/*
+ * Registers reset to zero after cold boot / FLR / bus reset. Firmware
+ * programs valid values before driver probe, so zero is only seen on
+ * hardware fault or register read failure. Initialize data=0 to guard
+ * against nbl_hw_read_mbx_regs() early-return on bounds-check failure.
+ */
+static void nbl_hw_get_board_info(struct nbl_hw_mgt *hw_mgt,
+				  struct nbl_board_port_info *board_info)
+{
+	u32 data = 0;
+
+	nbl_hw_read_mbx_regs(hw_mgt, NBL_FW_BOARD_DW3_OFFSET, &data,
+			     sizeof(data));
+	board_info->eth_num = FIELD_GET(NBL_FW_BOARD_DW3_PORT_NUM_MASK, data);
+	board_info->eth_speed =
+		FIELD_GET(NBL_FW_BOARD_DW3_PORT_SPEED_MASK, data);
+	board_info->p4_version =
+		FIELD_GET(NBL_FW_BOARD_DW3_P4_VERSION_MASK, data);
+}
+
+static struct nbl_hw_ops hw_ops = {
+	.init_module = nbl_hw_init_module,
+	.deinit_module = nbl_hw_deinit_module,
+
+	.cfg_msix_map = nbl_hw_cfg_msix_map,
+	.cfg_msix_info = nbl_hw_cfg_msix_info,
+	.flush_write = nbl_flush_writes,
+
+	.update_mailbox_queue_tail_ptr = nbl_hw_update_mailbox_queue_tail_ptr,
+	.config_mailbox_rxq = nbl_hw_config_mailbox_rxq,
+	.config_mailbox_txq = nbl_hw_config_mailbox_txq,
+	.stop_mailbox_rxq = nbl_hw_stop_mailbox_rxq,
+	.stop_mailbox_txq = nbl_hw_stop_mailbox_txq,
+	.get_host_pf_mask = nbl_hw_get_host_pf_mask,
+	.get_real_bus = nbl_hw_get_real_bus,
+
+	.cfg_mailbox_qinfo = nbl_hw_cfg_mailbox_qinfo,
+	.set_mailbox_irq = nbl_hw_set_mailbox_irq,
+
+	.get_fw_eth_map = nbl_hw_get_fw_eth_map,
+	.get_board_info = nbl_hw_get_board_info,
+};
+
 /* Structure starts here, adding an op should not modify anything below */
 static struct nbl_hw_mgt *nbl_hw_setup_hw_mgt(struct nbl_common_info *common)
 {
@@ -23,6 +1246,31 @@ static struct nbl_hw_mgt *nbl_hw_setup_hw_mgt(struct nbl_common_info *common)
 	return hw_mgt;
 }
 
+static struct nbl_hw_ops_tbl *nbl_hw_setup_ops(struct nbl_common_info *common,
+					       struct nbl_hw_mgt *hw_mgt)
+{
+	struct nbl_hw_ops_tbl *hw_ops_tbl;
+	struct device *dev;
+
+	dev = common->dev;
+	hw_ops_tbl = devm_kzalloc(dev, sizeof(*hw_ops_tbl), GFP_KERNEL);
+	if (!hw_ops_tbl)
+		return ERR_PTR(-ENOMEM);
+	if (!hw_ops.cfg_msix_map || !hw_ops.cfg_msix_info ||
+	    !hw_ops.flush_write || !hw_ops.update_mailbox_queue_tail_ptr ||
+	    !hw_ops.config_mailbox_rxq || !hw_ops.config_mailbox_txq ||
+	    !hw_ops.stop_mailbox_rxq || !hw_ops.stop_mailbox_txq ||
+	    !hw_ops.get_host_pf_mask || !hw_ops.get_real_bus ||
+	    !hw_ops.cfg_mailbox_qinfo || !hw_ops.set_mailbox_irq ||
+	    !hw_ops.get_fw_eth_map || !hw_ops.get_board_info ||
+	    !hw_ops.init_module || !hw_ops.deinit_module)
+		return ERR_PTR(-EINVAL);
+	hw_ops_tbl->ops = &hw_ops;
+	hw_ops_tbl->priv = hw_mgt;
+
+	return hw_ops_tbl;
+}
+
 static int nbl_pcim_request_selected_bars(struct pci_dev *pdev, u32 mask,
 					  const char *name)
 {
@@ -43,6 +1291,7 @@ int nbl_hw_init_leonis(struct nbl_adapter *adapter)
 {
 	resource_size_t expect_sz = NBL_MEM_BAR_TOTAL_SIZE;
 	struct nbl_common_info *common = &adapter->common;
+	struct nbl_hw_ops_tbl *hw_ops_tbl = NULL;
 	struct pci_dev *pdev = common->pdev;
 	struct nbl_hw_mgt *hw_mgt = NULL;
 	resource_size_t bar_len;
@@ -53,7 +1302,7 @@ int nbl_hw_init_leonis(struct nbl_adapter *adapter)
 	hw_mgt = nbl_hw_setup_hw_mgt(common);
 	if (IS_ERR(hw_mgt)) {
 		ret = PTR_ERR(hw_mgt);
-		goto setup_mgt_fail;
+		return ret;
 	}
 	bar_mask = BIT(NBL_MEMORY_BAR) | BIT(NBL_MAILBOX_BAR);
 	ret = nbl_pcim_request_selected_bars(pdev, bar_mask, NBL_DRIVER_NAME);
@@ -61,14 +1310,14 @@ int nbl_hw_init_leonis(struct nbl_adapter *adapter)
 		dev_err(&pdev->dev,
 			"Request memory bar failed, err = %d\n",
 			ret);
-		goto setup_mgt_fail;
+		return ret;
 	}
 
 	bar_len = pci_resource_len(pdev, NBL_MEMORY_BAR);
 	if (!(pci_resource_flags(pdev, NBL_MEMORY_BAR) & IORESOURCE_MEM)) {
 		dev_err(&pdev->dev, "MEMORY BAR is not memory resource\n");
 		ret = -EINVAL;
-		goto setup_mgt_fail;
+		return ret;
 	}
 	if (common->has_ctrl) {
 		/*
@@ -90,7 +1339,7 @@ int nbl_hw_init_leonis(struct nbl_adapter *adapter)
 				"MEMORY BAR len %pa smaller than expected %pa\n",
 				&bar_len, &expect_sz);
 			ret = -EINVAL;
-			goto setup_mgt_fail;
+			return ret;
 		}
 		hw_size = PAGE_ALIGN_DOWN(NBL_MEM_BAR_TOTAL_SIZE -
 					  NBL_RDMA_NOTIFY_LEN);
@@ -103,7 +1352,7 @@ int nbl_hw_init_leonis(struct nbl_adapter *adapter)
 				"MEMORY BAR len %pa too small for net only reg space\n",
 				&bar_len);
 			ret = -EINVAL;
-			goto setup_mgt_fail;
+			return ret;
 		}
 		hw_size = NBL_REG_NET_ONLY_LEN;
 		hw_mgt->hw_addr = pcim_iomap(pdev, NBL_MEMORY_BAR,
@@ -112,37 +1361,50 @@ int nbl_hw_init_leonis(struct nbl_adapter *adapter)
 	if (!hw_mgt->hw_addr) {
 		dev_err(&pdev->dev, "MEMORY BAR pcim_iomap failed\n");
 		ret = -EIO;
-		goto setup_mgt_fail;
+		return ret;
 	}
 
 	bar_len = pci_resource_len(pdev, NBL_MAILBOX_BAR);
 	if (!(pci_resource_flags(pdev, NBL_MAILBOX_BAR) & IORESOURCE_MEM)) {
 		dev_err(&pdev->dev, "MAILBOX BAR is not memory resource\n");
 		ret = -EINVAL;
-		goto setup_mgt_fail;
+		return ret;
 	}
 	if (bar_len < NBL_BAR2_MAX_LEN) {
 		dev_err(&pdev->dev, "MAILBOX BAR length %pa too small\n",
 			&bar_len);
 		ret = -EINVAL;
-		goto setup_mgt_fail;
+		return ret;
 	}
 	hw_mgt->mailbox_bar_hw_addr = pcim_iomap(pdev, NBL_MAILBOX_BAR,
 						 bar_len);
 	if (!hw_mgt->mailbox_bar_hw_addr) {
 		dev_err(&pdev->dev, "MAILBOX BAR pcim_iomap failed\n");
 		ret = -EIO;
-		goto setup_mgt_fail;
+		return ret;
 	}
 
 	hw_mgt->mailbox_bar_size = bar_len;
+	spin_lock_init(&hw_mgt->reg_lock);
 
+	/*
+	 * Clear a stale driver_status now, while this is the only register
+	 * access this probe has done: the firmware cleanup the 1 -> 0
+	 * transition starts reclaims the mailbox QINFO routing and qinfo
+	 * registers that nbl_dev_init() programs later, so it must not be
+	 * deferred to init_module(); see nbl_hw_clear_stale_status().
+	 */
+	nbl_hw_clear_stale_status(hw_mgt);
+
+	hw_ops_tbl = nbl_hw_setup_ops(common, hw_mgt);
+	if (IS_ERR(hw_ops_tbl)) {
+		ret = PTR_ERR(hw_ops_tbl);
+		return ret;
+	}
+	adapter->intf.hw_ops_tbl = hw_ops_tbl;
 	adapter->core.hw_mgt = hw_mgt;
 
 	return 0;
-
-setup_mgt_fail:
-	return ret;
 }
 
 void nbl_hw_remove_leonis(struct nbl_adapter *adapter)
diff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_leonis/nbl_hw_leonis.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_leonis/nbl_hw_leonis.h
index 1f9e509dc6312f..7d27ab19838e21 100644
--- a/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_leonis/nbl_hw_leonis.h
+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_leonis/nbl_hw_leonis.h
@@ -11,5 +11,347 @@
 #include "../../nbl_include/nbl_include.h"
 #include "../nbl_hw_reg.h"
 
+/*
+ * Chip-wide driver-present handshake with firmware.
+ *
+ * NBL_DRIVER_STATUS_BIT is the only bit the driver owns: setting it tells
+ * firmware that a driver is bound, and the 1 -> 0 transition starts the
+ * firmware pass that reclaims per-PF state (mailbox QINFO routing, qinfo
+ * registers).  Every other bit of this register belongs to firmware and
+ * the driver must leave it as it read it.
+ *
+ * Because the register is shared, bit 16 can only be changed with a
+ * read-modify-write: it has no documented W1S/W1C semantics that would
+ * let the driver set or clear that one bit atomically.  reg_lock makes
+ * the read-modify-write atomic with respect to other CPU writers, but it
+ * cannot serialise against firmware, so a firmware update to another bit
+ * landing between the read and the write would be written back stale.
+ * No such firmware update is known to race the three driver callers
+ * (the probe-time clear, init_module() and deinit_module()), but that is
+ * an observation about this hardware revision, not a guarantee the
+ * driver can enforce.
+ */
+#define NBL_DRIVER_STATUS_REG			0x1300444
+#define NBL_DRIVER_STATUS_BIT			16
+
+/*  ----------  REG BASE ADDR  ----------  */
+/* Interface modules base addr */
+#define NBL_INTF_HOST_PCOMPLETER_BASE		0x00f08000
+#define NBL_INTF_HOST_PADPT_BASE		0x00f4c000
+#define NBL_INTF_HOST_MAILBOX_BASE		0x00fb0000
+#define NBL_INTF_HOST_PCIE_BASE			0X01504000
+/* DP modules base addr */
+#define NBL_DP_USTORE_BASE			0x00104000
+#define NBL_DP_UQM_BASE				0x00114000
+#define NBL_DP_UPED_BASE			0x0015c000
+#define NBL_DP_UVN_BASE				0x00244000
+#define NBL_DP_DSCH_BASE			0x00404000
+#define NBL_DP_SHAPING_BASE			0x00504000
+#define NBL_DP_DVN_BASE				0x00514000
+#define NBL_DP_DSTORE_BASE			0x00704000
+#define NBL_DP_DPED_BASE			0x0075c000
+/*  --------  MAILBOX BAR2 -----  */
+#define NBL_MAILBOX_NOTIFY_ADDR			0x00000000
+#define NBL_MAILBOX_QINFO_CFG_RX_TABLE_ADDR	0x10
+#define NBL_MAILBOX_QINFO_CFG_TX_TABLE_ADDR	0x20
+
+/*  --------  MAILBOX  --------  */
+
+/* mailbox BAR qinfo_cfg_table */
+#define MAILBOX_QINFO_CFG_TABLE_DWLEN	4
+/* data[2] */
+#define NBL_MAILBOX_QINFO_CFG_QUEUE_SIZE_BWID_MASK	GENMASK(3, 0)
+/* data[3] */
+#define NBL_MAILBOX_QINFO_CFG_QUEUE_RST_MASK		BIT(0)
+#define NBL_MAILBOX_QINFO_CFG_QUEUE_EN_MASK		BIT(1)
+#define NBL_MAILBOX_QINFO_CFG_DIF_ERR_MASK		BIT(2)
+#define NBL_MAILBOX_QINFO_CFG_PTR_ERR_MASK		BIT(3)
+struct nbl_mailbox_qinfo_cfg_table {
+	u32 data[MAILBOX_QINFO_CFG_TABLE_DWLEN];
+};
+
+/*  --------  MAILBOX BAR0 -----  */
+/* mailbox qinfo_map_table */
+#define NBL_MAILBOX_QINFO_MAP_REG_ARR(func_id) \
+	(NBL_INTF_HOST_MAILBOX_BASE + 0x00001000 + (func_id) * sizeof(u32))
+
+/* MAILBOX qinfo_map_table */
+#define NBL_MAILBOX_QINFO_MAP_FUNCTION_MASK		GENMASK(2, 0)
+#define NBL_MAILBOX_QINFO_MAP_DEVID_MASK		GENMASK(7, 3)
+#define NBL_MAILBOX_QINFO_MAP_BUS_MASK			GENMASK(15, 8)
+#define NBL_MAILBOX_QINFO_MAP_MSIX_IDX_MASK		GENMASK(28, 16)
+#define NBL_MAILBOX_QINFO_MAP_MSIX_IDX_VALID_MASK	BIT(29)
+
+/*  --------  HOST_PCIE  --------  */
+#define NBL_PCIE_HOST_K_PF_MASK_REG (NBL_INTF_HOST_PCIE_BASE + 0x00001004)
+#define NBL_PCIE_HOST_TL_CFG_BUSDEV (NBL_INTF_HOST_PCIE_BASE + 0x11040)
+
+#define NBL_PCIE_BUS_MASK	GENMASK(12, 5)
+
+/*  --------  HOST_PADPT  --------  */
+#define NBL_HOST_PADPT_HOST_CFG_FC_PD_DN (NBL_INTF_HOST_PADPT_BASE + 0x00000160)
+#define NBL_HOST_PADPT_HOST_CFG_FC_PH_DN (NBL_INTF_HOST_PADPT_BASE + 0x00000164)
+#define NBL_HOST_PADPT_HOST_CFG_FC_NPH_DN \
+	(NBL_INTF_HOST_PADPT_BASE + 0x0000016C)
+#define NBL_HOST_PADPT_HOST_CFG_FC_CPLH_UP \
+	(NBL_INTF_HOST_PADPT_BASE + 0x00000170)
+
+#define NBL_HOST_PADPT_CFG_FC_CPLH_UP_VAL      0x10400
+#define NBL_HOST_PADPT_CFG_FC_PD_DN_VAL        0x10080
+#define NBL_HOST_PADPT_CFG_FC_PH_DN_VAL        0x10010
+#define NBL_HOST_PADPT_CFG_FC_NPH_DN_VAL       0x10010
+/* host_padpt host_msix_info */
+#define NBL_PADPT_HOST_MSIX_INFO_REG_ARR(vector_id) \
+	(NBL_INTF_HOST_PADPT_BASE + 0x00010000 +    \
+	 (vector_id) * sizeof(struct nbl_host_msix_info))
+
+#define NBL_HOST_MSIX_INFO_DWLEN	2
+/* data[0] */
+#define NBL_HOST_MSIX_INFO_INTRL_PNUM_MASK GENMASK(15, 0)
+#define NBL_HOST_MSIX_INFO_INTRL_RATE_MASK GENMASK(31, 16)
+/* data[1] */
+#define NBL_HOST_MSIX_INFO_FUNCTION_MASK GENMASK(2, 0)
+#define NBL_HOST_MSIX_INFO_DEVID_MASK GENMASK(7, 3)
+#define NBL_HOST_MSIX_INFO_BUS_MASK GENMASK(15, 8)
+#define NBL_HOST_MSIX_INFO_VALID_MASK BIT(16)
+#define NBL_HOST_MSIX_INFO_MSIX_MASK_EN_MASK BIT(17)
+struct nbl_host_msix_info {
+	u32 data[NBL_HOST_MSIX_INFO_DWLEN];
+};
+
+/*  --------  HOST_PCOMPLETER  --------  */
+/* pcompleter_host function_msix_map_table */
+#define NBL_PCOMPLETER_FUNCTION_MSIX_MAP(i)   \
+	(NBL_INTF_HOST_PCOMPLETER_BASE + 0x00004000 + \
+	 (i) * sizeof(struct nbl_function_msix_map))
+#define NBL_PCOMPLETER_HOST_MSIX_FID_TABLE(i) \
+	(NBL_INTF_HOST_PCOMPLETER_BASE + 0x0003a000 + (i) * sizeof(u32))
+
+#define NBL_PCOMPLETER_HOST_MSIX_FID_TABLE_FID_MASK  GENMASK(9, 0)
+#define NBL_PCOMPLETER_HOST_MSIX_FID_TABLE_VLD_MASK  BIT(10)
+
+#define NBL_FUNC_MSIX_MAP_DWLEN		4
+/* The VLD/BDF fields live in data[2]; it can be cleared on its own */
+#define NBL_FUNC_MSIX_MAP_VALID_DW	2
+/* data[2] */
+#define NBL_FUNCTION_MSIX_MAP_FUNCTION_MASK GENMASK(2, 0)
+#define NBL_FUNCTION_MSIX_MAP_DEVID_MASK GENMASK(7, 3)
+#define NBL_FUNCTION_MSIX_MAP_BUS_MASK GENMASK(15, 8)
+#define NBL_FUNCTION_MSIX_MAP_VALID_MASK BIT(16)
+struct nbl_function_msix_map {
+	u32 data[NBL_FUNC_MSIX_MAP_DWLEN];
+};
+
+/*  ----------  DPED  ----------  */
+#define NBL_DPED_VLAN_OFFSET		(NBL_DP_DPED_BASE + 0x000003F4)
+#define NBL_DPED_DSCP_OFFSET_0		(NBL_DP_DPED_BASE + 0x000003F8)
+#define NBL_DPED_DSCP_OFFSET_1		(NBL_DP_DPED_BASE + 0x000003FC)
+/* DPED hw_edt_prof/ UPED hw_edt_prof */
+
+#define NBL_DPED_L4_CK_CMD_40_ADDR	(NBL_DP_DPED_BASE + 0x00000338)
+
+#define NBL_DPED_L4_CK_CMD_40_EN_MASK		BIT(31)
+
+/*  ----------  UPED  ----------  */
+/* UPED uped_hw_edt_prof */
+#define NBL_UPED_HW_EDT_PROF_TABLE(i) \
+	(NBL_DP_UPED_BASE + 0x00001000 + (i) * sizeof(u32))
+#define NBL_UPED_V4_TCP_IDX		5
+#define NBL_UPED_V6_TCP_IDX		6
+#define NBL_PED_HW_EDIT_PROFILE_L3_LEN_MASK GENMASK(3, 2)
+
+/*  ----------  DSCH  ----------  */
+#define NBL_DSCH_PSHA_EN_MASK  GENMASK(3, 0)
+/* DSCH dsch maxqid */
+#define NBL_DSCH_HOST_QID_MAX (NBL_DP_DSCH_BASE + 0x00000118)
+#define NBL_DSCH_HOST_QID_MAX_MASK GENMASK(10, 0)
+#define NBL_DSCH_VN_QUANTA_ADDR (NBL_DP_DSCH_BASE + 0x00000134)
+
+#define NBL_MAX_QUEUE_ID	0x7ff
+#define NBL_HOST_QUANTA		0x8000
+#define NBL_ECPU_QUANTA		0x1000
+
+#define NBL_DSCH_VN_QUANTA_H_QUA_MASK GENMASK(15, 0)
+#define NBL_DSCH_VN_QUANTA_E_QUA_MASK GENMASK(31, 16)
+
+/*  ----------  DVN  ----------  */
+/* DVN dvn_queue_table */
+#define NBL_DVN_ECPU_QUEUE_NUM			(NBL_DP_DVN_BASE + 0x0000041C)
+#define NBL_DVN_DESCREQ_NUM_CFG			(NBL_DP_DVN_BASE + 0x00000430)
+#define NBL_DVN_DESC_WR_MERGE_TIMEOUT		(NBL_DP_DVN_BASE + 0x00000480)
+#define NBL_DVN_DIF_REQ_RD_RO_FLAG		(NBL_DP_DVN_BASE + 0x0000045C)
+
+#define DEFAULT_DVN_DESCREQ_NUMCFG		0x03
+#define DEFAULT_DVN_100G_DESCREQ_NUMCFG		0x07
+
+#define DEFAULT_DVN_DESC_WR_MERGE_TIMEOUT_MAX	0x3FF
+
+/* spilit ring descreq_num 0:8,1:16 */
+#define NBL_DVN_DESCREQ_NUM_CFG_AVRING_DESREQ_NUM_CFG_MASK BIT(0)
+/* packet ring descreq_num
+ * 0:8,1:12,2:16;3:20,4:24,5:26;6:32,7:32
+ */
+#define NBL_DVN_DESCREQ_NUM_CFG_PACKED_L1_NUM_MASK GENMASK(6, 4)
+
+#define NBL_DVN_DESC_WR_MERGE_TIMEOUT_CFG_CYCLE_MASK GENMASK(9, 0)
+
+#define NBL_DVN_DIF_REQ_RD_RO_FLAG_DESC_RO_EN_MASK BIT(0)
+#define NBL_DVN_DIF_REQ_RD_RO_FLAG_DATA_RO_EN_MASK BIT(1)
+#define NBL_DVN_DIF_REQ_RD_RO_FLAG_AVRING_RO_EN_MASK BIT(2)
+
+/*  ----------  UVN  ----------  */
+/* UVN uvn_queue_table */
+
+#define NBL_UVN_DESC_RD_WAIT			(NBL_DP_UVN_BASE + 0x0000020C)
+#define NBL_UVN_QUEUE_ERR_MASK			(NBL_DP_UVN_BASE + 0x00000224)
+#define NBL_UVN_ECPU_QUEUE_NUM			(NBL_DP_UVN_BASE + 0x0000023C)
+#define NBL_UVN_DESC_WR_TIMEOUT			(NBL_DP_UVN_BASE + 0x00000214)
+#define NBL_UVN_DIF_REQ_RO_FLAG			(NBL_DP_UVN_BASE + 0x00000250)
+#define NBL_UVN_DESC_PREFETCH_INIT		(NBL_DP_UVN_BASE + 0x00000204)
+#define NBL_UVN_DESC_PREFETCH_NUM		4
+
+#define NBL_UVN_DIF_REQ_RO_FLAG_AVAIL_RD_MASK BIT(0)
+#define NBL_UVN_DIF_REQ_RO_FLAG_DESC_RD_MASK BIT(1)
+#define NBL_UVN_DIF_REQ_RO_FLAG_PKT_WR_MASK BIT(2)
+#define NBL_UVN_DIF_REQ_RO_FLAG_DESC_WR_MASK BIT(3)
+
+#define NBL_UVN_DESC_WR_TIMEOUT_NUM_MASK GENMASK(14, 0)
+
+#define NBL_UVN_QUEUE_ERR_MASK_DIF_ERR_MASK BIT(5)
+
+#define NBL_UVN_DESC_PREFETCH_INIT_NUM_MASK GENMASK(7, 0)
+#define NBL_UVN_DESC_PREFETCH_INIT_SEL_MASK BIT(16)
+
+#define NBL_UVN_DESC_WR_TIMEOUT_VAL 0x12c
+/* 200us = 200000ns / 1.67ns per tick = 119760 ticks */
+#define NBL_UVN_DESC_RD_WAIT_TICKS 119760
+
+/*  --------  USTORE  --------  */
+#define NBL_USTORE_PKT_LEN_ADDR (NBL_DP_USTORE_BASE + 0x00000108)
+#define NBL_USTORE_PORT_DROP_TH_REG_ARR(port_id) \
+	(NBL_DP_USTORE_BASE + 0x00000150 + (port_id) * sizeof(u32))
+#define NBL_USTORE_BUF_PORT_DROP_PKT(eth_id) \
+	(NBL_DP_USTORE_BASE + 0x00002500 + (eth_id) * sizeof(u32))
+#define NBL_USTORE_BUF_PORT_TRUN_PKT(eth_id) \
+	(NBL_DP_USTORE_BASE + 0x00002540 + (eth_id) * sizeof(u32))
+
+#define NBL_USTORE_SINGLE_ETH_DROP_TH		0xC80
+#define NBL_USTORE_DUAL_ETH_DROP_TH		0x640
+#define NBL_USTORE_QUAD_ETH_DROP_TH		0x320
+
+/* USTORE pkt_len */
+#define NBL_USTORE_PKT_LEN_MIN_MASK GENMASK(6, 0)
+
+/* USTORE port_drop_th */
+#define NBL_USTORE_PORT_DROP_TH_DISC_TH_MASK GENMASK(11, 0)
+#define NBL_USTORE_PORT_DROP_TH_EN_MASK BIT(31)
+
+/* UQM*/
+#define NBL_UQM_QUE_TYPE			(NBL_DP_UQM_BASE + 0x0000013c)
+#define NBL_UQM_DROP_PKT_CNT			(NBL_DP_UQM_BASE + 0x000009C0)
+#define NBL_UQM_DROP_PKT_SLICE_CNT		(NBL_DP_UQM_BASE + 0x000009C4)
+#define NBL_UQM_DROP_PKT_LEN_ADD_CNT		(NBL_DP_UQM_BASE + 0x000009C8)
+#define NBL_UQM_DROP_HEAD_PNTR_ADD_CNT		(NBL_DP_UQM_BASE + 0x000009CC)
+#define NBL_UQM_DROP_WEIGHT_ADD_CNT		(NBL_DP_UQM_BASE + 0x000009D0)
+#define NBL_UQM_PORT_DROP_PKT_CNT		(NBL_DP_UQM_BASE + 0x000009D4)
+#define NBL_UQM_PORT_DROP_PKT_SLICE_CNT		(NBL_DP_UQM_BASE + 0x000009F4)
+#define NBL_UQM_PORT_DROP_PKT_LEN_ADD_CNT	(NBL_DP_UQM_BASE + 0x00000A14)
+#define NBL_UQM_PORT_DROP_HEAD_PNTR_ADD_CNT	(NBL_DP_UQM_BASE + 0x00000A34)
+#define NBL_UQM_PORT_DROP_WEIGHT_ADD_CNT	(NBL_DP_UQM_BASE + 0x00000A54)
+#define NBL_UQM_FWD_DROP_CNT			(NBL_DP_UQM_BASE + 0x00000A80)
+#define NBL_UQM_DPORT_DROP_CNT			(NBL_DP_UQM_BASE + 0x00000B74)
+
+#define NBL_UQM_PORT_DROP_DEPTH			6
+#define NBL_UQM_DPORT_DROP_DEPTH		16
+
+/*  ---------  SHAPING  ---------  */
+
+/* Shaping rate unit: 1 = 1 Mbit/s.
+ * e.g. 100000 = 100 Gbit/s, 25000 = 25 Gbit/s.
+ */
+#define NBL_LR_LEONIS_NET_BUCKET_DEPTH		9600
+#define NBL_SHAPING_DPORT_ADDR (NBL_DP_SHAPING_BASE + 0x700)
+#define NBL_SHAPING_DPORT_DWLEN 4
+#define NBL_SHAPING_DPORT_REG(r) \
+	(NBL_SHAPING_DPORT_ADDR + (NBL_SHAPING_DPORT_DWLEN * 4) * (r))
+#define NBL_SHAPING_DVN_DPORT_ADDR (NBL_DP_SHAPING_BASE + 0x750)
+#define NBL_SHAPING_DVN_DPORT_DWLEN 4
+#define NBL_SHAPING_DVN_DPORT_REG(r) \
+	(NBL_SHAPING_DVN_DPORT_ADDR + (NBL_SHAPING_DVN_DPORT_DWLEN * 4) * (r))
+#define NBL_DSCH_PSHA_EN_ADDR (NBL_DP_DSCH_BASE + 0x00000314)
+#define NBL_SHAPING_NET_ADDR (NBL_DP_SHAPING_BASE + 0x1800)
+#define NBL_SHAPING_NET_DWLEN 4
+#define NBL_SHAPING_NET_REG(r) \
+	(NBL_SHAPING_NET_ADDR + (NBL_SHAPING_NET_DWLEN * 4) * (r))
+
+#define NBL_DPORT_VALID_MASK		GENMASK_ULL(0, 0)
+#define NBL_DPORT_DEPTH_MASK		GENMASK_ULL(19, 1)
+#define NBL_DPORT_CIR_MASK		GENMASK_ULL(38, 20)
+#define NBL_DPORT_PIR_MASK		GENMASK_ULL(57, 39)
+#define NBL_DPORT_CBS_MASK_LOW		GENMASK_ULL(63, 58)
+#define NBL_DPORT_CBS_MASK_HIGH		GENMASK_ULL(14, 0)
+#define NBL_DPORT_PBS_MASK		GENMASK_ULL(35, 15)
+
+/* SHAPING shaping_net */
+struct nbl_shaping_net_u {
+	u32 data[NBL_SHAPING_NET_DWLEN];
+};
+
+struct nbl_shaping_dport_u {
+	u32 data[NBL_SHAPING_DPORT_DWLEN];
+};
+
+struct nbl_shaping_dvn_dport_u {
+	u32 data[NBL_SHAPING_DVN_DPORT_DWLEN];
+};
+
+/*  --------  DSTORE  --------  */
+#define NBL_DSTORE_D_DPORT_FC_TH_ADDR  (NBL_DP_DSTORE_BASE + 0x00000600)
+#define NBL_DSTORE_D_DPORT_FC_TH_DEPTH 5
+#define NBL_DSTORE_D_DPORT_FC_TH_WIDTH 32
+#define NBL_DSTORE_D_DPORT_FC_TH_DWLEN 1
+
+#define NBL_DSTORE_D_DPORT_FC_XOFF_TH_MASK GENMASK(10, 0)
+#define NBL_DSTORE_D_DPORT_FC_XON_TH_MASK GENMASK(26, 16)
+#define NBL_DSTORE_D_DPORT_FC_FC_EN_MASK BIT(31)
+
+#define NBL_DSTORE_D_DPORT_FC_TH_REG(r)  \
+	(NBL_DSTORE_D_DPORT_FC_TH_ADDR + \
+	 (NBL_DSTORE_D_DPORT_FC_TH_DWLEN * 4) * (r))
+#define NBL_DSTORE_PORT_DROP_TH_ADDR (NBL_DP_DSTORE_BASE + 0x00000150)
+#define NBL_DSTORE_PORT_DROP_TH_DEPTH 6
+#define NBL_DSTORE_PORT_DROP_TH_WIDTH 32
+#define NBL_DSTORE_PORT_DROP_TH_DWLEN 1
+
+#define NBL_DSTORE_PORT_DROP_EN_MASK BIT(31)
+
+#define NBL_DSTORE_DROP_XOFF_TH			0xC8
+#define NBL_DSTORE_DROP_XON_TH			0x64
+
+#define NBL_DSTORE_DROP_XOFF_TH_100G		0x1F4
+#define NBL_DSTORE_DROP_XON_TH_100G		0x12C
+
+#define NBL_DSTORE_DISC_BP_TH (NBL_DP_DSTORE_BASE + 0x00000630)
+
+#define NBL_DSTORE_DISC_BP_TH_EN_MASK BIT(31)
+
+#define NBL_DSTORE_PORT_DROP_TH_REG(r)  \
+	(NBL_DSTORE_PORT_DROP_TH_ADDR + \
+	 (NBL_DSTORE_PORT_DROP_TH_DWLEN * 4) * (r))
+
+#define NBL_FW_BOARD_CONFIG			0x200
+#define NBL_FW_BOARD_DW3_OFFSET			(NBL_FW_BOARD_CONFIG + 12)
+#define NBL_FW_BOARD_DW6_OFFSET			(NBL_FW_BOARD_CONFIG + 24)
+
+#define NBL_FW_BOARD_DW3_PORT_TYPE_MASK BIT(0)
+#define NBL_FW_BOARD_DW3_PORT_NUM_MASK GENMASK(7, 1)
+#define NBL_FW_BOARD_DW3_PORT_SPEED_MASK GENMASK(9, 8)
+#define NBL_FW_BOARD_DW3_GPIO_TYPE_MASK GENMASK(12, 10)
+#define NBL_FW_BOARD_DW3_P4_VERSION_MASK GENMASK(13, 13)
+
+#define NBL_FW_BOARD_DW6_LANE_BITMAP_MASK GENMASK(7, 0)
+#define NBL_FW_BOARD_DW6_ETH_BITMAP_MASK GENMASK(15, 8)
+
+#define NBL_LEONIS_QUIRKS_OFFSET	0x00000140
 #define NBL_BAR2_MAX_LEN		0x300
 #endif
diff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_leonis/nbl_resource_leonis.c b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_leonis/nbl_resource_leonis.c
new file mode 100644
index 00000000000000..ba589af55374b7
--- /dev/null
+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_leonis/nbl_resource_leonis.c
@@ -0,0 +1,370 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2026 Nebula Matrix Limited.
+ */
+#include <linux/device.h>
+#include <linux/pci.h>
+#include <linux/bits.h>
+#include "nbl_resource_leonis.h"
+
+static struct nbl_resource_ops res_ops = {
+	.get_vsi_id = nbl_res_func_id_to_vsi_id,
+	.get_eth_id = nbl_res_get_eth_id,
+	.cfg_msix_map = nbl_res_intr_cfg_msix_map,
+	.destroy_msix_map = nbl_res_intr_destroy_msix_map,
+	.set_mailbox_irq = nbl_res_intr_set_mailbox_irq,
+	.init_module = nbl_res_chip_init_module,
+	.deinit_module = nbl_res_chip_deinit_module,
+};
+
+static struct nbl_resource_mgt *
+nbl_res_setup_res_mgt(struct nbl_common_info *common)
+{
+	struct nbl_resource_info *resource_info;
+	struct nbl_resource_mgt *res_mgt;
+	struct device *dev = common->dev;
+
+	res_mgt = devm_kzalloc(dev, sizeof(*res_mgt), GFP_KERNEL);
+	if (!res_mgt)
+		return ERR_PTR(-ENOMEM);
+	res_mgt->common = common;
+
+	resource_info =
+		devm_kzalloc(dev, sizeof(*resource_info), GFP_KERNEL);
+	if (!resource_info)
+		return ERR_PTR(-ENOMEM);
+	res_mgt->resource_info = resource_info;
+
+	return res_mgt;
+}
+
+static struct nbl_resource_ops_tbl *
+nbl_res_setup_ops(struct device *dev, struct nbl_resource_mgt *res_mgt)
+{
+	struct nbl_resource_ops_tbl *res_ops_tbl;
+
+	res_ops_tbl = devm_kzalloc(dev, sizeof(*res_ops_tbl), GFP_KERNEL);
+	if (!res_ops_tbl)
+		return ERR_PTR(-ENOMEM);
+	if (!res_ops.get_vsi_id || !res_ops.get_eth_id ||
+	    !res_ops.cfg_msix_map || !res_ops.destroy_msix_map ||
+	    !res_ops.set_mailbox_irq || !res_ops.init_module ||
+	    !res_ops.deinit_module)
+		return ERR_PTR(-EINVAL);
+	res_ops_tbl->ops = &res_ops;
+	res_ops_tbl->priv = res_mgt;
+
+	return res_ops_tbl;
+}
+
+static int nbl_res_ctrl_dev_setup_eth_info(struct nbl_resource_mgt *res_mgt)
+{
+	struct nbl_hw_ops *hw_ops = res_mgt->hw_ops_tbl->ops;
+	struct device *dev = res_mgt->common->dev;
+	struct nbl_eth_info *eth_info;
+	u32 eth_bitmap = 0;
+	u32 eth_num = 0;
+	u32 fw_port_num;
+	int i;
+
+	eth_info = devm_kzalloc(dev, sizeof(*eth_info), GFP_KERNEL);
+	if (!eth_info)
+		return -ENOMEM;
+
+	res_mgt->resource_info->eth_info = eth_info;
+
+	fw_port_num = res_mgt->resource_info->board_info.eth_num;
+
+	hw_ops->get_fw_eth_map(res_mgt->hw_ops_tbl->priv, &eth_bitmap);
+	if (eth_bitmap & ~((1 << NBL_MAX_ETHERNET) - 1)) {
+		dev_err(dev, "FW reported invalid eth_bitmap 0x%x\n",
+			eth_bitmap);
+		return -EINVAL;
+	}
+	if (fw_port_num != hweight32(eth_bitmap)) {
+		dev_err(dev, "FW inconsistency: port_num=%u, bitmap=0x%x\n",
+			fw_port_num, eth_bitmap);
+		return -EINVAL;
+	}
+	/*
+	 * Firmware is ready before probe. Valid port counts are 1/2/4;
+	 * 0 (invalid config), 3 (unsupported topology), and >4 (exceeds
+	 * hardware max) are all rejected with -EINVAL.
+	 */
+	if (fw_port_num == 0 || fw_port_num == 3 ||
+	    fw_port_num > NBL_MAX_ETHERNET) {
+		dev_err(dev, "FW reports %u Ethernet ports, unsupported (valid: 1/2/4)\n",
+			fw_port_num);
+		return -EINVAL;
+	}
+	eth_info->eth_num = fw_port_num;
+	/* Intentional design constraint: each PF maps to exactly one
+	 * Ethernet port. This couples PF identity to port identity
+	 * and is required by nbl_res_get_eth_id() which indexes
+	 * eth_info->eth_id[] by relative PF id.
+	 */
+	if (res_mgt->common->max_pf != eth_info->eth_num) {
+		dev_err(dev, "Invalid PF-to-port topology: max_pf=%u, eth_num=%u\n",
+			res_mgt->common->max_pf, eth_info->eth_num);
+		return -EINVAL;
+	}
+
+	/*
+	 * Any subset of valid bitmap bits is accepted (e.g. 0/1, 0/2,
+	 * 1/3, etc.).  Firmware only needs to report the correct count
+	 * of active ports; no hard-coded fixed bit positions required.
+	 * eth_id[] is filled in ascending bitmap-bit order, so the Nth
+	 * relative PF owns the Nth logical port (max_pf == eth_num is
+	 * enforced above).
+	 */
+	for (i = 0; i < NBL_MAX_ETHERNET; i++) {
+		if ((1 << i) & eth_bitmap) {
+			eth_info->eth_id[eth_num] = i;
+			eth_num++;
+		}
+	}
+
+	return 0;
+}
+
+static int nbl_res_ctrl_dev_sriov_info_init(struct nbl_resource_mgt *res_mgt)
+{
+	struct nbl_hw_ops *hw_ops = res_mgt->hw_ops_tbl->ops;
+	struct nbl_hw_mgt *p = res_mgt->hw_ops_tbl->priv;
+	struct nbl_common_info *common = res_mgt->common;
+	struct nbl_sriov_info *sriov_info;
+	struct device *dev = common->dev;
+	u8 hw_bus = 0;
+	u16 function;
+	u16 func_id;
+
+	hw_ops->get_real_bus(p, &hw_bus);
+	if (common->function + common->max_pf > NBL_MAX_PF) {
+		dev_err(dev, "PF count exceeds available function space\n");
+		return -EINVAL;
+	}
+	sriov_info = devm_kcalloc(dev, common->max_pf,
+				  sizeof(*sriov_info), GFP_KERNEL);
+	if (!sriov_info)
+		return -ENOMEM;
+
+	res_mgt->resource_info->sriov_info = sriov_info;
+	/*
+	 * Real bus number of the control PF;
+	 * the BDF table below is built from it.
+	 */
+	common->hw_bus = hw_bus;
+
+	for (func_id = 0; func_id < common->max_pf; func_id++) {
+		sriov_info = res_mgt->resource_info->sriov_info + func_id;
+		function = common->function + func_id;
+		sriov_info->bdf = PCI_DEVID(common->hw_bus,
+					    PCI_DEVFN(common->devid, function));
+	}
+
+	return 0;
+}
+
+static int nbl_res_ctrl_dev_vsi_info_init(struct nbl_resource_mgt *res_mgt)
+{
+	struct nbl_eth_info *eth_info = res_mgt->resource_info->eth_info;
+	struct nbl_common_info *common = res_mgt->common;
+	struct device *dev = common->dev;
+	struct nbl_vsi_info *vsi_info;
+	int i;
+
+	vsi_info = devm_kzalloc(dev, sizeof(*vsi_info), GFP_KERNEL);
+	if (!vsi_info)
+		return -ENOMEM;
+
+	res_mgt->resource_info->vsi_info = vsi_info;
+	/*
+	 * case 1 one port(1pf)
+	 * pf0 (NBL_VSI_SERV_PF_DATA_TYPE) vsi is 0
+	 * case 2 two port(2pf)
+	 * pf0,pf1(NBL_VSI_SERV_PF_DATA_TYPE) vsi is 0,512
+	 * case 3 four port(4pf)
+	 * pf0,pf1,pf2,pf3(NBL_VSI_SERV_PF_DATA_TYPE) vsi is 0,256,512,768
+	 */
+
+	vsi_info->num = eth_info->eth_num;
+	/*
+	 * eth_num can be 1/2/4:
+	 * - 2/4 ports use dedicated gap constants;
+	 * - 1 port falls back to NBL_DEFAULT_VSI_ID_GAP (1024).
+	 * All three values produce valid base_id offsets.
+	 */
+	for (i = 0; i < vsi_info->num; i++) {
+		vsi_info->serv_info[i][NBL_VSI_SERV_PF_DATA_TYPE].base_id =
+			i * nbl_vsi_id_gap(vsi_info->num);
+		vsi_info->serv_info[i][NBL_VSI_SERV_PF_DATA_TYPE].num = 1;
+	}
+
+	return 0;
+}
+
+static int nbl_res_init_pf_num(struct nbl_resource_mgt *res_mgt)
+{
+	struct nbl_hw_ops *hw_ops = res_mgt->hw_ops_tbl->ops;
+	u32 exp_contiguous_mask = 0;
+	u32 pf_mask = 0;
+	u32 pf_num = 0;
+	int i;
+
+	hw_ops->get_host_pf_mask(res_mgt->hw_ops_tbl->priv, &pf_mask);
+
+	/*
+	 * k_pf_mask register rule:
+	 * bit N == 0  -> PF#N enabled; bit N == 1 -> PF#N masked out.
+	 * Hardware constraint: bit0 is PF0's mask bit; driver requires
+	 * PF0 enabled as management PF, so bit0 must be clear.
+	 * All-zero pf_mask means all PF0~PF7 are enabled, which is unsupported
+	 * by the driver
+	 *
+	 * Product firmware constraint: only 3 valid configurations supported:
+	 * 1 PF  (PF0 only): pf_num = 1, mask = 0xfe
+	 * 2 PFs (PF0,PF1):  pf_num = 2, mask = 0xfc
+	 * 4 PFs (PF0~PF3): pf_num = 4, mask = 0xf0
+	 * No other PF count or sparse/non-contiguous PF layout is allowed.
+	 *
+	 * This is the only validation of the mask in the driver.  The channel
+	 * layer reads the same raw register value to program the mailbox
+	 * QINFO table and assumes the layout checked here, so it depends on
+	 * this running first: resource init completes before the channel
+	 * layer is set up.
+	 */
+	for (i = 0; i < NBL_MAX_PF; i++) {
+		if (!(pf_mask & (1 << i)))
+			pf_num++;
+	}
+
+	/*
+	 * Sanity check: enabled PFs must be contiguous starting from PF0.
+	 * Current resource framework uses relative PF id, sparse PF layout
+	 * will cause mismatch between resource layer and hardware func_id.
+	 */
+	for (i = 0; i < pf_num; i++)
+		exp_contiguous_mask |= BIT(i);
+	if ((pf_mask & exp_contiguous_mask) != 0) {
+		dev_err(res_mgt->common->dev,
+			"pf_mask 0x%08x: non-contiguous enabled PF, unsupported\n",
+			pf_mask);
+		return -EINVAL;
+	}
+
+	/* Only allow product-specified PF count: 1 / 2 / 4 */
+	if (pf_num != 1 && pf_num != 2 && pf_num != 4) {
+		dev_err(res_mgt->common->dev,
+			"Invalid pf_num=%u (mask=0x%08x), only 1/2/4 PFs supported\n",
+			pf_num, pf_mask);
+		return -EINVAL;
+	}
+
+	res_mgt->common->max_pf = pf_num;
+
+	return 0;
+}
+
+static void nbl_res_init_board_info(struct nbl_resource_mgt *res_mgt)
+{
+	struct nbl_hw_ops *hw_ops = res_mgt->hw_ops_tbl->ops;
+
+	hw_ops->get_board_info(res_mgt->hw_ops_tbl->priv,
+			       &res_mgt->resource_info->board_info);
+}
+
+static int nbl_res_start(struct nbl_resource_mgt *res_mgt)
+{
+	struct nbl_common_info *common = res_mgt->common;
+	int ret = 0;
+
+	if (common->has_ctrl) {
+		nbl_res_init_board_info(res_mgt);
+
+		ret = nbl_res_init_pf_num(res_mgt);
+		if (ret)
+			return ret;
+
+		ret = nbl_res_ctrl_dev_sriov_info_init(res_mgt);
+		if (ret)
+			return ret;
+
+		ret = nbl_res_ctrl_dev_setup_eth_info(res_mgt);
+		if (ret)
+			return ret;
+
+		ret = nbl_res_ctrl_dev_vsi_info_init(res_mgt);
+		if (ret)
+			return ret;
+
+		ret = nbl_intr_mgt_start(res_mgt);
+		if (ret)
+			return ret;
+	}
+
+	return 0;
+}
+
+int nbl_res_init_leonis(struct nbl_adapter *adap)
+{
+	struct nbl_channel_ops_tbl *chan_ops_tbl = adap->intf.channel_ops_tbl;
+	struct nbl_hw_ops_tbl *hw_ops_tbl = adap->intf.hw_ops_tbl;
+	struct nbl_common_info *common = &adap->common;
+	struct nbl_resource_ops_tbl *res_ops_tbl;
+	struct device *dev = &adap->pdev->dev;
+	struct nbl_resource_mgt *res_mgt;
+	int ret;
+
+	res_mgt = nbl_res_setup_res_mgt(common);
+	if (IS_ERR(res_mgt)) {
+		ret = PTR_ERR(res_mgt);
+		return ret;
+	}
+	res_mgt->chan_ops_tbl = chan_ops_tbl;
+	res_mgt->hw_ops_tbl = hw_ops_tbl;
+
+	ret = nbl_res_start(res_mgt);
+	if (ret)
+		return ret;
+
+	res_ops_tbl = nbl_res_setup_ops(dev, res_mgt);
+	if (IS_ERR(res_ops_tbl)) {
+		ret = PTR_ERR(res_ops_tbl);
+		return ret;
+	}
+	adap->intf.resource_ops_tbl = res_ops_tbl;
+	adap->core.res_mgt = res_mgt;
+
+	return 0;
+}
+
+void nbl_res_remove_leonis(struct nbl_adapter *adap)
+{
+	struct nbl_resource_mgt *res_mgt = adap->core.res_mgt;
+	struct nbl_common_info *common = &adap->common;
+
+	if (!res_mgt)
+		return;
+
+	/*
+	 * Tear down all MSI-X maps before destroying coherent tables.
+	 * This is critical on the control PF, whose table carries the map
+	 * entries installed for every configured function: they must be
+	 * invalidated in hardware before the tables backing them are
+	 * freed, otherwise the device keeps DMA-reading released pages.
+	 * Sibling PFs are already unbound by the device-link ordering
+	 * so no remote PF still has a live map entry or a running
+	 * mailbox RPC at this point.
+	 */
+	if (common->has_ctrl && res_mgt->intr_mgt)
+		nbl_intr_mgt_stop(res_mgt);
+
+	/* Note:
+	 * per-function interrupts arrays (kcalloc) are freed by
+	 * nbl_intr_mgt_stop().
+	 * MSIX coherent tables are explicitly freed by dma_free_coherent()
+	 * inside the intr destroy path, before nbl_intr_mgt_stop() returns.
+	 * intr_mgt itself (devm_kzalloc) is released by devres after this
+	 * function returns
+	 */
+}
diff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_leonis/nbl_resource_leonis.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_leonis/nbl_resource_leonis.h
new file mode 100644
index 00000000000000..f1cb0f23240b0c
--- /dev/null
+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_leonis/nbl_resource_leonis.h
@@ -0,0 +1,12 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2026 Nebula Matrix Limited.
+ */
+
+#ifndef _NBL_RESOURCE_LEONIS_H_
+#define _NBL_RESOURCE_LEONIS_H_
+
+#include "../nbl_resource.h"
+#include "../nbl_interrupt.h"
+#include "../nbl_chip.h"
+#endif
diff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_reg.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_reg.h
index e281109f502e67..00fa33eeabaab9 100644
--- a/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_reg.h
+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_hw_reg.h
@@ -8,6 +8,7 @@
 
 #include <linux/types.h>
 
+#include "../nbl_include/nbl_def_channel.h"
 #include "../nbl_include/nbl_def_hw.h"
 #include "../nbl_include/nbl_def_common.h"
 #include "../nbl_core.h"
@@ -16,6 +17,7 @@
 #define NBL_MAILBOX_BAR				2
 #define NBL_RDMA_NOTIFY_LEN			(8ULL << 10)
 #define NBL_REG_NET_ONLY_LEN			(8ULL << 10)
+#define NBL_HW_DUMMY_REG			0x1300904
 /*
  * PCI MEMORY BAR total size: 64MiB.
  */
@@ -26,6 +28,37 @@ struct nbl_hw_mgt {
 	u8 __iomem *hw_addr;
 	u8 __iomem *mailbox_bar_hw_addr;
 	resource_size_t mailbox_bar_size;
+	spinlock_t reg_lock; /* Protect reg access */
 };
 
+static inline u32 rd32(u8 __iomem *addr, u64 reg)
+{
+	return readl(addr + reg);
+}
+
+static inline void wr32(u8 __iomem *addr, u64 reg, u32 value)
+{
+	writel(value, addr + reg);
+}
+
+static inline void nbl_hw_wr32(struct nbl_hw_mgt *hw_mgt, u64 reg, u32 value)
+{
+	wr32(hw_mgt->hw_addr, reg, value);
+}
+
+static inline u32 nbl_hw_rd32(struct nbl_hw_mgt *hw_mgt, u64 reg)
+{
+	return rd32(hw_mgt->hw_addr, reg);
+}
+
+static inline void nbl_mbx_wr32(struct nbl_hw_mgt *hw_mgt, u64 reg, u32 value)
+{
+	writel(value, hw_mgt->mailbox_bar_hw_addr + reg);
+}
+
+static inline u32 nbl_mbx_rd32(struct nbl_hw_mgt *hw_mgt, u64 reg)
+{
+	return readl(hw_mgt->mailbox_bar_hw_addr + reg);
+}
+
 #endif
diff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_interrupt.c b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_interrupt.c
new file mode 100644
index 00000000000000..3969392abdc64a
--- /dev/null
+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_interrupt.c
@@ -0,0 +1,851 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2026 Nebula Matrix Limited.
+ */
+#include <linux/device.h>
+#include <linux/delay.h>
+#include <linux/dma-mapping.h>
+#include <linux/bitfield.h>
+#include "nbl_interrupt.h"
+
+#define NBL_MSIX_DMA_SYNC_MIN_US	1000 /* us */
+#define NBL_MSIX_DMA_SYNC_MAX_US	1200 /* us */
+
+/*
+ * Bounded best-effort wait for in-flight pcompleter fetches of a MSI-X
+ * map table after that entry's VALID bit has been cleared and flushed.
+ *
+ * The chip exposes no idle/completion status for those fetches, so the
+ * only ordering available is: clear VALID -> flush_write() -> wait.  The
+ * window is sized well above the tables' worst-case fetch/response
+ * latency; it is not a guarantee, and it protects two things:
+ *   - the coherent table that __nbl_res_intr_complete_destroy_msix_map()
+ *     is about to dma_free_coherent();
+ *   - the vector set that a reconfiguration is about to hand to another
+ *     function.
+ * A future caller that cannot tolerate the residual risk must keep the
+ * table allocated for the lifetime of the function instead of relying on
+ * this delay.
+ *
+ * Both are bus-master reads tagged with the control PF's BDF, so
+ * pci_clear_master() is the only other lever, and nbl_remove() pulls it
+ * before any teardown runs: that keeps the device from starting a new
+ * fetch inside this window.  It cannot help a sibling PF's map destroyed
+ * at runtime, because those fetches carry the control PF's BDF and so
+ * are unaffected by clearing bus mastering on the sibling.  Runtime
+ * destroys of sibling maps only become reachable with the mailbox RPC
+ * added later in this series.
+ */
+static void nbl_intr_quiesce_wait(void)
+{
+	usleep_range(NBL_MSIX_DMA_SYNC_MIN_US, NBL_MSIX_DMA_SYNC_MAX_US);
+}
+
+/*
+ * Release global vector IDs back to intr_net_bmap / intr_other_bmap.
+ * Caller must hold intr_mgt->lock, passing the intr_mgt it locked:
+ * nbl_intr_mgt_stop() clears res_mgt->intr_mgt while a caller may
+ * already be blocked on that mutex, so it must not be re-read here.
+ */
+static void nbl_intr_release_bitmap(struct nbl_interrupt_mgt *intr_mgt,
+				    struct nbl_resource_mgt *res_mgt,
+				    u16 *vec_buf, u16 cnt)
+{
+	u16 bit;
+	u16 i;
+
+	lockdep_assert_held(&intr_mgt->lock);
+
+	if (!vec_buf || cnt == 0)
+		return;
+
+	for (i = 0; i < cnt; i++) {
+		u16 intr_index = vec_buf[i];
+
+		if (intr_index >= NBL_NET_INTR_BASE) {
+			bit = intr_index - NBL_NET_INTR_BASE;
+			if (bit < NBL_MAX_NET_INTERRUPT)
+				clear_bit(bit, intr_mgt->intr_net_bmap);
+			else
+				dev_warn(res_mgt->common->dev,
+					 "invalid net intr index %u\n",
+					 intr_index);
+		} else {
+			if (intr_index < NBL_MAX_OTHER_INTERRUPT)
+				clear_bit(intr_index,
+					  intr_mgt->intr_other_bmap);
+			else
+				dev_warn(res_mgt->common->dev,
+					 "invalid other intr index %u\n",
+					 intr_index);
+		}
+	}
+}
+
+/*
+ * Internal (unlocked) mailbox IRQ bind.  Caller must hold
+ * intr_mgt->lock, passing the intr_mgt it locked (see
+ * nbl_intr_release_bitmap()).
+ *
+ * The disable path deliberately carries no state check: the teardown
+ * sequence uses it to disarm routing while func_res->state is still
+ * CONFIGURED, and it stays harmless after nbl_intr_mgt_stop() because
+ * the hardware op ignores gvec when en_msix=false.  Only the enable
+ * path is gated by the stopping latch and the per-function state.
+ */
+static int __nbl_res_intr_set_mailbox_irq(struct nbl_interrupt_mgt *intr_mgt,
+					  struct nbl_resource_mgt *res_mgt,
+					  u16 func_id, u16 vector_id,
+					  bool en_msix)
+{
+	struct nbl_hw_ops *hw_ops = res_mgt->hw_ops_tbl->ops;
+	struct nbl_common_info *common = res_mgt->common;
+	struct device *dev = common->dev;
+	struct nbl_func_interrupt_resource_mng *func_res;
+	u16 gvec;
+
+	lockdep_assert_held(&intr_mgt->lock);
+
+	/*
+	 * func_intr_res[] is indexed by the raw hardware function id; this
+	 * bound check is the only guard on this path, so it is what keeps
+	 * VF ids out.  Whether the PF is present on this board is not
+	 * checked here: callers pass either common->mgt_pf or an id that
+	 * nbl_res_intr_cfg_msix_map() has validated.
+	 */
+	if (func_id >= NBL_MAX_PF) {
+		dev_err(dev, "func_id %u out of range\n", func_id);
+		return -EINVAL;
+	}
+
+	if (!en_msix) {
+		hw_ops->set_mailbox_irq(res_mgt->hw_ops_tbl->priv,
+					func_id, false, 0);
+		hw_ops->flush_write(res_mgt->hw_ops_tbl->priv);
+		return 0;
+	}
+
+	/*
+	 * Enable path: the map must be live and not under teardown,
+	 * otherwise routing would point at a vector that the complete
+	 * phase is about to release and never re-disables.
+	 */
+	if (intr_mgt->stopping)
+		return -ESHUTDOWN;
+
+	func_res = &intr_mgt->func_intr_res[func_id];
+	if (func_res->state != NBL_INTR_FUNC_CONFIGURED) {
+		dev_err(dev, "func %u MSIX map not configured (state %u)\n",
+			func_id, func_res->state);
+		return -ENODEV;
+	}
+
+	if (vector_id >= func_res->num_interrupts) {
+		dev_err(dev, "vector_id %u out of range (max %u)\n",
+			vector_id, func_res->num_interrupts - 1);
+		return -EINVAL;
+	}
+
+	gvec = func_res->interrupts[vector_id];
+	hw_ops->set_mailbox_irq(res_mgt->hw_ops_tbl->priv, func_id,
+				en_msix, gvec);
+	hw_ops->flush_write(res_mgt->hw_ops_tbl->priv);
+
+	return 0;
+}
+
+/*
+ * Internal (unlocked) MSI-X map teardown prepare phase: only hardware
+ * register operations. The DMA address is retained and only the VALID
+ * bit is cleared; zeroing the address (Stage 2) is deferred to the
+ * complete phase after the hardware-DMA quiesce window.
+ *
+ * Caller must hold intr_mgt->lock.
+ */
+static int
+__nbl_res_intr_prepare_destroy_msix_map(struct nbl_interrupt_mgt *intr_mgt,
+					struct nbl_resource_mgt *res_mgt,
+					u16 func)
+{
+	struct nbl_hw_ops *hw_ops = res_mgt->hw_ops_tbl->ops;
+	struct nbl_func_interrupt_resource_mng *func_res;
+	u16 *interrupts;
+	u16 intr_num, i;
+	int ret;
+
+	lockdep_assert_held(&intr_mgt->lock);
+
+	if (func >= NBL_MAX_PF) {
+		dev_err(res_mgt->common->dev, "Invalid func_id %u\n", func);
+		return -EINVAL;
+	}
+
+	func_res = &intr_mgt->func_intr_res[func];
+	if (func_res->state != NBL_INTR_FUNC_CONFIGURED)
+		return 0;
+
+	interrupts = func_res->interrupts;
+	intr_num = func_res->num_interrupts;
+
+	/* Step 0: disable mailbox IRQ routing before tearing down map */
+	ret = __nbl_res_intr_set_mailbox_irq(intr_mgt, res_mgt, func, 0, false);
+	if (ret) {
+		dev_err(res_mgt->common->dev,
+			"disable mailbox irq failed, func=%u ret=%d\n",
+			func, ret);
+		return ret;
+	}
+
+	/* Step 1: invalidate each MSIX info entry in hardware first */
+	for (i = 0; i < intr_num; i++) {
+		hw_ops->cfg_msix_info(res_mgt->hw_ops_tbl->priv,
+				      func, false, interrupts[i],
+				      0, 0, 0, false);
+	}
+	hw_ops->flush_write(res_mgt->hw_ops_tbl->priv);
+
+	/*
+	 * Stage 1: retain the DMA address, only clear the VALID bit.
+	 * Stage 2 runs after the quiesce window in the complete phase.
+	 */
+	hw_ops->cfg_msix_map(res_mgt->hw_ops_tbl->priv, func,
+			     false, func_res->msix_map_table.dma,
+			     0, 0, 0);
+	hw_ops->flush_write(res_mgt->hw_ops_tbl->priv);
+
+	func_res->state = NBL_INTR_FUNC_DESTROYING;
+
+	return 0;
+}
+
+/*
+ * __nbl_res_intr_complete_destroy_msix_map - finish hardware teardown and
+ * release vector bitmap, DMA memory and interrupt buffer after the
+ * hardware quiesce window has elapsed.
+ *
+ * Caller must hold intr_mgt->lock.
+ */
+static int
+__nbl_res_intr_complete_destroy_msix_map(struct nbl_interrupt_mgt *intr_mgt,
+					 struct nbl_resource_mgt *res_mgt,
+					 u16 func_id)
+{
+	struct nbl_hw_ops *hw_ops = res_mgt->hw_ops_tbl->ops;
+	struct nbl_func_interrupt_resource_mng *func_res;
+	struct nbl_msix_map_table *msix_map_table;
+	struct device *dev = res_mgt->common->dev;
+	u16 *interrupts;
+	u16 intr_num;
+
+	lockdep_assert_held(&intr_mgt->lock);
+
+	if (func_id >= NBL_MAX_PF) {
+		dev_err(dev, "Invalid func_id %u\n", func_id);
+		return -EINVAL;
+	}
+
+	func_res = &intr_mgt->func_intr_res[func_id];
+	if (func_res->state != NBL_INTR_FUNC_DESTROYING)
+		return 0;
+
+	/*
+	 * Stage 2: the quiesce window has elapsed, it is now safe to
+	 * zero the DMA base address in the hardware map register.
+	 */
+	hw_ops->cfg_msix_map(res_mgt->hw_ops_tbl->priv, func_id,
+			     false, 0, 0, 0, 0);
+	hw_ops->flush_write(res_mgt->hw_ops_tbl->priv);
+
+	interrupts = func_res->interrupts;
+	intr_num = func_res->num_interrupts;
+	msix_map_table = &func_res->msix_map_table;
+
+	if (interrupts) {
+		nbl_intr_release_bitmap(intr_mgt, res_mgt, interrupts,
+					intr_num);
+		kfree(interrupts);
+	}
+
+	/*
+	 * Release the coherent table independently of interrupts so a
+	 * partially built config (table allocated, vectors never
+	 * published) cannot leak coherent DMA memory.
+	 */
+	if (msix_map_table->base_addr) {
+		dma_free_coherent(dev, msix_map_table->size,
+				  msix_map_table->base_addr,
+				  msix_map_table->dma);
+		msix_map_table->base_addr = NULL;
+		msix_map_table->dma = 0;
+		msix_map_table->size = 0;
+	}
+
+	func_res->interrupts = NULL;
+	func_res->num_interrupts = 0;
+	func_res->num_net_interrupts = 0;
+	func_res->state = NBL_INTR_FUNC_IDLE;
+
+	return 0;
+}
+
+/*
+ * Internal (unlocked) MSI-X map teardown.  Caller must hold
+ * intr_mgt->lock for the whole sequence, including the hardware-DMA
+ * quiesce window: dropping the lock would let a concurrent caller (or
+ * nbl_intr_mgt_stop()) install/free state against this teardown.
+ *
+ * This is used for the single function synchronous destroy path.
+ */
+static int __nbl_res_intr_destroy_msix_map(struct nbl_interrupt_mgt *intr_mgt,
+					   struct nbl_resource_mgt *res_mgt,
+					   u16 func_id)
+{
+	int ret;
+
+	lockdep_assert_held(&intr_mgt->lock);
+
+	if (intr_mgt->stopping)
+		return -ESHUTDOWN;
+
+	ret = __nbl_res_intr_prepare_destroy_msix_map(intr_mgt, res_mgt,
+						      func_id);
+	if (ret)
+		return ret;
+	/*
+	 * prepare() only transitions CONFIGURED functions; an IDLE func
+	 * has nothing to wait for or complete.
+	 */
+	if (intr_mgt->func_intr_res[func_id].state !=
+	    NBL_INTR_FUNC_DESTROYING)
+		return 0;
+
+	nbl_intr_quiesce_wait();
+
+	return __nbl_res_intr_complete_destroy_msix_map(intr_mgt, res_mgt,
+							func_id);
+}
+
+int nbl_res_intr_destroy_msix_map(struct nbl_resource_mgt *res_mgt,
+				  u16 func_id)
+{
+	struct nbl_interrupt_mgt *intr_mgt = res_mgt->intr_mgt;
+	int ret;
+
+	if (!intr_mgt)
+		return -EINVAL;
+
+	mutex_lock(&intr_mgt->lock);
+	ret = __nbl_res_intr_destroy_msix_map(intr_mgt, res_mgt, func_id);
+	mutex_unlock(&intr_mgt->lock);
+
+	return ret;
+}
+
+/**
+ * nbl_res_intr_cfg_msix_map - allocate & program MSI-X mapping table
+ * @res_mgt: resource management instance
+ * @func_id: target function identifier
+ * @num_net_msix: required net data interrupt vectors
+ * @num_others_msix: required control interrupt vectors
+ * @net_msix_mask_en: enable mask for net interrupt entries
+ *
+ * Allocate interrupt vectors; MSIX coherent DMA table is allocated once
+ * per function on first configuration, entries are rewritten while the
+ * map is invalidated on subsequent reconfigurations. No free/realloc of
+ * DMA table on vector count changes. This removes the DMA table
+ * free/realloc cycle. On reconfiguration the map VALID bit is cleared
+ * and the hardware-DMA quiesce window is observed (lock held) before old
+ * vectors are recycled and the table is rewritten.
+ *
+ * Reconfiguration transiently needs room for the whole new vector set on
+ * top of the old one, because the old set is only released after the new
+ * allocation has succeeded (a failed reconfiguration must leave the old
+ * configuration intact).  The bitmap pools are therefore sized for peak,
+ * not maximum, concurrent use: a reconfigure of the same or a smaller
+ * size still fails with -EAGAIN when the pool is nearly exhausted.  The
+ * in-tree caller (nbl_dev_init_msix_cnt()) asks for one non-net vector
+ * per PF, far below NBL_MAX_OTHER_INTERRUPT.
+ *
+ * Serialization: this function takes intr_mgt->lock internally to
+ * protect the global vector bitmaps and per-function state against
+ * concurrent callers.
+ *
+ * Return: 0 on success, negative errno on failure
+ */
+int nbl_res_intr_cfg_msix_map(struct nbl_resource_mgt *res_mgt,
+			      u16 func_id, u16 num_net_msix,
+			      u16 num_others_msix,
+			      bool net_msix_mask_en)
+{
+	struct nbl_interrupt_mgt *intr_mgt = res_mgt->intr_mgt;
+	struct nbl_hw_ops *hw_ops = res_mgt->hw_ops_tbl->ops;
+	struct nbl_common_info *common = res_mgt->common;
+	struct nbl_msix_map_table *official_tbl;
+	struct nbl_msix_map *msix_map_entries;
+	struct device *dev = common->dev;
+	u16 requested, intr_index;
+	u8 bus, devid, function;
+	bool entry_masked = false;
+	u16 *tmp_interrupts = NULL;
+	u16 allocated_cnt = 0;
+	u16 *old_interrupts;
+	u16 old_num;
+	bool had_config;
+	int ret = 0;
+	u16 gvec;
+	u16 i, j;
+
+	if (!intr_mgt)
+		return -EINVAL;
+
+	if (!common->has_ctrl)
+		return -EINVAL;
+
+	if (func_id >= NBL_MAX_PF) {
+		dev_err(dev, "Invalid func_id %u\n", func_id);
+		return -EINVAL;
+	}
+
+	if (num_net_msix == 0 && num_others_msix == 0) {
+		dev_err(dev, "MSI-X vector count cannot both be zero\n");
+		return -EINVAL;
+	}
+
+	if (num_net_msix > NBL_MSIX_MAP_TABLE_MAX_ENTRIES ||
+	    num_others_msix > NBL_MSIX_MAP_TABLE_MAX_ENTRIES) {
+		dev_err(dev, "MSI-X count out of limit: net=%u, others=%u\n",
+			num_net_msix, num_others_msix);
+		return -EINVAL;
+	}
+
+	if (check_add_overflow(num_net_msix, num_others_msix, &requested) ||
+	    requested > NBL_MSIX_MAP_TABLE_MAX_ENTRIES) {
+		dev_err(dev, "Total MSI-X vectors %u exceeds maximum %u\n",
+			requested, NBL_MSIX_MAP_TABLE_MAX_ENTRIES);
+		return -EINVAL;
+	}
+
+	ret = nbl_res_func_id_to_bdf(res_mgt, func_id, &bus, &devid, &function);
+	if (ret) {
+		if (ret == -EOPNOTSUPP)
+			dev_err(dev,
+				"MSI-X mapping: func_id=%u is not a PF present on this board\n",
+				func_id);
+		return ret;
+	}
+
+	mutex_lock(&intr_mgt->lock);
+	official_tbl = &intr_mgt->func_intr_res[func_id].msix_map_table;
+
+	/* Reject new configs once global teardown has started */
+	if (intr_mgt->stopping) {
+		ret = -ESHUTDOWN;
+		goto out_unlock;
+	}
+
+	had_config = intr_mgt->func_intr_res[func_id].state ==
+		     NBL_INTR_FUNC_CONFIGURED;
+
+	/*
+	 * Phase1: allocate global vector array first.
+	 * Allocate the fixed-size MSIX DMA table only ONCE for this function.
+	 */
+	tmp_interrupts = kcalloc(requested, sizeof(*tmp_interrupts),
+				 GFP_KERNEL);
+	if (!tmp_interrupts) {
+		ret = -ENOMEM;
+		goto out_unlock;
+	}
+	/* Allocate MSIX DMA table once per function */
+	if (!official_tbl->base_addr) {
+		official_tbl->size =
+			sizeof(struct nbl_msix_map) *
+			NBL_MSIX_MAP_TABLE_MAX_ENTRIES;
+		official_tbl->base_addr = dma_alloc_coherent(dev,
+							     official_tbl->size,
+							     &official_tbl->dma,
+							     GFP_KERNEL);
+		if (!official_tbl->base_addr) {
+			dev_err(dev, "Failed to allocate DMA memory for MSIX table\n");
+			ret = -ENOMEM;
+			goto release_vecs_unlock;
+		}
+	}
+
+	/* Allocate net interrupt vectors */
+	for (i = 0; i < num_net_msix; i++) {
+		intr_index = find_first_zero_bit(intr_mgt->intr_net_bmap,
+						 NBL_MAX_NET_INTERRUPT);
+		if (intr_index == NBL_MAX_NET_INTERRUPT) {
+			dev_err(dev, "No free net interrupt vectors left\n");
+			ret = -EAGAIN;
+			goto release_vecs_unlock;
+		}
+		tmp_interrupts[i] = intr_index + NBL_NET_INTR_BASE;
+		set_bit(intr_index, intr_mgt->intr_net_bmap);
+		allocated_cnt++;
+	}
+
+	/* Allocate other interrupt vectors */
+	for (; i < requested; i++) {
+		intr_index =
+			find_first_zero_bit(intr_mgt->intr_other_bmap,
+					    NBL_MAX_OTHER_INTERRUPT);
+		if (intr_index == NBL_MAX_OTHER_INTERRUPT) {
+			dev_err(dev, "No free control interrupt vectors left\n");
+			ret = -EAGAIN;
+			goto release_vecs_unlock;
+		}
+		tmp_interrupts[i] = intr_index;
+		set_bit(intr_index, intr_mgt->intr_other_bmap);
+		allocated_cnt++;
+	}
+
+	/*
+	 * Phase2: quiesce the old hardware MSIX config before touching
+	 * the live DMA table. Same sequence as destroy:
+	 *   disable mailbox routing -> invalidate per-vector INFO ->
+	 *   clear map VALID -> flush -> wait for in-flight table fetches.
+	 * The lock stays held across the wait, so no concurrent caller
+	 * can program the quiesced function. Only then are old vectors
+	 * recycled.
+	 * NOTE: NO DMA table free here.
+	 */
+	if (had_config) {
+		old_interrupts =
+			intr_mgt->func_intr_res[func_id].interrupts;
+		old_num = intr_mgt->func_intr_res[func_id].num_interrupts;
+
+		ret = __nbl_res_intr_set_mailbox_irq(intr_mgt, res_mgt, func_id,
+						     0, false);
+		if (ret) {
+			dev_err(dev, "%s: disable old mailbox irq failed, keep old config\n",
+				__func__);
+			goto release_vecs_unlock;
+		}
+		for (j = 0; j < old_num; j++) {
+			hw_ops->cfg_msix_info(res_mgt->hw_ops_tbl->priv,
+					      func_id, false,
+					      old_interrupts[j],
+					      0, 0, 0, false);
+		}
+		hw_ops->cfg_msix_map(res_mgt->hw_ops_tbl->priv, func_id,
+				     false, official_tbl->dma,
+				     0, 0, 0);
+		hw_ops->flush_write(res_mgt->hw_ops_tbl->priv);
+
+		nbl_intr_quiesce_wait();
+
+		nbl_intr_release_bitmap(intr_mgt, res_mgt, old_interrupts,
+					old_num);
+		kfree(old_interrupts);
+		intr_mgt->func_intr_res[func_id].interrupts = NULL;
+		intr_mgt->func_intr_res[func_id].num_interrupts = 0;
+		intr_mgt->func_intr_res[func_id].num_net_interrupts = 0;
+	}
+
+	/* Swap new vector array into func state */
+	intr_mgt->func_intr_res[func_id].interrupts = tmp_interrupts;
+	intr_mgt->func_intr_res[func_id].num_interrupts = requested;
+	intr_mgt->func_intr_res[func_id].num_net_interrupts = num_net_msix;
+	tmp_interrupts = NULL;
+
+	/*
+	 * First configuration of this function.  Re-invalidate the entry
+	 * and take a quiesce window before the table below is rewritten:
+	 * while the entry is VALID the pcompleter could fetch a table
+	 * that is being written and route stale gvecs.
+	 *
+	 * This is a second, real quiesce window rather than part of the
+	 * reconfiguration path, and it costs one delay per function on the
+	 * first configuration only.  Reusing the table removes the
+	 * free/realloc cycle and the destroy-path quiesce window from
+	 * every *re*configuration; it does not remove this one.
+	 */
+	if (!had_config) {
+		hw_ops->cfg_msix_map(res_mgt->hw_ops_tbl->priv, func_id,
+				     false, 0, 0, 0, 0);
+		hw_ops->flush_write(res_mgt->hw_ops_tbl->priv);
+		nbl_intr_quiesce_wait();
+	}
+
+	/*
+	 * Rewrite the table in the pre-allocated DMA buffer while the
+	 * map is invalid, so the device cannot observe a torn old/new
+	 * mix. Only entries beyond requested count need explicit zeroing.
+	 */
+	msix_map_entries = official_tbl->base_addr;
+	memset(msix_map_entries + requested, 0,
+	       (NBL_MSIX_MAP_TABLE_MAX_ENTRIES - requested) *
+	       sizeof(*msix_map_entries));
+
+	for (i = 0; i < requested; i++) {
+		gvec = intr_mgt->func_intr_res[func_id].interrupts[i];
+		msix_map_entries[i].data =
+			cpu_to_le16(FIELD_PREP(NBL_MSIX_MAP_VALID_MASK, 1) |
+				    FIELD_PREP(NBL_MSIX_MAP_INDEX_MASK,
+					       gvec));
+	}
+
+	/* Ensure coherent table writes are visible before HW fetch/enable */
+	dma_wmb();
+
+	/* Enable per-vector INFO entries after the table is published */
+	for (i = 0; i < requested; i++) {
+		gvec = intr_mgt->func_intr_res[func_id].interrupts[i];
+		entry_masked = (i < num_net_msix && net_msix_mask_en);
+		hw_ops->cfg_msix_info(res_mgt->hw_ops_tbl->priv,
+				      func_id, true, gvec,
+				      bus, devid, function,
+				      entry_masked);
+	}
+	hw_ops->flush_write(res_mgt->hw_ops_tbl->priv);
+
+	/*
+	 * Point the map at the table last and set VALID.
+	 *
+	 * cfg_msix_map uses the control PF's own BDF (common->hw_bus etc.),
+	 * not the target function's BDF.  This BDF tags the pcompler DMA
+	 * read of the MSI-X map table as originating from the control PF.
+	 * The target function's BDF (bus/devid/function from
+	 * nbl_res_func_id_to_bdf) is used only in cfg_msix_info for the
+	 * host_msix_ctrl table entry BDF filtering.
+	 */
+	hw_ops->cfg_msix_map(res_mgt->hw_ops_tbl->priv, func_id,
+			     true, official_tbl->dma, common->hw_bus,
+			     common->devid, common->function);
+	hw_ops->flush_write(res_mgt->hw_ops_tbl->priv);
+
+	intr_mgt->func_intr_res[func_id].state = NBL_INTR_FUNC_CONFIGURED;
+	mutex_unlock(&intr_mgt->lock);
+	return 0;
+
+release_vecs_unlock:
+	nbl_intr_release_bitmap(intr_mgt, res_mgt, tmp_interrupts,
+				allocated_cnt);
+	kfree(tmp_interrupts);
+	/*
+	 * On a failed fresh configuration, release the DMA table
+	 * allocated during this call. On a failed reconfiguration the
+	 * old configuration is still intact (it is only torn down
+	 * after all vector allocations succeed) and owns the table.
+	 */
+	if (!had_config && official_tbl->base_addr) {
+		dma_free_coherent(dev, official_tbl->size,
+				  official_tbl->base_addr,
+				  official_tbl->dma);
+		official_tbl->base_addr = NULL;
+		official_tbl->dma = 0;
+		official_tbl->size = 0;
+	}
+out_unlock:
+	mutex_unlock(&intr_mgt->lock);
+	return ret;
+}
+
+/**
+ * nbl_res_intr_set_mailbox_irq - bind mailbox IRQ to specified vector
+ * @res_mgt: resource management instance
+ * @func_id: target function identifier
+ * @vector_id: index inside local interrupt array
+ * @en_msix: enable/disable mailbox interrupt
+ *
+ * Serialization: takes intr_mgt->lock internally and passes the
+ * pointer it read (before taking the lock) down to the helpers, which
+ * must not re-read res_mgt->intr_mgt: nbl_intr_mgt_stop() clears that
+ * pointer while a caller may already be blocked on this mutex.
+ *
+ * Return: 0 on success, negative errno on parameter or state check
+ * failure.  The hardware op is void and cannot report failure.
+ */
+int nbl_res_intr_set_mailbox_irq(struct nbl_resource_mgt *res_mgt,
+				 u16 func_id, u16 vector_id,
+				 bool en_msix)
+{
+	struct nbl_interrupt_mgt *intr_mgt = res_mgt->intr_mgt;
+	struct nbl_common_info *common = res_mgt->common;
+	int ret;
+
+	if (!intr_mgt)
+		return -EINVAL;
+
+	if (!common->has_ctrl)
+		return -EINVAL;
+
+	mutex_lock(&intr_mgt->lock);
+	ret = __nbl_res_intr_set_mailbox_irq(intr_mgt, res_mgt, func_id,
+					     vector_id, en_msix);
+	mutex_unlock(&intr_mgt->lock);
+
+	return ret;
+}
+
+/*
+ * Reset the software state and invalidate every function's hardware MSI-X
+ * map entry once.  The chip-internal tables (FUNCTION_MSIX_MAP,
+ * PADPT_HOST_MSIX_INFO, HOST_MSIX_FID_TABLE) can survive a kexec or a
+ * forced unload without FLR and there is no .shutdown callback to scrub
+ * them.  Only FUNCTION_MSIX_MAP is scrubbed here: it is indexed by
+ * func_id, so covering all NBL_MAX_PF entries costs one register write
+ * each, after which the pcompleter can no longer fetch a stale table that
+ * points into memory this kernel may have reallocated.  The INFO/FID
+ * tables are indexed by global vector id and are not cleared; instead a
+ * map entry is always invalidated by nbl_res_intr_cfg_msix_map() and by
+ * the teardown paths before it is armed for a new configuration.
+ *
+ * No sibling PF is live at this point: the device links make them probe
+ * only after this PF and remove before it, so clearing all NBL_MAX_PF
+ * entries cannot disturb a running function.
+ */
+static struct nbl_interrupt_mgt *
+nbl_intr_setup_mgt(struct nbl_resource_mgt *res_mgt)
+{
+	struct device *dev = res_mgt->common->dev;
+	struct nbl_hw_ops *hw_ops = res_mgt->hw_ops_tbl->ops;
+	struct nbl_interrupt_mgt *intr_mgt;
+	u16 func_id;
+	int err;
+
+	intr_mgt = devm_kzalloc(dev, sizeof(*intr_mgt), GFP_KERNEL);
+	if (!intr_mgt)
+		return ERR_PTR(-ENOMEM);
+
+	err = devm_mutex_init(dev, &intr_mgt->lock);
+	if (err)
+		return ERR_PTR(err);
+
+	intr_mgt->stopping = false;
+	bitmap_zero(intr_mgt->intr_net_bmap, NBL_MAX_NET_INTERRUPT);
+	bitmap_zero(intr_mgt->intr_other_bmap, NBL_MAX_OTHER_INTERRUPT);
+
+	/*
+	 * Invalidate any map entry left VALID by a previous kernel.  One
+	 * flush plus one quiesce window covers the outstanding table
+	 * fetches, same as the teardown paths do.
+	 *
+	 * This is the scrub that makes it safe to enable bus mastering:
+	 * nbl_probe() calls it with bus mastering still disabled, so no
+	 * new fetch can start while these entries are still VALID.
+	 */
+	for (func_id = 0; func_id < NBL_MAX_PF; func_id++)
+		hw_ops->cfg_msix_map(res_mgt->hw_ops_tbl->priv, func_id,
+				     false, 0, 0, 0, 0);
+	hw_ops->flush_write(res_mgt->hw_ops_tbl->priv);
+	nbl_intr_quiesce_wait();
+
+	return intr_mgt;
+}
+
+int nbl_intr_mgt_start(struct nbl_resource_mgt *res_mgt)
+{
+	struct nbl_interrupt_mgt *intr_mgt;
+	int ret;
+
+	intr_mgt = nbl_intr_setup_mgt(res_mgt);
+	if (IS_ERR(intr_mgt)) {
+		ret = PTR_ERR(intr_mgt);
+		return ret;
+	}
+
+	res_mgt->intr_mgt = intr_mgt;
+	return 0;
+}
+
+/*
+ * nbl_intr_mgt_stop - global control-PF interrupt teardown
+ *
+ * Phase 1 sets the stopping latch and invalidates every configured
+ * function's hardware map entry while holding the lock.  The lock is
+ * dropped for the global quiesce window, so a caller that races the
+ * window is rejected by the latch (checked under the lock), not by the
+ * lock being held.
+ *
+ * The latch is what rejects a caller that raced the window; it is not
+ * what makes the manager safe to touch.  No caller is in flight here by
+ * construction: nbl_remove() clears bus mastering and stops the device
+ * before reaching this point, and every sibling PF - the only remote
+ * source of these calls - is unbound first through its
+ * DL_FLAG_AUTOPROBE_CONSUMER device link.  A caller that did latch the
+ * pointer earlier and then gets in under the lock hits the latch
+ * instead:
+ *   - the config path, destroy_msix_map and the enable path of
+ *     nbl_res_intr_set_mailbox_irq() report -ESHUTDOWN.  The latch is
+ *     checked ahead of that path's per-function state check, so its
+ *     -ENODEV is only reachable while the manager is not stopping.
+ *   - the en_msix=false disable path consults neither and returns 0,
+ *     which is what lets teardown disarm routing after the latch is set.
+ * Once this returns res_mgt->intr_mgt is NULL and the public entry
+ * points report -EINVAL.
+ */
+void nbl_intr_mgt_stop(struct nbl_resource_mgt *res_mgt)
+{
+	struct nbl_interrupt_mgt *intr_mgt = res_mgt->intr_mgt;
+	u16 func_id;
+	int ret;
+
+	if (!intr_mgt)
+		return;
+
+	/*
+	 * Phase 1: set the stopping latch and invalidate all hardware
+	 * MSIX map entries.  A caller racing the quiesce window below is
+	 * either still waiting for the lock, in which case it observes
+	 * stopping == true at its next checkpoint once it is let in, or
+	 * it already holds the lock, in which case this phase waits for
+	 * it to finish.  The latch is never cleared.
+	 *
+	 * The en_msix=false mailbox path is exempt from the latch on
+	 * purpose: this phase and the per-function teardown use it to
+	 * disarm routing, and it only rewrites bits that are already
+	 * clear by then.
+	 */
+	mutex_lock(&intr_mgt->lock);
+	intr_mgt->stopping = true;
+	for (func_id = 0; func_id < NBL_MAX_PF; func_id++) {
+		if (intr_mgt->func_intr_res[func_id].state ==
+		    NBL_INTR_FUNC_CONFIGURED) {
+			dev_info(res_mgt->common->dev,
+				 "intr_mgt_stop: preparing destroy map for func %u\n",
+				 func_id);
+			ret = __nbl_res_intr_prepare_destroy_msix_map(intr_mgt,
+								      res_mgt,
+								      func_id);
+			if (ret)
+				dev_warn(res_mgt->common->dev,
+					 "intr_mgt_stop: prepare destroy map for func %u failed: %d\n",
+					 func_id, ret);
+		}
+	}
+	mutex_unlock(&intr_mgt->lock);
+
+	/*
+	 * Global quiesce: wait for straggler DMA table reads after all
+	 * MSIX map entries have been invalidated in hardware, before
+	 * freeing coherent memory.  Best-effort only, see
+	 * nbl_intr_quiesce_wait().
+	 */
+	nbl_intr_quiesce_wait();
+
+	/* Phase2: safely release MSIX coherent memory and intr resources */
+	mutex_lock(&intr_mgt->lock);
+	for (func_id = 0; func_id < NBL_MAX_PF; func_id++) {
+		if (intr_mgt->func_intr_res[func_id].state ==
+		    NBL_INTR_FUNC_DESTROYING) {
+			ret = __nbl_res_intr_complete_destroy_msix_map(intr_mgt,
+								       res_mgt,
+								       func_id);
+			if (ret)
+				dev_warn(res_mgt->common->dev,
+					 "intr_mgt_stop: complete destroy map for func %u failed: %d\n",
+					 func_id, ret);
+		}
+	}
+	/*
+	 * Unpublish last.  Readers do not synchronize on intr_mgt->lock,
+	 * so this only has to happen after every hardware access and
+	 * every free above; a caller that latched the pointer earlier is
+	 * rejected by the stopping latch.
+	 */
+	res_mgt->intr_mgt = NULL;
+	mutex_unlock(&intr_mgt->lock);
+}
diff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_interrupt.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_interrupt.h
new file mode 100644
index 00000000000000..9f66f5e19c984b
--- /dev/null
+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_interrupt.h
@@ -0,0 +1,21 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2026 Nebula Matrix Limited.
+ */
+
+#ifndef _NBL_INTERRUPT_H_
+#define _NBL_INTERRUPT_H_
+
+#include "nbl_resource.h"
+
+#define NBL_MSIX_MAP_TABLE_MAX_ENTRIES	1024
+int nbl_res_intr_destroy_msix_map(struct nbl_resource_mgt *res_mgt,
+				  u16 func_id);
+int nbl_res_intr_cfg_msix_map(struct nbl_resource_mgt *res_mgt,
+			      u16 func_id, u16 num_net_msix,
+			      u16 num_others_msix,
+			      bool net_msix_mask_en);
+int nbl_res_intr_set_mailbox_irq(struct nbl_resource_mgt *res_mgt,
+				 u16 func_id, u16 vector_id,
+				 bool en_msix);
+#endif
diff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_resource.c b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_resource.c
new file mode 100644
index 00000000000000..49c7d4412e38ad
--- /dev/null
+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_resource.c
@@ -0,0 +1,157 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) 2026 Nebula Matrix Limited.
+ */
+
+#include <linux/pci.h>
+#include "nbl_resource.h"
+
+int nbl_res_func_id_to_vsi_id(struct nbl_resource_mgt *res_mgt, u16 func_id,
+			      u16 type, u16 *vsi_id)
+{
+	struct nbl_vsi_info *vsi_info = res_mgt->resource_info->vsi_info;
+	enum nbl_vsi_serv_type dst_type = NBL_VSI_SERV_PF_DATA_TYPE;
+	struct nbl_common_info *common = res_mgt->common;
+	struct device *dev = res_mgt->common->dev;
+	int pfid = func_id;
+	u32 rel_pf_id;
+	int ret;
+
+	if (!common->has_ctrl || !vsi_id) {
+		dev_dbg(dev, "No control plane or null vsi output ptr\n");
+		return -EINVAL;
+	}
+	ret = nbl_common_func_id_to_rel_pf_id(common, pfid, &rel_pf_id);
+	if (ret)
+		return ret;
+	if (rel_pf_id >= vsi_info->num) {
+		dev_err(dev, "PF %d (diff=%u) exceeds vsi_info->num (%u)\n",
+			pfid, rel_pf_id, vsi_info->num);
+		return -EINVAL;
+	}
+
+	ret = nbl_res_pf_dev_vsi_type_to_hw_vsi_type(res_mgt, type, &dst_type);
+	if (ret) {
+		dev_err(dev, "Invalid vsi type %u func_id %u\n", type, func_id);
+		return ret;
+	}
+	*vsi_id = vsi_info->serv_info[rel_pf_id][dst_type].base_id;
+	return 0;
+}
+
+int nbl_res_vsi_id_to_pf_id(struct nbl_resource_mgt *res_mgt, u16 vsi_id)
+{
+	struct nbl_vsi_info *vsi_info = res_mgt->resource_info->vsi_info;
+	struct nbl_common_info *common = res_mgt->common;
+	struct device *dev = res_mgt->common->dev;
+	int j = NBL_VSI_SERV_PF_DATA_TYPE;
+	int pf_id, i;
+
+	if (!common->has_ctrl) {
+		dev_dbg(dev, "No control plane available\n");
+		return -EINVAL;
+	}
+	for (i = 0; i < vsi_info->num; i++) {
+		if (vsi_id >= vsi_info->serv_info[i][j].base_id &&
+		    (vsi_id < vsi_info->serv_info[i][j].base_id +
+					vsi_info->serv_info[i][j].num)) {
+			pf_id = i + common->mgt_pf;
+			if (pf_id >= NBL_MAX_PF) {
+				dev_err(dev, "PF ID overflow\n");
+				return -ERANGE;
+			}
+			return pf_id;
+		}
+	}
+
+	dev_dbg(dev, "VSI ID %u not found\n", vsi_id);
+	return -ENOENT;
+}
+
+int nbl_res_func_id_to_bdf(struct nbl_resource_mgt *res_mgt, u16 func_id,
+			   u8 *bus, u8 *dev, u8 *function)
+{
+	struct nbl_common_info *common = res_mgt->common;
+	struct nbl_sriov_info *sriov_info;
+	int pfid = func_id;
+	u8 pf_bus, devfn;
+	u32 rel_pf_id;
+	int ret;
+
+	if (!common->has_ctrl || !bus || !dev || !function)
+		return -EINVAL;
+	ret = nbl_common_func_id_to_rel_pf_id(common, pfid, &rel_pf_id);
+	if (ret)
+		return ret;
+	if (rel_pf_id >= common->max_pf) {
+		dev_err(common->dev,
+			"func_id=%u rel_pf_id=%u exceeds max_pf=%u, no such PF on this board\n",
+			pfid, rel_pf_id,
+			common->max_pf);
+		return -EOPNOTSUPP;
+	}
+	sriov_info = res_mgt->resource_info->sriov_info + rel_pf_id;
+	pf_bus = PCI_BUS_NUM(sriov_info->bdf);
+	devfn = sriov_info->bdf & 0xff;
+	*bus = pf_bus;
+	*dev = PCI_SLOT(devfn);
+	*function = PCI_FUNC(devfn);
+
+	return 0;
+}
+
+int nbl_res_get_eth_id(struct nbl_resource_mgt *res_mgt, u16 func_id,
+		       u16 vsi_id, u8 *eth_num, u8 *eth_id, u8 *logic_eth_id)
+{
+	struct nbl_eth_info *eth_info = res_mgt->resource_info->eth_info;
+	struct nbl_common_info *common = res_mgt->common;
+	struct device *dev = res_mgt->common->dev;
+	int pfid = func_id;
+	int rel_pf_id;
+	int abs_pf_id;
+
+	if (!common->has_ctrl || !eth_num || !eth_id || !logic_eth_id)
+		return -EINVAL;
+	abs_pf_id = nbl_res_vsi_id_to_pf_id(res_mgt, vsi_id);
+	if (abs_pf_id < 0) {
+		dev_err(dev, "Failed to get PF ID from VSI ID %u\n", vsi_id);
+		return -EINVAL;
+	}
+	if (abs_pf_id != pfid) {
+		dev_err(dev, "func_id %u does not match pf derived from vsi_id %u\n",
+			pfid, vsi_id);
+		return -EINVAL;
+	}
+	rel_pf_id = abs_pf_id - common->mgt_pf;
+
+	if (rel_pf_id >= eth_info->eth_num) {
+		dev_err(dev, "rel_pf_id %d out of range [0, %u)\n",
+			rel_pf_id, eth_info->eth_num);
+		return -ERANGE;
+	}
+
+	*eth_num = eth_info->eth_num;
+	*eth_id = eth_info->eth_id[rel_pf_id];
+	/*
+	 * Logical eth id equals the relative PF id: eth_info setup enforces
+	 * max_pf == eth_num and fills eth_id[] in ascending bitmap-bit
+	 * order, so the Nth PF owns the Nth logical port.
+	 */
+	*logic_eth_id = rel_pf_id;
+	return 0;
+}
+
+int nbl_res_pf_dev_vsi_type_to_hw_vsi_type(struct nbl_resource_mgt *res_mgt,
+					   u16 src_type,
+					   enum nbl_vsi_serv_type *dst_type)
+{
+	switch (src_type) {
+	case NBL_VSI_DATA:
+		*dst_type = NBL_VSI_SERV_PF_DATA_TYPE;
+		return 0;
+	default:
+		dev_err_once(res_mgt->common->dev,
+			     "Unsupported vsi src_type %u\n", src_type);
+		return -EINVAL;
+	}
+}
diff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_resource.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_resource.h
new file mode 100644
index 00000000000000..24cf49475bbd90
--- /dev/null
+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_hw/nbl_resource.h
@@ -0,0 +1,159 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2026 Nebula Matrix Limited.
+ */
+
+#ifndef _NBL_RESOURCE_H_
+#define _NBL_RESOURCE_H_
+
+#include <linux/types.h>
+
+#include "../nbl_include/nbl_include.h"
+#include "../nbl_include/nbl_def_channel.h"
+#include "../nbl_include/nbl_def_hw.h"
+#include "../nbl_include/nbl_def_resource.h"
+#include "../nbl_include/nbl_def_common.h"
+#include "../nbl_core.h"
+
+struct nbl_resource_mgt;
+
+/* --------- INTERRUPT ---------- */
+#define NBL_MAX_OTHER_INTERRUPT			1024
+#define NBL_MAX_NET_INTERRUPT			4096
+#define NBL_NET_INTR_BASE		NBL_MAX_OTHER_INTERRUPT
+
+#define NBL_MSIX_MAP_VALID_MASK		BIT(0)
+#define NBL_MSIX_MAP_INDEX_MASK		GENMASK(13, 1)
+#define NBL_MSIX_MAP_RSV_MASK		GENMASK(15, 14)
+
+struct nbl_msix_map {
+	__le16 data;
+};
+
+struct nbl_msix_map_table {
+	struct nbl_msix_map *base_addr;
+	dma_addr_t dma;
+	size_t size;
+};
+
+/*
+ * Per-function MSI-X resource state.
+ *
+ * IDLE       - nothing configured for this function.
+ * CONFIGURED - map valid in hardware, vectors owned by this function.
+ * DESTROYING - map invalidated, teardown in flight: neither the vector
+ *              bitmaps nor the coherent table may be released until the
+ *              hardware-DMA quiesce window has elapsed.
+ *
+ * The single-function destroy path holds intr_mgt->lock across that
+ * window.  nbl_intr_mgt_stop() cannot, because it quiesces every
+ * function at once: it drops the lock and relies on the stopping latch
+ * to keep a concurrent configuration from installing a map that the
+ * in-flight teardown would free.
+ */
+enum nbl_intr_func_state {
+	NBL_INTR_FUNC_IDLE = 0,
+	NBL_INTR_FUNC_CONFIGURED,
+	NBL_INTR_FUNC_DESTROYING,
+};
+
+struct nbl_func_interrupt_resource_mng {
+	u16 num_interrupts;
+	u16 num_net_interrupts;
+	u16 *interrupts;
+	struct nbl_msix_map_table msix_map_table;
+	u8 state; /* enum nbl_intr_func_state */
+};
+
+struct nbl_interrupt_mgt {
+	struct mutex lock; /* Protects the bitmaps + func_intr_res[] */
+	DECLARE_BITMAP(intr_net_bmap, NBL_MAX_NET_INTERRUPT);
+	DECLARE_BITMAP(intr_other_bmap, NBL_MAX_OTHER_INTERRUPT);
+	/* One-way latch: set on teardown, never cleared */
+	bool stopping;
+	/*
+	 * Indexed by the raw hardware function id.  Only PF entries are ever
+	 * used: the public entry points reject ids outside the PF range (VFs
+	 * included) with -EINVAL, and nbl_res_intr_cfg_msix_map() also
+	 * rejects a PF that is not present on this board.  The destroy and
+	 * mailbox paths only bound-check NBL_MAX_PF; they rely on their
+	 * callers passing either common->mgt_pf or an id that the config
+	 * path has already validated.
+	 */
+	struct nbl_func_interrupt_resource_mng func_intr_res[NBL_MAX_PF];
+};
+
+/* --------- INFO ---------- */
+struct nbl_sriov_info {
+	unsigned int bdf;
+};
+
+struct nbl_eth_info {
+	u8 eth_num;
+	u8 resv[3];
+	u8 eth_id[NBL_MAX_ETHERNET];
+};
+
+enum nbl_vsi_serv_type {
+	NBL_VSI_SERV_PF_DATA_TYPE,
+	NBL_VSI_SERV_MAX_TYPE,
+};
+
+struct nbl_vsi_serv_info {
+	u16 base_id;
+	u16 num;
+};
+
+struct nbl_vsi_info {
+	u16 num;
+	struct nbl_vsi_serv_info serv_info[NBL_MAX_ETHERNET]
+					  [NBL_VSI_SERV_MAX_TYPE];
+};
+
+struct nbl_resource_info {
+	struct nbl_sriov_info *sriov_info;
+	struct nbl_eth_info *eth_info;
+	struct nbl_vsi_info *vsi_info;
+	struct nbl_board_port_info board_info;
+};
+
+struct nbl_resource_mgt {
+	struct nbl_common_info *common;
+	struct nbl_resource_info *resource_info;
+	struct nbl_channel_ops_tbl *chan_ops_tbl;
+	struct nbl_hw_ops_tbl *hw_ops_tbl;
+	/*
+	 * Published interrupt manager, control PF only.  The public entry
+	 * points read this pointer without the manager lock and then pass
+	 * the value they read down to the internal helpers;
+	 * nbl_intr_mgt_stop() clears it under the manager lock as its last
+	 * step, after every hardware access and every free, so a caller
+	 * that reads it after that gets -EINVAL.
+	 *
+	 * The object is devres-owned and is released together with the
+	 * device rather than by nbl_intr_mgt_stop(), so a latched pointer
+	 * is never dereferenced into freed memory.  That is not what keeps
+	 * a concurrent caller safe, though: see nbl_intr_mgt_stop() for
+	 * the ordering that makes sure there is none.  A caller that
+	 * latched the pointer before it was cleared and then gets in under
+	 * the lock hits the stopping latch: the config path,
+	 * destroy_msix_map and the mailbox-enable path report -ESHUTDOWN,
+	 * while the mailbox-disable path (en_msix == false) returns 0
+	 * without consulting the latch.
+	 */
+	struct nbl_interrupt_mgt *intr_mgt;
+};
+
+int nbl_res_vsi_id_to_pf_id(struct nbl_resource_mgt *res_mgt, u16 vsi_id);
+int nbl_res_func_id_to_vsi_id(struct nbl_resource_mgt *res_mgt, u16 func_id,
+			      u16 type, u16 *vsi_id);
+int nbl_res_func_id_to_bdf(struct nbl_resource_mgt *res_mgt, u16 func_id,
+			   u8 *bus, u8 *dev, u8 *function);
+int nbl_res_get_eth_id(struct nbl_resource_mgt *res_mgt, u16 func_id,
+		       u16 vsi_id, u8 *eth_num, u8 *eth_id, u8 *logic_eth_id);
+int nbl_intr_mgt_start(struct nbl_resource_mgt *res_mgt);
+int nbl_res_pf_dev_vsi_type_to_hw_vsi_type(struct nbl_resource_mgt *res_mgt,
+					   u16 src_type,
+					   enum nbl_vsi_serv_type *dst_type);
+void nbl_intr_mgt_stop(struct nbl_resource_mgt *res_mgt);
+#endif
diff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_channel.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_channel.h
new file mode 100644
index 00000000000000..5aa16d4cd4321f
--- /dev/null
+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_channel.h
@@ -0,0 +1,175 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2026 Nebula Matrix Limited.
+ */
+
+#ifndef _NBL_DEF_CHANNEL_H_
+#define _NBL_DEF_CHANNEL_H_
+
+#include <linux/types.h>
+#include <linux/build_bug.h>
+
+struct nbl_channel_mgt;
+struct nbl_adapter;
+
+typedef void (*nbl_chan_resp)(void *, u16, u16, void *, u32);
+
+/*
+ * Mailbox wire opcodes, stable wire ABI shared between driver and firmware.
+ * Each opcode has a fixed assigned number to preserve compatibility.
+ * ABI compatibility rules:
+ * 1. New opcodes shall only be appended before NBL_CHAN_MSG_MAILBOX_MAX;
+ * 2. Reordering, inserting or deleting existing enumerators breaks driver-
+ *    firmware interoperability and must be avoided;
+ * 3. Modifications to existing opcodes require synchronized firmware ABI
+ *    updates.
+ *
+ * Only opcodes currently used by in-tree driver logic are defined here.
+ * Unimplemented feature opcodes (KTLS, IPsec, vDPA, mirror etc.) will be
+ * added incrementally together with their corresponding driver
+ * implementation patches.
+ */
+enum nbl_chan_msg_type {
+	NBL_CHAN_MSG_ACK = 0,
+	NBL_CHAN_MSG_CONFIGURE_MSIX_MAP = 17,
+	NBL_CHAN_MSG_DESTROY_MSIX_MAP = 18,
+	NBL_CHAN_MSG_MAILBOX_SET_IRQ = 19,
+	NBL_CHAN_MSG_GET_VSI_ID = 21,
+	NBL_CHAN_MSG_GET_ETH_ID = 67,
+	/* mailbox msg end */
+	NBL_CHAN_MSG_MAILBOX_MAX,
+};
+
+enum nbl_chan_state {
+	NBL_CHAN_IRQ_RDY,
+	NBL_CHAN_STATE_NBITS
+};
+
+struct nbl_chan_param_cfg_msix_map {
+	__le16 num_net_msix;
+	__le16 num_others_msix;
+	__le16 msix_mask_en;
+	__le16 rsvd;
+};
+
+struct nbl_chan_param_set_mailbox_irq {
+	__le16 vector_id;
+	u8 en_msix;
+	u8 rsvd;
+};
+
+struct nbl_chan_param_get_vsi_id {
+	__le16 vsi_id;
+	__le16 type;
+};
+
+struct nbl_chan_param_get_eth_id {
+	__le16 vsi_id;
+	u8 eth_num;
+	u8 eth_id;
+	u8 logic_eth_id;
+	u8 rsvd[3];
+};
+
+struct nbl_board_port_info {
+	u8 eth_num;
+	u8 eth_speed;
+	u8 p4_version;
+	u8 rsv[5];
+};
+
+static_assert(sizeof(struct nbl_chan_param_cfg_msix_map) == 8,
+	      "nbl_chan_param_cfg_msix_map size must be 8 bytes");
+static_assert(sizeof(struct nbl_chan_param_set_mailbox_irq) == 4,
+	      "nbl_chan_param_set_mailbox_irq size must be 4 bytes");
+static_assert(sizeof(struct nbl_chan_param_get_vsi_id) == 4,
+	      "nbl_chan_param_get_vsi_id size must be 4 bytes");
+static_assert(sizeof(struct nbl_chan_param_get_eth_id) == 8,
+	      "nbl_chan_param_get_eth_id size must be 8 bytes");
+static_assert(sizeof(struct nbl_board_port_info) == 8,
+	      "nbl_board_port_info size must be 8 bytes");
+
+struct nbl_chan_send_info {
+	void *arg;
+	size_t arg_len;
+	void *resp;
+	size_t resp_len;
+	u16 dstid;
+	u16 msg_type;
+	u16 ack;
+	u16 ack_len;
+};
+
+struct nbl_chan_ack_info {
+	void *data;
+	int err;
+	u32 data_len;
+	u16 dstid;
+	u16 msg_type;
+	u16 msgid;
+};
+
+enum nbl_channel_type {
+	NBL_CHAN_TYPE_MAILBOX,
+	NBL_CHAN_TYPE_MAX
+};
+
+static inline void
+nbl_chan_fill_send_info(struct nbl_chan_send_info *info,
+			u16 dst_id, u16 msg_type,
+			void *argument, u32 arg_length,
+			void *response, u32 resp_length,
+			bool need_ack)
+{
+	info->dstid = dst_id;
+	info->msg_type = msg_type;
+	info->arg = argument;
+	info->arg_len = arg_length;
+	info->resp = response;
+	info->resp_len = resp_length;
+	info->ack = need_ack;
+}
+
+static inline void
+nbl_chan_fill_ack_info(struct nbl_chan_ack_info *info,
+		       u16 dst_id, u16 msg_type, u16 msg_id,
+		       int err_code, void *ack_data, u32 data_length)
+{
+	info->dstid = dst_id;
+	info->msg_type = msg_type;
+	info->msgid = msg_id;
+	info->err = err_code;
+	info->data = ack_data;
+	info->data_len = data_length;
+}
+
+struct nbl_channel_ops {
+	int (*send_msg)(struct nbl_channel_mgt *chan_mgt,
+			struct nbl_chan_send_info *chan_send);
+	int (*send_ack)(struct nbl_channel_mgt *chan_mgt,
+			struct nbl_chan_ack_info *chan_ack);
+	int (*register_msg)(struct nbl_channel_mgt *chan_mgt, u16 msg_type,
+			    nbl_chan_resp func, void *callback_priv);
+	void (*cfg_chan_qinfo_map_table)(struct nbl_channel_mgt *chan_mgt,
+					 u8 bus, u8 devid);
+	bool (*check_queue_exist)(struct nbl_channel_mgt *chan_mgt,
+				  u8 chan_type);
+	int (*setup_queue)(struct nbl_channel_mgt *chan_mgt, u8 chan_type);
+	int (*teardown_queue)(struct nbl_channel_mgt *chan_mgt, u8 chan_type);
+	void (*clean_queue_subtask)(struct nbl_channel_mgt *chan_mgt,
+				    u8 chan_type);
+	void (*register_chan_task)(struct nbl_channel_mgt *chan_mgt,
+				   u8 chan_type, struct work_struct *task);
+	void (*set_queue_state)(struct nbl_channel_mgt *chan_mgt,
+				enum nbl_chan_state state, u8 chan_type,
+				u8 set);
+};
+
+struct nbl_channel_ops_tbl {
+	struct nbl_channel_ops *ops;
+	struct nbl_channel_mgt *priv;
+};
+
+int nbl_chan_init_common(struct nbl_adapter *adapter);
+void nbl_chan_remove_common(struct nbl_adapter *adapter);
+#endif
diff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_common.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_common.h
index ea646d1efc2dfb..3e18a2a97850e6 100644
--- a/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_common.h
+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_common.h
@@ -11,7 +11,24 @@
 #include <linux/device.h>
 #include "nbl_include.h"
 
+#define NBL_TWO_ETHERNET_PORT			2
+#define NBL_FOUR_ETHERNET_PORT			4
+#define NBL_DEFAULT_VSI_ID_GAP			1024
+#define NBL_TWO_ETHERNET_VSI_ID_GAP		512
+#define NBL_FOUR_ETHERNET_VSI_ID_GAP		256
+
+static inline u32 nbl_vsi_id_gap(u32 m)
+{
+	if (m == NBL_FOUR_ETHERNET_PORT)
+		return NBL_FOUR_ETHERNET_VSI_ID_GAP;
+	else if (m == NBL_TWO_ETHERNET_PORT)
+		return NBL_TWO_ETHERNET_VSI_ID_GAP;
+
+	return NBL_DEFAULT_VSI_ID_GAP;
+}
+
 struct nbl_common_info {
+	struct workqueue_struct *wq;
 	struct pci_dev *pdev;
 	struct device *dev;
 	u16 vsi_id;
@@ -23,9 +40,15 @@ struct nbl_common_info {
 	u8 devid;
 	u8 bus;
 	u8 hw_bus;
+	u16 mgt_pf;
 
 	u8 has_ctrl;
 	u8 has_net;
+	u8 max_pf;
 };
 
+void nbl_common_destroy_wq(struct nbl_common_info *common);
+int nbl_common_create_wq(struct nbl_common_info *common);
+int nbl_common_func_id_to_rel_pf_id(struct nbl_common_info *common, u32 pf_id,
+				    u32 *rel_pf_id);
 #endif
diff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_dev.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_dev.h
new file mode 100644
index 00000000000000..a66c633a0e7af7
--- /dev/null
+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_dev.h
@@ -0,0 +1,16 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2026 Nebula Matrix Limited.
+ */
+
+#ifndef _NBL_DEF_DEV_H_
+#define _NBL_DEF_DEV_H_
+
+struct nbl_adapter;
+
+int nbl_dev_init(struct nbl_adapter *adapter);
+void nbl_dev_remove(struct nbl_adapter *adapter);
+int nbl_dev_start(struct nbl_adapter *adapter);
+void nbl_dev_stop(struct nbl_adapter *adapter);
+
+#endif
diff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_dispatch.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_dispatch.h
new file mode 100644
index 00000000000000..7da9e242c621e4
--- /dev/null
+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_dispatch.h
@@ -0,0 +1,83 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2026 Nebula Matrix Limited.
+ */
+
+#ifndef _NBL_DEF_DISPATCH_H_
+#define _NBL_DEF_DISPATCH_H_
+
+#include <linux/types.h>
+
+struct nbl_dispatch_mgt;
+struct nbl_adapter;
+enum {
+	NBL_DISP_CTRL_LVL_MGT,
+	NBL_DISP_CTRL_LVL_NET,
+	NBL_DISP_CTRL_LVL_MAX,
+};
+
+/**
+ * struct nbl_dispatch_ops - dispatch control plane operation callbacks
+ * @init_module: chip-wide datapath init, forwarded to the resource layer
+ *               (res_ops->init_module() and then hw_ops->init_module());
+ *               on leonis it programs the chip-wide datapath registers and
+ *               sets driver_status=true.  Returns -EOPNOTSUPP if the
+ *               resource layer installs no init_module op.  Control-PF
+ *               exclusive: the pointer is NULL on a function without
+ *               has_ctrl, so the caller must check has_ctrl first.
+ * @deinit_module: chip-wide teardown, forwarded the same way; on leonis
+ *               it clears driver_status, which starts an asynchronous
+ *               firmware per-PF cleanup pass with no completion status
+ *               bit, waited for best-effort only.  The caller must have
+ *               stopped the mailbox channel and drained all in-flight
+ *               mailbox DMA before calling it, see nbl_dev_remove().
+ *               Returns void.  Control-PF exclusive, same NULL rule as
+ *               @init_module.
+ * @cfg_msix_map: configure function msix mapping table
+ * @destroy_msix_map: tear down msix mapping resource
+ * @set_mailbox_irq: bind mailbox interrupt to specified msix vector
+ * @get_vsi_id: resolve VSI ID by type
+ * @get_eth_id: resolve eth port info from VSI ID
+ *
+ * These callbacks are not dispatch-layer setup/teardown: the dispatch
+ * structures are devm-allocated in nbl_disp_init(), so nbl_disp_remove()
+ * has nothing to undo. The five responders registered by
+ * nbl_disp_setup_msg() are owned by the channel layer and go away with
+ * it, so they need no explicit unregistration either; the teardown order
+ * that makes that safe is documented with the code that performs it, in
+ * the next patch in this series. They forward chip/firmware control down
+ * to the layers below instead: a control-PF function calls the resource
+ * ops directly, a non-control function sends a mailbox request over the
+ * channel layer.
+ *
+ * Warning: init_module/deinit_module are control-PF exclusive. The five
+ * resource ops (cfg_msix_map, destroy_msix_map, set_mailbox_irq,
+ * get_vsi_id, get_eth_id) are PF-only and resolve to either a local
+ * resource call (control PF) or a mailbox RPC (non-control PF with
+ * has_net). A function with neither has_ctrl nor has_net leaves these
+ * pointers NULL; callers must not invoke them on such functions. VFs are
+ * rejected by the responders with -EPERM.
+ */
+struct nbl_dispatch_ops {
+	int (*init_module)(struct nbl_dispatch_mgt *disp_mgt);
+	void (*deinit_module)(struct nbl_dispatch_mgt *disp_mgt);
+	int (*cfg_msix_map)(struct nbl_dispatch_mgt *disp_mgt,
+			    u16 num_net_msix, u16 num_others_msix,
+			    bool net_msix_mask_en);
+	int (*destroy_msix_map)(struct nbl_dispatch_mgt *disp_mgt);
+	int (*set_mailbox_irq)(struct nbl_dispatch_mgt *disp_mgt,
+			       u16 vector_id, bool en_msix);
+	int (*get_vsi_id)(struct nbl_dispatch_mgt *disp_mgt, u16 type,
+			  u16 *vsi_id);
+	int (*get_eth_id)(struct nbl_dispatch_mgt *disp_mgt, u16 vsi_id,
+			  u8 *eth_num, u8 *eth_id, u8 *logic_eth_id);
+};
+
+struct nbl_dispatch_ops_tbl {
+	struct nbl_dispatch_ops *ops;
+	struct nbl_dispatch_mgt *priv;
+};
+
+int nbl_disp_init(struct nbl_adapter *adapter);
+void nbl_disp_remove(struct nbl_adapter *adapter);
+#endif
diff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_hw.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_hw.h
index ecbf440e436667..e9cd3136252d86 100644
--- a/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_hw.h
+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_hw.h
@@ -8,8 +8,88 @@
 
 #include <linux/types.h>
 
+struct nbl_board_port_info;
 struct nbl_hw_mgt;
 struct nbl_adapter;
+struct nbl_hw_ops {
+	int (*init_module)(struct nbl_hw_mgt *hw_mgt, u8 eth_speed, u8 eth_num);
+	void (*deinit_module)(struct nbl_hw_mgt *hw_mgt);
+
+	void (*cfg_msix_map)(struct nbl_hw_mgt *hw_mgt, u16 func_id,
+			     bool valid, dma_addr_t dma_addr, u8 bus,
+			     u8 devid, u8 function);
+	void (*cfg_msix_info)(struct nbl_hw_mgt *hw_mgt, u16 func_id,
+			      bool valid, u16 interrupt_id, u8 bus,
+			      u8 devid, u8 function,
+			      bool net_msix_mask_en);
+	/**
+	 * flush_write - Flush posted MEMORY-BAR writes
+	 * @hw_mgt: hardware management context
+	 *
+	 * Only valid on the control PF (common->has_ctrl != 0).  The
+	 * register read back to order the writes, NBL_HW_DUMMY_REG
+	 * (0x1300904), is covered by the ~64 MiB MEMORY-BAR mapping the
+	 * control PF gets, but sits outside the NBL_REG_NET_ONLY_LEN
+	 * (8 KiB) mapping a net-only PF gets.  The leonis implementation
+	 * enforces this with a WARN_ON_ONCE().
+	 */
+	void (*flush_write)(struct nbl_hw_mgt *hw_mgt);
+	void (*update_mailbox_queue_tail_ptr)(struct nbl_hw_mgt *hw_mgt,
+					      u16 tail_ptr, u8 txrx);
+	void (*config_mailbox_rxq)(struct nbl_hw_mgt *hw_mgt,
+				   dma_addr_t dma_addr, int size_bwid);
+	void (*config_mailbox_txq)(struct nbl_hw_mgt *hw_mgt,
+				   dma_addr_t dma_addr, int size_bwid);
+	void (*stop_mailbox_rxq)(struct nbl_hw_mgt *hw_mgt);
+	void (*stop_mailbox_txq)(struct nbl_hw_mgt *hw_mgt);
+	/**
+	 * get_host_pf_mask - Fetch host PF mask from firmware k_pf_mask reg
+	 * @hw_mgt: hardware management context
+	 * @pf_mask: output pointer for PF mask value
+	 *
+	 * k_pf_mask register rule:
+	 *   bit N == 0 -> PF#N enabled; bit N == 1 -> PF#N masked out.
+	 *   bit0 is PF0's mask bit (not reserved); PF0 can be masked but
+	 *   the driver requires at least PF0 enabled.
+	 *   Only 1/2/4 PFs are supported:
+	 *     1 PF  (PF0):     mask = 0xfe
+	 *     2 PFs (PF0,PF1): mask = 0xfc
+	 *     4 PFs (PF0~PF3): mask = 0xf0
+	 *   All-zero mask (0x00) means all 8 PFs enabled, which is
+	 *   unsupported by the driver.
+	 *
+	 * This op only returns the raw register value and does not validate
+	 * it.  Neither consumer exists yet at the commit that introduces it:
+	 * the resource layer, which validates the value, and the channel
+	 * layer, which programs the mailbox QINFO table from it, both arrive
+	 * later in this series, and validation runs before the channel layer
+	 * reads the mask.
+	 */
+	void (*get_host_pf_mask)(struct nbl_hw_mgt *hw_mgt, u32 *pf_mask);
+	void (*get_real_bus)(struct nbl_hw_mgt *hw_mgt, u8 *bus);
+
+	void (*cfg_mailbox_qinfo)(struct nbl_hw_mgt *hw_mgt, u16 func_id,
+				  u8 bus, u8 devid, u8 function);
+	void (*set_mailbox_irq)(struct nbl_hw_mgt *hw_mgt, u16 func_id,
+				bool en_msix, u16 gvec);
+	void (*get_fw_eth_map)(struct nbl_hw_mgt *hw_mgt, u32 *eth_map);
+	/**
+	 * get_board_info - Fetch board info from firmware
+	 * @hw_mgt: hardware management context
+	 * @board_info: output pointer for board info structure
+	 *
+	 * Firmware contract: board_info.eth_num MUST equal the number of
+	 * unmasked PFs from get_host_pf_mask(). See get_host_pf_mask for
+	 * details.
+	 */
+	void (*get_board_info)(struct nbl_hw_mgt *hw_mgt,
+			       struct nbl_board_port_info *board_info);
+};
+
+struct nbl_hw_ops_tbl {
+	struct nbl_hw_ops *ops;
+	struct nbl_hw_mgt *priv;
+};
 
 int nbl_hw_init_leonis(struct nbl_adapter *adapter);
 void nbl_hw_remove_leonis(struct nbl_adapter *adapter);
diff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_resource.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_resource.h
new file mode 100644
index 00000000000000..8dc64e806c1e2b
--- /dev/null
+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_def_resource.h
@@ -0,0 +1,38 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2026 Nebula Matrix Limited.
+ */
+
+#ifndef _NBL_DEF_RESOURCE_H_
+#define _NBL_DEF_RESOURCE_H_
+
+#include <linux/types.h>
+
+struct nbl_resource_mgt;
+struct nbl_adapter;
+
+struct nbl_resource_ops {
+	int (*init_module)(struct nbl_resource_mgt *res_mgt);
+	void (*deinit_module)(struct nbl_resource_mgt *res_mgt);
+
+	int (*cfg_msix_map)(struct nbl_resource_mgt *res_mgt, u16 func_id,
+			    u16 num_net_msix, u16 num_others_msix,
+			    bool net_msix_mask_en);
+	int (*destroy_msix_map)(struct nbl_resource_mgt *res_mgt, u16 func_id);
+	int (*set_mailbox_irq)(struct nbl_resource_mgt *res_mgt, u16 func_id,
+			       u16 vector_id, bool en_msix);
+	int (*get_vsi_id)(struct nbl_resource_mgt *res_mgt, u16 func_id,
+			  u16 type, u16 *vsi_id);
+	int (*get_eth_id)(struct nbl_resource_mgt *res_mgt, u16 func_id,
+			  u16 vsi_id, u8 *eth_num, u8 *eth_id,
+			  u8 *logic_eth_id);
+};
+
+struct nbl_resource_ops_tbl {
+	struct nbl_resource_ops *ops;
+	struct nbl_resource_mgt *priv;
+};
+
+int nbl_res_init_leonis(struct nbl_adapter *adapter);
+void nbl_res_remove_leonis(struct nbl_adapter *adapter);
+#endif
diff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_include.h b/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_include.h
index 14e7b19f9a4cf8..efc4c79a082d25 100644
--- a/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_include.h
+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_include/nbl_include.h
@@ -10,6 +10,15 @@
 
 /*  ------  Basic definitions  -------  */
 #define NBL_DRIVER_NAME					"nbl"
+#define NBL_MAX_PF					8
+#define NBL_NEXT_ID(id, max) (((id) + 1) % ((max) + 1))
+
+#define NBL_MAX_ETHERNET				4
+
+enum {
+	NBL_VSI_DATA = 0,
+};
+
 struct nbl_func_caps {
 	u32 has_ctrl:1;
 	u32 has_net:1;
@@ -20,4 +29,23 @@ struct nbl_init_param {
 	struct nbl_func_caps caps;
 };
 
+/*
+ * Firmware ABI defines port speed enum fixed, value 0 represents 10G, cannot
+ * reassign 0 to INVALID for compatibility
+ */
+enum nbl_fw_port_speed {
+	NBL_FW_PORT_SPEED_10G,
+	NBL_FW_PORT_SPEED_25G,
+	NBL_FW_PORT_SPEED_50G,
+	NBL_FW_PORT_SPEED_100G,
+};
+
+/*
+ * Firmware quirk word @ NBL_LEONIS_QUIRKS_OFFSET (0x140)
+ * Sentinel value: ~0U (0xFFFFFFFF) = firmware reports no active quirks
+ * BIT(1): NBL_QUIRK_UVN_PREFETCH_ALIGN – control UVN descriptor prefetch
+ * selection
+ */
+#define NBL_QUIRK_UVN_PREFETCH_ALIGN	BIT(1)
+
 #endif
diff --git a/drivers/net/ethernet/nebula-matrix/nbl/nbl_main.c b/drivers/net/ethernet/nebula-matrix/nbl/nbl_main.c
index f2552bc7329376..0bdc7f9940fcb2 100644
--- a/drivers/net/ethernet/nebula-matrix/nbl/nbl_main.c
+++ b/drivers/net/ethernet/nebula-matrix/nbl/nbl_main.c
@@ -7,11 +7,34 @@
 #include <linux/pci.h>
 #include <linux/module.h>
 #include <linux/bits.h>
+#include <linux/delay.h>
 #include "nbl_include/nbl_include.h"
+#include "nbl_include/nbl_def_channel.h"
 #include "nbl_include/nbl_def_hw.h"
+#include "nbl_include/nbl_def_resource.h"
+#include "nbl_include/nbl_def_dispatch.h"
+#include "nbl_include/nbl_def_dev.h"
 #include "nbl_include/nbl_def_common.h"
 #include "nbl_core.h"
 
+/*
+ * Budget for the device_trylock() loop that takes func 0's device lock in
+ * nbl_probe_chip_deps().  See the comment there for why a single failed
+ * trylock must not defer the probe, and why the wait has to stay bounded.
+ */
+#define NBL_MGT_LOCK_RETRY_MIN_US	1000
+#define NBL_MGT_LOCK_RETRY_MAX_US	2000
+#define NBL_MGT_LOCK_MAX_RETRIES	50
+int nbl_core_start(struct nbl_adapter *adapter)
+{
+	return nbl_dev_start(adapter);
+}
+
+void nbl_core_stop(struct nbl_adapter *adapter)
+{
+	nbl_dev_stop(adapter);
+}
+
 struct nbl_adapter *nbl_core_init(struct pci_dev *pdev,
 				  struct nbl_init_param *param)
 {
@@ -26,6 +49,7 @@ struct nbl_adapter *nbl_core_init(struct pci_dev *pdev,
 	adapter->pdev = pdev;
 	common = &adapter->common;
 
+	common->mgt_pf = 0;
 	common->pdev = pdev;
 	common->dev = &pdev->dev;
 	common->has_ctrl = param->caps.has_ctrl;
@@ -36,15 +60,43 @@ struct nbl_adapter *nbl_core_init(struct pci_dev *pdev,
 
 	ret = nbl_hw_init_leonis(adapter);
 	if (ret)
-		goto hw_init_fail;
+		goto out;
+
+	ret = nbl_chan_init_common(adapter);
+	if (ret)
+		goto out_hw_remove;
 
+	ret = nbl_res_init_leonis(adapter);
+	if (ret)
+		goto out_chan_remove;
+
+	ret = nbl_disp_init(adapter);
+	if (ret)
+		goto out_res_remove;
+
+	ret = nbl_dev_init(adapter);
+	if (ret)
+		goto out_disp_remove;
 	return adapter;
-hw_init_fail:
+
+out_disp_remove:
+	nbl_disp_remove(adapter);
+out_res_remove:
+	nbl_res_remove_leonis(adapter);
+out_chan_remove:
+	nbl_chan_remove_common(adapter);
+out_hw_remove:
+	nbl_hw_remove_leonis(adapter);
+out:
 	return ERR_PTR(ret);
 }
 
 void nbl_core_remove(struct nbl_adapter *adapter)
 {
+	nbl_dev_remove(adapter);
+	nbl_disp_remove(adapter);
+	nbl_res_remove_leonis(adapter);
+	nbl_chan_remove_common(adapter);
 	nbl_hw_remove_leonis(adapter);
 }
 
@@ -67,6 +119,164 @@ static void nbl_get_func_param(struct pci_dev *pdev, kernel_ulong_t driver_data,
 		param->caps.has_ctrl = 1;
 }
 
+/*
+ * Establish chip-wide dependencies for this PF:
+ *  - for every non-management PF, add a consumer->management PF device
+ *    link. The driver core then guarantees (sysfs unbind, driver
+ *    unregister, hot-unplug alike) that this consumer is released
+ *    BEFORE the func 0 supplier, which is the only teardown order in
+ *    which the chip-global firmware deinit is safe.
+ *
+ * The management PF is addressed by the deterministic identity
+ * (domain, bus, slot, func 0) instead of any name-based scan: hardware
+ * guarantees PFs are contiguous from func 0 in the same slot.
+ *
+ * The func 0 device must already be bound to this driver.  While func 0
+ * is still probing, its chip-global init (mailbox QINFO map programming,
+ * datapath setup) can race a sibling's mailbox setup, and a link created
+ * while the supplier has no driver starts DORMANT and is force-promoted
+ * (with a WARN backtrace) by device_links_driver_bound() when this probe
+ * ends, so it never delivers the unbind-order guarantee.
+ *
+ * The link is a persistent managed link with DL_FLAG_AUTOPROBE_CONSUMER:
+ * the core force-unbinds this consumer when func 0 unbinds (keeping the
+ * teardown order) and re-queues it for probe when func 0 binds again, so
+ * the sibling PFs and their ports come back without a manual rebind.
+ * AUTOREMOVE_CONSUMER would delete the link on unbind and cannot be
+ * combined with AUTOPROBE_CONSUMER (device_link_add() returns NULL for
+ * that pair).  The link is therefore not auto-purged on a failed probe:
+ * it lives with the two devices and is dropped when either is removed.
+ * The pointer device_link_add() returns is never dereferenced.
+ *
+ * Return: 0 on success, negative errno on failure.
+ */
+static int nbl_probe_chip_deps(struct pci_dev *pdev, bool has_ctrl)
+{
+	struct pci_dev *mgt;
+	int retries;
+
+	if (has_ctrl)
+		return 0;
+
+	mgt = pci_get_domain_bus_and_slot(pci_domain_nr(pdev->bus),
+					  pdev->bus->number,
+					  PCI_DEVFN(PCI_SLOT(pdev->devfn), 0));
+	if (!mgt) {
+		dev_err(&pdev->dev,
+			"management PF (func 0) not found on this chip\n");
+		return -ENODEV;
+	}
+
+	/*
+	 * That identity is meaningless for a VF: under ARI a VF's devfn
+	 * decodes to function 0 of a slot, so the lookup can resolve to the
+	 * VF itself.  A VF is never the management PF, so stop here instead
+	 * of taking our own device lock below.
+	 */
+	if (mgt == pdev) {
+		dev_err(&pdev->dev,
+			"management PF lookup matched this device\n");
+		pci_dev_put(mgt);
+		return -ENODEV;
+	}
+
+	/*
+	 * func 0's device lock is taken with a trylock.  This probe already
+	 * runs under device_lock(funcN), so a blocking device_lock(func0)
+	 * would establish the N -> 0 order, while __pci_bus_lock() (AER
+	 * recovery, slot reset) walks the bus in devfn order, i.e. 0 -> N,
+	 * and dev->mutex is lockdep_set_novalidate_class(), so lockdep
+	 * cannot report that inversion.  A trylock never blocks, so it
+	 * cannot take part in a deadlock cycle.
+	 *
+	 * Holding the lock is what makes the check and the link creation
+	 * below atomic with respect to func 0: it both gains its driver
+	 * (driver_bound() -> device_links_driver_bound()) and loses it
+	 * (__device_release_driver() -> device_links_driver_cleanup())
+	 * under device_lock(func0), so func 0 can change neither driver nor
+	 * links.status while we hold it.
+	 *
+	 * One failed trylock is nevertheless not enough to defer on.  Func 0
+	 * binds with device_lock(func0) held, and driver_bound() calls
+	 * driver_deferred_probe_trigger() before it releases that lock, so
+	 * the deferred probe work it queues can re-probe this PF while func
+	 * 0's lock is still taken.  driver_probe_device() only re-arms that
+	 * work when deferred_trigger_count changes *during* the probe, which
+	 * it does not here: func 0's increment happened before this probe
+	 * read the count.  A bare -EPROBE_DEFER would therefore leave this
+	 * PF on deferred_probe_pending_list with nothing scheduled to run
+	 * it again.  Retrying closes that window - driver_bound() has only a
+	 * bus_notify() and a kobject_uevent() left before it drops the lock.
+	 *
+	 * The retry budget is what keeps this safe.  device_lock(funcN) is
+	 * held for the whole loop, so waiting indefinitely would deadlock
+	 * against __pci_bus_lock(); giving up and deferring releases it and
+	 * lets the other side make progress.  A holder that is not part of a
+	 * bind - __pci_bus_lock() during AER recovery or a bus/slot reset,
+	 * pci_reset_function() through sysfs, an sriov_numvfs write - can
+	 * outlast the budget, and no bind event follows those, so this PF
+	 * then stays deferred until some later deferred-probe trigger from
+	 * anywhere in the system.  There is no way to force one from here.
+	 */
+	for (retries = 0; !device_trylock(&mgt->dev); retries++) {
+		if (retries == NBL_MGT_LOCK_MAX_RETRIES) {
+			dev_dbg(&pdev->dev,
+				"management PF device lock busy after %u retries, deferring probe\n",
+				retries);
+			pci_dev_put(mgt);
+			return -EPROBE_DEFER;
+		}
+		usleep_range(NBL_MGT_LOCK_RETRY_MIN_US,
+			     NBL_MGT_LOCK_RETRY_MAX_US);
+	}
+
+	/*
+	 * The management PF must be bound to *this* driver, with its probe
+	 * completed.  links.status == DL_DEV_DRIVER_BOUND alone only says
+	 * "bound to some driver", so func 0 on vfio-pci or pci-stub
+	 * (driver_override) would pass it and this PF would run with func
+	 * 0's chip-global init never performed.  pdev->dev.driver is our
+	 * own driver, set by the core before ->probe() is called.
+	 *
+	 * Deferring is what recovers from this: a successful bind
+	 * re-triggers deferred probing for every pending device
+	 * (driver_bound() -> driver_deferred_probe_trigger()), so this PF is
+	 * retried once func 0 is bound to this driver.  That covers a func 0
+	 * that is not bound yet.  It does not cover a func 0 that is bound
+	 * while something else holds its device lock, which is what the
+	 * retry loop above is for and what its budget cannot fully rule out.
+	 */
+	if (mgt->dev.driver != pdev->dev.driver ||
+	    mgt->dev.links.status != DL_DEV_DRIVER_BOUND) {
+		device_unlock(&mgt->dev);
+		pci_dev_put(mgt);
+		return -EPROBE_DEFER;
+	}
+
+	/*
+	 * Create the link while func 0's lock is still held, so that func 0
+	 * cannot change state in between.  With the supplier bound and this
+	 * consumer PROBING, device_link_init_status() starts the link in
+	 * DL_STATE_CONSUMER_PROBE: that is the state
+	 * device_links_driver_bound() expects when this probe finishes, and
+	 * it is what device_links_busy() reports to func 0's own unbind, so
+	 * func 0 cannot be released underneath this PF.
+	 */
+	if (!device_link_add(&pdev->dev, &mgt->dev,
+			     DL_FLAG_AUTOPROBE_CONSUMER)) {
+		device_unlock(&mgt->dev);
+		dev_err(&pdev->dev,
+			"failed to create device link to management PF\n");
+		pci_dev_put(mgt);
+		return -ENOMEM;
+	}
+
+	device_unlock(&mgt->dev);
+	pci_dev_put(mgt);
+
+	return 0;
+}
+
 static int nbl_probe(struct pci_dev *pdev,
 		     const struct pci_device_id *id)
 {
@@ -75,28 +285,68 @@ static int nbl_probe(struct pci_dev *pdev,
 	struct nbl_adapter *adapter;
 	int err;
 
+	nbl_get_func_param(pdev, id->driver_data, &param);
+	/*
+	 * Reject a function with neither capability before touching any
+	 * hardware.  nbl_disp_refresh_ctrl_ops() leaves every dispatch op NULL
+	 * for such a function, while nbl_dev_start() reaches cfg_msix_map,
+	 * get_vsi_id, get_eth_id and (on unwind) destroy_msix_map through
+	 * those pointers.  The state is reachable through driver_override:
+	 * binding this driver to a function that is not in nbl_id_table (a VF,
+	 * or a non-func-0 PF) makes pci_match_device() pass the dummy ID whose
+	 * driver_data 0 leaves has_ctrl and has_net both clear.  A foreign
+	 * device at func 0 is unaffected: nbl_get_func_param() gives any
+	 * func 0 device has_ctrl, so it is not rejected here and still runs
+	 * nbl_core_init() with the local control dispatch ops, as it did
+	 * before this check existed.
+	 */
+	if (!param.caps.has_ctrl && !param.caps.has_net) {
+		dev_err(dev, "Function has neither ctrl nor net capability\n");
+		return -EOPNOTSUPP;
+	}
+
 	err = pcim_enable_device(pdev);
 	if (err) {
 		dev_err(&pdev->dev, "Failed to enable PCI dev, err=%d\n", err);
 		return err;
 	}
 
-	nbl_get_func_param(pdev, id->driver_data, &param);
 	/* never return fail when DMA_BIT_MASK(64) */
 	dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
 
-	pci_set_master(pdev);
+	/*
+	 * Bus mastering is enabled only after nbl_core_init() returns.
+	 *
+	 * FUNCTION_MSIX_MAP survives a kexec or a forced unload without
+	 * FLR and there is no .shutdown callback to scrub it, so at this
+	 * point every map entry left VALID by a previous kernel still
+	 * holds a DMA address that this kernel has already reallocated.
+	 * Those entries tag their table fetches as bus-master reads, and
+	 * the only thing standing between such a fetch and a read of
+	 * reused memory is Bus Master Enable, which the PCI core cleared
+	 * on the way into this kernel.  nbl_intr_setup_mgt() invalidates
+	 * the entries from inside nbl_core_init(); enabling bus mastering
+	 * before that would re-open the window.  Everything up to that
+	 * point is host-side setup plus MMIO and nothing in it waits for
+	 * the device, so it does not need bus mastering.
+	 */
+	err = nbl_probe_chip_deps(pdev, param.caps.has_ctrl);
+	if (err)
+		return err;
 
 	adapter = nbl_core_init(pdev, &param);
 	if (IS_ERR(adapter)) {
 		dev_err(dev, "Nbl adapter init fail: %pe\n", adapter);
-		err = PTR_ERR(adapter);
-		goto adapter_init_err;
+		return PTR_ERR(adapter);
 	}
+
+	pci_set_master(pdev);
 	pci_set_drvdata(pdev, adapter);
-	return 0;
-adapter_init_err:
-	pci_clear_master(pdev);
+	err = nbl_core_start(adapter);
+	if (err) {
+		nbl_core_remove(adapter);
+		pci_clear_master(pdev);
+	}
 	return err;
 }
 
@@ -107,9 +357,28 @@ static void nbl_remove(struct pci_dev *pdev)
 	if (!adapter)
 		return;
 	pci_set_drvdata(pdev, NULL);
-	nbl_core_remove(adapter);
 
+	nbl_core_stop(adapter);
+	/*
+	 * Stop bus mastering before any teardown runs.
+	 *
+	 * nbl_core_remove() releases the coherent MSI-X map tables whose
+	 * DMA addresses are still programmed in FUNCTION_MSIX_MAP, and the
+	 * only thing standing between a late pcompleter fetch and a read
+	 * of freed memory is a bounded best-effort delay - the chip
+	 * exposes no completion status for those fetches (see
+	 * nbl_intr_quiesce_wait()).  Clearing Bus Master Enable first
+	 * keeps the device from starting a new fetch at all, so that
+	 * residual window only covers requests already in flight.
+	 *
+	 * nbl_core_stop() above has already torn down this function's own
+	 * map, so what is left to protect here is the control PF's table of
+	 * per-function entries, freed from nbl_intr_mgt_stop() inside
+	 * nbl_core_remove().
+	 */
 	pci_clear_master(pdev);
+
+	nbl_core_remove(adapter);
 }
 
 /*