AI Trajectory

Trajectory: (Expand/Collapse All)
Seq	Timestamp	Type	Name	Duration
0/0	2026/07/22 17:52	flow	patch-triage	0m Args: null Results: { "EnableConfigs": [ "MICROSOFT_MANA", "MANA_INFINIBAND" ], "FocusSymbols": [ "mana_ib_install_cq_cb", "mana_ib_remove_cq_cb", "mana_hwc_get_msg_index", "mana_hwc_handle_resp", "mana_hwc_rx_event_handler", "mana_hwc_create_channel", "mana_hwc_destroy_channel", "mana_hwc_send_request", "mana_gd_process_eqe", "mana_gd_destroy_cq" ], "Reasoning": "The patch makes functional changes to the Microsoft Azure Network Adapter (MANA) driver, including adding RCU protection for the CQ table, handling HWC timeouts, and adding refcounts for caller contexts. These changes affect the core logic of the driver and should be fuzzed.", "WorthFuzzing": true }
1/1	2026/07/22 17:52	action	read-patch-diff	0m Args: null Results: { "PatchDiff": "commit 7a7f673c5a49005405d55fc2540f242d3b89cd96\nAuthor: syz-cluster \u003ctriage@syzkaller.com\u003e\nDate: Wed Jul 22 17:52:19 2026 +0000\n\n syz-cluster: applied patch under review\n\ndiff --git a/drivers/infiniband/hw/mana/cq.c b/drivers/infiniband/hw/mana/cq.c\nindex f2547989f42290..2bf4be21cede79 100644\n--- a/drivers/infiniband/hw/mana/cq.c\n+++ b/drivers/infiniband/hw/mana/cq.c\n@@ -131,12 +131,20 @@ static void mana_ib_cq_handler(void ctx, struct gdma_queue gdma_cq)\n int mana_ib_install_cq_cb(struct mana_ib_dev mdev, struct mana_ib_cq cq)\n {\n \tstruct gdma_context gc = mdev_to_gc(mdev);\n+\tstruct gdma_queue __rcu cq_table;\n \tstruct gdma_queue gdma_cq;\n \n-\tif (cq-\u003equeue.id \u003e= gc-\u003emax_num_cqs)\n+\t/* No rcu_read_lock(): install/remove run within the IB device\n+\t * lifetime, which mana_rdma_remove() (ib_unregister_device) drains\n+\t * before the base cq_table can be freed. See gdma_context::cq_table\n+\t * in gdma.h for why \"true\" is sound.\n+\t /\n+\tcq_table = rcu_dereference_protected(gc-\u003ecq_table, true);\n+\tif (!cq_table \|\| cq-\u003equeue.id \u003e= gc-\u003emax_num_cqs)\n \t\treturn -EINVAL;\n+\n \t/ Create CQ table entry, sharing a CQ between WQs is not supported /\n-\tif (gc-\u003ecq_table[cq-\u003equeue.id])\n+\tif (rcu_access_pointer(cq_table[cq-\u003equeue.id]))\n \t\treturn -EINVAL;\n \tif (cq-\u003equeue.kmem)\n \t\tgdma_cq = cq-\u003equeue.kmem;\n@@ -149,23 +157,49 @@ int mana_ib_install_cq_cb(struct mana_ib_dev mdev, struct mana_ib_cq cq)\n \tgdma_cq-\u003etype = GDMA_CQ;\n \tgdma_cq-\u003ecq.callback = mana_ib_cq_handler;\n \tgdma_cq-\u003eid = cq-\u003equeue.id;\n-\tgc-\u003ecq_table[cq-\u003equeue.id] = gdma_cq;\n+\trcu_assign_pointer(cq_table[cq-\u003equeue.id], gdma_cq);\n \treturn 0;\n }\n \n void mana_ib_remove_cq_cb(struct mana_ib_dev mdev, struct mana_ib_cq cq)\n {\n \tstruct gdma_context gc = mdev_to_gc(mdev);\n+\tstruct gdma_queue __rcu *cq_table;\n+\tstruct gdma_queue gdma_cq;\n \n-\tif (cq-\u003equeue.id \u003e= gc-\u003emax_num_cqs \|\| cq-\u003equeue.id == INVALID_QUEUE_ID)\n+\tif (cq-\u003equeue.id == INVALID_QUEUE_ID)\n \t\treturn;\n \n \tif (cq-\u003equeue.kmem)\n \t/* Then it will be cleaned and removed by the mana /\n \t\treturn;\n \n-\tkfree(gc-\u003ecq_table[cq-\u003equeue.id]);\n-\tgc-\u003ecq_table[cq-\u003equeue.id] = NULL;\n+\t/ No rcu_read_lock(): like mana_ib_install_cq_cb(), this runs within\n+\t * the IB device lifetime that mana_rdma_remove() drains before the\n+\t * base cq_table can be freed. See gdma_context::cq_table in gdma.h.\n+\t /\n+\tcq_table = rcu_dereference_protected(gc-\u003ecq_table, true);\n+\tif (!cq_table \|\| cq-\u003equeue.id \u003e= gc-\u003emax_num_cqs)\n+\t\treturn;\n+\t/ Removers for a given CQ are serialized by the IB core, so the slot\n+\t * is read and cleared without rcu_read_lock() or atomicity: a CQ is\n+\t * never torn down while a live QP references it (cq-\u003eusecnt), nor\n+\t * while the QP-create that installed the entry is still running (that\n+\t * create holds a reference on the CQ uobject across its error path,\n+\t * before usecnt is taken). Any double-remove is therefore sequential\n+\t * -- the later caller sees the NULL stored below and returns.\n+\t /\n+\tgdma_cq = rcu_dereference_protected(cq_table[cq-\u003equeue.id], true);\n+\tif (!gdma_cq)\n+\t\treturn; / already removed by a prior teardown path /\n+\n+\trcu_assign_pointer(cq_table[cq-\u003equeue.id], NULL);\n+\n+\t/ Wait for in-flight EQ handlers that may have loaded the old\n+\t * pointer via rcu_dereference() to finish before freeing.\n+\t /\n+\tsynchronize_rcu();\n+\tkfree(gdma_cq);\n }\n \n int mana_ib_arm_cq(struct ib_cq ibcq, enum ib_cq_notify_flags flags)\ndiff --git a/drivers/net/ethernet/microsoft/mana/gdma_main.c b/drivers/net/ethernet/microsoft/mana/gdma_main.c\nindex a38d4bb74621f9..17711ee37c8be3 100644\n--- a/drivers/net/ethernet/microsoft/mana/gdma_main.c\n+++ b/drivers/net/ethernet/microsoft/mana/gdma_main.c\n@@ -161,6 +161,8 @@ static int mana_gd_init_registers(struct pci_dev pdev)\n bool mana_need_log(struct gdma_context gc, int err)\n {\n \tstruct hw_channel_context hwc;\n+\tbool need_log = true;\n+\tunsigned long flags;\n \n \tif (err != -ETIMEDOUT)\n \t\treturn true;\n@@ -168,11 +170,13 @@ bool mana_need_log(struct gdma_context gc, int err)\n \tif (!gc)\n \t\treturn true;\n \n+\tspin_lock_irqsave(\u0026gc-\u003ehwc_lock, flags);\n \thwc = gc-\u003ehwc.driver_data;\n \tif (hwc \u0026\u0026 hwc-\u003ehwc_timeout == 0)\n-\t\treturn false;\n+\t\tneed_log = false;\n+\tspin_unlock_irqrestore(\u0026gc-\u003ehwc_lock, flags);\n \n-\treturn true;\n+\treturn need_log;\n }\n \n static int mana_gd_query_max_resources(struct pci_dev pdev)\n@@ -367,9 +371,27 @@ static int mana_gd_detect_devices(struct pci_dev pdev)\n int mana_gd_send_request(struct gdma_context gc, u32 req_len, const void req,\n \t\t\t u32 resp_len, void resp)\n {\n-\tstruct hw_channel_context hwc = gc-\u003ehwc.driver_data;\n+\tstruct hw_channel_context hwc;\n+\tunsigned long flags;\n+\tint err;\n+\n+\tspin_lock_irqsave(\u0026gc-\u003ehwc_lock, flags);\n+\thwc = gc-\u003ehwc.driver_data;\n+\tif (!hwc) {\n+\t\tspin_unlock_irqrestore(\u0026gc-\u003ehwc_lock, flags);\n+\t\treturn -ENODEV;\n+\t}\n+\thwc-\u003eactive_senders++;\n+\tspin_unlock_irqrestore(\u0026gc-\u003ehwc_lock, flags);\n+\n+\terr = mana_hwc_send_request(hwc, req_len, req, resp_len, resp);\n \n-\treturn mana_hwc_send_request(hwc, req_len, req, resp_len, resp);\n+\tspin_lock_irqsave(\u0026gc-\u003ehwc_lock, flags);\n+\tif (--hwc-\u003eactive_senders == 0)\n+\t\twake_up(\u0026gc-\u003ehwc_drain_waitq);\n+\tspin_unlock_irqrestore(\u0026gc-\u003ehwc_lock, flags);\n+\n+\treturn err;\n }\n EXPORT_SYMBOL_NS(mana_gd_send_request, \"NET_MANA\");\n \n@@ -622,6 +644,7 @@ static void mana_serv_reset(struct pci_dev pdev)\n {\n \tstruct gdma_context gc = pci_get_drvdata(pdev);\n \tstruct hw_channel_context hwc;\n+\tunsigned long flags;\n \tint ret;\n \n \tif (!gc) {\n@@ -631,14 +654,17 @@ static void mana_serv_reset(struct pci_dev pdev)\n \t\treturn;\n \t}\n \n+\tspin_lock_irqsave(\u0026gc-\u003ehwc_lock, flags);\n \thwc = gc-\u003ehwc.driver_data;\n \tif (!hwc) {\n+\t\tspin_unlock_irqrestore(\u0026gc-\u003ehwc_lock, flags);\n \t\tdev_err(\u0026pdev-\u003edev, \"MANA service: no HWC\\n\");\n \t\tgoto out;\n \t}\n \n \t/ HWC is not responding in this case, so don't wait /\n \thwc-\u003ehwc_timeout = 0;\n+\tspin_unlock_irqrestore(\u0026gc-\u003ehwc_lock, flags);\n \n \tdev_info(\u0026pdev-\u003edev, \"MANA reset cycle start\\n\");\n \n@@ -761,6 +787,7 @@ static void mana_gd_process_eqe(struct gdma_queue eq)\n \tunion gdma_eqe_info eqe_info;\n \tenum gdma_eqe_type type;\n \tstruct gdma_event event;\n+\tstruct gdma_queue __rcu *cq_table;\n \tstruct gdma_queue cq;\n \tstruct gdma_eqe eqe;\n \tu32 cq_id;\n@@ -772,10 +799,11 @@ static void mana_gd_process_eqe(struct gdma_queue eq)\n \tswitch (type) {\n \tcase GDMA_EQE_COMPLETION:\n \t\tcq_id = eqe-\u003edetails[0] \u0026 0xFFFFFF;\n-\t\tif (WARN_ON_ONCE(cq_id \u003e= gc-\u003emax_num_cqs))\n+\t\tcq_table = rcu_dereference(gc-\u003ecq_table);\n+\t\tif (WARN_ON_ONCE(cq_id \u003e= gc-\u003emax_num_cqs \|\| !cq_table))\n \t\t\tbreak;\n \n-\t\tcq = gc-\u003ecq_table[cq_id];\n+\t\tcq = rcu_dereference(cq_table[cq_id]);\n \t\tif (WARN_ON_ONCE(!cq \|\| cq-\u003etype != GDMA_CQ \|\| cq-\u003eid != cq_id))\n \t\t\tbreak;\n \n@@ -1082,15 +1110,28 @@ static void mana_gd_create_cq(const struct gdma_queue_spec spec,\n static void mana_gd_destroy_cq(struct gdma_context gc,\n \t\t\t struct gdma_queue queue)\n {\n+\tstruct gdma_queue __rcu cq_table;\n \tu32 id = queue-\u003eid;\n \n-\tif (id \u003e= gc-\u003emax_num_cqs)\n+\t/ No rcu_read_lock() here: mana_gd_destroy_cq() runs only on the\n+\t * CQ-destroy/teardown path, where the base cq_table is stable. See\n+\t * the lifecycle note on gdma_context::cq_table in gdma.h for why the\n+\t * \"true\" predicate is sound.\n+\t /\n+\tcq_table = rcu_dereference_protected(gc-\u003ecq_table, true);\n+\tif (!cq_table \|\| id \u003e= gc-\u003emax_num_cqs)\n \t\treturn;\n \n-\tif (!gc-\u003ecq_table[id])\n+\tif (!rcu_access_pointer(cq_table[id]))\n \t\treturn;\n \n-\tgc-\u003ecq_table[id] = NULL;\n+\trcu_assign_pointer(cq_table[id], NULL);\n+\n+\t/ Wait for in-flight EQ handlers that may have loaded the old\n+\t * pointer via rcu_dereference() to finish before the caller\n+\t * frees the CQ memory.\n+\t /\n+\tsynchronize_rcu();\n }\n \n int mana_gd_create_hwc_queue(struct gdma_dev gd,\n@@ -1185,6 +1226,16 @@ static int mana_gd_create_dma_region(struct gdma_dev gd,\n \tif (!MANA_PAGE_ALIGNED(gmi-\u003evirt_addr))\n \t\treturn -EINVAL;\n \n+\t/ No RCU needed: this runs only on the data-path queue-creation\n+\t * path (mana_gd_create_mana_eq/mana_gd_create_mana_wq_cq, called\n+\t * by mana_en under RTNL and by mana_ib RDMA verbs, or during\n+\t * init). Every teardown path — mana_gd_remove, mana_gd_suspend,\n+\t * and the HWC reset/service path (which goes through\n+\t * mana_gd_suspend) — drains those consumers via mana_rdma_remove()\n+\t * + mana_remove() before mana_hwc_destroy_channel() clears\n+\t * gc-\u003ehwc.driver_data, so no concurrent destroy can race with\n+\t * this dereference.\n+\t /\n \thwc = gc-\u003ehwc.driver_data;\n \treq_msg_size = struct_size(req, page_addr_list, num_page);\n \tif (req_msg_size \u003e hwc-\u003emax_req_msg_size)\n@@ -1374,7 +1425,17 @@ int mana_gd_verify_vf_version(struct pci_dev pdev)\n \tstruct hw_channel_context hwc;\n \tint err;\n \n+\t/ No RCU needed: this runs only inside mana_gd_setup, on the\n+\t * probe and resume paths. The PCI/PM core holds device_lock\n+\t * across .probe/.resume and .remove/.suspend, so setup cannot\n+\t * overlap teardown of the same device. The HWC reset/service\n+\t * path is additionally serialized by GC_IN_SERVICE and runs\n+\t * suspend (destroy) then resume (this) sequentially in one work\n+\t * item. driver_data was just set by mana_hwc_create_channel\n+\t * earlier in this same setup call, so it is live here.\n+\t /\n \thwc = gc-\u003ehwc.driver_data;\n+\n \tmana_gd_init_req_hdr(\u0026req.hdr, GDMA_VERIFY_VF_DRIVER_VERSION,\n \t\t\t sizeof(req), sizeof(resp));\n \n@@ -2364,6 +2425,7 @@ static int mana_gd_probe(struct pci_dev pdev, const struct pci_device_id ent)\n \n \tmutex_init(\u0026gc-\u003eeq_test_event_mutex);\n \tmutex_init(\u0026gc-\u003egic_mutex);\n+\tspin_lock_init(\u0026gc-\u003ehwc_lock);\n \tpci_set_drvdata(pdev, gc);\n \tgc-\u003ebar0_pa = pci_resource_start(pdev, 0);\n \tgc-\u003ebar0_size = pci_resource_len(pdev, 0);\ndiff --git a/drivers/net/ethernet/microsoft/mana/hw_channel.c b/drivers/net/ethernet/microsoft/mana/hw_channel.c\nindex e3c24d50dad07c..0481d40443424f 100644\n--- a/drivers/net/ethernet/microsoft/mana/hw_channel.c\n+++ b/drivers/net/ethernet/microsoft/mana/hw_channel.c\n@@ -4,27 +4,55 @@\n #include \u003cnet/mana/gdma.h\u003e\n #include \u003cnet/mana/mana.h\u003e\n #include \u003cnet/mana/hw_channel.h\u003e\n+#include \u003clinux/pci.h\u003e\n #include \u003clinux/vmalloc.h\u003e\n \n+/ Acquire a free message slot from the inflight bitmap. Returns\n+ * -ENODEV if the channel is torn down, or -ETIMEDOUT if a prior HWC\n+ * command has timed out (preserving the error code callers expect).\n+ /\n static int mana_hwc_get_msg_index(struct hw_channel_context hwc, u16 msg_id)\n {\n \tstruct gdma_resource r = \u0026hwc-\u003einflight_msg_res;\n \tunsigned long flags;\n \tu32 index;\n \n-\tdown(\u0026hwc-\u003esema);\n+\tfor (;;) {\n+\t\tspin_lock_irqsave(\u0026r-\u003elock, flags);\n \n-\tspin_lock_irqsave(\u0026r-\u003elock, flags);\n+\t\tif (!hwc-\u003echannel_up \|\| hwc-\u003ehwc_timed_out) {\n+\t\t\tspin_unlock_irqrestore(\u0026r-\u003elock, flags);\n+\t\t\treturn hwc-\u003echannel_up ? -ETIMEDOUT : -ENODEV;\n+\t\t}\n \n-\tindex = find_first_zero_bit(hwc-\u003einflight_msg_res.map,\n-\t\t\t\t hwc-\u003einflight_msg_res.size);\n+\t\tindex = find_first_zero_bit(r-\u003emap, r-\u003esize);\n+\t\tif (index \u003c r-\u003esize) {\n+\t\t\tstruct hwc_caller_ctx ctx;\n+\n+\t\t\tbitmap_set(r-\u003emap, index, 1);\n+\t\t\tctx = \u0026hwc-\u003ecaller_ctx[index];\n+\t\t\treinit_completion(\u0026ctx-\u003ecomp_event);\n+\t\t\trefcount_set(\u0026ctx-\u003erefcnt, 1);\n+\t\t\tctx-\u003emsg_id = index;\n+\t\t\tctx-\u003eerror = -EINPROGRESS;\n+\t\t\tspin_unlock_irqrestore(\u0026r-\u003elock, flags);\n+\t\t\tbreak;\n+\t\t}\n+\t\tspin_unlock_irqrestore(\u0026r-\u003elock, flags);\n \n-\tbitmap_set(hwc-\u003einflight_msg_res.map, index, 1);\n+\t\twait_event(hwc-\u003emsg_waitq,\n+\t\t\t !hwc-\u003echannel_up \|\|\n+\t\t\t hwc-\u003ehwc_timed_out \|\|\n+\t\t\t !bitmap_full(r-\u003emap, r-\u003esize));\n \n-\tspin_unlock_irqrestore(\u0026r-\u003elock, flags);\n+\t\tif (!hwc-\u003echannel_up)\n+\t\t\treturn -ENODEV;\n \n-\tmsg_id = index;\n+\t\tif (hwc-\u003ehwc_timed_out)\n+\t\t\treturn -ETIMEDOUT;\n+\t}\n \n+\tmsg_id = index;\n \treturn 0;\n }\n \n@@ -34,10 +62,17 @@ static void mana_hwc_put_msg_index(struct hw_channel_context hwc, u16 msg_id)\n \tunsigned long flags;\n \n \tspin_lock_irqsave(\u0026r-\u003elock, flags);\n-\tbitmap_clear(hwc-\u003einflight_msg_res.map, msg_id, 1);\n+\tbitmap_clear(r-\u003emap, msg_id, 1);\n \tspin_unlock_irqrestore(\u0026r-\u003elock, flags);\n \n-\tup(\u0026hwc-\u003esema);\n+\twake_up(\u0026hwc-\u003emsg_waitq);\n+}\n+\n+static void hwc_ctx_put(struct hw_channel_context hwc,\n+\t\t\tstruct hwc_caller_ctx ctx)\n+{\n+\tif (refcount_dec_and_test(\u0026ctx-\u003erefcnt))\n+\t\tmana_hwc_put_msg_index(hwc, ctx-\u003emsg_id);\n }\n \n static int mana_hwc_verify_resp_msg(const struct hwc_caller_ctx caller_ctx,\n@@ -83,6 +118,17 @@ static void mana_hwc_handle_resp(struct hw_channel_context hwc, u32 resp_len,\n \tstruct hwc_caller_ctx ctx;\n \tint err;\n \n+\t/ Validate msg_id is in range before using it to index bitmap\n+\t * and caller_ctx array. Malicious firmware could send\n+\t * out-of-range msg_id causing out-of-bounds access.\n+\t /\n+\tif (msg_id \u003e= hwc-\u003enum_inflight_msg) {\n+\t\tdev_err(hwc-\u003edev, \"hwc_rx: msg_id %u \u003e= max %u\\n\",\n+\t\t\tmsg_id, hwc-\u003enum_inflight_msg);\n+\t\tmana_hwc_post_rx_wqe(hwc-\u003erxq, rx_req);\n+\t\treturn;\n+\t}\n+\n \tif (!test_bit(msg_id, hwc-\u003einflight_msg_res.map)) {\n \t\tdev_err(hwc-\u003edev, \"hwc_rx: invalid msg_id = %u\\n\", msg_id);\n \t\tmana_hwc_post_rx_wqe(hwc-\u003erxq, rx_req);\n@@ -90,22 +136,44 @@ static void mana_hwc_handle_resp(struct hw_channel_context hwc, u32 resp_len,\n \t}\n \n \tctx = hwc-\u003ecaller_ctx + msg_id;\n-\terr = mana_hwc_verify_resp_msg(ctx, resp_msg, resp_len);\n-\tif (err)\n-\t\tgoto out;\n \n-\tctx-\u003estatus_code = resp_msg-\u003estatus;\n+\t/* Reject responses larger than the RX DMA buffer — the SGE\n+\t * limits what hardware can DMA, so an oversized resp_len\n+\t * indicates a firmware bug. Fail rather than silently\n+\t * truncating.\n+\t /\n+\tif (resp_len \u003e rx_req-\u003ebuf_len) {\n+\t\tdev_err(hwc-\u003edev, \"HWC RX: resp_len %u \u003e buf_len %u\\n\",\n+\t\t\tresp_len, rx_req-\u003ebuf_len);\n+\t\tresp_len = 0;\n+\t}\n \n-\tmemcpy(ctx-\u003eoutput_buf, resp_msg, resp_len);\n-out:\n-\tctx-\u003eerror = err;\n+\tspin_lock(\u0026ctx-\u003elock);\n+\n+\terr = mana_hwc_verify_resp_msg(ctx, resp_msg, resp_len);\n \n-\t/ Must post rx wqe before complete(), otherwise the next rx may\n-\t * hit no_wqe error.\n+\tif (!err \u0026\u0026 ctx-\u003eoutput_buf) {\n+\t\tctx-\u003estatus_code = resp_msg-\u003estatus;\n+\t\tmemcpy(ctx-\u003eoutput_buf, resp_msg, resp_len);\n+\t\tctx-\u003eerror = 0;\n+\t} else if (ctx-\u003eoutput_buf) {\n+\t\t/* Only overwrite error if the sender hasn't timed out\n+\t\t * or been force-completed by destroy. When output_buf\n+\t\t * is NULL, a terminal error (-ENODEV or timeout) has\n+\t\t * already been set — preserve it so the sender doesn't\n+\t\t * see a spurious success.\n+\t\t /\n+\t\tctx-\u003eerror = err;\n+\t}\n+\n+\t/ Post RX WQE before completing — the next response may arrive\n+\t * immediately and needs a posted buffer.\n \t /\n \tmana_hwc_post_rx_wqe(hwc-\u003erxq, rx_req);\n-\n \tcomplete(\u0026ctx-\u003ecomp_event);\n+\tspin_unlock(\u0026ctx-\u003elock);\n+\n+\thwc_ctx_put(hwc, ctx);\n }\n \n static void mana_hwc_init_event_handler(void ctx, struct gdma_queue q_self,\n@@ -146,7 +214,12 @@ static void mana_hwc_init_event_handler(void ctx, struct gdma_queue q_self,\n \t\t\tbreak;\n \n \t\tcase HWC_INIT_DATA_QUEUE_DEPTH:\n-\t\t\thwc-\u003ehwc_init_q_depth_max = (u16)val;\n+\t\t\t/ HWC_INIT_DATA_QUEUE_DEPTH is a 24-bit field. Keep\n+\t\t\t * the full device-reported value here; it is clamped\n+\t\t\t * and validated in mana_hwc_create_channel() rather\n+\t\t\t * than silently truncated to u16.\n+\t\t\t /\n+\t\t\thwc-\u003ehwc_init_q_depth_max = val;\n \t\t\tbreak;\n \n \t\tcase HWC_INIT_DATA_MAX_REQUEST:\n@@ -261,19 +334,69 @@ static void mana_hwc_rx_event_handler(void ctx, u32 gdma_rxq_id,\n \n \tsge = (struct gdma_sge )(wqe + 8 + dma_oob-\u003einline_oob_size_div4 4);\n \n-\t/* Select the RX work request for virtual address and for reposting. /\n-\trq_base_addr = hwc_rxq-\u003emsg_buf-\u003emem_info.dma_handle;\n-\trx_req_idx = (sge-\u003eaddress - rq_base_addr) / hwc-\u003emax_req_msg_size;\n+\t/ inline_oob_size_div4 is read from the WQE in device-accessible RQ\n+\t * memory, so a malicious host in a CVM (or buggy firmware) could set\n+\t * it to push the SGE past the RQ buffer. Bounds-check the SGE it\n+\t * locates before dereferencing sge-\u003eaddress below. A validly posted\n+\t * WQE keeps the SGE inside the ring (worst case ends exactly at the\n+\t * buffer boundary); reject anything that would read past it. The\n+\t * slot cannot be trusted here, so leak this RX WQE rather than repost\n+\t * the wrong one -- as in the SGE-address mismatch path below.\n+\t /\n+\tif ((u8 )(sge + 1) \u003e (u8 )rq-\u003equeue_mem_ptr + rq-\u003equeue_size) {\n+\t\tdev_err(hwc-\u003edev, \"HWC RX: SGE past RQ buffer, oob_div4=%u\\n\",\n+\t\t\tdma_oob-\u003einline_oob_size_div4);\n+\t\treturn;\n+\t}\n \n-\tif (rx_req_idx \u003e= hwc_rxq-\u003emsg_buf-\u003enum_reqs) {\n-\t\tdev_err(hwc-\u003edev, \"HWC RX: wrong rx_req_idx=%llu, num_reqs=%u\\n\",\n-\t\t\trx_req_idx, hwc_rxq-\u003emsg_buf-\u003enum_reqs);\n+\t/ Recover the originating RX slot from the SGE address. Of the three\n+\t * terms here only sge-\u003eaddress lives in device-accessible RQ memory;\n+\t * rq_base_addr and max_resp_msg_size are driver-private constants. An\n+\t * in-range but wrong/unaligned SGE (corrupted WQE, or a malicious host\n+\t * in a CVM) would otherwise truncate onto a neighbouring slot, letting\n+\t * us read a stale response that could complete the wrong, reused\n+\t * in-flight request. Require the index to be in range AND the address\n+\t * to exactly match the value the driver posted for that slot.\n+\t /\n+\trq_base_addr = hwc_rxq-\u003emsg_buf-\u003emem_info.dma_handle;\n+\trx_req_idx = (sge-\u003eaddress - rq_base_addr) / hwc-\u003emax_resp_msg_size;\n+\n+\tif (rx_req_idx \u003e= hwc_rxq-\u003equeue_depth) {\n+\t\t/ Cannot trust which WQE this is, so we cannot safely repost\n+\t\t * it; leak one RX WQE and bail. An out-of-range index means\n+\t\t * a corrupted SGE from hardware (or host tampering), an\n+\t\t * unrecoverable device error.\n+\t\t /\n+\t\tdev_err(hwc-\u003edev, \"HWC RX: SGE idx %llu out of range\\n\",\n+\t\t\trx_req_idx);\n \t\treturn;\n \t}\n \n \trx_req = \u0026hwc_rxq-\u003emsg_buf-\u003ereqs[rx_req_idx];\n+\tif (sge-\u003eaddress != (u64)rx_req-\u003ebuf_sge_addr) {\n+\t\t/ In-range index but the address does not match what the\n+\t\t * driver posted for that slot; the same unrecoverable case,\n+\t\t * so leak this WQE rather than repost the wrong one.\n+\t\t /\n+\t\tdev_err(hwc-\u003edev, \"HWC RX: invalid SGE address %llx (idx=%llu)\\n\",\n+\t\t\tsge-\u003eaddress, rx_req_idx);\n+\t\treturn;\n+\t}\n+\n \tresp = (struct gdma_resp_hdr )rx_req-\u003ebuf_va;\n \n+\t/* Validate resp_len covers the response header before reading\n+\t * hwc_msg_id. A short response leaves stale data from the\n+\t * previous buffer occupant, which could match a live slot and\n+\t * complete the wrong request.\n+\t /\n+\tif (rx_oob-\u003etx_oob_data_size \u003c sizeof(resp)) {\n+\t\tdev_err(hwc-\u003edev, \"HWC RX: short resp_len=%u\\n\",\n+\t\t\trx_oob-\u003etx_oob_data_size);\n+\t\tmana_hwc_post_rx_wqe(hwc_rxq, rx_req);\n+\t\treturn;\n+\t}\n+\n \t/* Read msg_id once from DMA buffer to prevent TOCTOU:\n \t * DMA memory is shared/unencrypted in CVMs - host can\n \t * modify it between reads.\n@@ -281,6 +404,7 @@ static void mana_hwc_rx_event_handler(void ctx, u32 gdma_rxq_id,\n \tmsg_id = READ_ONCE(resp-\u003eresponse.hwc_msg_id);\n \tif (msg_id \u003e= hwc-\u003enum_inflight_msg) {\n \t\tdev_err(hwc-\u003edev, \"HWC RX: wrong msg_id=%u\\n\", msg_id);\n+\t\tmana_hwc_post_rx_wqe(hwc_rxq, rx_req);\n \t\treturn;\n \t}\n \n@@ -384,14 +508,20 @@ static void mana_hwc_comp_event(void ctx, struct gdma_queue q_self)\n \n static void mana_hwc_destroy_cq(struct gdma_context gc, struct hwc_cq hwc_cq)\n {\n-\tkfree(hwc_cq-\u003ecomp_buf);\n-\n \tif (hwc_cq-\u003egdma_cq)\n \t\tmana_gd_destroy_queue(gc, hwc_cq-\u003egdma_cq);\n \n+\t/ comp_buf is reached only by mana_hwc_comp_event(), which the\n+\t * EQ handler invokes via cq_table[id]. The CQ destroy above\n+\t * already cleared that slot and ran synchronize_rcu(), so no\n+\t * handler can reach comp_buf once it returns. Destroying the EQ\n+\t * here additionally tears down the IRQ (defense in depth) before\n+\t * comp_buf and hwc_cq are freed below.\n+\t /\n \tif (hwc_cq-\u003egdma_eq)\n \t\tmana_gd_destroy_queue(gc, hwc_cq-\u003egdma_eq);\n \n+\tkfree(hwc_cq-\u003ecomp_buf);\n \tkfree(hwc_cq);\n }\n \n@@ -560,6 +690,7 @@ static int mana_hwc_create_wq(struct hw_channel_context hwc,\n \thwc_wq-\u003egdma_wq = queue;\n \thwc_wq-\u003equeue_depth = q_depth;\n \thwc_wq-\u003ehwc_cq = hwc_cq;\n+\tspin_lock_init(\u0026hwc_wq-\u003elock);\n \n \terr = mana_hwc_alloc_dma_buf(hwc, q_depth, max_msg_size,\n \t\t\t\t \u0026hwc_wq-\u003emsg_buf);\n@@ -577,7 +708,7 @@ static int mana_hwc_create_wq(struct hw_channel_context hwc,\n \treturn err;\n }\n \n-static int mana_hwc_post_tx_wqe(const struct hwc_wq hwc_txq,\n+static int mana_hwc_post_tx_wqe(struct hwc_wq hwc_txq,\n \t\t\t\tstruct hwc_work_request req,\n \t\t\t\tu32 dest_virt_rq_id, u32 dest_virt_rcq_id,\n \t\t\t\tbool dest_pf)\n@@ -616,7 +747,11 @@ static int mana_hwc_post_tx_wqe(const struct hwc_wq hwc_txq,\n \treq-\u003ewqe_req.inline_oob_data = tx_oob;\n \treq-\u003ewqe_req.client_data_unit = 0;\n \n+\t/ Serialize WQE posting — multiple senders may call concurrently. /\n+\tspin_lock(\u0026hwc_txq-\u003elock);\n \terr = mana_gd_post_and_ring(hwc_txq-\u003egdma_wq, \u0026req-\u003ewqe_req, NULL);\n+\tspin_unlock(\u0026hwc_txq-\u003elock);\n+\n \tif (err)\n \t\tdev_err(dev, \"Failed to post WQE on HWC SQ: %d\\n\", err);\n \treturn err;\n@@ -627,7 +762,7 @@ static int mana_hwc_init_inflight_msg(struct hw_channel_context hwc,\n {\n \tint err;\n \n-\tsema_init(\u0026hwc-\u003esema, num_msg);\n+\tinit_waitqueue_head(\u0026hwc-\u003emsg_waitq);\n \n \terr = mana_gd_alloc_res_map(num_msg, \u0026hwc-\u003einflight_msg_res);\n \tif (err)\n@@ -657,23 +792,40 @@ static int mana_hwc_test_channel(struct hw_channel_context hwc, u16 q_depth,\n \tif (!ctx)\n \t\treturn -ENOMEM;\n \n-\tfor (i = 0; i \u003c q_depth; ++i)\n+\tfor (i = 0; i \u003c q_depth; ++i) {\n+\t\tspin_lock_init(\u0026ctx[i].lock);\n \t\tinit_completion(\u0026ctx[i].comp_event);\n+\t}\n \n \thwc-\u003ecaller_ctx = ctx;\n \n-\treturn mana_gd_test_eq(gc, hwc-\u003ecq-\u003egdma_eq);\n+\t/ channel_up must be set before the test EQ request, because\n+\t * the request goes through mana_hwc_get_msg_index() which\n+\t * checks channel_up. caller_ctx is allocated above, so\n+\t * concurrent access to a NULL caller_ctx is not possible.\n+\t /\n+\thwc-\u003echannel_up = true;\n+\n+\terr = mana_gd_test_eq(gc, hwc-\u003ecq-\u003egdma_eq);\n+\tif (err)\n+\t\thwc-\u003echannel_up = false;\n+\n+\treturn err;\n }\n \n-static int mana_hwc_establish_channel(struct gdma_context gc, u16 q_depth,\n+static int mana_hwc_establish_channel(struct gdma_context gc, u32 q_depth,\n \t\t\t\t u32 max_req_msg_size,\n \t\t\t\t u32 max_resp_msg_size)\n {\n+\t/ No RCU needed: called only from mana_hwc_create_channel\n+\t * during init, before the channel is published to senders.\n+\t /\n \tstruct hw_channel_context hwc = gc-\u003ehwc.driver_data;\n \tstruct gdma_queue rq = hwc-\u003erxq-\u003egdma_wq;\n \tstruct gdma_queue sq = hwc-\u003etxq-\u003egdma_wq;\n \tstruct gdma_queue eq = hwc-\u003ecq-\u003egdma_eq;\n \tstruct gdma_queue cq = hwc-\u003ecq-\u003egdma_cq;\n+\tstruct gdma_queue __rcu *cq_table;\n \tint err;\n \n \tinit_completion(\u0026hwc-\u003ehwc_init_eqe_comp);\n@@ -687,24 +839,51 @@ static int mana_hwc_establish_channel(struct gdma_context gc, u16 q_depth,\n \tif (err)\n \t\treturn err;\n \n-\tif (!wait_for_completion_timeout(\u0026hwc-\u003ehwc_init_eqe_comp, 60 HZ))\n-\t\treturn -ETIMEDOUT;\n+\t/* setup_hwc activated MST entries — hardware can now DMA into\n+\t * our queue buffers. If anything below fails, we must tear\n+\t * down before returning so the caller doesn't need to track\n+\t * whether setup_hwc succeeded.\n+\t /\n+\thwc-\u003esetup_active = true;\n+\n+\tif (!wait_for_completion_timeout(\u0026hwc-\u003ehwc_init_eqe_comp, 60 HZ)) {\n+\t\terr = -ETIMEDOUT;\n+\t\tgoto teardown;\n+\t}\n \n \tq_depth = hwc-\u003ehwc_init_q_depth_max;\n \tmax_req_msg_size = hwc-\u003ehwc_init_max_req_msg_size;\n \tmax_resp_msg_size = hwc-\u003ehwc_init_max_resp_msg_size;\n \n \t/ Both were set in mana_hwc_init_event_handler(). /\n-\tif (WARN_ON(cq-\u003eid \u003e= gc-\u003emax_num_cqs))\n-\t\treturn -EPROTO;\n+\tif (WARN_ON(cq-\u003eid \u003e= gc-\u003emax_num_cqs)) {\n+\t\terr = -EPROTO;\n+\t\tgoto teardown;\n+\t}\n \n-\tgc-\u003ecq_table = vcalloc(gc-\u003emax_num_cqs, sizeof(struct gdma_queue ));\n-\tif (!gc-\u003ecq_table)\n-\t\treturn -ENOMEM;\n+\tcq_table = vcalloc(gc-\u003emax_num_cqs, sizeof(cq_table));\n+\tif (!cq_table) {\n+\t\terr = -ENOMEM;\n+\t\tgoto teardown;\n+\t}\n \n-\tgc-\u003ecq_table[cq-\u003eid] = cq;\n+\trcu_assign_pointer(cq_table[cq-\u003eid], cq);\n+\t/ Publish the fully-initialised table last; pairs with the\n+\t * rcu_dereference(gc-\u003ecq_table) in mana_gd_process_eqe().\n+\t /\n+\trcu_assign_pointer(gc-\u003ecq_table, cq_table);\n \n \treturn 0;\n+\n+teardown:\n+\t{\n+\t\tint td_err = mana_smc_teardown_hwc(\u0026gc-\u003eshm_channel, false);\n+\n+\t\tif (!td_err)\n+\t\t\thwc-\u003esetup_active = false;\n+\n+\t\treturn td_err ? td_err : err;\n+\t}\n }\n \n static int mana_hwc_init_queues(struct hw_channel_context hwc, u16 q_depth,\n@@ -712,6 +891,12 @@ static int mana_hwc_init_queues(struct hw_channel_context hwc, u16 q_depth,\n {\n \tint err;\n \n+\t/ CQ depth is q_depth * 2 (SQ + RQ) passed as u16 to create_cq.\n+\t * Cap to prevent u16 truncation.\n+\t /\n+\tif (q_depth \u003e U16_MAX / 2)\n+\t\tq_depth = U16_MAX / 2;\n+\n \terr = mana_hwc_init_inflight_msg(hwc, q_depth);\n \tif (err)\n \t\treturn err;\n@@ -728,14 +913,14 @@ static int mana_hwc_init_queues(struct hw_channel_context hwc, u16 q_depth,\n \t\tgoto out;\n \t}\n \n-\terr = mana_hwc_create_wq(hwc, GDMA_RQ, q_depth, max_req_msg_size,\n+\terr = mana_hwc_create_wq(hwc, GDMA_RQ, q_depth, max_resp_msg_size,\n \t\t\t\t hwc-\u003ecq, \u0026hwc-\u003erxq);\n \tif (err) {\n \t\tdev_err(hwc-\u003edev, \"Failed to create HWC RQ: %d\\n\", err);\n \t\tgoto out;\n \t}\n \n-\terr = mana_hwc_create_wq(hwc, GDMA_SQ, q_depth, max_resp_msg_size,\n+\terr = mana_hwc_create_wq(hwc, GDMA_SQ, q_depth, max_req_msg_size,\n \t\t\t\t hwc-\u003ecq, \u0026hwc-\u003etxq);\n \tif (err) {\n \t\tdev_err(hwc-\u003edev, \"Failed to create HWC SQ: %d\\n\", err);\n@@ -744,6 +929,7 @@ static int mana_hwc_init_queues(struct hw_channel_context hwc, u16 q_depth,\n \n \thwc-\u003enum_inflight_msg = q_depth;\n \thwc-\u003emax_req_msg_size = max_req_msg_size;\n+\thwc-\u003emax_resp_msg_size = max_resp_msg_size;\n \n \treturn 0;\n out:\n@@ -751,12 +937,62 @@ static int mana_hwc_init_queues(struct hw_channel_context hwc, u16 q_depth,\n \treturn err;\n }\n \n+/* Tear down all HWC queues and free associated resources. Used on\n+ * the reinit-with-higher-queue-depth path and reinit fallback.\n+ \n+ PRECONDITION: must be called only during channel bring-up in\n+ * mana_hwc_create_channel(), before the channel is published to\n+ * senders. There the setup thread is effectively single-threaded\n+ * (serialized against teardown by the PCI/PM device_lock or, on the\n+ * service path, GC_IN_SERVICE, with the data path not yet probed),\n+ * channel_up is still false, caller_ctx is not yet allocated, and\n+ * active_senders is 0 — so no concurrent request/response user can\n+ * touch these queues. That is why this skips the hwc_lock-protected\n+ * driver_data clear + active_senders drain that\n+ * mana_hwc_destroy_channel() needs for the runtime teardown race;\n+ * only the CQ-first ordering below (to fence off a pending interrupt)\n+ * is required. Calling this on a live, published channel would be a\n+ * use-after-free.\n+ /\n+static void mana_hwc_destroy_queues(struct hw_channel_context hwc)\n+{\n+\tstruct gdma_context gc = hwc-\u003egdma_dev-\u003egdma_context;\n+\n+\t/ Destroy CQ first to deregister the EQ from the interrupt\n+\t * handler list before freeing caller_ctx, TXQ, or RXQ memory.\n+\t * A pending interrupt handler could still reach handle_resp()\n+\t * which dereferences caller_ctx.\n+\t /\n+\tif (hwc-\u003ecq) {\n+\t\tmana_hwc_destroy_cq(gc, hwc-\u003ecq);\n+\t\thwc-\u003ecq = NULL;\n+\t}\n+\n+\tkfree(hwc-\u003ecaller_ctx);\n+\thwc-\u003ecaller_ctx = NULL;\n+\n+\tif (hwc-\u003etxq) {\n+\t\tmana_hwc_destroy_wq(hwc, hwc-\u003etxq);\n+\t\thwc-\u003etxq = NULL;\n+\t}\n+\n+\tif (hwc-\u003erxq) {\n+\t\tmana_hwc_destroy_wq(hwc, hwc-\u003erxq);\n+\t\thwc-\u003erxq = NULL;\n+\t}\n+\n+\tmana_gd_free_res_map(\u0026hwc-\u003einflight_msg_res);\n+\thwc-\u003enum_inflight_msg = 0;\n+}\n+\n int mana_hwc_create_channel(struct gdma_context gc)\n {\n \tu32 max_req_msg_size, max_resp_msg_size;\n \tstruct gdma_dev gd = \u0026gc-\u003ehwc;\n \tstruct hw_channel_context hwc;\n-\tu16 q_depth_max;\n+\tstruct gdma_queue __rcu *old_cq_table;\n+\tunsigned long flags;\n+\tu32 q_depth_max;\n \tint err;\n \n \thwc = kzalloc_obj(hwc);\n@@ -764,10 +1000,11 @@ int mana_hwc_create_channel(struct gdma_context gc)\n \t\treturn -ENOMEM;\n \n \tgd-\u003egdma_context = gc;\n-\tgd-\u003edriver_data = hwc;\n \thwc-\u003egdma_dev = gd;\n \thwc-\u003edev = gc-\u003edev;\n \thwc-\u003ehwc_timeout = HW_CHANNEL_WAIT_RESOURCE_TIMEOUT_MS;\n+\thwc-\u003eactive_senders = 0;\n+\tinit_waitqueue_head(\u0026gc-\u003ehwc_drain_waitq);\n \n \t/ HWC's instance number is always 0. /\n \tgd-\u003edev_id.as_uint32 = 0;\n@@ -776,6 +1013,15 @@ int mana_hwc_create_channel(struct gdma_context gc)\n \tgd-\u003epdid = INVALID_PDID;\n \tgd-\u003edoorbell = INVALID_DOORBELL;\n \n+\t/* Publish driver_data last, under hwc_lock: the lock orders the hwc\n+\t * initialisation above before the pointer becomes visible and\n+\t * serialises the publish against the control-plane readers in\n+\t * mana_gd_send_request(), mana_need_log() and mana_serv_reset().\n+\t /\n+\tspin_lock_irqsave(\u0026gc-\u003ehwc_lock, flags);\n+\tgc-\u003ehwc.driver_data = hwc;\n+\tspin_unlock_irqrestore(\u0026gc-\u003ehwc_lock, flags);\n+\n \t/ mana_hwc_init_queues() only creates the required data structures,\n \t * and doesn't touch the HWC device.\n \t /\n@@ -794,8 +1040,130 @@ int mana_hwc_create_channel(struct gdma_context gc)\n \t\tgoto out;\n \t}\n \n+\t/* The channel was bootstrapped at a minimal queue depth. If the\n+\t * device reports a higher maximum, tear down and rebuild with\n+\t * the larger depth so more HWC commands can be in flight.\n+\t /\n+\tif (q_depth_max \u003e HW_CHANNEL_VF_BOOTSTRAP_QUEUE_DEPTH) {\n+\t\t/ q_depth_max now carries the full device-reported value\n+\t\t * (HWC_INIT_DATA_QUEUE_DEPTH is 24-bit). Clamp it to the\n+\t\t * depth the rest of the driver supports — create_cq() takes\n+\t\t * q_depth * 2 as a u16 — before the overflow check below, so\n+\t\t * an over-large but otherwise-valid depth is reduced to the\n+\t\t * maximum instead of wrapping or being rejected.\n+\t\t /\n+\t\tif (q_depth_max \u003e U16_MAX / 2)\n+\t\t\tq_depth_max = U16_MAX / 2;\n+\n+\t\t/ Sanity-check device-reported values before using them\n+\t\t * to size DMA allocations. Enforce protocol minimums\n+\t\t * for message sizes and check that q_depth * max_msg_size\n+\t\t * plus alignment headroom fits in u32 (for\n+\t\t * mana_hwc_alloc_dma_buf's MANA_PAGE_ALIGN).\n+\t\t /\n+\t\tif (!max_req_msg_size \|\| !max_resp_msg_size \|\|\n+\t\t max_req_msg_size \u003c sizeof(struct gdma_req_hdr) \|\|\n+\t\t max_resp_msg_size \u003c sizeof(struct gdma_resp_hdr) \|\|\n+\t\t (u64)q_depth_max max_req_msg_size \u003e\n+\t\t\tU32_MAX - MANA_PAGE_SIZE \|\|\n+\t\t (u64)q_depth_max * max_resp_msg_size \u003e\n+\t\t\tU32_MAX - MANA_PAGE_SIZE) {\n+\t\t\tdev_err(hwc-\u003edev,\n+\t\t\t\t\"HWC: invalid dims q=%u req=%u resp=%u\\n\",\n+\t\t\t\tq_depth_max, max_req_msg_size,\n+\t\t\t\tmax_resp_msg_size);\n+\t\t\tq_depth_max = HW_CHANNEL_VF_BOOTSTRAP_QUEUE_DEPTH;\n+\t\t\tgoto skip_reinit;\n+\t\t}\n+\n+\t\terr = mana_smc_teardown_hwc(\u0026gc-\u003eshm_channel, false);\n+\t\tif (err) {\n+\t\t\t/* Keep using the bootstrap-depth channel. /\n+\t\t\tdev_err(hwc-\u003edev,\n+\t\t\t\t\"Failed to teardown HWC for reinit: %d\\n\",\n+\t\t\t\terr);\n+\t\t\tq_depth_max = HW_CHANNEL_VF_BOOTSTRAP_QUEUE_DEPTH;\n+\t\t\tgoto skip_reinit;\n+\t\t}\n+\n+\t\thwc-\u003esetup_active = false;\n+\n+\t\t/ Destroy queues first — mana_gd_destroy_cq inside\n+\t\t * unpublishes the CQ from cq_table via\n+\t\t * rcu_assign_pointer(NULL) + synchronize_rcu.\n+\t\t * Must happen while cq_table is still valid.\n+\t\t /\n+\t\tmana_hwc_destroy_queues(hwc);\n+\n+\t\told_cq_table = rcu_replace_pointer(gc-\u003ecq_table, NULL, true);\n+\t\tsynchronize_rcu();\n+\t\tvfree(old_cq_table);\n+\n+\t\terr = mana_hwc_init_queues(hwc, q_depth_max,\n+\t\t\t\t\t max_req_msg_size,\n+\t\t\t\t\t max_resp_msg_size);\n+\t\tif (err) {\n+\t\t\tdev_err(hwc-\u003edev, \"Failed to reinit HWC: %d\\n\", err);\n+\t\t\tgoto reinit_fallback;\n+\t\t}\n+\n+\t\terr = mana_hwc_establish_channel(gc, \u0026q_depth_max,\n+\t\t\t\t\t\t \u0026max_req_msg_size,\n+\t\t\t\t\t\t \u0026max_resp_msg_size);\n+\t\tif (err) {\n+\t\t\tdev_err(hwc-\u003edev, \"Failed to re-establish HWC: %d\\n\",\n+\t\t\t\terr);\n+\t\t\t/ establish_channel does internal teardown on\n+\t\t\t * failure. If teardown succeeded (setup_active\n+\t\t\t * cleared), MST entries are invalidated and we\n+\t\t\t * can try the bootstrap fallback. If teardown\n+\t\t\t * also failed (setup_active still set), hardware\n+\t\t\t * mappings may still be active — skip fallback.\n+\t\t\t /\n+\t\t\tif (hwc-\u003esetup_active)\n+\t\t\t\tgoto out;\n+\t\t\tgoto reinit_fallback;\n+\t\t}\n+\t}\n+\n+\tgoto skip_reinit;\n+\n+reinit_fallback:\n+\t/ Restore bootstrap-depth channel so the device remains functional.\n+\t * Free cq_table if it was allocated by a partially successful\n+\t * establish attempt.\n+\t /\n+\tdev_warn(hwc-\u003edev, \"HWC reinit failed, falling back to bootstrap depth\\n\");\n+\n+\tmana_hwc_destroy_queues(hwc);\n+\n+\told_cq_table = rcu_replace_pointer(gc-\u003ecq_table, NULL, true);\n+\tsynchronize_rcu();\n+\tvfree(old_cq_table);\n+\n+\terr = mana_hwc_init_queues(hwc, HW_CHANNEL_VF_BOOTSTRAP_QUEUE_DEPTH,\n+\t\t\t\t HW_CHANNEL_MAX_REQUEST_SIZE,\n+\t\t\t\t HW_CHANNEL_MAX_RESPONSE_SIZE);\n+\tif (err) {\n+\t\tdev_err(hwc-\u003edev, \"Failed to restore bootstrap HWC: %d\\n\", err);\n+\t\tgoto out;\n+\t}\n+\n+\terr = mana_hwc_establish_channel(gc, \u0026q_depth_max, \u0026max_req_msg_size,\n+\t\t\t\t\t \u0026max_resp_msg_size);\n+\tif (err) {\n+\t\tdev_err(hwc-\u003edev, \"Failed to re-establish bootstrap HWC: %d\\n\",\n+\t\t\terr);\n+\t\tgoto out;\n+\t}\n+\n+skip_reinit:\n+\n+\t/ No RCU needed: still in mana_hwc_create_channel, the\n+\t * pointer has not been published to concurrent senders yet.\n+\t /\n \terr = mana_hwc_test_channel(gc-\u003ehwc.driver_data,\n-\t\t\t\t HW_CHANNEL_VF_BOOTSTRAP_QUEUE_DEPTH,\n+\t\t\t\t hwc-\u003enum_inflight_msg,\n \t\t\t\t max_req_msg_size, max_resp_msg_size);\n \tif (err) {\n \t\tdev_err(hwc-\u003edev, \"Failed to test HWC: %d\\n\", err);\n@@ -810,27 +1178,195 @@ int mana_hwc_create_channel(struct gdma_context gc)\n \n void mana_hwc_destroy_channel(struct gdma_context gc)\n {\n+\t/ This is the only destroy entry point. driver_data is read\n+\t * plainly here (teardown is serialised against other teardown);\n+\t * it is cleared under hwc_lock below before hwc is freed.\n+\t /\n \tstruct hw_channel_context hwc = gc-\u003ehwc.driver_data;\n+\tstruct gdma_queue __rcu *old_cq_table;\n+\tunsigned long flags;\n \n \tif (!hwc)\n \t\treturn;\n \n-\t/ gc-\u003emax_num_cqs is set in mana_hwc_init_event_handler(). If it's\n-\t * non-zero, the HWC worked and we should tear down the HWC here.\n+\t/* Prevent new requests from starting and wake any\n+\t * threads waiting for a free msg slot. Set channel_up under\n+\t * the bitmap lock so get_msg_index() cannot acquire a slot\n+\t * and increment active_senders after this point.\n+\t \n+\t If channel_up is already false (e.g. init failed before\n+\t * the channel was established), skip the lock — it may not\n+\t * have been initialized yet, and no senders can be active.\n \t /\n-\tif (gc-\u003emax_num_cqs \u003e 0) {\n-\t\tmana_smc_teardown_hwc(\u0026gc-\u003eshm_channel, false);\n-\t\tgc-\u003emax_num_cqs = 0;\n+\tif (hwc-\u003echannel_up) {\n+\t\tspin_lock_irqsave(\u0026hwc-\u003einflight_msg_res.lock, flags);\n+\t\thwc-\u003echannel_up = false;\n+\t\tspin_unlock_irqrestore(\u0026hwc-\u003einflight_msg_res.lock, flags);\n+\t\twake_up_all(\u0026hwc-\u003emsg_waitq);\n \t}\n \n+\t/ Clear the pointer under hwc_lock so new callers in\n+\t * mana_gd_send_request() see NULL and return -ENODEV. The lock\n+\t * makes the readers' \"load driver_data + active_senders++\"\n+\t * atomic against this store, so once it returns no new sender can\n+\t * take a reference; the active_senders drain below waits out those\n+\t * that already did, before their hwc is freed.\n+\t /\n+\tspin_lock_irqsave(\u0026gc-\u003ehwc_lock, flags);\n+\tgc-\u003ehwc.driver_data = NULL;\n+\tspin_unlock_irqrestore(\u0026gc-\u003ehwc_lock, flags);\n+\n+\t/ Force-complete any in-flight senders so they observe -ENODEV,\n+\t * return, and drop their references. This runs before the HWC\n+\t * hardware teardown below, so a live interrupt may still deliver\n+\t * a real response via handle_resp() concurrently — that is safe\n+\t * because the per-slot refcount model tolerates a concurrent\n+\t * complete() and both paths (handle_resp and this loop) drop\n+\t * their refs without double-releasing the slot.\n+\t /\n+\tif (hwc-\u003ecaller_ctx) {\n+\t\tstruct hwc_caller_ctx ctx;\n+\t\tint i;\n+\n+\t\tfor (i = 0; i \u003c hwc-\u003enum_inflight_msg; i++) {\n+\t\t\tif (!test_bit(i, hwc-\u003einflight_msg_res.map))\n+\t\t\t\tcontinue;\n+\n+\t\t\tctx = \u0026hwc-\u003ecaller_ctx[i];\n+\n+\t\t\t/* Wake senders blocked on wait_for_completion.\n+\t\t\t * Set error under lock to avoid racing with\n+\t\t\t * handle_resp() which writes error under the\n+\t\t\t * same lock. The sender NULLs output_buf\n+\t\t\t * after waking — doing it here would race\n+\t\t\t * with a sender that hasn't set output_buf yet.\n+\t\t\t /\n+\t\t\tspin_lock_irqsave(\u0026ctx-\u003elock, flags);\n+\t\t\tctx-\u003eerror = -ENODEV;\n+\t\t\tcomplete(\u0026ctx-\u003ecomp_event);\n+\t\t\tspin_unlock_irqrestore(\u0026ctx-\u003elock, flags);\n+\t\t}\n+\t}\n+\n+\t/ Wait for all sender threads to finish and drop their refs\n+\t * before touching the hardware or freeing anything. Draining\n+\t * here -- ahead of the teardown/FLR block below -- guarantees no\n+\t * in-flight sender is still running when this function returns on\n+\t * any path, including the FLR-failure early return; otherwise a\n+\t * stranded sender would dereference gc-\u003ehwc_lock /\n+\t * gc-\u003ehwc_drain_waitq after the caller frees gc.\n+\t * After this, only slots held by timed-out senders whose\n+\t * handle_resp() never ran remain in the bitmap.\n+\t \n+\t active_senders is only ever modified under hwc_lock, and the\n+\t * last sender's wake_up() runs under that lock before it is\n+\t * released. Evaluating the condition under hwc_lock therefore\n+\t * guarantees that once we observe 0 the waking sender has\n+\t * already dropped the lock -- i.e. finished touching gc -- so it\n+\t * cannot race the caller freeing gc after this returns.\n+\t /\n+\tspin_lock_irq(\u0026gc-\u003ehwc_lock);\n+\twait_event_lock_irq(gc-\u003ehwc_drain_waitq,\n+\t\t\t hwc-\u003eactive_senders == 0, gc-\u003ehwc_lock);\n+\tspin_unlock_irq(\u0026gc-\u003ehwc_lock);\n+\n+\t/ Tear down the HWC if setup_hwc previously activated MST entries.\n+\t * This is the definitive flag — unlike max_num_cqs which depends\n+\t * on the init EQE arriving.\n+\t \n+\t The return value is intentionally not checked. This is the\n+\t * terminal cleanup path (device removal, suspend, or init\n+\t * failure) — resources must be freed regardless. If teardown\n+\t * fails, hardware may still have active MST entries, but:\n+\t \n+\t - Interrupts: mana_hwc_destroy_cq() below calls\n+\t * mana_gd_deregister_irq() which removes the HWC EQ from\n+\t * the interrupt dispatch list via list_del_rcu() +\n+\t * synchronize_rcu(). After that, no interrupt handler can\n+\t * invoke handle_resp() or access CQ/RQ buffers — even if\n+\t * the IRQ is shared with data path queues.\n+\t \n+\t - DMA: mana_hwc_destroy_wq() frees DMA buffers via\n+\t * dma_free_coherent() which unmaps the IOVA from the\n+\t * IOMMU. Any stale hardware DMA to the old address\n+\t * faults at the IOMMU, not in kernel memory.\n+\t /\n+\tif (hwc-\u003esetup_active) {\n+\t\tint td_err = mana_smc_teardown_hwc(\u0026gc-\u003eshm_channel, false);\n+\n+\t\tif (td_err) {\n+\t\t\tdev_err(gc-\u003edev, \"HWC teardown failed: %d, issuing FLR\\n\",\n+\t\t\t\ttd_err);\n+\n+\t\t\t/ On systems without IOMMU, freeing DMA memory with\n+\t\t\t * active hardware MST mappings risks memory corruption.\n+\t\t\t * Issue FLR to force-reset the device and invalidate\n+\t\t\t * all hardware state including MST entries.\n+\t\t\t /\n+\t\t\ttd_err = pcie_flr(to_pci_dev(gc-\u003edev));\n+\t\t\tif (td_err) {\n+\t\t\t\t/ Device wedged: teardown and FLR\n+\t\t\t\t * both failed, so hardware may still\n+\t\t\t\t * DMA into our queue buffers. Leak\n+\t\t\t\t * all HWC DMA buffers -- and the EQ\n+\t\t\t\t * IRQ registration, whose safe removal\n+\t\t\t\t * needs those queues -- instead of\n+\t\t\t\t * freeing memory the device can still\n+\t\t\t\t * reach; without an IOMMU the freed\n+\t\t\t\t * pages could be reused under live\n+\t\t\t\t * DMA.\n+\t\t\t\t /\n+\n+\t\t\t\tdev_warn(gc-\u003edev,\n+\t\t\t\t\t \"Leaked HWC DMA buffers (CQ/RQ/TXQ) to prevent memory corruption. Device is no longer usable.\\n\");\n+\n+\t\t\t\t/ Do not call destroy_cq/wq below;\n+\t\t\t\t * they would dma_free_coherent().\n+\t\t\t\t * Leave hwc/cq/rxq/txq allocated.\n+\t\t\t\t /\n+\t\t\t\treturn;\n+\t\t\t}\n+\n+\t\t\tdev_info(gc-\u003edev, \"FLR succeeded, hardware state cleared\\n\");\n+\t\t}\n+\n+\t\thwc-\u003esetup_active = false;\n+\t}\n+\n+\t/ Tear down the HWC CQ object first — mana_hwc_destroy_cq()\n+\t * both unpublishes the CQ from cq_table (+synchronize_rcu) and\n+\t * deregisters the HWC EQ from the interrupt handler's RCU list\n+\t * (via mana_gd_deregister_irq + synchronize_rcu), guaranteeing\n+\t * no interrupt handler can access RQ/TXQ buffers after this\n+\t * point. The active_senders drain above ensures no sender is\n+\t * accessing the CQ via txq-\u003ehwc_cq when it is destroyed. Then\n+\t * destroy TXQ and RQ safely.\n+\t /\n+\tif (hwc-\u003ecq)\n+\t\tmana_hwc_destroy_cq(hwc-\u003egdma_dev-\u003egdma_context, hwc-\u003ecq);\n+\n \tif (hwc-\u003etxq)\n \t\tmana_hwc_destroy_wq(hwc, hwc-\u003etxq);\n \n \tif (hwc-\u003erxq)\n \t\tmana_hwc_destroy_wq(hwc, hwc-\u003erxq);\n \n-\tif (hwc-\u003ecq)\n-\t\tmana_hwc_destroy_cq(hwc-\u003egdma_dev-\u003egdma_context, hwc-\u003ecq);\n+\t/ Release any slots still held — these belong to timed-out\n+\t * senders where handle_resp() never ran (refcount = 1 with\n+\t * handle_resp's ref still outstanding).\n+\t /\n+\tif (hwc-\u003ecaller_ctx) {\n+\t\tstruct hwc_caller_ctx ctx;\n+\t\tint i;\n+\n+\t\tfor (i = 0; i \u003c hwc-\u003enum_inflight_msg; i++) {\n+\t\t\tif (!test_bit(i, hwc-\u003einflight_msg_res.map))\n+\t\t\t\tcontinue;\n+\n+\t\t\tctx = \u0026hwc-\u003ecaller_ctx[i];\n+\t\t\thwc_ctx_put(hwc, ctx);\n+\t\t}\n+\t}\n \n \tkfree(hwc-\u003ecaller_ctx);\n \thwc-\u003ecaller_ctx = NULL;\n@@ -845,11 +1381,11 @@ void mana_hwc_destroy_channel(struct gdma_context gc)\n \thwc-\u003ehwc_timeout = 0;\n \n \tkfree(hwc);\n-\tgc-\u003ehwc.driver_data = NULL;\n \tgc-\u003ehwc.gdma_context = NULL;\n \n-\tvfree(gc-\u003ecq_table);\n-\tgc-\u003ecq_table = NULL;\n+\told_cq_table = rcu_replace_pointer(gc-\u003ecq_table, NULL, true);\n+\tsynchronize_rcu();\n+\tvfree(old_cq_table);\n }\n \n int mana_hwc_send_request(struct hw_channel_context hwc, u32 req_len,\n@@ -860,13 +1396,17 @@ int mana_hwc_send_request(struct hw_channel_context hwc, u32 req_len,\n \tstruct hwc_wq txq = hwc-\u003etxq;\n \tstruct gdma_req_hdr req_msg;\n \tstruct hwc_caller_ctx ctx;\n+\tunsigned long flags;\n \tu32 dest_vrcq = 0;\n \tu32 dest_vrq = 0;\n \tu32 command;\n+\tu32 status;\n \tu16 msg_id;\n \tint err;\n \n-\tmana_hwc_get_msg_index(hwc, \u0026msg_id);\n+\terr = mana_hwc_get_msg_index(hwc, \u0026msg_id);\n+\tif (err)\n+\t\treturn err;\n \n \ttx_wr = \u0026txq-\u003emsg_buf-\u003ereqs[msg_id];\n \n@@ -878,8 +1418,11 @@ int mana_hwc_send_request(struct hw_channel_context hwc, u32 req_len,\n \t}\n \n \tctx = hwc-\u003ecaller_ctx + msg_id;\n+\n+\tspin_lock_irqsave(\u0026ctx-\u003elock, flags);\n \tctx-\u003eoutput_buf = resp;\n \tctx-\u003eoutput_buflen = resp_len;\n+\tspin_unlock_irqrestore(\u0026ctx-\u003elock, flags);\n \n \treq_msg = (struct gdma_req_hdr )tx_wr-\u003ebuf_va;\n \tif (req)\n@@ -895,8 +1438,14 @@ int mana_hwc_send_request(struct hw_channel_context hwc, u32 req_len,\n \t\tdest_vrcq = hwc-\u003epf_dest_vrcq_id;\n \t}\n \n+\t/ Take handle_resp's ref before posting — hardware can respond\n+\t * immediately after the doorbell ring.\n+\t /\n+\trefcount_inc(\u0026ctx-\u003erefcnt);\n+\n \terr = mana_hwc_post_tx_wqe(txq, tx_wr, dest_vrq, dest_vrcq, false);\n \tif (err) {\n+\t\trefcount_dec(\u0026ctx-\u003erefcnt);\n \t\tdev_err(hwc-\u003edev, \"HWC: Failed to post send WQE: %d\\n\", err);\n \t\tgoto out;\n \t}\n@@ -907,31 +1456,86 @@ int mana_hwc_send_request(struct hw_channel_context hwc, u32 req_len,\n \t\t\tdev_err(hwc-\u003edev, \"Command 0x%x timed out: %u ms\\n\",\n \t\t\t\tcommand, hwc-\u003ehwc_timeout);\n \n-\t\t/* Reduce further waiting if HWC no response /\n+\t\t/ NULL out output_buf so a late handle_resp() won't write\n+\t\t * into the caller's buffer after the sender returns, then\n+\t\t * check whether handle_resp() already delivered a valid\n+\t\t * response between the timeout firing and this lock\n+\t\t * acquisition — ctx-\u003eerror != -EINPROGRESS means it ran.\n+\t\t /\n+\t\tspin_lock_irqsave(\u0026ctx-\u003elock, flags);\n+\t\tctx-\u003eoutput_buf = NULL;\n+\t\terr = ctx-\u003eerror;\n+\t\tstatus = ctx-\u003estatus_code;\n+\t\tspin_unlock_irqrestore(\u0026ctx-\u003elock, flags);\n+\n+\t\tif (err != -EINPROGRESS) {\n+\t\t\t/ handle_resp() delivered a valid response just after\n+\t\t\t * the timeout fired. The hardware is alive, so use\n+\t\t\t * the response and leave the channel usable; do not\n+\t\t\t * latch hwc_timed_out or degrade hwc_timeout for what\n+\t\t\t * turned out to be a transient race.\n+\t\t\t /\n+\t\t\thwc_ctx_put(hwc, ctx);\n+\t\t\tgoto check_status;\n+\t\t}\n+\n+\t\t/ Genuine timeout: no response arrived. Reduce further\n+\t\t * waiting, and mark the channel timed out under the bitmap\n+\t\t * lock so get_msg_index() cannot acquire new slots after this.\n+\t\t /\n \t\tif (hwc-\u003ehwc_timeout \u003e 1)\n \t\t\thwc-\u003ehwc_timeout = 1;\n \n+\t\tspin_lock_irqsave(\u0026hwc-\u003einflight_msg_res.lock, flags);\n+\t\thwc-\u003ehwc_timed_out = true;\n+\t\tspin_unlock_irqrestore(\u0026hwc-\u003einflight_msg_res.lock, flags);\n+\t\twake_up_all(\u0026hwc-\u003emsg_waitq);\n+\n \t\terr = -ETIMEDOUT;\n-\t\tgoto out;\n+\t\thwc_ctx_put(hwc, ctx);\n+\t\tgoto done;\n \t}\n \n-\tif (ctx-\u003eerror) {\n-\t\terr = ctx-\u003eerror;\n-\t\tgoto out;\n-\t}\n+\t/ NULL output_buf so a late handle_resp() won't memcpy into\n+\t * the caller's buffer after the sender exits. Read error and\n+\t * status_code under the same lock — after hwc_ctx_put the slot\n+\t * may be reused and these fields overwritten.\n+\t /\n+\tspin_lock_irqsave(\u0026ctx-\u003elock, flags);\n+\tctx-\u003eoutput_buf = NULL;\n+\terr = ctx-\u003eerror;\n+\tstatus = ctx-\u003estatus_code;\n+\tspin_unlock_irqrestore(\u0026ctx-\u003elock, flags);\n+\thwc_ctx_put(hwc, ctx);\n+\n+check_status:\n+\tif (err)\n+\t\tgoto done;\n \n-\tif (ctx-\u003estatus_code \u0026\u0026 ctx-\u003estatus_code != GDMA_STATUS_MORE_ENTRIES) {\n-\t\tif (ctx-\u003estatus_code == GDMA_STATUS_CMD_UNSUPPORTED) {\n+\tif (status \u0026\u0026 status != GDMA_STATUS_MORE_ENTRIES) {\n+\t\tif (status == GDMA_STATUS_CMD_UNSUPPORTED) {\n \t\t\terr = -EOPNOTSUPP;\n-\t\t\tgoto out;\n+\t\t\tgoto done;\n \t\t}\n+\n \t\tif (command != MANA_QUERY_PHY_STAT)\n \t\t\tdev_err(hwc-\u003edev, \"Command 0x%x failed with status: 0x%x\\n\",\n-\t\t\t\tcommand, ctx-\u003estatus_code);\n+\t\t\t\tcommand, status);\n \t\terr = -EPROTO;\n-\t\tgoto out;\n+\t\tgoto done;\n \t}\n+\n+\terr = 0;\n+\tgoto done;\n out:\n-\tmana_hwc_put_msg_index(hwc, msg_id);\n+\t/ Pre-post error paths: no WQE was submitted so handle_resp()\n+\t * cannot race here. refcount is 1 (no second ref taken).\n+\t /\n+\tctx = hwc-\u003ecaller_ctx + msg_id;\n+\tspin_lock_irqsave(\u0026ctx-\u003elock, flags);\n+\tctx-\u003eoutput_buf = NULL;\n+\tspin_unlock_irqrestore(\u0026ctx-\u003elock, flags);\n+\thwc_ctx_put(hwc, ctx);\n+done:\n \treturn err;\n }\ndiff --git a/drivers/net/ethernet/microsoft/mana/mana_en.c b/drivers/net/ethernet/microsoft/mana/mana_en.c\nindex a8c329bdbacf83..5508746590e7cd 100644\n--- a/drivers/net/ethernet/microsoft/mana/mana_en.c\n+++ b/drivers/net/ethernet/microsoft/mana/mana_en.c\n@@ -2648,6 +2648,7 @@ static int mana_create_txq(struct mana_port_context apc,\n \tstruct mana_obj_spec cq_spec;\n \tstruct gdma_queue_spec spec;\n \tstruct gdma_context gc;\n+\tstruct gdma_queue __rcu cq_table;\n \tstruct mana_txq txq;\n \tstruct mana_cq cq;\n \tu32 txq_size;\n@@ -2753,12 +2754,18 @@ static int mana_create_txq(struct mana_port_context apc,\n \n \t\tcq-\u003egdma_id = cq-\u003egdma_cq-\u003eid;\n \n-\t\tif (WARN_ON(cq-\u003egdma_id \u003e= gc-\u003emax_num_cqs)) {\n+\t\t/* No rcu_read_lock(): mana_create_txq runs under RTNL during\n+\t\t * netdev bring-up, inside the netdev lifetime that\n+\t\t * mana_remove() drains before the base cq_table can be freed.\n+\t\t * See gdma_context::cq_table in gdma.h for why \"true\" is sound.\n+\t\t /\n+\t\tcq_table = rcu_dereference_protected(gc-\u003ecq_table, true);\n+\t\tif (WARN_ON(!cq_table \|\| cq-\u003egdma_id \u003e= gc-\u003emax_num_cqs)) {\n \t\t\terr = -EINVAL;\n \t\t\tgoto out;\n \t\t}\n \n-\t\tgc-\u003ecq_table[cq-\u003egdma_id] = cq-\u003egdma_cq;\n+\t\trcu_assign_pointer(cq_table[cq-\u003egdma_id], cq-\u003egdma_cq);\n \n \t\tmana_create_txq_debugfs(apc, i);\n \n@@ -2986,6 +2993,7 @@ static struct mana_rxq mana_create_rxq(struct mana_port_context apc,\n \tstruct gdma_queue_spec spec;\n \tstruct mana_cq cq = NULL;\n \tstruct gdma_context gc;\n+\tstruct gdma_queue __rcu cq_table;\n \tu32 cq_size, rq_size;\n \tstruct mana_rxq rxq;\n \tint err;\n@@ -3075,12 +3083,18 @@ static struct mana_rxq mana_create_rxq(struct mana_port_context apc,\n \tif (err)\n \t\tgoto out;\n \n-\tif (WARN_ON(cq-\u003egdma_id \u003e= gc-\u003emax_num_cqs)) {\n+\t/* No rcu_read_lock(): mana_create_rxq runs under RTNL during netdev\n+\t * bring-up, inside the netdev lifetime that mana_remove() drains\n+\t * before the base cq_table can be freed. See gdma_context::cq_table\n+\t * in gdma.h for why \"true\" is sound.\n+\t /\n+\tcq_table = rcu_dereference_protected(gc-\u003ecq_table, true);\n+\tif (WARN_ON(!cq_table \|\| cq-\u003egdma_id \u003e= gc-\u003emax_num_cqs)) {\n \t\terr = -EINVAL;\n \t\tgoto out;\n \t}\n \n-\tgc-\u003ecq_table[cq-\u003egdma_id] = cq-\u003egdma_cq;\n+\trcu_assign_pointer(cq_table[cq-\u003egdma_id], cq-\u003egdma_cq);\n \n \tnetif_napi_add_weight_locked(ndev, \u0026cq-\u003enapi, mana_poll, 1);\n \ndiff --git a/include/net/mana/gdma.h b/include/net/mana/gdma.h\nindex 8529cef0d7c452..a4eac6f7c36609 100644\n--- a/include/net/mana/gdma.h\n+++ b/include/net/mana/gdma.h\n@@ -428,9 +428,34 @@ struct gdma_context {\n \t/ L2 MTU /\n \tu16 adapter_mtu;\n \n-\t/ This maps a CQ index to the queue structure. /\n+\t/ Size of cq_table, i.e. the largest valid CQ index + 1. Set once\n+\t * when cq_table is allocated and treated as immutable for the\n+\t * table's lifetime (a bound only) -- it is never reset on teardown.\n+\t * cq_table == NULL is the sole \"table torn down\" signal, so every\n+\t * cq_table[id] access must guard with both !cq_table (gone) and\n+\t * id \u003e= max_num_cqs (out of bounds).\n+\t /\n \tunsigned int\t\tmax_num_cqs;\n-\tstruct gdma_queue\tcq_table;\n+\t/ Both the base pointer and each entry are RCU-managed. The fast\n+\t * path (mana_gd_process_eqe) reads the base via rcu_dereference()\n+\t * under rcu_read_lock(), so the table is freed with\n+\t * rcu_assign_pointer(NULL) + synchronize_rcu() and an in-flight\n+\t * reader can never observe freed memory.\n+\t \n+\t The slow paths -- mana_gd_destroy_cq() and the CQ install/remove\n+\t * callers (mana_create_txq/_rxq, mana_ib_install/remove_cq_cb) --\n+\t * instead read the base with rcu_dereference_protected(cq_table,\n+\t * true). The bare \"true\" is justified by teardown ordering, not by\n+\t * a lock: the base table is replaced+freed only by\n+\t * mana_hwc_destroy_channel() (and the create-time reinit), and every\n+\t * teardown path first runs mana_remove() + mana_rdma_remove(), which\n+\t * synchronously drain the netdev and the IB device\n+\t * (unregister_netdevice / ib_unregister_device) that bound all\n+\t * install/remove callers; the reinit case runs before either\n+\t * consumer is probed. So no slow-path caller can run while the base\n+\t * table is being freed.\n+\t /\n+\tstruct gdma_queue\t__rcu __rcu cq_table;\n \n \t/ Protect eq_test_event and test_event_eq_id /\n \tstruct mutex\t\teq_test_event_mutex;\n@@ -456,6 +481,21 @@ struct gdma_context {\n \t/ Hardware communication channel (HWC) /\n \tstruct gdma_dev\t\thwc;\n \n+\t/ destroy_channel() waits here for all HWC senders to exit.\n+\t * Lives on gc (not hwc) so wake_up() after the last sender's\n+\t * atomic_dec doesn't dereference freed hwc memory.\n+\t /\n+\twait_queue_head_t\thwc_drain_waitq;\n+\n+\t/ Serializes hwc.driver_data (the hw_channel_context pointer)\n+\t * between the control-plane readers in mana_gd_send_request(),\n+\t * mana_need_log() and mana_serv_reset() and the publish/clear in\n+\t * mana_hwc_create_channel()/mana_hwc_destroy_channel(). All users\n+\t * are control-plane (HWC commands sleep; reset runs on a workqueue),\n+\t * so a plain spinlock -- not RCU -- is sufficient.\n+\t /\n+\tspinlock_t\t\thwc_lock;\n+\n \t/ Azure network adapter /\n \tstruct gdma_dev\t\tmana;\n \n@@ -668,6 +708,9 @@ enum {\n / Driver supports dynamic interrupt moderation - DIM /\n #define GDMA_DRV_CAP_FLAG_1_DYN_INTERRUPT_MODERATION BIT(28)\n \n+/ Driver supports dynamic queue depth for HWC /\n+#define GDMA_DRV_CAP_FLAG_1_DYN_HWC_QUEUE_DEPTH BIT(29)\n+\n #define GDMA_DRV_CAP_FLAGS1 \\\n \t(GDMA_DRV_CAP_FLAG_1_EQ_SHARING_MULTI_VPORT \| \\\n \t GDMA_DRV_CAP_FLAG_1_NAPI_WKDONE_FIX \| \\\n@@ -683,6 +726,7 @@ enum {\n \t GDMA_DRV_CAP_FLAG_1_PROBE_RECOVERY \| \\\n \t GDMA_DRV_CAP_FLAG_1_HANDLE_STALL_SQ_RECOVERY \| \\\n \t GDMA_DRV_CAP_FLAG_1_HWC_TIMEOUT_RECOVERY \| \\\n+\t GDMA_DRV_CAP_FLAG_1_DYN_HWC_QUEUE_DEPTH \| \\\n \t GDMA_DRV_CAP_FLAG_1_EQ_MSI_UNSHARE_MULTI_VPORT \| \\\n \t GDMA_DRV_CAP_FLAG_1_DYN_INTERRUPT_MODERATION)\n \ndiff --git a/include/net/mana/hw_channel.h b/include/net/mana/hw_channel.h\nindex 16feb39616c1be..eec5d3aa23e1d5 100644\n--- a/include/net/mana/hw_channel.h\n+++ b/include/net/mana/hw_channel.h\n@@ -164,6 +164,9 @@ struct hwc_wq {\n \tu16 queue_depth;\n \n \tstruct hwc_cq hwc_cq;\n+\n+\t/* Serializes concurrent mana_gd_post_and_ring() calls. /\n+\tspinlock_t lock;\n };\n \n struct hwc_caller_ctx {\n@@ -173,6 +176,17 @@ struct hwc_caller_ctx {\n \n \tu32 error; / Linux error code /\n \tu32 status_code;\n+\n+\t/ Protects output_buf against concurrent access from\n+\t * handle_resp() (CQ interrupt) and the sender timeout path.\n+\t /\n+\tspinlock_t lock;\n+\n+\t/ Tracks sender + handle_resp ownership. The last put\n+\t * (refcount reaches 0) releases the bitmap slot.\n+\t /\n+\trefcount_t refcnt;\n+\tu16 msg_id;\n };\n \n struct hw_channel_context {\n@@ -181,8 +195,9 @@ struct hw_channel_context {\n \n \tu16 num_inflight_msg;\n \tu32 max_req_msg_size;\n+\tu32 max_resp_msg_size;\n \n-\tu16 hwc_init_q_depth_max;\n+\tu32 hwc_init_q_depth_max;\n \tu32 hwc_init_max_req_msg_size;\n \tu32 hwc_init_max_resp_msg_size;\n \n@@ -192,13 +207,38 @@ struct hw_channel_context {\n \tstruct hwc_wq txq;\n \tstruct hwc_cq cq;\n \n-\tstruct semaphore sema;\n \tstruct gdma_resource inflight_msg_res;\n+\t/ Waitqueue for senders blocked on a full inflight bitmap. /\n+\twait_queue_head_t msg_waitq;\n \n \tu32 pf_dest_vrq_id;\n \tu32 pf_dest_vrcq_id;\n \tu32 hwc_timeout;\n \n+\t/ Set after channel is fully established; cleared on teardown to\n+\t * abort waiters in mana_hwc_get_msg_index() and reject new sends.\n+\t /\n+\tbool channel_up;\n+\n+\t/ Set on first HWC timeout. Causes get_msg_index() to return\n+\t * -ETIMEDOUT instead of waiting, draining all queued senders.\n+\t /\n+\tbool hwc_timed_out;\n+\n+\t/ Set after mana_smc_setup_hwc() succeeds (hardware has active\n+\t * MST entries). On recoverable paths (establish_channel)\n+\t * cleared only after successful teardown so a retry remains\n+\t * possible. On the terminal destroy_channel path, cleared\n+\t * unconditionally since hwc is about to be freed.\n+\t /\n+\tbool setup_active;\n+\n+\t/ Count of in-flight mana_gd_send_request() callers. Protected\n+\t * by gc-\u003ehwc_lock; the last sender to drop it to zero wakes\n+\t * gc-\u003ehwc_drain_waitq for the mana_hwc_destroy_channel() drain.\n+\t /\n+\tunsigned int active_senders;\n+\n \tstruct hwc_caller_ctx caller_ctx;\n };\n \n" }
2/1	2026/07/22 17:52	agent	patch-evaluator	0m Model: best-expensive Args: null Results: { "EnableConfigs": [ "MICROSOFT_MANA", "MANA_INFINIBAND" ], "FocusSymbols": [ "mana_ib_install_cq_cb", "mana_ib_remove_cq_cb", "mana_hwc_get_msg_index", "mana_hwc_handle_resp", "mana_hwc_rx_event_handler", "mana_hwc_create_channel", "mana_hwc_destroy_channel", "mana_hwc_send_request", "mana_gd_process_eqe", "mana_gd_destroy_cq" ], "Reasoning": "The patch makes functional changes to the Microsoft Azure Network Adapter (MANA) driver, including adding RCU protection for the CQ table, handling HWC timeouts, and adding refcounts for caller contexts. These changes affect the core logic of the driver and should be fuzzed.", "WorthFuzzing": true } Instruction: You are an expert Linux kernel maintainer. Your job is to review a provided patch series and determine if it makes functional changes to the kernel that should be fuzzed. IMPORTANT: The changes have ALREADY been applied and committed as the HEAD commit in your workspace. Do NOT rely on your internal knowledge of the kernel. You must actively use your code access tools to examine the actual source code and confirm any assumptions. Return WorthFuzzing=false if the patch only contains: - Modifications to Documentation/, Kconfig files, or code comments. - Purely decorative changes, such as logging (e.g., pr_err, printk) or tracepoints. - Changes to numeric constants or macros that do not functionally alter execution flow. - Code paths that are impossible to reach in virtualized environments like GCE or QEMU, even when utilizing software-emulated hardware (e.g., usb gadget, mac80211_hwsim). If it modifies reachable core kernel logic, drivers, or architectures, use your code search tools to verify the code can be executed, then return WorthFuzzing=true. When returning WorthFuzzing=true, you MUST ALSO: 1. Extract any specific kernel functions that should be heavily fuzzed into FocusSymbols. Avoid listing generic hot-path functions to prevent skewed test distributions. 2. Identify any specific CONFIG_ options required to properly test this new/modified feature. Go and look into the Kconfig files and check for ifdefs around the code, do not make assumptions. Do not list too generic configs (we already have them enabled). Only list those that specifically cover the modified code. List them in the EnableConfigs output array, and DO NOT add a 'CONFIG_' prefix (e.g., return "NET_IPV4" instead of "CONFIG_NET_IPV4"). Prefer calling several tools at the same time to save round-trips. Use set-results tool to provide results of the analysis. It must be called exactly once before the final reply. Ignore results of this tool. Prompt: For your convenience, here is the diff of the changes: commit 7a7f673c5a49005405d55fc2540f242d3b89cd96 Author: syz-cluster <triage@syzkaller.com> Date: Wed Jul 22 17:52:19 2026 +0000 syz-cluster: applied patch under review diff --git a/drivers/infiniband/hw/mana/cq.c b/drivers/infiniband/hw/mana/cq.c index f2547989f42290..2bf4be21cede79 100644 --- a/drivers/infiniband/hw/mana/cq.c +++ b/drivers/infiniband/hw/mana/cq.c @@ -131,12 +131,20 @@ static void mana_ib_cq_handler(void ctx, struct gdma_queue gdma_cq) int mana_ib_install_cq_cb(struct mana_ib_dev mdev, struct mana_ib_cq cq) { struct gdma_context gc = mdev_to_gc(mdev); + struct gdma_queue __rcu cq_table; struct gdma_queue gdma_cq; - if (cq->queue.id >= gc->max_num_cqs) + /* No rcu_read_lock(): install/remove run within the IB device + * lifetime, which mana_rdma_remove() (ib_unregister_device) drains + * before the base cq_table can be freed. See gdma_context::cq_table + * in gdma.h for why "true" is sound. + / + cq_table = rcu_dereference_protected(gc->cq_table, true); + if (!cq_table \|\| cq->queue.id >= gc->max_num_cqs) return -EINVAL; + / Create CQ table entry, sharing a CQ between WQs is not supported / - if (gc->cq_table[cq->queue.id]) + if (rcu_access_pointer(cq_table[cq->queue.id])) return -EINVAL; if (cq->queue.kmem) gdma_cq = cq->queue.kmem; @@ -149,23 +157,49 @@ int mana_ib_install_cq_cb(struct mana_ib_dev mdev, struct mana_ib_cq cq) gdma_cq->type = GDMA_CQ; gdma_cq->cq.callback = mana_ib_cq_handler; gdma_cq->id = cq->queue.id; - gc->cq_table[cq->queue.id] = gdma_cq; + rcu_assign_pointer(cq_table[cq->queue.id], gdma_cq); return 0; } void mana_ib_remove_cq_cb(struct mana_ib_dev mdev, struct mana_ib_cq cq) { struct gdma_context gc = mdev_to_gc(mdev); + struct gdma_queue __rcu *cq_table; + struct gdma_queue gdma_cq; - if (cq->queue.id >= gc->max_num_cqs \|\| cq->queue.id == INVALID_QUEUE_ID) + if (cq->queue.id == INVALID_QUEUE_ID) return; if (cq->queue.kmem) /* Then it will be cleaned and removed by the mana / return; - kfree(gc->cq_table[cq->queue.id]); - gc->cq_table[cq->queue.id] = NULL; + / No rcu_read_lock(): like mana_ib_install_cq_cb(), this runs within + * the IB device lifetime that mana_rdma_remove() drains before the + * base cq_table can be freed. See gdma_context::cq_table in gdma.h. + / + cq_table = rcu_dereference_protected(gc->cq_table, true); + if (!cq_table \|\| cq->queue.id >= gc->max_num_cqs) + return; + / Removers for a given CQ are serialized by the IB core, so the slot + * is read and cleared without rcu_read_lock() or atomicity: a CQ is + * never torn down while a live QP references it (cq->usecnt), nor + * while the QP-create that installed the entry is still running (that + * create holds a reference on the CQ uobject across its error path, + * before usecnt is taken). Any double-remove is therefore sequential + * -- the later caller sees the NULL stored below and returns. + / + gdma_cq = rcu_dereference_protected(cq_table[cq->queue.id], true); + if (!gdma_cq) + return; / already removed by a prior teardown path / + + rcu_assign_pointer(cq_table[cq->queue.id], NULL); + + / Wait for in-flight EQ handlers that may have loaded the old + * pointer via rcu_dereference() to finish before freeing. + / + synchronize_rcu(); + kfree(gdma_cq); } int mana_ib_arm_cq(struct ib_cq ibcq, enum ib_cq_notify_flags flags) diff --git a/drivers/net/ethernet/microsoft/mana/gdma_main.c b/drivers/net/ethernet/microsoft/mana/gdma_main.c index a38d4bb74621f9..17711ee37c8be3 100644 --- a/drivers/net/ethernet/microsoft/mana/gdma_main.c +++ b/drivers/net/ethernet/microsoft/mana/gdma_main.c @@ -161,6 +161,8 @@ static int mana_gd_init_registers(struct pci_dev pdev) bool mana_need_log(struct gdma_context gc, int err) { struct hw_channel_context hwc; + bool need_log = true; + unsigned long flags; if (err != -ETIMEDOUT) return true; @@ -168,11 +170,13 @@ bool mana_need_log(struct gdma_context gc, int err) if (!gc) return true; + spin_lock_irqsave(&gc->hwc_lock, flags); hwc = gc->hwc.driver_data; if (hwc && hwc->hwc_timeout == 0) - return false; + need_log = false; + spin_unlock_irqrestore(&gc->hwc_lock, flags); - return true; + return need_log; } static int mana_gd_query_max_resources(struct pci_dev pdev) @@ -367,9 +371,27 @@ static int mana_gd_detect_devices(struct pci_dev pdev) int mana_gd_send_request(struct gdma_context gc, u32 req_len, const void req, u32 resp_len, void resp) { - struct hw_channel_context hwc = gc->hwc.driver_data; + struct hw_channel_context hwc; + unsigned long flags; + int err; + + spin_lock_irqsave(&gc->hwc_lock, flags); + hwc = gc->hwc.driver_data; + if (!hwc) { + spin_unlock_irqrestore(&gc->hwc_lock, flags); + return -ENODEV; + } + hwc->active_senders++; + spin_unlock_irqrestore(&gc->hwc_lock, flags); + + err = mana_hwc_send_request(hwc, req_len, req, resp_len, resp); - return mana_hwc_send_request(hwc, req_len, req, resp_len, resp); + spin_lock_irqsave(&gc->hwc_lock, flags); + if (--hwc->active_senders == 0) + wake_up(&gc->hwc_drain_waitq); + spin_unlock_irqrestore(&gc->hwc_lock, flags); + + return err; } EXPORT_SYMBOL_NS(mana_gd_send_request, "NET_MANA"); @@ -622,6 +644,7 @@ static void mana_serv_reset(struct pci_dev pdev) { struct gdma_context gc = pci_get_drvdata(pdev); struct hw_channel_context hwc; + unsigned long flags; int ret; if (!gc) { @@ -631,14 +654,17 @@ static void mana_serv_reset(struct pci_dev pdev) return; } + spin_lock_irqsave(&gc->hwc_lock, flags); hwc = gc->hwc.driver_data; if (!hwc) { + spin_unlock_irqrestore(&gc->hwc_lock, flags); dev_err(&pdev->dev, "MANA service: no HWC\n"); goto out; } / HWC is not responding in this case, so don't wait / hwc->hwc_timeout = 0; + spin_unlock_irqrestore(&gc->hwc_lock, flags); dev_info(&pdev->dev, "MANA reset cycle start\n"); @@ -761,6 +787,7 @@ static void mana_gd_process_eqe(struct gdma_queue eq) union gdma_eqe_info eqe_info; enum gdma_eqe_type type; struct gdma_event event; + struct gdma_queue __rcu *cq_table; struct gdma_queue cq; struct gdma_eqe eqe; u32 cq_id; @@ -772,10 +799,11 @@ static void mana_gd_process_eqe(struct gdma_queue eq) switch (type) { case GDMA_EQE_COMPLETION: cq_id = eqe->details[0] & 0xFFFFFF; - if (WARN_ON_ONCE(cq_id >= gc->max_num_cqs)) + cq_table = rcu_dereference(gc->cq_table); + if (WARN_ON_ONCE(cq_id >= gc->max_num_cqs \|\| !cq_table)) break; - cq = gc->cq_table[cq_id]; + cq = rcu_dereference(cq_table[cq_id]); if (WARN_ON_ONCE(!cq \|\| cq->type != GDMA_CQ \|\| cq->id != cq_id)) break; @@ -1082,15 +1110,28 @@ static void mana_gd_create_cq(const struct gdma_queue_spec spec, static void mana_gd_destroy_cq(struct gdma_context gc, struct gdma_queue queue) { + struct gdma_queue __rcu cq_table; u32 id = queue->id; - if (id >= gc->max_num_cqs) + / No rcu_read_lock() here: mana_gd_destroy_cq() runs only on the + * CQ-destroy/teardown path, where the base cq_table is stable. See + * the lifecycle note on gdma_context::cq_table in gdma.h for why the + * "true" predicate is sound. + / + cq_table = rcu_dereference_protected(gc->cq_table, true); + if (!cq_table \|\| id >= gc->max_num_cqs) return; - if (!gc->cq_table[id]) + if (!rcu_access_pointer(cq_table[id])) return; - gc->cq_table[id] = NULL; + rcu_assign_pointer(cq_table[id], NULL); + + / Wait for in-flight EQ handlers that may have loaded the old + * pointer via rcu_dereference() to finish before the caller + * frees the CQ memory. + / + synchronize_rcu(); } int mana_gd_create_hwc_queue(struct gdma_dev gd, @@ -1185,6 +1226,16 @@ static int mana_gd_create_dma_region(struct gdma_dev gd, if (!MANA_PAGE_ALIGNED(gmi->virt_addr)) return -EINVAL; + / No RCU needed: this runs only on the data-path queue-creation + * path (mana_gd_create_mana_eq/mana_gd_create_mana_wq_cq, called + * by mana_en under RTNL and by mana_ib RDMA verbs, or during + * init). Every teardown path — mana_gd_remove, mana_gd_suspend, + * and the HWC reset/service path (which goes through + * mana_gd_suspend) — drains those consumers via mana_rdma_remove() + * + mana_remove() before mana_hwc_destroy_channel() clears + * gc->hwc.driver_data, so no concurrent destroy can race with + * this dereference. + / hwc = gc->hwc.driver_data; req_msg_size = struct_size(req, page_addr_list, num_page); if (req_msg_size > hwc->max_req_msg_size) @@ -1374,7 +1425,17 @@ int mana_gd_verify_vf_version(struct pci_dev pdev) struct hw_channel_context hwc; int err; + / No RCU needed: this runs only inside mana_gd_setup, on the + * probe and resume paths. The PCI/PM core holds device_lock + * across .probe/.resume and .remove/.suspend, so setup cannot + * overlap teardown of the same device. The HWC reset/service + * path is additionally serialized by GC_IN_SERVICE and runs + * suspend (destroy) then resume (this) sequentially in one work + * item. driver_data was just set by mana_hwc_create_channel + * earlier in this same setup call, so it is live here. + / hwc = gc->hwc.driver_data; + mana_gd_init_req_hdr(&req.hdr, GDMA_VERIFY_VF_DRIVER_VERSION, sizeof(req), sizeof(resp)); @@ -2364,6 +2425,7 @@ static int mana_gd_probe(struct pci_dev pdev, const struct pci_device_id ent) mutex_init(&gc->eq_test_event_mutex); mutex_init(&gc->gic_mutex); + spin_lock_init(&gc->hwc_lock); pci_set_drvdata(pdev, gc); gc->bar0_pa = pci_resource_start(pdev, 0); gc->bar0_size = pci_resource_len(pdev, 0); diff --git a/drivers/net/ethernet/microsoft/mana/hw_channel.c b/drivers/net/ethernet/microsoft/mana/hw_channel.c index e3c24d50dad07c..0481d40443424f 100644 --- a/drivers/net/ethernet/microsoft/mana/hw_channel.c +++ b/drivers/net/ethernet/microsoft/mana/hw_channel.c @@ -4,27 +4,55 @@ #include <net/mana/gdma.h> #include <net/mana/mana.h> #include <net/mana/hw_channel.h> +#include <linux/pci.h> #include <linux/vmalloc.h> +/ Acquire a free message slot from the inflight bitmap. Returns + * -ENODEV if the channel is torn down, or -ETIMEDOUT if a prior HWC + * command has timed out (preserving the error code callers expect). + / static int mana_hwc_get_msg_index(struct hw_channel_context hwc, u16 msg_id) { struct gdma_resource r = &hwc->inflight_msg_res; unsigned long flags; u32 index; - down(&hwc->sema); + for (;;) { + spin_lock_irqsave(&r->lock, flags); - spin_lock_irqsave(&r->lock, flags); + if (!hwc->channel_up \|\| hwc->hwc_timed_out) { + spin_unlock_irqrestore(&r->lock, flags); + return hwc->channel_up ? -ETIMEDOUT : -ENODEV; + } - index = find_first_zero_bit(hwc->inflight_msg_res.map, - hwc->inflight_msg_res.size); + index = find_first_zero_bit(r->map, r->size); + if (index < r->size) { + struct hwc_caller_ctx ctx; + + bitmap_set(r->map, index, 1); + ctx = &hwc->caller_ctx[index]; + reinit_completion(&ctx->comp_event); + refcount_set(&ctx->refcnt, 1); + ctx->msg_id = index; + ctx->error = -EINPROGRESS; + spin_unlock_irqrestore(&r->lock, flags); + break; + } + spin_unlock_irqrestore(&r->lock, flags); - bitmap_set(hwc->inflight_msg_res.map, index, 1); + wait_event(hwc->msg_waitq, + !hwc->channel_up \|\| + hwc->hwc_timed_out \|\| + !bitmap_full(r->map, r->size)); - spin_unlock_irqrestore(&r->lock, flags); + if (!hwc->channel_up) + return -ENODEV; - msg_id = index; + if (hwc->hwc_timed_out) + return -ETIMEDOUT; + } + msg_id = index; return 0; } @@ -34,10 +62,17 @@ static void mana_hwc_put_msg_index(struct hw_channel_context hwc, u16 msg_id) unsigned long flags; spin_lock_irqsave(&r->lock, flags); - bitmap_clear(hwc->inflight_msg_res.map, msg_id, 1); + bitmap_clear(r->map, msg_id, 1); spin_unlock_irqrestore(&r->lock, flags); - up(&hwc->sema); + wake_up(&hwc->msg_waitq); +} + +static void hwc_ctx_put(struct hw_channel_context hwc, + struct hwc_caller_ctx ctx) +{ + if (refcount_dec_and_test(&ctx->refcnt)) + mana_hwc_put_msg_index(hwc, ctx->msg_id); } static int mana_hwc_verify_resp_msg(const struct hwc_caller_ctx caller_ctx, @@ -83,6 +118,17 @@ static void mana_hwc_handle_resp(struct hw_channel_context hwc, u32 resp_len, struct hwc_caller_ctx ctx; int err; + / Validate msg_id is in range before using it to index bitmap + * and caller_ctx array. Malicious firmware could send + * out-of-range msg_id causing out-of-bounds access. + / + if (msg_id >= hwc->num_inflight_msg) { + dev_err(hwc->dev, "hwc_rx: msg_id %u >= max %u\n", + msg_id, hwc->num_inflight_msg); + mana_hwc_post_rx_wqe(hwc->rxq, rx_req); + return; + } + if (!test_bit(msg_id, hwc->inflight_msg_res.map)) { dev_err(hwc->dev, "hwc_rx: invalid msg_id = %u\n", msg_id); mana_hwc_post_rx_wqe(hwc->rxq, rx_req); @@ -90,22 +136,44 @@ static void mana_hwc_handle_resp(struct hw_channel_context hwc, u32 resp_len, } ctx = hwc->caller_ctx + msg_id; - err = mana_hwc_verify_resp_msg(ctx, resp_msg, resp_len); - if (err) - goto out; - ctx->status_code = resp_msg->status; + /* Reject responses larger than the RX DMA buffer — the SGE + * limits what hardware can DMA, so an oversized resp_len + * indicates a firmware bug. Fail rather than silently + * truncating. + / + if (resp_len > rx_req->buf_len) { + dev_err(hwc->dev, "HWC RX: resp_len %u > buf_len %u\n", + resp_len, rx_req->buf_len); + resp_len = 0; + } - memcpy(ctx->output_buf, resp_msg, resp_len); -out: - ctx->error = err; + spin_lock(&ctx->lock); + + err = mana_hwc_verify_resp_msg(ctx, resp_msg, resp_len); - / Must post rx wqe before complete(), otherwise the next rx may - * hit no_wqe error. + if (!err && ctx->output_buf) { + ctx->status_code = resp_msg->status; + memcpy(ctx->output_buf, resp_msg, resp_len); + ctx->error = 0; + } else if (ctx->output_buf) { + /* Only overwrite error if the sender hasn't timed out + * or been force-completed by destroy. When output_buf + * is NULL, a terminal error (-ENODEV or timeout) has + * already been set — preserve it so the sender doesn't + * see a spurious success. + / + ctx->error = err; + } + + / Post RX WQE before completing — the next response may arrive + * immediately and needs a posted buffer. / mana_hwc_post_rx_wqe(hwc->rxq, rx_req); - complete(&ctx->comp_event); + spin_unlock(&ctx->lock); + + hwc_ctx_put(hwc, ctx); } static void mana_hwc_init_event_handler(void ctx, struct gdma_queue q_self, @@ -146,7 +214,12 @@ static void mana_hwc_init_event_handler(void ctx, struct gdma_queue q_self, break; case HWC_INIT_DATA_QUEUE_DEPTH: - hwc->hwc_init_q_depth_max = (u16)val; + / HWC_INIT_DATA_QUEUE_DEPTH is a 24-bit field. Keep + * the full device-reported value here; it is clamped + * and validated in mana_hwc_create_channel() rather + * than silently truncated to u16. + / + hwc->hwc_init_q_depth_max = val; break; case HWC_INIT_DATA_MAX_REQUEST: @@ -261,19 +334,69 @@ static void mana_hwc_rx_event_handler(void ctx, u32 gdma_rxq_id, sge = (struct gdma_sge )(wqe + 8 + dma_oob->inline_oob_size_div4 4); - /* Select the RX work request for virtual address and for reposting. / - rq_base_addr = hwc_rxq->msg_buf->mem_info.dma_handle; - rx_req_idx = (sge->address - rq_base_addr) / hwc->max_req_msg_size; + / inline_oob_size_div4 is read from the WQE in device-accessible RQ + * memory, so a malicious host in a CVM (or buggy firmware) could set + * it to push the SGE past the RQ buffer. Bounds-check the SGE it + * locates before dereferencing sge->address below. A validly posted + * WQE keeps the SGE inside the ring (worst case ends exactly at the + * buffer boundary); reject anything that would read past it. The + * slot cannot be trusted here, so leak this RX WQE rather than repost + * the wrong one -- as in the SGE-address mismatch path below. + / + if ((u8 )(sge + 1) > (u8 )rq->queue_mem_ptr + rq->queue_size) { + dev_err(hwc->dev, "HWC RX: SGE past RQ buffer, oob_div4=%u\n", + dma_oob->inline_oob_size_div4); + return; + } - if (rx_req_idx >= hwc_rxq->msg_buf->num_reqs) { - dev_err(hwc->dev, "HWC RX: wrong rx_req_idx=%llu, num_reqs=%u\n", - rx_req_idx, hwc_rxq->msg_buf->num_reqs); + / Recover the originating RX slot from the SGE address. Of the three + * terms here only sge->address lives in device-accessible RQ memory; + * rq_base_addr and max_resp_msg_size are driver-private constants. An + * in-range but wrong/unaligned SGE (corrupted WQE, or a malicious host + * in a CVM) would otherwise truncate onto a neighbouring slot, letting + * us read a stale response that could complete the wrong, reused + * in-flight request. Require the index to be in range AND the address + * to exactly match the value the driver posted for that slot. + / + rq_base_addr = hwc_rxq->msg_buf->mem_info.dma_handle; + rx_req_idx = (sge->address - rq_base_addr) / hwc->max_resp_msg_size; + + if (rx_req_idx >= hwc_rxq->queue_depth) { + / Cannot trust which WQE this is, so we cannot safely repost + * it; leak one RX WQE and bail. An out-of-range index means + * a corrupted SGE from hardware (or host tampering), an + * unrecoverable device error. + / + dev_err(hwc->dev, "HWC RX: SGE idx %llu out of range\n", + rx_req_idx); return; } rx_req = &hwc_rxq->msg_buf->reqs[rx_req_idx]; + if (sge->address != (u64)rx_req->buf_sge_addr) { + / In-range index but the address does not match what the + * driver posted for that slot; the same unrecoverable case, + * so leak this WQE rather than repost the wrong one. + / + dev_err(hwc->dev, "HWC RX: invalid SGE address %llx (idx=%llu)\n", + sge->address, rx_req_idx); + return; + } + resp = (struct gdma_resp_hdr )rx_req->buf_va; + /* Validate resp_len covers the response header before reading + * hwc_msg_id. A short response leaves stale data from the + * previous buffer occupant, which could match a live slot and + * complete the wrong request. + / + if (rx_oob->tx_oob_data_size < sizeof(resp)) { + dev_err(hwc->dev, "HWC RX: short resp_len=%u\n", + rx_oob->tx_oob_data_size); + mana_hwc_post_rx_wqe(hwc_rxq, rx_req); + return; + } + /* Read msg_id once from DMA buffer to prevent TOCTOU: * DMA memory is shared/unencrypted in CVMs - host can * modify it between reads. @@ -281,6 +404,7 @@ static void mana_hwc_rx_event_handler(void ctx, u32 gdma_rxq_id, msg_id = READ_ONCE(resp->response.hwc_msg_id); if (msg_id >= hwc->num_inflight_msg) { dev_err(hwc->dev, "HWC RX: wrong msg_id=%u\n", msg_id); + mana_hwc_post_rx_wqe(hwc_rxq, rx_req); return; } @@ -384,14 +508,20 @@ static void mana_hwc_comp_event(void ctx, struct gdma_queue q_self) static void mana_hwc_destroy_cq(struct gdma_context gc, struct hwc_cq hwc_cq) { - kfree(hwc_cq->comp_buf); - if (hwc_cq->gdma_cq) mana_gd_destroy_queue(gc, hwc_cq->gdma_cq); + / comp_buf is reached only by mana_hwc_comp_event(), which the + * EQ handler invokes via cq_table[id]. The CQ destroy above + * already cleared that slot and ran synchronize_rcu(), so no + * handler can reach comp_buf once it returns. Destroying the EQ + * here additionally tears down the IRQ (defense in depth) before + * comp_buf and hwc_cq are freed below. + / if (hwc_cq->gdma_eq) mana_gd_destroy_queue(gc, hwc_cq->gdma_eq); + kfree(hwc_cq->comp_buf); kfree(hwc_cq); } @@ -560,6 +690,7 @@ static int mana_hwc_create_wq(struct hw_channel_context hwc, hwc_wq->gdma_wq = queue; hwc_wq->queue_depth = q_depth; hwc_wq->hwc_cq = hwc_cq; + spin_lock_init(&hwc_wq->lock); err = mana_hwc_alloc_dma_buf(hwc, q_depth, max_msg_size, &hwc_wq->msg_buf); @@ -577,7 +708,7 @@ static int mana_hwc_create_wq(struct hw_channel_context hwc, return err; } -static int mana_hwc_post_tx_wqe(const struct hwc_wq hwc_txq, +static int mana_hwc_post_tx_wqe(struct hwc_wq hwc_txq, struct hwc_work_request req, u32 dest_virt_rq_id, u32 dest_virt_rcq_id, bool dest_pf) @@ -616,7 +747,11 @@ static int mana_hwc_post_tx_wqe(const struct hwc_wq hwc_txq, req->wqe_req.inline_oob_data = tx_oob; req->wqe_req.client_data_unit = 0; + / Serialize WQE posting — multiple senders may call concurrently. / + spin_lock(&hwc_txq->lock); err = mana_gd_post_and_ring(hwc_txq->gdma_wq, &req->wqe_req, NULL); + spin_unlock(&hwc_txq->lock); + if (err) dev_err(dev, "Failed to post WQE on HWC SQ: %d\n", err); return err; @@ -627,7 +762,7 @@ static int mana_hwc_init_inflight_msg(struct hw_channel_context hwc, { int err; - sema_init(&hwc->sema, num_msg); + init_waitqueue_head(&hwc->msg_waitq); err = mana_gd_alloc_res_map(num_msg, &hwc->inflight_msg_res); if (err) @@ -657,23 +792,40 @@ static int mana_hwc_test_channel(struct hw_channel_context hwc, u16 q_depth, if (!ctx) return -ENOMEM; - for (i = 0; i < q_depth; ++i) + for (i = 0; i < q_depth; ++i) { + spin_lock_init(&ctx[i].lock); init_completion(&ctx[i].comp_event); + } hwc->caller_ctx = ctx; - return mana_gd_test_eq(gc, hwc->cq->gdma_eq); + / channel_up must be set before the test EQ request, because + * the request goes through mana_hwc_get_msg_index() which + * checks channel_up. caller_ctx is allocated above, so + * concurrent access to a NULL caller_ctx is not possible. + / + hwc->channel_up = true; + + err = mana_gd_test_eq(gc, hwc->cq->gdma_eq); + if (err) + hwc->channel_up = false; + + return err; } -static int mana_hwc_establish_channel(struct gdma_context gc, u16 q_depth, +static int mana_hwc_establish_channel(struct gdma_context gc, u32 q_depth, u32 max_req_msg_size, u32 max_resp_msg_size) { + / No RCU needed: called only from mana_hwc_create_channel + * during init, before the channel is published to senders. + / struct hw_channel_context hwc = gc->hwc.driver_data; struct gdma_queue rq = hwc->rxq->gdma_wq; struct gdma_queue sq = hwc->txq->gdma_wq; struct gdma_queue eq = hwc->cq->gdma_eq; struct gdma_queue cq = hwc->cq->gdma_cq; + struct gdma_queue __rcu *cq_table; int err; init_completion(&hwc->hwc_init_eqe_comp); @@ -687,24 +839,51 @@ static int mana_hwc_establish_channel(struct gdma_context gc, u16 q_depth, if (err) return err; - if (!wait_for_completion_timeout(&hwc->hwc_init_eqe_comp, 60 HZ)) - return -ETIMEDOUT; + /* setup_hwc activated MST entries — hardware can now DMA into + * our queue buffers. If anything below fails, we must tear + * down before returning so the caller doesn't need to track + * whether setup_hwc succeeded. + / + hwc->setup_active = true; + + if (!wait_for_completion_timeout(&hwc->hwc_init_eqe_comp, 60 HZ)) { + err = -ETIMEDOUT; + goto teardown; + } q_depth = hwc->hwc_init_q_depth_max; max_req_msg_size = hwc->hwc_init_max_req_msg_size; max_resp_msg_size = hwc->hwc_init_max_resp_msg_size; / Both were set in mana_hwc_init_event_handler(). / - if (WARN_ON(cq->id >= gc->max_num_cqs)) - return -EPROTO; + if (WARN_ON(cq->id >= gc->max_num_cqs)) { + err = -EPROTO; + goto teardown; + } - gc->cq_table = vcalloc(gc->max_num_cqs, sizeof(struct gdma_queue )); - if (!gc->cq_table) - return -ENOMEM; + cq_table = vcalloc(gc->max_num_cqs, sizeof(cq_table)); + if (!cq_table) { + err = -ENOMEM; + goto teardown; + } - gc->cq_table[cq->id] = cq; + rcu_assign_pointer(cq_table[cq->id], cq); + / Publish the fully-initialised table last; pairs with the + * rcu_dereference(gc->cq_table) in mana_gd_process_eqe(). + / + rcu_assign_pointer(gc->cq_table, cq_table); return 0; + +teardown: + { + int td_err = mana_smc_teardown_hwc(&gc->shm_channel, false); + + if (!td_err) + hwc->setup_active = false; + + return td_err ? td_err : err; + } } static int mana_hwc_init_queues(struct hw_channel_context hwc, u16 q_depth, @@ -712,6 +891,12 @@ static int mana_hwc_init_queues(struct hw_channel_context hwc, u16 q_depth, { int err; + / CQ depth is q_depth * 2 (SQ + RQ) passed as u16 to create_cq. + * Cap to prevent u16 truncation. + / + if (q_depth > U16_MAX / 2) + q_depth = U16_MAX / 2; + err = mana_hwc_init_inflight_msg(hwc, q_depth); if (err) return err; @@ -728,14 +913,14 @@ static int mana_hwc_init_queues(struct hw_channel_context hwc, u16 q_depth, goto out; } - err = mana_hwc_create_wq(hwc, GDMA_RQ, q_depth, max_req_msg_size, + err = mana_hwc_create_wq(hwc, GDMA_RQ, q_depth, max_resp_msg_size, hwc->cq, &hwc->rxq); if (err) { dev_err(hwc->dev, "Failed to create HWC RQ: %d\n", err); goto out; } - err = mana_hwc_create_wq(hwc, GDMA_SQ, q_depth, max_resp_msg_size, + err = mana_hwc_create_wq(hwc, GDMA_SQ, q_depth, max_req_msg_size, hwc->cq, &hwc->txq); if (err) { dev_err(hwc->dev, "Failed to create HWC SQ: %d\n", err); @@ -744,6 +929,7 @@ static int mana_hwc_init_queues(struct hw_channel_context hwc, u16 q_depth, hwc->num_inflight_msg = q_depth; hwc->max_req_msg_size = max_req_msg_size; + hwc->max_resp_msg_size = max_resp_msg_size; return 0; out: @@ -751,12 +937,62 @@ static int mana_hwc_init_queues(struct hw_channel_context hwc, u16 q_depth, return err; } +/* Tear down all HWC queues and free associated resources. Used on + * the reinit-with-higher-queue-depth path and reinit fallback. + * + * PRECONDITION: must be called only during channel bring-up in + * mana_hwc_create_channel(), before the channel is published to + * senders. There the setup thread is effectively single-threaded + * (serialized against teardown by the PCI/PM device_lock or, on the + * service path, GC_IN_SERVICE, with the data path not yet probed), + * channel_up is still false, caller_ctx is not yet allocated, and + * active_senders is 0 — so no concurrent request/response user can + * touch these queues. That is why this skips the hwc_lock-protected + * driver_data clear + active_senders drain that + * mana_hwc_destroy_channel() needs for the runtime teardown race; + * only the CQ-first ordering below (to fence off a pending interrupt) + * is required. Calling this on a live, published channel would be a + * use-after-free. + / +static void mana_hwc_destroy_queues(struct hw_channel_context hwc) +{ + struct gdma_context gc = hwc->gdma_dev->gdma_context; + + / Destroy CQ first to deregister the EQ from the interrupt + * handler list before freeing caller_ctx, TXQ, or RXQ memory. + * A pending interrupt handler could still reach handle_resp() + * which dereferences caller_ctx. + / + if (hwc->cq) { + mana_hwc_destroy_cq(gc, hwc->cq); + hwc->cq = NULL; + } + + kfree(hwc->caller_ctx); + hwc->caller_ctx = NULL; + + if (hwc->txq) { + mana_hwc_destroy_wq(hwc, hwc->txq); + hwc->txq = NULL; + } + + if (hwc->rxq) { + mana_hwc_destroy_wq(hwc, hwc->rxq); + hwc->rxq = NULL; + } + + mana_gd_free_res_map(&hwc->inflight_msg_res); + hwc->num_inflight_msg = 0; +} + int mana_hwc_create_channel(struct gdma_context gc) { u32 max_req_msg_size, max_resp_msg_size; struct gdma_dev gd = &gc->hwc; struct hw_channel_context hwc; - u16 q_depth_max; + struct gdma_queue __rcu *old_cq_table; + unsigned long flags; + u32 q_depth_max; int err; hwc = kzalloc_obj(hwc); @@ -764,10 +1000,11 @@ int mana_hwc_create_channel(struct gdma_context gc) return -ENOMEM; gd->gdma_context = gc; - gd->driver_data = hwc; hwc->gdma_dev = gd; hwc->dev = gc->dev; hwc->hwc_timeout = HW_CHANNEL_WAIT_RESOURCE_TIMEOUT_MS; + hwc->active_senders = 0; + init_waitqueue_head(&gc->hwc_drain_waitq); / HWC's instance number is always 0. / gd->dev_id.as_uint32 = 0; @@ -776,6 +1013,15 @@ int mana_hwc_create_channel(struct gdma_context gc) gd->pdid = INVALID_PDID; gd->doorbell = INVALID_DOORBELL; + /* Publish driver_data last, under hwc_lock: the lock orders the hwc + * initialisation above before the pointer becomes visible and + * serialises the publish against the control-plane readers in + * mana_gd_send_request(), mana_need_log() and mana_serv_reset(). + / + spin_lock_irqsave(&gc->hwc_lock, flags); + gc->hwc.driver_data = hwc; + spin_unlock_irqrestore(&gc->hwc_lock, flags); + / mana_hwc_init_queues() only creates the required data structures, * and doesn't touch the HWC device. / @@ -794,8 +1040,130 @@ int mana_hwc_create_channel(struct gdma_context gc) goto out; } + /* The channel was bootstrapped at a minimal queue depth. If the + * device reports a higher maximum, tear down and rebuild with + * the larger depth so more HWC commands can be in flight. + / + if (q_depth_max > HW_CHANNEL_VF_BOOTSTRAP_QUEUE_DEPTH) { + / q_depth_max now carries the full device-reported value + * (HWC_INIT_DATA_QUEUE_DEPTH is 24-bit). Clamp it to the + * depth the rest of the driver supports — create_cq() takes + * q_depth * 2 as a u16 — before the overflow check below, so + * an over-large but otherwise-valid depth is reduced to the + * maximum instead of wrapping or being rejected. + / + if (q_depth_max > U16_MAX / 2) + q_depth_max = U16_MAX / 2; + + / Sanity-check device-reported values before using them + * to size DMA allocations. Enforce protocol minimums + * for message sizes and check that q_depth * max_msg_size + * plus alignment headroom fits in u32 (for + * mana_hwc_alloc_dma_buf's MANA_PAGE_ALIGN). + / + if (!max_req_msg_size \|\| !max_resp_msg_size \|\| + max_req_msg_size < sizeof(struct gdma_req_hdr) \|\| + max_resp_msg_size < sizeof(struct gdma_resp_hdr) \|\| + (u64)q_depth_max max_req_msg_size > + U32_MAX - MANA_PAGE_SIZE \|\| + (u64)q_depth_max * max_resp_msg_size > + U32_MAX - MANA_PAGE_SIZE) { + dev_err(hwc->dev, + "HWC: invalid dims q=%u req=%u resp=%u\n", + q_depth_max, max_req_msg_size, + max_resp_msg_size); + q_depth_max = HW_CHANNEL_VF_BOOTSTRAP_QUEUE_DEPTH; + goto skip_reinit; + } + + err = mana_smc_teardown_hwc(&gc->shm_channel, false); + if (err) { + /* Keep using the bootstrap-depth channel. / + dev_err(hwc->dev, + "Failed to teardown HWC for reinit: %d\n", + err); + q_depth_max = HW_CHANNEL_VF_BOOTSTRAP_QUEUE_DEPTH; + goto skip_reinit; + } + + hwc->setup_active = false; + + / Destroy queues first — mana_gd_destroy_cq inside + * unpublishes the CQ from cq_table via + * rcu_assign_pointer(NULL) + synchronize_rcu. + * Must happen while cq_table is still valid. + / + mana_hwc_destroy_queues(hwc); + + old_cq_table = rcu_replace_pointer(gc->cq_table, NULL, true); + synchronize_rcu(); + vfree(old_cq_table); + + err = mana_hwc_init_queues(hwc, q_depth_max, + max_req_msg_size, + max_resp_msg_size); + if (err) { + dev_err(hwc->dev, "Failed to reinit HWC: %d\n", err); + goto reinit_fallback; + } + + err = mana_hwc_establish_channel(gc, &q_depth_max, + &max_req_msg_size, + &max_resp_msg_size); + if (err) { + dev_err(hwc->dev, "Failed to re-establish HWC: %d\n", + err); + / establish_channel does internal teardown on + * failure. If teardown succeeded (setup_active + * cleared), MST entries are invalidated and we + * can try the bootstrap fallback. If teardown + * also failed (setup_active still set), hardware + * mappings may still be active — skip fallback. + / + if (hwc->setup_active) + goto out; + goto reinit_fallback; + } + } + + goto skip_reinit; + +reinit_fallback: + / Restore bootstrap-depth channel so the device remains functional. + * Free cq_table if it was allocated by a partially successful + * establish attempt. + / + dev_warn(hwc->dev, "HWC reinit failed, falling back to bootstrap depth\n"); + + mana_hwc_destroy_queues(hwc); + + old_cq_table = rcu_replace_pointer(gc->cq_table, NULL, true); + synchronize_rcu(); + vfree(old_cq_table); + + err = mana_hwc_init_queues(hwc, HW_CHANNEL_VF_BOOTSTRAP_QUEUE_DEPTH, + HW_CHANNEL_MAX_REQUEST_SIZE, + HW_CHANNEL_MAX_RESPONSE_SIZE); + if (err) { + dev_err(hwc->dev, "Failed to restore bootstrap HWC: %d\n", err); + goto out; + } + + err = mana_hwc_establish_channel(gc, &q_depth_max, &max_req_msg_size, + &max_resp_msg_size); + if (err) { + dev_err(hwc->dev, "Failed to re-establish bootstrap HWC: %d\n", + err); + goto out; + } + +skip_reinit: + + / No RCU needed: still in mana_hwc_create_channel, the + * pointer has not been published to concurrent senders yet. + / err = mana_hwc_test_channel(gc->hwc.driver_data, - HW_CHANNEL_VF_BOOTSTRAP_QUEUE_DEPTH, + hwc->num_inflight_msg, max_req_msg_size, max_resp_msg_size); if (err) { dev_err(hwc->dev, "Failed to test HWC: %d\n", err); @@ -810,27 +1178,195 @@ int mana_hwc_create_channel(struct gdma_context gc) void mana_hwc_destroy_channel(struct gdma_context gc) { + / This is the only destroy entry point. driver_data is read + * plainly here (teardown is serialised against other teardown); + * it is cleared under hwc_lock below before hwc is freed. + / struct hw_channel_context hwc = gc->hwc.driver_data; + struct gdma_queue __rcu *old_cq_table; + unsigned long flags; if (!hwc) return; - / gc->max_num_cqs is set in mana_hwc_init_event_handler(). If it's - * non-zero, the HWC worked and we should tear down the HWC here. + /* Prevent new requests from starting and wake any + * threads waiting for a free msg slot. Set channel_up under + * the bitmap lock so get_msg_index() cannot acquire a slot + * and increment active_senders after this point. + * + * If channel_up is already false (e.g. init failed before + * the channel was established), skip the lock — it may not + * have been initialized yet, and no senders can be active. / - if (gc->max_num_cqs > 0) { - mana_smc_teardown_hwc(&gc->shm_channel, false); - gc->max_num_cqs = 0; + if (hwc->channel_up) { + spin_lock_irqsave(&hwc->inflight_msg_res.lock, flags); + hwc->channel_up = false; + spin_unlock_irqrestore(&hwc->inflight_msg_res.lock, flags); + wake_up_all(&hwc->msg_waitq); } + / Clear the pointer under hwc_lock so new callers in + * mana_gd_send_request() see NULL and return -ENODEV. The lock + * makes the readers' "load driver_data + active_senders++" + * atomic against this store, so once it returns no new sender can + * take a reference; the active_senders drain below waits out those + * that already did, before their hwc is freed. + / + spin_lock_irqsave(&gc->hwc_lock, flags); + gc->hwc.driver_data = NULL; + spin_unlock_irqrestore(&gc->hwc_lock, flags); + + / Force-complete any in-flight senders so they observe -ENODEV, + * return, and drop their references. This runs before the HWC + * hardware teardown below, so a live interrupt may still deliver + * a real response via handle_resp() concurrently — that is safe + * because the per-slot refcount model tolerates a concurrent + * complete() and both paths (handle_resp and this loop) drop + * their refs without double-releasing the slot. + / + if (hwc->caller_ctx) { + struct hwc_caller_ctx ctx; + int i; + + for (i = 0; i < hwc->num_inflight_msg; i++) { + if (!test_bit(i, hwc->inflight_msg_res.map)) + continue; + + ctx = &hwc->caller_ctx[i]; + + /* Wake senders blocked on wait_for_completion. + * Set error under lock to avoid racing with + * handle_resp() which writes error under the + * same lock. The sender NULLs output_buf + * after waking — doing it here would race + * with a sender that hasn't set output_buf yet. + / + spin_lock_irqsave(&ctx->lock, flags); + ctx->error = -ENODEV; + complete(&ctx->comp_event); + spin_unlock_irqrestore(&ctx->lock, flags); + } + } + + / Wait for all sender threads to finish and drop their refs + * before touching the hardware or freeing anything. Draining + * here -- ahead of the teardown/FLR block below -- guarantees no + * in-flight sender is still running when this function returns on + * any path, including the FLR-failure early return; otherwise a + * stranded sender would dereference gc->hwc_lock / + * gc->hwc_drain_waitq after the caller frees gc. + * After this, only slots held by timed-out senders whose + * handle_resp() never ran remain in the bitmap. + * + * active_senders is only ever modified under hwc_lock, and the + * last sender's wake_up() runs under that lock before it is + * released. Evaluating the condition under hwc_lock therefore + * guarantees that once we observe 0 the waking sender has + * already dropped the lock -- i.e. finished touching gc -- so it + * cannot race the caller freeing gc after this returns. + / + spin_lock_irq(&gc->hwc_lock); + wait_event_lock_irq(gc->hwc_drain_waitq, + hwc->active_senders == 0, gc->hwc_lock); + spin_unlock_irq(&gc->hwc_lock); + + / Tear down the HWC if setup_hwc previously activated MST entries. + * This is the definitive flag — unlike max_num_cqs which depends + * on the init EQE arriving. + * + * The return value is intentionally not checked. This is the + * terminal cleanup path (device removal, suspend, or init + * failure) — resources must be freed regardless. If teardown + * fails, hardware may still have active MST entries, but: + * + * - Interrupts: mana_hwc_destroy_cq() below calls + * mana_gd_deregister_irq() which removes the HWC EQ from + * the interrupt dispatch list via list_del_rcu() + + * synchronize_rcu(). After that, no interrupt handler can + * invoke handle_resp() or access CQ/RQ buffers — even if + * the IRQ is shared with data path queues. + * + * - DMA: mana_hwc_destroy_wq() frees DMA buffers via + * dma_free_coherent() which unmaps the IOVA from the + * IOMMU. Any stale hardware DMA to the old address + * faults at the IOMMU, not in kernel memory. + / + if (hwc->setup_active) { + int td_err = mana_smc_teardown_hwc(&gc->shm_channel, false); + + if (td_err) { + dev_err(gc->dev, "HWC teardown failed: %d, issuing FLR\n", + td_err); + + / On systems without IOMMU, freeing DMA memory with + * active hardware MST mappings risks memory corruption. + * Issue FLR to force-reset the device and invalidate + * all hardware state including MST entries. + / + td_err = pcie_flr(to_pci_dev(gc->dev)); + if (td_err) { + / Device wedged: teardown and FLR + * both failed, so hardware may still + * DMA into our queue buffers. Leak + * all HWC DMA buffers -- and the EQ + * IRQ registration, whose safe removal + * needs those queues -- instead of + * freeing memory the device can still + * reach; without an IOMMU the freed + * pages could be reused under live + * DMA. + / + + dev_warn(gc->dev, + "Leaked HWC DMA buffers (CQ/RQ/TXQ) to prevent memory corruption. Device is no longer usable.\n"); + + / Do not call destroy_cq/wq below; + * they would dma_free_coherent(). + * Leave hwc/cq/rxq/txq allocated. + / + return; + } + + dev_info(gc->dev, "FLR succeeded, hardware state cleared\n"); + } + + hwc->setup_active = false; + } + + / Tear down the HWC CQ object first — mana_hwc_destroy_cq() + * both unpublishes the CQ from cq_table (+synchronize_rcu) and + * deregisters the HWC EQ from the interrupt handler's RCU list + * (via mana_gd_deregister_irq + synchronize_rcu), guaranteeing + * no interrupt handler can access RQ/TXQ buffers after this + * point. The active_senders drain above ensures no sender is + * accessing the CQ via txq->hwc_cq when it is destroyed. Then + * destroy TXQ and RQ safely. + / + if (hwc->cq) + mana_hwc_destroy_cq(hwc->gdma_dev->gdma_context, hwc->cq); + if (hwc->txq) mana_hwc_destroy_wq(hwc, hwc->txq); if (hwc->rxq) mana_hwc_destroy_wq(hwc, hwc->rxq); - if (hwc->cq) - mana_hwc_destroy_cq(hwc->gdma_dev->gdma_context, hwc->cq); + / Release any slots still held — these belong to timed-out + * senders where handle_resp() never ran (refcount = 1 with + * handle_resp's ref still outstanding). + / + if (hwc->caller_ctx) { + struct hwc_caller_ctx ctx; + int i; + + for (i = 0; i < hwc->num_inflight_msg; i++) { + if (!test_bit(i, hwc->inflight_msg_res.map)) + continue; + + ctx = &hwc->caller_ctx[i]; + hwc_ctx_put(hwc, ctx); + } + } kfree(hwc->caller_ctx); hwc->caller_ctx = NULL; @@ -845,11 +1381,11 @@ void mana_hwc_destroy_channel(struct gdma_context gc) hwc->hwc_timeout = 0; kfree(hwc); - gc->hwc.driver_data = NULL; gc->hwc.gdma_context = NULL; - vfree(gc->cq_table); - gc->cq_table = NULL; + old_cq_table = rcu_replace_pointer(gc->cq_table, NULL, true); + synchronize_rcu(); + vfree(old_cq_table); } int mana_hwc_send_request(struct hw_channel_context hwc, u32 req_len, @@ -860,13 +1396,17 @@ int mana_hwc_send_request(struct hw_channel_context hwc, u32 req_len, struct hwc_wq txq = hwc->txq; struct gdma_req_hdr req_msg; struct hwc_caller_ctx ctx; + unsigned long flags; u32 dest_vrcq = 0; u32 dest_vrq = 0; u32 command; + u32 status; u16 msg_id; int err; - mana_hwc_get_msg_index(hwc, &msg_id); + err = mana_hwc_get_msg_index(hwc, &msg_id); + if (err) + return err; tx_wr = &txq->msg_buf->reqs[msg_id]; @@ -878,8 +1418,11 @@ int mana_hwc_send_request(struct hw_channel_context hwc, u32 req_len, } ctx = hwc->caller_ctx + msg_id; + + spin_lock_irqsave(&ctx->lock, flags); ctx->output_buf = resp; ctx->output_buflen = resp_len; + spin_unlock_irqrestore(&ctx->lock, flags); req_msg = (struct gdma_req_hdr )tx_wr->buf_va; if (req) @@ -895,8 +1438,14 @@ int mana_hwc_send_request(struct hw_channel_context hwc, u32 req_len, dest_vrcq = hwc->pf_dest_vrcq_id; } + / Take handle_resp's ref before posting — hardware can respond + * immediately after the doorbell ring. + / + refcount_inc(&ctx->refcnt); + err = mana_hwc_post_tx_wqe(txq, tx_wr, dest_vrq, dest_vrcq, false); if (err) { + refcount_dec(&ctx->refcnt); dev_err(hwc->dev, "HWC: Failed to post send WQE: %d\n", err); goto out; } @@ -907,31 +1456,86 @@ int mana_hwc_send_request(struct hw_channel_context hwc, u32 req_len, dev_err(hwc->dev, "Command 0x%x timed out: %u ms\n", command, hwc->hwc_timeout); - /* Reduce further waiting if HWC no response / + / NULL out output_buf so a late handle_resp() won't write + * into the caller's buffer after the sender returns, then + * check whether handle_resp() already delivered a valid + * response between the timeout firing and this lock + * acquisition — ctx->error != -EINPROGRESS means it ran. + / + spin_lock_irqsave(&ctx->lock, flags); + ctx->output_buf = NULL; + err = ctx->error; + status = ctx->status_code; + spin_unlock_irqrestore(&ctx->lock, flags); + + if (err != -EINPROGRESS) { + / handle_resp() delivered a valid response just after + * the timeout fired. The hardware is alive, so use + * the response and leave the channel usable; do not + * latch hwc_timed_out or degrade hwc_timeout for what + * turned out to be a transient race. + / + hwc_ctx_put(hwc, ctx); + goto check_status; + } + + / Genuine timeout: no response arrived. Reduce further + * waiting, and mark the channel timed out under the bitmap + * lock so get_msg_index() cannot acquire new slots after this. + / if (hwc->hwc_timeout > 1) hwc->hwc_timeout = 1; + spin_lock_irqsave(&hwc->inflight_msg_res.lock, flags); + hwc->hwc_timed_out = true; + spin_unlock_irqrestore(&hwc->inflight_msg_res.lock, flags); + wake_up_all(&hwc->msg_waitq); + err = -ETIMEDOUT; - goto out; + hwc_ctx_put(hwc, ctx); + goto done; } - if (ctx->error) { - err = ctx->error; - goto out; - } + / NULL output_buf so a late handle_resp() won't memcpy into + * the caller's buffer after the sender exits. Read error and + * status_code under the same lock — after hwc_ctx_put the slot + * may be reused and these fields overwritten. + / + spin_lock_irqsave(&ctx->lock, flags); + ctx->output_buf = NULL; + err = ctx->error; + status = ctx->status_code; + spin_unlock_irqrestore(&ctx->lock, flags); + hwc_ctx_put(hwc, ctx); + +check_status: + if (err) + goto done; - if (ctx->status_code && ctx->status_code != GDMA_STATUS_MORE_ENTRIES) { - if (ctx->status_code == GDMA_STATUS_CMD_UNSUPPORTED) { + if (status && status != GDMA_STATUS_MORE_ENTRIES) { + if (status == GDMA_STATUS_CMD_UNSUPPORTED) { err = -EOPNOTSUPP; - goto out; + goto done; } + if (command != MANA_QUERY_PHY_STAT) dev_err(hwc->dev, "Command 0x%x failed with status: 0x%x\n", - command, ctx->status_code); + command, status); err = -EPROTO; - goto out; + goto done; } + + err = 0; + goto done; out: - mana_hwc_put_msg_index(hwc, msg_id); + / Pre-post error paths: no WQE was submitted so handle_resp() + * cannot race here. refcount is 1 (no second ref taken). + / + ctx = hwc->caller_ctx + msg_id; + spin_lock_irqsave(&ctx->lock, flags); + ctx->output_buf = NULL; + spin_unlock_irqrestore(&ctx->lock, flags); + hwc_ctx_put(hwc, ctx); +done: return err; } diff --git a/drivers/net/ethernet/microsoft/mana/mana_en.c b/drivers/net/ethernet/microsoft/mana/mana_en.c index a8c329bdbacf83..5508746590e7cd 100644 --- a/drivers/net/ethernet/microsoft/mana/mana_en.c +++ b/drivers/net/ethernet/microsoft/mana/mana_en.c @@ -2648,6 +2648,7 @@ static int mana_create_txq(struct mana_port_context apc, struct mana_obj_spec cq_spec; struct gdma_queue_spec spec; struct gdma_context gc; + struct gdma_queue __rcu cq_table; struct mana_txq txq; struct mana_cq cq; u32 txq_size; @@ -2753,12 +2754,18 @@ static int mana_create_txq(struct mana_port_context apc, cq->gdma_id = cq->gdma_cq->id; - if (WARN_ON(cq->gdma_id >= gc->max_num_cqs)) { + /* No rcu_read_lock(): mana_create_txq runs under RTNL during + * netdev bring-up, inside the netdev lifetime that + * mana_remove() drains before the base cq_table can be freed. + * See gdma_context::cq_table in gdma.h for why "true" is sound. + / + cq_table = rcu_dereference_protected(gc->cq_table, true); + if (WARN_ON(!cq_table \|\| cq->gdma_id >= gc->max_num_cqs)) { err = -EINVAL; goto out; } - gc->cq_table[cq->gdma_id] = cq->gdma_cq; + rcu_assign_pointer(cq_table[cq->gdma_id], cq->gdma_cq); mana_create_txq_debugfs(apc, i); @@ -2986,6 +2993,7 @@ static struct mana_rxq mana_create_rxq(struct mana_port_context apc, struct gdma_queue_spec spec; struct mana_cq cq = NULL; struct gdma_context gc; + struct gdma_queue __rcu cq_table; u32 cq_size, rq_size; struct mana_rxq rxq; int err; @@ -3075,12 +3083,18 @@ static struct mana_rxq mana_create_rxq(struct mana_port_context apc, if (err) goto out; - if (WARN_ON(cq->gdma_id >= gc->max_num_cqs)) { + /* No rcu_read_lock(): mana_create_rxq runs under RTNL during netdev + * bring-up, inside the netdev lifetime that mana_remove() drains + * before the base cq_table can be freed. See gdma_context::cq_table + * in gdma.h for why "true" is sound. + / + cq_table = rcu_dereference_protected(gc->cq_table, true); + if (WARN_ON(!cq_table \|\| cq->gdma_id >= gc->max_num_cqs)) { err = -EINVAL; goto out; } - gc->cq_table[cq->gdma_id] = cq->gdma_cq; + rcu_assign_pointer(cq_table[cq->gdma_id], cq->gdma_cq); netif_napi_add_weight_locked(ndev, &cq->napi, mana_poll, 1); diff --git a/include/net/mana/gdma.h b/include/net/mana/gdma.h index 8529cef0d7c452..a4eac6f7c36609 100644 --- a/include/net/mana/gdma.h +++ b/include/net/mana/gdma.h @@ -428,9 +428,34 @@ struct gdma_context { / L2 MTU / u16 adapter_mtu; - / This maps a CQ index to the queue structure. / + / Size of cq_table, i.e. the largest valid CQ index + 1. Set once + * when cq_table is allocated and treated as immutable for the + * table's lifetime (a bound only) -- it is never reset on teardown. + * cq_table == NULL is the sole "table torn down" signal, so every + * cq_table[id] access must guard with both !cq_table (gone) and + * id >= max_num_cqs (out of bounds). + / unsigned int max_num_cqs; - struct gdma_queue cq_table; + / Both the base pointer and each entry are RCU-managed. The fast + * path (mana_gd_process_eqe) reads the base via rcu_dereference() + * under rcu_read_lock(), so the table is freed with + * rcu_assign_pointer(NULL) + synchronize_rcu() and an in-flight + * reader can never observe freed memory. + * + * The slow paths -- mana_gd_destroy_cq() and the CQ install/remove + * callers (mana_create_txq/_rxq, mana_ib_install/remove_cq_cb) -- + * instead read the base with rcu_dereference_protected(cq_table, + * true). The bare "true" is justified by teardown ordering, not by + * a lock: the base table is replaced+freed only by + * mana_hwc_destroy_channel() (and the create-time reinit), and every + * teardown path first runs mana_remove() + mana_rdma_remove(), which + * synchronously drain the netdev and the IB device + * (unregister_netdevice / ib_unregister_device) that bound all + * install/remove callers; the reinit case runs before either + * consumer is probed. So no slow-path caller can run while the base + * table is being freed. + / + struct gdma_queue __rcu __rcu cq_table; / Protect eq_test_event and test_event_eq_id / struct mutex eq_test_event_mutex; @@ -456,6 +481,21 @@ struct gdma_context { / Hardware communication channel (HWC) / struct gdma_dev hwc; + / destroy_channel() waits here for all HWC senders to exit. + * Lives on gc (not hwc) so wake_up() after the last sender's + * atomic_dec doesn't dereference freed hwc memory. + / + wait_queue_head_t hwc_drain_waitq; + + / Serializes hwc.driver_data (the hw_channel_context pointer) + * between the control-plane readers in mana_gd_send_request(), + * mana_need_log() and mana_serv_reset() and the publish/clear in + * mana_hwc_create_channel()/mana_hwc_destroy_channel(). All users + * are control-plane (HWC commands sleep; reset runs on a workqueue), + * so a plain spinlock -- not RCU -- is sufficient. + / + spinlock_t hwc_lock; + / Azure network adapter / struct gdma_dev mana; @@ -668,6 +708,9 @@ enum { / Driver supports dynamic interrupt moderation - DIM / #define GDMA_DRV_CAP_FLAG_1_DYN_INTERRUPT_MODERATION BIT(28) +/ Driver supports dynamic queue depth for HWC / +#define GDMA_DRV_CAP_FLAG_1_DYN_HWC_QUEUE_DEPTH BIT(29) + #define GDMA_DRV_CAP_FLAGS1 \ (GDMA_DRV_CAP_FLAG_1_EQ_SHARING_MULTI_VPORT \| \ GDMA_DRV_CAP_FLAG_1_NAPI_WKDONE_FIX \| \ @@ -683,6 +726,7 @@ enum { GDMA_DRV_CAP_FLAG_1_PROBE_RECOVERY \| \ GDMA_DRV_CAP_FLAG_1_HANDLE_STALL_SQ_RECOVERY \| \ GDMA_DRV_CAP_FLAG_1_HWC_TIMEOUT_RECOVERY \| \ + GDMA_DRV_CAP_FLAG_1_DYN_HWC_QUEUE_DEPTH \| \ GDMA_DRV_CAP_FLAG_1_EQ_MSI_UNSHARE_MULTI_VPORT \| \ GDMA_DRV_CAP_FLAG_1_DYN_INTERRUPT_MODERATION) diff --git a/include/net/mana/hw_channel.h b/include/net/mana/hw_channel.h index 16feb39616c1be..eec5d3aa23e1d5 100644 --- a/include/net/mana/hw_channel.h +++ b/include/net/mana/hw_channel.h @@ -164,6 +164,9 @@ struct hwc_wq { u16 queue_depth; struct hwc_cq hwc_cq; + + /* Serializes concurrent mana_gd_post_and_ring() calls. / + spinlock_t lock; }; struct hwc_caller_ctx { @@ -173,6 +176,17 @@ struct hwc_caller_ctx { u32 error; / Linux error code / u32 status_code; + + / Protects output_buf against concurrent access from + * handle_resp() (CQ interrupt) and the sender timeout path. + / + spinlock_t lock; + + / Tracks sender + handle_resp ownership. The last put + * (refcount reaches 0) releases the bitmap slot. + / + refcount_t refcnt; + u16 msg_id; }; struct hw_channel_context { @@ -181,8 +195,9 @@ struct hw_channel_context { u16 num_inflight_msg; u32 max_req_msg_size; + u32 max_resp_msg_size; - u16 hwc_init_q_depth_max; + u32 hwc_init_q_depth_max; u32 hwc_init_max_req_msg_size; u32 hwc_init_max_resp_msg_size; @@ -192,13 +207,38 @@ struct hw_channel_context { struct hwc_wq txq; struct hwc_cq cq; - struct semaphore sema; struct gdma_resource inflight_msg_res; + / Waitqueue for senders blocked on a full inflight bitmap. / + wait_queue_head_t msg_waitq; u32 pf_dest_vrq_id; u32 pf_dest_vrcq_id; u32 hwc_timeout; + / Set after channel is fully established; cleared on teardown to + * abort waiters in mana_hwc_get_msg_index() and reject new sends. + / + bool channel_up; + + / Set on first HWC timeout. Causes get_msg_index() to return + * -ETIMEDOUT instead of waiting, draining all queued senders. + / + bool hwc_timed_out; + + / Set after mana_smc_setup_hwc() succeeds (hardware has active + * MST entries). On recoverable paths (establish_channel) + * cleared only after successful teardown so a retry remains + * possible. On the terminal destroy_channel path, cleared + * unconditionally since hwc is about to be freed. + / + bool setup_active; + + / Count of in-flight mana_gd_send_request() callers. Protected + * by gc->hwc_lock; the last sender to drop it to zero wakes + * gc->hwc_drain_waitq for the mana_hwc_destroy_channel() drain. + / + unsigned int active_senders; + struct hwc_caller_ctx caller_ctx; };
3/2	2026/07/22 17:52	llm	patch-evaluator	0m Model: gemini-3.1-pro-preview Args: null Results: null Tokens: input: 23162 output: 19 thoughts: 213 Thoughts: Analyzing the MANA Driver Patch Series Okay, so I've got this patch series in front of me that's making changes to the Microsoft Azure Network Adapter, the MANA driver. It looks like the core of the driver is being touched; I'm seeing modifications in `cq.c`, `gdma_main.c`, `hw_channel.c`, and `mana_en.c`, along with some header file updates in `gdma.h` and `hw_channel.h`. Given my experience, that list of files immediately tells me this is likely a substantial functional change. Let's quickly assess the nature of these changes to get a better handle on the scope. Ah, yes, I'm seeing RCU protection added for `cq_table`, changes to spinlocks, timeout handling improvements, refcount management for `hwc_caller_ctx`, and handling of HWC reinitialization. That's a pretty significant laundry list. This isn't just a minor tweak; this is a fundamental rework of some key synchronization and resource management components within the MANA driver. These types of changes almost always introduce the potential for subtle, hard-to-debug regressions, so thorough testing is absolutely critical. Next, I need to understand the configuration options associated with this driver. I'm going to search through the Kconfig files, specifically looking for something like `MICROSOFT_MANA` or a similar configuration symbol to figure out how this driver is enabled and if any new options have been introduced with this patch. Understanding the configuration is vital to properly integrate this patch series, or to even just understand the broader context.
4/2	2026/07/22 17:52	tool	grepper	0m Args: { "Expression": "MICROSOFT_MANA" } Results: { "Output": "drivers/infiniband/hw/mana/Kconfig=2=config MANA_INFINIBAND\ndrivers/infiniband/hw/mana/Kconfig-3-\ttristate \"Microsoft Azure Network Adapter support\"\ndrivers/infiniband/hw/mana/Kconfig:4:\tdepends on NETDEVICES \u0026\u0026 ETHERNET \u0026\u0026 PCI \u0026\u0026 MICROSOFT_MANA\ndrivers/infiniband/hw/mana/Kconfig-5-\thelp\n--\ndrivers/net/ethernet/microsoft/Kconfig=16=if NET_VENDOR_MICROSOFT\ndrivers/net/ethernet/microsoft/Kconfig-17-\ndrivers/net/ethernet/microsoft/Kconfig:18:config MICROSOFT_MANA\ndrivers/net/ethernet/microsoft/Kconfig-19-\ttristate \"Microsoft Azure Network Adapter (MANA) support\"\n--\ndrivers/net/ethernet/microsoft/Makefile-4-\ndrivers/net/ethernet/microsoft/Makefile:5:obj-$(CONFIG_MICROSOFT_MANA) += mana/\n--\ndrivers/net/ethernet/microsoft/mana/Makefile-4-\ndrivers/net/ethernet/microsoft/mana/Makefile:5:obj-$(CONFIG_MICROSOFT_MANA) += mana.o\ndrivers/net/ethernet/microsoft/mana/Makefile-6-mana-objs := gdma_main.o shm_channel.o hw_channel.o mana_en.o mana_ethtool.o mana_bpf.o\n" }
5/2	2026/07/22 17:52	llm	patch-evaluator	0m Model: gemini-3.1-pro-preview Args: null Results: null Tokens: input: 24098 output: 17 thoughts: 1097 Thoughts: Fuzzing Strategy for the MANA Driver Okay, so I'm looking at this patch and immediately, my mind jumps to fuzzing. The key here is the `MICROSOFT_MANA` and `MANA_INFINIBAND` configs, they will guide us on whether this is built as a module or built in. I need to get a handle on what the target functions are within the code, so let's list those. The patch touches a good chunk of the driver, specifically: `mana_ib_install_cq_cb`, `mana_ib_remove_cq_cb`, `mana_need_log`, `mana_gd_send_request`, `mana_serv_reset`, `mana_gd_process_eqe`, `mana_gd_destroy_cq`, `mana_gd_create_dma_region`, `mana_gd_verify_vf_version`, `mana_gd_probe`, `mana_hwc_get_msg_index`, `mana_hwc_put_msg_index`, `hwc_ctx_put`, `mana_hwc_handle_resp`, `mana_hwc_init_event_handler`, `mana_hwc_rx_event_handler`, `mana_hwc_destroy_cq`, `mana_hwc_create_wq`, `mana_hwc_post_tx_wqe`, `mana_hwc_init_inflight_msg`, `mana_hwc_test_channel`, `mana_hwc_establish_channel`, `mana_hwc_init_queues`, `mana_hwc_destroy_queues`, `mana_hwc_create_channel`, `mana_hwc_destroy_channel`, `mana_hwc_send_request`, `mana_create_txq`, and `mana_create_rxq`. That's a lot, so I'll need to prioritize. Let's pick a few promising candidates for fuzzing, and I should focus on entry points from userspace or those that handle complex interaction. I'll pick: `mana_hwc_send_request`, `mana_hwc_handle_resp`, `mana_hwc_rx_event_handler`, `mana_hwc_create_channel`, `mana_hwc_destroy_channel`, `mana_hwc_establish_channel`, `mana_hwc_test_channel`, `mana_hwc_get_msg_index`, `mana_hwc_put_msg_index`, `mana_gd_send_request`, `mana_gd_process_eqe`, `mana_ib_install_cq_cb`, and `mana_ib_remove_cq_cb`. Now, the big question: Is this driver even reachable in a virtualized environment? MANA is a Microsoft Azure Network Adapter, so it's a PCI device. Typically, I'd look for syzkaller support via vhci or a similar pseudo-device if the hardware is emulated. Syzkaller doesn't have a specific MANA emulator, so I was concerned it wouldn't be possible at first. But wait, PCI drivers can be fuzzed using `syz_pci_vpci` and similar. The text says we may be able to reach even software-emulated hardware. I'm seeing "syz-cluster" as the author, which strongly suggests that syzkaller is actively working on or has found something to fuzz in this code! Since it is a patch being tested by syzkaller it is worth it. Syzkaller can fuzz any PCI driver by binding it to the vhci or pci pseudo-device and providing random MMIO/DMA responses. Okay, let's confirm the configs again, because it's always good to be thorough. Yup, just `MICROSOFT_MANA` and `MANA_INFINIBAND`. Let's refine the FocusSymbols to the important ones: `mana_hwc_send_request`, `mana_hwc_handle_resp`, `mana_hwc_rx_event_handler`, `mana_hwc_create_channel`, `mana_hwc_destroy_channel`, `mana_hwc_establish_channel`, `mana_hwc_destroy_queues`, `mana_gd_process_eqe`, `mana_ib_install_cq_cb`, `mana_ib_remove_cq_cb`, and `mana_hwc_get_msg_index`. Good. Now, I should make sure there aren't any other configs that would influence the build or functionality I'm missing.
6/2	2026/07/22 17:52	tool	grepper	0m Args: { "Expression": "config MANA" } Results: { "Output": "drivers/infiniband/hw/mana/Kconfig-1-# SPDX-License-Identifier: GPL-2.0-only\ndrivers/infiniband/hw/mana/Kconfig:2:config MANA_INFINIBAND\ndrivers/infiniband/hw/mana/Kconfig-3-\ttristate \"Microsoft Azure Network Adapter support\"\n--\ndrivers/power/supply/Kconfig=293=config CHARGER_SBS\n--\ndrivers/power/supply/Kconfig-299-\ndrivers/power/supply/Kconfig:300:config MANAGER_SBS\ndrivers/power/supply/Kconfig-301-\ttristate \"Smart Battery System Manager\"\n" }
7/2	2026/07/22 17:52	llm	patch-evaluator	0m Model: gemini-3.1-pro-preview Args: null Results: null Tokens: input: 26385 output: 217 thoughts: 150 Thoughts: Okay, here's my take on those thoughts, framed from my expert perspective: Deep Dive into MANA Infiniband Functionality Alright, let's see what we're working with here. The configuration context is clearly focused on the `MICROSOFT_MANA` and `MANA_INFINIBAND` setups. I need to keep those in mind as the foundation for everything that's to come. I'm zeroing in on a critical subset of functions, or what they're calling `FocusSymbols`. These seem to be the core building blocks for handling Completion Queues (CQs), hardware communication, and general data processing within this Infiniband implementation. Let's break them down. * `mana_ib_install_cq_cb` and `mana_ib_remove_cq_cb`: These clearly relate to the installation and removal of Completion Queue callbacks. This suggests a sophisticated event-driven model for handling completions, which is standard but important to verify the nuances of. * `mana_hwc_get_msg_index`, `mana_hwc_handle_resp`, `mana_hwc_rx_event_handler`, `mana_hwc_create_channel`, `mana_hwc_destroy_channel`, and `mana_hwc_send_request`: These collectively paint a picture of Hardware Communication (HWC) management. We're looking at message indexing, response handling, event processing (particularly on the receive side), and the creation, destruction, and transmission aspects of communication channels. A good level of abstraction, critical to performance. * `mana_gd_process_eqe` and `mana_gd_destroy_cq`: These are likely related to generic data (GD) processing and destroying CQs. This implies the existence of a generic data model for data transfer, separate from the Infiniband-specific aspects. So, overall, a thorough set of considerations that indicates a well-structured approach to handling Infiniband operations with this `MANA` implementation. My next step is to call set-results. This indicates the next steps in this evaluation, likely gathering all available data points to establish a benchmark for performance.
8/2	2026/07/22 17:53	tool	set-results	0m Args: { "EnableConfigs": [ "MICROSOFT_MANA", "MANA_INFINIBAND" ], "FocusSymbols": [ "mana_ib_install_cq_cb", "mana_ib_remove_cq_cb", "mana_hwc_get_msg_index", "mana_hwc_handle_resp", "mana_hwc_rx_event_handler", "mana_hwc_create_channel", "mana_hwc_destroy_channel", "mana_hwc_send_request", "mana_gd_process_eqe", "mana_gd_destroy_cq" ], "Reasoning": "The patch makes functional changes to the Microsoft Azure Network Adapter (MANA) driver, including adding RCU protection for the CQ table, handling HWC timeouts, and adding refcounts for caller contexts. These changes affect the core logic of the driver and should be fuzzed.", "WorthFuzzing": true } Results: { "EnableConfigs": [ "MICROSOFT_MANA", "MANA_INFINIBAND" ], "FocusSymbols": [ "mana_ib_install_cq_cb", "mana_ib_remove_cq_cb", "mana_hwc_get_msg_index", "mana_hwc_handle_resp", "mana_hwc_rx_event_handler", "mana_hwc_create_channel", "mana_hwc_destroy_channel", "mana_hwc_send_request", "mana_gd_process_eqe", "mana_gd_destroy_cq" ], "Reasoning": "The patch makes functional changes to the Microsoft Azure Network Adapter (MANA) driver, including adding RCU protection for the CQ table, handling HWC timeouts, and adding refcounts for caller contexts. These changes affect the core logic of the driver and should be fuzzed.", "WorthFuzzing": true }

Trajectory: (Expand/Collapse All)

Seq

Timestamp

Type

Name

Duration

0/0

2026/07/22 17:52

flow

patch-triage

Args:

null

Results:

{
  "EnableConfigs": [
    "MICROSOFT_MANA",
    "MANA_INFINIBAND"
  ],
  "FocusSymbols": [
    "mana_ib_install_cq_cb",
    "mana_ib_remove_cq_cb",
    "mana_hwc_get_msg_index",
    "mana_hwc_handle_resp",
    "mana_hwc_rx_event_handler",
    "mana_hwc_create_channel",
    "mana_hwc_destroy_channel",
    "mana_hwc_send_request",
    "mana_gd_process_eqe",
    "mana_gd_destroy_cq"
  ],
  "Reasoning": "The patch makes functional changes to the Microsoft Azure Network Adapter (MANA) driver, including adding RCU protection for the CQ table, handling HWC timeouts, and adding refcounts for caller contexts. These changes affect the core logic of the driver and should be fuzzed.",
  "WorthFuzzing": true
}

1/1

2026/07/22 17:52

action

read-patch-diff

Args:

null

Results:

{
  "PatchDiff": "commit 7a7f673c5a49005405d55fc2540f242d3b89cd96\nAuthor: syz-cluster \u003ctriage@syzkaller.com\u003e\nDate:   Wed Jul 22 17:52:19 2026 +0000\n\n    syz-cluster: applied patch under review\n\ndiff --git a/drivers/infiniband/hw/mana/cq.c b/drivers/infiniband/hw/mana/cq.c\nindex f2547989f42290..2bf4be21cede79 100644\n--- a/drivers/infiniband/hw/mana/cq.c\n+++ b/drivers/infiniband/hw/mana/cq.c\n@@ -131,12 +131,20 @@ static void mana_ib_cq_handler(void *ctx, struct gdma_queue *gdma_cq)\n int mana_ib_install_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq *cq)\n {\n \tstruct gdma_context *gc = mdev_to_gc(mdev);\n+\tstruct gdma_queue __rcu **cq_table;\n \tstruct gdma_queue *gdma_cq;\n \n-\tif (cq-\u003equeue.id \u003e= gc-\u003emax_num_cqs)\n+\t/* No rcu_read_lock(): install/remove run within the IB device\n+\t * lifetime, which mana_rdma_remove() (ib_unregister_device) drains\n+\t * before the base cq_table can be freed.  See gdma_context::cq_table\n+\t * in gdma.h for why \"true\" is sound.\n+\t */\n+\tcq_table = rcu_dereference_protected(gc-\u003ecq_table, true);\n+\tif (!cq_table || cq-\u003equeue.id \u003e= gc-\u003emax_num_cqs)\n \t\treturn -EINVAL;\n+\n \t/* Create CQ table entry, sharing a CQ between WQs is not supported */\n-\tif (gc-\u003ecq_table[cq-\u003equeue.id])\n+\tif (rcu_access_pointer(cq_table[cq-\u003equeue.id]))\n \t\treturn -EINVAL;\n \tif (cq-\u003equeue.kmem)\n \t\tgdma_cq = cq-\u003equeue.kmem;\n@@ -149,23 +157,49 @@ int mana_ib_install_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq *cq)\n \tgdma_cq-\u003etype = GDMA_CQ;\n \tgdma_cq-\u003ecq.callback = mana_ib_cq_handler;\n \tgdma_cq-\u003eid = cq-\u003equeue.id;\n-\tgc-\u003ecq_table[cq-\u003equeue.id] = gdma_cq;\n+\trcu_assign_pointer(cq_table[cq-\u003equeue.id], gdma_cq);\n \treturn 0;\n }\n \n void mana_ib_remove_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq *cq)\n {\n \tstruct gdma_context *gc = mdev_to_gc(mdev);\n+\tstruct gdma_queue __rcu **cq_table;\n+\tstruct gdma_queue *gdma_cq;\n \n-\tif (cq-\u003equeue.id \u003e= gc-\u003emax_num_cqs || cq-\u003equeue.id == INVALID_QUEUE_ID)\n+\tif (cq-\u003equeue.id == INVALID_QUEUE_ID)\n \t\treturn;\n \n \tif (cq-\u003equeue.kmem)\n \t/* Then it will be cleaned and removed by the mana */\n \t\treturn;\n \n-\tkfree(gc-\u003ecq_table[cq-\u003equeue.id]);\n-\tgc-\u003ecq_table[cq-\u003equeue.id] = NULL;\n+\t/* No rcu_read_lock(): like mana_ib_install_cq_cb(), this runs within\n+\t * the IB device lifetime that mana_rdma_remove() drains before the\n+\t * base cq_table can be freed.  See gdma_context::cq_table in gdma.h.\n+\t */\n+\tcq_table = rcu_dereference_protected(gc-\u003ecq_table, true);\n+\tif (!cq_table || cq-\u003equeue.id \u003e= gc-\u003emax_num_cqs)\n+\t\treturn;\n+\t/* Removers for a given CQ are serialized by the IB core, so the slot\n+\t * is read and cleared without rcu_read_lock() or atomicity: a CQ is\n+\t * never torn down while a live QP references it (cq-\u003eusecnt), nor\n+\t * while the QP-create that installed the entry is still running (that\n+\t * create holds a reference on the CQ uobject across its error path,\n+\t * before usecnt is taken).  Any double-remove is therefore sequential\n+\t * -- the later caller sees the NULL stored below and returns.\n+\t */\n+\tgdma_cq = rcu_dereference_protected(cq_table[cq-\u003equeue.id], true);\n+\tif (!gdma_cq)\n+\t\treturn;  /* already removed by a prior teardown path */\n+\n+\trcu_assign_pointer(cq_table[cq-\u003equeue.id], NULL);\n+\n+\t/* Wait for in-flight EQ handlers that may have loaded the old\n+\t * pointer via rcu_dereference() to finish before freeing.\n+\t */\n+\tsynchronize_rcu();\n+\tkfree(gdma_cq);\n }\n \n int mana_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)\ndiff --git a/drivers/net/ethernet/microsoft/mana/gdma_main.c b/drivers/net/ethernet/microsoft/mana/gdma_main.c\nindex a38d4bb74621f9..17711ee37c8be3 100644\n--- a/drivers/net/ethernet/microsoft/mana/gdma_main.c\n+++ b/drivers/net/ethernet/microsoft/mana/gdma_main.c\n@@ -161,6 +161,8 @@ static int mana_gd_init_registers(struct pci_dev *pdev)\n bool mana_need_log(struct gdma_context *gc, int err)\n {\n \tstruct hw_channel_context *hwc;\n+\tbool need_log = true;\n+\tunsigned long flags;\n \n \tif (err != -ETIMEDOUT)\n \t\treturn true;\n@@ -168,11 +170,13 @@ bool mana_need_log(struct gdma_context *gc, int err)\n \tif (!gc)\n \t\treturn true;\n \n+\tspin_lock_irqsave(\u0026gc-\u003ehwc_lock, flags);\n \thwc = gc-\u003ehwc.driver_data;\n \tif (hwc \u0026\u0026 hwc-\u003ehwc_timeout == 0)\n-\t\treturn false;\n+\t\tneed_log = false;\n+\tspin_unlock_irqrestore(\u0026gc-\u003ehwc_lock, flags);\n \n-\treturn true;\n+\treturn need_log;\n }\n \n static int mana_gd_query_max_resources(struct pci_dev *pdev)\n@@ -367,9 +371,27 @@ static int mana_gd_detect_devices(struct pci_dev *pdev)\n int mana_gd_send_request(struct gdma_context *gc, u32 req_len, const void *req,\n \t\t\t u32 resp_len, void *resp)\n {\n-\tstruct hw_channel_context *hwc = gc-\u003ehwc.driver_data;\n+\tstruct hw_channel_context *hwc;\n+\tunsigned long flags;\n+\tint err;\n+\n+\tspin_lock_irqsave(\u0026gc-\u003ehwc_lock, flags);\n+\thwc = gc-\u003ehwc.driver_data;\n+\tif (!hwc) {\n+\t\tspin_unlock_irqrestore(\u0026gc-\u003ehwc_lock, flags);\n+\t\treturn -ENODEV;\n+\t}\n+\thwc-\u003eactive_senders++;\n+\tspin_unlock_irqrestore(\u0026gc-\u003ehwc_lock, flags);\n+\n+\terr = mana_hwc_send_request(hwc, req_len, req, resp_len, resp);\n \n-\treturn mana_hwc_send_request(hwc, req_len, req, resp_len, resp);\n+\tspin_lock_irqsave(\u0026gc-\u003ehwc_lock, flags);\n+\tif (--hwc-\u003eactive_senders == 0)\n+\t\twake_up(\u0026gc-\u003ehwc_drain_waitq);\n+\tspin_unlock_irqrestore(\u0026gc-\u003ehwc_lock, flags);\n+\n+\treturn err;\n }\n EXPORT_SYMBOL_NS(mana_gd_send_request, \"NET_MANA\");\n \n@@ -622,6 +644,7 @@ static void mana_serv_reset(struct pci_dev *pdev)\n {\n \tstruct gdma_context *gc = pci_get_drvdata(pdev);\n \tstruct hw_channel_context *hwc;\n+\tunsigned long flags;\n \tint ret;\n \n \tif (!gc) {\n@@ -631,14 +654,17 @@ static void mana_serv_reset(struct pci_dev *pdev)\n \t\treturn;\n \t}\n \n+\tspin_lock_irqsave(\u0026gc-\u003ehwc_lock, flags);\n \thwc = gc-\u003ehwc.driver_data;\n \tif (!hwc) {\n+\t\tspin_unlock_irqrestore(\u0026gc-\u003ehwc_lock, flags);\n \t\tdev_err(\u0026pdev-\u003edev, \"MANA service: no HWC\\n\");\n \t\tgoto out;\n \t}\n \n \t/* HWC is not responding in this case, so don't wait */\n \thwc-\u003ehwc_timeout = 0;\n+\tspin_unlock_irqrestore(\u0026gc-\u003ehwc_lock, flags);\n \n \tdev_info(\u0026pdev-\u003edev, \"MANA reset cycle start\\n\");\n \n@@ -761,6 +787,7 @@ static void mana_gd_process_eqe(struct gdma_queue *eq)\n \tunion gdma_eqe_info eqe_info;\n \tenum gdma_eqe_type type;\n \tstruct gdma_event event;\n+\tstruct gdma_queue __rcu **cq_table;\n \tstruct gdma_queue *cq;\n \tstruct gdma_eqe *eqe;\n \tu32 cq_id;\n@@ -772,10 +799,11 @@ static void mana_gd_process_eqe(struct gdma_queue *eq)\n \tswitch (type) {\n \tcase GDMA_EQE_COMPLETION:\n \t\tcq_id = eqe-\u003edetails[0] \u0026 0xFFFFFF;\n-\t\tif (WARN_ON_ONCE(cq_id \u003e= gc-\u003emax_num_cqs))\n+\t\tcq_table = rcu_dereference(gc-\u003ecq_table);\n+\t\tif (WARN_ON_ONCE(cq_id \u003e= gc-\u003emax_num_cqs || !cq_table))\n \t\t\tbreak;\n \n-\t\tcq = gc-\u003ecq_table[cq_id];\n+\t\tcq = rcu_dereference(cq_table[cq_id]);\n \t\tif (WARN_ON_ONCE(!cq || cq-\u003etype != GDMA_CQ || cq-\u003eid != cq_id))\n \t\t\tbreak;\n \n@@ -1082,15 +1110,28 @@ static void mana_gd_create_cq(const struct gdma_queue_spec *spec,\n static void mana_gd_destroy_cq(struct gdma_context *gc,\n \t\t\t       struct gdma_queue *queue)\n {\n+\tstruct gdma_queue __rcu **cq_table;\n \tu32 id = queue-\u003eid;\n \n-\tif (id \u003e= gc-\u003emax_num_cqs)\n+\t/* No rcu_read_lock() here: mana_gd_destroy_cq() runs only on the\n+\t * CQ-destroy/teardown path, where the base cq_table is stable.  See\n+\t * the lifecycle note on gdma_context::cq_table in gdma.h for why the\n+\t * \"true\" predicate is sound.\n+\t */\n+\tcq_table = rcu_dereference_protected(gc-\u003ecq_table, true);\n+\tif (!cq_table || id \u003e= gc-\u003emax_num_cqs)\n \t\treturn;\n \n-\tif (!gc-\u003ecq_table[id])\n+\tif (!rcu_access_pointer(cq_table[id]))\n \t\treturn;\n \n-\tgc-\u003ecq_table[id] = NULL;\n+\trcu_assign_pointer(cq_table[id], NULL);\n+\n+\t/* Wait for in-flight EQ handlers that may have loaded the old\n+\t * pointer via rcu_dereference() to finish before the caller\n+\t * frees the CQ memory.\n+\t */\n+\tsynchronize_rcu();\n }\n \n int mana_gd_create_hwc_queue(struct gdma_dev *gd,\n@@ -1185,6 +1226,16 @@ static int mana_gd_create_dma_region(struct gdma_dev *gd,\n \tif (!MANA_PAGE_ALIGNED(gmi-\u003evirt_addr))\n \t\treturn -EINVAL;\n \n+\t/* No RCU needed: this runs only on the data-path queue-creation\n+\t * path (mana_gd_create_mana_eq/mana_gd_create_mana_wq_cq, called\n+\t * by mana_en under RTNL and by mana_ib RDMA verbs, or during\n+\t * init).  Every teardown path — mana_gd_remove, mana_gd_suspend,\n+\t * and the HWC reset/service path (which goes through\n+\t * mana_gd_suspend) — drains those consumers via mana_rdma_remove()\n+\t * + mana_remove() before mana_hwc_destroy_channel() clears\n+\t * gc-\u003ehwc.driver_data, so no concurrent destroy can race with\n+\t * this dereference.\n+\t */\n \thwc = gc-\u003ehwc.driver_data;\n \treq_msg_size = struct_size(req, page_addr_list, num_page);\n \tif (req_msg_size \u003e hwc-\u003emax_req_msg_size)\n@@ -1374,7 +1425,17 @@ int mana_gd_verify_vf_version(struct pci_dev *pdev)\n \tstruct hw_channel_context *hwc;\n \tint err;\n \n+\t/* No RCU needed: this runs only inside mana_gd_setup, on the\n+\t * probe and resume paths.  The PCI/PM core holds device_lock\n+\t * across .probe/.resume and .remove/.suspend, so setup cannot\n+\t * overlap teardown of the same device.  The HWC reset/service\n+\t * path is additionally serialized by GC_IN_SERVICE and runs\n+\t * suspend (destroy) then resume (this) sequentially in one work\n+\t * item.  driver_data was just set by mana_hwc_create_channel\n+\t * earlier in this same setup call, so it is live here.\n+\t */\n \thwc = gc-\u003ehwc.driver_data;\n+\n \tmana_gd_init_req_hdr(\u0026req.hdr, GDMA_VERIFY_VF_DRIVER_VERSION,\n \t\t\t     sizeof(req), sizeof(resp));\n \n@@ -2364,6 +2425,7 @@ static int mana_gd_probe(struct pci_dev *pdev, const struct pci_device_id *ent)\n \n \tmutex_init(\u0026gc-\u003eeq_test_event_mutex);\n \tmutex_init(\u0026gc-\u003egic_mutex);\n+\tspin_lock_init(\u0026gc-\u003ehwc_lock);\n \tpci_set_drvdata(pdev, gc);\n \tgc-\u003ebar0_pa = pci_resource_start(pdev, 0);\n \tgc-\u003ebar0_size = pci_resource_len(pdev, 0);\ndiff --git a/drivers/net/ethernet/microsoft/mana/hw_channel.c b/drivers/net/ethernet/microsoft/mana/hw_channel.c\nindex e3c24d50dad07c..0481d40443424f 100644\n--- a/drivers/net/ethernet/microsoft/mana/hw_channel.c\n+++ b/drivers/net/ethernet/microsoft/mana/hw_channel.c\n@@ -4,27 +4,55 @@\n #include \u003cnet/mana/gdma.h\u003e\n #include \u003cnet/mana/mana.h\u003e\n #include \u003cnet/mana/hw_channel.h\u003e\n+#include \u003clinux/pci.h\u003e\n #include \u003clinux/vmalloc.h\u003e\n \n+/* Acquire a free message slot from the inflight bitmap.  Returns\n+ * -ENODEV if the channel is torn down, or -ETIMEDOUT if a prior HWC\n+ * command has timed out (preserving the error code callers expect).\n+ */\n static int mana_hwc_get_msg_index(struct hw_channel_context *hwc, u16 *msg_id)\n {\n \tstruct gdma_resource *r = \u0026hwc-\u003einflight_msg_res;\n \tunsigned long flags;\n \tu32 index;\n \n-\tdown(\u0026hwc-\u003esema);\n+\tfor (;;) {\n+\t\tspin_lock_irqsave(\u0026r-\u003elock, flags);\n \n-\tspin_lock_irqsave(\u0026r-\u003elock, flags);\n+\t\tif (!hwc-\u003echannel_up || hwc-\u003ehwc_timed_out) {\n+\t\t\tspin_unlock_irqrestore(\u0026r-\u003elock, flags);\n+\t\t\treturn hwc-\u003echannel_up ? -ETIMEDOUT : -ENODEV;\n+\t\t}\n \n-\tindex = find_first_zero_bit(hwc-\u003einflight_msg_res.map,\n-\t\t\t\t    hwc-\u003einflight_msg_res.size);\n+\t\tindex = find_first_zero_bit(r-\u003emap, r-\u003esize);\n+\t\tif (index \u003c r-\u003esize) {\n+\t\t\tstruct hwc_caller_ctx *ctx;\n+\n+\t\t\tbitmap_set(r-\u003emap, index, 1);\n+\t\t\tctx = \u0026hwc-\u003ecaller_ctx[index];\n+\t\t\treinit_completion(\u0026ctx-\u003ecomp_event);\n+\t\t\trefcount_set(\u0026ctx-\u003erefcnt, 1);\n+\t\t\tctx-\u003emsg_id = index;\n+\t\t\tctx-\u003eerror = -EINPROGRESS;\n+\t\t\tspin_unlock_irqrestore(\u0026r-\u003elock, flags);\n+\t\t\tbreak;\n+\t\t}\n+\t\tspin_unlock_irqrestore(\u0026r-\u003elock, flags);\n \n-\tbitmap_set(hwc-\u003einflight_msg_res.map, index, 1);\n+\t\twait_event(hwc-\u003emsg_waitq,\n+\t\t\t   !hwc-\u003echannel_up ||\n+\t\t\t   hwc-\u003ehwc_timed_out ||\n+\t\t\t   !bitmap_full(r-\u003emap, r-\u003esize));\n \n-\tspin_unlock_irqrestore(\u0026r-\u003elock, flags);\n+\t\tif (!hwc-\u003echannel_up)\n+\t\t\treturn -ENODEV;\n \n-\t*msg_id = index;\n+\t\tif (hwc-\u003ehwc_timed_out)\n+\t\t\treturn -ETIMEDOUT;\n+\t}\n \n+\t*msg_id = index;\n \treturn 0;\n }\n \n@@ -34,10 +62,17 @@ static void mana_hwc_put_msg_index(struct hw_channel_context *hwc, u16 msg_id)\n \tunsigned long flags;\n \n \tspin_lock_irqsave(\u0026r-\u003elock, flags);\n-\tbitmap_clear(hwc-\u003einflight_msg_res.map, msg_id, 1);\n+\tbitmap_clear(r-\u003emap, msg_id, 1);\n \tspin_unlock_irqrestore(\u0026r-\u003elock, flags);\n \n-\tup(\u0026hwc-\u003esema);\n+\twake_up(\u0026hwc-\u003emsg_waitq);\n+}\n+\n+static void hwc_ctx_put(struct hw_channel_context *hwc,\n+\t\t\tstruct hwc_caller_ctx *ctx)\n+{\n+\tif (refcount_dec_and_test(\u0026ctx-\u003erefcnt))\n+\t\tmana_hwc_put_msg_index(hwc, ctx-\u003emsg_id);\n }\n \n static int mana_hwc_verify_resp_msg(const struct hwc_caller_ctx *caller_ctx,\n@@ -83,6 +118,17 @@ static void mana_hwc_handle_resp(struct hw_channel_context *hwc, u32 resp_len,\n \tstruct hwc_caller_ctx *ctx;\n \tint err;\n \n+\t/* Validate msg_id is in range before using it to index bitmap\n+\t * and caller_ctx array.  Malicious firmware could send\n+\t * out-of-range msg_id causing out-of-bounds access.\n+\t */\n+\tif (msg_id \u003e= hwc-\u003enum_inflight_msg) {\n+\t\tdev_err(hwc-\u003edev, \"hwc_rx: msg_id %u \u003e= max %u\\n\",\n+\t\t\tmsg_id, hwc-\u003enum_inflight_msg);\n+\t\tmana_hwc_post_rx_wqe(hwc-\u003erxq, rx_req);\n+\t\treturn;\n+\t}\n+\n \tif (!test_bit(msg_id, hwc-\u003einflight_msg_res.map)) {\n \t\tdev_err(hwc-\u003edev, \"hwc_rx: invalid msg_id = %u\\n\", msg_id);\n \t\tmana_hwc_post_rx_wqe(hwc-\u003erxq, rx_req);\n@@ -90,22 +136,44 @@ static void mana_hwc_handle_resp(struct hw_channel_context *hwc, u32 resp_len,\n \t}\n \n \tctx = hwc-\u003ecaller_ctx + msg_id;\n-\terr = mana_hwc_verify_resp_msg(ctx, resp_msg, resp_len);\n-\tif (err)\n-\t\tgoto out;\n \n-\tctx-\u003estatus_code = resp_msg-\u003estatus;\n+\t/* Reject responses larger than the RX DMA buffer — the SGE\n+\t * limits what hardware can DMA, so an oversized resp_len\n+\t * indicates a firmware bug.  Fail rather than silently\n+\t * truncating.\n+\t */\n+\tif (resp_len \u003e rx_req-\u003ebuf_len) {\n+\t\tdev_err(hwc-\u003edev, \"HWC RX: resp_len %u \u003e buf_len %u\\n\",\n+\t\t\tresp_len, rx_req-\u003ebuf_len);\n+\t\tresp_len = 0;\n+\t}\n \n-\tmemcpy(ctx-\u003eoutput_buf, resp_msg, resp_len);\n-out:\n-\tctx-\u003eerror = err;\n+\tspin_lock(\u0026ctx-\u003elock);\n+\n+\terr = mana_hwc_verify_resp_msg(ctx, resp_msg, resp_len);\n \n-\t/* Must post rx wqe before complete(), otherwise the next rx may\n-\t * hit no_wqe error.\n+\tif (!err \u0026\u0026 ctx-\u003eoutput_buf) {\n+\t\tctx-\u003estatus_code = resp_msg-\u003estatus;\n+\t\tmemcpy(ctx-\u003eoutput_buf, resp_msg, resp_len);\n+\t\tctx-\u003eerror = 0;\n+\t} else if (ctx-\u003eoutput_buf) {\n+\t\t/* Only overwrite error if the sender hasn't timed out\n+\t\t * or been force-completed by destroy.  When output_buf\n+\t\t * is NULL, a terminal error (-ENODEV or timeout) has\n+\t\t * already been set — preserve it so the sender doesn't\n+\t\t * see a spurious success.\n+\t\t */\n+\t\tctx-\u003eerror = err;\n+\t}\n+\n+\t/* Post RX WQE before completing — the next response may arrive\n+\t * immediately and needs a posted buffer.\n \t */\n \tmana_hwc_post_rx_wqe(hwc-\u003erxq, rx_req);\n-\n \tcomplete(\u0026ctx-\u003ecomp_event);\n+\tspin_unlock(\u0026ctx-\u003elock);\n+\n+\thwc_ctx_put(hwc, ctx);\n }\n \n static void mana_hwc_init_event_handler(void *ctx, struct gdma_queue *q_self,\n@@ -146,7 +214,12 @@ static void mana_hwc_init_event_handler(void *ctx, struct gdma_queue *q_self,\n \t\t\tbreak;\n \n \t\tcase HWC_INIT_DATA_QUEUE_DEPTH:\n-\t\t\thwc-\u003ehwc_init_q_depth_max = (u16)val;\n+\t\t\t/* HWC_INIT_DATA_QUEUE_DEPTH is a 24-bit field.  Keep\n+\t\t\t * the full device-reported value here; it is clamped\n+\t\t\t * and validated in mana_hwc_create_channel() rather\n+\t\t\t * than silently truncated to u16.\n+\t\t\t */\n+\t\t\thwc-\u003ehwc_init_q_depth_max = val;\n \t\t\tbreak;\n \n \t\tcase HWC_INIT_DATA_MAX_REQUEST:\n@@ -261,19 +334,69 @@ static void mana_hwc_rx_event_handler(void *ctx, u32 gdma_rxq_id,\n \n \tsge = (struct gdma_sge *)(wqe + 8 + dma_oob-\u003einline_oob_size_div4 * 4);\n \n-\t/* Select the RX work request for virtual address and for reposting. */\n-\trq_base_addr = hwc_rxq-\u003emsg_buf-\u003emem_info.dma_handle;\n-\trx_req_idx = (sge-\u003eaddress - rq_base_addr) / hwc-\u003emax_req_msg_size;\n+\t/* inline_oob_size_div4 is read from the WQE in device-accessible RQ\n+\t * memory, so a malicious host in a CVM (or buggy firmware) could set\n+\t * it to push the SGE past the RQ buffer.  Bounds-check the SGE it\n+\t * locates before dereferencing sge-\u003eaddress below.  A validly posted\n+\t * WQE keeps the SGE inside the ring (worst case ends exactly at the\n+\t * buffer boundary); reject anything that would read past it.  The\n+\t * slot cannot be trusted here, so leak this RX WQE rather than repost\n+\t * the wrong one -- as in the SGE-address mismatch path below.\n+\t */\n+\tif ((u8 *)(sge + 1) \u003e (u8 *)rq-\u003equeue_mem_ptr + rq-\u003equeue_size) {\n+\t\tdev_err(hwc-\u003edev, \"HWC RX: SGE past RQ buffer, oob_div4=%u\\n\",\n+\t\t\tdma_oob-\u003einline_oob_size_div4);\n+\t\treturn;\n+\t}\n \n-\tif (rx_req_idx \u003e= hwc_rxq-\u003emsg_buf-\u003enum_reqs) {\n-\t\tdev_err(hwc-\u003edev, \"HWC RX: wrong rx_req_idx=%llu, num_reqs=%u\\n\",\n-\t\t\trx_req_idx, hwc_rxq-\u003emsg_buf-\u003enum_reqs);\n+\t/* Recover the originating RX slot from the SGE address.  Of the three\n+\t * terms here only sge-\u003eaddress lives in device-accessible RQ memory;\n+\t * rq_base_addr and max_resp_msg_size are driver-private constants.  An\n+\t * in-range but wrong/unaligned SGE (corrupted WQE, or a malicious host\n+\t * in a CVM) would otherwise truncate onto a neighbouring slot, letting\n+\t * us read a stale response that could complete the wrong, reused\n+\t * in-flight request.  Require the index to be in range AND the address\n+\t * to exactly match the value the driver posted for that slot.\n+\t */\n+\trq_base_addr = hwc_rxq-\u003emsg_buf-\u003emem_info.dma_handle;\n+\trx_req_idx = (sge-\u003eaddress - rq_base_addr) / hwc-\u003emax_resp_msg_size;\n+\n+\tif (rx_req_idx \u003e= hwc_rxq-\u003equeue_depth) {\n+\t\t/* Cannot trust which WQE this is, so we cannot safely repost\n+\t\t * it; leak one RX WQE and bail.  An out-of-range index means\n+\t\t * a corrupted SGE from hardware (or host tampering), an\n+\t\t * unrecoverable device error.\n+\t\t */\n+\t\tdev_err(hwc-\u003edev, \"HWC RX: SGE idx %llu out of range\\n\",\n+\t\t\trx_req_idx);\n \t\treturn;\n \t}\n \n \trx_req = \u0026hwc_rxq-\u003emsg_buf-\u003ereqs[rx_req_idx];\n+\tif (sge-\u003eaddress != (u64)rx_req-\u003ebuf_sge_addr) {\n+\t\t/* In-range index but the address does not match what the\n+\t\t * driver posted for that slot; the same unrecoverable case,\n+\t\t * so leak this WQE rather than repost the wrong one.\n+\t\t */\n+\t\tdev_err(hwc-\u003edev, \"HWC RX: invalid SGE address %llx (idx=%llu)\\n\",\n+\t\t\tsge-\u003eaddress, rx_req_idx);\n+\t\treturn;\n+\t}\n+\n \tresp = (struct gdma_resp_hdr *)rx_req-\u003ebuf_va;\n \n+\t/* Validate resp_len covers the response header before reading\n+\t * hwc_msg_id.  A short response leaves stale data from the\n+\t * previous buffer occupant, which could match a live slot and\n+\t * complete the wrong request.\n+\t */\n+\tif (rx_oob-\u003etx_oob_data_size \u003c sizeof(*resp)) {\n+\t\tdev_err(hwc-\u003edev, \"HWC RX: short resp_len=%u\\n\",\n+\t\t\trx_oob-\u003etx_oob_data_size);\n+\t\tmana_hwc_post_rx_wqe(hwc_rxq, rx_req);\n+\t\treturn;\n+\t}\n+\n \t/* Read msg_id once from DMA buffer to prevent TOCTOU:\n \t * DMA memory is shared/unencrypted in CVMs - host can\n \t * modify it between reads.\n@@ -281,6 +404,7 @@ static void mana_hwc_rx_event_handler(void *ctx, u32 gdma_rxq_id,\n \tmsg_id = READ_ONCE(resp-\u003eresponse.hwc_msg_id);\n \tif (msg_id \u003e= hwc-\u003enum_inflight_msg) {\n \t\tdev_err(hwc-\u003edev, \"HWC RX: wrong msg_id=%u\\n\", msg_id);\n+\t\tmana_hwc_post_rx_wqe(hwc_rxq, rx_req);\n \t\treturn;\n \t}\n \n@@ -384,14 +508,20 @@ static void mana_hwc_comp_event(void *ctx, struct gdma_queue *q_self)\n \n static void mana_hwc_destroy_cq(struct gdma_context *gc, struct hwc_cq *hwc_cq)\n {\n-\tkfree(hwc_cq-\u003ecomp_buf);\n-\n \tif (hwc_cq-\u003egdma_cq)\n \t\tmana_gd_destroy_queue(gc, hwc_cq-\u003egdma_cq);\n \n+\t/* comp_buf is reached only by mana_hwc_comp_event(), which the\n+\t * EQ handler invokes via cq_table[id].  The CQ destroy above\n+\t * already cleared that slot and ran synchronize_rcu(), so no\n+\t * handler can reach comp_buf once it returns.  Destroying the EQ\n+\t * here additionally tears down the IRQ (defense in depth) before\n+\t * comp_buf and hwc_cq are freed below.\n+\t */\n \tif (hwc_cq-\u003egdma_eq)\n \t\tmana_gd_destroy_queue(gc, hwc_cq-\u003egdma_eq);\n \n+\tkfree(hwc_cq-\u003ecomp_buf);\n \tkfree(hwc_cq);\n }\n \n@@ -560,6 +690,7 @@ static int mana_hwc_create_wq(struct hw_channel_context *hwc,\n \thwc_wq-\u003egdma_wq = queue;\n \thwc_wq-\u003equeue_depth = q_depth;\n \thwc_wq-\u003ehwc_cq = hwc_cq;\n+\tspin_lock_init(\u0026hwc_wq-\u003elock);\n \n \terr = mana_hwc_alloc_dma_buf(hwc, q_depth, max_msg_size,\n \t\t\t\t     \u0026hwc_wq-\u003emsg_buf);\n@@ -577,7 +708,7 @@ static int mana_hwc_create_wq(struct hw_channel_context *hwc,\n \treturn err;\n }\n \n-static int mana_hwc_post_tx_wqe(const struct hwc_wq *hwc_txq,\n+static int mana_hwc_post_tx_wqe(struct hwc_wq *hwc_txq,\n \t\t\t\tstruct hwc_work_request *req,\n \t\t\t\tu32 dest_virt_rq_id, u32 dest_virt_rcq_id,\n \t\t\t\tbool dest_pf)\n@@ -616,7 +747,11 @@ static int mana_hwc_post_tx_wqe(const struct hwc_wq *hwc_txq,\n \treq-\u003ewqe_req.inline_oob_data = tx_oob;\n \treq-\u003ewqe_req.client_data_unit = 0;\n \n+\t/* Serialize WQE posting — multiple senders may call concurrently. */\n+\tspin_lock(\u0026hwc_txq-\u003elock);\n \terr = mana_gd_post_and_ring(hwc_txq-\u003egdma_wq, \u0026req-\u003ewqe_req, NULL);\n+\tspin_unlock(\u0026hwc_txq-\u003elock);\n+\n \tif (err)\n \t\tdev_err(dev, \"Failed to post WQE on HWC SQ: %d\\n\", err);\n \treturn err;\n@@ -627,7 +762,7 @@ static int mana_hwc_init_inflight_msg(struct hw_channel_context *hwc,\n {\n \tint err;\n \n-\tsema_init(\u0026hwc-\u003esema, num_msg);\n+\tinit_waitqueue_head(\u0026hwc-\u003emsg_waitq);\n \n \terr = mana_gd_alloc_res_map(num_msg, \u0026hwc-\u003einflight_msg_res);\n \tif (err)\n@@ -657,23 +792,40 @@ static int mana_hwc_test_channel(struct hw_channel_context *hwc, u16 q_depth,\n \tif (!ctx)\n \t\treturn -ENOMEM;\n \n-\tfor (i = 0; i \u003c q_depth; ++i)\n+\tfor (i = 0; i \u003c q_depth; ++i) {\n+\t\tspin_lock_init(\u0026ctx[i].lock);\n \t\tinit_completion(\u0026ctx[i].comp_event);\n+\t}\n \n \thwc-\u003ecaller_ctx = ctx;\n \n-\treturn mana_gd_test_eq(gc, hwc-\u003ecq-\u003egdma_eq);\n+\t/* channel_up must be set before the test EQ request, because\n+\t * the request goes through mana_hwc_get_msg_index() which\n+\t * checks channel_up.  caller_ctx is allocated above, so\n+\t * concurrent access to a NULL caller_ctx is not possible.\n+\t */\n+\thwc-\u003echannel_up = true;\n+\n+\terr = mana_gd_test_eq(gc, hwc-\u003ecq-\u003egdma_eq);\n+\tif (err)\n+\t\thwc-\u003echannel_up = false;\n+\n+\treturn err;\n }\n \n-static int mana_hwc_establish_channel(struct gdma_context *gc, u16 *q_depth,\n+static int mana_hwc_establish_channel(struct gdma_context *gc, u32 *q_depth,\n \t\t\t\t      u32 *max_req_msg_size,\n \t\t\t\t      u32 *max_resp_msg_size)\n {\n+\t/* No RCU needed: called only from mana_hwc_create_channel\n+\t * during init, before the channel is published to senders.\n+\t */\n \tstruct hw_channel_context *hwc = gc-\u003ehwc.driver_data;\n \tstruct gdma_queue *rq = hwc-\u003erxq-\u003egdma_wq;\n \tstruct gdma_queue *sq = hwc-\u003etxq-\u003egdma_wq;\n \tstruct gdma_queue *eq = hwc-\u003ecq-\u003egdma_eq;\n \tstruct gdma_queue *cq = hwc-\u003ecq-\u003egdma_cq;\n+\tstruct gdma_queue __rcu **cq_table;\n \tint err;\n \n \tinit_completion(\u0026hwc-\u003ehwc_init_eqe_comp);\n@@ -687,24 +839,51 @@ static int mana_hwc_establish_channel(struct gdma_context *gc, u16 *q_depth,\n \tif (err)\n \t\treturn err;\n \n-\tif (!wait_for_completion_timeout(\u0026hwc-\u003ehwc_init_eqe_comp, 60 * HZ))\n-\t\treturn -ETIMEDOUT;\n+\t/* setup_hwc activated MST entries — hardware can now DMA into\n+\t * our queue buffers.  If anything below fails, we must tear\n+\t * down before returning so the caller doesn't need to track\n+\t * whether setup_hwc succeeded.\n+\t */\n+\thwc-\u003esetup_active = true;\n+\n+\tif (!wait_for_completion_timeout(\u0026hwc-\u003ehwc_init_eqe_comp, 60 * HZ)) {\n+\t\terr = -ETIMEDOUT;\n+\t\tgoto teardown;\n+\t}\n \n \t*q_depth = hwc-\u003ehwc_init_q_depth_max;\n \t*max_req_msg_size = hwc-\u003ehwc_init_max_req_msg_size;\n \t*max_resp_msg_size = hwc-\u003ehwc_init_max_resp_msg_size;\n \n \t/* Both were set in mana_hwc_init_event_handler(). */\n-\tif (WARN_ON(cq-\u003eid \u003e= gc-\u003emax_num_cqs))\n-\t\treturn -EPROTO;\n+\tif (WARN_ON(cq-\u003eid \u003e= gc-\u003emax_num_cqs)) {\n+\t\terr = -EPROTO;\n+\t\tgoto teardown;\n+\t}\n \n-\tgc-\u003ecq_table = vcalloc(gc-\u003emax_num_cqs, sizeof(struct gdma_queue *));\n-\tif (!gc-\u003ecq_table)\n-\t\treturn -ENOMEM;\n+\tcq_table = vcalloc(gc-\u003emax_num_cqs, sizeof(*cq_table));\n+\tif (!cq_table) {\n+\t\terr = -ENOMEM;\n+\t\tgoto teardown;\n+\t}\n \n-\tgc-\u003ecq_table[cq-\u003eid] = cq;\n+\trcu_assign_pointer(cq_table[cq-\u003eid], cq);\n+\t/* Publish the fully-initialised table last; pairs with the\n+\t * rcu_dereference(gc-\u003ecq_table) in mana_gd_process_eqe().\n+\t */\n+\trcu_assign_pointer(gc-\u003ecq_table, cq_table);\n \n \treturn 0;\n+\n+teardown:\n+\t{\n+\t\tint td_err = mana_smc_teardown_hwc(\u0026gc-\u003eshm_channel, false);\n+\n+\t\tif (!td_err)\n+\t\t\thwc-\u003esetup_active = false;\n+\n+\t\treturn td_err ? td_err : err;\n+\t}\n }\n \n static int mana_hwc_init_queues(struct hw_channel_context *hwc, u16 q_depth,\n@@ -712,6 +891,12 @@ static int mana_hwc_init_queues(struct hw_channel_context *hwc, u16 q_depth,\n {\n \tint err;\n \n+\t/* CQ depth is q_depth * 2 (SQ + RQ) passed as u16 to create_cq.\n+\t * Cap to prevent u16 truncation.\n+\t */\n+\tif (q_depth \u003e U16_MAX / 2)\n+\t\tq_depth = U16_MAX / 2;\n+\n \terr = mana_hwc_init_inflight_msg(hwc, q_depth);\n \tif (err)\n \t\treturn err;\n@@ -728,14 +913,14 @@ static int mana_hwc_init_queues(struct hw_channel_context *hwc, u16 q_depth,\n \t\tgoto out;\n \t}\n \n-\terr = mana_hwc_create_wq(hwc, GDMA_RQ, q_depth, max_req_msg_size,\n+\terr = mana_hwc_create_wq(hwc, GDMA_RQ, q_depth, max_resp_msg_size,\n \t\t\t\t hwc-\u003ecq, \u0026hwc-\u003erxq);\n \tif (err) {\n \t\tdev_err(hwc-\u003edev, \"Failed to create HWC RQ: %d\\n\", err);\n \t\tgoto out;\n \t}\n \n-\terr = mana_hwc_create_wq(hwc, GDMA_SQ, q_depth, max_resp_msg_size,\n+\terr = mana_hwc_create_wq(hwc, GDMA_SQ, q_depth, max_req_msg_size,\n \t\t\t\t hwc-\u003ecq, \u0026hwc-\u003etxq);\n \tif (err) {\n \t\tdev_err(hwc-\u003edev, \"Failed to create HWC SQ: %d\\n\", err);\n@@ -744,6 +929,7 @@ static int mana_hwc_init_queues(struct hw_channel_context *hwc, u16 q_depth,\n \n \thwc-\u003enum_inflight_msg = q_depth;\n \thwc-\u003emax_req_msg_size = max_req_msg_size;\n+\thwc-\u003emax_resp_msg_size = max_resp_msg_size;\n \n \treturn 0;\n out:\n@@ -751,12 +937,62 @@ static int mana_hwc_init_queues(struct hw_channel_context *hwc, u16 q_depth,\n \treturn err;\n }\n \n+/* Tear down all HWC queues and free associated resources.  Used on\n+ * the reinit-with-higher-queue-depth path and reinit fallback.\n+ *\n+ * PRECONDITION: must be called only during channel bring-up in\n+ * mana_hwc_create_channel(), before the channel is published to\n+ * senders.  There the setup thread is effectively single-threaded\n+ * (serialized against teardown by the PCI/PM device_lock or, on the\n+ * service path, GC_IN_SERVICE, with the data path not yet probed),\n+ * channel_up is still false, caller_ctx is not yet allocated, and\n+ * active_senders is 0 — so no concurrent request/response user can\n+ * touch these queues.  That is why this skips the hwc_lock-protected\n+ * driver_data clear + active_senders drain that\n+ * mana_hwc_destroy_channel() needs for the runtime teardown race;\n+ * only the CQ-first ordering below (to fence off a pending interrupt)\n+ * is required.  Calling this on a live, published channel would be a\n+ * use-after-free.\n+ */\n+static void mana_hwc_destroy_queues(struct hw_channel_context *hwc)\n+{\n+\tstruct gdma_context *gc = hwc-\u003egdma_dev-\u003egdma_context;\n+\n+\t/* Destroy CQ first to deregister the EQ from the interrupt\n+\t * handler list before freeing caller_ctx, TXQ, or RXQ memory.\n+\t * A pending interrupt handler could still reach handle_resp()\n+\t * which dereferences caller_ctx.\n+\t */\n+\tif (hwc-\u003ecq) {\n+\t\tmana_hwc_destroy_cq(gc, hwc-\u003ecq);\n+\t\thwc-\u003ecq = NULL;\n+\t}\n+\n+\tkfree(hwc-\u003ecaller_ctx);\n+\thwc-\u003ecaller_ctx = NULL;\n+\n+\tif (hwc-\u003etxq) {\n+\t\tmana_hwc_destroy_wq(hwc, hwc-\u003etxq);\n+\t\thwc-\u003etxq = NULL;\n+\t}\n+\n+\tif (hwc-\u003erxq) {\n+\t\tmana_hwc_destroy_wq(hwc, hwc-\u003erxq);\n+\t\thwc-\u003erxq = NULL;\n+\t}\n+\n+\tmana_gd_free_res_map(\u0026hwc-\u003einflight_msg_res);\n+\thwc-\u003enum_inflight_msg = 0;\n+}\n+\n int mana_hwc_create_channel(struct gdma_context *gc)\n {\n \tu32 max_req_msg_size, max_resp_msg_size;\n \tstruct gdma_dev *gd = \u0026gc-\u003ehwc;\n \tstruct hw_channel_context *hwc;\n-\tu16 q_depth_max;\n+\tstruct gdma_queue __rcu **old_cq_table;\n+\tunsigned long flags;\n+\tu32 q_depth_max;\n \tint err;\n \n \thwc = kzalloc_obj(*hwc);\n@@ -764,10 +1000,11 @@ int mana_hwc_create_channel(struct gdma_context *gc)\n \t\treturn -ENOMEM;\n \n \tgd-\u003egdma_context = gc;\n-\tgd-\u003edriver_data = hwc;\n \thwc-\u003egdma_dev = gd;\n \thwc-\u003edev = gc-\u003edev;\n \thwc-\u003ehwc_timeout = HW_CHANNEL_WAIT_RESOURCE_TIMEOUT_MS;\n+\thwc-\u003eactive_senders = 0;\n+\tinit_waitqueue_head(\u0026gc-\u003ehwc_drain_waitq);\n \n \t/* HWC's instance number is always 0. */\n \tgd-\u003edev_id.as_uint32 = 0;\n@@ -776,6 +1013,15 @@ int mana_hwc_create_channel(struct gdma_context *gc)\n \tgd-\u003epdid = INVALID_PDID;\n \tgd-\u003edoorbell = INVALID_DOORBELL;\n \n+\t/* Publish driver_data last, under hwc_lock: the lock orders the hwc\n+\t * initialisation above before the pointer becomes visible and\n+\t * serialises the publish against the control-plane readers in\n+\t * mana_gd_send_request(), mana_need_log() and mana_serv_reset().\n+\t */\n+\tspin_lock_irqsave(\u0026gc-\u003ehwc_lock, flags);\n+\tgc-\u003ehwc.driver_data = hwc;\n+\tspin_unlock_irqrestore(\u0026gc-\u003ehwc_lock, flags);\n+\n \t/* mana_hwc_init_queues() only creates the required data structures,\n \t * and doesn't touch the HWC device.\n \t */\n@@ -794,8 +1040,130 @@ int mana_hwc_create_channel(struct gdma_context *gc)\n \t\tgoto out;\n \t}\n \n+\t/* The channel was bootstrapped at a minimal queue depth.  If the\n+\t * device reports a higher maximum, tear down and rebuild with\n+\t * the larger depth so more HWC commands can be in flight.\n+\t */\n+\tif (q_depth_max \u003e HW_CHANNEL_VF_BOOTSTRAP_QUEUE_DEPTH) {\n+\t\t/* q_depth_max now carries the full device-reported value\n+\t\t * (HWC_INIT_DATA_QUEUE_DEPTH is 24-bit).  Clamp it to the\n+\t\t * depth the rest of the driver supports — create_cq() takes\n+\t\t * q_depth * 2 as a u16 — before the overflow check below, so\n+\t\t * an over-large but otherwise-valid depth is reduced to the\n+\t\t * maximum instead of wrapping or being rejected.\n+\t\t */\n+\t\tif (q_depth_max \u003e U16_MAX / 2)\n+\t\t\tq_depth_max = U16_MAX / 2;\n+\n+\t\t/* Sanity-check device-reported values before using them\n+\t\t * to size DMA allocations.  Enforce protocol minimums\n+\t\t * for message sizes and check that q_depth * max_msg_size\n+\t\t * plus alignment headroom fits in u32 (for\n+\t\t * mana_hwc_alloc_dma_buf's MANA_PAGE_ALIGN).\n+\t\t */\n+\t\tif (!max_req_msg_size || !max_resp_msg_size ||\n+\t\t    max_req_msg_size \u003c sizeof(struct gdma_req_hdr) ||\n+\t\t    max_resp_msg_size \u003c sizeof(struct gdma_resp_hdr) ||\n+\t\t    (u64)q_depth_max * max_req_msg_size \u003e\n+\t\t\tU32_MAX - MANA_PAGE_SIZE ||\n+\t\t    (u64)q_depth_max * max_resp_msg_size \u003e\n+\t\t\tU32_MAX - MANA_PAGE_SIZE) {\n+\t\t\tdev_err(hwc-\u003edev,\n+\t\t\t\t\"HWC: invalid dims q=%u req=%u resp=%u\\n\",\n+\t\t\t\tq_depth_max, max_req_msg_size,\n+\t\t\t\tmax_resp_msg_size);\n+\t\t\tq_depth_max = HW_CHANNEL_VF_BOOTSTRAP_QUEUE_DEPTH;\n+\t\t\tgoto skip_reinit;\n+\t\t}\n+\n+\t\terr = mana_smc_teardown_hwc(\u0026gc-\u003eshm_channel, false);\n+\t\tif (err) {\n+\t\t\t/* Keep using the bootstrap-depth channel. */\n+\t\t\tdev_err(hwc-\u003edev,\n+\t\t\t\t\"Failed to teardown HWC for reinit: %d\\n\",\n+\t\t\t\terr);\n+\t\t\tq_depth_max = HW_CHANNEL_VF_BOOTSTRAP_QUEUE_DEPTH;\n+\t\t\tgoto skip_reinit;\n+\t\t}\n+\n+\t\thwc-\u003esetup_active = false;\n+\n+\t\t/* Destroy queues first — mana_gd_destroy_cq inside\n+\t\t * unpublishes the CQ from cq_table via\n+\t\t * rcu_assign_pointer(NULL) + synchronize_rcu.\n+\t\t * Must happen while cq_table is still valid.\n+\t\t */\n+\t\tmana_hwc_destroy_queues(hwc);\n+\n+\t\told_cq_table = rcu_replace_pointer(gc-\u003ecq_table, NULL, true);\n+\t\tsynchronize_rcu();\n+\t\tvfree(old_cq_table);\n+\n+\t\terr = mana_hwc_init_queues(hwc, q_depth_max,\n+\t\t\t\t\t   max_req_msg_size,\n+\t\t\t\t\t   max_resp_msg_size);\n+\t\tif (err) {\n+\t\t\tdev_err(hwc-\u003edev, \"Failed to reinit HWC: %d\\n\", err);\n+\t\t\tgoto reinit_fallback;\n+\t\t}\n+\n+\t\terr = mana_hwc_establish_channel(gc, \u0026q_depth_max,\n+\t\t\t\t\t\t \u0026max_req_msg_size,\n+\t\t\t\t\t\t \u0026max_resp_msg_size);\n+\t\tif (err) {\n+\t\t\tdev_err(hwc-\u003edev, \"Failed to re-establish HWC: %d\\n\",\n+\t\t\t\terr);\n+\t\t\t/* establish_channel does internal teardown on\n+\t\t\t * failure.  If teardown succeeded (setup_active\n+\t\t\t * cleared), MST entries are invalidated and we\n+\t\t\t * can try the bootstrap fallback.  If teardown\n+\t\t\t * also failed (setup_active still set), hardware\n+\t\t\t * mappings may still be active — skip fallback.\n+\t\t\t */\n+\t\t\tif (hwc-\u003esetup_active)\n+\t\t\t\tgoto out;\n+\t\t\tgoto reinit_fallback;\n+\t\t}\n+\t}\n+\n+\tgoto skip_reinit;\n+\n+reinit_fallback:\n+\t/* Restore bootstrap-depth channel so the device remains functional.\n+\t * Free cq_table if it was allocated by a partially successful\n+\t * establish attempt.\n+\t */\n+\tdev_warn(hwc-\u003edev, \"HWC reinit failed, falling back to bootstrap depth\\n\");\n+\n+\tmana_hwc_destroy_queues(hwc);\n+\n+\told_cq_table = rcu_replace_pointer(gc-\u003ecq_table, NULL, true);\n+\tsynchronize_rcu();\n+\tvfree(old_cq_table);\n+\n+\terr = mana_hwc_init_queues(hwc, HW_CHANNEL_VF_BOOTSTRAP_QUEUE_DEPTH,\n+\t\t\t\t   HW_CHANNEL_MAX_REQUEST_SIZE,\n+\t\t\t\t   HW_CHANNEL_MAX_RESPONSE_SIZE);\n+\tif (err) {\n+\t\tdev_err(hwc-\u003edev, \"Failed to restore bootstrap HWC: %d\\n\", err);\n+\t\tgoto out;\n+\t}\n+\n+\terr = mana_hwc_establish_channel(gc, \u0026q_depth_max, \u0026max_req_msg_size,\n+\t\t\t\t\t \u0026max_resp_msg_size);\n+\tif (err) {\n+\t\tdev_err(hwc-\u003edev, \"Failed to re-establish bootstrap HWC: %d\\n\",\n+\t\t\terr);\n+\t\tgoto out;\n+\t}\n+\n+skip_reinit:\n+\n+\t/* No RCU needed: still in mana_hwc_create_channel, the\n+\t * pointer has not been published to concurrent senders yet.\n+\t */\n \terr = mana_hwc_test_channel(gc-\u003ehwc.driver_data,\n-\t\t\t\t    HW_CHANNEL_VF_BOOTSTRAP_QUEUE_DEPTH,\n+\t\t\t\t    hwc-\u003enum_inflight_msg,\n \t\t\t\t    max_req_msg_size, max_resp_msg_size);\n \tif (err) {\n \t\tdev_err(hwc-\u003edev, \"Failed to test HWC: %d\\n\", err);\n@@ -810,27 +1178,195 @@ int mana_hwc_create_channel(struct gdma_context *gc)\n \n void mana_hwc_destroy_channel(struct gdma_context *gc)\n {\n+\t/* This is the only destroy entry point.  driver_data is read\n+\t * plainly here (teardown is serialised against other teardown);\n+\t * it is cleared under hwc_lock below before hwc is freed.\n+\t */\n \tstruct hw_channel_context *hwc = gc-\u003ehwc.driver_data;\n+\tstruct gdma_queue __rcu **old_cq_table;\n+\tunsigned long flags;\n \n \tif (!hwc)\n \t\treturn;\n \n-\t/* gc-\u003emax_num_cqs is set in mana_hwc_init_event_handler(). If it's\n-\t * non-zero, the HWC worked and we should tear down the HWC here.\n+\t/* Prevent new requests from starting and wake any\n+\t * threads waiting for a free msg slot.  Set channel_up under\n+\t * the bitmap lock so get_msg_index() cannot acquire a slot\n+\t * and increment active_senders after this point.\n+\t *\n+\t * If channel_up is already false (e.g. init failed before\n+\t * the channel was established), skip the lock — it may not\n+\t * have been initialized yet, and no senders can be active.\n \t */\n-\tif (gc-\u003emax_num_cqs \u003e 0) {\n-\t\tmana_smc_teardown_hwc(\u0026gc-\u003eshm_channel, false);\n-\t\tgc-\u003emax_num_cqs = 0;\n+\tif (hwc-\u003echannel_up) {\n+\t\tspin_lock_irqsave(\u0026hwc-\u003einflight_msg_res.lock, flags);\n+\t\thwc-\u003echannel_up = false;\n+\t\tspin_unlock_irqrestore(\u0026hwc-\u003einflight_msg_res.lock, flags);\n+\t\twake_up_all(\u0026hwc-\u003emsg_waitq);\n \t}\n \n+\t/* Clear the pointer under hwc_lock so new callers in\n+\t * mana_gd_send_request() see NULL and return -ENODEV.  The lock\n+\t * makes the readers' \"load driver_data + active_senders++\"\n+\t * atomic against this store, so once it returns no new sender can\n+\t * take a reference; the active_senders drain below waits out those\n+\t * that already did, before their hwc is freed.\n+\t */\n+\tspin_lock_irqsave(\u0026gc-\u003ehwc_lock, flags);\n+\tgc-\u003ehwc.driver_data = NULL;\n+\tspin_unlock_irqrestore(\u0026gc-\u003ehwc_lock, flags);\n+\n+\t/* Force-complete any in-flight senders so they observe -ENODEV,\n+\t * return, and drop their references.  This runs before the HWC\n+\t * hardware teardown below, so a live interrupt may still deliver\n+\t * a real response via handle_resp() concurrently — that is safe\n+\t * because the per-slot refcount model tolerates a concurrent\n+\t * complete() and both paths (handle_resp and this loop) drop\n+\t * their refs without double-releasing the slot.\n+\t */\n+\tif (hwc-\u003ecaller_ctx) {\n+\t\tstruct hwc_caller_ctx *ctx;\n+\t\tint i;\n+\n+\t\tfor (i = 0; i \u003c hwc-\u003enum_inflight_msg; i++) {\n+\t\t\tif (!test_bit(i, hwc-\u003einflight_msg_res.map))\n+\t\t\t\tcontinue;\n+\n+\t\t\tctx = \u0026hwc-\u003ecaller_ctx[i];\n+\n+\t\t\t/* Wake senders blocked on wait_for_completion.\n+\t\t\t * Set error under lock to avoid racing with\n+\t\t\t * handle_resp() which writes error under the\n+\t\t\t * same lock.  The sender NULLs output_buf\n+\t\t\t * after waking — doing it here would race\n+\t\t\t * with a sender that hasn't set output_buf yet.\n+\t\t\t */\n+\t\t\tspin_lock_irqsave(\u0026ctx-\u003elock, flags);\n+\t\t\tctx-\u003eerror = -ENODEV;\n+\t\t\tcomplete(\u0026ctx-\u003ecomp_event);\n+\t\t\tspin_unlock_irqrestore(\u0026ctx-\u003elock, flags);\n+\t\t}\n+\t}\n+\n+\t/* Wait for all sender threads to finish and drop their refs\n+\t * before touching the hardware or freeing anything.  Draining\n+\t * here -- ahead of the teardown/FLR block below -- guarantees no\n+\t * in-flight sender is still running when this function returns on\n+\t * any path, including the FLR-failure early return; otherwise a\n+\t * stranded sender would dereference gc-\u003ehwc_lock /\n+\t * gc-\u003ehwc_drain_waitq after the caller frees gc.\n+\t * After this, only slots held by timed-out senders whose\n+\t * handle_resp() never ran remain in the bitmap.\n+\t *\n+\t * active_senders is only ever modified under hwc_lock, and the\n+\t * last sender's wake_up() runs under that lock before it is\n+\t * released.  Evaluating the condition under hwc_lock therefore\n+\t * guarantees that once we observe 0 the waking sender has\n+\t * already dropped the lock -- i.e. finished touching gc -- so it\n+\t * cannot race the caller freeing gc after this returns.\n+\t */\n+\tspin_lock_irq(\u0026gc-\u003ehwc_lock);\n+\twait_event_lock_irq(gc-\u003ehwc_drain_waitq,\n+\t\t\t    hwc-\u003eactive_senders == 0, gc-\u003ehwc_lock);\n+\tspin_unlock_irq(\u0026gc-\u003ehwc_lock);\n+\n+\t/* Tear down the HWC if setup_hwc previously activated MST entries.\n+\t * This is the definitive flag — unlike max_num_cqs which depends\n+\t * on the init EQE arriving.\n+\t *\n+\t * The return value is intentionally not checked.  This is the\n+\t * terminal cleanup path (device removal, suspend, or init\n+\t * failure) — resources must be freed regardless.  If teardown\n+\t * fails, hardware may still have active MST entries, but:\n+\t *\n+\t *  - Interrupts: mana_hwc_destroy_cq() below calls\n+\t *    mana_gd_deregister_irq() which removes the HWC EQ from\n+\t *    the interrupt dispatch list via list_del_rcu() +\n+\t *    synchronize_rcu().  After that, no interrupt handler can\n+\t *    invoke handle_resp() or access CQ/RQ buffers — even if\n+\t *    the IRQ is shared with data path queues.\n+\t *\n+\t *  - DMA: mana_hwc_destroy_wq() frees DMA buffers via\n+\t *    dma_free_coherent() which unmaps the IOVA from the\n+\t *    IOMMU.  Any stale hardware DMA to the old address\n+\t *    faults at the IOMMU, not in kernel memory.\n+\t */\n+\tif (hwc-\u003esetup_active) {\n+\t\tint td_err = mana_smc_teardown_hwc(\u0026gc-\u003eshm_channel, false);\n+\n+\t\tif (td_err) {\n+\t\t\tdev_err(gc-\u003edev, \"HWC teardown failed: %d, issuing FLR\\n\",\n+\t\t\t\ttd_err);\n+\n+\t\t\t/* On systems without IOMMU, freeing DMA memory with\n+\t\t\t * active hardware MST mappings risks memory corruption.\n+\t\t\t * Issue FLR to force-reset the device and invalidate\n+\t\t\t * all hardware state including MST entries.\n+\t\t\t */\n+\t\t\ttd_err = pcie_flr(to_pci_dev(gc-\u003edev));\n+\t\t\tif (td_err) {\n+\t\t\t\t/* Device wedged: teardown and FLR\n+\t\t\t\t * both failed, so hardware may still\n+\t\t\t\t * DMA into our queue buffers.  Leak\n+\t\t\t\t * all HWC DMA buffers -- and the EQ\n+\t\t\t\t * IRQ registration, whose safe removal\n+\t\t\t\t * needs those queues -- instead of\n+\t\t\t\t * freeing memory the device can still\n+\t\t\t\t * reach; without an IOMMU the freed\n+\t\t\t\t * pages could be reused under live\n+\t\t\t\t * DMA.\n+\t\t\t\t */\n+\n+\t\t\t\tdev_warn(gc-\u003edev,\n+\t\t\t\t\t \"Leaked HWC DMA buffers (CQ/RQ/TXQ) to prevent memory corruption. Device is no longer usable.\\n\");\n+\n+\t\t\t\t/* Do not call destroy_cq/wq below;\n+\t\t\t\t * they would dma_free_coherent().\n+\t\t\t\t * Leave hwc/cq/rxq/txq allocated.\n+\t\t\t\t */\n+\t\t\t\treturn;\n+\t\t\t}\n+\n+\t\t\tdev_info(gc-\u003edev, \"FLR succeeded, hardware state cleared\\n\");\n+\t\t}\n+\n+\t\thwc-\u003esetup_active = false;\n+\t}\n+\n+\t/* Tear down the HWC CQ object first — mana_hwc_destroy_cq()\n+\t * both unpublishes the CQ from cq_table (+synchronize_rcu) and\n+\t * deregisters the HWC EQ from the interrupt handler's RCU list\n+\t * (via mana_gd_deregister_irq + synchronize_rcu), guaranteeing\n+\t * no interrupt handler can access RQ/TXQ buffers after this\n+\t * point.  The active_senders drain above ensures no sender is\n+\t * accessing the CQ via txq-\u003ehwc_cq when it is destroyed.  Then\n+\t * destroy TXQ and RQ safely.\n+\t */\n+\tif (hwc-\u003ecq)\n+\t\tmana_hwc_destroy_cq(hwc-\u003egdma_dev-\u003egdma_context, hwc-\u003ecq);\n+\n \tif (hwc-\u003etxq)\n \t\tmana_hwc_destroy_wq(hwc, hwc-\u003etxq);\n \n \tif (hwc-\u003erxq)\n \t\tmana_hwc_destroy_wq(hwc, hwc-\u003erxq);\n \n-\tif (hwc-\u003ecq)\n-\t\tmana_hwc_destroy_cq(hwc-\u003egdma_dev-\u003egdma_context, hwc-\u003ecq);\n+\t/* Release any slots still held — these belong to timed-out\n+\t * senders where handle_resp() never ran (refcount = 1 with\n+\t * handle_resp's ref still outstanding).\n+\t */\n+\tif (hwc-\u003ecaller_ctx) {\n+\t\tstruct hwc_caller_ctx *ctx;\n+\t\tint i;\n+\n+\t\tfor (i = 0; i \u003c hwc-\u003enum_inflight_msg; i++) {\n+\t\t\tif (!test_bit(i, hwc-\u003einflight_msg_res.map))\n+\t\t\t\tcontinue;\n+\n+\t\t\tctx = \u0026hwc-\u003ecaller_ctx[i];\n+\t\t\thwc_ctx_put(hwc, ctx);\n+\t\t}\n+\t}\n \n \tkfree(hwc-\u003ecaller_ctx);\n \thwc-\u003ecaller_ctx = NULL;\n@@ -845,11 +1381,11 @@ void mana_hwc_destroy_channel(struct gdma_context *gc)\n \thwc-\u003ehwc_timeout = 0;\n \n \tkfree(hwc);\n-\tgc-\u003ehwc.driver_data = NULL;\n \tgc-\u003ehwc.gdma_context = NULL;\n \n-\tvfree(gc-\u003ecq_table);\n-\tgc-\u003ecq_table = NULL;\n+\told_cq_table = rcu_replace_pointer(gc-\u003ecq_table, NULL, true);\n+\tsynchronize_rcu();\n+\tvfree(old_cq_table);\n }\n \n int mana_hwc_send_request(struct hw_channel_context *hwc, u32 req_len,\n@@ -860,13 +1396,17 @@ int mana_hwc_send_request(struct hw_channel_context *hwc, u32 req_len,\n \tstruct hwc_wq *txq = hwc-\u003etxq;\n \tstruct gdma_req_hdr *req_msg;\n \tstruct hwc_caller_ctx *ctx;\n+\tunsigned long flags;\n \tu32 dest_vrcq = 0;\n \tu32 dest_vrq = 0;\n \tu32 command;\n+\tu32 status;\n \tu16 msg_id;\n \tint err;\n \n-\tmana_hwc_get_msg_index(hwc, \u0026msg_id);\n+\terr = mana_hwc_get_msg_index(hwc, \u0026msg_id);\n+\tif (err)\n+\t\treturn err;\n \n \ttx_wr = \u0026txq-\u003emsg_buf-\u003ereqs[msg_id];\n \n@@ -878,8 +1418,11 @@ int mana_hwc_send_request(struct hw_channel_context *hwc, u32 req_len,\n \t}\n \n \tctx = hwc-\u003ecaller_ctx + msg_id;\n+\n+\tspin_lock_irqsave(\u0026ctx-\u003elock, flags);\n \tctx-\u003eoutput_buf = resp;\n \tctx-\u003eoutput_buflen = resp_len;\n+\tspin_unlock_irqrestore(\u0026ctx-\u003elock, flags);\n \n \treq_msg = (struct gdma_req_hdr *)tx_wr-\u003ebuf_va;\n \tif (req)\n@@ -895,8 +1438,14 @@ int mana_hwc_send_request(struct hw_channel_context *hwc, u32 req_len,\n \t\tdest_vrcq = hwc-\u003epf_dest_vrcq_id;\n \t}\n \n+\t/* Take handle_resp's ref before posting — hardware can respond\n+\t * immediately after the doorbell ring.\n+\t */\n+\trefcount_inc(\u0026ctx-\u003erefcnt);\n+\n \terr = mana_hwc_post_tx_wqe(txq, tx_wr, dest_vrq, dest_vrcq, false);\n \tif (err) {\n+\t\trefcount_dec(\u0026ctx-\u003erefcnt);\n \t\tdev_err(hwc-\u003edev, \"HWC: Failed to post send WQE: %d\\n\", err);\n \t\tgoto out;\n \t}\n@@ -907,31 +1456,86 @@ int mana_hwc_send_request(struct hw_channel_context *hwc, u32 req_len,\n \t\t\tdev_err(hwc-\u003edev, \"Command 0x%x timed out: %u ms\\n\",\n \t\t\t\tcommand, hwc-\u003ehwc_timeout);\n \n-\t\t/* Reduce further waiting if HWC no response */\n+\t\t/* NULL out output_buf so a late handle_resp() won't write\n+\t\t * into the caller's buffer after the sender returns, then\n+\t\t * check whether handle_resp() already delivered a valid\n+\t\t * response between the timeout firing and this lock\n+\t\t * acquisition — ctx-\u003eerror != -EINPROGRESS means it ran.\n+\t\t */\n+\t\tspin_lock_irqsave(\u0026ctx-\u003elock, flags);\n+\t\tctx-\u003eoutput_buf = NULL;\n+\t\terr = ctx-\u003eerror;\n+\t\tstatus = ctx-\u003estatus_code;\n+\t\tspin_unlock_irqrestore(\u0026ctx-\u003elock, flags);\n+\n+\t\tif (err != -EINPROGRESS) {\n+\t\t\t/* handle_resp() delivered a valid response just after\n+\t\t\t * the timeout fired.  The hardware is alive, so use\n+\t\t\t * the response and leave the channel usable; do not\n+\t\t\t * latch hwc_timed_out or degrade hwc_timeout for what\n+\t\t\t * turned out to be a transient race.\n+\t\t\t */\n+\t\t\thwc_ctx_put(hwc, ctx);\n+\t\t\tgoto check_status;\n+\t\t}\n+\n+\t\t/* Genuine timeout: no response arrived.  Reduce further\n+\t\t * waiting, and mark the channel timed out under the bitmap\n+\t\t * lock so get_msg_index() cannot acquire new slots after this.\n+\t\t */\n \t\tif (hwc-\u003ehwc_timeout \u003e 1)\n \t\t\thwc-\u003ehwc_timeout = 1;\n \n+\t\tspin_lock_irqsave(\u0026hwc-\u003einflight_msg_res.lock, flags);\n+\t\thwc-\u003ehwc_timed_out = true;\n+\t\tspin_unlock_irqrestore(\u0026hwc-\u003einflight_msg_res.lock, flags);\n+\t\twake_up_all(\u0026hwc-\u003emsg_waitq);\n+\n \t\terr = -ETIMEDOUT;\n-\t\tgoto out;\n+\t\thwc_ctx_put(hwc, ctx);\n+\t\tgoto done;\n \t}\n \n-\tif (ctx-\u003eerror) {\n-\t\terr = ctx-\u003eerror;\n-\t\tgoto out;\n-\t}\n+\t/* NULL output_buf so a late handle_resp() won't memcpy into\n+\t * the caller's buffer after the sender exits.  Read error and\n+\t * status_code under the same lock — after hwc_ctx_put the slot\n+\t * may be reused and these fields overwritten.\n+\t */\n+\tspin_lock_irqsave(\u0026ctx-\u003elock, flags);\n+\tctx-\u003eoutput_buf = NULL;\n+\terr = ctx-\u003eerror;\n+\tstatus = ctx-\u003estatus_code;\n+\tspin_unlock_irqrestore(\u0026ctx-\u003elock, flags);\n+\thwc_ctx_put(hwc, ctx);\n+\n+check_status:\n+\tif (err)\n+\t\tgoto done;\n \n-\tif (ctx-\u003estatus_code \u0026\u0026 ctx-\u003estatus_code != GDMA_STATUS_MORE_ENTRIES) {\n-\t\tif (ctx-\u003estatus_code == GDMA_STATUS_CMD_UNSUPPORTED) {\n+\tif (status \u0026\u0026 status != GDMA_STATUS_MORE_ENTRIES) {\n+\t\tif (status == GDMA_STATUS_CMD_UNSUPPORTED) {\n \t\t\terr = -EOPNOTSUPP;\n-\t\t\tgoto out;\n+\t\t\tgoto done;\n \t\t}\n+\n \t\tif (command != MANA_QUERY_PHY_STAT)\n \t\t\tdev_err(hwc-\u003edev, \"Command 0x%x failed with status: 0x%x\\n\",\n-\t\t\t\tcommand, ctx-\u003estatus_code);\n+\t\t\t\tcommand, status);\n \t\terr = -EPROTO;\n-\t\tgoto out;\n+\t\tgoto done;\n \t}\n+\n+\terr = 0;\n+\tgoto done;\n out:\n-\tmana_hwc_put_msg_index(hwc, msg_id);\n+\t/* Pre-post error paths: no WQE was submitted so handle_resp()\n+\t * cannot race here.  refcount is 1 (no second ref taken).\n+\t */\n+\tctx = hwc-\u003ecaller_ctx + msg_id;\n+\tspin_lock_irqsave(\u0026ctx-\u003elock, flags);\n+\tctx-\u003eoutput_buf = NULL;\n+\tspin_unlock_irqrestore(\u0026ctx-\u003elock, flags);\n+\thwc_ctx_put(hwc, ctx);\n+done:\n \treturn err;\n }\ndiff --git a/drivers/net/ethernet/microsoft/mana/mana_en.c b/drivers/net/ethernet/microsoft/mana/mana_en.c\nindex a8c329bdbacf83..5508746590e7cd 100644\n--- a/drivers/net/ethernet/microsoft/mana/mana_en.c\n+++ b/drivers/net/ethernet/microsoft/mana/mana_en.c\n@@ -2648,6 +2648,7 @@ static int mana_create_txq(struct mana_port_context *apc,\n \tstruct mana_obj_spec cq_spec;\n \tstruct gdma_queue_spec spec;\n \tstruct gdma_context *gc;\n+\tstruct gdma_queue __rcu **cq_table;\n \tstruct mana_txq *txq;\n \tstruct mana_cq *cq;\n \tu32 txq_size;\n@@ -2753,12 +2754,18 @@ static int mana_create_txq(struct mana_port_context *apc,\n \n \t\tcq-\u003egdma_id = cq-\u003egdma_cq-\u003eid;\n \n-\t\tif (WARN_ON(cq-\u003egdma_id \u003e= gc-\u003emax_num_cqs)) {\n+\t\t/* No rcu_read_lock(): mana_create_txq runs under RTNL during\n+\t\t * netdev bring-up, inside the netdev lifetime that\n+\t\t * mana_remove() drains before the base cq_table can be freed.\n+\t\t * See gdma_context::cq_table in gdma.h for why \"true\" is sound.\n+\t\t */\n+\t\tcq_table = rcu_dereference_protected(gc-\u003ecq_table, true);\n+\t\tif (WARN_ON(!cq_table || cq-\u003egdma_id \u003e= gc-\u003emax_num_cqs)) {\n \t\t\terr = -EINVAL;\n \t\t\tgoto out;\n \t\t}\n \n-\t\tgc-\u003ecq_table[cq-\u003egdma_id] = cq-\u003egdma_cq;\n+\t\trcu_assign_pointer(cq_table[cq-\u003egdma_id], cq-\u003egdma_cq);\n \n \t\tmana_create_txq_debugfs(apc, i);\n \n@@ -2986,6 +2993,7 @@ static struct mana_rxq *mana_create_rxq(struct mana_port_context *apc,\n \tstruct gdma_queue_spec spec;\n \tstruct mana_cq *cq = NULL;\n \tstruct gdma_context *gc;\n+\tstruct gdma_queue __rcu **cq_table;\n \tu32 cq_size, rq_size;\n \tstruct mana_rxq *rxq;\n \tint err;\n@@ -3075,12 +3083,18 @@ static struct mana_rxq *mana_create_rxq(struct mana_port_context *apc,\n \tif (err)\n \t\tgoto out;\n \n-\tif (WARN_ON(cq-\u003egdma_id \u003e= gc-\u003emax_num_cqs)) {\n+\t/* No rcu_read_lock(): mana_create_rxq runs under RTNL during netdev\n+\t * bring-up, inside the netdev lifetime that mana_remove() drains\n+\t * before the base cq_table can be freed.  See gdma_context::cq_table\n+\t * in gdma.h for why \"true\" is sound.\n+\t */\n+\tcq_table = rcu_dereference_protected(gc-\u003ecq_table, true);\n+\tif (WARN_ON(!cq_table || cq-\u003egdma_id \u003e= gc-\u003emax_num_cqs)) {\n \t\terr = -EINVAL;\n \t\tgoto out;\n \t}\n \n-\tgc-\u003ecq_table[cq-\u003egdma_id] = cq-\u003egdma_cq;\n+\trcu_assign_pointer(cq_table[cq-\u003egdma_id], cq-\u003egdma_cq);\n \n \tnetif_napi_add_weight_locked(ndev, \u0026cq-\u003enapi, mana_poll, 1);\n \ndiff --git a/include/net/mana/gdma.h b/include/net/mana/gdma.h\nindex 8529cef0d7c452..a4eac6f7c36609 100644\n--- a/include/net/mana/gdma.h\n+++ b/include/net/mana/gdma.h\n@@ -428,9 +428,34 @@ struct gdma_context {\n \t/* L2 MTU */\n \tu16 adapter_mtu;\n \n-\t/* This maps a CQ index to the queue structure. */\n+\t/* Size of cq_table, i.e. the largest valid CQ index + 1.  Set once\n+\t * when cq_table is allocated and treated as immutable for the\n+\t * table's lifetime (a bound only) -- it is never reset on teardown.\n+\t * cq_table == NULL is the sole \"table torn down\" signal, so every\n+\t * cq_table[id] access must guard with both !cq_table (gone) and\n+\t * id \u003e= max_num_cqs (out of bounds).\n+\t */\n \tunsigned int\t\tmax_num_cqs;\n-\tstruct gdma_queue\t**cq_table;\n+\t/* Both the base pointer and each entry are RCU-managed.  The fast\n+\t * path (mana_gd_process_eqe) reads the base via rcu_dereference()\n+\t * under rcu_read_lock(), so the table is freed with\n+\t * rcu_assign_pointer(NULL) + synchronize_rcu() and an in-flight\n+\t * reader can never observe freed memory.\n+\t *\n+\t * The slow paths -- mana_gd_destroy_cq() and the CQ install/remove\n+\t * callers (mana_create_txq/_rxq, mana_ib_install/remove_cq_cb) --\n+\t * instead read the base with rcu_dereference_protected(cq_table,\n+\t * true).  The bare \"true\" is justified by teardown ordering, not by\n+\t * a lock: the base table is replaced+freed only by\n+\t * mana_hwc_destroy_channel() (and the create-time reinit), and every\n+\t * teardown path first runs mana_remove() + mana_rdma_remove(), which\n+\t * synchronously drain the netdev and the IB device\n+\t * (unregister_netdevice / ib_unregister_device) that bound all\n+\t * install/remove callers; the reinit case runs before either\n+\t * consumer is probed.  So no slow-path caller can run while the base\n+\t * table is being freed.\n+\t */\n+\tstruct gdma_queue\t__rcu * __rcu *cq_table;\n \n \t/* Protect eq_test_event and test_event_eq_id  */\n \tstruct mutex\t\teq_test_event_mutex;\n@@ -456,6 +481,21 @@ struct gdma_context {\n \t/* Hardware communication channel (HWC) */\n \tstruct gdma_dev\t\thwc;\n \n+\t/* destroy_channel() waits here for all HWC senders to exit.\n+\t * Lives on gc (not hwc) so wake_up() after the last sender's\n+\t * atomic_dec doesn't dereference freed hwc memory.\n+\t */\n+\twait_queue_head_t\thwc_drain_waitq;\n+\n+\t/* Serializes hwc.driver_data (the hw_channel_context pointer)\n+\t * between the control-plane readers in mana_gd_send_request(),\n+\t * mana_need_log() and mana_serv_reset() and the publish/clear in\n+\t * mana_hwc_create_channel()/mana_hwc_destroy_channel().  All users\n+\t * are control-plane (HWC commands sleep; reset runs on a workqueue),\n+\t * so a plain spinlock -- not RCU -- is sufficient.\n+\t */\n+\tspinlock_t\t\thwc_lock;\n+\n \t/* Azure network adapter */\n \tstruct gdma_dev\t\tmana;\n \n@@ -668,6 +708,9 @@ enum {\n /* Driver supports dynamic interrupt moderation - DIM */\n #define GDMA_DRV_CAP_FLAG_1_DYN_INTERRUPT_MODERATION BIT(28)\n \n+/* Driver supports dynamic queue depth for HWC */\n+#define GDMA_DRV_CAP_FLAG_1_DYN_HWC_QUEUE_DEPTH BIT(29)\n+\n #define GDMA_DRV_CAP_FLAGS1 \\\n \t(GDMA_DRV_CAP_FLAG_1_EQ_SHARING_MULTI_VPORT | \\\n \t GDMA_DRV_CAP_FLAG_1_NAPI_WKDONE_FIX | \\\n@@ -683,6 +726,7 @@ enum {\n \t GDMA_DRV_CAP_FLAG_1_PROBE_RECOVERY | \\\n \t GDMA_DRV_CAP_FLAG_1_HANDLE_STALL_SQ_RECOVERY | \\\n \t GDMA_DRV_CAP_FLAG_1_HWC_TIMEOUT_RECOVERY | \\\n+\t GDMA_DRV_CAP_FLAG_1_DYN_HWC_QUEUE_DEPTH | \\\n \t GDMA_DRV_CAP_FLAG_1_EQ_MSI_UNSHARE_MULTI_VPORT | \\\n \t GDMA_DRV_CAP_FLAG_1_DYN_INTERRUPT_MODERATION)\n \ndiff --git a/include/net/mana/hw_channel.h b/include/net/mana/hw_channel.h\nindex 16feb39616c1be..eec5d3aa23e1d5 100644\n--- a/include/net/mana/hw_channel.h\n+++ b/include/net/mana/hw_channel.h\n@@ -164,6 +164,9 @@ struct hwc_wq {\n \tu16 queue_depth;\n \n \tstruct hwc_cq *hwc_cq;\n+\n+\t/* Serializes concurrent mana_gd_post_and_ring() calls. */\n+\tspinlock_t lock;\n };\n \n struct hwc_caller_ctx {\n@@ -173,6 +176,17 @@ struct hwc_caller_ctx {\n \n \tu32 error; /* Linux error code */\n \tu32 status_code;\n+\n+\t/* Protects output_buf against concurrent access from\n+\t * handle_resp() (CQ interrupt) and the sender timeout path.\n+\t */\n+\tspinlock_t lock;\n+\n+\t/* Tracks sender + handle_resp ownership.  The last put\n+\t * (refcount reaches 0) releases the bitmap slot.\n+\t */\n+\trefcount_t refcnt;\n+\tu16 msg_id;\n };\n \n struct hw_channel_context {\n@@ -181,8 +195,9 @@ struct hw_channel_context {\n \n \tu16 num_inflight_msg;\n \tu32 max_req_msg_size;\n+\tu32 max_resp_msg_size;\n \n-\tu16 hwc_init_q_depth_max;\n+\tu32 hwc_init_q_depth_max;\n \tu32 hwc_init_max_req_msg_size;\n \tu32 hwc_init_max_resp_msg_size;\n \n@@ -192,13 +207,38 @@ struct hw_channel_context {\n \tstruct hwc_wq *txq;\n \tstruct hwc_cq *cq;\n \n-\tstruct semaphore sema;\n \tstruct gdma_resource inflight_msg_res;\n+\t/* Waitqueue for senders blocked on a full inflight bitmap. */\n+\twait_queue_head_t msg_waitq;\n \n \tu32 pf_dest_vrq_id;\n \tu32 pf_dest_vrcq_id;\n \tu32 hwc_timeout;\n \n+\t/* Set after channel is fully established; cleared on teardown to\n+\t * abort waiters in mana_hwc_get_msg_index() and reject new sends.\n+\t */\n+\tbool channel_up;\n+\n+\t/* Set on first HWC timeout.  Causes get_msg_index() to return\n+\t * -ETIMEDOUT instead of waiting, draining all queued senders.\n+\t */\n+\tbool hwc_timed_out;\n+\n+\t/* Set after mana_smc_setup_hwc() succeeds (hardware has active\n+\t * MST entries).  On recoverable paths (establish_channel)\n+\t * cleared only after successful teardown so a retry remains\n+\t * possible.  On the terminal destroy_channel path, cleared\n+\t * unconditionally since hwc is about to be freed.\n+\t */\n+\tbool setup_active;\n+\n+\t/* Count of in-flight mana_gd_send_request() callers.  Protected\n+\t * by gc-\u003ehwc_lock; the last sender to drop it to zero wakes\n+\t * gc-\u003ehwc_drain_waitq for the mana_hwc_destroy_channel() drain.\n+\t */\n+\tunsigned int active_senders;\n+\n \tstruct hwc_caller_ctx *caller_ctx;\n };\n \n"
}

2/1

2026/07/22 17:52

agent

patch-evaluator

Model:

best-expensive

Args:

null

Results:

{
  "EnableConfigs": [
    "MICROSOFT_MANA",
    "MANA_INFINIBAND"
  ],
  "FocusSymbols": [
    "mana_ib_install_cq_cb",
    "mana_ib_remove_cq_cb",
    "mana_hwc_get_msg_index",
    "mana_hwc_handle_resp",
    "mana_hwc_rx_event_handler",
    "mana_hwc_create_channel",
    "mana_hwc_destroy_channel",
    "mana_hwc_send_request",
    "mana_gd_process_eqe",
    "mana_gd_destroy_cq"
  ],
  "Reasoning": "The patch makes functional changes to the Microsoft Azure Network Adapter (MANA) driver, including adding RCU protection for the CQ table, handling HWC timeouts, and adding refcounts for caller contexts. These changes affect the core logic of the driver and should be fuzzed.",
  "WorthFuzzing": true
}

Instruction:

You are an expert Linux kernel maintainer.
Your job is to review a provided patch series and determine
if it makes functional changes to the kernel that should be fuzzed.

IMPORTANT: The changes have ALREADY been applied and committed as the HEAD commit in
your workspace. Do NOT rely on your internal knowledge of the kernel. You must actively
use your code access tools to examine the actual source code and confirm any assumptions.

Return WorthFuzzing=false if the patch only contains:
- Modifications to Documentation/, Kconfig files, or code comments.
- Purely decorative changes, such as logging (e.g., pr_err, printk) or tracepoints.
- Changes to numeric constants or macros that do not functionally alter execution flow.
- Code paths that are impossible to reach in virtualized environments like GCE or QEMU,
even when utilizing software-emulated hardware (e.g., usb gadget, mac80211_hwsim).

If it modifies reachable core kernel logic, drivers, or architectures, use your code search
tools to verify the code can be executed, then return WorthFuzzing=true.

When returning WorthFuzzing=true, you MUST ALSO:
1. Extract any specific kernel functions that should be heavily fuzzed into FocusSymbols.
   Avoid listing generic hot-path functions to prevent skewed test distributions.
2. Identify any specific CONFIG_ options required to properly test this new/modified feature.
   Go and look into the Kconfig files and check for ifdefs around the code, do not make assumptions.
   Do not list too generic configs (we already have them enabled). Only list those that
   specifically cover the modified code. List them in the EnableConfigs output array,
   and DO NOT add a 'CONFIG_' prefix (e.g., return "NET_IPV4" instead of "CONFIG_NET_IPV4").
Prefer calling several tools at the same time to save round-trips.


Use set-results tool to provide results of the analysis.
It must be called exactly once before the final reply.
Ignore results of this tool.

Prompt:

For your convenience, here is the diff of the changes:
commit 7a7f673c5a49005405d55fc2540f242d3b89cd96
Author: syz-cluster <triage@syzkaller.com>
Date:   Wed Jul 22 17:52:19 2026 +0000

    syz-cluster: applied patch under review

diff --git a/drivers/infiniband/hw/mana/cq.c b/drivers/infiniband/hw/mana/cq.c
index f2547989f42290..2bf4be21cede79 100644
--- a/drivers/infiniband/hw/mana/cq.c
+++ b/drivers/infiniband/hw/mana/cq.c
@@ -131,12 +131,20 @@ static void mana_ib_cq_handler(void *ctx, struct gdma_queue *gdma_cq)
 int mana_ib_install_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq *cq)
 {
 	struct gdma_context *gc = mdev_to_gc(mdev);
+	struct gdma_queue __rcu **cq_table;
 	struct gdma_queue *gdma_cq;
 
-	if (cq->queue.id >= gc->max_num_cqs)
+	/* No rcu_read_lock(): install/remove run within the IB device
+	 * lifetime, which mana_rdma_remove() (ib_unregister_device) drains
+	 * before the base cq_table can be freed.  See gdma_context::cq_table
+	 * in gdma.h for why "true" is sound.
+	 */
+	cq_table = rcu_dereference_protected(gc->cq_table, true);
+	if (!cq_table || cq->queue.id >= gc->max_num_cqs)
 		return -EINVAL;
+
 	/* Create CQ table entry, sharing a CQ between WQs is not supported */
-	if (gc->cq_table[cq->queue.id])
+	if (rcu_access_pointer(cq_table[cq->queue.id]))
 		return -EINVAL;
 	if (cq->queue.kmem)
 		gdma_cq = cq->queue.kmem;
@@ -149,23 +157,49 @@ int mana_ib_install_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq *cq)
 	gdma_cq->type = GDMA_CQ;
 	gdma_cq->cq.callback = mana_ib_cq_handler;
 	gdma_cq->id = cq->queue.id;
-	gc->cq_table[cq->queue.id] = gdma_cq;
+	rcu_assign_pointer(cq_table[cq->queue.id], gdma_cq);
 	return 0;
 }
 
 void mana_ib_remove_cq_cb(struct mana_ib_dev *mdev, struct mana_ib_cq *cq)
 {
 	struct gdma_context *gc = mdev_to_gc(mdev);
+	struct gdma_queue __rcu **cq_table;
+	struct gdma_queue *gdma_cq;
 
-	if (cq->queue.id >= gc->max_num_cqs || cq->queue.id == INVALID_QUEUE_ID)
+	if (cq->queue.id == INVALID_QUEUE_ID)
 		return;
 
 	if (cq->queue.kmem)
 	/* Then it will be cleaned and removed by the mana */
 		return;
 
-	kfree(gc->cq_table[cq->queue.id]);
-	gc->cq_table[cq->queue.id] = NULL;
+	/* No rcu_read_lock(): like mana_ib_install_cq_cb(), this runs within
+	 * the IB device lifetime that mana_rdma_remove() drains before the
+	 * base cq_table can be freed.  See gdma_context::cq_table in gdma.h.
+	 */
+	cq_table = rcu_dereference_protected(gc->cq_table, true);
+	if (!cq_table || cq->queue.id >= gc->max_num_cqs)
+		return;
+	/* Removers for a given CQ are serialized by the IB core, so the slot
+	 * is read and cleared without rcu_read_lock() or atomicity: a CQ is
+	 * never torn down while a live QP references it (cq->usecnt), nor
+	 * while the QP-create that installed the entry is still running (that
+	 * create holds a reference on the CQ uobject across its error path,
+	 * before usecnt is taken).  Any double-remove is therefore sequential
+	 * -- the later caller sees the NULL stored below and returns.
+	 */
+	gdma_cq = rcu_dereference_protected(cq_table[cq->queue.id], true);
+	if (!gdma_cq)
+		return;  /* already removed by a prior teardown path */
+
+	rcu_assign_pointer(cq_table[cq->queue.id], NULL);
+
+	/* Wait for in-flight EQ handlers that may have loaded the old
+	 * pointer via rcu_dereference() to finish before freeing.
+	 */
+	synchronize_rcu();
+	kfree(gdma_cq);
 }
 
 int mana_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
diff --git a/drivers/net/ethernet/microsoft/mana/gdma_main.c b/drivers/net/ethernet/microsoft/mana/gdma_main.c
index a38d4bb74621f9..17711ee37c8be3 100644
--- a/drivers/net/ethernet/microsoft/mana/gdma_main.c
+++ b/drivers/net/ethernet/microsoft/mana/gdma_main.c
@@ -161,6 +161,8 @@ static int mana_gd_init_registers(struct pci_dev *pdev)
 bool mana_need_log(struct gdma_context *gc, int err)
 {
 	struct hw_channel_context *hwc;
+	bool need_log = true;
+	unsigned long flags;
 
 	if (err != -ETIMEDOUT)
 		return true;
@@ -168,11 +170,13 @@ bool mana_need_log(struct gdma_context *gc, int err)
 	if (!gc)
 		return true;
 
+	spin_lock_irqsave(&gc->hwc_lock, flags);
 	hwc = gc->hwc.driver_data;
 	if (hwc && hwc->hwc_timeout == 0)
-		return false;
+		need_log = false;
+	spin_unlock_irqrestore(&gc->hwc_lock, flags);
 
-	return true;
+	return need_log;
 }
 
 static int mana_gd_query_max_resources(struct pci_dev *pdev)
@@ -367,9 +371,27 @@ static int mana_gd_detect_devices(struct pci_dev *pdev)
 int mana_gd_send_request(struct gdma_context *gc, u32 req_len, const void *req,
 			 u32 resp_len, void *resp)
 {
-	struct hw_channel_context *hwc = gc->hwc.driver_data;
+	struct hw_channel_context *hwc;
+	unsigned long flags;
+	int err;
+
+	spin_lock_irqsave(&gc->hwc_lock, flags);
+	hwc = gc->hwc.driver_data;
+	if (!hwc) {
+		spin_unlock_irqrestore(&gc->hwc_lock, flags);
+		return -ENODEV;
+	}
+	hwc->active_senders++;
+	spin_unlock_irqrestore(&gc->hwc_lock, flags);
+
+	err = mana_hwc_send_request(hwc, req_len, req, resp_len, resp);
 
-	return mana_hwc_send_request(hwc, req_len, req, resp_len, resp);
+	spin_lock_irqsave(&gc->hwc_lock, flags);
+	if (--hwc->active_senders == 0)
+		wake_up(&gc->hwc_drain_waitq);
+	spin_unlock_irqrestore(&gc->hwc_lock, flags);
+
+	return err;
 }
 EXPORT_SYMBOL_NS(mana_gd_send_request, "NET_MANA");
 
@@ -622,6 +644,7 @@ static void mana_serv_reset(struct pci_dev *pdev)
 {
 	struct gdma_context *gc = pci_get_drvdata(pdev);
 	struct hw_channel_context *hwc;
+	unsigned long flags;
 	int ret;
 
 	if (!gc) {
@@ -631,14 +654,17 @@ static void mana_serv_reset(struct pci_dev *pdev)
 		return;
 	}
 
+	spin_lock_irqsave(&gc->hwc_lock, flags);
 	hwc = gc->hwc.driver_data;
 	if (!hwc) {
+		spin_unlock_irqrestore(&gc->hwc_lock, flags);
 		dev_err(&pdev->dev, "MANA service: no HWC\n");
 		goto out;
 	}
 
 	/* HWC is not responding in this case, so don't wait */
 	hwc->hwc_timeout = 0;
+	spin_unlock_irqrestore(&gc->hwc_lock, flags);
 
 	dev_info(&pdev->dev, "MANA reset cycle start\n");
 
@@ -761,6 +787,7 @@ static void mana_gd_process_eqe(struct gdma_queue *eq)
 	union gdma_eqe_info eqe_info;
 	enum gdma_eqe_type type;
 	struct gdma_event event;
+	struct gdma_queue __rcu **cq_table;
 	struct gdma_queue *cq;
 	struct gdma_eqe *eqe;
 	u32 cq_id;
@@ -772,10 +799,11 @@ static void mana_gd_process_eqe(struct gdma_queue *eq)
 	switch (type) {
 	case GDMA_EQE_COMPLETION:
 		cq_id = eqe->details[0] & 0xFFFFFF;
-		if (WARN_ON_ONCE(cq_id >= gc->max_num_cqs))
+		cq_table = rcu_dereference(gc->cq_table);
+		if (WARN_ON_ONCE(cq_id >= gc->max_num_cqs || !cq_table))
 			break;
 
-		cq = gc->cq_table[cq_id];
+		cq = rcu_dereference(cq_table[cq_id]);
 		if (WARN_ON_ONCE(!cq || cq->type != GDMA_CQ || cq->id != cq_id))
 			break;
 
@@ -1082,15 +1110,28 @@ static void mana_gd_create_cq(const struct gdma_queue_spec *spec,
 static void mana_gd_destroy_cq(struct gdma_context *gc,
 			       struct gdma_queue *queue)
 {
+	struct gdma_queue __rcu **cq_table;
 	u32 id = queue->id;
 
-	if (id >= gc->max_num_cqs)
+	/* No rcu_read_lock() here: mana_gd_destroy_cq() runs only on the
+	 * CQ-destroy/teardown path, where the base cq_table is stable.  See
+	 * the lifecycle note on gdma_context::cq_table in gdma.h for why the
+	 * "true" predicate is sound.
+	 */
+	cq_table = rcu_dereference_protected(gc->cq_table, true);
+	if (!cq_table || id >= gc->max_num_cqs)
 		return;
 
-	if (!gc->cq_table[id])
+	if (!rcu_access_pointer(cq_table[id]))
 		return;
 
-	gc->cq_table[id] = NULL;
+	rcu_assign_pointer(cq_table[id], NULL);
+
+	/* Wait for in-flight EQ handlers that may have loaded the old
+	 * pointer via rcu_dereference() to finish before the caller
+	 * frees the CQ memory.
+	 */
+	synchronize_rcu();
 }
 
 int mana_gd_create_hwc_queue(struct gdma_dev *gd,
@@ -1185,6 +1226,16 @@ static int mana_gd_create_dma_region(struct gdma_dev *gd,
 	if (!MANA_PAGE_ALIGNED(gmi->virt_addr))
 		return -EINVAL;
 
+	/* No RCU needed: this runs only on the data-path queue-creation
+	 * path (mana_gd_create_mana_eq/mana_gd_create_mana_wq_cq, called
+	 * by mana_en under RTNL and by mana_ib RDMA verbs, or during
+	 * init).  Every teardown path — mana_gd_remove, mana_gd_suspend,
+	 * and the HWC reset/service path (which goes through
+	 * mana_gd_suspend) — drains those consumers via mana_rdma_remove()
+	 * + mana_remove() before mana_hwc_destroy_channel() clears
+	 * gc->hwc.driver_data, so no concurrent destroy can race with
+	 * this dereference.
+	 */
 	hwc = gc->hwc.driver_data;
 	req_msg_size = struct_size(req, page_addr_list, num_page);
 	if (req_msg_size > hwc->max_req_msg_size)
@@ -1374,7 +1425,17 @@ int mana_gd_verify_vf_version(struct pci_dev *pdev)
 	struct hw_channel_context *hwc;
 	int err;
 
+	/* No RCU needed: this runs only inside mana_gd_setup, on the
+	 * probe and resume paths.  The PCI/PM core holds device_lock
+	 * across .probe/.resume and .remove/.suspend, so setup cannot
+	 * overlap teardown of the same device.  The HWC reset/service
+	 * path is additionally serialized by GC_IN_SERVICE and runs
+	 * suspend (destroy) then resume (this) sequentially in one work
+	 * item.  driver_data was just set by mana_hwc_create_channel
+	 * earlier in this same setup call, so it is live here.
+	 */
 	hwc = gc->hwc.driver_data;
+
 	mana_gd_init_req_hdr(&req.hdr, GDMA_VERIFY_VF_DRIVER_VERSION,
 			     sizeof(req), sizeof(resp));
 
@@ -2364,6 +2425,7 @@ static int mana_gd_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
 
 	mutex_init(&gc->eq_test_event_mutex);
 	mutex_init(&gc->gic_mutex);
+	spin_lock_init(&gc->hwc_lock);
 	pci_set_drvdata(pdev, gc);
 	gc->bar0_pa = pci_resource_start(pdev, 0);
 	gc->bar0_size = pci_resource_len(pdev, 0);
diff --git a/drivers/net/ethernet/microsoft/mana/hw_channel.c b/drivers/net/ethernet/microsoft/mana/hw_channel.c
index e3c24d50dad07c..0481d40443424f 100644
--- a/drivers/net/ethernet/microsoft/mana/hw_channel.c
+++ b/drivers/net/ethernet/microsoft/mana/hw_channel.c
@@ -4,27 +4,55 @@
 #include <net/mana/gdma.h>
 #include <net/mana/mana.h>
 #include <net/mana/hw_channel.h>
+#include <linux/pci.h>
 #include <linux/vmalloc.h>
 
+/* Acquire a free message slot from the inflight bitmap.  Returns
+ * -ENODEV if the channel is torn down, or -ETIMEDOUT if a prior HWC
+ * command has timed out (preserving the error code callers expect).
+ */
 static int mana_hwc_get_msg_index(struct hw_channel_context *hwc, u16 *msg_id)
 {
 	struct gdma_resource *r = &hwc->inflight_msg_res;
 	unsigned long flags;
 	u32 index;
 
-	down(&hwc->sema);
+	for (;;) {
+		spin_lock_irqsave(&r->lock, flags);
 
-	spin_lock_irqsave(&r->lock, flags);
+		if (!hwc->channel_up || hwc->hwc_timed_out) {
+			spin_unlock_irqrestore(&r->lock, flags);
+			return hwc->channel_up ? -ETIMEDOUT : -ENODEV;
+		}
 
-	index = find_first_zero_bit(hwc->inflight_msg_res.map,
-				    hwc->inflight_msg_res.size);
+		index = find_first_zero_bit(r->map, r->size);
+		if (index < r->size) {
+			struct hwc_caller_ctx *ctx;
+
+			bitmap_set(r->map, index, 1);
+			ctx = &hwc->caller_ctx[index];
+			reinit_completion(&ctx->comp_event);
+			refcount_set(&ctx->refcnt, 1);
+			ctx->msg_id = index;
+			ctx->error = -EINPROGRESS;
+			spin_unlock_irqrestore(&r->lock, flags);
+			break;
+		}
+		spin_unlock_irqrestore(&r->lock, flags);
 
-	bitmap_set(hwc->inflight_msg_res.map, index, 1);
+		wait_event(hwc->msg_waitq,
+			   !hwc->channel_up ||
+			   hwc->hwc_timed_out ||
+			   !bitmap_full(r->map, r->size));
 
-	spin_unlock_irqrestore(&r->lock, flags);
+		if (!hwc->channel_up)
+			return -ENODEV;
 
-	*msg_id = index;
+		if (hwc->hwc_timed_out)
+			return -ETIMEDOUT;
+	}
 
+	*msg_id = index;
 	return 0;
 }
 
@@ -34,10 +62,17 @@ static void mana_hwc_put_msg_index(struct hw_channel_context *hwc, u16 msg_id)
 	unsigned long flags;
 
 	spin_lock_irqsave(&r->lock, flags);
-	bitmap_clear(hwc->inflight_msg_res.map, msg_id, 1);
+	bitmap_clear(r->map, msg_id, 1);
 	spin_unlock_irqrestore(&r->lock, flags);
 
-	up(&hwc->sema);
+	wake_up(&hwc->msg_waitq);
+}
+
+static void hwc_ctx_put(struct hw_channel_context *hwc,
+			struct hwc_caller_ctx *ctx)
+{
+	if (refcount_dec_and_test(&ctx->refcnt))
+		mana_hwc_put_msg_index(hwc, ctx->msg_id);
 }
 
 static int mana_hwc_verify_resp_msg(const struct hwc_caller_ctx *caller_ctx,
@@ -83,6 +118,17 @@ static void mana_hwc_handle_resp(struct hw_channel_context *hwc, u32 resp_len,
 	struct hwc_caller_ctx *ctx;
 	int err;
 
+	/* Validate msg_id is in range before using it to index bitmap
+	 * and caller_ctx array.  Malicious firmware could send
+	 * out-of-range msg_id causing out-of-bounds access.
+	 */
+	if (msg_id >= hwc->num_inflight_msg) {
+		dev_err(hwc->dev, "hwc_rx: msg_id %u >= max %u\n",
+			msg_id, hwc->num_inflight_msg);
+		mana_hwc_post_rx_wqe(hwc->rxq, rx_req);
+		return;
+	}
+
 	if (!test_bit(msg_id, hwc->inflight_msg_res.map)) {
 		dev_err(hwc->dev, "hwc_rx: invalid msg_id = %u\n", msg_id);
 		mana_hwc_post_rx_wqe(hwc->rxq, rx_req);
@@ -90,22 +136,44 @@ static void mana_hwc_handle_resp(struct hw_channel_context *hwc, u32 resp_len,
 	}
 
 	ctx = hwc->caller_ctx + msg_id;
-	err = mana_hwc_verify_resp_msg(ctx, resp_msg, resp_len);
-	if (err)
-		goto out;
 
-	ctx->status_code = resp_msg->status;
+	/* Reject responses larger than the RX DMA buffer — the SGE
+	 * limits what hardware can DMA, so an oversized resp_len
+	 * indicates a firmware bug.  Fail rather than silently
+	 * truncating.
+	 */
+	if (resp_len > rx_req->buf_len) {
+		dev_err(hwc->dev, "HWC RX: resp_len %u > buf_len %u\n",
+			resp_len, rx_req->buf_len);
+		resp_len = 0;
+	}
 
-	memcpy(ctx->output_buf, resp_msg, resp_len);
-out:
-	ctx->error = err;
+	spin_lock(&ctx->lock);
+
+	err = mana_hwc_verify_resp_msg(ctx, resp_msg, resp_len);
 
-	/* Must post rx wqe before complete(), otherwise the next rx may
-	 * hit no_wqe error.
+	if (!err && ctx->output_buf) {
+		ctx->status_code = resp_msg->status;
+		memcpy(ctx->output_buf, resp_msg, resp_len);
+		ctx->error = 0;
+	} else if (ctx->output_buf) {
+		/* Only overwrite error if the sender hasn't timed out
+		 * or been force-completed by destroy.  When output_buf
+		 * is NULL, a terminal error (-ENODEV or timeout) has
+		 * already been set — preserve it so the sender doesn't
+		 * see a spurious success.
+		 */
+		ctx->error = err;
+	}
+
+	/* Post RX WQE before completing — the next response may arrive
+	 * immediately and needs a posted buffer.
 	 */
 	mana_hwc_post_rx_wqe(hwc->rxq, rx_req);
-
 	complete(&ctx->comp_event);
+	spin_unlock(&ctx->lock);
+
+	hwc_ctx_put(hwc, ctx);
 }
 
 static void mana_hwc_init_event_handler(void *ctx, struct gdma_queue *q_self,
@@ -146,7 +214,12 @@ static void mana_hwc_init_event_handler(void *ctx, struct gdma_queue *q_self,
 			break;
 
 		case HWC_INIT_DATA_QUEUE_DEPTH:
-			hwc->hwc_init_q_depth_max = (u16)val;
+			/* HWC_INIT_DATA_QUEUE_DEPTH is a 24-bit field.  Keep
+			 * the full device-reported value here; it is clamped
+			 * and validated in mana_hwc_create_channel() rather
+			 * than silently truncated to u16.
+			 */
+			hwc->hwc_init_q_depth_max = val;
 			break;
 
 		case HWC_INIT_DATA_MAX_REQUEST:
@@ -261,19 +334,69 @@ static void mana_hwc_rx_event_handler(void *ctx, u32 gdma_rxq_id,
 
 	sge = (struct gdma_sge *)(wqe + 8 + dma_oob->inline_oob_size_div4 * 4);
 
-	/* Select the RX work request for virtual address and for reposting. */
-	rq_base_addr = hwc_rxq->msg_buf->mem_info.dma_handle;
-	rx_req_idx = (sge->address - rq_base_addr) / hwc->max_req_msg_size;
+	/* inline_oob_size_div4 is read from the WQE in device-accessible RQ
+	 * memory, so a malicious host in a CVM (or buggy firmware) could set
+	 * it to push the SGE past the RQ buffer.  Bounds-check the SGE it
+	 * locates before dereferencing sge->address below.  A validly posted
+	 * WQE keeps the SGE inside the ring (worst case ends exactly at the
+	 * buffer boundary); reject anything that would read past it.  The
+	 * slot cannot be trusted here, so leak this RX WQE rather than repost
+	 * the wrong one -- as in the SGE-address mismatch path below.
+	 */
+	if ((u8 *)(sge + 1) > (u8 *)rq->queue_mem_ptr + rq->queue_size) {
+		dev_err(hwc->dev, "HWC RX: SGE past RQ buffer, oob_div4=%u\n",
+			dma_oob->inline_oob_size_div4);
+		return;
+	}
 
-	if (rx_req_idx >= hwc_rxq->msg_buf->num_reqs) {
-		dev_err(hwc->dev, "HWC RX: wrong rx_req_idx=%llu, num_reqs=%u\n",
-			rx_req_idx, hwc_rxq->msg_buf->num_reqs);
+	/* Recover the originating RX slot from the SGE address.  Of the three
+	 * terms here only sge->address lives in device-accessible RQ memory;
+	 * rq_base_addr and max_resp_msg_size are driver-private constants.  An
+	 * in-range but wrong/unaligned SGE (corrupted WQE, or a malicious host
+	 * in a CVM) would otherwise truncate onto a neighbouring slot, letting
+	 * us read a stale response that could complete the wrong, reused
+	 * in-flight request.  Require the index to be in range AND the address
+	 * to exactly match the value the driver posted for that slot.
+	 */
+	rq_base_addr = hwc_rxq->msg_buf->mem_info.dma_handle;
+	rx_req_idx = (sge->address - rq_base_addr) / hwc->max_resp_msg_size;
+
+	if (rx_req_idx >= hwc_rxq->queue_depth) {
+		/* Cannot trust which WQE this is, so we cannot safely repost
+		 * it; leak one RX WQE and bail.  An out-of-range index means
+		 * a corrupted SGE from hardware (or host tampering), an
+		 * unrecoverable device error.
+		 */
+		dev_err(hwc->dev, "HWC RX: SGE idx %llu out of range\n",
+			rx_req_idx);
 		return;
 	}
 
 	rx_req = &hwc_rxq->msg_buf->reqs[rx_req_idx];
+	if (sge->address != (u64)rx_req->buf_sge_addr) {
+		/* In-range index but the address does not match what the
+		 * driver posted for that slot; the same unrecoverable case,
+		 * so leak this WQE rather than repost the wrong one.
+		 */
+		dev_err(hwc->dev, "HWC RX: invalid SGE address %llx (idx=%llu)\n",
+			sge->address, rx_req_idx);
+		return;
+	}
+
 	resp = (struct gdma_resp_hdr *)rx_req->buf_va;
 
+	/* Validate resp_len covers the response header before reading
+	 * hwc_msg_id.  A short response leaves stale data from the
+	 * previous buffer occupant, which could match a live slot and
+	 * complete the wrong request.
+	 */
+	if (rx_oob->tx_oob_data_size < sizeof(*resp)) {
+		dev_err(hwc->dev, "HWC RX: short resp_len=%u\n",
+			rx_oob->tx_oob_data_size);
+		mana_hwc_post_rx_wqe(hwc_rxq, rx_req);
+		return;
+	}
+
 	/* Read msg_id once from DMA buffer to prevent TOCTOU:
 	 * DMA memory is shared/unencrypted in CVMs - host can
 	 * modify it between reads.
@@ -281,6 +404,7 @@ static void mana_hwc_rx_event_handler(void *ctx, u32 gdma_rxq_id,
 	msg_id = READ_ONCE(resp->response.hwc_msg_id);
 	if (msg_id >= hwc->num_inflight_msg) {
 		dev_err(hwc->dev, "HWC RX: wrong msg_id=%u\n", msg_id);
+		mana_hwc_post_rx_wqe(hwc_rxq, rx_req);
 		return;
 	}
 
@@ -384,14 +508,20 @@ static void mana_hwc_comp_event(void *ctx, struct gdma_queue *q_self)
 
 static void mana_hwc_destroy_cq(struct gdma_context *gc, struct hwc_cq *hwc_cq)
 {
-	kfree(hwc_cq->comp_buf);
-
 	if (hwc_cq->gdma_cq)
 		mana_gd_destroy_queue(gc, hwc_cq->gdma_cq);
 
+	/* comp_buf is reached only by mana_hwc_comp_event(), which the
+	 * EQ handler invokes via cq_table[id].  The CQ destroy above
+	 * already cleared that slot and ran synchronize_rcu(), so no
+	 * handler can reach comp_buf once it returns.  Destroying the EQ
+	 * here additionally tears down the IRQ (defense in depth) before
+	 * comp_buf and hwc_cq are freed below.
+	 */
 	if (hwc_cq->gdma_eq)
 		mana_gd_destroy_queue(gc, hwc_cq->gdma_eq);
 
+	kfree(hwc_cq->comp_buf);
 	kfree(hwc_cq);
 }
 
@@ -560,6 +690,7 @@ static int mana_hwc_create_wq(struct hw_channel_context *hwc,
 	hwc_wq->gdma_wq = queue;
 	hwc_wq->queue_depth = q_depth;
 	hwc_wq->hwc_cq = hwc_cq;
+	spin_lock_init(&hwc_wq->lock);
 
 	err = mana_hwc_alloc_dma_buf(hwc, q_depth, max_msg_size,
 				     &hwc_wq->msg_buf);
@@ -577,7 +708,7 @@ static int mana_hwc_create_wq(struct hw_channel_context *hwc,
 	return err;
 }
 
-static int mana_hwc_post_tx_wqe(const struct hwc_wq *hwc_txq,
+static int mana_hwc_post_tx_wqe(struct hwc_wq *hwc_txq,
 				struct hwc_work_request *req,
 				u32 dest_virt_rq_id, u32 dest_virt_rcq_id,
 				bool dest_pf)
@@ -616,7 +747,11 @@ static int mana_hwc_post_tx_wqe(const struct hwc_wq *hwc_txq,
 	req->wqe_req.inline_oob_data = tx_oob;
 	req->wqe_req.client_data_unit = 0;
 
+	/* Serialize WQE posting — multiple senders may call concurrently. */
+	spin_lock(&hwc_txq->lock);
 	err = mana_gd_post_and_ring(hwc_txq->gdma_wq, &req->wqe_req, NULL);
+	spin_unlock(&hwc_txq->lock);
+
 	if (err)
 		dev_err(dev, "Failed to post WQE on HWC SQ: %d\n", err);
 	return err;
@@ -627,7 +762,7 @@ static int mana_hwc_init_inflight_msg(struct hw_channel_context *hwc,
 {
 	int err;
 
-	sema_init(&hwc->sema, num_msg);
+	init_waitqueue_head(&hwc->msg_waitq);
 
 	err = mana_gd_alloc_res_map(num_msg, &hwc->inflight_msg_res);
 	if (err)
@@ -657,23 +792,40 @@ static int mana_hwc_test_channel(struct hw_channel_context *hwc, u16 q_depth,
 	if (!ctx)
 		return -ENOMEM;
 
-	for (i = 0; i < q_depth; ++i)
+	for (i = 0; i < q_depth; ++i) {
+		spin_lock_init(&ctx[i].lock);
 		init_completion(&ctx[i].comp_event);
+	}
 
 	hwc->caller_ctx = ctx;
 
-	return mana_gd_test_eq(gc, hwc->cq->gdma_eq);
+	/* channel_up must be set before the test EQ request, because
+	 * the request goes through mana_hwc_get_msg_index() which
+	 * checks channel_up.  caller_ctx is allocated above, so
+	 * concurrent access to a NULL caller_ctx is not possible.
+	 */
+	hwc->channel_up = true;
+
+	err = mana_gd_test_eq(gc, hwc->cq->gdma_eq);
+	if (err)
+		hwc->channel_up = false;
+
+	return err;
 }
 
-static int mana_hwc_establish_channel(struct gdma_context *gc, u16 *q_depth,
+static int mana_hwc_establish_channel(struct gdma_context *gc, u32 *q_depth,
 				      u32 *max_req_msg_size,
 				      u32 *max_resp_msg_size)
 {
+	/* No RCU needed: called only from mana_hwc_create_channel
+	 * during init, before the channel is published to senders.
+	 */
 	struct hw_channel_context *hwc = gc->hwc.driver_data;
 	struct gdma_queue *rq = hwc->rxq->gdma_wq;
 	struct gdma_queue *sq = hwc->txq->gdma_wq;
 	struct gdma_queue *eq = hwc->cq->gdma_eq;
 	struct gdma_queue *cq = hwc->cq->gdma_cq;
+	struct gdma_queue __rcu **cq_table;
 	int err;
 
 	init_completion(&hwc->hwc_init_eqe_comp);
@@ -687,24 +839,51 @@ static int mana_hwc_establish_channel(struct gdma_context *gc, u16 *q_depth,
 	if (err)
 		return err;
 
-	if (!wait_for_completion_timeout(&hwc->hwc_init_eqe_comp, 60 * HZ))
-		return -ETIMEDOUT;
+	/* setup_hwc activated MST entries — hardware can now DMA into
+	 * our queue buffers.  If anything below fails, we must tear
+	 * down before returning so the caller doesn't need to track
+	 * whether setup_hwc succeeded.
+	 */
+	hwc->setup_active = true;
+
+	if (!wait_for_completion_timeout(&hwc->hwc_init_eqe_comp, 60 * HZ)) {
+		err = -ETIMEDOUT;
+		goto teardown;
+	}
 
 	*q_depth = hwc->hwc_init_q_depth_max;
 	*max_req_msg_size = hwc->hwc_init_max_req_msg_size;
 	*max_resp_msg_size = hwc->hwc_init_max_resp_msg_size;
 
 	/* Both were set in mana_hwc_init_event_handler(). */
-	if (WARN_ON(cq->id >= gc->max_num_cqs))
-		return -EPROTO;
+	if (WARN_ON(cq->id >= gc->max_num_cqs)) {
+		err = -EPROTO;
+		goto teardown;
+	}
 
-	gc->cq_table = vcalloc(gc->max_num_cqs, sizeof(struct gdma_queue *));
-	if (!gc->cq_table)
-		return -ENOMEM;
+	cq_table = vcalloc(gc->max_num_cqs, sizeof(*cq_table));
+	if (!cq_table) {
+		err = -ENOMEM;
+		goto teardown;
+	}
 
-	gc->cq_table[cq->id] = cq;
+	rcu_assign_pointer(cq_table[cq->id], cq);
+	/* Publish the fully-initialised table last; pairs with the
+	 * rcu_dereference(gc->cq_table) in mana_gd_process_eqe().
+	 */
+	rcu_assign_pointer(gc->cq_table, cq_table);
 
 	return 0;
+
+teardown:
+	{
+		int td_err = mana_smc_teardown_hwc(&gc->shm_channel, false);
+
+		if (!td_err)
+			hwc->setup_active = false;
+
+		return td_err ? td_err : err;
+	}
 }
 
 static int mana_hwc_init_queues(struct hw_channel_context *hwc, u16 q_depth,
@@ -712,6 +891,12 @@ static int mana_hwc_init_queues(struct hw_channel_context *hwc, u16 q_depth,
 {
 	int err;
 
+	/* CQ depth is q_depth * 2 (SQ + RQ) passed as u16 to create_cq.
+	 * Cap to prevent u16 truncation.
+	 */
+	if (q_depth > U16_MAX / 2)
+		q_depth = U16_MAX / 2;
+
 	err = mana_hwc_init_inflight_msg(hwc, q_depth);
 	if (err)
 		return err;
@@ -728,14 +913,14 @@ static int mana_hwc_init_queues(struct hw_channel_context *hwc, u16 q_depth,
 		goto out;
 	}
 
-	err = mana_hwc_create_wq(hwc, GDMA_RQ, q_depth, max_req_msg_size,
+	err = mana_hwc_create_wq(hwc, GDMA_RQ, q_depth, max_resp_msg_size,
 				 hwc->cq, &hwc->rxq);
 	if (err) {
 		dev_err(hwc->dev, "Failed to create HWC RQ: %d\n", err);
 		goto out;
 	}
 
-	err = mana_hwc_create_wq(hwc, GDMA_SQ, q_depth, max_resp_msg_size,
+	err = mana_hwc_create_wq(hwc, GDMA_SQ, q_depth, max_req_msg_size,
 				 hwc->cq, &hwc->txq);
 	if (err) {
 		dev_err(hwc->dev, "Failed to create HWC SQ: %d\n", err);
@@ -744,6 +929,7 @@ static int mana_hwc_init_queues(struct hw_channel_context *hwc, u16 q_depth,
 
 	hwc->num_inflight_msg = q_depth;
 	hwc->max_req_msg_size = max_req_msg_size;
+	hwc->max_resp_msg_size = max_resp_msg_size;
 
 	return 0;
 out:
@@ -751,12 +937,62 @@ static int mana_hwc_init_queues(struct hw_channel_context *hwc, u16 q_depth,
 	return err;
 }
 
+/* Tear down all HWC queues and free associated resources.  Used on
+ * the reinit-with-higher-queue-depth path and reinit fallback.
+ *
+ * PRECONDITION: must be called only during channel bring-up in
+ * mana_hwc_create_channel(), before the channel is published to
+ * senders.  There the setup thread is effectively single-threaded
+ * (serialized against teardown by the PCI/PM device_lock or, on the
+ * service path, GC_IN_SERVICE, with the data path not yet probed),
+ * channel_up is still false, caller_ctx is not yet allocated, and
+ * active_senders is 0 — so no concurrent request/response user can
+ * touch these queues.  That is why this skips the hwc_lock-protected
+ * driver_data clear + active_senders drain that
+ * mana_hwc_destroy_channel() needs for the runtime teardown race;
+ * only the CQ-first ordering below (to fence off a pending interrupt)
+ * is required.  Calling this on a live, published channel would be a
+ * use-after-free.
+ */
+static void mana_hwc_destroy_queues(struct hw_channel_context *hwc)
+{
+	struct gdma_context *gc = hwc->gdma_dev->gdma_context;
+
+	/* Destroy CQ first to deregister the EQ from the interrupt
+	 * handler list before freeing caller_ctx, TXQ, or RXQ memory.
+	 * A pending interrupt handler could still reach handle_resp()
+	 * which dereferences caller_ctx.
+	 */
+	if (hwc->cq) {
+		mana_hwc_destroy_cq(gc, hwc->cq);
+		hwc->cq = NULL;
+	}
+
+	kfree(hwc->caller_ctx);
+	hwc->caller_ctx = NULL;
+
+	if (hwc->txq) {
+		mana_hwc_destroy_wq(hwc, hwc->txq);
+		hwc->txq = NULL;
+	}
+
+	if (hwc->rxq) {
+		mana_hwc_destroy_wq(hwc, hwc->rxq);
+		hwc->rxq = NULL;
+	}
+
+	mana_gd_free_res_map(&hwc->inflight_msg_res);
+	hwc->num_inflight_msg = 0;
+}
+
 int mana_hwc_create_channel(struct gdma_context *gc)
 {
 	u32 max_req_msg_size, max_resp_msg_size;
 	struct gdma_dev *gd = &gc->hwc;
 	struct hw_channel_context *hwc;
-	u16 q_depth_max;
+	struct gdma_queue __rcu **old_cq_table;
+	unsigned long flags;
+	u32 q_depth_max;
 	int err;
 
 	hwc = kzalloc_obj(*hwc);
@@ -764,10 +1000,11 @@ int mana_hwc_create_channel(struct gdma_context *gc)
 		return -ENOMEM;
 
 	gd->gdma_context = gc;
-	gd->driver_data = hwc;
 	hwc->gdma_dev = gd;
 	hwc->dev = gc->dev;
 	hwc->hwc_timeout = HW_CHANNEL_WAIT_RESOURCE_TIMEOUT_MS;
+	hwc->active_senders = 0;
+	init_waitqueue_head(&gc->hwc_drain_waitq);
 
 	/* HWC's instance number is always 0. */
 	gd->dev_id.as_uint32 = 0;
@@ -776,6 +1013,15 @@ int mana_hwc_create_channel(struct gdma_context *gc)
 	gd->pdid = INVALID_PDID;
 	gd->doorbell = INVALID_DOORBELL;
 
+	/* Publish driver_data last, under hwc_lock: the lock orders the hwc
+	 * initialisation above before the pointer becomes visible and
+	 * serialises the publish against the control-plane readers in
+	 * mana_gd_send_request(), mana_need_log() and mana_serv_reset().
+	 */
+	spin_lock_irqsave(&gc->hwc_lock, flags);
+	gc->hwc.driver_data = hwc;
+	spin_unlock_irqrestore(&gc->hwc_lock, flags);
+
 	/* mana_hwc_init_queues() only creates the required data structures,
 	 * and doesn't touch the HWC device.
 	 */
@@ -794,8 +1040,130 @@ int mana_hwc_create_channel(struct gdma_context *gc)
 		goto out;
 	}
 
+	/* The channel was bootstrapped at a minimal queue depth.  If the
+	 * device reports a higher maximum, tear down and rebuild with
+	 * the larger depth so more HWC commands can be in flight.
+	 */
+	if (q_depth_max > HW_CHANNEL_VF_BOOTSTRAP_QUEUE_DEPTH) {
+		/* q_depth_max now carries the full device-reported value
+		 * (HWC_INIT_DATA_QUEUE_DEPTH is 24-bit).  Clamp it to the
+		 * depth the rest of the driver supports — create_cq() takes
+		 * q_depth * 2 as a u16 — before the overflow check below, so
+		 * an over-large but otherwise-valid depth is reduced to the
+		 * maximum instead of wrapping or being rejected.
+		 */
+		if (q_depth_max > U16_MAX / 2)
+			q_depth_max = U16_MAX / 2;
+
+		/* Sanity-check device-reported values before using them
+		 * to size DMA allocations.  Enforce protocol minimums
+		 * for message sizes and check that q_depth * max_msg_size
+		 * plus alignment headroom fits in u32 (for
+		 * mana_hwc_alloc_dma_buf's MANA_PAGE_ALIGN).
+		 */
+		if (!max_req_msg_size || !max_resp_msg_size ||
+		    max_req_msg_size < sizeof(struct gdma_req_hdr) ||
+		    max_resp_msg_size < sizeof(struct gdma_resp_hdr) ||
+		    (u64)q_depth_max * max_req_msg_size >
+			U32_MAX - MANA_PAGE_SIZE ||
+		    (u64)q_depth_max * max_resp_msg_size >
+			U32_MAX - MANA_PAGE_SIZE) {
+			dev_err(hwc->dev,
+				"HWC: invalid dims q=%u req=%u resp=%u\n",
+				q_depth_max, max_req_msg_size,
+				max_resp_msg_size);
+			q_depth_max = HW_CHANNEL_VF_BOOTSTRAP_QUEUE_DEPTH;
+			goto skip_reinit;
+		}
+
+		err = mana_smc_teardown_hwc(&gc->shm_channel, false);
+		if (err) {
+			/* Keep using the bootstrap-depth channel. */
+			dev_err(hwc->dev,
+				"Failed to teardown HWC for reinit: %d\n",
+				err);
+			q_depth_max = HW_CHANNEL_VF_BOOTSTRAP_QUEUE_DEPTH;
+			goto skip_reinit;
+		}
+
+		hwc->setup_active = false;
+
+		/* Destroy queues first — mana_gd_destroy_cq inside
+		 * unpublishes the CQ from cq_table via
+		 * rcu_assign_pointer(NULL) + synchronize_rcu.
+		 * Must happen while cq_table is still valid.
+		 */
+		mana_hwc_destroy_queues(hwc);
+
+		old_cq_table = rcu_replace_pointer(gc->cq_table, NULL, true);
+		synchronize_rcu();
+		vfree(old_cq_table);
+
+		err = mana_hwc_init_queues(hwc, q_depth_max,
+					   max_req_msg_size,
+					   max_resp_msg_size);
+		if (err) {
+			dev_err(hwc->dev, "Failed to reinit HWC: %d\n", err);
+			goto reinit_fallback;
+		}
+
+		err = mana_hwc_establish_channel(gc, &q_depth_max,
+						 &max_req_msg_size,
+						 &max_resp_msg_size);
+		if (err) {
+			dev_err(hwc->dev, "Failed to re-establish HWC: %d\n",
+				err);
+			/* establish_channel does internal teardown on
+			 * failure.  If teardown succeeded (setup_active
+			 * cleared), MST entries are invalidated and we
+			 * can try the bootstrap fallback.  If teardown
+			 * also failed (setup_active still set), hardware
+			 * mappings may still be active — skip fallback.
+			 */
+			if (hwc->setup_active)
+				goto out;
+			goto reinit_fallback;
+		}
+	}
+
+	goto skip_reinit;
+
+reinit_fallback:
+	/* Restore bootstrap-depth channel so the device remains functional.
+	 * Free cq_table if it was allocated by a partially successful
+	 * establish attempt.
+	 */
+	dev_warn(hwc->dev, "HWC reinit failed, falling back to bootstrap depth\n");
+
+	mana_hwc_destroy_queues(hwc);
+
+	old_cq_table = rcu_replace_pointer(gc->cq_table, NULL, true);
+	synchronize_rcu();
+	vfree(old_cq_table);
+
+	err = mana_hwc_init_queues(hwc, HW_CHANNEL_VF_BOOTSTRAP_QUEUE_DEPTH,
+				   HW_CHANNEL_MAX_REQUEST_SIZE,
+				   HW_CHANNEL_MAX_RESPONSE_SIZE);
+	if (err) {
+		dev_err(hwc->dev, "Failed to restore bootstrap HWC: %d\n", err);
+		goto out;
+	}
+
+	err = mana_hwc_establish_channel(gc, &q_depth_max, &max_req_msg_size,
+					 &max_resp_msg_size);
+	if (err) {
+		dev_err(hwc->dev, "Failed to re-establish bootstrap HWC: %d\n",
+			err);
+		goto out;
+	}
+
+skip_reinit:
+
+	/* No RCU needed: still in mana_hwc_create_channel, the
+	 * pointer has not been published to concurrent senders yet.
+	 */
 	err = mana_hwc_test_channel(gc->hwc.driver_data,
-				    HW_CHANNEL_VF_BOOTSTRAP_QUEUE_DEPTH,
+				    hwc->num_inflight_msg,
 				    max_req_msg_size, max_resp_msg_size);
 	if (err) {
 		dev_err(hwc->dev, "Failed to test HWC: %d\n", err);
@@ -810,27 +1178,195 @@ int mana_hwc_create_channel(struct gdma_context *gc)
 
 void mana_hwc_destroy_channel(struct gdma_context *gc)
 {
+	/* This is the only destroy entry point.  driver_data is read
+	 * plainly here (teardown is serialised against other teardown);
+	 * it is cleared under hwc_lock below before hwc is freed.
+	 */
 	struct hw_channel_context *hwc = gc->hwc.driver_data;
+	struct gdma_queue __rcu **old_cq_table;
+	unsigned long flags;
 
 	if (!hwc)
 		return;
 
-	/* gc->max_num_cqs is set in mana_hwc_init_event_handler(). If it's
-	 * non-zero, the HWC worked and we should tear down the HWC here.
+	/* Prevent new requests from starting and wake any
+	 * threads waiting for a free msg slot.  Set channel_up under
+	 * the bitmap lock so get_msg_index() cannot acquire a slot
+	 * and increment active_senders after this point.
+	 *
+	 * If channel_up is already false (e.g. init failed before
+	 * the channel was established), skip the lock — it may not
+	 * have been initialized yet, and no senders can be active.
 	 */
-	if (gc->max_num_cqs > 0) {
-		mana_smc_teardown_hwc(&gc->shm_channel, false);
-		gc->max_num_cqs = 0;
+	if (hwc->channel_up) {
+		spin_lock_irqsave(&hwc->inflight_msg_res.lock, flags);
+		hwc->channel_up = false;
+		spin_unlock_irqrestore(&hwc->inflight_msg_res.lock, flags);
+		wake_up_all(&hwc->msg_waitq);
 	}
 
+	/* Clear the pointer under hwc_lock so new callers in
+	 * mana_gd_send_request() see NULL and return -ENODEV.  The lock
+	 * makes the readers' "load driver_data + active_senders++"
+	 * atomic against this store, so once it returns no new sender can
+	 * take a reference; the active_senders drain below waits out those
+	 * that already did, before their hwc is freed.
+	 */
+	spin_lock_irqsave(&gc->hwc_lock, flags);
+	gc->hwc.driver_data = NULL;
+	spin_unlock_irqrestore(&gc->hwc_lock, flags);
+
+	/* Force-complete any in-flight senders so they observe -ENODEV,
+	 * return, and drop their references.  This runs before the HWC
+	 * hardware teardown below, so a live interrupt may still deliver
+	 * a real response via handle_resp() concurrently — that is safe
+	 * because the per-slot refcount model tolerates a concurrent
+	 * complete() and both paths (handle_resp and this loop) drop
+	 * their refs without double-releasing the slot.
+	 */
+	if (hwc->caller_ctx) {
+		struct hwc_caller_ctx *ctx;
+		int i;
+
+		for (i = 0; i < hwc->num_inflight_msg; i++) {
+			if (!test_bit(i, hwc->inflight_msg_res.map))
+				continue;
+
+			ctx = &hwc->caller_ctx[i];
+
+			/* Wake senders blocked on wait_for_completion.
+			 * Set error under lock to avoid racing with
+			 * handle_resp() which writes error under the
+			 * same lock.  The sender NULLs output_buf
+			 * after waking — doing it here would race
+			 * with a sender that hasn't set output_buf yet.
+			 */
+			spin_lock_irqsave(&ctx->lock, flags);
+			ctx->error = -ENODEV;
+			complete(&ctx->comp_event);
+			spin_unlock_irqrestore(&ctx->lock, flags);
+		}
+	}
+
+	/* Wait for all sender threads to finish and drop their refs
+	 * before touching the hardware or freeing anything.  Draining
+	 * here -- ahead of the teardown/FLR block below -- guarantees no
+	 * in-flight sender is still running when this function returns on
+	 * any path, including the FLR-failure early return; otherwise a
+	 * stranded sender would dereference gc->hwc_lock /
+	 * gc->hwc_drain_waitq after the caller frees gc.
+	 * After this, only slots held by timed-out senders whose
+	 * handle_resp() never ran remain in the bitmap.
+	 *
+	 * active_senders is only ever modified under hwc_lock, and the
+	 * last sender's wake_up() runs under that lock before it is
+	 * released.  Evaluating the condition under hwc_lock therefore
+	 * guarantees that once we observe 0 the waking sender has
+	 * already dropped the lock -- i.e. finished touching gc -- so it
+	 * cannot race the caller freeing gc after this returns.
+	 */
+	spin_lock_irq(&gc->hwc_lock);
+	wait_event_lock_irq(gc->hwc_drain_waitq,
+			    hwc->active_senders == 0, gc->hwc_lock);
+	spin_unlock_irq(&gc->hwc_lock);
+
+	/* Tear down the HWC if setup_hwc previously activated MST entries.
+	 * This is the definitive flag — unlike max_num_cqs which depends
+	 * on the init EQE arriving.
+	 *
+	 * The return value is intentionally not checked.  This is the
+	 * terminal cleanup path (device removal, suspend, or init
+	 * failure) — resources must be freed regardless.  If teardown
+	 * fails, hardware may still have active MST entries, but:
+	 *
+	 *  - Interrupts: mana_hwc_destroy_cq() below calls
+	 *    mana_gd_deregister_irq() which removes the HWC EQ from
+	 *    the interrupt dispatch list via list_del_rcu() +
+	 *    synchronize_rcu().  After that, no interrupt handler can
+	 *    invoke handle_resp() or access CQ/RQ buffers — even if
+	 *    the IRQ is shared with data path queues.
+	 *
+	 *  - DMA: mana_hwc_destroy_wq() frees DMA buffers via
+	 *    dma_free_coherent() which unmaps the IOVA from the
+	 *    IOMMU.  Any stale hardware DMA to the old address
+	 *    faults at the IOMMU, not in kernel memory.
+	 */
+	if (hwc->setup_active) {
+		int td_err = mana_smc_teardown_hwc(&gc->shm_channel, false);
+
+		if (td_err) {
+			dev_err(gc->dev, "HWC teardown failed: %d, issuing FLR\n",
+				td_err);
+
+			/* On systems without IOMMU, freeing DMA memory with
+			 * active hardware MST mappings risks memory corruption.
+			 * Issue FLR to force-reset the device and invalidate
+			 * all hardware state including MST entries.
+			 */
+			td_err = pcie_flr(to_pci_dev(gc->dev));
+			if (td_err) {
+				/* Device wedged: teardown and FLR
+				 * both failed, so hardware may still
+				 * DMA into our queue buffers.  Leak
+				 * all HWC DMA buffers -- and the EQ
+				 * IRQ registration, whose safe removal
+				 * needs those queues -- instead of
+				 * freeing memory the device can still
+				 * reach; without an IOMMU the freed
+				 * pages could be reused under live
+				 * DMA.
+				 */
+
+				dev_warn(gc->dev,
+					 "Leaked HWC DMA buffers (CQ/RQ/TXQ) to prevent memory corruption. Device is no longer usable.\n");
+
+				/* Do not call destroy_cq/wq below;
+				 * they would dma_free_coherent().
+				 * Leave hwc/cq/rxq/txq allocated.
+				 */
+				return;
+			}
+
+			dev_info(gc->dev, "FLR succeeded, hardware state cleared\n");
+		}
+
+		hwc->setup_active = false;
+	}
+
+	/* Tear down the HWC CQ object first — mana_hwc_destroy_cq()
+	 * both unpublishes the CQ from cq_table (+synchronize_rcu) and
+	 * deregisters the HWC EQ from the interrupt handler's RCU list
+	 * (via mana_gd_deregister_irq + synchronize_rcu), guaranteeing
+	 * no interrupt handler can access RQ/TXQ buffers after this
+	 * point.  The active_senders drain above ensures no sender is
+	 * accessing the CQ via txq->hwc_cq when it is destroyed.  Then
+	 * destroy TXQ and RQ safely.
+	 */
+	if (hwc->cq)
+		mana_hwc_destroy_cq(hwc->gdma_dev->gdma_context, hwc->cq);
+
 	if (hwc->txq)
 		mana_hwc_destroy_wq(hwc, hwc->txq);
 
 	if (hwc->rxq)
 		mana_hwc_destroy_wq(hwc, hwc->rxq);
 
-	if (hwc->cq)
-		mana_hwc_destroy_cq(hwc->gdma_dev->gdma_context, hwc->cq);
+	/* Release any slots still held — these belong to timed-out
+	 * senders where handle_resp() never ran (refcount = 1 with
+	 * handle_resp's ref still outstanding).
+	 */
+	if (hwc->caller_ctx) {
+		struct hwc_caller_ctx *ctx;
+		int i;
+
+		for (i = 0; i < hwc->num_inflight_msg; i++) {
+			if (!test_bit(i, hwc->inflight_msg_res.map))
+				continue;
+
+			ctx = &hwc->caller_ctx[i];
+			hwc_ctx_put(hwc, ctx);
+		}
+	}
 
 	kfree(hwc->caller_ctx);
 	hwc->caller_ctx = NULL;
@@ -845,11 +1381,11 @@ void mana_hwc_destroy_channel(struct gdma_context *gc)
 	hwc->hwc_timeout = 0;
 
 	kfree(hwc);
-	gc->hwc.driver_data = NULL;
 	gc->hwc.gdma_context = NULL;
 
-	vfree(gc->cq_table);
-	gc->cq_table = NULL;
+	old_cq_table = rcu_replace_pointer(gc->cq_table, NULL, true);
+	synchronize_rcu();
+	vfree(old_cq_table);
 }
 
 int mana_hwc_send_request(struct hw_channel_context *hwc, u32 req_len,
@@ -860,13 +1396,17 @@ int mana_hwc_send_request(struct hw_channel_context *hwc, u32 req_len,
 	struct hwc_wq *txq = hwc->txq;
 	struct gdma_req_hdr *req_msg;
 	struct hwc_caller_ctx *ctx;
+	unsigned long flags;
 	u32 dest_vrcq = 0;
 	u32 dest_vrq = 0;
 	u32 command;
+	u32 status;
 	u16 msg_id;
 	int err;
 
-	mana_hwc_get_msg_index(hwc, &msg_id);
+	err = mana_hwc_get_msg_index(hwc, &msg_id);
+	if (err)
+		return err;
 
 	tx_wr = &txq->msg_buf->reqs[msg_id];
 
@@ -878,8 +1418,11 @@ int mana_hwc_send_request(struct hw_channel_context *hwc, u32 req_len,
 	}
 
 	ctx = hwc->caller_ctx + msg_id;
+
+	spin_lock_irqsave(&ctx->lock, flags);
 	ctx->output_buf = resp;
 	ctx->output_buflen = resp_len;
+	spin_unlock_irqrestore(&ctx->lock, flags);
 
 	req_msg = (struct gdma_req_hdr *)tx_wr->buf_va;
 	if (req)
@@ -895,8 +1438,14 @@ int mana_hwc_send_request(struct hw_channel_context *hwc, u32 req_len,
 		dest_vrcq = hwc->pf_dest_vrcq_id;
 	}
 
+	/* Take handle_resp's ref before posting — hardware can respond
+	 * immediately after the doorbell ring.
+	 */
+	refcount_inc(&ctx->refcnt);
+
 	err = mana_hwc_post_tx_wqe(txq, tx_wr, dest_vrq, dest_vrcq, false);
 	if (err) {
+		refcount_dec(&ctx->refcnt);
 		dev_err(hwc->dev, "HWC: Failed to post send WQE: %d\n", err);
 		goto out;
 	}
@@ -907,31 +1456,86 @@ int mana_hwc_send_request(struct hw_channel_context *hwc, u32 req_len,
 			dev_err(hwc->dev, "Command 0x%x timed out: %u ms\n",
 				command, hwc->hwc_timeout);
 
-		/* Reduce further waiting if HWC no response */
+		/* NULL out output_buf so a late handle_resp() won't write
+		 * into the caller's buffer after the sender returns, then
+		 * check whether handle_resp() already delivered a valid
+		 * response between the timeout firing and this lock
+		 * acquisition — ctx->error != -EINPROGRESS means it ran.
+		 */
+		spin_lock_irqsave(&ctx->lock, flags);
+		ctx->output_buf = NULL;
+		err = ctx->error;
+		status = ctx->status_code;
+		spin_unlock_irqrestore(&ctx->lock, flags);
+
+		if (err != -EINPROGRESS) {
+			/* handle_resp() delivered a valid response just after
+			 * the timeout fired.  The hardware is alive, so use
+			 * the response and leave the channel usable; do not
+			 * latch hwc_timed_out or degrade hwc_timeout for what
+			 * turned out to be a transient race.
+			 */
+			hwc_ctx_put(hwc, ctx);
+			goto check_status;
+		}
+
+		/* Genuine timeout: no response arrived.  Reduce further
+		 * waiting, and mark the channel timed out under the bitmap
+		 * lock so get_msg_index() cannot acquire new slots after this.
+		 */
 		if (hwc->hwc_timeout > 1)
 			hwc->hwc_timeout = 1;
 
+		spin_lock_irqsave(&hwc->inflight_msg_res.lock, flags);
+		hwc->hwc_timed_out = true;
+		spin_unlock_irqrestore(&hwc->inflight_msg_res.lock, flags);
+		wake_up_all(&hwc->msg_waitq);
+
 		err = -ETIMEDOUT;
-		goto out;
+		hwc_ctx_put(hwc, ctx);
+		goto done;
 	}
 
-	if (ctx->error) {
-		err = ctx->error;
-		goto out;
-	}
+	/* NULL output_buf so a late handle_resp() won't memcpy into
+	 * the caller's buffer after the sender exits.  Read error and
+	 * status_code under the same lock — after hwc_ctx_put the slot
+	 * may be reused and these fields overwritten.
+	 */
+	spin_lock_irqsave(&ctx->lock, flags);
+	ctx->output_buf = NULL;
+	err = ctx->error;
+	status = ctx->status_code;
+	spin_unlock_irqrestore(&ctx->lock, flags);
+	hwc_ctx_put(hwc, ctx);
+
+check_status:
+	if (err)
+		goto done;
 
-	if (ctx->status_code && ctx->status_code != GDMA_STATUS_MORE_ENTRIES) {
-		if (ctx->status_code == GDMA_STATUS_CMD_UNSUPPORTED) {
+	if (status && status != GDMA_STATUS_MORE_ENTRIES) {
+		if (status == GDMA_STATUS_CMD_UNSUPPORTED) {
 			err = -EOPNOTSUPP;
-			goto out;
+			goto done;
 		}
+
 		if (command != MANA_QUERY_PHY_STAT)
 			dev_err(hwc->dev, "Command 0x%x failed with status: 0x%x\n",
-				command, ctx->status_code);
+				command, status);
 		err = -EPROTO;
-		goto out;
+		goto done;
 	}
+
+	err = 0;
+	goto done;
 out:
-	mana_hwc_put_msg_index(hwc, msg_id);
+	/* Pre-post error paths: no WQE was submitted so handle_resp()
+	 * cannot race here.  refcount is 1 (no second ref taken).
+	 */
+	ctx = hwc->caller_ctx + msg_id;
+	spin_lock_irqsave(&ctx->lock, flags);
+	ctx->output_buf = NULL;
+	spin_unlock_irqrestore(&ctx->lock, flags);
+	hwc_ctx_put(hwc, ctx);
+done:
 	return err;
 }
diff --git a/drivers/net/ethernet/microsoft/mana/mana_en.c b/drivers/net/ethernet/microsoft/mana/mana_en.c
index a8c329bdbacf83..5508746590e7cd 100644
--- a/drivers/net/ethernet/microsoft/mana/mana_en.c
+++ b/drivers/net/ethernet/microsoft/mana/mana_en.c
@@ -2648,6 +2648,7 @@ static int mana_create_txq(struct mana_port_context *apc,
 	struct mana_obj_spec cq_spec;
 	struct gdma_queue_spec spec;
 	struct gdma_context *gc;
+	struct gdma_queue __rcu **cq_table;
 	struct mana_txq *txq;
 	struct mana_cq *cq;
 	u32 txq_size;
@@ -2753,12 +2754,18 @@ static int mana_create_txq(struct mana_port_context *apc,
 
 		cq->gdma_id = cq->gdma_cq->id;
 
-		if (WARN_ON(cq->gdma_id >= gc->max_num_cqs)) {
+		/* No rcu_read_lock(): mana_create_txq runs under RTNL during
+		 * netdev bring-up, inside the netdev lifetime that
+		 * mana_remove() drains before the base cq_table can be freed.
+		 * See gdma_context::cq_table in gdma.h for why "true" is sound.
+		 */
+		cq_table = rcu_dereference_protected(gc->cq_table, true);
+		if (WARN_ON(!cq_table || cq->gdma_id >= gc->max_num_cqs)) {
 			err = -EINVAL;
 			goto out;
 		}
 
-		gc->cq_table[cq->gdma_id] = cq->gdma_cq;
+		rcu_assign_pointer(cq_table[cq->gdma_id], cq->gdma_cq);
 
 		mana_create_txq_debugfs(apc, i);
 
@@ -2986,6 +2993,7 @@ static struct mana_rxq *mana_create_rxq(struct mana_port_context *apc,
 	struct gdma_queue_spec spec;
 	struct mana_cq *cq = NULL;
 	struct gdma_context *gc;
+	struct gdma_queue __rcu **cq_table;
 	u32 cq_size, rq_size;
 	struct mana_rxq *rxq;
 	int err;
@@ -3075,12 +3083,18 @@ static struct mana_rxq *mana_create_rxq(struct mana_port_context *apc,
 	if (err)
 		goto out;
 
-	if (WARN_ON(cq->gdma_id >= gc->max_num_cqs)) {
+	/* No rcu_read_lock(): mana_create_rxq runs under RTNL during netdev
+	 * bring-up, inside the netdev lifetime that mana_remove() drains
+	 * before the base cq_table can be freed.  See gdma_context::cq_table
+	 * in gdma.h for why "true" is sound.
+	 */
+	cq_table = rcu_dereference_protected(gc->cq_table, true);
+	if (WARN_ON(!cq_table || cq->gdma_id >= gc->max_num_cqs)) {
 		err = -EINVAL;
 		goto out;
 	}
 
-	gc->cq_table[cq->gdma_id] = cq->gdma_cq;
+	rcu_assign_pointer(cq_table[cq->gdma_id], cq->gdma_cq);
 
 	netif_napi_add_weight_locked(ndev, &cq->napi, mana_poll, 1);
 
diff --git a/include/net/mana/gdma.h b/include/net/mana/gdma.h
index 8529cef0d7c452..a4eac6f7c36609 100644
--- a/include/net/mana/gdma.h
+++ b/include/net/mana/gdma.h
@@ -428,9 +428,34 @@ struct gdma_context {
 	/* L2 MTU */
 	u16 adapter_mtu;
 
-	/* This maps a CQ index to the queue structure. */
+	/* Size of cq_table, i.e. the largest valid CQ index + 1.  Set once
+	 * when cq_table is allocated and treated as immutable for the
+	 * table's lifetime (a bound only) -- it is never reset on teardown.
+	 * cq_table == NULL is the sole "table torn down" signal, so every
+	 * cq_table[id] access must guard with both !cq_table (gone) and
+	 * id >= max_num_cqs (out of bounds).
+	 */
 	unsigned int		max_num_cqs;
-	struct gdma_queue	**cq_table;
+	/* Both the base pointer and each entry are RCU-managed.  The fast
+	 * path (mana_gd_process_eqe) reads the base via rcu_dereference()
+	 * under rcu_read_lock(), so the table is freed with
+	 * rcu_assign_pointer(NULL) + synchronize_rcu() and an in-flight
+	 * reader can never observe freed memory.
+	 *
+	 * The slow paths -- mana_gd_destroy_cq() and the CQ install/remove
+	 * callers (mana_create_txq/_rxq, mana_ib_install/remove_cq_cb) --
+	 * instead read the base with rcu_dereference_protected(cq_table,
+	 * true).  The bare "true" is justified by teardown ordering, not by
+	 * a lock: the base table is replaced+freed only by
+	 * mana_hwc_destroy_channel() (and the create-time reinit), and every
+	 * teardown path first runs mana_remove() + mana_rdma_remove(), which
+	 * synchronously drain the netdev and the IB device
+	 * (unregister_netdevice / ib_unregister_device) that bound all
+	 * install/remove callers; the reinit case runs before either
+	 * consumer is probed.  So no slow-path caller can run while the base
+	 * table is being freed.
+	 */
+	struct gdma_queue	__rcu * __rcu *cq_table;
 
 	/* Protect eq_test_event and test_event_eq_id  */
 	struct mutex		eq_test_event_mutex;
@@ -456,6 +481,21 @@ struct gdma_context {
 	/* Hardware communication channel (HWC) */
 	struct gdma_dev		hwc;
 
+	/* destroy_channel() waits here for all HWC senders to exit.
+	 * Lives on gc (not hwc) so wake_up() after the last sender's
+	 * atomic_dec doesn't dereference freed hwc memory.
+	 */
+	wait_queue_head_t	hwc_drain_waitq;
+
+	/* Serializes hwc.driver_data (the hw_channel_context pointer)
+	 * between the control-plane readers in mana_gd_send_request(),
+	 * mana_need_log() and mana_serv_reset() and the publish/clear in
+	 * mana_hwc_create_channel()/mana_hwc_destroy_channel().  All users
+	 * are control-plane (HWC commands sleep; reset runs on a workqueue),
+	 * so a plain spinlock -- not RCU -- is sufficient.
+	 */
+	spinlock_t		hwc_lock;
+
 	/* Azure network adapter */
 	struct gdma_dev		mana;
 
@@ -668,6 +708,9 @@ enum {
 /* Driver supports dynamic interrupt moderation - DIM */
 #define GDMA_DRV_CAP_FLAG_1_DYN_INTERRUPT_MODERATION BIT(28)
 
+/* Driver supports dynamic queue depth for HWC */
+#define GDMA_DRV_CAP_FLAG_1_DYN_HWC_QUEUE_DEPTH BIT(29)
+
 #define GDMA_DRV_CAP_FLAGS1 \
 	(GDMA_DRV_CAP_FLAG_1_EQ_SHARING_MULTI_VPORT | \
 	 GDMA_DRV_CAP_FLAG_1_NAPI_WKDONE_FIX | \
@@ -683,6 +726,7 @@ enum {
 	 GDMA_DRV_CAP_FLAG_1_PROBE_RECOVERY | \
 	 GDMA_DRV_CAP_FLAG_1_HANDLE_STALL_SQ_RECOVERY | \
 	 GDMA_DRV_CAP_FLAG_1_HWC_TIMEOUT_RECOVERY | \
+	 GDMA_DRV_CAP_FLAG_1_DYN_HWC_QUEUE_DEPTH | \
 	 GDMA_DRV_CAP_FLAG_1_EQ_MSI_UNSHARE_MULTI_VPORT | \
 	 GDMA_DRV_CAP_FLAG_1_DYN_INTERRUPT_MODERATION)
 
diff --git a/include/net/mana/hw_channel.h b/include/net/mana/hw_channel.h
index 16feb39616c1be..eec5d3aa23e1d5 100644
--- a/include/net/mana/hw_channel.h
+++ b/include/net/mana/hw_channel.h
@@ -164,6 +164,9 @@ struct hwc_wq {
 	u16 queue_depth;
 
 	struct hwc_cq *hwc_cq;
+
+	/* Serializes concurrent mana_gd_post_and_ring() calls. */
+	spinlock_t lock;
 };
 
 struct hwc_caller_ctx {
@@ -173,6 +176,17 @@ struct hwc_caller_ctx {
 
 	u32 error; /* Linux error code */
 	u32 status_code;
+
+	/* Protects output_buf against concurrent access from
+	 * handle_resp() (CQ interrupt) and the sender timeout path.
+	 */
+	spinlock_t lock;
+
+	/* Tracks sender + handle_resp ownership.  The last put
+	 * (refcount reaches 0) releases the bitmap slot.
+	 */
+	refcount_t refcnt;
+	u16 msg_id;
 };
 
 struct hw_channel_context {
@@ -181,8 +195,9 @@ struct hw_channel_context {
 
 	u16 num_inflight_msg;
 	u32 max_req_msg_size;
+	u32 max_resp_msg_size;
 
-	u16 hwc_init_q_depth_max;
+	u32 hwc_init_q_depth_max;
 	u32 hwc_init_max_req_msg_size;
 	u32 hwc_init_max_resp_msg_size;
 
@@ -192,13 +207,38 @@ struct hw_channel_context {
 	struct hwc_wq *txq;
 	struct hwc_cq *cq;
 
-	struct semaphore sema;
 	struct gdma_resource inflight_msg_res;
+	/* Waitqueue for senders blocked on a full inflight bitmap. */
+	wait_queue_head_t msg_waitq;
 
 	u32 pf_dest_vrq_id;
 	u32 pf_dest_vrcq_id;
 	u32 hwc_timeout;
 
+	/* Set after channel is fully established; cleared on teardown to
+	 * abort waiters in mana_hwc_get_msg_index() and reject new sends.
+	 */
+	bool channel_up;
+
+	/* Set on first HWC timeout.  Causes get_msg_index() to return
+	 * -ETIMEDOUT instead of waiting, draining all queued senders.
+	 */
+	bool hwc_timed_out;
+
+	/* Set after mana_smc_setup_hwc() succeeds (hardware has active
+	 * MST entries).  On recoverable paths (establish_channel)
+	 * cleared only after successful teardown so a retry remains
+	 * possible.  On the terminal destroy_channel path, cleared
+	 * unconditionally since hwc is about to be freed.
+	 */
+	bool setup_active;
+
+	/* Count of in-flight mana_gd_send_request() callers.  Protected
+	 * by gc->hwc_lock; the last sender to drop it to zero wakes
+	 * gc->hwc_drain_waitq for the mana_hwc_destroy_channel() drain.
+	 */
+	unsigned int active_senders;
+
 	struct hwc_caller_ctx *caller_ctx;
 };

3/2

2026/07/22 17:52

llm

patch-evaluator

Model:

gemini-3.1-pro-preview

Args:

null

Results:

null

Tokens:

							input: 23162
							output: 19
							thoughts: 213

Thoughts:

**Analyzing the MANA Driver Patch Series**

Okay, so I've got this patch series in front of me that's making changes to the Microsoft Azure Network Adapter, the MANA driver. It looks like the core of the driver is being touched; I'm seeing modifications in `cq.c`, `gdma_main.c`, `hw_channel.c`, and `mana_en.c`, along with some header file updates in `gdma.h` and `hw_channel.h`. Given my experience, that list of files immediately tells me this is likely a substantial functional change.

Let's quickly assess the nature of these changes to get a better handle on the scope. Ah, yes, I'm seeing RCU protection added for `cq_table`, changes to spinlocks, timeout handling improvements, refcount management for `hwc_caller_ctx`, and handling of HWC reinitialization. That's a pretty significant laundry list. This isn't just a minor tweak; this is a fundamental rework of some key synchronization and resource management components within the MANA driver. These types of changes almost always introduce the potential for subtle, hard-to-debug regressions, so thorough testing is absolutely critical.

Next, I need to understand the configuration options associated with this driver. I'm going to search through the Kconfig files, specifically looking for something like `MICROSOFT_MANA` or a similar configuration symbol to figure out how this driver is enabled and if any new options have been introduced with this patch. Understanding the configuration is vital to properly integrate this patch series, or to even just understand the broader context.

4/2

2026/07/22 17:52

tool

grepper

Args:

{
  "Expression": "MICROSOFT_MANA"
}

Results:

{
  "Output": "drivers/infiniband/hw/mana/Kconfig=2=config MANA_INFINIBAND\ndrivers/infiniband/hw/mana/Kconfig-3-\ttristate \"Microsoft Azure Network Adapter support\"\ndrivers/infiniband/hw/mana/Kconfig:4:\tdepends on NETDEVICES \u0026\u0026 ETHERNET \u0026\u0026 PCI \u0026\u0026 MICROSOFT_MANA\ndrivers/infiniband/hw/mana/Kconfig-5-\thelp\n--\ndrivers/net/ethernet/microsoft/Kconfig=16=if NET_VENDOR_MICROSOFT\ndrivers/net/ethernet/microsoft/Kconfig-17-\ndrivers/net/ethernet/microsoft/Kconfig:18:config MICROSOFT_MANA\ndrivers/net/ethernet/microsoft/Kconfig-19-\ttristate \"Microsoft Azure Network Adapter (MANA) support\"\n--\ndrivers/net/ethernet/microsoft/Makefile-4-\ndrivers/net/ethernet/microsoft/Makefile:5:obj-$(CONFIG_MICROSOFT_MANA) += mana/\n--\ndrivers/net/ethernet/microsoft/mana/Makefile-4-\ndrivers/net/ethernet/microsoft/mana/Makefile:5:obj-$(CONFIG_MICROSOFT_MANA) += mana.o\ndrivers/net/ethernet/microsoft/mana/Makefile-6-mana-objs := gdma_main.o shm_channel.o hw_channel.o mana_en.o mana_ethtool.o mana_bpf.o\n"
}

5/2

2026/07/22 17:52

llm

patch-evaluator

Model:

gemini-3.1-pro-preview

Args:

null

Results:

null

Tokens:

							input: 24098
							output: 17
							thoughts: 1097

Thoughts:

**Fuzzing Strategy for the MANA Driver**

Okay, so I'm looking at this patch and immediately, my mind jumps to fuzzing. The key here is the `MICROSOFT_MANA` and `MANA_INFINIBAND` configs, they will guide us on whether this is built as a module or built in. I need to get a handle on what the target functions are within the code, so let's list those. The patch touches a good chunk of the driver, specifically: `mana_ib_install_cq_cb`, `mana_ib_remove_cq_cb`, `mana_need_log`, `mana_gd_send_request`, `mana_serv_reset`, `mana_gd_process_eqe`, `mana_gd_destroy_cq`, `mana_gd_create_dma_region`, `mana_gd_verify_vf_version`, `mana_gd_probe`, `mana_hwc_get_msg_index`, `mana_hwc_put_msg_index`, `hwc_ctx_put`, `mana_hwc_handle_resp`, `mana_hwc_init_event_handler`, `mana_hwc_rx_event_handler`, `mana_hwc_destroy_cq`, `mana_hwc_create_wq`, `mana_hwc_post_tx_wqe`, `mana_hwc_init_inflight_msg`, `mana_hwc_test_channel`, `mana_hwc_establish_channel`, `mana_hwc_init_queues`, `mana_hwc_destroy_queues`, `mana_hwc_create_channel`, `mana_hwc_destroy_channel`, `mana_hwc_send_request`, `mana_create_txq`, and `mana_create_rxq`. That's a lot, so I'll need to prioritize.

Let's pick a few promising candidates for fuzzing, and I should focus on entry points from userspace or those that handle complex interaction.  I'll pick: `mana_hwc_send_request`, `mana_hwc_handle_resp`, `mana_hwc_rx_event_handler`, `mana_hwc_create_channel`, `mana_hwc_destroy_channel`, `mana_hwc_establish_channel`, `mana_hwc_test_channel`, `mana_hwc_get_msg_index`, `mana_hwc_put_msg_index`, `mana_gd_send_request`, `mana_gd_process_eqe`, `mana_ib_install_cq_cb`, and `mana_ib_remove_cq_cb`.

Now, the big question: Is this driver even reachable in a virtualized environment? MANA is a Microsoft Azure Network Adapter, so it's a PCI device.  Typically, I'd look for syzkaller support via vhci or a similar pseudo-device if the hardware is emulated. Syzkaller doesn't have a specific MANA emulator, so I was concerned it wouldn't be possible at first.

But wait, PCI drivers can be fuzzed using `syz_pci_vpci` and similar. The text says we may be able to reach even software-emulated hardware. I'm seeing "syz-cluster" as the author, which strongly suggests that syzkaller is actively working on or has found something to fuzz in this code! Since it is a patch being tested by syzkaller it is worth it. Syzkaller can fuzz any PCI driver by binding it to the vhci or pci pseudo-device and providing random MMIO/DMA responses.

Okay, let's confirm the configs again, because it's always good to be thorough. Yup, just `MICROSOFT_MANA` and `MANA_INFINIBAND`.

Let's refine the FocusSymbols to the important ones: `mana_hwc_send_request`, `mana_hwc_handle_resp`, `mana_hwc_rx_event_handler`, `mana_hwc_create_channel`, `mana_hwc_destroy_channel`, `mana_hwc_establish_channel`, `mana_hwc_destroy_queues`, `mana_gd_process_eqe`, `mana_ib_install_cq_cb`, `mana_ib_remove_cq_cb`, and `mana_hwc_get_msg_index`.

Good. Now, I should make sure there aren't any other configs that would influence the build or functionality I'm missing.

6/2

2026/07/22 17:52

tool

grepper

Args:

{
  "Expression": "config MANA"
}

Results:

{
  "Output": "drivers/infiniband/hw/mana/Kconfig-1-# SPDX-License-Identifier: GPL-2.0-only\ndrivers/infiniband/hw/mana/Kconfig:2:config MANA_INFINIBAND\ndrivers/infiniband/hw/mana/Kconfig-3-\ttristate \"Microsoft Azure Network Adapter support\"\n--\ndrivers/power/supply/Kconfig=293=config CHARGER_SBS\n--\ndrivers/power/supply/Kconfig-299-\ndrivers/power/supply/Kconfig:300:config MANAGER_SBS\ndrivers/power/supply/Kconfig-301-\ttristate \"Smart Battery System Manager\"\n"
}

7/2

2026/07/22 17:52

llm

patch-evaluator

Model:

gemini-3.1-pro-preview

Args:

null

Results:

null

Tokens:

							input: 26385
							output: 217
							thoughts: 150

Thoughts:

Okay, here's my take on those thoughts, framed from my expert perspective:

**Deep Dive into MANA Infiniband Functionality**

Alright, let's see what we're working with here. The configuration context is clearly focused on the `MICROSOFT_MANA` and `MANA_INFINIBAND` setups. I need to keep those in mind as the foundation for everything that's to come.

I'm zeroing in on a critical subset of functions, or what they're calling `FocusSymbols`. These seem to be the core building blocks for handling Completion Queues (CQs), hardware communication, and general data processing within this Infiniband implementation. Let's break them down.

* `mana_ib_install_cq_cb` and `mana_ib_remove_cq_cb`: These clearly relate to the installation and removal of Completion Queue callbacks. This suggests a sophisticated event-driven model for handling completions, which is standard but important to verify the nuances of.
* `mana_hwc_get_msg_index`, `mana_hwc_handle_resp`, `mana_hwc_rx_event_handler`, `mana_hwc_create_channel`, `mana_hwc_destroy_channel`, and `mana_hwc_send_request`: These collectively paint a picture of Hardware Communication (HWC) management. We're looking at message indexing, response handling, event processing (particularly on the receive side), and the creation, destruction, and transmission aspects of communication channels. A good level of abstraction, critical to performance.
* `mana_gd_process_eqe` and `mana_gd_destroy_cq`: These are likely related to generic data (GD) processing and destroying CQs. This implies the existence of a generic data model for data transfer, separate from the Infiniband-specific aspects.

So, overall, a thorough set of considerations that indicates a well-structured approach to handling Infiniband operations with this `MANA` implementation.

My next step is to call set-results. This indicates the next steps in this evaluation, likely gathering all available data points to establish a benchmark for performance.

8/2

2026/07/22 17:53

tool

set-results

Args:

{
  "EnableConfigs": [
    "MICROSOFT_MANA",
    "MANA_INFINIBAND"
  ],
  "FocusSymbols": [
    "mana_ib_install_cq_cb",
    "mana_ib_remove_cq_cb",
    "mana_hwc_get_msg_index",
    "mana_hwc_handle_resp",
    "mana_hwc_rx_event_handler",
    "mana_hwc_create_channel",
    "mana_hwc_destroy_channel",
    "mana_hwc_send_request",
    "mana_gd_process_eqe",
    "mana_gd_destroy_cq"
  ],
  "Reasoning": "The patch makes functional changes to the Microsoft Azure Network Adapter (MANA) driver, including adding RCU protection for the CQ table, handling HWC timeouts, and adding refcounts for caller contexts. These changes affect the core logic of the driver and should be fuzzed.",
  "WorthFuzzing": true
}

Results:

{
  "EnableConfigs": [
    "MICROSOFT_MANA",
    "MANA_INFINIBAND"
  ],
  "FocusSymbols": [
    "mana_ib_install_cq_cb",
    "mana_ib_remove_cq_cb",
    "mana_hwc_get_msg_index",
    "mana_hwc_handle_resp",
    "mana_hwc_rx_event_handler",
    "mana_hwc_create_channel",
    "mana_hwc_destroy_channel",
    "mana_hwc_send_request",
    "mana_gd_process_eqe",
    "mana_gd_destroy_cq"
  ],
  "Reasoning": "The patch makes functional changes to the Microsoft Azure Network Adapter (MANA) driver, including adding RCU protection for the CQ table, handling HWC timeouts, and adding refcounts for caller contexts. These changes affect the core logic of the driver and should be fuzzed.",
  "WorthFuzzing": true
}

LLM Calls Summary:
Total Calls	Total Tokens	Avg Tokens	Total Duration (Seconds)	Avg Duration (Seconds)

Tool Calls Summary:
Total Calls	Total Duration (Seconds)	Avg Duration (Seconds)