AI Trajectory

Trajectory: (Expand/Collapse All)
Seq	Timestamp	Type	Name	Duration
0/0	2026/09/27 14:24	flow	patch-triage	0m Args: null Results: { "EnableConfigs": null, "FocusSymbols": [], "KMSANReasoning": "", "NeedsKMSAN": false, "Reasoning": "The patch series modifies the IBM Virtual Ethernet driver (ibmveth), PowerPC hypercall definitions in arch/powerpc/include/asm/hvcall.h, and a perf Python script under tools/. The ibmveth driver depends on CONFIG_PPC_PSERIES (which requires PPC64), making all modified kernel code exclusive to the PowerPC architecture. As the target architecture is amd64, the changes are structurally unreachable.", "WorthFuzzing": false }
1/1	2026/09/27 14:24	action	read-patch-diff	0m Args: null Results: { "PatchDiff": "commit 845350ec79fd7e1e12ceb4f4a49bb29fa44d8e9a\nAuthor: syz-cluster \u003ctriage@syzkaller.com\u003e\nDate: Sun Sep 27 14:24:02 2026 +0000\n\n syz-cluster: applied patch under review\n\ndiff --git a/arch/powerpc/include/asm/hvcall.h b/arch/powerpc/include/asm/hvcall.h\nindex dff90a7d7f70b..cb0ea53491e6c 100644\n--- a/arch/powerpc/include/asm/hvcall.h\n+++ b/arch/powerpc/include/asm/hvcall.h\n@@ -362,7 +362,11 @@\n #define H_GUEST_DELETE\t\t0x488\n #define H_PKS_WRAP_OBJECT\t0x490\n #define H_PKS_UNWRAP_OBJECT\t0x494\n-#define MAX_HCALL_OPCODE\tH_PKS_UNWRAP_OBJECT\n+/* 0x498 reserved; 0x4A4 = H_FREE_LOGICAL_LAN_BUFFER_QUEUE (unused here) /\n+#define H_REG_LOGICAL_LAN_QUEUE 0x49C\n+#define H_ADD_LOGICAL_LAN_BUFFERS_QUEUE 0x4A0\n+#define H_FREE_LOGICAL_LAN_QUEUE 0x4A8\n+#define MAX_HCALL_OPCODE\tH_FREE_LOGICAL_LAN_QUEUE\n \n / Scope args for H_SCM_UNBIND_ALL /\n #define H_UNBIND_SCOPE_ALL (0x1)\ndiff --git a/drivers/net/ethernet/ibm/ibmveth.c b/drivers/net/ethernet/ibm/ibmveth.c\nindex 73e051d26b9d8..da14c69152110 100644\n--- a/drivers/net/ethernet/ibm/ibmveth.c\n+++ b/drivers/net/ethernet/ibm/ibmveth.c\n@@ -21,6 +21,8 @@\n #include \u003clinux/skbuff.h\u003e\n #include \u003clinux/init.h\u003e\n #include \u003clinux/interrupt.h\u003e\n+#include \u003clinux/irq.h\u003e\n+#include \u003clinux/irqdomain.h\u003e\n #include \u003clinux/mm.h\u003e\n #include \u003clinux/pm.h\u003e\n #include \u003clinux/ethtool.h\u003e\n@@ -28,6 +30,8 @@\n #include \u003clinux/ip.h\u003e\n #include \u003clinux/ipv6.h\u003e\n #include \u003clinux/slab.h\u003e\n+#include \u003clinux/spinlock.h\u003e\n+#include \u003clinux/debugfs.h\u003e\n #include \u003casm/hvcall.h\u003e\n #include \u003clinux/atomic.h\u003e\n #include \u003casm/vio.h\u003e\n@@ -35,6 +39,7 @@\n #include \u003casm/firmware.h\u003e\n #include \u003cnet/tcp.h\u003e\n #include \u003cnet/ip6_checksum.h\u003e\n+#include \u003cnet/netdev_queues.h\u003e\n \n #include \"ibmveth.h\"\n \n@@ -72,748 +77,2940 @@ module_param(old_large_send, bool, 0444);\n MODULE_PARM_DESC(old_large_send,\n \t\"Use old large send method on firmware that supports the new method\");\n \n+/\n+ enum ibmveth_stat_src - where an ethtool -S counter is stored\n+ * @IBMVETH_STAT_ADAPTER: plain u64 in struct ibmveth_adapter\n+ * @IBMVETH_STAT_RX_QSUM: per-queue u64, summed over rx_qstats[]\n+ * @IBMVETH_STAT_TX_QSUM: per-queue u64, summed over tx_qstats[]\n+ * @IBMVETH_STAT_RX_NO_BUFFER: rx_qstats[] live-page absolute plus the\n+ \tabsolutes carried over from pages the queue has already retired\n+ \n+ * Counters live per-queue so multi-queue writers never share a field.\n+ * The adapter is only ever read from ethtool, so summing there is free.\n+ /\n+enum ibmveth_stat_src {\n+\tIBMVETH_STAT_ADAPTER,\n+\tIBMVETH_STAT_RX_QSUM,\n+\tIBMVETH_STAT_TX_QSUM,\n+\tIBMVETH_STAT_RX_NO_BUFFER,\n+};\n+\n struct ibmveth_stat {\n \tchar name[ETH_GSTRING_LEN];\n-\tint offset;\n+\tenum ibmveth_stat_src src;\n+\t/ Offset into the struct named by @src. /\n+\tsize_t off;\n };\n \n #define IBMVETH_STAT_OFF(stat) offsetof(struct ibmveth_adapter, stat)\n+#define IBMVETH_RXQ_OFF(stat) offsetof(struct ibmveth_rx_queue_stats, stat)\n+#define IBMVETH_TXQ_OFF(stat) offsetof(struct ibmveth_tx_queue_stats, stat)\n #define IBMVETH_GET_STAT(a, off) ((u64 )(((unsigned long)(a)) + off))\n \n+#define IBMVETH_ADAPTER_STAT(key, field) \\\n+\t{ key, IBMVETH_STAT_ADAPTER, IBMVETH_STAT_OFF(field) }\n+#define IBMVETH_RXQ_STAT(key, field) \\\n+\t{ key, IBMVETH_STAT_RX_QSUM, IBMVETH_RXQ_OFF(field) }\n+#define IBMVETH_TXQ_STAT(key, field) \\\n+\t{ key, IBMVETH_STAT_TX_QSUM, IBMVETH_TXQ_OFF(field) }\n+\n+/\n+ * Key names and their order are ABI. Do not reorder or rename; append\n+ * only, and only when the counter is worth a permanent interface.\n+ /\n static struct ibmveth_stat ibmveth_stats[] = {\n-\t{ \"replenish_task_cycles\", IBMVETH_STAT_OFF(replenish_task_cycles) },\n-\t{ \"replenish_no_mem\", IBMVETH_STAT_OFF(replenish_no_mem) },\n-\t{ \"replenish_add_buff_failure\",\n-\t\t\tIBMVETH_STAT_OFF(replenish_add_buff_failure) },\n-\t{ \"replenish_add_buff_success\",\n-\t\t\tIBMVETH_STAT_OFF(replenish_add_buff_success) },\n-\t{ \"rx_invalid_buffer\", IBMVETH_STAT_OFF(rx_invalid_buffer) },\n-\t{ \"rx_no_buffer\", IBMVETH_STAT_OFF(rx_no_buffer) },\n-\t{ \"tx_map_failed\", IBMVETH_STAT_OFF(tx_map_failed) },\n-\t{ \"tx_send_failed\", IBMVETH_STAT_OFF(tx_send_failed) },\n-\t{ \"fw_enabled_ipv4_csum\", IBMVETH_STAT_OFF(fw_ipv4_csum_support) },\n-\t{ \"fw_enabled_ipv6_csum\", IBMVETH_STAT_OFF(fw_ipv6_csum_support) },\n-\t{ \"tx_large_packets\", IBMVETH_STAT_OFF(tx_large_packets) },\n-\t{ \"rx_large_packets\", IBMVETH_STAT_OFF(rx_large_packets) },\n-\t{ \"fw_enabled_large_send\", IBMVETH_STAT_OFF(fw_large_send_support) }\n+\tIBMVETH_RXQ_STAT(\"replenish_task_cycles\", replenish_task_cycles),\n+\tIBMVETH_RXQ_STAT(\"replenish_no_mem\", replenish_no_mem),\n+\tIBMVETH_RXQ_STAT(\"replenish_add_buff_failure\",\n+\t\t\t replenish_add_buff_failure),\n+\tIBMVETH_RXQ_STAT(\"replenish_add_buff_success\",\n+\t\t\t replenish_add_buff_success),\n+\tIBMVETH_RXQ_STAT(\"rx_invalid_buffer\", invalid_buffers),\n+\t{ \"rx_no_buffer\", IBMVETH_STAT_RX_NO_BUFFER,\n+\t IBMVETH_RXQ_OFF(no_buffer_drops) },\n+\tIBMVETH_ADAPTER_STAT(\"tx_map_failed\", tx_map_failed),\n+\tIBMVETH_TXQ_STAT(\"tx_send_failed\", send_failures),\n+\tIBMVETH_ADAPTER_STAT(\"fw_enabled_ipv4_csum\", fw_ipv4_csum_support),\n+\tIBMVETH_ADAPTER_STAT(\"fw_enabled_ipv6_csum\", fw_ipv6_csum_support),\n+\tIBMVETH_TXQ_STAT(\"tx_large_packets\", large_packets),\n+\tIBMVETH_RXQ_STAT(\"rx_large_packets\", large_packets),\n+\tIBMVETH_ADAPTER_STAT(\"fw_enabled_large_send\", fw_large_send_support),\n+};\n+\n+/\n+ struct ibmveth_qstat - a per-queue counter exposed through ethtool -S\n+ * @fmt: key name, taking the queue index as its only argument\n+ * @off: offset into the matching per-queue stats struct\n+ \n+ Driving the strings and the values from one table keeps the two in\n+ * step; get_sset_count() derives its length from ARRAY_SIZE() so the\n+ * three cannot drift apart.\n+ /\n+struct ibmveth_qstat {\n+\tconst char fmt;\n+\tsize_t off;\n+};\n+\n+/\n+ Only counters with no home in the standard interfaces belong here.\n+ * packets, bytes and drops are reported through netdev_stat_ops.\n+ /\n+static const struct ibmveth_qstat ibmveth_rx_qstat_keys[] = {\n+\t{ \"rx%d_interrupts\", IBMVETH_RXQ_OFF(interrupts) },\n+\t{ \"rx%d_polls\", IBMVETH_RXQ_OFF(polls) },\n+\t{ \"rx%d_large_packets\", IBMVETH_RXQ_OFF(large_packets) },\n+\t{ \"rx%d_invalid_buffers\", IBMVETH_RXQ_OFF(invalid_buffers) },\n+\t{ \"rx%d_no_buffer_drops\", IBMVETH_RXQ_OFF(no_buffer_drops) },\n+};\n+\n+static const struct ibmveth_qstat ibmveth_tx_qstat_keys[] = {\n+\t{ \"tx%d_large_packets\", IBMVETH_TXQ_OFF(large_packets) },\n+\t{ \"tx%d_send_failures\", IBMVETH_TXQ_OFF(send_failures) },\n+\t{ \"tx%d_checksum_offload\", IBMVETH_TXQ_OFF(checksum_offload) },\n };\n \n+#define IBMVETH_NUM_RX_QSTATS ARRAY_SIZE(ibmveth_rx_qstat_keys)\n+#define IBMVETH_NUM_TX_QSTATS ARRAY_SIZE(ibmveth_tx_qstat_keys)\n+\n / simple methods of getting data from the current rxq entry /\n-static inline u32 ibmveth_rxq_flags(struct ibmveth_adapter adapter)\n+static u32 ibmveth_rxq_flags(struct ibmveth_adapter adapter,\n+\t\t\t int queue_index)\n {\n-\treturn be32_to_cpu(adapter-\u003erx_queue.queue_addr[adapter-\u003erx_queue.index].flags_off);\n+\tstruct ibmveth_rx_q rxq = \u0026adapter-\u003erx_queue[queue_index];\n+\n+\treturn be32_to_cpu(rxq-\u003equeue_addr[rxq-\u003eindex].flags_off);\n }\n \n-static inline int ibmveth_rxq_toggle(struct ibmveth_adapter adapter)\n+static int ibmveth_rxq_toggle(struct ibmveth_adapter adapter,\n+\t\t\t int queue_index)\n {\n-\treturn (ibmveth_rxq_flags(adapter) \u0026 IBMVETH_RXQ_TOGGLE) \u003e\u003e\n-\t\t\tIBMVETH_RXQ_TOGGLE_SHIFT;\n+\treturn (ibmveth_rxq_flags(adapter, queue_index) \u0026 IBMVETH_RXQ_TOGGLE) \u003e\u003e\n+\t\tIBMVETH_RXQ_TOGGLE_SHIFT;\n }\n \n-static inline int ibmveth_rxq_pending_buffer(struct ibmveth_adapter adapter)\n+static int ibmveth_rxq_pending_buffer(struct ibmveth_adapter adapter,\n+\t\t\t\t int queue_index)\n {\n-\treturn ibmveth_rxq_toggle(adapter) == adapter-\u003erx_queue.toggle;\n+\treturn ibmveth_rxq_toggle(adapter, queue_index) ==\n+\t\tadapter-\u003erx_queue[queue_index].toggle;\n }\n \n-static inline int ibmveth_rxq_buffer_valid(struct ibmveth_adapter adapter)\n+static int ibmveth_rxq_buffer_valid(struct ibmveth_adapter adapter,\n+\t\t\t\t int queue_index)\n {\n-\treturn ibmveth_rxq_flags(adapter) \u0026 IBMVETH_RXQ_VALID;\n+\treturn ibmveth_rxq_flags(adapter, queue_index) \u0026 IBMVETH_RXQ_VALID;\n }\n \n-static inline int ibmveth_rxq_frame_offset(struct ibmveth_adapter adapter)\n+static int ibmveth_rxq_frame_offset(struct ibmveth_adapter adapter,\n+\t\t\t\t int queue_index)\n {\n-\treturn ibmveth_rxq_flags(adapter) \u0026 IBMVETH_RXQ_OFF_MASK;\n+\treturn ibmveth_rxq_flags(adapter, queue_index) \u0026 IBMVETH_RXQ_OFF_MASK;\n }\n \n-static inline int ibmveth_rxq_large_packet(struct ibmveth_adapter adapter)\n+static int ibmveth_rxq_large_packet(struct ibmveth_adapter adapter,\n+\t\t\t\t int queue_index)\n {\n-\treturn ibmveth_rxq_flags(adapter) \u0026 IBMVETH_RXQ_LRG_PKT;\n+\treturn ibmveth_rxq_flags(adapter, queue_index) \u0026 IBMVETH_RXQ_LRG_PKT;\n }\n \n-static inline int ibmveth_rxq_frame_length(struct ibmveth_adapter adapter)\n+static int ibmveth_rxq_frame_length(struct ibmveth_adapter adapter,\n+\t\t\t\t int queue_index)\n {\n-\treturn be32_to_cpu(adapter-\u003erx_queue.queue_addr[adapter-\u003erx_queue.index].length);\n+\tstruct ibmveth_rx_q rxq = \u0026adapter-\u003erx_queue[queue_index];\n+\n+\treturn be32_to_cpu(rxq-\u003equeue_addr[rxq-\u003eindex].length);\n }\n \n-static inline int ibmveth_rxq_csum_good(struct ibmveth_adapter adapter)\n+static int ibmveth_rxq_csum_good(struct ibmveth_adapter adapter,\n+\t\t\t\t int queue_index)\n {\n-\treturn ibmveth_rxq_flags(adapter) \u0026 IBMVETH_RXQ_CSUM_GOOD;\n+\treturn ibmveth_rxq_flags(adapter, queue_index) \u0026 IBMVETH_RXQ_CSUM_GOOD;\n }\n \n-static unsigned int ibmveth_real_max_tx_queues(void)\n+/ Lockless IRQ/poll readers vs resize publishers. /\n+static unsigned int\n+ibmveth_get_num_rx_queues(const struct ibmveth_adapter adapter)\n {\n-\tunsigned int n_cpu = num_online_cpus();\n-\n-\treturn min(n_cpu, IBMVETH_MAX_QUEUES);\n+\t/\n+\t Pairs with the release in ibmveth_publish_num_rx_queues(): a reader\n+\t * that sees the new count also sees the per-queue state behind it.\n+\t /\n+\treturn smp_load_acquire(\u0026adapter-\u003enum_rx_queues);\n }\n \n-/ setup the initial settings for a buffer pool /\n-static void ibmveth_init_buffer_pool(struct ibmveth_buff_pool pool,\n-\t\t\t\t u32 pool_index, u32 pool_size,\n-\t\t\t\t u32 buff_size, u32 pool_active)\n+static void\n+ibmveth_publish_num_rx_queues(struct ibmveth_adapter adapter,\n+\t\t\t unsigned int num)\n {\n-\tpool-\u003esize = pool_size;\n-\tpool-\u003eindex = pool_index;\n-\tpool-\u003ebuff_size = buff_size;\n-\tpool-\u003ethreshold = pool_size 7 / 8;\n-\tpool-\u003eactive = pool_active;\n+\t/\n+\t Pairs with the acquire in ibmveth_get_num_rx_queues(): per-queue\n+\t * state must be visible to a reader before it observes the new count.\n+\t /\n+\tsmp_store_release(\u0026adapter-\u003enum_rx_queues, num);\n }\n \n-/ allocate and setup an buffer pool - called during open /\n-static int ibmveth_alloc_buffer_pool(struct ibmveth_buff_pool pool)\n+static unsigned int ibmveth_real_max_tx_queues(void)\n {\n-\tint i;\n+\tunsigned int n_cpu = num_online_cpus();\n \n-\tpool-\u003efree_map = kmalloc_array(pool-\u003esize, sizeof(u16), GFP_KERNEL);\n+\treturn min(n_cpu, IBMVETH_MAX_QUEUES);\n+}\n \n-\tif (!pool-\u003efree_map)\n-\t\treturn -1;\n+/*\n+ ibmveth_alloc_filter_list - Allocate and map filter list\n+ * @adapter: ibmveth adapter structure\n+ \n+ Return: 0 on success, negative error code on failure\n+ /\n+static int\n+ibmveth_alloc_filter_list(struct ibmveth_adapter adapter)\n+{\n+\tstruct device dev = \u0026adapter-\u003evdev-\u003edev;\n+\tstruct net_device netdev = adapter-\u003enetdev;\n \n-\tpool-\u003edma_addr = kzalloc_objs(dma_addr_t, pool-\u003esize);\n-\tif (!pool-\u003edma_addr) {\n-\t\tkfree(pool-\u003efree_map);\n-\t\tpool-\u003efree_map = NULL;\n-\t\treturn -1;\n+\tadapter-\u003efilter_list_addr = (void )get_zeroed_page(GFP_KERNEL);\n+\tif (!adapter-\u003efilter_list_addr) {\n+\t\tnetdev_err(netdev, \"unable to allocate filter pages\\n\");\n+\t\treturn -ENOMEM;\n \t}\n \n-\tpool-\u003eskbuff = kcalloc(pool-\u003esize, sizeof(void ), GFP_KERNEL);\n-\n-\tif (!pool-\u003eskbuff) {\n-\t\tkfree(pool-\u003edma_addr);\n-\t\tpool-\u003edma_addr = NULL;\n-\n-\t\tkfree(pool-\u003efree_map);\n-\t\tpool-\u003efree_map = NULL;\n-\t\treturn -1;\n+\tadapter-\u003efilter_list_dma = dma_map_single(dev,\n+\t\t\t\t\t\t adapter-\u003efilter_list_addr,\n+\t\t\t\t\t\t 4096, DMA_BIDIRECTIONAL);\n+\tif (dma_mapping_error(dev, adapter-\u003efilter_list_dma)) {\n+\t\tnetdev_err(netdev, \"unable to map filter list pages\\n\");\n+\t\tfree_page((unsigned long)adapter-\u003efilter_list_addr);\n+\t\tadapter-\u003efilter_list_addr = NULL;\n+\t\t/* Do not leave DMA_MAPPING_ERROR for free_filter_list(). /\n+\t\tadapter-\u003efilter_list_dma = 0;\n+\t\treturn -ENOMEM;\n \t}\n \n-\tfor (i = 0; i \u003c pool-\u003esize; ++i)\n-\t\tpool-\u003efree_map[i] = i;\n-\n-\tatomic_set(\u0026pool-\u003eavailable, 0);\n-\tpool-\u003eproducer_index = 0;\n-\tpool-\u003econsumer_index = 0;\n+\tnetdev_dbg(netdev, \"filter list @ 0x%p (DMA: 0x%llx)\\n\",\n+\t\t adapter-\u003efilter_list_addr,\n+\t\t (unsigned long long)adapter-\u003efilter_list_dma);\n \n \treturn 0;\n }\n \n-static inline void ibmveth_flush_buffer(void addr, unsigned long length)\n+/*\n+ ibmveth_free_filter_list - Free filter list resources\n+ * @adapter: ibmveth adapter structure\n+ /\n+static void\n+ibmveth_free_filter_list(struct ibmveth_adapter adapter)\n {\n-\tunsigned long offset;\n+\tstruct device dev = \u0026adapter-\u003evdev-\u003edev;\n \n-\tfor (offset = 0; offset \u003c length; offset += SMP_CACHE_BYTES)\n-\t\tasm(\"dcbf %0,%1,1\" :: \"b\" (addr), \"r\" (offset));\n+\t/ Unmap by CPU pointer: SPAPR can return DMA address 0. /\n+\tif (adapter-\u003efilter_list_addr) {\n+\t\tdma_unmap_single(dev, adapter-\u003efilter_list_dma, 4096,\n+\t\t\t\t DMA_BIDIRECTIONAL);\n+\t\tadapter-\u003efilter_list_dma = 0;\n+\t\tfree_page((unsigned long)adapter-\u003efilter_list_addr);\n+\t\tadapter-\u003efilter_list_addr = NULL;\n+\t}\n }\n \n-/ replenish the buffers for a pool. note that we don't need to\n- * skb_reserve these since they are used for incoming...\n+/*\n+ ibmveth_alloc_rx_qstats - Allocate per-queue RX statistics\n+ * @adapter: ibmveth adapter structure\n+ \n+ Return: 0 on success, -ENOMEM on failure\n /\n-static void ibmveth_replenish_buffer_pool(struct ibmveth_adapter adapter,\n-\t\t\t\t\t struct ibmveth_buff_pool pool)\n+static int ibmveth_alloc_rx_qstats(struct ibmveth_adapter adapter)\n {\n-\tunion ibmveth_buf_desc descs[IBMVETH_MAX_RX_PER_HCALL] = {0};\n-\tu32 remaining = pool-\u003esize - atomic_read(\u0026pool-\u003eavailable);\n-\tu64 correlators[IBMVETH_MAX_RX_PER_HCALL] = {0};\n-\tunsigned long lpar_rc;\n-\tu32 buffers_added = 0;\n-\tu32 i, filled, batch;\n-\tstruct vio_dev vdev;\n-\tdma_addr_t dma_addr;\n-\tstruct device dev;\n-\tu32 index;\n+\tadapter-\u003erx_qstats = kcalloc(IBMVETH_MAX_RX_QUEUES,\n+\t\t\t\t sizeof(adapter-\u003erx_qstats),\n+\t\t\t\t GFP_KERNEL);\n+\tif (!adapter-\u003erx_qstats)\n+\t\treturn -ENOMEM;\n \n-\tvdev = adapter-\u003evdev;\n-\tdev = \u0026vdev-\u003edev;\n+\treturn 0;\n+}\n \n-\tmb();\n+/\n+ ibmveth_free_rx_qstats - Free per-queue RX statistics\n+ * @adapter: ibmveth adapter structure\n+ /\n+static void ibmveth_free_rx_qstats(struct ibmveth_adapter adapter)\n+{\n+\tkfree(adapter-\u003erx_qstats);\n+\tadapter-\u003erx_qstats = NULL;\n+}\n \n-\tbatch = adapter-\u003erx_buffers_per_hcall;\n+/*\n+ ibmveth_alloc_tx_qstats - Allocate per-queue TX statistics\n+ * @adapter: ibmveth adapter structure\n+ \n+ Return: 0 on success, -ENOMEM on failure\n+ /\n+static int ibmveth_alloc_tx_qstats(struct ibmveth_adapter adapter)\n+{\n+\tadapter-\u003etx_qstats = kcalloc(IBMVETH_MAX_QUEUES,\n+\t\t\t\t sizeof(adapter-\u003etx_qstats),\n+\t\t\t\t GFP_KERNEL);\n+\tif (!adapter-\u003etx_qstats)\n+\t\treturn -ENOMEM;\n \n-\twhile (remaining \u003e 0) {\n-\t\tunsigned int free_index = pool-\u003econsumer_index;\n+\treturn 0;\n+}\n \n-\t\t/ Fill a batch of descriptors /\n-\t\tfor (filled = 0; filled \u003c min(remaining, batch); filled++) {\n-\t\t\tindex = pool-\u003efree_map[free_index];\n-\t\t\tif (WARN_ON(index == IBM_VETH_INVALID_MAP)) {\n-\t\t\t\tadapter-\u003ereplenish_add_buff_failure++;\n-\t\t\t\tnetdev_info(adapter-\u003enetdev,\n-\t\t\t\t\t \"Invalid map index %u, reset\\n\",\n-\t\t\t\t\t index);\n-\t\t\t\tschedule_work(\u0026adapter-\u003ework);\n-\t\t\t\tbreak;\n-\t\t\t}\n+/\n+ ibmveth_free_tx_qstats - Free per-queue TX statistics\n+ * @adapter: ibmveth adapter structure\n+ /\n+static void ibmveth_free_tx_qstats(struct ibmveth_adapter adapter)\n+{\n+\tkfree(adapter-\u003etx_qstats);\n+\tadapter-\u003etx_qstats = NULL;\n+}\n \n-\t\t\tif (!pool-\u003eskbuff[index]) {\n-\t\t\t\tstruct sk_buff skb = NULL;\n+/\n+ ibmveth_alloc_rx_queues - Allocate per-queue RX resources\n+ * @adapter: ibmveth adapter structure\n+ * @rxq_entries: Number of entries per RX queue\n+ \n+ Return: 0 on success, negative error code on failure\n+ /\n+static int\n+ibmveth_alloc_rx_queues(struct ibmveth_adapter adapter, int rxq_entries)\n+{\n+\tstruct device dev = \u0026adapter-\u003evdev-\u003edev;\n+\tstruct net_device netdev = adapter-\u003enetdev;\n+\tint i;\n \n-\t\t\t\tskb = netdev_alloc_skb(adapter-\u003enetdev,\n-\t\t\t\t\t\t pool-\u003ebuff_size);\n-\t\t\t\tif (!skb) {\n-\t\t\t\t\tadapter-\u003ereplenish_no_mem++;\n-\t\t\t\t\tadapter-\u003ereplenish_add_buff_failure++;\n-\t\t\t\t\tbreak;\n-\t\t\t\t}\n+\tfor (i = 0; i \u003c ibmveth_get_num_rx_queues(adapter); i++) {\n+\t\tadapter-\u003ebuffer_list_addr[i] =\n+\t\t\t(void )get_zeroed_page(GFP_KERNEL);\n+\t\tif (!adapter-\u003ebuffer_list_addr[i]) {\n+\t\t\tnetdev_err(netdev,\n+\t\t\t\t \"unable to allocate buffer list for queue %d\\n\",\n+\t\t\t\t i);\n+\t\t\tgoto err_cleanup;\n+\t\t}\n \n-\t\t\t\tdma_addr = dma_map_single(dev, skb-\u003edata,\n-\t\t\t\t\t\t\t pool-\u003ebuff_size,\n-\t\t\t\t\t\t\t DMA_FROM_DEVICE);\n-\t\t\t\tif (dma_mapping_error(dev, dma_addr)) {\n-\t\t\t\t\tdev_kfree_skb_any(skb);\n-\t\t\t\t\tadapter-\u003ereplenish_add_buff_failure++;\n-\t\t\t\t\tbreak;\n-\t\t\t\t}\n+\t\tadapter-\u003erx_queue[i].queue_len =\n+\t\t\tsizeof(struct ibmveth_rx_q_entry) rxq_entries;\n+\t\tadapter-\u003erx_queue[i].queue_addr =\n+\t\t\tdma_alloc_coherent(dev, adapter-\u003erx_queue[i].queue_len,\n+\t\t\t\t\t \u0026adapter-\u003erx_queue[i].queue_dma,\n+\t\t\t\t\t GFP_KERNEL);\n+\t\tif (!adapter-\u003erx_queue[i].queue_addr) {\n+\t\t\tnetdev_err(netdev,\n+\t\t\t\t \"unable to allocate RX queue for queue %d\\n\",\n+\t\t\t\t i);\n+\t\t\tfree_page((unsigned long)adapter-\u003ebuffer_list_addr[i]);\n+\t\t\tadapter-\u003ebuffer_list_addr[i] = NULL;\n+\t\t\tgoto err_cleanup;\n+\t\t}\n \n-\t\t\t\tpool-\u003edma_addr[index] = dma_addr;\n-\t\t\t\tpool-\u003eskbuff[index] = skb;\n-\t\t\t} else {\n-\t\t\t\t/* re-use case /\n-\t\t\t\tdma_addr = pool-\u003edma_addr[index];\n-\t\t\t}\n+\t\tadapter-\u003ebuffer_list_dma[i] =\n+\t\t\tdma_map_single(dev, adapter-\u003ebuffer_list_addr[i],\n+\t\t\t\t 4096, DMA_BIDIRECTIONAL);\n+\t\tif (dma_mapping_error(dev, adapter-\u003ebuffer_list_dma[i])) {\n+\t\t\tnetdev_err(netdev,\n+\t\t\t\t \"unable to map buffer list for queue %d\\n\",\n+\t\t\t\t i);\n+\t\t\tfree_page((unsigned long)adapter-\u003ebuffer_list_addr[i]);\n+\t\t\tadapter-\u003ebuffer_list_addr[i] = NULL;\n+\t\t\tadapter-\u003ebuffer_list_dma[i] = 0;\n+\t\t\tgoto err_cleanup;\n+\t\t}\n \n-\t\t\tif (rx_flush) {\n-\t\t\t\tunsigned int len;\n+\t\tadapter-\u003erx_queue[i].index = 0;\n+\t\tadapter-\u003erx_queue[i].num_slots = rxq_entries;\n+\t\tadapter-\u003erx_queue[i].toggle = 1;\n \n-\t\t\t\tlen = adapter-\u003enetdev-\u003emtu + IBMVETH_BUFF_OH;\n-\t\t\t\tlen = min(pool-\u003ebuff_size, len);\n-\t\t\t\tibmveth_flush_buffer(pool-\u003eskbuff[index]-\u003edata,\n-\t\t\t\t\t\t len);\n-\t\t\t}\n+\t\tnetdev_dbg(netdev, \"queue %d: buffer_list @ 0x%p (DMA: 0x%llx), rx_queue @ 0x%p (DMA: 0x%llx), %llu entries\\n\",\n+\t\t\t i, adapter-\u003ebuffer_list_addr[i],\n+\t\t\t (unsigned long long)adapter-\u003ebuffer_list_dma[i],\n+\t\t\t adapter-\u003erx_queue[i].queue_addr,\n+\t\t\t (unsigned long long)adapter-\u003erx_queue[i].queue_dma,\n+\t\t\t (unsigned long long)rxq_entries);\n+\t}\n \n-\t\t\tdescs[filled].fields.flags_len = IBMVETH_BUF_VALID \|\n-\t\t\t\t\t\t\t pool-\u003ebuff_size;\n-\t\t\tdescs[filled].fields.address = dma_addr;\n+\tnetdev_dbg(netdev, \"allocated %u RX queue(s) with %d entries each\\n\",\n+\t\t ibmveth_get_num_rx_queues(adapter), rxq_entries);\n \n-\t\t\tcorrelators[filled] = ((u64)pool-\u003eindex \u003c\u003c 32) \| index;\n-\t\t\t(u64 )pool-\u003eskbuff[index]-\u003edata = correlators[filled];\n+\treturn 0;\n \n-\t\t\tfree_index++;\n-\t\t\tif (free_index \u003e= pool-\u003esize)\n-\t\t\t\tfree_index = 0;\n+err_cleanup:\n+\t/\n+\t * Free each leftover resource by pointer presence. A\n+\t * dma_mapping_error on queue i has already dropped\n+\t * buffer_list_addr[i] but can leave rx_queue[i].queue_addr\n+\t * live, so this index may be mixed. Do not collapse the\n+\t * buffer_list and queue_addr checks.\n+\t /\n+\tfor (; i \u003e= 0; i--) {\n+\t\tif (adapter-\u003ebuffer_list_addr[i]) {\n+\t\t\tdma_unmap_single(dev, adapter-\u003ebuffer_list_dma[i],\n+\t\t\t\t\t 4096, DMA_BIDIRECTIONAL);\n+\t\t\tadapter-\u003ebuffer_list_dma[i] = 0;\n \t\t}\n-\n-\t\tif (!filled)\n-\t\t\tbreak;\n-\n-\t\t/ single buffer case/\n-\t\tif (filled == 1)\n-\t\t\tlpar_rc = h_add_logical_lan_buffer(vdev-\u003eunit_address,\n-\t\t\t\t\t\t\t descs[0].desc);\n-\t\telse\n-\t\t\t/ Multi-buffer hcall /\n-\t\t\tlpar_rc = h_add_logical_lan_buffers(vdev-\u003eunit_address,\n-\t\t\t\t\t\t\t descs[0].desc,\n-\t\t\t\t\t\t\t descs[1].desc,\n-\t\t\t\t\t\t\t descs[2].desc,\n-\t\t\t\t\t\t\t descs[3].desc,\n-\t\t\t\t\t\t\t descs[4].desc,\n-\t\t\t\t\t\t\t descs[5].desc,\n-\t\t\t\t\t\t\t descs[6].desc,\n-\t\t\t\t\t\t\t descs[7].desc);\n-\t\tif (lpar_rc != H_SUCCESS) {\n-\t\t\tdev_warn_ratelimited(dev,\n-\t\t\t\t\t \"RX h_add_logical_lan failed: filled=%u, rc=%lu, batch=%u\\n\",\n-\t\t\t\t\t filled, lpar_rc, batch);\n-\t\t\tgoto hcall_failure;\n+\t\tif (adapter-\u003erx_queue[i].queue_addr) {\n+\t\t\tdma_free_coherent(dev, adapter-\u003erx_queue[i].queue_len,\n+\t\t\t\t\t adapter-\u003erx_queue[i].queue_addr,\n+\t\t\t\t\t adapter-\u003erx_queue[i].queue_dma);\n+\t\t\tadapter-\u003erx_queue[i].queue_addr = NULL;\n+\t\t}\n+\t\tif (adapter-\u003ebuffer_list_addr[i]) {\n+\t\t\tfree_page((unsigned long)adapter-\u003ebuffer_list_addr[i]);\n+\t\t\tadapter-\u003ebuffer_list_addr[i] = NULL;\n \t\t}\n+\t}\n \n-\t\t/ Only update pool state after hcall succeeds /\n-\t\tfor (i = 0; i \u003c filled; i++) {\n-\t\t\tfree_index = pool-\u003econsumer_index;\n-\t\t\tpool-\u003efree_map[free_index] = IBM_VETH_INVALID_MAP;\n+\treturn -ENOMEM;\n+}\n \n-\t\t\tpool-\u003econsumer_index++;\n-\t\t\tif (pool-\u003econsumer_index \u003e= pool-\u003esize)\n-\t\t\t\tpool-\u003econsumer_index = 0;\n-\t\t}\n+/\n+ ibmveth_cleanup_rx_resources - Free all RX queue resources\n+ * @adapter: ibmveth adapter structure\n+ /\n+static void\n+ibmveth_cleanup_rx_resources(struct ibmveth_adapter adapter)\n+{\n+\tstruct device dev = \u0026adapter-\u003evdev-\u003edev;\n+\tint i;\n \n-\t\tbuffers_added += filled;\n-\t\tadapter-\u003ereplenish_add_buff_success += filled;\n-\t\tremaining -= filled;\n+\tnetdev_dbg(adapter-\u003enetdev, \"cleaning up %u RX queue(s)\\n\",\n+\t\t ibmveth_get_num_rx_queues(adapter));\n \n-\t\tmemset(\u0026descs, 0, sizeof(descs));\n-\t\tmemset(\u0026correlators, 0, sizeof(correlators));\n-\t\tcontinue;\n+\tfor (i = 0; i \u003c ibmveth_get_num_rx_queues(adapter); i++) {\n+\t\tif (adapter-\u003ebuffer_list_addr[i]) {\n+\t\t\tdma_unmap_single(dev, adapter-\u003ebuffer_list_dma[i],\n+\t\t\t\t\t 4096, DMA_BIDIRECTIONAL);\n+\t\t\tadapter-\u003ebuffer_list_dma[i] = 0;\n+\t\t}\n \n-hcall_failure:\n-\t\tfor (i = 0; i \u003c filled; i++) {\n-\t\t\tindex = correlators[i] \u0026 0xffffffffUL;\n-\t\t\tdma_addr = pool-\u003edma_addr[index];\n+\t\tif (adapter-\u003erx_queue[i].queue_addr) {\n+\t\t\tdma_free_coherent(dev, adapter-\u003erx_queue[i].queue_len,\n+\t\t\t\t\t adapter-\u003erx_queue[i].queue_addr,\n+\t\t\t\t\t adapter-\u003erx_queue[i].queue_dma);\n+\t\t\tadapter-\u003erx_queue[i].queue_addr = NULL;\n+\t\t}\n \n-\t\t\tif (pool-\u003eskbuff[index]) {\n-\t\t\t\tif (dma_addr \u0026\u0026\n-\t\t\t\t !dma_mapping_error(dev, dma_addr))\n-\t\t\t\t\tdma_unmap_single(dev, dma_addr,\n-\t\t\t\t\t\t\t pool-\u003ebuff_size,\n-\t\t\t\t\t\t\t DMA_FROM_DEVICE);\n+\t\tif (adapter-\u003ebuffer_list_addr[i]) {\n+\t\t\tfree_page((unsigned long)adapter-\u003ebuffer_list_addr[i]);\n+\t\t\tadapter-\u003ebuffer_list_addr[i] = NULL;\n+\t\t}\n+\t}\n+}\n \n-\t\t\t\tdev_kfree_skb_any(pool-\u003eskbuff[index]);\n-\t\t\t\tpool-\u003eskbuff[index] = NULL;\n-\t\t\t}\n+/\n+ ibmveth_toggle_irq - Common helper to enable/disable queue interrupts\n+ * @adapter: ibmveth adapter structure\n+ * @queue_index: Index of the queue (0 for primary, 1+ for subordinate)\n+ * @enable: true to enable, false to disable\n+ \n+ For queue 0 (primary), uses h_vio_signal() as it's registered via\n+ * h_register_logical_lan(). For subordinate queues (1+), uses H_VIOCTL\n+ * with H_ENABLE/DISABLE_VIO_INTERRUPT for per-queue interrupt control.\n+ \n+ Return: 0 on success, negative errno on failure (never raw H_).\n+ /\n+static int\n+ibmveth_toggle_irq(struct ibmveth_adapter adapter, int queue_index,\n+\t\t bool enable)\n+{\n+\tunsigned long h_rc;\n+\tunsigned long irq = adapter-\u003equeue_irq[queue_index];\n+\tconst char action = enable ? \"enable\" : \"disable\";\n+\n+\tif (queue_index == 0) {\n+\t\t/* Primary queue: use h_vio_signal() /\n+\t\th_rc = h_vio_signal(adapter-\u003evdev-\u003eunit_address,\n+\t\t\t\t enable ? VIO_IRQ_ENABLE : VIO_IRQ_DISABLE);\n+\t} else {\n+\t\t/ Subordinate queues: use H_VIOCTL with hardware IRQ /\n+\t\tstruct irq_data irq_data = irq_get_irq_data(irq);\n+\t\tirq_hw_number_t hwirq;\n+\t\tu64 vioctl_cmd = enable ? H_ENABLE_VIO_INTERRUPT :\n+\t\t\tH_DISABLE_VIO_INTERRUPT;\n+\n+\t\tif (!irq_data) {\n+\t\t\tnetdev_err(adapter-\u003enetdev,\n+\t\t\t\t \"Failed to get IRQ data for queue %d (virq=%lu)\\n\",\n+\t\t\t\t queue_index, irq);\n+\t\t\treturn -EINVAL;\n \t\t}\n-\t\tadapter-\u003ereplenish_add_buff_failure += filled;\n+\n+\t\thwirq = irqd_to_hwirq(irq_data);\n+\t\th_rc = plpar_hcall_norets(H_VIOCTL,\n+\t\t\t\t\t adapter-\u003evdev-\u003eunit_address,\n+\t\t\t\t\t vioctl_cmd,\n+\t\t\t\t\t hwirq, 0, 0);\n \n \t\t/\n-\t\t If multi rx buffers hcall is no longer supported by FW\n-\t\t * e.g. in the case of Live Partition Migration\n+\t\t * H_PARAMETER is ambiguous (already in requested state vs bad\n+\t\t * args). Fold only on disable as an idempotent mask. On enable\n+\t\t * keep it an error so a stuck-masked queue stays visible to\n+\t\t * poll/resize recovery.\n \t\t /\n-\t\tif (batch \u003e 1 \u0026\u0026 lpar_rc == H_FUNCTION) {\n-\t\t\t/\n-\t\t\t * Instead of retry submit single buffer individually\n-\t\t\t * here just set the max rx buffer per hcall to 1\n-\t\t\t * buffers will be respleshed next time\n-\t\t\t * when ibmveth_replenish_buffer_pool() is called again\n-\t\t\t * with single-buffer case\n-\t\t\t /\n-\t\t\tnetdev_info(adapter-\u003enetdev,\n-\t\t\t\t \"RX Multi buffers not supported by FW, rc=%lu\\n\",\n-\t\t\t\t lpar_rc);\n-\t\t\tadapter-\u003erx_buffers_per_hcall = 1;\n-\t\t\tnetdev_info(adapter-\u003enetdev,\n-\t\t\t\t \"Next rx replesh will fall back to single-buffer hcall\\n\");\n+\t\tif (h_rc == H_PARAMETER \u0026\u0026 !enable) {\n+\t\t\tdev_warn_ratelimited(\u0026adapter-\u003enetdev-\u003edev,\n+\t\t\t\t\t \"H_VIOCTL %s IRQ returned H_PARAMETER for queue %d (hwirq=%lu)\\n\",\n+\t\t\t\t\t action, queue_index, hwirq);\n+\t\t\treturn 0;\n \t\t}\n-\t\tbreak;\n \t}\n \n-\tmb();\n-\tatomic_add(buffers_added, \u0026(pool-\u003eavailable));\n+\tif (h_rc) {\n+\t\tdev_err_ratelimited(\u0026adapter-\u003enetdev-\u003edev,\n+\t\t\t\t \"Failed to %s IRQ for queue %d, rc=0x%lx\\n\",\n+\t\t\t\t action, queue_index, h_rc);\n+\t\treturn -EIO;\n+\t}\n+\treturn 0;\n }\n \n-/\n- * The final 8 bytes of the buffer list is a counter of frames dropped\n- * because there was not a buffer in the buffer list capable of holding\n- * the frame.\n+/*\n+ ibmveth_disable_irq - Disable interrupt for a specific queue\n+ * @adapter: ibmveth adapter structure\n+ * @queue_index: Index of the queue (0 for primary, 1+ for subordinate)\n+ \n+ Return: 0 on success, negative errno on failure\n /\n-static void ibmveth_update_rx_no_buffer(struct ibmveth_adapter adapter)\n+static int\n+ibmveth_disable_irq(struct ibmveth_adapter adapter, int queue_index)\n {\n-\t__be64 p = adapter-\u003ebuffer_list_addr + 4096 - 8;\n-\n-\tadapter-\u003erx_no_buffer = be64_to_cpup(p);\n+\treturn ibmveth_toggle_irq(adapter, queue_index, false);\n }\n \n-/* replenish routine /\n-static void ibmveth_replenish_task(struct ibmveth_adapter adapter)\n+/*\n+ ibmveth_enable_irq - Enable interrupt for a specific queue\n+ * @adapter: ibmveth adapter structure\n+ * @queue_index: Index of the queue (0 for primary, 1+ for subordinate)\n+ \n+ Return: 0 on success, negative errno on failure\n+ /\n+static int\n+ibmveth_enable_irq(struct ibmveth_adapter adapter, int queue_index)\n {\n-\tint i;\n-\n-\tadapter-\u003ereplenish_task_cycles++;\n+\treturn ibmveth_toggle_irq(adapter, queue_index, true);\n+}\n \n-\tfor (i = (IBMVETH_NUM_BUFF_POOLS - 1); i \u003e= 0; i--) {\n-\t\tstruct ibmveth_buff_pool pool = \u0026adapter-\u003erx_buff_pool[i];\n+/\n+ ibmveth_dispose_subordinate_irq_mapping - Drop one subordinate virq mapping\n+ * @adapter: ibmveth adapter structure\n+ * @queue_idx: RX queue index (1..N)\n+ \n+ Subordinate queues get mappings from irq_create_mapping() during PHYP\n+ * registration. Queue 0 uses netdev-\u003eirq from device tree and is left alone.\n+ \n+ Bound against IBMVETH_MAX_RX_QUEUES, not num_rx_queues: a caller may\n+ * dispose a queue that is no longer in the published live set but still\n+ * owns a virq in queue_irq[]. Contrast with the bulk helper, which only\n+ * walks 1..num_rx_queues-1 (close / open-fail cleanup of the live set).\n+ \n+ Linux virq lifetime is owned by interrupt cleanup helpers. Call this only\n+ * after free_irq() when a handler was installed, or from registration failure\n+ * cleanup before request_irq().\n+ /\n+static void\n+ibmveth_dispose_subordinate_irq_mapping(struct ibmveth_adapter adapter,\n+\t\t\t\t\tint queue_idx)\n+{\n+\tif (queue_idx \u003c= 0 \|\| queue_idx \u003e= IBMVETH_MAX_RX_QUEUES)\n+\t\treturn;\n+\n+\tif (adapter-\u003equeue_irq[queue_idx]) {\n+\t\tirq_dispose_mapping(adapter-\u003equeue_irq[queue_idx]);\n+\t\tadapter-\u003equeue_irq[queue_idx] = 0;\n+\t}\n+}\n+\n+/*\n+ ibmveth_dispose_subordinate_irq_mappings - Drop virq mappings for queues 1..N\n+ * @adapter: ibmveth adapter structure\n+ \n+ Bulk helper for close / open-fail cleanup of the published live set\n+ * (queues 1..num_rx_queues-1). Paths that need a retired or not-yet-published\n+ * queue must call ibmveth_dispose_subordinate_irq_mapping() directly.\n+ /\n+static void\n+ibmveth_dispose_subordinate_irq_mappings(struct ibmveth_adapter adapter)\n+{\n+\tint i;\n+\n+\tfor (i = 1; i \u003c ibmveth_get_num_rx_queues(adapter); i++)\n+\t\tibmveth_dispose_subordinate_irq_mapping(adapter, i);\n+}\n+\n+/*\n+ ibmveth_setup_rx_interrupts - Register IRQs and enable NAPI\n+ * @adapter: ibmveth adapter structure\n+ \n+ Registers interrupt handlers for all RX queues, enables NAPI, then\n+ * enables hypervisor interrupt delivery for multi-queue mode after\n+ * every queue has a Linux handler installed. The caller replenishes\n+ * every live queue before this helper so traffic during open is not\n+ * dropped (PHYP only interrupts after a successful enqueue, which\n+ * needs buffers). Single-queue open still leaves PHYP masked here;\n+ * restart_rx_queue then kicks NAPI.\n+ \n+ Return: 0 on success, negative error code on failure\n+ /\n+static int\n+ibmveth_setup_rx_interrupts(struct ibmveth_adapter adapter)\n+{\n+\tstruct net_device netdev = adapter-\u003enetdev;\n+\tint i, rc, num = ibmveth_get_num_rx_queues(adapter);\n+\n+\tfor (i = 0; i \u003c num; i++) {\n+\t\tif (!adapter-\u003equeue_irq[i]) {\n+\t\t\tnetdev_err(netdev, \"queue %d has invalid IRQ (0)\\n\", i);\n+\t\t\trc = -EINVAL;\n+\t\t\tgoto err_free_irqs;\n+\t\t}\n+\n+\t\trc = request_irq(adapter-\u003equeue_irq[i], ibmveth_interrupt,\n+\t\t\t\t 0, netdev-\u003ename, \u0026adapter-\u003enapi[i]);\n+\t\tif (rc) {\n+\t\t\tnetdev_err(netdev,\n+\t\t\t\t \"request_irq() failed for irq 0x%x queue %d: %d\\n\",\n+\t\t\t\t adapter-\u003equeue_irq[i], i, rc);\n+\t\t\tgoto err_free_irqs;\n+\t\t}\n+\t}\n+\n+\tfor (i = 0; i \u003c num; i++)\n+\t\tnapi_enable(\u0026adapter-\u003enapi[i]);\n+\n+\tif (adapter-\u003emulti_queue \u0026\u0026 num \u003e 1) {\n+\t\tfor (i = 0; i \u003c num; i++) {\n+\t\t\trc = ibmveth_enable_irq(adapter, i);\n+\t\t\tif (rc) {\n+\t\t\t\tnetdev_err(netdev,\n+\t\t\t\t\t \"Failed to enable IRQ for queue %d, rc=%d\\n\",\n+\t\t\t\t\t i, rc);\n+\t\t\t\tfor (; i \u003e= 0; i--) {\n+\t\t\t\t\tibmveth_disable_irq(adapter, i);\n+\t\t\t\t\tsynchronize_irq(adapter-\u003equeue_irq[i]);\n+\t\t\t\t}\n+\t\t\t\trc = -EIO;\n+\t\t\t\tgoto err_disable_napi;\n+\t\t\t}\n+\t\t}\n+\t}\n+\n+\t/ Set only on full success; fail paths leave this false so a later\n+\t * close() / cleanup is a no-op.\n+\t /\n+\tadapter-\u003erx_irq_setup = true;\n+\treturn 0;\n+\n+err_disable_napi:\n+\t/ STOP: remask after napi_disable; an in-flight poll can re-arm. /\n+\tfor (i = 0; i \u003c num; i++)\n+\t\tnapi_disable(\u0026adapter-\u003enapi[i]);\n+\tfor (i = 0; i \u003c num; i++) {\n+\t\tif (!adapter-\u003equeue_irq[i])\n+\t\t\tcontinue;\n+\t\tibmveth_disable_irq(adapter, i);\n+\t\tsynchronize_irq(adapter-\u003equeue_irq[i]);\n+\t}\n+\tfor (i = 0; i \u003c num; i++) {\n+\t\tif (adapter-\u003equeue_irq[i])\n+\t\t\tfree_irq(adapter-\u003equeue_irq[i], \u0026adapter-\u003enapi[i]);\n+\t}\n+\tgoto err_dispose_mappings;\n+\n+err_free_irqs:\n+\twhile (--i \u003e= 0)\n+\t\tfree_irq(adapter-\u003equeue_irq[i], \u0026adapter-\u003enapi[i]);\n+err_dispose_mappings:\n+\t/ Both setup failure paths own subordinate virq disposal. /\n+\tibmveth_dispose_subordinate_irq_mappings(adapter);\n+\treturn rc;\n+}\n+\n+/\n+ ibmveth_cleanup_rx_interrupts - Mask PHYP IRQs, stop NAPI, and free IRQs\n+ * @adapter: ibmveth adapter structure\n+ \n+ Mask and synchronize each queue IRQ before napi_disable() so the handler\n+ * cannot miss a PHYP mask while NAPI is already dead. Remask after\n+ * napi_disable() in case an in-flight poll re-armed PHYP while we waited.\n+ * free_irq() runs only after that. Safe for close and for open failure after\n+ * setup_rx_interrupts() already unmasked PHYP. No-op if setup never\n+ * succeeded (avoids double napi_disable / free_irq after a failed close+open\n+ * while IFF_UP remains set).\n+ /\n+static void\n+ibmveth_cleanup_rx_interrupts(struct ibmveth_adapter adapter)\n+{\n+\tint i;\n+\n+\tif (!adapter-\u003erx_irq_setup)\n+\t\treturn;\n+\n+\tfor (i = 0; i \u003c ibmveth_get_num_rx_queues(adapter); i++) {\n+\t\tif (!adapter-\u003equeue_irq[i])\n+\t\t\tcontinue;\n+\t\tibmveth_disable_irq(adapter, i);\n+\t\tsynchronize_irq(adapter-\u003equeue_irq[i]);\n+\t}\n+\n+\tfor (i = 0; i \u003c ibmveth_get_num_rx_queues(adapter); i++)\n+\t\tnapi_disable(\u0026adapter-\u003enapi[i]);\n+\n+\tfor (i = 0; i \u003c ibmveth_get_num_rx_queues(adapter); i++) {\n+\t\tif (!adapter-\u003equeue_irq[i])\n+\t\t\tcontinue;\n+\t\tibmveth_disable_irq(adapter, i);\n+\t\tsynchronize_irq(adapter-\u003equeue_irq[i]);\n+\t}\n+\n+\tfor (i = 0; i \u003c ibmveth_get_num_rx_queues(adapter); i++) {\n+\t\tif (adapter-\u003equeue_irq[i])\n+\t\t\tfree_irq(adapter-\u003equeue_irq[i], \u0026adapter-\u003enapi[i]);\n+\t}\n+\n+\tibmveth_dispose_subordinate_irq_mappings(adapter);\n+\n+\t/* Queue 0 uses netdev-\u003eirq; leave queue_irq[0] for next open. /\n+\tadapter-\u003erx_irq_setup = false;\n+}\n+\n+/\n+ ibmveth_setup_single_rx_interrupt - Setup interrupt for a single RX queue\n+ * @adapter: ibmveth adapter structure\n+ * @queue_idx: Queue index to setup\n+ \n+ Registers the IRQ handler for one queue. Used during incremental\n+ * scale-up when adding new RX queues. The caller publishes the queue,\n+ * replenishes buffers, enables NAPI, then unmasks PHYP delivery.\n+ \n+ Return: 0 on success, negative error code on failure\n+ /\n+static int\n+ibmveth_setup_single_rx_interrupt(struct ibmveth_adapter adapter,\n+\t\t\t\t int queue_idx)\n+{\n+\tstruct net_device netdev = adapter-\u003enetdev;\n+\tint rc;\n+\n+\trc = request_irq(adapter-\u003equeue_irq[queue_idx], ibmveth_interrupt,\n+\t\t\t 0, netdev-\u003ename, \u0026adapter-\u003enapi[queue_idx]);\n+\tif (rc) {\n+\t\tnetdev_err(netdev, \"request_irq() failed for queue %d: %d\\n\",\n+\t\t\t queue_idx, rc);\n+\t\treturn rc;\n+\t}\n+\n+\tnetdev_dbg(netdev, \"Setup IRQ %d for queue %d\\n\",\n+\t\t adapter-\u003equeue_irq[queue_idx], queue_idx);\n+\treturn 0;\n+}\n+\n+/\n+ ibmveth_cleanup_single_rx_interrupt - Cleanup interrupt for a single RX queue\n+ * @adapter: ibmveth adapter structure\n+ * @queue_idx: Queue index to cleanup\n+ \n+ Frees the IRQ handler for one queue and releases the subordinate virq\n+ * mapping. Used during incremental scale-down.\n+ /\n+static void\n+ibmveth_cleanup_single_rx_interrupt(struct ibmveth_adapter adapter,\n+\t\t\t\t int queue_idx)\n+{\n+\tif (adapter-\u003equeue_irq[queue_idx]) {\n+\t\tfree_irq(adapter-\u003equeue_irq[queue_idx],\n+\t\t\t \u0026adapter-\u003enapi[queue_idx]);\n+\t\tibmveth_dispose_subordinate_irq_mapping(adapter, queue_idx);\n+\t\tnetdev_dbg(adapter-\u003enetdev,\n+\t\t\t \"Freed IRQ for queue %d\\n\", queue_idx);\n+\t}\n+}\n+\n+/*\n+ ibmveth_schedule_rx_queue - Mask PHYP IRQ and schedule NAPI for one RX queue\n+ * @adapter: ibmveth adapter structure\n+ * @qindex: RX queue index\n+ \n+ Shared by the IRQ handler and process-context kick sites (open, resume,\n+ * pool sysfs, poll_controller).\n+ \n+ Return: true if napi_schedule_prep() succeeded and NAPI was scheduled.\n+ * Mask is attempted in that case; a failed disable_irq() is logged by the\n+ * helper and does not change the return (queue may still be unmasked).\n+ * false if the index is negative (WARN_ON), past the live count (quiet:\n+ * a shrink can race netpoll), or prep failed (including NAPI already\n+ * scheduled).\n+ /\n+static bool ibmveth_schedule_rx_queue(struct ibmveth_adapter adapter,\n+\t\t\t\t int qindex)\n+{\n+\tstruct napi_struct napi = \u0026adapter-\u003enapi[qindex];\n+\n+\tif (WARN_ON(qindex \u003c 0))\n+\t\treturn false;\n+\n+\t/\n+\t * A live shrink can publish a lower count while netpoll walks a\n+\t * snapshot of the old one, so an index past the end is expected\n+\t * here and must not splat. ibmveth_replenish_task() skips the\n+\t * same way; callers already treat false as \"queue is gone\".\n+\t /\n+\tif (qindex \u003e= ibmveth_get_num_rx_queues(adapter))\n+\t\treturn false;\n+\n+\t/\n+\t * Only mask PHYP when NAPI will run. Masking on prep failure can\n+\t * race a completing poll that already re-enabled the queue, leaving\n+\t * NAPI idle with the IRQ masked (TX works, RX stalls) until reload.\n+\t * Storm prevention on teardown remains in cleanup/disable paths.\n+\t /\n+\tif (napi_schedule_prep(napi)) {\n+\t\t/ Failure is already logged with the hcall rc by the helper. /\n+\t\tibmveth_disable_irq(adapter, qindex);\n+\t\t__napi_schedule(napi);\n+\t\treturn true;\n+\t}\n+\treturn false;\n+}\n+\n+/ setup the initial settings for a buffer pool /\n+static void ibmveth_init_buffer_pool(struct ibmveth_buff_pool pool,\n+\t\t\t\t u32 pool_index, u32 pool_size,\n+\t\t\t\t u32 buff_size, u32 pool_active)\n+{\n+\tpool-\u003esize = pool_size;\n+\tpool-\u003eindex = pool_index;\n+\tpool-\u003ebuff_size = buff_size;\n+\tpool-\u003ethreshold = pool_size * 7 / 8;\n+\tpool-\u003eactive = pool_active;\n+}\n+\n+/* allocate and setup an buffer pool - called during open /\n+static int ibmveth_alloc_buffer_pool(struct ibmveth_buff_pool pool)\n+{\n+\tint i;\n+\n+\tpool-\u003efree_map = kmalloc_array(pool-\u003esize, sizeof(u16), GFP_KERNEL);\n+\n+\tif (!pool-\u003efree_map)\n+\t\treturn -1;\n+\n+\tpool-\u003edma_addr = kzalloc_objs(dma_addr_t, pool-\u003esize);\n+\tif (!pool-\u003edma_addr) {\n+\t\tkfree(pool-\u003efree_map);\n+\t\tpool-\u003efree_map = NULL;\n+\t\treturn -1;\n+\t}\n+\n+\tpool-\u003eskbuff = kcalloc(pool-\u003esize, sizeof(void ), GFP_KERNEL);\n+\n+\tif (!pool-\u003eskbuff) {\n+\t\tkfree(pool-\u003edma_addr);\n+\t\tpool-\u003edma_addr = NULL;\n+\n+\t\tkfree(pool-\u003efree_map);\n+\t\tpool-\u003efree_map = NULL;\n+\t\treturn -1;\n+\t}\n+\n+\tfor (i = 0; i \u003c pool-\u003esize; ++i)\n+\t\tpool-\u003efree_map[i] = i;\n+\n+\tatomic_set(\u0026pool-\u003eavailable, 0);\n+\tpool-\u003eproducer_index = 0;\n+\tpool-\u003econsumer_index = 0;\n+\n+\treturn 0;\n+}\n+\n+static inline void ibmveth_flush_buffer(void addr, unsigned long length)\n+{\n+\tunsigned long offset;\n+\n+\tfor (offset = 0; offset \u003c length; offset += SMP_CACHE_BYTES)\n+\t\tasm(\"dcbf %0,%1,1\" :: \"b\" (addr), \"r\" (offset));\n+}\n+\n+/*\n+ ibmveth_add_logical_lan_buffers - Add receive buffers to hypervisor\n+ * @adapter: ibmveth adapter structure\n+ * @descs: array of buffer descriptors to add\n+ * @filled: number of valid descriptors in the array\n+ * @buff_size: size of each buffer (multi-queue mode only)\n+ * @queue_index: RX queue index\n+ \n+ Return: hypervisor return code\n+ /\n+static long ibmveth_add_logical_lan_buffers(struct ibmveth_adapter adapter,\n+\t\t\t\t\t union ibmveth_buf_desc descs,\n+\t\t\t\t\t int filled,\n+\t\t\t\t\t unsigned long buff_size,\n+\t\t\t\t\t int queue_index)\n+{\n+\tstruct vio_dev vdev = adapter-\u003evdev;\n+\tunsigned long rc;\n+\n+\t/\n+\t The MQ hcall takes six ioba words (12 packed addresses). The\n+\t * legacy hcall takes eight descriptors. The argument lists below\n+\t * are written out by hand; keep the defines matched to those lists.\n+\t /\n+\tstatic_assert(IBMVETH_MAX_RX_PER_HCALL == 12);\n+\tstatic_assert(IBMVETH_MAX_RX_REGULAR == 8);\n+\n+\tif (adapter-\u003emulti_queue) {\n+\t\tunsigned long buffersznum = (buff_size \u003c\u003c 32) \| filled;\n+\t\tunsigned long ioba[IBMVETH_MAX_RX_PER_HCALL / 2] = {0};\n+\t\tunsigned long handle = adapter-\u003equeue_handle[queue_index];\n+\t\tint i;\n+\n+\t\t/ Pack descriptor addresses into ioba pairs.\n+\t\t * Each ioba holds two 32-bit addresses packed into 64 bits:\n+\t\t * - Even descriptors (0,2,4...) go in high 32 bits\n+\t\t * - Odd descriptors (1,3,5...) go in low 32 bits\n+\t\t /\n+\t\tfor (i = 0; i \u003c filled \u0026\u0026 i \u003c IBMVETH_MAX_RX_PER_HCALL; i++) {\n+\t\t\tint pair_idx = i / 2;\n+\t\t\tint is_high = (i % 2 == 0);\n+\n+\t\t\tif (is_high)\n+\t\t\t\tioba[pair_idx] = (unsigned long)\n+\t\t\t\t\tdescs[i].fields.address \u003c\u003c 32;\n+\t\t\telse\n+\t\t\t\tioba[pair_idx] \|= descs[i].fields.address;\n+\t\t}\n+\n+\t\trc = h_add_logical_lan_buffers_queue(vdev-\u003eunit_address,\n+\t\t\t\t\t\t handle,\n+\t\t\t\t\t\t buffersznum,\n+\t\t\t\t\t\t ioba[0], ioba[1], ioba[2],\n+\t\t\t\t\t\t ioba[3], ioba[4], ioba[5]);\n+\t} else if (filled == 1) {\n+\t\trc = h_add_logical_lan_buffer(vdev-\u003eunit_address,\n+\t\t\t\t\t descs[0].desc);\n+\t} else {\n+\t\t/ Legacy 8-desc hcall; probe/mq_fallback keep batch \u003c=\n+\t\t * IBMVETH_MAX_RX_REGULAR.\n+\t\t /\n+\t\trc = h_add_logical_lan_buffers(vdev-\u003eunit_address,\n+\t\t\t\t\t descs[0].desc, descs[1].desc,\n+\t\t\t\t\t descs[2].desc, descs[3].desc,\n+\t\t\t\t\t descs[4].desc, descs[5].desc,\n+\t\t\t\t\t descs[6].desc, descs[7].desc);\n+\t}\n+\n+\treturn rc;\n+}\n+\n+/ Outcomes for ibmveth_replenish_buffer_pool(); logged after unlock. /\n+enum {\n+\tIBMVETH_REPLENISH_OK = 0,\n+\tIBMVETH_REPLENISH_RESET_MAP,\n+\tIBMVETH_REPLENISH_RESET_MQ,\n+\tIBMVETH_REPLENISH_HCALL_FAIL,\n+\tIBMVETH_REPLENISH_BATCH_FALLBACK,\n+};\n+\n+struct ibmveth_replenish_fail {\n+\tunsigned long lpar_rc;\n+\tu32 filled;\n+\tu32 batch;\n+};\n+\n+/ Replenish the buffers for a pool.\n+ * Caller must hold the per-queue replenish_lock. Do not printk here:\n+ * netconsole on the same device can re-enter replenish_task.\n+ /\n+static int ibmveth_replenish_buffer_pool(struct ibmveth_adapter adapter,\n+\t\t\t\t\t struct ibmveth_buff_pool pool,\n+\t\t\t\t\t int queue_index,\n+\t\t\t\t\t struct ibmveth_replenish_fail fail)\n+{\n+\tstruct ibmveth_rx_queue_stats qstats =\n+\t\t\u0026adapter-\u003erx_qstats[queue_index];\n+\tunion ibmveth_buf_desc descs[IBMVETH_MAX_RX_PER_HCALL] = {0};\n+\tu32 remaining = pool-\u003esize - atomic_read(\u0026pool-\u003eavailable);\n+\tu64 correlators[IBMVETH_MAX_RX_PER_HCALL] = {0};\n+\tunsigned long lpar_rc;\n+\tu32 buffers_added = 0;\n+\tu32 i, filled, batch;\n+\tstruct vio_dev vdev;\n+\tdma_addr_t dma_addr;\n+\tstruct device dev;\n+\tu32 index;\n+\tint outcome = IBMVETH_REPLENISH_OK;\n+\n+\tvdev = adapter-\u003evdev;\n+\tdev = \u0026vdev-\u003edev;\n+\n+\tmb();\n+\n+\tbatch = adapter-\u003erx_buffers_per_hcall;\n+\n+\twhile (remaining \u003e 0) {\n+\t\tunsigned int free_index = pool-\u003econsumer_index;\n+\n+\t\t/ Fill a batch of descriptors /\n+\t\tfor (filled = 0; filled \u003c min(remaining, batch); filled++) {\n+\t\t\tindex = pool-\u003efree_map[free_index];\n+\t\t\tif (index == IBM_VETH_INVALID_MAP) {\n+\t\t\t\tqstats-\u003ereplenish_add_buff_failure++;\n+\t\t\t\toutcome = IBMVETH_REPLENISH_RESET_MAP;\n+\t\t\t\tbreak;\n+\t\t\t}\n+\n+\t\t\tif (!pool-\u003eskbuff[index]) {\n+\t\t\t\tstruct sk_buff skb = NULL;\n+\n+\t\t\t\tskb = netdev_alloc_skb(adapter-\u003enetdev,\n+\t\t\t\t\t\t pool-\u003ebuff_size);\n+\t\t\t\tif (!skb) {\n+\t\t\t\t\tqstats-\u003ereplenish_no_mem++;\n+\t\t\t\t\tqstats-\u003ereplenish_add_buff_failure++;\n+\t\t\t\t\tbreak;\n+\t\t\t\t}\n+\n+\t\t\t\t/* NO_WARN: hold replenish_lock; iommu\n+\t\t\t\t * printk can re-enter via netconsole.\n+\t\t\t\t /\n+\t\t\t\tdma_addr =\n+\t\t\t\t\tdma_map_single_attrs(dev,\n+\t\t\t\t\t\t\t skb-\u003edata,\n+\t\t\t\t\t\t\t pool-\u003ebuff_size,\n+\t\t\t\t\t\t\t DMA_FROM_DEVICE,\n+\t\t\t\t\t\t\t DMA_ATTR_NO_WARN);\n+\t\t\t\tif (dma_mapping_error(dev, dma_addr)) {\n+\t\t\t\t\tdev_kfree_skb_any(skb);\n+\t\t\t\t\tqstats-\u003ereplenish_add_buff_failure++;\n+\t\t\t\t\tbreak;\n+\t\t\t\t}\n+\n+\t\t\t\tpool-\u003edma_addr[index] = dma_addr;\n+\t\t\t\tpool-\u003eskbuff[index] = skb;\n+\t\t\t} else {\n+\t\t\t\t/ re-use case /\n+\t\t\t\tdma_addr = pool-\u003edma_addr[index];\n+\t\t\t}\n+\n+\t\t\tif (rx_flush) {\n+\t\t\t\tunsigned int len;\n+\n+\t\t\t\tlen = adapter-\u003enetdev-\u003emtu + IBMVETH_BUFF_OH;\n+\t\t\t\tlen = min(pool-\u003ebuff_size, len);\n+\t\t\t\tibmveth_flush_buffer(pool-\u003eskbuff[index]-\u003edata,\n+\t\t\t\t\t\t len);\n+\t\t\t}\n+\n+\t\t\tdescs[filled].fields.flags_len = IBMVETH_BUF_VALID \|\n+\t\t\t\t\t\t\t pool-\u003ebuff_size;\n+\t\t\tdescs[filled].fields.address = dma_addr;\n+\n+\t\t\tcorrelators[filled] = ((u64)pool-\u003eindex \u003c\u003c 32) \| index;\n+\t\t\t(u64 )pool-\u003eskbuff[index]-\u003edata = correlators[filled];\n+\n+\t\t\tfree_index++;\n+\t\t\tif (free_index \u003e= pool-\u003esize)\n+\t\t\t\tfree_index = 0;\n+\t\t}\n+\n+\t\tif (outcome != IBMVETH_REPLENISH_OK)\n+\t\t\tbreak;\n+\n+\t\tif (!filled)\n+\t\t\tbreak;\n+\n+\t\tlpar_rc = ibmveth_add_logical_lan_buffers(adapter, descs,\n+\t\t\t\t\t\t\t filled,\n+\t\t\t\t\t\t\t pool-\u003ebuff_size,\n+\t\t\t\t\t\t\t queue_index);\n+\n+\t\tif (lpar_rc != H_SUCCESS) {\n+\t\t\tfail-\u003elpar_rc = lpar_rc;\n+\t\t\tfail-\u003efilled = filled;\n+\t\t\tfail-\u003ebatch = batch;\n+\t\t\tgoto hcall_failure;\n+\t\t}\n+\n+\t\t/ Only update pool state after hcall succeeds /\n+\t\tfor (i = 0; i \u003c filled; i++) {\n+\t\t\tfree_index = pool-\u003econsumer_index;\n+\t\t\tpool-\u003efree_map[free_index] = IBM_VETH_INVALID_MAP;\n+\n+\t\t\tpool-\u003econsumer_index++;\n+\t\t\tif (pool-\u003econsumer_index \u003e= pool-\u003esize)\n+\t\t\t\tpool-\u003econsumer_index = 0;\n+\t\t}\n+\n+\t\tbuffers_added += filled;\n+\t\tqstats-\u003ereplenish_add_buff_success += filled;\n+\t\tremaining -= filled;\n+\n+\t\tmemset(\u0026descs, 0, sizeof(descs));\n+\t\tmemset(\u0026correlators, 0, sizeof(correlators));\n+\t\tcontinue;\n+\n+hcall_failure:\n+\t\tfor (i = 0; i \u003c filled; i++) {\n+\t\t\tindex = correlators[i] \u0026 0xffffffffUL;\n+\t\t\tdma_addr = pool-\u003edma_addr[index];\n+\n+\t\t\tif (pool-\u003eskbuff[index]) {\n+\t\t\t\tif (dma_addr \u0026\u0026\n+\t\t\t\t !dma_mapping_error(dev, dma_addr))\n+\t\t\t\t\tdma_unmap_single(dev, dma_addr,\n+\t\t\t\t\t\t\t pool-\u003ebuff_size,\n+\t\t\t\t\t\t\t DMA_FROM_DEVICE);\n+\n+\t\t\t\tdev_kfree_skb_any(pool-\u003eskbuff[index]);\n+\t\t\t\tpool-\u003eskbuff[index] = NULL;\n+\t\t\t}\n+\t\t}\n+\t\tqstats-\u003ereplenish_add_buff_failure += filled;\n+\n+\t\tif (lpar_rc == H_FUNCTION) {\n+\t\t\tif (adapter-\u003emulti_queue) {\n+\t\t\t\t/\n+\t\t\t\t * LPM / firmware may drop MQ buffer hcalls.\n+\t\t\t\t * Schedule reset so we do not sit forever in\n+\t\t\t\t * no-buffer with the link still up.\n+\t\t\t\t /\n+\t\t\t\toutcome = IBMVETH_REPLENISH_RESET_MQ;\n+\t\t\t} else if (batch \u003e 1) {\n+\t\t\t\t/\n+\t\t\t\t * Live Partition Migration may drop multi-\n+\t\t\t\t * buffer support. Fall back to single-buffer\n+\t\t\t\t * on the next replenish; do not continue with\n+\t\t\t\t * a stale local batch size (infinite loop).\n+\t\t\t\t /\n+\t\t\t\tadapter-\u003erx_buffers_per_hcall = 1;\n+\t\t\t\toutcome = IBMVETH_REPLENISH_BATCH_FALLBACK;\n+\t\t\t} else {\n+\t\t\t\toutcome = IBMVETH_REPLENISH_HCALL_FAIL;\n+\t\t\t}\n+\t\t} else {\n+\t\t\toutcome = IBMVETH_REPLENISH_HCALL_FAIL;\n+\t\t}\n+\t\tbreak;\n+\t}\n+\n+\tmb();\n+\tatomic_add(buffers_added, \u0026(pool-\u003eavailable));\n+\treturn outcome;\n+}\n+\n+/\n+ * The final 8 bytes of the buffer list is a counter of frames dropped\n+ * because there was not a buffer in the buffer list capable of holding\n+ * the frame.\n+ /\n+static void ibmveth_update_rx_no_buffer(struct ibmveth_adapter adapter,\n+\t\t\t\t\tint queue_index)\n+{\n+\tstruct ibmveth_rx_queue_stats qstats;\n+\t__be64 p;\n+\tu64 drops;\n+\n+\tif (queue_index \u003c 0 \|\|\n+\t queue_index \u003e= ibmveth_get_num_rx_queues(adapter) \|\|\n+\t !adapter-\u003ebuffer_list_addr[queue_index])\n+\t\treturn;\n+\n+\tp = adapter-\u003ebuffer_list_addr[queue_index] + 4096 - 8;\n+\tdrops = be64_to_cpup(p);\n+\n+\t/\n+\t PHYP's buffer-list page counter is absolute for that page. A new\n+\t * page (reopen / queue reuse after -L) starts near zero; fold the\n+\t * previous absolute into this queue's retired carry so sums stay\n+\t * monotonic. Both fields belong to the queue being updated, so this\n+\t * stays single-writer under the queue's replenish_lock.\n+\t /\n+\tqstats = \u0026adapter-\u003erx_qstats[queue_index];\n+\n+\tif (drops \u003c qstats-\u003eno_buffer_drops)\n+\t\tqstats-\u003eno_buffer_retired += qstats-\u003eno_buffer_drops;\n+\tqstats-\u003eno_buffer_drops = drops;\n+}\n+\n+/ replenish routine /\n+static void ibmveth_replenish_task(struct ibmveth_adapter adapter,\n+\t\t\t\t int queue_index)\n+{\n+\tstruct ibmveth_rx_q rxq = \u0026adapter-\u003erx_queue[queue_index];\n+\tstruct ibmveth_replenish_fail fail = {};\n+\tunsigned long flags;\n+\tint i, rc;\n+\tint need_reset = 0;\n+\tint batch_fallback = 0;\n+\tint hcall_fail = 0;\n+\n+\tif (queue_index \u003e= ibmveth_get_num_rx_queues(adapter)) {\n+\t\tnetdev_dbg(adapter-\u003enetdev,\n+\t\t\t \"Skipping replenish for freed queue %d (num_queues=%u)\\n\",\n+\t\t\t queue_index, ibmveth_get_num_rx_queues(adapter));\n+\t\treturn;\n+\t}\n+\n+\tspin_lock_irqsave(\u0026rxq-\u003ereplenish_lock, flags);\n+\n+\tadapter-\u003erx_qstats[queue_index].replenish_task_cycles++;\n+\n+\tfor (i = (IBMVETH_NUM_BUFF_POOLS - 1); i \u003e= 0; i--) {\n+\t\tstruct ibmveth_buff_pool pool =\n+\t\t\t\u0026adapter-\u003erx_buff_pool[queue_index][i];\n+\n+\t\tif (pool-\u003eactive \u0026\u0026 pool-\u003efree_map \u0026\u0026\n+\t\t (atomic_read(\u0026pool-\u003eavailable) \u003c pool-\u003ethreshold)) {\n+\t\t\trc = ibmveth_replenish_buffer_pool(adapter, pool,\n+\t\t\t\t\t\t\t queue_index, \u0026fail);\n+\t\t\tswitch (rc) {\n+\t\t\tcase IBMVETH_REPLENISH_RESET_MAP:\n+\t\t\tcase IBMVETH_REPLENISH_RESET_MQ:\n+\t\t\t\tneed_reset = rc;\n+\t\t\t\tgoto out_unlock;\n+\t\t\tcase IBMVETH_REPLENISH_BATCH_FALLBACK:\n+\t\t\t\tbatch_fallback = 1;\n+\t\t\t\tbreak;\n+\t\t\tcase IBMVETH_REPLENISH_HCALL_FAIL:\n+\t\t\t\thcall_fail = 1;\n+\t\t\t\tbreak;\n+\t\t\tdefault:\n+\t\t\t\tbreak;\n+\t\t\t}\n+\t\t}\n+\t}\n+\n+out_unlock:\n+\tibmveth_update_rx_no_buffer(adapter, queue_index);\n+\n+\tspin_unlock_irqrestore(\u0026rxq-\u003ereplenish_lock, flags);\n+\n+\t/* Log and schedule reset only after dropping replenish_lock. /\n+\tif (need_reset == IBMVETH_REPLENISH_RESET_MAP) {\n+\t\tnetdev_info(adapter-\u003enetdev,\n+\t\t\t \"Invalid RX free_map entry on queue %d, reset\\n\",\n+\t\t\t queue_index);\n+\t\tschedule_work(\u0026adapter-\u003ework);\n+\t} else if (need_reset == IBMVETH_REPLENISH_RESET_MQ) {\n+\t\tdev_err_ratelimited(\u0026adapter-\u003enetdev-\u003edev,\n+\t\t\t\t \"MQ buffer add H_FUNCTION (q=%d, batch=%u), reset\\n\",\n+\t\t\t\t queue_index, fail.batch);\n+\t\tadapter-\u003emq_fallback = true;\n+\t\tschedule_work(\u0026adapter-\u003ework);\n+\t}\n+\n+\tif (batch_fallback)\n+\t\tdev_warn_ratelimited(\u0026adapter-\u003enetdev-\u003edev,\n+\t\t\t\t \"Legacy batch add H_FUNCTION (batch=%u), fallback\\n\",\n+\t\t\t\t fail.batch);\n+\n+\tif (hcall_fail)\n+\t\tdev_warn_ratelimited(\u0026adapter-\u003enetdev-\u003edev,\n+\t\t\t\t \"RX %s failed: filled=%u, rc=%lu, batch=%u\\n\",\n+\t\t\t\t adapter-\u003emulti_queue ?\n+\t\t\t\t \"h_add_logical_lan_buffers_queue\" :\n+\t\t\t\t (fail.filled == 1 ?\n+\t\t\t\t \"h_add_logical_lan_buffer\" :\n+\t\t\t\t \"h_add_logical_lan_buffers\"),\n+\t\t\t\t fail.filled, fail.lpar_rc, fail.batch);\n+}\n+\n+/\n+ ibmveth_restart_rx_queue - Post buffers and ensure Q can take RX\n+ * @adapter: ibmveth adapter\n+ * @qindex: RX queue index\n+ \n+ SQ open leaves PHYP masked until the first poll. If schedule_prep fails,\n+ * NAPI never runs and the queue stays masked (TX OK, RX/ARP dead) until\n+ * reload. Replenish first so an enable_irq fallback can actually deliver.\n+ * Also used after every open (SQ and MQ) and after scale-down rollback.\n+ /\n+static void ibmveth_restart_rx_queue(struct ibmveth_adapter adapter,\n+\t\t\t\t int qindex)\n+{\n+\tibmveth_replenish_task(adapter, qindex);\n+\tif (ibmveth_schedule_rx_queue(adapter, qindex))\n+\t\treturn;\n+\n+\tibmveth_enable_irq(adapter, qindex);\n+}\n+\n+/* empty and free ana buffer pool - also used to do cleanup in error paths /\n+static void ibmveth_free_buffer_pool(struct ibmveth_adapter adapter,\n+\t\t\t\t struct ibmveth_buff_pool pool)\n+{\n+\tint i;\n+\n+\tkfree(pool-\u003efree_map);\n+\tpool-\u003efree_map = NULL;\n+\n+\tif (pool-\u003eskbuff \u0026\u0026 pool-\u003edma_addr) {\n+\t\tfor (i = 0; i \u003c pool-\u003esize; ++i) {\n+\t\t\tstruct sk_buff skb = pool-\u003eskbuff[i];\n+\t\t\tif (skb) {\n+\t\t\t\tdma_unmap_single(\u0026adapter-\u003evdev-\u003edev,\n+\t\t\t\t\t\t pool-\u003edma_addr[i],\n+\t\t\t\t\t\t pool-\u003ebuff_size,\n+\t\t\t\t\t\t DMA_FROM_DEVICE);\n+\t\t\t\tdev_kfree_skb_any(skb);\n+\t\t\t\tpool-\u003eskbuff[i] = NULL;\n+\t\t\t}\n+\t\t}\n+\t}\n+\n+\tif (pool-\u003edma_addr) {\n+\t\tkfree(pool-\u003edma_addr);\n+\t\tpool-\u003edma_addr = NULL;\n+\t}\n+\n+\tif (pool-\u003eskbuff) {\n+\t\tkfree(pool-\u003eskbuff);\n+\t\tpool-\u003eskbuff = NULL;\n+\t}\n+\n+\t/\n+\t Keep probe/sysfs geometry (active, size, buff_size, threshold).\n+\t * Only tear down runtime allocations; open reuses active pools.\n+\t /\n+\tatomic_set(\u0026pool-\u003eavailable, 0);\n+}\n+\n+/\n+ ibmveth_free_queue_buffer_pools - Free buffer pools for a single queue\n+ * @adapter: ibmveth adapter structure\n+ * @queue: queue index\n+ \n+ Frees buffer pools that still hold allocations for the specified\n+ * queue (by free_map / dma_addr / skbuff presence), regardless of the\n+ * active flag.\n+ /\n+static void ibmveth_free_queue_buffer_pools(struct ibmveth_adapter adapter,\n+\t\t\t\t\t int queue)\n+{\n+\tint i;\n+\n+\tfor (i = 0; i \u003c IBMVETH_NUM_BUFF_POOLS; i++) {\n+\t\tstruct ibmveth_buff_pool pool =\n+\t\t\t\u0026adapter-\u003erx_buff_pool[queue][i];\n+\n+\t\t/ Free pool if it has allocated memory, regardless of\n+\t\t * active flag. Allocation and active can diverge on failure\n+\t\t * paths, so check for actual allocations.\n+\t\t /\n+\t\tif (pool-\u003efree_map \|\| pool-\u003edma_addr \|\| pool-\u003eskbuff)\n+\t\t\tibmveth_free_buffer_pool(adapter, pool);\n+\t}\n+}\n+\n+/\n+ ibmveth_alloc_queue_buffer_pools - Allocate buffer pools for a single queue\n+ * @adapter: ibmveth adapter structure\n+ * @queue: queue index\n+ \n+ Allocates backing storage for each active pool on @queue.\n+ * Inactive pools (!active) are skipped. Pool metadata must be\n+ * initialized before calling this function.\n+ \n+ Return: 0 on success, negative error code on failure\n+ /\n+static int ibmveth_alloc_queue_buffer_pools(struct ibmveth_adapter adapter,\n+\t\t\t\t\t int queue)\n+{\n+\tstruct net_device netdev = adapter-\u003enetdev;\n+\tint i;\n+\n+\tfor (i = 0; i \u003c IBMVETH_NUM_BUFF_POOLS; i++) {\n+\t\tstruct ibmveth_buff_pool bpool =\n+\t\t\t\u0026adapter-\u003erx_buff_pool[queue][i];\n+\n+\t\tif (!bpool-\u003eactive)\n+\t\t\tcontinue;\n+\n+\t\tif (ibmveth_alloc_buffer_pool(bpool)) {\n+\t\t\tnetdev_err(netdev,\n+\t\t\t\t \"queue %d pool %d alloc failed (size=%u count=%u)\\n\",\n+\t\t\t\t queue, i,\n+\t\t\t\t bpool-\u003ebuff_size,\n+\t\t\t\t bpool-\u003esize);\n+\t\t\tbpool-\u003eactive = 0;\n+\t\t\t/* Free by allocation presence, not pool-\u003eactive.\n+\t\t\t * Needed on later resize paths (a pool can still\n+\t\t\t * hold memory after active is cleared). This\n+\t\t\t * open-fail slot is already empty:\n+\t\t\t * alloc_buffer_pool() undoes its own partials.\n+\t\t\t /\n+\t\t\tibmveth_free_queue_buffer_pools(adapter, queue);\n+\t\t\treturn -ENOMEM;\n+\t\t}\n+\t}\n+\n+\treturn 0;\n+}\n+\n+/\n+ ibmveth_alloc_buffer_pools - Allocate buffer pools for all queues\n+ * @adapter: ibmveth adapter structure\n+ \n+ Initializes pool metadata for queues 1-N from queue 0 settings,\n+ * then allocates buffer pools for all queues using the helper function.\n+ \n+ Return: 0 on success, negative error code on failure\n+ /\n+static int\n+ibmveth_alloc_buffer_pools(struct ibmveth_adapter adapter)\n+{\n+\tstruct net_device netdev = adapter-\u003enetdev;\n+\tint i, q, rc;\n+\n+\t/ Initialize pool metadata for queues 1..N from queue 0 settings /\n+\tfor (q = 1; q \u003c ibmveth_get_num_rx_queues(adapter); q++) {\n+\t\tfor (i = 0; i \u003c IBMVETH_NUM_BUFF_POOLS; i++) {\n+\t\t\tstruct ibmveth_buff_pool src =\n+\t\t\t\t\u0026adapter-\u003erx_buff_pool[0][i];\n+\t\t\tstruct ibmveth_buff_pool dst =\n+\t\t\t\t\u0026adapter-\u003erx_buff_pool[q][i];\n+\n+\t\t\tdst-\u003esize = src-\u003esize;\n+\t\t\tdst-\u003eindex = src-\u003eindex;\n+\t\t\tdst-\u003ebuff_size = src-\u003ebuff_size;\n+\t\t\tdst-\u003ethreshold = src-\u003ethreshold;\n+\t\t\tdst-\u003eactive = src-\u003eactive;\n+\t\t}\n+\t}\n+\n+\t/ Allocate actual buffers for all queues /\n+\tfor (q = 0; q \u003c ibmveth_get_num_rx_queues(adapter); q++) {\n+\t\trc = ibmveth_alloc_queue_buffer_pools(adapter, q);\n+\t\tif (rc) {\n+\t\t\t/ Free pools for all previous queues /\n+\t\t\twhile (--q \u003e= 0)\n+\t\t\t\tibmveth_free_queue_buffer_pools(adapter, q);\n+\t\t\treturn rc;\n+\t\t}\n+\t}\n+\n+\tnetdev_dbg(netdev, \"allocated buffer pools for %u queue(s)\\n\",\n+\t\t ibmveth_get_num_rx_queues(adapter));\n+\treturn 0;\n+}\n+\n+/\n+ ibmveth_free_buffer_pools - Free buffer pools for all queues\n+ * @adapter: ibmveth adapter structure\n+ \n+ Frees buffer pools for all queues using the helper function.\n+ /\n+static void\n+ibmveth_free_buffer_pools(struct ibmveth_adapter adapter)\n+{\n+\tint q;\n+\n+\t/* Free buffer pools for all queues /\n+\tfor (q = 0; q \u003c ibmveth_get_num_rx_queues(adapter); q++)\n+\t\tibmveth_free_queue_buffer_pools(adapter, q);\n+\n+\tnetdev_dbg(adapter-\u003enetdev, \"freed buffer pools for %u queue(s)\\n\",\n+\t\t ibmveth_get_num_rx_queues(adapter));\n+}\n+\n+/\n+ ibmveth_alloc_single_rx_queue - Allocate resources for a single RX queue\n+ * @adapter: ibmveth adapter structure\n+ * @queue_idx: Queue index to allocate\n+ * @rxq_entries: Number of RX queue entries\n+ \n+ Allocates buffer list, RX queue, and per-queue buffer pools for one queue.\n+ * Used during incremental scale-up without affecting existing queues.\n+ \n+ Return: 0 on success, negative error code on failure\n+ /\n+static int\n+ibmveth_alloc_single_rx_queue(struct ibmveth_adapter adapter, int queue_idx,\n+\t\t\t int rxq_entries)\n+{\n+\tstruct device dev = \u0026adapter-\u003evdev-\u003edev;\n+\tstruct net_device netdev = adapter-\u003enetdev;\n+\tint i, rc = -ENOMEM;\n+\n+\tadapter-\u003ebuffer_list_addr[queue_idx] =\n+\t\t(void )get_zeroed_page(GFP_KERNEL);\n+\tif (!adapter-\u003ebuffer_list_addr[queue_idx]) {\n+\t\tnetdev_err(netdev, \"unable to allocate buffer list for queue %d\\n\",\n+\t\t\t queue_idx);\n+\t\treturn -ENOMEM;\n+\t}\n+\n+\tadapter-\u003erx_queue[queue_idx].queue_len =\n+\t\tsizeof(struct ibmveth_rx_q_entry) rxq_entries;\n+\tadapter-\u003erx_queue[queue_idx].queue_addr =\n+\t\tdma_alloc_coherent(dev, adapter-\u003erx_queue[queue_idx].queue_len,\n+\t\t\t\t \u0026adapter-\u003erx_queue[queue_idx].queue_dma,\n+\t\t\t\t GFP_KERNEL);\n+\tif (!adapter-\u003erx_queue[queue_idx].queue_addr) {\n+\t\tnetdev_err(netdev, \"unable to allocate RX queue for queue %d\\n\",\n+\t\t\t queue_idx);\n+\t\tgoto out_free_buflist;\n+\t}\n+\n+\tadapter-\u003ebuffer_list_dma[queue_idx] =\n+\t\tdma_map_single(dev, adapter-\u003ebuffer_list_addr[queue_idx],\n+\t\t\t 4096, DMA_BIDIRECTIONAL);\n+\tif (dma_mapping_error(dev, adapter-\u003ebuffer_list_dma[queue_idx])) {\n+\t\tnetdev_err(netdev, \"unable to map buffer list for queue %d\\n\",\n+\t\t\t queue_idx);\n+\t\tadapter-\u003ebuffer_list_dma[queue_idx] = 0;\n+\t\tgoto out_free_rxq;\n+\t}\n+\n+\tfor (i = 0; i \u003c IBMVETH_NUM_BUFF_POOLS; i++) {\n+\t\tstruct ibmveth_buff_pool src =\n+\t\t\t\u0026adapter-\u003erx_buff_pool[0][i];\n+\t\tstruct ibmveth_buff_pool dst =\n+\t\t\t\u0026adapter-\u003erx_buff_pool[queue_idx][i];\n+\n+\t\tdst-\u003esize = src-\u003esize;\n+\t\tdst-\u003eindex = src-\u003eindex;\n+\t\tdst-\u003ebuff_size = src-\u003ebuff_size;\n+\t\tdst-\u003ethreshold = src-\u003ethreshold;\n+\t\tdst-\u003eactive = src-\u003eactive;\n+\t}\n+\n+\trc = ibmveth_alloc_queue_buffer_pools(adapter, queue_idx);\n+\tif (rc) {\n+\t\tnetdev_err(netdev,\n+\t\t\t \"Failed to allocate buffer pools for queue %d\\n\",\n+\t\t\t queue_idx);\n+\t\tgoto out_unmap_buflist;\n+\t}\n+\n+\tadapter-\u003erx_queue[queue_idx].index = 0;\n+\tadapter-\u003erx_queue[queue_idx].num_slots = rxq_entries;\n+\tadapter-\u003erx_queue[queue_idx].toggle = 1;\n+\n+\tnetdev_dbg(netdev,\n+\t\t \"Allocated queue %d: buffer_list @ %p (DMA: 0x%llx), rx_queue @ %p (DMA: 0x%llx), %d entries\\n\",\n+\t\t queue_idx, adapter-\u003ebuffer_list_addr[queue_idx],\n+\t\t (unsigned long long)adapter-\u003ebuffer_list_dma[queue_idx],\n+\t\t adapter-\u003erx_queue[queue_idx].queue_addr,\n+\t\t (unsigned long long)adapter-\u003erx_queue[queue_idx].queue_dma,\n+\t\t rxq_entries);\n+\n+\treturn 0;\n+\n+out_unmap_buflist:\n+\tdma_unmap_single(dev, adapter-\u003ebuffer_list_dma[queue_idx],\n+\t\t\t 4096, DMA_BIDIRECTIONAL);\n+\tadapter-\u003ebuffer_list_dma[queue_idx] = 0;\n+out_free_rxq:\n+\tdma_free_coherent(dev, adapter-\u003erx_queue[queue_idx].queue_len,\n+\t\t\t adapter-\u003erx_queue[queue_idx].queue_addr,\n+\t\t\t adapter-\u003erx_queue[queue_idx].queue_dma);\n+\tadapter-\u003erx_queue[queue_idx].queue_addr = NULL;\n+out_free_buflist:\n+\tfree_page((unsigned long)adapter-\u003ebuffer_list_addr[queue_idx]);\n+\tadapter-\u003ebuffer_list_addr[queue_idx] = NULL;\n+\treturn rc;\n+}\n+\n+/*\n+ ibmveth_free_single_rx_queue - Free resources for a single RX queue\n+ * @adapter: ibmveth adapter structure\n+ * @queue_idx: Queue index to free\n+ \n+ Frees buffer list, RX queue, and per-queue buffer pools for one queue.\n+ * Used during incremental scale-down without affecting remaining queues.\n+ /\n+static void\n+ibmveth_free_single_rx_queue(struct ibmveth_adapter adapter, int queue_idx)\n+{\n+\tstruct device dev = \u0026adapter-\u003evdev-\u003edev;\n+\n+\tibmveth_free_queue_buffer_pools(adapter, queue_idx);\n+\n+\tif (adapter-\u003ebuffer_list_addr[queue_idx]) {\n+\t\tdma_unmap_single(dev, adapter-\u003ebuffer_list_dma[queue_idx],\n+\t\t\t\t 4096, DMA_BIDIRECTIONAL);\n+\t\tadapter-\u003ebuffer_list_dma[queue_idx] = 0;\n+\t}\n+\n+\tif (adapter-\u003erx_queue[queue_idx].queue_addr) {\n+\t\tdma_free_coherent(dev, adapter-\u003erx_queue[queue_idx].queue_len,\n+\t\t\t\t adapter-\u003erx_queue[queue_idx].queue_addr,\n+\t\t\t\t adapter-\u003erx_queue[queue_idx].queue_dma);\n+\t\tadapter-\u003erx_queue[queue_idx].queue_addr = NULL;\n+\t}\n+\n+\tif (adapter-\u003ebuffer_list_addr[queue_idx]) {\n+\t\tfree_page((unsigned long)adapter-\u003ebuffer_list_addr[queue_idx]);\n+\t\tadapter-\u003ebuffer_list_addr[queue_idx] = NULL;\n+\t}\n+\n+\tnetdev_dbg(adapter-\u003enetdev, \"Freed queue %d resources\\n\", queue_idx);\n+}\n+\n+static bool ibmveth_rxq_correlator_valid(struct ibmveth_adapter adapter,\n+\t\t\t\t\t int queue_index, u64 correlator)\n+{\n+\tunsigned int pool = correlator \u003e\u003e 32;\n+\tunsigned int index = correlator \u0026 0xffffffffUL;\n+\tstruct ibmveth_buff_pool bpool;\n+\n+\tif (pool \u003e= IBMVETH_NUM_BUFF_POOLS)\n+\t\treturn false;\n+\n+\tbpool = \u0026adapter-\u003erx_buff_pool[queue_index][pool];\n+\n+\t/ Require a live pool with allocated arrays before indexing.\n+\t * Inactive pools still have size from init; free clears skbuff.\n+\t /\n+\tif (!bpool-\u003eactive \|\| !bpool-\u003eskbuff \|\| !bpool-\u003efree_map)\n+\t\treturn false;\n+\n+\treturn index \u003c bpool-\u003esize;\n+}\n+\n+static void ibmveth_rxq_advance(struct ibmveth_rx_q rxq)\n+{\n+\tif (++rxq-\u003eindex == rxq-\u003enum_slots) {\n+\t\trxq-\u003eindex = 0;\n+\t\trxq-\u003etoggle = !rxq-\u003etoggle;\n+\t}\n+}\n+\n+/*\n+ ibmveth_remove_buffer_from_pool - remove a buffer from a pool\n+ * @adapter: adapter instance\n+ * @correlator: identifies pool and index\n+ * @queue_index: RX queue index (0..num_rx_queues-1)\n+ * @reuse: whether to reuse buffer\n+ \n+ Context: may run concurrently with netpoll replenish_task on the same\n+ * queue; takes per-queue replenish_lock to serialize free_map /\n+ * producer_index / available against the producer.\n+ \n+ Return:\n+ * * %0 - success\n+ * * %-EINVAL - pool or index out of range, or the pool is\n+ * inactive / not live (skbuff or free_map NULL)\n+ * * %-EFAULT - pool and index map to null skb\n+ /\n+static int ibmveth_remove_buffer_from_pool(struct ibmveth_adapter adapter,\n+\t\t\t\t\t u64 correlator, int queue_index,\n+\t\t\t\t\t bool reuse)\n+{\n+\tstruct ibmveth_rx_q rxq = \u0026adapter-\u003erx_queue[queue_index];\n+\tunsigned int pool = correlator \u003e\u003e 32;\n+\tunsigned int index = correlator \u0026 0xffffffffUL;\n+\tunsigned int free_index;\n+\tstruct sk_buff skb;\n+\tunsigned long flags;\n+\tint rc = 0;\n+\n+\tspin_lock_irqsave(\u0026rxq-\u003ereplenish_lock, flags);\n+\n+\tif (!ibmveth_rxq_correlator_valid(adapter, queue_index, correlator)) {\n+\t\trc = -EINVAL;\n+\t\tgoto out_unlock;\n+\t}\n+\n+\tskb = adapter-\u003erx_buff_pool[queue_index][pool].skbuff[index];\n+\tif (!skb) {\n+\t\trc = -EFAULT;\n+\t\tgoto out_unlock;\n+\t}\n+\n+\t/* if we are going to reuse the buffer then keep the pointers around\n+\t * but mark index as available. replenish will see the skb pointer and\n+\t * assume it is to be recycled.\n+\t /\n+\tif (!reuse) {\n+\t\t/ remove the skb pointer to mark free. actual freeing is done\n+\t\t * by upper level networking after gro_receive\n+\t\t /\n+\t\tstruct ibmveth_buff_pool bpool =\n+\t\t\t\u0026adapter-\u003erx_buff_pool[queue_index][pool];\n+\n+\t\tbpool-\u003eskbuff[index] = NULL;\n+\n+\t\tdma_unmap_single(\u0026adapter-\u003evdev-\u003edev,\n+\t\t\t\t bpool-\u003edma_addr[index],\n+\t\t\t\t bpool-\u003ebuff_size,\n+\t\t\t\t DMA_FROM_DEVICE);\n+\t}\n+\n+\tfree_index = adapter-\u003erx_buff_pool[queue_index][pool].producer_index;\n+\tadapter-\u003erx_buff_pool[queue_index][pool].producer_index++;\n+\tif (adapter-\u003erx_buff_pool[queue_index][pool].producer_index \u003e=\n+\t adapter-\u003erx_buff_pool[queue_index][pool].size)\n+\t\tadapter-\u003erx_buff_pool[queue_index][pool].producer_index = 0;\n+\tadapter-\u003erx_buff_pool[queue_index][pool].free_map[free_index] = index;\n+\n+\tmb();\n+\n+\tatomic_dec(\u0026adapter-\u003erx_buff_pool[queue_index][pool].available);\n+\n+out_unlock:\n+\tspin_unlock_irqrestore(\u0026rxq-\u003ereplenish_lock, flags);\n+\treturn rc;\n+}\n+\n+/* get the current buffer on the rx queue /\n+static struct sk_buff \n+ibmveth_rxq_get_buffer(struct ibmveth_adapter adapter,\n+\t\t int queue_index)\n+{\n+\tstruct ibmveth_rx_q rxq = \u0026adapter-\u003erx_queue[queue_index];\n+\tu64 correlator = rxq-\u003equeue_addr[rxq-\u003eindex].correlator;\n+\tunsigned int pool = correlator \u003e\u003e 32;\n+\tunsigned int index = correlator \u0026 0xffffffffUL;\n+\n+\tif (!ibmveth_rxq_correlator_valid(adapter, queue_index, correlator))\n+\t\treturn NULL;\n+\n+\treturn adapter-\u003erx_buff_pool[queue_index][pool].skbuff[index];\n+}\n+\n+/*\n+ ibmveth_rxq_harvest_buffer - Harvest buffer from pool\n+ \n+ @adapter: pointer to adapter\n+ * @queue_index: RX queue index to harvest from\n+ * @reuse: whether to reuse buffer\n+ \n+ Context: called from ibmveth_poll\n+ \n+ On a bad correlator (-EINVAL/-EFAULT) the ring is still advanced so poll\n+ * cannot spin forever on one slot. The error is still returned: callers must\n+ * not treat it as a successful take from the pool (especially reuse=false,\n+ * which would hand the SKB to the stack while it remains pool-owned).\n+ \n+ Return:\n+ * * %0 - buffer removed from pool (or marked for reuse) and ring advanced\n+ * * other - non-zero return from ibmveth_remove_buffer_from_pool; ring has\n+ * still been advanced for -EINVAL/-EFAULT\n+ /\n+static int ibmveth_rxq_harvest_buffer(struct ibmveth_adapter adapter,\n+\t\t\t\t int queue_index, bool reuse)\n+{\n+\tstruct ibmveth_rx_q rxq = \u0026adapter-\u003erx_queue[queue_index];\n+\tu64 cor;\n+\tint rc;\n+\n+\tcor = rxq-\u003equeue_addr[rxq-\u003eindex].correlator;\n+\trc = ibmveth_remove_buffer_from_pool(adapter, cor, queue_index, reuse);\n+\tif (unlikely(rc)) {\n+\t\t/ Skip a corrupt slot without claiming pool ownership. /\n+\t\tif (rc == -EINVAL \|\| rc == -EFAULT)\n+\t\t\tibmveth_rxq_advance(rxq);\n+\t\treturn rc;\n+\t}\n+\n+\tibmveth_rxq_advance(rxq);\n+\n+\treturn 0;\n+}\n+\n+/\n+ ibmveth_drain_rx_queue - Drain pending buffers from an RX queue\n+ * @adapter: ibmveth adapter structure\n+ * @queue_index: Queue index to drain\n+ \n+ Harvests pending completions back to the per-queue buffer pools.\n+ * A corrupt slot still advances the ring, so the return is slots\n+ * processed, not buffers recycled.\n+ * Must be called with NAPI disabled for this queue.\n+ \n+ Return: number of slots processed\n+ /\n+static int\n+ibmveth_drain_rx_queue(struct ibmveth_adapter adapter, int queue_index)\n+{\n+\tstruct net_device netdev = adapter-\u003enetdev;\n+\tint drained = 0;\n+\tint limit = adapter-\u003erx_queue[queue_index].num_slots;\n+\tint rc;\n+\n+\tnetdev_dbg(netdev, \"Draining RX queue %d (limit: %d slots)\\n\",\n+\t\t queue_index, limit);\n+\n+\twhile (drained \u003c limit \u0026\u0026\n+\t ibmveth_rxq_pending_buffer(adapter, queue_index)) {\n+\t\t/ Match poll-side order before harvesting completion state. /\n+\t\tsmp_rmb();\n+\t\trc = ibmveth_rxq_harvest_buffer(adapter, queue_index, true);\n+\t\tif (rc) {\n+\t\t\t/ -EINVAL/-EFAULT already advanced past the slot. /\n+\t\t\tif (rc == -EINVAL \|\| rc == -EFAULT) {\n+\t\t\t\tdrained++;\n+\t\t\t\tcontinue;\n+\t\t\t}\n+\t\t\tnetdev_err(netdev,\n+\t\t\t\t \"Failed to harvest buffer from queue %d during drain: %d\\n\",\n+\t\t\t\t queue_index, rc);\n+\t\t\tbreak;\n+\t\t}\n+\t\tdrained++;\n+\t}\n+\n+\tif (drained \u003e 0)\n+\t\tnetdev_dbg(netdev, \"Drained %d slot(s) from RX queue %d\\n\",\n+\t\t\t drained, queue_index);\n+\telse\n+\t\tnetdev_dbg(netdev, \"No slots to drain from RX queue %d\\n\",\n+\t\t\t queue_index);\n+\n+\treturn drained;\n+}\n+\n+static void ibmveth_free_tx_ltb(struct ibmveth_adapter adapter, int idx)\n+{\n+\tvoid ltb = adapter-\u003etx_ltb_ptr[idx];\n+\tdma_addr_t dma = adapter-\u003etx_ltb_dma[idx];\n+\n+\tif (!ltb)\n+\t\treturn;\n+\n+\t/\n+\t * Clear the slot before releasing it. start_xmit() tests\n+\t * tx_ltb_ptr[idx] to decide whether the LTB is usable.\n+\t /\n+\tadapter-\u003etx_ltb_ptr[idx] = NULL;\n+\tadapter-\u003etx_ltb_dma[idx] = 0;\n+\n+\tdma_unmap_single(\u0026adapter-\u003evdev-\u003edev, dma, adapter-\u003etx_ltb_size,\n+\t\t\t DMA_TO_DEVICE);\n+\tkfree(ltb);\n+}\n+\n+/ Caller must ensure tx_ltb_ptr[idx] is NULL. open() runs on\n+ * probe-zeroed slots; set_channels() skips populated indices.\n+ /\n+static int ibmveth_allocate_tx_ltb(struct ibmveth_adapter adapter, int idx)\n+{\n+\tadapter-\u003etx_ltb_ptr[idx] = kzalloc(adapter-\u003etx_ltb_size,\n+\t\t\t\t\t GFP_KERNEL);\n+\tif (!adapter-\u003etx_ltb_ptr[idx]) {\n+\t\tnetdev_err(adapter-\u003enetdev,\n+\t\t\t \"unable to allocate tx long term buffer\\n\");\n+\t\treturn -ENOMEM;\n+\t}\n+\tadapter-\u003etx_ltb_dma[idx] = dma_map_single(\u0026adapter-\u003evdev-\u003edev,\n+\t\t\t\t\t\t adapter-\u003etx_ltb_ptr[idx],\n+\t\t\t\t\t\t adapter-\u003etx_ltb_size,\n+\t\t\t\t\t\t DMA_TO_DEVICE);\n+\tif (dma_mapping_error(\u0026adapter-\u003evdev-\u003edev, adapter-\u003etx_ltb_dma[idx])) {\n+\t\tnetdev_err(adapter-\u003enetdev,\n+\t\t\t \"unable to DMA map tx long term buffer\\n\");\n+\t\tkfree(adapter-\u003etx_ltb_ptr[idx]);\n+\t\tadapter-\u003etx_ltb_ptr[idx] = NULL;\n+\t\tadapter-\u003etx_ltb_dma[idx] = 0;\n+\t\treturn -ENOMEM;\n+\t}\n+\n+\treturn 0;\n+}\n+\n+/*\n+ ibmveth_alloc_tx_resources - Allocate TX LTBs for real_num_tx_queues\n+ * @adapter: ibmveth adapter structure\n+ \n+ Allocates TX Long Term Buffers (LTBs) for real_num_tx_queues.\n+ \n+ Return: 0 on success, -ENOMEM on failure\n+ /\n+static int ibmveth_alloc_tx_resources(struct ibmveth_adapter adapter)\n+{\n+\tstruct net_device netdev = adapter-\u003enetdev;\n+\tint i;\n+\n+\tfor (i = 0; i \u003c netdev-\u003ereal_num_tx_queues; i++) {\n+\t\tif (ibmveth_allocate_tx_ltb(adapter, i))\n+\t\t\tgoto err_free_ltbs;\n+\t}\n+\n+\treturn 0;\n+\n+err_free_ltbs:\n+\twhile (--i \u003e= 0)\n+\t\tibmveth_free_tx_ltb(adapter, i);\n+\treturn -ENOMEM;\n+}\n+\n+/\n+ ibmveth_free_tx_resources - Free TX LTBs for real_num_tx_queues\n+ * @adapter: ibmveth adapter structure\n+ \n+ Frees TX Long Term Buffers (LTBs) for real_num_tx_queues.\n+ /\n+static void ibmveth_free_tx_resources(struct ibmveth_adapter adapter)\n+{\n+\tstruct net_device netdev = adapter-\u003enetdev;\n+\tint i;\n+\n+\tfor (i = 0; i \u003c netdev-\u003ereal_num_tx_queues; i++)\n+\t\tibmveth_free_tx_ltb(adapter, i);\n+}\n+\n+static int ibmveth_register_logical_lan(struct ibmveth_adapter adapter,\n+\t\t\t\t\tunion ibmveth_buf_desc rxq_desc,\n+\t\t\t\t\tu64 mac_address)\n+{\n+\tint rc, try_again = 1;\n+\tunsigned long ua = adapter-\u003evdev-\u003eunit_address;\n+\tunsigned long buf_dma = adapter-\u003ebuffer_list_dma[0];\n+\tunsigned long filter_dma = adapter-\u003efilter_list_dma;\n+\tunsigned long qh0;\n+\n+\t/\n+\t After a kexec the adapter will still be open, so our attempt to\n+\t * open it will fail. So if we get a failure we free the adapter and\n+\t * try again, but only once.\n+\t /\n+retry:\n+\t/ In multi-queue mode, obtain a queue handle for queue 0 so all RX\n+\t * queues can use the same per-queue buffer hypercalls.\n+\t /\n+\tif (adapter-\u003emulti_queue) {\n+\t\trc = h_register_logical_lan_with_handle(ua, buf_dma,\n+\t\t\t\t\t\t\trxq_desc.desc,\n+\t\t\t\t\t\t\tfilter_dma,\n+\t\t\t\t\t\t\tmac_address,\n+\t\t\t\t\t\t\t\u0026qh0);\n+\t\tif (rc == H_SUCCESS)\n+\t\t\tadapter-\u003equeue_handle[0] = qh0;\n+\t} else {\n+\t\trc = h_register_logical_lan(ua, buf_dma, rxq_desc.desc,\n+\t\t\t\t\t filter_dma, mac_address);\n+\t}\n+\tnetdev_dbg(adapter-\u003enetdev, \"h_register_logical_lan%s rc=%d\\n\",\n+\t\t adapter-\u003emulti_queue ? \"_with_handle\" : \"\", rc);\n+\n+\tif (rc != H_SUCCESS \u0026\u0026 try_again) {\n+\t\tdo {\n+\t\t\trc = h_free_logical_lan(adapter-\u003evdev-\u003eunit_address);\n+\t\t} while (H_IS_LONG_BUSY(rc) \|\| (rc == H_BUSY));\n+\t\tnetdev_dbg(adapter-\u003enetdev, \"h_free_logical_lan rc=%d\\n\", rc);\n+\n+\t\ttry_again = 0;\n+\t\tgoto retry;\n+\t}\n+\n+\treturn rc;\n+}\n+\n+/\n+ ibmveth_register_logical_lan_queue - Register subordinate queue with\n+ * hypervisor\n+ * @adapter: ibmveth adapter structure\n+ * @rxq_desc: Receive queue descriptor\n+ * @queue_index: RX queue index (1..N for subordinate queues)\n+ \n+ Registers a subordinate receive queue using H_REG_LOGICAL_LAN_QUEUE.\n+ * On success, stores the queue handle and virtual IRQ in the adapter.\n+ * If IRQ mapping fails after a successful hypervisor registration, the\n+ * queue is freed before returning. If that free fails, the handle is\n+ * kept so the caller can skip unmap.\n+ \n+ Return: H_SUCCESS on success, negative errno on IRQ mapping failure,\n+ * hypervisor error code otherwise\n+ /\n+static int\n+ibmveth_register_logical_lan_queue(struct ibmveth_adapter adapter,\n+\t\t\t\t union ibmveth_buf_desc rxq_desc,\n+\t\t\t\t int queue_index)\n+{\n+\tunsigned long handle, hwirq;\n+\tunsigned int virq;\n+\tlong lpar_rc;\n+\tunsigned long ua = adapter-\u003evdev-\u003eunit_address;\n+\tunsigned long bl = adapter-\u003ebuffer_list_dma[queue_index];\n+\n+\tnetdev_dbg(adapter-\u003enetdev,\n+\t\t \"register queue %d: ua=0x%lx bl=0x%lx rxq=0x%llx\\n\",\n+\t\t queue_index, ua, bl, rxq_desc.desc);\n+\tdo {\n+\t\tlpar_rc = h_register_logical_lan_queue(ua, bl,\n+\t\t\t\t\t\t rxq_desc.desc, \u0026handle,\n+\t\t\t\t\t\t \u0026hwirq);\n+\t} while (H_IS_LONG_BUSY(lpar_rc) \|\| (lpar_rc == H_BUSY));\n+\tnetdev_dbg(adapter-\u003enetdev,\n+\t\t \"h_register_logical_lan_queue queue %d rc=%ld\\n\",\n+\t\t queue_index, lpar_rc);\n+\n+\tif (lpar_rc == H_SUCCESS) {\n+\t\tvirq = irq_create_mapping(NULL, hwirq);\n+\t\tif (!virq) {\n+\t\t\tunsigned long free_rc;\n+\n+\t\t\tnetdev_err(adapter-\u003enetdev,\n+\t\t\t\t \"Failed to map IRQ for queue %d (hwirq=%lu)\\n\",\n+\t\t\t\t queue_index, hwirq);\n+\t\t\tdo {\n+\t\t\t\tfree_rc = h_free_logical_lan_queue(ua, handle);\n+\t\t\t} while (H_IS_LONG_BUSY(free_rc) \|\|\n+\t\t\t\t (free_rc == H_BUSY));\n+\t\t\tif (free_rc != H_SUCCESS) {\n+\t\t\t\tnetdev_err(adapter-\u003enetdev,\n+\t\t\t\t\t \"h_free_logical_lan_queue failed for queue %d after IRQ map failure: rc=0x%lx\\n\",\n+\t\t\t\t\t queue_index, free_rc);\n+\t\t\t\t/\n+\t\t\t\t PHYP still owns this queue. Keep the\n+\t\t\t\t * handle so the caller skips unmap.\n+\t\t\t\t /\n+\t\t\t\tadapter-\u003equeue_handle[queue_index] = handle;\n+\t\t\t\treturn -EIO;\n+\t\t\t}\n+\t\t\treturn -EINVAL;\n+\t\t}\n \n-\t\tif (pool-\u003eactive \u0026\u0026\n-\t\t (atomic_read(\u0026pool-\u003eavailable) \u003c pool-\u003ethreshold))\n-\t\t\tibmveth_replenish_buffer_pool(adapter, pool);\n+\t\tadapter-\u003equeue_handle[queue_index] = handle;\n+\t\tadapter-\u003equeue_irq[queue_index] = virq;\n+\n+\t\tnetdev_dbg(adapter-\u003enetdev,\n+\t\t\t \"queue %d registered: handle=0x%llx irq=%u\\n\",\n+\t\t\t queue_index, adapter-\u003equeue_handle[queue_index],\n+\t\t\t adapter-\u003equeue_irq[queue_index]);\n+\t\treturn H_SUCCESS;\n \t}\n \n-\tibmveth_update_rx_no_buffer(adapter);\n+\t/\n+\t * H_FUNCTION means firmware rejected this subordinate register\n+\t * (MQ unsupported / dropped after LPM). Caller fails this open and\n+\t * latches mq_fallback so the next open applies SQ; keep a specific\n+\t * log then the generic failure lines below.\n+\t /\n+\tif (lpar_rc == H_FUNCTION)\n+\t\tnetdev_err(adapter-\u003enetdev,\n+\t\t\t \"h_register_logical_lan_queue H_FUNCTION for queue %d (firmware MQ unsupported)\\n\",\n+\t\t\t queue_index);\n+\n+\tnetdev_err(adapter-\u003enetdev,\n+\t\t \"h_register_logical_lan_queue failed for queue %d with %ld\\n\",\n+\t\t queue_index, lpar_rc);\n+\tnetdev_err(adapter-\u003enetdev,\n+\t\t \"queue %d params: unit_addr=0x%x buffer_list_dma=0x%llx rxq_desc=0x%llx\\n\",\n+\t\t queue_index, adapter-\u003evdev-\u003eunit_address,\n+\t\t adapter-\u003ebuffer_list_dma[queue_index],\n+\t\t rxq_desc.desc);\n+\n+\treturn lpar_rc;\n }\n \n-/ empty and free ana buffer pool - also used to do cleanup in error paths /\n-static void ibmveth_free_buffer_pool(struct ibmveth_adapter adapter,\n-\t\t\t\t struct ibmveth_buff_pool pool)\n+/\n+ ibmveth_register_single_rx_queue - Register one subordinate RX queue\n+ * @adapter: ibmveth adapter structure\n+ * @queue_idx: Queue index to register (1..N)\n+ \n+ Builds the queue descriptor and registers with the hypervisor via\n+ * ibmveth_register_logical_lan_queue().\n+ \n+ Return: 0 on success, -EINVAL if @queue_idx is invalid, -EOPNOTSUPP if\n+ * firmware rejects MQ (H_FUNCTION), -EIO on other failures\n+ /\n+static int\n+ibmveth_register_single_rx_queue(struct ibmveth_adapter adapter,\n+\t\t\t\t int queue_idx)\n+{\n+\tstruct net_device netdev = adapter-\u003enetdev;\n+\tunion ibmveth_buf_desc rxq_desc;\n+\tlong lpar_rc;\n+\n+\tif (WARN_ON(queue_idx \u003c 1 \|\| queue_idx \u003e= IBMVETH_MAX_RX_QUEUES))\n+\t\treturn -EINVAL;\n+\n+\trxq_desc.fields.flags_len = IBMVETH_BUF_VALID \|\n+\t\t\t\t adapter-\u003erx_queue[queue_idx].queue_len;\n+\trxq_desc.fields.address = adapter-\u003erx_queue[queue_idx].queue_dma;\n+\n+\tlpar_rc = ibmveth_register_logical_lan_queue(adapter, rxq_desc,\n+\t\t\t\t\t\t queue_idx);\n+\tif (lpar_rc != H_SUCCESS) {\n+\t\tnetdev_err(netdev, \"Failed to register queue %d: rc=%ld\\n\",\n+\t\t\t queue_idx, lpar_rc);\n+\t\tif (lpar_rc == H_FUNCTION)\n+\t\t\treturn -EOPNOTSUPP;\n+\t\treturn -EIO;\n+\t}\n+\n+\tnetdev_dbg(netdev, \"Registered queue %d with handle 0x%llx\\n\",\n+\t\t queue_idx, adapter-\u003equeue_handle[queue_idx]);\n+\n+\treturn 0;\n+}\n+\n+/\n+ ibmveth_deregister_single_rx_queue - Deregister one subordinate RX queue\n+ * @adapter: ibmveth adapter structure\n+ * @queue_idx: Queue index to deregister (1..N)\n+ \n+ Deregisters a single queue via H_FREE_LOGICAL_LAN_QUEUE. Linux IRQ handler\n+ * teardown and subordinate virq mapping disposal are owned by interrupt\n+ * cleanup helpers; queue 0 is freed only through ibmveth_free_all_queues()\n+ * (H_FREE_LOGICAL_LAN).\n+ \n+ Return: 0 if the queue was freed or had no handle, -EIO if\n+ * H_FREE_LOGICAL_LAN_QUEUE failed. On -EIO the handle is\n+ * left set so the caller can skip unmap.\n+ /\n+static int\n+ibmveth_deregister_single_rx_queue(struct ibmveth_adapter adapter,\n+\t\t\t\t int queue_idx)\n+{\n+\tunsigned long lpar_rc;\n+\tunsigned long ua = adapter-\u003evdev-\u003eunit_address;\n+\tunsigned long qh = adapter-\u003equeue_handle[queue_idx];\n+\n+\tif (!qh)\n+\t\treturn 0;\n+\n+\tdo {\n+\t\tlpar_rc = h_free_logical_lan_queue(ua, qh);\n+\t} while (H_IS_LONG_BUSY(lpar_rc) \|\| (lpar_rc == H_BUSY));\n+\n+\tif (lpar_rc != H_SUCCESS) {\n+\t\tnetdev_err(adapter-\u003enetdev,\n+\t\t\t \"h_free_logical_lan_queue failed, queue %d rc=%ld\\n\",\n+\t\t\t queue_idx, lpar_rc);\n+\t\treturn -EIO;\n+\t}\n+\n+\tadapter-\u003equeue_handle[queue_idx] = 0;\n+\n+\tnetdev_dbg(adapter-\u003enetdev, \"Deregistered queue %d\\n\", queue_idx);\n+\treturn 0;\n+}\n+\n+/*\n+ ibmveth_destroy_subordinate_rx_queue - Tear down one subordinate RX queue\n+ * @adapter: ibmveth adapter structure\n+ * @queue_idx: Queue index to destroy (1..N)\n+ \n+ Deregister with PHYP before unmapping buffer pools so hypervisor buffer\n+ * ownership is released while queue metadata is still valid. If the\n+ * hcall fails, leave the queue mapped and keep the handle.\n+ \n+ Return: 0 on success, -EIO if deregister failed\n+ /\n+static int\n+ibmveth_destroy_subordinate_rx_queue(struct ibmveth_adapter adapter,\n+\t\t\t\t int queue_idx)\n+{\n+\tint rc;\n+\n+\trc = ibmveth_deregister_single_rx_queue(adapter, queue_idx);\n+\tif (rc)\n+\t\treturn rc;\n+\tibmveth_cleanup_single_rx_interrupt(adapter, queue_idx);\n+\tibmveth_free_single_rx_queue(adapter, queue_idx);\n+\treturn 0;\n+}\n+\n+/*\n+ ibmveth_desired_dma_for_rxqs - CMO entitlement for a given RX queue count\n+ * @adapter: ibmveth adapter\n+ * @rxqs: number of RX queues to size for\n+ \n+ Same math as ibmveth_get_desired_dma(), but uses @rxqs instead of the\n+ * live adapter-\u003enum_rx_queues. Scale-up raises desired for the target\n+ * count before allocating so vio_cmo_alloc cannot fail mid-resize.\n+ \n+ Return: bytes of IO memory desired for @rxqs RX queues\n+ /\n+static unsigned long\n+ibmveth_desired_dma_for_rxqs(struct ibmveth_adapter adapter,\n+\t\t\t unsigned int rxqs)\n+{\n+\tstruct net_device netdev = adapter-\u003enetdev;\n+\tstruct iommu_table tbl;\n+\tunsigned long ret;\n+\tint i, q;\n+\n+\ttbl = get_iommu_table_base(\u0026adapter-\u003evdev-\u003edev);\n+\n+\tret = IBMVETH_BUFF_LIST_SIZE * rxqs + IBMVETH_FILT_LIST_SIZE;\n+\tret += IOMMU_PAGE_ALIGN(netdev-\u003emtu, tbl);\n+\tret += IOMMU_PAGE_ALIGN(IBMVETH_MAX_TX_BUF_SIZE, tbl);\n+\n+\t/\n+\t Pool metadata for queues 1+ is copied from queue 0 at open.\n+\t * Always size from pool 0 x @rxqs.\n+\t \n+\t CMO (Power9 and earlier) and MQ firmware (Power11+) do not\n+\t * coexist, so a CMO partition always sizes one queue here. The\n+\t * MQ terms and the ethtool -L CMO refreshes are defensive only.\n+\t /\n+\tfor (q = 0; q \u003c rxqs; q++) {\n+\t\tint rxqentries = 1;\n+\n+\t\tfor (i = 0; i \u003c IBMVETH_NUM_BUFF_POOLS; i++) {\n+\t\t\tstruct ibmveth_buff_pool bpool =\n+\t\t\t\t\u0026adapter-\u003erx_buff_pool[0][i];\n+\n+\t\t\tif (bpool-\u003eactive)\n+\t\t\t\tret += bpool-\u003esize \n+\t\t\t\t\tIOMMU_PAGE_ALIGN(bpool-\u003ebuff_size, tbl);\n+\t\t\trxqentries += bpool-\u003esize;\n+\t\t}\n+\n+\t\tret += IOMMU_PAGE_ALIGN(rxqentries \n+\t\t\t\t\tsizeof(struct ibmveth_rx_q_entry), tbl);\n+\t}\n+\n+\treturn ret;\n+}\n+\n+/*\n+ ibmveth_free_stranded_rx_queues - Free queues stranded above the live count\n+ * @adapter: ibmveth adapter structure\n+ \n+ A scale-up register or IRQ-setup failure whose H_FREE_LOGICAL_LAN_QUEUE\n+ * also fails leaves that queue out of the live set with its handle set\n+ * and its memory still mapped. Call only after successful H_FREE_LOGICAL_LAN,\n+ * which drops every subordinate queue, so the memory can be released.\n+ /\n+static void ibmveth_free_stranded_rx_queues(struct ibmveth_adapter adapter)\n {\n \tint i;\n \n-\tkfree(pool-\u003efree_map);\n-\tpool-\u003efree_map = NULL;\n+\tfor (i = ibmveth_get_num_rx_queues(adapter);\n+\t i \u003c IBMVETH_MAX_RX_QUEUES; i++) {\n+\t\tif (!adapter-\u003equeue_handle[i])\n+\t\t\tcontinue;\n+\t\tadapter-\u003equeue_handle[i] = 0;\n+\t\tibmveth_free_single_rx_queue(adapter, i);\n+\t}\n+}\n \n-\tif (pool-\u003eskbuff \u0026\u0026 pool-\u003edma_addr) {\n-\t\tfor (i = 0; i \u003c pool-\u003esize; ++i) {\n-\t\t\tstruct sk_buff skb = pool-\u003eskbuff[i];\n-\t\t\tif (skb) {\n-\t\t\t\tdma_unmap_single(\u0026adapter-\u003evdev-\u003edev,\n-\t\t\t\t\t\t pool-\u003edma_addr[i],\n-\t\t\t\t\t\t pool-\u003ebuff_size,\n-\t\t\t\t\t\t DMA_FROM_DEVICE);\n-\t\t\t\tdev_kfree_skb_any(skb);\n-\t\t\t\tpool-\u003eskbuff[i] = NULL;\n+/ Set CMO desired entitlement from the live RX queue count. /\n+static void ibmveth_refresh_cmo_desired(struct ibmveth_adapter adapter)\n+{\n+\tif (firmware_has_feature(FW_FEATURE_CMO))\n+\t\tvio_cmo_set_dev_desired(adapter-\u003evdev,\n+\t\t\t\t\tibmveth_get_desired_dma(adapter-\u003evdev));\n+}\n+\n+/*\n+ ibmveth_scale_up_rx_queues - Add RX queues to a live adapter\n+ * @adapter: ibmveth adapter structure\n+ * @old_count: Current number of RX queues\n+ * @new_count: Target number of RX queues, above @old_count\n+ * @rxq_entries: Number of entries per RX queue\n+ \n+ Raises CMO entitlement and widens real_num_rx_queues before any new\n+ * queue is unmasked, then brings each queue up in turn. A failure\n+ * unwinds only the queues this call added; a queue PHYP refuses to\n+ * release stays in the live set instead.\n+ \n+ Return: 0 on success, negative error code on failure\n+ /\n+static int\n+ibmveth_scale_up_rx_queues(struct ibmveth_adapter adapter, int old_count,\n+\t\t\t int new_count, int rxq_entries)\n+{\n+\tstruct net_device netdev = adapter-\u003enetdev;\n+\tint failed_queue;\n+\tint rc, i;\n+\n+\tnetdev_dbg(netdev, \"Scale-up: adding queues %d-%d\\n\",\n+\t\t old_count, new_count - 1);\n+\n+\t/\n+\t * Raise CMO desired for the target count before dma_map /\n+\t * dma_alloc_coherent / replenish (same order as change_mtu).\n+\t * Do not bump live num_rx_queues here, only entitlement.\n+\t /\n+\tif (firmware_has_feature(FW_FEATURE_CMO)) {\n+\t\tunsigned long dma;\n+\n+\t\tdma = ibmveth_desired_dma_for_rxqs(adapter, new_count);\n+\t\tvio_cmo_set_dev_desired(adapter-\u003evdev, dma);\n+\t}\n+\n+\t/\n+\t * Widen real_num before any new queue is unmasked so\n+\t * skb_record_rx_queue() cannot trip get_rps_cpu()\n+\t * WARN_ONCE. Probe already allocated num_rx_queues.\n+\t /\n+\trc = netif_set_real_num_rx_queues(netdev, new_count);\n+\tif (rc) {\n+\t\tnetdev_err(netdev,\n+\t\t\t \"Failed to set real RX queues to %d: %d\\n\",\n+\t\t\t new_count, rc);\n+\t\tibmveth_refresh_cmo_desired(adapter);\n+\t\treturn rc;\n+\t}\n+\n+\tfor (i = old_count; i \u003c new_count; i++) {\n+\t\tif (adapter-\u003equeue_handle[i]) {\n+\t\t\t/ Left by a failed H_FREE; close frees it. /\n+\t\t\tnetdev_err(netdev,\n+\t\t\t\t \"RX queue %d still held by PHYP, reset pending\\n\",\n+\t\t\t\t i);\n+\t\t\trc = -EBUSY;\n+\t\t\tgoto cleanup_new_queues;\n+\t\t}\n+\t\trc = ibmveth_alloc_single_rx_queue(adapter, i,\n+\t\t\t\t\t\t rxq_entries);\n+\t\tif (rc) {\n+\t\t\tnetdev_err(netdev, \"Failed to allocate queue %d: %d\\n\",\n+\t\t\t\t i, rc);\n+\t\t\tgoto cleanup_new_queues;\n+\t\t}\n+\n+\t\trc = ibmveth_register_single_rx_queue(adapter, i);\n+\t\tif (rc) {\n+\t\t\tnetdev_err(netdev, \"Failed to register queue %d: %d\\n\",\n+\t\t\t\t i, rc);\n+\t\t\tif (rc == -EOPNOTSUPP)\n+\t\t\t\tadapter-\u003emq_fallback = true;\n+\t\t\tif (!ibmveth_deregister_single_rx_queue(adapter,\n+\t\t\t\t\t\t\t\ti))\n+\t\t\t\tibmveth_free_single_rx_queue(adapter,\n+\t\t\t\t\t\t\t i);\n+\t\t\telse\n+\t\t\t\tschedule_work(\u0026adapter-\u003ework);\n+\t\t\tgoto cleanup_new_queues;\n+\t\t}\n+\n+\t\trc = ibmveth_setup_single_rx_interrupt(adapter, i);\n+\t\tif (rc) {\n+\t\t\tnetdev_err(netdev,\n+\t\t\t\t \"Failed to setup IRQ for queue %d: %d\\n\",\n+\t\t\t\t i, rc);\n+\t\t\t/ request_irq failed: mapped but no handler /\n+\t\t\tibmveth_dispose_subordinate_irq_mapping(adapter,\n+\t\t\t\t\t\t\t\ti);\n+\t\t\tif (!ibmveth_deregister_single_rx_queue(adapter,\n+\t\t\t\t\t\t\t\ti))\n+\t\t\t\tibmveth_free_single_rx_queue(adapter,\n+\t\t\t\t\t\t\t i);\n+\t\t\telse\n+\t\t\t\tschedule_work(\u0026adapter-\u003ework);\n+\t\t\tgoto cleanup_new_queues;\n+\t\t}\n+\n+\t\t/\n+\t\t * Fully ready before PHYP delivery, matching open():\n+\t\t * publish -\u003e replenish -\u003e napi_enable -\u003e enable_irq.\n+\t\t * That way ibmveth_interrupt() cannot run on an\n+\t\t * unpublished, empty, or NAPI-disabled queue.\n+\t\t /\n+\t\tibmveth_publish_num_rx_queues(adapter, i + 1);\n+\t\tibmveth_replenish_task(adapter, i);\n+\t\tnapi_enable(\u0026adapter-\u003enapi[i]);\n+\n+\t\trc = ibmveth_enable_irq(adapter, i);\n+\t\tif (rc) {\n+\t\t\tnetdev_err(netdev,\n+\t\t\t\t \"Failed to enable IRQ for queue %d: %d\\n\",\n+\t\t\t\t i, rc);\n+\t\t\t/\n+\t\t\t * Published, replenished, and NAPI-enabled,\n+\t\t\t * but PHYP never unmasked. Match scale-down /\n+\t\t\t * shared cleanup: drain posted buffers, then\n+\t\t\t * deregister before unmap via\n+\t\t\t * destroy_subordinate.\n+\t\t\t \n+\t\t\t napi_disable() must come BEFORE the count\n+\t\t\t * is lowered, matching scale-down and\n+\t\t\t * cleanup_new_queues. Lowering it first does\n+\t\t\t * not hide queue i from netpoll: after\n+\t\t\t * ndo_poll_controller, netpoll_poll_dev()\n+\t\t\t * calls poll_napi(), which walks dev-\u003enapi_list\n+\t\t\t * unbounded by the queue count and skips a NAPI\n+\t\t\t * only once NAPI_STATE_NPSVC is set. Queue i is\n+\t\t\t * enabled here, so ibmveth_poll() would run and\n+\t\t\t * trip its queue_index \u003e= num_rx_queues\n+\t\t\t * WARN_ON. napi_disable() sets NPSVC, so\n+\t\t\t * poll_napi() skips the queue instead.\n+\t\t\t /\n+\t\t\tnapi_disable(\u0026adapter-\u003enapi[i]);\n+\t\t\t/ A poll may have unmasked PHYP; remask. /\n+\t\t\tibmveth_disable_irq(adapter, i);\n+\t\t\tsynchronize_irq(adapter-\u003equeue_irq[i]);\n+\t\t\tibmveth_publish_num_rx_queues(adapter, i);\n+\t\t\tibmveth_drain_rx_queue(adapter, i);\n+\t\t\tsynchronize_net();\n+\t\t}\n+\t\tif (rc \u0026\u0026\n+\t\t ibmveth_destroy_subordinate_rx_queue(adapter, i)) {\n+\t\t\tint keep = i + 1;\n+\n+\t\t\t/\n+\t\t\t * PHYP still owns queue i. Keep it in the\n+\t\t\t * live set with the queues this -L already\n+\t\t\t * added. cleanup_new_queues would publish\n+\t\t\t * below i and orphan this mapping.\n+\t\t\t /\n+\t\t\tibmveth_publish_num_rx_queues(adapter, keep);\n+\t\t\tif (netif_set_real_num_rx_queues(netdev,\n+\t\t\t\t\t\t\t keep))\n+\t\t\t\tschedule_work(\u0026adapter-\u003ework);\n+\t\t\tibmveth_replenish_task(adapter, i);\n+\t\t\tnapi_enable(\u0026adapter-\u003enapi[i]);\n+\t\t\trc = ibmveth_enable_irq(adapter, i);\n+\t\t\tif (rc) {\n+\t\t\t\tnetdev_err(netdev,\n+\t\t\t\t\t \"IRQ %d rc=%d\\n\",\n+\t\t\t\t\t i, rc);\n+\t\t\t\tschedule_work(\u0026adapter-\u003ework);\n+\t\t\t} else {\n+\t\t\t\tibmveth_restart_rx_queue(adapter, i);\n \t\t\t}\n+\t\t\treturn -EIO;\n+\t\t}\n+\t\tif (rc) {\n+\t\t\t/ enable_irq errno; keep -EIO. /\n+\t\t\trc = -EIO;\n+\t\t\tgoto cleanup_new_queues;\n \t\t}\n+\t\tibmveth_restart_rx_queue(adapter, i);\n \t}\n \n-\tif (pool-\u003edma_addr) {\n-\t\tkfree(pool-\u003edma_addr);\n-\t\tpool-\u003edma_addr = NULL;\n+\treturn 0;\n+\n+cleanup_new_queues:\n+\tfailed_queue = i;\n+\tif (failed_queue \u003e old_count)\n+\t\tnetdev_err(netdev,\n+\t\t\t \"Scale-up failed at queue %d, cleaning up queues %d-%d\\n\",\n+\t\t\t failed_queue, old_count, failed_queue - 1);\n+\telse\n+\t\tnetdev_err(netdev,\n+\t\t\t \"Scale-up failed at queue %d, nothing to clean up\\n\",\n+\t\t\t failed_queue);\n+\n+\tfor (i = old_count; i \u003c failed_queue; i++) {\n+\t\tibmveth_disable_irq(adapter, i);\n+\t\tsynchronize_irq(adapter-\u003equeue_irq[i]);\n \t}\n \n-\tif (pool-\u003eskbuff) {\n-\t\tkfree(pool-\u003eskbuff);\n-\t\tpool-\u003eskbuff = NULL;\n+\tfor (i = old_count; i \u003c failed_queue; i++)\n+\t\tnapi_disable(\u0026adapter-\u003enapi[i]);\n+\n+\t/ Same remask as scale-down: poll may have re-armed during disable. /\n+\tfor (i = old_count; i \u003c failed_queue; i++) {\n+\t\tibmveth_disable_irq(adapter, i);\n+\t\tsynchronize_irq(adapter-\u003equeue_irq[i]);\n+\t}\n+\n+\tfor (i = old_count; i \u003c failed_queue; i++)\n+\t\tibmveth_drain_rx_queue(adapter, i);\n+\n+\t/ Drop the live count before freeing the half-added queues. /\n+\tibmveth_publish_num_rx_queues(adapter, old_count);\n+\tif (netif_set_real_num_rx_queues(netdev, old_count))\n+\t\tschedule_work(\u0026adapter-\u003ework);\n+\tsynchronize_net();\n+\n+\tfor (i = failed_queue - 1; i \u003e= old_count; i--) {\n+\t\tint keep;\n+\n+\t\tif (!ibmveth_destroy_subordinate_rx_queue(adapter, i))\n+\t\t\tcontinue;\n+\n+\t\tkeep = i + 1;\n+\t\tibmveth_publish_num_rx_queues(adapter, keep);\n+\t\tif (netif_set_real_num_rx_queues(netdev, keep))\n+\t\t\tschedule_work(\u0026adapter-\u003ework);\n+\t\tfor (i = old_count; i \u003c keep; i++) {\n+\t\t\tint irq_rc;\n+\n+\t\t\tibmveth_replenish_task(adapter, i);\n+\t\t\t/ START: NAPI before PHYP unmask. /\n+\t\t\tnapi_enable(\u0026adapter-\u003enapi[i]);\n+\t\t\tirq_rc = ibmveth_enable_irq(adapter, i);\n+\t\t\tif (irq_rc) {\n+\t\t\t\tnetdev_err(netdev,\n+\t\t\t\t\t \"IRQ %d rc=%d\\n\",\n+\t\t\t\t\t i, irq_rc);\n+\t\t\t\tschedule_work(\u0026adapter-\u003ework);\n+\t\t\t\tcontinue;\n+\t\t\t}\n+\t\t\tibmveth_restart_rx_queue(adapter, i);\n+\t\t}\n+\t\tibmveth_refresh_cmo_desired(adapter);\n+\t\tnetdev_warn(netdev,\n+\t\t\t \"Keeping %d queues after scale-up failure\\n\",\n+\t\t\t keep);\n+\t\treturn rc;\n \t}\n+\n+\t/ Roll CMO desired back to the surviving queue count. /\n+\tibmveth_refresh_cmo_desired(adapter);\n+\n+\tnetdev_warn(netdev, \"Keeping %d queues after scale-up failure\\n\",\n+\t\t old_count);\n+\treturn rc;\n }\n \n /\n- ibmveth_remove_buffer_from_pool - remove a buffer from a pool\n- * @adapter: adapter instance\n- * @correlator: identifies pool and index\n- * @reuse: whether to reuse buffer\n+ * ibmveth_scale_down_rx_queues - Remove RX queues from a live adapter\n+ * @adapter: ibmveth adapter structure\n+ * @old_count: Current number of RX queues\n+ * @new_count: Target number of RX queues, below @old_count\n \n- Return:\n- * * %0 - success\n- * * %-EINVAL - correlator maps to pool or index out of range\n- * * %-EFAULT - pool and index map to null skb\n+ * Quiesces the retiring queues, drains them and harvests their final\n+ * no_buffer counts, publishes the surviving count, and only then\n+ * deregisters them. Walks high to low so a failed free leaves 0..i\n+ * live and republishes that count.\n+ \n+ Return: 0 on success, negative error code on failure\n /\n-static int ibmveth_remove_buffer_from_pool(struct ibmveth_adapter adapter,\n-\t\t\t\t\t u64 correlator, bool reuse)\n+static int\n+ibmveth_scale_down_rx_queues(struct ibmveth_adapter adapter, int old_count,\n+\t\t\t int new_count)\n {\n-\tunsigned int pool = correlator \u003e\u003e 32;\n-\tunsigned int index = correlator \u0026 0xffffffffUL;\n-\tunsigned int free_index;\n-\tstruct sk_buff skb;\n+\tstruct net_device netdev = adapter-\u003enetdev;\n+\tint rc, i;\n \n-\tif (WARN_ON(pool \u003e= IBMVETH_NUM_BUFF_POOLS) \|\|\n-\t WARN_ON(index \u003e= adapter-\u003erx_buff_pool[pool].size)) {\n-\t\tschedule_work(\u0026adapter-\u003ework);\n-\t\treturn -EINVAL;\n+\tnetdev_dbg(netdev, \"Scale-down: removing queues %d-%d\\n\",\n+\t\t new_count, old_count - 1);\n+\n+\t/\n+\t * Mask PHYP before napi_disable so the handler cannot miss\n+\t * a mask while NAPI is already dead. An in-flight poll can\n+\t * still re-arm PHYP while napi_disable() waits, so remask\n+\t * and sync again after NAPI is stopped. Then drain, harvest\n+\t * final no_buffer stats, publish the surviving count, and\n+\t * synchronize_net() before destroy so netpoll cannot walk\n+\t * dying queues (handler may still be registered until destroy).\n+\t /\n+\tfor (i = new_count; i \u003c old_count; i++) {\n+\t\tif (!adapter-\u003equeue_irq[i])\n+\t\t\tcontinue;\n+\t\tibmveth_disable_irq(adapter, i);\n+\t\tsynchronize_irq(adapter-\u003equeue_irq[i]);\n \t}\n \n-\tskb = adapter-\u003erx_buff_pool[pool].skbuff[index];\n-\tif (WARN_ON(!skb)) {\n-\t\tschedule_work(\u0026adapter-\u003ework);\n-\t\treturn -EFAULT;\n+\tfor (i = new_count; i \u003c old_count; i++)\n+\t\tnapi_disable(\u0026adapter-\u003enapi[i]);\n+\n+\tfor (i = new_count; i \u003c old_count; i++) {\n+\t\tif (!adapter-\u003equeue_irq[i])\n+\t\t\tcontinue;\n+\t\tibmveth_disable_irq(adapter, i);\n+\t\tsynchronize_irq(adapter-\u003equeue_irq[i]);\n \t}\n \n-\t/ if we are going to reuse the buffer then keep the pointers around\n-\t * but mark index as available. replenish will see the skb pointer and\n-\t * assume it is to be recycled.\n+\tfor (i = new_count; i \u003c old_count; i++)\n+\t\tibmveth_drain_rx_queue(adapter, i);\n+\n+\t/\n+\t netpoll still reaches these queues until the publish\n+\t * below, and ibmveth_replenish_task() folds the same\n+\t * counters under the queue's replenish_lock. Take it here\n+\t * so the harvest stays the single writer it documents.\n \t /\n-\tif (!reuse) {\n-\t\t/ remove the skb pointer to mark free. actual freeing is done\n-\t\t * by upper level networking after gro_receive\n+\tfor (i = new_count; i \u003c old_count; i++) {\n+\t\tstruct ibmveth_rx_q rxq = \u0026adapter-\u003erx_queue[i];\n+\t\tunsigned long flags;\n+\n+\t\tspin_lock_irqsave(\u0026rxq-\u003ereplenish_lock, flags);\n+\t\tibmveth_update_rx_no_buffer(adapter, i);\n+\t\tspin_unlock_irqrestore(\u0026rxq-\u003ereplenish_lock, flags);\n+\t}\n+\n+\tibmveth_publish_num_rx_queues(adapter, new_count);\n+\tsynchronize_net();\n+\n+\trc = netif_set_real_num_rx_queues(netdev, new_count);\n+\tif (rc) {\n+\t\tnetdev_err(netdev, \"Failed to set real RX queues to %d: %d\\n\",\n+\t\t\t new_count, rc);\n+\t\tibmveth_publish_num_rx_queues(adapter, old_count);\n+\t\tfor (i = new_count; i \u003c old_count; i++) {\n+\t\t\tint irq_rc;\n+\n+\t\t\tibmveth_replenish_task(adapter, i);\n+\t\t\t/ START: NAPI before PHYP unmask. /\n+\t\t\tnapi_enable(\u0026adapter-\u003enapi[i]);\n+\t\t\tirq_rc = ibmveth_enable_irq(adapter, i);\n+\t\t\tif (irq_rc) {\n+\t\t\t\tnetdev_err(netdev,\n+\t\t\t\t\t \"Failed to re-enable IRQ for queue %d during scale-down rollback (rc=%d), scheduling reset\\n\",\n+\t\t\t\t\t i, irq_rc);\n+\t\t\t\tschedule_work(\u0026adapter-\u003ework);\n+\t\t\t\tcontinue;\n+\t\t\t}\n+\t\t\tibmveth_restart_rx_queue(adapter, i);\n+\t\t}\n+\t\treturn rc;\n+\t}\n+\n+\tfor (i = old_count - 1; i \u003e= new_count; i--) {\n+\t\tint keep;\n+\n+\t\trc = ibmveth_destroy_subordinate_rx_queue(adapter, i);\n+\t\tif (!rc)\n+\t\t\tcontinue;\n+\n+\t\t/\n+\t\t * High-to-low so a failed free leaves 0..i live.\n+\t\t * Queues i+1..old_count-1 are already gone.\n \t\t /\n-\t\tadapter-\u003erx_buff_pool[pool].skbuff[index] = NULL;\n+\t\tkeep = i + 1;\n+\t\tibmveth_publish_num_rx_queues(adapter, keep);\n+\t\tif (netif_set_real_num_rx_queues(netdev, keep))\n+\t\t\tschedule_work(\u0026adapter-\u003ework);\n+\t\tfor (i = new_count; i \u003c keep; i++) {\n+\t\t\tint irq_rc;\n+\n+\t\t\tibmveth_replenish_task(adapter, i);\n+\t\t\t/ START: NAPI before PHYP unmask. /\n+\t\t\tnapi_enable(\u0026adapter-\u003enapi[i]);\n+\t\t\tirq_rc = ibmveth_enable_irq(adapter, i);\n+\t\t\tif (irq_rc) {\n+\t\t\t\tnetdev_err(netdev,\n+\t\t\t\t\t \"IRQ %d rc=%d\\n\",\n+\t\t\t\t\t i, irq_rc);\n+\t\t\t\tschedule_work(\u0026adapter-\u003ework);\n+\t\t\t\tcontinue;\n+\t\t\t}\n+\t\t\tibmveth_restart_rx_queue(adapter, i);\n+\t\t}\n+\t\treturn rc;\n+\t}\n \n-\t\tdma_unmap_single(\u0026adapter-\u003evdev-\u003edev,\n-\t\t\t\t adapter-\u003erx_buff_pool[pool].dma_addr[index],\n-\t\t\t\t adapter-\u003erx_buff_pool[pool].buff_size,\n-\t\t\t\t DMA_FROM_DEVICE);\n+\treturn 0;\n+}\n+\n+/\n+ ibmveth_resize_rx_queues_incremental - Resize RX queue count incrementally\n+ * @adapter: ibmveth adapter structure\n+ * @new_count: Target number of RX queues\n+ * @rxq_entries: Number of entries per RX queue\n+ \n+ Adds or removes RX queues without tearing down the entire adapter.\n+ * Active queues continue receiving during scale-up. Scale-up widens\n+ * real_num_rx_queues before unmasking a new queue. Scale-down drains\n+ * excess queues before deregistering them with the hypervisor.\n+ \n+ Return: 0 on success, negative error code on failure\n+ /\n+static int\n+ibmveth_resize_rx_queues_incremental(struct ibmveth_adapter adapter,\n+\t\t\t\t int new_count, int rxq_entries)\n+{\n+\tstruct net_device netdev = adapter-\u003enetdev;\n+\tint old_count = ibmveth_get_num_rx_queues(adapter);\n+\tint rc;\n+\n+\tif (old_count == new_count) {\n+\t\tnetdev_dbg(netdev, \"RX queue count unchanged (%d), nothing to do\\n\",\n+\t\t\t old_count);\n+\t\treturn 0;\n+\t}\n+\n+\tif (new_count \u003c 1 \|\| new_count \u003e IBMVETH_MAX_RX_QUEUES) {\n+\t\tnetdev_err(netdev, \"Invalid RX queue count %d (must be 1-%d)\\n\",\n+\t\t\t new_count, IBMVETH_MAX_RX_QUEUES);\n+\t\treturn -EINVAL;\n \t}\n \n-\tfree_index = adapter-\u003erx_buff_pool[pool].producer_index;\n-\tadapter-\u003erx_buff_pool[pool].producer_index++;\n-\tif (adapter-\u003erx_buff_pool[pool].producer_index \u003e=\n-\t adapter-\u003erx_buff_pool[pool].size)\n-\t\tadapter-\u003erx_buff_pool[pool].producer_index = 0;\n-\tadapter-\u003erx_buff_pool[pool].free_map[free_index] = index;\n+\tnetdev_info(netdev, \"Incrementally resizing RX queues: %d to %d\\n\",\n+\t\t old_count, new_count);\n \n-\tmb();\n+\tif (new_count \u003e old_count)\n+\t\trc = ibmveth_scale_up_rx_queues(adapter, old_count, new_count,\n+\t\t\t\t\t\trxq_entries);\n+\telse\n+\t\trc = ibmveth_scale_down_rx_queues(adapter, old_count,\n+\t\t\t\t\t\t new_count);\n+\tif (rc)\n+\t\treturn rc;\n \n-\tatomic_dec(\u0026(adapter-\u003erx_buff_pool[pool].available));\n+\tnetdev_info(netdev, \"Successfully resized to %u RX queues (incremental)\\n\",\n+\t\t ibmveth_get_num_rx_queues(adapter));\n+\n+\tibmveth_refresh_cmo_desired(adapter);\n \n \treturn 0;\n }\n \n-/ get the current buffer on the rx queue /\n-static inline struct sk_buff ibmveth_rxq_get_buffer(struct ibmveth_adapter adapter)\n+/\n+ ibmveth_free_all_queues - Free all RX queues at once\n+ * @adapter: ibmveth adapter structure\n+ \n+ Issues one H_FREE_LOGICAL_LAN for full adapter teardown. Per PAPR/PHYP,\n+ * that drops the primary LAN and any subordinate queues registered under\n+ * it. Incremental scale-down uses H_FREE_LOGICAL_LAN_QUEUE per queue\n+ * instead; do not use this helper for partial live-set shrink.\n+ \n+ Used during interface close and registration error cleanup.\n+ \n+ Retries only H_BUSY and H_IS_LONG_BUSY. On other failures, logs and\n+ * returns -EIO; queue_handle[] is cleared regardless. Callers still run\n+ * RX pool and DMA teardown afterward (same as pre-helper close()).\n+ \n+ Clears queue handles only; queue_irq[] is released by\n+ * ibmveth_cleanup_rx_interrupts() on close, or by\n+ * ibmveth_dispose_subordinate_irq_mappings() on partial register failure.\n+ \n+ Return: 0 on success, -EIO if H_FREE_LOGICAL_LAN failed\n+ /\n+static int ibmveth_free_all_queues(struct ibmveth_adapter adapter)\n {\n-\tu64 correlator = adapter-\u003erx_queue.queue_addr[adapter-\u003erx_queue.index].correlator;\n-\tunsigned int pool = correlator \u003e\u003e 32;\n-\tunsigned int index = correlator \u0026 0xffffffffUL;\n+\tunsigned long lpar_rc;\n+\tint i;\n \n-\tif (WARN_ON(pool \u003e= IBMVETH_NUM_BUFF_POOLS) \|\|\n-\t WARN_ON(index \u003e= adapter-\u003erx_buff_pool[pool].size)) {\n-\t\tschedule_work(\u0026adapter-\u003ework);\n-\t\treturn NULL;\n+\tnetdev_dbg(adapter-\u003enetdev, \"freeing all RX queues at once\\n\");\n+\n+\tdo {\n+\t\tlpar_rc = h_free_logical_lan(adapter-\u003evdev-\u003eunit_address);\n+\t} while (H_IS_LONG_BUSY(lpar_rc) \|\| (lpar_rc == H_BUSY));\n+\n+\tfor (i = 0; i \u003c ibmveth_get_num_rx_queues(adapter); i++)\n+\t\tadapter-\u003equeue_handle[i] = 0;\n+\n+\tif (lpar_rc != H_SUCCESS) {\n+\t\tnetdev_err(adapter-\u003enetdev,\n+\t\t\t \"h_free_logical_lan failed: %ld\\n\", lpar_rc);\n+\t\treturn -EIO;\n \t}\n \n-\treturn adapter-\u003erx_buff_pool[pool].skbuff[index];\n+\treturn 0;\n }\n \n /*\n- ibmveth_rxq_harvest_buffer - Harvest buffer from pool\n- \n- @adapter: pointer to adapter\n- * @reuse: whether to reuse buffer\n+ * ibmveth_register_rx_queues - Register RX queues with hypervisor\n+ * @adapter: ibmveth adapter structure\n+ * @mac_address: MAC address for device registration\n \n- Context: called from ibmveth_poll\n+ * Registers queue 0 via ibmveth_register_logical_lan(), then subordinate\n+ * queues 1..N when multi-queue mode is enabled.\n \n- Return:\n- * * %0 - success\n- * * other - non-zero return from ibmveth_remove_buffer_from_pool\n+ * Return: 0 on success, -ENONET if queue 0 registration fails,\n+ * -EOPNOTSUPP if firmware rejects a subordinate queue (H_FUNCTION;\n+ * this function latches mq_fallback), -EIO on other subordinate\n+ * failures\n /\n-static int ibmveth_rxq_harvest_buffer(struct ibmveth_adapter adapter,\n-\t\t\t\t bool reuse)\n+static int\n+ibmveth_register_rx_queues(struct ibmveth_adapter adapter, u64 mac_address)\n {\n-\tu64 cor;\n-\tint rc;\n+\tstruct net_device netdev = adapter-\u003enetdev;\n+\tunion ibmveth_buf_desc rxq_desc;\n+\tunsigned long lpar_rc;\n+\tunsigned int num;\n+\tint i, rc;\n \n-\tcor = adapter-\u003erx_queue.queue_addr[adapter-\u003erx_queue.index].correlator;\n-\trc = ibmveth_remove_buffer_from_pool(adapter, cor, reuse);\n-\tif (unlikely(rc))\n-\t\treturn rc;\n+\trxq_desc.fields.flags_len = IBMVETH_BUF_VALID \|\n+\t\t\t\t adapter-\u003erx_queue[0].queue_len;\n+\trxq_desc.fields.address = adapter-\u003erx_queue[0].queue_dma;\n+\tadapter-\u003equeue_irq[0] = netdev-\u003eirq;\n+\n+\trc = ibmveth_disable_irq(adapter, 0);\n+\tif (rc)\n+\t\tnetdev_dbg(netdev,\n+\t\t\t \"Failed to disable IRQ for queue 0 before registration, rc=%d\\n\",\n+\t\t\t rc);\n \n-\tif (++adapter-\u003erx_queue.index == adapter-\u003erx_queue.num_slots) {\n-\t\tadapter-\u003erx_queue.index = 0;\n-\t\tadapter-\u003erx_queue.toggle = !adapter-\u003erx_queue.toggle;\n+\tlpar_rc = ibmveth_register_logical_lan(adapter, rxq_desc, mac_address);\n+\tif (lpar_rc != H_SUCCESS) {\n+\t\tnetdev_err(netdev,\n+\t\t\t \"h_register_logical_lan failed: %ld\\n\", lpar_rc);\n+\t\tnetdev_err(netdev,\n+\t\t\t \"buffer TCE:0x%llx filter TCE:0x%llx rxq desc:0x%llx MAC:0x%llx\\n\",\n+\t\t\t adapter-\u003ebuffer_list_dma[0],\n+\t\t\t adapter-\u003efilter_list_dma,\n+\t\t\t rxq_desc.desc, mac_address);\n+\t\treturn -ENONET;\n \t}\n \n-\treturn 0;\n-}\n+\tnum = ibmveth_get_num_rx_queues(adapter);\n+\tif (num == 1 \|\| !adapter-\u003emulti_queue) {\n+\t\tnetdev_dbg(netdev,\n+\t\t\t \"registered 1 RX queue with hypervisor (single-queue mode)\\n\");\n+\t\treturn 0;\n+\t}\n \n-static void ibmveth_free_tx_ltb(struct ibmveth_adapter adapter, int idx)\n-{\n-\tdma_unmap_single(\u0026adapter-\u003evdev-\u003edev, adapter-\u003etx_ltb_dma[idx],\n-\t\t\t adapter-\u003etx_ltb_size, DMA_TO_DEVICE);\n-\tkfree(adapter-\u003etx_ltb_ptr[idx]);\n-\tadapter-\u003etx_ltb_ptr[idx] = NULL;\n-}\n+\tnetdev_dbg(netdev, \"Registering %u subordinate queues (1-%u)\\n\",\n+\t\t num - 1, num - 1);\n \n-static int ibmveth_allocate_tx_ltb(struct ibmveth_adapter adapter, int idx)\n-{\n-\tadapter-\u003etx_ltb_ptr[idx] = kzalloc(adapter-\u003etx_ltb_size,\n-\t\t\t\t\t GFP_KERNEL);\n-\tif (!adapter-\u003etx_ltb_ptr[idx]) {\n-\t\tnetdev_err(adapter-\u003enetdev,\n-\t\t\t \"unable to allocate tx long term buffer\\n\");\n-\t\treturn -ENOMEM;\n-\t}\n-\tadapter-\u003etx_ltb_dma[idx] = dma_map_single(\u0026adapter-\u003evdev-\u003edev,\n-\t\t\t\t\t\t adapter-\u003etx_ltb_ptr[idx],\n-\t\t\t\t\t\t adapter-\u003etx_ltb_size,\n-\t\t\t\t\t\t DMA_TO_DEVICE);\n-\tif (dma_mapping_error(\u0026adapter-\u003evdev-\u003edev, adapter-\u003etx_ltb_dma[idx])) {\n-\t\tnetdev_err(adapter-\u003enetdev,\n-\t\t\t \"unable to DMA map tx long term buffer\\n\");\n-\t\tkfree(adapter-\u003etx_ltb_ptr[idx]);\n-\t\tadapter-\u003etx_ltb_ptr[idx] = NULL;\n-\t\treturn -ENOMEM;\n+\tfor (i = 1; i \u003c num; i++) {\n+\t\trc = ibmveth_register_single_rx_queue(adapter, i);\n+\t\tif (rc) {\n+\t\t\t/* Firmware MQ gone: fall back to SQ on next open. /\n+\t\t\tif (rc == -EOPNOTSUPP)\n+\t\t\t\tadapter-\u003emq_fallback = true;\n+\t\t\tgoto err_unregister;\n+\t\t}\n \t}\n \n+\tnetdev_dbg(netdev,\n+\t\t \"registered %u RX queues with hypervisor (multi-queue mode)\\n\",\n+\t\t num);\n+\n \treturn 0;\n+\n+err_unregister:\n+\tibmveth_dispose_subordinate_irq_mappings(adapter);\n+\tibmveth_free_all_queues(adapter);\n+\treturn rc;\n }\n \n-static int ibmveth_register_logical_lan(struct ibmveth_adapter adapter,\n- union ibmveth_buf_desc rxq_desc, u64 mac_address)\n+/*\n+ ibmveth_apply_mq_fallback - Drop multi-queue mode after firmware rejection\n+ * @adapter: ibmveth adapter\n+ \n+ mq_fallback is set when firmware rejects MQ (subordinate register or\n+ * buffer-add H_FUNCTION). Apply only at the start of open after teardown so\n+ * num_rx_queues is not shrunk while IRQ/NAPI still reference higher queues.\n+ * Consumes the flag and clears multi_queue, which is what makes the\n+ * single-queue decision permanent for this device.\n+ /\n+static void ibmveth_apply_mq_fallback(struct ibmveth_adapter adapter)\n {\n-\tint rc, try_again = 1;\n+\tstruct net_device netdev = adapter-\u003enetdev;\n \n-\t/\n-\t * After a kexec the adapter will still be open, so our attempt to\n-\t * open it will fail. So if we get a failure we free the adapter and\n-\t * try again, but only once.\n-\t /\n-retry:\n-\trc = h_register_logical_lan(adapter-\u003evdev-\u003eunit_address,\n-\t\t\t\t adapter-\u003ebuffer_list_dma, rxq_desc.desc,\n-\t\t\t\t adapter-\u003efilter_list_dma, mac_address);\n+\tif (!adapter-\u003emq_fallback)\n+\t\treturn;\n \n-\tif (rc != H_SUCCESS \u0026\u0026 try_again) {\n-\t\tdo {\n-\t\t\trc = h_free_logical_lan(adapter-\u003evdev-\u003eunit_address);\n-\t\t} while (H_IS_LONG_BUSY(rc) \|\| (rc == H_BUSY));\n+\tadapter-\u003emq_fallback = false;\n \n-\t\ttry_again = 0;\n-\t\tgoto retry;\n-\t}\n+\tif (!adapter-\u003emulti_queue \u0026\u0026 ibmveth_get_num_rx_queues(adapter) == 1)\n+\t\treturn;\n \n-\treturn rc;\n+\tnetdev_warn(netdev,\n+\t\t \"Falling back to single RX queue (firmware MQ unavailable)\\n\");\n+\tadapter-\u003emulti_queue = false;\n+\tibmveth_publish_num_rx_queues(adapter, 1);\n+\t/ real_num_rx_queues is set later in open after resources exist. /\n+\tif (adapter-\u003erx_buffers_per_hcall \u003e IBMVETH_MAX_RX_REGULAR)\n+\t\tadapter-\u003erx_buffers_per_hcall = IBMVETH_MAX_RX_REGULAR;\n }\n \n static int ibmveth_open(struct net_device netdev)\n {\n \tstruct ibmveth_adapter adapter = netdev_priv(netdev);\n-\tu64 mac_address;\n+\tu64 mac_address = ether_addr_to_u64(netdev-\u003edev_addr);\n \tint rxq_entries = 1;\n-\tunsigned long lpar_rc;\n \tint rc;\n-\tunion ibmveth_buf_desc rxq_desc;\n \tint i;\n-\tstruct device dev;\n \n \tnetdev_dbg(netdev, \"open starting\\n\");\n \n-\tnapi_enable(\u0026adapter-\u003enapi);\n+\tibmveth_apply_mq_fallback(adapter);\n \n-\tfor(i = 0; i \u003c IBMVETH_NUM_BUFF_POOLS; i++)\n-\t\trxq_entries += adapter-\u003erx_buff_pool[i].size;\n+\tfor (i = 0; i \u003c IBMVETH_NUM_BUFF_POOLS; i++)\n+\t\trxq_entries += adapter-\u003erx_buff_pool[0][i].size;\n \n-\trc = -ENOMEM;\n-\tadapter-\u003ebuffer_list_addr = (void) get_zeroed_page(GFP_KERNEL);\n-\tif (!adapter-\u003ebuffer_list_addr) {\n-\t\tnetdev_err(netdev, \"unable to allocate list pages\\n\");\n+\trc = ibmveth_alloc_filter_list(adapter);\n+\tif (rc)\n \t\tgoto out;\n-\t}\n-\n-\tadapter-\u003efilter_list_addr = (void) get_zeroed_page(GFP_KERNEL);\n-\tif (!adapter-\u003efilter_list_addr) {\n-\t\tnetdev_err(netdev, \"unable to allocate filter pages\\n\");\n-\t\tgoto out_free_buffer_list;\n-\t}\n-\n-\tdev = \u0026adapter-\u003evdev-\u003edev;\n \n-\tadapter-\u003erx_queue.queue_len = sizeof(struct ibmveth_rx_q_entry) \n-\t\t\t\t\t\trxq_entries;\n-\tadapter-\u003erx_queue.queue_addr =\n-\t\tdma_alloc_coherent(dev, adapter-\u003erx_queue.queue_len,\n-\t\t\t\t \u0026adapter-\u003erx_queue.queue_dma, GFP_KERNEL);\n-\tif (!adapter-\u003erx_queue.queue_addr)\n+\trc = ibmveth_alloc_rx_queues(adapter, rxq_entries);\n+\tif (rc)\n \t\tgoto out_free_filter_list;\n \n-\tadapter-\u003ebuffer_list_dma = dma_map_single(dev,\n-\t\t\tadapter-\u003ebuffer_list_addr, 4096, DMA_BIDIRECTIONAL);\n-\tif (dma_mapping_error(dev, adapter-\u003ebuffer_list_dma)) {\n-\t\tnetdev_err(netdev, \"unable to map buffer list pages\\n\");\n+\trc = ibmveth_alloc_buffer_pools(adapter);\n+\tif (rc)\n \t\tgoto out_free_queue_mem;\n-\t}\n-\n-\tadapter-\u003efilter_list_dma = dma_map_single(dev,\n-\t\t\tadapter-\u003efilter_list_addr, 4096, DMA_BIDIRECTIONAL);\n-\tif (dma_mapping_error(dev, adapter-\u003efilter_list_dma)) {\n-\t\tnetdev_err(netdev, \"unable to map filter list pages\\n\");\n-\t\tgoto out_unmap_buffer_list;\n-\t}\n-\n-\tfor (i = 0; i \u003c netdev-\u003ereal_num_tx_queues; i++) {\n-\t\tif (ibmveth_allocate_tx_ltb(adapter, i))\n-\t\t\tgoto out_free_tx_ltb;\n-\t}\n-\n-\tadapter-\u003erx_queue.index = 0;\n-\tadapter-\u003erx_queue.num_slots = rxq_entries;\n-\tadapter-\u003erx_queue.toggle = 1;\n-\n-\tmac_address = ether_addr_to_u64(netdev-\u003edev_addr);\n \n-\trxq_desc.fields.flags_len = IBMVETH_BUF_VALID \|\n-\t\t\t\t\tadapter-\u003erx_queue.queue_len;\n-\trxq_desc.fields.address = adapter-\u003erx_queue.queue_dma;\n-\n-\tnetdev_dbg(netdev, \"buffer list @ 0x%p\\n\", adapter-\u003ebuffer_list_addr);\n-\tnetdev_dbg(netdev, \"filter list @ 0x%p\\n\", adapter-\u003efilter_list_addr);\n-\tnetdev_dbg(netdev, \"receive q @ 0x%p\\n\", adapter-\u003erx_queue.queue_addr);\n-\n-\th_vio_signal(adapter-\u003evdev-\u003eunit_address, VIO_IRQ_DISABLE);\n+\trc = ibmveth_register_rx_queues(adapter, mac_address);\n+\tif (rc)\n+\t\tgoto out_free_buffer_pools;\n \n-\tlpar_rc = ibmveth_register_logical_lan(adapter, rxq_desc, mac_address);\n+\trc = netif_set_real_num_rx_queues(netdev,\n+\t\t\t\t\t ibmveth_get_num_rx_queues(adapter));\n \n-\tif (lpar_rc != H_SUCCESS) {\n-\t\tnetdev_err(netdev, \"h_register_logical_lan failed with %ld\\n\",\n-\t\t\t lpar_rc);\n-\t\tnetdev_err(netdev, \"buffer TCE:0x%llx filter TCE:0x%llx rxq \"\n-\t\t\t \"desc:0x%llx MAC:0x%llx\\n\",\n-\t\t\t\t adapter-\u003ebuffer_list_dma,\n-\t\t\t\t adapter-\u003efilter_list_dma,\n-\t\t\t\t rxq_desc.desc,\n-\t\t\t\t mac_address);\n-\t\trc = -ENONET;\n-\t\tgoto out_unmap_filter_list;\n+\tif (rc) {\n+\t\tnetdev_err(netdev, \"failed to set number of rx queues\\n\");\n+\t\tgoto out_unregister_queues;\n \t}\n \n-\tfor (i = 0; i \u003c IBMVETH_NUM_BUFF_POOLS; i++) {\n-\t\tif (!adapter-\u003erx_buff_pool[i].active)\n-\t\t\tcontinue;\n-\t\tif (ibmveth_alloc_buffer_pool(\u0026adapter-\u003erx_buff_pool[i])) {\n-\t\t\tnetdev_err(netdev, \"unable to alloc pool\\n\");\n-\t\t\tadapter-\u003erx_buff_pool[i].active = 0;\n-\t\t\trc = -ENOMEM;\n-\t\t\tgoto out_free_buffer_pools;\n-\t\t}\n+\t/\n+\t * Post buffers before setup_rx_interrupts(). MQ setup then unmasks\n+\t * PHYP; SQ setup leaves PHYP masked. Scheduling NAPI only when a\n+\t * descriptor is already pending is not enough: after ifdown/up\n+\t * (RX=8, no -L) NAPI can be idle with nothing pending and the\n+\t * queue stays dead (TX OK, ARP/RX fail).\n+\t * restart_rx_queue() replenishes, schedules NAPI, and unmasks if\n+\t * prep fails.\n+\t /\n+\tfor (i = 0; i \u003c ibmveth_get_num_rx_queues(adapter); i++) {\n+\t\tnetdev_dbg(netdev, \"initial replenish cycle for queue %d\\n\", i);\n+\t\tibmveth_replenish_task(adapter, i);\n \t}\n \n-\tnetdev_dbg(netdev, \"registering irq 0x%x\\n\", netdev-\u003eirq);\n-\trc = request_irq(netdev-\u003eirq, ibmveth_interrupt, 0, netdev-\u003ename,\n-\t\t\t netdev);\n-\tif (rc != 0) {\n-\t\tnetdev_err(netdev, \"unable to request irq 0x%x, rc %d\\n\",\n-\t\t\t netdev-\u003eirq, rc);\n-\t\tdo {\n-\t\t\tlpar_rc = h_free_logical_lan(adapter-\u003evdev-\u003eunit_address);\n-\t\t} while (H_IS_LONG_BUSY(lpar_rc) \|\| (lpar_rc == H_BUSY));\n-\n-\t\tgoto out_free_buffer_pools;\n-\t}\n+\trc = ibmveth_setup_rx_interrupts(adapter);\n+\tif (rc)\n+\t\tgoto out_free_all_queues; / setup already disposed IRQs /\n \n-\trc = -ENOMEM;\n+\tfor (i = 0; i \u003c ibmveth_get_num_rx_queues(adapter); i++)\n+\t\tibmveth_restart_rx_queue(adapter, i);\n \n-\tnetdev_dbg(netdev, \"initial replenish cycle\\n\");\n-\tibmveth_interrupt(netdev-\u003eirq, netdev);\n+\trc = ibmveth_alloc_tx_resources(adapter);\n+\tif (rc)\n+\t\tgoto out_cleanup_rx_interrupts;\n \n \tnetif_tx_start_all_queues(netdev);\n \n+\tadapter-\u003eopened = true;\n \tnetdev_dbg(netdev, \"open complete\\n\");\n \n \treturn 0;\n \n+out_cleanup_rx_interrupts:\n+\tibmveth_cleanup_rx_interrupts(adapter);\n+\t/ As in close(): a poll past napi_complete_done() may still run. /\n+\tsynchronize_net();\n+\tgoto out_free_all_queues; / cleanup already disposed IRQs /\n+out_unregister_queues:\n+\tibmveth_dispose_subordinate_irq_mappings(adapter);\n+out_free_all_queues:\n+\tibmveth_free_all_queues(adapter);\n out_free_buffer_pools:\n-\twhile (--i \u003e= 0) {\n-\t\tif (adapter-\u003erx_buff_pool[i].active)\n-\t\t\tibmveth_free_buffer_pool(adapter,\n-\t\t\t\t\t\t \u0026adapter-\u003erx_buff_pool[i]);\n-\t}\n-out_unmap_filter_list:\n-\tdma_unmap_single(dev, adapter-\u003efilter_list_dma, 4096,\n-\t\t\t DMA_BIDIRECTIONAL);\n-\n-out_free_tx_ltb:\n-\twhile (--i \u003e= 0) {\n-\t\tibmveth_free_tx_ltb(adapter, i);\n-\t}\n-\n-out_unmap_buffer_list:\n-\tdma_unmap_single(dev, adapter-\u003ebuffer_list_dma, 4096,\n-\t\t\t DMA_BIDIRECTIONAL);\n+\tibmveth_free_buffer_pools(adapter);\n out_free_queue_mem:\n-\tdma_free_coherent(dev, adapter-\u003erx_queue.queue_len,\n-\t\t\t adapter-\u003erx_queue.queue_addr,\n-\t\t\t adapter-\u003erx_queue.queue_dma);\n+\tibmveth_cleanup_rx_resources(adapter);\n out_free_filter_list:\n-\tfree_page((unsigned long)adapter-\u003efilter_list_addr);\n-out_free_buffer_list:\n-\tfree_page((unsigned long)adapter-\u003ebuffer_list_addr);\n+\tibmveth_free_filter_list(adapter);\n out:\n-\tnapi_disable(\u0026adapter-\u003enapi);\n \treturn rc;\n }\n \n static int ibmveth_close(struct net_device netdev)\n {\n \tstruct ibmveth_adapter adapter = netdev_priv(netdev);\n-\tstruct device dev = \u0026adapter-\u003evdev-\u003edev;\n-\tlong lpar_rc;\n \tint i;\n \n-\tnetdev_dbg(netdev, \"close starting\\n\");\n-\n-\tnapi_disable(\u0026adapter-\u003enapi);\n-\n-\tnetif_tx_stop_all_queues(netdev);\n-\n-\th_vio_signal(adapter-\u003evdev-\u003eunit_address, VIO_IRQ_DISABLE);\n-\n-\tdo {\n-\t\tlpar_rc = h_free_logical_lan(adapter-\u003evdev-\u003eunit_address);\n-\t} while (H_IS_LONG_BUSY(lpar_rc) \|\| (lpar_rc == H_BUSY));\n-\n-\tif (lpar_rc != H_SUCCESS) {\n-\t\tnetdev_err(netdev, \"h_free_logical_lan failed with %lx, \"\n-\t\t\t \"continuing with close\\n\", lpar_rc);\n-\t}\n-\n-\tfree_irq(netdev-\u003eirq, netdev);\n+\t/* Gate on opened, not IFF_UP: pool_store/change_mtu close+open can\n+\t * leave IFF_UP set after a failed reopen.\n+\t /\n+\tif (!adapter-\u003eopened)\n+\t\treturn 0;\n \n-\tibmveth_update_rx_no_buffer(adapter);\n+\tadapter-\u003eopened = false;\n \n-\tdma_unmap_single(dev, adapter-\u003ebuffer_list_dma, 4096,\n-\t\t\t DMA_BIDIRECTIONAL);\n-\tfree_page((unsigned long)adapter-\u003ebuffer_list_addr);\n+\tnetdev_dbg(netdev, \"close starting\\n\");\n \n-\tdma_unmap_single(dev, adapter-\u003efilter_list_dma, 4096,\n-\t\t\t DMA_BIDIRECTIONAL);\n-\tfree_page((unsigned long)adapter-\u003efilter_list_addr);\n+\t/ Disable and wait for in-flight ndo_start_xmit (stop_all_queues\n+\t * alone does not). Direct close() callers bypass synchronize_net().\n+\t /\n+\tnetif_tx_disable(netdev);\n \n-\tdma_free_coherent(dev, adapter-\u003erx_queue.queue_len,\n-\t\t\t adapter-\u003erx_queue.queue_addr,\n-\t\t\t adapter-\u003erx_queue.queue_dma);\n+\tibmveth_cleanup_rx_interrupts(adapter);\n+\t/ Wait for softirq/poll that already passed shutdown checks. /\n+\tsynchronize_net();\n \n-\tfor (i = 0; i \u003c IBMVETH_NUM_BUFF_POOLS; i++)\n-\t\tif (adapter-\u003erx_buff_pool[i].active)\n-\t\t\tibmveth_free_buffer_pool(adapter,\n-\t\t\t\t\t\t \u0026adapter-\u003erx_buff_pool[i]);\n+\tfor (i = 0; i \u003c ibmveth_get_num_rx_queues(adapter); i++) {\n+\t\tstruct ibmveth_rx_q rxq = \u0026adapter-\u003erx_queue[i];\n+\t\tunsigned long flags;\n \n-\tfor (i = 0; i \u003c netdev-\u003ereal_num_tx_queues; i++)\n-\t\tibmveth_free_tx_ltb(adapter, i);\n+\t\tspin_lock_irqsave(\u0026rxq-\u003ereplenish_lock, flags);\n+\t\tibmveth_update_rx_no_buffer(adapter, i);\n+\t\tspin_unlock_irqrestore(\u0026rxq-\u003ereplenish_lock, flags);\n+\t}\n+\tif (!ibmveth_free_all_queues(adapter))\n+\t\tibmveth_free_stranded_rx_queues(adapter);\n+\t/* Free TX LTBs after quiesce and after H_FREE_LOGICAL_LAN so xmit\n+\t * cannot touch unmapped bounce buffers while the LAN is live.\n+\t /\n+\tibmveth_free_tx_resources(adapter);\n+\tibmveth_free_buffer_pools(adapter);\n+\tibmveth_cleanup_rx_resources(adapter);\n+\tibmveth_free_filter_list(adapter);\n \n \tnetdev_dbg(netdev, \"close complete\\n\");\n \n@@ -840,6 +3037,10 @@ static void ibmveth_reset(struct work_struct w)\n \tnetdev_dbg(netdev, \"reset starting\\n\");\n \n \trtnl_lock();\n+\tif (netdev-\u003ereg_state != NETREG_REGISTERED) {\n+\t\trtnl_unlock();\n+\t\treturn;\n+\t}\n \n \tdev_close(adapter-\u003enetdev);\n \tdev_open(adapter-\u003enetdev, NULL);\n@@ -1073,22 +3274,96 @@ static int ibmveth_set_features(struct net_device dev,\n \treturn rc1 ? rc1 : rc2;\n }\n \n-static void ibmveth_get_strings(struct net_device dev, u32 stringset, u8 data)\n+/\n+ * Sum per-queue counters for rare ethtool reads. The hot paths only ever\n+ * touch their own queue's slot, so nothing here needs an atomic; the cost\n+ * of aggregation is paid by the reader instead (ibmvnic-style).\n+ \n+ Every slot is summed, not just the live ones, so that shrinking the\n+ * queue count with ethtool -L cannot make a counter go backwards.\n+ /\n+static u64 ibmveth_sum_rx_qstat(struct ibmveth_adapter adapter, size_t off)\n+{\n+\tu64 total = 0;\n+\tint i;\n+\n+\tfor (i = 0; i \u003c IBMVETH_MAX_RX_QUEUES; i++)\n+\t\ttotal += (u64 )((u8 )\u0026adapter-\u003erx_qstats[i] + off);\n+\n+\treturn total;\n+}\n+\n+static u64 ibmveth_sum_tx_qstat(struct ibmveth_adapter adapter, size_t off)\n {\n+\tu64 total = 0;\n \tint i;\n \n+\tfor (i = 0; i \u003c IBMVETH_MAX_QUEUES; i++)\n+\t\ttotal += (u64 )((u8 )\u0026adapter-\u003etx_qstats[i] + off);\n+\n+\treturn total;\n+}\n+\n+static u64 ibmveth_ethtool_adapter_stat(struct ibmveth_adapter adapter,\n+\t\t\t\t\tint index)\n+{\n+\tconst struct ibmveth_stat stat = \u0026ibmveth_stats[index];\n+\n+\tswitch (stat-\u003esrc) {\n+\tcase IBMVETH_STAT_RX_QSUM:\n+\t\treturn ibmveth_sum_rx_qstat(adapter, stat-\u003eoff);\n+\tcase IBMVETH_STAT_TX_QSUM:\n+\t\treturn ibmveth_sum_tx_qstat(adapter, stat-\u003eoff);\n+\tcase IBMVETH_STAT_RX_NO_BUFFER:\n+\t\t/\n+\t\t * PHYP's page counter is absolute for the page currently\n+\t\t * mapped, so a reopen or queue reuse restarts it near zero.\n+\t\t * ibmveth_update_rx_no_buffer() folds each decrease into the\n+\t\t * queue's retired carry; add both back to stay monotonic.\n+\t\t /\n+\t\treturn ibmveth_sum_rx_qstat(adapter, stat-\u003eoff) +\n+\t\t ibmveth_sum_rx_qstat(adapter,\n+\t\t\t\t\t IBMVETH_RXQ_OFF(no_buffer_retired));\n+\tcase IBMVETH_STAT_ADAPTER:\n+\t\tbreak;\n+\t}\n+\n+\treturn IBMVETH_GET_STAT(adapter, stat-\u003eoff);\n+}\n+\n+static void ibmveth_get_strings(struct net_device dev, u32 stringset, u8 data)\n+{\n+\tstruct ibmveth_adapter adapter = netdev_priv(dev);\n+\tu8 p = data;\n+\tint i, j;\n+\n \tif (stringset != ETH_SS_STATS)\n \t\treturn;\n \n-\tfor (i = 0; i \u003c ARRAY_SIZE(ibmveth_stats); i++, data += ETH_GSTRING_LEN)\n-\t\tmemcpy(data, ibmveth_stats[i].name, ETH_GSTRING_LEN);\n+\tfor (i = 0; i \u003c ARRAY_SIZE(ibmveth_stats); i++) {\n+\t\tmemcpy(p, ibmveth_stats[i].name, ETH_GSTRING_LEN);\n+\t\tp += ETH_GSTRING_LEN;\n+\t}\n+\n+\tfor (i = 0; i \u003c ibmveth_get_num_rx_queues(adapter); i++)\n+\t\tfor (j = 0; j \u003c IBMVETH_NUM_RX_QSTATS; j++)\n+\t\t\tethtool_sprintf(\u0026p, ibmveth_rx_qstat_keys[j].fmt, i);\n+\n+\tfor (i = 0; i \u003c dev-\u003ereal_num_tx_queues; i++)\n+\t\tfor (j = 0; j \u003c IBMVETH_NUM_TX_QSTATS; j++)\n+\t\t\tethtool_sprintf(\u0026p, ibmveth_tx_qstat_keys[j].fmt, i);\n }\n \n static int ibmveth_get_sset_count(struct net_device dev, int sset)\n {\n+\tstruct ibmveth_adapter adapter = netdev_priv(dev);\n+\n \tswitch (sset) {\n \tcase ETH_SS_STATS:\n-\t\treturn ARRAY_SIZE(ibmveth_stats);\n+\t\treturn ARRAY_SIZE(ibmveth_stats) +\n+\t\t ibmveth_get_num_rx_queues(adapter) \n+\t\t IBMVETH_NUM_RX_QSTATS +\n+\t\t dev-\u003ereal_num_tx_queues * IBMVETH_NUM_TX_QSTATS;\n \tdefault:\n \t\treturn -EOPNOTSUPP;\n \t}\n@@ -1097,44 +3372,211 @@ static int ibmveth_get_sset_count(struct net_device dev, int sset)\n static void ibmveth_get_ethtool_stats(struct net_device dev,\n \t\t\t\t struct ethtool_stats stats, u64 data)\n {\n-\tint i;\n \tstruct ibmveth_adapter adapter = netdev_priv(dev);\n+\tint i, j, k;\n \n \tfor (i = 0; i \u003c ARRAY_SIZE(ibmveth_stats); i++)\n-\t\tdata[i] = IBMVETH_GET_STAT(adapter, ibmveth_stats[i].offset);\n+\t\tdata[i] = ibmveth_ethtool_adapter_stat(adapter, i);\n+\n+\tfor (j = 0; j \u003c ibmveth_get_num_rx_queues(adapter); j++) {\n+\t\tconst u8 q = (const u8 )\u0026adapter-\u003erx_qstats[j];\n+\n+\t\tfor (k = 0; k \u003c IBMVETH_NUM_RX_QSTATS; k++)\n+\t\t\tdata[i++] = (const u64 )\n+\t\t\t\t(q + ibmveth_rx_qstat_keys[k].off);\n+\t}\n+\n+\tfor (j = 0; j \u003c dev-\u003ereal_num_tx_queues; j++) {\n+\t\tconst u8 q = (const u8 )\u0026adapter-\u003etx_qstats[j];\n+\n+\t\tfor (k = 0; k \u003c IBMVETH_NUM_TX_QSTATS; k++)\n+\t\t\tdata[i++] = (const u64 )\n+\t\t\t\t(q + ibmveth_tx_qstat_keys[k].off);\n+\t}\n }\n \n static void ibmveth_get_channels(struct net_device netdev,\n \t\t\t\t struct ethtool_channels channels)\n {\n-\tchannels-\u003emax_tx = ibmveth_real_max_tx_queues();\n+\tstruct ibmveth_adapter adapter = netdev_priv(netdev);\n+\tunsigned int rx_count = ibmveth_get_num_rx_queues(adapter);\n+\n \tchannels-\u003etx_count = netdev-\u003ereal_num_tx_queues;\n+\t/\n+\t Never advertise max_tx below the live count. After CPU\n+\t * offline, real_max can drop below real_num_tx_queues;\n+\t * ethtool -L is read-modify-write and the core rejects\n+\t * tx_count \u003e max_tx, which would also block an RX-only\n+\t * request. set_channels uses this same ceiling.\n+\t /\n+\tchannels-\u003emax_tx = max_t(unsigned int, channels-\u003etx_count,\n+\t\t\t\t ibmveth_real_max_tx_queues());\n+\n+\t/\n+\t * Always report the live RX count. ethtool -L is read-modify-\n+\t * write, so a TX-only request echoes rx_count back at us; an\n+\t * understated value would be applied as a silent RX shrink.\n+\t * mq_fallback instead caps max_rx at the live count, which\n+\t * blocks growth in the core without misreporting what is\n+\t * currently configured.\n+\t /\n+\tchannels-\u003erx_count = rx_count;\n+\tif (adapter-\u003emulti_queue \u0026\u0026 !adapter-\u003emq_fallback)\n+\t\tchannels-\u003emax_rx = IBMVETH_MAX_RX_QUEUES;\n+\telse\n+\t\tchannels-\u003emax_rx = rx_count;\n+}\n+\n+/\n+ ibmveth_resize_rx_channels - Validate and apply a new RX queue count\n+ * @adapter: ibmveth adapter\n+ * @goal_rx: requested RX queue count\n+ \n+ Rejects rx \u003e 1 without MQ firmware (-EOPNOTSUPP) and rx outside\n+ * 1..IBMVETH_MAX_RX_QUEUES (-EINVAL). When RX resources are not live\n+ * (!opened), only validate; do not allocate. When up, apply via\n+ * ibmveth_resize_rx_queues_incremental().\n+ \n+ Return: 0 or negative errno\n+ /\n+static int ibmveth_resize_rx_channels(struct ibmveth_adapter adapter,\n+\t\t\t\t unsigned int goal_rx)\n+{\n+\tstruct net_device netdev = adapter-\u003enetdev;\n+\tunsigned int old_rx = ibmveth_get_num_rx_queues(adapter);\n+\tint rxq_entries;\n+\tint rc;\n+\n+\tif (goal_rx \u003c 1 \|\| goal_rx \u003e IBMVETH_MAX_RX_QUEUES) {\n+\t\tnetdev_err(netdev,\n+\t\t\t \"Invalid RX queue count %u (must be 1-%d)\\n\",\n+\t\t\t goal_rx, IBMVETH_MAX_RX_QUEUES);\n+\t\treturn -EINVAL;\n+\t}\n+\n+\t/\n+\t * Check for a no-op before the capability gate. ethtool -L is\n+\t * read-modify-write, so a TX-only request arrives carrying the\n+\t * current RX count; gating first would fail those with\n+\t * -EOPNOTSUPP once mq_fallback is set.\n+\t /\n+\tif (goal_rx == old_rx)\n+\t\treturn 0;\n+\n+\t/\n+\t * Refuse any rx \u003e 1, not just growth: once mq_fallback is set the\n+\t * next open comes up single-queue, so an intermediate count could\n+\t * not be honoured either, and accepting it would only repeat the\n+\t * silent clamp at open. max_rx stays at the live count so that\n+\t * read-modify-write TX-only requests still clear the core.\n+\t /\n+\tif (goal_rx \u003e 1 \u0026\u0026 (!adapter-\u003emulti_queue \|\| adapter-\u003emq_fallback)) {\n+\t\tnetdev_err(netdev,\n+\t\t\t \"Cannot resize to %u RX queues: multi-queue mode not supported by firmware\\n\",\n+\t\t\t goal_rx);\n+\t\treturn -EOPNOTSUPP;\n+\t}\n \n-\tchannels-\u003emax_rx = netdev-\u003ereal_num_rx_queues;\n-\tchannels-\u003erx_count = netdev-\u003ereal_num_rx_queues;\n+\t/ Down / failed-open: do not allocate. /\n+\tif (!adapter-\u003eopened)\n+\t\treturn 0;\n+\n+\trxq_entries = adapter-\u003erx_queue[0].num_slots;\n+\trc = ibmveth_resize_rx_queues_incremental(adapter, goal_rx,\n+\t\t\t\t\t\t rxq_entries);\n+\tif (rc)\n+\t\tnetdev_err(netdev, \"Failed to resize RX queues: %d\\n\", rc);\n+\treturn rc;\n }\n \n static int ibmveth_set_channels(struct net_device netdev,\n \t\t\t\tstruct ethtool_channels channels)\n {\n \tstruct ibmveth_adapter adapter = netdev_priv(netdev);\n-\tunsigned int old = netdev-\u003ereal_num_tx_queues,\n-\t\t goal = channels-\u003etx_count;\n+\tunsigned int old_rx = ibmveth_get_num_rx_queues(adapter);\n+\tunsigned int goal_rx = channels-\u003erx_count;\n+\tunsigned int old_tx = netdev-\u003ereal_num_tx_queues;\n+\tunsigned int goal_tx = channels-\u003etx_count;\n+\tunsigned int want_tx = goal_tx;\n+\tunsigned int max_tx;\n+\tbool rx_changed = false;\n \tint rc, i;\n \n-\t/* If ndo_open has not been called yet then don't allocate, just set\n-\t * desired netdev_queue's and return\n+\t/\n+\t Same ceiling get_channels reports: live count or the\n+\t * online-CPU cap, whichever is larger.\n+\t /\n+\tmax_tx = max_t(unsigned int, old_tx,\n+\t\t ibmveth_real_max_tx_queues());\n+\tif (goal_tx \u003c 1 \|\| goal_tx \u003e max_tx) {\n+\t\tnetdev_err(netdev,\n+\t\t\t \"Invalid TX queue count %u (must be 1-%u)\\n\",\n+\t\t\t goal_tx, max_tx);\n+\t\treturn -EINVAL;\n+\t}\n+\n+\t/ RX range / MQ checks live in ibmveth_resize_rx_channels(). /\n+\trc = ibmveth_resize_rx_channels(adapter, goal_rx);\n+\tif (rc)\n+\t\treturn rc;\n+\n+\t/ If RX resources are not live (never opened, or close+open failed\n+\t * while IFF_UP stayed set), publish desired queue counts without\n+\t * allocating.\n \t /\n-\tif (!(netdev-\u003eflags \u0026 IFF_UP))\n-\t\treturn netif_set_real_num_tx_queues(netdev, goal);\n+\tif (!adapter-\u003eopened) {\n+\t\t/ Apply TX first so a failure leaves the published RX\n+\t\t * count unchanged.\n+\t\t /\n+\t\trc = netif_set_real_num_tx_queues(netdev, goal_tx);\n+\t\tif (rc)\n+\t\t\treturn rc;\n+\n+\t\t/ Publish desired RX count for next open() and refresh CMO;\n+\t\t * do not allocate while down.\n+\t\t /\n+\t\tif (goal_rx != ibmveth_get_num_rx_queues(adapter)) {\n+\t\t\tibmveth_publish_num_rx_queues(adapter, goal_rx);\n+\t\t\trc = netif_set_real_num_rx_queues(netdev, goal_rx);\n+\t\t\tif (rc) {\n+\t\t\t\tint tx_rc;\n+\n+\t\t\t\tibmveth_publish_num_rx_queues(adapter, old_rx);\n+\t\t\t\ttx_rc = netif_set_real_num_tx_queues(netdev,\n+\t\t\t\t\t\t\t\t old_tx);\n+\t\t\t\tif (tx_rc)\n+\t\t\t\t\tnetdev_err(netdev,\n+\t\t\t\t\t\t \"Failed to restore TX queues to %u after RX failure: %d\\n\",\n+\t\t\t\t\t\t old_tx, tx_rc);\n+\t\t\t\treturn rc;\n+\t\t\t}\n+\t\t\tif (firmware_has_feature(FW_FEATURE_CMO)) {\n+\t\t\t\tunsigned long dma;\n+\n+\t\t\t\tdma = ibmveth_get_desired_dma(adapter-\u003evdev);\n+\t\t\t\tvio_cmo_set_dev_desired(adapter-\u003evdev, dma);\n+\t\t\t}\n+\t\t}\n+\t\treturn 0;\n+\t}\n+\n+\tif (goal_rx != old_rx)\n+\t\trx_changed = true;\n \n \t/ We have IBMVETH_MAX_QUEUES netdev_queue's allocated\n \t * but we may need to alloc/free the ltb's.\n \t /\n+\tif (goal_tx == old_tx)\n+\t\treturn 0;\n+\n \tnetif_tx_stop_all_queues(netdev);\n \n-\t/ Allocate any queue that we need /\n-\tfor (i = old; i \u003c goal; i++) {\n+\t/ Allocate any new TX LTBs. i starts at old_tx for the free walk\n+\t * below when this loop body never runs (goal_tx == old_tx already\n+\t * returned; goal_tx \u003c old_tx is scale-down).\n+\t /\n+\ti = old_tx;\n+\tfor (; i \u003c goal_tx; i++) {\n \t\tif (adapter-\u003etx_ltb_ptr[i])\n \t\t\tcontinue;\n \n@@ -1143,28 +3585,48 @@ static int ibmveth_set_channels(struct net_device netdev,\n \t\t\tcontinue;\n \n \t\t/* if something goes wrong, free everything we just allocated /\n-\t\tnetdev_err(netdev, \"Failed to allocate more tx queues, returning to %d queues\\n\",\n-\t\t\t old);\n-\t\tgoal = old;\n-\t\told = i;\n+\t\tnetdev_err(netdev, \"Failed to allocate more tx queues, returning to %u queues\\n\",\n+\t\t\t old_tx);\n+\t\tgoal_tx = old_tx;\n+\t\told_tx = i;\n \t\tbreak;\n \t}\n-\trc = netif_set_real_num_tx_queues(netdev, goal);\n+\trc = netif_set_real_num_tx_queues(netdev, goal_tx);\n \tif (rc) {\n-\t\tnetdev_err(netdev, \"Failed to set real tx queues, returning to %d queues\\n\",\n-\t\t\t old);\n-\t\tgoal = old;\n-\t\told = i;\n+\t\tnetdev_err(netdev, \"Failed to set real tx queues, returning to %u queues\\n\",\n+\t\t\t old_tx);\n+\t\tgoal_tx = old_tx;\n+\t\told_tx = i;\n \t}\n \t/ Free any that are no longer needed /\n-\tfor (i = old; i \u003e goal; i--) {\n+\tfor (i = old_tx; i \u003e goal_tx; i--) {\n \t\tif (adapter-\u003etx_ltb_ptr[i - 1])\n \t\t\tibmveth_free_tx_ltb(adapter, i - 1);\n \t}\n \n \tnetif_tx_wake_all_queues(netdev);\n \n-\treturn rc;\n+\tif (netdev-\u003ereal_num_tx_queues != want_tx) {\n+\t\tif (rx_changed) {\n+\t\t\t/\n+\t\t\t * Restore the RX count from before this -L.\n+\t\t\t * num_slots is the live size after that resize.\n+\t\t\t /\n+\t\t\tint rxq_entries = adapter-\u003erx_queue[0].num_slots;\n+\t\t\tint rb;\n+\n+\t\t\trb = ibmveth_resize_rx_queues_incremental(adapter,\n+\t\t\t\t\t\t\t\t old_rx,\n+\t\t\t\t\t\t\t\t rxq_entries);\n+\t\t\tif (rb)\n+\t\t\t\tnetdev_err(netdev,\n+\t\t\t\t\t \"Failed to roll back RX queues to %u after TX failure: %d\\n\",\n+\t\t\t\t\t old_rx, rb);\n+\t\t}\n+\t\treturn rc ? rc : -ENOMEM;\n+\t}\n+\n+\treturn 0;\n }\n \n static const struct ethtool_ops netdev_ethtool_ops = {\n@@ -1213,8 +3675,10 @@ static int ibmveth_send(struct ibmveth_adapter adapter,\n }\n \n static int ibmveth_is_packet_unsupported(struct sk_buff skb,\n-\t\t\t\t\t struct net_device netdev)\n+\t\t\t\t\t struct ibmveth_adapter adapter,\n+\t\t\t\t\t int queue_num)\n {\n+\tstruct net_device netdev = adapter-\u003enetdev;\n \tstruct ethhdr ether_header;\n \tint ret = 0;\n \n@@ -1222,7 +3686,7 @@ static int ibmveth_is_packet_unsupported(struct sk_buff skb,\n \n \tif (ether_addr_equal(ether_header-\u003eh_dest, netdev-\u003edev_addr)) {\n \t\tnetdev_dbg(netdev, \"veth doesn't support loopback packets, dropping packet.\\n\");\n-\t\tnetdev-\u003estats.tx_dropped++;\n+\t\tadapter-\u003etx_qstats[queue_num].dropped_packets++;\n \t\tret = -EOPNOTSUPP;\n \t}\n \n@@ -1238,7 +3702,13 @@ static netdev_tx_t ibmveth_start_xmit(struct sk_buff skb,\n \tint i, queue_num = skb_get_queue_mapping(skb);\n \tunsigned long mss = 0;\n \n-\tif (ibmveth_is_packet_unsupported(skb, netdev))\n+\t/ Close / failed reopen can free LTBs while IFF_UP is still set. /\n+\tif (unlikely(!adapter-\u003etx_ltb_ptr[queue_num])) {\n+\t\tadapter-\u003etx_qstats[queue_num].dropped_packets++;\n+\t\tgoto out;\n+\t}\n+\n+\tif (ibmveth_is_packet_unsupported(skb, adapter, queue_num))\n \t\tgoto out;\n \t/ veth can't checksum offload UDP /\n \tif (skb-\u003eip_summed == CHECKSUM_PARTIAL \u0026\u0026\n@@ -1249,7 +3719,7 @@ static netdev_tx_t ibmveth_start_xmit(struct sk_buff skb,\n \t skb_checksum_help(skb)) {\n \n \t\tnetdev_err(netdev, \"tx: failed to checksum packet\\n\");\n-\t\tnetdev-\u003estats.tx_dropped++;\n+\t\tadapter-\u003etx_qstats[queue_num].dropped_packets++;\n \t\tgoto out;\n \t}\n \n@@ -1261,6 +3731,8 @@ static netdev_tx_t ibmveth_start_xmit(struct sk_buff skb,\n \n \t\tdesc_flags \|= (IBMVETH_BUF_NO_CSUM \| IBMVETH_BUF_CSUM_GOOD);\n \n+\t\tadapter-\u003etx_qstats[queue_num].checksum_offload++;\n+\n \t\t/ Need to zero out the checksum /\n \t\tbuf[0] = 0;\n \t\tbuf[1] = 0;\n@@ -1272,7 +3744,7 @@ static netdev_tx_t ibmveth_start_xmit(struct sk_buff skb,\n \tif (skb-\u003eip_summed == CHECKSUM_PARTIAL \u0026\u0026 skb_is_gso(skb)) {\n \t\tif (adapter-\u003efw_large_send_support) {\n \t\t\tmss = (unsigned long)skb_shinfo(skb)-\u003egso_size;\n-\t\t\tadapter-\u003etx_large_packets++;\n+\t\t\tadapter-\u003etx_qstats[queue_num].large_packets++;\n \t\t} else if (!skb_is_gso_v6(skb)) {\n \t\t\t/* Put -1 in the IP checksum to tell phyp it\n \t\t\t * is a largesend packet. Put the mss in\n@@ -1281,7 +3753,7 @@ static netdev_tx_t ibmveth_start_xmit(struct sk_buff skb,\n \t\t\tip_hdr(skb)-\u003echeck = 0xffff;\n \t\t\ttcp_hdr(skb)-\u003echeck =\n \t\t\t\tcpu_to_be16(skb_shinfo(skb)-\u003egso_size);\n-\t\t\tadapter-\u003etx_large_packets++;\n+\t\t\tadapter-\u003etx_qstats[queue_num].large_packets++;\n \t\t}\n \t}\n \n@@ -1289,7 +3761,7 @@ static netdev_tx_t ibmveth_start_xmit(struct sk_buff skb,\n \tif (unlikely(skb-\u003elen \u003e adapter-\u003etx_ltb_size)) {\n \t\tnetdev_err(adapter-\u003enetdev, \"tx: packet size (%u) exceeds ltb (%u)\\n\",\n \t\t\t skb-\u003elen, adapter-\u003etx_ltb_size);\n-\t\tnetdev-\u003estats.tx_dropped++;\n+\t\tadapter-\u003etx_qstats[queue_num].dropped_packets++;\n \t\tgoto out;\n \t}\n \tmemcpy(adapter-\u003etx_ltb_ptr[queue_num], skb-\u003edata, skb_headlen(skb));\n@@ -1306,7 +3778,7 @@ static netdev_tx_t ibmveth_start_xmit(struct sk_buff skb,\n \tif (unlikely(total_bytes != skb-\u003elen)) {\n \t\tnetdev_err(adapter-\u003enetdev, \"tx: incorrect packet len copied into ltb (%u != %u)\\n\",\n \t\t\t skb-\u003elen, total_bytes);\n-\t\tnetdev-\u003estats.tx_dropped++;\n+\t\tadapter-\u003etx_qstats[queue_num].dropped_packets++;\n \t\tgoto out;\n \t}\n \tdesc.fields.flags_len = desc_flags \| skb-\u003elen;\n@@ -1315,11 +3787,11 @@ static netdev_tx_t ibmveth_start_xmit(struct sk_buff skb,\n \tdma_wmb();\n \n \tif (ibmveth_send(adapter, desc.desc, mss)) {\n-\t\tadapter-\u003etx_send_failed++;\n-\t\tnetdev-\u003estats.tx_dropped++;\n+\t\tadapter-\u003etx_qstats[queue_num].send_failures++;\n+\t\tadapter-\u003etx_qstats[queue_num].dropped_packets++;\n \t} else {\n-\t\tnetdev-\u003estats.tx_packets++;\n-\t\tnetdev-\u003estats.tx_bytes += skb-\u003elen;\n+\t\tadapter-\u003etx_qstats[queue_num].packets++;\n+\t\tadapter-\u003etx_qstats[queue_num].bytes += skb-\u003elen;\n \t}\n \n out:\n@@ -1445,102 +3917,226 @@ static void ibmveth_rx_csum_helper(struct sk_buff skb,\n \t}\n }\n \n-static int ibmveth_poll(struct napi_struct napi, int budget)\n+static void ibmveth_poll_bump_invalid(struct ibmveth_adapter adapter,\n+\t\t\t\t int queue_index)\n {\n-\tstruct ibmveth_adapter adapter =\n-\t\t\tcontainer_of(napi, struct ibmveth_adapter, napi);\n-\tstruct net_device netdev = adapter-\u003enetdev;\n-\tint frames_processed = 0;\n-\tunsigned long lpar_rc;\n+\tadapter-\u003erx_qstats[queue_index].invalid_buffers++;\n+}\n+\n+static bool ibmveth_poll_stopping(struct net_device netdev,\n+\t\t\t\t struct napi_struct napi)\n+{\n+\treturn !netif_running(netdev) \|\| napi_disable_pending(napi);\n+}\n+\n+static bool ibmveth_poll_harvest_slot(struct ibmveth_adapter adapter,\n+\t\t\t\t int queue_index, bool reuse)\n+{\n+\tint rc = ibmveth_rxq_harvest_buffer(adapter, queue_index, reuse);\n+\n+\treturn !rc \|\| rc == -EINVAL \|\| rc == -EFAULT;\n+}\n+\n+static bool ibmveth_poll_recycle_invalid(struct net_device netdev,\n+\t\t\t\t\t struct ibmveth_adapter adapter,\n+\t\t\t\t\t int queue_index)\n+{\n+\tnetdev_dbg(netdev, \"recycling invalid buffer\\n\");\n+\tibmveth_poll_bump_invalid(adapter, queue_index);\n+\treturn ibmveth_poll_harvest_slot(adapter, queue_index, true);\n+}\n+\n+static bool ibmveth_poll_skip_bad_correlator(struct net_device netdev,\n+\t\t\t\t\t struct ibmveth_adapter adapter,\n+\t\t\t\t\t int queue_index)\n+{\n+\tif (net_ratelimit())\n+\t\tnetdev_err(netdev,\n+\t\t\t \"bad correlator on queue %d, skipping slot\\n\",\n+\t\t\t queue_index);\n+\t/* Residual stale slot after resize: recover via reset rather\n+\t * than spinning forever. Always escalate; only the log is\n+\t * rate-limited.\n+\t /\n+\tschedule_work(\u0026adapter-\u003ework);\n+\tibmveth_poll_bump_invalid(adapter, queue_index);\n+\treturn ibmveth_poll_harvest_slot(adapter, queue_index, true);\n+}\n+\n+static bool ibmveth_poll_drop_oversize(struct net_device netdev,\n+\t\t\t\t struct ibmveth_adapter adapter,\n+\t\t\t\t int queue_index, unsigned int off,\n+\t\t\t\t unsigned int len, unsigned int room)\n+{\n+\tif (net_ratelimit())\n+\t\tnetdev_err(netdev,\n+\t\t\t \"RX frame %u+%u exceeds buffer %u on queue %d, dropping\\n\",\n+\t\t\t off, len, room, queue_index);\n+\tibmveth_poll_bump_invalid(adapter, queue_index);\n+\treturn ibmveth_poll_harvest_slot(adapter, queue_index, true);\n+}\n+\n+/\n+ ibmveth_poll_deliver_frame - Build SKB from one valid RX slot and GRO it\n+ * @napi: NAPI context for this RX queue\n+ * @adapter: ibmveth adapter\n+ * @netdev: net_device for @adapter\n+ * @queue_index: RX queue index\n+ \n+ Return: 1 frame delivered, 0 if the slot was skipped cleanly, -1 on error.\n+ /\n+static int ibmveth_poll_deliver_frame(struct napi_struct napi,\n+\t\t\t\t struct ibmveth_adapter adapter,\n+\t\t\t\t struct net_device netdev,\n+\t\t\t\t int queue_index)\n+{\n+\tstruct sk_buff skb, new_skb;\n+\tstruct ibmveth_rx_q rxq = \u0026adapter-\u003erx_queue[queue_index];\n+\tunsigned int room, off, len, pool;\n+\tint length, offset, csum_good, lrg_pkt;\n+\t__sum16 iph_check = 0;\n \tu16 mss = 0;\n+\tint rc;\n \n-restart_poll:\n-\twhile (frames_processed \u003c budget) {\n-\t\tif (!ibmveth_rxq_pending_buffer(adapter))\n-\t\t\tbreak;\n+\tlength = ibmveth_rxq_frame_length(adapter, queue_index);\n+\toffset = ibmveth_rxq_frame_offset(adapter, queue_index);\n+\tcsum_good = ibmveth_rxq_csum_good(adapter, queue_index);\n+\tlrg_pkt = ibmveth_rxq_large_packet(adapter, queue_index);\n+\n+\tskb = ibmveth_rxq_get_buffer(adapter, queue_index);\n+\tif (unlikely(!skb)) {\n+\t\tif (!ibmveth_poll_skip_bad_correlator(netdev, adapter,\n+\t\t\t\t\t\t queue_index))\n+\t\t\treturn -1;\n+\t\treturn 0;\n+\t}\n \n-\t\tsmp_rmb();\n-\t\tif (!ibmveth_rxq_buffer_valid(adapter)) {\n-\t\t\twmb(); / suggested by larson1 /\n-\t\t\tadapter-\u003erx_invalid_buffer++;\n-\t\t\tnetdev_dbg(netdev, \"recycling invalid buffer\\n\");\n-\t\t\tif (unlikely(ibmveth_rxq_harvest_buffer(adapter, true)))\n-\t\t\t\tbreak;\n-\t\t} else {\n-\t\t\tstruct sk_buff skb, new_skb;\n-\t\t\tint length = ibmveth_rxq_frame_length(adapter);\n-\t\t\tint offset = ibmveth_rxq_frame_offset(adapter);\n-\t\t\tint csum_good = ibmveth_rxq_csum_good(adapter);\n-\t\t\tint lrg_pkt = ibmveth_rxq_large_packet(adapter);\n-\t\t\t__sum16 iph_check = 0;\n-\n-\t\t\tskb = ibmveth_rxq_get_buffer(adapter);\n-\t\t\tif (unlikely(!skb))\n-\t\t\t\tbreak;\n+\tpool = rxq-\u003equeue_addr[rxq-\u003eindex].correlator \u003e\u003e 32;\n+\troom = min_t(unsigned int, skb_tailroom(skb),\n+\t\t adapter-\u003erx_buff_pool[queue_index][pool].buff_size);\n+\toff = offset;\n+\tlen = length;\n+\tif (unlikely(off \u003e= room \|\| len \u003e room - off)) {\n+\t\tif (!ibmveth_poll_drop_oversize(netdev, adapter, queue_index,\n+\t\t\t\t\t\toff, len, room))\n+\t\t\treturn -1;\n+\t\treturn 0;\n+\t}\n \n-\t\t\t/ if the large packet bit is set in the rx queue\n-\t\t\t * descriptor, the mss will be written by PHYP eight\n-\t\t\t * bytes from the start of the rx buffer, which is\n-\t\t\t * skb-\u003edata at this stage\n-\t\t\t /\n-\t\t\tif (lrg_pkt) {\n-\t\t\t\t__be64 rxmss = (__be64 )(skb-\u003edata + 8);\n+\tif (lrg_pkt) {\n+\t\t__be64 rxmss = (__be64 )(skb-\u003edata + 8);\n \n-\t\t\t\tmss = (u16)be64_to_cpu(rxmss);\n-\t\t\t}\n+\t\tmss = (u16)be64_to_cpu(rxmss);\n+\t}\n \n-\t\t\tnew_skb = NULL;\n-\t\t\tif (length \u003c rx_copybreak)\n-\t\t\t\tnew_skb = netdev_alloc_skb(netdev, length);\n-\n-\t\t\tif (new_skb) {\n-\t\t\t\tskb_copy_to_linear_data(new_skb,\n-\t\t\t\t\t\t\tskb-\u003edata + offset,\n-\t\t\t\t\t\t\tlength);\n-\t\t\t\tif (rx_flush)\n-\t\t\t\t\tibmveth_flush_buffer(skb-\u003edata,\n-\t\t\t\t\t\tlength + offset);\n-\t\t\t\tif (unlikely(ibmveth_rxq_harvest_buffer(adapter, true)))\n-\t\t\t\t\tbreak;\n-\t\t\t\tskb = new_skb;\n-\t\t\t} else {\n-\t\t\t\tif (unlikely(ibmveth_rxq_harvest_buffer(adapter, false)))\n-\t\t\t\t\tbreak;\n-\t\t\t\tskb_reserve(skb, offset);\n-\t\t\t}\n+\tnew_skb = NULL;\n+\tif (length \u003c rx_copybreak)\n+\t\tnew_skb = netdev_alloc_skb(netdev, length);\n+\n+\tif (new_skb) {\n+\t\tskb_copy_to_linear_data(new_skb, skb-\u003edata + offset, length);\n+\t\tif (rx_flush)\n+\t\t\tibmveth_flush_buffer(skb-\u003edata, length + offset);\n+\t\trc = ibmveth_rxq_harvest_buffer(adapter, queue_index, true);\n+\t\tif (unlikely(rc)) {\n+\t\t\tkfree_skb(new_skb);\n+\t\t\treturn -1;\n+\t\t}\n+\t\tskb = new_skb;\n+\t} else {\n+\t\trc = ibmveth_rxq_harvest_buffer(adapter, queue_index, false);\n+\t\tif (unlikely(rc))\n+\t\t\treturn -1;\n+\t\tskb_reserve(skb, offset);\n+\t}\n \n-\t\t\tskb_put(skb, length);\n-\t\t\tskb-\u003eprotocol = eth_type_trans(skb, netdev);\n+\tskb_put(skb, length);\n+\tskb-\u003eprotocol = eth_type_trans(skb, netdev);\n \n-\t\t\t/ PHYP without PLSO support places a -1 in the ip\n-\t\t\t * checksum for large send frames.\n-\t\t\t /\n-\t\t\tif (skb-\u003eprotocol == cpu_to_be16(ETH_P_IP)) {\n-\t\t\t\tstruct iphdr iph = (struct iphdr )skb-\u003edata;\n+\tif (skb-\u003eprotocol == cpu_to_be16(ETH_P_IP))\n+\t\tiph_check = ((struct iphdr )skb-\u003edata)-\u003echeck;\n \n-\t\t\t\tiph_check = iph-\u003echeck;\n-\t\t\t}\n+\tif ((length \u003e netdev-\u003emtu + ETH_HLEN) \|\| lrg_pkt \|\|\n+\t iph_check == 0xffff) {\n+\t\tibmveth_rx_mss_helper(skb, mss, lrg_pkt);\n+\t\tadapter-\u003erx_qstats[queue_index].large_packets++;\n+\t}\n \n-\t\t\tif ((length \u003e netdev-\u003emtu + ETH_HLEN) \|\|\n-\t\t\t lrg_pkt \|\| iph_check == 0xffff) {\n-\t\t\t\tibmveth_rx_mss_helper(skb, mss, lrg_pkt);\n-\t\t\t\tadapter-\u003erx_large_packets++;\n-\t\t\t}\n+\tif (csum_good) {\n+\t\tskb-\u003eip_summed = CHECKSUM_UNNECESSARY;\n+\t\tibmveth_rx_csum_helper(skb, adapter);\n+\t}\n \n-\t\t\tif (csum_good) {\n-\t\t\t\tskb-\u003eip_summed = CHECKSUM_UNNECESSARY;\n-\t\t\t\tibmveth_rx_csum_helper(skb, adapter);\n-\t\t\t}\n+\tskb_record_rx_queue(skb, queue_index);\n+\tnapi_gro_receive(napi, skb);\n+\n+\tadapter-\u003erx_qstats[queue_index].packets++;\n+\tadapter-\u003erx_qstats[queue_index].bytes += length;\n+\n+\treturn 1;\n+}\n+\n+static int ibmveth_poll(struct napi_struct napi, int budget)\n+{\n+\tstruct net_device netdev = napi-\u003edev;\n+\tstruct ibmveth_adapter adapter = netdev_priv(netdev);\n+\tint frames_processed = 0;\n+\tint queue_index, rc;\n+\n+\tqueue_index = napi - adapter-\u003enapi;\n+\n+\tif (WARN_ON(queue_index \u003c 0 \|\|\n+\t\t queue_index \u003e= ibmveth_get_num_rx_queues(adapter))) {\n+\t\tif (budget)\n+\t\t\tnapi_complete_done(napi, 0);\n+\t\treturn 0;\n+\t}\n+\n+\tif (ibmveth_poll_stopping(netdev, napi)) {\n+\t\tif (budget)\n+\t\t\tnapi_complete_done(napi, 0);\n+\t\treturn 0;\n+\t}\n+\n+\tadapter-\u003erx_qstats[queue_index].polls++;\n+\n+restart_poll:\n+\twhile (frames_processed \u003c budget) {\n+\t\tif (ibmveth_poll_stopping(netdev, napi))\n+\t\t\tbreak;\n \n-\t\t\tnapi_gro_receive(napi, skb);\t/ send it up /\n+\t\tif (!ibmveth_rxq_pending_buffer(adapter, queue_index))\n+\t\t\tbreak;\n \n-\t\t\tnetdev-\u003estats.rx_packets++;\n-\t\t\tnetdev-\u003estats.rx_bytes += length;\n-\t\t\tframes_processed++;\n+\t\tsmp_rmb();\n+\t\tif (!ibmveth_rxq_buffer_valid(adapter, queue_index)) {\n+\t\t\twmb(); / suggested by larson1 /\n+\t\t\tif (!ibmveth_poll_recycle_invalid(netdev, adapter,\n+\t\t\t\t\t\t\t queue_index))\n+\t\t\t\tbreak;\n+\t\t} else {\n+\t\t\trc = ibmveth_poll_deliver_frame(napi, adapter, netdev,\n+\t\t\t\t\t\t\tqueue_index);\n+\t\t\tif (rc \u003c 0)\n+\t\t\t\tbreak;\n+\t\t\tif (rc \u003e 0)\n+\t\t\t\tframes_processed++;\n \t\t}\n \t}\n \n-\tibmveth_replenish_task(adapter);\n+\tibmveth_replenish_task(adapter, queue_index);\n+\n+\tif (ibmveth_poll_stopping(netdev, napi)) {\n+\t\t/ budget 0 is netpoll, which must not complete NAPI.\n+\t\t * Otherwise returning budget after completing would ask\n+\t\t * NAPI to reschedule, so cap the return at budget - 1.\n+\t\t /\n+\t\tif (budget) {\n+\t\t\tnapi_complete_done(napi, frames_processed);\n+\t\t\treturn min(frames_processed, budget - 1);\n+\t\t}\n+\t\treturn 0;\n+\t}\n \n \tif (frames_processed == budget)\n \t\tgoto out;\n@@ -1548,18 +4144,27 @@ static int ibmveth_poll(struct napi_struct napi, int budget)\n \tif (!napi_complete_done(napi, frames_processed))\n \t\tgoto out;\n \n-\t/* We think we are done - reenable interrupts,\n-\t * then check once more to make sure we are done.\n+\t/\n+\t napi_disable() sets DISABLE then waits for this poll. Without a\n+\t * second stopping check here, enable_irq() can re-arm PHYP after\n+\t * resize already masked the queue; late IRQs then hit the handler\n+\t * after num_rx_queues was published lower (lab WARN at interrupt).\n \t /\n-\tlpar_rc = h_vio_signal(adapter-\u003evdev-\u003eunit_address, VIO_IRQ_ENABLE);\n-\tif (WARN_ON(lpar_rc != H_SUCCESS)) {\n+\tif (ibmveth_poll_stopping(netdev, napi))\n+\t\tgoto out;\n+\n+\trc = ibmveth_enable_irq(adapter, queue_index);\n+\tif (rc) {\n+\t\tnetdev_err(netdev,\n+\t\t\t \"Failed to enable IRQ for queue %d (rc=%d), scheduling reset\\n\",\n+\t\t\t queue_index, rc);\n \t\tschedule_work(\u0026adapter-\u003ework);\n \t\tgoto out;\n \t}\n \n-\tif (ibmveth_rxq_pending_buffer(adapter) \u0026\u0026 napi_schedule(napi)) {\n-\t\tlpar_rc = h_vio_signal(adapter-\u003evdev-\u003eunit_address,\n-\t\t\t\t VIO_IRQ_DISABLE);\n+\tif (ibmveth_rxq_pending_buffer(adapter, queue_index) \u0026\u0026\n+\t napi_schedule(napi)) {\n+\t\tibmveth_disable_irq(adapter, queue_index);\n \t\tgoto restart_poll;\n \t}\n \n@@ -1569,16 +4174,23 @@ static int ibmveth_poll(struct napi_struct napi, int budget)\n \n static irqreturn_t ibmveth_interrupt(int irq, void dev_instance)\n {\n-\tstruct net_device netdev = dev_instance;\n+\tstruct napi_struct napi = dev_instance;\n+\tstruct net_device netdev = napi-\u003edev;\n \tstruct ibmveth_adapter adapter = netdev_priv(netdev);\n-\tunsigned long lpar_rc;\n+\tint qindex;\n \n-\tif (napi_schedule_prep(\u0026adapter-\u003enapi)) {\n-\t\tlpar_rc = h_vio_signal(adapter-\u003evdev-\u003eunit_address,\n-\t\t\t\t VIO_IRQ_DISABLE);\n-\t\tWARN_ON(lpar_rc != H_SUCCESS);\n-\t\t__napi_schedule(\u0026adapter-\u003enapi);\n-\t}\n+\tqindex = napi - adapter-\u003enapi;\n+\t/\n+\t * Quiet on out-of-range: scale-down publishes a lower live count\n+\t * before free_irq(). A residual IRQ must not WARN-storm; return\n+\t * IRQ_NONE until the handler is removed.\n+\t /\n+\tif (qindex \u003c 0 \|\| qindex \u003e= ibmveth_get_num_rx_queues(adapter))\n+\t\treturn IRQ_NONE;\n+\n+\tadapter-\u003erx_qstats[qindex].interrupts++;\n+\n+\tibmveth_schedule_rx_queue(adapter, qindex);\n \treturn IRQ_HANDLED;\n }\n \n@@ -1645,7 +4257,7 @@ static int ibmveth_change_mtu(struct net_device dev, int new_mtu)\n \tint need_restart = 0;\n \n \tfor (i = 0; i \u003c IBMVETH_NUM_BUFF_POOLS; i++)\n-\t\tif (new_mtu_oh \u003c= adapter-\u003erx_buff_pool[i].buff_size)\n+\t\tif (new_mtu_oh \u003c= adapter-\u003erx_buff_pool[0][i].buff_size)\n \t\t\tbreak;\n \n \tif (i == IBMVETH_NUM_BUFF_POOLS)\n@@ -1660,9 +4272,9 @@ static int ibmveth_change_mtu(struct net_device dev, int new_mtu)\n \n \t/ Look for an active buffer pool that can hold the new MTU /\n \tfor (i = 0; i \u003c IBMVETH_NUM_BUFF_POOLS; i++) {\n-\t\tadapter-\u003erx_buff_pool[i].active = 1;\n+\t\tadapter-\u003erx_buff_pool[0][i].active = 1;\n \n-\t\tif (new_mtu_oh \u003c= adapter-\u003erx_buff_pool[i].buff_size) {\n+\t\tif (new_mtu_oh \u003c= adapter-\u003erx_buff_pool[0][i].buff_size) {\n \t\t\tWRITE_ONCE(dev-\u003emtu, new_mtu);\n \t\t\tvio_cmo_set_dev_desired(viodev,\n \t\t\t\t\t\tibmveth_get_desired_dma\n@@ -1683,8 +4295,20 @@ static int ibmveth_change_mtu(struct net_device dev, int new_mtu)\n #ifdef CONFIG_NET_POLL_CONTROLLER\n static void ibmveth_poll_controller(struct net_device dev)\n {\n-\tibmveth_replenish_task(netdev_priv(dev));\n-\tibmveth_interrupt(dev-\u003eirq, dev);\n+\tstruct ibmveth_adapter adapter = netdev_priv(dev);\n+\tunsigned int num;\n+\tint i;\n+\n+\tif (!adapter-\u003eopened)\n+\t\treturn;\n+\n+\tnum = ibmveth_get_num_rx_queues(adapter);\n+\n+\tfor (i = 0; i \u003c num; i++)\n+\t\tibmveth_replenish_task(adapter, i);\n+\n+\tfor (i = 0; i \u003c num; i++)\n+\t\tibmveth_schedule_rx_queue(adapter, i);\n }\n #endif\n \n@@ -1701,9 +4325,6 @@ static unsigned long ibmveth_get_desired_dma(struct vio_dev vdev)\n \tstruct net_device netdev = dev_get_drvdata(\u0026vdev-\u003edev);\n \tstruct ibmveth_adapter adapter;\n \tstruct iommu_table tbl;\n-\tunsigned long ret;\n-\tint i;\n-\tint rxqentries = 1;\n \n \ttbl = get_iommu_table_base(\u0026vdev-\u003edev);\n \n@@ -1712,26 +4333,8 @@ static unsigned long ibmveth_get_desired_dma(struct vio_dev vdev)\n \t\treturn IOMMU_PAGE_ALIGN(IBMVETH_IO_ENTITLEMENT_DEFAULT, tbl);\n \n \tadapter = netdev_priv(netdev);\n-\n-\tret = IBMVETH_BUFF_LIST_SIZE + IBMVETH_FILT_LIST_SIZE;\n-\tret += IOMMU_PAGE_ALIGN(netdev-\u003emtu, tbl);\n-\t/ add size of mapped tx buffers /\n-\tret += IOMMU_PAGE_ALIGN(IBMVETH_MAX_TX_BUF_SIZE, tbl);\n-\n-\tfor (i = 0; i \u003c IBMVETH_NUM_BUFF_POOLS; i++) {\n-\t\t/ add the size of the active receive buffers /\n-\t\tif (adapter-\u003erx_buff_pool[i].active)\n-\t\t\tret +=\n-\t\t\t adapter-\u003erx_buff_pool[i].size \n-\t\t\t IOMMU_PAGE_ALIGN(adapter-\u003erx_buff_pool[i].\n-\t\t\t\t\t buff_size, tbl);\n-\t\trxqentries += adapter-\u003erx_buff_pool[i].size;\n-\t}\n-\t/* add the size of the receive queue entries /\n-\tret += IOMMU_PAGE_ALIGN(\n-\t\trxqentries sizeof(struct ibmveth_rx_q_entry), tbl);\n-\n-\treturn ret;\n+\treturn ibmveth_desired_dma_for_rxqs(adapter,\n+\t\t\t\t\t ibmveth_get_num_rx_queues(adapter));\n }\n \n static int ibmveth_set_mac_addr(struct net_device dev, void p)\n@@ -1777,6 +4380,124 @@ static netdev_features_t ibmveth_features_check(struct sk_buff skb,\n \treturn vlan_features_check(skb, features);\n }\n \n+/\n+ ibmveth_get_stats64 - Return aggregated per-queue statistics\n+ * @dev: network device\n+ * @stats: rtnl link statistics storage\n+ \n+ Sums per-queue rx_qstats and tx_qstats into the rtnl counters.\n+ * Walk the full allocated arrays (not the live queue count) so shrinking\n+ * channels cannot make the totals go backwards.\n+ * Callers use ndo_get_stats64(); avoid updating netdev-\u003estats on the\n+ * xmit/poll paths to keep per-queue counters off the hot cache line.\n+ /\n+static void ibmveth_get_stats64(struct net_device dev,\n+\t\t\t\tstruct rtnl_link_stats64 stats)\n+{\n+\tstruct ibmveth_adapter adapter = netdev_priv(dev);\n+\tint i;\n+\n+\tfor (i = 0; i \u003c IBMVETH_MAX_RX_QUEUES; i++) {\n+\t\tstats-\u003erx_packets += adapter-\u003erx_qstats[i].packets;\n+\t\tstats-\u003erx_bytes += adapter-\u003erx_qstats[i].bytes;\n+\t}\n+\n+\tfor (i = 0; i \u003c IBMVETH_MAX_QUEUES; i++) {\n+\t\tstats-\u003etx_packets += adapter-\u003etx_qstats[i].packets;\n+\t\tstats-\u003etx_bytes += adapter-\u003etx_qstats[i].bytes;\n+\t\tstats-\u003etx_dropped += adapter-\u003etx_qstats[i].dropped_packets;\n+\t}\n+}\n+\n+static void ibmveth_get_queue_stats_rx(struct net_device dev, int idx,\n+\t\t\t\t struct netdev_queue_stats_rx stats)\n+{\n+\tstruct ibmveth_adapter adapter = netdev_priv(dev);\n+\n+\tstats-\u003epackets = adapter-\u003erx_qstats[idx].packets;\n+\tstats-\u003ebytes = adapter-\u003erx_qstats[idx].bytes;\n+\t/\n+\t * All three are frames that entered the device and never left it,\n+\t * which is what rx-hw-drops is specified to cover: no_buffer_drops\n+\t * is PHYP dropping for lack of buffer space on the page mapped now,\n+\t * no_buffer_retired the same for pages this queue has already\n+\t * released, and invalid_buffers is a processing error.\n+\t /\n+\tstats-\u003ehw_drops = adapter-\u003erx_qstats[idx].no_buffer_drops +\n+\t\t\t adapter-\u003erx_qstats[idx].no_buffer_retired +\n+\t\t\t adapter-\u003erx_qstats[idx].invalid_buffers;\n+\tstats-\u003ealloc_fail = adapter-\u003erx_qstats[idx].replenish_no_mem;\n+}\n+\n+static void ibmveth_get_queue_stats_tx(struct net_device dev, int idx,\n+\t\t\t\t struct netdev_queue_stats_tx stats)\n+{\n+\tstruct ibmveth_adapter adapter = netdev_priv(dev);\n+\n+\tstats-\u003epackets = adapter-\u003etx_qstats[idx].packets;\n+\tstats-\u003ebytes = adapter-\u003etx_qstats[idx].bytes;\n+\tstats-\u003ehw_drops = adapter-\u003etx_qstats[idx].dropped_packets;\n+}\n+\n+/*\n+ ibmveth_get_base_stats - account for traffic not on a live queue\n+ * @dev: network device\n+ * @rx: RX base statistics storage\n+ * @tx: TX base statistics storage\n+ \n+ get_queue_stats_{rx,tx}() only report queues the core still iterates,\n+ * i.e. below real_num_{rx,tx}_queues, while ibmveth_get_stats64() walks\n+ * the full arrays so device totals stay monotonic across a shrink.\n+ * Report the retired-queue remainder here, otherwise qstats and\n+ * rtnl_link_stats64 disagree by a delta that grows with every shrink.\n+ * Zeroing would not be neutral: per netdev_stat_ops it asserts the\n+ * per-queue sum is already exact.\n+ \n+ Bound the live side with real_num__queues rather than the adapter's\n+ own count, so the split lines up with the core's iteration exactly.\n+ \n+ Every field the per-queue callbacks fill must also be initialised\n+ * here: netdev_nl_stats_add() starts the sum at NETDEV_STAT_NOT_SET and\n+ * only accumulates while both sides are set, so a field left unset here\n+ * is dropped from the device total even though the queues report it.\n+ /\n+static void ibmveth_get_base_stats(struct net_device dev,\n+\t\t\t\t struct netdev_queue_stats_rx rx,\n+\t\t\t\t struct netdev_queue_stats_tx tx)\n+{\n+\tstruct ibmveth_adapter adapter = netdev_priv(dev);\n+\tunsigned int i;\n+\n+\trx-\u003epackets = 0;\n+\trx-\u003ebytes = 0;\n+\trx-\u003ealloc_fail = 0;\n+\trx-\u003ehw_drops = 0;\n+\ttx-\u003epackets = 0;\n+\ttx-\u003ebytes = 0;\n+\ttx-\u003ehw_drops = 0;\n+\n+\tfor (i = dev-\u003ereal_num_rx_queues; i \u003c IBMVETH_MAX_RX_QUEUES; i++) {\n+\t\trx-\u003epackets += adapter-\u003erx_qstats[i].packets;\n+\t\trx-\u003ebytes += adapter-\u003erx_qstats[i].bytes;\n+\t\trx-\u003ehw_drops += adapter-\u003erx_qstats[i].no_buffer_drops +\n+\t\t\t\tadapter-\u003erx_qstats[i].no_buffer_retired +\n+\t\t\t\tadapter-\u003erx_qstats[i].invalid_buffers;\n+\t\trx-\u003ealloc_fail += adapter-\u003erx_qstats[i].replenish_no_mem;\n+\t}\n+\n+\tfor (i = dev-\u003ereal_num_tx_queues; i \u003c IBMVETH_MAX_QUEUES; i++) {\n+\t\ttx-\u003epackets += adapter-\u003etx_qstats[i].packets;\n+\t\ttx-\u003ebytes += adapter-\u003etx_qstats[i].bytes;\n+\t\ttx-\u003ehw_drops += adapter-\u003etx_qstats[i].dropped_packets;\n+\t}\n+}\n+\n+static const struct netdev_stat_ops ibmveth_stat_ops = {\n+\t.get_queue_stats_rx\t= ibmveth_get_queue_stats_rx,\n+\t.get_queue_stats_tx\t= ibmveth_get_queue_stats_tx,\n+\t.get_base_stats\t\t= ibmveth_get_base_stats,\n+};\n+\n static const struct net_device_ops ibmveth_netdev_ops = {\n \t.ndo_open\t\t= ibmveth_open,\n \t.ndo_stop\t\t= ibmveth_close,\n@@ -1789,14 +4510,129 @@ static const struct net_device_ops ibmveth_netdev_ops = {\n \t.ndo_validate_addr\t= eth_validate_addr,\n \t.ndo_set_mac_address = ibmveth_set_mac_addr,\n \t.ndo_features_check\t= ibmveth_features_check,\n+\t.ndo_get_stats64\t= ibmveth_get_stats64,\n #ifdef CONFIG_NET_POLL_CONTROLLER\n \t.ndo_poll_controller\t= ibmveth_poll_controller,\n #endif\n };\n \n+static int ibmveth_buffer_pools_show(struct seq_file m, void v)\n+{\n+\tstruct ibmveth_adapter adapter = m-\u003eprivate;\n+\tint i, j;\n+\n+\t/\n+\t size / buff_size / pool-\u003eactive are written under RTNL\n+\t * (veth_pool_store, open template copy). Take the same lock so\n+\t * those columns are not a torn snapshot. available is updated\n+\t * from NAPI/softirq; only atomic_read() keeps it from tearing.\n+\t * Not required for memory safety; embedded arrays only.\n+\t /\n+\trtnl_lock();\n+\n+\tseq_puts(m, \"Queue Pool Count BuffSize Active Available\\n\");\n+\tseq_puts(m, \"----- ---- ----- -------- ------ ---------\\n\");\n+\tif (!adapter-\u003eopened) {\n+\t\tseq_puts(m, \"# down: Active/Available 0 unless allocated\\n\");\n+\t\tseq_puts(m, \"# down: geometry above queue 0 set at open\\n\");\n+\t}\n+\n+\tfor (i = 0; i \u003c ibmveth_get_num_rx_queues(adapter); i++) {\n+\t\tfor (j = 0; j \u003c IBMVETH_NUM_BUFF_POOLS; j++) {\n+\t\t\tstruct ibmveth_buff_pool pool =\n+\t\t\t\t\u0026adapter-\u003erx_buff_pool[i][j];\n+\t\t\tbool live = pool-\u003eskbuff \u0026\u0026 pool-\u003efree_map;\n+\t\t\tint active = live ? pool-\u003eactive : 0;\n+\t\t\tint available = live ? atomic_read(\u0026pool-\u003eavailable)\n+\t\t\t\t\t : 0;\n+\n+\t\t\tseq_printf(m, \"%5d %4d %5u %8u %6d %9d\\n\",\n+\t\t\t\t i, j, pool-\u003esize, pool-\u003ebuff_size,\n+\t\t\t\t active, available);\n+\t\t}\n+\t}\n+\n+\trtnl_unlock();\n+\treturn 0;\n+}\n+DEFINE_SHOW_ATTRIBUTE(ibmveth_buffer_pools);\n+\n+/* Driver-owned root so per-adapter dirs use a stable vio name, not the\n+ * mutable netdev-\u003ename (avoids stale names / eth0 collisions after rename).\n+ /\n+static struct dentry ibmveth_dbg_root;\n+\n+static void ibmveth_debugfs_init(struct ibmveth_adapter adapter)\n+{\n+\tadapter-\u003edebugfs_dir =\n+\t\tdebugfs_create_dir(dev_name(\u0026adapter-\u003evdev-\u003edev),\n+\t\t\t\t ibmveth_dbg_root);\n+\tdebugfs_create_file(\"buffer_pools\", 0400, adapter-\u003edebugfs_dir,\n+\t\t\t adapter, \u0026ibmveth_buffer_pools_fops);\n+}\n+\n+static void ibmveth_debugfs_exit(struct ibmveth_adapter adapter)\n+{\n+\tdebugfs_remove_recursive(adapter-\u003edebugfs_dir);\n+\tadapter-\u003edebugfs_dir = NULL;\n+}\n+\n+/*\n+ ibmveth_pool_kobj_release - Mark a pool kobject finished\n+ * @kobj: kobject embedded in the pool\n+ \n+ Pool kobjects live in netdev_priv(). Last put waits for this\n+ * before free_netdev() so DEBUG_KOBJECT_RELEASE delayed cleanup\n+ * does not run on freed memory.\n+ /\n+static void ibmveth_pool_kobj_release(struct kobject kobj)\n+{\n+\tstruct ibmveth_buff_pool pool = container_of(kobj,\n+\t\t\t\t\t\t struct ibmveth_buff_pool,\n+\t\t\t\t\t\t kobj);\n+\n+\tcomplete(\u0026pool-\u003ereleased);\n+}\n+\n+/\n+ ibmveth_put_pool_kobjs - Drop pool kobjects and wait for release\n+ * @adapter: ibmveth adapter\n+ * @pools_ready: number of queue-0 pools that were initialized\n+ \n+ Put every initialized pool kobject, then wait so free_netdev()\n+ * cannot race DEBUG_KOBJECT_RELEASE delayed cleanup.\n+ /\n+static void ibmveth_put_pool_kobjs(struct ibmveth_adapter adapter,\n+\t\t\t\t int pools_ready)\n+{\n+\tint i;\n+\n+\tfor (i = 0; i \u003c pools_ready; i++)\n+\t\tkobject_put(\u0026adapter-\u003erx_buff_pool[0][i].kobj);\n+\tfor (i = 0; i \u003c pools_ready; i++)\n+\t\twait_for_completion(\u0026adapter-\u003erx_buff_pool[0][i].released);\n+}\n+\n+static void ibmveth_probe_cleanup(struct ibmveth_adapter adapter,\n+\t\t\t\t int pools_ready)\n+{\n+\tstruct net_device netdev = adapter-\u003enetdev;\n+\n+\tcancel_work_sync(\u0026adapter-\u003ework);\n+\tibmveth_put_pool_kobjs(adapter, pools_ready);\n+\n+\tibmveth_free_tx_qstats(adapter);\n+\tibmveth_free_rx_qstats(adapter);\n+\t/* Probe failure never reaches ibmveth_remove(); clear before free so\n+\t * CMO get_desired_dma() cannot see a freed netdev on rebind.\n+\t /\n+\tdev_set_drvdata(\u0026adapter-\u003evdev-\u003edev, NULL);\n+\tfree_netdev(netdev);\n+}\n+\n static int ibmveth_probe(struct vio_dev dev, const struct vio_device_id id)\n {\n-\tint rc, i, mac_len;\n+\tint rc, i, mac_len, pools_ready = 0;\n \tstruct net_device netdev;\n \tstruct ibmveth_adapter adapter;\n \tunsigned char mac_addr_p;\n@@ -1831,7 +4667,8 @@ static int ibmveth_probe(struct vio_dev dev, const struct vio_device_id id)\n \t\treturn -EINVAL;\n \t}\n \n-\tnetdev = alloc_etherdev_mqs(sizeof(struct ibmveth_adapter), IBMVETH_MAX_QUEUES, 1);\n+\tnetdev = alloc_etherdev_mqs(sizeof(struct ibmveth_adapter),\n+\t\t\t\t IBMVETH_MAX_QUEUES, IBMVETH_MAX_RX_QUEUES);\n \tif (!netdev)\n \t\treturn -ENOMEM;\n \n@@ -1841,14 +4678,33 @@ static int ibmveth_probe(struct vio_dev dev, const struct vio_device_id id)\n \tadapter-\u003evdev = dev;\n \tadapter-\u003enetdev = netdev;\n \tINIT_WORK(\u0026adapter-\u003ework, ibmveth_reset);\n+\n+\t/\n+\t Initialise the replenish locks once. open() is re-entered on\n+\t * MTU and offload changes without netpoll_poll_disable(), so a\n+\t * lock set up there could be reinitialised while poll_controller()\n+\t * holds it.\n+\t /\n+\tfor (i = 0; i \u003c IBMVETH_MAX_RX_QUEUES; i++)\n+\t\tspin_lock_init(\u0026adapter-\u003erx_queue[i].replenish_lock);\n+\n \tadapter-\u003emcastFilterSize = be32_to_cpu(mcastFilterSize_p);\n \tibmveth_init_link_settings(netdev);\n \n-\tnetif_napi_add_weight(netdev, \u0026adapter-\u003enapi, ibmveth_poll, 16);\n+\tfor (i = 0; i \u003c IBMVETH_MAX_RX_QUEUES; i++)\n+\t\tnetif_napi_add_weight(netdev, \u0026adapter-\u003enapi[i],\n+\t\t\t\t ibmveth_poll, 16);\n+\n+\tif (ibmveth_alloc_rx_qstats(adapter) \|\|\n+\t ibmveth_alloc_tx_qstats(adapter)) {\n+\t\tibmveth_probe_cleanup(adapter, 0);\n+\t\treturn -ENOMEM;\n+\t}\n \n \tnetdev-\u003eirq = dev-\u003eirq;\n \tnetdev-\u003enetdev_ops = \u0026ibmveth_netdev_ops;\n \tnetdev-\u003eethtool_ops = \u0026netdev_ethtool_ops;\n+\tnetdev-\u003estat_ops = \u0026ibmveth_stat_ops;\n \tSET_NETDEV_DEV(netdev, \u0026dev-\u003edev);\n \tnetdev-\u003ehw_features = NETIF_F_SG;\n \tif (vio_get_attribute(dev, \"ibm,illan-options\", NULL) != NULL) {\n@@ -1876,12 +4732,30 @@ static int ibmveth_probe(struct vio_dev dev, const struct vio_device_id id)\n \t\tnetdev-\u003efeatures \|= NETIF_F_FRAGLIST;\n \t}\n \n+\tif (ret == H_SUCCESS \u0026\u0026\n+\t (ret_attr \u0026 IBMVETH_ILLAN_RX_MULTI_QUEUE_SUPPORT)) {\n+\t\tadapter-\u003emulti_queue = true;\n+\t\tibmveth_publish_num_rx_queues(adapter,\n+\t\t\t\t\t min(num_online_cpus(),\n+\t\t\t\t\t\t IBMVETH_DEFAULT_QUEUES));\n+\t\tnetdev_dbg(netdev, \"RX multi queue mode enabled: %u queues\\n\",\n+\t\t\t ibmveth_get_num_rx_queues(adapter));\n+\t} else {\n+\t\tadapter-\u003emulti_queue = false;\n+\t\tibmveth_publish_num_rx_queues(adapter,\n+\t\t\t\t\t IBMVETH_DEFAULT_RX_QUEUES);\n+\t}\n+\n \tif (ret == H_SUCCESS \u0026\u0026\n \t (ret_attr \u0026 IBMVETH_ILLAN_RX_MULTI_BUFF_SUPPORT)) {\n-\t\tadapter-\u003erx_buffers_per_hcall = IBMVETH_MAX_RX_PER_HCALL;\n+\t\tif (adapter-\u003emulti_queue)\n+\t\t\tadapter-\u003erx_buffers_per_hcall =\n+\t\t\t\tIBMVETH_MAX_RX_PER_HCALL;\n+\t\telse\n+\t\t\tadapter-\u003erx_buffers_per_hcall = IBMVETH_MAX_RX_REGULAR;\n \t\tnetdev_dbg(netdev,\n \t\t\t \"RX Multi-buffer hcall supported by FW, batch set to %u\\n\",\n-\t\t\t adapter-\u003erx_buffers_per_hcall);\n+\t\t\t adapter-\u003erx_buffers_per_hcall);\n \t} else {\n \t\tadapter-\u003erx_buffers_per_hcall = 1;\n \t\tnetdev_dbg(netdev,\n@@ -1898,16 +4772,29 @@ static int ibmveth_probe(struct vio_dev dev, const struct vio_device_id id)\n \t\tmemcpy(pool_count, pool_count_cmo, sizeof(pool_count));\n \n \tfor (i = 0; i \u003c IBMVETH_NUM_BUFF_POOLS; i++) {\n-\t\tstruct kobject kobj = \u0026adapter-\u003erx_buff_pool[i].kobj;\n-\t\tint error;\n+\t\tstruct kobject kobj = \u0026adapter-\u003erx_buff_pool[0][i].kobj;\n \n-\t\tibmveth_init_buffer_pool(\u0026adapter-\u003erx_buff_pool[i], i,\n+\t\tibmveth_init_buffer_pool(\u0026adapter-\u003erx_buff_pool[0][i], i,\n \t\t\t\t\t pool_count[i], pool_size[i],\n \t\t\t\t\t pool_active[i]);\n-\t\terror = kobject_init_and_add(kobj, \u0026ktype_veth_pool,\n-\t\t\t\t\t \u0026dev-\u003edev.kobj, \"pool%d\", i);\n-\t\tif (!error)\n-\t\t\tkobject_uevent(kobj, KOBJ_ADD);\n+\t\tinit_completion(\u0026adapter-\u003erx_buff_pool[0][i].released);\n+\t\trc = kobject_init_and_add(kobj, \u0026ktype_veth_pool,\n+\t\t\t\t\t \u0026dev-\u003edev.kobj, \"pool%d\", i);\n+\t\tif (rc) {\n+\t\t\tstruct ibmveth_buff_pool pool =\n+\t\t\t\t\u0026adapter-\u003erx_buff_pool[0][i];\n+\n+\t\t\tdev_err(\u0026dev-\u003edev,\n+\t\t\t\t\"failed to create pool%d kobject: %d\\n\", i, rc);\n+\t\t\t/ init_and_add takes a ref even on failure /\n+\t\t\tkobject_put(kobj);\n+\t\t\twait_for_completion(\u0026pool-\u003ereleased);\n+\t\t\tibmveth_probe_cleanup(adapter, pools_ready);\n+\t\t\treturn rc;\n+\t\t}\n+\n+\t\tpools_ready++;\n+\t\tkobject_uevent(kobj, KOBJ_ADD);\n \t}\n \n \trc = netif_set_real_num_tx_queues(netdev, min(num_online_cpus(),\n@@ -1915,9 +4802,25 @@ static int ibmveth_probe(struct vio_dev dev, const struct vio_device_id id)\n \tif (rc) {\n \t\tnetdev_dbg(netdev, \"failed to set number of tx queues rc=%d\\n\",\n \t\t\t rc);\n-\t\tfree_netdev(netdev);\n+\t\tibmveth_probe_cleanup(adapter, pools_ready);\n+\t\treturn rc;\n+\t}\n+\n+\t/\n+\t * alloc_etherdev_mqs() sized real_num_rx_queues to MAX. Match the\n+\t * advertised default (or SQ 1) before register_netdev so down-state\n+\t * readers agree with adapter-\u003enum_rx_queues / ethtool -l.\n+\t /\n+\trc = netif_set_real_num_rx_queues(netdev,\n+\t\t\t\t\t ibmveth_get_num_rx_queues(adapter));\n+\n+\tif (rc) {\n+\t\tnetdev_dbg(netdev, \"failed to set number of rx queues rc=%d\\n\",\n+\t\t\t rc);\n+\t\tibmveth_probe_cleanup(adapter, pools_ready);\n \t\treturn rc;\n \t}\n+\n \tadapter-\u003etx_ltb_size = PAGE_ALIGN(IBMVETH_MAX_TX_BUF_SIZE);\n \tfor (i = 0; i \u003c IBMVETH_MAX_QUEUES; i++)\n \t\tadapter-\u003etx_ltb_ptr[i] = NULL;\n@@ -1931,12 +4834,14 @@ static int ibmveth_probe(struct vio_dev dev, const struct vio_device_id id)\n \n \tif (rc) {\n \t\tnetdev_dbg(netdev, \"failed to register netdev rc=%d\\n\", rc);\n-\t\tfree_netdev(netdev);\n+\t\tibmveth_probe_cleanup(adapter, pools_ready);\n \t\treturn rc;\n \t}\n \n \tnetdev_dbg(netdev, \"registered\\n\");\n \n+\tibmveth_debugfs_init(adapter);\n+\n \treturn 0;\n }\n \n@@ -1944,14 +4849,22 @@ static void ibmveth_remove(struct vio_dev dev)\n {\n \tstruct net_device netdev = dev_get_drvdata(\u0026dev-\u003edev);\n \tstruct ibmveth_adapter adapter = netdev_priv(netdev);\n-\tint i;\n \n-\tcancel_work_sync(\u0026adapter-\u003ework);\n+\tibmveth_debugfs_exit(adapter);\n \n-\tfor (i = 0; i \u003c IBMVETH_NUM_BUFF_POOLS; i++)\n-\t\tkobject_put(\u0026adapter-\u003erx_buff_pool[i].kobj);\n+\tibmveth_put_pool_kobjs(adapter, IBMVETH_NUM_BUFF_POOLS);\n \n+\t/\n+\t Unregister first so NAPI/xmit cannot re-arm reset work after we\n+\t * cancel it. cancel_work_sync() before unregister left a window\n+\t * where poll could schedule_work() and the worker ran after\n+\t * free_netdev().\n+\t /\n \tunregister_netdev(netdev);\n+\tcancel_work_sync(\u0026adapter-\u003ework);\n+\n+\tibmveth_free_tx_qstats(adapter);\n+\tibmveth_free_rx_qstats(adapter);\n \n \tfree_netdev(netdev);\n \tdev_set_drvdata(\u0026dev-\u003edev, NULL);\n@@ -2035,11 +4948,12 @@ static ssize_t veth_pool_store(struct kobject kobj, struct attribute attr,\n \t\t\t/ Make sure there is a buffer pool with buffers that\n \t\t\t can hold a packet of the size of the MTU /\n \t\t\tfor (i = 0; i \u003c IBMVETH_NUM_BUFF_POOLS; i++) {\n-\t\t\t\tif (pool == \u0026adapter-\u003erx_buff_pool[i])\n+\t\t\t\tif (pool == \u0026adapter-\u003erx_buff_pool[0][i])\n \t\t\t\t\tcontinue;\n-\t\t\t\tif (!adapter-\u003erx_buff_pool[i].active)\n+\t\t\t\tif (!adapter-\u003erx_buff_pool[0][i].active)\n \t\t\t\t\tcontinue;\n-\t\t\t\tif (mtu \u003c= adapter-\u003erx_buff_pool[i].buff_size)\n+\t\t\t\tif (mtu \u003c=\n+\t\t\t\t adapter-\u003erx_buff_pool[0][i].buff_size)\n \t\t\t\t\tbreak;\n \t\t\t}\n \n@@ -2092,8 +5006,8 @@ static ssize_t veth_pool_store(struct kobject kobj, struct attribute attr,\n \t}\n \trtnl_unlock();\n \n-\t/ kick the interrupt handler to allocate/deallocate pools /\n-\tibmveth_interrupt(netdev-\u003eirq, netdev);\n+\t/ kick RX processing to allocate/deallocate pools /\n+\tibmveth_schedule_rx_queue(adapter, 0);\n \treturn count;\n \n unlock_err:\n@@ -2101,7 +5015,6 @@ static ssize_t veth_pool_store(struct kobject kobj, struct attribute attr,\n \treturn rc;\n }\n \n-\n #define ATTR(_name, _mode)\t\t\t\t\\\n \tstruct attribute veth_##_name##_attr = {\t\\\n \t.name = __stringify(_name), .mode = _mode,\t\\\n@@ -2125,7 +5038,7 @@ static const struct sysfs_ops veth_pool_ops = {\n };\n \n static struct kobj_type ktype_veth_pool = {\n-\t.release = NULL,\n+\t.release = ibmveth_pool_kobj_release,\n \t.sysfs_ops = \u0026veth_pool_ops,\n \t.default_groups = veth_pool_groups,\n };\n@@ -2133,7 +5046,13 @@ static struct kobj_type ktype_veth_pool = {\n static int ibmveth_resume(struct device dev)\n {\n \tstruct net_device netdev = dev_get_drvdata(dev);\n-\tibmveth_interrupt(netdev-\u003eirq, netdev);\n+\tstruct ibmveth_adapter adapter = netdev_priv(netdev);\n+\tunsigned int num = ibmveth_get_num_rx_queues(adapter);\n+\tint i;\n+\n+\tfor (i = 0; i \u003c num; i++)\n+\t\tibmveth_schedule_rx_queue(adapter, i);\n+\n \treturn 0;\n }\n \n@@ -2158,15 +5077,27 @@ static struct vio_driver ibmveth_driver = {\n \n static int __init ibmveth_module_init(void)\n {\n+\tint rc;\n+\n \tprintk(KERN_DEBUG \"%s: %s %s\\n\", ibmveth_driver_name,\n \t ibmveth_driver_string, ibmveth_driver_version);\n \n-\treturn vio_register_driver(\u0026ibmveth_driver);\n+\tibmveth_dbg_root = debugfs_create_dir(ibmveth_driver_name, NULL);\n+\n+\trc = vio_register_driver(\u0026ibmveth_driver);\n+\tif (rc) {\n+\t\tdebugfs_remove_recursive(ibmveth_dbg_root);\n+\t\tibmveth_dbg_root = NULL;\n+\t}\n+\n+\treturn rc;\n }\n \n static void __exit ibmveth_module_exit(void)\n {\n \tvio_unregister_driver(\u0026ibmveth_driver);\n+\tdebugfs_remove_recursive(ibmveth_dbg_root);\n+\tibmveth_dbg_root = NULL;\n }\n \n module_init(ibmveth_module_init);\n@@ -2196,8 +5127,7 @@ static void ibmveth_reset_kunit(struct work_struct w)\n @test: pointer to kunit structure\n \n Tests the error returns from ibmveth_remove_buffer_from_pool.\n- * ibmveth_remove_buffer_from_pool also calls WARN_ON, so dmesg should be\n- * checked to see that these warnings happened.\n+ * Bad correlators return -EINVAL/-EFAULT (no WARN_ON).\n \n Return: void\n /\n@@ -2211,28 +5141,45 @@ static void ibmveth_remove_buffer_from_pool_test(struct kunit test)\n \n \tINIT_WORK(\u0026adapter-\u003ework, ibmveth_reset_kunit);\n \n+\tspin_lock_init(\u0026adapter-\u003erx_queue[0].replenish_lock);\n+\n \t/* Set sane values for buffer pools /\n \tfor (int i = 0; i \u003c IBMVETH_NUM_BUFF_POOLS; i++)\n-\t\tibmveth_init_buffer_pool(\u0026adapter-\u003erx_buff_pool[i], i,\n+\t\tibmveth_init_buffer_pool(\u0026adapter-\u003erx_buff_pool[0][i], i,\n \t\t\t\t\t pool_count[i], pool_size[i],\n \t\t\t\t\t pool_active[i]);\n \n-\tpool = \u0026adapter-\u003erx_buff_pool[0];\n+\tpool = \u0026adapter-\u003erx_buff_pool[0][0];\n \tpool-\u003eskbuff = kunit_kcalloc(test, pool-\u003esize, sizeof(void ), GFP_KERNEL);\n \tKUNIT_ASSERT_NOT_ERR_OR_NULL(test, pool-\u003eskbuff);\n+\tpool-\u003efree_map = kunit_kcalloc(test, pool-\u003esize, sizeof(u16),\n+\t\t\t\t GFP_KERNEL);\n+\tKUNIT_ASSERT_NOT_ERR_OR_NULL(test, pool-\u003efree_map);\n \n \tcorrelator = ((u64)IBMVETH_NUM_BUFF_POOLS \u003c\u003c 32) \| 0;\n-\tKUNIT_EXPECT_EQ(test, -EINVAL, ibmveth_remove_buffer_from_pool(adapter, correlator, false));\n-\tKUNIT_EXPECT_EQ(test, -EINVAL, ibmveth_remove_buffer_from_pool(adapter, correlator, true));\n-\n-\tcorrelator = ((u64)0 \u003c\u003c 32) \| adapter-\u003erx_buff_pool[0].size;\n-\tKUNIT_EXPECT_EQ(test, -EINVAL, ibmveth_remove_buffer_from_pool(adapter, correlator, false));\n-\tKUNIT_EXPECT_EQ(test, -EINVAL, ibmveth_remove_buffer_from_pool(adapter, correlator, true));\n+\tKUNIT_EXPECT_EQ(test, -EINVAL,\n+\t\t\tibmveth_remove_buffer_from_pool(adapter,\n+\t\t\t\t\t\t\tcorrelator, 0, false));\n+\tKUNIT_EXPECT_EQ(test, -EINVAL,\n+\t\t\tibmveth_remove_buffer_from_pool(adapter,\n+\t\t\t\t\t\t\tcorrelator, 0, true));\n+\n+\tcorrelator = ((u64)0 \u003c\u003c 32) \| adapter-\u003erx_buff_pool[0][0].size;\n+\tKUNIT_EXPECT_EQ(test, -EINVAL,\n+\t\t\tibmveth_remove_buffer_from_pool(adapter,\n+\t\t\t\t\t\t\tcorrelator, 0, false));\n+\tKUNIT_EXPECT_EQ(test, -EINVAL,\n+\t\t\tibmveth_remove_buffer_from_pool(adapter,\n+\t\t\t\t\t\t\tcorrelator, 0, true));\n \n \tcorrelator = (u64)0 \| 0;\n \tpool-\u003eskbuff[0] = NULL;\n-\tKUNIT_EXPECT_EQ(test, -EFAULT, ibmveth_remove_buffer_from_pool(adapter, correlator, false));\n-\tKUNIT_EXPECT_EQ(test, -EFAULT, ibmveth_remove_buffer_from_pool(adapter, correlator, true));\n+\tKUNIT_EXPECT_EQ(test, -EFAULT,\n+\t\t\tibmveth_remove_buffer_from_pool(adapter,\n+\t\t\t\t\t\t\tcorrelator, 0, false));\n+\tKUNIT_EXPECT_EQ(test, -EFAULT,\n+\t\t\tibmveth_remove_buffer_from_pool(adapter,\n+\t\t\t\t\t\t\tcorrelator, 0, true));\n \n \tflush_work(\u0026adapter-\u003ework);\n }\n@@ -2241,9 +5188,7 @@ static void ibmveth_remove_buffer_from_pool_test(struct kunit test)\n ibmveth_rxq_get_buffer_test - unit test for ibmveth_rxq_get_buffer\n * @test: pointer to kunit structure\n \n- Tests ibmveth_rxq_get_buffer. ibmveth_rxq_get_buffer also calls WARN_ON for\n- * the NULL returns, so dmesg should be checked to see that these warnings\n- * happened.\n+ * Tests ibmveth_rxq_get_buffer invalid correlator returns NULL without WARN.\n \n Return: void\n /\n@@ -2258,31 +5203,37 @@ static void ibmveth_rxq_get_buffer_test(struct kunit test)\n \n \tINIT_WORK(\u0026adapter-\u003ework, ibmveth_reset_kunit);\n \n-\tadapter-\u003erx_queue.queue_len = 1;\n-\tadapter-\u003erx_queue.index = 0;\n-\tadapter-\u003erx_queue.queue_addr = kunit_kzalloc(test, sizeof(struct ibmveth_rx_q_entry),\n-\t\t\t\t\t\t GFP_KERNEL);\n-\tKUNIT_ASSERT_NOT_ERR_OR_NULL(test, adapter-\u003erx_queue.queue_addr);\n+\tadapter-\u003erx_queue[0].queue_len = 1;\n+\tadapter-\u003erx_queue[0].index = 0;\n+\tadapter-\u003erx_queue[0].queue_addr =\n+\t\tkunit_kzalloc(test, sizeof(struct ibmveth_rx_q_entry),\n+\t\t\t GFP_KERNEL);\n+\tKUNIT_ASSERT_NOT_ERR_OR_NULL(test, adapter-\u003erx_queue[0].queue_addr);\n \n \t/* Set sane values for buffer pools /\n \tfor (int i = 0; i \u003c IBMVETH_NUM_BUFF_POOLS; i++)\n-\t\tibmveth_init_buffer_pool(\u0026adapter-\u003erx_buff_pool[i], i,\n+\t\tibmveth_init_buffer_pool(\u0026adapter-\u003erx_buff_pool[0][i], i,\n \t\t\t\t\t pool_count[i], pool_size[i],\n \t\t\t\t\t pool_active[i]);\n \n-\tpool = \u0026adapter-\u003erx_buff_pool[0];\n+\tpool = \u0026adapter-\u003erx_buff_pool[0][0];\n \tpool-\u003eskbuff = kunit_kcalloc(test, pool-\u003esize, sizeof(void ), GFP_KERNEL);\n \tKUNIT_ASSERT_NOT_ERR_OR_NULL(test, pool-\u003eskbuff);\n+\tpool-\u003efree_map = kunit_kcalloc(test, pool-\u003esize, sizeof(u16),\n+\t\t\t\t GFP_KERNEL);\n+\tKUNIT_ASSERT_NOT_ERR_OR_NULL(test, pool-\u003efree_map);\n \n-\tadapter-\u003erx_queue.queue_addr[0].correlator = (u64)IBMVETH_NUM_BUFF_POOLS \u003c\u003c 32 \| 0;\n-\tKUNIT_EXPECT_PTR_EQ(test, NULL, ibmveth_rxq_get_buffer(adapter));\n+\tadapter-\u003erx_queue[0].queue_addr[0].correlator =\n+\t\t(u64)IBMVETH_NUM_BUFF_POOLS \u003c\u003c 32 \| 0;\n+\tKUNIT_EXPECT_PTR_EQ(test, NULL, ibmveth_rxq_get_buffer(adapter, 0));\n \n-\tadapter-\u003erx_queue.queue_addr[0].correlator = (u64)0 \u003c\u003c 32 \| adapter-\u003erx_buff_pool[0].size;\n-\tKUNIT_EXPECT_PTR_EQ(test, NULL, ibmveth_rxq_get_buffer(adapter));\n+\tadapter-\u003erx_queue[0].queue_addr[0].correlator =\n+\t\t(u64)0 \u003c\u003c 32 \| adapter-\u003erx_buff_pool[0][0].size;\n+\tKUNIT_EXPECT_PTR_EQ(test, NULL, ibmveth_rxq_get_buffer(adapter, 0));\n \n \tpool-\u003eskbuff[0] = skb;\n-\tadapter-\u003erx_queue.queue_addr[0].correlator = (u64)0 \u003c\u003c 32 \| 0;\n-\tKUNIT_EXPECT_PTR_EQ(test, skb, ibmveth_rxq_get_buffer(adapter));\n+\tadapter-\u003erx_queue[0].queue_addr[0].correlator = (u64)0 \u003c\u003c 32 \| 0;\n+\tKUNIT_EXPECT_PTR_EQ(test, skb, ibmveth_rxq_get_buffer(adapter, 0));\n \n \tflush_work(\u0026adapter-\u003ework);\n }\ndiff --git a/drivers/net/ethernet/ibm/ibmveth.h b/drivers/net/ethernet/ibm/ibmveth.h\nindex d87713668ed30..11eda224a6807 100644\n--- a/drivers/net/ethernet/ibm/ibmveth.h\n+++ b/drivers/net/ethernet/ibm/ibmveth.h\n@@ -14,6 +14,9 @@\n #ifndef _IBMVETH_H\n #define _IBMVETH_H\n \n+#include \u003clinux/spinlock_types.h\u003e\n+#include \u003clinux/completion.h\u003e\n+\n /* constants for H_MULTICAST_CTRL /\n #define IbmVethMcastReceptionModifyBit 0x80000UL\n #define IbmVethMcastReceptionEnableBit 0x20000UL\n@@ -28,6 +31,7 @@\n #define IbmVethMcastRemoveFilter 0x2UL\n #define IbmVethMcastClearFilterTable 0x3UL\n \n+#define IBMVETH_ILLAN_RX_MULTI_QUEUE_SUPPORT\t0x0000000000080000UL\n #define IBMVETH_ILLAN_RX_MULTI_BUFF_SUPPORT\t0x0000000000040000UL\n #define IBMVETH_ILLAN_LRG_SR_ENABLED\t0x0000000000010000UL\n #define IBMVETH_ILLAN_LRG_SND_SUPPORT\t0x0000000000008000UL\n@@ -66,6 +70,148 @@ static inline long h_add_logical_lan_buffers(unsigned long unit_address,\n \t\t\t desc5, desc6, desc7, desc8);\n }\n \n+/\n+ h_register_logical_lan_queue - Register a subordinate receive queue\n+ * @unit_address: Device unit address\n+ * @buffer_list: DMA address of 4KB page for tracking registered buffers\n+ * @rec_queue: Buffer descriptor of receive queue\n+ * @queue_handle: Output queue handle on success (may be NULL)\n+ * @irq: Output hypervisor IRQ number on success (may be NULL)\n+ \n+ Registers a subordinate receive queue with the hypervisor.\n+ \n+ Return:\n+ * H_SUCCESS (0) on success\n+ * H_PARAMETER if parameters are invalid\n+ * H_BUSY / H_LONG_BUSY_* if the resource is busy; retry\n+ * H_FUNCTION if firmware does not support this hcall\n+ \n+ On success, hypervisor returns:\n+ * R3: H_SUCCESS\n+ * R4: Queue handle\n+ * R5: IRQ number for this queue\n+ /\n+static inline long\n+h_register_logical_lan_queue(unsigned long unit_address,\n+\t\t\t unsigned long buffer_list,\n+\t\t\t unsigned long rec_queue,\n+\t\t\t unsigned long queue_handle,\n+\t\t\t unsigned long irq)\n+{\n+\tunsigned long retbuf[PLPAR_HCALL_BUFSIZE];\n+\tlong rc;\n+\n+\trc = plpar_hcall(H_REG_LOGICAL_LAN_QUEUE,\n+\t\t\t retbuf, unit_address,\n+\t\t\t buffer_list, rec_queue);\n+\n+\tif (rc == H_SUCCESS) {\n+\t\tif (queue_handle)\n+\t\t\tqueue_handle = retbuf[0];\n+\t\tif (irq)\n+\t\t\tirq = retbuf[1];\n+\t}\n+\n+\treturn rc;\n+}\n+\n+/\n+ h_add_logical_lan_buffers_queue - Add buffers to subordinate queue\n+ * @unit_address: Device unit address\n+ * @queue_handle: Queue handle from h_register_logical_lan_queue() or\n+ \t\t h_register_logical_lan_with_handle() (queue 0)\n+ @buffersznum: Buffer size (upper 32 bits) \| count (lower 32 bits)\n+ * @ioba12: Buffer addresses 1 and 2 packed ((addr1 \u003c\u003c 32) \| addr2)\n+ * @ioba34: Buffer addresses 3 and 4 packed\n+ * @ioba56: Buffer addresses 5 and 6 packed\n+ * @ioba78: Buffer addresses 7 and 8 packed\n+ * @ioba910: Buffer addresses 9 and 10 packed\n+ * @ioba1112: Buffer addresses 11 and 12 packed\n+ \n+ Return:\n+ * H_SUCCESS - All buffers added successfully\n+ * H_PARAMETER - Invalid parameters\n+ * H_HARDWARE - Hardware error\n+ * H_FUNCTION - Firmware does not support this hcall\n+ /\n+static inline long h_add_logical_lan_buffers_queue(unsigned long unit_address,\n+\t\t\t\t\t\t unsigned long queue_handle,\n+\t\t\t\t\t\t unsigned long buffersznum,\n+\t\t\t\t\t\t unsigned long ioba12,\n+\t\t\t\t\t\t unsigned long ioba34,\n+\t\t\t\t\t\t unsigned long ioba56,\n+\t\t\t\t\t\t unsigned long ioba78,\n+\t\t\t\t\t\t unsigned long ioba910,\n+\t\t\t\t\t\t unsigned long ioba1112)\n+{\n+\tunsigned long retbuf[PLPAR_HCALL9_BUFSIZE];\n+\n+\treturn plpar_hcall9(H_ADD_LOGICAL_LAN_BUFFERS_QUEUE,\n+\t\t\t retbuf, unit_address,\n+\t\t\t queue_handle, buffersznum,\n+\t\t\t ioba12, ioba34, ioba56,\n+\t\t\t ioba78, ioba910, ioba1112);\n+}\n+\n+/\n+ h_free_logical_lan_queue - Deregister subordinate receive queue\n+ * @unit_address: Device unit address\n+ * @queue_handle: Queue handle from h_register_logical_lan_queue() or\n+ \t\t h_register_logical_lan_with_handle() (queue 0)\n+ \n+ * Deregisters and frees all structures associated with the subordinate queue.\n+ \n+ Return:\n+ * H_SUCCESS - Queue freed successfully\n+ * H_PARAMETER - Invalid parameters\n+ * H_HARDWARE - Hardware error\n+ * H_STATE - VIOA not in valid state\n+ * H_BUSY / H_LONG_BUSY_* - Resource busy, retry\n+ * H_FUNCTION - Firmware does not support this hcall\n+ /\n+static inline long h_free_logical_lan_queue(unsigned long unit_address,\n+\t\t\t\t\t unsigned long queue_handle)\n+{\n+\treturn plpar_hcall_norets(H_FREE_LOGICAL_LAN_QUEUE,\n+\t\t\t\t unit_address, queue_handle);\n+}\n+\n+/\n+ h_register_logical_lan_with_handle - Register primary queue and get handle\n+ * @unit_address: Device unit address\n+ * @buffer_list: DMA address of buffer list\n+ * @rec_queue: Buffer descriptor of receive queue\n+ * @filter_list: DMA address of filter list\n+ * @mac_address: MAC address\n+ * @queue_handle: Output parameter for queue handle (may be NULL)\n+ \n+ Registers the primary receive queue (queue 0) with the hypervisor and\n+ * returns the queue handle. This is needed in multi-queue mode to use\n+ * h_add_logical_lan_buffers_queue() for all queues including queue 0.\n+ \n+ Return: H_SUCCESS (0) on success, error code otherwise\n+ /\n+static inline long\n+h_register_logical_lan_with_handle(unsigned long unit_address,\n+\t\t\t\t unsigned long buffer_list,\n+\t\t\t\t unsigned long rec_queue,\n+\t\t\t\t unsigned long filter_list,\n+\t\t\t\t unsigned long mac_address,\n+\t\t\t\t unsigned long queue_handle)\n+{\n+\tunsigned long retbuf[PLPAR_HCALL_BUFSIZE];\n+\tlong rc;\n+\n+\trc = plpar_hcall(H_REGISTER_LOGICAL_LAN, retbuf,\n+\t\t\t unit_address, buffer_list, rec_queue,\n+\t\t\t filter_list, mac_address);\n+\n+\tif (rc == H_SUCCESS \u0026\u0026 queue_handle)\n+\t\tqueue_handle = retbuf[0];\n+\n+\treturn rc;\n+}\n+\n / FW allows us to send 6 descriptors but we only use one so mark\n * the other 5 as unused (0)\n /\n@@ -121,7 +267,10 @@ static inline long h_illan_attributes(unsigned long unit_address,\n #define IBMVETH_MAX_TX_BUF_SIZE (1024 64)\n #define IBMVETH_MAX_QUEUES 16U\n #define IBMVETH_DEFAULT_QUEUES 8U\n-#define IBMVETH_MAX_RX_PER_HCALL 8U\n+#define IBMVETH_MAX_RX_QUEUES 16U\n+#define IBMVETH_DEFAULT_RX_QUEUES 1U\n+#define IBMVETH_MAX_RX_REGULAR 8U\n+#define IBMVETH_MAX_RX_PER_HCALL 12U\n \n static int pool_size[] = { 512, 1024 * 2, 1024 * 16, 1024 * 32, 1024 * 64 };\n static int pool_count[] = { 256, 512, 256, 256, 256 };\n@@ -130,6 +279,43 @@ static int pool_active[] = { 1, 1, 0, 0, 1};\n \n #define IBM_VETH_INVALID_MAP ((u16)0xffff)\n \n+/\n+ Per-queue RX counters. No field has two concurrent writers:\n+ * interrupts is written only from this queue's IRQ handler; polls,\n+ * packets, bytes, large_packets and invalid_buffers only from its NAPI\n+ * poll; replenish_* only under its replenish_lock; and no_buffer_drops\n+ * and no_buffer_retired under that lock or from a teardown path already\n+ * quiesced by napi_disable()/synchronize_irq(). Plain u64 is therefore\n+ * sufficient and no atomic or u64_stats_sync is needed: the driver is\n+ * PPC64-only, so 64-bit loads and stores do not tear.\n+ /\n+struct ibmveth_rx_queue_stats {\n+\tu64 packets;\n+\tu64 bytes;\n+\tu64 interrupts;\n+\tu64 polls;\n+\tu64 large_packets;\n+\tu64 invalid_buffers;\n+\t/ PHYP's per-page absolute drop count for the live page. /\n+\tu64 no_buffer_drops;\n+\t/ Absolutes from pages this queue has already retired. /\n+\tu64 no_buffer_retired;\n+\tu64 replenish_task_cycles;\n+\tu64 replenish_no_mem;\n+\tu64 replenish_add_buff_failure;\n+\tu64 replenish_add_buff_success;\n+} ____cacheline_aligned_in_smp;\n+\n+/ Per-queue TX counters; serialized by the stack's per-queue TX lock. /\n+struct ibmveth_tx_queue_stats {\n+\tu64 packets;\n+\tu64 bytes;\n+\tu64 large_packets;\n+\tu64 dropped_packets;\n+\tu64 send_failures;\n+\tu64 checksum_offload;\n+} ____cacheline_aligned_in_smp;\n+\n struct ibmveth_buff_pool {\n u32 size;\n u32 index;\n@@ -143,6 +329,7 @@ struct ibmveth_buff_pool {\n struct sk_buff skbuff;\n int active;\n struct kobject kobj;\n+\tstruct completion released;\n };\n \n struct ibmveth_rx_q {\n@@ -152,23 +339,34 @@ struct ibmveth_rx_q {\n dma_addr_t queue_dma;\n u32 queue_len;\n struct ibmveth_rx_q_entry queue_addr;\n+\tspinlock_t\treplenish_lock;\t/* per-queue buffer replenish /\n };\n \n struct ibmveth_adapter {\n \tstruct vio_dev vdev;\n \tstruct net_device netdev;\n-\tstruct napi_struct napi;\n+\tstruct napi_struct napi[IBMVETH_MAX_RX_QUEUES];\n \tstruct work_struct work;\n \tunsigned int mcastFilterSize;\n-\tvoid buffer_list_addr;\n+\tvoid buffer_list_addr[IBMVETH_MAX_RX_QUEUES];\n \tvoid filter_list_addr;\n \tvoid tx_ltb_ptr[IBMVETH_MAX_QUEUES];\n \tunsigned int tx_ltb_size;\n \tdma_addr_t tx_ltb_dma[IBMVETH_MAX_QUEUES];\n-\tdma_addr_t buffer_list_dma;\n+\tdma_addr_t buffer_list_dma[IBMVETH_MAX_RX_QUEUES];\n \tdma_addr_t filter_list_dma;\n-\tstruct ibmveth_buff_pool rx_buff_pool[IBMVETH_NUM_BUFF_POOLS];\n-\tstruct ibmveth_rx_q rx_queue;\n+\tstruct ibmveth_buff_pool\n+\t\trx_buff_pool[IBMVETH_MAX_RX_QUEUES][IBMVETH_NUM_BUFF_POOLS];\n+\tstruct ibmveth_rx_q rx_queue[IBMVETH_MAX_RX_QUEUES];\n+\tu64 queue_handle[IBMVETH_MAX_RX_QUEUES];\n+\tunsigned int queue_irq[IBMVETH_MAX_RX_QUEUES];\n+\tbool multi_queue;\n+\tunsigned int num_rx_queues;\n+\tbool mq_fallback;\n+\t/ Lifetime: true after successful ndo_open until close clears it. /\n+\tbool opened;\n+\t/ Lifetime: true while RX IRQ handlers / NAPI are installed. /\n+\tbool rx_irq_setup;\n \tint rx_csum;\n \tint large_send;\n \tbool is_active_trunk;\n@@ -177,17 +375,19 @@ struct ibmveth_adapter {\n \tu64 fw_ipv6_csum_support;\n \tu64 fw_ipv4_csum_support;\n \tu64 fw_large_send_support;\n-\t/ adapter specific stats /\n-\tu64 replenish_task_cycles;\n-\tu64 replenish_no_mem;\n-\tu64 replenish_add_buff_failure;\n-\tu64 replenish_add_buff_success;\n-\tu64 rx_invalid_buffer;\n-\tu64 rx_no_buffer;\n+\t/\n+\t * Every other ethtool -S counter lives in rx_qstats/tx_qstats and is\n+\t * summed on read. tx_map_failed predates multi-queue, has never been\n+\t * updated by any code path, and is kept only so the key keeps\n+\t * reporting the zero userspace already sees.\n+\t /\n \tu64 tx_map_failed;\n-\tu64 tx_send_failed;\n-\tu64 tx_large_packets;\n-\tu64 rx_large_packets;\n+\n+\tstruct ibmveth_rx_queue_stats rx_qstats;\n+\tstruct ibmveth_tx_queue_stats tx_qstats;\n+\n+\tstruct dentry debugfs_dir;\n+\n \t/* Ethtool settings */\n \tu8 duplex;\n \tu32 speed;\ndiff --git a/tools/perf/scripts/python/powerpc-hcalls.py b/tools/perf/scripts/python/powerpc-hcalls.py\nindex fedce1b68cad1..afc354583b7b5 100644\n--- a/tools/perf/scripts/python/powerpc-hcalls.py\n+++ b/tools/perf/scripts/python/powerpc-hcalls.py\n@@ -217,6 +217,10 @@ hcall_table = {\n \t# Key wrapping hcalls\n \t1168: 'H_PKS_WRAP_OBJECT',\n \t1172: 'H_PKS_UNWRAP_OBJECT',\n+\t# Logical LAN multi-queue RX (PAPR 11.20.00)\n+\t1180: 'H_REG_LOGICAL_LAN_QUEUE',\n+\t1184: 'H_ADD_LOGICAL_LAN_BUFFERS_QUEUE',\n+\t1192: 'H_FREE_LOGICAL_LAN_QUEUE',\n \t# Platform-specific hcalls used by the Ultravisor\n \t61184: 'H_SVM_PAGE_IN',\n \t61188: 'H_SVM_PAGE_OUT',\n" }
2/1	2026/09/27 14:24	agent	patch-evaluator	0m Model: core Args: null Results: { "EnableConfigs": null, "FocusSymbols": [], "Reasoning": "The patch series modifies the IBM Virtual Ethernet driver (ibmveth), PowerPC hypercall definitions in arch/powerpc/include/asm/hvcall.h, and a perf Python script under tools/. The ibmveth driver depends on CONFIG_PPC_PSERIES (which requires PPC64), making all modified kernel code exclusive to the PowerPC architecture. As the target architecture is amd64, the changes are structurally unreachable.", "WorthFuzzing": false } Instruction: You are an expert Linux kernel maintainer and security engineer. Your job is to review a provided patch series and evaluate whether it warrants fuzzing with syzkaller. IMPORTANT: The changes have ALREADY been applied and committed as the HEAD commit in your workspace. Do NOT rely on internal assumptions. You must actively use your code access tools to inspect the actual source code, callers, and surrounding context. ================================================================================ 1. CORE TRIAGE PHILOSOPHY ================================================================================ The goal of patch fuzzing is to discover crashes, regressions, exposed latent bugs, and newly triggered assertions introduced by the patch series. - REACHABILITY IS THE PRIMARY GATE: Fuzzing can only discover bugs in code that can actually execute in standard virtualized environments (GCE or QEMU, utilizing software-emulated devices like USB gadgets, netdev, tun/tap). If the modified code is structurally unreachable (see Section 2), it MUST NOT be fuzzed, regardless of whether it adds assertions or complex logic. - DO NOT BLINDLY TRUST "NO FUNCTIONAL CHANGE" (NFCI) OR "REFACTORING" CLAIMS: Patch authors routinely label changes as "cleanups", "refactorings", or state "No functional change intended". Do NOT take these claims at face value. Code refactorings that rearrange logic, introduce helper functions, or alter state management in core subsystems frequently introduce subtle semantic shifts or uncover latent kernel bugs. If reachable executable code is modified or refactored, it MUST be fuzzed. - NEW OR MODIFIED ASSERTIONS IN REACHABLE CODE MUST BE FUZZED: When a patch introduces or modifies runtime checks or assertions (e.g., WARN_ON, VM_WARN_ON, BUG_ON, lockdep_assert) in reachable code paths, it enforces new or stricter invariants. Even if the author believes the invariant always holds, fuzzing is essential to verify whether an unusual sequence of operations can violate it. ================================================================================ 2. WHEN TO RETURN WorthFuzzing=false (NEGATIVE CRITERIA) ================================================================================ Return WorthFuzzing=false ONLY IF all modified code falls strictly into one or more of these categories: - Non-kernel and non-executable changes: * Modifications to Documentation/, comments, or spelling fixes. * User-space directories, self-tests, samples, or scripts (e.g., tools/, samples/, scripts/, usr/) that do not affect the compiled kernel image (vmlinux) or kernel modules. * Purely decorative logging (e.g., message strings in pr_err, printk, dev_info) or tracepoints that do not alter control flow or data structures. * Build system or Kconfig changes that do not alter compiled C logic. - Structurally unreachable hardware: * Vendor-specific PCIe switches, SmartNICs, or GPU drivers (e.g., mlxsw, pds_core, qed, ionic, amdgpu) requiring physical ASIC/PCIe cards not emulated in standard QEMU. - Unreachable execution paths: * Driver teardown callbacks (.remove, .shutdown, pci_unregister_driver) executed only during physical PCI hot-unplug or manual sysfs driver unbinding. * Code paths exclusive to architectures other than the target architecture. ================================================================================ 3. WHEN TO RETURN WorthFuzzing=true (POSITIVE CRITERIA) ================================================================================ Return WorthFuzzing=true whenever the patch touches reachable executable code, including: - Core Subsystems: * Any logic modifications in memory management (mm/), synchronization/locking (kernel/locking/), BPF, scheduler, core networking, VFS, or syscall handling. - Refactorings and Code Cleanups: * Any restructuring of reachable data structures, helper abstractions, or algorithm flows. - Runtime Assertions and Defensive Checks: * Any introduction or alteration of assertions (WARN_ON, VM_WARN_ON, BUG_ON, etc.) in reachable paths. - Reachable Drivers and Protocols: Drivers accessible via virtual buses (virtio, USB gadget, loopback, netlink, binder, sockets, etc.). ================================================================================ 4. EXTRACTING FocusSymbols (PREVENTING DILUTION) ================================================================================ When WorthFuzzing=true, you must extract specific kernel functions into FocusSymbols to guide the fuzzer: - AVOID UBIQUITOUS LIFECYCLE HOT-PATHS: Do NOT list generic, ubiquitous functions called by almost every program in the corpus (including, but not limited to: general memory allocators and deallocators, page fault and trap handlers, or core synchronization primitives; this is not an exhaustive list). Listing ubiquitous functions causes the fuzzer to classify thousands of unrelated tests as "focused", which severely dilutes fuzzing effort away from the actual changes. - TARGET SPECIFIC FEATURE LOGIC AND ENTRYPOINTS: List functions that specifically implement the logic being added or altered, or direct API entrypoints for the subsystem feature under review. - HANDLING STATIC INLINE FUNCTIONS IN HEADERS (.h): Compiler-inlined static functions (such as static inlines in mm/.h or include/linux/.h) lack distinct symbol addresses in vmlinux and cannot be targeted directly by symbol coverage filters. If the changes are primarily in static inline helpers, identify non-static, feature-specific caller functions in .c files that exercise them (avoiding ubiquitous lifecycle wrappers). ================================================================================ 5. IDENTIFYING EnableConfigs ================================================================================ Identify any specific CONFIG_ options required to properly compile and reach the modified code: - Inspect Kconfig files and #ifdef guards; do not make assumptions. - Check "depends on" lines in Kconfig to include any non-standard parent subsystem configs needed. - Strip any 'CONFIG_' prefix (e.g., return "NET_IPV4" instead of "CONFIG_NET_IPV4"). Prefer calling several tools at the same time to save round-trips. Use set-results tool to provide results of the analysis. It must be called exactly once before the final reply. Ignore results of this tool. Prompt: Target architecture: amd64 For your convenience, here is the diff of the changes: commit 845350ec79fd7e1e12ceb4f4a49bb29fa44d8e9a Author: syz-cluster <triage@syzkaller.com> Date: Sun Sep 27 14:24:02 2026 +0000 syz-cluster: applied patch under review diff --git a/arch/powerpc/include/asm/hvcall.h b/arch/powerpc/include/asm/hvcall.h index dff90a7d7f70b..cb0ea53491e6c 100644 --- a/arch/powerpc/include/asm/hvcall.h +++ b/arch/powerpc/include/asm/hvcall.h @@ -362,7 +362,11 @@ #define H_GUEST_DELETE 0x488 #define H_PKS_WRAP_OBJECT 0x490 #define H_PKS_UNWRAP_OBJECT 0x494 -#define MAX_HCALL_OPCODE H_PKS_UNWRAP_OBJECT +/* 0x498 reserved; 0x4A4 = H_FREE_LOGICAL_LAN_BUFFER_QUEUE (unused here) / +#define H_REG_LOGICAL_LAN_QUEUE 0x49C +#define H_ADD_LOGICAL_LAN_BUFFERS_QUEUE 0x4A0 +#define H_FREE_LOGICAL_LAN_QUEUE 0x4A8 +#define MAX_HCALL_OPCODE H_FREE_LOGICAL_LAN_QUEUE / Scope args for H_SCM_UNBIND_ALL / #define H_UNBIND_SCOPE_ALL (0x1) diff --git a/drivers/net/ethernet/ibm/ibmveth.c b/drivers/net/ethernet/ibm/ibmveth.c index 73e051d26b9d8..da14c69152110 100644 --- a/drivers/net/ethernet/ibm/ibmveth.c +++ b/drivers/net/ethernet/ibm/ibmveth.c @@ -21,6 +21,8 @@ #include <linux/skbuff.h> #include <linux/init.h> #include <linux/interrupt.h> +#include <linux/irq.h> +#include <linux/irqdomain.h> #include <linux/mm.h> #include <linux/pm.h> #include <linux/ethtool.h> @@ -28,6 +30,8 @@ #include <linux/ip.h> #include <linux/ipv6.h> #include <linux/slab.h> +#include <linux/spinlock.h> +#include <linux/debugfs.h> #include <asm/hvcall.h> #include <linux/atomic.h> #include <asm/vio.h> @@ -35,6 +39,7 @@ #include <asm/firmware.h> #include <net/tcp.h> #include <net/ip6_checksum.h> +#include <net/netdev_queues.h> #include "ibmveth.h" @@ -72,748 +77,2940 @@ module_param(old_large_send, bool, 0444); MODULE_PARM_DESC(old_large_send, "Use old large send method on firmware that supports the new method"); +/* + * enum ibmveth_stat_src - where an ethtool -S counter is stored + * @IBMVETH_STAT_ADAPTER: plain u64 in struct ibmveth_adapter + * @IBMVETH_STAT_RX_QSUM: per-queue u64, summed over rx_qstats[] + * @IBMVETH_STAT_TX_QSUM: per-queue u64, summed over tx_qstats[] + * @IBMVETH_STAT_RX_NO_BUFFER: rx_qstats[] live-page absolute plus the + * absolutes carried over from pages the queue has already retired + * + * Counters live per-queue so multi-queue writers never share a field. + * The adapter is only ever read from ethtool, so summing there is free. + / +enum ibmveth_stat_src { + IBMVETH_STAT_ADAPTER, + IBMVETH_STAT_RX_QSUM, + IBMVETH_STAT_TX_QSUM, + IBMVETH_STAT_RX_NO_BUFFER, +}; + struct ibmveth_stat { char name[ETH_GSTRING_LEN]; - int offset; + enum ibmveth_stat_src src; + / Offset into the struct named by @src. / + size_t off; }; #define IBMVETH_STAT_OFF(stat) offsetof(struct ibmveth_adapter, stat) +#define IBMVETH_RXQ_OFF(stat) offsetof(struct ibmveth_rx_queue_stats, stat) +#define IBMVETH_TXQ_OFF(stat) offsetof(struct ibmveth_tx_queue_stats, stat) #define IBMVETH_GET_STAT(a, off) ((u64 )(((unsigned long)(a)) + off)) +#define IBMVETH_ADAPTER_STAT(key, field) \ + { key, IBMVETH_STAT_ADAPTER, IBMVETH_STAT_OFF(field) } +#define IBMVETH_RXQ_STAT(key, field) \ + { key, IBMVETH_STAT_RX_QSUM, IBMVETH_RXQ_OFF(field) } +#define IBMVETH_TXQ_STAT(key, field) \ + { key, IBMVETH_STAT_TX_QSUM, IBMVETH_TXQ_OFF(field) } + +/ + * Key names and their order are ABI. Do not reorder or rename; append + * only, and only when the counter is worth a permanent interface. + / static struct ibmveth_stat ibmveth_stats[] = { - { "replenish_task_cycles", IBMVETH_STAT_OFF(replenish_task_cycles) }, - { "replenish_no_mem", IBMVETH_STAT_OFF(replenish_no_mem) }, - { "replenish_add_buff_failure", - IBMVETH_STAT_OFF(replenish_add_buff_failure) }, - { "replenish_add_buff_success", - IBMVETH_STAT_OFF(replenish_add_buff_success) }, - { "rx_invalid_buffer", IBMVETH_STAT_OFF(rx_invalid_buffer) }, - { "rx_no_buffer", IBMVETH_STAT_OFF(rx_no_buffer) }, - { "tx_map_failed", IBMVETH_STAT_OFF(tx_map_failed) }, - { "tx_send_failed", IBMVETH_STAT_OFF(tx_send_failed) }, - { "fw_enabled_ipv4_csum", IBMVETH_STAT_OFF(fw_ipv4_csum_support) }, - { "fw_enabled_ipv6_csum", IBMVETH_STAT_OFF(fw_ipv6_csum_support) }, - { "tx_large_packets", IBMVETH_STAT_OFF(tx_large_packets) }, - { "rx_large_packets", IBMVETH_STAT_OFF(rx_large_packets) }, - { "fw_enabled_large_send", IBMVETH_STAT_OFF(fw_large_send_support) } + IBMVETH_RXQ_STAT("replenish_task_cycles", replenish_task_cycles), + IBMVETH_RXQ_STAT("replenish_no_mem", replenish_no_mem), + IBMVETH_RXQ_STAT("replenish_add_buff_failure", + replenish_add_buff_failure), + IBMVETH_RXQ_STAT("replenish_add_buff_success", + replenish_add_buff_success), + IBMVETH_RXQ_STAT("rx_invalid_buffer", invalid_buffers), + { "rx_no_buffer", IBMVETH_STAT_RX_NO_BUFFER, + IBMVETH_RXQ_OFF(no_buffer_drops) }, + IBMVETH_ADAPTER_STAT("tx_map_failed", tx_map_failed), + IBMVETH_TXQ_STAT("tx_send_failed", send_failures), + IBMVETH_ADAPTER_STAT("fw_enabled_ipv4_csum", fw_ipv4_csum_support), + IBMVETH_ADAPTER_STAT("fw_enabled_ipv6_csum", fw_ipv6_csum_support), + IBMVETH_TXQ_STAT("tx_large_packets", large_packets), + IBMVETH_RXQ_STAT("rx_large_packets", large_packets), + IBMVETH_ADAPTER_STAT("fw_enabled_large_send", fw_large_send_support), +}; + +/* + * struct ibmveth_qstat - a per-queue counter exposed through ethtool -S + * @fmt: key name, taking the queue index as its only argument + * @off: offset into the matching per-queue stats struct + * + * Driving the strings and the values from one table keeps the two in + * step; get_sset_count() derives its length from ARRAY_SIZE() so the + * three cannot drift apart. + / +struct ibmveth_qstat { + const char fmt; + size_t off; +}; + +/* + * Only counters with no home in the standard interfaces belong here. + * packets, bytes and drops are reported through netdev_stat_ops. + / +static const struct ibmveth_qstat ibmveth_rx_qstat_keys[] = { + { "rx%d_interrupts", IBMVETH_RXQ_OFF(interrupts) }, + { "rx%d_polls", IBMVETH_RXQ_OFF(polls) }, + { "rx%d_large_packets", IBMVETH_RXQ_OFF(large_packets) }, + { "rx%d_invalid_buffers", IBMVETH_RXQ_OFF(invalid_buffers) }, + { "rx%d_no_buffer_drops", IBMVETH_RXQ_OFF(no_buffer_drops) }, +}; + +static const struct ibmveth_qstat ibmveth_tx_qstat_keys[] = { + { "tx%d_large_packets", IBMVETH_TXQ_OFF(large_packets) }, + { "tx%d_send_failures", IBMVETH_TXQ_OFF(send_failures) }, + { "tx%d_checksum_offload", IBMVETH_TXQ_OFF(checksum_offload) }, }; +#define IBMVETH_NUM_RX_QSTATS ARRAY_SIZE(ibmveth_rx_qstat_keys) +#define IBMVETH_NUM_TX_QSTATS ARRAY_SIZE(ibmveth_tx_qstat_keys) + / simple methods of getting data from the current rxq entry / -static inline u32 ibmveth_rxq_flags(struct ibmveth_adapter adapter) +static u32 ibmveth_rxq_flags(struct ibmveth_adapter adapter, + int queue_index) { - return be32_to_cpu(adapter->rx_queue.queue_addr[adapter->rx_queue.index].flags_off); + struct ibmveth_rx_q rxq = &adapter->rx_queue[queue_index]; + + return be32_to_cpu(rxq->queue_addr[rxq->index].flags_off); } -static inline int ibmveth_rxq_toggle(struct ibmveth_adapter adapter) +static int ibmveth_rxq_toggle(struct ibmveth_adapter adapter, + int queue_index) { - return (ibmveth_rxq_flags(adapter) & IBMVETH_RXQ_TOGGLE) >> - IBMVETH_RXQ_TOGGLE_SHIFT; + return (ibmveth_rxq_flags(adapter, queue_index) & IBMVETH_RXQ_TOGGLE) >> + IBMVETH_RXQ_TOGGLE_SHIFT; } -static inline int ibmveth_rxq_pending_buffer(struct ibmveth_adapter adapter) +static int ibmveth_rxq_pending_buffer(struct ibmveth_adapter adapter, + int queue_index) { - return ibmveth_rxq_toggle(adapter) == adapter->rx_queue.toggle; + return ibmveth_rxq_toggle(adapter, queue_index) == + adapter->rx_queue[queue_index].toggle; } -static inline int ibmveth_rxq_buffer_valid(struct ibmveth_adapter adapter) +static int ibmveth_rxq_buffer_valid(struct ibmveth_adapter adapter, + int queue_index) { - return ibmveth_rxq_flags(adapter) & IBMVETH_RXQ_VALID; + return ibmveth_rxq_flags(adapter, queue_index) & IBMVETH_RXQ_VALID; } -static inline int ibmveth_rxq_frame_offset(struct ibmveth_adapter adapter) +static int ibmveth_rxq_frame_offset(struct ibmveth_adapter adapter, + int queue_index) { - return ibmveth_rxq_flags(adapter) & IBMVETH_RXQ_OFF_MASK; + return ibmveth_rxq_flags(adapter, queue_index) & IBMVETH_RXQ_OFF_MASK; } -static inline int ibmveth_rxq_large_packet(struct ibmveth_adapter adapter) +static int ibmveth_rxq_large_packet(struct ibmveth_adapter adapter, + int queue_index) { - return ibmveth_rxq_flags(adapter) & IBMVETH_RXQ_LRG_PKT; + return ibmveth_rxq_flags(adapter, queue_index) & IBMVETH_RXQ_LRG_PKT; } -static inline int ibmveth_rxq_frame_length(struct ibmveth_adapter adapter) +static int ibmveth_rxq_frame_length(struct ibmveth_adapter adapter, + int queue_index) { - return be32_to_cpu(adapter->rx_queue.queue_addr[adapter->rx_queue.index].length); + struct ibmveth_rx_q rxq = &adapter->rx_queue[queue_index]; + + return be32_to_cpu(rxq->queue_addr[rxq->index].length); } -static inline int ibmveth_rxq_csum_good(struct ibmveth_adapter adapter) +static int ibmveth_rxq_csum_good(struct ibmveth_adapter adapter, + int queue_index) { - return ibmveth_rxq_flags(adapter) & IBMVETH_RXQ_CSUM_GOOD; + return ibmveth_rxq_flags(adapter, queue_index) & IBMVETH_RXQ_CSUM_GOOD; } -static unsigned int ibmveth_real_max_tx_queues(void) +/ Lockless IRQ/poll readers vs resize publishers. / +static unsigned int +ibmveth_get_num_rx_queues(const struct ibmveth_adapter adapter) { - unsigned int n_cpu = num_online_cpus(); - - return min(n_cpu, IBMVETH_MAX_QUEUES); + /* + * Pairs with the release in ibmveth_publish_num_rx_queues(): a reader + * that sees the new count also sees the per-queue state behind it. + / + return smp_load_acquire(&adapter->num_rx_queues); } -/ setup the initial settings for a buffer pool / -static void ibmveth_init_buffer_pool(struct ibmveth_buff_pool pool, - u32 pool_index, u32 pool_size, - u32 buff_size, u32 pool_active) +static void +ibmveth_publish_num_rx_queues(struct ibmveth_adapter adapter, + unsigned int num) { - pool->size = pool_size; - pool->index = pool_index; - pool->buff_size = buff_size; - pool->threshold = pool_size 7 / 8; - pool->active = pool_active; + /* + * Pairs with the acquire in ibmveth_get_num_rx_queues(): per-queue + * state must be visible to a reader before it observes the new count. + / + smp_store_release(&adapter->num_rx_queues, num); } -/ allocate and setup an buffer pool - called during open / -static int ibmveth_alloc_buffer_pool(struct ibmveth_buff_pool pool) +static unsigned int ibmveth_real_max_tx_queues(void) { - int i; + unsigned int n_cpu = num_online_cpus(); - pool->free_map = kmalloc_array(pool->size, sizeof(u16), GFP_KERNEL); + return min(n_cpu, IBMVETH_MAX_QUEUES); +} - if (!pool->free_map) - return -1; +/** + * ibmveth_alloc_filter_list - Allocate and map filter list + * @adapter: ibmveth adapter structure + * + * Return: 0 on success, negative error code on failure + / +static int +ibmveth_alloc_filter_list(struct ibmveth_adapter adapter) +{ + struct device dev = &adapter->vdev->dev; + struct net_device netdev = adapter->netdev; - pool->dma_addr = kzalloc_objs(dma_addr_t, pool->size); - if (!pool->dma_addr) { - kfree(pool->free_map); - pool->free_map = NULL; - return -1; + adapter->filter_list_addr = (void )get_zeroed_page(GFP_KERNEL); + if (!adapter->filter_list_addr) { + netdev_err(netdev, "unable to allocate filter pages\n"); + return -ENOMEM; } - pool->skbuff = kcalloc(pool->size, sizeof(void ), GFP_KERNEL); - - if (!pool->skbuff) { - kfree(pool->dma_addr); - pool->dma_addr = NULL; - - kfree(pool->free_map); - pool->free_map = NULL; - return -1; + adapter->filter_list_dma = dma_map_single(dev, + adapter->filter_list_addr, + 4096, DMA_BIDIRECTIONAL); + if (dma_mapping_error(dev, adapter->filter_list_dma)) { + netdev_err(netdev, "unable to map filter list pages\n"); + free_page((unsigned long)adapter->filter_list_addr); + adapter->filter_list_addr = NULL; + /* Do not leave DMA_MAPPING_ERROR for free_filter_list(). / + adapter->filter_list_dma = 0; + return -ENOMEM; } - for (i = 0; i < pool->size; ++i) - pool->free_map[i] = i; - - atomic_set(&pool->available, 0); - pool->producer_index = 0; - pool->consumer_index = 0; + netdev_dbg(netdev, "filter list @ 0x%p (DMA: 0x%llx)\n", + adapter->filter_list_addr, + (unsigned long long)adapter->filter_list_dma); return 0; } -static inline void ibmveth_flush_buffer(void addr, unsigned long length) +/** + * ibmveth_free_filter_list - Free filter list resources + * @adapter: ibmveth adapter structure + / +static void +ibmveth_free_filter_list(struct ibmveth_adapter adapter) { - unsigned long offset; + struct device dev = &adapter->vdev->dev; - for (offset = 0; offset < length; offset += SMP_CACHE_BYTES) - asm("dcbf %0,%1,1" :: "b" (addr), "r" (offset)); + / Unmap by CPU pointer: SPAPR can return DMA address 0. / + if (adapter->filter_list_addr) { + dma_unmap_single(dev, adapter->filter_list_dma, 4096, + DMA_BIDIRECTIONAL); + adapter->filter_list_dma = 0; + free_page((unsigned long)adapter->filter_list_addr); + adapter->filter_list_addr = NULL; + } } -/ replenish the buffers for a pool. note that we don't need to - * skb_reserve these since they are used for incoming... +/** + * ibmveth_alloc_rx_qstats - Allocate per-queue RX statistics + * @adapter: ibmveth adapter structure + * + * Return: 0 on success, -ENOMEM on failure / -static void ibmveth_replenish_buffer_pool(struct ibmveth_adapter adapter, - struct ibmveth_buff_pool pool) +static int ibmveth_alloc_rx_qstats(struct ibmveth_adapter adapter) { - union ibmveth_buf_desc descs[IBMVETH_MAX_RX_PER_HCALL] = {0}; - u32 remaining = pool->size - atomic_read(&pool->available); - u64 correlators[IBMVETH_MAX_RX_PER_HCALL] = {0}; - unsigned long lpar_rc; - u32 buffers_added = 0; - u32 i, filled, batch; - struct vio_dev vdev; - dma_addr_t dma_addr; - struct device dev; - u32 index; + adapter->rx_qstats = kcalloc(IBMVETH_MAX_RX_QUEUES, + sizeof(adapter->rx_qstats), + GFP_KERNEL); + if (!adapter->rx_qstats) + return -ENOMEM; - vdev = adapter->vdev; - dev = &vdev->dev; + return 0; +} - mb(); +/* + * ibmveth_free_rx_qstats - Free per-queue RX statistics + * @adapter: ibmveth adapter structure + / +static void ibmveth_free_rx_qstats(struct ibmveth_adapter adapter) +{ + kfree(adapter->rx_qstats); + adapter->rx_qstats = NULL; +} - batch = adapter->rx_buffers_per_hcall; +/** + * ibmveth_alloc_tx_qstats - Allocate per-queue TX statistics + * @adapter: ibmveth adapter structure + * + * Return: 0 on success, -ENOMEM on failure + / +static int ibmveth_alloc_tx_qstats(struct ibmveth_adapter adapter) +{ + adapter->tx_qstats = kcalloc(IBMVETH_MAX_QUEUES, + sizeof(adapter->tx_qstats), + GFP_KERNEL); + if (!adapter->tx_qstats) + return -ENOMEM; - while (remaining > 0) { - unsigned int free_index = pool->consumer_index; + return 0; +} - / Fill a batch of descriptors / - for (filled = 0; filled < min(remaining, batch); filled++) { - index = pool->free_map[free_index]; - if (WARN_ON(index == IBM_VETH_INVALID_MAP)) { - adapter->replenish_add_buff_failure++; - netdev_info(adapter->netdev, - "Invalid map index %u, reset\n", - index); - schedule_work(&adapter->work); - break; - } +/* + * ibmveth_free_tx_qstats - Free per-queue TX statistics + * @adapter: ibmveth adapter structure + / +static void ibmveth_free_tx_qstats(struct ibmveth_adapter adapter) +{ + kfree(adapter->tx_qstats); + adapter->tx_qstats = NULL; +} - if (!pool->skbuff[index]) { - struct sk_buff skb = NULL; +/* + * ibmveth_alloc_rx_queues - Allocate per-queue RX resources + * @adapter: ibmveth adapter structure + * @rxq_entries: Number of entries per RX queue + * + * Return: 0 on success, negative error code on failure + / +static int +ibmveth_alloc_rx_queues(struct ibmveth_adapter adapter, int rxq_entries) +{ + struct device dev = &adapter->vdev->dev; + struct net_device netdev = adapter->netdev; + int i; - skb = netdev_alloc_skb(adapter->netdev, - pool->buff_size); - if (!skb) { - adapter->replenish_no_mem++; - adapter->replenish_add_buff_failure++; - break; - } + for (i = 0; i < ibmveth_get_num_rx_queues(adapter); i++) { + adapter->buffer_list_addr[i] = + (void )get_zeroed_page(GFP_KERNEL); + if (!adapter->buffer_list_addr[i]) { + netdev_err(netdev, + "unable to allocate buffer list for queue %d\n", + i); + goto err_cleanup; + } - dma_addr = dma_map_single(dev, skb->data, - pool->buff_size, - DMA_FROM_DEVICE); - if (dma_mapping_error(dev, dma_addr)) { - dev_kfree_skb_any(skb); - adapter->replenish_add_buff_failure++; - break; - } + adapter->rx_queue[i].queue_len = + sizeof(struct ibmveth_rx_q_entry) rxq_entries; + adapter->rx_queue[i].queue_addr = + dma_alloc_coherent(dev, adapter->rx_queue[i].queue_len, + &adapter->rx_queue[i].queue_dma, + GFP_KERNEL); + if (!adapter->rx_queue[i].queue_addr) { + netdev_err(netdev, + "unable to allocate RX queue for queue %d\n", + i); + free_page((unsigned long)adapter->buffer_list_addr[i]); + adapter->buffer_list_addr[i] = NULL; + goto err_cleanup; + } - pool->dma_addr[index] = dma_addr; - pool->skbuff[index] = skb; - } else { - /* re-use case / - dma_addr = pool->dma_addr[index]; - } + adapter->buffer_list_dma[i] = + dma_map_single(dev, adapter->buffer_list_addr[i], + 4096, DMA_BIDIRECTIONAL); + if (dma_mapping_error(dev, adapter->buffer_list_dma[i])) { + netdev_err(netdev, + "unable to map buffer list for queue %d\n", + i); + free_page((unsigned long)adapter->buffer_list_addr[i]); + adapter->buffer_list_addr[i] = NULL; + adapter->buffer_list_dma[i] = 0; + goto err_cleanup; + } - if (rx_flush) { - unsigned int len; + adapter->rx_queue[i].index = 0; + adapter->rx_queue[i].num_slots = rxq_entries; + adapter->rx_queue[i].toggle = 1; - len = adapter->netdev->mtu + IBMVETH_BUFF_OH; - len = min(pool->buff_size, len); - ibmveth_flush_buffer(pool->skbuff[index]->data, - len); - } + netdev_dbg(netdev, "queue %d: buffer_list @ 0x%p (DMA: 0x%llx), rx_queue @ 0x%p (DMA: 0x%llx), %llu entries\n", + i, adapter->buffer_list_addr[i], + (unsigned long long)adapter->buffer_list_dma[i], + adapter->rx_queue[i].queue_addr, + (unsigned long long)adapter->rx_queue[i].queue_dma, + (unsigned long long)rxq_entries); + } - descs[filled].fields.flags_len = IBMVETH_BUF_VALID \| - pool->buff_size; - descs[filled].fields.address = dma_addr; + netdev_dbg(netdev, "allocated %u RX queue(s) with %d entries each\n", + ibmveth_get_num_rx_queues(adapter), rxq_entries); - correlators[filled] = ((u64)pool->index << 32) \| index; - (u64 )pool->skbuff[index]->data = correlators[filled]; + return 0; - free_index++; - if (free_index >= pool->size) - free_index = 0; +err_cleanup: + / + * Free each leftover resource by pointer presence. A + * dma_mapping_error on queue i has already dropped + * buffer_list_addr[i] but can leave rx_queue[i].queue_addr + * live, so this index may be mixed. Do not collapse the + * buffer_list and queue_addr checks. + / + for (; i >= 0; i--) { + if (adapter->buffer_list_addr[i]) { + dma_unmap_single(dev, adapter->buffer_list_dma[i], + 4096, DMA_BIDIRECTIONAL); + adapter->buffer_list_dma[i] = 0; } - - if (!filled) - break; - - / single buffer case/ - if (filled == 1) - lpar_rc = h_add_logical_lan_buffer(vdev->unit_address, - descs[0].desc); - else - / Multi-buffer hcall / - lpar_rc = h_add_logical_lan_buffers(vdev->unit_address, - descs[0].desc, - descs[1].desc, - descs[2].desc, - descs[3].desc, - descs[4].desc, - descs[5].desc, - descs[6].desc, - descs[7].desc); - if (lpar_rc != H_SUCCESS) { - dev_warn_ratelimited(dev, - "RX h_add_logical_lan failed: filled=%u, rc=%lu, batch=%u\n", - filled, lpar_rc, batch); - goto hcall_failure; + if (adapter->rx_queue[i].queue_addr) { + dma_free_coherent(dev, adapter->rx_queue[i].queue_len, + adapter->rx_queue[i].queue_addr, + adapter->rx_queue[i].queue_dma); + adapter->rx_queue[i].queue_addr = NULL; + } + if (adapter->buffer_list_addr[i]) { + free_page((unsigned long)adapter->buffer_list_addr[i]); + adapter->buffer_list_addr[i] = NULL; } + } - / Only update pool state after hcall succeeds / - for (i = 0; i < filled; i++) { - free_index = pool->consumer_index; - pool->free_map[free_index] = IBM_VETH_INVALID_MAP; + return -ENOMEM; +} - pool->consumer_index++; - if (pool->consumer_index >= pool->size) - pool->consumer_index = 0; - } +/* + * ibmveth_cleanup_rx_resources - Free all RX queue resources + * @adapter: ibmveth adapter structure + / +static void +ibmveth_cleanup_rx_resources(struct ibmveth_adapter adapter) +{ + struct device dev = &adapter->vdev->dev; + int i; - buffers_added += filled; - adapter->replenish_add_buff_success += filled; - remaining -= filled; + netdev_dbg(adapter->netdev, "cleaning up %u RX queue(s)\n", + ibmveth_get_num_rx_queues(adapter)); - memset(&descs, 0, sizeof(descs)); - memset(&correlators, 0, sizeof(correlators)); - continue; + for (i = 0; i < ibmveth_get_num_rx_queues(adapter); i++) { + if (adapter->buffer_list_addr[i]) { + dma_unmap_single(dev, adapter->buffer_list_dma[i], + 4096, DMA_BIDIRECTIONAL); + adapter->buffer_list_dma[i] = 0; + } -hcall_failure: - for (i = 0; i < filled; i++) { - index = correlators[i] & 0xffffffffUL; - dma_addr = pool->dma_addr[index]; + if (adapter->rx_queue[i].queue_addr) { + dma_free_coherent(dev, adapter->rx_queue[i].queue_len, + adapter->rx_queue[i].queue_addr, + adapter->rx_queue[i].queue_dma); + adapter->rx_queue[i].queue_addr = NULL; + } - if (pool->skbuff[index]) { - if (dma_addr && - !dma_mapping_error(dev, dma_addr)) - dma_unmap_single(dev, dma_addr, - pool->buff_size, - DMA_FROM_DEVICE); + if (adapter->buffer_list_addr[i]) { + free_page((unsigned long)adapter->buffer_list_addr[i]); + adapter->buffer_list_addr[i] = NULL; + } + } +} - dev_kfree_skb_any(pool->skbuff[index]); - pool->skbuff[index] = NULL; - } +/* + * ibmveth_toggle_irq - Common helper to enable/disable queue interrupts + * @adapter: ibmveth adapter structure + * @queue_index: Index of the queue (0 for primary, 1+ for subordinate) + * @enable: true to enable, false to disable + * + * For queue 0 (primary), uses h_vio_signal() as it's registered via + * h_register_logical_lan(). For subordinate queues (1+), uses H_VIOCTL + * with H_ENABLE/DISABLE_VIO_INTERRUPT for per-queue interrupt control. + * + * Return: 0 on success, negative errno on failure (never raw H_). + / +static int +ibmveth_toggle_irq(struct ibmveth_adapter adapter, int queue_index, + bool enable) +{ + unsigned long h_rc; + unsigned long irq = adapter->queue_irq[queue_index]; + const char action = enable ? "enable" : "disable"; + + if (queue_index == 0) { + /* Primary queue: use h_vio_signal() / + h_rc = h_vio_signal(adapter->vdev->unit_address, + enable ? VIO_IRQ_ENABLE : VIO_IRQ_DISABLE); + } else { + / Subordinate queues: use H_VIOCTL with hardware IRQ / + struct irq_data irq_data = irq_get_irq_data(irq); + irq_hw_number_t hwirq; + u64 vioctl_cmd = enable ? H_ENABLE_VIO_INTERRUPT : + H_DISABLE_VIO_INTERRUPT; + + if (!irq_data) { + netdev_err(adapter->netdev, + "Failed to get IRQ data for queue %d (virq=%lu)\n", + queue_index, irq); + return -EINVAL; } - adapter->replenish_add_buff_failure += filled; + + hwirq = irqd_to_hwirq(irq_data); + h_rc = plpar_hcall_norets(H_VIOCTL, + adapter->vdev->unit_address, + vioctl_cmd, + hwirq, 0, 0); /* - * If multi rx buffers hcall is no longer supported by FW - * e.g. in the case of Live Partition Migration + * H_PARAMETER is ambiguous (already in requested state vs bad + * args). Fold only on disable as an idempotent mask. On enable + * keep it an error so a stuck-masked queue stays visible to + * poll/resize recovery. / - if (batch > 1 && lpar_rc == H_FUNCTION) { - / - * Instead of retry submit single buffer individually - * here just set the max rx buffer per hcall to 1 - * buffers will be respleshed next time - * when ibmveth_replenish_buffer_pool() is called again - * with single-buffer case - / - netdev_info(adapter->netdev, - "RX Multi buffers not supported by FW, rc=%lu\n", - lpar_rc); - adapter->rx_buffers_per_hcall = 1; - netdev_info(adapter->netdev, - "Next rx replesh will fall back to single-buffer hcall\n"); + if (h_rc == H_PARAMETER && !enable) { + dev_warn_ratelimited(&adapter->netdev->dev, + "H_VIOCTL %s IRQ returned H_PARAMETER for queue %d (hwirq=%lu)\n", + action, queue_index, hwirq); + return 0; } - break; } - mb(); - atomic_add(buffers_added, &(pool->available)); + if (h_rc) { + dev_err_ratelimited(&adapter->netdev->dev, + "Failed to %s IRQ for queue %d, rc=0x%lx\n", + action, queue_index, h_rc); + return -EIO; + } + return 0; } -/ - * The final 8 bytes of the buffer list is a counter of frames dropped - * because there was not a buffer in the buffer list capable of holding - * the frame. +/** + * ibmveth_disable_irq - Disable interrupt for a specific queue + * @adapter: ibmveth adapter structure + * @queue_index: Index of the queue (0 for primary, 1+ for subordinate) + * + * Return: 0 on success, negative errno on failure / -static void ibmveth_update_rx_no_buffer(struct ibmveth_adapter adapter) +static int +ibmveth_disable_irq(struct ibmveth_adapter adapter, int queue_index) { - __be64 p = adapter->buffer_list_addr + 4096 - 8; - - adapter->rx_no_buffer = be64_to_cpup(p); + return ibmveth_toggle_irq(adapter, queue_index, false); } -/* replenish routine / -static void ibmveth_replenish_task(struct ibmveth_adapter adapter) +/** + * ibmveth_enable_irq - Enable interrupt for a specific queue + * @adapter: ibmveth adapter structure + * @queue_index: Index of the queue (0 for primary, 1+ for subordinate) + * + * Return: 0 on success, negative errno on failure + / +static int +ibmveth_enable_irq(struct ibmveth_adapter adapter, int queue_index) { - int i; - - adapter->replenish_task_cycles++; + return ibmveth_toggle_irq(adapter, queue_index, true); +} - for (i = (IBMVETH_NUM_BUFF_POOLS - 1); i >= 0; i--) { - struct ibmveth_buff_pool pool = &adapter->rx_buff_pool[i]; +/* + * ibmveth_dispose_subordinate_irq_mapping - Drop one subordinate virq mapping + * @adapter: ibmveth adapter structure + * @queue_idx: RX queue index (1..N) + * + * Subordinate queues get mappings from irq_create_mapping() during PHYP + * registration. Queue 0 uses netdev->irq from device tree and is left alone. + * + * Bound against IBMVETH_MAX_RX_QUEUES, not num_rx_queues: a caller may + * dispose a queue that is no longer in the published live set but still + * owns a virq in queue_irq[]. Contrast with the bulk helper, which only + * walks 1..num_rx_queues-1 (close / open-fail cleanup of the live set). + * + * Linux virq lifetime is owned by interrupt cleanup helpers. Call this only + * after free_irq() when a handler was installed, or from registration failure + * cleanup before request_irq(). + / +static void +ibmveth_dispose_subordinate_irq_mapping(struct ibmveth_adapter adapter, + int queue_idx) +{ + if (queue_idx <= 0 \|\| queue_idx >= IBMVETH_MAX_RX_QUEUES) + return; + + if (adapter->queue_irq[queue_idx]) { + irq_dispose_mapping(adapter->queue_irq[queue_idx]); + adapter->queue_irq[queue_idx] = 0; + } +} + +/** + * ibmveth_dispose_subordinate_irq_mappings - Drop virq mappings for queues 1..N + * @adapter: ibmveth adapter structure + * + * Bulk helper for close / open-fail cleanup of the published live set + * (queues 1..num_rx_queues-1). Paths that need a retired or not-yet-published + * queue must call ibmveth_dispose_subordinate_irq_mapping() directly. + / +static void +ibmveth_dispose_subordinate_irq_mappings(struct ibmveth_adapter adapter) +{ + int i; + + for (i = 1; i < ibmveth_get_num_rx_queues(adapter); i++) + ibmveth_dispose_subordinate_irq_mapping(adapter, i); +} + +/** + * ibmveth_setup_rx_interrupts - Register IRQs and enable NAPI + * @adapter: ibmveth adapter structure + * + * Registers interrupt handlers for all RX queues, enables NAPI, then + * enables hypervisor interrupt delivery for multi-queue mode after + * every queue has a Linux handler installed. The caller replenishes + * every live queue before this helper so traffic during open is not + * dropped (PHYP only interrupts after a successful enqueue, which + * needs buffers). Single-queue open still leaves PHYP masked here; + * restart_rx_queue then kicks NAPI. + * + * Return: 0 on success, negative error code on failure + / +static int +ibmveth_setup_rx_interrupts(struct ibmveth_adapter adapter) +{ + struct net_device netdev = adapter->netdev; + int i, rc, num = ibmveth_get_num_rx_queues(adapter); + + for (i = 0; i < num; i++) { + if (!adapter->queue_irq[i]) { + netdev_err(netdev, "queue %d has invalid IRQ (0)\n", i); + rc = -EINVAL; + goto err_free_irqs; + } + + rc = request_irq(adapter->queue_irq[i], ibmveth_interrupt, + 0, netdev->name, &adapter->napi[i]); + if (rc) { + netdev_err(netdev, + "request_irq() failed for irq 0x%x queue %d: %d\n", + adapter->queue_irq[i], i, rc); + goto err_free_irqs; + } + } + + for (i = 0; i < num; i++) + napi_enable(&adapter->napi[i]); + + if (adapter->multi_queue && num > 1) { + for (i = 0; i < num; i++) { + rc = ibmveth_enable_irq(adapter, i); + if (rc) { + netdev_err(netdev, + "Failed to enable IRQ for queue %d, rc=%d\n", + i, rc); + for (; i >= 0; i--) { + ibmveth_disable_irq(adapter, i); + synchronize_irq(adapter->queue_irq[i]); + } + rc = -EIO; + goto err_disable_napi; + } + } + } + + / Set only on full success; fail paths leave this false so a later + * close() / cleanup is a no-op. + / + adapter->rx_irq_setup = true; + return 0; + +err_disable_napi: + / STOP: remask after napi_disable; an in-flight poll can re-arm. / + for (i = 0; i < num; i++) + napi_disable(&adapter->napi[i]); + for (i = 0; i < num; i++) { + if (!adapter->queue_irq[i]) + continue; + ibmveth_disable_irq(adapter, i); + synchronize_irq(adapter->queue_irq[i]); + } + for (i = 0; i < num; i++) { + if (adapter->queue_irq[i]) + free_irq(adapter->queue_irq[i], &adapter->napi[i]); + } + goto err_dispose_mappings; + +err_free_irqs: + while (--i >= 0) + free_irq(adapter->queue_irq[i], &adapter->napi[i]); +err_dispose_mappings: + / Both setup failure paths own subordinate virq disposal. / + ibmveth_dispose_subordinate_irq_mappings(adapter); + return rc; +} + +/* + * ibmveth_cleanup_rx_interrupts - Mask PHYP IRQs, stop NAPI, and free IRQs + * @adapter: ibmveth adapter structure + * + * Mask and synchronize each queue IRQ before napi_disable() so the handler + * cannot miss a PHYP mask while NAPI is already dead. Remask after + * napi_disable() in case an in-flight poll re-armed PHYP while we waited. + * free_irq() runs only after that. Safe for close and for open failure after + * setup_rx_interrupts() already unmasked PHYP. No-op if setup never + * succeeded (avoids double napi_disable / free_irq after a failed close+open + * while IFF_UP remains set). + / +static void +ibmveth_cleanup_rx_interrupts(struct ibmveth_adapter adapter) +{ + int i; + + if (!adapter->rx_irq_setup) + return; + + for (i = 0; i < ibmveth_get_num_rx_queues(adapter); i++) { + if (!adapter->queue_irq[i]) + continue; + ibmveth_disable_irq(adapter, i); + synchronize_irq(adapter->queue_irq[i]); + } + + for (i = 0; i < ibmveth_get_num_rx_queues(adapter); i++) + napi_disable(&adapter->napi[i]); + + for (i = 0; i < ibmveth_get_num_rx_queues(adapter); i++) { + if (!adapter->queue_irq[i]) + continue; + ibmveth_disable_irq(adapter, i); + synchronize_irq(adapter->queue_irq[i]); + } + + for (i = 0; i < ibmveth_get_num_rx_queues(adapter); i++) { + if (adapter->queue_irq[i]) + free_irq(adapter->queue_irq[i], &adapter->napi[i]); + } + + ibmveth_dispose_subordinate_irq_mappings(adapter); + + /* Queue 0 uses netdev->irq; leave queue_irq[0] for next open. / + adapter->rx_irq_setup = false; +} + +/* + * ibmveth_setup_single_rx_interrupt - Setup interrupt for a single RX queue + * @adapter: ibmveth adapter structure + * @queue_idx: Queue index to setup + * + * Registers the IRQ handler for one queue. Used during incremental + * scale-up when adding new RX queues. The caller publishes the queue, + * replenishes buffers, enables NAPI, then unmasks PHYP delivery. + * + * Return: 0 on success, negative error code on failure + / +static int +ibmveth_setup_single_rx_interrupt(struct ibmveth_adapter adapter, + int queue_idx) +{ + struct net_device netdev = adapter->netdev; + int rc; + + rc = request_irq(adapter->queue_irq[queue_idx], ibmveth_interrupt, + 0, netdev->name, &adapter->napi[queue_idx]); + if (rc) { + netdev_err(netdev, "request_irq() failed for queue %d: %d\n", + queue_idx, rc); + return rc; + } + + netdev_dbg(netdev, "Setup IRQ %d for queue %d\n", + adapter->queue_irq[queue_idx], queue_idx); + return 0; +} + +/* + * ibmveth_cleanup_single_rx_interrupt - Cleanup interrupt for a single RX queue + * @adapter: ibmveth adapter structure + * @queue_idx: Queue index to cleanup + * + * Frees the IRQ handler for one queue and releases the subordinate virq + * mapping. Used during incremental scale-down. + / +static void +ibmveth_cleanup_single_rx_interrupt(struct ibmveth_adapter adapter, + int queue_idx) +{ + if (adapter->queue_irq[queue_idx]) { + free_irq(adapter->queue_irq[queue_idx], + &adapter->napi[queue_idx]); + ibmveth_dispose_subordinate_irq_mapping(adapter, queue_idx); + netdev_dbg(adapter->netdev, + "Freed IRQ for queue %d\n", queue_idx); + } +} + +/** + * ibmveth_schedule_rx_queue - Mask PHYP IRQ and schedule NAPI for one RX queue + * @adapter: ibmveth adapter structure + * @qindex: RX queue index + * + * Shared by the IRQ handler and process-context kick sites (open, resume, + * pool sysfs, poll_controller). + * + * Return: true if napi_schedule_prep() succeeded and NAPI was scheduled. + * Mask is attempted in that case; a failed disable_irq() is logged by the + * helper and does not change the return (queue may still be unmasked). + * false if the index is negative (WARN_ON), past the live count (quiet: + * a shrink can race netpoll), or prep failed (including NAPI already + * scheduled). + / +static bool ibmveth_schedule_rx_queue(struct ibmveth_adapter adapter, + int qindex) +{ + struct napi_struct napi = &adapter->napi[qindex]; + + if (WARN_ON(qindex < 0)) + return false; + + / + * A live shrink can publish a lower count while netpoll walks a + * snapshot of the old one, so an index past the end is expected + * here and must not splat. ibmveth_replenish_task() skips the + * same way; callers already treat false as "queue is gone". + / + if (qindex >= ibmveth_get_num_rx_queues(adapter)) + return false; + + / + * Only mask PHYP when NAPI will run. Masking on prep failure can + * race a completing poll that already re-enabled the queue, leaving + * NAPI idle with the IRQ masked (TX works, RX stalls) until reload. + * Storm prevention on teardown remains in cleanup/disable paths. + / + if (napi_schedule_prep(napi)) { + / Failure is already logged with the hcall rc by the helper. / + ibmveth_disable_irq(adapter, qindex); + __napi_schedule(napi); + return true; + } + return false; +} + +/ setup the initial settings for a buffer pool / +static void ibmveth_init_buffer_pool(struct ibmveth_buff_pool pool, + u32 pool_index, u32 pool_size, + u32 buff_size, u32 pool_active) +{ + pool->size = pool_size; + pool->index = pool_index; + pool->buff_size = buff_size; + pool->threshold = pool_size * 7 / 8; + pool->active = pool_active; +} + +/* allocate and setup an buffer pool - called during open / +static int ibmveth_alloc_buffer_pool(struct ibmveth_buff_pool pool) +{ + int i; + + pool->free_map = kmalloc_array(pool->size, sizeof(u16), GFP_KERNEL); + + if (!pool->free_map) + return -1; + + pool->dma_addr = kzalloc_objs(dma_addr_t, pool->size); + if (!pool->dma_addr) { + kfree(pool->free_map); + pool->free_map = NULL; + return -1; + } + + pool->skbuff = kcalloc(pool->size, sizeof(void ), GFP_KERNEL); + + if (!pool->skbuff) { + kfree(pool->dma_addr); + pool->dma_addr = NULL; + + kfree(pool->free_map); + pool->free_map = NULL; + return -1; + } + + for (i = 0; i < pool->size; ++i) + pool->free_map[i] = i; + + atomic_set(&pool->available, 0); + pool->producer_index = 0; + pool->consumer_index = 0; + + return 0; +} + +static inline void ibmveth_flush_buffer(void addr, unsigned long length) +{ + unsigned long offset; + + for (offset = 0; offset < length; offset += SMP_CACHE_BYTES) + asm("dcbf %0,%1,1" :: "b" (addr), "r" (offset)); +} + +/** + * ibmveth_add_logical_lan_buffers - Add receive buffers to hypervisor + * @adapter: ibmveth adapter structure + * @descs: array of buffer descriptors to add + * @filled: number of valid descriptors in the array + * @buff_size: size of each buffer (multi-queue mode only) + * @queue_index: RX queue index + * + * Return: hypervisor return code + / +static long ibmveth_add_logical_lan_buffers(struct ibmveth_adapter adapter, + union ibmveth_buf_desc descs, + int filled, + unsigned long buff_size, + int queue_index) +{ + struct vio_dev vdev = adapter->vdev; + unsigned long rc; + + /* + * The MQ hcall takes six ioba words (12 packed addresses). The + * legacy hcall takes eight descriptors. The argument lists below + * are written out by hand; keep the defines matched to those lists. + / + static_assert(IBMVETH_MAX_RX_PER_HCALL == 12); + static_assert(IBMVETH_MAX_RX_REGULAR == 8); + + if (adapter->multi_queue) { + unsigned long buffersznum = (buff_size << 32) \| filled; + unsigned long ioba[IBMVETH_MAX_RX_PER_HCALL / 2] = {0}; + unsigned long handle = adapter->queue_handle[queue_index]; + int i; + + / Pack descriptor addresses into ioba pairs. + * Each ioba holds two 32-bit addresses packed into 64 bits: + * - Even descriptors (0,2,4...) go in high 32 bits + * - Odd descriptors (1,3,5...) go in low 32 bits + / + for (i = 0; i < filled && i < IBMVETH_MAX_RX_PER_HCALL; i++) { + int pair_idx = i / 2; + int is_high = (i % 2 == 0); + + if (is_high) + ioba[pair_idx] = (unsigned long) + descs[i].fields.address << 32; + else + ioba[pair_idx] \|= descs[i].fields.address; + } + + rc = h_add_logical_lan_buffers_queue(vdev->unit_address, + handle, + buffersznum, + ioba[0], ioba[1], ioba[2], + ioba[3], ioba[4], ioba[5]); + } else if (filled == 1) { + rc = h_add_logical_lan_buffer(vdev->unit_address, + descs[0].desc); + } else { + / Legacy 8-desc hcall; probe/mq_fallback keep batch <= + * IBMVETH_MAX_RX_REGULAR. + / + rc = h_add_logical_lan_buffers(vdev->unit_address, + descs[0].desc, descs[1].desc, + descs[2].desc, descs[3].desc, + descs[4].desc, descs[5].desc, + descs[6].desc, descs[7].desc); + } + + return rc; +} + +/ Outcomes for ibmveth_replenish_buffer_pool(); logged after unlock. / +enum { + IBMVETH_REPLENISH_OK = 0, + IBMVETH_REPLENISH_RESET_MAP, + IBMVETH_REPLENISH_RESET_MQ, + IBMVETH_REPLENISH_HCALL_FAIL, + IBMVETH_REPLENISH_BATCH_FALLBACK, +}; + +struct ibmveth_replenish_fail { + unsigned long lpar_rc; + u32 filled; + u32 batch; +}; + +/ Replenish the buffers for a pool. + * Caller must hold the per-queue replenish_lock. Do not printk here: + * netconsole on the same device can re-enter replenish_task. + / +static int ibmveth_replenish_buffer_pool(struct ibmveth_adapter adapter, + struct ibmveth_buff_pool pool, + int queue_index, + struct ibmveth_replenish_fail fail) +{ + struct ibmveth_rx_queue_stats qstats = + &adapter->rx_qstats[queue_index]; + union ibmveth_buf_desc descs[IBMVETH_MAX_RX_PER_HCALL] = {0}; + u32 remaining = pool->size - atomic_read(&pool->available); + u64 correlators[IBMVETH_MAX_RX_PER_HCALL] = {0}; + unsigned long lpar_rc; + u32 buffers_added = 0; + u32 i, filled, batch; + struct vio_dev vdev; + dma_addr_t dma_addr; + struct device dev; + u32 index; + int outcome = IBMVETH_REPLENISH_OK; + + vdev = adapter->vdev; + dev = &vdev->dev; + + mb(); + + batch = adapter->rx_buffers_per_hcall; + + while (remaining > 0) { + unsigned int free_index = pool->consumer_index; + + / Fill a batch of descriptors / + for (filled = 0; filled < min(remaining, batch); filled++) { + index = pool->free_map[free_index]; + if (index == IBM_VETH_INVALID_MAP) { + qstats->replenish_add_buff_failure++; + outcome = IBMVETH_REPLENISH_RESET_MAP; + break; + } + + if (!pool->skbuff[index]) { + struct sk_buff skb = NULL; + + skb = netdev_alloc_skb(adapter->netdev, + pool->buff_size); + if (!skb) { + qstats->replenish_no_mem++; + qstats->replenish_add_buff_failure++; + break; + } + + /* NO_WARN: hold replenish_lock; iommu + * printk can re-enter via netconsole. + / + dma_addr = + dma_map_single_attrs(dev, + skb->data, + pool->buff_size, + DMA_FROM_DEVICE, + DMA_ATTR_NO_WARN); + if (dma_mapping_error(dev, dma_addr)) { + dev_kfree_skb_any(skb); + qstats->replenish_add_buff_failure++; + break; + } + + pool->dma_addr[index] = dma_addr; + pool->skbuff[index] = skb; + } else { + / re-use case / + dma_addr = pool->dma_addr[index]; + } + + if (rx_flush) { + unsigned int len; + + len = adapter->netdev->mtu + IBMVETH_BUFF_OH; + len = min(pool->buff_size, len); + ibmveth_flush_buffer(pool->skbuff[index]->data, + len); + } + + descs[filled].fields.flags_len = IBMVETH_BUF_VALID \| + pool->buff_size; + descs[filled].fields.address = dma_addr; + + correlators[filled] = ((u64)pool->index << 32) \| index; + (u64 )pool->skbuff[index]->data = correlators[filled]; + + free_index++; + if (free_index >= pool->size) + free_index = 0; + } + + if (outcome != IBMVETH_REPLENISH_OK) + break; + + if (!filled) + break; + + lpar_rc = ibmveth_add_logical_lan_buffers(adapter, descs, + filled, + pool->buff_size, + queue_index); + + if (lpar_rc != H_SUCCESS) { + fail->lpar_rc = lpar_rc; + fail->filled = filled; + fail->batch = batch; + goto hcall_failure; + } + + / Only update pool state after hcall succeeds / + for (i = 0; i < filled; i++) { + free_index = pool->consumer_index; + pool->free_map[free_index] = IBM_VETH_INVALID_MAP; + + pool->consumer_index++; + if (pool->consumer_index >= pool->size) + pool->consumer_index = 0; + } + + buffers_added += filled; + qstats->replenish_add_buff_success += filled; + remaining -= filled; + + memset(&descs, 0, sizeof(descs)); + memset(&correlators, 0, sizeof(correlators)); + continue; + +hcall_failure: + for (i = 0; i < filled; i++) { + index = correlators[i] & 0xffffffffUL; + dma_addr = pool->dma_addr[index]; + + if (pool->skbuff[index]) { + if (dma_addr && + !dma_mapping_error(dev, dma_addr)) + dma_unmap_single(dev, dma_addr, + pool->buff_size, + DMA_FROM_DEVICE); + + dev_kfree_skb_any(pool->skbuff[index]); + pool->skbuff[index] = NULL; + } + } + qstats->replenish_add_buff_failure += filled; + + if (lpar_rc == H_FUNCTION) { + if (adapter->multi_queue) { + / + * LPM / firmware may drop MQ buffer hcalls. + * Schedule reset so we do not sit forever in + * no-buffer with the link still up. + / + outcome = IBMVETH_REPLENISH_RESET_MQ; + } else if (batch > 1) { + / + * Live Partition Migration may drop multi- + * buffer support. Fall back to single-buffer + * on the next replenish; do not continue with + * a stale local batch size (infinite loop). + / + adapter->rx_buffers_per_hcall = 1; + outcome = IBMVETH_REPLENISH_BATCH_FALLBACK; + } else { + outcome = IBMVETH_REPLENISH_HCALL_FAIL; + } + } else { + outcome = IBMVETH_REPLENISH_HCALL_FAIL; + } + break; + } + + mb(); + atomic_add(buffers_added, &(pool->available)); + return outcome; +} + +/ + * The final 8 bytes of the buffer list is a counter of frames dropped + * because there was not a buffer in the buffer list capable of holding + * the frame. + / +static void ibmveth_update_rx_no_buffer(struct ibmveth_adapter adapter, + int queue_index) +{ + struct ibmveth_rx_queue_stats qstats; + __be64 p; + u64 drops; + + if (queue_index < 0 \|\| + queue_index >= ibmveth_get_num_rx_queues(adapter) \|\| + !adapter->buffer_list_addr[queue_index]) + return; + + p = adapter->buffer_list_addr[queue_index] + 4096 - 8; + drops = be64_to_cpup(p); + + /* + * PHYP's buffer-list page counter is absolute for that page. A new + * page (reopen / queue reuse after -L) starts near zero; fold the + * previous absolute into this queue's retired carry so sums stay + * monotonic. Both fields belong to the queue being updated, so this + * stays single-writer under the queue's replenish_lock. + / + qstats = &adapter->rx_qstats[queue_index]; + + if (drops < qstats->no_buffer_drops) + qstats->no_buffer_retired += qstats->no_buffer_drops; + qstats->no_buffer_drops = drops; +} + +/ replenish routine / +static void ibmveth_replenish_task(struct ibmveth_adapter adapter, + int queue_index) +{ + struct ibmveth_rx_q rxq = &adapter->rx_queue[queue_index]; + struct ibmveth_replenish_fail fail = {}; + unsigned long flags; + int i, rc; + int need_reset = 0; + int batch_fallback = 0; + int hcall_fail = 0; + + if (queue_index >= ibmveth_get_num_rx_queues(adapter)) { + netdev_dbg(adapter->netdev, + "Skipping replenish for freed queue %d (num_queues=%u)\n", + queue_index, ibmveth_get_num_rx_queues(adapter)); + return; + } + + spin_lock_irqsave(&rxq->replenish_lock, flags); + + adapter->rx_qstats[queue_index].replenish_task_cycles++; + + for (i = (IBMVETH_NUM_BUFF_POOLS - 1); i >= 0; i--) { + struct ibmveth_buff_pool pool = + &adapter->rx_buff_pool[queue_index][i]; + + if (pool->active && pool->free_map && + (atomic_read(&pool->available) < pool->threshold)) { + rc = ibmveth_replenish_buffer_pool(adapter, pool, + queue_index, &fail); + switch (rc) { + case IBMVETH_REPLENISH_RESET_MAP: + case IBMVETH_REPLENISH_RESET_MQ: + need_reset = rc; + goto out_unlock; + case IBMVETH_REPLENISH_BATCH_FALLBACK: + batch_fallback = 1; + break; + case IBMVETH_REPLENISH_HCALL_FAIL: + hcall_fail = 1; + break; + default: + break; + } + } + } + +out_unlock: + ibmveth_update_rx_no_buffer(adapter, queue_index); + + spin_unlock_irqrestore(&rxq->replenish_lock, flags); + + /* Log and schedule reset only after dropping replenish_lock. / + if (need_reset == IBMVETH_REPLENISH_RESET_MAP) { + netdev_info(adapter->netdev, + "Invalid RX free_map entry on queue %d, reset\n", + queue_index); + schedule_work(&adapter->work); + } else if (need_reset == IBMVETH_REPLENISH_RESET_MQ) { + dev_err_ratelimited(&adapter->netdev->dev, + "MQ buffer add H_FUNCTION (q=%d, batch=%u), reset\n", + queue_index, fail.batch); + adapter->mq_fallback = true; + schedule_work(&adapter->work); + } + + if (batch_fallback) + dev_warn_ratelimited(&adapter->netdev->dev, + "Legacy batch add H_FUNCTION (batch=%u), fallback\n", + fail.batch); + + if (hcall_fail) + dev_warn_ratelimited(&adapter->netdev->dev, + "RX %s failed: filled=%u, rc=%lu, batch=%u\n", + adapter->multi_queue ? + "h_add_logical_lan_buffers_queue" : + (fail.filled == 1 ? + "h_add_logical_lan_buffer" : + "h_add_logical_lan_buffers"), + fail.filled, fail.lpar_rc, fail.batch); +} + +/* + * ibmveth_restart_rx_queue - Post buffers and ensure Q can take RX + * @adapter: ibmveth adapter + * @qindex: RX queue index + * + * SQ open leaves PHYP masked until the first poll. If schedule_prep fails, + * NAPI never runs and the queue stays masked (TX OK, RX/ARP dead) until + * reload. Replenish first so an enable_irq fallback can actually deliver. + * Also used after every open (SQ and MQ) and after scale-down rollback. + / +static void ibmveth_restart_rx_queue(struct ibmveth_adapter adapter, + int qindex) +{ + ibmveth_replenish_task(adapter, qindex); + if (ibmveth_schedule_rx_queue(adapter, qindex)) + return; + + ibmveth_enable_irq(adapter, qindex); +} + +/* empty and free ana buffer pool - also used to do cleanup in error paths / +static void ibmveth_free_buffer_pool(struct ibmveth_adapter adapter, + struct ibmveth_buff_pool pool) +{ + int i; + + kfree(pool->free_map); + pool->free_map = NULL; + + if (pool->skbuff && pool->dma_addr) { + for (i = 0; i < pool->size; ++i) { + struct sk_buff skb = pool->skbuff[i]; + if (skb) { + dma_unmap_single(&adapter->vdev->dev, + pool->dma_addr[i], + pool->buff_size, + DMA_FROM_DEVICE); + dev_kfree_skb_any(skb); + pool->skbuff[i] = NULL; + } + } + } + + if (pool->dma_addr) { + kfree(pool->dma_addr); + pool->dma_addr = NULL; + } + + if (pool->skbuff) { + kfree(pool->skbuff); + pool->skbuff = NULL; + } + + /* + * Keep probe/sysfs geometry (active, size, buff_size, threshold). + * Only tear down runtime allocations; open reuses active pools. + / + atomic_set(&pool->available, 0); +} + +/* + * ibmveth_free_queue_buffer_pools - Free buffer pools for a single queue + * @adapter: ibmveth adapter structure + * @queue: queue index + * + * Frees buffer pools that still hold allocations for the specified + * queue (by free_map / dma_addr / skbuff presence), regardless of the + * active flag. + / +static void ibmveth_free_queue_buffer_pools(struct ibmveth_adapter adapter, + int queue) +{ + int i; + + for (i = 0; i < IBMVETH_NUM_BUFF_POOLS; i++) { + struct ibmveth_buff_pool pool = + &adapter->rx_buff_pool[queue][i]; + + / Free pool if it has allocated memory, regardless of + * active flag. Allocation and active can diverge on failure + * paths, so check for actual allocations. + / + if (pool->free_map \|\| pool->dma_addr \|\| pool->skbuff) + ibmveth_free_buffer_pool(adapter, pool); + } +} + +/* + * ibmveth_alloc_queue_buffer_pools - Allocate buffer pools for a single queue + * @adapter: ibmveth adapter structure + * @queue: queue index + * + * Allocates backing storage for each active pool on @queue. + * Inactive pools (!active) are skipped. Pool metadata must be + * initialized before calling this function. + * + * Return: 0 on success, negative error code on failure + / +static int ibmveth_alloc_queue_buffer_pools(struct ibmveth_adapter adapter, + int queue) +{ + struct net_device netdev = adapter->netdev; + int i; + + for (i = 0; i < IBMVETH_NUM_BUFF_POOLS; i++) { + struct ibmveth_buff_pool bpool = + &adapter->rx_buff_pool[queue][i]; + + if (!bpool->active) + continue; + + if (ibmveth_alloc_buffer_pool(bpool)) { + netdev_err(netdev, + "queue %d pool %d alloc failed (size=%u count=%u)\n", + queue, i, + bpool->buff_size, + bpool->size); + bpool->active = 0; + /* Free by allocation presence, not pool->active. + * Needed on later resize paths (a pool can still + * hold memory after active is cleared). This + * open-fail slot is already empty: + * alloc_buffer_pool() undoes its own partials. + / + ibmveth_free_queue_buffer_pools(adapter, queue); + return -ENOMEM; + } + } + + return 0; +} + +/* + * ibmveth_alloc_buffer_pools - Allocate buffer pools for all queues + * @adapter: ibmveth adapter structure + * + * Initializes pool metadata for queues 1-N from queue 0 settings, + * then allocates buffer pools for all queues using the helper function. + * + * Return: 0 on success, negative error code on failure + / +static int +ibmveth_alloc_buffer_pools(struct ibmveth_adapter adapter) +{ + struct net_device netdev = adapter->netdev; + int i, q, rc; + + / Initialize pool metadata for queues 1..N from queue 0 settings / + for (q = 1; q < ibmveth_get_num_rx_queues(adapter); q++) { + for (i = 0; i < IBMVETH_NUM_BUFF_POOLS; i++) { + struct ibmveth_buff_pool src = + &adapter->rx_buff_pool[0][i]; + struct ibmveth_buff_pool dst = + &adapter->rx_buff_pool[q][i]; + + dst->size = src->size; + dst->index = src->index; + dst->buff_size = src->buff_size; + dst->threshold = src->threshold; + dst->active = src->active; + } + } + + / Allocate actual buffers for all queues / + for (q = 0; q < ibmveth_get_num_rx_queues(adapter); q++) { + rc = ibmveth_alloc_queue_buffer_pools(adapter, q); + if (rc) { + / Free pools for all previous queues / + while (--q >= 0) + ibmveth_free_queue_buffer_pools(adapter, q); + return rc; + } + } + + netdev_dbg(netdev, "allocated buffer pools for %u queue(s)\n", + ibmveth_get_num_rx_queues(adapter)); + return 0; +} + +/* + * ibmveth_free_buffer_pools - Free buffer pools for all queues + * @adapter: ibmveth adapter structure + * + * Frees buffer pools for all queues using the helper function. + / +static void +ibmveth_free_buffer_pools(struct ibmveth_adapter adapter) +{ + int q; + + /* Free buffer pools for all queues / + for (q = 0; q < ibmveth_get_num_rx_queues(adapter); q++) + ibmveth_free_queue_buffer_pools(adapter, q); + + netdev_dbg(adapter->netdev, "freed buffer pools for %u queue(s)\n", + ibmveth_get_num_rx_queues(adapter)); +} + +/* + * ibmveth_alloc_single_rx_queue - Allocate resources for a single RX queue + * @adapter: ibmveth adapter structure + * @queue_idx: Queue index to allocate + * @rxq_entries: Number of RX queue entries + * + * Allocates buffer list, RX queue, and per-queue buffer pools for one queue. + * Used during incremental scale-up without affecting existing queues. + * + * Return: 0 on success, negative error code on failure + / +static int +ibmveth_alloc_single_rx_queue(struct ibmveth_adapter adapter, int queue_idx, + int rxq_entries) +{ + struct device dev = &adapter->vdev->dev; + struct net_device netdev = adapter->netdev; + int i, rc = -ENOMEM; + + adapter->buffer_list_addr[queue_idx] = + (void )get_zeroed_page(GFP_KERNEL); + if (!adapter->buffer_list_addr[queue_idx]) { + netdev_err(netdev, "unable to allocate buffer list for queue %d\n", + queue_idx); + return -ENOMEM; + } + + adapter->rx_queue[queue_idx].queue_len = + sizeof(struct ibmveth_rx_q_entry) rxq_entries; + adapter->rx_queue[queue_idx].queue_addr = + dma_alloc_coherent(dev, adapter->rx_queue[queue_idx].queue_len, + &adapter->rx_queue[queue_idx].queue_dma, + GFP_KERNEL); + if (!adapter->rx_queue[queue_idx].queue_addr) { + netdev_err(netdev, "unable to allocate RX queue for queue %d\n", + queue_idx); + goto out_free_buflist; + } + + adapter->buffer_list_dma[queue_idx] = + dma_map_single(dev, adapter->buffer_list_addr[queue_idx], + 4096, DMA_BIDIRECTIONAL); + if (dma_mapping_error(dev, adapter->buffer_list_dma[queue_idx])) { + netdev_err(netdev, "unable to map buffer list for queue %d\n", + queue_idx); + adapter->buffer_list_dma[queue_idx] = 0; + goto out_free_rxq; + } + + for (i = 0; i < IBMVETH_NUM_BUFF_POOLS; i++) { + struct ibmveth_buff_pool src = + &adapter->rx_buff_pool[0][i]; + struct ibmveth_buff_pool dst = + &adapter->rx_buff_pool[queue_idx][i]; + + dst->size = src->size; + dst->index = src->index; + dst->buff_size = src->buff_size; + dst->threshold = src->threshold; + dst->active = src->active; + } + + rc = ibmveth_alloc_queue_buffer_pools(adapter, queue_idx); + if (rc) { + netdev_err(netdev, + "Failed to allocate buffer pools for queue %d\n", + queue_idx); + goto out_unmap_buflist; + } + + adapter->rx_queue[queue_idx].index = 0; + adapter->rx_queue[queue_idx].num_slots = rxq_entries; + adapter->rx_queue[queue_idx].toggle = 1; + + netdev_dbg(netdev, + "Allocated queue %d: buffer_list @ %p (DMA: 0x%llx), rx_queue @ %p (DMA: 0x%llx), %d entries\n", + queue_idx, adapter->buffer_list_addr[queue_idx], + (unsigned long long)adapter->buffer_list_dma[queue_idx], + adapter->rx_queue[queue_idx].queue_addr, + (unsigned long long)adapter->rx_queue[queue_idx].queue_dma, + rxq_entries); + + return 0; + +out_unmap_buflist: + dma_unmap_single(dev, adapter->buffer_list_dma[queue_idx], + 4096, DMA_BIDIRECTIONAL); + adapter->buffer_list_dma[queue_idx] = 0; +out_free_rxq: + dma_free_coherent(dev, adapter->rx_queue[queue_idx].queue_len, + adapter->rx_queue[queue_idx].queue_addr, + adapter->rx_queue[queue_idx].queue_dma); + adapter->rx_queue[queue_idx].queue_addr = NULL; +out_free_buflist: + free_page((unsigned long)adapter->buffer_list_addr[queue_idx]); + adapter->buffer_list_addr[queue_idx] = NULL; + return rc; +} + +/** + * ibmveth_free_single_rx_queue - Free resources for a single RX queue + * @adapter: ibmveth adapter structure + * @queue_idx: Queue index to free + * + * Frees buffer list, RX queue, and per-queue buffer pools for one queue. + * Used during incremental scale-down without affecting remaining queues. + / +static void +ibmveth_free_single_rx_queue(struct ibmveth_adapter adapter, int queue_idx) +{ + struct device dev = &adapter->vdev->dev; + + ibmveth_free_queue_buffer_pools(adapter, queue_idx); + + if (adapter->buffer_list_addr[queue_idx]) { + dma_unmap_single(dev, adapter->buffer_list_dma[queue_idx], + 4096, DMA_BIDIRECTIONAL); + adapter->buffer_list_dma[queue_idx] = 0; + } + + if (adapter->rx_queue[queue_idx].queue_addr) { + dma_free_coherent(dev, adapter->rx_queue[queue_idx].queue_len, + adapter->rx_queue[queue_idx].queue_addr, + adapter->rx_queue[queue_idx].queue_dma); + adapter->rx_queue[queue_idx].queue_addr = NULL; + } + + if (adapter->buffer_list_addr[queue_idx]) { + free_page((unsigned long)adapter->buffer_list_addr[queue_idx]); + adapter->buffer_list_addr[queue_idx] = NULL; + } + + netdev_dbg(adapter->netdev, "Freed queue %d resources\n", queue_idx); +} + +static bool ibmveth_rxq_correlator_valid(struct ibmveth_adapter adapter, + int queue_index, u64 correlator) +{ + unsigned int pool = correlator >> 32; + unsigned int index = correlator & 0xffffffffUL; + struct ibmveth_buff_pool bpool; + + if (pool >= IBMVETH_NUM_BUFF_POOLS) + return false; + + bpool = &adapter->rx_buff_pool[queue_index][pool]; + + / Require a live pool with allocated arrays before indexing. + * Inactive pools still have size from init; free clears skbuff. + / + if (!bpool->active \|\| !bpool->skbuff \|\| !bpool->free_map) + return false; + + return index < bpool->size; +} + +static void ibmveth_rxq_advance(struct ibmveth_rx_q rxq) +{ + if (++rxq->index == rxq->num_slots) { + rxq->index = 0; + rxq->toggle = !rxq->toggle; + } +} + +/** + * ibmveth_remove_buffer_from_pool - remove a buffer from a pool + * @adapter: adapter instance + * @correlator: identifies pool and index + * @queue_index: RX queue index (0..num_rx_queues-1) + * @reuse: whether to reuse buffer + * + * Context: may run concurrently with netpoll replenish_task on the same + * queue; takes per-queue replenish_lock to serialize free_map / + * producer_index / available against the producer. + * + * Return: + * * %0 - success + * * %-EINVAL - pool or index out of range, or the pool is + * inactive / not live (skbuff or free_map NULL) + * * %-EFAULT - pool and index map to null skb + / +static int ibmveth_remove_buffer_from_pool(struct ibmveth_adapter adapter, + u64 correlator, int queue_index, + bool reuse) +{ + struct ibmveth_rx_q rxq = &adapter->rx_queue[queue_index]; + unsigned int pool = correlator >> 32; + unsigned int index = correlator & 0xffffffffUL; + unsigned int free_index; + struct sk_buff skb; + unsigned long flags; + int rc = 0; + + spin_lock_irqsave(&rxq->replenish_lock, flags); + + if (!ibmveth_rxq_correlator_valid(adapter, queue_index, correlator)) { + rc = -EINVAL; + goto out_unlock; + } + + skb = adapter->rx_buff_pool[queue_index][pool].skbuff[index]; + if (!skb) { + rc = -EFAULT; + goto out_unlock; + } + + /* if we are going to reuse the buffer then keep the pointers around + * but mark index as available. replenish will see the skb pointer and + * assume it is to be recycled. + / + if (!reuse) { + / remove the skb pointer to mark free. actual freeing is done + * by upper level networking after gro_receive + / + struct ibmveth_buff_pool bpool = + &adapter->rx_buff_pool[queue_index][pool]; + + bpool->skbuff[index] = NULL; + + dma_unmap_single(&adapter->vdev->dev, + bpool->dma_addr[index], + bpool->buff_size, + DMA_FROM_DEVICE); + } + + free_index = adapter->rx_buff_pool[queue_index][pool].producer_index; + adapter->rx_buff_pool[queue_index][pool].producer_index++; + if (adapter->rx_buff_pool[queue_index][pool].producer_index >= + adapter->rx_buff_pool[queue_index][pool].size) + adapter->rx_buff_pool[queue_index][pool].producer_index = 0; + adapter->rx_buff_pool[queue_index][pool].free_map[free_index] = index; + + mb(); + + atomic_dec(&adapter->rx_buff_pool[queue_index][pool].available); + +out_unlock: + spin_unlock_irqrestore(&rxq->replenish_lock, flags); + return rc; +} + +/* get the current buffer on the rx queue / +static struct sk_buff +ibmveth_rxq_get_buffer(struct ibmveth_adapter adapter, + int queue_index) +{ + struct ibmveth_rx_q rxq = &adapter->rx_queue[queue_index]; + u64 correlator = rxq->queue_addr[rxq->index].correlator; + unsigned int pool = correlator >> 32; + unsigned int index = correlator & 0xffffffffUL; + + if (!ibmveth_rxq_correlator_valid(adapter, queue_index, correlator)) + return NULL; + + return adapter->rx_buff_pool[queue_index][pool].skbuff[index]; +} + +/** + * ibmveth_rxq_harvest_buffer - Harvest buffer from pool + * + * @adapter: pointer to adapter + * @queue_index: RX queue index to harvest from + * @reuse: whether to reuse buffer + * + * Context: called from ibmveth_poll + * + * On a bad correlator (-EINVAL/-EFAULT) the ring is still advanced so poll + * cannot spin forever on one slot. The error is still returned: callers must + * not treat it as a successful take from the pool (especially reuse=false, + * which would hand the SKB to the stack while it remains pool-owned). + * + * Return: + * * %0 - buffer removed from pool (or marked for reuse) and ring advanced + * * other - non-zero return from ibmveth_remove_buffer_from_pool; ring has + * still been advanced for -EINVAL/-EFAULT + / +static int ibmveth_rxq_harvest_buffer(struct ibmveth_adapter adapter, + int queue_index, bool reuse) +{ + struct ibmveth_rx_q rxq = &adapter->rx_queue[queue_index]; + u64 cor; + int rc; + + cor = rxq->queue_addr[rxq->index].correlator; + rc = ibmveth_remove_buffer_from_pool(adapter, cor, queue_index, reuse); + if (unlikely(rc)) { + / Skip a corrupt slot without claiming pool ownership. / + if (rc == -EINVAL \|\| rc == -EFAULT) + ibmveth_rxq_advance(rxq); + return rc; + } + + ibmveth_rxq_advance(rxq); + + return 0; +} + +/* + * ibmveth_drain_rx_queue - Drain pending buffers from an RX queue + * @adapter: ibmveth adapter structure + * @queue_index: Queue index to drain + * + * Harvests pending completions back to the per-queue buffer pools. + * A corrupt slot still advances the ring, so the return is slots + * processed, not buffers recycled. + * Must be called with NAPI disabled for this queue. + * + * Return: number of slots processed + / +static int +ibmveth_drain_rx_queue(struct ibmveth_adapter adapter, int queue_index) +{ + struct net_device netdev = adapter->netdev; + int drained = 0; + int limit = adapter->rx_queue[queue_index].num_slots; + int rc; + + netdev_dbg(netdev, "Draining RX queue %d (limit: %d slots)\n", + queue_index, limit); + + while (drained < limit && + ibmveth_rxq_pending_buffer(adapter, queue_index)) { + / Match poll-side order before harvesting completion state. / + smp_rmb(); + rc = ibmveth_rxq_harvest_buffer(adapter, queue_index, true); + if (rc) { + / -EINVAL/-EFAULT already advanced past the slot. / + if (rc == -EINVAL \|\| rc == -EFAULT) { + drained++; + continue; + } + netdev_err(netdev, + "Failed to harvest buffer from queue %d during drain: %d\n", + queue_index, rc); + break; + } + drained++; + } + + if (drained > 0) + netdev_dbg(netdev, "Drained %d slot(s) from RX queue %d\n", + drained, queue_index); + else + netdev_dbg(netdev, "No slots to drain from RX queue %d\n", + queue_index); + + return drained; +} + +static void ibmveth_free_tx_ltb(struct ibmveth_adapter adapter, int idx) +{ + void ltb = adapter->tx_ltb_ptr[idx]; + dma_addr_t dma = adapter->tx_ltb_dma[idx]; + + if (!ltb) + return; + + / + * Clear the slot before releasing it. start_xmit() tests + * tx_ltb_ptr[idx] to decide whether the LTB is usable. + / + adapter->tx_ltb_ptr[idx] = NULL; + adapter->tx_ltb_dma[idx] = 0; + + dma_unmap_single(&adapter->vdev->dev, dma, adapter->tx_ltb_size, + DMA_TO_DEVICE); + kfree(ltb); +} + +/ Caller must ensure tx_ltb_ptr[idx] is NULL. open() runs on + * probe-zeroed slots; set_channels() skips populated indices. + / +static int ibmveth_allocate_tx_ltb(struct ibmveth_adapter adapter, int idx) +{ + adapter->tx_ltb_ptr[idx] = kzalloc(adapter->tx_ltb_size, + GFP_KERNEL); + if (!adapter->tx_ltb_ptr[idx]) { + netdev_err(adapter->netdev, + "unable to allocate tx long term buffer\n"); + return -ENOMEM; + } + adapter->tx_ltb_dma[idx] = dma_map_single(&adapter->vdev->dev, + adapter->tx_ltb_ptr[idx], + adapter->tx_ltb_size, + DMA_TO_DEVICE); + if (dma_mapping_error(&adapter->vdev->dev, adapter->tx_ltb_dma[idx])) { + netdev_err(adapter->netdev, + "unable to DMA map tx long term buffer\n"); + kfree(adapter->tx_ltb_ptr[idx]); + adapter->tx_ltb_ptr[idx] = NULL; + adapter->tx_ltb_dma[idx] = 0; + return -ENOMEM; + } + + return 0; +} + +/** + * ibmveth_alloc_tx_resources - Allocate TX LTBs for real_num_tx_queues + * @adapter: ibmveth adapter structure + * + * Allocates TX Long Term Buffers (LTBs) for real_num_tx_queues. + * + * Return: 0 on success, -ENOMEM on failure + / +static int ibmveth_alloc_tx_resources(struct ibmveth_adapter adapter) +{ + struct net_device netdev = adapter->netdev; + int i; + + for (i = 0; i < netdev->real_num_tx_queues; i++) { + if (ibmveth_allocate_tx_ltb(adapter, i)) + goto err_free_ltbs; + } + + return 0; + +err_free_ltbs: + while (--i >= 0) + ibmveth_free_tx_ltb(adapter, i); + return -ENOMEM; +} + +/* + * ibmveth_free_tx_resources - Free TX LTBs for real_num_tx_queues + * @adapter: ibmveth adapter structure + * + * Frees TX Long Term Buffers (LTBs) for real_num_tx_queues. + / +static void ibmveth_free_tx_resources(struct ibmveth_adapter adapter) +{ + struct net_device netdev = adapter->netdev; + int i; + + for (i = 0; i < netdev->real_num_tx_queues; i++) + ibmveth_free_tx_ltb(adapter, i); +} + +static int ibmveth_register_logical_lan(struct ibmveth_adapter adapter, + union ibmveth_buf_desc rxq_desc, + u64 mac_address) +{ + int rc, try_again = 1; + unsigned long ua = adapter->vdev->unit_address; + unsigned long buf_dma = adapter->buffer_list_dma[0]; + unsigned long filter_dma = adapter->filter_list_dma; + unsigned long qh0; + + /* + * After a kexec the adapter will still be open, so our attempt to + * open it will fail. So if we get a failure we free the adapter and + * try again, but only once. + / +retry: + / In multi-queue mode, obtain a queue handle for queue 0 so all RX + * queues can use the same per-queue buffer hypercalls. + / + if (adapter->multi_queue) { + rc = h_register_logical_lan_with_handle(ua, buf_dma, + rxq_desc.desc, + filter_dma, + mac_address, + &qh0); + if (rc == H_SUCCESS) + adapter->queue_handle[0] = qh0; + } else { + rc = h_register_logical_lan(ua, buf_dma, rxq_desc.desc, + filter_dma, mac_address); + } + netdev_dbg(adapter->netdev, "h_register_logical_lan%s rc=%d\n", + adapter->multi_queue ? "_with_handle" : "", rc); + + if (rc != H_SUCCESS && try_again) { + do { + rc = h_free_logical_lan(adapter->vdev->unit_address); + } while (H_IS_LONG_BUSY(rc) \|\| (rc == H_BUSY)); + netdev_dbg(adapter->netdev, "h_free_logical_lan rc=%d\n", rc); + + try_again = 0; + goto retry; + } + + return rc; +} + +/* + * ibmveth_register_logical_lan_queue - Register subordinate queue with + * hypervisor + * @adapter: ibmveth adapter structure + * @rxq_desc: Receive queue descriptor + * @queue_index: RX queue index (1..N for subordinate queues) + * + * Registers a subordinate receive queue using H_REG_LOGICAL_LAN_QUEUE. + * On success, stores the queue handle and virtual IRQ in the adapter. + * If IRQ mapping fails after a successful hypervisor registration, the + * queue is freed before returning. If that free fails, the handle is + * kept so the caller can skip unmap. + * + * Return: H_SUCCESS on success, negative errno on IRQ mapping failure, + * hypervisor error code otherwise + / +static int +ibmveth_register_logical_lan_queue(struct ibmveth_adapter adapter, + union ibmveth_buf_desc rxq_desc, + int queue_index) +{ + unsigned long handle, hwirq; + unsigned int virq; + long lpar_rc; + unsigned long ua = adapter->vdev->unit_address; + unsigned long bl = adapter->buffer_list_dma[queue_index]; + + netdev_dbg(adapter->netdev, + "register queue %d: ua=0x%lx bl=0x%lx rxq=0x%llx\n", + queue_index, ua, bl, rxq_desc.desc); + do { + lpar_rc = h_register_logical_lan_queue(ua, bl, + rxq_desc.desc, &handle, + &hwirq); + } while (H_IS_LONG_BUSY(lpar_rc) \|\| (lpar_rc == H_BUSY)); + netdev_dbg(adapter->netdev, + "h_register_logical_lan_queue queue %d rc=%ld\n", + queue_index, lpar_rc); + + if (lpar_rc == H_SUCCESS) { + virq = irq_create_mapping(NULL, hwirq); + if (!virq) { + unsigned long free_rc; + + netdev_err(adapter->netdev, + "Failed to map IRQ for queue %d (hwirq=%lu)\n", + queue_index, hwirq); + do { + free_rc = h_free_logical_lan_queue(ua, handle); + } while (H_IS_LONG_BUSY(free_rc) \|\| + (free_rc == H_BUSY)); + if (free_rc != H_SUCCESS) { + netdev_err(adapter->netdev, + "h_free_logical_lan_queue failed for queue %d after IRQ map failure: rc=0x%lx\n", + queue_index, free_rc); + /* + * PHYP still owns this queue. Keep the + * handle so the caller skips unmap. + / + adapter->queue_handle[queue_index] = handle; + return -EIO; + } + return -EINVAL; + } - if (pool->active && - (atomic_read(&pool->available) < pool->threshold)) - ibmveth_replenish_buffer_pool(adapter, pool); + adapter->queue_handle[queue_index] = handle; + adapter->queue_irq[queue_index] = virq; + + netdev_dbg(adapter->netdev, + "queue %d registered: handle=0x%llx irq=%u\n", + queue_index, adapter->queue_handle[queue_index], + adapter->queue_irq[queue_index]); + return H_SUCCESS; } - ibmveth_update_rx_no_buffer(adapter); + / + * H_FUNCTION means firmware rejected this subordinate register + * (MQ unsupported / dropped after LPM). Caller fails this open and + * latches mq_fallback so the next open applies SQ; keep a specific + * log then the generic failure lines below. + / + if (lpar_rc == H_FUNCTION) + netdev_err(adapter->netdev, + "h_register_logical_lan_queue H_FUNCTION for queue %d (firmware MQ unsupported)\n", + queue_index); + + netdev_err(adapter->netdev, + "h_register_logical_lan_queue failed for queue %d with %ld\n", + queue_index, lpar_rc); + netdev_err(adapter->netdev, + "queue %d params: unit_addr=0x%x buffer_list_dma=0x%llx rxq_desc=0x%llx\n", + queue_index, adapter->vdev->unit_address, + adapter->buffer_list_dma[queue_index], + rxq_desc.desc); + + return lpar_rc; } -/ empty and free ana buffer pool - also used to do cleanup in error paths / -static void ibmveth_free_buffer_pool(struct ibmveth_adapter adapter, - struct ibmveth_buff_pool pool) +/* + * ibmveth_register_single_rx_queue - Register one subordinate RX queue + * @adapter: ibmveth adapter structure + * @queue_idx: Queue index to register (1..N) + * + * Builds the queue descriptor and registers with the hypervisor via + * ibmveth_register_logical_lan_queue(). + * + * Return: 0 on success, -EINVAL if @queue_idx is invalid, -EOPNOTSUPP if + * firmware rejects MQ (H_FUNCTION), -EIO on other failures + / +static int +ibmveth_register_single_rx_queue(struct ibmveth_adapter adapter, + int queue_idx) +{ + struct net_device netdev = adapter->netdev; + union ibmveth_buf_desc rxq_desc; + long lpar_rc; + + if (WARN_ON(queue_idx < 1 \|\| queue_idx >= IBMVETH_MAX_RX_QUEUES)) + return -EINVAL; + + rxq_desc.fields.flags_len = IBMVETH_BUF_VALID \| + adapter->rx_queue[queue_idx].queue_len; + rxq_desc.fields.address = adapter->rx_queue[queue_idx].queue_dma; + + lpar_rc = ibmveth_register_logical_lan_queue(adapter, rxq_desc, + queue_idx); + if (lpar_rc != H_SUCCESS) { + netdev_err(netdev, "Failed to register queue %d: rc=%ld\n", + queue_idx, lpar_rc); + if (lpar_rc == H_FUNCTION) + return -EOPNOTSUPP; + return -EIO; + } + + netdev_dbg(netdev, "Registered queue %d with handle 0x%llx\n", + queue_idx, adapter->queue_handle[queue_idx]); + + return 0; +} + +/* + * ibmveth_deregister_single_rx_queue - Deregister one subordinate RX queue + * @adapter: ibmveth adapter structure + * @queue_idx: Queue index to deregister (1..N) + * + * Deregisters a single queue via H_FREE_LOGICAL_LAN_QUEUE. Linux IRQ handler + * teardown and subordinate virq mapping disposal are owned by interrupt + * cleanup helpers; queue 0 is freed only through ibmveth_free_all_queues() + * (H_FREE_LOGICAL_LAN). + * + * Return: 0 if the queue was freed or had no handle, -EIO if + * H_FREE_LOGICAL_LAN_QUEUE failed. On -EIO the handle is + * left set so the caller can skip unmap. + / +static int +ibmveth_deregister_single_rx_queue(struct ibmveth_adapter adapter, + int queue_idx) +{ + unsigned long lpar_rc; + unsigned long ua = adapter->vdev->unit_address; + unsigned long qh = adapter->queue_handle[queue_idx]; + + if (!qh) + return 0; + + do { + lpar_rc = h_free_logical_lan_queue(ua, qh); + } while (H_IS_LONG_BUSY(lpar_rc) \|\| (lpar_rc == H_BUSY)); + + if (lpar_rc != H_SUCCESS) { + netdev_err(adapter->netdev, + "h_free_logical_lan_queue failed, queue %d rc=%ld\n", + queue_idx, lpar_rc); + return -EIO; + } + + adapter->queue_handle[queue_idx] = 0; + + netdev_dbg(adapter->netdev, "Deregistered queue %d\n", queue_idx); + return 0; +} + +/** + * ibmveth_destroy_subordinate_rx_queue - Tear down one subordinate RX queue + * @adapter: ibmveth adapter structure + * @queue_idx: Queue index to destroy (1..N) + * + * Deregister with PHYP before unmapping buffer pools so hypervisor buffer + * ownership is released while queue metadata is still valid. If the + * hcall fails, leave the queue mapped and keep the handle. + * + * Return: 0 on success, -EIO if deregister failed + / +static int +ibmveth_destroy_subordinate_rx_queue(struct ibmveth_adapter adapter, + int queue_idx) +{ + int rc; + + rc = ibmveth_deregister_single_rx_queue(adapter, queue_idx); + if (rc) + return rc; + ibmveth_cleanup_single_rx_interrupt(adapter, queue_idx); + ibmveth_free_single_rx_queue(adapter, queue_idx); + return 0; +} + +/** + * ibmveth_desired_dma_for_rxqs - CMO entitlement for a given RX queue count + * @adapter: ibmveth adapter + * @rxqs: number of RX queues to size for + * + * Same math as ibmveth_get_desired_dma(), but uses @rxqs instead of the + * live adapter->num_rx_queues. Scale-up raises desired for the target + * count before allocating so vio_cmo_alloc cannot fail mid-resize. + * + * Return: bytes of IO memory desired for @rxqs RX queues + / +static unsigned long +ibmveth_desired_dma_for_rxqs(struct ibmveth_adapter adapter, + unsigned int rxqs) +{ + struct net_device netdev = adapter->netdev; + struct iommu_table tbl; + unsigned long ret; + int i, q; + + tbl = get_iommu_table_base(&adapter->vdev->dev); + + ret = IBMVETH_BUFF_LIST_SIZE * rxqs + IBMVETH_FILT_LIST_SIZE; + ret += IOMMU_PAGE_ALIGN(netdev->mtu, tbl); + ret += IOMMU_PAGE_ALIGN(IBMVETH_MAX_TX_BUF_SIZE, tbl); + + /* + * Pool metadata for queues 1+ is copied from queue 0 at open. + * Always size from pool 0 x @rxqs. + * + * CMO (Power9 and earlier) and MQ firmware (Power11+) do not + * coexist, so a CMO partition always sizes one queue here. The + * MQ terms and the ethtool -L CMO refreshes are defensive only. + / + for (q = 0; q < rxqs; q++) { + int rxqentries = 1; + + for (i = 0; i < IBMVETH_NUM_BUFF_POOLS; i++) { + struct ibmveth_buff_pool bpool = + &adapter->rx_buff_pool[0][i]; + + if (bpool->active) + ret += bpool->size * + IOMMU_PAGE_ALIGN(bpool->buff_size, tbl); + rxqentries += bpool->size; + } + + ret += IOMMU_PAGE_ALIGN(rxqentries * + sizeof(struct ibmveth_rx_q_entry), tbl); + } + + return ret; +} + +/** + * ibmveth_free_stranded_rx_queues - Free queues stranded above the live count + * @adapter: ibmveth adapter structure + * + * A scale-up register or IRQ-setup failure whose H_FREE_LOGICAL_LAN_QUEUE + * also fails leaves that queue out of the live set with its handle set + * and its memory still mapped. Call only after successful H_FREE_LOGICAL_LAN, + * which drops every subordinate queue, so the memory can be released. + / +static void ibmveth_free_stranded_rx_queues(struct ibmveth_adapter adapter) { int i; - kfree(pool->free_map); - pool->free_map = NULL; + for (i = ibmveth_get_num_rx_queues(adapter); + i < IBMVETH_MAX_RX_QUEUES; i++) { + if (!adapter->queue_handle[i]) + continue; + adapter->queue_handle[i] = 0; + ibmveth_free_single_rx_queue(adapter, i); + } +} - if (pool->skbuff && pool->dma_addr) { - for (i = 0; i < pool->size; ++i) { - struct sk_buff skb = pool->skbuff[i]; - if (skb) { - dma_unmap_single(&adapter->vdev->dev, - pool->dma_addr[i], - pool->buff_size, - DMA_FROM_DEVICE); - dev_kfree_skb_any(skb); - pool->skbuff[i] = NULL; +/ Set CMO desired entitlement from the live RX queue count. / +static void ibmveth_refresh_cmo_desired(struct ibmveth_adapter adapter) +{ + if (firmware_has_feature(FW_FEATURE_CMO)) + vio_cmo_set_dev_desired(adapter->vdev, + ibmveth_get_desired_dma(adapter->vdev)); +} + +/** + * ibmveth_scale_up_rx_queues - Add RX queues to a live adapter + * @adapter: ibmveth adapter structure + * @old_count: Current number of RX queues + * @new_count: Target number of RX queues, above @old_count + * @rxq_entries: Number of entries per RX queue + * + * Raises CMO entitlement and widens real_num_rx_queues before any new + * queue is unmasked, then brings each queue up in turn. A failure + * unwinds only the queues this call added; a queue PHYP refuses to + * release stays in the live set instead. + * + * Return: 0 on success, negative error code on failure + / +static int +ibmveth_scale_up_rx_queues(struct ibmveth_adapter adapter, int old_count, + int new_count, int rxq_entries) +{ + struct net_device netdev = adapter->netdev; + int failed_queue; + int rc, i; + + netdev_dbg(netdev, "Scale-up: adding queues %d-%d\n", + old_count, new_count - 1); + + / + * Raise CMO desired for the target count before dma_map / + * dma_alloc_coherent / replenish (same order as change_mtu). + * Do not bump live num_rx_queues here, only entitlement. + / + if (firmware_has_feature(FW_FEATURE_CMO)) { + unsigned long dma; + + dma = ibmveth_desired_dma_for_rxqs(adapter, new_count); + vio_cmo_set_dev_desired(adapter->vdev, dma); + } + + / + * Widen real_num before any new queue is unmasked so + * skb_record_rx_queue() cannot trip get_rps_cpu() + * WARN_ONCE. Probe already allocated num_rx_queues. + / + rc = netif_set_real_num_rx_queues(netdev, new_count); + if (rc) { + netdev_err(netdev, + "Failed to set real RX queues to %d: %d\n", + new_count, rc); + ibmveth_refresh_cmo_desired(adapter); + return rc; + } + + for (i = old_count; i < new_count; i++) { + if (adapter->queue_handle[i]) { + / Left by a failed H_FREE; close frees it. / + netdev_err(netdev, + "RX queue %d still held by PHYP, reset pending\n", + i); + rc = -EBUSY; + goto cleanup_new_queues; + } + rc = ibmveth_alloc_single_rx_queue(adapter, i, + rxq_entries); + if (rc) { + netdev_err(netdev, "Failed to allocate queue %d: %d\n", + i, rc); + goto cleanup_new_queues; + } + + rc = ibmveth_register_single_rx_queue(adapter, i); + if (rc) { + netdev_err(netdev, "Failed to register queue %d: %d\n", + i, rc); + if (rc == -EOPNOTSUPP) + adapter->mq_fallback = true; + if (!ibmveth_deregister_single_rx_queue(adapter, + i)) + ibmveth_free_single_rx_queue(adapter, + i); + else + schedule_work(&adapter->work); + goto cleanup_new_queues; + } + + rc = ibmveth_setup_single_rx_interrupt(adapter, i); + if (rc) { + netdev_err(netdev, + "Failed to setup IRQ for queue %d: %d\n", + i, rc); + / request_irq failed: mapped but no handler / + ibmveth_dispose_subordinate_irq_mapping(adapter, + i); + if (!ibmveth_deregister_single_rx_queue(adapter, + i)) + ibmveth_free_single_rx_queue(adapter, + i); + else + schedule_work(&adapter->work); + goto cleanup_new_queues; + } + + / + * Fully ready before PHYP delivery, matching open(): + * publish -> replenish -> napi_enable -> enable_irq. + * That way ibmveth_interrupt() cannot run on an + * unpublished, empty, or NAPI-disabled queue. + / + ibmveth_publish_num_rx_queues(adapter, i + 1); + ibmveth_replenish_task(adapter, i); + napi_enable(&adapter->napi[i]); + + rc = ibmveth_enable_irq(adapter, i); + if (rc) { + netdev_err(netdev, + "Failed to enable IRQ for queue %d: %d\n", + i, rc); + / + * Published, replenished, and NAPI-enabled, + * but PHYP never unmasked. Match scale-down / + * shared cleanup: drain posted buffers, then + * deregister before unmap via + * destroy_subordinate. + * + * napi_disable() must come BEFORE the count + * is lowered, matching scale-down and + * cleanup_new_queues. Lowering it first does + * not hide queue i from netpoll: after + * ndo_poll_controller, netpoll_poll_dev() + * calls poll_napi(), which walks dev->napi_list + * unbounded by the queue count and skips a NAPI + * only once NAPI_STATE_NPSVC is set. Queue i is + * enabled here, so ibmveth_poll() would run and + * trip its queue_index >= num_rx_queues + * WARN_ON. napi_disable() sets NPSVC, so + * poll_napi() skips the queue instead. + / + napi_disable(&adapter->napi[i]); + / A poll may have unmasked PHYP; remask. / + ibmveth_disable_irq(adapter, i); + synchronize_irq(adapter->queue_irq[i]); + ibmveth_publish_num_rx_queues(adapter, i); + ibmveth_drain_rx_queue(adapter, i); + synchronize_net(); + } + if (rc && + ibmveth_destroy_subordinate_rx_queue(adapter, i)) { + int keep = i + 1; + + / + * PHYP still owns queue i. Keep it in the + * live set with the queues this -L already + * added. cleanup_new_queues would publish + * below i and orphan this mapping. + / + ibmveth_publish_num_rx_queues(adapter, keep); + if (netif_set_real_num_rx_queues(netdev, + keep)) + schedule_work(&adapter->work); + ibmveth_replenish_task(adapter, i); + napi_enable(&adapter->napi[i]); + rc = ibmveth_enable_irq(adapter, i); + if (rc) { + netdev_err(netdev, + "IRQ %d rc=%d\n", + i, rc); + schedule_work(&adapter->work); + } else { + ibmveth_restart_rx_queue(adapter, i); } + return -EIO; + } + if (rc) { + / enable_irq errno; keep -EIO. / + rc = -EIO; + goto cleanup_new_queues; } + ibmveth_restart_rx_queue(adapter, i); } - if (pool->dma_addr) { - kfree(pool->dma_addr); - pool->dma_addr = NULL; + return 0; + +cleanup_new_queues: + failed_queue = i; + if (failed_queue > old_count) + netdev_err(netdev, + "Scale-up failed at queue %d, cleaning up queues %d-%d\n", + failed_queue, old_count, failed_queue - 1); + else + netdev_err(netdev, + "Scale-up failed at queue %d, nothing to clean up\n", + failed_queue); + + for (i = old_count; i < failed_queue; i++) { + ibmveth_disable_irq(adapter, i); + synchronize_irq(adapter->queue_irq[i]); } - if (pool->skbuff) { - kfree(pool->skbuff); - pool->skbuff = NULL; + for (i = old_count; i < failed_queue; i++) + napi_disable(&adapter->napi[i]); + + / Same remask as scale-down: poll may have re-armed during disable. / + for (i = old_count; i < failed_queue; i++) { + ibmveth_disable_irq(adapter, i); + synchronize_irq(adapter->queue_irq[i]); + } + + for (i = old_count; i < failed_queue; i++) + ibmveth_drain_rx_queue(adapter, i); + + / Drop the live count before freeing the half-added queues. / + ibmveth_publish_num_rx_queues(adapter, old_count); + if (netif_set_real_num_rx_queues(netdev, old_count)) + schedule_work(&adapter->work); + synchronize_net(); + + for (i = failed_queue - 1; i >= old_count; i--) { + int keep; + + if (!ibmveth_destroy_subordinate_rx_queue(adapter, i)) + continue; + + keep = i + 1; + ibmveth_publish_num_rx_queues(adapter, keep); + if (netif_set_real_num_rx_queues(netdev, keep)) + schedule_work(&adapter->work); + for (i = old_count; i < keep; i++) { + int irq_rc; + + ibmveth_replenish_task(adapter, i); + / START: NAPI before PHYP unmask. / + napi_enable(&adapter->napi[i]); + irq_rc = ibmveth_enable_irq(adapter, i); + if (irq_rc) { + netdev_err(netdev, + "IRQ %d rc=%d\n", + i, irq_rc); + schedule_work(&adapter->work); + continue; + } + ibmveth_restart_rx_queue(adapter, i); + } + ibmveth_refresh_cmo_desired(adapter); + netdev_warn(netdev, + "Keeping %d queues after scale-up failure\n", + keep); + return rc; } + + / Roll CMO desired back to the surviving queue count. / + ibmveth_refresh_cmo_desired(adapter); + + netdev_warn(netdev, "Keeping %d queues after scale-up failure\n", + old_count); + return rc; } /* - * ibmveth_remove_buffer_from_pool - remove a buffer from a pool - * @adapter: adapter instance - * @correlator: identifies pool and index - * @reuse: whether to reuse buffer + * ibmveth_scale_down_rx_queues - Remove RX queues from a live adapter + * @adapter: ibmveth adapter structure + * @old_count: Current number of RX queues + * @new_count: Target number of RX queues, below @old_count * - * Return: - * * %0 - success - * * %-EINVAL - correlator maps to pool or index out of range - * * %-EFAULT - pool and index map to null skb + * Quiesces the retiring queues, drains them and harvests their final + * no_buffer counts, publishes the surviving count, and only then + * deregisters them. Walks high to low so a failed free leaves 0..i + * live and republishes that count. + * + * Return: 0 on success, negative error code on failure / -static int ibmveth_remove_buffer_from_pool(struct ibmveth_adapter adapter, - u64 correlator, bool reuse) +static int +ibmveth_scale_down_rx_queues(struct ibmveth_adapter adapter, int old_count, + int new_count) { - unsigned int pool = correlator >> 32; - unsigned int index = correlator & 0xffffffffUL; - unsigned int free_index; - struct sk_buff skb; + struct net_device netdev = adapter->netdev; + int rc, i; - if (WARN_ON(pool >= IBMVETH_NUM_BUFF_POOLS) \|\| - WARN_ON(index >= adapter->rx_buff_pool[pool].size)) { - schedule_work(&adapter->work); - return -EINVAL; + netdev_dbg(netdev, "Scale-down: removing queues %d-%d\n", + new_count, old_count - 1); + + / + * Mask PHYP before napi_disable so the handler cannot miss + * a mask while NAPI is already dead. An in-flight poll can + * still re-arm PHYP while napi_disable() waits, so remask + * and sync again after NAPI is stopped. Then drain, harvest + * final no_buffer stats, publish the surviving count, and + * synchronize_net() before destroy so netpoll cannot walk + * dying queues (handler may still be registered until destroy). + / + for (i = new_count; i < old_count; i++) { + if (!adapter->queue_irq[i]) + continue; + ibmveth_disable_irq(adapter, i); + synchronize_irq(adapter->queue_irq[i]); } - skb = adapter->rx_buff_pool[pool].skbuff[index]; - if (WARN_ON(!skb)) { - schedule_work(&adapter->work); - return -EFAULT; + for (i = new_count; i < old_count; i++) + napi_disable(&adapter->napi[i]); + + for (i = new_count; i < old_count; i++) { + if (!adapter->queue_irq[i]) + continue; + ibmveth_disable_irq(adapter, i); + synchronize_irq(adapter->queue_irq[i]); } - / if we are going to reuse the buffer then keep the pointers around - * but mark index as available. replenish will see the skb pointer and - * assume it is to be recycled. + for (i = new_count; i < old_count; i++) + ibmveth_drain_rx_queue(adapter, i); + + /* + * netpoll still reaches these queues until the publish + * below, and ibmveth_replenish_task() folds the same + * counters under the queue's replenish_lock. Take it here + * so the harvest stays the single writer it documents. / - if (!reuse) { - / remove the skb pointer to mark free. actual freeing is done - * by upper level networking after gro_receive + for (i = new_count; i < old_count; i++) { + struct ibmveth_rx_q rxq = &adapter->rx_queue[i]; + unsigned long flags; + + spin_lock_irqsave(&rxq->replenish_lock, flags); + ibmveth_update_rx_no_buffer(adapter, i); + spin_unlock_irqrestore(&rxq->replenish_lock, flags); + } + + ibmveth_publish_num_rx_queues(adapter, new_count); + synchronize_net(); + + rc = netif_set_real_num_rx_queues(netdev, new_count); + if (rc) { + netdev_err(netdev, "Failed to set real RX queues to %d: %d\n", + new_count, rc); + ibmveth_publish_num_rx_queues(adapter, old_count); + for (i = new_count; i < old_count; i++) { + int irq_rc; + + ibmveth_replenish_task(adapter, i); + / START: NAPI before PHYP unmask. / + napi_enable(&adapter->napi[i]); + irq_rc = ibmveth_enable_irq(adapter, i); + if (irq_rc) { + netdev_err(netdev, + "Failed to re-enable IRQ for queue %d during scale-down rollback (rc=%d), scheduling reset\n", + i, irq_rc); + schedule_work(&adapter->work); + continue; + } + ibmveth_restart_rx_queue(adapter, i); + } + return rc; + } + + for (i = old_count - 1; i >= new_count; i--) { + int keep; + + rc = ibmveth_destroy_subordinate_rx_queue(adapter, i); + if (!rc) + continue; + + / + * High-to-low so a failed free leaves 0..i live. + * Queues i+1..old_count-1 are already gone. / - adapter->rx_buff_pool[pool].skbuff[index] = NULL; + keep = i + 1; + ibmveth_publish_num_rx_queues(adapter, keep); + if (netif_set_real_num_rx_queues(netdev, keep)) + schedule_work(&adapter->work); + for (i = new_count; i < keep; i++) { + int irq_rc; + + ibmveth_replenish_task(adapter, i); + / START: NAPI before PHYP unmask. / + napi_enable(&adapter->napi[i]); + irq_rc = ibmveth_enable_irq(adapter, i); + if (irq_rc) { + netdev_err(netdev, + "IRQ %d rc=%d\n", + i, irq_rc); + schedule_work(&adapter->work); + continue; + } + ibmveth_restart_rx_queue(adapter, i); + } + return rc; + } - dma_unmap_single(&adapter->vdev->dev, - adapter->rx_buff_pool[pool].dma_addr[index], - adapter->rx_buff_pool[pool].buff_size, - DMA_FROM_DEVICE); + return 0; +} + +/* + * ibmveth_resize_rx_queues_incremental - Resize RX queue count incrementally + * @adapter: ibmveth adapter structure + * @new_count: Target number of RX queues + * @rxq_entries: Number of entries per RX queue + * + * Adds or removes RX queues without tearing down the entire adapter. + * Active queues continue receiving during scale-up. Scale-up widens + * real_num_rx_queues before unmasking a new queue. Scale-down drains + * excess queues before deregistering them with the hypervisor. + * + * Return: 0 on success, negative error code on failure + / +static int +ibmveth_resize_rx_queues_incremental(struct ibmveth_adapter adapter, + int new_count, int rxq_entries) +{ + struct net_device netdev = adapter->netdev; + int old_count = ibmveth_get_num_rx_queues(adapter); + int rc; + + if (old_count == new_count) { + netdev_dbg(netdev, "RX queue count unchanged (%d), nothing to do\n", + old_count); + return 0; + } + + if (new_count < 1 \|\| new_count > IBMVETH_MAX_RX_QUEUES) { + netdev_err(netdev, "Invalid RX queue count %d (must be 1-%d)\n", + new_count, IBMVETH_MAX_RX_QUEUES); + return -EINVAL; } - free_index = adapter->rx_buff_pool[pool].producer_index; - adapter->rx_buff_pool[pool].producer_index++; - if (adapter->rx_buff_pool[pool].producer_index >= - adapter->rx_buff_pool[pool].size) - adapter->rx_buff_pool[pool].producer_index = 0; - adapter->rx_buff_pool[pool].free_map[free_index] = index; + netdev_info(netdev, "Incrementally resizing RX queues: %d to %d\n", + old_count, new_count); - mb(); + if (new_count > old_count) + rc = ibmveth_scale_up_rx_queues(adapter, old_count, new_count, + rxq_entries); + else + rc = ibmveth_scale_down_rx_queues(adapter, old_count, + new_count); + if (rc) + return rc; - atomic_dec(&(adapter->rx_buff_pool[pool].available)); + netdev_info(netdev, "Successfully resized to %u RX queues (incremental)\n", + ibmveth_get_num_rx_queues(adapter)); + + ibmveth_refresh_cmo_desired(adapter); return 0; } -/ get the current buffer on the rx queue / -static inline struct sk_buff ibmveth_rxq_get_buffer(struct ibmveth_adapter adapter) +/* + * ibmveth_free_all_queues - Free all RX queues at once + * @adapter: ibmveth adapter structure + * + * Issues one H_FREE_LOGICAL_LAN for full adapter teardown. Per PAPR/PHYP, + * that drops the primary LAN and any subordinate queues registered under + * it. Incremental scale-down uses H_FREE_LOGICAL_LAN_QUEUE per queue + * instead; do not use this helper for partial live-set shrink. + * + * Used during interface close and registration error cleanup. + * + * Retries only H_BUSY and H_IS_LONG_BUSY. On other failures, logs and + * returns -EIO; queue_handle[] is cleared regardless. Callers still run + * RX pool and DMA teardown afterward (same as pre-helper close()). + * + * Clears queue handles only; queue_irq[] is released by + * ibmveth_cleanup_rx_interrupts() on close, or by + * ibmveth_dispose_subordinate_irq_mappings() on partial register failure. + * + * Return: 0 on success, -EIO if H_FREE_LOGICAL_LAN failed + / +static int ibmveth_free_all_queues(struct ibmveth_adapter adapter) { - u64 correlator = adapter->rx_queue.queue_addr[adapter->rx_queue.index].correlator; - unsigned int pool = correlator >> 32; - unsigned int index = correlator & 0xffffffffUL; + unsigned long lpar_rc; + int i; - if (WARN_ON(pool >= IBMVETH_NUM_BUFF_POOLS) \|\| - WARN_ON(index >= adapter->rx_buff_pool[pool].size)) { - schedule_work(&adapter->work); - return NULL; + netdev_dbg(adapter->netdev, "freeing all RX queues at once\n"); + + do { + lpar_rc = h_free_logical_lan(adapter->vdev->unit_address); + } while (H_IS_LONG_BUSY(lpar_rc) \|\| (lpar_rc == H_BUSY)); + + for (i = 0; i < ibmveth_get_num_rx_queues(adapter); i++) + adapter->queue_handle[i] = 0; + + if (lpar_rc != H_SUCCESS) { + netdev_err(adapter->netdev, + "h_free_logical_lan failed: %ld\n", lpar_rc); + return -EIO; } - return adapter->rx_buff_pool[pool].skbuff[index]; + return 0; } /** - * ibmveth_rxq_harvest_buffer - Harvest buffer from pool - * - * @adapter: pointer to adapter - * @reuse: whether to reuse buffer + * ibmveth_register_rx_queues - Register RX queues with hypervisor + * @adapter: ibmveth adapter structure + * @mac_address: MAC address for device registration * - * Context: called from ibmveth_poll + * Registers queue 0 via ibmveth_register_logical_lan(), then subordinate + * queues 1..N when multi-queue mode is enabled. * - * Return: - * * %0 - success - * * other - non-zero return from ibmveth_remove_buffer_from_pool + * Return: 0 on success, -ENONET if queue 0 registration fails, + * -EOPNOTSUPP if firmware rejects a subordinate queue (H_FUNCTION; + * this function latches mq_fallback), -EIO on other subordinate + * failures / -static int ibmveth_rxq_harvest_buffer(struct ibmveth_adapter adapter, - bool reuse) +static int +ibmveth_register_rx_queues(struct ibmveth_adapter adapter, u64 mac_address) { - u64 cor; - int rc; + struct net_device netdev = adapter->netdev; + union ibmveth_buf_desc rxq_desc; + unsigned long lpar_rc; + unsigned int num; + int i, rc; - cor = adapter->rx_queue.queue_addr[adapter->rx_queue.index].correlator; - rc = ibmveth_remove_buffer_from_pool(adapter, cor, reuse); - if (unlikely(rc)) - return rc; + rxq_desc.fields.flags_len = IBMVETH_BUF_VALID \| + adapter->rx_queue[0].queue_len; + rxq_desc.fields.address = adapter->rx_queue[0].queue_dma; + adapter->queue_irq[0] = netdev->irq; + + rc = ibmveth_disable_irq(adapter, 0); + if (rc) + netdev_dbg(netdev, + "Failed to disable IRQ for queue 0 before registration, rc=%d\n", + rc); - if (++adapter->rx_queue.index == adapter->rx_queue.num_slots) { - adapter->rx_queue.index = 0; - adapter->rx_queue.toggle = !adapter->rx_queue.toggle; + lpar_rc = ibmveth_register_logical_lan(adapter, rxq_desc, mac_address); + if (lpar_rc != H_SUCCESS) { + netdev_err(netdev, + "h_register_logical_lan failed: %ld\n", lpar_rc); + netdev_err(netdev, + "buffer TCE:0x%llx filter TCE:0x%llx rxq desc:0x%llx MAC:0x%llx\n", + adapter->buffer_list_dma[0], + adapter->filter_list_dma, + rxq_desc.desc, mac_address); + return -ENONET; } - return 0; -} + num = ibmveth_get_num_rx_queues(adapter); + if (num == 1 \|\| !adapter->multi_queue) { + netdev_dbg(netdev, + "registered 1 RX queue with hypervisor (single-queue mode)\n"); + return 0; + } -static void ibmveth_free_tx_ltb(struct ibmveth_adapter adapter, int idx) -{ - dma_unmap_single(&adapter->vdev->dev, adapter->tx_ltb_dma[idx], - adapter->tx_ltb_size, DMA_TO_DEVICE); - kfree(adapter->tx_ltb_ptr[idx]); - adapter->tx_ltb_ptr[idx] = NULL; -} + netdev_dbg(netdev, "Registering %u subordinate queues (1-%u)\n", + num - 1, num - 1); -static int ibmveth_allocate_tx_ltb(struct ibmveth_adapter adapter, int idx) -{ - adapter->tx_ltb_ptr[idx] = kzalloc(adapter->tx_ltb_size, - GFP_KERNEL); - if (!adapter->tx_ltb_ptr[idx]) { - netdev_err(adapter->netdev, - "unable to allocate tx long term buffer\n"); - return -ENOMEM; - } - adapter->tx_ltb_dma[idx] = dma_map_single(&adapter->vdev->dev, - adapter->tx_ltb_ptr[idx], - adapter->tx_ltb_size, - DMA_TO_DEVICE); - if (dma_mapping_error(&adapter->vdev->dev, adapter->tx_ltb_dma[idx])) { - netdev_err(adapter->netdev, - "unable to DMA map tx long term buffer\n"); - kfree(adapter->tx_ltb_ptr[idx]); - adapter->tx_ltb_ptr[idx] = NULL; - return -ENOMEM; + for (i = 1; i < num; i++) { + rc = ibmveth_register_single_rx_queue(adapter, i); + if (rc) { + /* Firmware MQ gone: fall back to SQ on next open. / + if (rc == -EOPNOTSUPP) + adapter->mq_fallback = true; + goto err_unregister; + } } + netdev_dbg(netdev, + "registered %u RX queues with hypervisor (multi-queue mode)\n", + num); + return 0; + +err_unregister: + ibmveth_dispose_subordinate_irq_mappings(adapter); + ibmveth_free_all_queues(adapter); + return rc; } -static int ibmveth_register_logical_lan(struct ibmveth_adapter adapter, - union ibmveth_buf_desc rxq_desc, u64 mac_address) +/** + * ibmveth_apply_mq_fallback - Drop multi-queue mode after firmware rejection + * @adapter: ibmveth adapter + * + * mq_fallback is set when firmware rejects MQ (subordinate register or + * buffer-add H_FUNCTION). Apply only at the start of open after teardown so + * num_rx_queues is not shrunk while IRQ/NAPI still reference higher queues. + * Consumes the flag and clears multi_queue, which is what makes the + * single-queue decision permanent for this device. + / +static void ibmveth_apply_mq_fallback(struct ibmveth_adapter adapter) { - int rc, try_again = 1; + struct net_device netdev = adapter->netdev; - / - * After a kexec the adapter will still be open, so our attempt to - * open it will fail. So if we get a failure we free the adapter and - * try again, but only once. - / -retry: - rc = h_register_logical_lan(adapter->vdev->unit_address, - adapter->buffer_list_dma, rxq_desc.desc, - adapter->filter_list_dma, mac_address); + if (!adapter->mq_fallback) + return; - if (rc != H_SUCCESS && try_again) { - do { - rc = h_free_logical_lan(adapter->vdev->unit_address); - } while (H_IS_LONG_BUSY(rc) \|\| (rc == H_BUSY)); + adapter->mq_fallback = false; - try_again = 0; - goto retry; - } + if (!adapter->multi_queue && ibmveth_get_num_rx_queues(adapter) == 1) + return; - return rc; + netdev_warn(netdev, + "Falling back to single RX queue (firmware MQ unavailable)\n"); + adapter->multi_queue = false; + ibmveth_publish_num_rx_queues(adapter, 1); + / real_num_rx_queues is set later in open after resources exist. / + if (adapter->rx_buffers_per_hcall > IBMVETH_MAX_RX_REGULAR) + adapter->rx_buffers_per_hcall = IBMVETH_MAX_RX_REGULAR; } static int ibmveth_open(struct net_device netdev) { struct ibmveth_adapter adapter = netdev_priv(netdev); - u64 mac_address; + u64 mac_address = ether_addr_to_u64(netdev->dev_addr); int rxq_entries = 1; - unsigned long lpar_rc; int rc; - union ibmveth_buf_desc rxq_desc; int i; - struct device dev; netdev_dbg(netdev, "open starting\n"); - napi_enable(&adapter->napi); + ibmveth_apply_mq_fallback(adapter); - for(i = 0; i < IBMVETH_NUM_BUFF_POOLS; i++) - rxq_entries += adapter->rx_buff_pool[i].size; + for (i = 0; i < IBMVETH_NUM_BUFF_POOLS; i++) + rxq_entries += adapter->rx_buff_pool[0][i].size; - rc = -ENOMEM; - adapter->buffer_list_addr = (void) get_zeroed_page(GFP_KERNEL); - if (!adapter->buffer_list_addr) { - netdev_err(netdev, "unable to allocate list pages\n"); + rc = ibmveth_alloc_filter_list(adapter); + if (rc) goto out; - } - - adapter->filter_list_addr = (void) get_zeroed_page(GFP_KERNEL); - if (!adapter->filter_list_addr) { - netdev_err(netdev, "unable to allocate filter pages\n"); - goto out_free_buffer_list; - } - - dev = &adapter->vdev->dev; - adapter->rx_queue.queue_len = sizeof(struct ibmveth_rx_q_entry) * - rxq_entries; - adapter->rx_queue.queue_addr = - dma_alloc_coherent(dev, adapter->rx_queue.queue_len, - &adapter->rx_queue.queue_dma, GFP_KERNEL); - if (!adapter->rx_queue.queue_addr) + rc = ibmveth_alloc_rx_queues(adapter, rxq_entries); + if (rc) goto out_free_filter_list; - adapter->buffer_list_dma = dma_map_single(dev, - adapter->buffer_list_addr, 4096, DMA_BIDIRECTIONAL); - if (dma_mapping_error(dev, adapter->buffer_list_dma)) { - netdev_err(netdev, "unable to map buffer list pages\n"); + rc = ibmveth_alloc_buffer_pools(adapter); + if (rc) goto out_free_queue_mem; - } - - adapter->filter_list_dma = dma_map_single(dev, - adapter->filter_list_addr, 4096, DMA_BIDIRECTIONAL); - if (dma_mapping_error(dev, adapter->filter_list_dma)) { - netdev_err(netdev, "unable to map filter list pages\n"); - goto out_unmap_buffer_list; - } - - for (i = 0; i < netdev->real_num_tx_queues; i++) { - if (ibmveth_allocate_tx_ltb(adapter, i)) - goto out_free_tx_ltb; - } - - adapter->rx_queue.index = 0; - adapter->rx_queue.num_slots = rxq_entries; - adapter->rx_queue.toggle = 1; - - mac_address = ether_addr_to_u64(netdev->dev_addr); - rxq_desc.fields.flags_len = IBMVETH_BUF_VALID \| - adapter->rx_queue.queue_len; - rxq_desc.fields.address = adapter->rx_queue.queue_dma; - - netdev_dbg(netdev, "buffer list @ 0x%p\n", adapter->buffer_list_addr); - netdev_dbg(netdev, "filter list @ 0x%p\n", adapter->filter_list_addr); - netdev_dbg(netdev, "receive q @ 0x%p\n", adapter->rx_queue.queue_addr); - - h_vio_signal(adapter->vdev->unit_address, VIO_IRQ_DISABLE); + rc = ibmveth_register_rx_queues(adapter, mac_address); + if (rc) + goto out_free_buffer_pools; - lpar_rc = ibmveth_register_logical_lan(adapter, rxq_desc, mac_address); + rc = netif_set_real_num_rx_queues(netdev, + ibmveth_get_num_rx_queues(adapter)); - if (lpar_rc != H_SUCCESS) { - netdev_err(netdev, "h_register_logical_lan failed with %ld\n", - lpar_rc); - netdev_err(netdev, "buffer TCE:0x%llx filter TCE:0x%llx rxq " - "desc:0x%llx MAC:0x%llx\n", - adapter->buffer_list_dma, - adapter->filter_list_dma, - rxq_desc.desc, - mac_address); - rc = -ENONET; - goto out_unmap_filter_list; + if (rc) { + netdev_err(netdev, "failed to set number of rx queues\n"); + goto out_unregister_queues; } - for (i = 0; i < IBMVETH_NUM_BUFF_POOLS; i++) { - if (!adapter->rx_buff_pool[i].active) - continue; - if (ibmveth_alloc_buffer_pool(&adapter->rx_buff_pool[i])) { - netdev_err(netdev, "unable to alloc pool\n"); - adapter->rx_buff_pool[i].active = 0; - rc = -ENOMEM; - goto out_free_buffer_pools; - } + /* + * Post buffers before setup_rx_interrupts(). MQ setup then unmasks + * PHYP; SQ setup leaves PHYP masked. Scheduling NAPI only when a + * descriptor is already pending is not enough: after ifdown/up + * (RX=8, no -L) NAPI can be idle with nothing pending and the + * queue stays dead (TX OK, ARP/RX fail). + * restart_rx_queue() replenishes, schedules NAPI, and unmasks if + * prep fails. + / + for (i = 0; i < ibmveth_get_num_rx_queues(adapter); i++) { + netdev_dbg(netdev, "initial replenish cycle for queue %d\n", i); + ibmveth_replenish_task(adapter, i); } - netdev_dbg(netdev, "registering irq 0x%x\n", netdev->irq); - rc = request_irq(netdev->irq, ibmveth_interrupt, 0, netdev->name, - netdev); - if (rc != 0) { - netdev_err(netdev, "unable to request irq 0x%x, rc %d\n", - netdev->irq, rc); - do { - lpar_rc = h_free_logical_lan(adapter->vdev->unit_address); - } while (H_IS_LONG_BUSY(lpar_rc) \|\| (lpar_rc == H_BUSY)); - - goto out_free_buffer_pools; - } + rc = ibmveth_setup_rx_interrupts(adapter); + if (rc) + goto out_free_all_queues; / setup already disposed IRQs / - rc = -ENOMEM; + for (i = 0; i < ibmveth_get_num_rx_queues(adapter); i++) + ibmveth_restart_rx_queue(adapter, i); - netdev_dbg(netdev, "initial replenish cycle\n"); - ibmveth_interrupt(netdev->irq, netdev); + rc = ibmveth_alloc_tx_resources(adapter); + if (rc) + goto out_cleanup_rx_interrupts; netif_tx_start_all_queues(netdev); + adapter->opened = true; netdev_dbg(netdev, "open complete\n"); return 0; +out_cleanup_rx_interrupts: + ibmveth_cleanup_rx_interrupts(adapter); + / As in close(): a poll past napi_complete_done() may still run. / + synchronize_net(); + goto out_free_all_queues; / cleanup already disposed IRQs / +out_unregister_queues: + ibmveth_dispose_subordinate_irq_mappings(adapter); +out_free_all_queues: + ibmveth_free_all_queues(adapter); out_free_buffer_pools: - while (--i >= 0) { - if (adapter->rx_buff_pool[i].active) - ibmveth_free_buffer_pool(adapter, - &adapter->rx_buff_pool[i]); - } -out_unmap_filter_list: - dma_unmap_single(dev, adapter->filter_list_dma, 4096, - DMA_BIDIRECTIONAL); - -out_free_tx_ltb: - while (--i >= 0) { - ibmveth_free_tx_ltb(adapter, i); - } - -out_unmap_buffer_list: - dma_unmap_single(dev, adapter->buffer_list_dma, 4096, - DMA_BIDIRECTIONAL); + ibmveth_free_buffer_pools(adapter); out_free_queue_mem: - dma_free_coherent(dev, adapter->rx_queue.queue_len, - adapter->rx_queue.queue_addr, - adapter->rx_queue.queue_dma); + ibmveth_cleanup_rx_resources(adapter); out_free_filter_list: - free_page((unsigned long)adapter->filter_list_addr); -out_free_buffer_list: - free_page((unsigned long)adapter->buffer_list_addr); + ibmveth_free_filter_list(adapter); out: - napi_disable(&adapter->napi); return rc; } static int ibmveth_close(struct net_device netdev) { struct ibmveth_adapter adapter = netdev_priv(netdev); - struct device dev = &adapter->vdev->dev; - long lpar_rc; int i; - netdev_dbg(netdev, "close starting\n"); - - napi_disable(&adapter->napi); - - netif_tx_stop_all_queues(netdev); - - h_vio_signal(adapter->vdev->unit_address, VIO_IRQ_DISABLE); - - do { - lpar_rc = h_free_logical_lan(adapter->vdev->unit_address); - } while (H_IS_LONG_BUSY(lpar_rc) \|\| (lpar_rc == H_BUSY)); - - if (lpar_rc != H_SUCCESS) { - netdev_err(netdev, "h_free_logical_lan failed with %lx, " - "continuing with close\n", lpar_rc); - } - - free_irq(netdev->irq, netdev); + /* Gate on opened, not IFF_UP: pool_store/change_mtu close+open can + * leave IFF_UP set after a failed reopen. + / + if (!adapter->opened) + return 0; - ibmveth_update_rx_no_buffer(adapter); + adapter->opened = false; - dma_unmap_single(dev, adapter->buffer_list_dma, 4096, - DMA_BIDIRECTIONAL); - free_page((unsigned long)adapter->buffer_list_addr); + netdev_dbg(netdev, "close starting\n"); - dma_unmap_single(dev, adapter->filter_list_dma, 4096, - DMA_BIDIRECTIONAL); - free_page((unsigned long)adapter->filter_list_addr); + / Disable and wait for in-flight ndo_start_xmit (stop_all_queues + * alone does not). Direct close() callers bypass synchronize_net(). + / + netif_tx_disable(netdev); - dma_free_coherent(dev, adapter->rx_queue.queue_len, - adapter->rx_queue.queue_addr, - adapter->rx_queue.queue_dma); + ibmveth_cleanup_rx_interrupts(adapter); + / Wait for softirq/poll that already passed shutdown checks. / + synchronize_net(); - for (i = 0; i < IBMVETH_NUM_BUFF_POOLS; i++) - if (adapter->rx_buff_pool[i].active) - ibmveth_free_buffer_pool(adapter, - &adapter->rx_buff_pool[i]); + for (i = 0; i < ibmveth_get_num_rx_queues(adapter); i++) { + struct ibmveth_rx_q rxq = &adapter->rx_queue[i]; + unsigned long flags; - for (i = 0; i < netdev->real_num_tx_queues; i++) - ibmveth_free_tx_ltb(adapter, i); + spin_lock_irqsave(&rxq->replenish_lock, flags); + ibmveth_update_rx_no_buffer(adapter, i); + spin_unlock_irqrestore(&rxq->replenish_lock, flags); + } + if (!ibmveth_free_all_queues(adapter)) + ibmveth_free_stranded_rx_queues(adapter); + /* Free TX LTBs after quiesce and after H_FREE_LOGICAL_LAN so xmit + * cannot touch unmapped bounce buffers while the LAN is live. + / + ibmveth_free_tx_resources(adapter); + ibmveth_free_buffer_pools(adapter); + ibmveth_cleanup_rx_resources(adapter); + ibmveth_free_filter_list(adapter); netdev_dbg(netdev, "close complete\n"); @@ -840,6 +3037,10 @@ static void ibmveth_reset(struct work_struct w) netdev_dbg(netdev, "reset starting\n"); rtnl_lock(); + if (netdev->reg_state != NETREG_REGISTERED) { + rtnl_unlock(); + return; + } dev_close(adapter->netdev); dev_open(adapter->netdev, NULL); @@ -1073,22 +3274,96 @@ static int ibmveth_set_features(struct net_device dev, return rc1 ? rc1 : rc2; } -static void ibmveth_get_strings(struct net_device dev, u32 stringset, u8 data) +/ + * Sum per-queue counters for rare ethtool reads. The hot paths only ever + * touch their own queue's slot, so nothing here needs an atomic; the cost + * of aggregation is paid by the reader instead (ibmvnic-style). + * + * Every slot is summed, not just the live ones, so that shrinking the + * queue count with ethtool -L cannot make a counter go backwards. + / +static u64 ibmveth_sum_rx_qstat(struct ibmveth_adapter adapter, size_t off) +{ + u64 total = 0; + int i; + + for (i = 0; i < IBMVETH_MAX_RX_QUEUES; i++) + total += (u64 )((u8 )&adapter->rx_qstats[i] + off); + + return total; +} + +static u64 ibmveth_sum_tx_qstat(struct ibmveth_adapter adapter, size_t off) { + u64 total = 0; int i; + for (i = 0; i < IBMVETH_MAX_QUEUES; i++) + total += (u64 )((u8 )&adapter->tx_qstats[i] + off); + + return total; +} + +static u64 ibmveth_ethtool_adapter_stat(struct ibmveth_adapter adapter, + int index) +{ + const struct ibmveth_stat stat = &ibmveth_stats[index]; + + switch (stat->src) { + case IBMVETH_STAT_RX_QSUM: + return ibmveth_sum_rx_qstat(adapter, stat->off); + case IBMVETH_STAT_TX_QSUM: + return ibmveth_sum_tx_qstat(adapter, stat->off); + case IBMVETH_STAT_RX_NO_BUFFER: + / + * PHYP's page counter is absolute for the page currently + * mapped, so a reopen or queue reuse restarts it near zero. + * ibmveth_update_rx_no_buffer() folds each decrease into the + * queue's retired carry; add both back to stay monotonic. + / + return ibmveth_sum_rx_qstat(adapter, stat->off) + + ibmveth_sum_rx_qstat(adapter, + IBMVETH_RXQ_OFF(no_buffer_retired)); + case IBMVETH_STAT_ADAPTER: + break; + } + + return IBMVETH_GET_STAT(adapter, stat->off); +} + +static void ibmveth_get_strings(struct net_device dev, u32 stringset, u8 data) +{ + struct ibmveth_adapter adapter = netdev_priv(dev); + u8 p = data; + int i, j; + if (stringset != ETH_SS_STATS) return; - for (i = 0; i < ARRAY_SIZE(ibmveth_stats); i++, data += ETH_GSTRING_LEN) - memcpy(data, ibmveth_stats[i].name, ETH_GSTRING_LEN); + for (i = 0; i < ARRAY_SIZE(ibmveth_stats); i++) { + memcpy(p, ibmveth_stats[i].name, ETH_GSTRING_LEN); + p += ETH_GSTRING_LEN; + } + + for (i = 0; i < ibmveth_get_num_rx_queues(adapter); i++) + for (j = 0; j < IBMVETH_NUM_RX_QSTATS; j++) + ethtool_sprintf(&p, ibmveth_rx_qstat_keys[j].fmt, i); + + for (i = 0; i < dev->real_num_tx_queues; i++) + for (j = 0; j < IBMVETH_NUM_TX_QSTATS; j++) + ethtool_sprintf(&p, ibmveth_tx_qstat_keys[j].fmt, i); } static int ibmveth_get_sset_count(struct net_device dev, int sset) { + struct ibmveth_adapter adapter = netdev_priv(dev); + switch (sset) { case ETH_SS_STATS: - return ARRAY_SIZE(ibmveth_stats); + return ARRAY_SIZE(ibmveth_stats) + + ibmveth_get_num_rx_queues(adapter) + IBMVETH_NUM_RX_QSTATS + + dev->real_num_tx_queues * IBMVETH_NUM_TX_QSTATS; default: return -EOPNOTSUPP; } @@ -1097,44 +3372,211 @@ static int ibmveth_get_sset_count(struct net_device dev, int sset) static void ibmveth_get_ethtool_stats(struct net_device dev, struct ethtool_stats stats, u64 data) { - int i; struct ibmveth_adapter adapter = netdev_priv(dev); + int i, j, k; for (i = 0; i < ARRAY_SIZE(ibmveth_stats); i++) - data[i] = IBMVETH_GET_STAT(adapter, ibmveth_stats[i].offset); + data[i] = ibmveth_ethtool_adapter_stat(adapter, i); + + for (j = 0; j < ibmveth_get_num_rx_queues(adapter); j++) { + const u8 q = (const u8 )&adapter->rx_qstats[j]; + + for (k = 0; k < IBMVETH_NUM_RX_QSTATS; k++) + data[i++] = (const u64 ) + (q + ibmveth_rx_qstat_keys[k].off); + } + + for (j = 0; j < dev->real_num_tx_queues; j++) { + const u8 q = (const u8 )&adapter->tx_qstats[j]; + + for (k = 0; k < IBMVETH_NUM_TX_QSTATS; k++) + data[i++] = (const u64 ) + (q + ibmveth_tx_qstat_keys[k].off); + } } static void ibmveth_get_channels(struct net_device netdev, struct ethtool_channels channels) { - channels->max_tx = ibmveth_real_max_tx_queues(); + struct ibmveth_adapter adapter = netdev_priv(netdev); + unsigned int rx_count = ibmveth_get_num_rx_queues(adapter); + channels->tx_count = netdev->real_num_tx_queues; + /* + * Never advertise max_tx below the live count. After CPU + * offline, real_max can drop below real_num_tx_queues; + * ethtool -L is read-modify-write and the core rejects + * tx_count > max_tx, which would also block an RX-only + * request. set_channels uses this same ceiling. + / + channels->max_tx = max_t(unsigned int, channels->tx_count, + ibmveth_real_max_tx_queues()); + + / + * Always report the live RX count. ethtool -L is read-modify- + * write, so a TX-only request echoes rx_count back at us; an + * understated value would be applied as a silent RX shrink. + * mq_fallback instead caps max_rx at the live count, which + * blocks growth in the core without misreporting what is + * currently configured. + / + channels->rx_count = rx_count; + if (adapter->multi_queue && !adapter->mq_fallback) + channels->max_rx = IBMVETH_MAX_RX_QUEUES; + else + channels->max_rx = rx_count; +} + +/* + * ibmveth_resize_rx_channels - Validate and apply a new RX queue count + * @adapter: ibmveth adapter + * @goal_rx: requested RX queue count + * + * Rejects rx > 1 without MQ firmware (-EOPNOTSUPP) and rx outside + * 1..IBMVETH_MAX_RX_QUEUES (-EINVAL). When RX resources are not live + * (!opened), only validate; do not allocate. When up, apply via + * ibmveth_resize_rx_queues_incremental(). + * + * Return: 0 or negative errno + / +static int ibmveth_resize_rx_channels(struct ibmveth_adapter adapter, + unsigned int goal_rx) +{ + struct net_device netdev = adapter->netdev; + unsigned int old_rx = ibmveth_get_num_rx_queues(adapter); + int rxq_entries; + int rc; + + if (goal_rx < 1 \|\| goal_rx > IBMVETH_MAX_RX_QUEUES) { + netdev_err(netdev, + "Invalid RX queue count %u (must be 1-%d)\n", + goal_rx, IBMVETH_MAX_RX_QUEUES); + return -EINVAL; + } + + / + * Check for a no-op before the capability gate. ethtool -L is + * read-modify-write, so a TX-only request arrives carrying the + * current RX count; gating first would fail those with + * -EOPNOTSUPP once mq_fallback is set. + / + if (goal_rx == old_rx) + return 0; + + / + * Refuse any rx > 1, not just growth: once mq_fallback is set the + * next open comes up single-queue, so an intermediate count could + * not be honoured either, and accepting it would only repeat the + * silent clamp at open. max_rx stays at the live count so that + * read-modify-write TX-only requests still clear the core. + / + if (goal_rx > 1 && (!adapter->multi_queue \|\| adapter->mq_fallback)) { + netdev_err(netdev, + "Cannot resize to %u RX queues: multi-queue mode not supported by firmware\n", + goal_rx); + return -EOPNOTSUPP; + } - channels->max_rx = netdev->real_num_rx_queues; - channels->rx_count = netdev->real_num_rx_queues; + / Down / failed-open: do not allocate. / + if (!adapter->opened) + return 0; + + rxq_entries = adapter->rx_queue[0].num_slots; + rc = ibmveth_resize_rx_queues_incremental(adapter, goal_rx, + rxq_entries); + if (rc) + netdev_err(netdev, "Failed to resize RX queues: %d\n", rc); + return rc; } static int ibmveth_set_channels(struct net_device netdev, struct ethtool_channels channels) { struct ibmveth_adapter adapter = netdev_priv(netdev); - unsigned int old = netdev->real_num_tx_queues, - goal = channels->tx_count; + unsigned int old_rx = ibmveth_get_num_rx_queues(adapter); + unsigned int goal_rx = channels->rx_count; + unsigned int old_tx = netdev->real_num_tx_queues; + unsigned int goal_tx = channels->tx_count; + unsigned int want_tx = goal_tx; + unsigned int max_tx; + bool rx_changed = false; int rc, i; - /* If ndo_open has not been called yet then don't allocate, just set - * desired netdev_queue's and return + /* + * Same ceiling get_channels reports: live count or the + * online-CPU cap, whichever is larger. + / + max_tx = max_t(unsigned int, old_tx, + ibmveth_real_max_tx_queues()); + if (goal_tx < 1 \|\| goal_tx > max_tx) { + netdev_err(netdev, + "Invalid TX queue count %u (must be 1-%u)\n", + goal_tx, max_tx); + return -EINVAL; + } + + / RX range / MQ checks live in ibmveth_resize_rx_channels(). / + rc = ibmveth_resize_rx_channels(adapter, goal_rx); + if (rc) + return rc; + + / If RX resources are not live (never opened, or close+open failed + * while IFF_UP stayed set), publish desired queue counts without + * allocating. / - if (!(netdev->flags & IFF_UP)) - return netif_set_real_num_tx_queues(netdev, goal); + if (!adapter->opened) { + / Apply TX first so a failure leaves the published RX + * count unchanged. + / + rc = netif_set_real_num_tx_queues(netdev, goal_tx); + if (rc) + return rc; + + / Publish desired RX count for next open() and refresh CMO; + * do not allocate while down. + / + if (goal_rx != ibmveth_get_num_rx_queues(adapter)) { + ibmveth_publish_num_rx_queues(adapter, goal_rx); + rc = netif_set_real_num_rx_queues(netdev, goal_rx); + if (rc) { + int tx_rc; + + ibmveth_publish_num_rx_queues(adapter, old_rx); + tx_rc = netif_set_real_num_tx_queues(netdev, + old_tx); + if (tx_rc) + netdev_err(netdev, + "Failed to restore TX queues to %u after RX failure: %d\n", + old_tx, tx_rc); + return rc; + } + if (firmware_has_feature(FW_FEATURE_CMO)) { + unsigned long dma; + + dma = ibmveth_get_desired_dma(adapter->vdev); + vio_cmo_set_dev_desired(adapter->vdev, dma); + } + } + return 0; + } + + if (goal_rx != old_rx) + rx_changed = true; / We have IBMVETH_MAX_QUEUES netdev_queue's allocated * but we may need to alloc/free the ltb's. / + if (goal_tx == old_tx) + return 0; + netif_tx_stop_all_queues(netdev); - / Allocate any queue that we need / - for (i = old; i < goal; i++) { + / Allocate any new TX LTBs. i starts at old_tx for the free walk + * below when this loop body never runs (goal_tx == old_tx already + * returned; goal_tx < old_tx is scale-down). + / + i = old_tx; + for (; i < goal_tx; i++) { if (adapter->tx_ltb_ptr[i]) continue; @@ -1143,28 +3585,48 @@ static int ibmveth_set_channels(struct net_device netdev, continue; /* if something goes wrong, free everything we just allocated / - netdev_err(netdev, "Failed to allocate more tx queues, returning to %d queues\n", - old); - goal = old; - old = i; + netdev_err(netdev, "Failed to allocate more tx queues, returning to %u queues\n", + old_tx); + goal_tx = old_tx; + old_tx = i; break; } - rc = netif_set_real_num_tx_queues(netdev, goal); + rc = netif_set_real_num_tx_queues(netdev, goal_tx); if (rc) { - netdev_err(netdev, "Failed to set real tx queues, returning to %d queues\n", - old); - goal = old; - old = i; + netdev_err(netdev, "Failed to set real tx queues, returning to %u queues\n", + old_tx); + goal_tx = old_tx; + old_tx = i; } / Free any that are no longer needed / - for (i = old; i > goal; i--) { + for (i = old_tx; i > goal_tx; i--) { if (adapter->tx_ltb_ptr[i - 1]) ibmveth_free_tx_ltb(adapter, i - 1); } netif_tx_wake_all_queues(netdev); - return rc; + if (netdev->real_num_tx_queues != want_tx) { + if (rx_changed) { + / + * Restore the RX count from before this -L. + * num_slots is the live size after that resize. + / + int rxq_entries = adapter->rx_queue[0].num_slots; + int rb; + + rb = ibmveth_resize_rx_queues_incremental(adapter, + old_rx, + rxq_entries); + if (rb) + netdev_err(netdev, + "Failed to roll back RX queues to %u after TX failure: %d\n", + old_rx, rb); + } + return rc ? rc : -ENOMEM; + } + + return 0; } static const struct ethtool_ops netdev_ethtool_ops = { @@ -1213,8 +3675,10 @@ static int ibmveth_send(struct ibmveth_adapter adapter, } static int ibmveth_is_packet_unsupported(struct sk_buff skb, - struct net_device netdev) + struct ibmveth_adapter adapter, + int queue_num) { + struct net_device netdev = adapter->netdev; struct ethhdr ether_header; int ret = 0; @@ -1222,7 +3686,7 @@ static int ibmveth_is_packet_unsupported(struct sk_buff skb, if (ether_addr_equal(ether_header->h_dest, netdev->dev_addr)) { netdev_dbg(netdev, "veth doesn't support loopback packets, dropping packet.\n"); - netdev->stats.tx_dropped++; + adapter->tx_qstats[queue_num].dropped_packets++; ret = -EOPNOTSUPP; } @@ -1238,7 +3702,13 @@ static netdev_tx_t ibmveth_start_xmit(struct sk_buff skb, int i, queue_num = skb_get_queue_mapping(skb); unsigned long mss = 0; - if (ibmveth_is_packet_unsupported(skb, netdev)) + / Close / failed reopen can free LTBs while IFF_UP is still set. / + if (unlikely(!adapter->tx_ltb_ptr[queue_num])) { + adapter->tx_qstats[queue_num].dropped_packets++; + goto out; + } + + if (ibmveth_is_packet_unsupported(skb, adapter, queue_num)) goto out; / veth can't checksum offload UDP / if (skb->ip_summed == CHECKSUM_PARTIAL && @@ -1249,7 +3719,7 @@ static netdev_tx_t ibmveth_start_xmit(struct sk_buff skb, skb_checksum_help(skb)) { netdev_err(netdev, "tx: failed to checksum packet\n"); - netdev->stats.tx_dropped++; + adapter->tx_qstats[queue_num].dropped_packets++; goto out; } @@ -1261,6 +3731,8 @@ static netdev_tx_t ibmveth_start_xmit(struct sk_buff skb, desc_flags \|= (IBMVETH_BUF_NO_CSUM \| IBMVETH_BUF_CSUM_GOOD); + adapter->tx_qstats[queue_num].checksum_offload++; + / Need to zero out the checksum / buf[0] = 0; buf[1] = 0; @@ -1272,7 +3744,7 @@ static netdev_tx_t ibmveth_start_xmit(struct sk_buff skb, if (skb->ip_summed == CHECKSUM_PARTIAL && skb_is_gso(skb)) { if (adapter->fw_large_send_support) { mss = (unsigned long)skb_shinfo(skb)->gso_size; - adapter->tx_large_packets++; + adapter->tx_qstats[queue_num].large_packets++; } else if (!skb_is_gso_v6(skb)) { /* Put -1 in the IP checksum to tell phyp it * is a largesend packet. Put the mss in @@ -1281,7 +3753,7 @@ static netdev_tx_t ibmveth_start_xmit(struct sk_buff skb, ip_hdr(skb)->check = 0xffff; tcp_hdr(skb)->check = cpu_to_be16(skb_shinfo(skb)->gso_size); - adapter->tx_large_packets++; + adapter->tx_qstats[queue_num].large_packets++; } } @@ -1289,7 +3761,7 @@ static netdev_tx_t ibmveth_start_xmit(struct sk_buff skb, if (unlikely(skb->len > adapter->tx_ltb_size)) { netdev_err(adapter->netdev, "tx: packet size (%u) exceeds ltb (%u)\n", skb->len, adapter->tx_ltb_size); - netdev->stats.tx_dropped++; + adapter->tx_qstats[queue_num].dropped_packets++; goto out; } memcpy(adapter->tx_ltb_ptr[queue_num], skb->data, skb_headlen(skb)); @@ -1306,7 +3778,7 @@ static netdev_tx_t ibmveth_start_xmit(struct sk_buff skb, if (unlikely(total_bytes != skb->len)) { netdev_err(adapter->netdev, "tx: incorrect packet len copied into ltb (%u != %u)\n", skb->len, total_bytes); - netdev->stats.tx_dropped++; + adapter->tx_qstats[queue_num].dropped_packets++; goto out; } desc.fields.flags_len = desc_flags \| skb->len; @@ -1315,11 +3787,11 @@ static netdev_tx_t ibmveth_start_xmit(struct sk_buff skb, dma_wmb(); if (ibmveth_send(adapter, desc.desc, mss)) { - adapter->tx_send_failed++; - netdev->stats.tx_dropped++; + adapter->tx_qstats[queue_num].send_failures++; + adapter->tx_qstats[queue_num].dropped_packets++; } else { - netdev->stats.tx_packets++; - netdev->stats.tx_bytes += skb->len; + adapter->tx_qstats[queue_num].packets++; + adapter->tx_qstats[queue_num].bytes += skb->len; } out: @@ -1445,102 +3917,226 @@ static void ibmveth_rx_csum_helper(struct sk_buff skb, } } -static int ibmveth_poll(struct napi_struct napi, int budget) +static void ibmveth_poll_bump_invalid(struct ibmveth_adapter adapter, + int queue_index) { - struct ibmveth_adapter adapter = - container_of(napi, struct ibmveth_adapter, napi); - struct net_device netdev = adapter->netdev; - int frames_processed = 0; - unsigned long lpar_rc; + adapter->rx_qstats[queue_index].invalid_buffers++; +} + +static bool ibmveth_poll_stopping(struct net_device netdev, + struct napi_struct napi) +{ + return !netif_running(netdev) \|\| napi_disable_pending(napi); +} + +static bool ibmveth_poll_harvest_slot(struct ibmveth_adapter adapter, + int queue_index, bool reuse) +{ + int rc = ibmveth_rxq_harvest_buffer(adapter, queue_index, reuse); + + return !rc \|\| rc == -EINVAL \|\| rc == -EFAULT; +} + +static bool ibmveth_poll_recycle_invalid(struct net_device netdev, + struct ibmveth_adapter adapter, + int queue_index) +{ + netdev_dbg(netdev, "recycling invalid buffer\n"); + ibmveth_poll_bump_invalid(adapter, queue_index); + return ibmveth_poll_harvest_slot(adapter, queue_index, true); +} + +static bool ibmveth_poll_skip_bad_correlator(struct net_device netdev, + struct ibmveth_adapter adapter, + int queue_index) +{ + if (net_ratelimit()) + netdev_err(netdev, + "bad correlator on queue %d, skipping slot\n", + queue_index); + /* Residual stale slot after resize: recover via reset rather + * than spinning forever. Always escalate; only the log is + * rate-limited. + / + schedule_work(&adapter->work); + ibmveth_poll_bump_invalid(adapter, queue_index); + return ibmveth_poll_harvest_slot(adapter, queue_index, true); +} + +static bool ibmveth_poll_drop_oversize(struct net_device netdev, + struct ibmveth_adapter adapter, + int queue_index, unsigned int off, + unsigned int len, unsigned int room) +{ + if (net_ratelimit()) + netdev_err(netdev, + "RX frame %u+%u exceeds buffer %u on queue %d, dropping\n", + off, len, room, queue_index); + ibmveth_poll_bump_invalid(adapter, queue_index); + return ibmveth_poll_harvest_slot(adapter, queue_index, true); +} + +/* + * ibmveth_poll_deliver_frame - Build SKB from one valid RX slot and GRO it + * @napi: NAPI context for this RX queue + * @adapter: ibmveth adapter + * @netdev: net_device for @adapter + * @queue_index: RX queue index + * + * Return: 1 frame delivered, 0 if the slot was skipped cleanly, -1 on error. + / +static int ibmveth_poll_deliver_frame(struct napi_struct napi, + struct ibmveth_adapter adapter, + struct net_device netdev, + int queue_index) +{ + struct sk_buff skb, new_skb; + struct ibmveth_rx_q rxq = &adapter->rx_queue[queue_index]; + unsigned int room, off, len, pool; + int length, offset, csum_good, lrg_pkt; + __sum16 iph_check = 0; u16 mss = 0; + int rc; -restart_poll: - while (frames_processed < budget) { - if (!ibmveth_rxq_pending_buffer(adapter)) - break; + length = ibmveth_rxq_frame_length(adapter, queue_index); + offset = ibmveth_rxq_frame_offset(adapter, queue_index); + csum_good = ibmveth_rxq_csum_good(adapter, queue_index); + lrg_pkt = ibmveth_rxq_large_packet(adapter, queue_index); + + skb = ibmveth_rxq_get_buffer(adapter, queue_index); + if (unlikely(!skb)) { + if (!ibmveth_poll_skip_bad_correlator(netdev, adapter, + queue_index)) + return -1; + return 0; + } - smp_rmb(); - if (!ibmveth_rxq_buffer_valid(adapter)) { - wmb(); / suggested by larson1 / - adapter->rx_invalid_buffer++; - netdev_dbg(netdev, "recycling invalid buffer\n"); - if (unlikely(ibmveth_rxq_harvest_buffer(adapter, true))) - break; - } else { - struct sk_buff skb, new_skb; - int length = ibmveth_rxq_frame_length(adapter); - int offset = ibmveth_rxq_frame_offset(adapter); - int csum_good = ibmveth_rxq_csum_good(adapter); - int lrg_pkt = ibmveth_rxq_large_packet(adapter); - __sum16 iph_check = 0; - - skb = ibmveth_rxq_get_buffer(adapter); - if (unlikely(!skb)) - break; + pool = rxq->queue_addr[rxq->index].correlator >> 32; + room = min_t(unsigned int, skb_tailroom(skb), + adapter->rx_buff_pool[queue_index][pool].buff_size); + off = offset; + len = length; + if (unlikely(off >= room \|\| len > room - off)) { + if (!ibmveth_poll_drop_oversize(netdev, adapter, queue_index, + off, len, room)) + return -1; + return 0; + } - / if the large packet bit is set in the rx queue - * descriptor, the mss will be written by PHYP eight - * bytes from the start of the rx buffer, which is - * skb->data at this stage - / - if (lrg_pkt) { - __be64 rxmss = (__be64 )(skb->data + 8); + if (lrg_pkt) { + __be64 rxmss = (__be64 )(skb->data + 8); - mss = (u16)be64_to_cpu(rxmss); - } + mss = (u16)be64_to_cpu(rxmss); + } - new_skb = NULL; - if (length < rx_copybreak) - new_skb = netdev_alloc_skb(netdev, length); - - if (new_skb) { - skb_copy_to_linear_data(new_skb, - skb->data + offset, - length); - if (rx_flush) - ibmveth_flush_buffer(skb->data, - length + offset); - if (unlikely(ibmveth_rxq_harvest_buffer(adapter, true))) - break; - skb = new_skb; - } else { - if (unlikely(ibmveth_rxq_harvest_buffer(adapter, false))) - break; - skb_reserve(skb, offset); - } + new_skb = NULL; + if (length < rx_copybreak) + new_skb = netdev_alloc_skb(netdev, length); + + if (new_skb) { + skb_copy_to_linear_data(new_skb, skb->data + offset, length); + if (rx_flush) + ibmveth_flush_buffer(skb->data, length + offset); + rc = ibmveth_rxq_harvest_buffer(adapter, queue_index, true); + if (unlikely(rc)) { + kfree_skb(new_skb); + return -1; + } + skb = new_skb; + } else { + rc = ibmveth_rxq_harvest_buffer(adapter, queue_index, false); + if (unlikely(rc)) + return -1; + skb_reserve(skb, offset); + } - skb_put(skb, length); - skb->protocol = eth_type_trans(skb, netdev); + skb_put(skb, length); + skb->protocol = eth_type_trans(skb, netdev); - / PHYP without PLSO support places a -1 in the ip - * checksum for large send frames. - / - if (skb->protocol == cpu_to_be16(ETH_P_IP)) { - struct iphdr iph = (struct iphdr )skb->data; + if (skb->protocol == cpu_to_be16(ETH_P_IP)) + iph_check = ((struct iphdr )skb->data)->check; - iph_check = iph->check; - } + if ((length > netdev->mtu + ETH_HLEN) \|\| lrg_pkt \|\| + iph_check == 0xffff) { + ibmveth_rx_mss_helper(skb, mss, lrg_pkt); + adapter->rx_qstats[queue_index].large_packets++; + } - if ((length > netdev->mtu + ETH_HLEN) \|\| - lrg_pkt \|\| iph_check == 0xffff) { - ibmveth_rx_mss_helper(skb, mss, lrg_pkt); - adapter->rx_large_packets++; - } + if (csum_good) { + skb->ip_summed = CHECKSUM_UNNECESSARY; + ibmveth_rx_csum_helper(skb, adapter); + } - if (csum_good) { - skb->ip_summed = CHECKSUM_UNNECESSARY; - ibmveth_rx_csum_helper(skb, adapter); - } + skb_record_rx_queue(skb, queue_index); + napi_gro_receive(napi, skb); + + adapter->rx_qstats[queue_index].packets++; + adapter->rx_qstats[queue_index].bytes += length; + + return 1; +} + +static int ibmveth_poll(struct napi_struct napi, int budget) +{ + struct net_device netdev = napi->dev; + struct ibmveth_adapter adapter = netdev_priv(netdev); + int frames_processed = 0; + int queue_index, rc; + + queue_index = napi - adapter->napi; + + if (WARN_ON(queue_index < 0 \|\| + queue_index >= ibmveth_get_num_rx_queues(adapter))) { + if (budget) + napi_complete_done(napi, 0); + return 0; + } + + if (ibmveth_poll_stopping(netdev, napi)) { + if (budget) + napi_complete_done(napi, 0); + return 0; + } + + adapter->rx_qstats[queue_index].polls++; + +restart_poll: + while (frames_processed < budget) { + if (ibmveth_poll_stopping(netdev, napi)) + break; - napi_gro_receive(napi, skb); / send it up / + if (!ibmveth_rxq_pending_buffer(adapter, queue_index)) + break; - netdev->stats.rx_packets++; - netdev->stats.rx_bytes += length; - frames_processed++; + smp_rmb(); + if (!ibmveth_rxq_buffer_valid(adapter, queue_index)) { + wmb(); / suggested by larson1 / + if (!ibmveth_poll_recycle_invalid(netdev, adapter, + queue_index)) + break; + } else { + rc = ibmveth_poll_deliver_frame(napi, adapter, netdev, + queue_index); + if (rc < 0) + break; + if (rc > 0) + frames_processed++; } } - ibmveth_replenish_task(adapter); + ibmveth_replenish_task(adapter, queue_index); + + if (ibmveth_poll_stopping(netdev, napi)) { + / budget 0 is netpoll, which must not complete NAPI. + * Otherwise returning budget after completing would ask + * NAPI to reschedule, so cap the return at budget - 1. + / + if (budget) { + napi_complete_done(napi, frames_processed); + return min(frames_processed, budget - 1); + } + return 0; + } if (frames_processed == budget) goto out; @@ -1548,18 +4144,27 @@ static int ibmveth_poll(struct napi_struct napi, int budget) if (!napi_complete_done(napi, frames_processed)) goto out; - /* We think we are done - reenable interrupts, - * then check once more to make sure we are done. + /* + * napi_disable() sets DISABLE then waits for this poll. Without a + * second stopping check here, enable_irq() can re-arm PHYP after + * resize already masked the queue; late IRQs then hit the handler + * after num_rx_queues was published lower (lab WARN at interrupt). / - lpar_rc = h_vio_signal(adapter->vdev->unit_address, VIO_IRQ_ENABLE); - if (WARN_ON(lpar_rc != H_SUCCESS)) { + if (ibmveth_poll_stopping(netdev, napi)) + goto out; + + rc = ibmveth_enable_irq(adapter, queue_index); + if (rc) { + netdev_err(netdev, + "Failed to enable IRQ for queue %d (rc=%d), scheduling reset\n", + queue_index, rc); schedule_work(&adapter->work); goto out; } - if (ibmveth_rxq_pending_buffer(adapter) && napi_schedule(napi)) { - lpar_rc = h_vio_signal(adapter->vdev->unit_address, - VIO_IRQ_DISABLE); + if (ibmveth_rxq_pending_buffer(adapter, queue_index) && + napi_schedule(napi)) { + ibmveth_disable_irq(adapter, queue_index); goto restart_poll; } @@ -1569,16 +4174,23 @@ static int ibmveth_poll(struct napi_struct napi, int budget) static irqreturn_t ibmveth_interrupt(int irq, void dev_instance) { - struct net_device netdev = dev_instance; + struct napi_struct napi = dev_instance; + struct net_device netdev = napi->dev; struct ibmveth_adapter adapter = netdev_priv(netdev); - unsigned long lpar_rc; + int qindex; - if (napi_schedule_prep(&adapter->napi)) { - lpar_rc = h_vio_signal(adapter->vdev->unit_address, - VIO_IRQ_DISABLE); - WARN_ON(lpar_rc != H_SUCCESS); - __napi_schedule(&adapter->napi); - } + qindex = napi - adapter->napi; + / + * Quiet on out-of-range: scale-down publishes a lower live count + * before free_irq(). A residual IRQ must not WARN-storm; return + * IRQ_NONE until the handler is removed. + / + if (qindex < 0 \|\| qindex >= ibmveth_get_num_rx_queues(adapter)) + return IRQ_NONE; + + adapter->rx_qstats[qindex].interrupts++; + + ibmveth_schedule_rx_queue(adapter, qindex); return IRQ_HANDLED; } @@ -1645,7 +4257,7 @@ static int ibmveth_change_mtu(struct net_device dev, int new_mtu) int need_restart = 0; for (i = 0; i < IBMVETH_NUM_BUFF_POOLS; i++) - if (new_mtu_oh <= adapter->rx_buff_pool[i].buff_size) + if (new_mtu_oh <= adapter->rx_buff_pool[0][i].buff_size) break; if (i == IBMVETH_NUM_BUFF_POOLS) @@ -1660,9 +4272,9 @@ static int ibmveth_change_mtu(struct net_device dev, int new_mtu) / Look for an active buffer pool that can hold the new MTU / for (i = 0; i < IBMVETH_NUM_BUFF_POOLS; i++) { - adapter->rx_buff_pool[i].active = 1; + adapter->rx_buff_pool[0][i].active = 1; - if (new_mtu_oh <= adapter->rx_buff_pool[i].buff_size) { + if (new_mtu_oh <= adapter->rx_buff_pool[0][i].buff_size) { WRITE_ONCE(dev->mtu, new_mtu); vio_cmo_set_dev_desired(viodev, ibmveth_get_desired_dma @@ -1683,8 +4295,20 @@ static int ibmveth_change_mtu(struct net_device dev, int new_mtu) #ifdef CONFIG_NET_POLL_CONTROLLER static void ibmveth_poll_controller(struct net_device dev) { - ibmveth_replenish_task(netdev_priv(dev)); - ibmveth_interrupt(dev->irq, dev); + struct ibmveth_adapter adapter = netdev_priv(dev); + unsigned int num; + int i; + + if (!adapter->opened) + return; + + num = ibmveth_get_num_rx_queues(adapter); + + for (i = 0; i < num; i++) + ibmveth_replenish_task(adapter, i); + + for (i = 0; i < num; i++) + ibmveth_schedule_rx_queue(adapter, i); } #endif @@ -1701,9 +4325,6 @@ static unsigned long ibmveth_get_desired_dma(struct vio_dev vdev) struct net_device netdev = dev_get_drvdata(&vdev->dev); struct ibmveth_adapter adapter; struct iommu_table tbl; - unsigned long ret; - int i; - int rxqentries = 1; tbl = get_iommu_table_base(&vdev->dev); @@ -1712,26 +4333,8 @@ static unsigned long ibmveth_get_desired_dma(struct vio_dev vdev) return IOMMU_PAGE_ALIGN(IBMVETH_IO_ENTITLEMENT_DEFAULT, tbl); adapter = netdev_priv(netdev); - - ret = IBMVETH_BUFF_LIST_SIZE + IBMVETH_FILT_LIST_SIZE; - ret += IOMMU_PAGE_ALIGN(netdev->mtu, tbl); - / add size of mapped tx buffers / - ret += IOMMU_PAGE_ALIGN(IBMVETH_MAX_TX_BUF_SIZE, tbl); - - for (i = 0; i < IBMVETH_NUM_BUFF_POOLS; i++) { - / add the size of the active receive buffers / - if (adapter->rx_buff_pool[i].active) - ret += - adapter->rx_buff_pool[i].size - IOMMU_PAGE_ALIGN(adapter->rx_buff_pool[i]. - buff_size, tbl); - rxqentries += adapter->rx_buff_pool[i].size; - } - /* add the size of the receive queue entries / - ret += IOMMU_PAGE_ALIGN( - rxqentries sizeof(struct ibmveth_rx_q_entry), tbl); - - return ret; + return ibmveth_desired_dma_for_rxqs(adapter, + ibmveth_get_num_rx_queues(adapter)); } static int ibmveth_set_mac_addr(struct net_device dev, void p) @@ -1777,6 +4380,124 @@ static netdev_features_t ibmveth_features_check(struct sk_buff skb, return vlan_features_check(skb, features); } +/* + * ibmveth_get_stats64 - Return aggregated per-queue statistics + * @dev: network device + * @stats: rtnl link statistics storage + * + * Sums per-queue rx_qstats and tx_qstats into the rtnl counters. + * Walk the full allocated arrays (not the live queue count) so shrinking + * channels cannot make the totals go backwards. + * Callers use ndo_get_stats64(); avoid updating netdev->stats on the + * xmit/poll paths to keep per-queue counters off the hot cache line. + / +static void ibmveth_get_stats64(struct net_device dev, + struct rtnl_link_stats64 stats) +{ + struct ibmveth_adapter adapter = netdev_priv(dev); + int i; + + for (i = 0; i < IBMVETH_MAX_RX_QUEUES; i++) { + stats->rx_packets += adapter->rx_qstats[i].packets; + stats->rx_bytes += adapter->rx_qstats[i].bytes; + } + + for (i = 0; i < IBMVETH_MAX_QUEUES; i++) { + stats->tx_packets += adapter->tx_qstats[i].packets; + stats->tx_bytes += adapter->tx_qstats[i].bytes; + stats->tx_dropped += adapter->tx_qstats[i].dropped_packets; + } +} + +static void ibmveth_get_queue_stats_rx(struct net_device dev, int idx, + struct netdev_queue_stats_rx stats) +{ + struct ibmveth_adapter adapter = netdev_priv(dev); + + stats->packets = adapter->rx_qstats[idx].packets; + stats->bytes = adapter->rx_qstats[idx].bytes; + / + * All three are frames that entered the device and never left it, + * which is what rx-hw-drops is specified to cover: no_buffer_drops + * is PHYP dropping for lack of buffer space on the page mapped now, + * no_buffer_retired the same for pages this queue has already + * released, and invalid_buffers is a processing error. + / + stats->hw_drops = adapter->rx_qstats[idx].no_buffer_drops + + adapter->rx_qstats[idx].no_buffer_retired + + adapter->rx_qstats[idx].invalid_buffers; + stats->alloc_fail = adapter->rx_qstats[idx].replenish_no_mem; +} + +static void ibmveth_get_queue_stats_tx(struct net_device dev, int idx, + struct netdev_queue_stats_tx stats) +{ + struct ibmveth_adapter adapter = netdev_priv(dev); + + stats->packets = adapter->tx_qstats[idx].packets; + stats->bytes = adapter->tx_qstats[idx].bytes; + stats->hw_drops = adapter->tx_qstats[idx].dropped_packets; +} + +/** + * ibmveth_get_base_stats - account for traffic not on a live queue + * @dev: network device + * @rx: RX base statistics storage + * @tx: TX base statistics storage + * + * get_queue_stats_{rx,tx}() only report queues the core still iterates, + * i.e. below real_num_{rx,tx}_queues, while ibmveth_get_stats64() walks + * the full arrays so device totals stay monotonic across a shrink. + * Report the retired-queue remainder here, otherwise qstats and + * rtnl_link_stats64 disagree by a delta that grows with every shrink. + * Zeroing would not be neutral: per netdev_stat_ops it asserts the + * per-queue sum is already exact. + * + * Bound the live side with real_num__queues rather than the adapter's + own count, so the split lines up with the core's iteration exactly. + * + * Every field the per-queue callbacks fill must also be initialised + * here: netdev_nl_stats_add() starts the sum at NETDEV_STAT_NOT_SET and + * only accumulates while both sides are set, so a field left unset here + * is dropped from the device total even though the queues report it. + / +static void ibmveth_get_base_stats(struct net_device dev, + struct netdev_queue_stats_rx rx, + struct netdev_queue_stats_tx tx) +{ + struct ibmveth_adapter adapter = netdev_priv(dev); + unsigned int i; + + rx->packets = 0; + rx->bytes = 0; + rx->alloc_fail = 0; + rx->hw_drops = 0; + tx->packets = 0; + tx->bytes = 0; + tx->hw_drops = 0; + + for (i = dev->real_num_rx_queues; i < IBMVETH_MAX_RX_QUEUES; i++) { + rx->packets += adapter->rx_qstats[i].packets; + rx->bytes += adapter->rx_qstats[i].bytes; + rx->hw_drops += adapter->rx_qstats[i].no_buffer_drops + + adapter->rx_qstats[i].no_buffer_retired + + adapter->rx_qstats[i].invalid_buffers; + rx->alloc_fail += adapter->rx_qstats[i].replenish_no_mem; + } + + for (i = dev->real_num_tx_queues; i < IBMVETH_MAX_QUEUES; i++) { + tx->packets += adapter->tx_qstats[i].packets; + tx->bytes += adapter->tx_qstats[i].bytes; + tx->hw_drops += adapter->tx_qstats[i].dropped_packets; + } +} + +static const struct netdev_stat_ops ibmveth_stat_ops = { + .get_queue_stats_rx = ibmveth_get_queue_stats_rx, + .get_queue_stats_tx = ibmveth_get_queue_stats_tx, + .get_base_stats = ibmveth_get_base_stats, +}; + static const struct net_device_ops ibmveth_netdev_ops = { .ndo_open = ibmveth_open, .ndo_stop = ibmveth_close, @@ -1789,14 +4510,129 @@ static const struct net_device_ops ibmveth_netdev_ops = { .ndo_validate_addr = eth_validate_addr, .ndo_set_mac_address = ibmveth_set_mac_addr, .ndo_features_check = ibmveth_features_check, + .ndo_get_stats64 = ibmveth_get_stats64, #ifdef CONFIG_NET_POLL_CONTROLLER .ndo_poll_controller = ibmveth_poll_controller, #endif }; +static int ibmveth_buffer_pools_show(struct seq_file m, void v) +{ + struct ibmveth_adapter adapter = m->private; + int i, j; + + /* + * size / buff_size / pool->active are written under RTNL + * (veth_pool_store, open template copy). Take the same lock so + * those columns are not a torn snapshot. available is updated + * from NAPI/softirq; only atomic_read() keeps it from tearing. + * Not required for memory safety; embedded arrays only. + / + rtnl_lock(); + + seq_puts(m, "Queue Pool Count BuffSize Active Available\n"); + seq_puts(m, "----- ---- ----- -------- ------ ---------\n"); + if (!adapter->opened) { + seq_puts(m, "# down: Active/Available 0 unless allocated\n"); + seq_puts(m, "# down: geometry above queue 0 set at open\n"); + } + + for (i = 0; i < ibmveth_get_num_rx_queues(adapter); i++) { + for (j = 0; j < IBMVETH_NUM_BUFF_POOLS; j++) { + struct ibmveth_buff_pool pool = + &adapter->rx_buff_pool[i][j]; + bool live = pool->skbuff && pool->free_map; + int active = live ? pool->active : 0; + int available = live ? atomic_read(&pool->available) + : 0; + + seq_printf(m, "%5d %4d %5u %8u %6d %9d\n", + i, j, pool->size, pool->buff_size, + active, available); + } + } + + rtnl_unlock(); + return 0; +} +DEFINE_SHOW_ATTRIBUTE(ibmveth_buffer_pools); + +/* Driver-owned root so per-adapter dirs use a stable vio name, not the + * mutable netdev->name (avoids stale names / eth0 collisions after rename). + / +static struct dentry ibmveth_dbg_root; + +static void ibmveth_debugfs_init(struct ibmveth_adapter adapter) +{ + adapter->debugfs_dir = + debugfs_create_dir(dev_name(&adapter->vdev->dev), + ibmveth_dbg_root); + debugfs_create_file("buffer_pools", 0400, adapter->debugfs_dir, + adapter, &ibmveth_buffer_pools_fops); +} + +static void ibmveth_debugfs_exit(struct ibmveth_adapter adapter) +{ + debugfs_remove_recursive(adapter->debugfs_dir); + adapter->debugfs_dir = NULL; +} + +/** + * ibmveth_pool_kobj_release - Mark a pool kobject finished + * @kobj: kobject embedded in the pool + * + * Pool kobjects live in netdev_priv(). Last put waits for this + * before free_netdev() so DEBUG_KOBJECT_RELEASE delayed cleanup + * does not run on freed memory. + / +static void ibmveth_pool_kobj_release(struct kobject kobj) +{ + struct ibmveth_buff_pool pool = container_of(kobj, + struct ibmveth_buff_pool, + kobj); + + complete(&pool->released); +} + +/* + * ibmveth_put_pool_kobjs - Drop pool kobjects and wait for release + * @adapter: ibmveth adapter + * @pools_ready: number of queue-0 pools that were initialized + * + * Put every initialized pool kobject, then wait so free_netdev() + * cannot race DEBUG_KOBJECT_RELEASE delayed cleanup. + / +static void ibmveth_put_pool_kobjs(struct ibmveth_adapter adapter, + int pools_ready) +{ + int i; + + for (i = 0; i < pools_ready; i++) + kobject_put(&adapter->rx_buff_pool[0][i].kobj); + for (i = 0; i < pools_ready; i++) + wait_for_completion(&adapter->rx_buff_pool[0][i].released); +} + +static void ibmveth_probe_cleanup(struct ibmveth_adapter adapter, + int pools_ready) +{ + struct net_device netdev = adapter->netdev; + + cancel_work_sync(&adapter->work); + ibmveth_put_pool_kobjs(adapter, pools_ready); + + ibmveth_free_tx_qstats(adapter); + ibmveth_free_rx_qstats(adapter); + /* Probe failure never reaches ibmveth_remove(); clear before free so + * CMO get_desired_dma() cannot see a freed netdev on rebind. + / + dev_set_drvdata(&adapter->vdev->dev, NULL); + free_netdev(netdev); +} + static int ibmveth_probe(struct vio_dev dev, const struct vio_device_id id) { - int rc, i, mac_len; + int rc, i, mac_len, pools_ready = 0; struct net_device netdev; struct ibmveth_adapter adapter; unsigned char mac_addr_p; @@ -1831,7 +4667,8 @@ static int ibmveth_probe(struct vio_dev dev, const struct vio_device_id id) return -EINVAL; } - netdev = alloc_etherdev_mqs(sizeof(struct ibmveth_adapter), IBMVETH_MAX_QUEUES, 1); + netdev = alloc_etherdev_mqs(sizeof(struct ibmveth_adapter), + IBMVETH_MAX_QUEUES, IBMVETH_MAX_RX_QUEUES); if (!netdev) return -ENOMEM; @@ -1841,14 +4678,33 @@ static int ibmveth_probe(struct vio_dev dev, const struct vio_device_id id) adapter->vdev = dev; adapter->netdev = netdev; INIT_WORK(&adapter->work, ibmveth_reset); + + /* + * Initialise the replenish locks once. open() is re-entered on + * MTU and offload changes without netpoll_poll_disable(), so a + * lock set up there could be reinitialised while poll_controller() + * holds it. + / + for (i = 0; i < IBMVETH_MAX_RX_QUEUES; i++) + spin_lock_init(&adapter->rx_queue[i].replenish_lock); + adapter->mcastFilterSize = be32_to_cpu(mcastFilterSize_p); ibmveth_init_link_settings(netdev); - netif_napi_add_weight(netdev, &adapter->napi, ibmveth_poll, 16); + for (i = 0; i < IBMVETH_MAX_RX_QUEUES; i++) + netif_napi_add_weight(netdev, &adapter->napi[i], + ibmveth_poll, 16); + + if (ibmveth_alloc_rx_qstats(adapter) \|\| + ibmveth_alloc_tx_qstats(adapter)) { + ibmveth_probe_cleanup(adapter, 0); + return -ENOMEM; + } netdev->irq = dev->irq; netdev->netdev_ops = &ibmveth_netdev_ops; netdev->ethtool_ops = &netdev_ethtool_ops; + netdev->stat_ops = &ibmveth_stat_ops; SET_NETDEV_DEV(netdev, &dev->dev); netdev->hw_features = NETIF_F_SG; if (vio_get_attribute(dev, "ibm,illan-options", NULL) != NULL) { @@ -1876,12 +4732,30 @@ static int ibmveth_probe(struct vio_dev dev, const struct vio_device_id id) netdev->features \|= NETIF_F_FRAGLIST; } + if (ret == H_SUCCESS && + (ret_attr & IBMVETH_ILLAN_RX_MULTI_QUEUE_SUPPORT)) { + adapter->multi_queue = true; + ibmveth_publish_num_rx_queues(adapter, + min(num_online_cpus(), + IBMVETH_DEFAULT_QUEUES)); + netdev_dbg(netdev, "RX multi queue mode enabled: %u queues\n", + ibmveth_get_num_rx_queues(adapter)); + } else { + adapter->multi_queue = false; + ibmveth_publish_num_rx_queues(adapter, + IBMVETH_DEFAULT_RX_QUEUES); + } + if (ret == H_SUCCESS && (ret_attr & IBMVETH_ILLAN_RX_MULTI_BUFF_SUPPORT)) { - adapter->rx_buffers_per_hcall = IBMVETH_MAX_RX_PER_HCALL; + if (adapter->multi_queue) + adapter->rx_buffers_per_hcall = + IBMVETH_MAX_RX_PER_HCALL; + else + adapter->rx_buffers_per_hcall = IBMVETH_MAX_RX_REGULAR; netdev_dbg(netdev, "RX Multi-buffer hcall supported by FW, batch set to %u\n", - adapter->rx_buffers_per_hcall); + adapter->rx_buffers_per_hcall); } else { adapter->rx_buffers_per_hcall = 1; netdev_dbg(netdev, @@ -1898,16 +4772,29 @@ static int ibmveth_probe(struct vio_dev dev, const struct vio_device_id id) memcpy(pool_count, pool_count_cmo, sizeof(pool_count)); for (i = 0; i < IBMVETH_NUM_BUFF_POOLS; i++) { - struct kobject kobj = &adapter->rx_buff_pool[i].kobj; - int error; + struct kobject kobj = &adapter->rx_buff_pool[0][i].kobj; - ibmveth_init_buffer_pool(&adapter->rx_buff_pool[i], i, + ibmveth_init_buffer_pool(&adapter->rx_buff_pool[0][i], i, pool_count[i], pool_size[i], pool_active[i]); - error = kobject_init_and_add(kobj, &ktype_veth_pool, - &dev->dev.kobj, "pool%d", i); - if (!error) - kobject_uevent(kobj, KOBJ_ADD); + init_completion(&adapter->rx_buff_pool[0][i].released); + rc = kobject_init_and_add(kobj, &ktype_veth_pool, + &dev->dev.kobj, "pool%d", i); + if (rc) { + struct ibmveth_buff_pool pool = + &adapter->rx_buff_pool[0][i]; + + dev_err(&dev->dev, + "failed to create pool%d kobject: %d\n", i, rc); + / init_and_add takes a ref even on failure / + kobject_put(kobj); + wait_for_completion(&pool->released); + ibmveth_probe_cleanup(adapter, pools_ready); + return rc; + } + + pools_ready++; + kobject_uevent(kobj, KOBJ_ADD); } rc = netif_set_real_num_tx_queues(netdev, min(num_online_cpus(), @@ -1915,9 +4802,25 @@ static int ibmveth_probe(struct vio_dev dev, const struct vio_device_id id) if (rc) { netdev_dbg(netdev, "failed to set number of tx queues rc=%d\n", rc); - free_netdev(netdev); + ibmveth_probe_cleanup(adapter, pools_ready); + return rc; + } + + / + * alloc_etherdev_mqs() sized real_num_rx_queues to MAX. Match the + * advertised default (or SQ 1) before register_netdev so down-state + * readers agree with adapter->num_rx_queues / ethtool -l. + / + rc = netif_set_real_num_rx_queues(netdev, + ibmveth_get_num_rx_queues(adapter)); + + if (rc) { + netdev_dbg(netdev, "failed to set number of rx queues rc=%d\n", + rc); + ibmveth_probe_cleanup(adapter, pools_ready); return rc; } + adapter->tx_ltb_size = PAGE_ALIGN(IBMVETH_MAX_TX_BUF_SIZE); for (i = 0; i < IBMVETH_MAX_QUEUES; i++) adapter->tx_ltb_ptr[i] = NULL; @@ -1931,12 +4834,14 @@ static int ibmveth_probe(struct vio_dev dev, const struct vio_device_id id) if (rc) { netdev_dbg(netdev, "failed to register netdev rc=%d\n", rc); - free_netdev(netdev); + ibmveth_probe_cleanup(adapter, pools_ready); return rc; } netdev_dbg(netdev, "registered\n"); + ibmveth_debugfs_init(adapter); + return 0; } @@ -1944,14 +4849,22 @@ static void ibmveth_remove(struct vio_dev dev) { struct net_device netdev = dev_get_drvdata(&dev->dev); struct ibmveth_adapter adapter = netdev_priv(netdev); - int i; - cancel_work_sync(&adapter->work); + ibmveth_debugfs_exit(adapter); - for (i = 0; i < IBMVETH_NUM_BUFF_POOLS; i++) - kobject_put(&adapter->rx_buff_pool[i].kobj); + ibmveth_put_pool_kobjs(adapter, IBMVETH_NUM_BUFF_POOLS); + /* + * Unregister first so NAPI/xmit cannot re-arm reset work after we + * cancel it. cancel_work_sync() before unregister left a window + * where poll could schedule_work() and the worker ran after + * free_netdev(). + / unregister_netdev(netdev); + cancel_work_sync(&adapter->work); + + ibmveth_free_tx_qstats(adapter); + ibmveth_free_rx_qstats(adapter); free_netdev(netdev); dev_set_drvdata(&dev->dev, NULL); @@ -2035,11 +4948,12 @@ static ssize_t veth_pool_store(struct kobject kobj, struct attribute attr, / Make sure there is a buffer pool with buffers that can hold a packet of the size of the MTU / for (i = 0; i < IBMVETH_NUM_BUFF_POOLS; i++) { - if (pool == &adapter->rx_buff_pool[i]) + if (pool == &adapter->rx_buff_pool[0][i]) continue; - if (!adapter->rx_buff_pool[i].active) + if (!adapter->rx_buff_pool[0][i].active) continue; - if (mtu <= adapter->rx_buff_pool[i].buff_size) + if (mtu <= + adapter->rx_buff_pool[0][i].buff_size) break; } @@ -2092,8 +5006,8 @@ static ssize_t veth_pool_store(struct kobject kobj, struct attribute attr, } rtnl_unlock(); - / kick the interrupt handler to allocate/deallocate pools / - ibmveth_interrupt(netdev->irq, netdev); + / kick RX processing to allocate/deallocate pools / + ibmveth_schedule_rx_queue(adapter, 0); return count; unlock_err: @@ -2101,7 +5015,6 @@ static ssize_t veth_pool_store(struct kobject kobj, struct attribute attr, return rc; } - #define ATTR(_name, _mode) \ struct attribute veth_##_name##_attr = { \ .name = __stringify(_name), .mode = _mode, \ @@ -2125,7 +5038,7 @@ static const struct sysfs_ops veth_pool_ops = { }; static struct kobj_type ktype_veth_pool = { - .release = NULL, + .release = ibmveth_pool_kobj_release, .sysfs_ops = &veth_pool_ops, .default_groups = veth_pool_groups, }; @@ -2133,7 +5046,13 @@ static struct kobj_type ktype_veth_pool = { static int ibmveth_resume(struct device dev) { struct net_device netdev = dev_get_drvdata(dev); - ibmveth_interrupt(netdev->irq, netdev); + struct ibmveth_adapter adapter = netdev_priv(netdev); + unsigned int num = ibmveth_get_num_rx_queues(adapter); + int i; + + for (i = 0; i < num; i++) + ibmveth_schedule_rx_queue(adapter, i); + return 0; } @@ -2158,15 +5077,27 @@ static struct vio_driver ibmveth_driver = { static int __init ibmveth_module_init(void) { + int rc; + printk(KERN_DEBUG "%s: %s %s\n", ibmveth_driver_name, ibmveth_driver_string, ibmveth_driver_version); - return vio_register_driver(&ibmveth_driver); + ibmveth_dbg_root = debugfs_create_dir(ibmveth_driver_name, NULL); + + rc = vio_register_driver(&ibmveth_driver); + if (rc) { + debugfs_remove_recursive(ibmveth_dbg_root); + ibmveth_dbg_root = NULL; + } + + return rc; } static void __exit ibmveth_module_exit(void) { vio_unregister_driver(&ibmveth_driver); + debugfs_remove_recursive(ibmveth_dbg_root); + ibmveth_dbg_root = NULL; } module_init(ibmveth_module_init); @@ -2196,8 +5127,7 @@ static void ibmveth_reset_kunit(struct work_struct w) @test: pointer to kunit structure * * Tests the error returns from ibmveth_remove_buffer_from_pool. - * ibmveth_remove_buffer_from_pool also calls WARN_ON, so dmesg should be - * checked to see that these warnings happened. + * Bad correlators return -EINVAL/-EFAULT (no WARN_ON). * * Return: void / @@ -2211,28 +5141,45 @@ static void ibmveth_remove_buffer_from_pool_test(struct kunit test) INIT_WORK(&adapter->work, ibmveth_reset_kunit); + spin_lock_init(&adapter->rx_queue[0].replenish_lock); + /* Set sane values for buffer pools / for (int i = 0; i < IBMVETH_NUM_BUFF_POOLS; i++) - ibmveth_init_buffer_pool(&adapter->rx_buff_pool[i], i, + ibmveth_init_buffer_pool(&adapter->rx_buff_pool[0][i], i, pool_count[i], pool_size[i], pool_active[i]); - pool = &adapter->rx_buff_pool[0]; + pool = &adapter->rx_buff_pool[0][0]; pool->skbuff = kunit_kcalloc(test, pool->size, sizeof(void ), GFP_KERNEL); KUNIT_ASSERT_NOT_ERR_OR_NULL(test, pool->skbuff); + pool->free_map = kunit_kcalloc(test, pool->size, sizeof(u16), + GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, pool->free_map); correlator = ((u64)IBMVETH_NUM_BUFF_POOLS << 32) \| 0; - KUNIT_EXPECT_EQ(test, -EINVAL, ibmveth_remove_buffer_from_pool(adapter, correlator, false)); - KUNIT_EXPECT_EQ(test, -EINVAL, ibmveth_remove_buffer_from_pool(adapter, correlator, true)); - - correlator = ((u64)0 << 32) \| adapter->rx_buff_pool[0].size; - KUNIT_EXPECT_EQ(test, -EINVAL, ibmveth_remove_buffer_from_pool(adapter, correlator, false)); - KUNIT_EXPECT_EQ(test, -EINVAL, ibmveth_remove_buffer_from_pool(adapter, correlator, true)); + KUNIT_EXPECT_EQ(test, -EINVAL, + ibmveth_remove_buffer_from_pool(adapter, + correlator, 0, false)); + KUNIT_EXPECT_EQ(test, -EINVAL, + ibmveth_remove_buffer_from_pool(adapter, + correlator, 0, true)); + + correlator = ((u64)0 << 32) \| adapter->rx_buff_pool[0][0].size; + KUNIT_EXPECT_EQ(test, -EINVAL, + ibmveth_remove_buffer_from_pool(adapter, + correlator, 0, false)); + KUNIT_EXPECT_EQ(test, -EINVAL, + ibmveth_remove_buffer_from_pool(adapter, + correlator, 0, true)); correlator = (u64)0 \| 0; pool->skbuff[0] = NULL; - KUNIT_EXPECT_EQ(test, -EFAULT, ibmveth_remove_buffer_from_pool(adapter, correlator, false)); - KUNIT_EXPECT_EQ(test, -EFAULT, ibmveth_remove_buffer_from_pool(adapter, correlator, true)); + KUNIT_EXPECT_EQ(test, -EFAULT, + ibmveth_remove_buffer_from_pool(adapter, + correlator, 0, false)); + KUNIT_EXPECT_EQ(test, -EFAULT, + ibmveth_remove_buffer_from_pool(adapter, + correlator, 0, true)); flush_work(&adapter->work); } @@ -2241,9 +5188,7 @@ static void ibmveth_remove_buffer_from_pool_test(struct kunit test) ibmveth_rxq_get_buffer_test - unit test for ibmveth_rxq_get_buffer * @test: pointer to kunit structure * - * Tests ibmveth_rxq_get_buffer. ibmveth_rxq_get_buffer also calls WARN_ON for - * the NULL returns, so dmesg should be checked to see that these warnings - * happened. + * Tests ibmveth_rxq_get_buffer invalid correlator returns NULL without WARN. * * Return: void / @@ -2258,31 +5203,37 @@ static void ibmveth_rxq_get_buffer_test(struct kunit test) INIT_WORK(&adapter->work, ibmveth_reset_kunit); - adapter->rx_queue.queue_len = 1; - adapter->rx_queue.index = 0; - adapter->rx_queue.queue_addr = kunit_kzalloc(test, sizeof(struct ibmveth_rx_q_entry), - GFP_KERNEL); - KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adapter->rx_queue.queue_addr); + adapter->rx_queue[0].queue_len = 1; + adapter->rx_queue[0].index = 0; + adapter->rx_queue[0].queue_addr = + kunit_kzalloc(test, sizeof(struct ibmveth_rx_q_entry), + GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adapter->rx_queue[0].queue_addr); /* Set sane values for buffer pools / for (int i = 0; i < IBMVETH_NUM_BUFF_POOLS; i++) - ibmveth_init_buffer_pool(&adapter->rx_buff_pool[i], i, + ibmveth_init_buffer_pool(&adapter->rx_buff_pool[0][i], i, pool_count[i], pool_size[i], pool_active[i]); - pool = &adapter->rx_buff_pool[0]; + pool = &adapter->rx_buff_pool[0][0]; pool->skbuff = kunit_kcalloc(test, pool->size, sizeof(void ), GFP_KERNEL); KUNIT_ASSERT_NOT_ERR_OR_NULL(test, pool->skbuff); + pool->free_map = kunit_kcalloc(test, pool->size, sizeof(u16), + GFP_KERNEL); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, pool->free_map); - adapter->rx_queue.queue_addr[0].correlator = (u64)IBMVETH_NUM_BUFF_POOLS << 32 \| 0; - KUNIT_EXPECT_PTR_EQ(test, NULL, ibmveth_rxq_get_buffer(adapter)); + adapter->rx_queue[0].queue_addr[0].correlator = + (u64)IBMVETH_NUM_BUFF_POOLS << 32 \| 0; + KUNIT_EXPECT_PTR_EQ(test, NULL, ibmveth_rxq_get_buffer(adapter, 0)); - adapter->rx_queue.queue_addr[0].correlator = (u64)0 << 32 \| adapter->rx_buff_pool[0].size; - KUNIT_EXPECT_PTR_EQ(test, NULL, ibmveth_rxq_get_buffer(adapter)); + adapter->rx_queue[0].queue_addr[0].correlator = + (u64)0 << 32 \| adapter->rx_buff_pool[0][0].size; + KUNIT_EXPECT_PTR_EQ(test, NULL, ibmveth_rxq_get_buffer(adapter, 0)); pool->skbuff[0] = skb; - adapter->rx_queue.queue_addr[0].correlator = (u64)0 << 32 \| 0; - KUNIT_EXPECT_PTR_EQ(test, skb, ibmveth_rxq_get_buffer(adapter)); + adapter->rx_queue[0].queue_addr[0].correlator = (u64)0 << 32 \| 0; + KUNIT_EXPECT_PTR_EQ(test, skb, ibmveth_rxq_get_buffer(adapter, 0)); flush_work(&adapter->work); } diff --git a/drivers/net/ethernet/ibm/ibmveth.h b/drivers/net/ethernet/ibm/ibmveth.h index d87713668ed30..11eda224a6807 100644 --- a/drivers/net/ethernet/ibm/ibmveth.h +++ b/drivers/net/ethernet/ibm/ibmveth.h @@ -14,6 +14,9 @@ #ifndef _IBMVETH_H #define _IBMVETH_H +#include <linux/spinlock_types.h> +#include <linux/completion.h> + /* constants for H_MULTICAST_CTRL / #define IbmVethMcastReceptionModifyBit 0x80000UL #define IbmVethMcastReceptionEnableBit 0x20000UL @@ -28,6 +31,7 @@ #define IbmVethMcastRemoveFilter 0x2UL #define IbmVethMcastClearFilterTable 0x3UL +#define IBMVETH_ILLAN_RX_MULTI_QUEUE_SUPPORT 0x0000000000080000UL #define IBMVETH_ILLAN_RX_MULTI_BUFF_SUPPORT 0x0000000000040000UL #define IBMVETH_ILLAN_LRG_SR_ENABLED 0x0000000000010000UL #define IBMVETH_ILLAN_LRG_SND_SUPPORT 0x0000000000008000UL @@ -66,6 +70,148 @@ static inline long h_add_logical_lan_buffers(unsigned long unit_address, desc5, desc6, desc7, desc8); } +/* + * h_register_logical_lan_queue - Register a subordinate receive queue + * @unit_address: Device unit address + * @buffer_list: DMA address of 4KB page for tracking registered buffers + * @rec_queue: Buffer descriptor of receive queue + * @queue_handle: Output queue handle on success (may be NULL) + * @irq: Output hypervisor IRQ number on success (may be NULL) + * + * Registers a subordinate receive queue with the hypervisor. + * + * Return: + * H_SUCCESS (0) on success + * H_PARAMETER if parameters are invalid + * H_BUSY / H_LONG_BUSY_* if the resource is busy; retry + * H_FUNCTION if firmware does not support this hcall + * + * On success, hypervisor returns: + * R3: H_SUCCESS + * R4: Queue handle + * R5: IRQ number for this queue + / +static inline long +h_register_logical_lan_queue(unsigned long unit_address, + unsigned long buffer_list, + unsigned long rec_queue, + unsigned long queue_handle, + unsigned long irq) +{ + unsigned long retbuf[PLPAR_HCALL_BUFSIZE]; + long rc; + + rc = plpar_hcall(H_REG_LOGICAL_LAN_QUEUE, + retbuf, unit_address, + buffer_list, rec_queue); + + if (rc == H_SUCCESS) { + if (queue_handle) + queue_handle = retbuf[0]; + if (irq) + irq = retbuf[1]; + } + + return rc; +} + +/* + * h_add_logical_lan_buffers_queue - Add buffers to subordinate queue + * @unit_address: Device unit address + * @queue_handle: Queue handle from h_register_logical_lan_queue() or + * h_register_logical_lan_with_handle() (queue 0) + * @buffersznum: Buffer size (upper 32 bits) \| count (lower 32 bits) + * @ioba12: Buffer addresses 1 and 2 packed ((addr1 << 32) \| addr2) + * @ioba34: Buffer addresses 3 and 4 packed + * @ioba56: Buffer addresses 5 and 6 packed + * @ioba78: Buffer addresses 7 and 8 packed + * @ioba910: Buffer addresses 9 and 10 packed + * @ioba1112: Buffer addresses 11 and 12 packed + * + * Return: + * H_SUCCESS - All buffers added successfully + * H_PARAMETER - Invalid parameters + * H_HARDWARE - Hardware error + * H_FUNCTION - Firmware does not support this hcall + / +static inline long h_add_logical_lan_buffers_queue(unsigned long unit_address, + unsigned long queue_handle, + unsigned long buffersznum, + unsigned long ioba12, + unsigned long ioba34, + unsigned long ioba56, + unsigned long ioba78, + unsigned long ioba910, + unsigned long ioba1112) +{ + unsigned long retbuf[PLPAR_HCALL9_BUFSIZE]; + + return plpar_hcall9(H_ADD_LOGICAL_LAN_BUFFERS_QUEUE, + retbuf, unit_address, + queue_handle, buffersznum, + ioba12, ioba34, ioba56, + ioba78, ioba910, ioba1112); +} + +/* + * h_free_logical_lan_queue - Deregister subordinate receive queue + * @unit_address: Device unit address + * @queue_handle: Queue handle from h_register_logical_lan_queue() or + * h_register_logical_lan_with_handle() (queue 0) + * + * Deregisters and frees all structures associated with the subordinate queue. + * + * Return: + * H_SUCCESS - Queue freed successfully + * H_PARAMETER - Invalid parameters + * H_HARDWARE - Hardware error + * H_STATE - VIOA not in valid state + * H_BUSY / H_LONG_BUSY_* - Resource busy, retry + * H_FUNCTION - Firmware does not support this hcall + / +static inline long h_free_logical_lan_queue(unsigned long unit_address, + unsigned long queue_handle) +{ + return plpar_hcall_norets(H_FREE_LOGICAL_LAN_QUEUE, + unit_address, queue_handle); +} + +/* + * h_register_logical_lan_with_handle - Register primary queue and get handle + * @unit_address: Device unit address + * @buffer_list: DMA address of buffer list + * @rec_queue: Buffer descriptor of receive queue + * @filter_list: DMA address of filter list + * @mac_address: MAC address + * @queue_handle: Output parameter for queue handle (may be NULL) + * + * Registers the primary receive queue (queue 0) with the hypervisor and + * returns the queue handle. This is needed in multi-queue mode to use + * h_add_logical_lan_buffers_queue() for all queues including queue 0. + * + * Return: H_SUCCESS (0) on success, error code otherwise + / +static inline long +h_register_logical_lan_with_handle(unsigned long unit_address, + unsigned long buffer_list, + unsigned long rec_queue, + unsigned long filter_list, + unsigned long mac_address, + unsigned long queue_handle) +{ + unsigned long retbuf[PLPAR_HCALL_BUFSIZE]; + long rc; + + rc = plpar_hcall(H_REGISTER_LOGICAL_LAN, retbuf, + unit_address, buffer_list, rec_queue, + filter_list, mac_address); + + if (rc == H_SUCCESS && queue_handle) + queue_handle = retbuf[0]; + + return rc; +} + / FW allows us to send 6 descriptors but we only use one so mark * the other 5 as unused (0) / @@ -121,7 +267,10 @@ static inline long h_illan_attributes(unsigned long unit_address, #define IBMVETH_MAX_TX_BUF_SIZE (1024 64) #define IBMVETH_MAX_QUEUES 16U #define IBMVETH_DEFAULT_QUEUES 8U -#define IBMVETH_MAX_RX_PER_HCALL 8U +#define IBMVETH_MAX_RX_QUEUES 16U +#define IBMVETH_DEFAULT_RX_QUEUES 1U +#define IBMVETH_MAX_RX_REGULAR 8U +#define IBMVETH_MAX_RX_PER_HCALL 12U static int pool_size[] = { 512, 1024 * 2, 1024 * 16, 1024 * 32, 1024 * 64 }; static int pool_count[] = { 256, 512, 256, 256, 256 }; @@ -130,6 +279,43 @@ static int pool_active[] = { 1, 1, 0, 0, 1}; #define IBM_VETH_INVALID_MAP ((u16)0xffff) +/* + * Per-queue RX counters. No field has two concurrent writers: + * interrupts is written only from this queue's IRQ handler; polls, + * packets, bytes, large_packets and invalid_buffers only from its NAPI + * poll; replenish_* only under its replenish_lock; and no_buffer_drops + * and no_buffer_retired under that lock or from a teardown path already + * quiesced by napi_disable()/synchronize_irq(). Plain u64 is therefore + * sufficient and no atomic or u64_stats_sync is needed: the driver is + * PPC64-only, so 64-bit loads and stores do not tear. + / +struct ibmveth_rx_queue_stats { + u64 packets; + u64 bytes; + u64 interrupts; + u64 polls; + u64 large_packets; + u64 invalid_buffers; + / PHYP's per-page absolute drop count for the live page. / + u64 no_buffer_drops; + / Absolutes from pages this queue has already retired. / + u64 no_buffer_retired; + u64 replenish_task_cycles; + u64 replenish_no_mem; + u64 replenish_add_buff_failure; + u64 replenish_add_buff_success; +} ____cacheline_aligned_in_smp; + +/ Per-queue TX counters; serialized by the stack's per-queue TX lock. / +struct ibmveth_tx_queue_stats { + u64 packets; + u64 bytes; + u64 large_packets; + u64 dropped_packets; + u64 send_failures; + u64 checksum_offload; +} ____cacheline_aligned_in_smp; + struct ibmveth_buff_pool { u32 size; u32 index; @@ -143,6 +329,7 @@ struct ibmveth_buff_pool { struct sk_buff skbuff; int active; struct kobject kobj; + struct completion released; }; struct ibmveth_rx_q { @@ -152,23 +339,34 @@ struct ibmveth_rx_q { dma_addr_t queue_dma; u32 queue_len; struct ibmveth_rx_q_entry queue_addr; + spinlock_t replenish_lock; /* per-queue buffer replenish / }; struct ibmveth_adapter { struct vio_dev vdev; struct net_device netdev; - struct napi_struct napi; + struct napi_struct napi[IBMVETH_MAX_RX_QUEUES]; struct work_struct work; unsigned int mcastFilterSize; - void buffer_list_addr; + void buffer_list_addr[IBMVETH_MAX_RX_QUEUES]; void filter_list_addr; void tx_ltb_ptr[IBMVETH_MAX_QUEUES]; unsigned int tx_ltb_size; dma_addr_t tx_ltb_dma[IBMVETH_MAX_QUEUES]; - dma_addr_t buffer_list_dma; + dma_addr_t buffer_list_dma[IBMVETH_MAX_RX_QUEUES]; dma_addr_t filter_list_dma; - struct ibmveth_buff_pool rx_buff_pool[IBMVETH_NUM_BUFF_POOLS]; - struct ibmveth_rx_q rx_queue; + struct ibmveth_buff_pool + rx_buff_pool[IBMVETH_MAX_RX_QUEUES][IBMVETH_NUM_BUFF_POOLS]; + struct ibmveth_rx_q rx_queue[IBMVETH_MAX_RX_QUEUES]; + u64 queue_handle[IBMVETH_MAX_RX_QUEUES]; + unsigned int queue_irq[IBMVETH_MAX_RX_QUEUES]; + bool multi_queue; + unsigned int num_rx_queues; + bool mq_fallback; + / Lifetime: true after successful ndo_open until close clears it. / + bool opened; + / Lifetime: true while RX IRQ handlers / NAPI are installed. / + bool rx_irq_setup; int rx_csum; int large_send; bool is_active_trunk; @@ -177,17 +375,19 @@ struct ibmveth_adapter { u64 fw_ipv6_csum_support; u64 fw_ipv4_csum_support; u64 fw_large_send_support; - / adapter specific stats / - u64 replenish_task_cycles; - u64 replenish_no_mem; - u64 replenish_add_buff_failure; - u64 replenish_add_buff_success; - u64 rx_invalid_buffer; - u64 rx_no_buffer; + / + * Every other ethtool -S counter lives in rx_qstats/tx_qstats and is + * summed on read. tx_map_failed predates multi-queue, has never been + * updated by any code path, and is kept only so the key keeps + * reporting the zero userspace already sees. + / u64 tx_map_failed; - u64 tx_send_failed; - u64 tx_large_packets; - u64 rx_large_packets; + + struct ibmveth_rx_queue_stats rx_qstats; + struct ibmveth_tx_queue_stats tx_qstats; + + struct dentry debugfs_dir; + /* Ethtool settings */ u8 duplex; u32 speed; diff --git a/tools/perf/scripts/python/powerpc-hcalls.py b/tools/perf/scripts/python/powerpc-hcalls.py index fedce1b68cad1..afc354583b7b5 100644 --- a/tools/perf/scripts/python/powerpc-hcalls.py +++ b/tools/perf/scripts/python/powerpc-hcalls.py @@ -217,6 +217,10 @@ hcall_table = { # Key wrapping hcalls 1168: 'H_PKS_WRAP_OBJECT', 1172: 'H_PKS_UNWRAP_OBJECT', + # Logical LAN multi-queue RX (PAPR 11.20.00) + 1180: 'H_REG_LOGICAL_LAN_QUEUE', + 1184: 'H_ADD_LOGICAL_LAN_BUFFERS_QUEUE', + 1192: 'H_FREE_LOGICAL_LAN_QUEUE', # Platform-specific hcalls used by the Ultravisor 61184: 'H_SVM_PAGE_IN', 61188: 'H_SVM_PAGE_OUT',
3/2	2026/09/27 14:24	llm	patch-evaluator	0m Model: gemini-3.8-flash Args: null Results: null Tokens: input: 71932 output: 32 thoughts: 213 Thoughts: Analyzing a Patch: Potential Incompatibility Okay, so I'm looking at this patch and here's the breakdown. It's touching three areas. First, there's `arch/powerpc/include/asm/hvcall.h`, which suggests changes to hypercall definitions, specifically related to something called "logical LAN queues" – `H_REG_LOGICAL_LAN_QUEUE`, `H_ADD_LOGICAL_LAN_BUFFERS_QUEUE`, and `H_FREE_LOGICAL_LAN_QUEUE`. This is intriguing, likely involving a network interface on PowerPC. Then we have `drivers/net/ethernet/ibm/ibmveth.c` and `ibmveth.h`. This is the IBM Virtual Ethernet driver. It's adding multiqueue RX support, which is good to see; hypercall handling for that multiqueue setup, per-queue stats, and ethtool channel configuration. All seems standard for enhancing network performance. Finally, a `tools/perf/scripts/python/powerpc-hcalls.py` file, which probably updates a perf script to handle these new hypercalls. It's related to the hypercalls, so it makes sense. Now, wait a minute... let's see. This patch is targeted at the PowerPC architecture, right? Oh, but the build target is amd64! This seems like a problem. Am I building this for the right platform? The key question is: does `ibmveth` even compile or apply on amd64? I need to check the Kconfig files for `IBMVETH` and make sure it's conditionally included based on the architecture. It's highly unlikely that `ibmveth` is designed to run on amd64. This is a potential architectural incompatibility that I need to investigate further.
4/2	2026/09/27 14:24	tool	grepper	0m Args: { "Expression": "config IBMVETH", "PathPrefix": "drivers/net/ethernet/ibm" } Results: { "Output": "drivers/net/ethernet/ibm/Kconfig=18=if NET_VENDOR_IBM\ndrivers/net/ethernet/ibm/Kconfig-19-\ndrivers/net/ethernet/ibm/Kconfig:20:config IBMVETH\ndrivers/net/ethernet/ibm/Kconfig-21-\ttristate \"IBM LAN Virtual Ethernet support\"\n--\ndrivers/net/ethernet/ibm/Kconfig-29-\ndrivers/net/ethernet/ibm/Kconfig:30:config IBMVETH_KUNIT_TEST\ndrivers/net/ethernet/ibm/Kconfig-31-\tbool \"KUnit test for IBM LAN Virtual Ethernet support\" if !KUNIT_ALL_TESTS\n" }
5/2	2026/09/27 14:24	llm	patch-evaluator	0m Model: gemini-3.8-flash Args: null Results: null Tokens: input: 72745 output: 38 thoughts: 25 Thoughts: Okay, let's dive into this. I'm going to take a look at the `drivers/net/ethernet/ibm/Kconfig` file, specifically lines 18 through 40. I expect this section to contain configuration options for IBM Ethernet drivers, likely relating to hardware features, different card models, or maybe even architectural choices. Based on the file location, it's probably going to define how these drivers are incorporated into the kernel build process. I'll be looking for things like: * Menu structure: How are the options organized? Is it a single flat list, or nested? Are there dependencies? * Dependencies: What other kernel features or hardware settings are required for these Ethernet drivers to be built? Are there conditionals? * Default values: What default settings are being proposed? Is the driver enabled by default, or disabled? * Help text: This is important. What's the documentation like? Is it clear about which options to choose and why? I'm hoping for some insightful descriptions. * Triggers and constraints: Are there settings that automatically enable or disable other options? Any conflicting settings? * Target hardware: What specific IBM hardware models are covered? Are there any clear delineations for different adapters, or is it a general configuration? I'll be paying close attention to any conditional statements and any `bool`, `tristate`, or `int` variables, as these will define how the configuration choices are made. I'm expecting to get a pretty good handle on the IBM Ethernet driver configuration landscape. I'll keep the context in mind, and interpret each option in the most relevant way to maximize understanding. Now let's see what's there...
6/2	2026/09/27 14:24	tool	read-file	0m Args: { "File": "drivers/net/ethernet/ibm/Kconfig", "FirstLine": 18, "LineCount": 35 } Results: { "Contents": " 18:\tif NET_VENDOR_IBM\n 19:\t\n 20:\tconfig IBMVETH\n 21:\t\ttristate \"IBM LAN Virtual Ethernet support\"\n 22:\t\tdepends on PPC_PSERIES\n 23:\t\thelp\n 24:\t\t This driver supports virtual ethernet adapters on newer IBM iSeries\n 25:\t\t and pSeries systems.\n 26:\t\n 27:\t\t To compile this driver as a module, choose M here. The module will\n 28:\t\t be called ibmveth.\n 29:\t\n 30:\tconfig IBMVETH_KUNIT_TEST\n 31:\t\tbool \"KUnit test for IBM LAN Virtual Ethernet support\" if !KUNIT_ALL_TESTS\n 32:\t\tdepends on KUNIT\n 33:\t\tdepends on KUNIT=y \u0026\u0026 IBMVETH=y\n 34:\t\tdefault KUNIT_ALL_TESTS\n 35:\t\thelp\n 36:\t\t This builds unit tests for the IBM LAN Virtual Ethernet driver.\n 37:\t\n 38:\t\t For more information on KUnit and unit tests in general, please refer\n 39:\t\t to the KUnit documentation in Documentation/dev-tools/kunit/.\n 40:\t\n 41:\t\t If unsure, say N.\n 42:\t\n 43:\tsource \"drivers/net/ethernet/ibm/emac/Kconfig\"\n 44:\t\n 45:\tconfig IBMVNIC\n 46:\t\ttristate \"IBM Virtual NIC support\"\n 47:\t\tdepends on PPC_PSERIES\n 48:\t\thelp\n 49:\t\t This driver supports Virtual NIC adapters on IBM i and IBM System p\n 50:\t\t systems.\n 51:\t\n 52:\t\t To compile this driver as a module, choose M here. The module will\n" }
7/2	2026/09/27 14:24	llm	patch-evaluator	0m Model: gemini-3.8-flash Args: null Results: null Tokens: input: 73645 output: 20 thoughts: 62 Thoughts: Here's my summary, as though I'm thinking through this myself: Investigating the IBMVETH Dependency Chain Okay, so `config IBMVETH`... that's where I need to start. The Kconfig in the kernel has it depending on `PPC_PSERIES`. Right, that makes sense – it's an IBM virtual Ethernet driver, and that's likely a platform-specific feature. Now, I need to figure out what `PPC_PSERIES` is tied to, and how deep this dependency chain goes. Let me see... I expect it'll lead to some architecture level stuff. Time to jump into the `Kconfig` files. I'll either check the file directly at `arch/powerpc/platforms/pseries/Kconfig` to see what depends on it or, maybe grep through the kernel source for `config PPC_PSERIES`. That's usually faster. Let's dig in and figure out the full picture.
8/2	2026/09/27 14:24	tool	grepper	0m Args: { "Expression": "config PPC_PSERIES" } Results: { "Output": "arch/powerpc/platforms/pseries/Kconfig-1-# SPDX-License-Identifier: GPL-2.0\narch/powerpc/platforms/pseries/Kconfig:2:config PPC_PSERIES\narch/powerpc/platforms/pseries/Kconfig-3-\tdepends on PPC64 \u0026\u0026 PPC_BOOK3S\n--\narch/powerpc/platforms/pseries/Kconfig=92=config LPARCFG\n--\narch/powerpc/platforms/pseries/Kconfig-98-\narch/powerpc/platforms/pseries/Kconfig:99:config PPC_PSERIES_DEBUG\narch/powerpc/platforms/pseries/Kconfig-100-\tdepends on PPC_PSERIES \u0026\u0026 PPC_EARLY_DEBUG\n" }
9/2	2026/09/27 14:24	llm	patch-evaluator	0m Model: gemini-3.8-flash Args: null Results: null Tokens: input: 74131 output: 122 thoughts: 888 Thoughts: Analysis of Patch Applicability for Fuzzing Okay, let's break this down. First, I see that the patch touches several components: `ibmveth` (IBM Virtual Ethernet), `arch/powerpc/include/asm/hvcall.h`, and a user-space script in `tools/perf/scripts/`. Immediately, the dependency on `PPC_PSERIES` for `IBMVETH` is a red flag. `PPC_PSERIES` in turn depends on `PPC64` and `PPC_BOOK3S`. That means `IBMVETH` cannot possibly be enabled or built on `amd64`, our target architecture. Now, let's go back and carefully review the criteria for determining if something is "WorthFuzzing." I need to consider when to return `WorthFuzzing=false`. The relevant section says to do so if all modified code falls into certain categories. The key category here is: "Code paths exclusive to architectures other than the target architecture." Considering the elements, `arch/powerpc/include/asm/hvcall.h` is explicitly PowerPC-specific. `ibmveth.c` and `ibmveth.h` are also exclusively PowerPC due to the aforementioned dependency chain. And the Python script in `tools/` is user-space and doesn't impact the kernel image. Since all modified code is completely exclusive to PowerPC and user space (unreachable on amd64), it is structurally unreachable on the target architecture. It’s impossible to execute or even build these code paths on an amd64 system. To be absolutely sure, I checked the `drivers/net/ethernet/ibm/Kconfig` file. There's no chance of enabling `IBMVETH` for compilation on amd64 using something like `COMPILE_TEST`. Even if that were present, the use of `asm/hvcall.h` and the vio bus, which are PPC-specific, would prevent compilation. Even the `KUNIT` tests related to `IBMVETH` depend on `PPC_PSERIES`, so there's no way around it. Thus, `WorthFuzzing` should definitely be `false`. Let's summarize the `set-results`: - `WorthFuzzing`: false - `Reasoning`: Correct. It points out the PowerPC-exclusive dependencies and the target architecture. - `FocusSymbols`: None required. - `EnableConfigs`: None required. This all aligns with the instructions, especially the "reachability" gate. Everything matches perfectly. Time to call `set-results`.
10/2	2026/09/27 14:24	tool	set-results	0m Args: { "EnableConfigs": [], "FocusSymbols": [], "Reasoning": "The patch series modifies the IBM Virtual Ethernet driver (ibmveth), PowerPC hypercall definitions in arch/powerpc/include/asm/hvcall.h, and a perf Python script under tools/. The ibmveth driver depends on CONFIG_PPC_PSERIES (which requires PPC64), making all modified kernel code exclusive to the PowerPC architecture. As the target architecture is amd64, the changes are structurally unreachable.", "WorthFuzzing": false } Results: { "EnableConfigs": null, "FocusSymbols": [], "Reasoning": "The patch series modifies the IBM Virtual Ethernet driver (ibmveth), PowerPC hypercall definitions in arch/powerpc/include/asm/hvcall.h, and a perf Python script under tools/. The ibmveth driver depends on CONFIG_PPC_PSERIES (which requires PPC64), making all modified kernel code exclusive to the PowerPC architecture. As the target architecture is amd64, the changes are structurally unreachable.", "WorthFuzzing": false }
11/1	2026/09/27 14:24	action	If	0m Args: { "WorthFuzzing": false } Results: null

{
  "PatchDiff": "commit 845350ec79fd7e1e12ceb4f4a49bb29fa44d8e9a\nAuthor: syz-cluster \u003ctriage@syzkaller.com\u003e\nDate:   Sun Sep 27 14:24:02 2026 +0000\n\n    syz-cluster: applied patch under review\n\ndiff --git a/arch/powerpc/include/asm/hvcall.h b/arch/powerpc/include/asm/hvcall.h\nindex dff90a7d7f70b..cb0ea53491e6c 100644\n--- a/arch/powerpc/include/asm/hvcall.h\n+++ b/arch/powerpc/include/asm/hvcall.h\n@@ -362,7 +362,11 @@\n #define H_GUEST_DELETE\t\t0x488\n #define H_PKS_WRAP_OBJECT\t0x490\n #define H_PKS_UNWRAP_OBJECT\t0x494\n-#define MAX_HCALL_OPCODE\tH_PKS_UNWRAP_OBJECT\n+/* 0x498 reserved; 0x4A4 = H_FREE_LOGICAL_LAN_BUFFER_QUEUE (unused here) */\n+#define H_REG_LOGICAL_LAN_QUEUE 0x49C\n+#define H_ADD_LOGICAL_LAN_BUFFERS_QUEUE 0x4A0\n+#define H_FREE_LOGICAL_LAN_QUEUE 0x4A8\n+#define MAX_HCALL_OPCODE\tH_FREE_LOGICAL_LAN_QUEUE\n \n /* Scope args for H_SCM_UNBIND_ALL */\n #define H_UNBIND_SCOPE_ALL (0x1)\ndiff --git a/drivers/net/ethernet/ibm/ibmveth.c b/drivers/net/ethernet/ibm/ibmveth.c\nindex 73e051d26b9d8..da14c69152110 100644\n--- a/drivers/net/ethernet/ibm/ibmveth.c\n+++ b/drivers/net/ethernet/ibm/ibmveth.c\n@@ -21,6 +21,8 @@\n #include \u003clinux/skbuff.h\u003e\n #include \u003clinux/init.h\u003e\n #include \u003clinux/interrupt.h\u003e\n+#include \u003clinux/irq.h\u003e\n+#include \u003clinux/irqdomain.h\u003e\n #include \u003clinux/mm.h\u003e\n #include \u003clinux/pm.h\u003e\n #include \u003clinux/ethtool.h\u003e\n@@ -28,6 +30,8 @@\n #include \u003clinux/ip.h\u003e\n #include \u003clinux/ipv6.h\u003e\n #include \u003clinux/slab.h\u003e\n+#include \u003clinux/spinlock.h\u003e\n+#include \u003clinux/debugfs.h\u003e\n #include \u003casm/hvcall.h\u003e\n #include \u003clinux/atomic.h\u003e\n #include \u003casm/vio.h\u003e\n@@ -35,6 +39,7 @@\n #include \u003casm/firmware.h\u003e\n #include \u003cnet/tcp.h\u003e\n #include \u003cnet/ip6_checksum.h\u003e\n+#include \u003cnet/netdev_queues.h\u003e\n \n #include \"ibmveth.h\"\n \n@@ -72,748 +77,2940 @@ module_param(old_large_send, bool, 0444);\n MODULE_PARM_DESC(old_large_send,\n \t\"Use old large send method on firmware that supports the new method\");\n \n+/**\n+ * enum ibmveth_stat_src - where an ethtool -S counter is stored\n+ * @IBMVETH_STAT_ADAPTER: plain u64 in struct ibmveth_adapter\n+ * @IBMVETH_STAT_RX_QSUM: per-queue u64, summed over rx_qstats[]\n+ * @IBMVETH_STAT_TX_QSUM: per-queue u64, summed over tx_qstats[]\n+ * @IBMVETH_STAT_RX_NO_BUFFER: rx_qstats[] live-page absolute plus the\n+ *\tabsolutes carried over from pages the queue has already retired\n+ *\n+ * Counters live per-queue so multi-queue writers never share a field.\n+ * The adapter is only ever read from ethtool, so summing there is free.\n+ */\n+enum ibmveth_stat_src {\n+\tIBMVETH_STAT_ADAPTER,\n+\tIBMVETH_STAT_RX_QSUM,\n+\tIBMVETH_STAT_TX_QSUM,\n+\tIBMVETH_STAT_RX_NO_BUFFER,\n+};\n+\n struct ibmveth_stat {\n \tchar name[ETH_GSTRING_LEN];\n-\tint offset;\n+\tenum ibmveth_stat_src src;\n+\t/* Offset into the struct named by @src. */\n+\tsize_t off;\n };\n \n #define IBMVETH_STAT_OFF(stat) offsetof(struct ibmveth_adapter, stat)\n+#define IBMVETH_RXQ_OFF(stat) offsetof(struct ibmveth_rx_queue_stats, stat)\n+#define IBMVETH_TXQ_OFF(stat) offsetof(struct ibmveth_tx_queue_stats, stat)\n #define IBMVETH_GET_STAT(a, off) *((u64 *)(((unsigned long)(a)) + off))\n \n+#define IBMVETH_ADAPTER_STAT(key, field) \\\n+\t{ key, IBMVETH_STAT_ADAPTER, IBMVETH_STAT_OFF(field) }\n+#define IBMVETH_RXQ_STAT(key, field) \\\n+\t{ key, IBMVETH_STAT_RX_QSUM, IBMVETH_RXQ_OFF(field) }\n+#define IBMVETH_TXQ_STAT(key, field) \\\n+\t{ key, IBMVETH_STAT_TX_QSUM, IBMVETH_TXQ_OFF(field) }\n+\n+/*\n+ * Key names and their order are ABI. Do not reorder or rename; append\n+ * only, and only when the counter is worth a permanent interface.\n+ */\n static struct ibmveth_stat ibmveth_stats[] = {\n-\t{ \"replenish_task_cycles\", IBMVETH_STAT_OFF(replenish_task_cycles) },\n-\t{ \"replenish_no_mem\", IBMVETH_STAT_OFF(replenish_no_mem) },\n-\t{ \"replenish_add_buff_failure\",\n-\t\t\tIBMVETH_STAT_OFF(replenish_add_buff_failure) },\n-\t{ \"replenish_add_buff_success\",\n-\t\t\tIBMVETH_STAT_OFF(replenish_add_buff_success) },\n-\t{ \"rx_invalid_buffer\", IBMVETH_STAT_OFF(rx_invalid_buffer) },\n-\t{ \"rx_no_buffer\", IBMVETH_STAT_OFF(rx_no_buffer) },\n-\t{ \"tx_map_failed\", IBMVETH_STAT_OFF(tx_map_failed) },\n-\t{ \"tx_send_failed\", IBMVETH_STAT_OFF(tx_send_failed) },\n-\t{ \"fw_enabled_ipv4_csum\", IBMVETH_STAT_OFF(fw_ipv4_csum_support) },\n-\t{ \"fw_enabled_ipv6_csum\", IBMVETH_STAT_OFF(fw_ipv6_csum_support) },\n-\t{ \"tx_large_packets\", IBMVETH_STAT_OFF(tx_large_packets) },\n-\t{ \"rx_large_packets\", IBMVETH_STAT_OFF(rx_large_packets) },\n-\t{ \"fw_enabled_large_send\", IBMVETH_STAT_OFF(fw_large_send_support) }\n+\tIBMVETH_RXQ_STAT(\"replenish_task_cycles\", replenish_task_cycles),\n+\tIBMVETH_RXQ_STAT(\"replenish_no_mem\", replenish_no_mem),\n+\tIBMVETH_RXQ_STAT(\"replenish_add_buff_failure\",\n+\t\t\t replenish_add_buff_failure),\n+\tIBMVETH_RXQ_STAT(\"replenish_add_buff_success\",\n+\t\t\t replenish_add_buff_success),\n+\tIBMVETH_RXQ_STAT(\"rx_invalid_buffer\", invalid_buffers),\n+\t{ \"rx_no_buffer\", IBMVETH_STAT_RX_NO_BUFFER,\n+\t  IBMVETH_RXQ_OFF(no_buffer_drops) },\n+\tIBMVETH_ADAPTER_STAT(\"tx_map_failed\", tx_map_failed),\n+\tIBMVETH_TXQ_STAT(\"tx_send_failed\", send_failures),\n+\tIBMVETH_ADAPTER_STAT(\"fw_enabled_ipv4_csum\", fw_ipv4_csum_support),\n+\tIBMVETH_ADAPTER_STAT(\"fw_enabled_ipv6_csum\", fw_ipv6_csum_support),\n+\tIBMVETH_TXQ_STAT(\"tx_large_packets\", large_packets),\n+\tIBMVETH_RXQ_STAT(\"rx_large_packets\", large_packets),\n+\tIBMVETH_ADAPTER_STAT(\"fw_enabled_large_send\", fw_large_send_support),\n+};\n+\n+/**\n+ * struct ibmveth_qstat - a per-queue counter exposed through ethtool -S\n+ * @fmt: key name, taking the queue index as its only argument\n+ * @off: offset into the matching per-queue stats struct\n+ *\n+ * Driving the strings and the values from one table keeps the two in\n+ * step; get_sset_count() derives its length from ARRAY_SIZE() so the\n+ * three cannot drift apart.\n+ */\n+struct ibmveth_qstat {\n+\tconst char *fmt;\n+\tsize_t off;\n+};\n+\n+/*\n+ * Only counters with no home in the standard interfaces belong here.\n+ * packets, bytes and drops are reported through netdev_stat_ops.\n+ */\n+static const struct ibmveth_qstat ibmveth_rx_qstat_keys[] = {\n+\t{ \"rx%d_interrupts\", IBMVETH_RXQ_OFF(interrupts) },\n+\t{ \"rx%d_polls\", IBMVETH_RXQ_OFF(polls) },\n+\t{ \"rx%d_large_packets\", IBMVETH_RXQ_OFF(large_packets) },\n+\t{ \"rx%d_invalid_buffers\", IBMVETH_RXQ_OFF(invalid_buffers) },\n+\t{ \"rx%d_no_buffer_drops\", IBMVETH_RXQ_OFF(no_buffer_drops) },\n+};\n+\n+static const struct ibmveth_qstat ibmveth_tx_qstat_keys[] = {\n+\t{ \"tx%d_large_packets\", IBMVETH_TXQ_OFF(large_packets) },\n+\t{ \"tx%d_send_failures\", IBMVETH_TXQ_OFF(send_failures) },\n+\t{ \"tx%d_checksum_offload\", IBMVETH_TXQ_OFF(checksum_offload) },\n };\n \n+#define IBMVETH_NUM_RX_QSTATS ARRAY_SIZE(ibmveth_rx_qstat_keys)\n+#define IBMVETH_NUM_TX_QSTATS ARRAY_SIZE(ibmveth_tx_qstat_keys)\n+\n /* simple methods of getting data from the current rxq entry */\n-static inline u32 ibmveth_rxq_flags(struct ibmveth_adapter *adapter)\n+static u32 ibmveth_rxq_flags(struct ibmveth_adapter *adapter,\n+\t\t\t     int queue_index)\n {\n-\treturn be32_to_cpu(adapter-\u003erx_queue.queue_addr[adapter-\u003erx_queue.index].flags_off);\n+\tstruct ibmveth_rx_q *rxq = \u0026adapter-\u003erx_queue[queue_index];\n+\n+\treturn be32_to_cpu(rxq-\u003equeue_addr[rxq-\u003eindex].flags_off);\n }\n \n-static inline int ibmveth_rxq_toggle(struct ibmveth_adapter *adapter)\n+static int ibmveth_rxq_toggle(struct ibmveth_adapter *adapter,\n+\t\t\t      int queue_index)\n {\n-\treturn (ibmveth_rxq_flags(adapter) \u0026 IBMVETH_RXQ_TOGGLE) \u003e\u003e\n-\t\t\tIBMVETH_RXQ_TOGGLE_SHIFT;\n+\treturn (ibmveth_rxq_flags(adapter, queue_index) \u0026 IBMVETH_RXQ_TOGGLE) \u003e\u003e\n+\t\tIBMVETH_RXQ_TOGGLE_SHIFT;\n }\n \n-static inline int ibmveth_rxq_pending_buffer(struct ibmveth_adapter *adapter)\n+static int ibmveth_rxq_pending_buffer(struct ibmveth_adapter *adapter,\n+\t\t\t\t      int queue_index)\n {\n-\treturn ibmveth_rxq_toggle(adapter) == adapter-\u003erx_queue.toggle;\n+\treturn ibmveth_rxq_toggle(adapter, queue_index) ==\n+\t\tadapter-\u003erx_queue[queue_index].toggle;\n }\n \n-static inline int ibmveth_rxq_buffer_valid(struct ibmveth_adapter *adapter)\n+static int ibmveth_rxq_buffer_valid(struct ibmveth_adapter *adapter,\n+\t\t\t\t    int queue_index)\n {\n-\treturn ibmveth_rxq_flags(adapter) \u0026 IBMVETH_RXQ_VALID;\n+\treturn ibmveth_rxq_flags(adapter, queue_index) \u0026 IBMVETH_RXQ_VALID;\n }\n \n-static inline int ibmveth_rxq_frame_offset(struct ibmveth_adapter *adapter)\n+static int ibmveth_rxq_frame_offset(struct ibmveth_adapter *adapter,\n+\t\t\t\t    int queue_index)\n {\n-\treturn ibmveth_rxq_flags(adapter) \u0026 IBMVETH_RXQ_OFF_MASK;\n+\treturn ibmveth_rxq_flags(adapter, queue_index) \u0026 IBMVETH_RXQ_OFF_MASK;\n }\n \n-static inline int ibmveth_rxq_large_packet(struct ibmveth_adapter *adapter)\n+static int ibmveth_rxq_large_packet(struct ibmveth_adapter *adapter,\n+\t\t\t\t    int queue_index)\n {\n-\treturn ibmveth_rxq_flags(adapter) \u0026 IBMVETH_RXQ_LRG_PKT;\n+\treturn ibmveth_rxq_flags(adapter, queue_index) \u0026 IBMVETH_RXQ_LRG_PKT;\n }\n \n-static inline int ibmveth_rxq_frame_length(struct ibmveth_adapter *adapter)\n+static int ibmveth_rxq_frame_length(struct ibmveth_adapter *adapter,\n+\t\t\t\t    int queue_index)\n {\n-\treturn be32_to_cpu(adapter-\u003erx_queue.queue_addr[adapter-\u003erx_queue.index].length);\n+\tstruct ibmveth_rx_q *rxq = \u0026adapter-\u003erx_queue[queue_index];\n+\n+\treturn be32_to_cpu(rxq-\u003equeue_addr[rxq-\u003eindex].length);\n }\n \n-static inline int ibmveth_rxq_csum_good(struct ibmveth_adapter *adapter)\n+static int ibmveth_rxq_csum_good(struct ibmveth_adapter *adapter,\n+\t\t\t\t int queue_index)\n {\n-\treturn ibmveth_rxq_flags(adapter) \u0026 IBMVETH_RXQ_CSUM_GOOD;\n+\treturn ibmveth_rxq_flags(adapter, queue_index) \u0026 IBMVETH_RXQ_CSUM_GOOD;\n }\n \n-static unsigned int ibmveth_real_max_tx_queues(void)\n+/* Lockless IRQ/poll readers vs resize publishers. */\n+static unsigned int\n+ibmveth_get_num_rx_queues(const struct ibmveth_adapter *adapter)\n {\n-\tunsigned int n_cpu = num_online_cpus();\n-\n-\treturn min(n_cpu, IBMVETH_MAX_QUEUES);\n+\t/*\n+\t * Pairs with the release in ibmveth_publish_num_rx_queues(): a reader\n+\t * that sees the new count also sees the per-queue state behind it.\n+\t */\n+\treturn smp_load_acquire(\u0026adapter-\u003enum_rx_queues);\n }\n \n-/* setup the initial settings for a buffer pool */\n-static void ibmveth_init_buffer_pool(struct ibmveth_buff_pool *pool,\n-\t\t\t\t     u32 pool_index, u32 pool_size,\n-\t\t\t\t     u32 buff_size, u32 pool_active)\n+static void\n+ibmveth_publish_num_rx_queues(struct ibmveth_adapter *adapter,\n+\t\t\t      unsigned int num)\n {\n-\tpool-\u003esize = pool_size;\n-\tpool-\u003eindex = pool_index;\n-\tpool-\u003ebuff_size = buff_size;\n-\tpool-\u003ethreshold = pool_size * 7 / 8;\n-\tpool-\u003eactive = pool_active;\n+\t/*\n+\t * Pairs with the acquire in ibmveth_get_num_rx_queues(): per-queue\n+\t * state must be visible to a reader before it observes the new count.\n+\t */\n+\tsmp_store_release(\u0026adapter-\u003enum_rx_queues, num);\n }\n \n-/* allocate and setup an buffer pool - called during open */\n-static int ibmveth_alloc_buffer_pool(struct ibmveth_buff_pool *pool)\n+static unsigned int ibmveth_real_max_tx_queues(void)\n {\n-\tint i;\n+\tunsigned int n_cpu = num_online_cpus();\n \n-\tpool-\u003efree_map = kmalloc_array(pool-\u003esize, sizeof(u16), GFP_KERNEL);\n+\treturn min(n_cpu, IBMVETH_MAX_QUEUES);\n+}\n \n-\tif (!pool-\u003efree_map)\n-\t\treturn -1;\n+/**\n+ * ibmveth_alloc_filter_list - Allocate and map filter list\n+ * @adapter: ibmveth adapter structure\n+ *\n+ * Return: 0 on success, negative error code on failure\n+ */\n+static int\n+ibmveth_alloc_filter_list(struct ibmveth_adapter *adapter)\n+{\n+\tstruct device *dev = \u0026adapter-\u003evdev-\u003edev;\n+\tstruct net_device *netdev = adapter-\u003enetdev;\n \n-\tpool-\u003edma_addr = kzalloc_objs(dma_addr_t, pool-\u003esize);\n-\tif (!pool-\u003edma_addr) {\n-\t\tkfree(pool-\u003efree_map);\n-\t\tpool-\u003efree_map = NULL;\n-\t\treturn -1;\n+\tadapter-\u003efilter_list_addr = (void *)get_zeroed_page(GFP_KERNEL);\n+\tif (!adapter-\u003efilter_list_addr) {\n+\t\tnetdev_err(netdev, \"unable to allocate filter pages\\n\");\n+\t\treturn -ENOMEM;\n \t}\n \n-\tpool-\u003eskbuff = kcalloc(pool-\u003esize, sizeof(void *), GFP_KERNEL);\n-\n-\tif (!pool-\u003eskbuff) {\n-\t\tkfree(pool-\u003edma_addr);\n-\t\tpool-\u003edma_addr = NULL;\n-\n-\t\tkfree(pool-\u003efree_map);\n-\t\tpool-\u003efree_map = NULL;\n-\t\treturn -1;\n+\tadapter-\u003efilter_list_dma = dma_map_single(dev,\n+\t\t\t\t\t\t  adapter-\u003efilter_list_addr,\n+\t\t\t\t\t\t  4096, DMA_BIDIRECTIONAL);\n+\tif (dma_mapping_error(dev, adapter-\u003efilter_list_dma)) {\n+\t\tnetdev_err(netdev, \"unable to map filter list pages\\n\");\n+\t\tfree_page((unsigned long)adapter-\u003efilter_list_addr);\n+\t\tadapter-\u003efilter_list_addr = NULL;\n+\t\t/* Do not leave DMA_MAPPING_ERROR for free_filter_list(). */\n+\t\tadapter-\u003efilter_list_dma = 0;\n+\t\treturn -ENOMEM;\n \t}\n \n-\tfor (i = 0; i \u003c pool-\u003esize; ++i)\n-\t\tpool-\u003efree_map[i] = i;\n-\n-\tatomic_set(\u0026pool-\u003eavailable, 0);\n-\tpool-\u003eproducer_index = 0;\n-\tpool-\u003econsumer_index = 0;\n+\tnetdev_dbg(netdev, \"filter list @ 0x%p (DMA: 0x%llx)\\n\",\n+\t\t   adapter-\u003efilter_list_addr,\n+\t\t   (unsigned long long)adapter-\u003efilter_list_dma);\n \n \treturn 0;\n }\n \n-static inline void ibmveth_flush_buffer(void *addr, unsigned long length)\n+/**\n+ * ibmveth_free_filter_list - Free filter list resources\n+ * @adapter: ibmveth adapter structure\n+ */\n+static void\n+ibmveth_free_filter_list(struct ibmveth_adapter *adapter)\n {\n-\tunsigned long offset;\n+\tstruct device *dev = \u0026adapter-\u003evdev-\u003edev;\n \n-\tfor (offset = 0; offset \u003c length; offset += SMP_CACHE_BYTES)\n-\t\tasm(\"dcbf %0,%1,1\" :: \"b\" (addr), \"r\" (offset));\n+\t/* Unmap by CPU pointer: SPAPR can return DMA address 0. */\n+\tif (adapter-\u003efilter_list_addr) {\n+\t\tdma_unmap_single(dev, adapter-\u003efilter_list_dma, 4096,\n+\t\t\t\t DMA_BIDIRECTIONAL);\n+\t\tadapter-\u003efilter_list_dma = 0;\n+\t\tfree_page((unsigned long)adapter-\u003efilter_list_addr);\n+\t\tadapter-\u003efilter_list_addr = NULL;\n+\t}\n }\n \n-/* replenish the buffers for a pool.  note that we don't need to\n- * skb_reserve these since they are used for incoming...\n+/**\n+ * ibmveth_alloc_rx_qstats - Allocate per-queue RX statistics\n+ * @adapter: ibmveth adapter structure\n+ *\n+ * Return: 0 on success, -ENOMEM on failure\n  */\n-static void ibmveth_replenish_buffer_pool(struct ibmveth_adapter *adapter,\n-\t\t\t\t\t  struct ibmveth_buff_pool *pool)\n+static int ibmveth_alloc_rx_qstats(struct ibmveth_adapter *adapter)\n {\n-\tunion ibmveth_buf_desc descs[IBMVETH_MAX_RX_PER_HCALL] = {0};\n-\tu32 remaining = pool-\u003esize - atomic_read(\u0026pool-\u003eavailable);\n-\tu64 correlators[IBMVETH_MAX_RX_PER_HCALL] = {0};\n-\tunsigned long lpar_rc;\n-\tu32 buffers_added = 0;\n-\tu32 i, filled, batch;\n-\tstruct vio_dev *vdev;\n-\tdma_addr_t dma_addr;\n-\tstruct device *dev;\n-\tu32 index;\n+\tadapter-\u003erx_qstats = kcalloc(IBMVETH_MAX_RX_QUEUES,\n+\t\t\t\t     sizeof(*adapter-\u003erx_qstats),\n+\t\t\t\t     GFP_KERNEL);\n+\tif (!adapter-\u003erx_qstats)\n+\t\treturn -ENOMEM;\n \n-\tvdev = adapter-\u003evdev;\n-\tdev = \u0026vdev-\u003edev;\n+\treturn 0;\n+}\n \n-\tmb();\n+/**\n+ * ibmveth_free_rx_qstats - Free per-queue RX statistics\n+ * @adapter: ibmveth adapter structure\n+ */\n+static void ibmveth_free_rx_qstats(struct ibmveth_adapter *adapter)\n+{\n+\tkfree(adapter-\u003erx_qstats);\n+\tadapter-\u003erx_qstats = NULL;\n+}\n \n-\tbatch = adapter-\u003erx_buffers_per_hcall;\n+/**\n+ * ibmveth_alloc_tx_qstats - Allocate per-queue TX statistics\n+ * @adapter: ibmveth adapter structure\n+ *\n+ * Return: 0 on success, -ENOMEM on failure\n+ */\n+static int ibmveth_alloc_tx_qstats(struct ibmveth_adapter *adapter)\n+{\n+\tadapter-\u003etx_qstats = kcalloc(IBMVETH_MAX_QUEUES,\n+\t\t\t\t     sizeof(*adapter-\u003etx_qstats),\n+\t\t\t\t     GFP_KERNEL);\n+\tif (!adapter-\u003etx_qstats)\n+\t\treturn -ENOMEM;\n \n-\twhile (remaining \u003e 0) {\n-\t\tunsigned int free_index = pool-\u003econsumer_index;\n+\treturn 0;\n+}\n \n-\t\t/* Fill a batch of descriptors */\n-\t\tfor (filled = 0; filled \u003c min(remaining, batch); filled++) {\n-\t\t\tindex = pool-\u003efree_map[free_index];\n-\t\t\tif (WARN_ON(index == IBM_VETH_INVALID_MAP)) {\n-\t\t\t\tadapter-\u003ereplenish_add_buff_failure++;\n-\t\t\t\tnetdev_info(adapter-\u003enetdev,\n-\t\t\t\t\t    \"Invalid map index %u, reset\\n\",\n-\t\t\t\t\t    index);\n-\t\t\t\tschedule_work(\u0026adapter-\u003ework);\n-\t\t\t\tbreak;\n-\t\t\t}\n+/**\n+ * ibmveth_free_tx_qstats - Free per-queue TX statistics\n+ * @adapter: ibmveth adapter structure\n+ */\n+static void ibmveth_free_tx_qstats(struct ibmveth_adapter *adapter)\n+{\n+\tkfree(adapter-\u003etx_qstats);\n+\tadapter-\u003etx_qstats = NULL;\n+}\n \n-\t\t\tif (!pool-\u003eskbuff[index]) {\n-\t\t\t\tstruct sk_buff *skb = NULL;\n+/**\n+ * ibmveth_alloc_rx_queues - Allocate per-queue RX resources\n+ * @adapter: ibmveth adapter structure\n+ * @rxq_entries: Number of entries per RX queue\n+ *\n+ * Return: 0 on success, negative error code on failure\n+ */\n+static int\n+ibmveth_alloc_rx_queues(struct ibmveth_adapter *adapter, int rxq_entries)\n+{\n+\tstruct device *dev = \u0026adapter-\u003evdev-\u003edev;\n+\tstruct net_device *netdev = adapter-\u003enetdev;\n+\tint i;\n \n-\t\t\t\tskb = netdev_alloc_skb(adapter-\u003enetdev,\n-\t\t\t\t\t\t       pool-\u003ebuff_size);\n-\t\t\t\tif (!skb) {\n-\t\t\t\t\tadapter-\u003ereplenish_no_mem++;\n-\t\t\t\t\tadapter-\u003ereplenish_add_buff_failure++;\n-\t\t\t\t\tbreak;\n-\t\t\t\t}\n+\tfor (i = 0; i \u003c ibmveth_get_num_rx_queues(adapter); i++) {\n+\t\tadapter-\u003ebuffer_list_addr[i] =\n+\t\t\t(void *)get_zeroed_page(GFP_KERNEL);\n+\t\tif (!adapter-\u003ebuffer_list_addr[i]) {\n+\t\t\tnetdev_err(netdev,\n+\t\t\t\t   \"unable to allocate buffer list for queue %d\\n\",\n+\t\t\t\t   i);\n+\t\t\tgoto err_cleanup;\n+\t\t}\n \n-\t\t\t\tdma_addr = dma_map_single(dev, skb-\u003edata,\n-\t\t\t\t\t\t\t  pool-\u003ebuff_size,\n-\t\t\t\t\t\t\t  DMA_FROM_DEVICE);\n-\t\t\t\tif (dma_mapping_error(dev, dma_addr)) {\n-\t\t\t\t\tdev_kfree_skb_any(skb);\n-\t\t\t\t\tadapter-\u003ereplenish_add_buff_failure++;\n-\t\t\t\t\tbreak;\n-\t\t\t\t}\n+\t\tadapter-\u003erx_queue[i].queue_len =\n+\t\t\tsizeof(struct ibmveth_rx_q_entry) * rxq_entries;\n+\t\tadapter-\u003erx_queue[i].queue_addr =\n+\t\t\tdma_alloc_coherent(dev, adapter-\u003erx_queue[i].queue_len,\n+\t\t\t\t\t   \u0026adapter-\u003erx_queue[i].queue_dma,\n+\t\t\t\t\t   GFP_KERNEL);\n+\t\tif (!adapter-\u003erx_queue[i].queue_addr) {\n+\t\t\tnetdev_err(netdev,\n+\t\t\t\t   \"unable to allocate RX queue for queue %d\\n\",\n+\t\t\t\t   i);\n+\t\t\tfree_page((unsigned long)adapter-\u003ebuffer_list_addr[i]);\n+\t\t\tadapter-\u003ebuffer_list_addr[i] = NULL;\n+\t\t\tgoto err_cleanup;\n+\t\t}\n \n-\t\t\t\tpool-\u003edma_addr[index] = dma_addr;\n-\t\t\t\tpool-\u003eskbuff[index] = skb;\n-\t\t\t} else {\n-\t\t\t\t/* re-use case */\n-\t\t\t\tdma_addr = pool-\u003edma_addr[index];\n-\t\t\t}\n+\t\tadapter-\u003ebuffer_list_dma[i] =\n+\t\t\tdma_map_single(dev, adapter-\u003ebuffer_list_addr[i],\n+\t\t\t\t       4096, DMA_BIDIRECTIONAL);\n+\t\tif (dma_mapping_error(dev, adapter-\u003ebuffer_list_dma[i])) {\n+\t\t\tnetdev_err(netdev,\n+\t\t\t\t   \"unable to map buffer list for queue %d\\n\",\n+\t\t\t\t   i);\n+\t\t\tfree_page((unsigned long)adapter-\u003ebuffer_list_addr[i]);\n+\t\t\tadapter-\u003ebuffer_list_addr[i] = NULL;\n+\t\t\tadapter-\u003ebuffer_list_dma[i] = 0;\n+\t\t\tgoto err_cleanup;\n+\t\t}\n \n-\t\t\tif (rx_flush) {\n-\t\t\t\tunsigned int len;\n+\t\tadapter-\u003erx_queue[i].index = 0;\n+\t\tadapter-\u003erx_queue[i].num_slots = rxq_entries;\n+\t\tadapter-\u003erx_queue[i].toggle = 1;\n \n-\t\t\t\tlen = adapter-\u003enetdev-\u003emtu + IBMVETH_BUFF_OH;\n-\t\t\t\tlen = min(pool-\u003ebuff_size, len);\n-\t\t\t\tibmveth_flush_buffer(pool-\u003eskbuff[index]-\u003edata,\n-\t\t\t\t\t\t     len);\n-\t\t\t}\n+\t\tnetdev_dbg(netdev, \"queue %d: buffer_list @ 0x%p (DMA: 0x%llx), rx_queue @ 0x%p (DMA: 0x%llx), %llu entries\\n\",\n+\t\t\t   i, adapter-\u003ebuffer_list_addr[i],\n+\t\t\t   (unsigned long long)adapter-\u003ebuffer_list_dma[i],\n+\t\t\t   adapter-\u003erx_queue[i].queue_addr,\n+\t\t\t   (unsigned long long)adapter-\u003erx_queue[i].queue_dma,\n+\t\t\t   (unsigned long long)rxq_entries);\n+\t}\n \n-\t\t\tdescs[filled].fields.flags_len = IBMVETH_BUF_VALID |\n-\t\t\t\t\t\t\t  pool-\u003ebuff_size;\n-\t\t\tdescs[filled].fields.address = dma_addr;\n+\tnetdev_dbg(netdev, \"allocated %u RX queue(s) with %d entries each\\n\",\n+\t\t   ibmveth_get_num_rx_queues(adapter), rxq_entries);\n \n-\t\t\tcorrelators[filled] = ((u64)pool-\u003eindex \u003c\u003c 32) | index;\n-\t\t\t*(u64 *)pool-\u003eskbuff[index]-\u003edata = correlators[filled];\n+\treturn 0;\n \n-\t\t\tfree_index++;\n-\t\t\tif (free_index \u003e= pool-\u003esize)\n-\t\t\t\tfree_index = 0;\n+err_cleanup:\n+\t/*\n+\t * Free each leftover resource by pointer presence. A\n+\t * dma_mapping_error on queue i has already dropped\n+\t * buffer_list_addr[i] but can leave rx_queue[i].queue_addr\n+\t * live, so this index may be mixed. Do not collapse the\n+\t * buffer_list and queue_addr checks.\n+\t */\n+\tfor (; i \u003e= 0; i--) {\n+\t\tif (adapter-\u003ebuffer_list_addr[i]) {\n+\t\t\tdma_unmap_single(dev, adapter-\u003ebuffer_list_dma[i],\n+\t\t\t\t\t 4096, DMA_BIDIRECTIONAL);\n+\t\t\tadapter-\u003ebuffer_list_dma[i] = 0;\n \t\t}\n-\n-\t\tif (!filled)\n-\t\t\tbreak;\n-\n-\t\t/* single buffer case*/\n-\t\tif (filled == 1)\n-\t\t\tlpar_rc = h_add_logical_lan_buffer(vdev-\u003eunit_address,\n-\t\t\t\t\t\t\t   descs[0].desc);\n-\t\telse\n-\t\t\t/* Multi-buffer hcall */\n-\t\t\tlpar_rc = h_add_logical_lan_buffers(vdev-\u003eunit_address,\n-\t\t\t\t\t\t\t    descs[0].desc,\n-\t\t\t\t\t\t\t    descs[1].desc,\n-\t\t\t\t\t\t\t    descs[2].desc,\n-\t\t\t\t\t\t\t    descs[3].desc,\n-\t\t\t\t\t\t\t    descs[4].desc,\n-\t\t\t\t\t\t\t    descs[5].desc,\n-\t\t\t\t\t\t\t    descs[6].desc,\n-\t\t\t\t\t\t\t    descs[7].desc);\n-\t\tif (lpar_rc != H_SUCCESS) {\n-\t\t\tdev_warn_ratelimited(dev,\n-\t\t\t\t\t     \"RX h_add_logical_lan failed: filled=%u, rc=%lu, batch=%u\\n\",\n-\t\t\t\t\t     filled, lpar_rc, batch);\n-\t\t\tgoto hcall_failure;\n+\t\tif (adapter-\u003erx_queue[i].queue_addr) {\n+\t\t\tdma_free_coherent(dev, adapter-\u003erx_queue[i].queue_len,\n+\t\t\t\t\t  adapter-\u003erx_queue[i].queue_addr,\n+\t\t\t\t\t  adapter-\u003erx_queue[i].queue_dma);\n+\t\t\tadapter-\u003erx_queue[i].queue_addr = NULL;\n+\t\t}\n+\t\tif (adapter-\u003ebuffer_list_addr[i]) {\n+\t\t\tfree_page((unsigned long)adapter-\u003ebuffer_list_addr[i]);\n+\t\t\tadapter-\u003ebuffer_list_addr[i] = NULL;\n \t\t}\n+\t}\n \n-\t\t/* Only update pool state after hcall succeeds */\n-\t\tfor (i = 0; i \u003c filled; i++) {\n-\t\t\tfree_index = pool-\u003econsumer_index;\n-\t\t\tpool-\u003efree_map[free_index] = IBM_VETH_INVALID_MAP;\n+\treturn -ENOMEM;\n+}\n \n-\t\t\tpool-\u003econsumer_index++;\n-\t\t\tif (pool-\u003econsumer_index \u003e= pool-\u003esize)\n-\t\t\t\tpool-\u003econsumer_index = 0;\n-\t\t}\n+/**\n+ * ibmveth_cleanup_rx_resources - Free all RX queue resources\n+ * @adapter: ibmveth adapter structure\n+ */\n+static void\n+ibmveth_cleanup_rx_resources(struct ibmveth_adapter *adapter)\n+{\n+\tstruct device *dev = \u0026adapter-\u003evdev-\u003edev;\n+\tint i;\n \n-\t\tbuffers_added += filled;\n-\t\tadapter-\u003ereplenish_add_buff_success += filled;\n-\t\tremaining -= filled;\n+\tnetdev_dbg(adapter-\u003enetdev, \"cleaning up %u RX queue(s)\\n\",\n+\t\t   ibmveth_get_num_rx_queues(adapter));\n \n-\t\tmemset(\u0026descs, 0, sizeof(descs));\n-\t\tmemset(\u0026correlators, 0, sizeof(correlators));\n-\t\tcontinue;\n+\tfor (i = 0; i \u003c ibmveth_get_num_rx_queues(adapter); i++) {\n+\t\tif (adapter-\u003ebuffer_list_addr[i]) {\n+\t\t\tdma_unmap_single(dev, adapter-\u003ebuffer_list_dma[i],\n+\t\t\t\t\t 4096, DMA_BIDIRECTIONAL);\n+\t\t\tadapter-\u003ebuffer_list_dma[i] = 0;\n+\t\t}\n \n-hcall_failure:\n-\t\tfor (i = 0; i \u003c filled; i++) {\n-\t\t\tindex = correlators[i] \u0026 0xffffffffUL;\n-\t\t\tdma_addr =  pool-\u003edma_addr[index];\n+\t\tif (adapter-\u003erx_queue[i].queue_addr) {\n+\t\t\tdma_free_coherent(dev, adapter-\u003erx_queue[i].queue_len,\n+\t\t\t\t\t  adapter-\u003erx_queue[i].queue_addr,\n+\t\t\t\t\t  adapter-\u003erx_queue[i].queue_dma);\n+\t\t\tadapter-\u003erx_queue[i].queue_addr = NULL;\n+\t\t}\n \n-\t\t\tif (pool-\u003eskbuff[index]) {\n-\t\t\t\tif (dma_addr \u0026\u0026\n-\t\t\t\t    !dma_mapping_error(dev, dma_addr))\n-\t\t\t\t\tdma_unmap_single(dev, dma_addr,\n-\t\t\t\t\t\t\t pool-\u003ebuff_size,\n-\t\t\t\t\t\t\t DMA_FROM_DEVICE);\n+\t\tif (adapter-\u003ebuffer_list_addr[i]) {\n+\t\t\tfree_page((unsigned long)adapter-\u003ebuffer_list_addr[i]);\n+\t\t\tadapter-\u003ebuffer_list_addr[i] = NULL;\n+\t\t}\n+\t}\n+}\n \n-\t\t\t\tdev_kfree_skb_any(pool-\u003eskbuff[index]);\n-\t\t\t\tpool-\u003eskbuff[index] = NULL;\n-\t\t\t}\n+/**\n+ * ibmveth_toggle_irq - Common helper to enable/disable queue interrupts\n+ * @adapter: ibmveth adapter structure\n+ * @queue_index: Index of the queue (0 for primary, 1+ for subordinate)\n+ * @enable: true to enable, false to disable\n+ *\n+ * For queue 0 (primary), uses h_vio_signal() as it's registered via\n+ * h_register_logical_lan(). For subordinate queues (1+), uses H_VIOCTL\n+ * with H_ENABLE/DISABLE_VIO_INTERRUPT for per-queue interrupt control.\n+ *\n+ * Return: 0 on success, negative errno on failure (never raw H_*).\n+ */\n+static int\n+ibmveth_toggle_irq(struct ibmveth_adapter *adapter, int queue_index,\n+\t\t   bool enable)\n+{\n+\tunsigned long h_rc;\n+\tunsigned long irq = adapter-\u003equeue_irq[queue_index];\n+\tconst char *action = enable ? \"enable\" : \"disable\";\n+\n+\tif (queue_index == 0) {\n+\t\t/* Primary queue: use h_vio_signal() */\n+\t\th_rc = h_vio_signal(adapter-\u003evdev-\u003eunit_address,\n+\t\t\t\t    enable ? VIO_IRQ_ENABLE : VIO_IRQ_DISABLE);\n+\t} else {\n+\t\t/* Subordinate queues: use H_VIOCTL with hardware IRQ */\n+\t\tstruct irq_data *irq_data = irq_get_irq_data(irq);\n+\t\tirq_hw_number_t hwirq;\n+\t\tu64 vioctl_cmd = enable ? H_ENABLE_VIO_INTERRUPT :\n+\t\t\tH_DISABLE_VIO_INTERRUPT;\n+\n+\t\tif (!irq_data) {\n+\t\t\tnetdev_err(adapter-\u003enetdev,\n+\t\t\t\t   \"Failed to get IRQ data for queue %d (virq=%lu)\\n\",\n+\t\t\t\t   queue_index, irq);\n+\t\t\treturn -EINVAL;\n \t\t}\n-\t\tadapter-\u003ereplenish_add_buff_failure += filled;\n+\n+\t\thwirq = irqd_to_hwirq(irq_data);\n+\t\th_rc = plpar_hcall_norets(H_VIOCTL,\n+\t\t\t\t\t  adapter-\u003evdev-\u003eunit_address,\n+\t\t\t\t\t  vioctl_cmd,\n+\t\t\t\t\t  hwirq, 0, 0);\n \n \t\t/*\n-\t\t * If multi rx buffers hcall is no longer supported by FW\n-\t\t * e.g. in the case of Live Partition Migration\n+\t\t * H_PARAMETER is ambiguous (already in requested state vs bad\n+\t\t * args). Fold only on disable as an idempotent mask. On enable\n+\t\t * keep it an error so a stuck-masked queue stays visible to\n+\t\t * poll/resize recovery.\n \t\t */\n-\t\tif (batch \u003e 1 \u0026\u0026 lpar_rc == H_FUNCTION) {\n-\t\t\t/*\n-\t\t\t * Instead of retry submit single buffer individually\n-\t\t\t * here just set the max rx buffer per hcall to 1\n-\t\t\t * buffers will be respleshed next time\n-\t\t\t * when ibmveth_replenish_buffer_pool() is called again\n-\t\t\t * with single-buffer case\n-\t\t\t */\n-\t\t\tnetdev_info(adapter-\u003enetdev,\n-\t\t\t\t    \"RX Multi buffers not supported by FW, rc=%lu\\n\",\n-\t\t\t\t    lpar_rc);\n-\t\t\tadapter-\u003erx_buffers_per_hcall = 1;\n-\t\t\tnetdev_info(adapter-\u003enetdev,\n-\t\t\t\t    \"Next rx replesh will fall back to single-buffer hcall\\n\");\n+\t\tif (h_rc == H_PARAMETER \u0026\u0026 !enable) {\n+\t\t\tdev_warn_ratelimited(\u0026adapter-\u003enetdev-\u003edev,\n+\t\t\t\t\t     \"H_VIOCTL %s IRQ returned H_PARAMETER for queue %d (hwirq=%lu)\\n\",\n+\t\t\t\t\t     action, queue_index, hwirq);\n+\t\t\treturn 0;\n \t\t}\n-\t\tbreak;\n \t}\n \n-\tmb();\n-\tatomic_add(buffers_added, \u0026(pool-\u003eavailable));\n+\tif (h_rc) {\n+\t\tdev_err_ratelimited(\u0026adapter-\u003enetdev-\u003edev,\n+\t\t\t\t    \"Failed to %s IRQ for queue %d, rc=0x%lx\\n\",\n+\t\t\t\t    action, queue_index, h_rc);\n+\t\treturn -EIO;\n+\t}\n+\treturn 0;\n }\n \n-/*\n- * The final 8 bytes of the buffer list is a counter of frames dropped\n- * because there was not a buffer in the buffer list capable of holding\n- * the frame.\n+/**\n+ * ibmveth_disable_irq - Disable interrupt for a specific queue\n+ * @adapter: ibmveth adapter structure\n+ * @queue_index: Index of the queue (0 for primary, 1+ for subordinate)\n+ *\n+ * Return: 0 on success, negative errno on failure\n  */\n-static void ibmveth_update_rx_no_buffer(struct ibmveth_adapter *adapter)\n+static int\n+ibmveth_disable_irq(struct ibmveth_adapter *adapter, int queue_index)\n {\n-\t__be64 *p = adapter-\u003ebuffer_list_addr + 4096 - 8;\n-\n-\tadapter-\u003erx_no_buffer = be64_to_cpup(p);\n+\treturn ibmveth_toggle_irq(adapter, queue_index, false);\n }\n \n-/* replenish routine */\n-static void ibmveth_replenish_task(struct ibmveth_adapter *adapter)\n+/**\n+ * ibmveth_enable_irq - Enable interrupt for a specific queue\n+ * @adapter: ibmveth adapter structure\n+ * @queue_index: Index of the queue (0 for primary, 1+ for subordinate)\n+ *\n+ * Return: 0 on success, negative errno on failure\n+ */\n+static int\n+ibmveth_enable_irq(struct ibmveth_adapter *adapter, int queue_index)\n {\n-\tint i;\n-\n-\tadapter-\u003ereplenish_task_cycles++;\n+\treturn ibmveth_toggle_irq(adapter, queue_index, true);\n+}\n \n-\tfor (i = (IBMVETH_NUM_BUFF_POOLS - 1); i \u003e= 0; i--) {\n-\t\tstruct ibmveth_buff_pool *pool = \u0026adapter-\u003erx_buff_pool[i];\n+/**\n+ * ibmveth_dispose_subordinate_irq_mapping - Drop one subordinate virq mapping\n+ * @adapter: ibmveth adapter structure\n+ * @queue_idx: RX queue index (1..N)\n+ *\n+ * Subordinate queues get mappings from irq_create_mapping() during PHYP\n+ * registration. Queue 0 uses netdev-\u003eirq from device tree and is left alone.\n+ *\n+ * Bound against IBMVETH_MAX_RX_QUEUES, not num_rx_queues: a caller may\n+ * dispose a queue that is no longer in the published live set but still\n+ * owns a virq in queue_irq[]. Contrast with the bulk helper, which only\n+ * walks 1..num_rx_queues-1 (close / open-fail cleanup of the live set).\n+ *\n+ * Linux virq lifetime is owned by interrupt cleanup helpers. Call this only\n+ * after free_irq() when a handler was installed, or from registration failure\n+ * cleanup before request_irq().\n+ */\n+static void\n+ibmveth_dispose_subordinate_irq_mapping(struct ibmveth_adapter *adapter,\n+\t\t\t\t\tint queue_idx)\n+{\n+\tif (queue_idx \u003c= 0 || queue_idx \u003e= IBMVETH_MAX_RX_QUEUES)\n+\t\treturn;\n+\n+\tif (adapter-\u003equeue_irq[queue_idx]) {\n+\t\tirq_dispose_mapping(adapter-\u003equeue_irq[queue_idx]);\n+\t\tadapter-\u003equeue_irq[queue_idx] = 0;\n+\t}\n+}\n+\n+/**\n+ * ibmveth_dispose_subordinate_irq_mappings - Drop virq mappings for queues 1..N\n+ * @adapter: ibmveth adapter structure\n+ *\n+ * Bulk helper for close / open-fail cleanup of the published live set\n+ * (queues 1..num_rx_queues-1). Paths that need a retired or not-yet-published\n+ * queue must call ibmveth_dispose_subordinate_irq_mapping() directly.\n+ */\n+static void\n+ibmveth_dispose_subordinate_irq_mappings(struct ibmveth_adapter *adapter)\n+{\n+\tint i;\n+\n+\tfor (i = 1; i \u003c ibmveth_get_num_rx_queues(adapter); i++)\n+\t\tibmveth_dispose_subordinate_irq_mapping(adapter, i);\n+}\n+\n+/**\n+ * ibmveth_setup_rx_interrupts - Register IRQs and enable NAPI\n+ * @adapter: ibmveth adapter structure\n+ *\n+ * Registers interrupt handlers for all RX queues, enables NAPI, then\n+ * enables hypervisor interrupt delivery for multi-queue mode after\n+ * every queue has a Linux handler installed. The caller replenishes\n+ * every live queue before this helper so traffic during open is not\n+ * dropped (PHYP only interrupts after a successful enqueue, which\n+ * needs buffers). Single-queue open still leaves PHYP masked here;\n+ * restart_rx_queue then kicks NAPI.\n+ *\n+ * Return: 0 on success, negative error code on failure\n+ */\n+static int\n+ibmveth_setup_rx_interrupts(struct ibmveth_adapter *adapter)\n+{\n+\tstruct net_device *netdev = adapter-\u003enetdev;\n+\tint i, rc, num = ibmveth_get_num_rx_queues(adapter);\n+\n+\tfor (i = 0; i \u003c num; i++) {\n+\t\tif (!adapter-\u003equeue_irq[i]) {\n+\t\t\tnetdev_err(netdev, \"queue %d has invalid IRQ (0)\\n\", i);\n+\t\t\trc = -EINVAL;\n+\t\t\tgoto err_free_irqs;\n+\t\t}\n+\n+\t\trc = request_irq(adapter-\u003equeue_irq[i], ibmveth_interrupt,\n+\t\t\t\t 0, netdev-\u003ename, \u0026adapter-\u003enapi[i]);\n+\t\tif (rc) {\n+\t\t\tnetdev_err(netdev,\n+\t\t\t\t   \"request_irq() failed for irq 0x%x queue %d: %d\\n\",\n+\t\t\t\t   adapter-\u003equeue_irq[i], i, rc);\n+\t\t\tgoto err_free_irqs;\n+\t\t}\n+\t}\n+\n+\tfor (i = 0; i \u003c num; i++)\n+\t\tnapi_enable(\u0026adapter-\u003enapi[i]);\n+\n+\tif (adapter-\u003emulti_queue \u0026\u0026 num \u003e 1) {\n+\t\tfor (i = 0; i \u003c num; i++) {\n+\t\t\trc = ibmveth_enable_irq(adapter, i);\n+\t\t\tif (rc) {\n+\t\t\t\tnetdev_err(netdev,\n+\t\t\t\t\t   \"Failed to enable IRQ for queue %d, rc=%d\\n\",\n+\t\t\t\t\t   i, rc);\n+\t\t\t\tfor (; i \u003e= 0; i--) {\n+\t\t\t\t\tibmveth_disable_irq(adapter, i);\n+\t\t\t\t\tsynchronize_irq(adapter-\u003equeue_irq[i]);\n+\t\t\t\t}\n+\t\t\t\trc = -EIO;\n+\t\t\t\tgoto err_disable_napi;\n+\t\t\t}\n+\t\t}\n+\t}\n+\n+\t/* Set only on full success; fail paths leave this false so a later\n+\t * close() / cleanup is a no-op.\n+\t */\n+\tadapter-\u003erx_irq_setup = true;\n+\treturn 0;\n+\n+err_disable_napi:\n+\t/* STOP: remask after napi_disable; an in-flight poll can re-arm. */\n+\tfor (i = 0; i \u003c num; i++)\n+\t\tnapi_disable(\u0026adapter-\u003enapi[i]);\n+\tfor (i = 0; i \u003c num; i++) {\n+\t\tif (!adapter-\u003equeue_irq[i])\n+\t\t\tcontinue;\n+\t\tibmveth_disable_irq(adapter, i);\n+\t\tsynchronize_irq(adapter-\u003equeue_irq[i]);\n+\t}\n+\tfor (i = 0; i \u003c num; i++) {\n+\t\tif (adapter-\u003equeue_irq[i])\n+\t\t\tfree_irq(adapter-\u003equeue_irq[i], \u0026adapter-\u003enapi[i]);\n+\t}\n+\tgoto err_dispose_mappings;\n+\n+err_free_irqs:\n+\twhile (--i \u003e= 0)\n+\t\tfree_irq(adapter-\u003equeue_irq[i], \u0026adapter-\u003enapi[i]);\n+err_dispose_mappings:\n+\t/* Both setup failure paths own subordinate virq disposal. */\n+\tibmveth_dispose_subordinate_irq_mappings(adapter);\n+\treturn rc;\n+}\n+\n+/**\n+ * ibmveth_cleanup_rx_interrupts - Mask PHYP IRQs, stop NAPI, and free IRQs\n+ * @adapter: ibmveth adapter structure\n+ *\n+ * Mask and synchronize each queue IRQ before napi_disable() so the handler\n+ * cannot miss a PHYP mask while NAPI is already dead. Remask after\n+ * napi_disable() in case an in-flight poll re-armed PHYP while we waited.\n+ * free_irq() runs only after that. Safe for close and for open failure after\n+ * setup_rx_interrupts() already unmasked PHYP. No-op if setup never\n+ * succeeded (avoids double napi_disable / free_irq after a failed close+open\n+ * while IFF_UP remains set).\n+ */\n+static void\n+ibmveth_cleanup_rx_interrupts(struct ibmveth_adapter *adapter)\n+{\n+\tint i;\n+\n+\tif (!adapter-\u003erx_irq_setup)\n+\t\treturn;\n+\n+\tfor (i = 0; i \u003c ibmveth_get_num_rx_queues(adapter); i++) {\n+\t\tif (!adapter-\u003equeue_irq[i])\n+\t\t\tcontinue;\n+\t\tibmveth_disable_irq(adapter, i);\n+\t\tsynchronize_irq(adapter-\u003equeue_irq[i]);\n+\t}\n+\n+\tfor (i = 0; i \u003c ibmveth_get_num_rx_queues(adapter); i++)\n+\t\tnapi_disable(\u0026adapter-\u003enapi[i]);\n+\n+\tfor (i = 0; i \u003c ibmveth_get_num_rx_queues(adapter); i++) {\n+\t\tif (!adapter-\u003equeue_irq[i])\n+\t\t\tcontinue;\n+\t\tibmveth_disable_irq(adapter, i);\n+\t\tsynchronize_irq(adapter-\u003equeue_irq[i]);\n+\t}\n+\n+\tfor (i = 0; i \u003c ibmveth_get_num_rx_queues(adapter); i++) {\n+\t\tif (adapter-\u003equeue_irq[i])\n+\t\t\tfree_irq(adapter-\u003equeue_irq[i], \u0026adapter-\u003enapi[i]);\n+\t}\n+\n+\tibmveth_dispose_subordinate_irq_mappings(adapter);\n+\n+\t/* Queue 0 uses netdev-\u003eirq; leave queue_irq[0] for next open. */\n+\tadapter-\u003erx_irq_setup = false;\n+}\n+\n+/**\n+ * ibmveth_setup_single_rx_interrupt - Setup interrupt for a single RX queue\n+ * @adapter: ibmveth adapter structure\n+ * @queue_idx: Queue index to setup\n+ *\n+ * Registers the IRQ handler for one queue. Used during incremental\n+ * scale-up when adding new RX queues. The caller publishes the queue,\n+ * replenishes buffers, enables NAPI, then unmasks PHYP delivery.\n+ *\n+ * Return: 0 on success, negative error code on failure\n+ */\n+static int\n+ibmveth_setup_single_rx_interrupt(struct ibmveth_adapter *adapter,\n+\t\t\t\t  int queue_idx)\n+{\n+\tstruct net_device *netdev = adapter-\u003enetdev;\n+\tint rc;\n+\n+\trc = request_irq(adapter-\u003equeue_irq[queue_idx], ibmveth_interrupt,\n+\t\t\t 0, netdev-\u003ename, \u0026adapter-\u003enapi[queue_idx]);\n+\tif (rc) {\n+\t\tnetdev_err(netdev, \"request_irq() failed for queue %d: %d\\n\",\n+\t\t\t   queue_idx, rc);\n+\t\treturn rc;\n+\t}\n+\n+\tnetdev_dbg(netdev, \"Setup IRQ %d for queue %d\\n\",\n+\t\t   adapter-\u003equeue_irq[queue_idx], queue_idx);\n+\treturn 0;\n+}\n+\n+/**\n+ * ibmveth_cleanup_single_rx_interrupt - Cleanup interrupt for a single RX queue\n+ * @adapter: ibmveth adapter structure\n+ * @queue_idx: Queue index to cleanup\n+ *\n+ * Frees the IRQ handler for one queue and releases the subordinate virq\n+ * mapping. Used during incremental scale-down.\n+ */\n+static void\n+ibmveth_cleanup_single_rx_interrupt(struct ibmveth_adapter *adapter,\n+\t\t\t\t    int queue_idx)\n+{\n+\tif (adapter-\u003equeue_irq[queue_idx]) {\n+\t\tfree_irq(adapter-\u003equeue_irq[queue_idx],\n+\t\t\t \u0026adapter-\u003enapi[queue_idx]);\n+\t\tibmveth_dispose_subordinate_irq_mapping(adapter, queue_idx);\n+\t\tnetdev_dbg(adapter-\u003enetdev,\n+\t\t\t   \"Freed IRQ for queue %d\\n\", queue_idx);\n+\t}\n+}\n+\n+/**\n+ * ibmveth_schedule_rx_queue - Mask PHYP IRQ and schedule NAPI for one RX queue\n+ * @adapter: ibmveth adapter structure\n+ * @qindex: RX queue index\n+ *\n+ * Shared by the IRQ handler and process-context kick sites (open, resume,\n+ * pool sysfs, poll_controller).\n+ *\n+ * Return: true if napi_schedule_prep() succeeded and NAPI was scheduled.\n+ * Mask is attempted in that case; a failed disable_irq() is logged by the\n+ * helper and does not change the return (queue may still be unmasked).\n+ * false if the index is negative (WARN_ON), past the live count (quiet:\n+ * a shrink can race netpoll), or prep failed (including NAPI already\n+ * scheduled).\n+ */\n+static bool ibmveth_schedule_rx_queue(struct ibmveth_adapter *adapter,\n+\t\t\t\t      int qindex)\n+{\n+\tstruct napi_struct *napi = \u0026adapter-\u003enapi[qindex];\n+\n+\tif (WARN_ON(qindex \u003c 0))\n+\t\treturn false;\n+\n+\t/*\n+\t * A live shrink can publish a lower count while netpoll walks a\n+\t * snapshot of the old one, so an index past the end is expected\n+\t * here and must not splat. ibmveth_replenish_task() skips the\n+\t * same way; callers already treat false as \"queue is gone\".\n+\t */\n+\tif (qindex \u003e= ibmveth_get_num_rx_queues(adapter))\n+\t\treturn false;\n+\n+\t/*\n+\t * Only mask PHYP when NAPI will run. Masking on prep failure can\n+\t * race a completing poll that already re-enabled the queue, leaving\n+\t * NAPI idle with the IRQ masked (TX works, RX stalls) until reload.\n+\t * Storm prevention on teardown remains in cleanup/disable paths.\n+\t */\n+\tif (napi_schedule_prep(napi)) {\n+\t\t/* Failure is already logged with the hcall rc by the helper. */\n+\t\tibmveth_disable_irq(adapter, qindex);\n+\t\t__napi_schedule(napi);\n+\t\treturn true;\n+\t}\n+\treturn false;\n+}\n+\n+/* setup the initial settings for a buffer pool */\n+static void ibmveth_init_buffer_pool(struct ibmveth_buff_pool *pool,\n+\t\t\t\t     u32 pool_index, u32 pool_size,\n+\t\t\t\t     u32 buff_size, u32 pool_active)\n+{\n+\tpool-\u003esize = pool_size;\n+\tpool-\u003eindex = pool_index;\n+\tpool-\u003ebuff_size = buff_size;\n+\tpool-\u003ethreshold = pool_size * 7 / 8;\n+\tpool-\u003eactive = pool_active;\n+}\n+\n+/* allocate and setup an buffer pool - called during open */\n+static int ibmveth_alloc_buffer_pool(struct ibmveth_buff_pool *pool)\n+{\n+\tint i;\n+\n+\tpool-\u003efree_map = kmalloc_array(pool-\u003esize, sizeof(u16), GFP_KERNEL);\n+\n+\tif (!pool-\u003efree_map)\n+\t\treturn -1;\n+\n+\tpool-\u003edma_addr = kzalloc_objs(dma_addr_t, pool-\u003esize);\n+\tif (!pool-\u003edma_addr) {\n+\t\tkfree(pool-\u003efree_map);\n+\t\tpool-\u003efree_map = NULL;\n+\t\treturn -1;\n+\t}\n+\n+\tpool-\u003eskbuff = kcalloc(pool-\u003esize, sizeof(void *), GFP_KERNEL);\n+\n+\tif (!pool-\u003eskbuff) {\n+\t\tkfree(pool-\u003edma_addr);\n+\t\tpool-\u003edma_addr = NULL;\n+\n+\t\tkfree(pool-\u003efree_map);\n+\t\tpool-\u003efree_map = NULL;\n+\t\treturn -1;\n+\t}\n+\n+\tfor (i = 0; i \u003c pool-\u003esize; ++i)\n+\t\tpool-\u003efree_map[i] = i;\n+\n+\tatomic_set(\u0026pool-\u003eavailable, 0);\n+\tpool-\u003eproducer_index = 0;\n+\tpool-\u003econsumer_index = 0;\n+\n+\treturn 0;\n+}\n+\n+static inline void ibmveth_flush_buffer(void *addr, unsigned long length)\n+{\n+\tunsigned long offset;\n+\n+\tfor (offset = 0; offset \u003c length; offset += SMP_CACHE_BYTES)\n+\t\tasm(\"dcbf %0,%1,1\" :: \"b\" (addr), \"r\" (offset));\n+}\n+\n+/**\n+ * ibmveth_add_logical_lan_buffers - Add receive buffers to hypervisor\n+ * @adapter: ibmveth adapter structure\n+ * @descs: array of buffer descriptors to add\n+ * @filled: number of valid descriptors in the array\n+ * @buff_size: size of each buffer (multi-queue mode only)\n+ * @queue_index: RX queue index\n+ *\n+ * Return: hypervisor return code\n+ */\n+static long ibmveth_add_logical_lan_buffers(struct ibmveth_adapter *adapter,\n+\t\t\t\t\t    union ibmveth_buf_desc *descs,\n+\t\t\t\t\t    int filled,\n+\t\t\t\t\t    unsigned long buff_size,\n+\t\t\t\t\t    int queue_index)\n+{\n+\tstruct vio_dev *vdev = adapter-\u003evdev;\n+\tunsigned long rc;\n+\n+\t/*\n+\t * The MQ hcall takes six ioba words (12 packed addresses). The\n+\t * legacy hcall takes eight descriptors. The argument lists below\n+\t * are written out by hand; keep the defines matched to those lists.\n+\t */\n+\tstatic_assert(IBMVETH_MAX_RX_PER_HCALL == 12);\n+\tstatic_assert(IBMVETH_MAX_RX_REGULAR == 8);\n+\n+\tif (adapter-\u003emulti_queue) {\n+\t\tunsigned long buffersznum = (buff_size \u003c\u003c 32) | filled;\n+\t\tunsigned long ioba[IBMVETH_MAX_RX_PER_HCALL / 2] = {0};\n+\t\tunsigned long handle = adapter-\u003equeue_handle[queue_index];\n+\t\tint i;\n+\n+\t\t/* Pack descriptor addresses into ioba pairs.\n+\t\t * Each ioba holds two 32-bit addresses packed into 64 bits:\n+\t\t * - Even descriptors (0,2,4...) go in high 32 bits\n+\t\t * - Odd descriptors (1,3,5...) go in low 32 bits\n+\t\t */\n+\t\tfor (i = 0; i \u003c filled \u0026\u0026 i \u003c IBMVETH_MAX_RX_PER_HCALL; i++) {\n+\t\t\tint pair_idx = i / 2;\n+\t\t\tint is_high = (i % 2 == 0);\n+\n+\t\t\tif (is_high)\n+\t\t\t\tioba[pair_idx] = (unsigned long)\n+\t\t\t\t\tdescs[i].fields.address \u003c\u003c 32;\n+\t\t\telse\n+\t\t\t\tioba[pair_idx] |= descs[i].fields.address;\n+\t\t}\n+\n+\t\trc = h_add_logical_lan_buffers_queue(vdev-\u003eunit_address,\n+\t\t\t\t\t\t     handle,\n+\t\t\t\t\t\t     buffersznum,\n+\t\t\t\t\t\t     ioba[0], ioba[1], ioba[2],\n+\t\t\t\t\t\t     ioba[3], ioba[4], ioba[5]);\n+\t} else if (filled == 1) {\n+\t\trc = h_add_logical_lan_buffer(vdev-\u003eunit_address,\n+\t\t\t\t\t      descs[0].desc);\n+\t} else {\n+\t\t/* Legacy 8-desc hcall; probe/mq_fallback keep batch \u003c=\n+\t\t * IBMVETH_MAX_RX_REGULAR.\n+\t\t */\n+\t\trc = h_add_logical_lan_buffers(vdev-\u003eunit_address,\n+\t\t\t\t\t       descs[0].desc, descs[1].desc,\n+\t\t\t\t\t       descs[2].desc, descs[3].desc,\n+\t\t\t\t\t       descs[4].desc, descs[5].desc,\n+\t\t\t\t\t       descs[6].desc, descs[7].desc);\n+\t}\n+\n+\treturn rc;\n+}\n+\n+/* Outcomes for ibmveth_replenish_buffer_pool(); logged after unlock. */\n+enum {\n+\tIBMVETH_REPLENISH_OK = 0,\n+\tIBMVETH_REPLENISH_RESET_MAP,\n+\tIBMVETH_REPLENISH_RESET_MQ,\n+\tIBMVETH_REPLENISH_HCALL_FAIL,\n+\tIBMVETH_REPLENISH_BATCH_FALLBACK,\n+};\n+\n+struct ibmveth_replenish_fail {\n+\tunsigned long lpar_rc;\n+\tu32 filled;\n+\tu32 batch;\n+};\n+\n+/* Replenish the buffers for a pool.\n+ * Caller must hold the per-queue replenish_lock. Do not printk here:\n+ * netconsole on the same device can re-enter replenish_task.\n+ */\n+static int ibmveth_replenish_buffer_pool(struct ibmveth_adapter *adapter,\n+\t\t\t\t\t struct ibmveth_buff_pool *pool,\n+\t\t\t\t\t int queue_index,\n+\t\t\t\t\t struct ibmveth_replenish_fail *fail)\n+{\n+\tstruct ibmveth_rx_queue_stats *qstats =\n+\t\t\u0026adapter-\u003erx_qstats[queue_index];\n+\tunion ibmveth_buf_desc descs[IBMVETH_MAX_RX_PER_HCALL] = {0};\n+\tu32 remaining = pool-\u003esize - atomic_read(\u0026pool-\u003eavailable);\n+\tu64 correlators[IBMVETH_MAX_RX_PER_HCALL] = {0};\n+\tunsigned long lpar_rc;\n+\tu32 buffers_added = 0;\n+\tu32 i, filled, batch;\n+\tstruct vio_dev *vdev;\n+\tdma_addr_t dma_addr;\n+\tstruct device *dev;\n+\tu32 index;\n+\tint outcome = IBMVETH_REPLENISH_OK;\n+\n+\tvdev = adapter-\u003evdev;\n+\tdev = \u0026vdev-\u003edev;\n+\n+\tmb();\n+\n+\tbatch = adapter-\u003erx_buffers_per_hcall;\n+\n+\twhile (remaining \u003e 0) {\n+\t\tunsigned int free_index = pool-\u003econsumer_index;\n+\n+\t\t/* Fill a batch of descriptors */\n+\t\tfor (filled = 0; filled \u003c min(remaining, batch); filled++) {\n+\t\t\tindex = pool-\u003efree_map[free_index];\n+\t\t\tif (index == IBM_VETH_INVALID_MAP) {\n+\t\t\t\tqstats-\u003ereplenish_add_buff_failure++;\n+\t\t\t\toutcome = IBMVETH_REPLENISH_RESET_MAP;\n+\t\t\t\tbreak;\n+\t\t\t}\n+\n+\t\t\tif (!pool-\u003eskbuff[index]) {\n+\t\t\t\tstruct sk_buff *skb = NULL;\n+\n+\t\t\t\tskb = netdev_alloc_skb(adapter-\u003enetdev,\n+\t\t\t\t\t\t       pool-\u003ebuff_size);\n+\t\t\t\tif (!skb) {\n+\t\t\t\t\tqstats-\u003ereplenish_no_mem++;\n+\t\t\t\t\tqstats-\u003ereplenish_add_buff_failure++;\n+\t\t\t\t\tbreak;\n+\t\t\t\t}\n+\n+\t\t\t\t/* NO_WARN: hold replenish_lock; iommu\n+\t\t\t\t * printk can re-enter via netconsole.\n+\t\t\t\t */\n+\t\t\t\tdma_addr =\n+\t\t\t\t\tdma_map_single_attrs(dev,\n+\t\t\t\t\t\t\t     skb-\u003edata,\n+\t\t\t\t\t\t\t     pool-\u003ebuff_size,\n+\t\t\t\t\t\t\t     DMA_FROM_DEVICE,\n+\t\t\t\t\t\t\t     DMA_ATTR_NO_WARN);\n+\t\t\t\tif (dma_mapping_error(dev, dma_addr)) {\n+\t\t\t\t\tdev_kfree_skb_any(skb);\n+\t\t\t\t\tqstats-\u003ereplenish_add_buff_failure++;\n+\t\t\t\t\tbreak;\n+\t\t\t\t}\n+\n+\t\t\t\tpool-\u003edma_addr[index] = dma_addr;\n+\t\t\t\tpool-\u003eskbuff[index] = skb;\n+\t\t\t} else {\n+\t\t\t\t/* re-use case */\n+\t\t\t\tdma_addr = pool-\u003edma_addr[index];\n+\t\t\t}\n+\n+\t\t\tif (rx_flush) {\n+\t\t\t\tunsigned int len;\n+\n+\t\t\t\tlen = adapter-\u003enetdev-\u003emtu + IBMVETH_BUFF_OH;\n+\t\t\t\tlen = min(pool-\u003ebuff_size, len);\n+\t\t\t\tibmveth_flush_buffer(pool-\u003eskbuff[index]-\u003edata,\n+\t\t\t\t\t\t     len);\n+\t\t\t}\n+\n+\t\t\tdescs[filled].fields.flags_len = IBMVETH_BUF_VALID |\n+\t\t\t\t\t\t\t  pool-\u003ebuff_size;\n+\t\t\tdescs[filled].fields.address = dma_addr;\n+\n+\t\t\tcorrelators[filled] = ((u64)pool-\u003eindex \u003c\u003c 32) | index;\n+\t\t\t*(u64 *)pool-\u003eskbuff[index]-\u003edata = correlators[filled];\n+\n+\t\t\tfree_index++;\n+\t\t\tif (free_index \u003e= pool-\u003esize)\n+\t\t\t\tfree_index = 0;\n+\t\t}\n+\n+\t\tif (outcome != IBMVETH_REPLENISH_OK)\n+\t\t\tbreak;\n+\n+\t\tif (!filled)\n+\t\t\tbreak;\n+\n+\t\tlpar_rc = ibmveth_add_logical_lan_buffers(adapter, descs,\n+\t\t\t\t\t\t\t  filled,\n+\t\t\t\t\t\t\t  pool-\u003ebuff_size,\n+\t\t\t\t\t\t\t  queue_index);\n+\n+\t\tif (lpar_rc != H_SUCCESS) {\n+\t\t\tfail-\u003elpar_rc = lpar_rc;\n+\t\t\tfail-\u003efilled = filled;\n+\t\t\tfail-\u003ebatch = batch;\n+\t\t\tgoto hcall_failure;\n+\t\t}\n+\n+\t\t/* Only update pool state after hcall succeeds */\n+\t\tfor (i = 0; i \u003c filled; i++) {\n+\t\t\tfree_index = pool-\u003econsumer_index;\n+\t\t\tpool-\u003efree_map[free_index] = IBM_VETH_INVALID_MAP;\n+\n+\t\t\tpool-\u003econsumer_index++;\n+\t\t\tif (pool-\u003econsumer_index \u003e= pool-\u003esize)\n+\t\t\t\tpool-\u003econsumer_index = 0;\n+\t\t}\n+\n+\t\tbuffers_added += filled;\n+\t\tqstats-\u003ereplenish_add_buff_success += filled;\n+\t\tremaining -= filled;\n+\n+\t\tmemset(\u0026descs, 0, sizeof(descs));\n+\t\tmemset(\u0026correlators, 0, sizeof(correlators));\n+\t\tcontinue;\n+\n+hcall_failure:\n+\t\tfor (i = 0; i \u003c filled; i++) {\n+\t\t\tindex = correlators[i] \u0026 0xffffffffUL;\n+\t\t\tdma_addr =  pool-\u003edma_addr[index];\n+\n+\t\t\tif (pool-\u003eskbuff[index]) {\n+\t\t\t\tif (dma_addr \u0026\u0026\n+\t\t\t\t    !dma_mapping_error(dev, dma_addr))\n+\t\t\t\t\tdma_unmap_single(dev, dma_addr,\n+\t\t\t\t\t\t\t pool-\u003ebuff_size,\n+\t\t\t\t\t\t\t DMA_FROM_DEVICE);\n+\n+\t\t\t\tdev_kfree_skb_any(pool-\u003eskbuff[index]);\n+\t\t\t\tpool-\u003eskbuff[index] = NULL;\n+\t\t\t}\n+\t\t}\n+\t\tqstats-\u003ereplenish_add_buff_failure += filled;\n+\n+\t\tif (lpar_rc == H_FUNCTION) {\n+\t\t\tif (adapter-\u003emulti_queue) {\n+\t\t\t\t/*\n+\t\t\t\t * LPM / firmware may drop MQ buffer hcalls.\n+\t\t\t\t * Schedule reset so we do not sit forever in\n+\t\t\t\t * no-buffer with the link still up.\n+\t\t\t\t */\n+\t\t\t\toutcome = IBMVETH_REPLENISH_RESET_MQ;\n+\t\t\t} else if (batch \u003e 1) {\n+\t\t\t\t/*\n+\t\t\t\t * Live Partition Migration may drop multi-\n+\t\t\t\t * buffer support. Fall back to single-buffer\n+\t\t\t\t * on the next replenish; do not continue with\n+\t\t\t\t * a stale local batch size (infinite loop).\n+\t\t\t\t */\n+\t\t\t\tadapter-\u003erx_buffers_per_hcall = 1;\n+\t\t\t\toutcome = IBMVETH_REPLENISH_BATCH_FALLBACK;\n+\t\t\t} else {\n+\t\t\t\toutcome = IBMVETH_REPLENISH_HCALL_FAIL;\n+\t\t\t}\n+\t\t} else {\n+\t\t\toutcome = IBMVETH_REPLENISH_HCALL_FAIL;\n+\t\t}\n+\t\tbreak;\n+\t}\n+\n+\tmb();\n+\tatomic_add(buffers_added, \u0026(pool-\u003eavailable));\n+\treturn outcome;\n+}\n+\n+/*\n+ * The final 8 bytes of the buffer list is a counter of frames dropped\n+ * because there was not a buffer in the buffer list capable of holding\n+ * the frame.\n+ */\n+static void ibmveth_update_rx_no_buffer(struct ibmveth_adapter *adapter,\n+\t\t\t\t\tint queue_index)\n+{\n+\tstruct ibmveth_rx_queue_stats *qstats;\n+\t__be64 *p;\n+\tu64 drops;\n+\n+\tif (queue_index \u003c 0 ||\n+\t    queue_index \u003e= ibmveth_get_num_rx_queues(adapter) ||\n+\t    !adapter-\u003ebuffer_list_addr[queue_index])\n+\t\treturn;\n+\n+\tp = adapter-\u003ebuffer_list_addr[queue_index] + 4096 - 8;\n+\tdrops = be64_to_cpup(p);\n+\n+\t/*\n+\t * PHYP's buffer-list page counter is absolute for that page. A new\n+\t * page (reopen / queue reuse after -L) starts near zero; fold the\n+\t * previous absolute into this queue's retired carry so sums stay\n+\t * monotonic. Both fields belong to the queue being updated, so this\n+\t * stays single-writer under the queue's replenish_lock.\n+\t */\n+\tqstats = \u0026adapter-\u003erx_qstats[queue_index];\n+\n+\tif (drops \u003c qstats-\u003eno_buffer_drops)\n+\t\tqstats-\u003eno_buffer_retired += qstats-\u003eno_buffer_drops;\n+\tqstats-\u003eno_buffer_drops = drops;\n+}\n+\n+/* replenish routine */\n+static void ibmveth_replenish_task(struct ibmveth_adapter *adapter,\n+\t\t\t\t   int queue_index)\n+{\n+\tstruct ibmveth_rx_q *rxq = \u0026adapter-\u003erx_queue[queue_index];\n+\tstruct ibmveth_replenish_fail fail = {};\n+\tunsigned long flags;\n+\tint i, rc;\n+\tint need_reset = 0;\n+\tint batch_fallback = 0;\n+\tint hcall_fail = 0;\n+\n+\tif (queue_index \u003e= ibmveth_get_num_rx_queues(adapter)) {\n+\t\tnetdev_dbg(adapter-\u003enetdev,\n+\t\t\t   \"Skipping replenish for freed queue %d (num_queues=%u)\\n\",\n+\t\t\t   queue_index, ibmveth_get_num_rx_queues(adapter));\n+\t\treturn;\n+\t}\n+\n+\tspin_lock_irqsave(\u0026rxq-\u003ereplenish_lock, flags);\n+\n+\tadapter-\u003erx_qstats[queue_index].replenish_task_cycles++;\n+\n+\tfor (i = (IBMVETH_NUM_BUFF_POOLS - 1); i \u003e= 0; i--) {\n+\t\tstruct ibmveth_buff_pool *pool =\n+\t\t\t\u0026adapter-\u003erx_buff_pool[queue_index][i];\n+\n+\t\tif (pool-\u003eactive \u0026\u0026 pool-\u003efree_map \u0026\u0026\n+\t\t    (atomic_read(\u0026pool-\u003eavailable) \u003c pool-\u003ethreshold)) {\n+\t\t\trc = ibmveth_replenish_buffer_pool(adapter, pool,\n+\t\t\t\t\t\t\t   queue_index, \u0026fail);\n+\t\t\tswitch (rc) {\n+\t\t\tcase IBMVETH_REPLENISH_RESET_MAP:\n+\t\t\tcase IBMVETH_REPLENISH_RESET_MQ:\n+\t\t\t\tneed_reset = rc;\n+\t\t\t\tgoto out_unlock;\n+\t\t\tcase IBMVETH_REPLENISH_BATCH_FALLBACK:\n+\t\t\t\tbatch_fallback = 1;\n+\t\t\t\tbreak;\n+\t\t\tcase IBMVETH_REPLENISH_HCALL_FAIL:\n+\t\t\t\thcall_fail = 1;\n+\t\t\t\tbreak;\n+\t\t\tdefault:\n+\t\t\t\tbreak;\n+\t\t\t}\n+\t\t}\n+\t}\n+\n+out_unlock:\n+\tibmveth_update_rx_no_buffer(adapter, queue_index);\n+\n+\tspin_unlock_irqrestore(\u0026rxq-\u003ereplenish_lock, flags);\n+\n+\t/* Log and schedule reset only after dropping replenish_lock. */\n+\tif (need_reset == IBMVETH_REPLENISH_RESET_MAP) {\n+\t\tnetdev_info(adapter-\u003enetdev,\n+\t\t\t    \"Invalid RX free_map entry on queue %d, reset\\n\",\n+\t\t\t    queue_index);\n+\t\tschedule_work(\u0026adapter-\u003ework);\n+\t} else if (need_reset == IBMVETH_REPLENISH_RESET_MQ) {\n+\t\tdev_err_ratelimited(\u0026adapter-\u003enetdev-\u003edev,\n+\t\t\t\t    \"MQ buffer add H_FUNCTION (q=%d, batch=%u), reset\\n\",\n+\t\t\t\t    queue_index, fail.batch);\n+\t\tadapter-\u003emq_fallback = true;\n+\t\tschedule_work(\u0026adapter-\u003ework);\n+\t}\n+\n+\tif (batch_fallback)\n+\t\tdev_warn_ratelimited(\u0026adapter-\u003enetdev-\u003edev,\n+\t\t\t\t     \"Legacy batch add H_FUNCTION (batch=%u), fallback\\n\",\n+\t\t\t\t     fail.batch);\n+\n+\tif (hcall_fail)\n+\t\tdev_warn_ratelimited(\u0026adapter-\u003enetdev-\u003edev,\n+\t\t\t\t     \"RX %s failed: filled=%u, rc=%lu, batch=%u\\n\",\n+\t\t\t\t     adapter-\u003emulti_queue ?\n+\t\t\t\t     \"h_add_logical_lan_buffers_queue\" :\n+\t\t\t\t     (fail.filled == 1 ?\n+\t\t\t\t      \"h_add_logical_lan_buffer\" :\n+\t\t\t\t      \"h_add_logical_lan_buffers\"),\n+\t\t\t\t     fail.filled, fail.lpar_rc, fail.batch);\n+}\n+\n+/**\n+ * ibmveth_restart_rx_queue - Post buffers and ensure Q can take RX\n+ * @adapter: ibmveth adapter\n+ * @qindex: RX queue index\n+ *\n+ * SQ open leaves PHYP masked until the first poll. If schedule_prep fails,\n+ * NAPI never runs and the queue stays masked (TX OK, RX/ARP dead) until\n+ * reload. Replenish first so an enable_irq fallback can actually deliver.\n+ * Also used after every open (SQ and MQ) and after scale-down rollback.\n+ */\n+static void ibmveth_restart_rx_queue(struct ibmveth_adapter *adapter,\n+\t\t\t\t     int qindex)\n+{\n+\tibmveth_replenish_task(adapter, qindex);\n+\tif (ibmveth_schedule_rx_queue(adapter, qindex))\n+\t\treturn;\n+\n+\tibmveth_enable_irq(adapter, qindex);\n+}\n+\n+/* empty and free ana buffer pool - also used to do cleanup in error paths */\n+static void ibmveth_free_buffer_pool(struct ibmveth_adapter *adapter,\n+\t\t\t\t     struct ibmveth_buff_pool *pool)\n+{\n+\tint i;\n+\n+\tkfree(pool-\u003efree_map);\n+\tpool-\u003efree_map = NULL;\n+\n+\tif (pool-\u003eskbuff \u0026\u0026 pool-\u003edma_addr) {\n+\t\tfor (i = 0; i \u003c pool-\u003esize; ++i) {\n+\t\t\tstruct sk_buff *skb = pool-\u003eskbuff[i];\n+\t\t\tif (skb) {\n+\t\t\t\tdma_unmap_single(\u0026adapter-\u003evdev-\u003edev,\n+\t\t\t\t\t\t pool-\u003edma_addr[i],\n+\t\t\t\t\t\t pool-\u003ebuff_size,\n+\t\t\t\t\t\t DMA_FROM_DEVICE);\n+\t\t\t\tdev_kfree_skb_any(skb);\n+\t\t\t\tpool-\u003eskbuff[i] = NULL;\n+\t\t\t}\n+\t\t}\n+\t}\n+\n+\tif (pool-\u003edma_addr) {\n+\t\tkfree(pool-\u003edma_addr);\n+\t\tpool-\u003edma_addr = NULL;\n+\t}\n+\n+\tif (pool-\u003eskbuff) {\n+\t\tkfree(pool-\u003eskbuff);\n+\t\tpool-\u003eskbuff = NULL;\n+\t}\n+\n+\t/*\n+\t * Keep probe/sysfs geometry (active, size, buff_size, threshold).\n+\t * Only tear down runtime allocations; open reuses active pools.\n+\t */\n+\tatomic_set(\u0026pool-\u003eavailable, 0);\n+}\n+\n+/**\n+ * ibmveth_free_queue_buffer_pools - Free buffer pools for a single queue\n+ * @adapter: ibmveth adapter structure\n+ * @queue: queue index\n+ *\n+ * Frees buffer pools that still hold allocations for the specified\n+ * queue (by free_map / dma_addr / skbuff presence), regardless of the\n+ * active flag.\n+ */\n+static void ibmveth_free_queue_buffer_pools(struct ibmveth_adapter *adapter,\n+\t\t\t\t\t    int queue)\n+{\n+\tint i;\n+\n+\tfor (i = 0; i \u003c IBMVETH_NUM_BUFF_POOLS; i++) {\n+\t\tstruct ibmveth_buff_pool *pool =\n+\t\t\t\u0026adapter-\u003erx_buff_pool[queue][i];\n+\n+\t\t/* Free pool if it has allocated memory, regardless of\n+\t\t * active flag. Allocation and active can diverge on failure\n+\t\t * paths, so check for actual allocations.\n+\t\t */\n+\t\tif (pool-\u003efree_map || pool-\u003edma_addr || pool-\u003eskbuff)\n+\t\t\tibmveth_free_buffer_pool(adapter, pool);\n+\t}\n+}\n+\n+/**\n+ * ibmveth_alloc_queue_buffer_pools - Allocate buffer pools for a single queue\n+ * @adapter: ibmveth adapter structure\n+ * @queue: queue index\n+ *\n+ * Allocates backing storage for each active pool on @queue.\n+ * Inactive pools (!active) are skipped. Pool metadata must be\n+ * initialized before calling this function.\n+ *\n+ * Return: 0 on success, negative error code on failure\n+ */\n+static int ibmveth_alloc_queue_buffer_pools(struct ibmveth_adapter *adapter,\n+\t\t\t\t\t    int queue)\n+{\n+\tstruct net_device *netdev = adapter-\u003enetdev;\n+\tint i;\n+\n+\tfor (i = 0; i \u003c IBMVETH_NUM_BUFF_POOLS; i++) {\n+\t\tstruct ibmveth_buff_pool *bpool =\n+\t\t\t\u0026adapter-\u003erx_buff_pool[queue][i];\n+\n+\t\tif (!bpool-\u003eactive)\n+\t\t\tcontinue;\n+\n+\t\tif (ibmveth_alloc_buffer_pool(bpool)) {\n+\t\t\tnetdev_err(netdev,\n+\t\t\t\t   \"queue %d pool %d alloc failed (size=%u count=%u)\\n\",\n+\t\t\t\t   queue, i,\n+\t\t\t\t   bpool-\u003ebuff_size,\n+\t\t\t\t   bpool-\u003esize);\n+\t\t\tbpool-\u003eactive = 0;\n+\t\t\t/* Free by allocation presence, not pool-\u003eactive.\n+\t\t\t * Needed on later resize paths (a pool can still\n+\t\t\t * hold memory after active is cleared). This\n+\t\t\t * open-fail slot is already empty:\n+\t\t\t * alloc_buffer_pool() undoes its own partials.\n+\t\t\t */\n+\t\t\tibmveth_free_queue_buffer_pools(adapter, queue);\n+\t\t\treturn -ENOMEM;\n+\t\t}\n+\t}\n+\n+\treturn 0;\n+}\n+\n+/**\n+ * ibmveth_alloc_buffer_pools - Allocate buffer pools for all queues\n+ * @adapter: ibmveth adapter structure\n+ *\n+ * Initializes pool metadata for queues 1-N from queue 0 settings,\n+ * then allocates buffer pools for all queues using the helper function.\n+ *\n+ * Return: 0 on success, negative error code on failure\n+ */\n+static int\n+ibmveth_alloc_buffer_pools(struct ibmveth_adapter *adapter)\n+{\n+\tstruct net_device *netdev = adapter-\u003enetdev;\n+\tint i, q, rc;\n+\n+\t/* Initialize pool metadata for queues 1..N from queue 0 settings */\n+\tfor (q = 1; q \u003c ibmveth_get_num_rx_queues(adapter); q++) {\n+\t\tfor (i = 0; i \u003c IBMVETH_NUM_BUFF_POOLS; i++) {\n+\t\t\tstruct ibmveth_buff_pool *src =\n+\t\t\t\t\u0026adapter-\u003erx_buff_pool[0][i];\n+\t\t\tstruct ibmveth_buff_pool *dst =\n+\t\t\t\t\u0026adapter-\u003erx_buff_pool[q][i];\n+\n+\t\t\tdst-\u003esize = src-\u003esize;\n+\t\t\tdst-\u003eindex = src-\u003eindex;\n+\t\t\tdst-\u003ebuff_size = src-\u003ebuff_size;\n+\t\t\tdst-\u003ethreshold = src-\u003ethreshold;\n+\t\t\tdst-\u003eactive = src-\u003eactive;\n+\t\t}\n+\t}\n+\n+\t/* Allocate actual buffers for all queues */\n+\tfor (q = 0; q \u003c ibmveth_get_num_rx_queues(adapter); q++) {\n+\t\trc = ibmveth_alloc_queue_buffer_pools(adapter, q);\n+\t\tif (rc) {\n+\t\t\t/* Free pools for all previous queues */\n+\t\t\twhile (--q \u003e= 0)\n+\t\t\t\tibmveth_free_queue_buffer_pools(adapter, q);\n+\t\t\treturn rc;\n+\t\t}\n+\t}\n+\n+\tnetdev_dbg(netdev, \"allocated buffer pools for %u queue(s)\\n\",\n+\t\t   ibmveth_get_num_rx_queues(adapter));\n+\treturn 0;\n+}\n+\n+/**\n+ * ibmveth_free_buffer_pools - Free buffer pools for all queues\n+ * @adapter: ibmveth adapter structure\n+ *\n+ * Frees buffer pools for all queues using the helper function.\n+ */\n+static void\n+ibmveth_free_buffer_pools(struct ibmveth_adapter *adapter)\n+{\n+\tint q;\n+\n+\t/* Free buffer pools for all queues */\n+\tfor (q = 0; q \u003c ibmveth_get_num_rx_queues(adapter); q++)\n+\t\tibmveth_free_queue_buffer_pools(adapter, q);\n+\n+\tnetdev_dbg(adapter-\u003enetdev, \"freed buffer pools for %u queue(s)\\n\",\n+\t\t   ibmveth_get_num_rx_queues(adapter));\n+}\n+\n+/**\n+ * ibmveth_alloc_single_rx_queue - Allocate resources for a single RX queue\n+ * @adapter: ibmveth adapter structure\n+ * @queue_idx: Queue index to allocate\n+ * @rxq_entries: Number of RX queue entries\n+ *\n+ * Allocates buffer list, RX queue, and per-queue buffer pools for one queue.\n+ * Used during incremental scale-up without affecting existing queues.\n+ *\n+ * Return: 0 on success, negative error code on failure\n+ */\n+static int\n+ibmveth_alloc_single_rx_queue(struct ibmveth_adapter *adapter, int queue_idx,\n+\t\t\t      int rxq_entries)\n+{\n+\tstruct device *dev = \u0026adapter-\u003evdev-\u003edev;\n+\tstruct net_device *netdev = adapter-\u003enetdev;\n+\tint i, rc = -ENOMEM;\n+\n+\tadapter-\u003ebuffer_list_addr[queue_idx] =\n+\t\t(void *)get_zeroed_page(GFP_KERNEL);\n+\tif (!adapter-\u003ebuffer_list_addr[queue_idx]) {\n+\t\tnetdev_err(netdev, \"unable to allocate buffer list for queue %d\\n\",\n+\t\t\t   queue_idx);\n+\t\treturn -ENOMEM;\n+\t}\n+\n+\tadapter-\u003erx_queue[queue_idx].queue_len =\n+\t\tsizeof(struct ibmveth_rx_q_entry) * rxq_entries;\n+\tadapter-\u003erx_queue[queue_idx].queue_addr =\n+\t\tdma_alloc_coherent(dev, adapter-\u003erx_queue[queue_idx].queue_len,\n+\t\t\t\t   \u0026adapter-\u003erx_queue[queue_idx].queue_dma,\n+\t\t\t\t   GFP_KERNEL);\n+\tif (!adapter-\u003erx_queue[queue_idx].queue_addr) {\n+\t\tnetdev_err(netdev, \"unable to allocate RX queue for queue %d\\n\",\n+\t\t\t   queue_idx);\n+\t\tgoto out_free_buflist;\n+\t}\n+\n+\tadapter-\u003ebuffer_list_dma[queue_idx] =\n+\t\tdma_map_single(dev, adapter-\u003ebuffer_list_addr[queue_idx],\n+\t\t\t       4096, DMA_BIDIRECTIONAL);\n+\tif (dma_mapping_error(dev, adapter-\u003ebuffer_list_dma[queue_idx])) {\n+\t\tnetdev_err(netdev, \"unable to map buffer list for queue %d\\n\",\n+\t\t\t   queue_idx);\n+\t\tadapter-\u003ebuffer_list_dma[queue_idx] = 0;\n+\t\tgoto out_free_rxq;\n+\t}\n+\n+\tfor (i = 0; i \u003c IBMVETH_NUM_BUFF_POOLS; i++) {\n+\t\tstruct ibmveth_buff_pool *src =\n+\t\t\t\u0026adapter-\u003erx_buff_pool[0][i];\n+\t\tstruct ibmveth_buff_pool *dst =\n+\t\t\t\u0026adapter-\u003erx_buff_pool[queue_idx][i];\n+\n+\t\tdst-\u003esize = src-\u003esize;\n+\t\tdst-\u003eindex = src-\u003eindex;\n+\t\tdst-\u003ebuff_size = src-\u003ebuff_size;\n+\t\tdst-\u003ethreshold = src-\u003ethreshold;\n+\t\tdst-\u003eactive = src-\u003eactive;\n+\t}\n+\n+\trc = ibmveth_alloc_queue_buffer_pools(adapter, queue_idx);\n+\tif (rc) {\n+\t\tnetdev_err(netdev,\n+\t\t\t   \"Failed to allocate buffer pools for queue %d\\n\",\n+\t\t\t   queue_idx);\n+\t\tgoto out_unmap_buflist;\n+\t}\n+\n+\tadapter-\u003erx_queue[queue_idx].index = 0;\n+\tadapter-\u003erx_queue[queue_idx].num_slots = rxq_entries;\n+\tadapter-\u003erx_queue[queue_idx].toggle = 1;\n+\n+\tnetdev_dbg(netdev,\n+\t\t   \"Allocated queue %d: buffer_list @ %p (DMA: 0x%llx), rx_queue @ %p (DMA: 0x%llx), %d entries\\n\",\n+\t\t   queue_idx, adapter-\u003ebuffer_list_addr[queue_idx],\n+\t\t   (unsigned long long)adapter-\u003ebuffer_list_dma[queue_idx],\n+\t\t   adapter-\u003erx_queue[queue_idx].queue_addr,\n+\t\t   (unsigned long long)adapter-\u003erx_queue[queue_idx].queue_dma,\n+\t\t   rxq_entries);\n+\n+\treturn 0;\n+\n+out_unmap_buflist:\n+\tdma_unmap_single(dev, adapter-\u003ebuffer_list_dma[queue_idx],\n+\t\t\t 4096, DMA_BIDIRECTIONAL);\n+\tadapter-\u003ebuffer_list_dma[queue_idx] = 0;\n+out_free_rxq:\n+\tdma_free_coherent(dev, adapter-\u003erx_queue[queue_idx].queue_len,\n+\t\t\t  adapter-\u003erx_queue[queue_idx].queue_addr,\n+\t\t\t  adapter-\u003erx_queue[queue_idx].queue_dma);\n+\tadapter-\u003erx_queue[queue_idx].queue_addr = NULL;\n+out_free_buflist:\n+\tfree_page((unsigned long)adapter-\u003ebuffer_list_addr[queue_idx]);\n+\tadapter-\u003ebuffer_list_addr[queue_idx] = NULL;\n+\treturn rc;\n+}\n+\n+/**\n+ * ibmveth_free_single_rx_queue - Free resources for a single RX queue\n+ * @adapter: ibmveth adapter structure\n+ * @queue_idx: Queue index to free\n+ *\n+ * Frees buffer list, RX queue, and per-queue buffer pools for one queue.\n+ * Used during incremental scale-down without affecting remaining queues.\n+ */\n+static void\n+ibmveth_free_single_rx_queue(struct ibmveth_adapter *adapter, int queue_idx)\n+{\n+\tstruct device *dev = \u0026adapter-\u003evdev-\u003edev;\n+\n+\tibmveth_free_queue_buffer_pools(adapter, queue_idx);\n+\n+\tif (adapter-\u003ebuffer_list_addr[queue_idx]) {\n+\t\tdma_unmap_single(dev, adapter-\u003ebuffer_list_dma[queue_idx],\n+\t\t\t\t 4096, DMA_BIDIRECTIONAL);\n+\t\tadapter-\u003ebuffer_list_dma[queue_idx] = 0;\n+\t}\n+\n+\tif (adapter-\u003erx_queue[queue_idx].queue_addr) {\n+\t\tdma_free_coherent(dev, adapter-\u003erx_queue[queue_idx].queue_len,\n+\t\t\t\t  adapter-\u003erx_queue[queue_idx].queue_addr,\n+\t\t\t\t  adapter-\u003erx_queue[queue_idx].queue_dma);\n+\t\tadapter-\u003erx_queue[queue_idx].queue_addr = NULL;\n+\t}\n+\n+\tif (adapter-\u003ebuffer_list_addr[queue_idx]) {\n+\t\tfree_page((unsigned long)adapter-\u003ebuffer_list_addr[queue_idx]);\n+\t\tadapter-\u003ebuffer_list_addr[queue_idx] = NULL;\n+\t}\n+\n+\tnetdev_dbg(adapter-\u003enetdev, \"Freed queue %d resources\\n\", queue_idx);\n+}\n+\n+static bool ibmveth_rxq_correlator_valid(struct ibmveth_adapter *adapter,\n+\t\t\t\t\t int queue_index, u64 correlator)\n+{\n+\tunsigned int pool = correlator \u003e\u003e 32;\n+\tunsigned int index = correlator \u0026 0xffffffffUL;\n+\tstruct ibmveth_buff_pool *bpool;\n+\n+\tif (pool \u003e= IBMVETH_NUM_BUFF_POOLS)\n+\t\treturn false;\n+\n+\tbpool = \u0026adapter-\u003erx_buff_pool[queue_index][pool];\n+\n+\t/* Require a live pool with allocated arrays before indexing.\n+\t * Inactive pools still have size from init; free clears skbuff.\n+\t */\n+\tif (!bpool-\u003eactive || !bpool-\u003eskbuff || !bpool-\u003efree_map)\n+\t\treturn false;\n+\n+\treturn index \u003c bpool-\u003esize;\n+}\n+\n+static void ibmveth_rxq_advance(struct ibmveth_rx_q *rxq)\n+{\n+\tif (++rxq-\u003eindex == rxq-\u003enum_slots) {\n+\t\trxq-\u003eindex = 0;\n+\t\trxq-\u003etoggle = !rxq-\u003etoggle;\n+\t}\n+}\n+\n+/**\n+ * ibmveth_remove_buffer_from_pool - remove a buffer from a pool\n+ * @adapter: adapter instance\n+ * @correlator: identifies pool and index\n+ * @queue_index: RX queue index (0..num_rx_queues-1)\n+ * @reuse: whether to reuse buffer\n+ *\n+ * Context: may run concurrently with netpoll replenish_task on the same\n+ * queue; takes per-queue replenish_lock to serialize free_map /\n+ * producer_index / available against the producer.\n+ *\n+ * Return:\n+ * * %0       - success\n+ * * %-EINVAL - pool or index out of range, or the pool is\n+ *              inactive / not live (skbuff or free_map NULL)\n+ * * %-EFAULT - pool and index map to null skb\n+ */\n+static int ibmveth_remove_buffer_from_pool(struct ibmveth_adapter *adapter,\n+\t\t\t\t\t   u64 correlator, int queue_index,\n+\t\t\t\t\t   bool reuse)\n+{\n+\tstruct ibmveth_rx_q *rxq = \u0026adapter-\u003erx_queue[queue_index];\n+\tunsigned int pool  = correlator \u003e\u003e 32;\n+\tunsigned int index = correlator \u0026 0xffffffffUL;\n+\tunsigned int free_index;\n+\tstruct sk_buff *skb;\n+\tunsigned long flags;\n+\tint rc = 0;\n+\n+\tspin_lock_irqsave(\u0026rxq-\u003ereplenish_lock, flags);\n+\n+\tif (!ibmveth_rxq_correlator_valid(adapter, queue_index, correlator)) {\n+\t\trc = -EINVAL;\n+\t\tgoto out_unlock;\n+\t}\n+\n+\tskb = adapter-\u003erx_buff_pool[queue_index][pool].skbuff[index];\n+\tif (!skb) {\n+\t\trc = -EFAULT;\n+\t\tgoto out_unlock;\n+\t}\n+\n+\t/* if we are going to reuse the buffer then keep the pointers around\n+\t * but mark index as available. replenish will see the skb pointer and\n+\t * assume it is to be recycled.\n+\t */\n+\tif (!reuse) {\n+\t\t/* remove the skb pointer to mark free. actual freeing is done\n+\t\t * by upper level networking after gro_receive\n+\t\t */\n+\t\tstruct ibmveth_buff_pool *bpool =\n+\t\t\t\u0026adapter-\u003erx_buff_pool[queue_index][pool];\n+\n+\t\tbpool-\u003eskbuff[index] = NULL;\n+\n+\t\tdma_unmap_single(\u0026adapter-\u003evdev-\u003edev,\n+\t\t\t\t bpool-\u003edma_addr[index],\n+\t\t\t\t bpool-\u003ebuff_size,\n+\t\t\t\t DMA_FROM_DEVICE);\n+\t}\n+\n+\tfree_index = adapter-\u003erx_buff_pool[queue_index][pool].producer_index;\n+\tadapter-\u003erx_buff_pool[queue_index][pool].producer_index++;\n+\tif (adapter-\u003erx_buff_pool[queue_index][pool].producer_index \u003e=\n+\t    adapter-\u003erx_buff_pool[queue_index][pool].size)\n+\t\tadapter-\u003erx_buff_pool[queue_index][pool].producer_index = 0;\n+\tadapter-\u003erx_buff_pool[queue_index][pool].free_map[free_index] = index;\n+\n+\tmb();\n+\n+\tatomic_dec(\u0026adapter-\u003erx_buff_pool[queue_index][pool].available);\n+\n+out_unlock:\n+\tspin_unlock_irqrestore(\u0026rxq-\u003ereplenish_lock, flags);\n+\treturn rc;\n+}\n+\n+/* get the current buffer on the rx queue */\n+static struct sk_buff *\n+ibmveth_rxq_get_buffer(struct ibmveth_adapter *adapter,\n+\t\t       int queue_index)\n+{\n+\tstruct ibmveth_rx_q *rxq = \u0026adapter-\u003erx_queue[queue_index];\n+\tu64 correlator = rxq-\u003equeue_addr[rxq-\u003eindex].correlator;\n+\tunsigned int pool = correlator \u003e\u003e 32;\n+\tunsigned int index = correlator \u0026 0xffffffffUL;\n+\n+\tif (!ibmveth_rxq_correlator_valid(adapter, queue_index, correlator))\n+\t\treturn NULL;\n+\n+\treturn adapter-\u003erx_buff_pool[queue_index][pool].skbuff[index];\n+}\n+\n+/**\n+ * ibmveth_rxq_harvest_buffer - Harvest buffer from pool\n+ *\n+ * @adapter: pointer to adapter\n+ * @queue_index: RX queue index to harvest from\n+ * @reuse:   whether to reuse buffer\n+ *\n+ * Context: called from ibmveth_poll\n+ *\n+ * On a bad correlator (-EINVAL/-EFAULT) the ring is still advanced so poll\n+ * cannot spin forever on one slot. The error is still returned: callers must\n+ * not treat it as a successful take from the pool (especially reuse=false,\n+ * which would hand the SKB to the stack while it remains pool-owned).\n+ *\n+ * Return:\n+ * * %0    - buffer removed from pool (or marked for reuse) and ring advanced\n+ * * other - non-zero return from ibmveth_remove_buffer_from_pool; ring has\n+ *           still been advanced for -EINVAL/-EFAULT\n+ */\n+static int ibmveth_rxq_harvest_buffer(struct ibmveth_adapter *adapter,\n+\t\t\t\t      int queue_index, bool reuse)\n+{\n+\tstruct ibmveth_rx_q *rxq = \u0026adapter-\u003erx_queue[queue_index];\n+\tu64 cor;\n+\tint rc;\n+\n+\tcor = rxq-\u003equeue_addr[rxq-\u003eindex].correlator;\n+\trc = ibmveth_remove_buffer_from_pool(adapter, cor, queue_index, reuse);\n+\tif (unlikely(rc)) {\n+\t\t/* Skip a corrupt slot without claiming pool ownership. */\n+\t\tif (rc == -EINVAL || rc == -EFAULT)\n+\t\t\tibmveth_rxq_advance(rxq);\n+\t\treturn rc;\n+\t}\n+\n+\tibmveth_rxq_advance(rxq);\n+\n+\treturn 0;\n+}\n+\n+/**\n+ * ibmveth_drain_rx_queue - Drain pending buffers from an RX queue\n+ * @adapter: ibmveth adapter structure\n+ * @queue_index: Queue index to drain\n+ *\n+ * Harvests pending completions back to the per-queue buffer pools.\n+ * A corrupt slot still advances the ring, so the return is slots\n+ * processed, not buffers recycled.\n+ * Must be called with NAPI disabled for this queue.\n+ *\n+ * Return: number of slots processed\n+ */\n+static int\n+ibmveth_drain_rx_queue(struct ibmveth_adapter *adapter, int queue_index)\n+{\n+\tstruct net_device *netdev = adapter-\u003enetdev;\n+\tint drained = 0;\n+\tint limit = adapter-\u003erx_queue[queue_index].num_slots;\n+\tint rc;\n+\n+\tnetdev_dbg(netdev, \"Draining RX queue %d (limit: %d slots)\\n\",\n+\t\t   queue_index, limit);\n+\n+\twhile (drained \u003c limit \u0026\u0026\n+\t       ibmveth_rxq_pending_buffer(adapter, queue_index)) {\n+\t\t/* Match poll-side order before harvesting completion state. */\n+\t\tsmp_rmb();\n+\t\trc = ibmveth_rxq_harvest_buffer(adapter, queue_index, true);\n+\t\tif (rc) {\n+\t\t\t/* -EINVAL/-EFAULT already advanced past the slot. */\n+\t\t\tif (rc == -EINVAL || rc == -EFAULT) {\n+\t\t\t\tdrained++;\n+\t\t\t\tcontinue;\n+\t\t\t}\n+\t\t\tnetdev_err(netdev,\n+\t\t\t\t   \"Failed to harvest buffer from queue %d during drain: %d\\n\",\n+\t\t\t\t   queue_index, rc);\n+\t\t\tbreak;\n+\t\t}\n+\t\tdrained++;\n+\t}\n+\n+\tif (drained \u003e 0)\n+\t\tnetdev_dbg(netdev, \"Drained %d slot(s) from RX queue %d\\n\",\n+\t\t\t   drained, queue_index);\n+\telse\n+\t\tnetdev_dbg(netdev, \"No slots to drain from RX queue %d\\n\",\n+\t\t\t   queue_index);\n+\n+\treturn drained;\n+}\n+\n+static void ibmveth_free_tx_ltb(struct ibmveth_adapter *adapter, int idx)\n+{\n+\tvoid *ltb = adapter-\u003etx_ltb_ptr[idx];\n+\tdma_addr_t dma = adapter-\u003etx_ltb_dma[idx];\n+\n+\tif (!ltb)\n+\t\treturn;\n+\n+\t/*\n+\t * Clear the slot before releasing it. start_xmit() tests\n+\t * tx_ltb_ptr[idx] to decide whether the LTB is usable.\n+\t */\n+\tadapter-\u003etx_ltb_ptr[idx] = NULL;\n+\tadapter-\u003etx_ltb_dma[idx] = 0;\n+\n+\tdma_unmap_single(\u0026adapter-\u003evdev-\u003edev, dma, adapter-\u003etx_ltb_size,\n+\t\t\t DMA_TO_DEVICE);\n+\tkfree(ltb);\n+}\n+\n+/* Caller must ensure tx_ltb_ptr[idx] is NULL. open() runs on\n+ * probe-zeroed slots; set_channels() skips populated indices.\n+ */\n+static int ibmveth_allocate_tx_ltb(struct ibmveth_adapter *adapter, int idx)\n+{\n+\tadapter-\u003etx_ltb_ptr[idx] = kzalloc(adapter-\u003etx_ltb_size,\n+\t\t\t\t\t   GFP_KERNEL);\n+\tif (!adapter-\u003etx_ltb_ptr[idx]) {\n+\t\tnetdev_err(adapter-\u003enetdev,\n+\t\t\t   \"unable to allocate tx long term buffer\\n\");\n+\t\treturn -ENOMEM;\n+\t}\n+\tadapter-\u003etx_ltb_dma[idx] = dma_map_single(\u0026adapter-\u003evdev-\u003edev,\n+\t\t\t\t\t\t  adapter-\u003etx_ltb_ptr[idx],\n+\t\t\t\t\t\t  adapter-\u003etx_ltb_size,\n+\t\t\t\t\t\t  DMA_TO_DEVICE);\n+\tif (dma_mapping_error(\u0026adapter-\u003evdev-\u003edev, adapter-\u003etx_ltb_dma[idx])) {\n+\t\tnetdev_err(adapter-\u003enetdev,\n+\t\t\t   \"unable to DMA map tx long term buffer\\n\");\n+\t\tkfree(adapter-\u003etx_ltb_ptr[idx]);\n+\t\tadapter-\u003etx_ltb_ptr[idx] = NULL;\n+\t\tadapter-\u003etx_ltb_dma[idx] = 0;\n+\t\treturn -ENOMEM;\n+\t}\n+\n+\treturn 0;\n+}\n+\n+/**\n+ * ibmveth_alloc_tx_resources - Allocate TX LTBs for real_num_tx_queues\n+ * @adapter: ibmveth adapter structure\n+ *\n+ * Allocates TX Long Term Buffers (LTBs) for real_num_tx_queues.\n+ *\n+ * Return: 0 on success, -ENOMEM on failure\n+ */\n+static int ibmveth_alloc_tx_resources(struct ibmveth_adapter *adapter)\n+{\n+\tstruct net_device *netdev = adapter-\u003enetdev;\n+\tint i;\n+\n+\tfor (i = 0; i \u003c netdev-\u003ereal_num_tx_queues; i++) {\n+\t\tif (ibmveth_allocate_tx_ltb(adapter, i))\n+\t\t\tgoto err_free_ltbs;\n+\t}\n+\n+\treturn 0;\n+\n+err_free_ltbs:\n+\twhile (--i \u003e= 0)\n+\t\tibmveth_free_tx_ltb(adapter, i);\n+\treturn -ENOMEM;\n+}\n+\n+/**\n+ * ibmveth_free_tx_resources - Free TX LTBs for real_num_tx_queues\n+ * @adapter: ibmveth adapter structure\n+ *\n+ * Frees TX Long Term Buffers (LTBs) for real_num_tx_queues.\n+ */\n+static void ibmveth_free_tx_resources(struct ibmveth_adapter *adapter)\n+{\n+\tstruct net_device *netdev = adapter-\u003enetdev;\n+\tint i;\n+\n+\tfor (i = 0; i \u003c netdev-\u003ereal_num_tx_queues; i++)\n+\t\tibmveth_free_tx_ltb(adapter, i);\n+}\n+\n+static int ibmveth_register_logical_lan(struct ibmveth_adapter *adapter,\n+\t\t\t\t\tunion ibmveth_buf_desc rxq_desc,\n+\t\t\t\t\tu64 mac_address)\n+{\n+\tint rc, try_again = 1;\n+\tunsigned long ua = adapter-\u003evdev-\u003eunit_address;\n+\tunsigned long buf_dma = adapter-\u003ebuffer_list_dma[0];\n+\tunsigned long filter_dma = adapter-\u003efilter_list_dma;\n+\tunsigned long qh0;\n+\n+\t/*\n+\t * After a kexec the adapter will still be open, so our attempt to\n+\t * open it will fail. So if we get a failure we free the adapter and\n+\t * try again, but only once.\n+\t */\n+retry:\n+\t/* In multi-queue mode, obtain a queue handle for queue 0 so all RX\n+\t * queues can use the same per-queue buffer hypercalls.\n+\t */\n+\tif (adapter-\u003emulti_queue) {\n+\t\trc = h_register_logical_lan_with_handle(ua, buf_dma,\n+\t\t\t\t\t\t\trxq_desc.desc,\n+\t\t\t\t\t\t\tfilter_dma,\n+\t\t\t\t\t\t\tmac_address,\n+\t\t\t\t\t\t\t\u0026qh0);\n+\t\tif (rc == H_SUCCESS)\n+\t\t\tadapter-\u003equeue_handle[0] = qh0;\n+\t} else {\n+\t\trc = h_register_logical_lan(ua, buf_dma, rxq_desc.desc,\n+\t\t\t\t\t    filter_dma, mac_address);\n+\t}\n+\tnetdev_dbg(adapter-\u003enetdev, \"h_register_logical_lan%s rc=%d\\n\",\n+\t\t   adapter-\u003emulti_queue ? \"_with_handle\" : \"\", rc);\n+\n+\tif (rc != H_SUCCESS \u0026\u0026 try_again) {\n+\t\tdo {\n+\t\t\trc = h_free_logical_lan(adapter-\u003evdev-\u003eunit_address);\n+\t\t} while (H_IS_LONG_BUSY(rc) || (rc == H_BUSY));\n+\t\tnetdev_dbg(adapter-\u003enetdev, \"h_free_logical_lan rc=%d\\n\", rc);\n+\n+\t\ttry_again = 0;\n+\t\tgoto retry;\n+\t}\n+\n+\treturn rc;\n+}\n+\n+/**\n+ * ibmveth_register_logical_lan_queue - Register subordinate queue with\n+ * hypervisor\n+ * @adapter: ibmveth adapter structure\n+ * @rxq_desc: Receive queue descriptor\n+ * @queue_index: RX queue index (1..N for subordinate queues)\n+ *\n+ * Registers a subordinate receive queue using H_REG_LOGICAL_LAN_QUEUE.\n+ * On success, stores the queue handle and virtual IRQ in the adapter.\n+ * If IRQ mapping fails after a successful hypervisor registration, the\n+ * queue is freed before returning. If that free fails, the handle is\n+ * kept so the caller can skip unmap.\n+ *\n+ * Return: H_SUCCESS on success, negative errno on IRQ mapping failure,\n+ *         hypervisor error code otherwise\n+ */\n+static int\n+ibmveth_register_logical_lan_queue(struct ibmveth_adapter *adapter,\n+\t\t\t\t   union ibmveth_buf_desc rxq_desc,\n+\t\t\t\t   int queue_index)\n+{\n+\tunsigned long handle, hwirq;\n+\tunsigned int virq;\n+\tlong lpar_rc;\n+\tunsigned long ua = adapter-\u003evdev-\u003eunit_address;\n+\tunsigned long bl = adapter-\u003ebuffer_list_dma[queue_index];\n+\n+\tnetdev_dbg(adapter-\u003enetdev,\n+\t\t   \"register queue %d: ua=0x%lx bl=0x%lx rxq=0x%llx\\n\",\n+\t\t   queue_index, ua, bl, rxq_desc.desc);\n+\tdo {\n+\t\tlpar_rc = h_register_logical_lan_queue(ua, bl,\n+\t\t\t\t\t\t       rxq_desc.desc, \u0026handle,\n+\t\t\t\t\t\t       \u0026hwirq);\n+\t} while (H_IS_LONG_BUSY(lpar_rc) || (lpar_rc == H_BUSY));\n+\tnetdev_dbg(adapter-\u003enetdev,\n+\t\t   \"h_register_logical_lan_queue queue %d rc=%ld\\n\",\n+\t\t   queue_index, lpar_rc);\n+\n+\tif (lpar_rc == H_SUCCESS) {\n+\t\tvirq = irq_create_mapping(NULL, hwirq);\n+\t\tif (!virq) {\n+\t\t\tunsigned long free_rc;\n+\n+\t\t\tnetdev_err(adapter-\u003enetdev,\n+\t\t\t\t   \"Failed to map IRQ for queue %d (hwirq=%lu)\\n\",\n+\t\t\t\t   queue_index, hwirq);\n+\t\t\tdo {\n+\t\t\t\tfree_rc = h_free_logical_lan_queue(ua, handle);\n+\t\t\t} while (H_IS_LONG_BUSY(free_rc) ||\n+\t\t\t\t  (free_rc == H_BUSY));\n+\t\t\tif (free_rc != H_SUCCESS) {\n+\t\t\t\tnetdev_err(adapter-\u003enetdev,\n+\t\t\t\t\t   \"h_free_logical_lan_queue failed for queue %d after IRQ map failure: rc=0x%lx\\n\",\n+\t\t\t\t\t   queue_index, free_rc);\n+\t\t\t\t/*\n+\t\t\t\t * PHYP still owns this queue. Keep the\n+\t\t\t\t * handle so the caller skips unmap.\n+\t\t\t\t */\n+\t\t\t\tadapter-\u003equeue_handle[queue_index] = handle;\n+\t\t\t\treturn -EIO;\n+\t\t\t}\n+\t\t\treturn -EINVAL;\n+\t\t}\n \n-\t\tif (pool-\u003eactive \u0026\u0026\n-\t\t    (atomic_read(\u0026pool-\u003eavailable) \u003c pool-\u003ethreshold))\n-\t\t\tibmveth_replenish_buffer_pool(adapter, pool);\n+\t\tadapter-\u003equeue_handle[queue_index] = handle;\n+\t\tadapter-\u003equeue_irq[queue_index] = virq;\n+\n+\t\tnetdev_dbg(adapter-\u003enetdev,\n+\t\t\t   \"queue %d registered: handle=0x%llx irq=%u\\n\",\n+\t\t\t   queue_index, adapter-\u003equeue_handle[queue_index],\n+\t\t\t   adapter-\u003equeue_irq[queue_index]);\n+\t\treturn H_SUCCESS;\n \t}\n \n-\tibmveth_update_rx_no_buffer(adapter);\n+\t/*\n+\t * H_FUNCTION means firmware rejected this subordinate register\n+\t * (MQ unsupported / dropped after LPM). Caller fails this open and\n+\t * latches mq_fallback so the next open applies SQ; keep a specific\n+\t * log then the generic failure lines below.\n+\t */\n+\tif (lpar_rc == H_FUNCTION)\n+\t\tnetdev_err(adapter-\u003enetdev,\n+\t\t\t   \"h_register_logical_lan_queue H_FUNCTION for queue %d (firmware MQ unsupported)\\n\",\n+\t\t\t   queue_index);\n+\n+\tnetdev_err(adapter-\u003enetdev,\n+\t\t   \"h_register_logical_lan_queue failed for queue %d with %ld\\n\",\n+\t\t   queue_index, lpar_rc);\n+\tnetdev_err(adapter-\u003enetdev,\n+\t\t   \"queue %d params: unit_addr=0x%x buffer_list_dma=0x%llx rxq_desc=0x%llx\\n\",\n+\t\t   queue_index, adapter-\u003evdev-\u003eunit_address,\n+\t\t   adapter-\u003ebuffer_list_dma[queue_index],\n+\t\t   rxq_desc.desc);\n+\n+\treturn lpar_rc;\n }\n \n-/* empty and free ana buffer pool - also used to do cleanup in error paths */\n-static void ibmveth_free_buffer_pool(struct ibmveth_adapter *adapter,\n-\t\t\t\t     struct ibmveth_buff_pool *pool)\n+/**\n+ * ibmveth_register_single_rx_queue - Register one subordinate RX queue\n+ * @adapter: ibmveth adapter structure\n+ * @queue_idx: Queue index to register (1..N)\n+ *\n+ * Builds the queue descriptor and registers with the hypervisor via\n+ * ibmveth_register_logical_lan_queue().\n+ *\n+ * Return: 0 on success, -EINVAL if @queue_idx is invalid, -EOPNOTSUPP if\n+ *         firmware rejects MQ (H_FUNCTION), -EIO on other failures\n+ */\n+static int\n+ibmveth_register_single_rx_queue(struct ibmveth_adapter *adapter,\n+\t\t\t\t int queue_idx)\n+{\n+\tstruct net_device *netdev = adapter-\u003enetdev;\n+\tunion ibmveth_buf_desc rxq_desc;\n+\tlong lpar_rc;\n+\n+\tif (WARN_ON(queue_idx \u003c 1 || queue_idx \u003e= IBMVETH_MAX_RX_QUEUES))\n+\t\treturn -EINVAL;\n+\n+\trxq_desc.fields.flags_len = IBMVETH_BUF_VALID |\n+\t\t\t\t    adapter-\u003erx_queue[queue_idx].queue_len;\n+\trxq_desc.fields.address = adapter-\u003erx_queue[queue_idx].queue_dma;\n+\n+\tlpar_rc = ibmveth_register_logical_lan_queue(adapter, rxq_desc,\n+\t\t\t\t\t\t     queue_idx);\n+\tif (lpar_rc != H_SUCCESS) {\n+\t\tnetdev_err(netdev, \"Failed to register queue %d: rc=%ld\\n\",\n+\t\t\t   queue_idx, lpar_rc);\n+\t\tif (lpar_rc == H_FUNCTION)\n+\t\t\treturn -EOPNOTSUPP;\n+\t\treturn -EIO;\n+\t}\n+\n+\tnetdev_dbg(netdev, \"Registered queue %d with handle 0x%llx\\n\",\n+\t\t   queue_idx, adapter-\u003equeue_handle[queue_idx]);\n+\n+\treturn 0;\n+}\n+\n+/**\n+ * ibmveth_deregister_single_rx_queue - Deregister one subordinate RX queue\n+ * @adapter: ibmveth adapter structure\n+ * @queue_idx: Queue index to deregister (1..N)\n+ *\n+ * Deregisters a single queue via H_FREE_LOGICAL_LAN_QUEUE. Linux IRQ handler\n+ * teardown and subordinate virq mapping disposal are owned by interrupt\n+ * cleanup helpers; queue 0 is freed only through ibmveth_free_all_queues()\n+ * (H_FREE_LOGICAL_LAN).\n+ *\n+ * Return: 0 if the queue was freed or had no handle, -EIO if\n+ *         H_FREE_LOGICAL_LAN_QUEUE failed. On -EIO the handle is\n+ *         left set so the caller can skip unmap.\n+ */\n+static int\n+ibmveth_deregister_single_rx_queue(struct ibmveth_adapter *adapter,\n+\t\t\t\t   int queue_idx)\n+{\n+\tunsigned long lpar_rc;\n+\tunsigned long ua = adapter-\u003evdev-\u003eunit_address;\n+\tunsigned long qh = adapter-\u003equeue_handle[queue_idx];\n+\n+\tif (!qh)\n+\t\treturn 0;\n+\n+\tdo {\n+\t\tlpar_rc = h_free_logical_lan_queue(ua, qh);\n+\t} while (H_IS_LONG_BUSY(lpar_rc) || (lpar_rc == H_BUSY));\n+\n+\tif (lpar_rc != H_SUCCESS) {\n+\t\tnetdev_err(adapter-\u003enetdev,\n+\t\t\t   \"h_free_logical_lan_queue failed, queue %d rc=%ld\\n\",\n+\t\t\t   queue_idx, lpar_rc);\n+\t\treturn -EIO;\n+\t}\n+\n+\tadapter-\u003equeue_handle[queue_idx] = 0;\n+\n+\tnetdev_dbg(adapter-\u003enetdev, \"Deregistered queue %d\\n\", queue_idx);\n+\treturn 0;\n+}\n+\n+/**\n+ * ibmveth_destroy_subordinate_rx_queue - Tear down one subordinate RX queue\n+ * @adapter: ibmveth adapter structure\n+ * @queue_idx: Queue index to destroy (1..N)\n+ *\n+ * Deregister with PHYP before unmapping buffer pools so hypervisor buffer\n+ * ownership is released while queue metadata is still valid. If the\n+ * hcall fails, leave the queue mapped and keep the handle.\n+ *\n+ * Return: 0 on success, -EIO if deregister failed\n+ */\n+static int\n+ibmveth_destroy_subordinate_rx_queue(struct ibmveth_adapter *adapter,\n+\t\t\t\t     int queue_idx)\n+{\n+\tint rc;\n+\n+\trc = ibmveth_deregister_single_rx_queue(adapter, queue_idx);\n+\tif (rc)\n+\t\treturn rc;\n+\tibmveth_cleanup_single_rx_interrupt(adapter, queue_idx);\n+\tibmveth_free_single_rx_queue(adapter, queue_idx);\n+\treturn 0;\n+}\n+\n+/**\n+ * ibmveth_desired_dma_for_rxqs - CMO entitlement for a given RX queue count\n+ * @adapter: ibmveth adapter\n+ * @rxqs: number of RX queues to size for\n+ *\n+ * Same math as ibmveth_get_desired_dma(), but uses @rxqs instead of the\n+ * live adapter-\u003enum_rx_queues. Scale-up raises desired for the *target*\n+ * count before allocating so vio_cmo_alloc cannot fail mid-resize.\n+ *\n+ * Return: bytes of IO memory desired for @rxqs RX queues\n+ */\n+static unsigned long\n+ibmveth_desired_dma_for_rxqs(struct ibmveth_adapter *adapter,\n+\t\t\t     unsigned int rxqs)\n+{\n+\tstruct net_device *netdev = adapter-\u003enetdev;\n+\tstruct iommu_table *tbl;\n+\tunsigned long ret;\n+\tint i, q;\n+\n+\ttbl = get_iommu_table_base(\u0026adapter-\u003evdev-\u003edev);\n+\n+\tret = IBMVETH_BUFF_LIST_SIZE * rxqs + IBMVETH_FILT_LIST_SIZE;\n+\tret += IOMMU_PAGE_ALIGN(netdev-\u003emtu, tbl);\n+\tret += IOMMU_PAGE_ALIGN(IBMVETH_MAX_TX_BUF_SIZE, tbl);\n+\n+\t/*\n+\t * Pool metadata for queues 1+ is copied from queue 0 at open.\n+\t * Always size from pool 0 x @rxqs.\n+\t *\n+\t * CMO (Power9 and earlier) and MQ firmware (Power11+) do not\n+\t * coexist, so a CMO partition always sizes one queue here. The\n+\t * MQ terms and the ethtool -L CMO refreshes are defensive only.\n+\t */\n+\tfor (q = 0; q \u003c rxqs; q++) {\n+\t\tint rxqentries = 1;\n+\n+\t\tfor (i = 0; i \u003c IBMVETH_NUM_BUFF_POOLS; i++) {\n+\t\t\tstruct ibmveth_buff_pool *bpool =\n+\t\t\t\t\u0026adapter-\u003erx_buff_pool[0][i];\n+\n+\t\t\tif (bpool-\u003eactive)\n+\t\t\t\tret += bpool-\u003esize *\n+\t\t\t\t\tIOMMU_PAGE_ALIGN(bpool-\u003ebuff_size, tbl);\n+\t\t\trxqentries += bpool-\u003esize;\n+\t\t}\n+\n+\t\tret += IOMMU_PAGE_ALIGN(rxqentries *\n+\t\t\t\t\tsizeof(struct ibmveth_rx_q_entry), tbl);\n+\t}\n+\n+\treturn ret;\n+}\n+\n+/**\n+ * ibmveth_free_stranded_rx_queues - Free queues stranded above the live count\n+ * @adapter: ibmveth adapter structure\n+ *\n+ * A scale-up register or IRQ-setup failure whose H_FREE_LOGICAL_LAN_QUEUE\n+ * also fails leaves that queue out of the live set with its handle set\n+ * and its memory still mapped. Call only after successful H_FREE_LOGICAL_LAN,\n+ * which drops every subordinate queue, so the memory can be released.\n+ */\n+static void ibmveth_free_stranded_rx_queues(struct ibmveth_adapter *adapter)\n {\n \tint i;\n \n-\tkfree(pool-\u003efree_map);\n-\tpool-\u003efree_map = NULL;\n+\tfor (i = ibmveth_get_num_rx_queues(adapter);\n+\t     i \u003c IBMVETH_MAX_RX_QUEUES; i++) {\n+\t\tif (!adapter-\u003equeue_handle[i])\n+\t\t\tcontinue;\n+\t\tadapter-\u003equeue_handle[i] = 0;\n+\t\tibmveth_free_single_rx_queue(adapter, i);\n+\t}\n+}\n \n-\tif (pool-\u003eskbuff \u0026\u0026 pool-\u003edma_addr) {\n-\t\tfor (i = 0; i \u003c pool-\u003esize; ++i) {\n-\t\t\tstruct sk_buff *skb = pool-\u003eskbuff[i];\n-\t\t\tif (skb) {\n-\t\t\t\tdma_unmap_single(\u0026adapter-\u003evdev-\u003edev,\n-\t\t\t\t\t\t pool-\u003edma_addr[i],\n-\t\t\t\t\t\t pool-\u003ebuff_size,\n-\t\t\t\t\t\t DMA_FROM_DEVICE);\n-\t\t\t\tdev_kfree_skb_any(skb);\n-\t\t\t\tpool-\u003eskbuff[i] = NULL;\n+/* Set CMO desired entitlement from the live RX queue count. */\n+static void ibmveth_refresh_cmo_desired(struct ibmveth_adapter *adapter)\n+{\n+\tif (firmware_has_feature(FW_FEATURE_CMO))\n+\t\tvio_cmo_set_dev_desired(adapter-\u003evdev,\n+\t\t\t\t\tibmveth_get_desired_dma(adapter-\u003evdev));\n+}\n+\n+/**\n+ * ibmveth_scale_up_rx_queues - Add RX queues to a live adapter\n+ * @adapter: ibmveth adapter structure\n+ * @old_count: Current number of RX queues\n+ * @new_count: Target number of RX queues, above @old_count\n+ * @rxq_entries: Number of entries per RX queue\n+ *\n+ * Raises CMO entitlement and widens real_num_rx_queues before any new\n+ * queue is unmasked, then brings each queue up in turn. A failure\n+ * unwinds only the queues this call added; a queue PHYP refuses to\n+ * release stays in the live set instead.\n+ *\n+ * Return: 0 on success, negative error code on failure\n+ */\n+static int\n+ibmveth_scale_up_rx_queues(struct ibmveth_adapter *adapter, int old_count,\n+\t\t\t   int new_count, int rxq_entries)\n+{\n+\tstruct net_device *netdev = adapter-\u003enetdev;\n+\tint failed_queue;\n+\tint rc, i;\n+\n+\tnetdev_dbg(netdev, \"Scale-up: adding queues %d-%d\\n\",\n+\t\t   old_count, new_count - 1);\n+\n+\t/*\n+\t * Raise CMO desired for the target count before dma_map /\n+\t * dma_alloc_coherent / replenish (same order as change_mtu).\n+\t * Do not bump live num_rx_queues here, only entitlement.\n+\t */\n+\tif (firmware_has_feature(FW_FEATURE_CMO)) {\n+\t\tunsigned long dma;\n+\n+\t\tdma = ibmveth_desired_dma_for_rxqs(adapter, new_count);\n+\t\tvio_cmo_set_dev_desired(adapter-\u003evdev, dma);\n+\t}\n+\n+\t/*\n+\t * Widen real_num before any new queue is unmasked so\n+\t * skb_record_rx_queue() cannot trip get_rps_cpu()\n+\t * WARN_ONCE. Probe already allocated num_rx_queues.\n+\t */\n+\trc = netif_set_real_num_rx_queues(netdev, new_count);\n+\tif (rc) {\n+\t\tnetdev_err(netdev,\n+\t\t\t   \"Failed to set real RX queues to %d: %d\\n\",\n+\t\t\t   new_count, rc);\n+\t\tibmveth_refresh_cmo_desired(adapter);\n+\t\treturn rc;\n+\t}\n+\n+\tfor (i = old_count; i \u003c new_count; i++) {\n+\t\tif (adapter-\u003equeue_handle[i]) {\n+\t\t\t/* Left by a failed H_FREE; close frees it. */\n+\t\t\tnetdev_err(netdev,\n+\t\t\t\t   \"RX queue %d still held by PHYP, reset pending\\n\",\n+\t\t\t\t   i);\n+\t\t\trc = -EBUSY;\n+\t\t\tgoto cleanup_new_queues;\n+\t\t}\n+\t\trc = ibmveth_alloc_single_rx_queue(adapter, i,\n+\t\t\t\t\t\t   rxq_entries);\n+\t\tif (rc) {\n+\t\t\tnetdev_err(netdev, \"Failed to allocate queue %d: %d\\n\",\n+\t\t\t\t   i, rc);\n+\t\t\tgoto cleanup_new_queues;\n+\t\t}\n+\n+\t\trc = ibmveth_register_single_rx_queue(adapter, i);\n+\t\tif (rc) {\n+\t\t\tnetdev_err(netdev, \"Failed to register queue %d: %d\\n\",\n+\t\t\t\t   i, rc);\n+\t\t\tif (rc == -EOPNOTSUPP)\n+\t\t\t\tadapter-\u003emq_fallback = true;\n+\t\t\tif (!ibmveth_deregister_single_rx_queue(adapter,\n+\t\t\t\t\t\t\t\ti))\n+\t\t\t\tibmveth_free_single_rx_queue(adapter,\n+\t\t\t\t\t\t\t     i);\n+\t\t\telse\n+\t\t\t\tschedule_work(\u0026adapter-\u003ework);\n+\t\t\tgoto cleanup_new_queues;\n+\t\t}\n+\n+\t\trc = ibmveth_setup_single_rx_interrupt(adapter, i);\n+\t\tif (rc) {\n+\t\t\tnetdev_err(netdev,\n+\t\t\t\t   \"Failed to setup IRQ for queue %d: %d\\n\",\n+\t\t\t\t   i, rc);\n+\t\t\t/* request_irq failed: mapped but no handler */\n+\t\t\tibmveth_dispose_subordinate_irq_mapping(adapter,\n+\t\t\t\t\t\t\t\ti);\n+\t\t\tif (!ibmveth_deregister_single_rx_queue(adapter,\n+\t\t\t\t\t\t\t\ti))\n+\t\t\t\tibmveth_free_single_rx_queue(adapter,\n+\t\t\t\t\t\t\t     i);\n+\t\t\telse\n+\t\t\t\tschedule_work(\u0026adapter-\u003ework);\n+\t\t\tgoto cleanup_new_queues;\n+\t\t}\n+\n+\t\t/*\n+\t\t * Fully ready before PHYP delivery, matching open():\n+\t\t * publish -\u003e replenish -\u003e napi_enable -\u003e enable_irq.\n+\t\t * That way ibmveth_interrupt() cannot run on an\n+\t\t * unpublished, empty, or NAPI-disabled queue.\n+\t\t */\n+\t\tibmveth_publish_num_rx_queues(adapter, i + 1);\n+\t\tibmveth_replenish_task(adapter, i);\n+\t\tnapi_enable(\u0026adapter-\u003enapi[i]);\n+\n+\t\trc = ibmveth_enable_irq(adapter, i);\n+\t\tif (rc) {\n+\t\t\tnetdev_err(netdev,\n+\t\t\t\t   \"Failed to enable IRQ for queue %d: %d\\n\",\n+\t\t\t\t   i, rc);\n+\t\t\t/*\n+\t\t\t * Published, replenished, and NAPI-enabled,\n+\t\t\t * but PHYP never unmasked. Match scale-down /\n+\t\t\t * shared cleanup: drain posted buffers, then\n+\t\t\t * deregister before unmap via\n+\t\t\t * destroy_subordinate.\n+\t\t\t *\n+\t\t\t * napi_disable() must come BEFORE the count\n+\t\t\t * is lowered, matching scale-down and\n+\t\t\t * cleanup_new_queues. Lowering it first does\n+\t\t\t * not hide queue i from netpoll: after\n+\t\t\t * ndo_poll_controller, netpoll_poll_dev()\n+\t\t\t * calls poll_napi(), which walks dev-\u003enapi_list\n+\t\t\t * unbounded by the queue count and skips a NAPI\n+\t\t\t * only once NAPI_STATE_NPSVC is set. Queue i is\n+\t\t\t * enabled here, so ibmveth_poll() would run and\n+\t\t\t * trip its queue_index \u003e= num_rx_queues\n+\t\t\t * WARN_ON. napi_disable() sets NPSVC, so\n+\t\t\t * poll_napi() skips the queue instead.\n+\t\t\t */\n+\t\t\tnapi_disable(\u0026adapter-\u003enapi[i]);\n+\t\t\t/* A poll may have unmasked PHYP; remask. */\n+\t\t\tibmveth_disable_irq(adapter, i);\n+\t\t\tsynchronize_irq(adapter-\u003equeue_irq[i]);\n+\t\t\tibmveth_publish_num_rx_queues(adapter, i);\n+\t\t\tibmveth_drain_rx_queue(adapter, i);\n+\t\t\tsynchronize_net();\n+\t\t}\n+\t\tif (rc \u0026\u0026\n+\t\t    ibmveth_destroy_subordinate_rx_queue(adapter, i)) {\n+\t\t\tint keep = i + 1;\n+\n+\t\t\t/*\n+\t\t\t * PHYP still owns queue i. Keep it in the\n+\t\t\t * live set with the queues this -L already\n+\t\t\t * added. cleanup_new_queues would publish\n+\t\t\t * below i and orphan this mapping.\n+\t\t\t */\n+\t\t\tibmveth_publish_num_rx_queues(adapter, keep);\n+\t\t\tif (netif_set_real_num_rx_queues(netdev,\n+\t\t\t\t\t\t\t keep))\n+\t\t\t\tschedule_work(\u0026adapter-\u003ework);\n+\t\t\tibmveth_replenish_task(adapter, i);\n+\t\t\tnapi_enable(\u0026adapter-\u003enapi[i]);\n+\t\t\trc = ibmveth_enable_irq(adapter, i);\n+\t\t\tif (rc) {\n+\t\t\t\tnetdev_err(netdev,\n+\t\t\t\t\t   \"IRQ %d rc=%d\\n\",\n+\t\t\t\t\t   i, rc);\n+\t\t\t\tschedule_work(\u0026adapter-\u003ework);\n+\t\t\t} else {\n+\t\t\t\tibmveth_restart_rx_queue(adapter, i);\n \t\t\t}\n+\t\t\treturn -EIO;\n+\t\t}\n+\t\tif (rc) {\n+\t\t\t/* enable_irq errno; keep -EIO. */\n+\t\t\trc = -EIO;\n+\t\t\tgoto cleanup_new_queues;\n \t\t}\n+\t\tibmveth_restart_rx_queue(adapter, i);\n \t}\n \n-\tif (pool-\u003edma_addr) {\n-\t\tkfree(pool-\u003edma_addr);\n-\t\tpool-\u003edma_addr = NULL;\n+\treturn 0;\n+\n+cleanup_new_queues:\n+\tfailed_queue = i;\n+\tif (failed_queue \u003e old_count)\n+\t\tnetdev_err(netdev,\n+\t\t\t   \"Scale-up failed at queue %d, cleaning up queues %d-%d\\n\",\n+\t\t\t   failed_queue, old_count, failed_queue - 1);\n+\telse\n+\t\tnetdev_err(netdev,\n+\t\t\t   \"Scale-up failed at queue %d, nothing to clean up\\n\",\n+\t\t\t   failed_queue);\n+\n+\tfor (i = old_count; i \u003c failed_queue; i++) {\n+\t\tibmveth_disable_irq(adapter, i);\n+\t\tsynchronize_irq(adapter-\u003equeue_irq[i]);\n \t}\n \n-\tif (pool-\u003eskbuff) {\n-\t\tkfree(pool-\u003eskbuff);\n-\t\tpool-\u003eskbuff = NULL;\n+\tfor (i = old_count; i \u003c failed_queue; i++)\n+\t\tnapi_disable(\u0026adapter-\u003enapi[i]);\n+\n+\t/* Same remask as scale-down: poll may have re-armed during disable. */\n+\tfor (i = old_count; i \u003c failed_queue; i++) {\n+\t\tibmveth_disable_irq(adapter, i);\n+\t\tsynchronize_irq(adapter-\u003equeue_irq[i]);\n+\t}\n+\n+\tfor (i = old_count; i \u003c failed_queue; i++)\n+\t\tibmveth_drain_rx_queue(adapter, i);\n+\n+\t/* Drop the live count before freeing the half-added queues. */\n+\tibmveth_publish_num_rx_queues(adapter, old_count);\n+\tif (netif_set_real_num_rx_queues(netdev, old_count))\n+\t\tschedule_work(\u0026adapter-\u003ework);\n+\tsynchronize_net();\n+\n+\tfor (i = failed_queue - 1; i \u003e= old_count; i--) {\n+\t\tint keep;\n+\n+\t\tif (!ibmveth_destroy_subordinate_rx_queue(adapter, i))\n+\t\t\tcontinue;\n+\n+\t\tkeep = i + 1;\n+\t\tibmveth_publish_num_rx_queues(adapter, keep);\n+\t\tif (netif_set_real_num_rx_queues(netdev, keep))\n+\t\t\tschedule_work(\u0026adapter-\u003ework);\n+\t\tfor (i = old_count; i \u003c keep; i++) {\n+\t\t\tint irq_rc;\n+\n+\t\t\tibmveth_replenish_task(adapter, i);\n+\t\t\t/* START: NAPI before PHYP unmask. */\n+\t\t\tnapi_enable(\u0026adapter-\u003enapi[i]);\n+\t\t\tirq_rc = ibmveth_enable_irq(adapter, i);\n+\t\t\tif (irq_rc) {\n+\t\t\t\tnetdev_err(netdev,\n+\t\t\t\t\t   \"IRQ %d rc=%d\\n\",\n+\t\t\t\t\t   i, irq_rc);\n+\t\t\t\tschedule_work(\u0026adapter-\u003ework);\n+\t\t\t\tcontinue;\n+\t\t\t}\n+\t\t\tibmveth_restart_rx_queue(adapter, i);\n+\t\t}\n+\t\tibmveth_refresh_cmo_desired(adapter);\n+\t\tnetdev_warn(netdev,\n+\t\t\t    \"Keeping %d queues after scale-up failure\\n\",\n+\t\t\t    keep);\n+\t\treturn rc;\n \t}\n+\n+\t/* Roll CMO desired back to the surviving queue count. */\n+\tibmveth_refresh_cmo_desired(adapter);\n+\n+\tnetdev_warn(netdev, \"Keeping %d queues after scale-up failure\\n\",\n+\t\t    old_count);\n+\treturn rc;\n }\n \n /**\n- * ibmveth_remove_buffer_from_pool - remove a buffer from a pool\n- * @adapter: adapter instance\n- * @correlator: identifies pool and index\n- * @reuse: whether to reuse buffer\n+ * ibmveth_scale_down_rx_queues - Remove RX queues from a live adapter\n+ * @adapter: ibmveth adapter structure\n+ * @old_count: Current number of RX queues\n+ * @new_count: Target number of RX queues, below @old_count\n  *\n- * Return:\n- * * %0       - success\n- * * %-EINVAL - correlator maps to pool or index out of range\n- * * %-EFAULT - pool and index map to null skb\n+ * Quiesces the retiring queues, drains them and harvests their final\n+ * no_buffer counts, publishes the surviving count, and only then\n+ * deregisters them. Walks high to low so a failed free leaves 0..i\n+ * live and republishes that count.\n+ *\n+ * Return: 0 on success, negative error code on failure\n  */\n-static int ibmveth_remove_buffer_from_pool(struct ibmveth_adapter *adapter,\n-\t\t\t\t\t   u64 correlator, bool reuse)\n+static int\n+ibmveth_scale_down_rx_queues(struct ibmveth_adapter *adapter, int old_count,\n+\t\t\t     int new_count)\n {\n-\tunsigned int pool  = correlator \u003e\u003e 32;\n-\tunsigned int index = correlator \u0026 0xffffffffUL;\n-\tunsigned int free_index;\n-\tstruct sk_buff *skb;\n+\tstruct net_device *netdev = adapter-\u003enetdev;\n+\tint rc, i;\n \n-\tif (WARN_ON(pool \u003e= IBMVETH_NUM_BUFF_POOLS) ||\n-\t    WARN_ON(index \u003e= adapter-\u003erx_buff_pool[pool].size)) {\n-\t\tschedule_work(\u0026adapter-\u003ework);\n-\t\treturn -EINVAL;\n+\tnetdev_dbg(netdev, \"Scale-down: removing queues %d-%d\\n\",\n+\t\t   new_count, old_count - 1);\n+\n+\t/*\n+\t * Mask PHYP before napi_disable so the handler cannot miss\n+\t * a mask while NAPI is already dead. An in-flight poll can\n+\t * still re-arm PHYP while napi_disable() waits, so remask\n+\t * and sync again after NAPI is stopped. Then drain, harvest\n+\t * final no_buffer stats, publish the surviving count, and\n+\t * synchronize_net() before destroy so netpoll cannot walk\n+\t * dying queues (handler may still be registered until destroy).\n+\t */\n+\tfor (i = new_count; i \u003c old_count; i++) {\n+\t\tif (!adapter-\u003equeue_irq[i])\n+\t\t\tcontinue;\n+\t\tibmveth_disable_irq(adapter, i);\n+\t\tsynchronize_irq(adapter-\u003equeue_irq[i]);\n \t}\n \n-\tskb = adapter-\u003erx_buff_pool[pool].skbuff[index];\n-\tif (WARN_ON(!skb)) {\n-\t\tschedule_work(\u0026adapter-\u003ework);\n-\t\treturn -EFAULT;\n+\tfor (i = new_count; i \u003c old_count; i++)\n+\t\tnapi_disable(\u0026adapter-\u003enapi[i]);\n+\n+\tfor (i = new_count; i \u003c old_count; i++) {\n+\t\tif (!adapter-\u003equeue_irq[i])\n+\t\t\tcontinue;\n+\t\tibmveth_disable_irq(adapter, i);\n+\t\tsynchronize_irq(adapter-\u003equeue_irq[i]);\n \t}\n \n-\t/* if we are going to reuse the buffer then keep the pointers around\n-\t * but mark index as available. replenish will see the skb pointer and\n-\t * assume it is to be recycled.\n+\tfor (i = new_count; i \u003c old_count; i++)\n+\t\tibmveth_drain_rx_queue(adapter, i);\n+\n+\t/*\n+\t * netpoll still reaches these queues until the publish\n+\t * below, and ibmveth_replenish_task() folds the same\n+\t * counters under the queue's replenish_lock. Take it here\n+\t * so the harvest stays the single writer it documents.\n \t */\n-\tif (!reuse) {\n-\t\t/* remove the skb pointer to mark free. actual freeing is done\n-\t\t * by upper level networking after gro_receive\n+\tfor (i = new_count; i \u003c old_count; i++) {\n+\t\tstruct ibmveth_rx_q *rxq = \u0026adapter-\u003erx_queue[i];\n+\t\tunsigned long flags;\n+\n+\t\tspin_lock_irqsave(\u0026rxq-\u003ereplenish_lock, flags);\n+\t\tibmveth_update_rx_no_buffer(adapter, i);\n+\t\tspin_unlock_irqrestore(\u0026rxq-\u003ereplenish_lock, flags);\n+\t}\n+\n+\tibmveth_publish_num_rx_queues(adapter, new_count);\n+\tsynchronize_net();\n+\n+\trc = netif_set_real_num_rx_queues(netdev, new_count);\n+\tif (rc) {\n+\t\tnetdev_err(netdev, \"Failed to set real RX queues to %d: %d\\n\",\n+\t\t\t   new_count, rc);\n+\t\tibmveth_publish_num_rx_queues(adapter, old_count);\n+\t\tfor (i = new_count; i \u003c old_count; i++) {\n+\t\t\tint irq_rc;\n+\n+\t\t\tibmveth_replenish_task(adapter, i);\n+\t\t\t/* START: NAPI before PHYP unmask. */\n+\t\t\tnapi_enable(\u0026adapter-\u003enapi[i]);\n+\t\t\tirq_rc = ibmveth_enable_irq(adapter, i);\n+\t\t\tif (irq_rc) {\n+\t\t\t\tnetdev_err(netdev,\n+\t\t\t\t\t   \"Failed to re-enable IRQ for queue %d during scale-down rollback (rc=%d), scheduling reset\\n\",\n+\t\t\t\t\t   i, irq_rc);\n+\t\t\t\tschedule_work(\u0026adapter-\u003ework);\n+\t\t\t\tcontinue;\n+\t\t\t}\n+\t\t\tibmveth_restart_rx_queue(adapter, i);\n+\t\t}\n+\t\treturn rc;\n+\t}\n+\n+\tfor (i = old_count - 1; i \u003e= new_count; i--) {\n+\t\tint keep;\n+\n+\t\trc = ibmveth_destroy_subordinate_rx_queue(adapter, i);\n+\t\tif (!rc)\n+\t\t\tcontinue;\n+\n+\t\t/*\n+\t\t * High-to-low so a failed free leaves 0..i live.\n+\t\t * Queues i+1..old_count-1 are already gone.\n \t\t */\n-\t\tadapter-\u003erx_buff_pool[pool].skbuff[index] = NULL;\n+\t\tkeep = i + 1;\n+\t\tibmveth_publish_num_rx_queues(adapter, keep);\n+\t\tif (netif_set_real_num_rx_queues(netdev, keep))\n+\t\t\tschedule_work(\u0026adapter-\u003ework);\n+\t\tfor (i = new_count; i \u003c keep; i++) {\n+\t\t\tint irq_rc;\n+\n+\t\t\tibmveth_replenish_task(adapter, i);\n+\t\t\t/* START: NAPI before PHYP unmask. */\n+\t\t\tnapi_enable(\u0026adapter-\u003enapi[i]);\n+\t\t\tirq_rc = ibmveth_enable_irq(adapter, i);\n+\t\t\tif (irq_rc) {\n+\t\t\t\tnetdev_err(netdev,\n+\t\t\t\t\t   \"IRQ %d rc=%d\\n\",\n+\t\t\t\t\t   i, irq_rc);\n+\t\t\t\tschedule_work(\u0026adapter-\u003ework);\n+\t\t\t\tcontinue;\n+\t\t\t}\n+\t\t\tibmveth_restart_rx_queue(adapter, i);\n+\t\t}\n+\t\treturn rc;\n+\t}\n \n-\t\tdma_unmap_single(\u0026adapter-\u003evdev-\u003edev,\n-\t\t\t\t adapter-\u003erx_buff_pool[pool].dma_addr[index],\n-\t\t\t\t adapter-\u003erx_buff_pool[pool].buff_size,\n-\t\t\t\t DMA_FROM_DEVICE);\n+\treturn 0;\n+}\n+\n+/**\n+ * ibmveth_resize_rx_queues_incremental - Resize RX queue count incrementally\n+ * @adapter: ibmveth adapter structure\n+ * @new_count: Target number of RX queues\n+ * @rxq_entries: Number of entries per RX queue\n+ *\n+ * Adds or removes RX queues without tearing down the entire adapter.\n+ * Active queues continue receiving during scale-up. Scale-up widens\n+ * real_num_rx_queues before unmasking a new queue. Scale-down drains\n+ * excess queues before deregistering them with the hypervisor.\n+ *\n+ * Return: 0 on success, negative error code on failure\n+ */\n+static int\n+ibmveth_resize_rx_queues_incremental(struct ibmveth_adapter *adapter,\n+\t\t\t\t     int new_count, int rxq_entries)\n+{\n+\tstruct net_device *netdev = adapter-\u003enetdev;\n+\tint old_count = ibmveth_get_num_rx_queues(adapter);\n+\tint rc;\n+\n+\tif (old_count == new_count) {\n+\t\tnetdev_dbg(netdev, \"RX queue count unchanged (%d), nothing to do\\n\",\n+\t\t\t   old_count);\n+\t\treturn 0;\n+\t}\n+\n+\tif (new_count \u003c 1 || new_count \u003e IBMVETH_MAX_RX_QUEUES) {\n+\t\tnetdev_err(netdev, \"Invalid RX queue count %d (must be 1-%d)\\n\",\n+\t\t\t   new_count, IBMVETH_MAX_RX_QUEUES);\n+\t\treturn -EINVAL;\n \t}\n \n-\tfree_index = adapter-\u003erx_buff_pool[pool].producer_index;\n-\tadapter-\u003erx_buff_pool[pool].producer_index++;\n-\tif (adapter-\u003erx_buff_pool[pool].producer_index \u003e=\n-\t    adapter-\u003erx_buff_pool[pool].size)\n-\t\tadapter-\u003erx_buff_pool[pool].producer_index = 0;\n-\tadapter-\u003erx_buff_pool[pool].free_map[free_index] = index;\n+\tnetdev_info(netdev, \"Incrementally resizing RX queues: %d to %d\\n\",\n+\t\t    old_count, new_count);\n \n-\tmb();\n+\tif (new_count \u003e old_count)\n+\t\trc = ibmveth_scale_up_rx_queues(adapter, old_count, new_count,\n+\t\t\t\t\t\trxq_entries);\n+\telse\n+\t\trc = ibmveth_scale_down_rx_queues(adapter, old_count,\n+\t\t\t\t\t\t  new_count);\n+\tif (rc)\n+\t\treturn rc;\n \n-\tatomic_dec(\u0026(adapter-\u003erx_buff_pool[pool].available));\n+\tnetdev_info(netdev, \"Successfully resized to %u RX queues (incremental)\\n\",\n+\t\t    ibmveth_get_num_rx_queues(adapter));\n+\n+\tibmveth_refresh_cmo_desired(adapter);\n \n \treturn 0;\n }\n \n-/* get the current buffer on the rx queue */\n-static inline struct sk_buff *ibmveth_rxq_get_buffer(struct ibmveth_adapter *adapter)\n+/**\n+ * ibmveth_free_all_queues - Free all RX queues at once\n+ * @adapter: ibmveth adapter structure\n+ *\n+ * Issues one H_FREE_LOGICAL_LAN for full adapter teardown. Per PAPR/PHYP,\n+ * that drops the primary LAN and any subordinate queues registered under\n+ * it. Incremental scale-down uses H_FREE_LOGICAL_LAN_QUEUE per queue\n+ * instead; do not use this helper for partial live-set shrink.\n+ *\n+ * Used during interface close and registration error cleanup.\n+ *\n+ * Retries only H_BUSY and H_IS_LONG_BUSY. On other failures, logs and\n+ * returns -EIO; queue_handle[] is cleared regardless. Callers still run\n+ * RX pool and DMA teardown afterward (same as pre-helper close()).\n+ *\n+ * Clears queue handles only; queue_irq[] is released by\n+ * ibmveth_cleanup_rx_interrupts() on close, or by\n+ * ibmveth_dispose_subordinate_irq_mappings() on partial register failure.\n+ *\n+ * Return: 0 on success, -EIO if H_FREE_LOGICAL_LAN failed\n+ */\n+static int ibmveth_free_all_queues(struct ibmveth_adapter *adapter)\n {\n-\tu64 correlator = adapter-\u003erx_queue.queue_addr[adapter-\u003erx_queue.index].correlator;\n-\tunsigned int pool = correlator \u003e\u003e 32;\n-\tunsigned int index = correlator \u0026 0xffffffffUL;\n+\tunsigned long lpar_rc;\n+\tint i;\n \n-\tif (WARN_ON(pool \u003e= IBMVETH_NUM_BUFF_POOLS) ||\n-\t    WARN_ON(index \u003e= adapter-\u003erx_buff_pool[pool].size)) {\n-\t\tschedule_work(\u0026adapter-\u003ework);\n-\t\treturn NULL;\n+\tnetdev_dbg(adapter-\u003enetdev, \"freeing all RX queues at once\\n\");\n+\n+\tdo {\n+\t\tlpar_rc = h_free_logical_lan(adapter-\u003evdev-\u003eunit_address);\n+\t} while (H_IS_LONG_BUSY(lpar_rc) || (lpar_rc == H_BUSY));\n+\n+\tfor (i = 0; i \u003c ibmveth_get_num_rx_queues(adapter); i++)\n+\t\tadapter-\u003equeue_handle[i] = 0;\n+\n+\tif (lpar_rc != H_SUCCESS) {\n+\t\tnetdev_err(adapter-\u003enetdev,\n+\t\t\t   \"h_free_logical_lan failed: %ld\\n\", lpar_rc);\n+\t\treturn -EIO;\n \t}\n \n-\treturn adapter-\u003erx_buff_pool[pool].skbuff[index];\n+\treturn 0;\n }\n \n /**\n- * ibmveth_rxq_harvest_buffer - Harvest buffer from pool\n- *\n- * @adapter: pointer to adapter\n- * @reuse:   whether to reuse buffer\n+ * ibmveth_register_rx_queues - Register RX queues with hypervisor\n+ * @adapter: ibmveth adapter structure\n+ * @mac_address: MAC address for device registration\n  *\n- * Context: called from ibmveth_poll\n+ * Registers queue 0 via ibmveth_register_logical_lan(), then subordinate\n+ * queues 1..N when multi-queue mode is enabled.\n  *\n- * Return:\n- * * %0    - success\n- * * other - non-zero return from ibmveth_remove_buffer_from_pool\n+ * Return: 0 on success, -ENONET if queue 0 registration fails,\n+ *         -EOPNOTSUPP if firmware rejects a subordinate queue (H_FUNCTION;\n+ *         this function latches mq_fallback), -EIO on other subordinate\n+ *         failures\n  */\n-static int ibmveth_rxq_harvest_buffer(struct ibmveth_adapter *adapter,\n-\t\t\t\t      bool reuse)\n+static int\n+ibmveth_register_rx_queues(struct ibmveth_adapter *adapter, u64 mac_address)\n {\n-\tu64 cor;\n-\tint rc;\n+\tstruct net_device *netdev = adapter-\u003enetdev;\n+\tunion ibmveth_buf_desc rxq_desc;\n+\tunsigned long lpar_rc;\n+\tunsigned int num;\n+\tint i, rc;\n \n-\tcor = adapter-\u003erx_queue.queue_addr[adapter-\u003erx_queue.index].correlator;\n-\trc = ibmveth_remove_buffer_from_pool(adapter, cor, reuse);\n-\tif (unlikely(rc))\n-\t\treturn rc;\n+\trxq_desc.fields.flags_len = IBMVETH_BUF_VALID |\n+\t\t\t\t    adapter-\u003erx_queue[0].queue_len;\n+\trxq_desc.fields.address = adapter-\u003erx_queue[0].queue_dma;\n+\tadapter-\u003equeue_irq[0] = netdev-\u003eirq;\n+\n+\trc = ibmveth_disable_irq(adapter, 0);\n+\tif (rc)\n+\t\tnetdev_dbg(netdev,\n+\t\t\t   \"Failed to disable IRQ for queue 0 before registration, rc=%d\\n\",\n+\t\t\t   rc);\n \n-\tif (++adapter-\u003erx_queue.index == adapter-\u003erx_queue.num_slots) {\n-\t\tadapter-\u003erx_queue.index = 0;\n-\t\tadapter-\u003erx_queue.toggle = !adapter-\u003erx_queue.toggle;\n+\tlpar_rc = ibmveth_register_logical_lan(adapter, rxq_desc, mac_address);\n+\tif (lpar_rc != H_SUCCESS) {\n+\t\tnetdev_err(netdev,\n+\t\t\t   \"h_register_logical_lan failed: %ld\\n\", lpar_rc);\n+\t\tnetdev_err(netdev,\n+\t\t\t   \"buffer TCE:0x%llx filter TCE:0x%llx rxq desc:0x%llx MAC:0x%llx\\n\",\n+\t\t\t   adapter-\u003ebuffer_list_dma[0],\n+\t\t\t   adapter-\u003efilter_list_dma,\n+\t\t\t   rxq_desc.desc, mac_address);\n+\t\treturn -ENONET;\n \t}\n \n-\treturn 0;\n-}\n+\tnum = ibmveth_get_num_rx_queues(adapter);\n+\tif (num == 1 || !adapter-\u003emulti_queue) {\n+\t\tnetdev_dbg(netdev,\n+\t\t\t   \"registered 1 RX queue with hypervisor (single-queue mode)\\n\");\n+\t\treturn 0;\n+\t}\n \n-static void ibmveth_free_tx_ltb(struct ibmveth_adapter *adapter, int idx)\n-{\n-\tdma_unmap_single(\u0026adapter-\u003evdev-\u003edev, adapter-\u003etx_ltb_dma[idx],\n-\t\t\t adapter-\u003etx_ltb_size, DMA_TO_DEVICE);\n-\tkfree(adapter-\u003etx_ltb_ptr[idx]);\n-\tadapter-\u003etx_ltb_ptr[idx] = NULL;\n-}\n+\tnetdev_dbg(netdev, \"Registering %u subordinate queues (1-%u)\\n\",\n+\t\t   num - 1, num - 1);\n \n-static int ibmveth_allocate_tx_ltb(struct ibmveth_adapter *adapter, int idx)\n-{\n-\tadapter-\u003etx_ltb_ptr[idx] = kzalloc(adapter-\u003etx_ltb_size,\n-\t\t\t\t\t   GFP_KERNEL);\n-\tif (!adapter-\u003etx_ltb_ptr[idx]) {\n-\t\tnetdev_err(adapter-\u003enetdev,\n-\t\t\t   \"unable to allocate tx long term buffer\\n\");\n-\t\treturn -ENOMEM;\n-\t}\n-\tadapter-\u003etx_ltb_dma[idx] = dma_map_single(\u0026adapter-\u003evdev-\u003edev,\n-\t\t\t\t\t\t  adapter-\u003etx_ltb_ptr[idx],\n-\t\t\t\t\t\t  adapter-\u003etx_ltb_size,\n-\t\t\t\t\t\t  DMA_TO_DEVICE);\n-\tif (dma_mapping_error(\u0026adapter-\u003evdev-\u003edev, adapter-\u003etx_ltb_dma[idx])) {\n-\t\tnetdev_err(adapter-\u003enetdev,\n-\t\t\t   \"unable to DMA map tx long term buffer\\n\");\n-\t\tkfree(adapter-\u003etx_ltb_ptr[idx]);\n-\t\tadapter-\u003etx_ltb_ptr[idx] = NULL;\n-\t\treturn -ENOMEM;\n+\tfor (i = 1; i \u003c num; i++) {\n+\t\trc = ibmveth_register_single_rx_queue(adapter, i);\n+\t\tif (rc) {\n+\t\t\t/* Firmware MQ gone: fall back to SQ on next open. */\n+\t\t\tif (rc == -EOPNOTSUPP)\n+\t\t\t\tadapter-\u003emq_fallback = true;\n+\t\t\tgoto err_unregister;\n+\t\t}\n \t}\n \n+\tnetdev_dbg(netdev,\n+\t\t   \"registered %u RX queues with hypervisor (multi-queue mode)\\n\",\n+\t\t   num);\n+\n \treturn 0;\n+\n+err_unregister:\n+\tibmveth_dispose_subordinate_irq_mappings(adapter);\n+\tibmveth_free_all_queues(adapter);\n+\treturn rc;\n }\n \n-static int ibmveth_register_logical_lan(struct ibmveth_adapter *adapter,\n-        union ibmveth_buf_desc rxq_desc, u64 mac_address)\n+/**\n+ * ibmveth_apply_mq_fallback - Drop multi-queue mode after firmware rejection\n+ * @adapter: ibmveth adapter\n+ *\n+ * mq_fallback is set when firmware rejects MQ (subordinate register or\n+ * buffer-add H_FUNCTION). Apply only at the start of open after teardown so\n+ * num_rx_queues is not shrunk while IRQ/NAPI still reference higher queues.\n+ * Consumes the flag and clears multi_queue, which is what makes the\n+ * single-queue decision permanent for this device.\n+ */\n+static void ibmveth_apply_mq_fallback(struct ibmveth_adapter *adapter)\n {\n-\tint rc, try_again = 1;\n+\tstruct net_device *netdev = adapter-\u003enetdev;\n \n-\t/*\n-\t * After a kexec the adapter will still be open, so our attempt to\n-\t * open it will fail. So if we get a failure we free the adapter and\n-\t * try again, but only once.\n-\t */\n-retry:\n-\trc = h_register_logical_lan(adapter-\u003evdev-\u003eunit_address,\n-\t\t\t\t    adapter-\u003ebuffer_list_dma, rxq_desc.desc,\n-\t\t\t\t    adapter-\u003efilter_list_dma, mac_address);\n+\tif (!adapter-\u003emq_fallback)\n+\t\treturn;\n \n-\tif (rc != H_SUCCESS \u0026\u0026 try_again) {\n-\t\tdo {\n-\t\t\trc = h_free_logical_lan(adapter-\u003evdev-\u003eunit_address);\n-\t\t} while (H_IS_LONG_BUSY(rc) || (rc == H_BUSY));\n+\tadapter-\u003emq_fallback = false;\n \n-\t\ttry_again = 0;\n-\t\tgoto retry;\n-\t}\n+\tif (!adapter-\u003emulti_queue \u0026\u0026 ibmveth_get_num_rx_queues(adapter) == 1)\n+\t\treturn;\n \n-\treturn rc;\n+\tnetdev_warn(netdev,\n+\t\t    \"Falling back to single RX queue (firmware MQ unavailable)\\n\");\n+\tadapter-\u003emulti_queue = false;\n+\tibmveth_publish_num_rx_queues(adapter, 1);\n+\t/* real_num_rx_queues is set later in open after resources exist. */\n+\tif (adapter-\u003erx_buffers_per_hcall \u003e IBMVETH_MAX_RX_REGULAR)\n+\t\tadapter-\u003erx_buffers_per_hcall = IBMVETH_MAX_RX_REGULAR;\n }\n \n static int ibmveth_open(struct net_device *netdev)\n {\n \tstruct ibmveth_adapter *adapter = netdev_priv(netdev);\n-\tu64 mac_address;\n+\tu64 mac_address = ether_addr_to_u64(netdev-\u003edev_addr);\n \tint rxq_entries = 1;\n-\tunsigned long lpar_rc;\n \tint rc;\n-\tunion ibmveth_buf_desc rxq_desc;\n \tint i;\n-\tstruct device *dev;\n \n \tnetdev_dbg(netdev, \"open starting\\n\");\n \n-\tnapi_enable(\u0026adapter-\u003enapi);\n+\tibmveth_apply_mq_fallback(adapter);\n \n-\tfor(i = 0; i \u003c IBMVETH_NUM_BUFF_POOLS; i++)\n-\t\trxq_entries += adapter-\u003erx_buff_pool[i].size;\n+\tfor (i = 0; i \u003c IBMVETH_NUM_BUFF_POOLS; i++)\n+\t\trxq_entries += adapter-\u003erx_buff_pool[0][i].size;\n \n-\trc = -ENOMEM;\n-\tadapter-\u003ebuffer_list_addr = (void*) get_zeroed_page(GFP_KERNEL);\n-\tif (!adapter-\u003ebuffer_list_addr) {\n-\t\tnetdev_err(netdev, \"unable to allocate list pages\\n\");\n+\trc = ibmveth_alloc_filter_list(adapter);\n+\tif (rc)\n \t\tgoto out;\n-\t}\n-\n-\tadapter-\u003efilter_list_addr = (void*) get_zeroed_page(GFP_KERNEL);\n-\tif (!adapter-\u003efilter_list_addr) {\n-\t\tnetdev_err(netdev, \"unable to allocate filter pages\\n\");\n-\t\tgoto out_free_buffer_list;\n-\t}\n-\n-\tdev = \u0026adapter-\u003evdev-\u003edev;\n \n-\tadapter-\u003erx_queue.queue_len = sizeof(struct ibmveth_rx_q_entry) *\n-\t\t\t\t\t\trxq_entries;\n-\tadapter-\u003erx_queue.queue_addr =\n-\t\tdma_alloc_coherent(dev, adapter-\u003erx_queue.queue_len,\n-\t\t\t\t   \u0026adapter-\u003erx_queue.queue_dma, GFP_KERNEL);\n-\tif (!adapter-\u003erx_queue.queue_addr)\n+\trc = ibmveth_alloc_rx_queues(adapter, rxq_entries);\n+\tif (rc)\n \t\tgoto out_free_filter_list;\n \n-\tadapter-\u003ebuffer_list_dma = dma_map_single(dev,\n-\t\t\tadapter-\u003ebuffer_list_addr, 4096, DMA_BIDIRECTIONAL);\n-\tif (dma_mapping_error(dev, adapter-\u003ebuffer_list_dma)) {\n-\t\tnetdev_err(netdev, \"unable to map buffer list pages\\n\");\n+\trc = ibmveth_alloc_buffer_pools(adapter);\n+\tif (rc)\n \t\tgoto out_free_queue_mem;\n-\t}\n-\n-\tadapter-\u003efilter_list_dma = dma_map_single(dev,\n-\t\t\tadapter-\u003efilter_list_addr, 4096, DMA_BIDIRECTIONAL);\n-\tif (dma_mapping_error(dev, adapter-\u003efilter_list_dma)) {\n-\t\tnetdev_err(netdev, \"unable to map filter list pages\\n\");\n-\t\tgoto out_unmap_buffer_list;\n-\t}\n-\n-\tfor (i = 0; i \u003c netdev-\u003ereal_num_tx_queues; i++) {\n-\t\tif (ibmveth_allocate_tx_ltb(adapter, i))\n-\t\t\tgoto out_free_tx_ltb;\n-\t}\n-\n-\tadapter-\u003erx_queue.index = 0;\n-\tadapter-\u003erx_queue.num_slots = rxq_entries;\n-\tadapter-\u003erx_queue.toggle = 1;\n-\n-\tmac_address = ether_addr_to_u64(netdev-\u003edev_addr);\n \n-\trxq_desc.fields.flags_len = IBMVETH_BUF_VALID |\n-\t\t\t\t\tadapter-\u003erx_queue.queue_len;\n-\trxq_desc.fields.address = adapter-\u003erx_queue.queue_dma;\n-\n-\tnetdev_dbg(netdev, \"buffer list @ 0x%p\\n\", adapter-\u003ebuffer_list_addr);\n-\tnetdev_dbg(netdev, \"filter list @ 0x%p\\n\", adapter-\u003efilter_list_addr);\n-\tnetdev_dbg(netdev, \"receive q   @ 0x%p\\n\", adapter-\u003erx_queue.queue_addr);\n-\n-\th_vio_signal(adapter-\u003evdev-\u003eunit_address, VIO_IRQ_DISABLE);\n+\trc = ibmveth_register_rx_queues(adapter, mac_address);\n+\tif (rc)\n+\t\tgoto out_free_buffer_pools;\n \n-\tlpar_rc = ibmveth_register_logical_lan(adapter, rxq_desc, mac_address);\n+\trc = netif_set_real_num_rx_queues(netdev,\n+\t\t\t\t\t  ibmveth_get_num_rx_queues(adapter));\n \n-\tif (lpar_rc != H_SUCCESS) {\n-\t\tnetdev_err(netdev, \"h_register_logical_lan failed with %ld\\n\",\n-\t\t\t   lpar_rc);\n-\t\tnetdev_err(netdev, \"buffer TCE:0x%llx filter TCE:0x%llx rxq \"\n-\t\t\t   \"desc:0x%llx MAC:0x%llx\\n\",\n-\t\t\t\t     adapter-\u003ebuffer_list_dma,\n-\t\t\t\t     adapter-\u003efilter_list_dma,\n-\t\t\t\t     rxq_desc.desc,\n-\t\t\t\t     mac_address);\n-\t\trc = -ENONET;\n-\t\tgoto out_unmap_filter_list;\n+\tif (rc) {\n+\t\tnetdev_err(netdev, \"failed to set number of rx queues\\n\");\n+\t\tgoto out_unregister_queues;\n \t}\n \n-\tfor (i = 0; i \u003c IBMVETH_NUM_BUFF_POOLS; i++) {\n-\t\tif (!adapter-\u003erx_buff_pool[i].active)\n-\t\t\tcontinue;\n-\t\tif (ibmveth_alloc_buffer_pool(\u0026adapter-\u003erx_buff_pool[i])) {\n-\t\t\tnetdev_err(netdev, \"unable to alloc pool\\n\");\n-\t\t\tadapter-\u003erx_buff_pool[i].active = 0;\n-\t\t\trc = -ENOMEM;\n-\t\t\tgoto out_free_buffer_pools;\n-\t\t}\n+\t/*\n+\t * Post buffers before setup_rx_interrupts(). MQ setup then unmasks\n+\t * PHYP; SQ setup leaves PHYP masked. Scheduling NAPI only when a\n+\t * descriptor is already pending is not enough: after ifdown/up\n+\t * (RX=8, no -L) NAPI can be idle with nothing pending and the\n+\t * queue stays dead (TX OK, ARP/RX fail).\n+\t * restart_rx_queue() replenishes, schedules NAPI, and unmasks if\n+\t * prep fails.\n+\t */\n+\tfor (i = 0; i \u003c ibmveth_get_num_rx_queues(adapter); i++) {\n+\t\tnetdev_dbg(netdev, \"initial replenish cycle for queue %d\\n\", i);\n+\t\tibmveth_replenish_task(adapter, i);\n \t}\n \n-\tnetdev_dbg(netdev, \"registering irq 0x%x\\n\", netdev-\u003eirq);\n-\trc = request_irq(netdev-\u003eirq, ibmveth_interrupt, 0, netdev-\u003ename,\n-\t\t\t netdev);\n-\tif (rc != 0) {\n-\t\tnetdev_err(netdev, \"unable to request irq 0x%x, rc %d\\n\",\n-\t\t\t   netdev-\u003eirq, rc);\n-\t\tdo {\n-\t\t\tlpar_rc = h_free_logical_lan(adapter-\u003evdev-\u003eunit_address);\n-\t\t} while (H_IS_LONG_BUSY(lpar_rc) || (lpar_rc == H_BUSY));\n-\n-\t\tgoto out_free_buffer_pools;\n-\t}\n+\trc = ibmveth_setup_rx_interrupts(adapter);\n+\tif (rc)\n+\t\tgoto out_free_all_queues; /* setup already disposed IRQs */\n \n-\trc = -ENOMEM;\n+\tfor (i = 0; i \u003c ibmveth_get_num_rx_queues(adapter); i++)\n+\t\tibmveth_restart_rx_queue(adapter, i);\n \n-\tnetdev_dbg(netdev, \"initial replenish cycle\\n\");\n-\tibmveth_interrupt(netdev-\u003eirq, netdev);\n+\trc = ibmveth_alloc_tx_resources(adapter);\n+\tif (rc)\n+\t\tgoto out_cleanup_rx_interrupts;\n \n \tnetif_tx_start_all_queues(netdev);\n \n+\tadapter-\u003eopened = true;\n \tnetdev_dbg(netdev, \"open complete\\n\");\n \n \treturn 0;\n \n+out_cleanup_rx_interrupts:\n+\tibmveth_cleanup_rx_interrupts(adapter);\n+\t/* As in close(): a poll past napi_complete_done() may still run. */\n+\tsynchronize_net();\n+\tgoto out_free_all_queues; /* cleanup already disposed IRQs */\n+out_unregister_queues:\n+\tibmveth_dispose_subordinate_irq_mappings(adapter);\n+out_free_all_queues:\n+\tibmveth_free_all_queues(adapter);\n out_free_buffer_pools:\n-\twhile (--i \u003e= 0) {\n-\t\tif (adapter-\u003erx_buff_pool[i].active)\n-\t\t\tibmveth_free_buffer_pool(adapter,\n-\t\t\t\t\t\t \u0026adapter-\u003erx_buff_pool[i]);\n-\t}\n-out_unmap_filter_list:\n-\tdma_unmap_single(dev, adapter-\u003efilter_list_dma, 4096,\n-\t\t\t DMA_BIDIRECTIONAL);\n-\n-out_free_tx_ltb:\n-\twhile (--i \u003e= 0) {\n-\t\tibmveth_free_tx_ltb(adapter, i);\n-\t}\n-\n-out_unmap_buffer_list:\n-\tdma_unmap_single(dev, adapter-\u003ebuffer_list_dma, 4096,\n-\t\t\t DMA_BIDIRECTIONAL);\n+\tibmveth_free_buffer_pools(adapter);\n out_free_queue_mem:\n-\tdma_free_coherent(dev, adapter-\u003erx_queue.queue_len,\n-\t\t\t  adapter-\u003erx_queue.queue_addr,\n-\t\t\t  adapter-\u003erx_queue.queue_dma);\n+\tibmveth_cleanup_rx_resources(adapter);\n out_free_filter_list:\n-\tfree_page((unsigned long)adapter-\u003efilter_list_addr);\n-out_free_buffer_list:\n-\tfree_page((unsigned long)adapter-\u003ebuffer_list_addr);\n+\tibmveth_free_filter_list(adapter);\n out:\n-\tnapi_disable(\u0026adapter-\u003enapi);\n \treturn rc;\n }\n \n static int ibmveth_close(struct net_device *netdev)\n {\n \tstruct ibmveth_adapter *adapter = netdev_priv(netdev);\n-\tstruct device *dev = \u0026adapter-\u003evdev-\u003edev;\n-\tlong lpar_rc;\n \tint i;\n \n-\tnetdev_dbg(netdev, \"close starting\\n\");\n-\n-\tnapi_disable(\u0026adapter-\u003enapi);\n-\n-\tnetif_tx_stop_all_queues(netdev);\n-\n-\th_vio_signal(adapter-\u003evdev-\u003eunit_address, VIO_IRQ_DISABLE);\n-\n-\tdo {\n-\t\tlpar_rc = h_free_logical_lan(adapter-\u003evdev-\u003eunit_address);\n-\t} while (H_IS_LONG_BUSY(lpar_rc) || (lpar_rc == H_BUSY));\n-\n-\tif (lpar_rc != H_SUCCESS) {\n-\t\tnetdev_err(netdev, \"h_free_logical_lan failed with %lx, \"\n-\t\t\t   \"continuing with close\\n\", lpar_rc);\n-\t}\n-\n-\tfree_irq(netdev-\u003eirq, netdev);\n+\t/* Gate on opened, not IFF_UP: pool_store/change_mtu close+open can\n+\t * leave IFF_UP set after a failed reopen.\n+\t */\n+\tif (!adapter-\u003eopened)\n+\t\treturn 0;\n \n-\tibmveth_update_rx_no_buffer(adapter);\n+\tadapter-\u003eopened = false;\n \n-\tdma_unmap_single(dev, adapter-\u003ebuffer_list_dma, 4096,\n-\t\t\t DMA_BIDIRECTIONAL);\n-\tfree_page((unsigned long)adapter-\u003ebuffer_list_addr);\n+\tnetdev_dbg(netdev, \"close starting\\n\");\n \n-\tdma_unmap_single(dev, adapter-\u003efilter_list_dma, 4096,\n-\t\t\t DMA_BIDIRECTIONAL);\n-\tfree_page((unsigned long)adapter-\u003efilter_list_addr);\n+\t/* Disable and wait for in-flight ndo_start_xmit (stop_all_queues\n+\t * alone does not). Direct close() callers bypass synchronize_net().\n+\t */\n+\tnetif_tx_disable(netdev);\n \n-\tdma_free_coherent(dev, adapter-\u003erx_queue.queue_len,\n-\t\t\t  adapter-\u003erx_queue.queue_addr,\n-\t\t\t  adapter-\u003erx_queue.queue_dma);\n+\tibmveth_cleanup_rx_interrupts(adapter);\n+\t/* Wait for softirq/poll that already passed shutdown checks. */\n+\tsynchronize_net();\n \n-\tfor (i = 0; i \u003c IBMVETH_NUM_BUFF_POOLS; i++)\n-\t\tif (adapter-\u003erx_buff_pool[i].active)\n-\t\t\tibmveth_free_buffer_pool(adapter,\n-\t\t\t\t\t\t \u0026adapter-\u003erx_buff_pool[i]);\n+\tfor (i = 0; i \u003c ibmveth_get_num_rx_queues(adapter); i++) {\n+\t\tstruct ibmveth_rx_q *rxq = \u0026adapter-\u003erx_queue[i];\n+\t\tunsigned long flags;\n \n-\tfor (i = 0; i \u003c netdev-\u003ereal_num_tx_queues; i++)\n-\t\tibmveth_free_tx_ltb(adapter, i);\n+\t\tspin_lock_irqsave(\u0026rxq-\u003ereplenish_lock, flags);\n+\t\tibmveth_update_rx_no_buffer(adapter, i);\n+\t\tspin_unlock_irqrestore(\u0026rxq-\u003ereplenish_lock, flags);\n+\t}\n+\tif (!ibmveth_free_all_queues(adapter))\n+\t\tibmveth_free_stranded_rx_queues(adapter);\n+\t/* Free TX LTBs after quiesce and after H_FREE_LOGICAL_LAN so xmit\n+\t * cannot touch unmapped bounce buffers while the LAN is live.\n+\t */\n+\tibmveth_free_tx_resources(adapter);\n+\tibmveth_free_buffer_pools(adapter);\n+\tibmveth_cleanup_rx_resources(adapter);\n+\tibmveth_free_filter_list(adapter);\n \n \tnetdev_dbg(netdev, \"close complete\\n\");\n \n@@ -840,6 +3037,10 @@ static void ibmveth_reset(struct work_struct *w)\n \tnetdev_dbg(netdev, \"reset starting\\n\");\n \n \trtnl_lock();\n+\tif (netdev-\u003ereg_state != NETREG_REGISTERED) {\n+\t\trtnl_unlock();\n+\t\treturn;\n+\t}\n \n \tdev_close(adapter-\u003enetdev);\n \tdev_open(adapter-\u003enetdev, NULL);\n@@ -1073,22 +3274,96 @@ static int ibmveth_set_features(struct net_device *dev,\n \treturn rc1 ? rc1 : rc2;\n }\n \n-static void ibmveth_get_strings(struct net_device *dev, u32 stringset, u8 *data)\n+/*\n+ * Sum per-queue counters for rare ethtool reads. The hot paths only ever\n+ * touch their own queue's slot, so nothing here needs an atomic; the cost\n+ * of aggregation is paid by the reader instead (ibmvnic-style).\n+ *\n+ * Every slot is summed, not just the live ones, so that shrinking the\n+ * queue count with ethtool -L cannot make a counter go backwards.\n+ */\n+static u64 ibmveth_sum_rx_qstat(struct ibmveth_adapter *adapter, size_t off)\n+{\n+\tu64 total = 0;\n+\tint i;\n+\n+\tfor (i = 0; i \u003c IBMVETH_MAX_RX_QUEUES; i++)\n+\t\ttotal += *(u64 *)((u8 *)\u0026adapter-\u003erx_qstats[i] + off);\n+\n+\treturn total;\n+}\n+\n+static u64 ibmveth_sum_tx_qstat(struct ibmveth_adapter *adapter, size_t off)\n {\n+\tu64 total = 0;\n \tint i;\n \n+\tfor (i = 0; i \u003c IBMVETH_MAX_QUEUES; i++)\n+\t\ttotal += *(u64 *)((u8 *)\u0026adapter-\u003etx_qstats[i] + off);\n+\n+\treturn total;\n+}\n+\n+static u64 ibmveth_ethtool_adapter_stat(struct ibmveth_adapter *adapter,\n+\t\t\t\t\tint index)\n+{\n+\tconst struct ibmveth_stat *stat = \u0026ibmveth_stats[index];\n+\n+\tswitch (stat-\u003esrc) {\n+\tcase IBMVETH_STAT_RX_QSUM:\n+\t\treturn ibmveth_sum_rx_qstat(adapter, stat-\u003eoff);\n+\tcase IBMVETH_STAT_TX_QSUM:\n+\t\treturn ibmveth_sum_tx_qstat(adapter, stat-\u003eoff);\n+\tcase IBMVETH_STAT_RX_NO_BUFFER:\n+\t\t/*\n+\t\t * PHYP's page counter is absolute for the page currently\n+\t\t * mapped, so a reopen or queue reuse restarts it near zero.\n+\t\t * ibmveth_update_rx_no_buffer() folds each decrease into the\n+\t\t * queue's retired carry; add both back to stay monotonic.\n+\t\t */\n+\t\treturn ibmveth_sum_rx_qstat(adapter, stat-\u003eoff) +\n+\t\t       ibmveth_sum_rx_qstat(adapter,\n+\t\t\t\t\t    IBMVETH_RXQ_OFF(no_buffer_retired));\n+\tcase IBMVETH_STAT_ADAPTER:\n+\t\tbreak;\n+\t}\n+\n+\treturn IBMVETH_GET_STAT(adapter, stat-\u003eoff);\n+}\n+\n+static void ibmveth_get_strings(struct net_device *dev, u32 stringset, u8 *data)\n+{\n+\tstruct ibmveth_adapter *adapter = netdev_priv(dev);\n+\tu8 *p = data;\n+\tint i, j;\n+\n \tif (stringset != ETH_SS_STATS)\n \t\treturn;\n \n-\tfor (i = 0; i \u003c ARRAY_SIZE(ibmveth_stats); i++, data += ETH_GSTRING_LEN)\n-\t\tmemcpy(data, ibmveth_stats[i].name, ETH_GSTRING_LEN);\n+\tfor (i = 0; i \u003c ARRAY_SIZE(ibmveth_stats); i++) {\n+\t\tmemcpy(p, ibmveth_stats[i].name, ETH_GSTRING_LEN);\n+\t\tp += ETH_GSTRING_LEN;\n+\t}\n+\n+\tfor (i = 0; i \u003c ibmveth_get_num_rx_queues(adapter); i++)\n+\t\tfor (j = 0; j \u003c IBMVETH_NUM_RX_QSTATS; j++)\n+\t\t\tethtool_sprintf(\u0026p, ibmveth_rx_qstat_keys[j].fmt, i);\n+\n+\tfor (i = 0; i \u003c dev-\u003ereal_num_tx_queues; i++)\n+\t\tfor (j = 0; j \u003c IBMVETH_NUM_TX_QSTATS; j++)\n+\t\t\tethtool_sprintf(\u0026p, ibmveth_tx_qstat_keys[j].fmt, i);\n }\n \n static int ibmveth_get_sset_count(struct net_device *dev, int sset)\n {\n+\tstruct ibmveth_adapter *adapter = netdev_priv(dev);\n+\n \tswitch (sset) {\n \tcase ETH_SS_STATS:\n-\t\treturn ARRAY_SIZE(ibmveth_stats);\n+\t\treturn ARRAY_SIZE(ibmveth_stats) +\n+\t\t       ibmveth_get_num_rx_queues(adapter) *\n+\t\t       IBMVETH_NUM_RX_QSTATS +\n+\t\t       dev-\u003ereal_num_tx_queues * IBMVETH_NUM_TX_QSTATS;\n \tdefault:\n \t\treturn -EOPNOTSUPP;\n \t}\n@@ -1097,44 +3372,211 @@ static int ibmveth_get_sset_count(struct net_device *dev, int sset)\n static void ibmveth_get_ethtool_stats(struct net_device *dev,\n \t\t\t\t      struct ethtool_stats *stats, u64 *data)\n {\n-\tint i;\n \tstruct ibmveth_adapter *adapter = netdev_priv(dev);\n+\tint i, j, k;\n \n \tfor (i = 0; i \u003c ARRAY_SIZE(ibmveth_stats); i++)\n-\t\tdata[i] = IBMVETH_GET_STAT(adapter, ibmveth_stats[i].offset);\n+\t\tdata[i] = ibmveth_ethtool_adapter_stat(adapter, i);\n+\n+\tfor (j = 0; j \u003c ibmveth_get_num_rx_queues(adapter); j++) {\n+\t\tconst u8 *q = (const u8 *)\u0026adapter-\u003erx_qstats[j];\n+\n+\t\tfor (k = 0; k \u003c IBMVETH_NUM_RX_QSTATS; k++)\n+\t\t\tdata[i++] = *(const u64 *)\n+\t\t\t\t(q + ibmveth_rx_qstat_keys[k].off);\n+\t}\n+\n+\tfor (j = 0; j \u003c dev-\u003ereal_num_tx_queues; j++) {\n+\t\tconst u8 *q = (const u8 *)\u0026adapter-\u003etx_qstats[j];\n+\n+\t\tfor (k = 0; k \u003c IBMVETH_NUM_TX_QSTATS; k++)\n+\t\t\tdata[i++] = *(const u64 *)\n+\t\t\t\t(q + ibmveth_tx_qstat_keys[k].off);\n+\t}\n }\n \n static void ibmveth_get_channels(struct net_device *netdev,\n \t\t\t\t struct ethtool_channels *channels)\n {\n-\tchannels-\u003emax_tx = ibmveth_real_max_tx_queues();\n+\tstruct ibmveth_adapter *adapter = netdev_priv(netdev);\n+\tunsigned int rx_count = ibmveth_get_num_rx_queues(adapter);\n+\n \tchannels-\u003etx_count = netdev-\u003ereal_num_tx_queues;\n+\t/*\n+\t * Never advertise max_tx below the live count. After CPU\n+\t * offline, real_max can drop below real_num_tx_queues;\n+\t * ethtool -L is read-modify-write and the core rejects\n+\t * tx_count \u003e max_tx, which would also block an RX-only\n+\t * request. set_channels uses this same ceiling.\n+\t */\n+\tchannels-\u003emax_tx = max_t(unsigned int, channels-\u003etx_count,\n+\t\t\t\t ibmveth_real_max_tx_queues());\n+\n+\t/*\n+\t * Always report the live RX count. ethtool -L is read-modify-\n+\t * write, so a TX-only request echoes rx_count back at us; an\n+\t * understated value would be applied as a silent RX shrink.\n+\t * mq_fallback instead caps max_rx at the live count, which\n+\t * blocks growth in the core without misreporting what is\n+\t * currently configured.\n+\t */\n+\tchannels-\u003erx_count = rx_count;\n+\tif (adapter-\u003emulti_queue \u0026\u0026 !adapter-\u003emq_fallback)\n+\t\tchannels-\u003emax_rx = IBMVETH_MAX_RX_QUEUES;\n+\telse\n+\t\tchannels-\u003emax_rx = rx_count;\n+}\n+\n+/**\n+ * ibmveth_resize_rx_channels - Validate and apply a new RX queue count\n+ * @adapter: ibmveth adapter\n+ * @goal_rx: requested RX queue count\n+ *\n+ * Rejects rx \u003e 1 without MQ firmware (-EOPNOTSUPP) and rx outside\n+ * 1..IBMVETH_MAX_RX_QUEUES (-EINVAL). When RX resources are not live\n+ * (!opened), only validate; do not allocate. When up, apply via\n+ * ibmveth_resize_rx_queues_incremental().\n+ *\n+ * Return: 0 or negative errno\n+ */\n+static int ibmveth_resize_rx_channels(struct ibmveth_adapter *adapter,\n+\t\t\t\t      unsigned int goal_rx)\n+{\n+\tstruct net_device *netdev = adapter-\u003enetdev;\n+\tunsigned int old_rx = ibmveth_get_num_rx_queues(adapter);\n+\tint rxq_entries;\n+\tint rc;\n+\n+\tif (goal_rx \u003c 1 || goal_rx \u003e IBMVETH_MAX_RX_QUEUES) {\n+\t\tnetdev_err(netdev,\n+\t\t\t   \"Invalid RX queue count %u (must be 1-%d)\\n\",\n+\t\t\t   goal_rx, IBMVETH_MAX_RX_QUEUES);\n+\t\treturn -EINVAL;\n+\t}\n+\n+\t/*\n+\t * Check for a no-op before the capability gate. ethtool -L is\n+\t * read-modify-write, so a TX-only request arrives carrying the\n+\t * current RX count; gating first would fail those with\n+\t * -EOPNOTSUPP once mq_fallback is set.\n+\t */\n+\tif (goal_rx == old_rx)\n+\t\treturn 0;\n+\n+\t/*\n+\t * Refuse any rx \u003e 1, not just growth: once mq_fallback is set the\n+\t * next open comes up single-queue, so an intermediate count could\n+\t * not be honoured either, and accepting it would only repeat the\n+\t * silent clamp at open. max_rx stays at the live count so that\n+\t * read-modify-write TX-only requests still clear the core.\n+\t */\n+\tif (goal_rx \u003e 1 \u0026\u0026 (!adapter-\u003emulti_queue || adapter-\u003emq_fallback)) {\n+\t\tnetdev_err(netdev,\n+\t\t\t   \"Cannot resize to %u RX queues: multi-queue mode not supported by firmware\\n\",\n+\t\t\t   goal_rx);\n+\t\treturn -EOPNOTSUPP;\n+\t}\n \n-\tchannels-\u003emax_rx = netdev-\u003ereal_num_rx_queues;\n-\tchannels-\u003erx_count = netdev-\u003ereal_num_rx_queues;\n+\t/* Down / failed-open: do not allocate. */\n+\tif (!adapter-\u003eopened)\n+\t\treturn 0;\n+\n+\trxq_entries = adapter-\u003erx_queue[0].num_slots;\n+\trc = ibmveth_resize_rx_queues_incremental(adapter, goal_rx,\n+\t\t\t\t\t\t  rxq_entries);\n+\tif (rc)\n+\t\tnetdev_err(netdev, \"Failed to resize RX queues: %d\\n\", rc);\n+\treturn rc;\n }\n \n static int ibmveth_set_channels(struct net_device *netdev,\n \t\t\t\tstruct ethtool_channels *channels)\n {\n \tstruct ibmveth_adapter *adapter = netdev_priv(netdev);\n-\tunsigned int old = netdev-\u003ereal_num_tx_queues,\n-\t\t     goal = channels-\u003etx_count;\n+\tunsigned int old_rx = ibmveth_get_num_rx_queues(adapter);\n+\tunsigned int goal_rx = channels-\u003erx_count;\n+\tunsigned int old_tx = netdev-\u003ereal_num_tx_queues;\n+\tunsigned int goal_tx = channels-\u003etx_count;\n+\tunsigned int want_tx = goal_tx;\n+\tunsigned int max_tx;\n+\tbool rx_changed = false;\n \tint rc, i;\n \n-\t/* If ndo_open has not been called yet then don't allocate, just set\n-\t * desired netdev_queue's and return\n+\t/*\n+\t * Same ceiling get_channels reports: live count or the\n+\t * online-CPU cap, whichever is larger.\n+\t */\n+\tmax_tx = max_t(unsigned int, old_tx,\n+\t\t       ibmveth_real_max_tx_queues());\n+\tif (goal_tx \u003c 1 || goal_tx \u003e max_tx) {\n+\t\tnetdev_err(netdev,\n+\t\t\t   \"Invalid TX queue count %u (must be 1-%u)\\n\",\n+\t\t\t   goal_tx, max_tx);\n+\t\treturn -EINVAL;\n+\t}\n+\n+\t/* RX range / MQ checks live in ibmveth_resize_rx_channels(). */\n+\trc = ibmveth_resize_rx_channels(adapter, goal_rx);\n+\tif (rc)\n+\t\treturn rc;\n+\n+\t/* If RX resources are not live (never opened, or close+open failed\n+\t * while IFF_UP stayed set), publish desired queue counts without\n+\t * allocating.\n \t */\n-\tif (!(netdev-\u003eflags \u0026 IFF_UP))\n-\t\treturn netif_set_real_num_tx_queues(netdev, goal);\n+\tif (!adapter-\u003eopened) {\n+\t\t/* Apply TX first so a failure leaves the published RX\n+\t\t * count unchanged.\n+\t\t */\n+\t\trc = netif_set_real_num_tx_queues(netdev, goal_tx);\n+\t\tif (rc)\n+\t\t\treturn rc;\n+\n+\t\t/* Publish desired RX count for next open() and refresh CMO;\n+\t\t * do not allocate while down.\n+\t\t */\n+\t\tif (goal_rx != ibmveth_get_num_rx_queues(adapter)) {\n+\t\t\tibmveth_publish_num_rx_queues(adapter, goal_rx);\n+\t\t\trc = netif_set_real_num_rx_queues(netdev, goal_rx);\n+\t\t\tif (rc) {\n+\t\t\t\tint tx_rc;\n+\n+\t\t\t\tibmveth_publish_num_rx_queues(adapter, old_rx);\n+\t\t\t\ttx_rc = netif_set_real_num_tx_queues(netdev,\n+\t\t\t\t\t\t\t\t     old_tx);\n+\t\t\t\tif (tx_rc)\n+\t\t\t\t\tnetdev_err(netdev,\n+\t\t\t\t\t\t   \"Failed to restore TX queues to %u after RX failure: %d\\n\",\n+\t\t\t\t\t\t   old_tx, tx_rc);\n+\t\t\t\treturn rc;\n+\t\t\t}\n+\t\t\tif (firmware_has_feature(FW_FEATURE_CMO)) {\n+\t\t\t\tunsigned long dma;\n+\n+\t\t\t\tdma = ibmveth_get_desired_dma(adapter-\u003evdev);\n+\t\t\t\tvio_cmo_set_dev_desired(adapter-\u003evdev, dma);\n+\t\t\t}\n+\t\t}\n+\t\treturn 0;\n+\t}\n+\n+\tif (goal_rx != old_rx)\n+\t\trx_changed = true;\n \n \t/* We have IBMVETH_MAX_QUEUES netdev_queue's allocated\n \t * but we may need to alloc/free the ltb's.\n \t */\n+\tif (goal_tx == old_tx)\n+\t\treturn 0;\n+\n \tnetif_tx_stop_all_queues(netdev);\n \n-\t/* Allocate any queue that we need */\n-\tfor (i = old; i \u003c goal; i++) {\n+\t/* Allocate any new TX LTBs. i starts at old_tx for the free walk\n+\t * below when this loop body never runs (goal_tx == old_tx already\n+\t * returned; goal_tx \u003c old_tx is scale-down).\n+\t */\n+\ti = old_tx;\n+\tfor (; i \u003c goal_tx; i++) {\n \t\tif (adapter-\u003etx_ltb_ptr[i])\n \t\t\tcontinue;\n \n@@ -1143,28 +3585,48 @@ static int ibmveth_set_channels(struct net_device *netdev,\n \t\t\tcontinue;\n \n \t\t/* if something goes wrong, free everything we just allocated */\n-\t\tnetdev_err(netdev, \"Failed to allocate more tx queues, returning to %d queues\\n\",\n-\t\t\t   old);\n-\t\tgoal = old;\n-\t\told = i;\n+\t\tnetdev_err(netdev, \"Failed to allocate more tx queues, returning to %u queues\\n\",\n+\t\t\t   old_tx);\n+\t\tgoal_tx = old_tx;\n+\t\told_tx = i;\n \t\tbreak;\n \t}\n-\trc = netif_set_real_num_tx_queues(netdev, goal);\n+\trc = netif_set_real_num_tx_queues(netdev, goal_tx);\n \tif (rc) {\n-\t\tnetdev_err(netdev, \"Failed to set real tx queues, returning to %d queues\\n\",\n-\t\t\t   old);\n-\t\tgoal = old;\n-\t\told = i;\n+\t\tnetdev_err(netdev, \"Failed to set real tx queues, returning to %u queues\\n\",\n+\t\t\t   old_tx);\n+\t\tgoal_tx = old_tx;\n+\t\told_tx = i;\n \t}\n \t/* Free any that are no longer needed */\n-\tfor (i = old; i \u003e goal; i--) {\n+\tfor (i = old_tx; i \u003e goal_tx; i--) {\n \t\tif (adapter-\u003etx_ltb_ptr[i - 1])\n \t\t\tibmveth_free_tx_ltb(adapter, i - 1);\n \t}\n \n \tnetif_tx_wake_all_queues(netdev);\n \n-\treturn rc;\n+\tif (netdev-\u003ereal_num_tx_queues != want_tx) {\n+\t\tif (rx_changed) {\n+\t\t\t/*\n+\t\t\t * Restore the RX count from before this -L.\n+\t\t\t * num_slots is the live size after that resize.\n+\t\t\t */\n+\t\t\tint rxq_entries = adapter-\u003erx_queue[0].num_slots;\n+\t\t\tint rb;\n+\n+\t\t\trb = ibmveth_resize_rx_queues_incremental(adapter,\n+\t\t\t\t\t\t\t\t  old_rx,\n+\t\t\t\t\t\t\t\t  rxq_entries);\n+\t\t\tif (rb)\n+\t\t\t\tnetdev_err(netdev,\n+\t\t\t\t\t   \"Failed to roll back RX queues to %u after TX failure: %d\\n\",\n+\t\t\t\t\t   old_rx, rb);\n+\t\t}\n+\t\treturn rc ? rc : -ENOMEM;\n+\t}\n+\n+\treturn 0;\n }\n \n static const struct ethtool_ops netdev_ethtool_ops = {\n@@ -1213,8 +3675,10 @@ static int ibmveth_send(struct ibmveth_adapter *adapter,\n }\n \n static int ibmveth_is_packet_unsupported(struct sk_buff *skb,\n-\t\t\t\t\t struct net_device *netdev)\n+\t\t\t\t\t struct ibmveth_adapter *adapter,\n+\t\t\t\t\t int queue_num)\n {\n+\tstruct net_device *netdev = adapter-\u003enetdev;\n \tstruct ethhdr *ether_header;\n \tint ret = 0;\n \n@@ -1222,7 +3686,7 @@ static int ibmveth_is_packet_unsupported(struct sk_buff *skb,\n \n \tif (ether_addr_equal(ether_header-\u003eh_dest, netdev-\u003edev_addr)) {\n \t\tnetdev_dbg(netdev, \"veth doesn't support loopback packets, dropping packet.\\n\");\n-\t\tnetdev-\u003estats.tx_dropped++;\n+\t\tadapter-\u003etx_qstats[queue_num].dropped_packets++;\n \t\tret = -EOPNOTSUPP;\n \t}\n \n@@ -1238,7 +3702,13 @@ static netdev_tx_t ibmveth_start_xmit(struct sk_buff *skb,\n \tint i, queue_num = skb_get_queue_mapping(skb);\n \tunsigned long mss = 0;\n \n-\tif (ibmveth_is_packet_unsupported(skb, netdev))\n+\t/* Close / failed reopen can free LTBs while IFF_UP is still set. */\n+\tif (unlikely(!adapter-\u003etx_ltb_ptr[queue_num])) {\n+\t\tadapter-\u003etx_qstats[queue_num].dropped_packets++;\n+\t\tgoto out;\n+\t}\n+\n+\tif (ibmveth_is_packet_unsupported(skb, adapter, queue_num))\n \t\tgoto out;\n \t/* veth can't checksum offload UDP */\n \tif (skb-\u003eip_summed == CHECKSUM_PARTIAL \u0026\u0026\n@@ -1249,7 +3719,7 @@ static netdev_tx_t ibmveth_start_xmit(struct sk_buff *skb,\n \t    skb_checksum_help(skb)) {\n \n \t\tnetdev_err(netdev, \"tx: failed to checksum packet\\n\");\n-\t\tnetdev-\u003estats.tx_dropped++;\n+\t\tadapter-\u003etx_qstats[queue_num].dropped_packets++;\n \t\tgoto out;\n \t}\n \n@@ -1261,6 +3731,8 @@ static netdev_tx_t ibmveth_start_xmit(struct sk_buff *skb,\n \n \t\tdesc_flags |= (IBMVETH_BUF_NO_CSUM | IBMVETH_BUF_CSUM_GOOD);\n \n+\t\tadapter-\u003etx_qstats[queue_num].checksum_offload++;\n+\n \t\t/* Need to zero out the checksum */\n \t\tbuf[0] = 0;\n \t\tbuf[1] = 0;\n@@ -1272,7 +3744,7 @@ static netdev_tx_t ibmveth_start_xmit(struct sk_buff *skb,\n \tif (skb-\u003eip_summed == CHECKSUM_PARTIAL \u0026\u0026 skb_is_gso(skb)) {\n \t\tif (adapter-\u003efw_large_send_support) {\n \t\t\tmss = (unsigned long)skb_shinfo(skb)-\u003egso_size;\n-\t\t\tadapter-\u003etx_large_packets++;\n+\t\t\tadapter-\u003etx_qstats[queue_num].large_packets++;\n \t\t} else if (!skb_is_gso_v6(skb)) {\n \t\t\t/* Put -1 in the IP checksum to tell phyp it\n \t\t\t * is a largesend packet. Put the mss in\n@@ -1281,7 +3753,7 @@ static netdev_tx_t ibmveth_start_xmit(struct sk_buff *skb,\n \t\t\tip_hdr(skb)-\u003echeck = 0xffff;\n \t\t\ttcp_hdr(skb)-\u003echeck =\n \t\t\t\tcpu_to_be16(skb_shinfo(skb)-\u003egso_size);\n-\t\t\tadapter-\u003etx_large_packets++;\n+\t\t\tadapter-\u003etx_qstats[queue_num].large_packets++;\n \t\t}\n \t}\n \n@@ -1289,7 +3761,7 @@ static netdev_tx_t ibmveth_start_xmit(struct sk_buff *skb,\n \tif (unlikely(skb-\u003elen \u003e adapter-\u003etx_ltb_size)) {\n \t\tnetdev_err(adapter-\u003enetdev, \"tx: packet size (%u) exceeds ltb (%u)\\n\",\n \t\t\t   skb-\u003elen, adapter-\u003etx_ltb_size);\n-\t\tnetdev-\u003estats.tx_dropped++;\n+\t\tadapter-\u003etx_qstats[queue_num].dropped_packets++;\n \t\tgoto out;\n \t}\n \tmemcpy(adapter-\u003etx_ltb_ptr[queue_num], skb-\u003edata, skb_headlen(skb));\n@@ -1306,7 +3778,7 @@ static netdev_tx_t ibmveth_start_xmit(struct sk_buff *skb,\n \tif (unlikely(total_bytes != skb-\u003elen)) {\n \t\tnetdev_err(adapter-\u003enetdev, \"tx: incorrect packet len copied into ltb (%u != %u)\\n\",\n \t\t\t   skb-\u003elen, total_bytes);\n-\t\tnetdev-\u003estats.tx_dropped++;\n+\t\tadapter-\u003etx_qstats[queue_num].dropped_packets++;\n \t\tgoto out;\n \t}\n \tdesc.fields.flags_len = desc_flags | skb-\u003elen;\n@@ -1315,11 +3787,11 @@ static netdev_tx_t ibmveth_start_xmit(struct sk_buff *skb,\n \tdma_wmb();\n \n \tif (ibmveth_send(adapter, desc.desc, mss)) {\n-\t\tadapter-\u003etx_send_failed++;\n-\t\tnetdev-\u003estats.tx_dropped++;\n+\t\tadapter-\u003etx_qstats[queue_num].send_failures++;\n+\t\tadapter-\u003etx_qstats[queue_num].dropped_packets++;\n \t} else {\n-\t\tnetdev-\u003estats.tx_packets++;\n-\t\tnetdev-\u003estats.tx_bytes += skb-\u003elen;\n+\t\tadapter-\u003etx_qstats[queue_num].packets++;\n+\t\tadapter-\u003etx_qstats[queue_num].bytes += skb-\u003elen;\n \t}\n \n out:\n@@ -1445,102 +3917,226 @@ static void ibmveth_rx_csum_helper(struct sk_buff *skb,\n \t}\n }\n \n-static int ibmveth_poll(struct napi_struct *napi, int budget)\n+static void ibmveth_poll_bump_invalid(struct ibmveth_adapter *adapter,\n+\t\t\t\t      int queue_index)\n {\n-\tstruct ibmveth_adapter *adapter =\n-\t\t\tcontainer_of(napi, struct ibmveth_adapter, napi);\n-\tstruct net_device *netdev = adapter-\u003enetdev;\n-\tint frames_processed = 0;\n-\tunsigned long lpar_rc;\n+\tadapter-\u003erx_qstats[queue_index].invalid_buffers++;\n+}\n+\n+static bool ibmveth_poll_stopping(struct net_device *netdev,\n+\t\t\t\t  struct napi_struct *napi)\n+{\n+\treturn !netif_running(netdev) || napi_disable_pending(napi);\n+}\n+\n+static bool ibmveth_poll_harvest_slot(struct ibmveth_adapter *adapter,\n+\t\t\t\t      int queue_index, bool reuse)\n+{\n+\tint rc = ibmveth_rxq_harvest_buffer(adapter, queue_index, reuse);\n+\n+\treturn !rc || rc == -EINVAL || rc == -EFAULT;\n+}\n+\n+static bool ibmveth_poll_recycle_invalid(struct net_device *netdev,\n+\t\t\t\t\t struct ibmveth_adapter *adapter,\n+\t\t\t\t\t int queue_index)\n+{\n+\tnetdev_dbg(netdev, \"recycling invalid buffer\\n\");\n+\tibmveth_poll_bump_invalid(adapter, queue_index);\n+\treturn ibmveth_poll_harvest_slot(adapter, queue_index, true);\n+}\n+\n+static bool ibmveth_poll_skip_bad_correlator(struct net_device *netdev,\n+\t\t\t\t\t     struct ibmveth_adapter *adapter,\n+\t\t\t\t\t     int queue_index)\n+{\n+\tif (net_ratelimit())\n+\t\tnetdev_err(netdev,\n+\t\t\t   \"bad correlator on queue %d, skipping slot\\n\",\n+\t\t\t   queue_index);\n+\t/* Residual stale slot after resize: recover via reset rather\n+\t * than spinning forever. Always escalate; only the log is\n+\t * rate-limited.\n+\t */\n+\tschedule_work(\u0026adapter-\u003ework);\n+\tibmveth_poll_bump_invalid(adapter, queue_index);\n+\treturn ibmveth_poll_harvest_slot(adapter, queue_index, true);\n+}\n+\n+static bool ibmveth_poll_drop_oversize(struct net_device *netdev,\n+\t\t\t\t       struct ibmveth_adapter *adapter,\n+\t\t\t\t     int queue_index, unsigned int off,\n+\t\t\t\t     unsigned int len, unsigned int room)\n+{\n+\tif (net_ratelimit())\n+\t\tnetdev_err(netdev,\n+\t\t\t   \"RX frame %u+%u exceeds buffer %u on queue %d, dropping\\n\",\n+\t\t\t   off, len, room, queue_index);\n+\tibmveth_poll_bump_invalid(adapter, queue_index);\n+\treturn ibmveth_poll_harvest_slot(adapter, queue_index, true);\n+}\n+\n+/**\n+ * ibmveth_poll_deliver_frame - Build SKB from one valid RX slot and GRO it\n+ * @napi: NAPI context for this RX queue\n+ * @adapter: ibmveth adapter\n+ * @netdev: net_device for @adapter\n+ * @queue_index: RX queue index\n+ *\n+ * Return: 1 frame delivered, 0 if the slot was skipped cleanly, -1 on error.\n+ */\n+static int ibmveth_poll_deliver_frame(struct napi_struct *napi,\n+\t\t\t\t      struct ibmveth_adapter *adapter,\n+\t\t\t\t      struct net_device *netdev,\n+\t\t\t\t      int queue_index)\n+{\n+\tstruct sk_buff *skb, *new_skb;\n+\tstruct ibmveth_rx_q *rxq = \u0026adapter-\u003erx_queue[queue_index];\n+\tunsigned int room, off, len, pool;\n+\tint length, offset, csum_good, lrg_pkt;\n+\t__sum16 iph_check = 0;\n \tu16 mss = 0;\n+\tint rc;\n \n-restart_poll:\n-\twhile (frames_processed \u003c budget) {\n-\t\tif (!ibmveth_rxq_pending_buffer(adapter))\n-\t\t\tbreak;\n+\tlength = ibmveth_rxq_frame_length(adapter, queue_index);\n+\toffset = ibmveth_rxq_frame_offset(adapter, queue_index);\n+\tcsum_good = ibmveth_rxq_csum_good(adapter, queue_index);\n+\tlrg_pkt = ibmveth_rxq_large_packet(adapter, queue_index);\n+\n+\tskb = ibmveth_rxq_get_buffer(adapter, queue_index);\n+\tif (unlikely(!skb)) {\n+\t\tif (!ibmveth_poll_skip_bad_correlator(netdev, adapter,\n+\t\t\t\t\t\t      queue_index))\n+\t\t\treturn -1;\n+\t\treturn 0;\n+\t}\n \n-\t\tsmp_rmb();\n-\t\tif (!ibmveth_rxq_buffer_valid(adapter)) {\n-\t\t\twmb(); /* suggested by larson1 */\n-\t\t\tadapter-\u003erx_invalid_buffer++;\n-\t\t\tnetdev_dbg(netdev, \"recycling invalid buffer\\n\");\n-\t\t\tif (unlikely(ibmveth_rxq_harvest_buffer(adapter, true)))\n-\t\t\t\tbreak;\n-\t\t} else {\n-\t\t\tstruct sk_buff *skb, *new_skb;\n-\t\t\tint length = ibmveth_rxq_frame_length(adapter);\n-\t\t\tint offset = ibmveth_rxq_frame_offset(adapter);\n-\t\t\tint csum_good = ibmveth_rxq_csum_good(adapter);\n-\t\t\tint lrg_pkt = ibmveth_rxq_large_packet(adapter);\n-\t\t\t__sum16 iph_check = 0;\n-\n-\t\t\tskb = ibmveth_rxq_get_buffer(adapter);\n-\t\t\tif (unlikely(!skb))\n-\t\t\t\tbreak;\n+\tpool = rxq-\u003equeue_addr[rxq-\u003eindex].correlator \u003e\u003e 32;\n+\troom = min_t(unsigned int, skb_tailroom(skb),\n+\t\t     adapter-\u003erx_buff_pool[queue_index][pool].buff_size);\n+\toff = offset;\n+\tlen = length;\n+\tif (unlikely(off \u003e= room || len \u003e room - off)) {\n+\t\tif (!ibmveth_poll_drop_oversize(netdev, adapter, queue_index,\n+\t\t\t\t\t\toff, len, room))\n+\t\t\treturn -1;\n+\t\treturn 0;\n+\t}\n \n-\t\t\t/* if the large packet bit is set in the rx queue\n-\t\t\t * descriptor, the mss will be written by PHYP eight\n-\t\t\t * bytes from the start of the rx buffer, which is\n-\t\t\t * skb-\u003edata at this stage\n-\t\t\t */\n-\t\t\tif (lrg_pkt) {\n-\t\t\t\t__be64 *rxmss = (__be64 *)(skb-\u003edata + 8);\n+\tif (lrg_pkt) {\n+\t\t__be64 *rxmss = (__be64 *)(skb-\u003edata + 8);\n \n-\t\t\t\tmss = (u16)be64_to_cpu(*rxmss);\n-\t\t\t}\n+\t\tmss = (u16)be64_to_cpu(*rxmss);\n+\t}\n \n-\t\t\tnew_skb = NULL;\n-\t\t\tif (length \u003c rx_copybreak)\n-\t\t\t\tnew_skb = netdev_alloc_skb(netdev, length);\n-\n-\t\t\tif (new_skb) {\n-\t\t\t\tskb_copy_to_linear_data(new_skb,\n-\t\t\t\t\t\t\tskb-\u003edata + offset,\n-\t\t\t\t\t\t\tlength);\n-\t\t\t\tif (rx_flush)\n-\t\t\t\t\tibmveth_flush_buffer(skb-\u003edata,\n-\t\t\t\t\t\tlength + offset);\n-\t\t\t\tif (unlikely(ibmveth_rxq_harvest_buffer(adapter, true)))\n-\t\t\t\t\tbreak;\n-\t\t\t\tskb = new_skb;\n-\t\t\t} else {\n-\t\t\t\tif (unlikely(ibmveth_rxq_harvest_buffer(adapter, false)))\n-\t\t\t\t\tbreak;\n-\t\t\t\tskb_reserve(skb, offset);\n-\t\t\t}\n+\tnew_skb = NULL;\n+\tif (length \u003c rx_copybreak)\n+\t\tnew_skb = netdev_alloc_skb(netdev, length);\n+\n+\tif (new_skb) {\n+\t\tskb_copy_to_linear_data(new_skb, skb-\u003edata + offset, length);\n+\t\tif (rx_flush)\n+\t\t\tibmveth_flush_buffer(skb-\u003edata, length + offset);\n+\t\trc = ibmveth_rxq_harvest_buffer(adapter, queue_index, true);\n+\t\tif (unlikely(rc)) {\n+\t\t\tkfree_skb(new_skb);\n+\t\t\treturn -1;\n+\t\t}\n+\t\tskb = new_skb;\n+\t} else {\n+\t\trc = ibmveth_rxq_harvest_buffer(adapter, queue_index, false);\n+\t\tif (unlikely(rc))\n+\t\t\treturn -1;\n+\t\tskb_reserve(skb, offset);\n+\t}\n \n-\t\t\tskb_put(skb, length);\n-\t\t\tskb-\u003eprotocol = eth_type_trans(skb, netdev);\n+\tskb_put(skb, length);\n+\tskb-\u003eprotocol = eth_type_trans(skb, netdev);\n \n-\t\t\t/* PHYP without PLSO support places a -1 in the ip\n-\t\t\t * checksum for large send frames.\n-\t\t\t */\n-\t\t\tif (skb-\u003eprotocol == cpu_to_be16(ETH_P_IP)) {\n-\t\t\t\tstruct iphdr *iph = (struct iphdr *)skb-\u003edata;\n+\tif (skb-\u003eprotocol == cpu_to_be16(ETH_P_IP))\n+\t\tiph_check = ((struct iphdr *)skb-\u003edata)-\u003echeck;\n \n-\t\t\t\tiph_check = iph-\u003echeck;\n-\t\t\t}\n+\tif ((length \u003e netdev-\u003emtu + ETH_HLEN) || lrg_pkt ||\n+\t    iph_check == 0xffff) {\n+\t\tibmveth_rx_mss_helper(skb, mss, lrg_pkt);\n+\t\tadapter-\u003erx_qstats[queue_index].large_packets++;\n+\t}\n \n-\t\t\tif ((length \u003e netdev-\u003emtu + ETH_HLEN) ||\n-\t\t\t    lrg_pkt || iph_check == 0xffff) {\n-\t\t\t\tibmveth_rx_mss_helper(skb, mss, lrg_pkt);\n-\t\t\t\tadapter-\u003erx_large_packets++;\n-\t\t\t}\n+\tif (csum_good) {\n+\t\tskb-\u003eip_summed = CHECKSUM_UNNECESSARY;\n+\t\tibmveth_rx_csum_helper(skb, adapter);\n+\t}\n \n-\t\t\tif (csum_good) {\n-\t\t\t\tskb-\u003eip_summed = CHECKSUM_UNNECESSARY;\n-\t\t\t\tibmveth_rx_csum_helper(skb, adapter);\n-\t\t\t}\n+\tskb_record_rx_queue(skb, queue_index);\n+\tnapi_gro_receive(napi, skb);\n+\n+\tadapter-\u003erx_qstats[queue_index].packets++;\n+\tadapter-\u003erx_qstats[queue_index].bytes += length;\n+\n+\treturn 1;\n+}\n+\n+static int ibmveth_poll(struct napi_struct *napi, int budget)\n+{\n+\tstruct net_device *netdev = napi-\u003edev;\n+\tstruct ibmveth_adapter *adapter = netdev_priv(netdev);\n+\tint frames_processed = 0;\n+\tint queue_index, rc;\n+\n+\tqueue_index = napi - adapter-\u003enapi;\n+\n+\tif (WARN_ON(queue_index \u003c 0 ||\n+\t\t    queue_index \u003e= ibmveth_get_num_rx_queues(adapter))) {\n+\t\tif (budget)\n+\t\t\tnapi_complete_done(napi, 0);\n+\t\treturn 0;\n+\t}\n+\n+\tif (ibmveth_poll_stopping(netdev, napi)) {\n+\t\tif (budget)\n+\t\t\tnapi_complete_done(napi, 0);\n+\t\treturn 0;\n+\t}\n+\n+\tadapter-\u003erx_qstats[queue_index].polls++;\n+\n+restart_poll:\n+\twhile (frames_processed \u003c budget) {\n+\t\tif (ibmveth_poll_stopping(netdev, napi))\n+\t\t\tbreak;\n \n-\t\t\tnapi_gro_receive(napi, skb);\t/* send it up */\n+\t\tif (!ibmveth_rxq_pending_buffer(adapter, queue_index))\n+\t\t\tbreak;\n \n-\t\t\tnetdev-\u003estats.rx_packets++;\n-\t\t\tnetdev-\u003estats.rx_bytes += length;\n-\t\t\tframes_processed++;\n+\t\tsmp_rmb();\n+\t\tif (!ibmveth_rxq_buffer_valid(adapter, queue_index)) {\n+\t\t\twmb(); /* suggested by larson1 */\n+\t\t\tif (!ibmveth_poll_recycle_invalid(netdev, adapter,\n+\t\t\t\t\t\t\t  queue_index))\n+\t\t\t\tbreak;\n+\t\t} else {\n+\t\t\trc = ibmveth_poll_deliver_frame(napi, adapter, netdev,\n+\t\t\t\t\t\t\tqueue_index);\n+\t\t\tif (rc \u003c 0)\n+\t\t\t\tbreak;\n+\t\t\tif (rc \u003e 0)\n+\t\t\t\tframes_processed++;\n \t\t}\n \t}\n \n-\tibmveth_replenish_task(adapter);\n+\tibmveth_replenish_task(adapter, queue_index);\n+\n+\tif (ibmveth_poll_stopping(netdev, napi)) {\n+\t\t/* budget 0 is netpoll, which must not complete NAPI.\n+\t\t * Otherwise returning budget after completing would ask\n+\t\t * NAPI to reschedule, so cap the return at budget - 1.\n+\t\t */\n+\t\tif (budget) {\n+\t\t\tnapi_complete_done(napi, frames_processed);\n+\t\t\treturn min(frames_processed, budget - 1);\n+\t\t}\n+\t\treturn 0;\n+\t}\n \n \tif (frames_processed == budget)\n \t\tgoto out;\n@@ -1548,18 +4144,27 @@ static int ibmveth_poll(struct napi_struct *napi, int budget)\n \tif (!napi_complete_done(napi, frames_processed))\n \t\tgoto out;\n \n-\t/* We think we are done - reenable interrupts,\n-\t * then check once more to make sure we are done.\n+\t/*\n+\t * napi_disable() sets DISABLE then waits for this poll. Without a\n+\t * second stopping check here, enable_irq() can re-arm PHYP after\n+\t * resize already masked the queue; late IRQs then hit the handler\n+\t * after num_rx_queues was published lower (lab WARN at interrupt).\n \t */\n-\tlpar_rc = h_vio_signal(adapter-\u003evdev-\u003eunit_address, VIO_IRQ_ENABLE);\n-\tif (WARN_ON(lpar_rc != H_SUCCESS)) {\n+\tif (ibmveth_poll_stopping(netdev, napi))\n+\t\tgoto out;\n+\n+\trc = ibmveth_enable_irq(adapter, queue_index);\n+\tif (rc) {\n+\t\tnetdev_err(netdev,\n+\t\t\t   \"Failed to enable IRQ for queue %d (rc=%d), scheduling reset\\n\",\n+\t\t\t   queue_index, rc);\n \t\tschedule_work(\u0026adapter-\u003ework);\n \t\tgoto out;\n \t}\n \n-\tif (ibmveth_rxq_pending_buffer(adapter) \u0026\u0026 napi_schedule(napi)) {\n-\t\tlpar_rc = h_vio_signal(adapter-\u003evdev-\u003eunit_address,\n-\t\t\t\t       VIO_IRQ_DISABLE);\n+\tif (ibmveth_rxq_pending_buffer(adapter, queue_index) \u0026\u0026\n+\t    napi_schedule(napi)) {\n+\t\tibmveth_disable_irq(adapter, queue_index);\n \t\tgoto restart_poll;\n \t}\n \n@@ -1569,16 +4174,23 @@ static int ibmveth_poll(struct napi_struct *napi, int budget)\n \n static irqreturn_t ibmveth_interrupt(int irq, void *dev_instance)\n {\n-\tstruct net_device *netdev = dev_instance;\n+\tstruct napi_struct *napi = dev_instance;\n+\tstruct net_device *netdev = napi-\u003edev;\n \tstruct ibmveth_adapter *adapter = netdev_priv(netdev);\n-\tunsigned long lpar_rc;\n+\tint qindex;\n \n-\tif (napi_schedule_prep(\u0026adapter-\u003enapi)) {\n-\t\tlpar_rc = h_vio_signal(adapter-\u003evdev-\u003eunit_address,\n-\t\t\t\t       VIO_IRQ_DISABLE);\n-\t\tWARN_ON(lpar_rc != H_SUCCESS);\n-\t\t__napi_schedule(\u0026adapter-\u003enapi);\n-\t}\n+\tqindex = napi - adapter-\u003enapi;\n+\t/*\n+\t * Quiet on out-of-range: scale-down publishes a lower live count\n+\t * before free_irq(). A residual IRQ must not WARN-storm; return\n+\t * IRQ_NONE until the handler is removed.\n+\t */\n+\tif (qindex \u003c 0 || qindex \u003e= ibmveth_get_num_rx_queues(adapter))\n+\t\treturn IRQ_NONE;\n+\n+\tadapter-\u003erx_qstats[qindex].interrupts++;\n+\n+\tibmveth_schedule_rx_queue(adapter, qindex);\n \treturn IRQ_HANDLED;\n }\n \n@@ -1645,7 +4257,7 @@ static int ibmveth_change_mtu(struct net_device *dev, int new_mtu)\n \tint need_restart = 0;\n \n \tfor (i = 0; i \u003c IBMVETH_NUM_BUFF_POOLS; i++)\n-\t\tif (new_mtu_oh \u003c= adapter-\u003erx_buff_pool[i].buff_size)\n+\t\tif (new_mtu_oh \u003c= adapter-\u003erx_buff_pool[0][i].buff_size)\n \t\t\tbreak;\n \n \tif (i == IBMVETH_NUM_BUFF_POOLS)\n@@ -1660,9 +4272,9 @@ static int ibmveth_change_mtu(struct net_device *dev, int new_mtu)\n \n \t/* Look for an active buffer pool that can hold the new MTU */\n \tfor (i = 0; i \u003c IBMVETH_NUM_BUFF_POOLS; i++) {\n-\t\tadapter-\u003erx_buff_pool[i].active = 1;\n+\t\tadapter-\u003erx_buff_pool[0][i].active = 1;\n \n-\t\tif (new_mtu_oh \u003c= adapter-\u003erx_buff_pool[i].buff_size) {\n+\t\tif (new_mtu_oh \u003c= adapter-\u003erx_buff_pool[0][i].buff_size) {\n \t\t\tWRITE_ONCE(dev-\u003emtu, new_mtu);\n \t\t\tvio_cmo_set_dev_desired(viodev,\n \t\t\t\t\t\tibmveth_get_desired_dma\n@@ -1683,8 +4295,20 @@ static int ibmveth_change_mtu(struct net_device *dev, int new_mtu)\n #ifdef CONFIG_NET_POLL_CONTROLLER\n static void ibmveth_poll_controller(struct net_device *dev)\n {\n-\tibmveth_replenish_task(netdev_priv(dev));\n-\tibmveth_interrupt(dev-\u003eirq, dev);\n+\tstruct ibmveth_adapter *adapter = netdev_priv(dev);\n+\tunsigned int num;\n+\tint i;\n+\n+\tif (!adapter-\u003eopened)\n+\t\treturn;\n+\n+\tnum = ibmveth_get_num_rx_queues(adapter);\n+\n+\tfor (i = 0; i \u003c num; i++)\n+\t\tibmveth_replenish_task(adapter, i);\n+\n+\tfor (i = 0; i \u003c num; i++)\n+\t\tibmveth_schedule_rx_queue(adapter, i);\n }\n #endif\n \n@@ -1701,9 +4325,6 @@ static unsigned long ibmveth_get_desired_dma(struct vio_dev *vdev)\n \tstruct net_device *netdev = dev_get_drvdata(\u0026vdev-\u003edev);\n \tstruct ibmveth_adapter *adapter;\n \tstruct iommu_table *tbl;\n-\tunsigned long ret;\n-\tint i;\n-\tint rxqentries = 1;\n \n \ttbl = get_iommu_table_base(\u0026vdev-\u003edev);\n \n@@ -1712,26 +4333,8 @@ static unsigned long ibmveth_get_desired_dma(struct vio_dev *vdev)\n \t\treturn IOMMU_PAGE_ALIGN(IBMVETH_IO_ENTITLEMENT_DEFAULT, tbl);\n \n \tadapter = netdev_priv(netdev);\n-\n-\tret = IBMVETH_BUFF_LIST_SIZE + IBMVETH_FILT_LIST_SIZE;\n-\tret += IOMMU_PAGE_ALIGN(netdev-\u003emtu, tbl);\n-\t/* add size of mapped tx buffers */\n-\tret += IOMMU_PAGE_ALIGN(IBMVETH_MAX_TX_BUF_SIZE, tbl);\n-\n-\tfor (i = 0; i \u003c IBMVETH_NUM_BUFF_POOLS; i++) {\n-\t\t/* add the size of the active receive buffers */\n-\t\tif (adapter-\u003erx_buff_pool[i].active)\n-\t\t\tret +=\n-\t\t\t    adapter-\u003erx_buff_pool[i].size *\n-\t\t\t    IOMMU_PAGE_ALIGN(adapter-\u003erx_buff_pool[i].\n-\t\t\t\t\t     buff_size, tbl);\n-\t\trxqentries += adapter-\u003erx_buff_pool[i].size;\n-\t}\n-\t/* add the size of the receive queue entries */\n-\tret += IOMMU_PAGE_ALIGN(\n-\t\trxqentries * sizeof(struct ibmveth_rx_q_entry), tbl);\n-\n-\treturn ret;\n+\treturn ibmveth_desired_dma_for_rxqs(adapter,\n+\t\t\t\t\t    ibmveth_get_num_rx_queues(adapter));\n }\n \n static int ibmveth_set_mac_addr(struct net_device *dev, void *p)\n@@ -1777,6 +4380,124 @@ static netdev_features_t ibmveth_features_check(struct sk_buff *skb,\n \treturn vlan_features_check(skb, features);\n }\n \n+/**\n+ * ibmveth_get_stats64 - Return aggregated per-queue statistics\n+ * @dev: network device\n+ * @stats: rtnl link statistics storage\n+ *\n+ * Sums per-queue rx_qstats and tx_qstats into the rtnl counters.\n+ * Walk the full allocated arrays (not the live queue count) so shrinking\n+ * channels cannot make the totals go backwards.\n+ * Callers use ndo_get_stats64(); avoid updating netdev-\u003estats on the\n+ * xmit/poll paths to keep per-queue counters off the hot cache line.\n+ */\n+static void ibmveth_get_stats64(struct net_device *dev,\n+\t\t\t\tstruct rtnl_link_stats64 *stats)\n+{\n+\tstruct ibmveth_adapter *adapter = netdev_priv(dev);\n+\tint i;\n+\n+\tfor (i = 0; i \u003c IBMVETH_MAX_RX_QUEUES; i++) {\n+\t\tstats-\u003erx_packets += adapter-\u003erx_qstats[i].packets;\n+\t\tstats-\u003erx_bytes += adapter-\u003erx_qstats[i].bytes;\n+\t}\n+\n+\tfor (i = 0; i \u003c IBMVETH_MAX_QUEUES; i++) {\n+\t\tstats-\u003etx_packets += adapter-\u003etx_qstats[i].packets;\n+\t\tstats-\u003etx_bytes += adapter-\u003etx_qstats[i].bytes;\n+\t\tstats-\u003etx_dropped += adapter-\u003etx_qstats[i].dropped_packets;\n+\t}\n+}\n+\n+static void ibmveth_get_queue_stats_rx(struct net_device *dev, int idx,\n+\t\t\t\t       struct netdev_queue_stats_rx *stats)\n+{\n+\tstruct ibmveth_adapter *adapter = netdev_priv(dev);\n+\n+\tstats-\u003epackets = adapter-\u003erx_qstats[idx].packets;\n+\tstats-\u003ebytes = adapter-\u003erx_qstats[idx].bytes;\n+\t/*\n+\t * All three are frames that entered the device and never left it,\n+\t * which is what rx-hw-drops is specified to cover: no_buffer_drops\n+\t * is PHYP dropping for lack of buffer space on the page mapped now,\n+\t * no_buffer_retired the same for pages this queue has already\n+\t * released, and invalid_buffers is a processing error.\n+\t */\n+\tstats-\u003ehw_drops = adapter-\u003erx_qstats[idx].no_buffer_drops +\n+\t\t\t  adapter-\u003erx_qstats[idx].no_buffer_retired +\n+\t\t\t  adapter-\u003erx_qstats[idx].invalid_buffers;\n+\tstats-\u003ealloc_fail = adapter-\u003erx_qstats[idx].replenish_no_mem;\n+}\n+\n+static void ibmveth_get_queue_stats_tx(struct net_device *dev, int idx,\n+\t\t\t\t       struct netdev_queue_stats_tx *stats)\n+{\n+\tstruct ibmveth_adapter *adapter = netdev_priv(dev);\n+\n+\tstats-\u003epackets = adapter-\u003etx_qstats[idx].packets;\n+\tstats-\u003ebytes = adapter-\u003etx_qstats[idx].bytes;\n+\tstats-\u003ehw_drops = adapter-\u003etx_qstats[idx].dropped_packets;\n+}\n+\n+/**\n+ * ibmveth_get_base_stats - account for traffic not on a live queue\n+ * @dev: network device\n+ * @rx: RX base statistics storage\n+ * @tx: TX base statistics storage\n+ *\n+ * get_queue_stats_{rx,tx}() only report queues the core still iterates,\n+ * i.e. below real_num_{rx,tx}_queues, while ibmveth_get_stats64() walks\n+ * the full arrays so device totals stay monotonic across a shrink.\n+ * Report the retired-queue remainder here, otherwise qstats and\n+ * rtnl_link_stats64 disagree by a delta that grows with every shrink.\n+ * Zeroing would not be neutral: per netdev_stat_ops it asserts the\n+ * per-queue sum is already exact.\n+ *\n+ * Bound the live side with real_num_*_queues rather than the adapter's\n+ * own count, so the split lines up with the core's iteration exactly.\n+ *\n+ * Every field the per-queue callbacks fill must also be initialised\n+ * here: netdev_nl_stats_add() starts the sum at NETDEV_STAT_NOT_SET and\n+ * only accumulates while both sides are set, so a field left unset here\n+ * is dropped from the device total even though the queues report it.\n+ */\n+static void ibmveth_get_base_stats(struct net_device *dev,\n+\t\t\t\t   struct netdev_queue_stats_rx *rx,\n+\t\t\t\t   struct netdev_queue_stats_tx *tx)\n+{\n+\tstruct ibmveth_adapter *adapter = netdev_priv(dev);\n+\tunsigned int i;\n+\n+\trx-\u003epackets = 0;\n+\trx-\u003ebytes = 0;\n+\trx-\u003ealloc_fail = 0;\n+\trx-\u003ehw_drops = 0;\n+\ttx-\u003epackets = 0;\n+\ttx-\u003ebytes = 0;\n+\ttx-\u003ehw_drops = 0;\n+\n+\tfor (i = dev-\u003ereal_num_rx_queues; i \u003c IBMVETH_MAX_RX_QUEUES; i++) {\n+\t\trx-\u003epackets += adapter-\u003erx_qstats[i].packets;\n+\t\trx-\u003ebytes += adapter-\u003erx_qstats[i].bytes;\n+\t\trx-\u003ehw_drops += adapter-\u003erx_qstats[i].no_buffer_drops +\n+\t\t\t\tadapter-\u003erx_qstats[i].no_buffer_retired +\n+\t\t\t\tadapter-\u003erx_qstats[i].invalid_buffers;\n+\t\trx-\u003ealloc_fail += adapter-\u003erx_qstats[i].replenish_no_mem;\n+\t}\n+\n+\tfor (i = dev-\u003ereal_num_tx_queues; i \u003c IBMVETH_MAX_QUEUES; i++) {\n+\t\ttx-\u003epackets += adapter-\u003etx_qstats[i].packets;\n+\t\ttx-\u003ebytes += adapter-\u003etx_qstats[i].bytes;\n+\t\ttx-\u003ehw_drops += adapter-\u003etx_qstats[i].dropped_packets;\n+\t}\n+}\n+\n+static const struct netdev_stat_ops ibmveth_stat_ops = {\n+\t.get_queue_stats_rx\t= ibmveth_get_queue_stats_rx,\n+\t.get_queue_stats_tx\t= ibmveth_get_queue_stats_tx,\n+\t.get_base_stats\t\t= ibmveth_get_base_stats,\n+};\n+\n static const struct net_device_ops ibmveth_netdev_ops = {\n \t.ndo_open\t\t= ibmveth_open,\n \t.ndo_stop\t\t= ibmveth_close,\n@@ -1789,14 +4510,129 @@ static const struct net_device_ops ibmveth_netdev_ops = {\n \t.ndo_validate_addr\t= eth_validate_addr,\n \t.ndo_set_mac_address    = ibmveth_set_mac_addr,\n \t.ndo_features_check\t= ibmveth_features_check,\n+\t.ndo_get_stats64\t= ibmveth_get_stats64,\n #ifdef CONFIG_NET_POLL_CONTROLLER\n \t.ndo_poll_controller\t= ibmveth_poll_controller,\n #endif\n };\n \n+static int ibmveth_buffer_pools_show(struct seq_file *m, void *v)\n+{\n+\tstruct ibmveth_adapter *adapter = m-\u003eprivate;\n+\tint i, j;\n+\n+\t/*\n+\t * size / buff_size / pool-\u003eactive are written under RTNL\n+\t * (veth_pool_store, open template copy). Take the same lock so\n+\t * those columns are not a torn snapshot. available is updated\n+\t * from NAPI/softirq; only atomic_read() keeps it from tearing.\n+\t * Not required for memory safety; embedded arrays only.\n+\t */\n+\trtnl_lock();\n+\n+\tseq_puts(m, \"Queue  Pool  Count  BuffSize  Active  Available\\n\");\n+\tseq_puts(m, \"-----  ----  -----  --------  ------  ---------\\n\");\n+\tif (!adapter-\u003eopened) {\n+\t\tseq_puts(m, \"# down: Active/Available 0 unless allocated\\n\");\n+\t\tseq_puts(m, \"# down: geometry above queue 0 set at open\\n\");\n+\t}\n+\n+\tfor (i = 0; i \u003c ibmveth_get_num_rx_queues(adapter); i++) {\n+\t\tfor (j = 0; j \u003c IBMVETH_NUM_BUFF_POOLS; j++) {\n+\t\t\tstruct ibmveth_buff_pool *pool =\n+\t\t\t\t\u0026adapter-\u003erx_buff_pool[i][j];\n+\t\t\tbool live = pool-\u003eskbuff \u0026\u0026 pool-\u003efree_map;\n+\t\t\tint active = live ? pool-\u003eactive : 0;\n+\t\t\tint available = live ? atomic_read(\u0026pool-\u003eavailable)\n+\t\t\t\t\t     : 0;\n+\n+\t\t\tseq_printf(m, \"%5d  %4d  %5u  %8u  %6d  %9d\\n\",\n+\t\t\t\t   i, j, pool-\u003esize, pool-\u003ebuff_size,\n+\t\t\t\t   active, available);\n+\t\t}\n+\t}\n+\n+\trtnl_unlock();\n+\treturn 0;\n+}\n+DEFINE_SHOW_ATTRIBUTE(ibmveth_buffer_pools);\n+\n+/* Driver-owned root so per-adapter dirs use a stable vio name, not the\n+ * mutable netdev-\u003ename (avoids stale names / eth0 collisions after rename).\n+ */\n+static struct dentry *ibmveth_dbg_root;\n+\n+static void ibmveth_debugfs_init(struct ibmveth_adapter *adapter)\n+{\n+\tadapter-\u003edebugfs_dir =\n+\t\tdebugfs_create_dir(dev_name(\u0026adapter-\u003evdev-\u003edev),\n+\t\t\t\t   ibmveth_dbg_root);\n+\tdebugfs_create_file(\"buffer_pools\", 0400, adapter-\u003edebugfs_dir,\n+\t\t\t    adapter, \u0026ibmveth_buffer_pools_fops);\n+}\n+\n+static void ibmveth_debugfs_exit(struct ibmveth_adapter *adapter)\n+{\n+\tdebugfs_remove_recursive(adapter-\u003edebugfs_dir);\n+\tadapter-\u003edebugfs_dir = NULL;\n+}\n+\n+/**\n+ * ibmveth_pool_kobj_release - Mark a pool kobject finished\n+ * @kobj: kobject embedded in the pool\n+ *\n+ * Pool kobjects live in netdev_priv(). Last put waits for this\n+ * before free_netdev() so DEBUG_KOBJECT_RELEASE delayed cleanup\n+ * does not run on freed memory.\n+ */\n+static void ibmveth_pool_kobj_release(struct kobject *kobj)\n+{\n+\tstruct ibmveth_buff_pool *pool = container_of(kobj,\n+\t\t\t\t\t\t      struct ibmveth_buff_pool,\n+\t\t\t\t\t\t      kobj);\n+\n+\tcomplete(\u0026pool-\u003ereleased);\n+}\n+\n+/**\n+ * ibmveth_put_pool_kobjs - Drop pool kobjects and wait for release\n+ * @adapter: ibmveth adapter\n+ * @pools_ready: number of queue-0 pools that were initialized\n+ *\n+ * Put every initialized pool kobject, then wait so free_netdev()\n+ * cannot race DEBUG_KOBJECT_RELEASE delayed cleanup.\n+ */\n+static void ibmveth_put_pool_kobjs(struct ibmveth_adapter *adapter,\n+\t\t\t\t   int pools_ready)\n+{\n+\tint i;\n+\n+\tfor (i = 0; i \u003c pools_ready; i++)\n+\t\tkobject_put(\u0026adapter-\u003erx_buff_pool[0][i].kobj);\n+\tfor (i = 0; i \u003c pools_ready; i++)\n+\t\twait_for_completion(\u0026adapter-\u003erx_buff_pool[0][i].released);\n+}\n+\n+static void ibmveth_probe_cleanup(struct ibmveth_adapter *adapter,\n+\t\t\t\t  int pools_ready)\n+{\n+\tstruct net_device *netdev = adapter-\u003enetdev;\n+\n+\tcancel_work_sync(\u0026adapter-\u003ework);\n+\tibmveth_put_pool_kobjs(adapter, pools_ready);\n+\n+\tibmveth_free_tx_qstats(adapter);\n+\tibmveth_free_rx_qstats(adapter);\n+\t/* Probe failure never reaches ibmveth_remove(); clear before free so\n+\t * CMO get_desired_dma() cannot see a freed netdev on rebind.\n+\t */\n+\tdev_set_drvdata(\u0026adapter-\u003evdev-\u003edev, NULL);\n+\tfree_netdev(netdev);\n+}\n+\n static int ibmveth_probe(struct vio_dev *dev, const struct vio_device_id *id)\n {\n-\tint rc, i, mac_len;\n+\tint rc, i, mac_len, pools_ready = 0;\n \tstruct net_device *netdev;\n \tstruct ibmveth_adapter *adapter;\n \tunsigned char *mac_addr_p;\n@@ -1831,7 +4667,8 @@ static int ibmveth_probe(struct vio_dev *dev, const struct vio_device_id *id)\n \t\treturn -EINVAL;\n \t}\n \n-\tnetdev = alloc_etherdev_mqs(sizeof(struct ibmveth_adapter), IBMVETH_MAX_QUEUES, 1);\n+\tnetdev = alloc_etherdev_mqs(sizeof(struct ibmveth_adapter),\n+\t\t\t\t    IBMVETH_MAX_QUEUES, IBMVETH_MAX_RX_QUEUES);\n \tif (!netdev)\n \t\treturn -ENOMEM;\n \n@@ -1841,14 +4678,33 @@ static int ibmveth_probe(struct vio_dev *dev, const struct vio_device_id *id)\n \tadapter-\u003evdev = dev;\n \tadapter-\u003enetdev = netdev;\n \tINIT_WORK(\u0026adapter-\u003ework, ibmveth_reset);\n+\n+\t/*\n+\t * Initialise the replenish locks once. open() is re-entered on\n+\t * MTU and offload changes without netpoll_poll_disable(), so a\n+\t * lock set up there could be reinitialised while poll_controller()\n+\t * holds it.\n+\t */\n+\tfor (i = 0; i \u003c IBMVETH_MAX_RX_QUEUES; i++)\n+\t\tspin_lock_init(\u0026adapter-\u003erx_queue[i].replenish_lock);\n+\n \tadapter-\u003emcastFilterSize = be32_to_cpu(*mcastFilterSize_p);\n \tibmveth_init_link_settings(netdev);\n \n-\tnetif_napi_add_weight(netdev, \u0026adapter-\u003enapi, ibmveth_poll, 16);\n+\tfor (i = 0; i \u003c IBMVETH_MAX_RX_QUEUES; i++)\n+\t\tnetif_napi_add_weight(netdev, \u0026adapter-\u003enapi[i],\n+\t\t\t\t      ibmveth_poll, 16);\n+\n+\tif (ibmveth_alloc_rx_qstats(adapter) ||\n+\t    ibmveth_alloc_tx_qstats(adapter)) {\n+\t\tibmveth_probe_cleanup(adapter, 0);\n+\t\treturn -ENOMEM;\n+\t}\n \n \tnetdev-\u003eirq = dev-\u003eirq;\n \tnetdev-\u003enetdev_ops = \u0026ibmveth_netdev_ops;\n \tnetdev-\u003eethtool_ops = \u0026netdev_ethtool_ops;\n+\tnetdev-\u003estat_ops = \u0026ibmveth_stat_ops;\n \tSET_NETDEV_DEV(netdev, \u0026dev-\u003edev);\n \tnetdev-\u003ehw_features = NETIF_F_SG;\n \tif (vio_get_attribute(dev, \"ibm,illan-options\", NULL) != NULL) {\n@@ -1876,12 +4732,30 @@ static int ibmveth_probe(struct vio_dev *dev, const struct vio_device_id *id)\n \t\tnetdev-\u003efeatures |= NETIF_F_FRAGLIST;\n \t}\n \n+\tif (ret == H_SUCCESS \u0026\u0026\n+\t    (ret_attr \u0026 IBMVETH_ILLAN_RX_MULTI_QUEUE_SUPPORT)) {\n+\t\tadapter-\u003emulti_queue = true;\n+\t\tibmveth_publish_num_rx_queues(adapter,\n+\t\t\t\t\t      min(num_online_cpus(),\n+\t\t\t\t\t\t  IBMVETH_DEFAULT_QUEUES));\n+\t\tnetdev_dbg(netdev, \"RX multi queue mode enabled: %u queues\\n\",\n+\t\t\t   ibmveth_get_num_rx_queues(adapter));\n+\t} else {\n+\t\tadapter-\u003emulti_queue = false;\n+\t\tibmveth_publish_num_rx_queues(adapter,\n+\t\t\t\t\t      IBMVETH_DEFAULT_RX_QUEUES);\n+\t}\n+\n \tif (ret == H_SUCCESS \u0026\u0026\n \t    (ret_attr \u0026 IBMVETH_ILLAN_RX_MULTI_BUFF_SUPPORT)) {\n-\t\tadapter-\u003erx_buffers_per_hcall = IBMVETH_MAX_RX_PER_HCALL;\n+\t\tif (adapter-\u003emulti_queue)\n+\t\t\tadapter-\u003erx_buffers_per_hcall =\n+\t\t\t\tIBMVETH_MAX_RX_PER_HCALL;\n+\t\telse\n+\t\t\tadapter-\u003erx_buffers_per_hcall = IBMVETH_MAX_RX_REGULAR;\n \t\tnetdev_dbg(netdev,\n \t\t\t   \"RX Multi-buffer hcall supported by FW, batch set to %u\\n\",\n-\t\t\t    adapter-\u003erx_buffers_per_hcall);\n+\t\t\t   adapter-\u003erx_buffers_per_hcall);\n \t} else {\n \t\tadapter-\u003erx_buffers_per_hcall = 1;\n \t\tnetdev_dbg(netdev,\n@@ -1898,16 +4772,29 @@ static int ibmveth_probe(struct vio_dev *dev, const struct vio_device_id *id)\n \t\tmemcpy(pool_count, pool_count_cmo, sizeof(pool_count));\n \n \tfor (i = 0; i \u003c IBMVETH_NUM_BUFF_POOLS; i++) {\n-\t\tstruct kobject *kobj = \u0026adapter-\u003erx_buff_pool[i].kobj;\n-\t\tint error;\n+\t\tstruct kobject *kobj = \u0026adapter-\u003erx_buff_pool[0][i].kobj;\n \n-\t\tibmveth_init_buffer_pool(\u0026adapter-\u003erx_buff_pool[i], i,\n+\t\tibmveth_init_buffer_pool(\u0026adapter-\u003erx_buff_pool[0][i], i,\n \t\t\t\t\t pool_count[i], pool_size[i],\n \t\t\t\t\t pool_active[i]);\n-\t\terror = kobject_init_and_add(kobj, \u0026ktype_veth_pool,\n-\t\t\t\t\t     \u0026dev-\u003edev.kobj, \"pool%d\", i);\n-\t\tif (!error)\n-\t\t\tkobject_uevent(kobj, KOBJ_ADD);\n+\t\tinit_completion(\u0026adapter-\u003erx_buff_pool[0][i].released);\n+\t\trc = kobject_init_and_add(kobj, \u0026ktype_veth_pool,\n+\t\t\t\t\t  \u0026dev-\u003edev.kobj, \"pool%d\", i);\n+\t\tif (rc) {\n+\t\t\tstruct ibmveth_buff_pool *pool =\n+\t\t\t\t\u0026adapter-\u003erx_buff_pool[0][i];\n+\n+\t\t\tdev_err(\u0026dev-\u003edev,\n+\t\t\t\t\"failed to create pool%d kobject: %d\\n\", i, rc);\n+\t\t\t/* init_and_add takes a ref even on failure */\n+\t\t\tkobject_put(kobj);\n+\t\t\twait_for_completion(\u0026pool-\u003ereleased);\n+\t\t\tibmveth_probe_cleanup(adapter, pools_ready);\n+\t\t\treturn rc;\n+\t\t}\n+\n+\t\tpools_ready++;\n+\t\tkobject_uevent(kobj, KOBJ_ADD);\n \t}\n \n \trc = netif_set_real_num_tx_queues(netdev, min(num_online_cpus(),\n@@ -1915,9 +4802,25 @@ static int ibmveth_probe(struct vio_dev *dev, const struct vio_device_id *id)\n \tif (rc) {\n \t\tnetdev_dbg(netdev, \"failed to set number of tx queues rc=%d\\n\",\n \t\t\t   rc);\n-\t\tfree_netdev(netdev);\n+\t\tibmveth_probe_cleanup(adapter, pools_ready);\n+\t\treturn rc;\n+\t}\n+\n+\t/*\n+\t * alloc_etherdev_mqs() sized real_num_rx_queues to MAX. Match the\n+\t * advertised default (or SQ 1) before register_netdev so down-state\n+\t * readers agree with adapter-\u003enum_rx_queues / ethtool -l.\n+\t */\n+\trc = netif_set_real_num_rx_queues(netdev,\n+\t\t\t\t\t  ibmveth_get_num_rx_queues(adapter));\n+\n+\tif (rc) {\n+\t\tnetdev_dbg(netdev, \"failed to set number of rx queues rc=%d\\n\",\n+\t\t\t   rc);\n+\t\tibmveth_probe_cleanup(adapter, pools_ready);\n \t\treturn rc;\n \t}\n+\n \tadapter-\u003etx_ltb_size = PAGE_ALIGN(IBMVETH_MAX_TX_BUF_SIZE);\n \tfor (i = 0; i \u003c IBMVETH_MAX_QUEUES; i++)\n \t\tadapter-\u003etx_ltb_ptr[i] = NULL;\n@@ -1931,12 +4834,14 @@ static int ibmveth_probe(struct vio_dev *dev, const struct vio_device_id *id)\n \n \tif (rc) {\n \t\tnetdev_dbg(netdev, \"failed to register netdev rc=%d\\n\", rc);\n-\t\tfree_netdev(netdev);\n+\t\tibmveth_probe_cleanup(adapter, pools_ready);\n \t\treturn rc;\n \t}\n \n \tnetdev_dbg(netdev, \"registered\\n\");\n \n+\tibmveth_debugfs_init(adapter);\n+\n \treturn 0;\n }\n \n@@ -1944,14 +4849,22 @@ static void ibmveth_remove(struct vio_dev *dev)\n {\n \tstruct net_device *netdev = dev_get_drvdata(\u0026dev-\u003edev);\n \tstruct ibmveth_adapter *adapter = netdev_priv(netdev);\n-\tint i;\n \n-\tcancel_work_sync(\u0026adapter-\u003ework);\n+\tibmveth_debugfs_exit(adapter);\n \n-\tfor (i = 0; i \u003c IBMVETH_NUM_BUFF_POOLS; i++)\n-\t\tkobject_put(\u0026adapter-\u003erx_buff_pool[i].kobj);\n+\tibmveth_put_pool_kobjs(adapter, IBMVETH_NUM_BUFF_POOLS);\n \n+\t/*\n+\t * Unregister first so NAPI/xmit cannot re-arm reset work after we\n+\t * cancel it. cancel_work_sync() before unregister left a window\n+\t * where poll could schedule_work() and the worker ran after\n+\t * free_netdev().\n+\t */\n \tunregister_netdev(netdev);\n+\tcancel_work_sync(\u0026adapter-\u003ework);\n+\n+\tibmveth_free_tx_qstats(adapter);\n+\tibmveth_free_rx_qstats(adapter);\n \n \tfree_netdev(netdev);\n \tdev_set_drvdata(\u0026dev-\u003edev, NULL);\n@@ -2035,11 +4948,12 @@ static ssize_t veth_pool_store(struct kobject *kobj, struct attribute *attr,\n \t\t\t/* Make sure there is a buffer pool with buffers that\n \t\t\t   can hold a packet of the size of the MTU */\n \t\t\tfor (i = 0; i \u003c IBMVETH_NUM_BUFF_POOLS; i++) {\n-\t\t\t\tif (pool == \u0026adapter-\u003erx_buff_pool[i])\n+\t\t\t\tif (pool == \u0026adapter-\u003erx_buff_pool[0][i])\n \t\t\t\t\tcontinue;\n-\t\t\t\tif (!adapter-\u003erx_buff_pool[i].active)\n+\t\t\t\tif (!adapter-\u003erx_buff_pool[0][i].active)\n \t\t\t\t\tcontinue;\n-\t\t\t\tif (mtu \u003c= adapter-\u003erx_buff_pool[i].buff_size)\n+\t\t\t\tif (mtu \u003c=\n+\t\t\t\t    adapter-\u003erx_buff_pool[0][i].buff_size)\n \t\t\t\t\tbreak;\n \t\t\t}\n \n@@ -2092,8 +5006,8 @@ static ssize_t veth_pool_store(struct kobject *kobj, struct attribute *attr,\n \t}\n \trtnl_unlock();\n \n-\t/* kick the interrupt handler to allocate/deallocate pools */\n-\tibmveth_interrupt(netdev-\u003eirq, netdev);\n+\t/* kick RX processing to allocate/deallocate pools */\n+\tibmveth_schedule_rx_queue(adapter, 0);\n \treturn count;\n \n unlock_err:\n@@ -2101,7 +5015,6 @@ static ssize_t veth_pool_store(struct kobject *kobj, struct attribute *attr,\n \treturn rc;\n }\n \n-\n #define ATTR(_name, _mode)\t\t\t\t\\\n \tstruct attribute veth_##_name##_attr = {\t\\\n \t.name = __stringify(_name), .mode = _mode,\t\\\n@@ -2125,7 +5038,7 @@ static const struct sysfs_ops veth_pool_ops = {\n };\n \n static struct kobj_type ktype_veth_pool = {\n-\t.release        = NULL,\n+\t.release        = ibmveth_pool_kobj_release,\n \t.sysfs_ops      = \u0026veth_pool_ops,\n \t.default_groups = veth_pool_groups,\n };\n@@ -2133,7 +5046,13 @@ static struct kobj_type ktype_veth_pool = {\n static int ibmveth_resume(struct device *dev)\n {\n \tstruct net_device *netdev = dev_get_drvdata(dev);\n-\tibmveth_interrupt(netdev-\u003eirq, netdev);\n+\tstruct ibmveth_adapter *adapter = netdev_priv(netdev);\n+\tunsigned int num = ibmveth_get_num_rx_queues(adapter);\n+\tint i;\n+\n+\tfor (i = 0; i \u003c num; i++)\n+\t\tibmveth_schedule_rx_queue(adapter, i);\n+\n \treturn 0;\n }\n \n@@ -2158,15 +5077,27 @@ static struct vio_driver ibmveth_driver = {\n \n static int __init ibmveth_module_init(void)\n {\n+\tint rc;\n+\n \tprintk(KERN_DEBUG \"%s: %s %s\\n\", ibmveth_driver_name,\n \t       ibmveth_driver_string, ibmveth_driver_version);\n \n-\treturn vio_register_driver(\u0026ibmveth_driver);\n+\tibmveth_dbg_root = debugfs_create_dir(ibmveth_driver_name, NULL);\n+\n+\trc = vio_register_driver(\u0026ibmveth_driver);\n+\tif (rc) {\n+\t\tdebugfs_remove_recursive(ibmveth_dbg_root);\n+\t\tibmveth_dbg_root = NULL;\n+\t}\n+\n+\treturn rc;\n }\n \n static void __exit ibmveth_module_exit(void)\n {\n \tvio_unregister_driver(\u0026ibmveth_driver);\n+\tdebugfs_remove_recursive(ibmveth_dbg_root);\n+\tibmveth_dbg_root = NULL;\n }\n \n module_init(ibmveth_module_init);\n@@ -2196,8 +5127,7 @@ static void ibmveth_reset_kunit(struct work_struct *w)\n  * @test: pointer to kunit structure\n  *\n  * Tests the error returns from ibmveth_remove_buffer_from_pool.\n- * ibmveth_remove_buffer_from_pool also calls WARN_ON, so dmesg should be\n- * checked to see that these warnings happened.\n+ * Bad correlators return -EINVAL/-EFAULT (no WARN_ON).\n  *\n  * Return: void\n  */\n@@ -2211,28 +5141,45 @@ static void ibmveth_remove_buffer_from_pool_test(struct kunit *test)\n \n \tINIT_WORK(\u0026adapter-\u003ework, ibmveth_reset_kunit);\n \n+\tspin_lock_init(\u0026adapter-\u003erx_queue[0].replenish_lock);\n+\n \t/* Set sane values for buffer pools */\n \tfor (int i = 0; i \u003c IBMVETH_NUM_BUFF_POOLS; i++)\n-\t\tibmveth_init_buffer_pool(\u0026adapter-\u003erx_buff_pool[i], i,\n+\t\tibmveth_init_buffer_pool(\u0026adapter-\u003erx_buff_pool[0][i], i,\n \t\t\t\t\t pool_count[i], pool_size[i],\n \t\t\t\t\t pool_active[i]);\n \n-\tpool = \u0026adapter-\u003erx_buff_pool[0];\n+\tpool = \u0026adapter-\u003erx_buff_pool[0][0];\n \tpool-\u003eskbuff = kunit_kcalloc(test, pool-\u003esize, sizeof(void *), GFP_KERNEL);\n \tKUNIT_ASSERT_NOT_ERR_OR_NULL(test, pool-\u003eskbuff);\n+\tpool-\u003efree_map = kunit_kcalloc(test, pool-\u003esize, sizeof(u16),\n+\t\t\t\t       GFP_KERNEL);\n+\tKUNIT_ASSERT_NOT_ERR_OR_NULL(test, pool-\u003efree_map);\n \n \tcorrelator = ((u64)IBMVETH_NUM_BUFF_POOLS \u003c\u003c 32) | 0;\n-\tKUNIT_EXPECT_EQ(test, -EINVAL, ibmveth_remove_buffer_from_pool(adapter, correlator, false));\n-\tKUNIT_EXPECT_EQ(test, -EINVAL, ibmveth_remove_buffer_from_pool(adapter, correlator, true));\n-\n-\tcorrelator = ((u64)0 \u003c\u003c 32) | adapter-\u003erx_buff_pool[0].size;\n-\tKUNIT_EXPECT_EQ(test, -EINVAL, ibmveth_remove_buffer_from_pool(adapter, correlator, false));\n-\tKUNIT_EXPECT_EQ(test, -EINVAL, ibmveth_remove_buffer_from_pool(adapter, correlator, true));\n+\tKUNIT_EXPECT_EQ(test, -EINVAL,\n+\t\t\tibmveth_remove_buffer_from_pool(adapter,\n+\t\t\t\t\t\t\tcorrelator, 0, false));\n+\tKUNIT_EXPECT_EQ(test, -EINVAL,\n+\t\t\tibmveth_remove_buffer_from_pool(adapter,\n+\t\t\t\t\t\t\tcorrelator, 0, true));\n+\n+\tcorrelator = ((u64)0 \u003c\u003c 32) | adapter-\u003erx_buff_pool[0][0].size;\n+\tKUNIT_EXPECT_EQ(test, -EINVAL,\n+\t\t\tibmveth_remove_buffer_from_pool(adapter,\n+\t\t\t\t\t\t\tcorrelator, 0, false));\n+\tKUNIT_EXPECT_EQ(test, -EINVAL,\n+\t\t\tibmveth_remove_buffer_from_pool(adapter,\n+\t\t\t\t\t\t\tcorrelator, 0, true));\n \n \tcorrelator = (u64)0 | 0;\n \tpool-\u003eskbuff[0] = NULL;\n-\tKUNIT_EXPECT_EQ(test, -EFAULT, ibmveth_remove_buffer_from_pool(adapter, correlator, false));\n-\tKUNIT_EXPECT_EQ(test, -EFAULT, ibmveth_remove_buffer_from_pool(adapter, correlator, true));\n+\tKUNIT_EXPECT_EQ(test, -EFAULT,\n+\t\t\tibmveth_remove_buffer_from_pool(adapter,\n+\t\t\t\t\t\t\tcorrelator, 0, false));\n+\tKUNIT_EXPECT_EQ(test, -EFAULT,\n+\t\t\tibmveth_remove_buffer_from_pool(adapter,\n+\t\t\t\t\t\t\tcorrelator, 0, true));\n \n \tflush_work(\u0026adapter-\u003ework);\n }\n@@ -2241,9 +5188,7 @@ static void ibmveth_remove_buffer_from_pool_test(struct kunit *test)\n  * ibmveth_rxq_get_buffer_test - unit test for ibmveth_rxq_get_buffer\n  * @test: pointer to kunit structure\n  *\n- * Tests ibmveth_rxq_get_buffer. ibmveth_rxq_get_buffer also calls WARN_ON for\n- * the NULL returns, so dmesg should be checked to see that these warnings\n- * happened.\n+ * Tests ibmveth_rxq_get_buffer invalid correlator returns NULL without WARN.\n  *\n  * Return: void\n  */\n@@ -2258,31 +5203,37 @@ static void ibmveth_rxq_get_buffer_test(struct kunit *test)\n \n \tINIT_WORK(\u0026adapter-\u003ework, ibmveth_reset_kunit);\n \n-\tadapter-\u003erx_queue.queue_len = 1;\n-\tadapter-\u003erx_queue.index = 0;\n-\tadapter-\u003erx_queue.queue_addr = kunit_kzalloc(test, sizeof(struct ibmveth_rx_q_entry),\n-\t\t\t\t\t\t     GFP_KERNEL);\n-\tKUNIT_ASSERT_NOT_ERR_OR_NULL(test, adapter-\u003erx_queue.queue_addr);\n+\tadapter-\u003erx_queue[0].queue_len = 1;\n+\tadapter-\u003erx_queue[0].index = 0;\n+\tadapter-\u003erx_queue[0].queue_addr =\n+\t\tkunit_kzalloc(test, sizeof(struct ibmveth_rx_q_entry),\n+\t\t\t      GFP_KERNEL);\n+\tKUNIT_ASSERT_NOT_ERR_OR_NULL(test, adapter-\u003erx_queue[0].queue_addr);\n \n \t/* Set sane values for buffer pools */\n \tfor (int i = 0; i \u003c IBMVETH_NUM_BUFF_POOLS; i++)\n-\t\tibmveth_init_buffer_pool(\u0026adapter-\u003erx_buff_pool[i], i,\n+\t\tibmveth_init_buffer_pool(\u0026adapter-\u003erx_buff_pool[0][i], i,\n \t\t\t\t\t pool_count[i], pool_size[i],\n \t\t\t\t\t pool_active[i]);\n \n-\tpool = \u0026adapter-\u003erx_buff_pool[0];\n+\tpool = \u0026adapter-\u003erx_buff_pool[0][0];\n \tpool-\u003eskbuff = kunit_kcalloc(test, pool-\u003esize, sizeof(void *), GFP_KERNEL);\n \tKUNIT_ASSERT_NOT_ERR_OR_NULL(test, pool-\u003eskbuff);\n+\tpool-\u003efree_map = kunit_kcalloc(test, pool-\u003esize, sizeof(u16),\n+\t\t\t\t       GFP_KERNEL);\n+\tKUNIT_ASSERT_NOT_ERR_OR_NULL(test, pool-\u003efree_map);\n \n-\tadapter-\u003erx_queue.queue_addr[0].correlator = (u64)IBMVETH_NUM_BUFF_POOLS \u003c\u003c 32 | 0;\n-\tKUNIT_EXPECT_PTR_EQ(test, NULL, ibmveth_rxq_get_buffer(adapter));\n+\tadapter-\u003erx_queue[0].queue_addr[0].correlator =\n+\t\t(u64)IBMVETH_NUM_BUFF_POOLS \u003c\u003c 32 | 0;\n+\tKUNIT_EXPECT_PTR_EQ(test, NULL, ibmveth_rxq_get_buffer(adapter, 0));\n \n-\tadapter-\u003erx_queue.queue_addr[0].correlator = (u64)0 \u003c\u003c 32 | adapter-\u003erx_buff_pool[0].size;\n-\tKUNIT_EXPECT_PTR_EQ(test, NULL, ibmveth_rxq_get_buffer(adapter));\n+\tadapter-\u003erx_queue[0].queue_addr[0].correlator =\n+\t\t(u64)0 \u003c\u003c 32 | adapter-\u003erx_buff_pool[0][0].size;\n+\tKUNIT_EXPECT_PTR_EQ(test, NULL, ibmveth_rxq_get_buffer(adapter, 0));\n \n \tpool-\u003eskbuff[0] = skb;\n-\tadapter-\u003erx_queue.queue_addr[0].correlator = (u64)0 \u003c\u003c 32 | 0;\n-\tKUNIT_EXPECT_PTR_EQ(test, skb, ibmveth_rxq_get_buffer(adapter));\n+\tadapter-\u003erx_queue[0].queue_addr[0].correlator = (u64)0 \u003c\u003c 32 | 0;\n+\tKUNIT_EXPECT_PTR_EQ(test, skb, ibmveth_rxq_get_buffer(adapter, 0));\n \n \tflush_work(\u0026adapter-\u003ework);\n }\ndiff --git a/drivers/net/ethernet/ibm/ibmveth.h b/drivers/net/ethernet/ibm/ibmveth.h\nindex d87713668ed30..11eda224a6807 100644\n--- a/drivers/net/ethernet/ibm/ibmveth.h\n+++ b/drivers/net/ethernet/ibm/ibmveth.h\n@@ -14,6 +14,9 @@\n #ifndef _IBMVETH_H\n #define _IBMVETH_H\n \n+#include \u003clinux/spinlock_types.h\u003e\n+#include \u003clinux/completion.h\u003e\n+\n /* constants for H_MULTICAST_CTRL */\n #define IbmVethMcastReceptionModifyBit     0x80000UL\n #define IbmVethMcastReceptionEnableBit     0x20000UL\n@@ -28,6 +31,7 @@\n #define IbmVethMcastRemoveFilter     0x2UL\n #define IbmVethMcastClearFilterTable 0x3UL\n \n+#define IBMVETH_ILLAN_RX_MULTI_QUEUE_SUPPORT\t0x0000000000080000UL\n #define IBMVETH_ILLAN_RX_MULTI_BUFF_SUPPORT\t0x0000000000040000UL\n #define IBMVETH_ILLAN_LRG_SR_ENABLED\t0x0000000000010000UL\n #define IBMVETH_ILLAN_LRG_SND_SUPPORT\t0x0000000000008000UL\n@@ -66,6 +70,148 @@ static inline long h_add_logical_lan_buffers(unsigned long unit_address,\n \t\t\t    desc5, desc6, desc7, desc8);\n }\n \n+/**\n+ * h_register_logical_lan_queue - Register a subordinate receive queue\n+ * @unit_address: Device unit address\n+ * @buffer_list: DMA address of 4KB page for tracking registered buffers\n+ * @rec_queue: Buffer descriptor of receive queue\n+ * @queue_handle: Output queue handle on success (may be NULL)\n+ * @irq: Output hypervisor IRQ number on success (may be NULL)\n+ *\n+ * Registers a subordinate receive queue with the hypervisor.\n+ *\n+ * Return:\n+ *   H_SUCCESS (0) on success\n+ *   H_PARAMETER if parameters are invalid\n+ *   H_BUSY / H_LONG_BUSY_* if the resource is busy; retry\n+ *   H_FUNCTION if firmware does not support this hcall\n+ *\n+ * On success, hypervisor returns:\n+ *   R3: H_SUCCESS\n+ *   R4: Queue handle\n+ *   R5: IRQ number for this queue\n+ */\n+static inline long\n+h_register_logical_lan_queue(unsigned long unit_address,\n+\t\t\t     unsigned long buffer_list,\n+\t\t\t     unsigned long rec_queue,\n+\t\t\t     unsigned long *queue_handle,\n+\t\t\t     unsigned long *irq)\n+{\n+\tunsigned long retbuf[PLPAR_HCALL_BUFSIZE];\n+\tlong rc;\n+\n+\trc = plpar_hcall(H_REG_LOGICAL_LAN_QUEUE,\n+\t\t\t retbuf, unit_address,\n+\t\t\t buffer_list, rec_queue);\n+\n+\tif (rc == H_SUCCESS) {\n+\t\tif (queue_handle)\n+\t\t\t*queue_handle = retbuf[0];\n+\t\tif (irq)\n+\t\t\t*irq = retbuf[1];\n+\t}\n+\n+\treturn rc;\n+}\n+\n+/**\n+ * h_add_logical_lan_buffers_queue - Add buffers to subordinate queue\n+ * @unit_address: Device unit address\n+ * @queue_handle: Queue handle from h_register_logical_lan_queue() or\n+ *\t\t  h_register_logical_lan_with_handle() (queue 0)\n+ * @buffersznum: Buffer size (upper 32 bits) | count (lower 32 bits)\n+ * @ioba12: Buffer addresses 1 and 2 packed ((addr1 \u003c\u003c 32) | addr2)\n+ * @ioba34: Buffer addresses 3 and 4 packed\n+ * @ioba56: Buffer addresses 5 and 6 packed\n+ * @ioba78: Buffer addresses 7 and 8 packed\n+ * @ioba910: Buffer addresses 9 and 10 packed\n+ * @ioba1112: Buffer addresses 11 and 12 packed\n+ *\n+ * Return:\n+ *   H_SUCCESS - All buffers added successfully\n+ *   H_PARAMETER - Invalid parameters\n+ *   H_HARDWARE - Hardware error\n+ *   H_FUNCTION - Firmware does not support this hcall\n+ */\n+static inline long h_add_logical_lan_buffers_queue(unsigned long unit_address,\n+\t\t\t\t\t\t   unsigned long queue_handle,\n+\t\t\t\t\t\t   unsigned long buffersznum,\n+\t\t\t\t\t\t   unsigned long ioba12,\n+\t\t\t\t\t\t   unsigned long ioba34,\n+\t\t\t\t\t\t   unsigned long ioba56,\n+\t\t\t\t\t\t   unsigned long ioba78,\n+\t\t\t\t\t\t   unsigned long ioba910,\n+\t\t\t\t\t\t   unsigned long ioba1112)\n+{\n+\tunsigned long retbuf[PLPAR_HCALL9_BUFSIZE];\n+\n+\treturn plpar_hcall9(H_ADD_LOGICAL_LAN_BUFFERS_QUEUE,\n+\t\t\t    retbuf, unit_address,\n+\t\t\t    queue_handle, buffersznum,\n+\t\t\t    ioba12, ioba34, ioba56,\n+\t\t\t    ioba78, ioba910, ioba1112);\n+}\n+\n+/**\n+ * h_free_logical_lan_queue - Deregister subordinate receive queue\n+ * @unit_address: Device unit address\n+ * @queue_handle: Queue handle from h_register_logical_lan_queue() or\n+ *\t\t  h_register_logical_lan_with_handle() (queue 0)\n+ *\n+ * Deregisters and frees all structures associated with the subordinate queue.\n+ *\n+ * Return:\n+ *   H_SUCCESS - Queue freed successfully\n+ *   H_PARAMETER - Invalid parameters\n+ *   H_HARDWARE - Hardware error\n+ *   H_STATE - VIOA not in valid state\n+ *   H_BUSY / H_LONG_BUSY_* - Resource busy, retry\n+ *   H_FUNCTION - Firmware does not support this hcall\n+ */\n+static inline long h_free_logical_lan_queue(unsigned long unit_address,\n+\t\t\t\t\t    unsigned long queue_handle)\n+{\n+\treturn plpar_hcall_norets(H_FREE_LOGICAL_LAN_QUEUE,\n+\t\t\t\t  unit_address, queue_handle);\n+}\n+\n+/**\n+ * h_register_logical_lan_with_handle - Register primary queue and get handle\n+ * @unit_address: Device unit address\n+ * @buffer_list: DMA address of buffer list\n+ * @rec_queue: Buffer descriptor of receive queue\n+ * @filter_list: DMA address of filter list\n+ * @mac_address: MAC address\n+ * @queue_handle: Output parameter for queue handle (may be NULL)\n+ *\n+ * Registers the primary receive queue (queue 0) with the hypervisor and\n+ * returns the queue handle. This is needed in multi-queue mode to use\n+ * h_add_logical_lan_buffers_queue() for all queues including queue 0.\n+ *\n+ * Return: H_SUCCESS (0) on success, error code otherwise\n+ */\n+static inline long\n+h_register_logical_lan_with_handle(unsigned long unit_address,\n+\t\t\t\t   unsigned long buffer_list,\n+\t\t\t\t   unsigned long rec_queue,\n+\t\t\t\t   unsigned long filter_list,\n+\t\t\t\t   unsigned long mac_address,\n+\t\t\t\t   unsigned long *queue_handle)\n+{\n+\tunsigned long retbuf[PLPAR_HCALL_BUFSIZE];\n+\tlong rc;\n+\n+\trc = plpar_hcall(H_REGISTER_LOGICAL_LAN, retbuf,\n+\t\t\t unit_address, buffer_list, rec_queue,\n+\t\t\t filter_list, mac_address);\n+\n+\tif (rc == H_SUCCESS \u0026\u0026 queue_handle)\n+\t\t*queue_handle = retbuf[0];\n+\n+\treturn rc;\n+}\n+\n /* FW allows us to send 6 descriptors but we only use one so mark\n  * the other 5 as unused (0)\n  */\n@@ -121,7 +267,10 @@ static inline long h_illan_attributes(unsigned long unit_address,\n #define IBMVETH_MAX_TX_BUF_SIZE (1024 * 64)\n #define IBMVETH_MAX_QUEUES 16U\n #define IBMVETH_DEFAULT_QUEUES 8U\n-#define IBMVETH_MAX_RX_PER_HCALL 8U\n+#define IBMVETH_MAX_RX_QUEUES 16U\n+#define IBMVETH_DEFAULT_RX_QUEUES 1U\n+#define IBMVETH_MAX_RX_REGULAR 8U\n+#define IBMVETH_MAX_RX_PER_HCALL 12U\n \n static int pool_size[] = { 512, 1024 * 2, 1024 * 16, 1024 * 32, 1024 * 64 };\n static int pool_count[] = { 256, 512, 256, 256, 256 };\n@@ -130,6 +279,43 @@ static int pool_active[] = { 1, 1, 0, 0, 1};\n \n #define IBM_VETH_INVALID_MAP ((u16)0xffff)\n \n+/*\n+ * Per-queue RX counters. No field has two concurrent writers:\n+ * interrupts is written only from this queue's IRQ handler; polls,\n+ * packets, bytes, large_packets and invalid_buffers only from its NAPI\n+ * poll; replenish_* only under its replenish_lock; and no_buffer_drops\n+ * and no_buffer_retired under that lock or from a teardown path already\n+ * quiesced by napi_disable()/synchronize_irq(). Plain u64 is therefore\n+ * sufficient and no atomic or u64_stats_sync is needed: the driver is\n+ * PPC64-only, so 64-bit loads and stores do not tear.\n+ */\n+struct ibmveth_rx_queue_stats {\n+\tu64 packets;\n+\tu64 bytes;\n+\tu64 interrupts;\n+\tu64 polls;\n+\tu64 large_packets;\n+\tu64 invalid_buffers;\n+\t/* PHYP's per-page absolute drop count for the live page. */\n+\tu64 no_buffer_drops;\n+\t/* Absolutes from pages this queue has already retired. */\n+\tu64 no_buffer_retired;\n+\tu64 replenish_task_cycles;\n+\tu64 replenish_no_mem;\n+\tu64 replenish_add_buff_failure;\n+\tu64 replenish_add_buff_success;\n+} ____cacheline_aligned_in_smp;\n+\n+/* Per-queue TX counters; serialized by the stack's per-queue TX lock. */\n+struct ibmveth_tx_queue_stats {\n+\tu64 packets;\n+\tu64 bytes;\n+\tu64 large_packets;\n+\tu64 dropped_packets;\n+\tu64 send_failures;\n+\tu64 checksum_offload;\n+} ____cacheline_aligned_in_smp;\n+\n struct ibmveth_buff_pool {\n     u32 size;\n     u32 index;\n@@ -143,6 +329,7 @@ struct ibmveth_buff_pool {\n     struct sk_buff **skbuff;\n     int active;\n     struct kobject kobj;\n+\tstruct completion released;\n };\n \n struct ibmveth_rx_q {\n@@ -152,23 +339,34 @@ struct ibmveth_rx_q {\n     dma_addr_t queue_dma;\n     u32        queue_len;\n     struct ibmveth_rx_q_entry *queue_addr;\n+\tspinlock_t\treplenish_lock;\t/* per-queue buffer replenish */\n };\n \n struct ibmveth_adapter {\n \tstruct vio_dev *vdev;\n \tstruct net_device *netdev;\n-\tstruct napi_struct napi;\n+\tstruct napi_struct napi[IBMVETH_MAX_RX_QUEUES];\n \tstruct work_struct work;\n \tunsigned int mcastFilterSize;\n-\tvoid *buffer_list_addr;\n+\tvoid *buffer_list_addr[IBMVETH_MAX_RX_QUEUES];\n \tvoid *filter_list_addr;\n \tvoid *tx_ltb_ptr[IBMVETH_MAX_QUEUES];\n \tunsigned int tx_ltb_size;\n \tdma_addr_t tx_ltb_dma[IBMVETH_MAX_QUEUES];\n-\tdma_addr_t buffer_list_dma;\n+\tdma_addr_t buffer_list_dma[IBMVETH_MAX_RX_QUEUES];\n \tdma_addr_t filter_list_dma;\n-\tstruct ibmveth_buff_pool rx_buff_pool[IBMVETH_NUM_BUFF_POOLS];\n-\tstruct ibmveth_rx_q rx_queue;\n+\tstruct ibmveth_buff_pool\n+\t\trx_buff_pool[IBMVETH_MAX_RX_QUEUES][IBMVETH_NUM_BUFF_POOLS];\n+\tstruct ibmveth_rx_q rx_queue[IBMVETH_MAX_RX_QUEUES];\n+\tu64 queue_handle[IBMVETH_MAX_RX_QUEUES];\n+\tunsigned int queue_irq[IBMVETH_MAX_RX_QUEUES];\n+\tbool multi_queue;\n+\tunsigned int num_rx_queues;\n+\tbool mq_fallback;\n+\t/* Lifetime: true after successful ndo_open until close clears it. */\n+\tbool opened;\n+\t/* Lifetime: true while RX IRQ handlers / NAPI are installed. */\n+\tbool rx_irq_setup;\n \tint rx_csum;\n \tint large_send;\n \tbool is_active_trunk;\n@@ -177,17 +375,19 @@ struct ibmveth_adapter {\n \tu64 fw_ipv6_csum_support;\n \tu64 fw_ipv4_csum_support;\n \tu64 fw_large_send_support;\n-\t/* adapter specific stats */\n-\tu64 replenish_task_cycles;\n-\tu64 replenish_no_mem;\n-\tu64 replenish_add_buff_failure;\n-\tu64 replenish_add_buff_success;\n-\tu64 rx_invalid_buffer;\n-\tu64 rx_no_buffer;\n+\t/*\n+\t * Every other ethtool -S counter lives in rx_qstats/tx_qstats and is\n+\t * summed on read. tx_map_failed predates multi-queue, has never been\n+\t * updated by any code path, and is kept only so the key keeps\n+\t * reporting the zero userspace already sees.\n+\t */\n \tu64 tx_map_failed;\n-\tu64 tx_send_failed;\n-\tu64 tx_large_packets;\n-\tu64 rx_large_packets;\n+\n+\tstruct ibmveth_rx_queue_stats *rx_qstats;\n+\tstruct ibmveth_tx_queue_stats *tx_qstats;\n+\n+\tstruct dentry *debugfs_dir;\n+\n \t/* Ethtool settings */\n \tu8 duplex;\n \tu32 speed;\ndiff --git a/tools/perf/scripts/python/powerpc-hcalls.py b/tools/perf/scripts/python/powerpc-hcalls.py\nindex fedce1b68cad1..afc354583b7b5 100644\n--- a/tools/perf/scripts/python/powerpc-hcalls.py\n+++ b/tools/perf/scripts/python/powerpc-hcalls.py\n@@ -217,6 +217,10 @@ hcall_table = {\n \t# Key wrapping hcalls\n \t1168: 'H_PKS_WRAP_OBJECT',\n \t1172: 'H_PKS_UNWRAP_OBJECT',\n+\t# Logical LAN multi-queue RX (PAPR 11.20.00)\n+\t1180: 'H_REG_LOGICAL_LAN_QUEUE',\n+\t1184: 'H_ADD_LOGICAL_LAN_BUFFERS_QUEUE',\n+\t1192: 'H_FREE_LOGICAL_LAN_QUEUE',\n \t# Platform-specific hcalls used by the Ultravisor\n \t61184: 'H_SVM_PAGE_IN',\n \t61188: 'H_SVM_PAGE_OUT',\n"
}

Target architecture: amd64

For your convenience, here is the diff of the changes:
commit 845350ec79fd7e1e12ceb4f4a49bb29fa44d8e9a
Author: syz-cluster <triage@syzkaller.com>
Date:   Sun Sep 27 14:24:02 2026 +0000

    syz-cluster: applied patch under review

diff --git a/arch/powerpc/include/asm/hvcall.h b/arch/powerpc/include/asm/hvcall.h
index dff90a7d7f70b..cb0ea53491e6c 100644
--- a/arch/powerpc/include/asm/hvcall.h
+++ b/arch/powerpc/include/asm/hvcall.h
@@ -362,7 +362,11 @@
 #define H_GUEST_DELETE		0x488
 #define H_PKS_WRAP_OBJECT	0x490
 #define H_PKS_UNWRAP_OBJECT	0x494
-#define MAX_HCALL_OPCODE	H_PKS_UNWRAP_OBJECT
+/* 0x498 reserved; 0x4A4 = H_FREE_LOGICAL_LAN_BUFFER_QUEUE (unused here) */
+#define H_REG_LOGICAL_LAN_QUEUE 0x49C
+#define H_ADD_LOGICAL_LAN_BUFFERS_QUEUE 0x4A0
+#define H_FREE_LOGICAL_LAN_QUEUE 0x4A8
+#define MAX_HCALL_OPCODE	H_FREE_LOGICAL_LAN_QUEUE
 
 /* Scope args for H_SCM_UNBIND_ALL */
 #define H_UNBIND_SCOPE_ALL (0x1)
diff --git a/drivers/net/ethernet/ibm/ibmveth.c b/drivers/net/ethernet/ibm/ibmveth.c
index 73e051d26b9d8..da14c69152110 100644
--- a/drivers/net/ethernet/ibm/ibmveth.c
+++ b/drivers/net/ethernet/ibm/ibmveth.c
@@ -21,6 +21,8 @@
 #include <linux/skbuff.h>
 #include <linux/init.h>
 #include <linux/interrupt.h>
+#include <linux/irq.h>
+#include <linux/irqdomain.h>
 #include <linux/mm.h>
 #include <linux/pm.h>
 #include <linux/ethtool.h>
@@ -28,6 +30,8 @@
 #include <linux/ip.h>
 #include <linux/ipv6.h>
 #include <linux/slab.h>
+#include <linux/spinlock.h>
+#include <linux/debugfs.h>
 #include <asm/hvcall.h>
 #include <linux/atomic.h>
 #include <asm/vio.h>
@@ -35,6 +39,7 @@
 #include <asm/firmware.h>
 #include <net/tcp.h>
 #include <net/ip6_checksum.h>
+#include <net/netdev_queues.h>
 
 #include "ibmveth.h"
 
@@ -72,748 +77,2940 @@ module_param(old_large_send, bool, 0444);
 MODULE_PARM_DESC(old_large_send,
 	"Use old large send method on firmware that supports the new method");
 
+/**
+ * enum ibmveth_stat_src - where an ethtool -S counter is stored
+ * @IBMVETH_STAT_ADAPTER: plain u64 in struct ibmveth_adapter
+ * @IBMVETH_STAT_RX_QSUM: per-queue u64, summed over rx_qstats[]
+ * @IBMVETH_STAT_TX_QSUM: per-queue u64, summed over tx_qstats[]
+ * @IBMVETH_STAT_RX_NO_BUFFER: rx_qstats[] live-page absolute plus the
+ *	absolutes carried over from pages the queue has already retired
+ *
+ * Counters live per-queue so multi-queue writers never share a field.
+ * The adapter is only ever read from ethtool, so summing there is free.
+ */
+enum ibmveth_stat_src {
+	IBMVETH_STAT_ADAPTER,
+	IBMVETH_STAT_RX_QSUM,
+	IBMVETH_STAT_TX_QSUM,
+	IBMVETH_STAT_RX_NO_BUFFER,
+};
+
 struct ibmveth_stat {
 	char name[ETH_GSTRING_LEN];
-	int offset;
+	enum ibmveth_stat_src src;
+	/* Offset into the struct named by @src. */
+	size_t off;
 };
 
 #define IBMVETH_STAT_OFF(stat) offsetof(struct ibmveth_adapter, stat)
+#define IBMVETH_RXQ_OFF(stat) offsetof(struct ibmveth_rx_queue_stats, stat)
+#define IBMVETH_TXQ_OFF(stat) offsetof(struct ibmveth_tx_queue_stats, stat)
 #define IBMVETH_GET_STAT(a, off) *((u64 *)(((unsigned long)(a)) + off))
 
+#define IBMVETH_ADAPTER_STAT(key, field) \
+	{ key, IBMVETH_STAT_ADAPTER, IBMVETH_STAT_OFF(field) }
+#define IBMVETH_RXQ_STAT(key, field) \
+	{ key, IBMVETH_STAT_RX_QSUM, IBMVETH_RXQ_OFF(field) }
+#define IBMVETH_TXQ_STAT(key, field) \
+	{ key, IBMVETH_STAT_TX_QSUM, IBMVETH_TXQ_OFF(field) }
+
+/*
+ * Key names and their order are ABI. Do not reorder or rename; append
+ * only, and only when the counter is worth a permanent interface.
+ */
 static struct ibmveth_stat ibmveth_stats[] = {
-	{ "replenish_task_cycles", IBMVETH_STAT_OFF(replenish_task_cycles) },
-	{ "replenish_no_mem", IBMVETH_STAT_OFF(replenish_no_mem) },
-	{ "replenish_add_buff_failure",
-			IBMVETH_STAT_OFF(replenish_add_buff_failure) },
-	{ "replenish_add_buff_success",
-			IBMVETH_STAT_OFF(replenish_add_buff_success) },
-	{ "rx_invalid_buffer", IBMVETH_STAT_OFF(rx_invalid_buffer) },
-	{ "rx_no_buffer", IBMVETH_STAT_OFF(rx_no_buffer) },
-	{ "tx_map_failed", IBMVETH_STAT_OFF(tx_map_failed) },
-	{ "tx_send_failed", IBMVETH_STAT_OFF(tx_send_failed) },
-	{ "fw_enabled_ipv4_csum", IBMVETH_STAT_OFF(fw_ipv4_csum_support) },
-	{ "fw_enabled_ipv6_csum", IBMVETH_STAT_OFF(fw_ipv6_csum_support) },
-	{ "tx_large_packets", IBMVETH_STAT_OFF(tx_large_packets) },
-	{ "rx_large_packets", IBMVETH_STAT_OFF(rx_large_packets) },
-	{ "fw_enabled_large_send", IBMVETH_STAT_OFF(fw_large_send_support) }
+	IBMVETH_RXQ_STAT("replenish_task_cycles", replenish_task_cycles),
+	IBMVETH_RXQ_STAT("replenish_no_mem", replenish_no_mem),
+	IBMVETH_RXQ_STAT("replenish_add_buff_failure",
+			 replenish_add_buff_failure),
+	IBMVETH_RXQ_STAT("replenish_add_buff_success",
+			 replenish_add_buff_success),
+	IBMVETH_RXQ_STAT("rx_invalid_buffer", invalid_buffers),
+	{ "rx_no_buffer", IBMVETH_STAT_RX_NO_BUFFER,
+	  IBMVETH_RXQ_OFF(no_buffer_drops) },
+	IBMVETH_ADAPTER_STAT("tx_map_failed", tx_map_failed),
+	IBMVETH_TXQ_STAT("tx_send_failed", send_failures),
+	IBMVETH_ADAPTER_STAT("fw_enabled_ipv4_csum", fw_ipv4_csum_support),
+	IBMVETH_ADAPTER_STAT("fw_enabled_ipv6_csum", fw_ipv6_csum_support),
+	IBMVETH_TXQ_STAT("tx_large_packets", large_packets),
+	IBMVETH_RXQ_STAT("rx_large_packets", large_packets),
+	IBMVETH_ADAPTER_STAT("fw_enabled_large_send", fw_large_send_support),
+};
+
+/**
+ * struct ibmveth_qstat - a per-queue counter exposed through ethtool -S
+ * @fmt: key name, taking the queue index as its only argument
+ * @off: offset into the matching per-queue stats struct
+ *
+ * Driving the strings and the values from one table keeps the two in
+ * step; get_sset_count() derives its length from ARRAY_SIZE() so the
+ * three cannot drift apart.
+ */
+struct ibmveth_qstat {
+	const char *fmt;
+	size_t off;
+};
+
+/*
+ * Only counters with no home in the standard interfaces belong here.
+ * packets, bytes and drops are reported through netdev_stat_ops.
+ */
+static const struct ibmveth_qstat ibmveth_rx_qstat_keys[] = {
+	{ "rx%d_interrupts", IBMVETH_RXQ_OFF(interrupts) },
+	{ "rx%d_polls", IBMVETH_RXQ_OFF(polls) },
+	{ "rx%d_large_packets", IBMVETH_RXQ_OFF(large_packets) },
+	{ "rx%d_invalid_buffers", IBMVETH_RXQ_OFF(invalid_buffers) },
+	{ "rx%d_no_buffer_drops", IBMVETH_RXQ_OFF(no_buffer_drops) },
+};
+
+static const struct ibmveth_qstat ibmveth_tx_qstat_keys[] = {
+	{ "tx%d_large_packets", IBMVETH_TXQ_OFF(large_packets) },
+	{ "tx%d_send_failures", IBMVETH_TXQ_OFF(send_failures) },
+	{ "tx%d_checksum_offload", IBMVETH_TXQ_OFF(checksum_offload) },
 };
 
+#define IBMVETH_NUM_RX_QSTATS ARRAY_SIZE(ibmveth_rx_qstat_keys)
+#define IBMVETH_NUM_TX_QSTATS ARRAY_SIZE(ibmveth_tx_qstat_keys)
+
 /* simple methods of getting data from the current rxq entry */
-static inline u32 ibmveth_rxq_flags(struct ibmveth_adapter *adapter)
+static u32 ibmveth_rxq_flags(struct ibmveth_adapter *adapter,
+			     int queue_index)
 {
-	return be32_to_cpu(adapter->rx_queue.queue_addr[adapter->rx_queue.index].flags_off);
+	struct ibmveth_rx_q *rxq = &adapter->rx_queue[queue_index];
+
+	return be32_to_cpu(rxq->queue_addr[rxq->index].flags_off);
 }
 
-static inline int ibmveth_rxq_toggle(struct ibmveth_adapter *adapter)
+static int ibmveth_rxq_toggle(struct ibmveth_adapter *adapter,
+			      int queue_index)
 {
-	return (ibmveth_rxq_flags(adapter) & IBMVETH_RXQ_TOGGLE) >>
-			IBMVETH_RXQ_TOGGLE_SHIFT;
+	return (ibmveth_rxq_flags(adapter, queue_index) & IBMVETH_RXQ_TOGGLE) >>
+		IBMVETH_RXQ_TOGGLE_SHIFT;
 }
 
-static inline int ibmveth_rxq_pending_buffer(struct ibmveth_adapter *adapter)
+static int ibmveth_rxq_pending_buffer(struct ibmveth_adapter *adapter,
+				      int queue_index)
 {
-	return ibmveth_rxq_toggle(adapter) == adapter->rx_queue.toggle;
+	return ibmveth_rxq_toggle(adapter, queue_index) ==
+		adapter->rx_queue[queue_index].toggle;
 }
 
-static inline int ibmveth_rxq_buffer_valid(struct ibmveth_adapter *adapter)
+static int ibmveth_rxq_buffer_valid(struct ibmveth_adapter *adapter,
+				    int queue_index)
 {
-	return ibmveth_rxq_flags(adapter) & IBMVETH_RXQ_VALID;
+	return ibmveth_rxq_flags(adapter, queue_index) & IBMVETH_RXQ_VALID;
 }
 
-static inline int ibmveth_rxq_frame_offset(struct ibmveth_adapter *adapter)
+static int ibmveth_rxq_frame_offset(struct ibmveth_adapter *adapter,
+				    int queue_index)
 {
-	return ibmveth_rxq_flags(adapter) & IBMVETH_RXQ_OFF_MASK;
+	return ibmveth_rxq_flags(adapter, queue_index) & IBMVETH_RXQ_OFF_MASK;
 }
 
-static inline int ibmveth_rxq_large_packet(struct ibmveth_adapter *adapter)
+static int ibmveth_rxq_large_packet(struct ibmveth_adapter *adapter,
+				    int queue_index)
 {
-	return ibmveth_rxq_flags(adapter) & IBMVETH_RXQ_LRG_PKT;
+	return ibmveth_rxq_flags(adapter, queue_index) & IBMVETH_RXQ_LRG_PKT;
 }
 
-static inline int ibmveth_rxq_frame_length(struct ibmveth_adapter *adapter)
+static int ibmveth_rxq_frame_length(struct ibmveth_adapter *adapter,
+				    int queue_index)
 {
-	return be32_to_cpu(adapter->rx_queue.queue_addr[adapter->rx_queue.index].length);
+	struct ibmveth_rx_q *rxq = &adapter->rx_queue[queue_index];
+
+	return be32_to_cpu(rxq->queue_addr[rxq->index].length);
 }
 
-static inline int ibmveth_rxq_csum_good(struct ibmveth_adapter *adapter)
+static int ibmveth_rxq_csum_good(struct ibmveth_adapter *adapter,
+				 int queue_index)
 {
-	return ibmveth_rxq_flags(adapter) & IBMVETH_RXQ_CSUM_GOOD;
+	return ibmveth_rxq_flags(adapter, queue_index) & IBMVETH_RXQ_CSUM_GOOD;
 }
 
-static unsigned int ibmveth_real_max_tx_queues(void)
+/* Lockless IRQ/poll readers vs resize publishers. */
+static unsigned int
+ibmveth_get_num_rx_queues(const struct ibmveth_adapter *adapter)
 {
-	unsigned int n_cpu = num_online_cpus();
-
-	return min(n_cpu, IBMVETH_MAX_QUEUES);
+	/*
+	 * Pairs with the release in ibmveth_publish_num_rx_queues(): a reader
+	 * that sees the new count also sees the per-queue state behind it.
+	 */
+	return smp_load_acquire(&adapter->num_rx_queues);
 }
 
-/* setup the initial settings for a buffer pool */
-static void ibmveth_init_buffer_pool(struct ibmveth_buff_pool *pool,
-				     u32 pool_index, u32 pool_size,
-				     u32 buff_size, u32 pool_active)
+static void
+ibmveth_publish_num_rx_queues(struct ibmveth_adapter *adapter,
+			      unsigned int num)
 {
-	pool->size = pool_size;
-	pool->index = pool_index;
-	pool->buff_size = buff_size;
-	pool->threshold = pool_size * 7 / 8;
-	pool->active = pool_active;
+	/*
+	 * Pairs with the acquire in ibmveth_get_num_rx_queues(): per-queue
+	 * state must be visible to a reader before it observes the new count.
+	 */
+	smp_store_release(&adapter->num_rx_queues, num);
 }
 
-/* allocate and setup an buffer pool - called during open */
-static int ibmveth_alloc_buffer_pool(struct ibmveth_buff_pool *pool)
+static unsigned int ibmveth_real_max_tx_queues(void)
 {
-	int i;
+	unsigned int n_cpu = num_online_cpus();
 
-	pool->free_map = kmalloc_array(pool->size, sizeof(u16), GFP_KERNEL);
+	return min(n_cpu, IBMVETH_MAX_QUEUES);
+}
 
-	if (!pool->free_map)
-		return -1;
+/**
+ * ibmveth_alloc_filter_list - Allocate and map filter list
+ * @adapter: ibmveth adapter structure
+ *
+ * Return: 0 on success, negative error code on failure
+ */
+static int
+ibmveth_alloc_filter_list(struct ibmveth_adapter *adapter)
+{
+	struct device *dev = &adapter->vdev->dev;
+	struct net_device *netdev = adapter->netdev;
 
-	pool->dma_addr = kzalloc_objs(dma_addr_t, pool->size);
-	if (!pool->dma_addr) {
-		kfree(pool->free_map);
-		pool->free_map = NULL;
-		return -1;
+	adapter->filter_list_addr = (void *)get_zeroed_page(GFP_KERNEL);
+	if (!adapter->filter_list_addr) {
+		netdev_err(netdev, "unable to allocate filter pages\n");
+		return -ENOMEM;
 	}
 
-	pool->skbuff = kcalloc(pool->size, sizeof(void *), GFP_KERNEL);
-
-	if (!pool->skbuff) {
-		kfree(pool->dma_addr);
-		pool->dma_addr = NULL;
-
-		kfree(pool->free_map);
-		pool->free_map = NULL;
-		return -1;
+	adapter->filter_list_dma = dma_map_single(dev,
+						  adapter->filter_list_addr,
+						  4096, DMA_BIDIRECTIONAL);
+	if (dma_mapping_error(dev, adapter->filter_list_dma)) {
+		netdev_err(netdev, "unable to map filter list pages\n");
+		free_page((unsigned long)adapter->filter_list_addr);
+		adapter->filter_list_addr = NULL;
+		/* Do not leave DMA_MAPPING_ERROR for free_filter_list(). */
+		adapter->filter_list_dma = 0;
+		return -ENOMEM;
 	}
 
-	for (i = 0; i < pool->size; ++i)
-		pool->free_map[i] = i;
-
-	atomic_set(&pool->available, 0);
-	pool->producer_index = 0;
-	pool->consumer_index = 0;
+	netdev_dbg(netdev, "filter list @ 0x%p (DMA: 0x%llx)\n",
+		   adapter->filter_list_addr,
+		   (unsigned long long)adapter->filter_list_dma);
 
 	return 0;
 }
 
-static inline void ibmveth_flush_buffer(void *addr, unsigned long length)
+/**
+ * ibmveth_free_filter_list - Free filter list resources
+ * @adapter: ibmveth adapter structure
+ */
+static void
+ibmveth_free_filter_list(struct ibmveth_adapter *adapter)
 {
-	unsigned long offset;
+	struct device *dev = &adapter->vdev->dev;
 
-	for (offset = 0; offset < length; offset += SMP_CACHE_BYTES)
-		asm("dcbf %0,%1,1" :: "b" (addr), "r" (offset));
+	/* Unmap by CPU pointer: SPAPR can return DMA address 0. */
+	if (adapter->filter_list_addr) {
+		dma_unmap_single(dev, adapter->filter_list_dma, 4096,
+				 DMA_BIDIRECTIONAL);
+		adapter->filter_list_dma = 0;
+		free_page((unsigned long)adapter->filter_list_addr);
+		adapter->filter_list_addr = NULL;
+	}
 }
 
-/* replenish the buffers for a pool.  note that we don't need to
- * skb_reserve these since they are used for incoming...
+/**
+ * ibmveth_alloc_rx_qstats - Allocate per-queue RX statistics
+ * @adapter: ibmveth adapter structure
+ *
+ * Return: 0 on success, -ENOMEM on failure
  */
-static void ibmveth_replenish_buffer_pool(struct ibmveth_adapter *adapter,
-					  struct ibmveth_buff_pool *pool)
+static int ibmveth_alloc_rx_qstats(struct ibmveth_adapter *adapter)
 {
-	union ibmveth_buf_desc descs[IBMVETH_MAX_RX_PER_HCALL] = {0};
-	u32 remaining = pool->size - atomic_read(&pool->available);
-	u64 correlators[IBMVETH_MAX_RX_PER_HCALL] = {0};
-	unsigned long lpar_rc;
-	u32 buffers_added = 0;
-	u32 i, filled, batch;
-	struct vio_dev *vdev;
-	dma_addr_t dma_addr;
-	struct device *dev;
-	u32 index;
+	adapter->rx_qstats = kcalloc(IBMVETH_MAX_RX_QUEUES,
+				     sizeof(*adapter->rx_qstats),
+				     GFP_KERNEL);
+	if (!adapter->rx_qstats)
+		return -ENOMEM;
 
-	vdev = adapter->vdev;
-	dev = &vdev->dev;
+	return 0;
+}
 
-	mb();
+/**
+ * ibmveth_free_rx_qstats - Free per-queue RX statistics
+ * @adapter: ibmveth adapter structure
+ */
+static void ibmveth_free_rx_qstats(struct ibmveth_adapter *adapter)
+{
+	kfree(adapter->rx_qstats);
+	adapter->rx_qstats = NULL;
+}
 
-	batch = adapter->rx_buffers_per_hcall;
+/**
+ * ibmveth_alloc_tx_qstats - Allocate per-queue TX statistics
+ * @adapter: ibmveth adapter structure
+ *
+ * Return: 0 on success, -ENOMEM on failure
+ */
+static int ibmveth_alloc_tx_qstats(struct ibmveth_adapter *adapter)
+{
+	adapter->tx_qstats = kcalloc(IBMVETH_MAX_QUEUES,
+				     sizeof(*adapter->tx_qstats),
+				     GFP_KERNEL);
+	if (!adapter->tx_qstats)
+		return -ENOMEM;
 
-	while (remaining > 0) {
-		unsigned int free_index = pool->consumer_index;
+	return 0;
+}
 
-		/* Fill a batch of descriptors */
-		for (filled = 0; filled < min(remaining, batch); filled++) {
-			index = pool->free_map[free_index];
-			if (WARN_ON(index == IBM_VETH_INVALID_MAP)) {
-				adapter->replenish_add_buff_failure++;
-				netdev_info(adapter->netdev,
-					    "Invalid map index %u, reset\n",
-					    index);
-				schedule_work(&adapter->work);
-				break;
-			}
+/**
+ * ibmveth_free_tx_qstats - Free per-queue TX statistics
+ * @adapter: ibmveth adapter structure
+ */
+static void ibmveth_free_tx_qstats(struct ibmveth_adapter *adapter)
+{
+	kfree(adapter->tx_qstats);
+	adapter->tx_qstats = NULL;
+}
 
-			if (!pool->skbuff[index]) {
-				struct sk_buff *skb = NULL;
+/**
+ * ibmveth_alloc_rx_queues - Allocate per-queue RX resources
+ * @adapter: ibmveth adapter structure
+ * @rxq_entries: Number of entries per RX queue
+ *
+ * Return: 0 on success, negative error code on failure
+ */
+static int
+ibmveth_alloc_rx_queues(struct ibmveth_adapter *adapter, int rxq_entries)
+{
+	struct device *dev = &adapter->vdev->dev;
+	struct net_device *netdev = adapter->netdev;
+	int i;
 
-				skb = netdev_alloc_skb(adapter->netdev,
-						       pool->buff_size);
-				if (!skb) {
-					adapter->replenish_no_mem++;
-					adapter->replenish_add_buff_failure++;
-					break;
-				}
+	for (i = 0; i < ibmveth_get_num_rx_queues(adapter); i++) {
+		adapter->buffer_list_addr[i] =
+			(void *)get_zeroed_page(GFP_KERNEL);
+		if (!adapter->buffer_list_addr[i]) {
+			netdev_err(netdev,
+				   "unable to allocate buffer list for queue %d\n",
+				   i);
+			goto err_cleanup;
+		}
 
-				dma_addr = dma_map_single(dev, skb->data,
-							  pool->buff_size,
-							  DMA_FROM_DEVICE);
-				if (dma_mapping_error(dev, dma_addr)) {
-					dev_kfree_skb_any(skb);
-					adapter->replenish_add_buff_failure++;
-					break;
-				}
+		adapter->rx_queue[i].queue_len =
+			sizeof(struct ibmveth_rx_q_entry) * rxq_entries;
+		adapter->rx_queue[i].queue_addr =
+			dma_alloc_coherent(dev, adapter->rx_queue[i].queue_len,
+					   &adapter->rx_queue[i].queue_dma,
+					   GFP_KERNEL);
+		if (!adapter->rx_queue[i].queue_addr) {
+			netdev_err(netdev,
+				   "unable to allocate RX queue for queue %d\n",
+				   i);
+			free_page((unsigned long)adapter->buffer_list_addr[i]);
+			adapter->buffer_list_addr[i] = NULL;
+			goto err_cleanup;
+		}
 
-				pool->dma_addr[index] = dma_addr;
-				pool->skbuff[index] = skb;
-			} else {
-				/* re-use case */
-				dma_addr = pool->dma_addr[index];
-			}
+		adapter->buffer_list_dma[i] =
+			dma_map_single(dev, adapter->buffer_list_addr[i],
+				       4096, DMA_BIDIRECTIONAL);
+		if (dma_mapping_error(dev, adapter->buffer_list_dma[i])) {
+			netdev_err(netdev,
+				   "unable to map buffer list for queue %d\n",
+				   i);
+			free_page((unsigned long)adapter->buffer_list_addr[i]);
+			adapter->buffer_list_addr[i] = NULL;
+			adapter->buffer_list_dma[i] = 0;
+			goto err_cleanup;
+		}
 
-			if (rx_flush) {
-				unsigned int len;
+		adapter->rx_queue[i].index = 0;
+		adapter->rx_queue[i].num_slots = rxq_entries;
+		adapter->rx_queue[i].toggle = 1;
 
-				len = adapter->netdev->mtu + IBMVETH_BUFF_OH;
-				len = min(pool->buff_size, len);
-				ibmveth_flush_buffer(pool->skbuff[index]->data,
-						     len);
-			}
+		netdev_dbg(netdev, "queue %d: buffer_list @ 0x%p (DMA: 0x%llx), rx_queue @ 0x%p (DMA: 0x%llx), %llu entries\n",
+			   i, adapter->buffer_list_addr[i],
+			   (unsigned long long)adapter->buffer_list_dma[i],
+			   adapter->rx_queue[i].queue_addr,
+			   (unsigned long long)adapter->rx_queue[i].queue_dma,
+			   (unsigned long long)rxq_entries);
+	}
 
-			descs[filled].fields.flags_len = IBMVETH_BUF_VALID |
-							  pool->buff_size;
-			descs[filled].fields.address = dma_addr;
+	netdev_dbg(netdev, "allocated %u RX queue(s) with %d entries each\n",
+		   ibmveth_get_num_rx_queues(adapter), rxq_entries);
 
-			correlators[filled] = ((u64)pool->index << 32) | index;
-			*(u64 *)pool->skbuff[index]->data = correlators[filled];
+	return 0;
 
-			free_index++;
-			if (free_index >= pool->size)
-				free_index = 0;
+err_cleanup:
+	/*
+	 * Free each leftover resource by pointer presence. A
+	 * dma_mapping_error on queue i has already dropped
+	 * buffer_list_addr[i] but can leave rx_queue[i].queue_addr
+	 * live, so this index may be mixed. Do not collapse the
+	 * buffer_list and queue_addr checks.
+	 */
+	for (; i >= 0; i--) {
+		if (adapter->buffer_list_addr[i]) {
+			dma_unmap_single(dev, adapter->buffer_list_dma[i],
+					 4096, DMA_BIDIRECTIONAL);
+			adapter->buffer_list_dma[i] = 0;
 		}
-
-		if (!filled)
-			break;
-
-		/* single buffer case*/
-		if (filled == 1)
-			lpar_rc = h_add_logical_lan_buffer(vdev->unit_address,
-							   descs[0].desc);
-		else
-			/* Multi-buffer hcall */
-			lpar_rc = h_add_logical_lan_buffers(vdev->unit_address,
-							    descs[0].desc,
-							    descs[1].desc,
-							    descs[2].desc,
-							    descs[3].desc,
-							    descs[4].desc,
-							    descs[5].desc,
-							    descs[6].desc,
-							    descs[7].desc);
-		if (lpar_rc != H_SUCCESS) {
-			dev_warn_ratelimited(dev,
-					     "RX h_add_logical_lan failed: filled=%u, rc=%lu, batch=%u\n",
-					     filled, lpar_rc, batch);
-			goto hcall_failure;
+		if (adapter->rx_queue[i].queue_addr) {
+			dma_free_coherent(dev, adapter->rx_queue[i].queue_len,
+					  adapter->rx_queue[i].queue_addr,
+					  adapter->rx_queue[i].queue_dma);
+			adapter->rx_queue[i].queue_addr = NULL;
+		}
+		if (adapter->buffer_list_addr[i]) {
+			free_page((unsigned long)adapter->buffer_list_addr[i]);
+			adapter->buffer_list_addr[i] = NULL;
 		}
+	}
 
-		/* Only update pool state after hcall succeeds */
-		for (i = 0; i < filled; i++) {
-			free_index = pool->consumer_index;
-			pool->free_map[free_index] = IBM_VETH_INVALID_MAP;
+	return -ENOMEM;
+}
 
-			pool->consumer_index++;
-			if (pool->consumer_index >= pool->size)
-				pool->consumer_index = 0;
-		}
+/**
+ * ibmveth_cleanup_rx_resources - Free all RX queue resources
+ * @adapter: ibmveth adapter structure
+ */
+static void
+ibmveth_cleanup_rx_resources(struct ibmveth_adapter *adapter)
+{
+	struct device *dev = &adapter->vdev->dev;
+	int i;
 
-		buffers_added += filled;
-		adapter->replenish_add_buff_success += filled;
-		remaining -= filled;
+	netdev_dbg(adapter->netdev, "cleaning up %u RX queue(s)\n",
+		   ibmveth_get_num_rx_queues(adapter));
 
-		memset(&descs, 0, sizeof(descs));
-		memset(&correlators, 0, sizeof(correlators));
-		continue;
+	for (i = 0; i < ibmveth_get_num_rx_queues(adapter); i++) {
+		if (adapter->buffer_list_addr[i]) {
+			dma_unmap_single(dev, adapter->buffer_list_dma[i],
+					 4096, DMA_BIDIRECTIONAL);
+			adapter->buffer_list_dma[i] = 0;
+		}
 
-hcall_failure:
-		for (i = 0; i < filled; i++) {
-			index = correlators[i] & 0xffffffffUL;
-			dma_addr =  pool->dma_addr[index];
+		if (adapter->rx_queue[i].queue_addr) {
+			dma_free_coherent(dev, adapter->rx_queue[i].queue_len,
+					  adapter->rx_queue[i].queue_addr,
+					  adapter->rx_queue[i].queue_dma);
+			adapter->rx_queue[i].queue_addr = NULL;
+		}
 
-			if (pool->skbuff[index]) {
-				if (dma_addr &&
-				    !dma_mapping_error(dev, dma_addr))
-					dma_unmap_single(dev, dma_addr,
-							 pool->buff_size,
-							 DMA_FROM_DEVICE);
+		if (adapter->buffer_list_addr[i]) {
+			free_page((unsigned long)adapter->buffer_list_addr[i]);
+			adapter->buffer_list_addr[i] = NULL;
+		}
+	}
+}
 
-				dev_kfree_skb_any(pool->skbuff[index]);
-				pool->skbuff[index] = NULL;
-			}
+/**
+ * ibmveth_toggle_irq - Common helper to enable/disable queue interrupts
+ * @adapter: ibmveth adapter structure
+ * @queue_index: Index of the queue (0 for primary, 1+ for subordinate)
+ * @enable: true to enable, false to disable
+ *
+ * For queue 0 (primary), uses h_vio_signal() as it's registered via
+ * h_register_logical_lan(). For subordinate queues (1+), uses H_VIOCTL
+ * with H_ENABLE/DISABLE_VIO_INTERRUPT for per-queue interrupt control.
+ *
+ * Return: 0 on success, negative errno on failure (never raw H_*).
+ */
+static int
+ibmveth_toggle_irq(struct ibmveth_adapter *adapter, int queue_index,
+		   bool enable)
+{
+	unsigned long h_rc;
+	unsigned long irq = adapter->queue_irq[queue_index];
+	const char *action = enable ? "enable" : "disable";
+
+	if (queue_index == 0) {
+		/* Primary queue: use h_vio_signal() */
+		h_rc = h_vio_signal(adapter->vdev->unit_address,
+				    enable ? VIO_IRQ_ENABLE : VIO_IRQ_DISABLE);
+	} else {
+		/* Subordinate queues: use H_VIOCTL with hardware IRQ */
+		struct irq_data *irq_data = irq_get_irq_data(irq);
+		irq_hw_number_t hwirq;
+		u64 vioctl_cmd = enable ? H_ENABLE_VIO_INTERRUPT :
+			H_DISABLE_VIO_INTERRUPT;
+
+		if (!irq_data) {
+			netdev_err(adapter->netdev,
+				   "Failed to get IRQ data for queue %d (virq=%lu)\n",
+				   queue_index, irq);
+			return -EINVAL;
 		}
-		adapter->replenish_add_buff_failure += filled;
+
+		hwirq = irqd_to_hwirq(irq_data);
+		h_rc = plpar_hcall_norets(H_VIOCTL,
+					  adapter->vdev->unit_address,
+					  vioctl_cmd,
+					  hwirq, 0, 0);
 
 		/*
-		 * If multi rx buffers hcall is no longer supported by FW
-		 * e.g. in the case of Live Partition Migration
+		 * H_PARAMETER is ambiguous (already in requested state vs bad
+		 * args). Fold only on disable as an idempotent mask. On enable
+		 * keep it an error so a stuck-masked queue stays visible to
+		 * poll/resize recovery.
 		 */
-		if (batch > 1 && lpar_rc == H_FUNCTION) {
-			/*
-			 * Instead of retry submit single buffer individually
-			 * here just set the max rx buffer per hcall to 1
-			 * buffers will be respleshed next time
-			 * when ibmveth_replenish_buffer_pool() is called again
-			 * with single-buffer case
-			 */
-			netdev_info(adapter->netdev,
-				    "RX Multi buffers not supported by FW, rc=%lu\n",
-				    lpar_rc);
-			adapter->rx_buffers_per_hcall = 1;
-			netdev_info(adapter->netdev,
-				    "Next rx replesh will fall back to single-buffer hcall\n");
+		if (h_rc == H_PARAMETER && !enable) {
+			dev_warn_ratelimited(&adapter->netdev->dev,
+					     "H_VIOCTL %s IRQ returned H_PARAMETER for queue %d (hwirq=%lu)\n",
+					     action, queue_index, hwirq);
+			return 0;
 		}
-		break;
 	}
 
-	mb();
-	atomic_add(buffers_added, &(pool->available));
+	if (h_rc) {
+		dev_err_ratelimited(&adapter->netdev->dev,
+				    "Failed to %s IRQ for queue %d, rc=0x%lx\n",
+				    action, queue_index, h_rc);
+		return -EIO;
+	}
+	return 0;
 }
 
-/*
- * The final 8 bytes of the buffer list is a counter of frames dropped
- * because there was not a buffer in the buffer list capable of holding
- * the frame.
+/**
+ * ibmveth_disable_irq - Disable interrupt for a specific queue
+ * @adapter: ibmveth adapter structure
+ * @queue_index: Index of the queue (0 for primary, 1+ for subordinate)
+ *
+ * Return: 0 on success, negative errno on failure
  */
-static void ibmveth_update_rx_no_buffer(struct ibmveth_adapter *adapter)
+static int
+ibmveth_disable_irq(struct ibmveth_adapter *adapter, int queue_index)
 {
-	__be64 *p = adapter->buffer_list_addr + 4096 - 8;
-
-	adapter->rx_no_buffer = be64_to_cpup(p);
+	return ibmveth_toggle_irq(adapter, queue_index, false);
 }
 
-/* replenish routine */
-static void ibmveth_replenish_task(struct ibmveth_adapter *adapter)
+/**
+ * ibmveth_enable_irq - Enable interrupt for a specific queue
+ * @adapter: ibmveth adapter structure
+ * @queue_index: Index of the queue (0 for primary, 1+ for subordinate)
+ *
+ * Return: 0 on success, negative errno on failure
+ */
+static int
+ibmveth_enable_irq(struct ibmveth_adapter *adapter, int queue_index)
 {
-	int i;
-
-	adapter->replenish_task_cycles++;
+	return ibmveth_toggle_irq(adapter, queue_index, true);
+}
 
-	for (i = (IBMVETH_NUM_BUFF_POOLS - 1); i >= 0; i--) {
-		struct ibmveth_buff_pool *pool = &adapter->rx_buff_pool[i];
+/**
+ * ibmveth_dispose_subordinate_irq_mapping - Drop one subordinate virq mapping
+ * @adapter: ibmveth adapter structure
+ * @queue_idx: RX queue index (1..N)
+ *
+ * Subordinate queues get mappings from irq_create_mapping() during PHYP
+ * registration. Queue 0 uses netdev->irq from device tree and is left alone.
+ *
+ * Bound against IBMVETH_MAX_RX_QUEUES, not num_rx_queues: a caller may
+ * dispose a queue that is no longer in the published live set but still
+ * owns a virq in queue_irq[]. Contrast with the bulk helper, which only
+ * walks 1..num_rx_queues-1 (close / open-fail cleanup of the live set).
+ *
+ * Linux virq lifetime is owned by interrupt cleanup helpers. Call this only
+ * after free_irq() when a handler was installed, or from registration failure
+ * cleanup before request_irq().
+ */
+static void
+ibmveth_dispose_subordinate_irq_mapping(struct ibmveth_adapter *adapter,
+					int queue_idx)
+{
+	if (queue_idx <= 0 || queue_idx >= IBMVETH_MAX_RX_QUEUES)
+		return;
+
+	if (adapter->queue_irq[queue_idx]) {
+		irq_dispose_mapping(adapter->queue_irq[queue_idx]);
+		adapter->queue_irq[queue_idx] = 0;
+	}
+}
+
+/**
+ * ibmveth_dispose_subordinate_irq_mappings - Drop virq mappings for queues 1..N
+ * @adapter: ibmveth adapter structure
+ *
+ * Bulk helper for close / open-fail cleanup of the published live set
+ * (queues 1..num_rx_queues-1). Paths that need a retired or not-yet-published
+ * queue must call ibmveth_dispose_subordinate_irq_mapping() directly.
+ */
+static void
+ibmveth_dispose_subordinate_irq_mappings(struct ibmveth_adapter *adapter)
+{
+	int i;
+
+	for (i = 1; i < ibmveth_get_num_rx_queues(adapter); i++)
+		ibmveth_dispose_subordinate_irq_mapping(adapter, i);
+}
+
+/**
+ * ibmveth_setup_rx_interrupts - Register IRQs and enable NAPI
+ * @adapter: ibmveth adapter structure
+ *
+ * Registers interrupt handlers for all RX queues, enables NAPI, then
+ * enables hypervisor interrupt delivery for multi-queue mode after
+ * every queue has a Linux handler installed. The caller replenishes
+ * every live queue before this helper so traffic during open is not
+ * dropped (PHYP only interrupts after a successful enqueue, which
+ * needs buffers). Single-queue open still leaves PHYP masked here;
+ * restart_rx_queue then kicks NAPI.
+ *
+ * Return: 0 on success, negative error code on failure
+ */
+static int
+ibmveth_setup_rx_interrupts(struct ibmveth_adapter *adapter)
+{
+	struct net_device *netdev = adapter->netdev;
+	int i, rc, num = ibmveth_get_num_rx_queues(adapter);
+
+	for (i = 0; i < num; i++) {
+		if (!adapter->queue_irq[i]) {
+			netdev_err(netdev, "queue %d has invalid IRQ (0)\n", i);
+			rc = -EINVAL;
+			goto err_free_irqs;
+		}
+
+		rc = request_irq(adapter->queue_irq[i], ibmveth_interrupt,
+				 0, netdev->name, &adapter->napi[i]);
+		if (rc) {
+			netdev_err(netdev,
+				   "request_irq() failed for irq 0x%x queue %d: %d\n",
+				   adapter->queue_irq[i], i, rc);
+			goto err_free_irqs;
+		}
+	}
+
+	for (i = 0; i < num; i++)
+		napi_enable(&adapter->napi[i]);
+
+	if (adapter->multi_queue && num > 1) {
+		for (i = 0; i < num; i++) {
+			rc = ibmveth_enable_irq(adapter, i);
+			if (rc) {
+				netdev_err(netdev,
+					   "Failed to enable IRQ for queue %d, rc=%d\n",
+					   i, rc);
+				for (; i >= 0; i--) {
+					ibmveth_disable_irq(adapter, i);
+					synchronize_irq(adapter->queue_irq[i]);
+				}
+				rc = -EIO;
+				goto err_disable_napi;
+			}
+		}
+	}
+
+	/* Set only on full success; fail paths leave this false so a later
+	 * close() / cleanup is a no-op.
+	 */
+	adapter->rx_irq_setup = true;
+	return 0;
+
+err_disable_napi:
+	/* STOP: remask after napi_disable; an in-flight poll can re-arm. */
+	for (i = 0; i < num; i++)
+		napi_disable(&adapter->napi[i]);
+	for (i = 0; i < num; i++) {
+		if (!adapter->queue_irq[i])
+			continue;
+		ibmveth_disable_irq(adapter, i);
+		synchronize_irq(adapter->queue_irq[i]);
+	}
+	for (i = 0; i < num; i++) {
+		if (adapter->queue_irq[i])
+			free_irq(adapter->queue_irq[i], &adapter->napi[i]);
+	}
+	goto err_dispose_mappings;
+
+err_free_irqs:
+	while (--i >= 0)
+		free_irq(adapter->queue_irq[i], &adapter->napi[i]);
+err_dispose_mappings:
+	/* Both setup failure paths own subordinate virq disposal. */
+	ibmveth_dispose_subordinate_irq_mappings(adapter);
+	return rc;
+}
+
+/**
+ * ibmveth_cleanup_rx_interrupts - Mask PHYP IRQs, stop NAPI, and free IRQs
+ * @adapter: ibmveth adapter structure
+ *
+ * Mask and synchronize each queue IRQ before napi_disable() so the handler
+ * cannot miss a PHYP mask while NAPI is already dead. Remask after
+ * napi_disable() in case an in-flight poll re-armed PHYP while we waited.
+ * free_irq() runs only after that. Safe for close and for open failure after
+ * setup_rx_interrupts() already unmasked PHYP. No-op if setup never
+ * succeeded (avoids double napi_disable / free_irq after a failed close+open
+ * while IFF_UP remains set).
+ */
+static void
+ibmveth_cleanup_rx_interrupts(struct ibmveth_adapter *adapter)
+{
+	int i;
+
+	if (!adapter->rx_irq_setup)
+		return;
+
+	for (i = 0; i < ibmveth_get_num_rx_queues(adapter); i++) {
+		if (!adapter->queue_irq[i])
+			continue;
+		ibmveth_disable_irq(adapter, i);
+		synchronize_irq(adapter->queue_irq[i]);
+	}
+
+	for (i = 0; i < ibmveth_get_num_rx_queues(adapter); i++)
+		napi_disable(&adapter->napi[i]);
+
+	for (i = 0; i < ibmveth_get_num_rx_queues(adapter); i++) {
+		if (!adapter->queue_irq[i])
+			continue;
+		ibmveth_disable_irq(adapter, i);
+		synchronize_irq(adapter->queue_irq[i]);
+	}
+
+	for (i = 0; i < ibmveth_get_num_rx_queues(adapter); i++) {
+		if (adapter->queue_irq[i])
+			free_irq(adapter->queue_irq[i], &adapter->napi[i]);
+	}
+
+	ibmveth_dispose_subordinate_irq_mappings(adapter);
+
+	/* Queue 0 uses netdev->irq; leave queue_irq[0] for next open. */
+	adapter->rx_irq_setup = false;
+}
+
+/**
+ * ibmveth_setup_single_rx_interrupt - Setup interrupt for a single RX queue
+ * @adapter: ibmveth adapter structure
+ * @queue_idx: Queue index to setup
+ *
+ * Registers the IRQ handler for one queue. Used during incremental
+ * scale-up when adding new RX queues. The caller publishes the queue,
+ * replenishes buffers, enables NAPI, then unmasks PHYP delivery.
+ *
+ * Return: 0 on success, negative error code on failure
+ */
+static int
+ibmveth_setup_single_rx_interrupt(struct ibmveth_adapter *adapter,
+				  int queue_idx)
+{
+	struct net_device *netdev = adapter->netdev;
+	int rc;
+
+	rc = request_irq(adapter->queue_irq[queue_idx], ibmveth_interrupt,
+			 0, netdev->name, &adapter->napi[queue_idx]);
+	if (rc) {
+		netdev_err(netdev, "request_irq() failed for queue %d: %d\n",
+			   queue_idx, rc);
+		return rc;
+	}
+
+	netdev_dbg(netdev, "Setup IRQ %d for queue %d\n",
+		   adapter->queue_irq[queue_idx], queue_idx);
+	return 0;
+}
+
+/**
+ * ibmveth_cleanup_single_rx_interrupt - Cleanup interrupt for a single RX queue
+ * @adapter: ibmveth adapter structure
+ * @queue_idx: Queue index to cleanup
+ *
+ * Frees the IRQ handler for one queue and releases the subordinate virq
+ * mapping. Used during incremental scale-down.
+ */
+static void
+ibmveth_cleanup_single_rx_interrupt(struct ibmveth_adapter *adapter,
+				    int queue_idx)
+{
+	if (adapter->queue_irq[queue_idx]) {
+		free_irq(adapter->queue_irq[queue_idx],
+			 &adapter->napi[queue_idx]);
+		ibmveth_dispose_subordinate_irq_mapping(adapter, queue_idx);
+		netdev_dbg(adapter->netdev,
+			   "Freed IRQ for queue %d\n", queue_idx);
+	}
+}
+
+/**
+ * ibmveth_schedule_rx_queue - Mask PHYP IRQ and schedule NAPI for one RX queue
+ * @adapter: ibmveth adapter structure
+ * @qindex: RX queue index
+ *
+ * Shared by the IRQ handler and process-context kick sites (open, resume,
+ * pool sysfs, poll_controller).
+ *
+ * Return: true if napi_schedule_prep() succeeded and NAPI was scheduled.
+ * Mask is attempted in that case; a failed disable_irq() is logged by the
+ * helper and does not change the return (queue may still be unmasked).
+ * false if the index is negative (WARN_ON), past the live count (quiet:
+ * a shrink can race netpoll), or prep failed (including NAPI already
+ * scheduled).
+ */
+static bool ibmveth_schedule_rx_queue(struct ibmveth_adapter *adapter,
+				      int qindex)
+{
+	struct napi_struct *napi = &adapter->napi[qindex];
+
+	if (WARN_ON(qindex < 0))
+		return false;
+
+	/*
+	 * A live shrink can publish a lower count while netpoll walks a
+	 * snapshot of the old one, so an index past the end is expected
+	 * here and must not splat. ibmveth_replenish_task() skips the
+	 * same way; callers already treat false as "queue is gone".
+	 */
+	if (qindex >= ibmveth_get_num_rx_queues(adapter))
+		return false;
+
+	/*
+	 * Only mask PHYP when NAPI will run. Masking on prep failure can
+	 * race a completing poll that already re-enabled the queue, leaving
+	 * NAPI idle with the IRQ masked (TX works, RX stalls) until reload.
+	 * Storm prevention on teardown remains in cleanup/disable paths.
+	 */
+	if (napi_schedule_prep(napi)) {
+		/* Failure is already logged with the hcall rc by the helper. */
+		ibmveth_disable_irq(adapter, qindex);
+		__napi_schedule(napi);
+		return true;
+	}
+	return false;
+}
+
+/* setup the initial settings for a buffer pool */
+static void ibmveth_init_buffer_pool(struct ibmveth_buff_pool *pool,
+				     u32 pool_index, u32 pool_size,
+				     u32 buff_size, u32 pool_active)
+{
+	pool->size = pool_size;
+	pool->index = pool_index;
+	pool->buff_size = buff_size;
+	pool->threshold = pool_size * 7 / 8;
+	pool->active = pool_active;
+}
+
+/* allocate and setup an buffer pool - called during open */
+static int ibmveth_alloc_buffer_pool(struct ibmveth_buff_pool *pool)
+{
+	int i;
+
+	pool->free_map = kmalloc_array(pool->size, sizeof(u16), GFP_KERNEL);
+
+	if (!pool->free_map)
+		return -1;
+
+	pool->dma_addr = kzalloc_objs(dma_addr_t, pool->size);
+	if (!pool->dma_addr) {
+		kfree(pool->free_map);
+		pool->free_map = NULL;
+		return -1;
+	}
+
+	pool->skbuff = kcalloc(pool->size, sizeof(void *), GFP_KERNEL);
+
+	if (!pool->skbuff) {
+		kfree(pool->dma_addr);
+		pool->dma_addr = NULL;
+
+		kfree(pool->free_map);
+		pool->free_map = NULL;
+		return -1;
+	}
+
+	for (i = 0; i < pool->size; ++i)
+		pool->free_map[i] = i;
+
+	atomic_set(&pool->available, 0);
+	pool->producer_index = 0;
+	pool->consumer_index = 0;
+
+	return 0;
+}
+
+static inline void ibmveth_flush_buffer(void *addr, unsigned long length)
+{
+	unsigned long offset;
+
+	for (offset = 0; offset < length; offset += SMP_CACHE_BYTES)
+		asm("dcbf %0,%1,1" :: "b" (addr), "r" (offset));
+}
+
+/**
+ * ibmveth_add_logical_lan_buffers - Add receive buffers to hypervisor
+ * @adapter: ibmveth adapter structure
+ * @descs: array of buffer descriptors to add
+ * @filled: number of valid descriptors in the array
+ * @buff_size: size of each buffer (multi-queue mode only)
+ * @queue_index: RX queue index
+ *
+ * Return: hypervisor return code
+ */
+static long ibmveth_add_logical_lan_buffers(struct ibmveth_adapter *adapter,
+					    union ibmveth_buf_desc *descs,
+					    int filled,
+					    unsigned long buff_size,
+					    int queue_index)
+{
+	struct vio_dev *vdev = adapter->vdev;
+	unsigned long rc;
+
+	/*
+	 * The MQ hcall takes six ioba words (12 packed addresses). The
+	 * legacy hcall takes eight descriptors. The argument lists below
+	 * are written out by hand; keep the defines matched to those lists.
+	 */
+	static_assert(IBMVETH_MAX_RX_PER_HCALL == 12);
+	static_assert(IBMVETH_MAX_RX_REGULAR == 8);
+
+	if (adapter->multi_queue) {
+		unsigned long buffersznum = (buff_size << 32) | filled;
+		unsigned long ioba[IBMVETH_MAX_RX_PER_HCALL / 2] = {0};
+		unsigned long handle = adapter->queue_handle[queue_index];
+		int i;
+
+		/* Pack descriptor addresses into ioba pairs.
+		 * Each ioba holds two 32-bit addresses packed into 64 bits:
+		 * - Even descriptors (0,2,4...) go in high 32 bits
+		 * - Odd descriptors (1,3,5...) go in low 32 bits
+		 */
+		for (i = 0; i < filled && i < IBMVETH_MAX_RX_PER_HCALL; i++) {
+			int pair_idx = i / 2;
+			int is_high = (i % 2 == 0);
+
+			if (is_high)
+				ioba[pair_idx] = (unsigned long)
+					descs[i].fields.address << 32;
+			else
+				ioba[pair_idx] |= descs[i].fields.address;
+		}
+
+		rc = h_add_logical_lan_buffers_queue(vdev->unit_address,
+						     handle,
+						     buffersznum,
+						     ioba[0], ioba[1], ioba[2],
+						     ioba[3], ioba[4], ioba[5]);
+	} else if (filled == 1) {
+		rc = h_add_logical_lan_buffer(vdev->unit_address,
+					      descs[0].desc);
+	} else {
+		/* Legacy 8-desc hcall; probe/mq_fallback keep batch <=
+		 * IBMVETH_MAX_RX_REGULAR.
+		 */
+		rc = h_add_logical_lan_buffers(vdev->unit_address,
+					       descs[0].desc, descs[1].desc,
+					       descs[2].desc, descs[3].desc,
+					       descs[4].desc, descs[5].desc,
+					       descs[6].desc, descs[7].desc);
+	}
+
+	return rc;
+}
+
+/* Outcomes for ibmveth_replenish_buffer_pool(); logged after unlock. */
+enum {
+	IBMVETH_REPLENISH_OK = 0,
+	IBMVETH_REPLENISH_RESET_MAP,
+	IBMVETH_REPLENISH_RESET_MQ,
+	IBMVETH_REPLENISH_HCALL_FAIL,
+	IBMVETH_REPLENISH_BATCH_FALLBACK,
+};
+
+struct ibmveth_replenish_fail {
+	unsigned long lpar_rc;
+	u32 filled;
+	u32 batch;
+};
+
+/* Replenish the buffers for a pool.
+ * Caller must hold the per-queue replenish_lock. Do not printk here:
+ * netconsole on the same device can re-enter replenish_task.
+ */
+static int ibmveth_replenish_buffer_pool(struct ibmveth_adapter *adapter,
+					 struct ibmveth_buff_pool *pool,
+					 int queue_index,
+					 struct ibmveth_replenish_fail *fail)
+{
+	struct ibmveth_rx_queue_stats *qstats =
+		&adapter->rx_qstats[queue_index];
+	union ibmveth_buf_desc descs[IBMVETH_MAX_RX_PER_HCALL] = {0};
+	u32 remaining = pool->size - atomic_read(&pool->available);
+	u64 correlators[IBMVETH_MAX_RX_PER_HCALL] = {0};
+	unsigned long lpar_rc;
+	u32 buffers_added = 0;
+	u32 i, filled, batch;
+	struct vio_dev *vdev;
+	dma_addr_t dma_addr;
+	struct device *dev;
+	u32 index;
+	int outcome = IBMVETH_REPLENISH_OK;
+
+	vdev = adapter->vdev;
+	dev = &vdev->dev;
+
+	mb();
+
+	batch = adapter->rx_buffers_per_hcall;
+
+	while (remaining > 0) {
+		unsigned int free_index = pool->consumer_index;
+
+		/* Fill a batch of descriptors */
+		for (filled = 0; filled < min(remaining, batch); filled++) {
+			index = pool->free_map[free_index];
+			if (index == IBM_VETH_INVALID_MAP) {
+				qstats->replenish_add_buff_failure++;
+				outcome = IBMVETH_REPLENISH_RESET_MAP;
+				break;
+			}
+
+			if (!pool->skbuff[index]) {
+				struct sk_buff *skb = NULL;
+
+				skb = netdev_alloc_skb(adapter->netdev,
+						       pool->buff_size);
+				if (!skb) {
+					qstats->replenish_no_mem++;
+					qstats->replenish_add_buff_failure++;
+					break;
+				}
+
+				/* NO_WARN: hold replenish_lock; iommu
+				 * printk can re-enter via netconsole.
+				 */
+				dma_addr =
+					dma_map_single_attrs(dev,
+							     skb->data,
+							     pool->buff_size,
+							     DMA_FROM_DEVICE,
+							     DMA_ATTR_NO_WARN);
+				if (dma_mapping_error(dev, dma_addr)) {
+					dev_kfree_skb_any(skb);
+					qstats->replenish_add_buff_failure++;
+					break;
+				}
+
+				pool->dma_addr[index] = dma_addr;
+				pool->skbuff[index] = skb;
+			} else {
+				/* re-use case */
+				dma_addr = pool->dma_addr[index];
+			}
+
+			if (rx_flush) {
+				unsigned int len;
+
+				len = adapter->netdev->mtu + IBMVETH_BUFF_OH;
+				len = min(pool->buff_size, len);
+				ibmveth_flush_buffer(pool->skbuff[index]->data,
+						     len);
+			}
+
+			descs[filled].fields.flags_len = IBMVETH_BUF_VALID |
+							  pool->buff_size;
+			descs[filled].fields.address = dma_addr;
+
+			correlators[filled] = ((u64)pool->index << 32) | index;
+			*(u64 *)pool->skbuff[index]->data = correlators[filled];
+
+			free_index++;
+			if (free_index >= pool->size)
+				free_index = 0;
+		}
+
+		if (outcome != IBMVETH_REPLENISH_OK)
+			break;
+
+		if (!filled)
+			break;
+
+		lpar_rc = ibmveth_add_logical_lan_buffers(adapter, descs,
+							  filled,
+							  pool->buff_size,
+							  queue_index);
+
+		if (lpar_rc != H_SUCCESS) {
+			fail->lpar_rc = lpar_rc;
+			fail->filled = filled;
+			fail->batch = batch;
+			goto hcall_failure;
+		}
+
+		/* Only update pool state after hcall succeeds */
+		for (i = 0; i < filled; i++) {
+			free_index = pool->consumer_index;
+			pool->free_map[free_index] = IBM_VETH_INVALID_MAP;
+
+			pool->consumer_index++;
+			if (pool->consumer_index >= pool->size)
+				pool->consumer_index = 0;
+		}
+
+		buffers_added += filled;
+		qstats->replenish_add_buff_success += filled;
+		remaining -= filled;
+
+		memset(&descs, 0, sizeof(descs));
+		memset(&correlators, 0, sizeof(correlators));
+		continue;
+
+hcall_failure:
+		for (i = 0; i < filled; i++) {
+			index = correlators[i] & 0xffffffffUL;
+			dma_addr =  pool->dma_addr[index];
+
+			if (pool->skbuff[index]) {
+				if (dma_addr &&
+				    !dma_mapping_error(dev, dma_addr))
+					dma_unmap_single(dev, dma_addr,
+							 pool->buff_size,
+							 DMA_FROM_DEVICE);
+
+				dev_kfree_skb_any(pool->skbuff[index]);
+				pool->skbuff[index] = NULL;
+			}
+		}
+		qstats->replenish_add_buff_failure += filled;
+
+		if (lpar_rc == H_FUNCTION) {
+			if (adapter->multi_queue) {
+				/*
+				 * LPM / firmware may drop MQ buffer hcalls.
+				 * Schedule reset so we do not sit forever in
+				 * no-buffer with the link still up.
+				 */
+				outcome = IBMVETH_REPLENISH_RESET_MQ;
+			} else if (batch > 1) {
+				/*
+				 * Live Partition Migration may drop multi-
+				 * buffer support. Fall back to single-buffer
+				 * on the next replenish; do not continue with
+				 * a stale local batch size (infinite loop).
+				 */
+				adapter->rx_buffers_per_hcall = 1;
+				outcome = IBMVETH_REPLENISH_BATCH_FALLBACK;
+			} else {
+				outcome = IBMVETH_REPLENISH_HCALL_FAIL;
+			}
+		} else {
+			outcome = IBMVETH_REPLENISH_HCALL_FAIL;
+		}
+		break;
+	}
+
+	mb();
+	atomic_add(buffers_added, &(pool->available));
+	return outcome;
+}
+
+/*
+ * The final 8 bytes of the buffer list is a counter of frames dropped
+ * because there was not a buffer in the buffer list capable of holding
+ * the frame.
+ */
+static void ibmveth_update_rx_no_buffer(struct ibmveth_adapter *adapter,
+					int queue_index)
+{
+	struct ibmveth_rx_queue_stats *qstats;
+	__be64 *p;
+	u64 drops;
+
+	if (queue_index < 0 ||
+	    queue_index >= ibmveth_get_num_rx_queues(adapter) ||
+	    !adapter->buffer_list_addr[queue_index])
+		return;
+
+	p = adapter->buffer_list_addr[queue_index] + 4096 - 8;
+	drops = be64_to_cpup(p);
+
+	/*
+	 * PHYP's buffer-list page counter is absolute for that page. A new
+	 * page (reopen / queue reuse after -L) starts near zero; fold the
+	 * previous absolute into this queue's retired carry so sums stay
+	 * monotonic. Both fields belong to the queue being updated, so this
+	 * stays single-writer under the queue's replenish_lock.
+	 */
+	qstats = &adapter->rx_qstats[queue_index];
+
+	if (drops < qstats->no_buffer_drops)
+		qstats->no_buffer_retired += qstats->no_buffer_drops;
+	qstats->no_buffer_drops = drops;
+}
+
+/* replenish routine */
+static void ibmveth_replenish_task(struct ibmveth_adapter *adapter,
+				   int queue_index)
+{
+	struct ibmveth_rx_q *rxq = &adapter->rx_queue[queue_index];
+	struct ibmveth_replenish_fail fail = {};
+	unsigned long flags;
+	int i, rc;
+	int need_reset = 0;
+	int batch_fallback = 0;
+	int hcall_fail = 0;
+
+	if (queue_index >= ibmveth_get_num_rx_queues(adapter)) {
+		netdev_dbg(adapter->netdev,
+			   "Skipping replenish for freed queue %d (num_queues=%u)\n",
+			   queue_index, ibmveth_get_num_rx_queues(adapter));
+		return;
+	}
+
+	spin_lock_irqsave(&rxq->replenish_lock, flags);
+
+	adapter->rx_qstats[queue_index].replenish_task_cycles++;
+
+	for (i = (IBMVETH_NUM_BUFF_POOLS - 1); i >= 0; i--) {
+		struct ibmveth_buff_pool *pool =
+			&adapter->rx_buff_pool[queue_index][i];
+
+		if (pool->active && pool->free_map &&
+		    (atomic_read(&pool->available) < pool->threshold)) {
+			rc = ibmveth_replenish_buffer_pool(adapter, pool,
+							   queue_index, &fail);
+			switch (rc) {
+			case IBMVETH_REPLENISH_RESET_MAP:
+			case IBMVETH_REPLENISH_RESET_MQ:
+				need_reset = rc;
+				goto out_unlock;
+			case IBMVETH_REPLENISH_BATCH_FALLBACK:
+				batch_fallback = 1;
+				break;
+			case IBMVETH_REPLENISH_HCALL_FAIL:
+				hcall_fail = 1;
+				break;
+			default:
+				break;
+			}
+		}
+	}
+
+out_unlock:
+	ibmveth_update_rx_no_buffer(adapter, queue_index);
+
+	spin_unlock_irqrestore(&rxq->replenish_lock, flags);
+
+	/* Log and schedule reset only after dropping replenish_lock. */
+	if (need_reset == IBMVETH_REPLENISH_RESET_MAP) {
+		netdev_info(adapter->netdev,
+			    "Invalid RX free_map entry on queue %d, reset\n",
+			    queue_index);
+		schedule_work(&adapter->work);
+	} else if (need_reset == IBMVETH_REPLENISH_RESET_MQ) {
+		dev_err_ratelimited(&adapter->netdev->dev,
+				    "MQ buffer add H_FUNCTION (q=%d, batch=%u), reset\n",
+				    queue_index, fail.batch);
+		adapter->mq_fallback = true;
+		schedule_work(&adapter->work);
+	}
+
+	if (batch_fallback)
+		dev_warn_ratelimited(&adapter->netdev->dev,
+				     "Legacy batch add H_FUNCTION (batch=%u), fallback\n",
+				     fail.batch);
+
+	if (hcall_fail)
+		dev_warn_ratelimited(&adapter->netdev->dev,
+				     "RX %s failed: filled=%u, rc=%lu, batch=%u\n",
+				     adapter->multi_queue ?
+				     "h_add_logical_lan_buffers_queue" :
+				     (fail.filled == 1 ?
+				      "h_add_logical_lan_buffer" :
+				      "h_add_logical_lan_buffers"),
+				     fail.filled, fail.lpar_rc, fail.batch);
+}
+
+/**
+ * ibmveth_restart_rx_queue - Post buffers and ensure Q can take RX
+ * @adapter: ibmveth adapter
+ * @qindex: RX queue index
+ *
+ * SQ open leaves PHYP masked until the first poll. If schedule_prep fails,
+ * NAPI never runs and the queue stays masked (TX OK, RX/ARP dead) until
+ * reload. Replenish first so an enable_irq fallback can actually deliver.
+ * Also used after every open (SQ and MQ) and after scale-down rollback.
+ */
+static void ibmveth_restart_rx_queue(struct ibmveth_adapter *adapter,
+				     int qindex)
+{
+	ibmveth_replenish_task(adapter, qindex);
+	if (ibmveth_schedule_rx_queue(adapter, qindex))
+		return;
+
+	ibmveth_enable_irq(adapter, qindex);
+}
+
+/* empty and free ana buffer pool - also used to do cleanup in error paths */
+static void ibmveth_free_buffer_pool(struct ibmveth_adapter *adapter,
+				     struct ibmveth_buff_pool *pool)
+{
+	int i;
+
+	kfree(pool->free_map);
+	pool->free_map = NULL;
+
+	if (pool->skbuff && pool->dma_addr) {
+		for (i = 0; i < pool->size; ++i) {
+			struct sk_buff *skb = pool->skbuff[i];
+			if (skb) {
+				dma_unmap_single(&adapter->vdev->dev,
+						 pool->dma_addr[i],
+						 pool->buff_size,
+						 DMA_FROM_DEVICE);
+				dev_kfree_skb_any(skb);
+				pool->skbuff[i] = NULL;
+			}
+		}
+	}
+
+	if (pool->dma_addr) {
+		kfree(pool->dma_addr);
+		pool->dma_addr = NULL;
+	}
+
+	if (pool->skbuff) {
+		kfree(pool->skbuff);
+		pool->skbuff = NULL;
+	}
+
+	/*
+	 * Keep probe/sysfs geometry (active, size, buff_size, threshold).
+	 * Only tear down runtime allocations; open reuses active pools.
+	 */
+	atomic_set(&pool->available, 0);
+}
+
+/**
+ * ibmveth_free_queue_buffer_pools - Free buffer pools for a single queue
+ * @adapter: ibmveth adapter structure
+ * @queue: queue index
+ *
+ * Frees buffer pools that still hold allocations for the specified
+ * queue (by free_map / dma_addr / skbuff presence), regardless of the
+ * active flag.
+ */
+static void ibmveth_free_queue_buffer_pools(struct ibmveth_adapter *adapter,
+					    int queue)
+{
+	int i;
+
+	for (i = 0; i < IBMVETH_NUM_BUFF_POOLS; i++) {
+		struct ibmveth_buff_pool *pool =
+			&adapter->rx_buff_pool[queue][i];
+
+		/* Free pool if it has allocated memory, regardless of
+		 * active flag. Allocation and active can diverge on failure
+		 * paths, so check for actual allocations.
+		 */
+		if (pool->free_map || pool->dma_addr || pool->skbuff)
+			ibmveth_free_buffer_pool(adapter, pool);
+	}
+}
+
+/**
+ * ibmveth_alloc_queue_buffer_pools - Allocate buffer pools for a single queue
+ * @adapter: ibmveth adapter structure
+ * @queue: queue index
+ *
+ * Allocates backing storage for each active pool on @queue.
+ * Inactive pools (!active) are skipped. Pool metadata must be
+ * initialized before calling this function.
+ *
+ * Return: 0 on success, negative error code on failure
+ */
+static int ibmveth_alloc_queue_buffer_pools(struct ibmveth_adapter *adapter,
+					    int queue)
+{
+	struct net_device *netdev = adapter->netdev;
+	int i;
+
+	for (i = 0; i < IBMVETH_NUM_BUFF_POOLS; i++) {
+		struct ibmveth_buff_pool *bpool =
+			&adapter->rx_buff_pool[queue][i];
+
+		if (!bpool->active)
+			continue;
+
+		if (ibmveth_alloc_buffer_pool(bpool)) {
+			netdev_err(netdev,
+				   "queue %d pool %d alloc failed (size=%u count=%u)\n",
+				   queue, i,
+				   bpool->buff_size,
+				   bpool->size);
+			bpool->active = 0;
+			/* Free by allocation presence, not pool->active.
+			 * Needed on later resize paths (a pool can still
+			 * hold memory after active is cleared). This
+			 * open-fail slot is already empty:
+			 * alloc_buffer_pool() undoes its own partials.
+			 */
+			ibmveth_free_queue_buffer_pools(adapter, queue);
+			return -ENOMEM;
+		}
+	}
+
+	return 0;
+}
+
+/**
+ * ibmveth_alloc_buffer_pools - Allocate buffer pools for all queues
+ * @adapter: ibmveth adapter structure
+ *
+ * Initializes pool metadata for queues 1-N from queue 0 settings,
+ * then allocates buffer pools for all queues using the helper function.
+ *
+ * Return: 0 on success, negative error code on failure
+ */
+static int
+ibmveth_alloc_buffer_pools(struct ibmveth_adapter *adapter)
+{
+	struct net_device *netdev = adapter->netdev;
+	int i, q, rc;
+
+	/* Initialize pool metadata for queues 1..N from queue 0 settings */
+	for (q = 1; q < ibmveth_get_num_rx_queues(adapter); q++) {
+		for (i = 0; i < IBMVETH_NUM_BUFF_POOLS; i++) {
+			struct ibmveth_buff_pool *src =
+				&adapter->rx_buff_pool[0][i];
+			struct ibmveth_buff_pool *dst =
+				&adapter->rx_buff_pool[q][i];
+
+			dst->size = src->size;
+			dst->index = src->index;
+			dst->buff_size = src->buff_size;
+			dst->threshold = src->threshold;
+			dst->active = src->active;
+		}
+	}
+
+	/* Allocate actual buffers for all queues */
+	for (q = 0; q < ibmveth_get_num_rx_queues(adapter); q++) {
+		rc = ibmveth_alloc_queue_buffer_pools(adapter, q);
+		if (rc) {
+			/* Free pools for all previous queues */
+			while (--q >= 0)
+				ibmveth_free_queue_buffer_pools(adapter, q);
+			return rc;
+		}
+	}
+
+	netdev_dbg(netdev, "allocated buffer pools for %u queue(s)\n",
+		   ibmveth_get_num_rx_queues(adapter));
+	return 0;
+}
+
+/**
+ * ibmveth_free_buffer_pools - Free buffer pools for all queues
+ * @adapter: ibmveth adapter structure
+ *
+ * Frees buffer pools for all queues using the helper function.
+ */
+static void
+ibmveth_free_buffer_pools(struct ibmveth_adapter *adapter)
+{
+	int q;
+
+	/* Free buffer pools for all queues */
+	for (q = 0; q < ibmveth_get_num_rx_queues(adapter); q++)
+		ibmveth_free_queue_buffer_pools(adapter, q);
+
+	netdev_dbg(adapter->netdev, "freed buffer pools for %u queue(s)\n",
+		   ibmveth_get_num_rx_queues(adapter));
+}
+
+/**
+ * ibmveth_alloc_single_rx_queue - Allocate resources for a single RX queue
+ * @adapter: ibmveth adapter structure
+ * @queue_idx: Queue index to allocate
+ * @rxq_entries: Number of RX queue entries
+ *
+ * Allocates buffer list, RX queue, and per-queue buffer pools for one queue.
+ * Used during incremental scale-up without affecting existing queues.
+ *
+ * Return: 0 on success, negative error code on failure
+ */
+static int
+ibmveth_alloc_single_rx_queue(struct ibmveth_adapter *adapter, int queue_idx,
+			      int rxq_entries)
+{
+	struct device *dev = &adapter->vdev->dev;
+	struct net_device *netdev = adapter->netdev;
+	int i, rc = -ENOMEM;
+
+	adapter->buffer_list_addr[queue_idx] =
+		(void *)get_zeroed_page(GFP_KERNEL);
+	if (!adapter->buffer_list_addr[queue_idx]) {
+		netdev_err(netdev, "unable to allocate buffer list for queue %d\n",
+			   queue_idx);
+		return -ENOMEM;
+	}
+
+	adapter->rx_queue[queue_idx].queue_len =
+		sizeof(struct ibmveth_rx_q_entry) * rxq_entries;
+	adapter->rx_queue[queue_idx].queue_addr =
+		dma_alloc_coherent(dev, adapter->rx_queue[queue_idx].queue_len,
+				   &adapter->rx_queue[queue_idx].queue_dma,
+				   GFP_KERNEL);
+	if (!adapter->rx_queue[queue_idx].queue_addr) {
+		netdev_err(netdev, "unable to allocate RX queue for queue %d\n",
+			   queue_idx);
+		goto out_free_buflist;
+	}
+
+	adapter->buffer_list_dma[queue_idx] =
+		dma_map_single(dev, adapter->buffer_list_addr[queue_idx],
+			       4096, DMA_BIDIRECTIONAL);
+	if (dma_mapping_error(dev, adapter->buffer_list_dma[queue_idx])) {
+		netdev_err(netdev, "unable to map buffer list for queue %d\n",
+			   queue_idx);
+		adapter->buffer_list_dma[queue_idx] = 0;
+		goto out_free_rxq;
+	}
+
+	for (i = 0; i < IBMVETH_NUM_BUFF_POOLS; i++) {
+		struct ibmveth_buff_pool *src =
+			&adapter->rx_buff_pool[0][i];
+		struct ibmveth_buff_pool *dst =
+			&adapter->rx_buff_pool[queue_idx][i];
+
+		dst->size = src->size;
+		dst->index = src->index;
+		dst->buff_size = src->buff_size;
+		dst->threshold = src->threshold;
+		dst->active = src->active;
+	}
+
+	rc = ibmveth_alloc_queue_buffer_pools(adapter, queue_idx);
+	if (rc) {
+		netdev_err(netdev,
+			   "Failed to allocate buffer pools for queue %d\n",
+			   queue_idx);
+		goto out_unmap_buflist;
+	}
+
+	adapter->rx_queue[queue_idx].index = 0;
+	adapter->rx_queue[queue_idx].num_slots = rxq_entries;
+	adapter->rx_queue[queue_idx].toggle = 1;
+
+	netdev_dbg(netdev,
+		   "Allocated queue %d: buffer_list @ %p (DMA: 0x%llx), rx_queue @ %p (DMA: 0x%llx), %d entries\n",
+		   queue_idx, adapter->buffer_list_addr[queue_idx],
+		   (unsigned long long)adapter->buffer_list_dma[queue_idx],
+		   adapter->rx_queue[queue_idx].queue_addr,
+		   (unsigned long long)adapter->rx_queue[queue_idx].queue_dma,
+		   rxq_entries);
+
+	return 0;
+
+out_unmap_buflist:
+	dma_unmap_single(dev, adapter->buffer_list_dma[queue_idx],
+			 4096, DMA_BIDIRECTIONAL);
+	adapter->buffer_list_dma[queue_idx] = 0;
+out_free_rxq:
+	dma_free_coherent(dev, adapter->rx_queue[queue_idx].queue_len,
+			  adapter->rx_queue[queue_idx].queue_addr,
+			  adapter->rx_queue[queue_idx].queue_dma);
+	adapter->rx_queue[queue_idx].queue_addr = NULL;
+out_free_buflist:
+	free_page((unsigned long)adapter->buffer_list_addr[queue_idx]);
+	adapter->buffer_list_addr[queue_idx] = NULL;
+	return rc;
+}
+
+/**
+ * ibmveth_free_single_rx_queue - Free resources for a single RX queue
+ * @adapter: ibmveth adapter structure
+ * @queue_idx: Queue index to free
+ *
+ * Frees buffer list, RX queue, and per-queue buffer pools for one queue.
+ * Used during incremental scale-down without affecting remaining queues.
+ */
+static void
+ibmveth_free_single_rx_queue(struct ibmveth_adapter *adapter, int queue_idx)
+{
+	struct device *dev = &adapter->vdev->dev;
+
+	ibmveth_free_queue_buffer_pools(adapter, queue_idx);
+
+	if (adapter->buffer_list_addr[queue_idx]) {
+		dma_unmap_single(dev, adapter->buffer_list_dma[queue_idx],
+				 4096, DMA_BIDIRECTIONAL);
+		adapter->buffer_list_dma[queue_idx] = 0;
+	}
+
+	if (adapter->rx_queue[queue_idx].queue_addr) {
+		dma_free_coherent(dev, adapter->rx_queue[queue_idx].queue_len,
+				  adapter->rx_queue[queue_idx].queue_addr,
+				  adapter->rx_queue[queue_idx].queue_dma);
+		adapter->rx_queue[queue_idx].queue_addr = NULL;
+	}
+
+	if (adapter->buffer_list_addr[queue_idx]) {
+		free_page((unsigned long)adapter->buffer_list_addr[queue_idx]);
+		adapter->buffer_list_addr[queue_idx] = NULL;
+	}
+
+	netdev_dbg(adapter->netdev, "Freed queue %d resources\n", queue_idx);
+}
+
+static bool ibmveth_rxq_correlator_valid(struct ibmveth_adapter *adapter,
+					 int queue_index, u64 correlator)
+{
+	unsigned int pool = correlator >> 32;
+	unsigned int index = correlator & 0xffffffffUL;
+	struct ibmveth_buff_pool *bpool;
+
+	if (pool >= IBMVETH_NUM_BUFF_POOLS)
+		return false;
+
+	bpool = &adapter->rx_buff_pool[queue_index][pool];
+
+	/* Require a live pool with allocated arrays before indexing.
+	 * Inactive pools still have size from init; free clears skbuff.
+	 */
+	if (!bpool->active || !bpool->skbuff || !bpool->free_map)
+		return false;
+
+	return index < bpool->size;
+}
+
+static void ibmveth_rxq_advance(struct ibmveth_rx_q *rxq)
+{
+	if (++rxq->index == rxq->num_slots) {
+		rxq->index = 0;
+		rxq->toggle = !rxq->toggle;
+	}
+}
+
+/**
+ * ibmveth_remove_buffer_from_pool - remove a buffer from a pool
+ * @adapter: adapter instance
+ * @correlator: identifies pool and index
+ * @queue_index: RX queue index (0..num_rx_queues-1)
+ * @reuse: whether to reuse buffer
+ *
+ * Context: may run concurrently with netpoll replenish_task on the same
+ * queue; takes per-queue replenish_lock to serialize free_map /
+ * producer_index / available against the producer.
+ *
+ * Return:
+ * * %0       - success
+ * * %-EINVAL - pool or index out of range, or the pool is
+ *              inactive / not live (skbuff or free_map NULL)
+ * * %-EFAULT - pool and index map to null skb
+ */
+static int ibmveth_remove_buffer_from_pool(struct ibmveth_adapter *adapter,
+					   u64 correlator, int queue_index,
+					   bool reuse)
+{
+	struct ibmveth_rx_q *rxq = &adapter->rx_queue[queue_index];
+	unsigned int pool  = correlator >> 32;
+	unsigned int index = correlator & 0xffffffffUL;
+	unsigned int free_index;
+	struct sk_buff *skb;
+	unsigned long flags;
+	int rc = 0;
+
+	spin_lock_irqsave(&rxq->replenish_lock, flags);
+
+	if (!ibmveth_rxq_correlator_valid(adapter, queue_index, correlator)) {
+		rc = -EINVAL;
+		goto out_unlock;
+	}
+
+	skb = adapter->rx_buff_pool[queue_index][pool].skbuff[index];
+	if (!skb) {
+		rc = -EFAULT;
+		goto out_unlock;
+	}
+
+	/* if we are going to reuse the buffer then keep the pointers around
+	 * but mark index as available. replenish will see the skb pointer and
+	 * assume it is to be recycled.
+	 */
+	if (!reuse) {
+		/* remove the skb pointer to mark free. actual freeing is done
+		 * by upper level networking after gro_receive
+		 */
+		struct ibmveth_buff_pool *bpool =
+			&adapter->rx_buff_pool[queue_index][pool];
+
+		bpool->skbuff[index] = NULL;
+
+		dma_unmap_single(&adapter->vdev->dev,
+				 bpool->dma_addr[index],
+				 bpool->buff_size,
+				 DMA_FROM_DEVICE);
+	}
+
+	free_index = adapter->rx_buff_pool[queue_index][pool].producer_index;
+	adapter->rx_buff_pool[queue_index][pool].producer_index++;
+	if (adapter->rx_buff_pool[queue_index][pool].producer_index >=
+	    adapter->rx_buff_pool[queue_index][pool].size)
+		adapter->rx_buff_pool[queue_index][pool].producer_index = 0;
+	adapter->rx_buff_pool[queue_index][pool].free_map[free_index] = index;
+
+	mb();
+
+	atomic_dec(&adapter->rx_buff_pool[queue_index][pool].available);
+
+out_unlock:
+	spin_unlock_irqrestore(&rxq->replenish_lock, flags);
+	return rc;
+}
+
+/* get the current buffer on the rx queue */
+static struct sk_buff *
+ibmveth_rxq_get_buffer(struct ibmveth_adapter *adapter,
+		       int queue_index)
+{
+	struct ibmveth_rx_q *rxq = &adapter->rx_queue[queue_index];
+	u64 correlator = rxq->queue_addr[rxq->index].correlator;
+	unsigned int pool = correlator >> 32;
+	unsigned int index = correlator & 0xffffffffUL;
+
+	if (!ibmveth_rxq_correlator_valid(adapter, queue_index, correlator))
+		return NULL;
+
+	return adapter->rx_buff_pool[queue_index][pool].skbuff[index];
+}
+
+/**
+ * ibmveth_rxq_harvest_buffer - Harvest buffer from pool
+ *
+ * @adapter: pointer to adapter
+ * @queue_index: RX queue index to harvest from
+ * @reuse:   whether to reuse buffer
+ *
+ * Context: called from ibmveth_poll
+ *
+ * On a bad correlator (-EINVAL/-EFAULT) the ring is still advanced so poll
+ * cannot spin forever on one slot. The error is still returned: callers must
+ * not treat it as a successful take from the pool (especially reuse=false,
+ * which would hand the SKB to the stack while it remains pool-owned).
+ *
+ * Return:
+ * * %0    - buffer removed from pool (or marked for reuse) and ring advanced
+ * * other - non-zero return from ibmveth_remove_buffer_from_pool; ring has
+ *           still been advanced for -EINVAL/-EFAULT
+ */
+static int ibmveth_rxq_harvest_buffer(struct ibmveth_adapter *adapter,
+				      int queue_index, bool reuse)
+{
+	struct ibmveth_rx_q *rxq = &adapter->rx_queue[queue_index];
+	u64 cor;
+	int rc;
+
+	cor = rxq->queue_addr[rxq->index].correlator;
+	rc = ibmveth_remove_buffer_from_pool(adapter, cor, queue_index, reuse);
+	if (unlikely(rc)) {
+		/* Skip a corrupt slot without claiming pool ownership. */
+		if (rc == -EINVAL || rc == -EFAULT)
+			ibmveth_rxq_advance(rxq);
+		return rc;
+	}
+
+	ibmveth_rxq_advance(rxq);
+
+	return 0;
+}
+
+/**
+ * ibmveth_drain_rx_queue - Drain pending buffers from an RX queue
+ * @adapter: ibmveth adapter structure
+ * @queue_index: Queue index to drain
+ *
+ * Harvests pending completions back to the per-queue buffer pools.
+ * A corrupt slot still advances the ring, so the return is slots
+ * processed, not buffers recycled.
+ * Must be called with NAPI disabled for this queue.
+ *
+ * Return: number of slots processed
+ */
+static int
+ibmveth_drain_rx_queue(struct ibmveth_adapter *adapter, int queue_index)
+{
+	struct net_device *netdev = adapter->netdev;
+	int drained = 0;
+	int limit = adapter->rx_queue[queue_index].num_slots;
+	int rc;
+
+	netdev_dbg(netdev, "Draining RX queue %d (limit: %d slots)\n",
+		   queue_index, limit);
+
+	while (drained < limit &&
+	       ibmveth_rxq_pending_buffer(adapter, queue_index)) {
+		/* Match poll-side order before harvesting completion state. */
+		smp_rmb();
+		rc = ibmveth_rxq_harvest_buffer(adapter, queue_index, true);
+		if (rc) {
+			/* -EINVAL/-EFAULT already advanced past the slot. */
+			if (rc == -EINVAL || rc == -EFAULT) {
+				drained++;
+				continue;
+			}
+			netdev_err(netdev,
+				   "Failed to harvest buffer from queue %d during drain: %d\n",
+				   queue_index, rc);
+			break;
+		}
+		drained++;
+	}
+
+	if (drained > 0)
+		netdev_dbg(netdev, "Drained %d slot(s) from RX queue %d\n",
+			   drained, queue_index);
+	else
+		netdev_dbg(netdev, "No slots to drain from RX queue %d\n",
+			   queue_index);
+
+	return drained;
+}
+
+static void ibmveth_free_tx_ltb(struct ibmveth_adapter *adapter, int idx)
+{
+	void *ltb = adapter->tx_ltb_ptr[idx];
+	dma_addr_t dma = adapter->tx_ltb_dma[idx];
+
+	if (!ltb)
+		return;
+
+	/*
+	 * Clear the slot before releasing it. start_xmit() tests
+	 * tx_ltb_ptr[idx] to decide whether the LTB is usable.
+	 */
+	adapter->tx_ltb_ptr[idx] = NULL;
+	adapter->tx_ltb_dma[idx] = 0;
+
+	dma_unmap_single(&adapter->vdev->dev, dma, adapter->tx_ltb_size,
+			 DMA_TO_DEVICE);
+	kfree(ltb);
+}
+
+/* Caller must ensure tx_ltb_ptr[idx] is NULL. open() runs on
+ * probe-zeroed slots; set_channels() skips populated indices.
+ */
+static int ibmveth_allocate_tx_ltb(struct ibmveth_adapter *adapter, int idx)
+{
+	adapter->tx_ltb_ptr[idx] = kzalloc(adapter->tx_ltb_size,
+					   GFP_KERNEL);
+	if (!adapter->tx_ltb_ptr[idx]) {
+		netdev_err(adapter->netdev,
+			   "unable to allocate tx long term buffer\n");
+		return -ENOMEM;
+	}
+	adapter->tx_ltb_dma[idx] = dma_map_single(&adapter->vdev->dev,
+						  adapter->tx_ltb_ptr[idx],
+						  adapter->tx_ltb_size,
+						  DMA_TO_DEVICE);
+	if (dma_mapping_error(&adapter->vdev->dev, adapter->tx_ltb_dma[idx])) {
+		netdev_err(adapter->netdev,
+			   "unable to DMA map tx long term buffer\n");
+		kfree(adapter->tx_ltb_ptr[idx]);
+		adapter->tx_ltb_ptr[idx] = NULL;
+		adapter->tx_ltb_dma[idx] = 0;
+		return -ENOMEM;
+	}
+
+	return 0;
+}
+
+/**
+ * ibmveth_alloc_tx_resources - Allocate TX LTBs for real_num_tx_queues
+ * @adapter: ibmveth adapter structure
+ *
+ * Allocates TX Long Term Buffers (LTBs) for real_num_tx_queues.
+ *
+ * Return: 0 on success, -ENOMEM on failure
+ */
+static int ibmveth_alloc_tx_resources(struct ibmveth_adapter *adapter)
+{
+	struct net_device *netdev = adapter->netdev;
+	int i;
+
+	for (i = 0; i < netdev->real_num_tx_queues; i++) {
+		if (ibmveth_allocate_tx_ltb(adapter, i))
+			goto err_free_ltbs;
+	}
+
+	return 0;
+
+err_free_ltbs:
+	while (--i >= 0)
+		ibmveth_free_tx_ltb(adapter, i);
+	return -ENOMEM;
+}
+
+/**
+ * ibmveth_free_tx_resources - Free TX LTBs for real_num_tx_queues
+ * @adapter: ibmveth adapter structure
+ *
+ * Frees TX Long Term Buffers (LTBs) for real_num_tx_queues.
+ */
+static void ibmveth_free_tx_resources(struct ibmveth_adapter *adapter)
+{
+	struct net_device *netdev = adapter->netdev;
+	int i;
+
+	for (i = 0; i < netdev->real_num_tx_queues; i++)
+		ibmveth_free_tx_ltb(adapter, i);
+}
+
+static int ibmveth_register_logical_lan(struct ibmveth_adapter *adapter,
+					union ibmveth_buf_desc rxq_desc,
+					u64 mac_address)
+{
+	int rc, try_again = 1;
+	unsigned long ua = adapter->vdev->unit_address;
+	unsigned long buf_dma = adapter->buffer_list_dma[0];
+	unsigned long filter_dma = adapter->filter_list_dma;
+	unsigned long qh0;
+
+	/*
+	 * After a kexec the adapter will still be open, so our attempt to
+	 * open it will fail. So if we get a failure we free the adapter and
+	 * try again, but only once.
+	 */
+retry:
+	/* In multi-queue mode, obtain a queue handle for queue 0 so all RX
+	 * queues can use the same per-queue buffer hypercalls.
+	 */
+	if (adapter->multi_queue) {
+		rc = h_register_logical_lan_with_handle(ua, buf_dma,
+							rxq_desc.desc,
+							filter_dma,
+							mac_address,
+							&qh0);
+		if (rc == H_SUCCESS)
+			adapter->queue_handle[0] = qh0;
+	} else {
+		rc = h_register_logical_lan(ua, buf_dma, rxq_desc.desc,
+					    filter_dma, mac_address);
+	}
+	netdev_dbg(adapter->netdev, "h_register_logical_lan%s rc=%d\n",
+		   adapter->multi_queue ? "_with_handle" : "", rc);
+
+	if (rc != H_SUCCESS && try_again) {
+		do {
+			rc = h_free_logical_lan(adapter->vdev->unit_address);
+		} while (H_IS_LONG_BUSY(rc) || (rc == H_BUSY));
+		netdev_dbg(adapter->netdev, "h_free_logical_lan rc=%d\n", rc);
+
+		try_again = 0;
+		goto retry;
+	}
+
+	return rc;
+}
+
+/**
+ * ibmveth_register_logical_lan_queue - Register subordinate queue with
+ * hypervisor
+ * @adapter: ibmveth adapter structure
+ * @rxq_desc: Receive queue descriptor
+ * @queue_index: RX queue index (1..N for subordinate queues)
+ *
+ * Registers a subordinate receive queue using H_REG_LOGICAL_LAN_QUEUE.
+ * On success, stores the queue handle and virtual IRQ in the adapter.
+ * If IRQ mapping fails after a successful hypervisor registration, the
+ * queue is freed before returning. If that free fails, the handle is
+ * kept so the caller can skip unmap.
+ *
+ * Return: H_SUCCESS on success, negative errno on IRQ mapping failure,
+ *         hypervisor error code otherwise
+ */
+static int
+ibmveth_register_logical_lan_queue(struct ibmveth_adapter *adapter,
+				   union ibmveth_buf_desc rxq_desc,
+				   int queue_index)
+{
+	unsigned long handle, hwirq;
+	unsigned int virq;
+	long lpar_rc;
+	unsigned long ua = adapter->vdev->unit_address;
+	unsigned long bl = adapter->buffer_list_dma[queue_index];
+
+	netdev_dbg(adapter->netdev,
+		   "register queue %d: ua=0x%lx bl=0x%lx rxq=0x%llx\n",
+		   queue_index, ua, bl, rxq_desc.desc);
+	do {
+		lpar_rc = h_register_logical_lan_queue(ua, bl,
+						       rxq_desc.desc, &handle,
+						       &hwirq);
+	} while (H_IS_LONG_BUSY(lpar_rc) || (lpar_rc == H_BUSY));
+	netdev_dbg(adapter->netdev,
+		   "h_register_logical_lan_queue queue %d rc=%ld\n",
+		   queue_index, lpar_rc);
+
+	if (lpar_rc == H_SUCCESS) {
+		virq = irq_create_mapping(NULL, hwirq);
+		if (!virq) {
+			unsigned long free_rc;
+
+			netdev_err(adapter->netdev,
+				   "Failed to map IRQ for queue %d (hwirq=%lu)\n",
+				   queue_index, hwirq);
+			do {
+				free_rc = h_free_logical_lan_queue(ua, handle);
+			} while (H_IS_LONG_BUSY(free_rc) ||
+				  (free_rc == H_BUSY));
+			if (free_rc != H_SUCCESS) {
+				netdev_err(adapter->netdev,
+					   "h_free_logical_lan_queue failed for queue %d after IRQ map failure: rc=0x%lx\n",
+					   queue_index, free_rc);
+				/*
+				 * PHYP still owns this queue. Keep the
+				 * handle so the caller skips unmap.
+				 */
+				adapter->queue_handle[queue_index] = handle;
+				return -EIO;
+			}
+			return -EINVAL;
+		}
 
-		if (pool->active &&
-		    (atomic_read(&pool->available) < pool->threshold))
-			ibmveth_replenish_buffer_pool(adapter, pool);
+		adapter->queue_handle[queue_index] = handle;
+		adapter->queue_irq[queue_index] = virq;
+
+		netdev_dbg(adapter->netdev,
+			   "queue %d registered: handle=0x%llx irq=%u\n",
+			   queue_index, adapter->queue_handle[queue_index],
+			   adapter->queue_irq[queue_index]);
+		return H_SUCCESS;
 	}
 
-	ibmveth_update_rx_no_buffer(adapter);
+	/*
+	 * H_FUNCTION means firmware rejected this subordinate register
+	 * (MQ unsupported / dropped after LPM). Caller fails this open and
+	 * latches mq_fallback so the next open applies SQ; keep a specific
+	 * log then the generic failure lines below.
+	 */
+	if (lpar_rc == H_FUNCTION)
+		netdev_err(adapter->netdev,
+			   "h_register_logical_lan_queue H_FUNCTION for queue %d (firmware MQ unsupported)\n",
+			   queue_index);
+
+	netdev_err(adapter->netdev,
+		   "h_register_logical_lan_queue failed for queue %d with %ld\n",
+		   queue_index, lpar_rc);
+	netdev_err(adapter->netdev,
+		   "queue %d params: unit_addr=0x%x buffer_list_dma=0x%llx rxq_desc=0x%llx\n",
+		   queue_index, adapter->vdev->unit_address,
+		   adapter->buffer_list_dma[queue_index],
+		   rxq_desc.desc);
+
+	return lpar_rc;
 }
 
-/* empty and free ana buffer pool - also used to do cleanup in error paths */
-static void ibmveth_free_buffer_pool(struct ibmveth_adapter *adapter,
-				     struct ibmveth_buff_pool *pool)
+/**
+ * ibmveth_register_single_rx_queue - Register one subordinate RX queue
+ * @adapter: ibmveth adapter structure
+ * @queue_idx: Queue index to register (1..N)
+ *
+ * Builds the queue descriptor and registers with the hypervisor via
+ * ibmveth_register_logical_lan_queue().
+ *
+ * Return: 0 on success, -EINVAL if @queue_idx is invalid, -EOPNOTSUPP if
+ *         firmware rejects MQ (H_FUNCTION), -EIO on other failures
+ */
+static int
+ibmveth_register_single_rx_queue(struct ibmveth_adapter *adapter,
+				 int queue_idx)
+{
+	struct net_device *netdev = adapter->netdev;
+	union ibmveth_buf_desc rxq_desc;
+	long lpar_rc;
+
+	if (WARN_ON(queue_idx < 1 || queue_idx >= IBMVETH_MAX_RX_QUEUES))
+		return -EINVAL;
+
+	rxq_desc.fields.flags_len = IBMVETH_BUF_VALID |
+				    adapter->rx_queue[queue_idx].queue_len;
+	rxq_desc.fields.address = adapter->rx_queue[queue_idx].queue_dma;
+
+	lpar_rc = ibmveth_register_logical_lan_queue(adapter, rxq_desc,
+						     queue_idx);
+	if (lpar_rc != H_SUCCESS) {
+		netdev_err(netdev, "Failed to register queue %d: rc=%ld\n",
+			   queue_idx, lpar_rc);
+		if (lpar_rc == H_FUNCTION)
+			return -EOPNOTSUPP;
+		return -EIO;
+	}
+
+	netdev_dbg(netdev, "Registered queue %d with handle 0x%llx\n",
+		   queue_idx, adapter->queue_handle[queue_idx]);
+
+	return 0;
+}
+
+/**
+ * ibmveth_deregister_single_rx_queue - Deregister one subordinate RX queue
+ * @adapter: ibmveth adapter structure
+ * @queue_idx: Queue index to deregister (1..N)
+ *
+ * Deregisters a single queue via H_FREE_LOGICAL_LAN_QUEUE. Linux IRQ handler
+ * teardown and subordinate virq mapping disposal are owned by interrupt
+ * cleanup helpers; queue 0 is freed only through ibmveth_free_all_queues()
+ * (H_FREE_LOGICAL_LAN).
+ *
+ * Return: 0 if the queue was freed or had no handle, -EIO if
+ *         H_FREE_LOGICAL_LAN_QUEUE failed. On -EIO the handle is
+ *         left set so the caller can skip unmap.
+ */
+static int
+ibmveth_deregister_single_rx_queue(struct ibmveth_adapter *adapter,
+				   int queue_idx)
+{
+	unsigned long lpar_rc;
+	unsigned long ua = adapter->vdev->unit_address;
+	unsigned long qh = adapter->queue_handle[queue_idx];
+
+	if (!qh)
+		return 0;
+
+	do {
+		lpar_rc = h_free_logical_lan_queue(ua, qh);
+	} while (H_IS_LONG_BUSY(lpar_rc) || (lpar_rc == H_BUSY));
+
+	if (lpar_rc != H_SUCCESS) {
+		netdev_err(adapter->netdev,
+			   "h_free_logical_lan_queue failed, queue %d rc=%ld\n",
+			   queue_idx, lpar_rc);
+		return -EIO;
+	}
+
+	adapter->queue_handle[queue_idx] = 0;
+
+	netdev_dbg(adapter->netdev, "Deregistered queue %d\n", queue_idx);
+	return 0;
+}
+
+/**
+ * ibmveth_destroy_subordinate_rx_queue - Tear down one subordinate RX queue
+ * @adapter: ibmveth adapter structure
+ * @queue_idx: Queue index to destroy (1..N)
+ *
+ * Deregister with PHYP before unmapping buffer pools so hypervisor buffer
+ * ownership is released while queue metadata is still valid. If the
+ * hcall fails, leave the queue mapped and keep the handle.
+ *
+ * Return: 0 on success, -EIO if deregister failed
+ */
+static int
+ibmveth_destroy_subordinate_rx_queue(struct ibmveth_adapter *adapter,
+				     int queue_idx)
+{
+	int rc;
+
+	rc = ibmveth_deregister_single_rx_queue(adapter, queue_idx);
+	if (rc)
+		return rc;
+	ibmveth_cleanup_single_rx_interrupt(adapter, queue_idx);
+	ibmveth_free_single_rx_queue(adapter, queue_idx);
+	return 0;
+}
+
+/**
+ * ibmveth_desired_dma_for_rxqs - CMO entitlement for a given RX queue count
+ * @adapter: ibmveth adapter
+ * @rxqs: number of RX queues to size for
+ *
+ * Same math as ibmveth_get_desired_dma(), but uses @rxqs instead of the
+ * live adapter->num_rx_queues. Scale-up raises desired for the *target*
+ * count before allocating so vio_cmo_alloc cannot fail mid-resize.
+ *
+ * Return: bytes of IO memory desired for @rxqs RX queues
+ */
+static unsigned long
+ibmveth_desired_dma_for_rxqs(struct ibmveth_adapter *adapter,
+			     unsigned int rxqs)
+{
+	struct net_device *netdev = adapter->netdev;
+	struct iommu_table *tbl;
+	unsigned long ret;
+	int i, q;
+
+	tbl = get_iommu_table_base(&adapter->vdev->dev);
+
+	ret = IBMVETH_BUFF_LIST_SIZE * rxqs + IBMVETH_FILT_LIST_SIZE;
+	ret += IOMMU_PAGE_ALIGN(netdev->mtu, tbl);
+	ret += IOMMU_PAGE_ALIGN(IBMVETH_MAX_TX_BUF_SIZE, tbl);
+
+	/*
+	 * Pool metadata for queues 1+ is copied from queue 0 at open.
+	 * Always size from pool 0 x @rxqs.
+	 *
+	 * CMO (Power9 and earlier) and MQ firmware (Power11+) do not
+	 * coexist, so a CMO partition always sizes one queue here. The
+	 * MQ terms and the ethtool -L CMO refreshes are defensive only.
+	 */
+	for (q = 0; q < rxqs; q++) {
+		int rxqentries = 1;
+
+		for (i = 0; i < IBMVETH_NUM_BUFF_POOLS; i++) {
+			struct ibmveth_buff_pool *bpool =
+				&adapter->rx_buff_pool[0][i];
+
+			if (bpool->active)
+				ret += bpool->size *
+					IOMMU_PAGE_ALIGN(bpool->buff_size, tbl);
+			rxqentries += bpool->size;
+		}
+
+		ret += IOMMU_PAGE_ALIGN(rxqentries *
+					sizeof(struct ibmveth_rx_q_entry), tbl);
+	}
+
+	return ret;
+}
+
+/**
+ * ibmveth_free_stranded_rx_queues - Free queues stranded above the live count
+ * @adapter: ibmveth adapter structure
+ *
+ * A scale-up register or IRQ-setup failure whose H_FREE_LOGICAL_LAN_QUEUE
+ * also fails leaves that queue out of the live set with its handle set
+ * and its memory still mapped. Call only after successful H_FREE_LOGICAL_LAN,
+ * which drops every subordinate queue, so the memory can be released.
+ */
+static void ibmveth_free_stranded_rx_queues(struct ibmveth_adapter *adapter)
 {
 	int i;
 
-	kfree(pool->free_map);
-	pool->free_map = NULL;
+	for (i = ibmveth_get_num_rx_queues(adapter);
+	     i < IBMVETH_MAX_RX_QUEUES; i++) {
+		if (!adapter->queue_handle[i])
+			continue;
+		adapter->queue_handle[i] = 0;
+		ibmveth_free_single_rx_queue(adapter, i);
+	}
+}
 
-	if (pool->skbuff && pool->dma_addr) {
-		for (i = 0; i < pool->size; ++i) {
-			struct sk_buff *skb = pool->skbuff[i];
-			if (skb) {
-				dma_unmap_single(&adapter->vdev->dev,
-						 pool->dma_addr[i],
-						 pool->buff_size,
-						 DMA_FROM_DEVICE);
-				dev_kfree_skb_any(skb);
-				pool->skbuff[i] = NULL;
+/* Set CMO desired entitlement from the live RX queue count. */
+static void ibmveth_refresh_cmo_desired(struct ibmveth_adapter *adapter)
+{
+	if (firmware_has_feature(FW_FEATURE_CMO))
+		vio_cmo_set_dev_desired(adapter->vdev,
+					ibmveth_get_desired_dma(adapter->vdev));
+}
+
+/**
+ * ibmveth_scale_up_rx_queues - Add RX queues to a live adapter
+ * @adapter: ibmveth adapter structure
+ * @old_count: Current number of RX queues
+ * @new_count: Target number of RX queues, above @old_count
+ * @rxq_entries: Number of entries per RX queue
+ *
+ * Raises CMO entitlement and widens real_num_rx_queues before any new
+ * queue is unmasked, then brings each queue up in turn. A failure
+ * unwinds only the queues this call added; a queue PHYP refuses to
+ * release stays in the live set instead.
+ *
+ * Return: 0 on success, negative error code on failure
+ */
+static int
+ibmveth_scale_up_rx_queues(struct ibmveth_adapter *adapter, int old_count,
+			   int new_count, int rxq_entries)
+{
+	struct net_device *netdev = adapter->netdev;
+	int failed_queue;
+	int rc, i;
+
+	netdev_dbg(netdev, "Scale-up: adding queues %d-%d\n",
+		   old_count, new_count - 1);
+
+	/*
+	 * Raise CMO desired for the target count before dma_map /
+	 * dma_alloc_coherent / replenish (same order as change_mtu).
+	 * Do not bump live num_rx_queues here, only entitlement.
+	 */
+	if (firmware_has_feature(FW_FEATURE_CMO)) {
+		unsigned long dma;
+
+		dma = ibmveth_desired_dma_for_rxqs(adapter, new_count);
+		vio_cmo_set_dev_desired(adapter->vdev, dma);
+	}
+
+	/*
+	 * Widen real_num before any new queue is unmasked so
+	 * skb_record_rx_queue() cannot trip get_rps_cpu()
+	 * WARN_ONCE. Probe already allocated num_rx_queues.
+	 */
+	rc = netif_set_real_num_rx_queues(netdev, new_count);
+	if (rc) {
+		netdev_err(netdev,
+			   "Failed to set real RX queues to %d: %d\n",
+			   new_count, rc);
+		ibmveth_refresh_cmo_desired(adapter);
+		return rc;
+	}
+
+	for (i = old_count; i < new_count; i++) {
+		if (adapter->queue_handle[i]) {
+			/* Left by a failed H_FREE; close frees it. */
+			netdev_err(netdev,
+				   "RX queue %d still held by PHYP, reset pending\n",
+				   i);
+			rc = -EBUSY;
+			goto cleanup_new_queues;
+		}
+		rc = ibmveth_alloc_single_rx_queue(adapter, i,
+						   rxq_entries);
+		if (rc) {
+			netdev_err(netdev, "Failed to allocate queue %d: %d\n",
+				   i, rc);
+			goto cleanup_new_queues;
+		}
+
+		rc = ibmveth_register_single_rx_queue(adapter, i);
+		if (rc) {
+			netdev_err(netdev, "Failed to register queue %d: %d\n",
+				   i, rc);
+			if (rc == -EOPNOTSUPP)
+				adapter->mq_fallback = true;
+			if (!ibmveth_deregister_single_rx_queue(adapter,
+								i))
+				ibmveth_free_single_rx_queue(adapter,
+							     i);
+			else
+				schedule_work(&adapter->work);
+			goto cleanup_new_queues;
+		}
+
+		rc = ibmveth_setup_single_rx_interrupt(adapter, i);
+		if (rc) {
+			netdev_err(netdev,
+				   "Failed to setup IRQ for queue %d: %d\n",
+				   i, rc);
+			/* request_irq failed: mapped but no handler */
+			ibmveth_dispose_subordinate_irq_mapping(adapter,
+								i);
+			if (!ibmveth_deregister_single_rx_queue(adapter,
+								i))
+				ibmveth_free_single_rx_queue(adapter,
+							     i);
+			else
+				schedule_work(&adapter->work);
+			goto cleanup_new_queues;
+		}
+
+		/*
+		 * Fully ready before PHYP delivery, matching open():
+		 * publish -> replenish -> napi_enable -> enable_irq.
+		 * That way ibmveth_interrupt() cannot run on an
+		 * unpublished, empty, or NAPI-disabled queue.
+		 */
+		ibmveth_publish_num_rx_queues(adapter, i + 1);
+		ibmveth_replenish_task(adapter, i);
+		napi_enable(&adapter->napi[i]);
+
+		rc = ibmveth_enable_irq(adapter, i);
+		if (rc) {
+			netdev_err(netdev,
+				   "Failed to enable IRQ for queue %d: %d\n",
+				   i, rc);
+			/*
+			 * Published, replenished, and NAPI-enabled,
+			 * but PHYP never unmasked. Match scale-down /
+			 * shared cleanup: drain posted buffers, then
+			 * deregister before unmap via
+			 * destroy_subordinate.
+			 *
+			 * napi_disable() must come BEFORE the count
+			 * is lowered, matching scale-down and
+			 * cleanup_new_queues. Lowering it first does
+			 * not hide queue i from netpoll: after
+			 * ndo_poll_controller, netpoll_poll_dev()
+			 * calls poll_napi(), which walks dev->napi_list
+			 * unbounded by the queue count and skips a NAPI
+			 * only once NAPI_STATE_NPSVC is set. Queue i is
+			 * enabled here, so ibmveth_poll() would run and
+			 * trip its queue_index >= num_rx_queues
+			 * WARN_ON. napi_disable() sets NPSVC, so
+			 * poll_napi() skips the queue instead.
+			 */
+			napi_disable(&adapter->napi[i]);
+			/* A poll may have unmasked PHYP; remask. */
+			ibmveth_disable_irq(adapter, i);
+			synchronize_irq(adapter->queue_irq[i]);
+			ibmveth_publish_num_rx_queues(adapter, i);
+			ibmveth_drain_rx_queue(adapter, i);
+			synchronize_net();
+		}
+		if (rc &&
+		    ibmveth_destroy_subordinate_rx_queue(adapter, i)) {
+			int keep = i + 1;
+
+			/*
+			 * PHYP still owns queue i. Keep it in the
+			 * live set with the queues this -L already
+			 * added. cleanup_new_queues would publish
+			 * below i and orphan this mapping.
+			 */
+			ibmveth_publish_num_rx_queues(adapter, keep);
+			if (netif_set_real_num_rx_queues(netdev,
+							 keep))
+				schedule_work(&adapter->work);
+			ibmveth_replenish_task(adapter, i);
+			napi_enable(&adapter->napi[i]);
+			rc = ibmveth_enable_irq(adapter, i);
+			if (rc) {
+				netdev_err(netdev,
+					   "IRQ %d rc=%d\n",
+					   i, rc);
+				schedule_work(&adapter->work);
+			} else {
+				ibmveth_restart_rx_queue(adapter, i);
 			}
+			return -EIO;
+		}
+		if (rc) {
+			/* enable_irq errno; keep -EIO. */
+			rc = -EIO;
+			goto cleanup_new_queues;
 		}
+		ibmveth_restart_rx_queue(adapter, i);
 	}
 
-	if (pool->dma_addr) {
-		kfree(pool->dma_addr);
-		pool->dma_addr = NULL;
+	return 0;
+
+cleanup_new_queues:
+	failed_queue = i;
+	if (failed_queue > old_count)
+		netdev_err(netdev,
+			   "Scale-up failed at queue %d, cleaning up queues %d-%d\n",
+			   failed_queue, old_count, failed_queue - 1);
+	else
+		netdev_err(netdev,
+			   "Scale-up failed at queue %d, nothing to clean up\n",
+			   failed_queue);
+
+	for (i = old_count; i < failed_queue; i++) {
+		ibmveth_disable_irq(adapter, i);
+		synchronize_irq(adapter->queue_irq[i]);
 	}
 
-	if (pool->skbuff) {
-		kfree(pool->skbuff);
-		pool->skbuff = NULL;
+	for (i = old_count; i < failed_queue; i++)
+		napi_disable(&adapter->napi[i]);
+
+	/* Same remask as scale-down: poll may have re-armed during disable. */
+	for (i = old_count; i < failed_queue; i++) {
+		ibmveth_disable_irq(adapter, i);
+		synchronize_irq(adapter->queue_irq[i]);
+	}
+
+	for (i = old_count; i < failed_queue; i++)
+		ibmveth_drain_rx_queue(adapter, i);
+
+	/* Drop the live count before freeing the half-added queues. */
+	ibmveth_publish_num_rx_queues(adapter, old_count);
+	if (netif_set_real_num_rx_queues(netdev, old_count))
+		schedule_work(&adapter->work);
+	synchronize_net();
+
+	for (i = failed_queue - 1; i >= old_count; i--) {
+		int keep;
+
+		if (!ibmveth_destroy_subordinate_rx_queue(adapter, i))
+			continue;
+
+		keep = i + 1;
+		ibmveth_publish_num_rx_queues(adapter, keep);
+		if (netif_set_real_num_rx_queues(netdev, keep))
+			schedule_work(&adapter->work);
+		for (i = old_count; i < keep; i++) {
+			int irq_rc;
+
+			ibmveth_replenish_task(adapter, i);
+			/* START: NAPI before PHYP unmask. */
+			napi_enable(&adapter->napi[i]);
+			irq_rc = ibmveth_enable_irq(adapter, i);
+			if (irq_rc) {
+				netdev_err(netdev,
+					   "IRQ %d rc=%d\n",
+					   i, irq_rc);
+				schedule_work(&adapter->work);
+				continue;
+			}
+			ibmveth_restart_rx_queue(adapter, i);
+		}
+		ibmveth_refresh_cmo_desired(adapter);
+		netdev_warn(netdev,
+			    "Keeping %d queues after scale-up failure\n",
+			    keep);
+		return rc;
 	}
+
+	/* Roll CMO desired back to the surviving queue count. */
+	ibmveth_refresh_cmo_desired(adapter);
+
+	netdev_warn(netdev, "Keeping %d queues after scale-up failure\n",
+		    old_count);
+	return rc;
 }
 
 /**
- * ibmveth_remove_buffer_from_pool - remove a buffer from a pool
- * @adapter: adapter instance
- * @correlator: identifies pool and index
- * @reuse: whether to reuse buffer
+ * ibmveth_scale_down_rx_queues - Remove RX queues from a live adapter
+ * @adapter: ibmveth adapter structure
+ * @old_count: Current number of RX queues
+ * @new_count: Target number of RX queues, below @old_count
  *
- * Return:
- * * %0       - success
- * * %-EINVAL - correlator maps to pool or index out of range
- * * %-EFAULT - pool and index map to null skb
+ * Quiesces the retiring queues, drains them and harvests their final
+ * no_buffer counts, publishes the surviving count, and only then
+ * deregisters them. Walks high to low so a failed free leaves 0..i
+ * live and republishes that count.
+ *
+ * Return: 0 on success, negative error code on failure
  */
-static int ibmveth_remove_buffer_from_pool(struct ibmveth_adapter *adapter,
-					   u64 correlator, bool reuse)
+static int
+ibmveth_scale_down_rx_queues(struct ibmveth_adapter *adapter, int old_count,
+			     int new_count)
 {
-	unsigned int pool  = correlator >> 32;
-	unsigned int index = correlator & 0xffffffffUL;
-	unsigned int free_index;
-	struct sk_buff *skb;
+	struct net_device *netdev = adapter->netdev;
+	int rc, i;
 
-	if (WARN_ON(pool >= IBMVETH_NUM_BUFF_POOLS) ||
-	    WARN_ON(index >= adapter->rx_buff_pool[pool].size)) {
-		schedule_work(&adapter->work);
-		return -EINVAL;
+	netdev_dbg(netdev, "Scale-down: removing queues %d-%d\n",
+		   new_count, old_count - 1);
+
+	/*
+	 * Mask PHYP before napi_disable so the handler cannot miss
+	 * a mask while NAPI is already dead. An in-flight poll can
+	 * still re-arm PHYP while napi_disable() waits, so remask
+	 * and sync again after NAPI is stopped. Then drain, harvest
+	 * final no_buffer stats, publish the surviving count, and
+	 * synchronize_net() before destroy so netpoll cannot walk
+	 * dying queues (handler may still be registered until destroy).
+	 */
+	for (i = new_count; i < old_count; i++) {
+		if (!adapter->queue_irq[i])
+			continue;
+		ibmveth_disable_irq(adapter, i);
+		synchronize_irq(adapter->queue_irq[i]);
 	}
 
-	skb = adapter->rx_buff_pool[pool].skbuff[index];
-	if (WARN_ON(!skb)) {
-		schedule_work(&adapter->work);
-		return -EFAULT;
+	for (i = new_count; i < old_count; i++)
+		napi_disable(&adapter->napi[i]);
+
+	for (i = new_count; i < old_count; i++) {
+		if (!adapter->queue_irq[i])
+			continue;
+		ibmveth_disable_irq(adapter, i);
+		synchronize_irq(adapter->queue_irq[i]);
 	}
 
-	/* if we are going to reuse the buffer then keep the pointers around
-	 * but mark index as available. replenish will see the skb pointer and
-	 * assume it is to be recycled.
+	for (i = new_count; i < old_count; i++)
+		ibmveth_drain_rx_queue(adapter, i);
+
+	/*
+	 * netpoll still reaches these queues until the publish
+	 * below, and ibmveth_replenish_task() folds the same
+	 * counters under the queue's replenish_lock. Take it here
+	 * so the harvest stays the single writer it documents.
 	 */
-	if (!reuse) {
-		/* remove the skb pointer to mark free. actual freeing is done
-		 * by upper level networking after gro_receive
+	for (i = new_count; i < old_count; i++) {
+		struct ibmveth_rx_q *rxq = &adapter->rx_queue[i];
+		unsigned long flags;
+
+		spin_lock_irqsave(&rxq->replenish_lock, flags);
+		ibmveth_update_rx_no_buffer(adapter, i);
+		spin_unlock_irqrestore(&rxq->replenish_lock, flags);
+	}
+
+	ibmveth_publish_num_rx_queues(adapter, new_count);
+	synchronize_net();
+
+	rc = netif_set_real_num_rx_queues(netdev, new_count);
+	if (rc) {
+		netdev_err(netdev, "Failed to set real RX queues to %d: %d\n",
+			   new_count, rc);
+		ibmveth_publish_num_rx_queues(adapter, old_count);
+		for (i = new_count; i < old_count; i++) {
+			int irq_rc;
+
+			ibmveth_replenish_task(adapter, i);
+			/* START: NAPI before PHYP unmask. */
+			napi_enable(&adapter->napi[i]);
+			irq_rc = ibmveth_enable_irq(adapter, i);
+			if (irq_rc) {
+				netdev_err(netdev,
+					   "Failed to re-enable IRQ for queue %d during scale-down rollback (rc=%d), scheduling reset\n",
+					   i, irq_rc);
+				schedule_work(&adapter->work);
+				continue;
+			}
+			ibmveth_restart_rx_queue(adapter, i);
+		}
+		return rc;
+	}
+
+	for (i = old_count - 1; i >= new_count; i--) {
+		int keep;
+
+		rc = ibmveth_destroy_subordinate_rx_queue(adapter, i);
+		if (!rc)
+			continue;
+
+		/*
+		 * High-to-low so a failed free leaves 0..i live.
+		 * Queues i+1..old_count-1 are already gone.
 		 */
-		adapter->rx_buff_pool[pool].skbuff[index] = NULL;
+		keep = i + 1;
+		ibmveth_publish_num_rx_queues(adapter, keep);
+		if (netif_set_real_num_rx_queues(netdev, keep))
+			schedule_work(&adapter->work);
+		for (i = new_count; i < keep; i++) {
+			int irq_rc;
+
+			ibmveth_replenish_task(adapter, i);
+			/* START: NAPI before PHYP unmask. */
+			napi_enable(&adapter->napi[i]);
+			irq_rc = ibmveth_enable_irq(adapter, i);
+			if (irq_rc) {
+				netdev_err(netdev,
+					   "IRQ %d rc=%d\n",
+					   i, irq_rc);
+				schedule_work(&adapter->work);
+				continue;
+			}
+			ibmveth_restart_rx_queue(adapter, i);
+		}
+		return rc;
+	}
 
-		dma_unmap_single(&adapter->vdev->dev,
-				 adapter->rx_buff_pool[pool].dma_addr[index],
-				 adapter->rx_buff_pool[pool].buff_size,
-				 DMA_FROM_DEVICE);
+	return 0;
+}
+
+/**
+ * ibmveth_resize_rx_queues_incremental - Resize RX queue count incrementally
+ * @adapter: ibmveth adapter structure
+ * @new_count: Target number of RX queues
+ * @rxq_entries: Number of entries per RX queue
+ *
+ * Adds or removes RX queues without tearing down the entire adapter.
+ * Active queues continue receiving during scale-up. Scale-up widens
+ * real_num_rx_queues before unmasking a new queue. Scale-down drains
+ * excess queues before deregistering them with the hypervisor.
+ *
+ * Return: 0 on success, negative error code on failure
+ */
+static int
+ibmveth_resize_rx_queues_incremental(struct ibmveth_adapter *adapter,
+				     int new_count, int rxq_entries)
+{
+	struct net_device *netdev = adapter->netdev;
+	int old_count = ibmveth_get_num_rx_queues(adapter);
+	int rc;
+
+	if (old_count == new_count) {
+		netdev_dbg(netdev, "RX queue count unchanged (%d), nothing to do\n",
+			   old_count);
+		return 0;
+	}
+
+	if (new_count < 1 || new_count > IBMVETH_MAX_RX_QUEUES) {
+		netdev_err(netdev, "Invalid RX queue count %d (must be 1-%d)\n",
+			   new_count, IBMVETH_MAX_RX_QUEUES);
+		return -EINVAL;
 	}
 
-	free_index = adapter->rx_buff_pool[pool].producer_index;
-	adapter->rx_buff_pool[pool].producer_index++;
-	if (adapter->rx_buff_pool[pool].producer_index >=
-	    adapter->rx_buff_pool[pool].size)
-		adapter->rx_buff_pool[pool].producer_index = 0;
-	adapter->rx_buff_pool[pool].free_map[free_index] = index;
+	netdev_info(netdev, "Incrementally resizing RX queues: %d to %d\n",
+		    old_count, new_count);
 
-	mb();
+	if (new_count > old_count)
+		rc = ibmveth_scale_up_rx_queues(adapter, old_count, new_count,
+						rxq_entries);
+	else
+		rc = ibmveth_scale_down_rx_queues(adapter, old_count,
+						  new_count);
+	if (rc)
+		return rc;
 
-	atomic_dec(&(adapter->rx_buff_pool[pool].available));
+	netdev_info(netdev, "Successfully resized to %u RX queues (incremental)\n",
+		    ibmveth_get_num_rx_queues(adapter));
+
+	ibmveth_refresh_cmo_desired(adapter);
 
 	return 0;
 }
 
-/* get the current buffer on the rx queue */
-static inline struct sk_buff *ibmveth_rxq_get_buffer(struct ibmveth_adapter *adapter)
+/**
+ * ibmveth_free_all_queues - Free all RX queues at once
+ * @adapter: ibmveth adapter structure
+ *
+ * Issues one H_FREE_LOGICAL_LAN for full adapter teardown. Per PAPR/PHYP,
+ * that drops the primary LAN and any subordinate queues registered under
+ * it. Incremental scale-down uses H_FREE_LOGICAL_LAN_QUEUE per queue
+ * instead; do not use this helper for partial live-set shrink.
+ *
+ * Used during interface close and registration error cleanup.
+ *
+ * Retries only H_BUSY and H_IS_LONG_BUSY. On other failures, logs and
+ * returns -EIO; queue_handle[] is cleared regardless. Callers still run
+ * RX pool and DMA teardown afterward (same as pre-helper close()).
+ *
+ * Clears queue handles only; queue_irq[] is released by
+ * ibmveth_cleanup_rx_interrupts() on close, or by
+ * ibmveth_dispose_subordinate_irq_mappings() on partial register failure.
+ *
+ * Return: 0 on success, -EIO if H_FREE_LOGICAL_LAN failed
+ */
+static int ibmveth_free_all_queues(struct ibmveth_adapter *adapter)
 {
-	u64 correlator = adapter->rx_queue.queue_addr[adapter->rx_queue.index].correlator;
-	unsigned int pool = correlator >> 32;
-	unsigned int index = correlator & 0xffffffffUL;
+	unsigned long lpar_rc;
+	int i;
 
-	if (WARN_ON(pool >= IBMVETH_NUM_BUFF_POOLS) ||
-	    WARN_ON(index >= adapter->rx_buff_pool[pool].size)) {
-		schedule_work(&adapter->work);
-		return NULL;
+	netdev_dbg(adapter->netdev, "freeing all RX queues at once\n");
+
+	do {
+		lpar_rc = h_free_logical_lan(adapter->vdev->unit_address);
+	} while (H_IS_LONG_BUSY(lpar_rc) || (lpar_rc == H_BUSY));
+
+	for (i = 0; i < ibmveth_get_num_rx_queues(adapter); i++)
+		adapter->queue_handle[i] = 0;
+
+	if (lpar_rc != H_SUCCESS) {
+		netdev_err(adapter->netdev,
+			   "h_free_logical_lan failed: %ld\n", lpar_rc);
+		return -EIO;
 	}
 
-	return adapter->rx_buff_pool[pool].skbuff[index];
+	return 0;
 }
 
 /**
- * ibmveth_rxq_harvest_buffer - Harvest buffer from pool
- *
- * @adapter: pointer to adapter
- * @reuse:   whether to reuse buffer
+ * ibmveth_register_rx_queues - Register RX queues with hypervisor
+ * @adapter: ibmveth adapter structure
+ * @mac_address: MAC address for device registration
  *
- * Context: called from ibmveth_poll
+ * Registers queue 0 via ibmveth_register_logical_lan(), then subordinate
+ * queues 1..N when multi-queue mode is enabled.
  *
- * Return:
- * * %0    - success
- * * other - non-zero return from ibmveth_remove_buffer_from_pool
+ * Return: 0 on success, -ENONET if queue 0 registration fails,
+ *         -EOPNOTSUPP if firmware rejects a subordinate queue (H_FUNCTION;
+ *         this function latches mq_fallback), -EIO on other subordinate
+ *         failures
  */
-static int ibmveth_rxq_harvest_buffer(struct ibmveth_adapter *adapter,
-				      bool reuse)
+static int
+ibmveth_register_rx_queues(struct ibmveth_adapter *adapter, u64 mac_address)
 {
-	u64 cor;
-	int rc;
+	struct net_device *netdev = adapter->netdev;
+	union ibmveth_buf_desc rxq_desc;
+	unsigned long lpar_rc;
+	unsigned int num;
+	int i, rc;
 
-	cor = adapter->rx_queue.queue_addr[adapter->rx_queue.index].correlator;
-	rc = ibmveth_remove_buffer_from_pool(adapter, cor, reuse);
-	if (unlikely(rc))
-		return rc;
+	rxq_desc.fields.flags_len = IBMVETH_BUF_VALID |
+				    adapter->rx_queue[0].queue_len;
+	rxq_desc.fields.address = adapter->rx_queue[0].queue_dma;
+	adapter->queue_irq[0] = netdev->irq;
+
+	rc = ibmveth_disable_irq(adapter, 0);
+	if (rc)
+		netdev_dbg(netdev,
+			   "Failed to disable IRQ for queue 0 before registration, rc=%d\n",
+			   rc);
 
-	if (++adapter->rx_queue.index == adapter->rx_queue.num_slots) {
-		adapter->rx_queue.index = 0;
-		adapter->rx_queue.toggle = !adapter->rx_queue.toggle;
+	lpar_rc = ibmveth_register_logical_lan(adapter, rxq_desc, mac_address);
+	if (lpar_rc != H_SUCCESS) {
+		netdev_err(netdev,
+			   "h_register_logical_lan failed: %ld\n", lpar_rc);
+		netdev_err(netdev,
+			   "buffer TCE:0x%llx filter TCE:0x%llx rxq desc:0x%llx MAC:0x%llx\n",
+			   adapter->buffer_list_dma[0],
+			   adapter->filter_list_dma,
+			   rxq_desc.desc, mac_address);
+		return -ENONET;
 	}
 
-	return 0;
-}
+	num = ibmveth_get_num_rx_queues(adapter);
+	if (num == 1 || !adapter->multi_queue) {
+		netdev_dbg(netdev,
+			   "registered 1 RX queue with hypervisor (single-queue mode)\n");
+		return 0;
+	}
 
-static void ibmveth_free_tx_ltb(struct ibmveth_adapter *adapter, int idx)
-{
-	dma_unmap_single(&adapter->vdev->dev, adapter->tx_ltb_dma[idx],
-			 adapter->tx_ltb_size, DMA_TO_DEVICE);
-	kfree(adapter->tx_ltb_ptr[idx]);
-	adapter->tx_ltb_ptr[idx] = NULL;
-}
+	netdev_dbg(netdev, "Registering %u subordinate queues (1-%u)\n",
+		   num - 1, num - 1);
 
-static int ibmveth_allocate_tx_ltb(struct ibmveth_adapter *adapter, int idx)
-{
-	adapter->tx_ltb_ptr[idx] = kzalloc(adapter->tx_ltb_size,
-					   GFP_KERNEL);
-	if (!adapter->tx_ltb_ptr[idx]) {
-		netdev_err(adapter->netdev,
-			   "unable to allocate tx long term buffer\n");
-		return -ENOMEM;
-	}
-	adapter->tx_ltb_dma[idx] = dma_map_single(&adapter->vdev->dev,
-						  adapter->tx_ltb_ptr[idx],
-						  adapter->tx_ltb_size,
-						  DMA_TO_DEVICE);
-	if (dma_mapping_error(&adapter->vdev->dev, adapter->tx_ltb_dma[idx])) {
-		netdev_err(adapter->netdev,
-			   "unable to DMA map tx long term buffer\n");
-		kfree(adapter->tx_ltb_ptr[idx]);
-		adapter->tx_ltb_ptr[idx] = NULL;
-		return -ENOMEM;
+	for (i = 1; i < num; i++) {
+		rc = ibmveth_register_single_rx_queue(adapter, i);
+		if (rc) {
+			/* Firmware MQ gone: fall back to SQ on next open. */
+			if (rc == -EOPNOTSUPP)
+				adapter->mq_fallback = true;
+			goto err_unregister;
+		}
 	}
 
+	netdev_dbg(netdev,
+		   "registered %u RX queues with hypervisor (multi-queue mode)\n",
+		   num);
+
 	return 0;
+
+err_unregister:
+	ibmveth_dispose_subordinate_irq_mappings(adapter);
+	ibmveth_free_all_queues(adapter);
+	return rc;
 }
 
-static int ibmveth_register_logical_lan(struct ibmveth_adapter *adapter,
-        union ibmveth_buf_desc rxq_desc, u64 mac_address)
+/**
+ * ibmveth_apply_mq_fallback - Drop multi-queue mode after firmware rejection
+ * @adapter: ibmveth adapter
+ *
+ * mq_fallback is set when firmware rejects MQ (subordinate register or
+ * buffer-add H_FUNCTION). Apply only at the start of open after teardown so
+ * num_rx_queues is not shrunk while IRQ/NAPI still reference higher queues.
+ * Consumes the flag and clears multi_queue, which is what makes the
+ * single-queue decision permanent for this device.
+ */
+static void ibmveth_apply_mq_fallback(struct ibmveth_adapter *adapter)
 {
-	int rc, try_again = 1;
+	struct net_device *netdev = adapter->netdev;
 
-	/*
-	 * After a kexec the adapter will still be open, so our attempt to
-	 * open it will fail. So if we get a failure we free the adapter and
-	 * try again, but only once.
-	 */
-retry:
-	rc = h_register_logical_lan(adapter->vdev->unit_address,
-				    adapter->buffer_list_dma, rxq_desc.desc,
-				    adapter->filter_list_dma, mac_address);
+	if (!adapter->mq_fallback)
+		return;
 
-	if (rc != H_SUCCESS && try_again) {
-		do {
-			rc = h_free_logical_lan(adapter->vdev->unit_address);
-		} while (H_IS_LONG_BUSY(rc) || (rc == H_BUSY));
+	adapter->mq_fallback = false;
 
-		try_again = 0;
-		goto retry;
-	}
+	if (!adapter->multi_queue && ibmveth_get_num_rx_queues(adapter) == 1)
+		return;
 
-	return rc;
+	netdev_warn(netdev,
+		    "Falling back to single RX queue (firmware MQ unavailable)\n");
+	adapter->multi_queue = false;
+	ibmveth_publish_num_rx_queues(adapter, 1);
+	/* real_num_rx_queues is set later in open after resources exist. */
+	if (adapter->rx_buffers_per_hcall > IBMVETH_MAX_RX_REGULAR)
+		adapter->rx_buffers_per_hcall = IBMVETH_MAX_RX_REGULAR;
 }
 
 static int ibmveth_open(struct net_device *netdev)
 {
 	struct ibmveth_adapter *adapter = netdev_priv(netdev);
-	u64 mac_address;
+	u64 mac_address = ether_addr_to_u64(netdev->dev_addr);
 	int rxq_entries = 1;
-	unsigned long lpar_rc;
 	int rc;
-	union ibmveth_buf_desc rxq_desc;
 	int i;
-	struct device *dev;
 
 	netdev_dbg(netdev, "open starting\n");
 
-	napi_enable(&adapter->napi);
+	ibmveth_apply_mq_fallback(adapter);
 
-	for(i = 0; i < IBMVETH_NUM_BUFF_POOLS; i++)
-		rxq_entries += adapter->rx_buff_pool[i].size;
+	for (i = 0; i < IBMVETH_NUM_BUFF_POOLS; i++)
+		rxq_entries += adapter->rx_buff_pool[0][i].size;
 
-	rc = -ENOMEM;
-	adapter->buffer_list_addr = (void*) get_zeroed_page(GFP_KERNEL);
-	if (!adapter->buffer_list_addr) {
-		netdev_err(netdev, "unable to allocate list pages\n");
+	rc = ibmveth_alloc_filter_list(adapter);
+	if (rc)
 		goto out;
-	}
-
-	adapter->filter_list_addr = (void*) get_zeroed_page(GFP_KERNEL);
-	if (!adapter->filter_list_addr) {
-		netdev_err(netdev, "unable to allocate filter pages\n");
-		goto out_free_buffer_list;
-	}
-
-	dev = &adapter->vdev->dev;
 
-	adapter->rx_queue.queue_len = sizeof(struct ibmveth_rx_q_entry) *
-						rxq_entries;
-	adapter->rx_queue.queue_addr =
-		dma_alloc_coherent(dev, adapter->rx_queue.queue_len,
-				   &adapter->rx_queue.queue_dma, GFP_KERNEL);
-	if (!adapter->rx_queue.queue_addr)
+	rc = ibmveth_alloc_rx_queues(adapter, rxq_entries);
+	if (rc)
 		goto out_free_filter_list;
 
-	adapter->buffer_list_dma = dma_map_single(dev,
-			adapter->buffer_list_addr, 4096, DMA_BIDIRECTIONAL);
-	if (dma_mapping_error(dev, adapter->buffer_list_dma)) {
-		netdev_err(netdev, "unable to map buffer list pages\n");
+	rc = ibmveth_alloc_buffer_pools(adapter);
+	if (rc)
 		goto out_free_queue_mem;
-	}
-
-	adapter->filter_list_dma = dma_map_single(dev,
-			adapter->filter_list_addr, 4096, DMA_BIDIRECTIONAL);
-	if (dma_mapping_error(dev, adapter->filter_list_dma)) {
-		netdev_err(netdev, "unable to map filter list pages\n");
-		goto out_unmap_buffer_list;
-	}
-
-	for (i = 0; i < netdev->real_num_tx_queues; i++) {
-		if (ibmveth_allocate_tx_ltb(adapter, i))
-			goto out_free_tx_ltb;
-	}
-
-	adapter->rx_queue.index = 0;
-	adapter->rx_queue.num_slots = rxq_entries;
-	adapter->rx_queue.toggle = 1;
-
-	mac_address = ether_addr_to_u64(netdev->dev_addr);
 
-	rxq_desc.fields.flags_len = IBMVETH_BUF_VALID |
-					adapter->rx_queue.queue_len;
-	rxq_desc.fields.address = adapter->rx_queue.queue_dma;
-
-	netdev_dbg(netdev, "buffer list @ 0x%p\n", adapter->buffer_list_addr);
-	netdev_dbg(netdev, "filter list @ 0x%p\n", adapter->filter_list_addr);
-	netdev_dbg(netdev, "receive q   @ 0x%p\n", adapter->rx_queue.queue_addr);
-
-	h_vio_signal(adapter->vdev->unit_address, VIO_IRQ_DISABLE);
+	rc = ibmveth_register_rx_queues(adapter, mac_address);
+	if (rc)
+		goto out_free_buffer_pools;
 
-	lpar_rc = ibmveth_register_logical_lan(adapter, rxq_desc, mac_address);
+	rc = netif_set_real_num_rx_queues(netdev,
+					  ibmveth_get_num_rx_queues(adapter));
 
-	if (lpar_rc != H_SUCCESS) {
-		netdev_err(netdev, "h_register_logical_lan failed with %ld\n",
-			   lpar_rc);
-		netdev_err(netdev, "buffer TCE:0x%llx filter TCE:0x%llx rxq "
-			   "desc:0x%llx MAC:0x%llx\n",
-				     adapter->buffer_list_dma,
-				     adapter->filter_list_dma,
-				     rxq_desc.desc,
-				     mac_address);
-		rc = -ENONET;
-		goto out_unmap_filter_list;
+	if (rc) {
+		netdev_err(netdev, "failed to set number of rx queues\n");
+		goto out_unregister_queues;
 	}
 
-	for (i = 0; i < IBMVETH_NUM_BUFF_POOLS; i++) {
-		if (!adapter->rx_buff_pool[i].active)
-			continue;
-		if (ibmveth_alloc_buffer_pool(&adapter->rx_buff_pool[i])) {
-			netdev_err(netdev, "unable to alloc pool\n");
-			adapter->rx_buff_pool[i].active = 0;
-			rc = -ENOMEM;
-			goto out_free_buffer_pools;
-		}
+	/*
+	 * Post buffers before setup_rx_interrupts(). MQ setup then unmasks
+	 * PHYP; SQ setup leaves PHYP masked. Scheduling NAPI only when a
+	 * descriptor is already pending is not enough: after ifdown/up
+	 * (RX=8, no -L) NAPI can be idle with nothing pending and the
+	 * queue stays dead (TX OK, ARP/RX fail).
+	 * restart_rx_queue() replenishes, schedules NAPI, and unmasks if
+	 * prep fails.
+	 */
+	for (i = 0; i < ibmveth_get_num_rx_queues(adapter); i++) {
+		netdev_dbg(netdev, "initial replenish cycle for queue %d\n", i);
+		ibmveth_replenish_task(adapter, i);
 	}
 
-	netdev_dbg(netdev, "registering irq 0x%x\n", netdev->irq);
-	rc = request_irq(netdev->irq, ibmveth_interrupt, 0, netdev->name,
-			 netdev);
-	if (rc != 0) {
-		netdev_err(netdev, "unable to request irq 0x%x, rc %d\n",
-			   netdev->irq, rc);
-		do {
-			lpar_rc = h_free_logical_lan(adapter->vdev->unit_address);
-		} while (H_IS_LONG_BUSY(lpar_rc) || (lpar_rc == H_BUSY));
-
-		goto out_free_buffer_pools;
-	}
+	rc = ibmveth_setup_rx_interrupts(adapter);
+	if (rc)
+		goto out_free_all_queues; /* setup already disposed IRQs */
 
-	rc = -ENOMEM;
+	for (i = 0; i < ibmveth_get_num_rx_queues(adapter); i++)
+		ibmveth_restart_rx_queue(adapter, i);
 
-	netdev_dbg(netdev, "initial replenish cycle\n");
-	ibmveth_interrupt(netdev->irq, netdev);
+	rc = ibmveth_alloc_tx_resources(adapter);
+	if (rc)
+		goto out_cleanup_rx_interrupts;
 
 	netif_tx_start_all_queues(netdev);
 
+	adapter->opened = true;
 	netdev_dbg(netdev, "open complete\n");
 
 	return 0;
 
+out_cleanup_rx_interrupts:
+	ibmveth_cleanup_rx_interrupts(adapter);
+	/* As in close(): a poll past napi_complete_done() may still run. */
+	synchronize_net();
+	goto out_free_all_queues; /* cleanup already disposed IRQs */
+out_unregister_queues:
+	ibmveth_dispose_subordinate_irq_mappings(adapter);
+out_free_all_queues:
+	ibmveth_free_all_queues(adapter);
 out_free_buffer_pools:
-	while (--i >= 0) {
-		if (adapter->rx_buff_pool[i].active)
-			ibmveth_free_buffer_pool(adapter,
-						 &adapter->rx_buff_pool[i]);
-	}
-out_unmap_filter_list:
-	dma_unmap_single(dev, adapter->filter_list_dma, 4096,
-			 DMA_BIDIRECTIONAL);
-
-out_free_tx_ltb:
-	while (--i >= 0) {
-		ibmveth_free_tx_ltb(adapter, i);
-	}
-
-out_unmap_buffer_list:
-	dma_unmap_single(dev, adapter->buffer_list_dma, 4096,
-			 DMA_BIDIRECTIONAL);
+	ibmveth_free_buffer_pools(adapter);
 out_free_queue_mem:
-	dma_free_coherent(dev, adapter->rx_queue.queue_len,
-			  adapter->rx_queue.queue_addr,
-			  adapter->rx_queue.queue_dma);
+	ibmveth_cleanup_rx_resources(adapter);
 out_free_filter_list:
-	free_page((unsigned long)adapter->filter_list_addr);
-out_free_buffer_list:
-	free_page((unsigned long)adapter->buffer_list_addr);
+	ibmveth_free_filter_list(adapter);
 out:
-	napi_disable(&adapter->napi);
 	return rc;
 }
 
 static int ibmveth_close(struct net_device *netdev)
 {
 	struct ibmveth_adapter *adapter = netdev_priv(netdev);
-	struct device *dev = &adapter->vdev->dev;
-	long lpar_rc;
 	int i;
 
-	netdev_dbg(netdev, "close starting\n");
-
-	napi_disable(&adapter->napi);
-
-	netif_tx_stop_all_queues(netdev);
-
-	h_vio_signal(adapter->vdev->unit_address, VIO_IRQ_DISABLE);
-
-	do {
-		lpar_rc = h_free_logical_lan(adapter->vdev->unit_address);
-	} while (H_IS_LONG_BUSY(lpar_rc) || (lpar_rc == H_BUSY));
-
-	if (lpar_rc != H_SUCCESS) {
-		netdev_err(netdev, "h_free_logical_lan failed with %lx, "
-			   "continuing with close\n", lpar_rc);
-	}
-
-	free_irq(netdev->irq, netdev);
+	/* Gate on opened, not IFF_UP: pool_store/change_mtu close+open can
+	 * leave IFF_UP set after a failed reopen.
+	 */
+	if (!adapter->opened)
+		return 0;
 
-	ibmveth_update_rx_no_buffer(adapter);
+	adapter->opened = false;
 
-	dma_unmap_single(dev, adapter->buffer_list_dma, 4096,
-			 DMA_BIDIRECTIONAL);
-	free_page((unsigned long)adapter->buffer_list_addr);
+	netdev_dbg(netdev, "close starting\n");
 
-	dma_unmap_single(dev, adapter->filter_list_dma, 4096,
-			 DMA_BIDIRECTIONAL);
-	free_page((unsigned long)adapter->filter_list_addr);
+	/* Disable and wait for in-flight ndo_start_xmit (stop_all_queues
+	 * alone does not). Direct close() callers bypass synchronize_net().
+	 */
+	netif_tx_disable(netdev);
 
-	dma_free_coherent(dev, adapter->rx_queue.queue_len,
-			  adapter->rx_queue.queue_addr,
-			  adapter->rx_queue.queue_dma);
+	ibmveth_cleanup_rx_interrupts(adapter);
+	/* Wait for softirq/poll that already passed shutdown checks. */
+	synchronize_net();
 
-	for (i = 0; i < IBMVETH_NUM_BUFF_POOLS; i++)
-		if (adapter->rx_buff_pool[i].active)
-			ibmveth_free_buffer_pool(adapter,
-						 &adapter->rx_buff_pool[i]);
+	for (i = 0; i < ibmveth_get_num_rx_queues(adapter); i++) {
+		struct ibmveth_rx_q *rxq = &adapter->rx_queue[i];
+		unsigned long flags;
 
-	for (i = 0; i < netdev->real_num_tx_queues; i++)
-		ibmveth_free_tx_ltb(adapter, i);
+		spin_lock_irqsave(&rxq->replenish_lock, flags);
+		ibmveth_update_rx_no_buffer(adapter, i);
+		spin_unlock_irqrestore(&rxq->replenish_lock, flags);
+	}
+	if (!ibmveth_free_all_queues(adapter))
+		ibmveth_free_stranded_rx_queues(adapter);
+	/* Free TX LTBs after quiesce and after H_FREE_LOGICAL_LAN so xmit
+	 * cannot touch unmapped bounce buffers while the LAN is live.
+	 */
+	ibmveth_free_tx_resources(adapter);
+	ibmveth_free_buffer_pools(adapter);
+	ibmveth_cleanup_rx_resources(adapter);
+	ibmveth_free_filter_list(adapter);
 
 	netdev_dbg(netdev, "close complete\n");
 
@@ -840,6 +3037,10 @@ static void ibmveth_reset(struct work_struct *w)
 	netdev_dbg(netdev, "reset starting\n");
 
 	rtnl_lock();
+	if (netdev->reg_state != NETREG_REGISTERED) {
+		rtnl_unlock();
+		return;
+	}
 
 	dev_close(adapter->netdev);
 	dev_open(adapter->netdev, NULL);
@@ -1073,22 +3274,96 @@ static int ibmveth_set_features(struct net_device *dev,
 	return rc1 ? rc1 : rc2;
 }
 
-static void ibmveth_get_strings(struct net_device *dev, u32 stringset, u8 *data)
+/*
+ * Sum per-queue counters for rare ethtool reads. The hot paths only ever
+ * touch their own queue's slot, so nothing here needs an atomic; the cost
+ * of aggregation is paid by the reader instead (ibmvnic-style).
+ *
+ * Every slot is summed, not just the live ones, so that shrinking the
+ * queue count with ethtool -L cannot make a counter go backwards.
+ */
+static u64 ibmveth_sum_rx_qstat(struct ibmveth_adapter *adapter, size_t off)
+{
+	u64 total = 0;
+	int i;
+
+	for (i = 0; i < IBMVETH_MAX_RX_QUEUES; i++)
+		total += *(u64 *)((u8 *)&adapter->rx_qstats[i] + off);
+
+	return total;
+}
+
+static u64 ibmveth_sum_tx_qstat(struct ibmveth_adapter *adapter, size_t off)
 {
+	u64 total = 0;
 	int i;
 
+	for (i = 0; i < IBMVETH_MAX_QUEUES; i++)
+		total += *(u64 *)((u8 *)&adapter->tx_qstats[i] + off);
+
+	return total;
+}
+
+static u64 ibmveth_ethtool_adapter_stat(struct ibmveth_adapter *adapter,
+					int index)
+{
+	const struct ibmveth_stat *stat = &ibmveth_stats[index];
+
+	switch (stat->src) {
+	case IBMVETH_STAT_RX_QSUM:
+		return ibmveth_sum_rx_qstat(adapter, stat->off);
+	case IBMVETH_STAT_TX_QSUM:
+		return ibmveth_sum_tx_qstat(adapter, stat->off);
+	case IBMVETH_STAT_RX_NO_BUFFER:
+		/*
+		 * PHYP's page counter is absolute for the page currently
+		 * mapped, so a reopen or queue reuse restarts it near zero.
+		 * ibmveth_update_rx_no_buffer() folds each decrease into the
+		 * queue's retired carry; add both back to stay monotonic.
+		 */
+		return ibmveth_sum_rx_qstat(adapter, stat->off) +
+		       ibmveth_sum_rx_qstat(adapter,
+					    IBMVETH_RXQ_OFF(no_buffer_retired));
+	case IBMVETH_STAT_ADAPTER:
+		break;
+	}
+
+	return IBMVETH_GET_STAT(adapter, stat->off);
+}
+
+static void ibmveth_get_strings(struct net_device *dev, u32 stringset, u8 *data)
+{
+	struct ibmveth_adapter *adapter = netdev_priv(dev);
+	u8 *p = data;
+	int i, j;
+
 	if (stringset != ETH_SS_STATS)
 		return;
 
-	for (i = 0; i < ARRAY_SIZE(ibmveth_stats); i++, data += ETH_GSTRING_LEN)
-		memcpy(data, ibmveth_stats[i].name, ETH_GSTRING_LEN);
+	for (i = 0; i < ARRAY_SIZE(ibmveth_stats); i++) {
+		memcpy(p, ibmveth_stats[i].name, ETH_GSTRING_LEN);
+		p += ETH_GSTRING_LEN;
+	}
+
+	for (i = 0; i < ibmveth_get_num_rx_queues(adapter); i++)
+		for (j = 0; j < IBMVETH_NUM_RX_QSTATS; j++)
+			ethtool_sprintf(&p, ibmveth_rx_qstat_keys[j].fmt, i);
+
+	for (i = 0; i < dev->real_num_tx_queues; i++)
+		for (j = 0; j < IBMVETH_NUM_TX_QSTATS; j++)
+			ethtool_sprintf(&p, ibmveth_tx_qstat_keys[j].fmt, i);
 }
 
 static int ibmveth_get_sset_count(struct net_device *dev, int sset)
 {
+	struct ibmveth_adapter *adapter = netdev_priv(dev);
+
 	switch (sset) {
 	case ETH_SS_STATS:
-		return ARRAY_SIZE(ibmveth_stats);
+		return ARRAY_SIZE(ibmveth_stats) +
+		       ibmveth_get_num_rx_queues(adapter) *
+		       IBMVETH_NUM_RX_QSTATS +
+		       dev->real_num_tx_queues * IBMVETH_NUM_TX_QSTATS;
 	default:
 		return -EOPNOTSUPP;
 	}
@@ -1097,44 +3372,211 @@ static int ibmveth_get_sset_count(struct net_device *dev, int sset)
 static void ibmveth_get_ethtool_stats(struct net_device *dev,
 				      struct ethtool_stats *stats, u64 *data)
 {
-	int i;
 	struct ibmveth_adapter *adapter = netdev_priv(dev);
+	int i, j, k;
 
 	for (i = 0; i < ARRAY_SIZE(ibmveth_stats); i++)
-		data[i] = IBMVETH_GET_STAT(adapter, ibmveth_stats[i].offset);
+		data[i] = ibmveth_ethtool_adapter_stat(adapter, i);
+
+	for (j = 0; j < ibmveth_get_num_rx_queues(adapter); j++) {
+		const u8 *q = (const u8 *)&adapter->rx_qstats[j];
+
+		for (k = 0; k < IBMVETH_NUM_RX_QSTATS; k++)
+			data[i++] = *(const u64 *)
+				(q + ibmveth_rx_qstat_keys[k].off);
+	}
+
+	for (j = 0; j < dev->real_num_tx_queues; j++) {
+		const u8 *q = (const u8 *)&adapter->tx_qstats[j];
+
+		for (k = 0; k < IBMVETH_NUM_TX_QSTATS; k++)
+			data[i++] = *(const u64 *)
+				(q + ibmveth_tx_qstat_keys[k].off);
+	}
 }
 
 static void ibmveth_get_channels(struct net_device *netdev,
 				 struct ethtool_channels *channels)
 {
-	channels->max_tx = ibmveth_real_max_tx_queues();
+	struct ibmveth_adapter *adapter = netdev_priv(netdev);
+	unsigned int rx_count = ibmveth_get_num_rx_queues(adapter);
+
 	channels->tx_count = netdev->real_num_tx_queues;
+	/*
+	 * Never advertise max_tx below the live count. After CPU
+	 * offline, real_max can drop below real_num_tx_queues;
+	 * ethtool -L is read-modify-write and the core rejects
+	 * tx_count > max_tx, which would also block an RX-only
+	 * request. set_channels uses this same ceiling.
+	 */
+	channels->max_tx = max_t(unsigned int, channels->tx_count,
+				 ibmveth_real_max_tx_queues());
+
+	/*
+	 * Always report the live RX count. ethtool -L is read-modify-
+	 * write, so a TX-only request echoes rx_count back at us; an
+	 * understated value would be applied as a silent RX shrink.
+	 * mq_fallback instead caps max_rx at the live count, which
+	 * blocks growth in the core without misreporting what is
+	 * currently configured.
+	 */
+	channels->rx_count = rx_count;
+	if (adapter->multi_queue && !adapter->mq_fallback)
+		channels->max_rx = IBMVETH_MAX_RX_QUEUES;
+	else
+		channels->max_rx = rx_count;
+}
+
+/**
+ * ibmveth_resize_rx_channels - Validate and apply a new RX queue count
+ * @adapter: ibmveth adapter
+ * @goal_rx: requested RX queue count
+ *
+ * Rejects rx > 1 without MQ firmware (-EOPNOTSUPP) and rx outside
+ * 1..IBMVETH_MAX_RX_QUEUES (-EINVAL). When RX resources are not live
+ * (!opened), only validate; do not allocate. When up, apply via
+ * ibmveth_resize_rx_queues_incremental().
+ *
+ * Return: 0 or negative errno
+ */
+static int ibmveth_resize_rx_channels(struct ibmveth_adapter *adapter,
+				      unsigned int goal_rx)
+{
+	struct net_device *netdev = adapter->netdev;
+	unsigned int old_rx = ibmveth_get_num_rx_queues(adapter);
+	int rxq_entries;
+	int rc;
+
+	if (goal_rx < 1 || goal_rx > IBMVETH_MAX_RX_QUEUES) {
+		netdev_err(netdev,
+			   "Invalid RX queue count %u (must be 1-%d)\n",
+			   goal_rx, IBMVETH_MAX_RX_QUEUES);
+		return -EINVAL;
+	}
+
+	/*
+	 * Check for a no-op before the capability gate. ethtool -L is
+	 * read-modify-write, so a TX-only request arrives carrying the
+	 * current RX count; gating first would fail those with
+	 * -EOPNOTSUPP once mq_fallback is set.
+	 */
+	if (goal_rx == old_rx)
+		return 0;
+
+	/*
+	 * Refuse any rx > 1, not just growth: once mq_fallback is set the
+	 * next open comes up single-queue, so an intermediate count could
+	 * not be honoured either, and accepting it would only repeat the
+	 * silent clamp at open. max_rx stays at the live count so that
+	 * read-modify-write TX-only requests still clear the core.
+	 */
+	if (goal_rx > 1 && (!adapter->multi_queue || adapter->mq_fallback)) {
+		netdev_err(netdev,
+			   "Cannot resize to %u RX queues: multi-queue mode not supported by firmware\n",
+			   goal_rx);
+		return -EOPNOTSUPP;
+	}
 
-	channels->max_rx = netdev->real_num_rx_queues;
-	channels->rx_count = netdev->real_num_rx_queues;
+	/* Down / failed-open: do not allocate. */
+	if (!adapter->opened)
+		return 0;
+
+	rxq_entries = adapter->rx_queue[0].num_slots;
+	rc = ibmveth_resize_rx_queues_incremental(adapter, goal_rx,
+						  rxq_entries);
+	if (rc)
+		netdev_err(netdev, "Failed to resize RX queues: %d\n", rc);
+	return rc;
 }
 
 static int ibmveth_set_channels(struct net_device *netdev,
 				struct ethtool_channels *channels)
 {
 	struct ibmveth_adapter *adapter = netdev_priv(netdev);
-	unsigned int old = netdev->real_num_tx_queues,
-		     goal = channels->tx_count;
+	unsigned int old_rx = ibmveth_get_num_rx_queues(adapter);
+	unsigned int goal_rx = channels->rx_count;
+	unsigned int old_tx = netdev->real_num_tx_queues;
+	unsigned int goal_tx = channels->tx_count;
+	unsigned int want_tx = goal_tx;
+	unsigned int max_tx;
+	bool rx_changed = false;
 	int rc, i;
 
-	/* If ndo_open has not been called yet then don't allocate, just set
-	 * desired netdev_queue's and return
+	/*
+	 * Same ceiling get_channels reports: live count or the
+	 * online-CPU cap, whichever is larger.
+	 */
+	max_tx = max_t(unsigned int, old_tx,
+		       ibmveth_real_max_tx_queues());
+	if (goal_tx < 1 || goal_tx > max_tx) {
+		netdev_err(netdev,
+			   "Invalid TX queue count %u (must be 1-%u)\n",
+			   goal_tx, max_tx);
+		return -EINVAL;
+	}
+
+	/* RX range / MQ checks live in ibmveth_resize_rx_channels(). */
+	rc = ibmveth_resize_rx_channels(adapter, goal_rx);
+	if (rc)
+		return rc;
+
+	/* If RX resources are not live (never opened, or close+open failed
+	 * while IFF_UP stayed set), publish desired queue counts without
+	 * allocating.
 	 */
-	if (!(netdev->flags & IFF_UP))
-		return netif_set_real_num_tx_queues(netdev, goal);
+	if (!adapter->opened) {
+		/* Apply TX first so a failure leaves the published RX
+		 * count unchanged.
+		 */
+		rc = netif_set_real_num_tx_queues(netdev, goal_tx);
+		if (rc)
+			return rc;
+
+		/* Publish desired RX count for next open() and refresh CMO;
+		 * do not allocate while down.
+		 */
+		if (goal_rx != ibmveth_get_num_rx_queues(adapter)) {
+			ibmveth_publish_num_rx_queues(adapter, goal_rx);
+			rc = netif_set_real_num_rx_queues(netdev, goal_rx);
+			if (rc) {
+				int tx_rc;
+
+				ibmveth_publish_num_rx_queues(adapter, old_rx);
+				tx_rc = netif_set_real_num_tx_queues(netdev,
+								     old_tx);
+				if (tx_rc)
+					netdev_err(netdev,
+						   "Failed to restore TX queues to %u after RX failure: %d\n",
+						   old_tx, tx_rc);
+				return rc;
+			}
+			if (firmware_has_feature(FW_FEATURE_CMO)) {
+				unsigned long dma;
+
+				dma = ibmveth_get_desired_dma(adapter->vdev);
+				vio_cmo_set_dev_desired(adapter->vdev, dma);
+			}
+		}
+		return 0;
+	}
+
+	if (goal_rx != old_rx)
+		rx_changed = true;
 
 	/* We have IBMVETH_MAX_QUEUES netdev_queue's allocated
 	 * but we may need to alloc/free the ltb's.
 	 */
+	if (goal_tx == old_tx)
+		return 0;
+
 	netif_tx_stop_all_queues(netdev);
 
-	/* Allocate any queue that we need */
-	for (i = old; i < goal; i++) {
+	/* Allocate any new TX LTBs. i starts at old_tx for the free walk
+	 * below when this loop body never runs (goal_tx == old_tx already
+	 * returned; goal_tx < old_tx is scale-down).
+	 */
+	i = old_tx;
+	for (; i < goal_tx; i++) {
 		if (adapter->tx_ltb_ptr[i])
 			continue;
 
@@ -1143,28 +3585,48 @@ static int ibmveth_set_channels(struct net_device *netdev,
 			continue;
 
 		/* if something goes wrong, free everything we just allocated */
-		netdev_err(netdev, "Failed to allocate more tx queues, returning to %d queues\n",
-			   old);
-		goal = old;
-		old = i;
+		netdev_err(netdev, "Failed to allocate more tx queues, returning to %u queues\n",
+			   old_tx);
+		goal_tx = old_tx;
+		old_tx = i;
 		break;
 	}
-	rc = netif_set_real_num_tx_queues(netdev, goal);
+	rc = netif_set_real_num_tx_queues(netdev, goal_tx);
 	if (rc) {
-		netdev_err(netdev, "Failed to set real tx queues, returning to %d queues\n",
-			   old);
-		goal = old;
-		old = i;
+		netdev_err(netdev, "Failed to set real tx queues, returning to %u queues\n",
+			   old_tx);
+		goal_tx = old_tx;
+		old_tx = i;
 	}
 	/* Free any that are no longer needed */
-	for (i = old; i > goal; i--) {
+	for (i = old_tx; i > goal_tx; i--) {
 		if (adapter->tx_ltb_ptr[i - 1])
 			ibmveth_free_tx_ltb(adapter, i - 1);
 	}
 
 	netif_tx_wake_all_queues(netdev);
 
-	return rc;
+	if (netdev->real_num_tx_queues != want_tx) {
+		if (rx_changed) {
+			/*
+			 * Restore the RX count from before this -L.
+			 * num_slots is the live size after that resize.
+			 */
+			int rxq_entries = adapter->rx_queue[0].num_slots;
+			int rb;
+
+			rb = ibmveth_resize_rx_queues_incremental(adapter,
+								  old_rx,
+								  rxq_entries);
+			if (rb)
+				netdev_err(netdev,
+					   "Failed to roll back RX queues to %u after TX failure: %d\n",
+					   old_rx, rb);
+		}
+		return rc ? rc : -ENOMEM;
+	}
+
+	return 0;
 }
 
 static const struct ethtool_ops netdev_ethtool_ops = {
@@ -1213,8 +3675,10 @@ static int ibmveth_send(struct ibmveth_adapter *adapter,
 }
 
 static int ibmveth_is_packet_unsupported(struct sk_buff *skb,
-					 struct net_device *netdev)
+					 struct ibmveth_adapter *adapter,
+					 int queue_num)
 {
+	struct net_device *netdev = adapter->netdev;
 	struct ethhdr *ether_header;
 	int ret = 0;
 
@@ -1222,7 +3686,7 @@ static int ibmveth_is_packet_unsupported(struct sk_buff *skb,
 
 	if (ether_addr_equal(ether_header->h_dest, netdev->dev_addr)) {
 		netdev_dbg(netdev, "veth doesn't support loopback packets, dropping packet.\n");
-		netdev->stats.tx_dropped++;
+		adapter->tx_qstats[queue_num].dropped_packets++;
 		ret = -EOPNOTSUPP;
 	}
 
@@ -1238,7 +3702,13 @@ static netdev_tx_t ibmveth_start_xmit(struct sk_buff *skb,
 	int i, queue_num = skb_get_queue_mapping(skb);
 	unsigned long mss = 0;
 
-	if (ibmveth_is_packet_unsupported(skb, netdev))
+	/* Close / failed reopen can free LTBs while IFF_UP is still set. */
+	if (unlikely(!adapter->tx_ltb_ptr[queue_num])) {
+		adapter->tx_qstats[queue_num].dropped_packets++;
+		goto out;
+	}
+
+	if (ibmveth_is_packet_unsupported(skb, adapter, queue_num))
 		goto out;
 	/* veth can't checksum offload UDP */
 	if (skb->ip_summed == CHECKSUM_PARTIAL &&
@@ -1249,7 +3719,7 @@ static netdev_tx_t ibmveth_start_xmit(struct sk_buff *skb,
 	    skb_checksum_help(skb)) {
 
 		netdev_err(netdev, "tx: failed to checksum packet\n");
-		netdev->stats.tx_dropped++;
+		adapter->tx_qstats[queue_num].dropped_packets++;
 		goto out;
 	}
 
@@ -1261,6 +3731,8 @@ static netdev_tx_t ibmveth_start_xmit(struct sk_buff *skb,
 
 		desc_flags |= (IBMVETH_BUF_NO_CSUM | IBMVETH_BUF_CSUM_GOOD);
 
+		adapter->tx_qstats[queue_num].checksum_offload++;
+
 		/* Need to zero out the checksum */
 		buf[0] = 0;
 		buf[1] = 0;
@@ -1272,7 +3744,7 @@ static netdev_tx_t ibmveth_start_xmit(struct sk_buff *skb,
 	if (skb->ip_summed == CHECKSUM_PARTIAL && skb_is_gso(skb)) {
 		if (adapter->fw_large_send_support) {
 			mss = (unsigned long)skb_shinfo(skb)->gso_size;
-			adapter->tx_large_packets++;
+			adapter->tx_qstats[queue_num].large_packets++;
 		} else if (!skb_is_gso_v6(skb)) {
 			/* Put -1 in the IP checksum to tell phyp it
 			 * is a largesend packet. Put the mss in
@@ -1281,7 +3753,7 @@ static netdev_tx_t ibmveth_start_xmit(struct sk_buff *skb,
 			ip_hdr(skb)->check = 0xffff;
 			tcp_hdr(skb)->check =
 				cpu_to_be16(skb_shinfo(skb)->gso_size);
-			adapter->tx_large_packets++;
+			adapter->tx_qstats[queue_num].large_packets++;
 		}
 	}
 
@@ -1289,7 +3761,7 @@ static netdev_tx_t ibmveth_start_xmit(struct sk_buff *skb,
 	if (unlikely(skb->len > adapter->tx_ltb_size)) {
 		netdev_err(adapter->netdev, "tx: packet size (%u) exceeds ltb (%u)\n",
 			   skb->len, adapter->tx_ltb_size);
-		netdev->stats.tx_dropped++;
+		adapter->tx_qstats[queue_num].dropped_packets++;
 		goto out;
 	}
 	memcpy(adapter->tx_ltb_ptr[queue_num], skb->data, skb_headlen(skb));
@@ -1306,7 +3778,7 @@ static netdev_tx_t ibmveth_start_xmit(struct sk_buff *skb,
 	if (unlikely(total_bytes != skb->len)) {
 		netdev_err(adapter->netdev, "tx: incorrect packet len copied into ltb (%u != %u)\n",
 			   skb->len, total_bytes);
-		netdev->stats.tx_dropped++;
+		adapter->tx_qstats[queue_num].dropped_packets++;
 		goto out;
 	}
 	desc.fields.flags_len = desc_flags | skb->len;
@@ -1315,11 +3787,11 @@ static netdev_tx_t ibmveth_start_xmit(struct sk_buff *skb,
 	dma_wmb();
 
 	if (ibmveth_send(adapter, desc.desc, mss)) {
-		adapter->tx_send_failed++;
-		netdev->stats.tx_dropped++;
+		adapter->tx_qstats[queue_num].send_failures++;
+		adapter->tx_qstats[queue_num].dropped_packets++;
 	} else {
-		netdev->stats.tx_packets++;
-		netdev->stats.tx_bytes += skb->len;
+		adapter->tx_qstats[queue_num].packets++;
+		adapter->tx_qstats[queue_num].bytes += skb->len;
 	}
 
 out:
@@ -1445,102 +3917,226 @@ static void ibmveth_rx_csum_helper(struct sk_buff *skb,
 	}
 }
 
-static int ibmveth_poll(struct napi_struct *napi, int budget)
+static void ibmveth_poll_bump_invalid(struct ibmveth_adapter *adapter,
+				      int queue_index)
 {
-	struct ibmveth_adapter *adapter =
-			container_of(napi, struct ibmveth_adapter, napi);
-	struct net_device *netdev = adapter->netdev;
-	int frames_processed = 0;
-	unsigned long lpar_rc;
+	adapter->rx_qstats[queue_index].invalid_buffers++;
+}
+
+static bool ibmveth_poll_stopping(struct net_device *netdev,
+				  struct napi_struct *napi)
+{
+	return !netif_running(netdev) || napi_disable_pending(napi);
+}
+
+static bool ibmveth_poll_harvest_slot(struct ibmveth_adapter *adapter,
+				      int queue_index, bool reuse)
+{
+	int rc = ibmveth_rxq_harvest_buffer(adapter, queue_index, reuse);
+
+	return !rc || rc == -EINVAL || rc == -EFAULT;
+}
+
+static bool ibmveth_poll_recycle_invalid(struct net_device *netdev,
+					 struct ibmveth_adapter *adapter,
+					 int queue_index)
+{
+	netdev_dbg(netdev, "recycling invalid buffer\n");
+	ibmveth_poll_bump_invalid(adapter, queue_index);
+	return ibmveth_poll_harvest_slot(adapter, queue_index, true);
+}
+
+static bool ibmveth_poll_skip_bad_correlator(struct net_device *netdev,
+					     struct ibmveth_adapter *adapter,
+					     int queue_index)
+{
+	if (net_ratelimit())
+		netdev_err(netdev,
+			   "bad correlator on queue %d, skipping slot\n",
+			   queue_index);
+	/* Residual stale slot after resize: recover via reset rather
+	 * than spinning forever. Always escalate; only the log is
+	 * rate-limited.
+	 */
+	schedule_work(&adapter->work);
+	ibmveth_poll_bump_invalid(adapter, queue_index);
+	return ibmveth_poll_harvest_slot(adapter, queue_index, true);
+}
+
+static bool ibmveth_poll_drop_oversize(struct net_device *netdev,
+				       struct ibmveth_adapter *adapter,
+				     int queue_index, unsigned int off,
+				     unsigned int len, unsigned int room)
+{
+	if (net_ratelimit())
+		netdev_err(netdev,
+			   "RX frame %u+%u exceeds buffer %u on queue %d, dropping\n",
+			   off, len, room, queue_index);
+	ibmveth_poll_bump_invalid(adapter, queue_index);
+	return ibmveth_poll_harvest_slot(adapter, queue_index, true);
+}
+
+/**
+ * ibmveth_poll_deliver_frame - Build SKB from one valid RX slot and GRO it
+ * @napi: NAPI context for this RX queue
+ * @adapter: ibmveth adapter
+ * @netdev: net_device for @adapter
+ * @queue_index: RX queue index
+ *
+ * Return: 1 frame delivered, 0 if the slot was skipped cleanly, -1 on error.
+ */
+static int ibmveth_poll_deliver_frame(struct napi_struct *napi,
+				      struct ibmveth_adapter *adapter,
+				      struct net_device *netdev,
+				      int queue_index)
+{
+	struct sk_buff *skb, *new_skb;
+	struct ibmveth_rx_q *rxq = &adapter->rx_queue[queue_index];
+	unsigned int room, off, len, pool;
+	int length, offset, csum_good, lrg_pkt;
+	__sum16 iph_check = 0;
 	u16 mss = 0;
+	int rc;
 
-restart_poll:
-	while (frames_processed < budget) {
-		if (!ibmveth_rxq_pending_buffer(adapter))
-			break;
+	length = ibmveth_rxq_frame_length(adapter, queue_index);
+	offset = ibmveth_rxq_frame_offset(adapter, queue_index);
+	csum_good = ibmveth_rxq_csum_good(adapter, queue_index);
+	lrg_pkt = ibmveth_rxq_large_packet(adapter, queue_index);
+
+	skb = ibmveth_rxq_get_buffer(adapter, queue_index);
+	if (unlikely(!skb)) {
+		if (!ibmveth_poll_skip_bad_correlator(netdev, adapter,
+						      queue_index))
+			return -1;
+		return 0;
+	}
 
-		smp_rmb();
-		if (!ibmveth_rxq_buffer_valid(adapter)) {
-			wmb(); /* suggested by larson1 */
-			adapter->rx_invalid_buffer++;
-			netdev_dbg(netdev, "recycling invalid buffer\n");
-			if (unlikely(ibmveth_rxq_harvest_buffer(adapter, true)))
-				break;
-		} else {
-			struct sk_buff *skb, *new_skb;
-			int length = ibmveth_rxq_frame_length(adapter);
-			int offset = ibmveth_rxq_frame_offset(adapter);
-			int csum_good = ibmveth_rxq_csum_good(adapter);
-			int lrg_pkt = ibmveth_rxq_large_packet(adapter);
-			__sum16 iph_check = 0;
-
-			skb = ibmveth_rxq_get_buffer(adapter);
-			if (unlikely(!skb))
-				break;
+	pool = rxq->queue_addr[rxq->index].correlator >> 32;
+	room = min_t(unsigned int, skb_tailroom(skb),
+		     adapter->rx_buff_pool[queue_index][pool].buff_size);
+	off = offset;
+	len = length;
+	if (unlikely(off >= room || len > room - off)) {
+		if (!ibmveth_poll_drop_oversize(netdev, adapter, queue_index,
+						off, len, room))
+			return -1;
+		return 0;
+	}
 
-			/* if the large packet bit is set in the rx queue
-			 * descriptor, the mss will be written by PHYP eight
-			 * bytes from the start of the rx buffer, which is
-			 * skb->data at this stage
-			 */
-			if (lrg_pkt) {
-				__be64 *rxmss = (__be64 *)(skb->data + 8);
+	if (lrg_pkt) {
+		__be64 *rxmss = (__be64 *)(skb->data + 8);
 
-				mss = (u16)be64_to_cpu(*rxmss);
-			}
+		mss = (u16)be64_to_cpu(*rxmss);
+	}
 
-			new_skb = NULL;
-			if (length < rx_copybreak)
-				new_skb = netdev_alloc_skb(netdev, length);
-
-			if (new_skb) {
-				skb_copy_to_linear_data(new_skb,
-							skb->data + offset,
-							length);
-				if (rx_flush)
-					ibmveth_flush_buffer(skb->data,
-						length + offset);
-				if (unlikely(ibmveth_rxq_harvest_buffer(adapter, true)))
-					break;
-				skb = new_skb;
-			} else {
-				if (unlikely(ibmveth_rxq_harvest_buffer(adapter, false)))
-					break;
-				skb_reserve(skb, offset);
-			}
+	new_skb = NULL;
+	if (length < rx_copybreak)
+		new_skb = netdev_alloc_skb(netdev, length);
+
+	if (new_skb) {
+		skb_copy_to_linear_data(new_skb, skb->data + offset, length);
+		if (rx_flush)
+			ibmveth_flush_buffer(skb->data, length + offset);
+		rc = ibmveth_rxq_harvest_buffer(adapter, queue_index, true);
+		if (unlikely(rc)) {
+			kfree_skb(new_skb);
+			return -1;
+		}
+		skb = new_skb;
+	} else {
+		rc = ibmveth_rxq_harvest_buffer(adapter, queue_index, false);
+		if (unlikely(rc))
+			return -1;
+		skb_reserve(skb, offset);
+	}
 
-			skb_put(skb, length);
-			skb->protocol = eth_type_trans(skb, netdev);
+	skb_put(skb, length);
+	skb->protocol = eth_type_trans(skb, netdev);
 
-			/* PHYP without PLSO support places a -1 in the ip
-			 * checksum for large send frames.
-			 */
-			if (skb->protocol == cpu_to_be16(ETH_P_IP)) {
-				struct iphdr *iph = (struct iphdr *)skb->data;
+	if (skb->protocol == cpu_to_be16(ETH_P_IP))
+		iph_check = ((struct iphdr *)skb->data)->check;
 
-				iph_check = iph->check;
-			}
+	if ((length > netdev->mtu + ETH_HLEN) || lrg_pkt ||
+	    iph_check == 0xffff) {
+		ibmveth_rx_mss_helper(skb, mss, lrg_pkt);
+		adapter->rx_qstats[queue_index].large_packets++;
+	}
 
-			if ((length > netdev->mtu + ETH_HLEN) ||
-			    lrg_pkt || iph_check == 0xffff) {
-				ibmveth_rx_mss_helper(skb, mss, lrg_pkt);
-				adapter->rx_large_packets++;
-			}
+	if (csum_good) {
+		skb->ip_summed = CHECKSUM_UNNECESSARY;
+		ibmveth_rx_csum_helper(skb, adapter);
+	}
 
-			if (csum_good) {
-				skb->ip_summed = CHECKSUM_UNNECESSARY;
-				ibmveth_rx_csum_helper(skb, adapter);
-			}
+	skb_record_rx_queue(skb, queue_index);
+	napi_gro_receive(napi, skb);
+
+	adapter->rx_qstats[queue_index].packets++;
+	adapter->rx_qstats[queue_index].bytes += length;
+
+	return 1;
+}
+
+static int ibmveth_poll(struct napi_struct *napi, int budget)
+{
+	struct net_device *netdev = napi->dev;
+	struct ibmveth_adapter *adapter = netdev_priv(netdev);
+	int frames_processed = 0;
+	int queue_index, rc;
+
+	queue_index = napi - adapter->napi;
+
+	if (WARN_ON(queue_index < 0 ||
+		    queue_index >= ibmveth_get_num_rx_queues(adapter))) {
+		if (budget)
+			napi_complete_done(napi, 0);
+		return 0;
+	}
+
+	if (ibmveth_poll_stopping(netdev, napi)) {
+		if (budget)
+			napi_complete_done(napi, 0);
+		return 0;
+	}
+
+	adapter->rx_qstats[queue_index].polls++;
+
+restart_poll:
+	while (frames_processed < budget) {
+		if (ibmveth_poll_stopping(netdev, napi))
+			break;
 
-			napi_gro_receive(napi, skb);	/* send it up */
+		if (!ibmveth_rxq_pending_buffer(adapter, queue_index))
+			break;
 
-			netdev->stats.rx_packets++;
-			netdev->stats.rx_bytes += length;
-			frames_processed++;
+		smp_rmb();
+		if (!ibmveth_rxq_buffer_valid(adapter, queue_index)) {
+			wmb(); /* suggested by larson1 */
+			if (!ibmveth_poll_recycle_invalid(netdev, adapter,
+							  queue_index))
+				break;
+		} else {
+			rc = ibmveth_poll_deliver_frame(napi, adapter, netdev,
+							queue_index);
+			if (rc < 0)
+				break;
+			if (rc > 0)
+				frames_processed++;
 		}
 	}
 
-	ibmveth_replenish_task(adapter);
+	ibmveth_replenish_task(adapter, queue_index);
+
+	if (ibmveth_poll_stopping(netdev, napi)) {
+		/* budget 0 is netpoll, which must not complete NAPI.
+		 * Otherwise returning budget after completing would ask
+		 * NAPI to reschedule, so cap the return at budget - 1.
+		 */
+		if (budget) {
+			napi_complete_done(napi, frames_processed);
+			return min(frames_processed, budget - 1);
+		}
+		return 0;
+	}
 
 	if (frames_processed == budget)
 		goto out;
@@ -1548,18 +4144,27 @@ static int ibmveth_poll(struct napi_struct *napi, int budget)
 	if (!napi_complete_done(napi, frames_processed))
 		goto out;
 
-	/* We think we are done - reenable interrupts,
-	 * then check once more to make sure we are done.
+	/*
+	 * napi_disable() sets DISABLE then waits for this poll. Without a
+	 * second stopping check here, enable_irq() can re-arm PHYP after
+	 * resize already masked the queue; late IRQs then hit the handler
+	 * after num_rx_queues was published lower (lab WARN at interrupt).
 	 */
-	lpar_rc = h_vio_signal(adapter->vdev->unit_address, VIO_IRQ_ENABLE);
-	if (WARN_ON(lpar_rc != H_SUCCESS)) {
+	if (ibmveth_poll_stopping(netdev, napi))
+		goto out;
+
+	rc = ibmveth_enable_irq(adapter, queue_index);
+	if (rc) {
+		netdev_err(netdev,
+			   "Failed to enable IRQ for queue %d (rc=%d), scheduling reset\n",
+			   queue_index, rc);
 		schedule_work(&adapter->work);
 		goto out;
 	}
 
-	if (ibmveth_rxq_pending_buffer(adapter) && napi_schedule(napi)) {
-		lpar_rc = h_vio_signal(adapter->vdev->unit_address,
-				       VIO_IRQ_DISABLE);
+	if (ibmveth_rxq_pending_buffer(adapter, queue_index) &&
+	    napi_schedule(napi)) {
+		ibmveth_disable_irq(adapter, queue_index);
 		goto restart_poll;
 	}
 
@@ -1569,16 +4174,23 @@ static int ibmveth_poll(struct napi_struct *napi, int budget)
 
 static irqreturn_t ibmveth_interrupt(int irq, void *dev_instance)
 {
-	struct net_device *netdev = dev_instance;
+	struct napi_struct *napi = dev_instance;
+	struct net_device *netdev = napi->dev;
 	struct ibmveth_adapter *adapter = netdev_priv(netdev);
-	unsigned long lpar_rc;
+	int qindex;
 
-	if (napi_schedule_prep(&adapter->napi)) {
-		lpar_rc = h_vio_signal(adapter->vdev->unit_address,
-				       VIO_IRQ_DISABLE);
-		WARN_ON(lpar_rc != H_SUCCESS);
-		__napi_schedule(&adapter->napi);
-	}
+	qindex = napi - adapter->napi;
+	/*
+	 * Quiet on out-of-range: scale-down publishes a lower live count
+	 * before free_irq(). A residual IRQ must not WARN-storm; return
+	 * IRQ_NONE until the handler is removed.
+	 */
+	if (qindex < 0 || qindex >= ibmveth_get_num_rx_queues(adapter))
+		return IRQ_NONE;
+
+	adapter->rx_qstats[qindex].interrupts++;
+
+	ibmveth_schedule_rx_queue(adapter, qindex);
 	return IRQ_HANDLED;
 }
 
@@ -1645,7 +4257,7 @@ static int ibmveth_change_mtu(struct net_device *dev, int new_mtu)
 	int need_restart = 0;
 
 	for (i = 0; i < IBMVETH_NUM_BUFF_POOLS; i++)
-		if (new_mtu_oh <= adapter->rx_buff_pool[i].buff_size)
+		if (new_mtu_oh <= adapter->rx_buff_pool[0][i].buff_size)
 			break;
 
 	if (i == IBMVETH_NUM_BUFF_POOLS)
@@ -1660,9 +4272,9 @@ static int ibmveth_change_mtu(struct net_device *dev, int new_mtu)
 
 	/* Look for an active buffer pool that can hold the new MTU */
 	for (i = 0; i < IBMVETH_NUM_BUFF_POOLS; i++) {
-		adapter->rx_buff_pool[i].active = 1;
+		adapter->rx_buff_pool[0][i].active = 1;
 
-		if (new_mtu_oh <= adapter->rx_buff_pool[i].buff_size) {
+		if (new_mtu_oh <= adapter->rx_buff_pool[0][i].buff_size) {
 			WRITE_ONCE(dev->mtu, new_mtu);
 			vio_cmo_set_dev_desired(viodev,
 						ibmveth_get_desired_dma
@@ -1683,8 +4295,20 @@ static int ibmveth_change_mtu(struct net_device *dev, int new_mtu)
 #ifdef CONFIG_NET_POLL_CONTROLLER
 static void ibmveth_poll_controller(struct net_device *dev)
 {
-	ibmveth_replenish_task(netdev_priv(dev));
-	ibmveth_interrupt(dev->irq, dev);
+	struct ibmveth_adapter *adapter = netdev_priv(dev);
+	unsigned int num;
+	int i;
+
+	if (!adapter->opened)
+		return;
+
+	num = ibmveth_get_num_rx_queues(adapter);
+
+	for (i = 0; i < num; i++)
+		ibmveth_replenish_task(adapter, i);
+
+	for (i = 0; i < num; i++)
+		ibmveth_schedule_rx_queue(adapter, i);
 }
 #endif
 
@@ -1701,9 +4325,6 @@ static unsigned long ibmveth_get_desired_dma(struct vio_dev *vdev)
 	struct net_device *netdev = dev_get_drvdata(&vdev->dev);
 	struct ibmveth_adapter *adapter;
 	struct iommu_table *tbl;
-	unsigned long ret;
-	int i;
-	int rxqentries = 1;
 
 	tbl = get_iommu_table_base(&vdev->dev);
 
@@ -1712,26 +4333,8 @@ static unsigned long ibmveth_get_desired_dma(struct vio_dev *vdev)
 		return IOMMU_PAGE_ALIGN(IBMVETH_IO_ENTITLEMENT_DEFAULT, tbl);
 
 	adapter = netdev_priv(netdev);
-
-	ret = IBMVETH_BUFF_LIST_SIZE + IBMVETH_FILT_LIST_SIZE;
-	ret += IOMMU_PAGE_ALIGN(netdev->mtu, tbl);
-	/* add size of mapped tx buffers */
-	ret += IOMMU_PAGE_ALIGN(IBMVETH_MAX_TX_BUF_SIZE, tbl);
-
-	for (i = 0; i < IBMVETH_NUM_BUFF_POOLS; i++) {
-		/* add the size of the active receive buffers */
-		if (adapter->rx_buff_pool[i].active)
-			ret +=
-			    adapter->rx_buff_pool[i].size *
-			    IOMMU_PAGE_ALIGN(adapter->rx_buff_pool[i].
-					     buff_size, tbl);
-		rxqentries += adapter->rx_buff_pool[i].size;
-	}
-	/* add the size of the receive queue entries */
-	ret += IOMMU_PAGE_ALIGN(
-		rxqentries * sizeof(struct ibmveth_rx_q_entry), tbl);
-
-	return ret;
+	return ibmveth_desired_dma_for_rxqs(adapter,
+					    ibmveth_get_num_rx_queues(adapter));
 }
 
 static int ibmveth_set_mac_addr(struct net_device *dev, void *p)
@@ -1777,6 +4380,124 @@ static netdev_features_t ibmveth_features_check(struct sk_buff *skb,
 	return vlan_features_check(skb, features);
 }
 
+/**
+ * ibmveth_get_stats64 - Return aggregated per-queue statistics
+ * @dev: network device
+ * @stats: rtnl link statistics storage
+ *
+ * Sums per-queue rx_qstats and tx_qstats into the rtnl counters.
+ * Walk the full allocated arrays (not the live queue count) so shrinking
+ * channels cannot make the totals go backwards.
+ * Callers use ndo_get_stats64(); avoid updating netdev->stats on the
+ * xmit/poll paths to keep per-queue counters off the hot cache line.
+ */
+static void ibmveth_get_stats64(struct net_device *dev,
+				struct rtnl_link_stats64 *stats)
+{
+	struct ibmveth_adapter *adapter = netdev_priv(dev);
+	int i;
+
+	for (i = 0; i < IBMVETH_MAX_RX_QUEUES; i++) {
+		stats->rx_packets += adapter->rx_qstats[i].packets;
+		stats->rx_bytes += adapter->rx_qstats[i].bytes;
+	}
+
+	for (i = 0; i < IBMVETH_MAX_QUEUES; i++) {
+		stats->tx_packets += adapter->tx_qstats[i].packets;
+		stats->tx_bytes += adapter->tx_qstats[i].bytes;
+		stats->tx_dropped += adapter->tx_qstats[i].dropped_packets;
+	}
+}
+
+static void ibmveth_get_queue_stats_rx(struct net_device *dev, int idx,
+				       struct netdev_queue_stats_rx *stats)
+{
+	struct ibmveth_adapter *adapter = netdev_priv(dev);
+
+	stats->packets = adapter->rx_qstats[idx].packets;
+	stats->bytes = adapter->rx_qstats[idx].bytes;
+	/*
+	 * All three are frames that entered the device and never left it,
+	 * which is what rx-hw-drops is specified to cover: no_buffer_drops
+	 * is PHYP dropping for lack of buffer space on the page mapped now,
+	 * no_buffer_retired the same for pages this queue has already
+	 * released, and invalid_buffers is a processing error.
+	 */
+	stats->hw_drops = adapter->rx_qstats[idx].no_buffer_drops +
+			  adapter->rx_qstats[idx].no_buffer_retired +
+			  adapter->rx_qstats[idx].invalid_buffers;
+	stats->alloc_fail = adapter->rx_qstats[idx].replenish_no_mem;
+}
+
+static void ibmveth_get_queue_stats_tx(struct net_device *dev, int idx,
+				       struct netdev_queue_stats_tx *stats)
+{
+	struct ibmveth_adapter *adapter = netdev_priv(dev);
+
+	stats->packets = adapter->tx_qstats[idx].packets;
+	stats->bytes = adapter->tx_qstats[idx].bytes;
+	stats->hw_drops = adapter->tx_qstats[idx].dropped_packets;
+}
+
+/**
+ * ibmveth_get_base_stats - account for traffic not on a live queue
+ * @dev: network device
+ * @rx: RX base statistics storage
+ * @tx: TX base statistics storage
+ *
+ * get_queue_stats_{rx,tx}() only report queues the core still iterates,
+ * i.e. below real_num_{rx,tx}_queues, while ibmveth_get_stats64() walks
+ * the full arrays so device totals stay monotonic across a shrink.
+ * Report the retired-queue remainder here, otherwise qstats and
+ * rtnl_link_stats64 disagree by a delta that grows with every shrink.
+ * Zeroing would not be neutral: per netdev_stat_ops it asserts the
+ * per-queue sum is already exact.
+ *
+ * Bound the live side with real_num_*_queues rather than the adapter's
+ * own count, so the split lines up with the core's iteration exactly.
+ *
+ * Every field the per-queue callbacks fill must also be initialised
+ * here: netdev_nl_stats_add() starts the sum at NETDEV_STAT_NOT_SET and
+ * only accumulates while both sides are set, so a field left unset here
+ * is dropped from the device total even though the queues report it.
+ */
+static void ibmveth_get_base_stats(struct net_device *dev,
+				   struct netdev_queue_stats_rx *rx,
+				   struct netdev_queue_stats_tx *tx)
+{
+	struct ibmveth_adapter *adapter = netdev_priv(dev);
+	unsigned int i;
+
+	rx->packets = 0;
+	rx->bytes = 0;
+	rx->alloc_fail = 0;
+	rx->hw_drops = 0;
+	tx->packets = 0;
+	tx->bytes = 0;
+	tx->hw_drops = 0;
+
+	for (i = dev->real_num_rx_queues; i < IBMVETH_MAX_RX_QUEUES; i++) {
+		rx->packets += adapter->rx_qstats[i].packets;
+		rx->bytes += adapter->rx_qstats[i].bytes;
+		rx->hw_drops += adapter->rx_qstats[i].no_buffer_drops +
+				adapter->rx_qstats[i].no_buffer_retired +
+				adapter->rx_qstats[i].invalid_buffers;
+		rx->alloc_fail += adapter->rx_qstats[i].replenish_no_mem;
+	}
+
+	for (i = dev->real_num_tx_queues; i < IBMVETH_MAX_QUEUES; i++) {
+		tx->packets += adapter->tx_qstats[i].packets;
+		tx->bytes += adapter->tx_qstats[i].bytes;
+		tx->hw_drops += adapter->tx_qstats[i].dropped_packets;
+	}
+}
+
+static const struct netdev_stat_ops ibmveth_stat_ops = {
+	.get_queue_stats_rx	= ibmveth_get_queue_stats_rx,
+	.get_queue_stats_tx	= ibmveth_get_queue_stats_tx,
+	.get_base_stats		= ibmveth_get_base_stats,
+};
+
 static const struct net_device_ops ibmveth_netdev_ops = {
 	.ndo_open		= ibmveth_open,
 	.ndo_stop		= ibmveth_close,
@@ -1789,14 +4510,129 @@ static const struct net_device_ops ibmveth_netdev_ops = {
 	.ndo_validate_addr	= eth_validate_addr,
 	.ndo_set_mac_address    = ibmveth_set_mac_addr,
 	.ndo_features_check	= ibmveth_features_check,
+	.ndo_get_stats64	= ibmveth_get_stats64,
 #ifdef CONFIG_NET_POLL_CONTROLLER
 	.ndo_poll_controller	= ibmveth_poll_controller,
 #endif
 };
 
+static int ibmveth_buffer_pools_show(struct seq_file *m, void *v)
+{
+	struct ibmveth_adapter *adapter = m->private;
+	int i, j;
+
+	/*
+	 * size / buff_size / pool->active are written under RTNL
+	 * (veth_pool_store, open template copy). Take the same lock so
+	 * those columns are not a torn snapshot. available is updated
+	 * from NAPI/softirq; only atomic_read() keeps it from tearing.
+	 * Not required for memory safety; embedded arrays only.
+	 */
+	rtnl_lock();
+
+	seq_puts(m, "Queue  Pool  Count  BuffSize  Active  Available\n");
+	seq_puts(m, "-----  ----  -----  --------  ------  ---------\n");
+	if (!adapter->opened) {
+		seq_puts(m, "# down: Active/Available 0 unless allocated\n");
+		seq_puts(m, "# down: geometry above queue 0 set at open\n");
+	}
+
+	for (i = 0; i < ibmveth_get_num_rx_queues(adapter); i++) {
+		for (j = 0; j < IBMVETH_NUM_BUFF_POOLS; j++) {
+			struct ibmveth_buff_pool *pool =
+				&adapter->rx_buff_pool[i][j];
+			bool live = pool->skbuff && pool->free_map;
+			int active = live ? pool->active : 0;
+			int available = live ? atomic_read(&pool->available)
+					     : 0;
+
+			seq_printf(m, "%5d  %4d  %5u  %8u  %6d  %9d\n",
+				   i, j, pool->size, pool->buff_size,
+				   active, available);
+		}
+	}
+
+	rtnl_unlock();
+	return 0;
+}
+DEFINE_SHOW_ATTRIBUTE(ibmveth_buffer_pools);
+
+/* Driver-owned root so per-adapter dirs use a stable vio name, not the
+ * mutable netdev->name (avoids stale names / eth0 collisions after rename).
+ */
+static struct dentry *ibmveth_dbg_root;
+
+static void ibmveth_debugfs_init(struct ibmveth_adapter *adapter)
+{
+	adapter->debugfs_dir =
+		debugfs_create_dir(dev_name(&adapter->vdev->dev),
+				   ibmveth_dbg_root);
+	debugfs_create_file("buffer_pools", 0400, adapter->debugfs_dir,
+			    adapter, &ibmveth_buffer_pools_fops);
+}
+
+static void ibmveth_debugfs_exit(struct ibmveth_adapter *adapter)
+{
+	debugfs_remove_recursive(adapter->debugfs_dir);
+	adapter->debugfs_dir = NULL;
+}
+
+/**
+ * ibmveth_pool_kobj_release - Mark a pool kobject finished
+ * @kobj: kobject embedded in the pool
+ *
+ * Pool kobjects live in netdev_priv(). Last put waits for this
+ * before free_netdev() so DEBUG_KOBJECT_RELEASE delayed cleanup
+ * does not run on freed memory.
+ */
+static void ibmveth_pool_kobj_release(struct kobject *kobj)
+{
+	struct ibmveth_buff_pool *pool = container_of(kobj,
+						      struct ibmveth_buff_pool,
+						      kobj);
+
+	complete(&pool->released);
+}
+
+/**
+ * ibmveth_put_pool_kobjs - Drop pool kobjects and wait for release
+ * @adapter: ibmveth adapter
+ * @pools_ready: number of queue-0 pools that were initialized
+ *
+ * Put every initialized pool kobject, then wait so free_netdev()
+ * cannot race DEBUG_KOBJECT_RELEASE delayed cleanup.
+ */
+static void ibmveth_put_pool_kobjs(struct ibmveth_adapter *adapter,
+				   int pools_ready)
+{
+	int i;
+
+	for (i = 0; i < pools_ready; i++)
+		kobject_put(&adapter->rx_buff_pool[0][i].kobj);
+	for (i = 0; i < pools_ready; i++)
+		wait_for_completion(&adapter->rx_buff_pool[0][i].released);
+}
+
+static void ibmveth_probe_cleanup(struct ibmveth_adapter *adapter,
+				  int pools_ready)
+{
+	struct net_device *netdev = adapter->netdev;
+
+	cancel_work_sync(&adapter->work);
+	ibmveth_put_pool_kobjs(adapter, pools_ready);
+
+	ibmveth_free_tx_qstats(adapter);
+	ibmveth_free_rx_qstats(adapter);
+	/* Probe failure never reaches ibmveth_remove(); clear before free so
+	 * CMO get_desired_dma() cannot see a freed netdev on rebind.
+	 */
+	dev_set_drvdata(&adapter->vdev->dev, NULL);
+	free_netdev(netdev);
+}
+
 static int ibmveth_probe(struct vio_dev *dev, const struct vio_device_id *id)
 {
-	int rc, i, mac_len;
+	int rc, i, mac_len, pools_ready = 0;
 	struct net_device *netdev;
 	struct ibmveth_adapter *adapter;
 	unsigned char *mac_addr_p;
@@ -1831,7 +4667,8 @@ static int ibmveth_probe(struct vio_dev *dev, const struct vio_device_id *id)
 		return -EINVAL;
 	}
 
-	netdev = alloc_etherdev_mqs(sizeof(struct ibmveth_adapter), IBMVETH_MAX_QUEUES, 1);
+	netdev = alloc_etherdev_mqs(sizeof(struct ibmveth_adapter),
+				    IBMVETH_MAX_QUEUES, IBMVETH_MAX_RX_QUEUES);
 	if (!netdev)
 		return -ENOMEM;
 
@@ -1841,14 +4678,33 @@ static int ibmveth_probe(struct vio_dev *dev, const struct vio_device_id *id)
 	adapter->vdev = dev;
 	adapter->netdev = netdev;
 	INIT_WORK(&adapter->work, ibmveth_reset);
+
+	/*
+	 * Initialise the replenish locks once. open() is re-entered on
+	 * MTU and offload changes without netpoll_poll_disable(), so a
+	 * lock set up there could be reinitialised while poll_controller()
+	 * holds it.
+	 */
+	for (i = 0; i < IBMVETH_MAX_RX_QUEUES; i++)
+		spin_lock_init(&adapter->rx_queue[i].replenish_lock);
+
 	adapter->mcastFilterSize = be32_to_cpu(*mcastFilterSize_p);
 	ibmveth_init_link_settings(netdev);
 
-	netif_napi_add_weight(netdev, &adapter->napi, ibmveth_poll, 16);
+	for (i = 0; i < IBMVETH_MAX_RX_QUEUES; i++)
+		netif_napi_add_weight(netdev, &adapter->napi[i],
+				      ibmveth_poll, 16);
+
+	if (ibmveth_alloc_rx_qstats(adapter) ||
+	    ibmveth_alloc_tx_qstats(adapter)) {
+		ibmveth_probe_cleanup(adapter, 0);
+		return -ENOMEM;
+	}
 
 	netdev->irq = dev->irq;
 	netdev->netdev_ops = &ibmveth_netdev_ops;
 	netdev->ethtool_ops = &netdev_ethtool_ops;
+	netdev->stat_ops = &ibmveth_stat_ops;
 	SET_NETDEV_DEV(netdev, &dev->dev);
 	netdev->hw_features = NETIF_F_SG;
 	if (vio_get_attribute(dev, "ibm,illan-options", NULL) != NULL) {
@@ -1876,12 +4732,30 @@ static int ibmveth_probe(struct vio_dev *dev, const struct vio_device_id *id)
 		netdev->features |= NETIF_F_FRAGLIST;
 	}
 
+	if (ret == H_SUCCESS &&
+	    (ret_attr & IBMVETH_ILLAN_RX_MULTI_QUEUE_SUPPORT)) {
+		adapter->multi_queue = true;
+		ibmveth_publish_num_rx_queues(adapter,
+					      min(num_online_cpus(),
+						  IBMVETH_DEFAULT_QUEUES));
+		netdev_dbg(netdev, "RX multi queue mode enabled: %u queues\n",
+			   ibmveth_get_num_rx_queues(adapter));
+	} else {
+		adapter->multi_queue = false;
+		ibmveth_publish_num_rx_queues(adapter,
+					      IBMVETH_DEFAULT_RX_QUEUES);
+	}
+
 	if (ret == H_SUCCESS &&
 	    (ret_attr & IBMVETH_ILLAN_RX_MULTI_BUFF_SUPPORT)) {
-		adapter->rx_buffers_per_hcall = IBMVETH_MAX_RX_PER_HCALL;
+		if (adapter->multi_queue)
+			adapter->rx_buffers_per_hcall =
+				IBMVETH_MAX_RX_PER_HCALL;
+		else
+			adapter->rx_buffers_per_hcall = IBMVETH_MAX_RX_REGULAR;
 		netdev_dbg(netdev,
 			   "RX Multi-buffer hcall supported by FW, batch set to %u\n",
-			    adapter->rx_buffers_per_hcall);
+			   adapter->rx_buffers_per_hcall);
 	} else {
 		adapter->rx_buffers_per_hcall = 1;
 		netdev_dbg(netdev,
@@ -1898,16 +4772,29 @@ static int ibmveth_probe(struct vio_dev *dev, const struct vio_device_id *id)
 		memcpy(pool_count, pool_count_cmo, sizeof(pool_count));
 
 	for (i = 0; i < IBMVETH_NUM_BUFF_POOLS; i++) {
-		struct kobject *kobj = &adapter->rx_buff_pool[i].kobj;
-		int error;
+		struct kobject *kobj = &adapter->rx_buff_pool[0][i].kobj;
 
-		ibmveth_init_buffer_pool(&adapter->rx_buff_pool[i], i,
+		ibmveth_init_buffer_pool(&adapter->rx_buff_pool[0][i], i,
 					 pool_count[i], pool_size[i],
 					 pool_active[i]);
-		error = kobject_init_and_add(kobj, &ktype_veth_pool,
-					     &dev->dev.kobj, "pool%d", i);
-		if (!error)
-			kobject_uevent(kobj, KOBJ_ADD);
+		init_completion(&adapter->rx_buff_pool[0][i].released);
+		rc = kobject_init_and_add(kobj, &ktype_veth_pool,
+					  &dev->dev.kobj, "pool%d", i);
+		if (rc) {
+			struct ibmveth_buff_pool *pool =
+				&adapter->rx_buff_pool[0][i];
+
+			dev_err(&dev->dev,
+				"failed to create pool%d kobject: %d\n", i, rc);
+			/* init_and_add takes a ref even on failure */
+			kobject_put(kobj);
+			wait_for_completion(&pool->released);
+			ibmveth_probe_cleanup(adapter, pools_ready);
+			return rc;
+		}
+
+		pools_ready++;
+		kobject_uevent(kobj, KOBJ_ADD);
 	}
 
 	rc = netif_set_real_num_tx_queues(netdev, min(num_online_cpus(),
@@ -1915,9 +4802,25 @@ static int ibmveth_probe(struct vio_dev *dev, const struct vio_device_id *id)
 	if (rc) {
 		netdev_dbg(netdev, "failed to set number of tx queues rc=%d\n",
 			   rc);
-		free_netdev(netdev);
+		ibmveth_probe_cleanup(adapter, pools_ready);
+		return rc;
+	}
+
+	/*
+	 * alloc_etherdev_mqs() sized real_num_rx_queues to MAX. Match the
+	 * advertised default (or SQ 1) before register_netdev so down-state
+	 * readers agree with adapter->num_rx_queues / ethtool -l.
+	 */
+	rc = netif_set_real_num_rx_queues(netdev,
+					  ibmveth_get_num_rx_queues(adapter));
+
+	if (rc) {
+		netdev_dbg(netdev, "failed to set number of rx queues rc=%d\n",
+			   rc);
+		ibmveth_probe_cleanup(adapter, pools_ready);
 		return rc;
 	}
+
 	adapter->tx_ltb_size = PAGE_ALIGN(IBMVETH_MAX_TX_BUF_SIZE);
 	for (i = 0; i < IBMVETH_MAX_QUEUES; i++)
 		adapter->tx_ltb_ptr[i] = NULL;
@@ -1931,12 +4834,14 @@ static int ibmveth_probe(struct vio_dev *dev, const struct vio_device_id *id)
 
 	if (rc) {
 		netdev_dbg(netdev, "failed to register netdev rc=%d\n", rc);
-		free_netdev(netdev);
+		ibmveth_probe_cleanup(adapter, pools_ready);
 		return rc;
 	}
 
 	netdev_dbg(netdev, "registered\n");
 
+	ibmveth_debugfs_init(adapter);
+
 	return 0;
 }
 
@@ -1944,14 +4849,22 @@ static void ibmveth_remove(struct vio_dev *dev)
 {
 	struct net_device *netdev = dev_get_drvdata(&dev->dev);
 	struct ibmveth_adapter *adapter = netdev_priv(netdev);
-	int i;
 
-	cancel_work_sync(&adapter->work);
+	ibmveth_debugfs_exit(adapter);
 
-	for (i = 0; i < IBMVETH_NUM_BUFF_POOLS; i++)
-		kobject_put(&adapter->rx_buff_pool[i].kobj);
+	ibmveth_put_pool_kobjs(adapter, IBMVETH_NUM_BUFF_POOLS);
 
+	/*
+	 * Unregister first so NAPI/xmit cannot re-arm reset work after we
+	 * cancel it. cancel_work_sync() before unregister left a window
+	 * where poll could schedule_work() and the worker ran after
+	 * free_netdev().
+	 */
 	unregister_netdev(netdev);
+	cancel_work_sync(&adapter->work);
+
+	ibmveth_free_tx_qstats(adapter);
+	ibmveth_free_rx_qstats(adapter);
 
 	free_netdev(netdev);
 	dev_set_drvdata(&dev->dev, NULL);
@@ -2035,11 +4948,12 @@ static ssize_t veth_pool_store(struct kobject *kobj, struct attribute *attr,
 			/* Make sure there is a buffer pool with buffers that
 			   can hold a packet of the size of the MTU */
 			for (i = 0; i < IBMVETH_NUM_BUFF_POOLS; i++) {
-				if (pool == &adapter->rx_buff_pool[i])
+				if (pool == &adapter->rx_buff_pool[0][i])
 					continue;
-				if (!adapter->rx_buff_pool[i].active)
+				if (!adapter->rx_buff_pool[0][i].active)
 					continue;
-				if (mtu <= adapter->rx_buff_pool[i].buff_size)
+				if (mtu <=
+				    adapter->rx_buff_pool[0][i].buff_size)
 					break;
 			}
 
@@ -2092,8 +5006,8 @@ static ssize_t veth_pool_store(struct kobject *kobj, struct attribute *attr,
 	}
 	rtnl_unlock();
 
-	/* kick the interrupt handler to allocate/deallocate pools */
-	ibmveth_interrupt(netdev->irq, netdev);
+	/* kick RX processing to allocate/deallocate pools */
+	ibmveth_schedule_rx_queue(adapter, 0);
 	return count;
 
 unlock_err:
@@ -2101,7 +5015,6 @@ static ssize_t veth_pool_store(struct kobject *kobj, struct attribute *attr,
 	return rc;
 }
 
-
 #define ATTR(_name, _mode)				\
 	struct attribute veth_##_name##_attr = {	\
 	.name = __stringify(_name), .mode = _mode,	\
@@ -2125,7 +5038,7 @@ static const struct sysfs_ops veth_pool_ops = {
 };
 
 static struct kobj_type ktype_veth_pool = {
-	.release        = NULL,
+	.release        = ibmveth_pool_kobj_release,
 	.sysfs_ops      = &veth_pool_ops,
 	.default_groups = veth_pool_groups,
 };
@@ -2133,7 +5046,13 @@ static struct kobj_type ktype_veth_pool = {
 static int ibmveth_resume(struct device *dev)
 {
 	struct net_device *netdev = dev_get_drvdata(dev);
-	ibmveth_interrupt(netdev->irq, netdev);
+	struct ibmveth_adapter *adapter = netdev_priv(netdev);
+	unsigned int num = ibmveth_get_num_rx_queues(adapter);
+	int i;
+
+	for (i = 0; i < num; i++)
+		ibmveth_schedule_rx_queue(adapter, i);
+
 	return 0;
 }
 
@@ -2158,15 +5077,27 @@ static struct vio_driver ibmveth_driver = {
 
 static int __init ibmveth_module_init(void)
 {
+	int rc;
+
 	printk(KERN_DEBUG "%s: %s %s\n", ibmveth_driver_name,
 	       ibmveth_driver_string, ibmveth_driver_version);
 
-	return vio_register_driver(&ibmveth_driver);
+	ibmveth_dbg_root = debugfs_create_dir(ibmveth_driver_name, NULL);
+
+	rc = vio_register_driver(&ibmveth_driver);
+	if (rc) {
+		debugfs_remove_recursive(ibmveth_dbg_root);
+		ibmveth_dbg_root = NULL;
+	}
+
+	return rc;
 }
 
 static void __exit ibmveth_module_exit(void)
 {
 	vio_unregister_driver(&ibmveth_driver);
+	debugfs_remove_recursive(ibmveth_dbg_root);
+	ibmveth_dbg_root = NULL;
 }
 
 module_init(ibmveth_module_init);
@@ -2196,8 +5127,7 @@ static void ibmveth_reset_kunit(struct work_struct *w)
  * @test: pointer to kunit structure
  *
  * Tests the error returns from ibmveth_remove_buffer_from_pool.
- * ibmveth_remove_buffer_from_pool also calls WARN_ON, so dmesg should be
- * checked to see that these warnings happened.
+ * Bad correlators return -EINVAL/-EFAULT (no WARN_ON).
  *
  * Return: void
  */
@@ -2211,28 +5141,45 @@ static void ibmveth_remove_buffer_from_pool_test(struct kunit *test)
 
 	INIT_WORK(&adapter->work, ibmveth_reset_kunit);
 
+	spin_lock_init(&adapter->rx_queue[0].replenish_lock);
+
 	/* Set sane values for buffer pools */
 	for (int i = 0; i < IBMVETH_NUM_BUFF_POOLS; i++)
-		ibmveth_init_buffer_pool(&adapter->rx_buff_pool[i], i,
+		ibmveth_init_buffer_pool(&adapter->rx_buff_pool[0][i], i,
 					 pool_count[i], pool_size[i],
 					 pool_active[i]);
 
-	pool = &adapter->rx_buff_pool[0];
+	pool = &adapter->rx_buff_pool[0][0];
 	pool->skbuff = kunit_kcalloc(test, pool->size, sizeof(void *), GFP_KERNEL);
 	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, pool->skbuff);
+	pool->free_map = kunit_kcalloc(test, pool->size, sizeof(u16),
+				       GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, pool->free_map);
 
 	correlator = ((u64)IBMVETH_NUM_BUFF_POOLS << 32) | 0;
-	KUNIT_EXPECT_EQ(test, -EINVAL, ibmveth_remove_buffer_from_pool(adapter, correlator, false));
-	KUNIT_EXPECT_EQ(test, -EINVAL, ibmveth_remove_buffer_from_pool(adapter, correlator, true));
-
-	correlator = ((u64)0 << 32) | adapter->rx_buff_pool[0].size;
-	KUNIT_EXPECT_EQ(test, -EINVAL, ibmveth_remove_buffer_from_pool(adapter, correlator, false));
-	KUNIT_EXPECT_EQ(test, -EINVAL, ibmveth_remove_buffer_from_pool(adapter, correlator, true));
+	KUNIT_EXPECT_EQ(test, -EINVAL,
+			ibmveth_remove_buffer_from_pool(adapter,
+							correlator, 0, false));
+	KUNIT_EXPECT_EQ(test, -EINVAL,
+			ibmveth_remove_buffer_from_pool(adapter,
+							correlator, 0, true));
+
+	correlator = ((u64)0 << 32) | adapter->rx_buff_pool[0][0].size;
+	KUNIT_EXPECT_EQ(test, -EINVAL,
+			ibmveth_remove_buffer_from_pool(adapter,
+							correlator, 0, false));
+	KUNIT_EXPECT_EQ(test, -EINVAL,
+			ibmveth_remove_buffer_from_pool(adapter,
+							correlator, 0, true));
 
 	correlator = (u64)0 | 0;
 	pool->skbuff[0] = NULL;
-	KUNIT_EXPECT_EQ(test, -EFAULT, ibmveth_remove_buffer_from_pool(adapter, correlator, false));
-	KUNIT_EXPECT_EQ(test, -EFAULT, ibmveth_remove_buffer_from_pool(adapter, correlator, true));
+	KUNIT_EXPECT_EQ(test, -EFAULT,
+			ibmveth_remove_buffer_from_pool(adapter,
+							correlator, 0, false));
+	KUNIT_EXPECT_EQ(test, -EFAULT,
+			ibmveth_remove_buffer_from_pool(adapter,
+							correlator, 0, true));
 
 	flush_work(&adapter->work);
 }
@@ -2241,9 +5188,7 @@ static void ibmveth_remove_buffer_from_pool_test(struct kunit *test)
  * ibmveth_rxq_get_buffer_test - unit test for ibmveth_rxq_get_buffer
  * @test: pointer to kunit structure
  *
- * Tests ibmveth_rxq_get_buffer. ibmveth_rxq_get_buffer also calls WARN_ON for
- * the NULL returns, so dmesg should be checked to see that these warnings
- * happened.
+ * Tests ibmveth_rxq_get_buffer invalid correlator returns NULL without WARN.
  *
  * Return: void
  */
@@ -2258,31 +5203,37 @@ static void ibmveth_rxq_get_buffer_test(struct kunit *test)
 
 	INIT_WORK(&adapter->work, ibmveth_reset_kunit);
 
-	adapter->rx_queue.queue_len = 1;
-	adapter->rx_queue.index = 0;
-	adapter->rx_queue.queue_addr = kunit_kzalloc(test, sizeof(struct ibmveth_rx_q_entry),
-						     GFP_KERNEL);
-	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adapter->rx_queue.queue_addr);
+	adapter->rx_queue[0].queue_len = 1;
+	adapter->rx_queue[0].index = 0;
+	adapter->rx_queue[0].queue_addr =
+		kunit_kzalloc(test, sizeof(struct ibmveth_rx_q_entry),
+			      GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adapter->rx_queue[0].queue_addr);
 
 	/* Set sane values for buffer pools */
 	for (int i = 0; i < IBMVETH_NUM_BUFF_POOLS; i++)
-		ibmveth_init_buffer_pool(&adapter->rx_buff_pool[i], i,
+		ibmveth_init_buffer_pool(&adapter->rx_buff_pool[0][i], i,
 					 pool_count[i], pool_size[i],
 					 pool_active[i]);
 
-	pool = &adapter->rx_buff_pool[0];
+	pool = &adapter->rx_buff_pool[0][0];
 	pool->skbuff = kunit_kcalloc(test, pool->size, sizeof(void *), GFP_KERNEL);
 	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, pool->skbuff);
+	pool->free_map = kunit_kcalloc(test, pool->size, sizeof(u16),
+				       GFP_KERNEL);
+	KUNIT_ASSERT_NOT_ERR_OR_NULL(test, pool->free_map);
 
-	adapter->rx_queue.queue_addr[0].correlator = (u64)IBMVETH_NUM_BUFF_POOLS << 32 | 0;
-	KUNIT_EXPECT_PTR_EQ(test, NULL, ibmveth_rxq_get_buffer(adapter));
+	adapter->rx_queue[0].queue_addr[0].correlator =
+		(u64)IBMVETH_NUM_BUFF_POOLS << 32 | 0;
+	KUNIT_EXPECT_PTR_EQ(test, NULL, ibmveth_rxq_get_buffer(adapter, 0));
 
-	adapter->rx_queue.queue_addr[0].correlator = (u64)0 << 32 | adapter->rx_buff_pool[0].size;
-	KUNIT_EXPECT_PTR_EQ(test, NULL, ibmveth_rxq_get_buffer(adapter));
+	adapter->rx_queue[0].queue_addr[0].correlator =
+		(u64)0 << 32 | adapter->rx_buff_pool[0][0].size;
+	KUNIT_EXPECT_PTR_EQ(test, NULL, ibmveth_rxq_get_buffer(adapter, 0));
 
 	pool->skbuff[0] = skb;
-	adapter->rx_queue.queue_addr[0].correlator = (u64)0 << 32 | 0;
-	KUNIT_EXPECT_PTR_EQ(test, skb, ibmveth_rxq_get_buffer(adapter));
+	adapter->rx_queue[0].queue_addr[0].correlator = (u64)0 << 32 | 0;
+	KUNIT_EXPECT_PTR_EQ(test, skb, ibmveth_rxq_get_buffer(adapter, 0));
 
 	flush_work(&adapter->work);
 }
diff --git a/drivers/net/ethernet/ibm/ibmveth.h b/drivers/net/ethernet/ibm/ibmveth.h
index d87713668ed30..11eda224a6807 100644
--- a/drivers/net/ethernet/ibm/ibmveth.h
+++ b/drivers/net/ethernet/ibm/ibmveth.h
@@ -14,6 +14,9 @@
 #ifndef _IBMVETH_H
 #define _IBMVETH_H
 
+#include <linux/spinlock_types.h>
+#include <linux/completion.h>
+
 /* constants for H_MULTICAST_CTRL */
 #define IbmVethMcastReceptionModifyBit     0x80000UL
 #define IbmVethMcastReceptionEnableBit     0x20000UL
@@ -28,6 +31,7 @@
 #define IbmVethMcastRemoveFilter     0x2UL
 #define IbmVethMcastClearFilterTable 0x3UL
 
+#define IBMVETH_ILLAN_RX_MULTI_QUEUE_SUPPORT	0x0000000000080000UL
 #define IBMVETH_ILLAN_RX_MULTI_BUFF_SUPPORT	0x0000000000040000UL
 #define IBMVETH_ILLAN_LRG_SR_ENABLED	0x0000000000010000UL
 #define IBMVETH_ILLAN_LRG_SND_SUPPORT	0x0000000000008000UL
@@ -66,6 +70,148 @@ static inline long h_add_logical_lan_buffers(unsigned long unit_address,
 			    desc5, desc6, desc7, desc8);
 }
 
+/**
+ * h_register_logical_lan_queue - Register a subordinate receive queue
+ * @unit_address: Device unit address
+ * @buffer_list: DMA address of 4KB page for tracking registered buffers
+ * @rec_queue: Buffer descriptor of receive queue
+ * @queue_handle: Output queue handle on success (may be NULL)
+ * @irq: Output hypervisor IRQ number on success (may be NULL)
+ *
+ * Registers a subordinate receive queue with the hypervisor.
+ *
+ * Return:
+ *   H_SUCCESS (0) on success
+ *   H_PARAMETER if parameters are invalid
+ *   H_BUSY / H_LONG_BUSY_* if the resource is busy; retry
+ *   H_FUNCTION if firmware does not support this hcall
+ *
+ * On success, hypervisor returns:
+ *   R3: H_SUCCESS
+ *   R4: Queue handle
+ *   R5: IRQ number for this queue
+ */
+static inline long
+h_register_logical_lan_queue(unsigned long unit_address,
+			     unsigned long buffer_list,
+			     unsigned long rec_queue,
+			     unsigned long *queue_handle,
+			     unsigned long *irq)
+{
+	unsigned long retbuf[PLPAR_HCALL_BUFSIZE];
+	long rc;
+
+	rc = plpar_hcall(H_REG_LOGICAL_LAN_QUEUE,
+			 retbuf, unit_address,
+			 buffer_list, rec_queue);
+
+	if (rc == H_SUCCESS) {
+		if (queue_handle)
+			*queue_handle = retbuf[0];
+		if (irq)
+			*irq = retbuf[1];
+	}
+
+	return rc;
+}
+
+/**
+ * h_add_logical_lan_buffers_queue - Add buffers to subordinate queue
+ * @unit_address: Device unit address
+ * @queue_handle: Queue handle from h_register_logical_lan_queue() or
+ *		  h_register_logical_lan_with_handle() (queue 0)
+ * @buffersznum: Buffer size (upper 32 bits) | count (lower 32 bits)
+ * @ioba12: Buffer addresses 1 and 2 packed ((addr1 << 32) | addr2)
+ * @ioba34: Buffer addresses 3 and 4 packed
+ * @ioba56: Buffer addresses 5 and 6 packed
+ * @ioba78: Buffer addresses 7 and 8 packed
+ * @ioba910: Buffer addresses 9 and 10 packed
+ * @ioba1112: Buffer addresses 11 and 12 packed
+ *
+ * Return:
+ *   H_SUCCESS - All buffers added successfully
+ *   H_PARAMETER - Invalid parameters
+ *   H_HARDWARE - Hardware error
+ *   H_FUNCTION - Firmware does not support this hcall
+ */
+static inline long h_add_logical_lan_buffers_queue(unsigned long unit_address,
+						   unsigned long queue_handle,
+						   unsigned long buffersznum,
+						   unsigned long ioba12,
+						   unsigned long ioba34,
+						   unsigned long ioba56,
+						   unsigned long ioba78,
+						   unsigned long ioba910,
+						   unsigned long ioba1112)
+{
+	unsigned long retbuf[PLPAR_HCALL9_BUFSIZE];
+
+	return plpar_hcall9(H_ADD_LOGICAL_LAN_BUFFERS_QUEUE,
+			    retbuf, unit_address,
+			    queue_handle, buffersznum,
+			    ioba12, ioba34, ioba56,
+			    ioba78, ioba910, ioba1112);
+}
+
+/**
+ * h_free_logical_lan_queue - Deregister subordinate receive queue
+ * @unit_address: Device unit address
+ * @queue_handle: Queue handle from h_register_logical_lan_queue() or
+ *		  h_register_logical_lan_with_handle() (queue 0)
+ *
+ * Deregisters and frees all structures associated with the subordinate queue.
+ *
+ * Return:
+ *   H_SUCCESS - Queue freed successfully
+ *   H_PARAMETER - Invalid parameters
+ *   H_HARDWARE - Hardware error
+ *   H_STATE - VIOA not in valid state
+ *   H_BUSY / H_LONG_BUSY_* - Resource busy, retry
+ *   H_FUNCTION - Firmware does not support this hcall
+ */
+static inline long h_free_logical_lan_queue(unsigned long unit_address,
+					    unsigned long queue_handle)
+{
+	return plpar_hcall_norets(H_FREE_LOGICAL_LAN_QUEUE,
+				  unit_address, queue_handle);
+}
+
+/**
+ * h_register_logical_lan_with_handle - Register primary queue and get handle
+ * @unit_address: Device unit address
+ * @buffer_list: DMA address of buffer list
+ * @rec_queue: Buffer descriptor of receive queue
+ * @filter_list: DMA address of filter list
+ * @mac_address: MAC address
+ * @queue_handle: Output parameter for queue handle (may be NULL)
+ *
+ * Registers the primary receive queue (queue 0) with the hypervisor and
+ * returns the queue handle. This is needed in multi-queue mode to use
+ * h_add_logical_lan_buffers_queue() for all queues including queue 0.
+ *
+ * Return: H_SUCCESS (0) on success, error code otherwise
+ */
+static inline long
+h_register_logical_lan_with_handle(unsigned long unit_address,
+				   unsigned long buffer_list,
+				   unsigned long rec_queue,
+				   unsigned long filter_list,
+				   unsigned long mac_address,
+				   unsigned long *queue_handle)
+{
+	unsigned long retbuf[PLPAR_HCALL_BUFSIZE];
+	long rc;
+
+	rc = plpar_hcall(H_REGISTER_LOGICAL_LAN, retbuf,
+			 unit_address, buffer_list, rec_queue,
+			 filter_list, mac_address);
+
+	if (rc == H_SUCCESS && queue_handle)
+		*queue_handle = retbuf[0];
+
+	return rc;
+}
+
 /* FW allows us to send 6 descriptors but we only use one so mark
  * the other 5 as unused (0)
  */
@@ -121,7 +267,10 @@ static inline long h_illan_attributes(unsigned long unit_address,
 #define IBMVETH_MAX_TX_BUF_SIZE (1024 * 64)
 #define IBMVETH_MAX_QUEUES 16U
 #define IBMVETH_DEFAULT_QUEUES 8U
-#define IBMVETH_MAX_RX_PER_HCALL 8U
+#define IBMVETH_MAX_RX_QUEUES 16U
+#define IBMVETH_DEFAULT_RX_QUEUES 1U
+#define IBMVETH_MAX_RX_REGULAR 8U
+#define IBMVETH_MAX_RX_PER_HCALL 12U
 
 static int pool_size[] = { 512, 1024 * 2, 1024 * 16, 1024 * 32, 1024 * 64 };
 static int pool_count[] = { 256, 512, 256, 256, 256 };
@@ -130,6 +279,43 @@ static int pool_active[] = { 1, 1, 0, 0, 1};
 
 #define IBM_VETH_INVALID_MAP ((u16)0xffff)
 
+/*
+ * Per-queue RX counters. No field has two concurrent writers:
+ * interrupts is written only from this queue's IRQ handler; polls,
+ * packets, bytes, large_packets and invalid_buffers only from its NAPI
+ * poll; replenish_* only under its replenish_lock; and no_buffer_drops
+ * and no_buffer_retired under that lock or from a teardown path already
+ * quiesced by napi_disable()/synchronize_irq(). Plain u64 is therefore
+ * sufficient and no atomic or u64_stats_sync is needed: the driver is
+ * PPC64-only, so 64-bit loads and stores do not tear.
+ */
+struct ibmveth_rx_queue_stats {
+	u64 packets;
+	u64 bytes;
+	u64 interrupts;
+	u64 polls;
+	u64 large_packets;
+	u64 invalid_buffers;
+	/* PHYP's per-page absolute drop count for the live page. */
+	u64 no_buffer_drops;
+	/* Absolutes from pages this queue has already retired. */
+	u64 no_buffer_retired;
+	u64 replenish_task_cycles;
+	u64 replenish_no_mem;
+	u64 replenish_add_buff_failure;
+	u64 replenish_add_buff_success;
+} ____cacheline_aligned_in_smp;
+
+/* Per-queue TX counters; serialized by the stack's per-queue TX lock. */
+struct ibmveth_tx_queue_stats {
+	u64 packets;
+	u64 bytes;
+	u64 large_packets;
+	u64 dropped_packets;
+	u64 send_failures;
+	u64 checksum_offload;
+} ____cacheline_aligned_in_smp;
+
 struct ibmveth_buff_pool {
     u32 size;
     u32 index;
@@ -143,6 +329,7 @@ struct ibmveth_buff_pool {
     struct sk_buff **skbuff;
     int active;
     struct kobject kobj;
+	struct completion released;
 };
 
 struct ibmveth_rx_q {
@@ -152,23 +339,34 @@ struct ibmveth_rx_q {
     dma_addr_t queue_dma;
     u32        queue_len;
     struct ibmveth_rx_q_entry *queue_addr;
+	spinlock_t	replenish_lock;	/* per-queue buffer replenish */
 };
 
 struct ibmveth_adapter {
 	struct vio_dev *vdev;
 	struct net_device *netdev;
-	struct napi_struct napi;
+	struct napi_struct napi[IBMVETH_MAX_RX_QUEUES];
 	struct work_struct work;
 	unsigned int mcastFilterSize;
-	void *buffer_list_addr;
+	void *buffer_list_addr[IBMVETH_MAX_RX_QUEUES];
 	void *filter_list_addr;
 	void *tx_ltb_ptr[IBMVETH_MAX_QUEUES];
 	unsigned int tx_ltb_size;
 	dma_addr_t tx_ltb_dma[IBMVETH_MAX_QUEUES];
-	dma_addr_t buffer_list_dma;
+	dma_addr_t buffer_list_dma[IBMVETH_MAX_RX_QUEUES];
 	dma_addr_t filter_list_dma;
-	struct ibmveth_buff_pool rx_buff_pool[IBMVETH_NUM_BUFF_POOLS];
-	struct ibmveth_rx_q rx_queue;
+	struct ibmveth_buff_pool
+		rx_buff_pool[IBMVETH_MAX_RX_QUEUES][IBMVETH_NUM_BUFF_POOLS];
+	struct ibmveth_rx_q rx_queue[IBMVETH_MAX_RX_QUEUES];
+	u64 queue_handle[IBMVETH_MAX_RX_QUEUES];
+	unsigned int queue_irq[IBMVETH_MAX_RX_QUEUES];
+	bool multi_queue;
+	unsigned int num_rx_queues;
+	bool mq_fallback;
+	/* Lifetime: true after successful ndo_open until close clears it. */
+	bool opened;
+	/* Lifetime: true while RX IRQ handlers / NAPI are installed. */
+	bool rx_irq_setup;
 	int rx_csum;
 	int large_send;
 	bool is_active_trunk;
@@ -177,17 +375,19 @@ struct ibmveth_adapter {
 	u64 fw_ipv6_csum_support;
 	u64 fw_ipv4_csum_support;
 	u64 fw_large_send_support;
-	/* adapter specific stats */
-	u64 replenish_task_cycles;
-	u64 replenish_no_mem;
-	u64 replenish_add_buff_failure;
-	u64 replenish_add_buff_success;
-	u64 rx_invalid_buffer;
-	u64 rx_no_buffer;
+	/*
+	 * Every other ethtool -S counter lives in rx_qstats/tx_qstats and is
+	 * summed on read. tx_map_failed predates multi-queue, has never been
+	 * updated by any code path, and is kept only so the key keeps
+	 * reporting the zero userspace already sees.
+	 */
 	u64 tx_map_failed;
-	u64 tx_send_failed;
-	u64 tx_large_packets;
-	u64 rx_large_packets;
+
+	struct ibmveth_rx_queue_stats *rx_qstats;
+	struct ibmveth_tx_queue_stats *tx_qstats;
+
+	struct dentry *debugfs_dir;
+
 	/* Ethtool settings */
 	u8 duplex;
 	u32 speed;
diff --git a/tools/perf/scripts/python/powerpc-hcalls.py b/tools/perf/scripts/python/powerpc-hcalls.py
index fedce1b68cad1..afc354583b7b5 100644
--- a/tools/perf/scripts/python/powerpc-hcalls.py
+++ b/tools/perf/scripts/python/powerpc-hcalls.py
@@ -217,6 +217,10 @@ hcall_table = {
 	# Key wrapping hcalls
 	1168: 'H_PKS_WRAP_OBJECT',
 	1172: 'H_PKS_UNWRAP_OBJECT',
+	# Logical LAN multi-queue RX (PAPR 11.20.00)
+	1180: 'H_REG_LOGICAL_LAN_QUEUE',
+	1184: 'H_ADD_LOGICAL_LAN_BUFFERS_QUEUE',
+	1192: 'H_FREE_LOGICAL_LAN_QUEUE',
 	# Platform-specific hcalls used by the Ultravisor
 	61184: 'H_SVM_PAGE_IN',
 	61188: 'H_SVM_PAGE_OUT',