From: Benjamin Coddington A transport that needs a connection accepted, closed, or a handshake run waits its turn like a transport with data, behind every transport of its client queued before it, and a client with many connections can hold a close back for k rounds of all the busy clients. None of those events costs the service a request's worth of work, and a close frees resources the sooner it is served. Give each client a second per-pool queue for transports with XPT_CONN, XPT_CLOSE or XPT_HANDSHAKE set when they are enqueued, and have the dispatcher serve that queue before the client's data queue. The client's turn in the round-robin is unchanged; only which of its transports goes first. A transport that is already queued with data when it is closed is served in its original order, as before. Suggested-by: Chuck Lever Signed-off-by: Benjamin Coddington --- include/linux/sunrpc/svc.h | 2 ++ include/linux/sunrpc/svc_xprt.h | 3 ++ net/sunrpc/svc.c | 1 + net/sunrpc/svc_xprt.c | 60 +++++++++++++++++++++++++++------ 4 files changed, 56 insertions(+), 10 deletions(-) diff --git a/include/linux/sunrpc/svc.h b/include/linux/sunrpc/svc.h index 5d8e35c5379e..02a8eb5d3357 100644 --- a/include/linux/sunrpc/svc.h +++ b/include/linux/sunrpc/svc.h @@ -66,6 +66,7 @@ enum { * of the service does not grow with its connection count. */ struct svc_client_pool { + struct lwq cp_ctrl; /* ready transports with a control event */ struct lwq cp_xprts; /* ready transports */ struct lwq_node cp_ready; /* link in svc_pool.sp_clients */ unsigned long cp_flags; @@ -74,6 +75,7 @@ struct svc_client_pool { enum { SVC_CP_QUEUED, /* on sp_clients, or held by the thread that took it */ + SVC_CP_DATA_NEXT, /* last turn served a control event; data is owed */ }; struct svc_client { diff --git a/include/linux/sunrpc/svc_xprt.h b/include/linux/sunrpc/svc_xprt.h index 10c18e948b42..e05d7c8610ff 100644 --- a/include/linux/sunrpc/svc_xprt.h +++ b/include/linux/sunrpc/svc_xprt.h @@ -116,6 +116,9 @@ enum { */ }; +/* events a client's queue serves ahead of its data */ +#define SVC_XPT_CTRL_EVENTS (BIT(XPT_CONN) | BIT(XPT_CLOSE) | BIT(XPT_HANDSHAKE)) + /* * Maximum number of "tmp" connections - those without XPT_PEER_VALID - * permitted on any service. diff --git a/net/sunrpc/svc.c b/net/sunrpc/svc.c index 4adf113d067b..92c6737e5905 100644 --- a/net/sunrpc/svc.c +++ b/net/sunrpc/svc.c @@ -405,6 +405,7 @@ struct svc_client *svc_client_alloc(unsigned int nrpools, gfp_t gfp) refcount_set(&cl->cl_ref, 1); INIT_HLIST_NODE(&cl->cl_hash); for (i = 0; i < nrpools; i++) { + lwq_init(&cl->cl_pool[i].cp_ctrl); lwq_init(&cl->cl_pool[i].cp_xprts); cl->cl_pool[i].cp_client = cl; } diff --git a/net/sunrpc/svc_xprt.c b/net/sunrpc/svc_xprt.c index 735f4d92bf5d..14a7483886fc 100644 --- a/net/sunrpc/svc_xprt.c +++ b/net/sunrpc/svc_xprt.c @@ -646,7 +646,10 @@ void svc_xprt_enqueue(struct svc_xprt *xprt) percpu_counter_inc(&pool->sp_sockets_queued); xprt->xpt_qtime = ktime_get(); - lwq_enqueue(&xprt->xpt_ready, &cp->cp_xprts); + if (READ_ONCE(xprt->xpt_flags) & SVC_XPT_CTRL_EVENTS) + lwq_enqueue(&xprt->xpt_ready, &cp->cp_ctrl); + else + lwq_enqueue(&xprt->xpt_ready, &cp->cp_xprts); if (!test_and_set_bit(SVC_CP_QUEUED, &cp->cp_flags)) lwq_enqueue(&cp->cp_ready, &pool->sp_clients); @@ -654,6 +657,11 @@ void svc_xprt_enqueue(struct svc_xprt *xprt) } EXPORT_SYMBOL_GPL(svc_xprt_enqueue); +static bool svc_client_pool_empty(struct svc_client_pool *cp) +{ + return lwq_empty(&cp->cp_ctrl) && lwq_empty(&cp->cp_xprts); +} + /* * Put @cp back on the pool's client queue if it still has ready * transports. Otherwise clear SVC_CP_QUEUED and look once more: an @@ -662,21 +670,47 @@ EXPORT_SYMBOL_GPL(svc_xprt_enqueue); static void svc_client_pool_requeue(struct svc_pool *pool, struct svc_client_pool *cp) { - if (!lwq_empty(&cp->cp_xprts)) { + if (!svc_client_pool_empty(cp)) { lwq_enqueue(&cp->cp_ready, &pool->sp_clients); return; } clear_bit(SVC_CP_QUEUED, &cp->cp_flags); /* the clear is visible before the re-test; see svc_xprt_enqueue() */ smp_mb__after_atomic(); - if (!lwq_empty(&cp->cp_xprts) && + if (!svc_client_pool_empty(cp) && !test_and_set_bit(SVC_CP_QUEUED, &cp->cp_flags)) lwq_enqueue(&cp->cp_ready, &pool->sp_clients); } /* - * Dequeue the next transport: one from the client at the head of the - * pool's client queue, which then goes to the tail if it has more. + * A client's control events go before its data, but not two turns in a + * row while data waits: a listener re-arms XPT_CONN after every accept + * and would otherwise hold the anonymous client's turns for as long as + * connections keep arriving. Only the thread holding @cp touches + * SVC_CP_DATA_NEXT. + */ +static struct svc_xprt *svc_client_pool_dequeue(struct svc_client_pool *cp) +{ + struct svc_xprt *xprt = NULL; + bool data_owed; + + data_owed = test_and_clear_bit(SVC_CP_DATA_NEXT, &cp->cp_flags); + if (!data_owed) + xprt = lwq_dequeue(&cp->cp_ctrl, struct svc_xprt, xpt_ready); + if (!xprt) { + xprt = lwq_dequeue(&cp->cp_xprts, struct svc_xprt, xpt_ready); + if (xprt || !data_owed) + return xprt; + xprt = lwq_dequeue(&cp->cp_ctrl, struct svc_xprt, xpt_ready); + } + if (xprt && !lwq_empty(&cp->cp_xprts)) + set_bit(SVC_CP_DATA_NEXT, &cp->cp_flags); + return xprt; +} + +/* + * Dequeue the next transport from the pool's client queue: one from the + * client at the head, which then goes to the tail if it has more. */ static struct svc_xprt *svc_xprt_dequeue(struct svc_pool *pool) { @@ -688,7 +722,7 @@ static struct svc_xprt *svc_xprt_dequeue(struct svc_pool *pool) cp_ready); if (!cp) return NULL; - xprt = lwq_dequeue(&cp->cp_xprts, struct svc_xprt, xpt_ready); + xprt = svc_client_pool_dequeue(cp); svc_client_pool_requeue(pool, cp); } while (!xprt); svc_xprt_get(xprt); @@ -1371,15 +1405,15 @@ static int svc_close_list(struct svc_serv *serv, struct list_head *xprt_list, st } /* - * Delete the ready transports of @net that are queued on @cp; the rest go + * Delete the ready transports of @net that are queued on @lq; the rest go * back in their order. */ -static void svc_client_pool_clean(struct svc_client_pool *cp, struct net *net) +static void svc_xprt_lwq_clean(struct lwq *lq, struct net *net) { struct llist_node *q, **t1, *t2; struct svc_xprt *xprt; - q = lwq_dequeue_all(&cp->cp_xprts); + q = lwq_dequeue_all(lq); lwq_for_each_safe(xprt, t1, t2, &q, xpt_ready) { if (xprt->xpt_net == net) { set_bit(XPT_CLOSE, &xprt->xpt_flags); @@ -1388,7 +1422,13 @@ static void svc_client_pool_clean(struct svc_client_pool *cp, struct net *net) } } if (q) - lwq_enqueue_batch(q, &cp->cp_xprts); + lwq_enqueue_batch(q, lq); +} + +static void svc_client_pool_clean(struct svc_client_pool *cp, struct net *net) +{ + svc_xprt_lwq_clean(&cp->cp_ctrl, net); + svc_xprt_lwq_clean(&cp->cp_xprts, net); } static void svc_clean_up_xprts(struct svc_serv *serv, struct net *net) -- 2.53.0