From 5edd1051f4f3bd7cddcc91e346fdd22eee639c72 Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Wed, 25 Feb 2015 14:49:16 -0500 Subject: SUNRPC: Reorder rpc_task to put waitqueue related info in same cachelines Try to group all the data required by the waitqueues, their timers and timer callbacks into the same cachelines for performance. With this reordering, "pahole" reports the following structure on x86_64: struct rpc_task { atomic_t tk_count; /* 0 4 */ int tk_status; /* 4 4 */ struct list_head tk_task; /* 8 16 */ void (*tk_callback)(struct rpc_task *); /* 24 void (*tk_action)(struct rpc_task *); /* 32 long unsigned int tk_timeout; /* 40 8 */ long unsigned int tk_runstate; /* 48 8 */ struct rpc_wait_queue * tk_waitqueue; /* 56 8 */ /* --- cacheline 1 boundary (64 bytes) --- */ union { struct work_struct tk_work; /* 64 */ struct rpc_wait tk_wait; /* 56 */ } u; /* 64 64 */ /* --- cacheline 2 boundary (128 bytes) --- */ struct rpc_message tk_msg; /* 128 32 */ void * tk_calldata; /* 160 8 */ const struct rpc_call_ops * tk_ops; /* 168 8 */ struct rpc_clnt * tk_client; /* 176 8 */ struct rpc_rqst * tk_rqstp; /* 184 8 */ /* --- cacheline 3 boundary (192 bytes) --- */ struct workqueue_struct * tk_workqueue; /* 192 8 */ ktime_t tk_start; /* 200 8 */ pid_t tk_owner; /* 208 4 */ short unsigned int tk_flags; /* 212 2 */ short unsigned int tk_timeouts; /* 214 2 */ short unsigned int tk_pid; /* 216 2 */ unsigned char tk_priority:2; /* 218: 6 1 */ unsigned char tk_garb_retry:2; /* 218: 4 1 */ unsigned char tk_cred_retry:2; /* 218: 2 1 */ unsigned char tk_rebind_retry:2; /* 218: 0 1 */ /* size: 224, cachelines: 4, members: 24 */ /* padding: 5 */ /* last cacheline: 32 bytes */ }; whereas on i386, it reports everything fitting into the 1st cacheline. Signed-off-by: Trond Myklebust --- include/linux/sunrpc/sched.h | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) (limited to 'include/linux/sunrpc/sched.h') diff --git a/include/linux/sunrpc/sched.h b/include/linux/sunrpc/sched.h index d703f0ef37d8..ee0fbcf9b02e 100644 --- a/include/linux/sunrpc/sched.h +++ b/include/linux/sunrpc/sched.h @@ -42,40 +42,41 @@ struct rpc_wait { */ struct rpc_task { atomic_t tk_count; /* Reference count */ + int tk_status; /* result of last operation */ struct list_head tk_task; /* global list of tasks */ - struct rpc_clnt * tk_client; /* RPC client */ - struct rpc_rqst * tk_rqstp; /* RPC request */ - - /* - * RPC call state - */ - struct rpc_message tk_msg; /* RPC call info */ /* * callback to be executed after waking up * action next procedure for async tasks - * tk_ops caller callbacks */ void (*tk_callback)(struct rpc_task *); void (*tk_action)(struct rpc_task *); - const struct rpc_call_ops *tk_ops; - void * tk_calldata; unsigned long tk_timeout; /* timeout for rpc_sleep() */ unsigned long tk_runstate; /* Task run status */ - struct workqueue_struct *tk_workqueue; /* Normally rpciod, but could - * be any workqueue - */ + struct rpc_wait_queue *tk_waitqueue; /* RPC wait queue we're on */ union { struct work_struct tk_work; /* Async task work queue */ struct rpc_wait tk_wait; /* RPC wait */ } u; + /* + * RPC call state + */ + struct rpc_message tk_msg; /* RPC call info */ + void * tk_calldata; /* Caller private data */ + const struct rpc_call_ops *tk_ops; /* Caller callbacks */ + + struct rpc_clnt * tk_client; /* RPC client */ + struct rpc_rqst * tk_rqstp; /* RPC request */ + + struct workqueue_struct *tk_workqueue; /* Normally rpciod, but could + * be any workqueue + */ ktime_t tk_start; /* RPC task init timestamp */ pid_t tk_owner; /* Process id for batching tasks */ - int tk_status; /* result of last operation */ unsigned short tk_flags; /* misc flags */ unsigned short tk_timeouts; /* maj timeouts */ -- cgit v1.2.3 From fb43d17210baa538e58fc83d2d0f8a32399db73b Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Sat, 30 Jan 2016 16:39:26 -0500 Subject: SUNRPC: Use the multipath iterator to assign a transport to each task Signed-off-by: Trond Myklebust --- include/linux/sunrpc/sched.h | 2 ++ net/sunrpc/clnt.c | 38 ++++++++++++++++++++------------------ net/sunrpc/rpcb_clnt.c | 6 ++---- net/sunrpc/xprt.c | 14 +++----------- net/sunrpc/xprtsock.c | 4 +--- 5 files changed, 28 insertions(+), 36 deletions(-) (limited to 'include/linux/sunrpc/sched.h') diff --git a/include/linux/sunrpc/sched.h b/include/linux/sunrpc/sched.h index ee0fbcf9b02e..0b248e98ee3b 100644 --- a/include/linux/sunrpc/sched.h +++ b/include/linux/sunrpc/sched.h @@ -69,6 +69,8 @@ struct rpc_task { const struct rpc_call_ops *tk_ops; /* Caller callbacks */ struct rpc_clnt * tk_client; /* RPC client */ + struct rpc_xprt * tk_xprt; /* Transport */ + struct rpc_rqst * tk_rqstp; /* RPC request */ struct workqueue_struct *tk_workqueue; /* Normally rpciod, but could diff --git a/net/sunrpc/clnt.c b/net/sunrpc/clnt.c index 625fb8a184c6..8e46fa5a2ab1 100644 --- a/net/sunrpc/clnt.c +++ b/net/sunrpc/clnt.c @@ -894,6 +894,7 @@ EXPORT_SYMBOL_GPL(rpc_bind_new_program); void rpc_task_release_client(struct rpc_task *task) { struct rpc_clnt *clnt = task->tk_client; + struct rpc_xprt *xprt = task->tk_xprt; if (clnt != NULL) { /* Remove from client task list */ @@ -904,13 +905,22 @@ void rpc_task_release_client(struct rpc_task *task) rpc_release_client(clnt); } + + if (xprt != NULL) { + task->tk_xprt = NULL; + + xprt_put(xprt); + } } static void rpc_task_set_client(struct rpc_task *task, struct rpc_clnt *clnt) { + if (clnt != NULL) { rpc_task_release_client(task); + if (task->tk_xprt == NULL) + task->tk_xprt = xprt_iter_get_next(&clnt->cl_xpi); task->tk_client = clnt; atomic_inc(&clnt->cl_count); if (clnt->cl_softrtry) @@ -2122,11 +2132,9 @@ call_timeout(struct rpc_task *task) } if (RPC_IS_SOFT(task)) { if (clnt->cl_chatty) { - rcu_read_lock(); printk(KERN_NOTICE "%s: server %s not responding, timed out\n", clnt->cl_program->name, - rcu_dereference(clnt->cl_xprt)->servername); - rcu_read_unlock(); + task->tk_xprt->servername); } if (task->tk_flags & RPC_TASK_TIMEOUT) rpc_exit(task, -ETIMEDOUT); @@ -2138,11 +2146,9 @@ call_timeout(struct rpc_task *task) if (!(task->tk_flags & RPC_CALL_MAJORSEEN)) { task->tk_flags |= RPC_CALL_MAJORSEEN; if (clnt->cl_chatty) { - rcu_read_lock(); printk(KERN_NOTICE "%s: server %s not responding, still trying\n", clnt->cl_program->name, - rcu_dereference(clnt->cl_xprt)->servername); - rcu_read_unlock(); + task->tk_xprt->servername); } } rpc_force_rebind(clnt); @@ -2172,11 +2178,9 @@ call_decode(struct rpc_task *task) if (task->tk_flags & RPC_CALL_MAJORSEEN) { if (clnt->cl_chatty) { - rcu_read_lock(); printk(KERN_NOTICE "%s: server %s OK\n", clnt->cl_program->name, - rcu_dereference(clnt->cl_xprt)->servername); - rcu_read_unlock(); + task->tk_xprt->servername); } task->tk_flags &= ~RPC_CALL_MAJORSEEN; } @@ -2330,11 +2334,9 @@ rpc_verify_header(struct rpc_task *task) task->tk_action = call_bind; goto out_retry; case RPC_AUTH_TOOWEAK: - rcu_read_lock(); printk(KERN_NOTICE "RPC: server %s requires stronger " "authentication.\n", - rcu_dereference(clnt->cl_xprt)->servername); - rcu_read_unlock(); + task->tk_xprt->servername); break; default: dprintk("RPC: %5u %s: unknown auth error: %x\n", @@ -2359,27 +2361,27 @@ rpc_verify_header(struct rpc_task *task) case RPC_SUCCESS: return p; case RPC_PROG_UNAVAIL: - dprintk_rcu("RPC: %5u %s: program %u is unsupported " + dprintk("RPC: %5u %s: program %u is unsupported " "by server %s\n", task->tk_pid, __func__, (unsigned int)clnt->cl_prog, - rcu_dereference(clnt->cl_xprt)->servername); + task->tk_xprt->servername); error = -EPFNOSUPPORT; goto out_err; case RPC_PROG_MISMATCH: - dprintk_rcu("RPC: %5u %s: program %u, version %u unsupported " + dprintk("RPC: %5u %s: program %u, version %u unsupported " "by server %s\n", task->tk_pid, __func__, (unsigned int)clnt->cl_prog, (unsigned int)clnt->cl_vers, - rcu_dereference(clnt->cl_xprt)->servername); + task->tk_xprt->servername); error = -EPROTONOSUPPORT; goto out_err; case RPC_PROC_UNAVAIL: - dprintk_rcu("RPC: %5u %s: proc %s unsupported by program %u, " + dprintk("RPC: %5u %s: proc %s unsupported by program %u, " "version %u on server %s\n", task->tk_pid, __func__, rpc_proc_name(task), clnt->cl_prog, clnt->cl_vers, - rcu_dereference(clnt->cl_xprt)->servername); + task->tk_xprt->servername); error = -EOPNOTSUPP; goto out_err; case RPC_GARBAGE_ARGS: diff --git a/net/sunrpc/rpcb_clnt.c b/net/sunrpc/rpcb_clnt.c index 44f025c150d8..5b30603596d0 100644 --- a/net/sunrpc/rpcb_clnt.c +++ b/net/sunrpc/rpcb_clnt.c @@ -683,11 +683,9 @@ void rpcb_getport_async(struct rpc_task *task) int status; rcu_read_lock(); - do { - clnt = rpcb_find_transport_owner(task->tk_client); - xprt = xprt_get(rcu_dereference(clnt->cl_xprt)); - } while (xprt == NULL); + clnt = rpcb_find_transport_owner(task->tk_client); rcu_read_unlock(); + xprt = xprt_get(task->tk_xprt); dprintk("RPC: %5u %s(%s, %u, %u, %d)\n", task->tk_pid, __func__, diff --git a/net/sunrpc/xprt.c b/net/sunrpc/xprt.c index 323b332f8f7c..216a1385718a 100644 --- a/net/sunrpc/xprt.c +++ b/net/sunrpc/xprt.c @@ -1181,7 +1181,7 @@ EXPORT_SYMBOL_GPL(xprt_free); */ void xprt_reserve(struct rpc_task *task) { - struct rpc_xprt *xprt; + struct rpc_xprt *xprt = task->tk_xprt; task->tk_status = 0; if (task->tk_rqstp != NULL) @@ -1189,11 +1189,8 @@ void xprt_reserve(struct rpc_task *task) task->tk_timeout = 0; task->tk_status = -EAGAIN; - rcu_read_lock(); - xprt = rcu_dereference(task->tk_client->cl_xprt); if (!xprt_throttle_congested(xprt, task)) xprt->ops->alloc_slot(xprt, task); - rcu_read_unlock(); } /** @@ -1207,7 +1204,7 @@ void xprt_reserve(struct rpc_task *task) */ void xprt_retry_reserve(struct rpc_task *task) { - struct rpc_xprt *xprt; + struct rpc_xprt *xprt = task->tk_xprt; task->tk_status = 0; if (task->tk_rqstp != NULL) @@ -1215,10 +1212,7 @@ void xprt_retry_reserve(struct rpc_task *task) task->tk_timeout = 0; task->tk_status = -EAGAIN; - rcu_read_lock(); - xprt = rcu_dereference(task->tk_client->cl_xprt); xprt->ops->alloc_slot(xprt, task); - rcu_read_unlock(); } static inline __be32 xprt_alloc_xid(struct rpc_xprt *xprt) @@ -1265,11 +1259,9 @@ void xprt_release(struct rpc_task *task) if (req == NULL) { if (task->tk_client) { - rcu_read_lock(); - xprt = rcu_dereference(task->tk_client->cl_xprt); + xprt = task->tk_xprt; if (xprt->snd_task == task) xprt_release_write(xprt, task); - rcu_read_unlock(); } return; } diff --git a/net/sunrpc/xprtsock.c b/net/sunrpc/xprtsock.c index fde2138b81e7..65e759569e48 100644 --- a/net/sunrpc/xprtsock.c +++ b/net/sunrpc/xprtsock.c @@ -1844,9 +1844,7 @@ static int xs_bind(struct sock_xprt *transport, struct socket *sock) */ static void xs_local_rpcbind(struct rpc_task *task) { - rcu_read_lock(); - xprt_set_bound(rcu_dereference(task->tk_client->cl_xprt)); - rcu_read_unlock(); + xprt_set_bound(task->tk_xprt); } static void xs_local_set_port(struct rpc_xprt *xprt, unsigned short port) -- cgit v1.2.3 From 9d61498d5f6cde68a708781bf2cd33cae21121dc Mon Sep 17 00:00:00 2001 From: Trond Myklebust Date: Sat, 30 Jan 2016 18:13:05 -0500 Subject: SUNRPC: Allow caller to specify the transport to use This is needed in order to allow the NFSv4.1 backchannel and BIND_CONN_TO_SESSION function to work. Signed-off-by: Trond Myklebust --- include/linux/sunrpc/sched.h | 1 + net/sunrpc/sched.c | 2 ++ 2 files changed, 3 insertions(+) (limited to 'include/linux/sunrpc/sched.h') diff --git a/include/linux/sunrpc/sched.h b/include/linux/sunrpc/sched.h index 0b248e98ee3b..05a1809c44d9 100644 --- a/include/linux/sunrpc/sched.h +++ b/include/linux/sunrpc/sched.h @@ -103,6 +103,7 @@ struct rpc_call_ops { struct rpc_task_setup { struct rpc_task *task; struct rpc_clnt *rpc_client; + struct rpc_xprt *rpc_xprt; const struct rpc_message *rpc_message; const struct rpc_call_ops *callback_ops; void *callback_data; diff --git a/net/sunrpc/sched.c b/net/sunrpc/sched.c index 73ad57a59989..fcfd48d263f6 100644 --- a/net/sunrpc/sched.c +++ b/net/sunrpc/sched.c @@ -909,6 +909,8 @@ static void rpc_init_task(struct rpc_task *task, const struct rpc_task_setup *ta /* Initialize workqueue for async tasks */ task->tk_workqueue = task_setup_data->workqueue; + task->tk_xprt = xprt_get(task_setup_data->rpc_xprt); + if (task->tk_ops->rpc_call_prepare != NULL) task->tk_action = rpc_prepare_task; -- cgit v1.2.3