svn commit: r361270 - projects/nfs-over-tls/sys/rpc
Rick Macklem
rmacklem at FreeBSD.org
Wed May 20 01:25:46 UTC 2020
Author: rmacklem
Date: Wed May 20 01:25:46 2020
New Revision: 361270
URL: https://svnweb.freebsd.org/changeset/base/361270
Log:
Modify the client side non-application data record upcall so that it is
done via a kthread.
There was a glitch in the upcall handling without this patch, in that
the upcall was done by a thread doing an RPC.
The problem with this was that there might not be an RPC done for
minutes, hours,...
This patch changes the code so that a kthread runs for each client
side TLS connection and does the upcalls.
Modified:
projects/nfs-over-tls/sys/rpc/clnt_vc.c
projects/nfs-over-tls/sys/rpc/krpc.h
Modified: projects/nfs-over-tls/sys/rpc/clnt_vc.c
==============================================================================
--- projects/nfs-over-tls/sys/rpc/clnt_vc.c Tue May 19 22:09:59 2020 (r361269)
+++ projects/nfs-over-tls/sys/rpc/clnt_vc.c Wed May 20 01:25:46 2020 (r361270)
@@ -62,6 +62,7 @@ __FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kernel.h>
+#include <sys/kthread.h>
#include <sys/ktls.h>
#include <sys/lock.h>
#include <sys/malloc.h>
@@ -101,7 +102,7 @@ static void clnt_vc_close(CLIENT *);
static void clnt_vc_destroy(CLIENT *);
static bool_t time_not_ok(struct timeval *);
static int clnt_vc_soupcall(struct socket *so, void *arg, int waitflag);
-static void clnt_vc_dotlsupcall(struct ct_data *ct);
+static void clnt_vc_dotlsupcall(void *data);
static struct clnt_ops clnt_vc_ops = {
.cl_call = clnt_vc_call,
@@ -418,13 +419,7 @@ call_again:
goto out;
}
- /*
- * For TLS, do an upcall, as required.
- * clnt_vc_dotlsupcall() will just return unless an
- * an upcall is needed.
- * Wait until any upcall is completed.
- */
- clnt_vc_dotlsupcall(ct);
+ /* For TLS, wait for an upcall to be done, as required. */
while ((ct->ct_rcvstate & (RPCRCVSTATE_NORMAL |
RPCRCVSTATE_NONAPPDATA)) == 0)
msleep(&ct->ct_rcvstate, &ct->ct_lock, 0, "rpcrcvst", hz);
@@ -502,30 +497,8 @@ printf("TRY AGAIN!!\n");
goto out;
}
- /*
- * For TLS, msleep() can be awakened to handle
- * an upcall via a call to clnt_vc_dotlsupcall().
- * If there was no error, it needs to loop
- * around and wait for the reply.
- */
- do {
- /*
- * Call clnt_vc_dotlsupcall() both before
- * and after the msleep(). If there is
- * no upcall to do, clnt_vc_dotlsupcall()
- * simply returns.
- * Call before msleep() in case ct_rcvstate
- * is already set to UPCALLNEEDED and the
- * wakeup(ct) has already been done.
- * Call after msleep() in case it has its
- * reply and will not be looping.
- */
- clnt_vc_dotlsupcall(ct);
- error = msleep(cr, &ct->ct_lock, ct->ct_waitflag,
- ct->ct_waitchan, tvtohz(&timeout));
- clnt_vc_dotlsupcall(ct);
- } while (cr->cr_mrep == NULL && error == 0 &&
- cr->cr_error == 0);
+ error = msleep(cr, &ct->ct_lock, ct->ct_waitflag, ct->ct_waitchan,
+ tvtohz(&timeout));
TAILQ_REMOVE(&ct->ct_pending, cr, cr_link);
@@ -673,6 +646,8 @@ clnt_vc_control(CLIENT *cl, u_int request, void *info)
void *infop = info;
SVCXPRT *xprt;
uint64_t *p;
+ int error;
+ static u_int thrdnum = 0;
mtx_lock(&ct->ct_lock);
@@ -800,7 +775,13 @@ printf("backch tls=0x%x xprt=%p\n", xprt->xp_tls, xprt
ct->ct_sslsec = *p++;
ct->ct_sslusec = *p++;
ct->ct_sslrefno = *p;
- break;
+ mtx_unlock(&ct->ct_lock);
+ /* Start the kthread that handles upcalls. */
+ error = kthread_add(clnt_vc_dotlsupcall, ct,
+ NULL, NULL, 0, 0, "krpctls%u", thrdnum++);
+ if (error != 0)
+ panic("Can't add KRPC thread error %d", error);
+ return (TRUE);
case CLSET_BLOCKRCV:
if (*(int *) info) {
@@ -868,6 +849,7 @@ clnt_vc_close(CLIENT *cl)
ct->ct_closing = FALSE;
ct->ct_closed = TRUE;
+ wakeup(&ct->ct_sslrefno);
mtx_unlock(&ct->ct_lock);
wakeup(ct);
}
@@ -900,6 +882,10 @@ clnt_vc_destroy(CLIENT *cl)
}
}
+ /* Wait for the upcall kthread to terminate. */
+ while ((ct->ct_rcvstate & RPCRCVSTATE_UPCALLTHREAD) != 0)
+ msleep(&ct->ct_sslrefno, &ct->ct_lock, 0,
+ "clntvccl", hz);
mtx_unlock(&ct->ct_lock);
mtx_destroy(&ct->ct_lock);
@@ -1036,13 +1022,10 @@ clnt_vc_soupcall(struct socket *so, void *arg, int wai
mtx_lock(&ct->ct_lock);
ct->ct_rcvstate |= RPCRCVSTATE_UPCALLNEEDED;
/*
- * If an upcall in needed, wake up all thread(s)
- * in clnt_vc_call() so that one of them can do it.
- * Not efficient, but this should not happen
- * frequently.
+ * If an upcall in needed, wake up the kthread
+ * that runs clnt_vc_dotlsupcall().
*/
- TAILQ_FOREACH(cr, &ct->ct_pending, cr_link)
- wakeup(cr);
+ wakeup(&ct->ct_sslrefno);
mtx_unlock(&ct->ct_lock);
printf("Mark upcallneeded\n");
break;
@@ -1296,16 +1279,19 @@ clnt_vc_upcallsdone(struct ct_data *ct)
/*
* Do a TLS upcall to the rpctlscd daemon, as required.
+ * This function runs as a kthread.
*/
static void
-clnt_vc_dotlsupcall(struct ct_data *ct)
+clnt_vc_dotlsupcall(void *data)
{
+ struct ct_data *ct = (struct ct_data *)data;
enum clnt_stat ret;
uint32_t reterr;
- mtx_assert(&ct->ct_lock, MA_OWNED);
- while ((ct->ct_rcvstate & (RPCRCVSTATE_UPCALLNEEDED |
- RPCRCVSTATE_SOUPCALLNEEDED)) != 0) {
+printf("TLSupcall started\n");
+ mtx_lock(&ct->ct_lock);
+ ct->ct_rcvstate |= RPCRCVSTATE_UPCALLTHREAD;
+ while (!ct->ct_closed) {
if ((ct->ct_rcvstate & RPCRCVSTATE_UPCALLNEEDED) != 0) {
ct->ct_rcvstate &= ~RPCRCVSTATE_UPCALLNEEDED;
ct->ct_rcvstate |= RPCRCVSTATE_UPCALLINPROG;
@@ -1328,5 +1314,11 @@ clnt_vc_dotlsupcall(struct ct_data *ct)
SOCKBUF_UNLOCK(&ct->ct_socket->so_rcv);
mtx_lock(&ct->ct_lock);
}
+ msleep(&ct->ct_sslrefno, &ct->ct_lock, 0, "clntvcdu", hz);
}
+ ct->ct_rcvstate &= ~RPCRCVSTATE_UPCALLTHREAD;
+ wakeup(&ct->ct_sslrefno);
+ mtx_unlock(&ct->ct_lock);
+printf("TLSupcall exit\n");
+ kthread_exit();
}
Modified: projects/nfs-over-tls/sys/rpc/krpc.h
==============================================================================
--- projects/nfs-over-tls/sys/rpc/krpc.h Tue May 19 22:09:59 2020 (r361269)
+++ projects/nfs-over-tls/sys/rpc/krpc.h Wed May 20 01:25:46 2020 (r361270)
@@ -89,6 +89,7 @@ struct rc_data {
#define RPCRCVSTATE_UPCALLNEEDED 0x08 /* Upcall to rpctlscd needed. */
#define RPCRCVSTATE_UPCALLINPROG 0x10 /* Upcall to rpctlscd in progress. */
#define RPCRCVSTATE_SOUPCALLNEEDED 0x20 /* Socket upcall needed. */
+#define RPCRCVSTATE_UPCALLTHREAD 0x40 /* Upcall kthread running. */
struct ct_data {
struct mtx ct_lock;
More information about the svn-src-projects
mailing list