svn commit: r361062 - in projects/nfs-over-tls: sys/kern sys/rpc sys/rpc/rpcsec_tls usr.sbin/rpctlscd usr.sbin/rpctlssd
Rick Macklem
rmacklem at FreeBSD.org
Thu May 14 21:46:14 UTC 2020
Author: rmacklem
Date: Thu May 14 21:46:12 2020
New Revision: 361062
URL: https://svnweb.freebsd.org/changeset/base/361062
Log:
Add code that does upcalls to the daemons when an non-application data record
is at the top of the socket receive queue.
Define a new socket flag MSG_TLSAPPDATA which tells soreceive_generic()
to return ENXIO if the first record in the receive queue is not an
application data record.
ENXIO then triggers svc_vc_recv() to do an upcall to the rpctlssd daemon,
so the daemon can do a SSL_read() for 0 bytes to handle the record.
This patch adds some similar code to the client side rpctlscd daemon,
but the kernel code to do the upcall still needs to be written.
For the client side, I do not currently know how to test it, because
it is in disconnect when the "close alert" is sent via SSL_shutdown().
Testing of this client code may need to wait until TLS1.3 support
is in the KERN_TLS.
Modified:
projects/nfs-over-tls/sys/kern/uipc_socket.c
projects/nfs-over-tls/sys/rpc/rpcsec_tls.h
projects/nfs-over-tls/sys/rpc/rpcsec_tls/rpctls_impl.c
projects/nfs-over-tls/sys/rpc/rpcsec_tls/rpctlscd.x
projects/nfs-over-tls/sys/rpc/rpcsec_tls/rpctlssd.x
projects/nfs-over-tls/sys/rpc/svc_vc.c
projects/nfs-over-tls/usr.sbin/rpctlscd/rpctlscd.c
projects/nfs-over-tls/usr.sbin/rpctlssd/rpctlssd.c
Modified: projects/nfs-over-tls/sys/kern/uipc_socket.c
==============================================================================
--- projects/nfs-over-tls/sys/kern/uipc_socket.c Thu May 14 21:36:59 2020 (r361061)
+++ projects/nfs-over-tls/sys/kern/uipc_socket.c Thu May 14 21:46:12 2020 (r361062)
@@ -2052,6 +2052,31 @@ dontblock:
if (m != NULL && m->m_type == MT_CONTROL) {
struct mbuf *cm = NULL, *cmn;
struct mbuf **cme = &cm;
+ struct cmsghdr *cmsg;
+ struct tls_get_record tgr;
+
+ /*
+ * For MSG_TLSAPPDATA, check for a non-application data
+ * record. If found, then return ENXIO without removing
+ * it from the receive queue. This allows a subsequent
+ * call without MSG_TLSAPPDATA to receive it.
+ * Note that, for TLS, there should only be a single
+ * control mbuf with the TLS_GET_RECORD message in it.
+ */
+ if (flags & MSG_TLSAPPDATA) {
+ cmsg = mtod(m, struct cmsghdr *);
+ if (cmsg->cmsg_type == TLS_GET_RECORD &&
+ cmsg->cmsg_len == CMSG_LEN(sizeof(tgr))) {
+ memcpy(&tgr, CMSG_DATA(cmsg), sizeof(tgr));
+ /* This will need to change for TLS 1.3. */
+ if (tgr.tls_type != TLS_RLTYPE_APP) {
+ SOCKBUF_UNLOCK(&so->so_rcv);
+printf("fnd non-app rec=%d\n", tgr.tls_type);
+ error = ENXIO;
+ goto release;
+ }
+ }
+ }
do {
if (flags & MSG_PEEK) {
Modified: projects/nfs-over-tls/sys/rpc/rpcsec_tls.h
==============================================================================
--- projects/nfs-over-tls/sys/rpc/rpcsec_tls.h Thu May 14 21:36:59 2020 (r361061)
+++ projects/nfs-over-tls/sys/rpc/rpcsec_tls.h Thu May 14 21:46:12 2020 (r361062)
@@ -47,6 +47,10 @@
/* Functions that perform upcalls to the rpctlsd daemon. */
enum clnt_stat rpctls_connect(CLIENT *newclient, struct socket *so,
uint64_t *sslp);
+enum clnt_stat rpctls_cl_handlerecord(uint64_t sec, uint64_t usec,
+ uint64_t ssl);
+enum clnt_stat rpctls_srv_handlerecord(uint64_t sec, uint64_t usec,
+ uint64_t ssl);
enum clnt_stat rpctls_cl_disconnect(uint64_t sec, uint64_t usec, uint64_t ssl);
enum clnt_stat rpctls_srv_disconnect(uint64_t sec, uint64_t usec,
uint64_t ssl);
Modified: projects/nfs-over-tls/sys/rpc/rpcsec_tls/rpctls_impl.c
==============================================================================
--- projects/nfs-over-tls/sys/rpc/rpcsec_tls/rpctls_impl.c Thu May 14 21:36:59 2020 (r361061)
+++ projects/nfs-over-tls/sys/rpc/rpcsec_tls/rpctls_impl.c Thu May 14 21:46:12 2020 (r361062)
@@ -385,6 +385,53 @@ printf("aft wakeup\n");
return (stat);
}
+/* Do an upcall to handle an non-application data record using TLS. */
+enum clnt_stat
+rpctls_cl_handlerecord(uint64_t sec, uint64_t usec, uint64_t ssl)
+{
+ struct rpctlscd_handlerecord_arg arg;
+ enum clnt_stat stat;
+ CLIENT *cl;
+
+printf("In rpctls_cl_handlerecord\n");
+ cl = rpctls_connect_client();
+printf("handlerecord_client=%p\n", cl);
+ if (cl == NULL)
+ return (RPC_FAILED);
+
+ /* Do the handlerecord upcall. */
+ arg.sec = sec;
+ arg.usec = usec;
+ arg.ssl = ssl;
+ stat = rpctlscd_handlerecord_1(&arg, NULL, cl);
+printf("aft handlerecord upcall=%d\n", stat);
+ CLNT_RELEASE(cl);
+ return (stat);
+}
+
+enum clnt_stat
+rpctls_srv_handlerecord(uint64_t sec, uint64_t usec, uint64_t ssl)
+{
+ struct rpctlssd_handlerecord_arg arg;
+ enum clnt_stat stat;
+ CLIENT *cl;
+
+printf("In rpctls_srv_handlerecord\n");
+ cl = rpctls_server_client();
+printf("srv handlerecord_client=%p\n", cl);
+ if (cl == NULL)
+ return (RPC_FAILED);
+
+ /* Do the handlerecord upcall. */
+ arg.sec = sec;
+ arg.usec = usec;
+ arg.ssl = ssl;
+ stat = rpctlssd_handlerecord_1(&arg, NULL, cl);
+printf("aft srv handlerecord upcall=%d\n", stat);
+ CLNT_RELEASE(cl);
+ return (stat);
+}
+
/* Do an upcall to shut down a socket using TLS. */
enum clnt_stat
rpctls_cl_disconnect(uint64_t sec, uint64_t usec, uint64_t ssl)
Modified: projects/nfs-over-tls/sys/rpc/rpcsec_tls/rpctlscd.x
==============================================================================
--- projects/nfs-over-tls/sys/rpc/rpcsec_tls/rpctlscd.x Thu May 14 21:36:59 2020 (r361061)
+++ projects/nfs-over-tls/sys/rpc/rpcsec_tls/rpctlscd.x Thu May 14 21:46:12 2020 (r361062)
@@ -35,6 +35,12 @@ struct rpctlscd_connect_res {
uint64_t ssl;
};
+struct rpctlscd_handlerecord_arg {
+ uint64_t sec;
+ uint64_t usec;
+ uint64_t ssl;
+};
+
struct rpctlscd_disconnect_arg {
uint64_t sec;
uint64_t usec;
@@ -48,6 +54,8 @@ program RPCTLSCD {
rpctlscd_connect_res
RPCTLSCD_CONNECT(void) = 1;
- void RPCTLSCD_DISCONNECT(rpctlscd_disconnect_arg) = 2;
+ void RPCTLSCD_HANDLERECORD(rpctlscd_handlerecord_arg) = 2;
+
+ void RPCTLSCD_DISCONNECT(rpctlscd_disconnect_arg) = 3;
} = 1;
} = 0x40677374;
Modified: projects/nfs-over-tls/sys/rpc/rpcsec_tls/rpctlssd.x
==============================================================================
--- projects/nfs-over-tls/sys/rpc/rpcsec_tls/rpctlssd.x Thu May 14 21:36:59 2020 (r361061)
+++ projects/nfs-over-tls/sys/rpc/rpcsec_tls/rpctlssd.x Thu May 14 21:46:12 2020 (r361062)
@@ -38,6 +38,12 @@ struct rpctlssd_connect_res {
uint32_t gid<>;
};
+struct rpctlssd_handlerecord_arg {
+ uint64_t sec;
+ uint64_t usec;
+ uint64_t ssl;
+};
+
struct rpctlssd_disconnect_arg {
uint64_t sec;
uint64_t usec;
@@ -51,6 +57,8 @@ program RPCTLSSD {
rpctlssd_connect_res
RPCTLSSD_CONNECT(void) = 1;
- void RPCTLSSD_DISCONNECT(rpctlssd_disconnect_arg) = 2;
+ void RPCTLSSD_HANDLERECORD(rpctlssd_handlerecord_arg) = 2;
+
+ void RPCTLSSD_DISCONNECT(rpctlssd_disconnect_arg) = 3;
} = 1;
} = 0x40677375;
Modified: projects/nfs-over-tls/sys/rpc/svc_vc.c
==============================================================================
--- projects/nfs-over-tls/sys/rpc/svc_vc.c Thu May 14 21:36:59 2020 (r361061)
+++ projects/nfs-over-tls/sys/rpc/svc_vc.c Thu May 14 21:46:12 2020 (r361062)
@@ -674,6 +674,7 @@ svc_vc_recv(SVCXPRT *xprt, struct rpc_msg *msg,
uint32_t xid_plus_direction[2];
struct cmsghdr *cmsg;
struct tls_get_record tgr;
+ enum clnt_stat ret;
/*
* Serialise access to the socket and our own record parsing
@@ -748,6 +749,9 @@ svc_vc_recv(SVCXPRT *xprt, struct rpc_msg *msg,
* If receiving is disabled so that a TLS handshake can be
* done by the rpctlssd daemon, return FALSE here.
*/
+ rcvflag = MSG_DONTWAIT;
+ if ((xprt->xp_tls & RPCTLS_FLAGS_HANDSHAKE) != 0)
+ rcvflag |= MSG_TLSAPPDATA;
tryagain:
if (xprt->xp_dontrcv) {
sx_xunlock(&xprt->xp_lock);
@@ -765,7 +769,6 @@ tryagain:
uio.uio_resid = 1000000000;
uio.uio_td = curthread;
ctrl = m = NULL;
- rcvflag = MSG_DONTWAIT;
error = soreceive(so, NULL, &uio, &m, &ctrl, &rcvflag);
if (error == EWOULDBLOCK) {
@@ -784,6 +787,36 @@ tryagain:
return (FALSE);
}
+ /*
+ * A return of ENXIO indicates that there is a
+ * non-application data record at the head of the
+ * socket's receive queue, for TLS connections.
+ * This record needs to be handled in userland
+ * via an SSL_read() call, so do an upcall to the daemon.
+ */
+ if ((xprt->xp_tls & RPCTLS_FLAGS_HANDSHAKE) != 0 &&
+ error == ENXIO) {
+ /* Disable reception. */
+ xprt->xp_dontrcv = TRUE;
+ sx_xunlock(&xprt->xp_lock);
+printf("Call rpctls_srv_handlerecord\n");
+ ret = rpctls_srv_handlerecord(xprt->xp_sslsec,
+ xprt->xp_sslusec, xprt->xp_sslrefno);
+ sx_xlock(&xprt->xp_lock);
+ xprt->xp_dontrcv = FALSE;
+ if (ret != RPC_SUCCESS) {
+ /*
+ * All we can do is soreceive() it and
+ * then toss it.
+ */
+ rcvflag = MSG_DONTWAIT;
+ goto tryagain;
+ }
+ sx_xunlock(&xprt->xp_lock);
+ xprt_active(xprt); /* Harmless if already active. */
+ return (FALSE);
+ }
+
if (error) {
SOCKBUF_LOCK(&so->so_rcv);
if (xprt->xp_upcallset) {
@@ -815,15 +848,16 @@ if (ctrl->m_next != NULL) printf("EEK! svc list of con
cmsg->cmsg_len == CMSG_LEN(sizeof(tgr))) {
memcpy(&tgr, CMSG_DATA(cmsg), sizeof(tgr));
/*
- * For now, just toss non-application
- * data records.
- * In the future, there may need to be
- * an upcall done to the daemon.
+ * This should have been handled by
+ * the rpctls_svc_handlerecord()
+ * upcall. If not, all we can do is
+ * toss it away.
*/
if (tgr.tls_type != TLS_RLTYPE_APP) {
printf("Got weird type=%d\n", tgr.tls_type);
m_freem(m);
m_free(ctrl);
+ rcvflag = MSG_DONTWAIT | MSG_TLSAPPDATA;
goto tryagain;
}
}
Modified: projects/nfs-over-tls/usr.sbin/rpctlscd/rpctlscd.c
==============================================================================
--- projects/nfs-over-tls/usr.sbin/rpctlscd/rpctlscd.c Thu May 14 21:36:59 2020 (r361061)
+++ projects/nfs-over-tls/usr.sbin/rpctlscd/rpctlscd.c Thu May 14 21:46:12 2020 (r361062)
@@ -371,10 +371,47 @@ rpctlscd_verbose_out("rpctlsd_connect s=%d\n", s);
}
bool_t
+rpctlscd_handlerecord_1_svc(struct rpctlscd_handlerecord_arg *argp,
+ void *result, struct svc_req *rqstp)
+{
+ struct ssl_entry *slp;
+ int ret;
+ char junk;
+
+ slp = NULL;
+ if (argp->sec == rpctls_ssl_sec && argp->usec ==
+ rpctls_ssl_usec) {
+ LIST_FOREACH(slp, &rpctls_ssllist, next) {
+ if (slp->refno == argp->ssl)
+ break;
+ }
+ }
+
+ if (slp != NULL) {
+ rpctlscd_verbose_out("rpctlscd_handlerecord fd=%d\n",
+ slp->s);
+ /*
+ * An SSL_read() of 0 bytes should fail, but it should
+ * handle the non-application data record before doing so.
+ */
+ ret = SSL_read(slp->ssl, &junk, 0);
+ if (ret > 0) {
+ if (rpctls_debug_level == 0)
+ syslog(LOG_ERR, "SSL_read returned %d", ret);
+ else
+ fprintf(stderr, "SSL_read returned %d\n", ret);
+ }
+ } else
+ return (FALSE);
+ return (TRUE);
+}
+
+bool_t
rpctlscd_disconnect_1_svc(struct rpctlscd_disconnect_arg *argp,
void *result, struct svc_req *rqstp)
{
struct ssl_entry *slp;
+ int ret;
slp = NULL;
if (argp->sec == rpctls_ssl_sec && argp->usec ==
@@ -389,6 +426,13 @@ rpctlscd_disconnect_1_svc(struct rpctlscd_disconnect_a
rpctlscd_verbose_out("rpctlscd_disconnect: fd=%d closed\n",
slp->s);
LIST_REMOVE(slp, next);
+ SSL_shutdown(slp->ssl);
+ /* Check to see if the peer has sent a close alert. */
+ ret = SSL_get_shutdown(slp->ssl);
+rpctlscd_verbose_out("get_shutdown=%d\n", ret);
+ if ((ret & (SSL_SENT_SHUTDOWN | SSL_RECEIVED_SHUTDOWN)) ==
+ SSL_SENT_SHUTDOWN)
+ SSL_shutdown(slp->ssl);
SSL_free(slp->ssl);
/*
* For RPC-over-TLS, this upcall is expected
Modified: projects/nfs-over-tls/usr.sbin/rpctlssd/rpctlssd.c
==============================================================================
--- projects/nfs-over-tls/usr.sbin/rpctlssd/rpctlssd.c Thu May 14 21:36:59 2020 (r361061)
+++ projects/nfs-over-tls/usr.sbin/rpctlssd/rpctlssd.c Thu May 14 21:46:12 2020 (r361062)
@@ -419,6 +419,49 @@ rpctlssd_verbose_out("rpctlsd_connect_svc s=%d\n", s);
}
bool_t
+rpctlssd_handlerecord_1_svc(struct rpctlssd_handlerecord_arg *argp,
+ void *result, struct svc_req *rqstp)
+{
+ struct ssl_entry *slp;
+ int ret;
+ char junk;
+
+ slp = NULL;
+ if (argp->sec == rpctls_ssl_sec && argp->usec ==
+ rpctls_ssl_usec) {
+ LIST_FOREACH(slp, &rpctls_ssllist, next) {
+ if (slp->refno == argp->ssl)
+ break;
+ }
+ }
+
+ if (slp != NULL) {
+ rpctlssd_verbose_out("rpctlssd_handlerecord fd=%d\n",
+ slp->s);
+ /*
+ * An SSL_read() of 0 bytes should fail, but it should
+ * handle the non-application data record before doing so.
+ */
+ ret = SSL_read(slp->ssl, &junk, 0);
+ if (ret <= 0) {
+ /* Check to see if this was a close alert. */
+ ret = SSL_get_shutdown(slp->ssl);
+rpctlssd_verbose_out("get_shutdown=%d\n", ret);
+ if ((ret & (SSL_SENT_SHUTDOWN |
+ SSL_RECEIVED_SHUTDOWN)) == SSL_RECEIVED_SHUTDOWN)
+ SSL_shutdown(slp->ssl);
+ } else {
+ if (rpctls_debug_level == 0)
+ syslog(LOG_ERR, "SSL_read returned %d", ret);
+ else
+ fprintf(stderr, "SSL_read returned %d\n", ret);
+ }
+ } else
+ return (FALSE);
+ return (TRUE);
+}
+
+bool_t
rpctlssd_disconnect_1_svc(struct rpctlssd_disconnect_arg *argp,
void *result, struct svc_req *rqstp)
{
@@ -442,6 +485,7 @@ rpctlssd_disconnect_1_svc(struct rpctlssd_disconnect_a
* For RPC-over-TLS, this upcall is expected
* to close off the socket.
*/
+ shutdown(slp->s, SHUT_WR);
close(slp->s);
free(slp);
} else
@@ -679,7 +723,7 @@ rpctlssd_verbose_out("%s\n", cp2);
}
if (ret == 0) {
if (rpctls_debug_level == 0)
- syslog(LOG_ERR, "ktls not working\n");
+ syslog(LOG_ERR, "ktls not working");
else
fprintf(stderr, "ktls not working\n");
/*
More information about the svn-src-projects
mailing list