git: 1a878807006c - main - krpc: Display stats of TLS usage

From: Rick Macklem <rmacklem_at_FreeBSD.org>
Date: Thu, 02 Nov 2023 21:08:14 UTC
The branch main has been updated by rmacklem:

URL: https://cgit.FreeBSD.org/src/commit/?id=1a878807006cc10a5698cbca9e24a38b3412d7ed

commit 1a878807006cc10a5698cbca9e24a38b3412d7ed
Author:     Rick Macklem <rmacklem@FreeBSD.org>
AuthorDate: 2023-11-02 21:07:01 +0000
Commit:     Rick Macklem <rmacklem@FreeBSD.org>
CommitDate: 2023-11-02 21:07:01 +0000

    krpc: Display stats of TLS usage
    
    This patch adds some sysctls:
    kern.rpc.unenc.tx_msgcnt
    kern.rpc.unenc.tx_msgbytes
    kern.rpc.unenc.rx_msgcnt
    kern.rpc.unenc.rx_msgbytes
    kern.rpc.tls.tx_msgcnt
    kern.rpc.tls.tx_msgbytes
    kern.rpc.tls.rx_msgcnt
    kern.rpc.tls.rx_msgbytes
    kern.rpc.tls.handshake_success
    kern.rpc.tls.handshake_failed
    kern.rpc.tls.alerts
    which allow a NFS server sysadmin to determine how much
    NFS-over-TLS is being used.  A large number of failed
    handshakes might also indicate an NFS confirguration
    problem.
    
    This patch moves the definition of "kern.rpc" from the
    kgssapi module to the krpc module.  As such, both modules
    need to be rebuilt from sources.  Since __FreeBSD_version
    was bumped yesterday, I will not bump it again.
    
    Suggested by:   gwollman
    Discussed on:   freebsd-current
    MFC after:      1 month
---
 sys/rpc/rpcsec_gss/svc_rpcsec_gss.c |  3 +-
 sys/rpc/rpcsec_tls.h                |  4 ++
 sys/rpc/rpcsec_tls/rpctls_impl.c    | 21 ++++++++--
 sys/rpc/svc_vc.c                    | 81 +++++++++++++++++++++++++++++++++++++
 4 files changed, 104 insertions(+), 5 deletions(-)

diff --git a/sys/rpc/rpcsec_gss/svc_rpcsec_gss.c b/sys/rpc/rpcsec_gss/svc_rpcsec_gss.c
index 90aa9e0d7d4f..89526544639a 100644
--- a/sys/rpc/rpcsec_gss/svc_rpcsec_gss.c
+++ b/sys/rpc/rpcsec_gss/svc_rpcsec_gss.c
@@ -174,8 +174,7 @@ struct svc_rpc_gss_cookedcred {
 u_int svc_rpc_gss_client_max = CLIENT_MAX;
 u_int svc_rpc_gss_client_hash_size = CLIENT_HASH_SIZE;
 
-SYSCTL_NODE(_kern, OID_AUTO, rpc, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
-    "RPC");
+SYSCTL_DECL(_kern_rpc);
 SYSCTL_NODE(_kern_rpc, OID_AUTO, gss, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
     "GSS");
 
diff --git a/sys/rpc/rpcsec_tls.h b/sys/rpc/rpcsec_tls.h
index 1445c5c35f19..e3eed64863a1 100644
--- a/sys/rpc/rpcsec_tls.h
+++ b/sys/rpc/rpcsec_tls.h
@@ -86,10 +86,14 @@ bool		rpctls_getinfo(u_int *maxlen, bool rpctlscd_run,
 
 /* Macros for VIMAGE. */
 /* Just define the KRPC_VNETxxx() macros as VNETxxx() macros. */
+#define	KRPC_VNET_NAME(n)		VNET_NAME(n)
+#define	KRPC_VNET_DECLARE(t, n)		VNET_DECLARE(t, n)
 #define	KRPC_VNET_DEFINE(t, n)		VNET_DEFINE(t, n)
 #define	KRPC_VNET_DEFINE_STATIC(t, n)	VNET_DEFINE_STATIC(t, n)
 #define	KRPC_VNET(n)			VNET(n)
 
+#define	CTLFLAG_KRPC_VNET		CTLFLAG_VNET
+
 #define	KRPC_CURVNET_SET(n)		CURVNET_SET(n)
 #define	KRPC_CURVNET_SET_QUIET(n)	CURVNET_SET_QUIET(n)
 #define	KRPC_CURVNET_RESTORE()		CURVNET_RESTORE()
diff --git a/sys/rpc/rpcsec_tls/rpctls_impl.c b/sys/rpc/rpcsec_tls/rpctls_impl.c
index c0e269e55932..64111eed62c0 100644
--- a/sys/rpc/rpcsec_tls/rpctls_impl.c
+++ b/sys/rpc/rpcsec_tls/rpctls_impl.c
@@ -78,6 +78,9 @@ static CLIENT		*rpctls_connect_cl = NULL;
 static struct mtx	rpctls_server_lock;
 static struct opaque_auth rpctls_null_verf;
 
+KRPC_VNET_DECLARE(uint64_t, svc_vc_tls_handshake_success);
+KRPC_VNET_DECLARE(uint64_t, svc_vc_tls_handshake_failed);
+
 KRPC_VNET_DEFINE_STATIC(CLIENT **, rpctls_server_handle);
 KRPC_VNET_DEFINE_STATIC(struct socket *, rpctls_server_so) = NULL;
 KRPC_VNET_DEFINE_STATIC(SVCXPRT *, rpctls_server_xprt) = NULL;
@@ -748,25 +751,33 @@ _svcauth_rpcsec_tls(struct svc_req *rqst, struct rpc_msg *msg)
 	u_int maxlen;
 #endif
 	
+	KRPC_CURVNET_SET_QUIET(KRPC_TD_TO_VNET(curthread));
+	KRPC_VNET(svc_vc_tls_handshake_failed)++;
 	/* Initialize reply. */
 	rqst->rq_verf = rpctls_null_verf;
 
 	/* Check client credentials. */
 	if (rqst->rq_cred.oa_length != 0 ||
 	    msg->rm_call.cb_verf.oa_length != 0 ||
-	    msg->rm_call.cb_verf.oa_flavor != AUTH_NULL)
+	    msg->rm_call.cb_verf.oa_flavor != AUTH_NULL) {
+		KRPC_CURVNET_RESTORE();
 		return (AUTH_BADCRED);
+	}
 	
-	if (rqst->rq_proc != NULLPROC)
+	if (rqst->rq_proc != NULLPROC) {
+		KRPC_CURVNET_RESTORE();
 		return (AUTH_REJECTEDCRED);
+	}
 
 	call_stat = FALSE;
 #ifdef KERN_TLS
 	if (rpctls_getinfo(&maxlen, false, true))
 		call_stat = TRUE;
 #endif
-	if (!call_stat)
+	if (!call_stat) {
+		KRPC_CURVNET_RESTORE();
 		return (AUTH_REJECTEDCRED);
+	}
 
 	/*
 	 * Disable reception for the krpc so that the TLS handshake can
@@ -787,6 +798,7 @@ _svcauth_rpcsec_tls(struct svc_req *rqst, struct rpc_msg *msg)
 		xprt->xp_dontrcv = FALSE;
 		sx_xunlock(&xprt->xp_lock);
 		xprt_active(xprt);	/* Harmless if already active. */
+		KRPC_CURVNET_RESTORE();
 		return (AUTH_REJECTEDCRED);
 	}
 
@@ -809,9 +821,12 @@ _svcauth_rpcsec_tls(struct svc_req *rqst, struct rpc_msg *msg)
 			xprt->xp_uid = uid;
 			xprt->xp_gidp = gidp;
 		}
+		KRPC_VNET(svc_vc_tls_handshake_failed)--;
+		KRPC_VNET(svc_vc_tls_handshake_success)++;
 	}
 	sx_xunlock(&xprt->xp_lock);
 	xprt_active(xprt);		/* Harmless if already active. */
+	KRPC_CURVNET_RESTORE();
 
 	return (RPCSEC_GSS_NODISPATCH);
 }
diff --git a/sys/rpc/svc_vc.c b/sys/rpc/svc_vc.c
index e98e28de6982..69960079883b 100644
--- a/sys/rpc/svc_vc.c
+++ b/sys/rpc/svc_vc.c
@@ -74,6 +74,66 @@ static char *sccsid = "@(#)svc_tcp.c	2.2 88/08/01 4.0 RPCSRC";
 
 #include <security/mac/mac_framework.h>
 
+SYSCTL_NODE(_kern, OID_AUTO, rpc, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
+    "RPC");
+SYSCTL_NODE(_kern_rpc, OID_AUTO, tls, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
+    "TLS");
+SYSCTL_NODE(_kern_rpc, OID_AUTO, unenc, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
+    "unencrypted");
+
+KRPC_VNET_DEFINE_STATIC(uint64_t, svc_vc_rx_msgbytes) = 0;
+SYSCTL_U64(_kern_rpc_unenc, OID_AUTO, rx_msgbytes, CTLFLAG_KRPC_VNET | CTLFLAG_RW,
+    &KRPC_VNET_NAME(svc_vc_rx_msgbytes), 0, "Count of non-TLS rx bytes");
+
+KRPC_VNET_DEFINE_STATIC(uint64_t, svc_vc_rx_msgcnt) = 0;
+SYSCTL_U64(_kern_rpc_unenc, OID_AUTO, rx_msgcnt, CTLFLAG_KRPC_VNET | CTLFLAG_RW,
+    &KRPC_VNET_NAME(svc_vc_rx_msgcnt), 0, "Count of non-TLS rx messages");
+
+KRPC_VNET_DEFINE_STATIC(uint64_t, svc_vc_tx_msgbytes) = 0;
+SYSCTL_U64(_kern_rpc_unenc, OID_AUTO, tx_msgbytes, CTLFLAG_KRPC_VNET | CTLFLAG_RW,
+    &KRPC_VNET_NAME(svc_vc_tx_msgbytes), 0, "Count of non-TLS tx bytes");
+
+KRPC_VNET_DEFINE_STATIC(uint64_t, svc_vc_tx_msgcnt) = 0;
+SYSCTL_U64(_kern_rpc_unenc, OID_AUTO, tx_msgcnt, CTLFLAG_KRPC_VNET | CTLFLAG_RW,
+    &KRPC_VNET_NAME(svc_vc_tx_msgcnt), 0, "Count of non-TLS tx messages");
+
+KRPC_VNET_DEFINE_STATIC(uint64_t, svc_vc_tls_alerts) = 0;
+SYSCTL_U64(_kern_rpc_tls, OID_AUTO, alerts,
+    CTLFLAG_KRPC_VNET | CTLFLAG_RW, &KRPC_VNET_NAME(svc_vc_tls_alerts), 0,
+    "Count of TLS alert messages");
+
+KRPC_VNET_DEFINE(uint64_t, svc_vc_tls_handshake_failed) = 0;
+SYSCTL_U64(_kern_rpc_tls, OID_AUTO, handshake_failed,
+    CTLFLAG_KRPC_VNET | CTLFLAG_RW,
+    &KRPC_VNET_NAME(svc_vc_tls_handshake_failed), 0,
+    "Count of TLS failed handshakes");
+
+KRPC_VNET_DEFINE(uint64_t, svc_vc_tls_handshake_success) = 0;
+SYSCTL_U64(_kern_rpc_tls, OID_AUTO, handshake_success,
+    CTLFLAG_KRPC_VNET | CTLFLAG_RW,
+    &KRPC_VNET_NAME(svc_vc_tls_handshake_success), 0,
+    "Count of TLS successful handshakes");
+
+KRPC_VNET_DEFINE_STATIC(uint64_t, svc_vc_tls_rx_msgbytes) = 0;
+SYSCTL_U64(_kern_rpc_tls, OID_AUTO, rx_msgbytes,
+    CTLFLAG_KRPC_VNET | CTLFLAG_RW, &KRPC_VNET_NAME(svc_vc_tls_rx_msgbytes), 0,
+    "Count of TLS rx bytes");
+
+KRPC_VNET_DEFINE_STATIC(uint64_t, svc_vc_tls_rx_msgcnt) = 0;
+SYSCTL_U64(_kern_rpc_tls, OID_AUTO, rx_msgcnt,
+    CTLFLAG_KRPC_VNET | CTLFLAG_RW, &KRPC_VNET_NAME(svc_vc_tls_rx_msgcnt), 0,
+    "Count of TLS rx messages");
+
+KRPC_VNET_DEFINE_STATIC(uint64_t, svc_vc_tls_tx_msgbytes) = 0;
+SYSCTL_U64(_kern_rpc_tls, OID_AUTO, tx_msgbytes,
+    CTLFLAG_KRPC_VNET | CTLFLAG_RW, &KRPC_VNET_NAME(svc_vc_tls_tx_msgbytes), 0,
+    "Count of TLS tx bytes");
+
+KRPC_VNET_DEFINE_STATIC(uint64_t, svc_vc_tls_tx_msgcnt) = 0;
+SYSCTL_U64(_kern_rpc_tls, OID_AUTO, tx_msgcnt,
+    CTLFLAG_KRPC_VNET | CTLFLAG_RW, &KRPC_VNET_NAME(svc_vc_tls_tx_msgcnt), 0,
+    "Count of TLS tx messages");
+
 static bool_t svc_vc_rendezvous_recv(SVCXPRT *, struct rpc_msg *,
     struct sockaddr **, struct mbuf **);
 static enum xprt_stat svc_vc_rendezvous_stat(SVCXPRT *);
@@ -808,8 +868,11 @@ tryagain:
 		 * This record needs to be handled in userland
 		 * via an SSL_read() call, so do an upcall to the daemon.
 		 */
+		KRPC_CURVNET_SET(so->so_vnet);
 		if ((xprt->xp_tls & RPCTLS_FLAGS_HANDSHAKE) != 0 &&
 		    error == ENXIO) {
+			KRPC_VNET(svc_vc_tls_alerts)++;
+			KRPC_CURVNET_RESTORE();
 			/* Disable reception. */
 			xprt->xp_dontrcv = TRUE;
 			sx_xunlock(&xprt->xp_lock);
@@ -832,6 +895,7 @@ tryagain:
 		}
 
 		if (error) {
+			KRPC_CURVNET_RESTORE();
 			SOCKBUF_LOCK(&so->so_rcv);
 			if (xprt->xp_upcallset) {
 				xprt->xp_upcallset = 0;
@@ -845,6 +909,7 @@ tryagain:
 		}
 
 		if (!m) {
+			KRPC_CURVNET_RESTORE();
 			/*
 			 * EOF - the other end has closed the socket.
 			 */
@@ -870,11 +935,20 @@ tryagain:
 					m_freem(m);
 					m_free(ctrl);
 					rcvflag = MSG_DONTWAIT | MSG_TLSAPPDATA;
+					KRPC_CURVNET_RESTORE();
 					goto tryagain;
 				}
+				KRPC_VNET(svc_vc_tls_rx_msgcnt)++;
+				KRPC_VNET(svc_vc_tls_rx_msgbytes) +=
+				    1000000000 - uio.uio_resid;
 			}
 			m_free(ctrl);
+		} else {
+			KRPC_VNET(svc_vc_rx_msgcnt)++;
+			KRPC_VNET(svc_vc_rx_msgbytes) += 1000000000 -
+			    uio.uio_resid;
 		}
+		KRPC_CURVNET_RESTORE();
 
 		if (cd->mpending)
 			m_last(cd->mpending)->m_next = m;
@@ -963,6 +1037,7 @@ svc_vc_reply(SVCXPRT *xprt, struct rpc_msg *msg,
 			htonl(0x80000000 | (len - sizeof(uint32_t)));
 
 		/* For RPC-over-TLS, copy mrep to a chain of ext_pgs. */
+		KRPC_CURVNET_SET(xprt->xp_socket->so_vnet);
 		if ((xprt->xp_tls & RPCTLS_FLAGS_HANDSHAKE) != 0) {
 			/*
 			 * Copy the mbuf chain to a chain of
@@ -974,7 +1049,13 @@ svc_vc_reply(SVCXPRT *xprt, struct rpc_msg *msg,
 				maxextsiz = min(maxextsiz, maxlen);
 #endif
 			mrep = _rpc_copym_into_ext_pgs(mrep, maxextsiz);
+			KRPC_VNET(svc_vc_tls_tx_msgcnt)++;
+			KRPC_VNET(svc_vc_tls_tx_msgbytes) += len;
+		} else {
+			KRPC_VNET(svc_vc_tx_msgcnt)++;
+			KRPC_VNET(svc_vc_tx_msgbytes) += len;
 		}
+		KRPC_CURVNET_RESTORE();
 		atomic_add_32(&xprt->xp_snd_cnt, len);
 		/*
 		 * sosend consumes mreq.