git: c67f3b8b78e5 - main - socket: Move sockbuf mutexes into the owning socket

[Mark Johnston]

The branch main has been updated by markj:

URL: https://cgit.FreeBSD.org/src/commit/?id=c67f3b8b78e50c6df7c057d6cf108e4d6b4312d0

commit c67f3b8b78e50c6df7c057d6cf108e4d6b4312d0
Author:     Mark Johnston <markj at FreeBSD.org>
AuthorDate: 2021-09-07 18:49:40 +0000
Commit:     Mark Johnston <markj at FreeBSD.org>
CommitDate: 2021-09-07 19:09:02 +0000

    socket: Move sockbuf mutexes into the owning socket
    
    This is necessary to provide proper interlocking with listen(2), which
    destroys the socket buffers.  Otherwise, code must lock the socket
    itself and check SOLISTENING(so), but most I/O paths do not otherwise
    need to acquire the socket lock, so the extra overhead needed to check a
    rare error case is undesirable.
    
    listen(2) calls are relatively rare.  Thus, the strategy is to have it
    acquire all socket buffer locks when transitioning to a listening
    socket.  To do this safely, these locks must be stable, and not
    destroyed during listen(2) as they are today.  So, move them out of the
    sockbuf and into the owning socket.  For the sockbuf mutexes, keep a
    pointer to the mutex in the sockbuf itself, for now.  This can be
    removed by replacing SOCKBUF_LOCK() etc. with macros which operate on
    the socket itself, as was done for the sockbuf I/O locks.
    
    Reviewed by:    tuexen, gallatin
    MFC after:      1 month
    Sponsored by:   The FreeBSD Foundation
    Differential Revision:  https://reviews.freebsd.org/D31658
---
 sys/sys/sockbuf.h   |  7 +++----
 sys/sys/socketvar.h | 19 +++++++++++++++----
 2 files changed, 18 insertions(+), 8 deletions(-)

diff --git a/sys/sys/sockbuf.h b/sys/sys/sockbuf.h
index 3d74a6d953f7..3b345870bd5f 100644
--- a/sys/sys/sockbuf.h
+++ b/sys/sys/sockbuf.h
@@ -79,9 +79,8 @@ struct selinfo;
  * Locking key to struct sockbuf:
  * (a) locked by SOCKBUF_LOCK().
  */
-struct	sockbuf {
-	struct	mtx sb_mtx;		/* sockbuf lock */
-	struct	sx sb_sx;		/* prevent I/O interlacing */
+struct sockbuf {
+	struct	mtx *sb_mtx;		/* sockbuf lock */
 	struct	selinfo *sb_sel;	/* process selecting read/write */
 	short	sb_state;	/* (a) socket state on sockbuf */
 #define	sb_startzero	sb_flags
@@ -122,7 +121,7 @@ struct	sockbuf {
  * Per-socket buffer mutex used to protect most fields in the socket
  * buffer.
  */
-#define	SOCKBUF_MTX(_sb)		(&(_sb)->sb_mtx)
+#define	SOCKBUF_MTX(_sb)		((_sb)->sb_mtx)
 #define	SOCKBUF_LOCK_INIT(_sb, _name) \
 	mtx_init(SOCKBUF_MTX(_sb), _name, NULL, MTX_DEF)
 #define	SOCKBUF_LOCK_DESTROY(_sb)	mtx_destroy(SOCKBUF_MTX(_sb))
diff --git a/sys/sys/socketvar.h b/sys/sys/socketvar.h
index a12e60e1b5c6..69e182dfa9a5 100644
--- a/sys/sys/socketvar.h
+++ b/sys/sys/socketvar.h
@@ -121,6 +121,17 @@ struct socket {
 
 	int so_ts_clock;	/* type of the clock used for timestamps */
 	uint32_t so_max_pacing_rate;	/* (f) TX rate limit in bytes/s */
+
+	/*
+	 * Mutexes to prevent interleaving of socket I/O.  These have to be
+	 * outside of the socket buffers in order to interlock with listen(2).
+	 */
+	struct sx so_snd_sx __aligned(CACHE_LINE_SIZE);
+	struct mtx so_snd_mtx;
+
+	struct sx so_rcv_sx __aligned(CACHE_LINE_SIZE);
+	struct mtx so_rcv_mtx;
+
 	union {
 		/* Regular (data flow) socket. */
 		struct {
@@ -256,13 +267,13 @@ struct socket {
 #define	SBL_VALID	(SBL_WAIT | SBL_NOINTR)
 
 #define	SOCK_IO_SEND_LOCK(so, flags)					\
-	soiolock((so), &(so)->so_snd.sb_sx, (flags))
+	soiolock((so), &(so)->so_snd_sx, (flags))
 #define	SOCK_IO_SEND_UNLOCK(so)						\
-	soiounlock(&(so)->so_snd.sb_sx)
+	soiounlock(&(so)->so_snd_sx)
 #define	SOCK_IO_RECV_LOCK(so, flags)					\
-	soiolock((so), &(so)->so_rcv.sb_sx, (flags))
+	soiolock((so), &(so)->so_rcv_sx, (flags))
 #define	SOCK_IO_RECV_UNLOCK(so)						\
-	soiounlock(&(so)->so_rcv.sb_sx)
+	soiounlock(&(so)->so_rcv_sx)
 
 /*
  * Do we need to notify the other side when I/O is possible?