git: 8c511e47f421 - main - nfs: Cache reply pages in an NFS-owned UMA zone

From: Rick Macklem <rmacklem_at_FreeBSD.org>
Date: Sat, 21 Nov 2026 23:25:17 UTC
The branch main has been updated by rmacklem:

URL: https://cgit.FreeBSD.org/src/commit/?id=8c511e47f42155c534ba1ee08771d02f50e42662

commit 8c511e47f42155c534ba1ee08771d02f50e42662
Author:     Vinicius Ferrao <ferrao@versatushpc.com.br>
AuthorDate: 2026-11-21 23:22:07 +0000
Commit:     Rick Macklem <rmacklem@FreeBSD.org>
CommitDate: 2026-11-21 23:22:07 +0000

    nfs: Cache reply pages in an NFS-owned UMA zone
    
    Allocate anonymous reply pages through a dedicated UMA cache and
    return them in batches when the last M_EXTPG reference is released.
    This reduced page-cache bucket shortages in the tested
    concurrent-read workloads.  The pool uses the existing UMA and
    mbuf interfaces.
    
    Once nfsd enables the pool, nfscommon refuses unload so delayed
    callbacks remain valid. The usual nfsd/nfslockd dependency chain
    already retains nfscommon; the explicit guard also enforces this
    requirement directly.  Loading nfscommon alone does not activate
    that guard.
    
    Mark Johnston suggested using a dedicated zone in the D61234 discussion.
    That feedback led to this NFS implementation instead of changing UMA.
    Gleb Smirnoff provided feedback on the original allocator proposal.
    
    Reviewed by:    rmacklem
    Discussed with: markj, glebius
    MFC after:      1 month
    Fixes:  7144a1d58c5c ("nfsd: Add glue for the nfsrdma.ko module")
---
 sys/fs/nfs/nfs_commonport.c     | 147 +++++++++++++++++++++++++++++++++++++++-
 sys/fs/nfs/nfs_commonsubs.c     |   7 +-
 sys/fs/nfs/nfs_var.h            |   4 ++
 sys/fs/nfsserver/nfs_nfsdport.c |  17 +++--
 sys/fs/nfsserver/nfs_nfsdsubs.c |   5 +-
 sys/modules/nfscommon/Makefile  |   3 +-
 6 files changed, 165 insertions(+), 18 deletions(-)

diff --git a/sys/fs/nfs/nfs_commonport.c b/sys/fs/nfs/nfs_commonport.c
index 03c6688b1406..30ab162af07c 100644
--- a/sys/fs/nfs/nfs_commonport.c
+++ b/sys/fs/nfs/nfs_commonport.c
@@ -54,6 +54,9 @@
 #include <vm/vm_extern.h>
 #include <vm/uma.h>
 
+static uma_zone_t nfs_extpg_zone;
+static u_int nfs_extpg_enabled;
+
 extern int nfscl_ticks;
 extern void (*nfsd_call_recall)(struct vnode *, int, struct ucred *,
     struct thread *);
@@ -918,6 +921,139 @@ VNET_SYSUNINIT(nfs_cleanup, SI_SUB_VNET_DONE, SI_ORDER_FIRST,
 
 extern int (*nfsd_call_nfscommon)(struct thread *, struct nfssvc_args *);
 
+static int
+nfs_extpg_import(void *arg __unused, void **store, int count, int domain,
+    int how)
+{
+	vm_page_t pg;
+	int i, req;
+
+	req = VM_ALLOC_WIRED | VM_ALLOC_NODUMP |
+	    ((how & M_WAITOK) != 0 ? VM_ALLOC_WAITOK : VM_ALLOC_NOWAIT);
+	for (i = 0; i < count; i++) {
+		if (domain == UMA_ANYDOMAIN)
+			pg = vm_page_alloc_noobj(req);
+		else
+			pg = vm_page_alloc_noobj_domain(domain, req);
+		if (pg == NULL)
+			break;
+		store[i] = pg;
+		req &= ~VM_ALLOC_WAITOK;
+	}
+	return (i);
+}
+
+static void
+nfs_extpg_release(void *arg __unused, void **store, int count)
+{
+	vm_page_t pg;
+	int i;
+
+	for (i = 0; i < count; i++) {
+		pg = store[i];
+		vm_page_unwire_noq(pg);
+		vm_page_free(pg);
+	}
+}
+
+static void
+nfs_free_extpg_cached(struct mbuf *m)
+{
+	int i;
+
+	M_ASSERTEXTPG(m);
+	for (i = 0; i < m->m_epg_npgs; i++)
+		uma_zfree(nfs_extpg_zone,
+		    PHYS_TO_VM_PAGE(m->m_epg_pa[i]));
+}
+
+bool
+nfs_alloc_extpg_page(struct mbuf *m, int how)
+{
+	vm_page_t pg;
+	vm_paddr_t pa;
+
+	M_ASSERTEXTPG(m);
+	KASSERT((m->m_ext.ext_flags & EXT_FLAG_EMBREF) != 0 &&
+	    m_extrefcnt(m) == 1, ("shared NFS external pages"));
+	KASSERT(m->m_epg_npgs < MBUF_PEXT_MAX_PGS,
+	    ("NFS external page array full"));
+	if (m->m_ext.ext_free == nfs_free_extpg_cached) {
+		pg = uma_zalloc(nfs_extpg_zone, how & ~M_ZERO);
+		if (pg == NULL)
+			return (false);
+		if ((how & M_ZERO) != 0)
+			pmap_zero_page(pg);
+		pa = VM_PAGE_TO_PHYS(pg);
+	} else {
+		KASSERT(m->m_ext.ext_free == mb_free_mext_pgs,
+		    ("unexpected NFS external page owner"));
+		pg = vm_page_alloc_noobj(VM_ALLOC_NODUMP | VM_ALLOC_WIRED |
+		    ((how & M_WAITOK) != 0 ? VM_ALLOC_WAITOK :
+		    VM_ALLOC_NOWAIT));
+		if (pg == NULL)
+			return (false);
+		pa = VM_PAGE_TO_PHYS(pg);
+	}
+	m->m_epg_pa[m->m_epg_npgs++] = pa;
+	return (true);
+}
+
+void
+nfs_free_extpg_page(struct mbuf *m, int index)
+{
+	vm_page_t pg;
+
+	M_ASSERTEXTPG(m);
+	KASSERT((m->m_ext.ext_flags & EXT_FLAG_EMBREF) != 0 &&
+	    m_extrefcnt(m) == 1, ("shared NFS external pages"));
+	KASSERT(index >= 0 && index < m->m_epg_npgs,
+	    ("NFS external page index out of range"));
+	if (m->m_ext.ext_free == nfs_free_extpg_cached) {
+		uma_zfree(nfs_extpg_zone,
+		    PHYS_TO_VM_PAGE(m->m_epg_pa[index]));
+	} else {
+		KASSERT(m->m_ext.ext_free == mb_free_mext_pgs,
+		    ("unexpected NFS external page owner"));
+		pg = PHYS_TO_VM_PAGE(m->m_epg_pa[index]);
+		vm_page_unwire_noq(pg);
+		vm_page_free(pg);
+	}
+}
+
+void
+nfs_enable_extpg_cache(void)
+{
+
+	/* Enabling the pool also prevents nfscommon unload. */
+	atomic_store_rel_int(&nfs_extpg_enabled, 1);
+}
+
+struct mbuf *
+nfs_alloc_extpg(int len, int how)
+{
+	struct mbuf *m;
+	int i, npgs;
+
+	if (atomic_load_acq_int(&nfs_extpg_enabled) == 0 ||
+	    nfs_extpg_zone == NULL)
+		return (mb_alloc_ext_plus_pages(len, how));
+	npgs = howmany(len, PAGE_SIZE);
+	KASSERT(npgs > 0 && npgs <= MBUF_PEXT_MAX_PGS,
+	    ("invalid NFS external page count %d", npgs));
+	m = mb_alloc_ext_pgs(how, nfs_free_extpg_cached, 0);
+	if (m == NULL)
+		return (NULL);
+	m->m_epg_flags |= EPG_FLAG_ANON;
+	for (i = 0; i < npgs; i++) {
+		if (!nfs_alloc_extpg_page(m, how)) {
+			m_free(m);
+			return (NULL);
+		}
+	}
+	return (m);
+}
+
 /*
  * Called once to initialize data structures...
  */
@@ -931,6 +1067,10 @@ nfscommon_modevent(module_t mod, int type, void *data)
 	case MOD_LOAD:
 		if (loaded)
 			goto out;
+		nfs_extpg_zone = uma_zcache_create("nfs reply pages", PAGE_SIZE,
+		    NULL, NULL, NULL, NULL, nfs_extpg_import,
+		    nfs_extpg_release, NULL, 0);
+		uma_zone_set_maxcache(nfs_extpg_zone, 16384);
 		newnfs_portinit();
 		mtx_init(&nfs_nameid_mutex, "nfs_nameid_mutex", NULL, MTX_DEF);
 		mtx_init(&nfs_sockl_mutex, "nfs_sockl_mutex", NULL, MTX_DEF);
@@ -944,13 +1084,18 @@ nfscommon_modevent(module_t mod, int type, void *data)
 		break;
 
 	case MOD_UNLOAD:
-		if (newnfs_numnfsd != 0 ||
+		if (atomic_load_acq_int(&nfs_extpg_enabled) != 0 ||
+		    newnfs_numnfsd != 0 ||
 		    VNET(nfsrv_nfsuserd) != NOTRUNNING ||
 		    nfs_numnfscbd != 0) {
 			error = EBUSY;
 			break;
 		}
 
+		if (nfs_extpg_zone != NULL) {
+			uma_zdestroy(nfs_extpg_zone);
+			nfs_extpg_zone = NULL;
+		}
 		nfsd_call_nfscommon = NULL;
 		/* and get rid of the mutexes */
 		mtx_destroy(&nfs_nameid_mutex);
diff --git a/sys/fs/nfs/nfs_commonsubs.c b/sys/fs/nfs/nfs_commonsubs.c
index a11b55b11c43..4ebc2a937a6c 100644
--- a/sys/fs/nfs/nfs_commonsubs.c
+++ b/sys/fs/nfs/nfs_commonsubs.c
@@ -5557,18 +5557,15 @@ struct mbuf *
 nfsm_add_ext_pgs(struct mbuf *m, int maxextsiz, int *bextpg)
 {
 	struct mbuf *mp;
-	vm_page_t pg;
 
 	if ((m->m_epg_npgs + 1) * PAGE_SIZE > maxextsiz) {
 		mp = mb_alloc_ext_plus_pages(PAGE_SIZE, M_WAITOK);
 		*bextpg = 0;
 		m->m_next = mp;
 	} else {
-		pg = vm_page_alloc_noobj(VM_ALLOC_WAITOK | VM_ALLOC_NODUMP |
-		    VM_ALLOC_WIRED);
-		m->m_epg_pa[m->m_epg_npgs] = VM_PAGE_TO_PHYS(pg);
 		*bextpg = m->m_epg_npgs;
-		m->m_epg_npgs++;
+		if (!nfs_alloc_extpg_page(m, M_WAITOK))
+			panic("nfsm_add_ext_pgs: page allocation failed");
 		m->m_epg_last_len = 0;
 		mp = m;
 	}
diff --git a/sys/fs/nfs/nfs_var.h b/sys/fs/nfs/nfs_var.h
index 488ddbd1c582..35c92b63bc4a 100644
--- a/sys/fs/nfs/nfs_var.h
+++ b/sys/fs/nfs/nfs_var.h
@@ -367,6 +367,10 @@ void nfsv4_freeslot(struct nfsclsession *, int, bool);
 struct ucred *nfsrv_getgrpscred(struct ucred *);
 struct nfsdevice *nfsv4_findmirror(struct nfsmount *);
 void nfsm_set(struct nfsrv_descript *, u_int);
+void nfs_enable_extpg_cache(void);
+struct mbuf *nfs_alloc_extpg(int, int);
+bool nfs_alloc_extpg_page(struct mbuf *, int);
+void nfs_free_extpg_page(struct mbuf *, int);
 struct mbuf *nfsm_add_ext_pgs(struct mbuf *, int, int *);
 int nfsrpc_destroysession(struct nfsmount *, struct nfsclsession *,
     struct ucred *, NFSPROC_T *);
diff --git a/sys/fs/nfsserver/nfs_nfsdport.c b/sys/fs/nfsserver/nfs_nfsdport.c
index fbbdef03f440..bda047d2d226 100644
--- a/sys/fs/nfsserver/nfs_nfsdport.c
+++ b/sys/fs/nfsserver/nfs_nfsdport.c
@@ -186,6 +186,9 @@ int nfs_pnfsio(task_fn_t *, void *);
 
 SYSCTL_NODE(_vfs, OID_AUTO, nfsd, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
     "NFS server");
+static bool nfsrv_extpg_cache = true;
+SYSCTL_BOOL(_vfs_nfsd, OID_AUTO, extpg_cache, CTLFLAG_RWTUN,
+    &nfsrv_extpg_cache, true, "Cache NFS reply pages");
 SYSCTL_INT(_vfs_nfsd, OID_AUTO, mirrormnt, CTLFLAG_RW,
     &nfsrv_enable_crossmntpt, 0, "Enable nfsd to cross mount points");
 SYSCTL_INT(_vfs_nfsd, OID_AUTO, commit_blks, CTLFLAG_RW, &nfs_commit_blks,
@@ -1100,7 +1103,10 @@ nfsrv_createiovec_extpgs(int len, int maxextsiz, struct mbuf **mpp,
 	i = 0;
 	while (left > 0) {
 		siz = min(left, maxextsiz);
-		m = mb_alloc_ext_plus_pages(siz, M_WAITOK);
+		if (nfsrv_extpg_cache)
+			m = nfs_alloc_extpg(siz, M_WAITOK);
+		else
+			m = mb_alloc_ext_plus_pages(siz, M_WAITOK);
 		left -= siz;
 		i += m->m_epg_npgs;
 		if (m3 != NULL)
@@ -8477,7 +8483,6 @@ void
 nfsm_trimtrailing(struct nfsrv_descript *nd, struct mbuf *mb, char *bpos,
     int bextpg, int bextpgsiz)
 {
-	vm_page_t pg;
 	int fullpgsiz, i;
 
 	if (mb->m_next != NULL) {
@@ -8492,11 +8497,8 @@ nfsm_trimtrailing(struct nfsrv_descript *nd, struct mbuf *mb, char *bpos,
 		    ("nfsm_trimtrailing: bextpgsiz bad!"));
 
 		/* First, get rid of any pages after this position. */
-		for (i = mb->m_epg_npgs - 1; i > bextpg; i--) {
-			pg = PHYS_TO_VM_PAGE(mb->m_epg_pa[i]);
-			vm_page_unwire_noq(pg);
-			vm_page_free(pg);
-		}
+		for (i = mb->m_epg_npgs - 1; i > bextpg; i--)
+			nfs_free_extpg_page(mb, i);
 		mb->m_epg_npgs = bextpg + 1;
 		if (bextpg == 0)
 			fullpgsiz = PAGE_SIZE - mb->m_epg_1st_off;
@@ -8701,6 +8703,7 @@ nfsd_modevent(module_t mod, int type, void *data)
 		vn_deleg_ops.vndeleg_disable = nfsd_disabledelegation;
 #endif
 		nfsd_call_nfsd = nfssvc_nfsd;
+		nfs_enable_extpg_cache();
 		loaded = 1;
 		break;
 
diff --git a/sys/fs/nfsserver/nfs_nfsdsubs.c b/sys/fs/nfsserver/nfs_nfsdsubs.c
index f7fd81ef4e1e..be7da2a1fb0a 100644
--- a/sys/fs/nfsserver/nfs_nfsdsubs.c
+++ b/sys/fs/nfsserver/nfs_nfsdsubs.c
@@ -1284,7 +1284,6 @@ struct mbuf *
 nfsrv_adj(struct mbuf *mp, int len, int nul)
 {
 	struct mbuf *m, *m2;
-	vm_page_t pg;
 	int i, lastlen, pgno, plen, tlen, trim;
 	uint16_t off;
 	char *cp;
@@ -1334,9 +1333,7 @@ nfsrv_adj(struct mbuf *mp, int len, int nul)
 				KASSERT(pgno > 0,
 				    ("nfsrv_adj: freeing page 0"));
 				/* Free page. */
-				pg = PHYS_TO_VM_PAGE(m->m_epg_pa[pgno]);
-				vm_page_unwire_noq(pg);
-				vm_page_free(pg);
+				nfs_free_extpg_page(m, pgno);
 				trim -= plen;
 				m->m_epg_npgs--;
 				pgno--;
diff --git a/sys/modules/nfscommon/Makefile b/sys/modules/nfscommon/Makefile
index 317b14d3fbd4..da9789b5bee2 100644
--- a/sys/modules/nfscommon/Makefile
+++ b/sys/modules/nfscommon/Makefile
@@ -10,7 +10,8 @@ SRCS=	vnode_if.h \
 	opt_kern_tls.h \
 	opt_kgssapi.h \
 	opt_nfs.h \
-	opt_ufs.h
+	opt_ufs.h \
+	opt_vm.h
 
 EXPORT_SYMS=	YES