git: 8c511e47f421 - main - nfs: Cache reply pages in an NFS-owned UMA zone
- Go to: [ bottom of page ] [ top of archives ] [ this month ]
Date: Sat, 21 Nov 2026 23:25:17 UTC
The branch main has been updated by rmacklem:
URL: https://cgit.FreeBSD.org/src/commit/?id=8c511e47f42155c534ba1ee08771d02f50e42662
commit 8c511e47f42155c534ba1ee08771d02f50e42662
Author: Vinicius Ferrao <ferrao@versatushpc.com.br>
AuthorDate: 2026-11-21 23:22:07 +0000
Commit: Rick Macklem <rmacklem@FreeBSD.org>
CommitDate: 2026-11-21 23:22:07 +0000
nfs: Cache reply pages in an NFS-owned UMA zone
Allocate anonymous reply pages through a dedicated UMA cache and
return them in batches when the last M_EXTPG reference is released.
This reduced page-cache bucket shortages in the tested
concurrent-read workloads. The pool uses the existing UMA and
mbuf interfaces.
Once nfsd enables the pool, nfscommon refuses unload so delayed
callbacks remain valid. The usual nfsd/nfslockd dependency chain
already retains nfscommon; the explicit guard also enforces this
requirement directly. Loading nfscommon alone does not activate
that guard.
Mark Johnston suggested using a dedicated zone in the D61234 discussion.
That feedback led to this NFS implementation instead of changing UMA.
Gleb Smirnoff provided feedback on the original allocator proposal.
Reviewed by: rmacklem
Discussed with: markj, glebius
MFC after: 1 month
Fixes: 7144a1d58c5c ("nfsd: Add glue for the nfsrdma.ko module")
---
sys/fs/nfs/nfs_commonport.c | 147 +++++++++++++++++++++++++++++++++++++++-
sys/fs/nfs/nfs_commonsubs.c | 7 +-
sys/fs/nfs/nfs_var.h | 4 ++
sys/fs/nfsserver/nfs_nfsdport.c | 17 +++--
sys/fs/nfsserver/nfs_nfsdsubs.c | 5 +-
sys/modules/nfscommon/Makefile | 3 +-
6 files changed, 165 insertions(+), 18 deletions(-)
diff --git a/sys/fs/nfs/nfs_commonport.c b/sys/fs/nfs/nfs_commonport.c
index 03c6688b1406..30ab162af07c 100644
--- a/sys/fs/nfs/nfs_commonport.c
+++ b/sys/fs/nfs/nfs_commonport.c
@@ -54,6 +54,9 @@
#include <vm/vm_extern.h>
#include <vm/uma.h>
+static uma_zone_t nfs_extpg_zone;
+static u_int nfs_extpg_enabled;
+
extern int nfscl_ticks;
extern void (*nfsd_call_recall)(struct vnode *, int, struct ucred *,
struct thread *);
@@ -918,6 +921,139 @@ VNET_SYSUNINIT(nfs_cleanup, SI_SUB_VNET_DONE, SI_ORDER_FIRST,
extern int (*nfsd_call_nfscommon)(struct thread *, struct nfssvc_args *);
+static int
+nfs_extpg_import(void *arg __unused, void **store, int count, int domain,
+ int how)
+{
+ vm_page_t pg;
+ int i, req;
+
+ req = VM_ALLOC_WIRED | VM_ALLOC_NODUMP |
+ ((how & M_WAITOK) != 0 ? VM_ALLOC_WAITOK : VM_ALLOC_NOWAIT);
+ for (i = 0; i < count; i++) {
+ if (domain == UMA_ANYDOMAIN)
+ pg = vm_page_alloc_noobj(req);
+ else
+ pg = vm_page_alloc_noobj_domain(domain, req);
+ if (pg == NULL)
+ break;
+ store[i] = pg;
+ req &= ~VM_ALLOC_WAITOK;
+ }
+ return (i);
+}
+
+static void
+nfs_extpg_release(void *arg __unused, void **store, int count)
+{
+ vm_page_t pg;
+ int i;
+
+ for (i = 0; i < count; i++) {
+ pg = store[i];
+ vm_page_unwire_noq(pg);
+ vm_page_free(pg);
+ }
+}
+
+static void
+nfs_free_extpg_cached(struct mbuf *m)
+{
+ int i;
+
+ M_ASSERTEXTPG(m);
+ for (i = 0; i < m->m_epg_npgs; i++)
+ uma_zfree(nfs_extpg_zone,
+ PHYS_TO_VM_PAGE(m->m_epg_pa[i]));
+}
+
+bool
+nfs_alloc_extpg_page(struct mbuf *m, int how)
+{
+ vm_page_t pg;
+ vm_paddr_t pa;
+
+ M_ASSERTEXTPG(m);
+ KASSERT((m->m_ext.ext_flags & EXT_FLAG_EMBREF) != 0 &&
+ m_extrefcnt(m) == 1, ("shared NFS external pages"));
+ KASSERT(m->m_epg_npgs < MBUF_PEXT_MAX_PGS,
+ ("NFS external page array full"));
+ if (m->m_ext.ext_free == nfs_free_extpg_cached) {
+ pg = uma_zalloc(nfs_extpg_zone, how & ~M_ZERO);
+ if (pg == NULL)
+ return (false);
+ if ((how & M_ZERO) != 0)
+ pmap_zero_page(pg);
+ pa = VM_PAGE_TO_PHYS(pg);
+ } else {
+ KASSERT(m->m_ext.ext_free == mb_free_mext_pgs,
+ ("unexpected NFS external page owner"));
+ pg = vm_page_alloc_noobj(VM_ALLOC_NODUMP | VM_ALLOC_WIRED |
+ ((how & M_WAITOK) != 0 ? VM_ALLOC_WAITOK :
+ VM_ALLOC_NOWAIT));
+ if (pg == NULL)
+ return (false);
+ pa = VM_PAGE_TO_PHYS(pg);
+ }
+ m->m_epg_pa[m->m_epg_npgs++] = pa;
+ return (true);
+}
+
+void
+nfs_free_extpg_page(struct mbuf *m, int index)
+{
+ vm_page_t pg;
+
+ M_ASSERTEXTPG(m);
+ KASSERT((m->m_ext.ext_flags & EXT_FLAG_EMBREF) != 0 &&
+ m_extrefcnt(m) == 1, ("shared NFS external pages"));
+ KASSERT(index >= 0 && index < m->m_epg_npgs,
+ ("NFS external page index out of range"));
+ if (m->m_ext.ext_free == nfs_free_extpg_cached) {
+ uma_zfree(nfs_extpg_zone,
+ PHYS_TO_VM_PAGE(m->m_epg_pa[index]));
+ } else {
+ KASSERT(m->m_ext.ext_free == mb_free_mext_pgs,
+ ("unexpected NFS external page owner"));
+ pg = PHYS_TO_VM_PAGE(m->m_epg_pa[index]);
+ vm_page_unwire_noq(pg);
+ vm_page_free(pg);
+ }
+}
+
+void
+nfs_enable_extpg_cache(void)
+{
+
+ /* Enabling the pool also prevents nfscommon unload. */
+ atomic_store_rel_int(&nfs_extpg_enabled, 1);
+}
+
+struct mbuf *
+nfs_alloc_extpg(int len, int how)
+{
+ struct mbuf *m;
+ int i, npgs;
+
+ if (atomic_load_acq_int(&nfs_extpg_enabled) == 0 ||
+ nfs_extpg_zone == NULL)
+ return (mb_alloc_ext_plus_pages(len, how));
+ npgs = howmany(len, PAGE_SIZE);
+ KASSERT(npgs > 0 && npgs <= MBUF_PEXT_MAX_PGS,
+ ("invalid NFS external page count %d", npgs));
+ m = mb_alloc_ext_pgs(how, nfs_free_extpg_cached, 0);
+ if (m == NULL)
+ return (NULL);
+ m->m_epg_flags |= EPG_FLAG_ANON;
+ for (i = 0; i < npgs; i++) {
+ if (!nfs_alloc_extpg_page(m, how)) {
+ m_free(m);
+ return (NULL);
+ }
+ }
+ return (m);
+}
+
/*
* Called once to initialize data structures...
*/
@@ -931,6 +1067,10 @@ nfscommon_modevent(module_t mod, int type, void *data)
case MOD_LOAD:
if (loaded)
goto out;
+ nfs_extpg_zone = uma_zcache_create("nfs reply pages", PAGE_SIZE,
+ NULL, NULL, NULL, NULL, nfs_extpg_import,
+ nfs_extpg_release, NULL, 0);
+ uma_zone_set_maxcache(nfs_extpg_zone, 16384);
newnfs_portinit();
mtx_init(&nfs_nameid_mutex, "nfs_nameid_mutex", NULL, MTX_DEF);
mtx_init(&nfs_sockl_mutex, "nfs_sockl_mutex", NULL, MTX_DEF);
@@ -944,13 +1084,18 @@ nfscommon_modevent(module_t mod, int type, void *data)
break;
case MOD_UNLOAD:
- if (newnfs_numnfsd != 0 ||
+ if (atomic_load_acq_int(&nfs_extpg_enabled) != 0 ||
+ newnfs_numnfsd != 0 ||
VNET(nfsrv_nfsuserd) != NOTRUNNING ||
nfs_numnfscbd != 0) {
error = EBUSY;
break;
}
+ if (nfs_extpg_zone != NULL) {
+ uma_zdestroy(nfs_extpg_zone);
+ nfs_extpg_zone = NULL;
+ }
nfsd_call_nfscommon = NULL;
/* and get rid of the mutexes */
mtx_destroy(&nfs_nameid_mutex);
diff --git a/sys/fs/nfs/nfs_commonsubs.c b/sys/fs/nfs/nfs_commonsubs.c
index a11b55b11c43..4ebc2a937a6c 100644
--- a/sys/fs/nfs/nfs_commonsubs.c
+++ b/sys/fs/nfs/nfs_commonsubs.c
@@ -5557,18 +5557,15 @@ struct mbuf *
nfsm_add_ext_pgs(struct mbuf *m, int maxextsiz, int *bextpg)
{
struct mbuf *mp;
- vm_page_t pg;
if ((m->m_epg_npgs + 1) * PAGE_SIZE > maxextsiz) {
mp = mb_alloc_ext_plus_pages(PAGE_SIZE, M_WAITOK);
*bextpg = 0;
m->m_next = mp;
} else {
- pg = vm_page_alloc_noobj(VM_ALLOC_WAITOK | VM_ALLOC_NODUMP |
- VM_ALLOC_WIRED);
- m->m_epg_pa[m->m_epg_npgs] = VM_PAGE_TO_PHYS(pg);
*bextpg = m->m_epg_npgs;
- m->m_epg_npgs++;
+ if (!nfs_alloc_extpg_page(m, M_WAITOK))
+ panic("nfsm_add_ext_pgs: page allocation failed");
m->m_epg_last_len = 0;
mp = m;
}
diff --git a/sys/fs/nfs/nfs_var.h b/sys/fs/nfs/nfs_var.h
index 488ddbd1c582..35c92b63bc4a 100644
--- a/sys/fs/nfs/nfs_var.h
+++ b/sys/fs/nfs/nfs_var.h
@@ -367,6 +367,10 @@ void nfsv4_freeslot(struct nfsclsession *, int, bool);
struct ucred *nfsrv_getgrpscred(struct ucred *);
struct nfsdevice *nfsv4_findmirror(struct nfsmount *);
void nfsm_set(struct nfsrv_descript *, u_int);
+void nfs_enable_extpg_cache(void);
+struct mbuf *nfs_alloc_extpg(int, int);
+bool nfs_alloc_extpg_page(struct mbuf *, int);
+void nfs_free_extpg_page(struct mbuf *, int);
struct mbuf *nfsm_add_ext_pgs(struct mbuf *, int, int *);
int nfsrpc_destroysession(struct nfsmount *, struct nfsclsession *,
struct ucred *, NFSPROC_T *);
diff --git a/sys/fs/nfsserver/nfs_nfsdport.c b/sys/fs/nfsserver/nfs_nfsdport.c
index fbbdef03f440..bda047d2d226 100644
--- a/sys/fs/nfsserver/nfs_nfsdport.c
+++ b/sys/fs/nfsserver/nfs_nfsdport.c
@@ -186,6 +186,9 @@ int nfs_pnfsio(task_fn_t *, void *);
SYSCTL_NODE(_vfs, OID_AUTO, nfsd, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
"NFS server");
+static bool nfsrv_extpg_cache = true;
+SYSCTL_BOOL(_vfs_nfsd, OID_AUTO, extpg_cache, CTLFLAG_RWTUN,
+ &nfsrv_extpg_cache, true, "Cache NFS reply pages");
SYSCTL_INT(_vfs_nfsd, OID_AUTO, mirrormnt, CTLFLAG_RW,
&nfsrv_enable_crossmntpt, 0, "Enable nfsd to cross mount points");
SYSCTL_INT(_vfs_nfsd, OID_AUTO, commit_blks, CTLFLAG_RW, &nfs_commit_blks,
@@ -1100,7 +1103,10 @@ nfsrv_createiovec_extpgs(int len, int maxextsiz, struct mbuf **mpp,
i = 0;
while (left > 0) {
siz = min(left, maxextsiz);
- m = mb_alloc_ext_plus_pages(siz, M_WAITOK);
+ if (nfsrv_extpg_cache)
+ m = nfs_alloc_extpg(siz, M_WAITOK);
+ else
+ m = mb_alloc_ext_plus_pages(siz, M_WAITOK);
left -= siz;
i += m->m_epg_npgs;
if (m3 != NULL)
@@ -8477,7 +8483,6 @@ void
nfsm_trimtrailing(struct nfsrv_descript *nd, struct mbuf *mb, char *bpos,
int bextpg, int bextpgsiz)
{
- vm_page_t pg;
int fullpgsiz, i;
if (mb->m_next != NULL) {
@@ -8492,11 +8497,8 @@ nfsm_trimtrailing(struct nfsrv_descript *nd, struct mbuf *mb, char *bpos,
("nfsm_trimtrailing: bextpgsiz bad!"));
/* First, get rid of any pages after this position. */
- for (i = mb->m_epg_npgs - 1; i > bextpg; i--) {
- pg = PHYS_TO_VM_PAGE(mb->m_epg_pa[i]);
- vm_page_unwire_noq(pg);
- vm_page_free(pg);
- }
+ for (i = mb->m_epg_npgs - 1; i > bextpg; i--)
+ nfs_free_extpg_page(mb, i);
mb->m_epg_npgs = bextpg + 1;
if (bextpg == 0)
fullpgsiz = PAGE_SIZE - mb->m_epg_1st_off;
@@ -8701,6 +8703,7 @@ nfsd_modevent(module_t mod, int type, void *data)
vn_deleg_ops.vndeleg_disable = nfsd_disabledelegation;
#endif
nfsd_call_nfsd = nfssvc_nfsd;
+ nfs_enable_extpg_cache();
loaded = 1;
break;
diff --git a/sys/fs/nfsserver/nfs_nfsdsubs.c b/sys/fs/nfsserver/nfs_nfsdsubs.c
index f7fd81ef4e1e..be7da2a1fb0a 100644
--- a/sys/fs/nfsserver/nfs_nfsdsubs.c
+++ b/sys/fs/nfsserver/nfs_nfsdsubs.c
@@ -1284,7 +1284,6 @@ struct mbuf *
nfsrv_adj(struct mbuf *mp, int len, int nul)
{
struct mbuf *m, *m2;
- vm_page_t pg;
int i, lastlen, pgno, plen, tlen, trim;
uint16_t off;
char *cp;
@@ -1334,9 +1333,7 @@ nfsrv_adj(struct mbuf *mp, int len, int nul)
KASSERT(pgno > 0,
("nfsrv_adj: freeing page 0"));
/* Free page. */
- pg = PHYS_TO_VM_PAGE(m->m_epg_pa[pgno]);
- vm_page_unwire_noq(pg);
- vm_page_free(pg);
+ nfs_free_extpg_page(m, pgno);
trim -= plen;
m->m_epg_npgs--;
pgno--;
diff --git a/sys/modules/nfscommon/Makefile b/sys/modules/nfscommon/Makefile
index 317b14d3fbd4..da9789b5bee2 100644
--- a/sys/modules/nfscommon/Makefile
+++ b/sys/modules/nfscommon/Makefile
@@ -10,7 +10,8 @@ SRCS= vnode_if.h \
opt_kern_tls.h \
opt_kgssapi.h \
opt_nfs.h \
- opt_ufs.h
+ opt_ufs.h \
+ opt_vm.h
EXPORT_SYMS= YES