git: c2089b654ad2 - main - nfscl: Add support for b_pages to be used by RDMA
- Go to: [ bottom of page ] [ top of archives ] [ this month ]
Date: Sun, 04 Oct 2026 00:45:01 UTC
The branch main has been updated by rmacklem:
URL: https://cgit.FreeBSD.org/src/commit/?id=c2089b654ad2568cdd5ddc7baa9ac47db7c3b00c
commit c2089b654ad2568cdd5ddc7baa9ac47db7c3b00c
Author: Rick Macklem <rmacklem@FreeBSD.org>
AuthorDate: 2026-10-04 00:40:04 +0000
Commit: Rick Macklem <rmacklem@FreeBSD.org>
CommitDate: 2026-10-04 00:43:21 +0000
nfscl: Add support for b_pages to be used by RDMA
This patch updates the NFS client RDMA glue so that I/O
can be done directly to/from b_pages for buffer cache
blocks.
It also adds a flag to disable read reduction, that might
be needed against some non-FreeBSD servers and sets readahead
to 8 for RDMA unless the "readahead" option has been
specified.
This commit should not affect non-RDMA behaviour.
MFC after: 3 months
Fixes: 884ee8d6c9b4 ("nfscl: Add some glue for client side NFS over RDMA")
---
sys/fs/nfs/nfs_var.h | 4 +--
sys/fs/nfs/nfsport.h | 1 +
sys/fs/nfsclient/nfs.h | 6 ++---
sys/fs/nfsclient/nfs_clbio.c | 12 ++++-----
sys/fs/nfsclient/nfs_clrpcops.c | 41 +++++++++++++++--------------
sys/fs/nfsclient/nfs_clvfsops.c | 5 ++++
sys/fs/nfsclient/nfs_clvnops.c | 15 ++++++-----
sys/fs/nfsclient/nfsmount.h | 1 +
sys/rpc/clntrdma.h | 4 +--
sys/rpc/rpc_generic.c | 58 +++++++++++++++++++++++++----------------
10 files changed, 86 insertions(+), 61 deletions(-)
diff --git a/sys/fs/nfs/nfs_var.h b/sys/fs/nfs/nfs_var.h
index 57cf1566a14d..38f0ff90c5da 100644
--- a/sys/fs/nfs/nfs_var.h
+++ b/sys/fs/nfs/nfs_var.h
@@ -478,9 +478,9 @@ int nfsrpc_lookup(vnode_t, char *, int, struct ucred *, NFSPROC_T *,
uint32_t);
int nfsrpc_readlink(vnode_t, struct uio *, struct ucred *,
NFSPROC_T *, struct nfsvattr *, int *);
-int nfsrpc_read(vnode_t, struct uio *, struct ucred *, NFSPROC_T *,
+int nfsrpc_read(vnode_t, struct uio *, vm_page_t *, struct ucred *, NFSPROC_T *,
struct nfsvattr *, int *);
-int nfsrpc_write(vnode_t, struct uio *, int *, int *,
+int nfsrpc_write(vnode_t, struct uio *, vm_page_t *, int *, int *,
struct ucred *, NFSPROC_T *, struct nfsvattr *, int *, int, int);
int nfsrpc_mknod(vnode_t, char *, int, struct vattr *, u_int32_t,
__enum_uint8(vtype), struct ucred *, NFSPROC_T *, struct nfsvattr *,
diff --git a/sys/fs/nfs/nfsport.h b/sys/fs/nfs/nfsport.h
index cd50d12dbe5f..196226577d6c 100644
--- a/sys/fs/nfs/nfsport.h
+++ b/sys/fs/nfs/nfsport.h
@@ -1067,6 +1067,7 @@ void ncl_copy_vattr(struct vnode *vp, struct vattr *dst, struct vattr *src);
#define NFSHASSYSKRB5(n) (((n)->nm_newflag & NFSMNT_SYSKRB5) != 0)
#define NFSHASRDMA(n) (((n)->nm_newflag & NFSMNT_RDMA) != 0)
#define NFSHASNOWRITEREDUCE(n) (((n)->nm_newflag & NFSMNT_NOWRITEREDUCE) != 0)
+#define NFSHASNOREADREDUCE(n) (((n)->nm_newflag & NFSMNT_NOREADREDUCE) != 0)
/*
* Set boottime.
diff --git a/sys/fs/nfsclient/nfs.h b/sys/fs/nfsclient/nfs.h
index eeb68a434a6b..007f522c8adf 100644
--- a/sys/fs/nfsclient/nfs.h
+++ b/sys/fs/nfsclient/nfs.h
@@ -102,9 +102,9 @@ void ncl_nhuninit(void);
void ncl_nodelock(struct nfsnode *);
void ncl_nodeunlock(struct nfsnode *);
int ncl_getattrcache(struct vnode *, struct vattr *);
-int ncl_readrpc(struct vnode *, struct uio *, struct ucred *);
-int ncl_writerpc(struct vnode *, struct uio *, struct ucred *, int *, int *,
- int, int);
+int ncl_readrpc(struct vnode *, struct uio *, vm_page_t *, struct ucred *);
+int ncl_writerpc(struct vnode *, struct uio *, vm_page_t *, struct ucred *,
+ int *, int *, int, int);
int ncl_readlinkrpc(struct vnode *, struct uio *, struct ucred *);
int ncl_readdirrpc(struct vnode *, struct uio *, struct ucred *,
struct thread *);
diff --git a/sys/fs/nfsclient/nfs_clbio.c b/sys/fs/nfsclient/nfs_clbio.c
index 421fb08761b1..f26da2598c45 100644
--- a/sys/fs/nfsclient/nfs_clbio.c
+++ b/sys/fs/nfsclient/nfs_clbio.c
@@ -196,7 +196,7 @@ ncl_getpages(struct vop_getpages_args *ap)
uio.uio_rw = UIO_READ;
uio.uio_td = td;
- error = ncl_readrpc(vp, &uio, cred);
+ error = ncl_readrpc(vp, &uio, NULL, cred);
pmap_qremove(kva, npages);
uma_zfree(ncl_pbuf_zone, bp);
@@ -467,7 +467,7 @@ ncl_bioread(struct vnode *vp, struct uio *uio, int ioflag, struct ucred *cred)
if (newnfs_directio_enable && (ioflag & IO_DIRECT) && (vp->v_type == VREG))
/* No caching/ no readaheads. Just read data into the user buffer */
- return ncl_readrpc(vp, uio, cred);
+ return ncl_readrpc(vp, uio, NULL, cred);
n = 0;
on = 0;
@@ -798,7 +798,7 @@ nfs_directio_write(struct vnode *vp, struct uio *uiop, struct ucred *cred,
* verifier on the mount point.
*/
must_commit = 2;
- error = ncl_writerpc(vp, &uio, cred, &iomode,
+ error = ncl_writerpc(vp, &uio, NULL, cred, &iomode,
&must_commit, 0, ioflag);
KASSERT(must_commit == 2,
("ncl_directio_write: Updated write verifier"));
@@ -1586,7 +1586,7 @@ ncl_doio(struct vnode *vp, struct buf *bp, struct ucred *cr, struct thread *td,
case VREG:
uiop->uio_offset = ((off_t)bp->b_blkno) * DEV_BSIZE;
NFSINCRGLOBAL(nfsstatsv1.read_bios);
- error = ncl_readrpc(vp, uiop, cr);
+ error = ncl_readrpc(vp, uiop, bp->b_pages, cr);
if (!error) {
if (uiop->uio_resid) {
@@ -1693,8 +1693,8 @@ ncl_doio(struct vnode *vp, struct buf *bp, struct ucred *cr, struct thread *td,
else
iomode = NFSWRITE_FILESYNC;
- error = ncl_writerpc(vp, uiop, cr, &iomode, &must_commit,
- called_from_strategy, 0);
+ error = ncl_writerpc(vp, uiop, bp->b_pages, cr, &iomode,
+ &must_commit, called_from_strategy, 0);
/*
* When setting B_NEEDCOMMIT also set B_CLUSTEROK to try
diff --git a/sys/fs/nfsclient/nfs_clrpcops.c b/sys/fs/nfsclient/nfs_clrpcops.c
index d69c630f5e47..547771e4594f 100644
--- a/sys/fs/nfsclient/nfs_clrpcops.c
+++ b/sys/fs/nfsclient/nfs_clrpcops.c
@@ -129,9 +129,9 @@ struct nfsclwritedsdorpc {
static int nfsrpc_setattrrpc(vnode_t , struct vattr *, nfsv4stateid_t *,
struct ucred *, NFSPROC_T *, struct nfsvattr *, int *);
-static int nfsrpc_readrpc(vnode_t , struct uio *, struct ucred *,
+static int nfsrpc_readrpc(vnode_t , struct uio *, vm_page_t *, struct ucred *,
nfsv4stateid_t *, NFSPROC_T *, struct nfsvattr *, int *);
-static int nfsrpc_writerpc(vnode_t , struct uio *, int *, int *,
+static int nfsrpc_writerpc(vnode_t , struct uio *, vm_page_t *, int *, int *,
struct ucred *, nfsv4stateid_t *, NFSPROC_T *, struct nfsvattr *, int *,
int);
static int nfsrpc_deallocaterpc(vnode_t, off_t, off_t, nfsv4stateid_t *,
@@ -237,8 +237,8 @@ static int nfsrpc_seekrpc(vnode_t, off_t *, nfsv4stateid_t *, bool *,
int, struct nfsvattr *, int *, struct ucred *);
static struct mbuf *nfsm_split(struct mbuf *, uint64_t);
static void nfscl_statfs(struct vnode *, struct ucred *, NFSPROC_T *);
-static struct mbuf *nfsm_build_rdma_reduction(struct nfsrv_descript *nd,
- int len, int pos, bool to_mem);
+static struct mbuf *nfsm_build_rdma_reduction(vm_page_t *pgp,
+ struct nfsrv_descript *nd, int len, int pos, bool to_mem);
int nfs_pnfsio(task_fn_t *, void *);
@@ -1750,7 +1750,7 @@ nfsmout:
* Read operation.
*/
int
-nfsrpc_read(vnode_t vp, struct uio *uiop, struct ucred *cred,
+nfsrpc_read(vnode_t vp, struct uio *uiop, vm_page_t *pgp, struct ucred *cred,
NFSPROC_T *p, struct nfsvattr *nap, int *attrflagp)
{
int error, expireret = 0, retrycnt;
@@ -1776,7 +1776,7 @@ nfsrpc_read(vnode_t vp, struct uio *uiop, struct ucred *cred,
(void)nfscl_getstateid(vp, nfhp->nfh_fh, nfhp->nfh_len,
NFSV4OPEN_ACCESSREAD, 0, newcred, p, &stateid,
&lckp);
- error = nfsrpc_readrpc(vp, uiop, newcred, &stateid, p, nap,
+ error = nfsrpc_readrpc(vp, uiop, pgp, newcred, &stateid, p, nap,
attrflagp);
if (error == NFSERR_OPENMODE) {
NFSLOCKMNT(nmp);
@@ -1818,7 +1818,7 @@ nfsrpc_read(vnode_t vp, struct uio *uiop, struct ucred *cred,
* The actual read RPC.
*/
static int
-nfsrpc_readrpc(vnode_t vp, struct uio *uiop, struct ucred *cred,
+nfsrpc_readrpc(vnode_t vp, struct uio *uiop, vm_page_t *pgp, struct ucred *cred,
nfsv4stateid_t *stateidp, NFSPROC_T *p, struct nfsvattr *nap,
int *attrflagp)
{
@@ -1839,7 +1839,8 @@ nfsrpc_readrpc(vnode_t vp, struct uio *uiop, struct ucred *cred,
if (NFSHASRDMA(nmp) && tsiz > 0) {
/* Assume the rest of the RPC without data is <= 1024 bytes. */
if ((uiop->uio_offset & PAGE_MASK) == 0 &&
- uiop->uio_segflg == UIO_SYSSPACE)
+ uiop->uio_segflg == UIO_SYSSPACE &&
+ !NFSHASNOREADREDUCE(nmp))
did_rdma = true;
else if (tsiz <= PAGE_SIZE - 1024)
mbflag = M_PROTO7;
@@ -1876,7 +1877,7 @@ nfsrpc_readrpc(vnode_t vp, struct uio *uiop, struct ucred *cred,
}
/* For RDMA, make the data a separate chunk. */
if (did_rdma)
- mr = nfsm_build_rdma_reduction(nd, len, 0, true);
+ mr = nfsm_build_rdma_reduction(pgp, nd, len, 0, true);
/*
* Since I can't do a Getattr for NFSv4 for Write, there
* doesn't seem any point in doing one here, either.
@@ -1951,9 +1952,9 @@ nfsmout:
* will then deadlock.
*/
int
-nfsrpc_write(vnode_t vp, struct uio *uiop, int *iomode, int *must_commit,
- struct ucred *cred, NFSPROC_T *p, struct nfsvattr *nap, int *attrflagp,
- int called_from_strategy, int ioflag)
+nfsrpc_write(vnode_t vp, struct uio *uiop, vm_page_t *pgp, int *iomode,
+ int *must_commit, struct ucred *cred, NFSPROC_T *p, struct nfsvattr *nap,
+ int *attrflagp, int called_from_strategy, int ioflag)
{
int error, expireret = 0, retrycnt, nostateid;
u_int32_t clidrev = 0;
@@ -1996,8 +1997,9 @@ nfsrpc_write(vnode_t vp, struct uio *uiop, int *iomode, int *must_commit,
if (nostateid)
error = 0;
else
- error = nfsrpc_writerpc(vp, uiop, iomode, must_commit,
- newcred, &stateid, p, nap, attrflagp, ioflag);
+ error = nfsrpc_writerpc(vp, uiop, pgp, iomode,
+ must_commit, newcred, &stateid, p, nap, attrflagp,
+ ioflag);
if (error == NFSERR_STALESTATEID)
nfscl_initiate_recovery(nmp->nm_clp);
if (lckp != NULL)
@@ -2034,7 +2036,7 @@ nfsrpc_write(vnode_t vp, struct uio *uiop, int *iomode, int *must_commit,
* The actual write RPC.
*/
static int
-nfsrpc_writerpc(vnode_t vp, struct uio *uiop, int *iomode,
+nfsrpc_writerpc(vnode_t vp, struct uio *uiop, vm_page_t *pgp, int *iomode,
int *must_commit, struct ucred *cred, nfsv4stateid_t *stateidp,
NFSPROC_T *p, struct nfsvattr *nap, int *attrflagp, int ioflag)
{
@@ -2125,7 +2127,8 @@ nfsrpc_writerpc(vnode_t vp, struct uio *uiop, int *iomode,
/* For RDMA, make the data a separate chunk. */
if (did_rdma && !NFSHASNOWRITEREDUCE(nmp)) {
rlen = m_length(nd->nd_mreq, NULL);
- mr = nfsm_build_rdma_reduction(nd, len, rlen, false);
+ mr = nfsm_build_rdma_reduction(pgp, nd, len, rlen,
+ false);
error = rpc_copy_uio_pages(mr, uiop, len, false);
if (error != 0) {
rpc_free_rdma_reduction(mr);
@@ -10136,12 +10139,12 @@ nfscl_statfs(struct vnode *vp, struct ucred *cred, NFSPROC_T *td)
* Set up the RDMA reduction mbuf in the build list.
*/
static struct mbuf *
-nfsm_build_rdma_reduction(struct nfsrv_descript *nd, int len, int pos,
- bool to_mem)
+nfsm_build_rdma_reduction(vm_page_t *pgp, struct nfsrv_descript *nd, int len,
+ int pos, bool to_mem)
{
struct mbuf *m, *mr;
- mr = rpc_reduce_pg(len, pos, to_mem);
+ mr = rpc_reduce_pg(pgp, len, pos, to_mem);
nd->nd_mb->m_next = mr;
nd->nd_mb = mr;
NFSMCLGET(m, M_NOWAIT);
diff --git a/sys/fs/nfsclient/nfs_clvfsops.c b/sys/fs/nfsclient/nfs_clvfsops.c
index f336e320b04c..5aada2bab09c 100644
--- a/sys/fs/nfsclient/nfs_clvfsops.c
+++ b/sys/fs/nfsclient/nfs_clvfsops.c
@@ -1288,6 +1288,11 @@ nfs_mount(struct mount *mp)
goto out;
}
newflag |= NFSMNT_RDMA;
+ /* Set readahead to 8, if not already set. */
+ if ((args.flags & NFSMNT_READAHEAD) == 0) {
+ args.readahead = 8;
+ args.flags |= NFSMNT_READAHEAD;
+ }
}
if (vfs_getopt(mp->mnt_optnew, "sec",
(void **) &secname, NULL) == 0)
diff --git a/sys/fs/nfsclient/nfs_clvnops.c b/sys/fs/nfsclient/nfs_clvnops.c
index 5081e442882e..c7d533c49348 100644
--- a/sys/fs/nfsclient/nfs_clvnops.c
+++ b/sys/fs/nfsclient/nfs_clvnops.c
@@ -549,7 +549,8 @@ nfs_access(struct vop_access_args *ap)
auio.uio_td = ap->a_td;
if (vp->v_type == VREG)
- error = ncl_readrpc(vp, &auio, ap->a_cred);
+ error = ncl_readrpc(vp, &auio, NULL,
+ ap->a_cred);
else if (vp->v_type == VDIR) {
char* bp;
bp = malloc(NFS_DIRBLKSIZ, M_TEMP, M_WAITOK);
@@ -1648,7 +1649,8 @@ ncl_readlinkrpc(struct vnode *vp, struct uio *uiop, struct ucred *cred)
* Ditto above
*/
int
-ncl_readrpc(struct vnode *vp, struct uio *uiop, struct ucred *cred)
+ncl_readrpc(struct vnode *vp, struct uio *uiop, vm_page_t *pgp,
+ struct ucred *cred)
{
int error, ret, attrflag;
struct nfsvattr nfsva;
@@ -1662,7 +1664,7 @@ ncl_readrpc(struct vnode *vp, struct uio *uiop, struct ucred *cred)
NFSV4OPEN_ACCESSREAD, 0, cred, uiop->uio_td);
NFSCL_DEBUG(4, "readrpc: aft doiods=%d\n", error);
if (error != 0 && error != EFAULT)
- error = nfsrpc_read(vp, uiop, cred, uiop->uio_td, &nfsva,
+ error = nfsrpc_read(vp, uiop, pgp, cred, uiop->uio_td, &nfsva,
&attrflag);
if (attrflag) {
ret = nfscl_loadattrcache(&vp, &nfsva, NULL, 0, 1);
@@ -1678,8 +1680,9 @@ ncl_readrpc(struct vnode *vp, struct uio *uiop, struct ucred *cred)
* nfs write call
*/
int
-ncl_writerpc(struct vnode *vp, struct uio *uiop, struct ucred *cred,
- int *iomode, int *must_commit, int called_from_strategy, int ioflag)
+ncl_writerpc(struct vnode *vp, struct uio *uiop, vm_page_t *pgp,
+ struct ucred *cred, int *iomode, int *must_commit, int called_from_strategy,
+ int ioflag)
{
struct nfsvattr nfsva;
int error, attrflag, ret;
@@ -1693,7 +1696,7 @@ ncl_writerpc(struct vnode *vp, struct uio *uiop, struct ucred *cred,
NFSV4OPEN_ACCESSWRITE, 0, cred, uiop->uio_td);
NFSCL_DEBUG(4, "writerpc: aft doiods=%d\n", error);
if (error != 0 && error != EFAULT)
- error = nfsrpc_write(vp, uiop, iomode, must_commit, cred,
+ error = nfsrpc_write(vp, uiop, pgp, iomode, must_commit, cred,
uiop->uio_td, &nfsva, &attrflag, called_from_strategy,
ioflag);
if (attrflag) {
diff --git a/sys/fs/nfsclient/nfsmount.h b/sys/fs/nfsclient/nfsmount.h
index 800aaeb7f46d..9eae88033fa7 100644
--- a/sys/fs/nfsclient/nfsmount.h
+++ b/sys/fs/nfsclient/nfsmount.h
@@ -131,6 +131,7 @@ struct nfsmount {
#define NFSMNT_SYSKRB5 0x00000002
#define NFSMNT_RDMA 0x00000004
#define NFSMNT_NOWRITEREDUCE 0x00000008
+#define NFSMNT_NOREADREDUCE 0x00000010
#define NFSMNT_DIRPATH(m) (&((m)->nm_name[(m)->nm_krbnamelen + 1]))
#define NFSMNT_SRVKRBNAME(m) \
diff --git a/sys/rpc/clntrdma.h b/sys/rpc/clntrdma.h
index 80836fb26fa5..d3ebba399563 100644
--- a/sys/rpc/clntrdma.h
+++ b/sys/rpc/clntrdma.h
@@ -16,6 +16,7 @@ struct rpcrdma_reduce_pg {
uint32_t len;
uint32_t npg;
uint32_t into_mem;
+ uint32_t pg_allocd;
vm_page_t pg[];
};
@@ -28,7 +29,6 @@ struct rpcrdma_xprt {
uint32_t maxio;
uint32_t maxsge;
uint32_t maxsegs;
- uint32_t use_bounce;
void *ep;
};
@@ -106,7 +106,7 @@ int xprt_rdma_acquire_buf(struct rpcrdma_xprt *xp, int start, int end);
void xprt_rdma_unmap_chunk(struct rpcrdma_xprt *xp, struct rpcrdma_chunk *chp);
-struct mbuf *rpc_reduce_pg(int len, int pos, bool to_mem);
+struct mbuf *rpc_reduce_pg(vm_page_t *pgp, int len, int pos, bool to_mem);
void rpc_free_rdma_reduction(struct mbuf *mr);
diff --git a/sys/rpc/rpc_generic.c b/sys/rpc/rpc_generic.c
index c540f86df032..96abe6a39080 100644
--- a/sys/rpc/rpc_generic.c
+++ b/sys/rpc/rpc_generic.c
@@ -946,7 +946,7 @@ _rpc_copym_into_ext_pgs(struct mbuf *mp, int maxextsiz)
* Create an mr mbuf and associated pages.
*/
struct mbuf *
-rpc_reduce_pg(int len, int pos, bool to_mem)
+rpc_reduce_pg(vm_page_t *pgp, int len, int pos, bool to_mem)
{
struct mbuf *mr;
struct rpcrdma_reduce_pg *rb;
@@ -960,9 +960,14 @@ rpc_reduce_pg(int len, int pos, bool to_mem)
rb->len = len;
rb->npg = i;
rb->pos = pos;
- for (i = 0; i < rb->npg; i++)
- rb->pg[i] = vm_page_alloc_noobj(VM_ALLOC_WAITOK |
- VM_ALLOC_NODUMP | VM_ALLOC_WIRED);
+ rb->pg_allocd = (pgp != NULL) ? 0 : 1;
+ for (i = 0; i < rb->npg; i++) {
+ if (pgp != NULL)
+ rb->pg[i] = *pgp++;
+ else
+ rb->pg[i] = vm_page_alloc_noobj(VM_ALLOC_WAITOK |
+ VM_ALLOC_NODUMP | VM_ALLOC_WIRED);
+ }
rb->into_mem = (to_mem) ? 1 : 0;
return (mr);
}
@@ -974,7 +979,7 @@ rpc_free_rdma_reduction(struct mbuf *mr)
int i;
rb = mtod(mr, struct rpcrdma_reduce_pg *);
- for (i = 0; i < rb->npg; i++) {
+ for (i = 0; i < rb->npg && rb->pg_allocd != 0; i++) {
vm_page_unwire_noq(rb->pg[i]);
vm_page_free(rb->pg[i]);
}
@@ -995,7 +1000,8 @@ rpc_copy_uio_pages(struct mbuf *mr, struct uio *uiop, int siz, bool from_pages)
if (siz > rb->len)
return (EBADRPC);
i = 0;
- cp = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(rb->pg[i]));
+ if (rb->pg_allocd != 0)
+ cp = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(rb->pg[i]));
len = PAGE_SIZE;
if (i == rb->npg - 1 && siz < PAGE_SIZE)
len = siz;
@@ -1012,33 +1018,39 @@ rpc_copy_uio_pages(struct mbuf *mr, struct uio *uiop, int siz, bool from_pages)
if (i == rb->npg - 1)
return (EBADRPC);
i++;
- cp = PHYS_TO_DMAP(
- VM_PAGE_TO_PHYS(rb->pg[i]));
+ if (rb->pg_allocd != 0)
+ cp = PHYS_TO_DMAP(
+ VM_PAGE_TO_PHYS(rb->pg[i]));
len = PAGE_SIZE;
if (i == rb->npg - 1 && siz < PAGE_SIZE)
len = siz;
}
xfer = (left > len) ? len : left;
- if (from_pages) {
- if (uiop->uio_segflg == UIO_SYSSPACE) {
- memcpy(uiocp, cp, xfer);
- } else {
- error = copyout(cp, uiocp, xfer);
- if (error != 0)
- return (EBADRPC);
- }
- } else {
- if (uiop->uio_segflg == UIO_SYSSPACE) {
- memcpy(cp, uiocp, xfer);
+ if (rb->pg_allocd != 0) {
+ if (from_pages) {
+ if (uiop->uio_segflg == UIO_SYSSPACE) {
+ memcpy(uiocp, cp, xfer);
+ } else {
+ error = copyout(cp, uiocp,
+ xfer);
+ if (error != 0)
+ return (EBADRPC);
+ }
} else {
- error = copyout(uiocp, cp, xfer);
- if (error != 0)
- return (EBADRPC);
+ if (uiop->uio_segflg == UIO_SYSSPACE) {
+ memcpy(cp, uiocp, xfer);
+ } else {
+ error = copyout(uiocp, cp,
+ xfer);
+ if (error != 0)
+ return (EBADRPC);
+ }
}
}
left -= xfer;
len -= xfer;
- cp += xfer;
+ if (rb->pg_allocd != 0)
+ cp += xfer;
uiocp += xfer;
uiop->uio_offset += xfer;
uiop->uio_resid -= xfer;