git: a311bd18a6fb - main - krpc: Add some glue for client side NFS over RDMA
- Go to: [ bottom of page ] [ top of archives ] [ this month ]
Date: Sun, 30 Aug 2026 21:34:05 UTC
The branch main has been updated by rmacklem:
URL: https://cgit.FreeBSD.org/src/commit/?id=a311bd18a6fb57c6d3a19ab5bb53bb6f1c5fd056
commit a311bd18a6fb57c6d3a19ab5bb53bb6f1c5fd056
Author: Rick Macklem <rmacklem@FreeBSD.org>
AuthorDate: 2026-08-30 21:32:11 +0000
Commit: Rick Macklem <rmacklem@FreeBSD.org>
CommitDate: 2026-08-30 21:32:11 +0000
krpc: Add some glue for client side NFS over RDMA
This patch adds assorted bits needed by the nfsclrdma.ko
module that implements client side NFS over RDMA.
It should not affect non-RDMA operation.
Some additional glue is needed for the nfsclrdma.ko
module within the NFS code. That will be added as
a separate commit.
I've specified a long MFC, since the module still
requires extensive testing and, hopefully, a review.
MFC after: 3 months
---
sys/rpc/clnt.h | 4 ++
sys/rpc/clnt_rc.c | 185 ++++++++++++++++++++++++++++++++-----------------
sys/rpc/clntrdma.h | 104 +++++++++++++++++++++++++++
sys/rpc/getnetconfig.c | 14 ++++
sys/rpc/krpc.h | 20 +++++-
sys/rpc/netconfig.h | 1 +
sys/rpc/rpc.h | 15 ++++
sys/rpc/rpc_generic.c | 136 ++++++++++++++++++++++++++++++++++++
sys/rpc/svc_vc.c | 12 +++-
9 files changed, 423 insertions(+), 68 deletions(-)
diff --git a/sys/rpc/clnt.h b/sys/rpc/clnt.h
index a237b00928e0..f90c15083347 100644
--- a/sys/rpc/clnt.h
+++ b/sys/rpc/clnt.h
@@ -321,6 +321,10 @@ struct rpc_reconupcall {
void *arg;
};
#define CLSET_RECONUPCALL 33 /* Reconnect upcall */
+#define CLSET_RDMASMALL_REPLY 34 /* Max. size of a small reply */
+#define CLSET_RDMAMAX_IO 35 /* Max. size of any reducible I/O */
+#define CLGET_RDMAMAX_IO 36 /* Max. size of any reducible I/O */
+#define CLSET_RDMA_CBSLOTS 37 /* Max. number of callbacks */
/*
* void
diff --git a/sys/rpc/clnt_rc.c b/sys/rpc/clnt_rc.c
index 44b63e38a8e6..1a743d2be1e3 100644
--- a/sys/rpc/clnt_rc.c
+++ b/sys/rpc/clnt_rc.c
@@ -49,6 +49,10 @@
#include <rpc/krpc.h>
#include <rpc/rpcsec_tls.h>
+xprt_rdma_check_route_ftype *rdma_check_route = NULL;
+clnt_rdma_create_ftype *clnt_rdma_create_call = NULL;
+clnt_rdma_bcksend_ftype *clnt_rdma_bcksend_call = NULL;
+
static enum clnt_stat clnt_reconnect_call(CLIENT *, struct rpc_callextra *,
rpcproc_t, struct mbuf *, struct mbuf **, struct timeval);
static void clnt_reconnect_geterr(CLIENT *, struct rpc_err *);
@@ -134,7 +138,9 @@ clnt_reconnect_connect(CLIENT *cl)
struct ucred *oldcred;
CLIENT *newclient = NULL;
uint32_t reterr;
+ bool dordma;
+ dordma = false;
mtx_lock(&rc->rc_lock);
while (rc->rc_connecting) {
error = msleep(rc, &rc->rc_lock,
@@ -164,81 +170,110 @@ clnt_reconnect_connect(CLIENT *cl)
oldcred = td->td_ucred;
td->td_ucred = rc->rc_ucred;
- so = __rpc_nconf2socket(rc->rc_nconf);
- if (!so) {
- stat = rpc_createerr.cf_stat = RPC_TLIERROR;
- rpc_createerr.cf_error.re_errno = 0;
- td->td_ucred = oldcred;
- goto out;
- }
- if (rc->rc_privport)
- bindresvport(so, NULL);
+ /* Handle RDMA. */
+ if (strcmp("rdma", rc->rc_nconf->nc_netid) == 0 ||
+ strcmp("rdma6", rc->rc_nconf->nc_netid) == 0)
+ dordma = true;
+ if (dordma) {
+ if (clnt_rdma_create_call != NULL) {
+ /* Set up the QP. */
+ stat = RPC_SUCCESS;
+ newclient = clnt_rdma_create_call(
+ (struct sockaddr *)&rc->rc_addr, rc->rc_prog,
+ rc->rc_vers, rc->rc_intr, rc->rc_rdmasmall_reply,
+ rc->rc_rdmamax_io, rc->rc_rdma_cbslots,
+ &rc->rc_err);
+ } else {
+ stat = RPC_FAILED;
+ newclient = NULL;
+ }
+ } else {
+ so = __rpc_nconf2socket(rc->rc_nconf);
+ if (!so) {
+ stat = rpc_createerr.cf_stat = RPC_TLIERROR;
+ rpc_createerr.cf_error.re_errno = 0;
+ td->td_ucred = oldcred;
+ goto out;
+ }
- if (rc->rc_nconf->nc_semantics == NC_TPI_CLTS)
- newclient = clnt_dg_create(so,
- (struct sockaddr *) &rc->rc_addr, rc->rc_prog, rc->rc_vers,
- rc->rc_sendsz, rc->rc_recvsz);
- else {
- /*
- * I do not believe a timeout of less than 1sec would make
- * sense here since short delays can occur when a server is
- * temporarily overloaded.
- */
- if (rc->rc_timeout.tv_sec > 0 && rc->rc_timeout.tv_usec >= 0) {
- error = so_setsockopt(so, SOL_SOCKET, SO_SNDTIMEO,
- &rc->rc_timeout, sizeof(struct timeval));
- if (error != 0) {
- stat = rpc_createerr.cf_stat = RPC_CANTSEND;
- rpc_createerr.cf_error.re_errno = error;
- td->td_ucred = oldcred;
- goto out;
+ if (rc->rc_privport)
+ bindresvport(so, NULL);
+
+ if (rc->rc_nconf->nc_semantics == NC_TPI_CLTS)
+ newclient = clnt_dg_create(so,
+ (struct sockaddr *) &rc->rc_addr, rc->rc_prog,
+ rc->rc_vers, rc->rc_sendsz, rc->rc_recvsz);
+ else {
+ /*
+ * I do not believe a timeout of less than 1sec would
+ * make sense here since short delays can occur when a
+ * server is temporarily overloaded.
+ */
+ if (rc->rc_timeout.tv_sec > 0 &&
+ rc->rc_timeout.tv_usec >= 0) {
+ error = so_setsockopt(so, SOL_SOCKET,
+ SO_SNDTIMEO, &rc->rc_timeout,
+ sizeof(struct timeval));
+ if (error != 0) {
+ stat = rpc_createerr.cf_stat =
+ RPC_CANTSEND;
+ rpc_createerr.cf_error.re_errno = error;
+ td->td_ucred = oldcred;
+ goto out;
+ }
}
- }
- newclient = clnt_vc_create(so,
- (struct sockaddr *) &rc->rc_addr, rc->rc_prog, rc->rc_vers,
- rc->rc_sendsz, rc->rc_recvsz, rc->rc_intr);
- /*
- * CLSET_FD_CLOSE must be done now, in case rpctls_connect()
- * fails just below.
- */
- if (newclient != NULL)
- CLNT_CONTROL(newclient, CLSET_FD_CLOSE, 0);
- if (rc->rc_tls && newclient != NULL) {
- CURVNET_SET(so->so_vnet);
- stat = rpctls_connect(newclient, rc->rc_tlscertname, so,
- &reterr);
- CURVNET_RESTORE();
- if (stat != RPC_SUCCESS || reterr != RPCTLSERR_OK) {
- if (stat == RPC_SUCCESS)
- stat = RPC_FAILED;
- stat = rpc_createerr.cf_stat = stat;
- rpc_createerr.cf_error.re_errno = 0;
- CLNT_CLOSE(newclient);
- CLNT_RELEASE(newclient);
- newclient = NULL;
- td->td_ucred = oldcred;
- goto out;
+ newclient = clnt_vc_create(so,
+ (struct sockaddr *) &rc->rc_addr, rc->rc_prog,
+ rc->rc_vers, rc->rc_sendsz, rc->rc_recvsz,
+ rc->rc_intr);
+ /*
+ * CLSET_FD_CLOSE must be done now, in case
+ * rpctls_connect() fails just below.
+ */
+ if (newclient != NULL)
+ CLNT_CONTROL(newclient, CLSET_FD_CLOSE, 0);
+ if (rc->rc_tls && newclient != NULL) {
+ CURVNET_SET(so->so_vnet);
+ stat = rpctls_connect(newclient,
+ rc->rc_tlscertname, so, &reterr);
+ CURVNET_RESTORE();
+ if (stat != RPC_SUCCESS ||
+ reterr != RPCTLSERR_OK) {
+ if (stat == RPC_SUCCESS)
+ stat = RPC_FAILED;
+ stat = rpc_createerr.cf_stat = stat;
+ rpc_createerr.cf_error.re_errno = 0;
+ CLNT_CLOSE(newclient);
+ CLNT_RELEASE(newclient);
+ newclient = NULL;
+ td->td_ucred = oldcred;
+ goto out;
+ }
+ CLNT_CONTROL(newclient, CLSET_TLS,
+ &(int){RPCTLS_COMPLETE});
}
- CLNT_CONTROL(newclient, CLSET_TLS,
- &(int){RPCTLS_COMPLETE});
- }
- if (newclient != NULL) {
- int optval = 1;
+ if (newclient != NULL) {
+ int optval = 1;
- (void)so_setsockopt(so, IPPROTO_TCP, TCP_USE_DDP,
- &optval, sizeof(optval));
+ (void)so_setsockopt(so, IPPROTO_TCP,
+ TCP_USE_DDP, &optval, sizeof(optval));
+ }
}
- if (newclient != NULL && rc->rc_reconcall != NULL)
- (*rc->rc_reconcall)(newclient, rc->rc_reconarg,
- rc->rc_ucred);
}
+ if (newclient != NULL && rc->rc_reconcall != NULL)
+ (*rc->rc_reconcall)(newclient, rc->rc_reconarg,
+ rc->rc_ucred);
td->td_ucred = oldcred;
if (!newclient) {
- soclose(so);
- rc->rc_err = rpc_createerr.cf_error;
- stat = rpc_createerr.cf_stat;
+ if (!dordma) {
+ soclose(so);
+ rc->rc_err = rpc_createerr.cf_error;
+ stat = rpc_createerr.cf_stat;
+ } else {
+ stat = rc->rc_err.re_status;
+ }
goto out;
}
@@ -536,6 +571,26 @@ clnt_reconnect_control(CLIENT *cl, u_int request, void *info)
rc->rc_reconarg = upcp->arg;
break;
+ case CLSET_RDMASMALL_REPLY:
+ rc->rc_rdmasmall_reply = *(int *)info;
+ break;
+
+ case CLSET_RDMA_CBSLOTS:
+ rc->rc_rdma_cbslots = *(int *)info;
+ break;
+
+
+ case CLSET_RDMAMAX_IO:
+ rc->rc_rdmamax_io = *(int *)info;
+ if (rc->rc_client)
+ CLNT_CONTROL(rc->rc_client, request, info);
+ break;
+
+ case CLGET_RDMAMAX_IO:
+ if (rc->rc_client)
+ CLNT_CONTROL(rc->rc_client, request, info);
+ break;
+
default:
return (FALSE);
}
diff --git a/sys/rpc/clntrdma.h b/sys/rpc/clntrdma.h
new file mode 100644
index 000000000000..09445faa25a1
--- /dev/null
+++ b/sys/rpc/clntrdma.h
@@ -0,0 +1,104 @@
+/*
+ * Copyright (c) 2026 Rick Macklem
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#ifndef _RPC_CLNTRDMA_H_
+#define _RPC_CLNTRDMA_H_
+
+#ifdef _KERNEL
+#define RPCRDMA_IO_NUMBUFS 64 /* Max. # of concurrent RPCs. */
+
+struct rpcrdma_reduce_pg {
+ uint32_t xid;
+ uint32_t pos;
+ uint32_t len;
+ uint32_t npg;
+ uint32_t into_mem;
+ vm_page_t pg[];
+};
+
+/* Structure for a connection. */
+struct rpcrdma_xprt {
+ struct mtx mtx;
+ uint32_t credits;
+ uint32_t maxrpc;
+ uint32_t maxbck;
+ uint32_t maxio;
+ uint32_t maxsge;
+ struct mbuf *reply_small[RPCRDMA_IO_NUMBUFS];
+ struct mbuf *reply_large[RPCRDMA_IO_NUMBUFS];
+ void *ep;
+};
+
+#define RPCRDMA_MAX_SEGMENTS 16 /* Limit from RFC8267. */
+#define RPCRDMA_MAX_INLINE 1024 /* Limit from RFC8267. */
+
+#define RPCRDMA_MAX_SGE (16 + 2)
+struct rpcrdma_chunk {
+ int ind;
+ unsigned int first_off;
+ uint32_t last_len;
+ uint32_t num_segment;
+ uint32_t into_mem;
+ uint32_t sge_cnt[RPCRDMA_MAX_SEGMENTS];
+ uint32_t handle[RPCRDMA_MAX_SEGMENTS];
+ uint32_t length[RPCRDMA_MAX_SEGMENTS];
+ uint64_t offset[RPCRDMA_MAX_SEGMENTS];
+ struct mbuf *mextpg;
+};
+
+#define RPCRDMA_DEBUG(level, ...) do { \
+ if (rpcrdma_debuglevel >= (level)) \
+ printf(__VA_ARGS__); \
+ } while (0)
+
+extern int rpcrdma_debuglevel;
+
+/*
+ * Upcall to krpc or similar. Passes mbuf chain and context up.
+ */
+typedef void (*xprt_rdma_upcall)(struct rpcrdma_xprt *xp, struct mbuf *mp);
+
+void xprt_rdma_init(struct rpcrdma_xprt *xp, xprt_rdma_upcall rdma_upcall);
+
+int xprt_rdma_check_route(struct vnet *net, struct sockaddr *saddr,
+ uint32_t cbslots);
+
+int xprt_rdma_connect(struct vnet *net, struct sockaddr *saddr,
+ struct rpcrdma_xprt *xp, size_t buflen, uint32_t cbslots);
+
+void xprt_rdma_disconnect(struct rpcrdma_xprt *xp);
+
+int xprt_rdma_send(struct rpcrdma_xprt *xp, struct mbuf *mreq, int ind);
+
+void xprt_rdma_release_send(struct rpcrdma_xprt *xp, int ind,
+ struct rpcrdma_chunk *extern_chp, struct rpcrdma_chunk *extern_reply_chp,
+ struct rpcrdma_chunk *extern_request_chp);
+
+int xprt_rdma_recv(struct rpcrdma_xprt *xp);
+
+void xprt_rdma_release_ep(struct rpcrdma_xprt *xp);
+
+struct rpcrdma_chunk *xprt_rdma_create_chunk(struct rpcrdma_xprt *xp,
+ uint32_t num_pg, struct rpcrdma_reduce_pg *rb, struct mbuf *mextpg,
+ bool into_mem, int ind);
+
+int xprt_rdma_disconnected(struct rpcrdma_xprt *xp);
+
+int xprt_rdma_acquire_buf(struct rpcrdma_xprt *xp, int start, int end);
+
+int xprt_rdma_rekey_chunk(struct rpcrdma_xprt *xp, struct rpcrdma_chunk *chp);
+
+struct mbuf *rpc_reduce_pg(int len, int pos, bool to_mem);
+
+void rpc_free_rdma_reduction(struct mbuf *mr);
+
+int rpc_copy_uio_pages(struct mbuf *mr, struct uio *uiop, int len,
+ bool from_pages);
+
+void rpc_copy_mbuf_to_rb(struct mbuf *m, struct rpcrdma_reduce_pg *rb);
+#endif /* _KERNEL */
+
+#endif /* _RPC_CLNTRDMA_H_ */
diff --git a/sys/rpc/getnetconfig.c b/sys/rpc/getnetconfig.c
index eebe9dcecf6b..21d9921c4f47 100644
--- a/sys/rpc/getnetconfig.c
+++ b/sys/rpc/getnetconfig.c
@@ -55,6 +55,13 @@ static struct netconfig netconfigs[] = {
.nc_protofmly = "inet6",
.nc_proto = "tcp",
},
+ {
+ .nc_netid = "rdma6",
+ .nc_semantics = NC_TPI_COTS_ORD,
+ .nc_flag = NC_VISIBLE,
+ .nc_protofmly = "inet6",
+ .nc_proto = "rdma",
+ },
#endif
{
.nc_netid = "udp",
@@ -70,6 +77,13 @@ static struct netconfig netconfigs[] = {
.nc_protofmly = "inet",
.nc_proto = "tcp",
},
+ {
+ .nc_netid = "rdma",
+ .nc_semantics = NC_TPI_COTS_ORD,
+ .nc_flag = NC_VISIBLE,
+ .nc_protofmly = "inet",
+ .nc_proto = "rdma",
+ },
{
.nc_netid = "local",
.nc_semantics = NC_TPI_COTS_ORD,
diff --git a/sys/rpc/krpc.h b/sys/rpc/krpc.h
index f53b07ed86b6..e0dea323d190 100644
--- a/sys/rpc/krpc.h
+++ b/sys/rpc/krpc.h
@@ -42,10 +42,24 @@ enum clnt_stat clnt_bck_call(CLIENT *, struct rpc_callextra *, rpcproc_t,
struct mbuf *, struct mbuf **, struct timeval, SVCXPRT *);
struct mbuf *_rpc_copym_into_ext_pgs(struct mbuf *, int);
-/* Callback function for server side RDMA. */
+/* Callback functions for server side RDMA. */
typedef int clnt_bck_rdma_send_ftype(SVCXPRT *xprt, struct mbuf *m);
extern clnt_bck_rdma_send_ftype *clnt_bck_rdma_send;
+/* Functions for client side RDMA. */
+typedef int xprt_rdma_check_route_ftype(struct vnet *vnet,
+ struct sockaddr *dstaddr, uint32_t cbslots);
+extern xprt_rdma_check_route_ftype *rdma_check_route;
+
+typedef bool_t clnt_rdma_bcksend_ftype(SVCXPRT *xprt, struct mbuf *m);
+extern clnt_rdma_bcksend_ftype *clnt_rdma_bcksend_call;
+
+typedef CLIENT *clnt_rdma_create_ftype(struct sockaddr *raddr,
+ const rpcprog_t prog, const rpcvers_t vers, int intrflag,
+ uint32_t small_reply, uint32_t max_io, uint32_t cblots,
+ struct rpc_err *err);
+extern clnt_rdma_create_ftype *clnt_rdma_create_call;
+
/*
* A pending RPC request which awaits a reply. Requests which have
* received their reply will have cr_xid set to zero and cr_mrep to
@@ -89,6 +103,9 @@ struct rc_data {
void (*rc_reconcall)(CLIENT *, void *,
struct ucred *); /* reconection upcall */
void *rc_reconarg; /* upcall arg */
+ uint32_t rc_rdmasmall_reply; /* size of small reply */
+ uint32_t rc_rdmamax_io; /* size of largest I/O */
+ uint32_t rc_rdma_cbslots; /* Max. # of callbacks */
};
/* Bits for ct_rcvstate. */
@@ -136,6 +153,7 @@ struct cf_conn { /* kept in xprt->xp_p1 for actual connection */
struct mbuf *mreq; /* current record being built from mpending */
uint32_t resid; /* number of bytes needed for fragment */
bool_t eor; /* reading last fragment of current record */
+ bool_t rdma; /* On an RDMA connection. */
};
void rpcnl_init(void);
diff --git a/sys/rpc/netconfig.h b/sys/rpc/netconfig.h
index f492a1326a8d..c877f20ca899 100644
--- a/sys/rpc/netconfig.h
+++ b/sys/rpc/netconfig.h
@@ -76,6 +76,7 @@ typedef struct {
#define NC_TCP "tcp"
#define NC_UDP "udp"
#define NC_ICMP "icmp"
+#define NC_RDMA "rdma"
__BEGIN_DECLS
void *setnetconfig(void);
diff --git a/sys/rpc/rpc.h b/sys/rpc/rpc.h
index 3ca19868d5d3..64ac7c907c8f 100644
--- a/sys/rpc/rpc.h
+++ b/sys/rpc/rpc.h
@@ -115,6 +115,21 @@ int __rpc_fd2sockinfo(int, struct __rpc_sockinfo *);
struct socket *__rpc_nconf2socket(const struct netconfig *);
int __rpc_nconf2sockinfo(const struct netconfig *, struct __rpc_sockinfo *);
int __rpc_socket2sockinfo(struct socket *, struct __rpc_sockinfo *);
+
+/* Largest small RDMA message is 1Kbytes per RFC-8166. */
+#define RPCRDMA_MAX_SMALL_MSG 1024
+
+CLIENT *
+clnt_rdma_create(
+ struct sockaddr *raddr, /* servers address */
+ const rpcprog_t prog, /* program number */
+ const rpcvers_t vers, /* version number */
+ int intrflag, /* interruptible */
+ uint32_t small_reply, /* Max. small reply */
+ uint32_t max_io, /* size of largest RPC I/O */
+ uint32_t cbslots, /* Max. # of callbacks */
+ struct rpc_err *err); /* Return error and stat */
+
#endif
u_int __rpc_get_t_size(int, int, int);
__END_DECLS
diff --git a/sys/rpc/rpc_generic.c b/sys/rpc/rpc_generic.c
index b3cf2612bdbb..585d89a6931c 100644
--- a/sys/rpc/rpc_generic.c
+++ b/sys/rpc/rpc_generic.c
@@ -53,6 +53,7 @@
#include <sys/socket.h>
#include <sys/socketvar.h>
#include <sys/syslog.h>
+#include <sys/uio.h>
#include <net/vnet.h>
@@ -60,6 +61,7 @@
#include <rpc/nettype.h>
#include <rpc/rpcsec_gss.h>
#include <rpc/rpcsec_tls.h>
+#include <rpc/clntrdma.h>
#include <rpc/rpc_com.h>
#include <rpc/krpc.h>
@@ -67,6 +69,7 @@
#include <vm/vm.h>
#include <vm/pmap.h>
#include <vm/vm_param.h>
+#include <vm/vm_page.h>
extern u_long sb_max_adj; /* not defined in socketvar.h */
@@ -939,6 +942,139 @@ _rpc_copym_into_ext_pgs(struct mbuf *mp, int maxextsiz)
return (mhead);
}
+/*
+ * Create an mr mbuf and associated pages.
+ */
+struct mbuf *
+rpc_reduce_pg(int len, int pos, bool to_mem)
+{
+ struct mbuf *mr;
+ struct rpcrdma_reduce_pg *rb;
+ int i;
+
+ i = howmany(len, PAGE_SIZE);
+ mr = m_get2(sizeof(*rb) + sizeof(vm_page_t) * i, M_WAITOK, MT_DATA, 0);
+ mr->m_flags |= M_PROTO10;
+ mr->m_len = sizeof(*rb) + sizeof(vm_page_t) * i;
+ rb = mtod(mr, struct rpcrdma_reduce_pg *);
+ rb->len = len;
+ rb->npg = i;
+ rb->pos = pos;
+ for (i = 0; i < rb->npg; i++)
+ rb->pg[i] = vm_page_alloc_noobj(VM_ALLOC_WAITOK |
+ VM_ALLOC_NODUMP | VM_ALLOC_WIRED);
+ rb->into_mem = (to_mem) ? 1 : 0;
+ return (mr);
+}
+
+void
+rpc_free_rdma_reduction(struct mbuf *mr)
+{
+ struct rpcrdma_reduce_pg *rb;
+ int i;
+
+ rb = mtod(mr, struct rpcrdma_reduce_pg *);
+ for (i = 0; i < rb->npg; i++) {
+ vm_page_unwire_noq(rb->pg[i]);
+ vm_page_free(rb->pg[i]);
+ }
+ m_free(mr);
+}
+
+/*
+ * Copy data between anonymous pages and uiop.
+ */
+int
+rpc_copy_uio_pages(struct mbuf *mr, struct uio *uiop, int len, bool from_pages)
+{
+ struct rpcrdma_reduce_pg *rb;
+ struct iovec *iov;
+ int cplen, error, lastlen, i, xfer;
+ char *cp;
+
+ rb = mtod(mr, struct rpcrdma_reduce_pg *);
+ iov = uiop->uio_iov;
+ lastlen = rb->len % PAGE_SIZE;
+ if (lastlen == 0)
+ lastlen = PAGE_SIZE;
+ for (i = 0; i < rb->npg && len > 0; i++) {
+ xfer = (i == rb->npg - 1) ? lastlen : PAGE_SIZE;
+ xfer = MIN(xfer, len);
+ cp = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(rb->pg[i]));
+ while (xfer > 0) {
+ while (iov->iov_len == 0) {
+ if (uiop->uio_iovcnt > 0) {
+ iov++;
+ uiop->uio_iovcnt--;
+ } else {
+ return (ENOMEM);
+ }
+ }
+ cplen = MIN(iov->iov_len, xfer);
+ if (from_pages) {
+ if (uiop->uio_segflg == UIO_SYSSPACE) {
+ memcpy(iov->iov_base, cp, cplen);
+ } else {
+ error = copyout(cp, iov->iov_base,
+ cplen);
+ if (error != 0)
+ return (error);
+ }
+ } else {
+ if (uiop->uio_segflg == UIO_SYSSPACE) {
+ memcpy(cp, iov->iov_base, cplen);
+ } else {
+ error = copyin(iov->iov_base, cp,
+ cplen);
+ if (error != 0)
+ return (error);
+ }
+ }
+ iov->iov_len -= cplen;
+ iov->iov_base = (char *)iov->iov_base + cplen;
+ uiop->uio_offset += cplen;
+ uiop->uio_resid -= cplen;
+ xfer -= cplen;
+ cp += cplen;
+ len -= xfer;
+ }
+ }
+ return (0);
+}
+
+/*
+ * Copy data from an mbuf list into a list of pages in an rb.
+ */
+void
+rpc_copy_mbuf_to_rb(struct mbuf *m, struct rpcrdma_reduce_pg *rb)
+{
+ int i, j, k, plen;
+ char *cp, *pgp;
+
+ j = m->m_len;
+ cp = mtod(m, char *);
+ for (i = 0; i < rb->npg && m != NULL; i++) {
+ pgp = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(rb->pg[i]));
+ for (plen = PAGE_SIZE; plen > 0; ) {
+ k = MIN(j, plen);
+ memcpy(pgp, cp, k);
+ j -= k;
+ plen -= k;
+ pgp += k;
+ cp += k;
+ if (j == 0) {
+ m = m->m_next;
+ while (m != NULL && m->m_len == 0)
+ m = m->m_next;
+ if (m == NULL)
+ break;
+ cp = mtod(m, char *);
+ j = m->m_len;
+ }
+ }
+ }
+}
+
/*
* Kernel module glue
*/
diff --git a/sys/rpc/svc_vc.c b/sys/rpc/svc_vc.c
index 24c09f182174..49a58b567e8f 100644
--- a/sys/rpc/svc_vc.c
+++ b/sys/rpc/svc_vc.c
@@ -1063,6 +1063,7 @@ svc_vc_backchannel_reply(SVCXPRT *xprt, struct rpc_msg *msg,
struct sockaddr *addr, struct mbuf *m, uint32_t *seq)
{
struct ct_data *ct;
+ struct cf_conn *cd = (struct cf_conn *)xprt->xp_p1;
XDR xdrs;
struct mbuf *mrep;
bool_t stat = TRUE;
@@ -1075,7 +1076,8 @@ svc_vc_backchannel_reply(SVCXPRT *xprt, struct rpc_msg *msg,
* Leave space for record mark.
*/
mrep = m_gethdr(M_WAITOK, MT_DATA);
- mrep->m_data += sizeof(uint32_t);
+ if (!cd->rdma)
+ mrep->m_data += sizeof(uint32_t);
xdrmbuf_create(&xdrs, mrep, XDR_ENCODE);
@@ -1089,7 +1091,13 @@ svc_vc_backchannel_reply(SVCXPRT *xprt, struct rpc_msg *msg,
stat = xdr_replymsg(&xdrs, msg);
}
- if (stat) {
+ if (stat && cd->rdma) {
+ KASSERT(clnt_rdma_bcksend_call != NULL,
+ ("svc_vc_backchannel_reply: RDMA set, but "
+ "clnt_rdma_bcksend_call NULL"));
+
+ stat = clnt_rdma_bcksend_call(xprt, mrep);
+ } else if (stat) {
m_fixhdr(mrep);
/*