svn commit: r187829 - in user/alc/zerocopy/sys: kern sys
Alan Cox
alc at FreeBSD.org
Wed Jan 28 08:38:37 PST 2009
Author: alc
Date: Wed Jan 28 16:37:49 2009
New Revision: 187829
URL: http://svn.freebsd.org/changeset/base/187829
Log:
Adapt the zero-copy transmit mechanism to work with a new explicit API.
At the moment, the API is a straw man, and not a serious or complete
proposal. However, it does allow for testing the implementation.
Modified:
user/alc/zerocopy/sys/kern/uipc_cow.c
user/alc/zerocopy/sys/kern/uipc_socket.c
user/alc/zerocopy/sys/kern/uipc_syscalls.c
user/alc/zerocopy/sys/sys/socket.h
user/alc/zerocopy/sys/sys/socketvar.h
Modified: user/alc/zerocopy/sys/kern/uipc_cow.c
==============================================================================
--- user/alc/zerocopy/sys/kern/uipc_cow.c Wed Jan 28 16:23:21 2009 (r187828)
+++ user/alc/zerocopy/sys/kern/uipc_cow.c Wed Jan 28 16:37:49 2009 (r187829)
@@ -63,39 +63,12 @@ struct netsend_cow_stats {
int fail_not_mapped;
int fail_sf_buf;
int success;
- int iodone;
};
static struct netsend_cow_stats socow_stats;
-static void socow_iodone(void *addr, void *args);
-
-static void
-socow_iodone(void *addr, void *args)
-{
- struct sf_buf *sf;
- vm_page_t pp;
-
- sf = args;
- pp = sf_buf_page(sf);
- sf_buf_free(sf);
- /* remove COW mapping */
- vm_page_lock_queues();
- vm_page_cowclear(pp);
- vm_page_unwire(pp, 0);
- /*
- * Check for the object going away on us. This can
- * happen since we don't hold a reference to it.
- * If so, we're responsible for freeing the page.
- */
- if (pp->wire_count == 0 && pp->object == NULL)
- vm_page_free(pp);
- vm_page_unlock_queues();
- socow_stats.iodone++;
-}
-
int
-socow_setup(struct mbuf *m0, struct uio *uio)
+socow_setup(struct mbuf *m0, struct uio *uio, struct sendfile_sync *sfs)
{
struct sf_buf *sf;
vm_page_t pp;
@@ -125,19 +98,10 @@ socow_setup(struct mbuf *m0, struct uio
return(0);
}
- /*
- * set up COW
- */
- vm_page_lock_queues();
- if (vm_page_cowsetup(pp) != 0) {
- vm_page_unhold(pp);
- vm_page_unlock_queues();
- return (0);
- }
-
/*
* wire the page for I/O
*/
+ vm_page_lock_queues();
vm_page_wire(pp);
vm_page_unhold(pp);
vm_page_unlock_queues();
@@ -148,7 +112,6 @@ socow_setup(struct mbuf *m0, struct uio
sf = sf_buf_alloc(pp, SFB_CATCH);
if (!sf) {
vm_page_lock_queues();
- vm_page_cowclear(pp);
vm_page_unwire(pp, 0);
/*
* Check for the object going away on us. This can
@@ -164,12 +127,17 @@ socow_setup(struct mbuf *m0, struct uio
/*
* attach to mbuf
*/
- MEXTADD(m0, sf_buf_kva(sf), PAGE_SIZE, socow_iodone,
- (void*)sf_buf_kva(sf), sf, M_RDONLY, EXT_SFBUF);
+ MEXTADD(m0, sf_buf_kva(sf), PAGE_SIZE, sf_buf_mext,
+ sfs, sf, M_RDONLY, EXT_SFBUF);
m0->m_len = PAGE_SIZE - offset;
m0->m_data = (caddr_t)sf_buf_kva(sf) + offset;
socow_stats.success++;
+ if (sfs != NULL) {
+ mtx_lock(&sfs->mtx);
+ sfs->count++;
+ mtx_unlock(&sfs->mtx);
+ }
iov = uio->uio_iov;
iov->iov_base = (char *)iov->iov_base + m0->m_len;
iov->iov_len -= m0->m_len;
Modified: user/alc/zerocopy/sys/kern/uipc_socket.c
==============================================================================
--- user/alc/zerocopy/sys/kern/uipc_socket.c Wed Jan 28 16:23:21 2009 (r187828)
+++ user/alc/zerocopy/sys/kern/uipc_socket.c Wed Jan 28 16:37:49 2009 (r187829)
@@ -105,6 +105,7 @@ __FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/systm.h>
+#include <sys/condvar.h>
#include <sys/fcntl.h>
#include <sys/limits.h>
#include <sys/lock.h>
@@ -176,14 +177,11 @@ SYSCTL_INT(_kern_ipc, OID_AUTO, numopens
&numopensockets, 0, "Number of open sockets");
#ifdef ZERO_COPY_SOCKETS
/* These aren't static because they're used in other files. */
-int so_zero_copy_send = 1;
int so_zero_copy_receive = 1;
SYSCTL_NODE(_kern_ipc, OID_AUTO, zero_copy, CTLFLAG_RD, 0,
"Zero copy controls");
SYSCTL_INT(_kern_ipc_zero_copy, OID_AUTO, receive, CTLFLAG_RW,
&so_zero_copy_receive, 0, "Enable zero copy receive");
-SYSCTL_INT(_kern_ipc_zero_copy, OID_AUTO, send, CTLFLAG_RW,
- &so_zero_copy_send, 0, "Enable zero copy send");
#endif /* ZERO_COPY_SOCKETS */
/*
@@ -831,7 +829,7 @@ struct so_zerocopy_stats so_zerocp_stats
*/
static int
sosend_copyin(struct uio *uio, struct mbuf **retmp, int atomic, long *space,
- int flags)
+ int flags, struct sendfile_sync *sfs)
{
struct mbuf *m, **mp, *top;
long len, resid;
@@ -857,13 +855,13 @@ sosend_copyin(struct uio *uio, struct mb
m->m_pkthdr.rcvif = NULL;
} else
m = m_get(M_WAITOK, MT_DATA);
- if (so_zero_copy_send &&
+ if ((flags & (MSG_AZCS | MSG_SZCS)) != 0 &&
resid>=PAGE_SIZE &&
*space>=PAGE_SIZE &&
uio->uio_iov->iov_len>=PAGE_SIZE) {
so_zerocp_stats.size_ok++;
so_zerocp_stats.align_ok++;
- cow_send = socow_setup(m, uio);
+ cow_send = socow_setup(m, uio, sfs);
len = cow_send;
}
if (!cow_send) {
@@ -934,6 +932,7 @@ int
sosend_dgram(struct socket *so, struct sockaddr *addr, struct uio *uio,
struct mbuf *top, struct mbuf *control, int flags, struct thread *td)
{
+ struct sendfile_sync *sfs = NULL;
long space, resid;
int clen = 0, error, dontroute;
#ifdef ZERO_COPY_SOCKETS
@@ -963,6 +962,11 @@ sosend_dgram(struct socket *so, struct s
goto out;
}
+ if (flags & MSG_SZCS) {
+ sfs = malloc(sizeof *sfs, M_TEMP, M_WAITOK | M_ZERO);
+ mtx_init(&sfs->mtx, "sendfile", MTX_DEF, 0);
+ cv_init(&sfs->cv, "sendfile");
+ }
dontroute =
(flags & MSG_DONTROUTE) && (so->so_options & SO_DONTROUTE) == 0;
if (td != NULL)
@@ -1025,7 +1029,7 @@ sosend_dgram(struct socket *so, struct s
top->m_flags |= M_EOR;
} else {
#ifdef ZERO_COPY_SOCKETS
- error = sosend_copyin(uio, &top, atomic, &space, flags);
+ error = sosend_copyin(uio, &top, atomic, &space, flags, sfs);
if (error)
goto out;
#else
@@ -1088,6 +1092,15 @@ out:
m_freem(top);
if (control != NULL)
m_freem(control);
+ if (sfs != NULL) {
+ mtx_lock(&sfs->mtx);
+ if (sfs->count != 0)
+ cv_wait(&sfs->cv, &sfs->mtx);
+ KASSERT(sfs->count == 0, ("sendfile sync still busy"));
+ cv_destroy(&sfs->cv);
+ mtx_destroy(&sfs->mtx);
+ free(sfs, M_TEMP);
+ }
return (error);
}
@@ -1108,6 +1121,7 @@ int
sosend_generic(struct socket *so, struct sockaddr *addr, struct uio *uio,
struct mbuf *top, struct mbuf *control, int flags, struct thread *td)
{
+ struct sendfile_sync *sfs = NULL;
long space, resid;
int clen = 0, error, dontroute;
int atomic = sosendallatonce(so) || top;
@@ -1131,6 +1145,11 @@ sosend_generic(struct socket *so, struct
goto out;
}
+ if (flags & MSG_SZCS) {
+ sfs = malloc(sizeof(*sfs), M_TEMP, M_WAITOK | M_ZERO);
+ mtx_init(&sfs->mtx, "sendfile", MTX_DEF, 0);
+ cv_init(&sfs->cv, "sendfile");
+ }
dontroute =
(flags & MSG_DONTROUTE) && (so->so_options & SO_DONTROUTE) == 0 &&
(so->so_proto->pr_flags & PR_ATOMIC);
@@ -1213,7 +1232,7 @@ restart:
} else {
#ifdef ZERO_COPY_SOCKETS
error = sosend_copyin(uio, &top, atomic,
- &space, flags);
+ &space, flags, sfs);
if (error != 0)
goto release;
#else
@@ -1283,6 +1302,15 @@ out:
m_freem(top);
if (control != NULL)
m_freem(control);
+ if (sfs != NULL) {
+ mtx_lock(&sfs->mtx);
+ if (sfs->count != 0)
+ cv_wait(&sfs->cv, &sfs->mtx);
+ KASSERT(sfs->count == 0, ("sendfile sync still busy"));
+ cv_destroy(&sfs->cv);
+ mtx_destroy(&sfs->mtx);
+ free(sfs, M_TEMP);
+ }
return (error);
}
Modified: user/alc/zerocopy/sys/kern/uipc_syscalls.c
==============================================================================
--- user/alc/zerocopy/sys/kern/uipc_syscalls.c Wed Jan 28 16:23:21 2009 (r187828)
+++ user/alc/zerocopy/sys/kern/uipc_syscalls.c Wed Jan 28 16:37:49 2009 (r187829)
@@ -42,6 +42,7 @@ __FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/systm.h>
+#include <sys/condvar.h>
#include <sys/kernel.h>
#include <sys/lock.h>
#include <sys/mutex.h>
@@ -1676,14 +1677,6 @@ getsockaddr(namp, uaddr, len)
return (error);
}
-#include <sys/condvar.h>
-
-struct sendfile_sync {
- struct mtx mtx;
- struct cv cv;
- unsigned count;
-};
-
/*
* Detach mapped page and release resources back to the system.
*/
@@ -1867,8 +1860,7 @@ kern_sendfile(struct thread *td, struct
mnw = 1;
if (uap->flags & SF_SYNC) {
- sfs = malloc(sizeof *sfs, M_TEMP, M_WAITOK);
- memset(sfs, 0, sizeof *sfs);
+ sfs = malloc(sizeof *sfs, M_TEMP, M_WAITOK | M_ZERO);
mtx_init(&sfs->mtx, "sendfile", MTX_DEF, 0);
cv_init(&sfs->cv, "sendfile");
}
Modified: user/alc/zerocopy/sys/sys/socket.h
==============================================================================
--- user/alc/zerocopy/sys/sys/socket.h Wed Jan 28 16:23:21 2009 (r187828)
+++ user/alc/zerocopy/sys/sys/socket.h Wed Jan 28 16:37:49 2009 (r187829)
@@ -465,6 +465,8 @@ struct msghdr {
#endif
#if __BSD_VISIBLE
#define MSG_NOSIGNAL 0x20000 /* do not generate SIGPIPE on EOF */
+#define MSG_AZCS 0x100000 /* asynchronous zero-copy send */
+#define MSG_SZCS 0x200000 /* synchronous zero-copy send */
#endif
/*
Modified: user/alc/zerocopy/sys/sys/socketvar.h
==============================================================================
--- user/alc/zerocopy/sys/sys/socketvar.h Wed Jan 28 16:23:21 2009 (r187828)
+++ user/alc/zerocopy/sys/sys/socketvar.h Wed Jan 28 16:37:49 2009 (r187829)
@@ -39,6 +39,7 @@
#include <sys/_lock.h>
#include <sys/_mutex.h>
#include <sys/_sx.h>
+#include <sys/condvar.h>
#include <sys/sockbuf.h>
#include <sys/sockstate.h>
#ifdef _KERNEL
@@ -286,6 +287,12 @@ struct accept_filter {
SLIST_ENTRY(accept_filter) accf_next;
};
+struct sendfile_sync {
+ struct mtx mtx;
+ struct cv cv;
+ unsigned count;
+};
+
#ifdef MALLOC_DECLARE
MALLOC_DECLARE(M_ACCF);
MALLOC_DECLARE(M_PCB);
@@ -314,7 +321,8 @@ int sobind(struct socket *so, struct soc
int soclose(struct socket *so);
int soconnect(struct socket *so, struct sockaddr *nam, struct thread *td);
int soconnect2(struct socket *so1, struct socket *so2);
-int socow_setup(struct mbuf *m0, struct uio *uio);
+int socow_setup(struct mbuf *m0, struct uio *uio,
+ struct sendfile_sync *sfs);
int socreate(int dom, struct socket **aso, int type, int proto,
struct ucred *cred, struct thread *td);
int sodisconnect(struct socket *so);
More information about the svn-src-user
mailing list