svn commit: r196875 - in user/kmacy/releng_7_2_fcs_1/sys:
compat/freebsd32 kern sys
Kip Macy
kmacy at FreeBSD.org
Sun Sep 6 01:03:20 UTC 2009
Author: kmacy
Date: Sun Sep 6 01:03:19 2009
New Revision: 196875
URL: http://svn.freebsd.org/changeset/base/196875
Log:
- make parallel sendfile request handling more robust
- eliminate per-write memory allocation
Modified:
user/kmacy/releng_7_2_fcs_1/sys/compat/freebsd32/freebsd32_misc.c
user/kmacy/releng_7_2_fcs_1/sys/kern/uipc_sockbuf.c
user/kmacy/releng_7_2_fcs_1/sys/kern/uipc_socket.c
user/kmacy/releng_7_2_fcs_1/sys/kern/uipc_syscalls.c
user/kmacy/releng_7_2_fcs_1/sys/sys/sockbuf.h
user/kmacy/releng_7_2_fcs_1/sys/sys/socket.h
user/kmacy/releng_7_2_fcs_1/sys/sys/sockstate.h
user/kmacy/releng_7_2_fcs_1/sys/sys/syscallsubr.h
Modified: user/kmacy/releng_7_2_fcs_1/sys/compat/freebsd32/freebsd32_misc.c
==============================================================================
--- user/kmacy/releng_7_2_fcs_1/sys/compat/freebsd32/freebsd32_misc.c Sat Sep 5 23:23:49 2009 (r196874)
+++ user/kmacy/releng_7_2_fcs_1/sys/compat/freebsd32/freebsd32_misc.c Sun Sep 6 01:03:19 2009 (r196875)
@@ -1861,7 +1861,8 @@ freebsd32_do_sendfile(struct thread *td,
}
}
- error = kern_sendfile(td, &ap, hdr_uio, trl_uio, compat);
+ error = kern_sendfile(td, &ap, hdr_uio, trl_uio, compat,
+ NULL, NULL, NULL);
out:
if (hdr_uio)
free(hdr_uio, M_IOV);
Modified: user/kmacy/releng_7_2_fcs_1/sys/kern/uipc_sockbuf.c
==============================================================================
--- user/kmacy/releng_7_2_fcs_1/sys/kern/uipc_sockbuf.c Sat Sep 5 23:23:49 2009 (r196874)
+++ user/kmacy/releng_7_2_fcs_1/sys/kern/uipc_sockbuf.c Sun Sep 6 01:03:19 2009 (r196875)
@@ -133,7 +133,7 @@ sbwait(struct sockbuf *sb)
}
int
-sblock(struct sockbuf *sb, int flags)
+_sblock(struct sockbuf *sb, int flags, const char *file, int line)
{
KASSERT((flags & SBL_VALID) == flags,
@@ -142,12 +142,12 @@ sblock(struct sockbuf *sb, int flags)
if (flags & SBL_WAIT) {
if ((sb->sb_flags & SB_NOINTR) ||
(flags & SBL_NOINTR)) {
- sx_xlock(&sb->sb_sx);
+ _sx_xlock(&sb->sb_sx, 0, file, line);
return (0);
}
- return (sx_xlock_sig(&sb->sb_sx));
+ return (_sx_xlock(&sb->sb_sx, SX_INTERRUPTIBLE, file, line));
} else {
- if (sx_try_xlock(&sb->sb_sx) == 0)
+ if (_sx_try_xlock(&sb->sb_sx, file, line) == 0)
return (EWOULDBLOCK);
return (0);
}
@@ -882,8 +882,6 @@ sbdrop_internal(struct sockbuf *sb, int
}
}
-extern void sosendingwakeup(void *unused __unused);
-
/*
* Drop data from (the front of) a sockbuf.
*/
@@ -895,7 +893,7 @@ sbdrop_locked(struct sockbuf *sb, int le
sbdrop_internal(sb, len);
if (sb->sb_flags & SB_SENDING)
- sosendingwakeup(NULL);
+ sosendingwakeup(sb);
}
void
Modified: user/kmacy/releng_7_2_fcs_1/sys/kern/uipc_socket.c
==============================================================================
--- user/kmacy/releng_7_2_fcs_1/sys/kern/uipc_socket.c Sat Sep 5 23:23:49 2009 (r196874)
+++ user/kmacy/releng_7_2_fcs_1/sys/kern/uipc_socket.c Sun Sep 6 01:03:19 2009 (r196875)
@@ -133,6 +133,7 @@ __FBSDID("$FreeBSD$");
#include <sys/sysproto.h>
#include <sys/taskqueue.h>
#include <sys/uio.h>
+#include <sys/vnode.h>
#include <sys/jail.h>
#include <security/mac/mac_framework.h>
@@ -3108,45 +3109,77 @@ soisdisconnected(struct socket *so)
struct socketref {
struct proc *sr_proc;
+ struct ucred *sr_ucred;
struct file *sr_sock_fp;
struct file *sr_fp;
+ struct socket *sr_so;
struct sendfile_args sr_uap;
struct uio sr_hdr_uio;
struct uio sr_trl_uio;
- int sr_compat;
+ short sr_compat;
int sr_magic;
+ struct task sr_task;
TAILQ_ENTRY(socketref) entry;
};
TAILQ_HEAD(srq, socketref);
-struct socketref_object {
- struct srq sro_srh;
- struct task sro_task;
-};
-
struct srq *sendfile_bg_queue;
struct mtx sendfile_bg_lock;
struct callout *sendfile_callout;
struct taskqueue *sendfile_tq;
extern int getsock(struct filedesc *fdp, int fd,
struct file **fpp, u_int *fflagp);
+static void sendfile_task_func(void *context, int pending __unused);
+static int srsendingwakeup(struct socketref *sr);
MALLOC_DEFINE(M_SOCKREF, "sockref", "socket reference memory");
+#define SOCKBUF_LOCK_COND(sb, lockflag) do { \
+ if ((lockflag)) \
+ SOCKBUF_LOCK((sb)); \
+} while (0)
+
+#define SOCKBUF_UNLOCK_COND(sb, lockflag) do { \
+ if ((lockflag)) \
+ SOCKBUF_UNLOCK((sb)); \
+} while (0)
+
+
+static void
+socketref_free(struct socketref *sr)
+{
+ struct thread *td;
+ struct file *fp = sr->sr_fp;
+ struct file *sock_fp = sr->sr_sock_fp;
+ struct proc *p = sr->sr_proc;
+ struct ucred *cred = sr->sr_ucred;
+
+ if (cred != NULL)
+ crfree(cred);
+ vrele(fp->f_vnode);
+ fdrop(fp, NULL);
+ fdrop(sock_fp, NULL);
+ PRELE(p);
+#ifdef INVARIANTS
+ bzero(sr, sizeof(*sr));
+#endif
+ free(sr, M_SOCKREF);
+}
+
void
soissending(struct socket *so, struct thread *td,
struct sendfile_args *uap, struct uio *hdr_uio,
- struct uio *trl_uio, int compat)
+ struct uio *trl_uio, int compat, int sbytes)
{
struct socketref *ref;
- struct srq *srh;
int error;
struct socket *refso;
+ struct vnode *vp;
SOCKBUF_LOCK_ASSERT(&so->so_snd);
ref = malloc(sizeof(struct socketref),
- M_SOCKREF, M_NOWAIT);
+ M_SOCKREF, M_NOWAIT|M_ZERO);
if (ref == NULL)
return;
/*
@@ -3154,36 +3187,43 @@ soissending(struct socket *so, struct th
* drop when done sending
*/
so->so_snd.sb_flags |= SB_SENDING;
+ PROC_LOCK(td->td_proc);
+ td->td_proc->p_lock++;
+ PROC_UNLOCK(td->td_proc);
+
ref->sr_proc = td->td_proc;
if ((error = getsock(td->td_proc->p_fd, uap->s, &ref->sr_sock_fp,
NULL)) != 0) {
- free(ref, M_DEVBUF);
- return;
+ goto error;
}
if (ref->sr_sock_fp->f_type != DTYPE_SOCKET) {
printf("socket descriptor s=%d is not socket", uap->s);
- free(ref, M_DEVBUF);
- return;
+ goto error;
}
refso = ref->sr_sock_fp->f_data;
if (refso != so) {
printf("socket mismatch between refso: %p so: %p\n",
refso, so);
- free(ref, M_DEVBUF);
- return;
+ goto error_sock_fp;
}
-
+ ref->sr_so = refso;
+
if ((error = fget(td, uap->fd, &ref->sr_fp)) != 0) {
- fdrop(ref->sr_sock_fp, td);
- free(ref, M_DEVBUF);
- return;
+ goto error_sock_fp;
+ } else if (ref->sr_fp->f_vnode != NULL) {
+ vp = ref->sr_fp->f_vnode;
+ vref(vp);
+ } else {
+ goto error_fp;
}
bcopy(uap, &ref->sr_uap, sizeof(*uap));
ref->sr_uap.sbytes = NULL;
-
+ ref->sr_uap.offset += sbytes;
+ if (uap->nbytes)
+ ref->sr_uap.nbytes -= sbytes;
/*
* XXX
* We have to malloc memory for the uio data
@@ -3196,231 +3236,192 @@ soissending(struct socket *so, struct th
sizeof(*trl_uio));
ref->sr_compat = compat;
ref->sr_magic = 0xCAFEBABE;
+ TASK_INIT(&ref->sr_task, 0, sendfile_task_func, ref);
+
CTR3(KTR_SPARE2, "enqueueing socket %p sock_fp %p s %d", so, ref->sr_sock_fp, uap->s);
mtx_lock(&sendfile_bg_lock);
- srh = sendfile_bg_queue;
- TAILQ_INSERT_HEAD(srh, ref, entry);
+ TAILQ_INSERT_TAIL(sendfile_bg_queue, ref, entry);
mtx_unlock(&sendfile_bg_lock);
-}
-
-static void
-socketref_free(struct socketref *sr)
-{
- struct thread *td = curthread;
-
- fdrop(sr->sr_sock_fp, td);
- fdrop(sr->sr_fp, td);
- free(sr, M_SOCKREF);
+ return;
+error_fp:
+ fdrop(ref->sr_fp, td);
+error_sock_fp:
+ fdrop(ref->sr_sock_fp, td);
+error:
+ free(ref, M_DEVBUF);
}
static void
sendfile_task_func(void *context, int pending __unused)
{
- struct socketref_object *sro;
- struct srq *sh;
- struct socketref *sr, *srtmp;
+ struct socketref *sr;
struct socket *so;
struct sockbuf *sb;
- struct proc *p;
- struct thread *td;
struct file *sock_fp, *fp;
int error, writeable;
+ struct uio *hdr_uio = NULL, *trl_uio = NULL;
+ off_t sbytes;
- sro = context;
- sh = &sro->sro_srh;
- td = curthread;
-
+ sr = context;
CTR0(KTR_SPARE2, "task_func running");
- while (!TAILQ_EMPTY(sh)) {
- sr = TAILQ_FIRST(sh);
- TAILQ_REMOVE(sh, sr, entry);
- if (sr->sr_magic != 0xCAFEBABE) {
- printf("bad magic! 0x%x\n", sr->sr_magic);
- continue;
- }
- p = td->td_proc;
- td->td_proc = sr->sr_proc;
- sock_fp = sr->sr_sock_fp;
-
- CTR2(KTR_SPARE2, "processing sr %p sock_fp %p", sr, sock_fp);
- if (sock_fp->f_type != DTYPE_SOCKET)
- goto done;
+ if (sr->sr_magic != 0xCAFEBABE) {
+ printf("bad magic! 0x%x\n", sr->sr_magic);
+ /* XXX memory leak */
+ return;
+ }
+
+ sock_fp = sr->sr_sock_fp;
+ fp = sr->sr_fp;
+ CTR2(KTR_SPARE2, "processing sr %p sock_fp %p", sr, sock_fp);
+ if (sock_fp->f_type != DTYPE_SOCKET)
+ goto done;
- so = sock_fp->f_data;
- CTR1(KTR_SPARE2, "task processing socket %p", so);
+ so = sock_fp->f_data;
+ CTR1(KTR_SPARE2, "task processing socket %p", so);
- if ((so->so_state & SS_ISCONNECTED) == 0)
- goto done;
- sb = &so->so_snd;
- fp = sr->sr_fp;
+ if ((so->so_state & SS_ISCONNECTED) == 0)
+ goto done;
- SOCKBUF_LOCK(sb);
- sb->sb_flags &= ~SB_SENDING;
- if (so->so_snd.sb_state & SBS_CANTSENDMORE) {
- CTR1(KTR_SPARE2, "task expired socket %p", so);
- sowwakeup_locked(so);
- } else if (sowriteable(so)) {
- off_t sbytes;
+ if (sr->sr_ucred == NULL &&
+ (sr->sr_ucred = crdup(sr->sr_proc->p_ucred)) == NULL)
+ goto done;
- sb->sb_flags |= SB_SENDING;
+ sb = &so->so_snd;
+ SOCKBUF_UNLOCK_ASSERT(sb);
+ SOCKBUF_LOCK(sb);
+ sb->sb_flags &= ~SB_SENDING;
+ if (sb->sb_state & SBS_CANTSENDMORE) {
+ CTR1(KTR_SPARE2, "SBS_CANTSENDMORE - socket %p", so);
+ sowwakeup_locked(so);
+ goto done;
+ } else if (sowriteable(so)) {
+ sb->sb_flags |= SB_SENDING;
+ SOCKBUF_UNLOCK(sb);
+ if (sr->sr_hdr_uio.uio_td != NULL)
+ hdr_uio = &sr->sr_hdr_uio;
+ if (sr->sr_trl_uio.uio_td != NULL)
+ trl_uio = &sr->sr_trl_uio;
+
+ sr->sr_uap.sbytes = &sbytes;
+ sr->sr_uap.flags |= SF_TASKQ;
+ CTR1(KTR_SPARE2, "task sending on socket %p", so);
+
+ error = kern_sendfile(curthread, &sr->sr_uap,
+ hdr_uio, trl_uio,
+ sr->sr_compat, fp, so, sr->sr_ucred);
+ atomic_add_long(&fp->f_sfbytes, sbytes);
+ sr->sr_uap.offset += sbytes;
+ if (sr->sr_uap.nbytes)
+ sr->sr_uap.nbytes -= sbytes;
+ /*
+ * XXX we have a race here
+ * - if sbdrop is called before a re-enqueue,
+ * we'll have a lost wakeup ... maybe call
+ * sosendingwakup? Or check for sowriteable(so)
+ */
+ SOCKBUF_LOCK(sb);
+ if (error == EAGAIN && srsendingwakeup(sr) != ENOTCONN) {
SOCKBUF_UNLOCK(sb);
- sr->sr_uap.sbytes = &sbytes;
- CTR1(KTR_SPARE2, "task sending on socket %p", so);
- error = kern_sendfile(td, &sr->sr_uap,
- &sr->sr_hdr_uio, &sr->sr_trl_uio,
- sr->sr_compat);
- atomic_add_long(&fp->f_sfbytes, sbytes);
- if (error != EAGAIN) {
- SOCKBUF_LOCK(sb);
- sb->sb_flags &= ~SB_SENDING;
- sowwakeup_locked(so);
- }
+ return;
}
- td->td_proc = p;
- done:
- fdrop(fp, td);
- fdrop(sr->sr_sock_fp, td);
- free(sr, M_DEVBUF);
- }
- free(sro, M_DEVBUF);
+ }
+ sb->sb_flags &= ~SB_SENDING;
+ sowwakeup_locked(so);
+done:
+ SOCKBUF_UNLOCK_ASSERT(sb);
+ socketref_free(sr);
}
-#define SOCKBUF_LOCK_COND(sb, lockflag) do { \
- if ((lockflag)) \
- SOCKBUF_LOCK((sb)); \
-} while (0)
-
-#define SOCKBUF_UNLOCK_COND(sb, lockflag) do { \
- if ((lockflag)) \
- SOCKBUF_UNLOCK((sb)); \
-} while (0)
-
-
-void
-sosendingwakeup(void *unused __unused)
+static int
+srsendingwakeup(struct socketref *sr)
{
- struct socketref *sr, *srtmp;
- struct srq *srh_local, *srh_global, srh_tmp;
- struct socketref_object *sro;
- struct task *srh_task;
struct socket *so;
- struct sockbuf *sb;
struct file *fp;
- struct proc *p;
- struct thread *td;
- int writeable, sblockneeded;
+ struct sockbuf *sb;
+
+ if (sr->sr_magic != 0xCAFEBABE) {
+ printf("bad magic! sr: %p magic : 0x%x in %s\n",
+ sr, sr->sr_magic, __FUNCTION__);
+ /*
+ * XXX leak - should be assert perhaps
+ *
+ */
+ return (0);
+ }
- srh_global = sendfile_bg_queue;
- if (!TAILQ_EMPTY(srh_global)) {
- TAILQ_INIT(&srh_tmp);
+ fp = sr->sr_sock_fp;
+ CTR2(KTR_SPARE2, "processing s %d sock_fp %p", sr->sr_uap.s, fp);
+ if (fp->f_type != DTYPE_SOCKET) {
+ CTR1(KTR_SPARE2, "not socket - type %d", fp->f_type);
+ goto error;
+ }
+ so = fp->f_data;
+ if ((so->so_state & SS_ISCONNECTED) == 0) {
+ CTR0(KTR_SPARE2, "not connected %p");
+ goto error;
+ }
+
+ CTR1(KTR_SPARE2, "processing socket %p", so);
+ sb = &so->so_snd;
+ SOCKBUF_LOCK_ASSERT(sb);
+ if (sb->sb_state & SBS_CANTSENDMORE) {
+ ;
+ } else if (sowriteable(so)) {
+ CTR2(KTR_SPARE2, "enqueue socket to task %p sr %p", so, sr);
+ sb->sb_flags |= SB_SENDING;
+ taskqueue_enqueue(sendfile_tq, &sr->sr_task);
+ } else {
mtx_lock(&sendfile_bg_lock);
- TAILQ_CONCAT(&srh_tmp, srh_global, entry);
+ TAILQ_INSERT_TAIL(sendfile_bg_queue, sr, entry);
mtx_unlock(&sendfile_bg_lock);
- if (TAILQ_EMPTY(&srh_tmp))
- goto done;
+ }
+ return (0);
+error:
+ return (ENOTCONN);
+}
- if ((sro = malloc(sizeof(struct socketref_object),
- M_DEVBUF, M_NOWAIT)) == NULL)
- goto done;
-
- srh_local = &sro->sro_srh;
- srh_task = &sro->sro_task;
- TAILQ_INIT(srh_local);
- TASK_INIT(srh_task, 0, sendfile_task_func, sro);
- CTR0(KTR_SPARE2, "processing pcpu list");
- } else
- goto done;
+void
+sosendingwakeup(struct sockbuf *sb)
+{
+ struct socketref *sr = NULL;
- td = curthread;
- p = td->td_proc;
- TAILQ_FOREACH_SAFE(sr, &srh_tmp, entry, srtmp) {
- fp = sr->sr_sock_fp;
- td->td_proc = sr->sr_proc;
- CTR2(KTR_SPARE2, "processing s %d sock_fp %p", sr->sr_uap.s, fp);
-
- if (fp->f_type != DTYPE_SOCKET) {
- CTR1(KTR_SPARE2, "not socket - type %d", fp->f_type);
- goto next;
- }
- so = fp->f_data;
- if ((so->so_state & SS_ISCONNECTED) == 0) {
- CTR0(KTR_SPARE2, "not connected %p");
- goto next;
+ SOCKBUF_LOCK_ASSERT(sb);
+ mtx_lock(&sendfile_bg_lock);
+ if (!TAILQ_EMPTY(sendfile_bg_queue)) {
+ TAILQ_FOREACH(sr, sendfile_bg_queue, entry) {
+ if (sb == &sr->sr_so->so_snd) {
+ sb->sb_flags &= ~SB_SENDING;
+ TAILQ_REMOVE(sendfile_bg_queue, sr, entry);
+ break;
+ }
}
- CTR1(KTR_SPARE2, "processing socket %p", so);
- sb = &so->so_snd;
- sblockneeded = !SOCKBUF_OWNED(sb);
- writeable = 0;
- SOCKBUF_LOCK_COND(sb, sblockneeded);
- sb->sb_flags &= ~SB_SENDING;
- if (sb->sb_state & SBS_CANTSENDMORE) {
- SOCKBUF_UNLOCK_COND(sb, sblockneeded);
- goto next;
- } else {
- writeable = sowriteable(so);
- sb->sb_flags |= SB_SENDING;
- SOCKBUF_UNLOCK_COND(sb, sblockneeded);
- }
-
- if (writeable) {
- CTR2(KTR_SPARE2, "enqueue socket to task %p sr %p", so, sr);
- TAILQ_REMOVE(&srh_tmp, sr, entry);
- TAILQ_INSERT_HEAD(srh_local, sr, entry);
- }
- if (sr->sr_magic != 0xCAFEBABE)
- printf("bad magic! 0x%x in %s\n",
- sr->sr_magic, __FUNCTION__);
-
- continue;
- next:
- CTR1(KTR_SPARE2, "freeing expired socket %p", so);
- TAILQ_REMOVE(&srh_tmp, sr, entry);
- socketref_free(sr);
- }
- td->td_proc = p;
- if (!TAILQ_EMPTY(&srh_tmp)) {
- mtx_lock(&sendfile_bg_lock);
- TAILQ_CONCAT(srh_global, &srh_tmp, entry);
- mtx_unlock(&sendfile_bg_lock);
}
+ mtx_unlock(&sendfile_bg_lock);
- if (!TAILQ_EMPTY(srh_local)) {
- taskqueue_enqueue(sendfile_tq, srh_task);
- } else {
- free(sro, M_DEVBUF);
+ /*
+ * Buffer in flight
+ */
+ if (sr != NULL && srsendingwakeup(sr) == ENOTCONN) {
+ CTR2(KTR_SPARE2, "freeing expired socket %p ref %p",
+ sr->sr_so, sr);
+ socketref_free(sr);
}
-done:
- if (!callout_pending(sendfile_callout))
- callout_reset(sendfile_callout, MAX(hz/10, 1),
- sosendingwakeup, NULL);
}
static void
init_bgsend(void *unused __unused)
{
- struct srq *srh;
sendfile_tq = taskqueue_create("sendfile background taskq", M_NOWAIT,
taskqueue_thread_enqueue, &sendfile_tq);
- taskqueue_start_threads(&sendfile_tq, 1, PI_NET,
+ taskqueue_start_threads(&sendfile_tq, 4, PI_SOFT,
"sendfile background taskq");
- printf("init_bgsend mp_maxid: %d all_cpus 0x%x\n",
- mp_maxid, all_cpus);
-
mtx_init(&sendfile_bg_lock, "sendfile bg", NULL, MTX_DEF);
- sendfile_callout = malloc(sizeof(struct callout),
+ sendfile_bg_queue = malloc(sizeof(struct srq),
M_DEVBUF, M_NOWAIT);
- srh = sendfile_bg_queue = malloc(sizeof(struct srq),
- M_DEVBUF, M_NOWAIT);
- TAILQ_INIT(srh);
-
- callout_init(sendfile_callout, TRUE);
- callout_reset(sendfile_callout, MAX(hz/10, 1),
- sosendingwakeup, NULL);
-
- printf("init_bgsend done\n");
+ TAILQ_INIT(sendfile_bg_queue);
}
SYSINIT(init_bgsend, SI_SUB_SMP, SI_ORDER_ANY, init_bgsend, NULL);
Modified: user/kmacy/releng_7_2_fcs_1/sys/kern/uipc_syscalls.c
==============================================================================
--- user/kmacy/releng_7_2_fcs_1/sys/kern/uipc_syscalls.c Sat Sep 5 23:23:49 2009 (r196874)
+++ user/kmacy/releng_7_2_fcs_1/sys/kern/uipc_syscalls.c Sun Sep 6 01:03:19 2009 (r196875)
@@ -1747,7 +1747,8 @@ do_sendfile(struct thread *td, struct se
}
}
- error = kern_sendfile(td, uap, hdr_uio, trl_uio, compat);
+ error = kern_sendfile(td, uap, hdr_uio, trl_uio, compat,
+ NULL, NULL, NULL);
out:
if (hdr_uio)
free(hdr_uio, M_IOV);
@@ -1776,7 +1777,8 @@ freebsd4_sendfile(struct thread *td, str
int
kern_sendfile(struct thread *td, struct sendfile_args *uap,
- struct uio *hdr_uio, struct uio *trl_uio, int compat)
+ struct uio *hdr_uio, struct uio *trl_uio, int compat,
+ struct file *bgfp, struct socket *bgso, struct ucred *bgcred)
{
struct file *sock_fp, *fp = NULL;
struct vnode *vp;
@@ -1785,30 +1787,39 @@ kern_sendfile(struct thread *td, struct
struct mbuf *m = NULL;
struct sf_buf *sf;
struct vm_page *pg;
+ struct ucred *cred;
off_t off, xfsize, fsbytes = 0, sbytes = 0, rem = 0;
int error, hdrlen = 0, mnw = 0;
int vfslocked;
+ if (bgcred != NULL)
+ cred = bgcred;
+ else
+ cred = td->td_ucred;
+
/*
* The file descriptor must be a regular file and have a
* backing VM object.
* File offset must be positive. If it goes beyond EOF
* we send only the header/trailer and no payload data.
*/
- if ((error = fget_read(td, uap->fd, &fp)) != 0)
- goto out;
- else {
- if (fp->f_vnode == NULL) {
- fdrop(fp, td);
- error = EINVAL;
+ if ((uap->flags & SF_TASKQ) == 0) {
+ if ((error = fget_read(td, uap->fd, &fp)) != 0)
goto out;
- } else {
- vp = fp->f_vnode;
- vref(vp);
+ else {
+ if (fp->f_vnode == NULL) {
+ fdrop(fp, td);
+ error = EINVAL;
+ goto out;
+ } else {
+ vp = fp->f_vnode;
+ vref(vp);
+ }
}
+ } else {
+ vp = bgfp->f_vnode;
}
-
-
+
vfslocked = VFS_LOCK_GIANT(vp->v_mount);
vn_lock(vp, LK_SHARED | LK_RETRY, curthread);
if (vp->v_type == VREG) {
@@ -1841,30 +1852,35 @@ kern_sendfile(struct thread *td, struct
goto out;
}
- /*
- * The socket must be a stream socket and connected.
- * Remember if it a blocking or non-blocking socket.
- */
- if ((error = getsock(td->td_proc->p_fd, uap->s, &sock_fp,
- NULL)) != 0)
- goto out;
- so = sock_fp->f_data;
- if (so->so_type != SOCK_STREAM) {
- error = EINVAL;
- goto out;
- }
- if ((so->so_state & SS_ISCONNECTED) == 0) {
- error = ENOTCONN;
- goto out;
+ if ((uap->flags & SF_TASKQ) == 0) {
+ /*
+ * The socket must be a stream socket and connected.
+ * Remember if it a blocking or non-blocking socket.
+ */
+ if ((error = getsock(td->td_proc->p_fd, uap->s, &sock_fp,
+ NULL)) != 0)
+ goto out;
+ so = sock_fp->f_data;
+ if (so->so_type != SOCK_STREAM) {
+ error = EINVAL;
+ goto out;
+ }
+ if ((so->so_state & SS_ISCONNECTED) == 0) {
+ error = ENOTCONN;
+ goto out;
+ }
+ } else {
+ so = bgso;
}
SOCKBUF_LOCK(&so->so_snd);
- if (((so->so_snd.sb_flags & SB_SENDING) == 0) && fp->f_sfbytes != 0) {
+ if ((uap->flags & SF_TASKQ) == 0 &&
+ sock_fp->f_sfbytes != 0) {
SOCKBUF_UNLOCK(&so->so_snd);
if (uap->sbytes != NULL) {
copyout(&sbytes, uap->sbytes, sizeof(off_t));
+ sock_fp->f_sfbytes = 0;
}
- fp->f_sfbytes = 0;
error = 0;
goto out;
}
@@ -1880,7 +1896,7 @@ kern_sendfile(struct thread *td, struct
#ifdef MAC
SOCK_LOCK(so);
- error = mac_check_socket_send(td->td_ucred, so);
+ error = mac_check_socket_send(cred, so);
SOCK_UNLOCK(so);
if (error)
goto out;
@@ -1971,7 +1987,8 @@ retry_space:
(space <= 0 ||
space < so->so_snd.sb_lowat)) {
if (so->so_state & SS_NBIO) {
- soissending(so, td, uap, hdr_uio, trl_uio, compat);
+ if ((so->so_snd.sb_flags & SB_SENDING) == 0)
+ soissending(so, td, uap, hdr_uio, trl_uio, compat, sbytes);
SOCKBUF_UNLOCK(&so->so_snd);
error = EAGAIN;
goto done;
@@ -2090,7 +2107,7 @@ retry_space:
error = vn_rdwr(UIO_READ, vp, NULL, MAXBSIZE,
trunc_page(off), UIO_NOCOPY, IO_NODELOCKED |
IO_VMIO | ((MAXBSIZE / bsize) << IO_SEQSHIFT),
- td->td_ucred, NOCRED, &resid, td);
+ cred, NOCRED, &resid, td);
VOP_UNLOCK(vp, 0, curthread);
VFS_UNLOCK_GIANT(vfslocked);
VM_OBJECT_LOCK(obj);
@@ -2233,15 +2250,17 @@ out:
}
if (obj != NULL)
vm_object_deallocate(obj);
- if (vp != NULL) {
- vfslocked = VFS_LOCK_GIANT(vp->v_mount);
- vrele(vp);
- VFS_UNLOCK_GIANT(vfslocked);
+ if ((uap->flags & SF_TASKQ) == 0) {
+ if (vp != NULL) {
+ vfslocked = VFS_LOCK_GIANT(vp->v_mount);
+ vrele(vp);
+ VFS_UNLOCK_GIANT(vfslocked);
+ }
+ if (so)
+ fdrop(sock_fp, td);
+ if (fp)
+ fdrop(fp, td);
}
- if (so)
- fdrop(sock_fp, td);
- if (fp)
- fdrop(fp, td);
if (m)
m_freem(m);
Modified: user/kmacy/releng_7_2_fcs_1/sys/sys/sockbuf.h
==============================================================================
--- user/kmacy/releng_7_2_fcs_1/sys/sys/sockbuf.h Sat Sep 5 23:23:49 2009 (r196874)
+++ user/kmacy/releng_7_2_fcs_1/sys/sys/sockbuf.h Sun Sep 6 01:03:19 2009 (r196875)
@@ -153,9 +153,12 @@ struct mbuf *
sbsndptr(struct sockbuf *sb, u_int off, u_int len, u_int *moff);
void sbtoxsockbuf(struct sockbuf *sb, struct xsockbuf *xsb);
int sbwait(struct sockbuf *sb);
-int sblock(struct sockbuf *sb, int flags);
+int _sblock(struct sockbuf *sb, int flags, const char *file, int line);
void sbunlock(struct sockbuf *sb);
+#define sblock(sb, flags) \
+ _sblock((sb), (flags), __FILE__, __LINE__)
+
/*
* How much space is there in a socket buffer (so->so_snd or so->so_rcv)?
* This is problematical if the fields are unsigned, as the space might
Modified: user/kmacy/releng_7_2_fcs_1/sys/sys/socket.h
==============================================================================
--- user/kmacy/releng_7_2_fcs_1/sys/sys/socket.h Sat Sep 5 23:23:49 2009 (r196874)
+++ user/kmacy/releng_7_2_fcs_1/sys/sys/socket.h Sun Sep 6 01:03:19 2009 (r196875)
@@ -616,6 +616,7 @@ struct sf_hdtr {
*/
#define SF_NODISKIO 0x00000001
#define SF_MNOWAIT 0x00000002
+#define SF_TASKQ 0x00000004
#endif
#ifndef _KERNEL
Modified: user/kmacy/releng_7_2_fcs_1/sys/sys/sockstate.h
==============================================================================
--- user/kmacy/releng_7_2_fcs_1/sys/sys/sockstate.h Sat Sep 5 23:23:49 2009 (r196874)
+++ user/kmacy/releng_7_2_fcs_1/sys/sys/sockstate.h Sun Sep 6 01:03:19 2009 (r196875)
@@ -80,7 +80,9 @@ void soisdisconnected(struct socket *so)
void soisdisconnecting(struct socket *so);
void soissending(struct socket *so,
struct thread *td, struct sendfile_args *uap,
- struct uio *hdr_uio, struct uio *trl_uio, int compat);
+ struct uio *hdr_uio, struct uio *trl_uio,
+ int compat, int sbytes);
+void sosendingwakeup(struct sockbuf *sb);
void socantrcvmore(struct socket *so);
void socantrcvmore_locked(struct socket *so);
void socantsendmore(struct socket *so);
Modified: user/kmacy/releng_7_2_fcs_1/sys/sys/syscallsubr.h
==============================================================================
--- user/kmacy/releng_7_2_fcs_1/sys/sys/syscallsubr.h Sat Sep 5 23:23:49 2009 (r196874)
+++ user/kmacy/releng_7_2_fcs_1/sys/sys/syscallsubr.h Sun Sep 6 01:03:19 2009 (r196875)
@@ -144,7 +144,8 @@ int kern_semctl(struct thread *td, int s
int kern_select(struct thread *td, int nd, fd_set *fd_in, fd_set *fd_ou,
fd_set *fd_ex, struct timeval *tvp);
int kern_sendfile(struct thread *td, struct sendfile_args *uap,
- struct uio *hdr_uio, struct uio *trl_uio, int compat);
+ struct uio *hdr_uio, struct uio *trl_uio, int compat,
+ struct file *bgfp, struct socket *bgso, struct ucred *bgcred);
int kern_sendit(struct thread *td, int s, struct msghdr *mp, int flags,
struct mbuf *control, enum uio_seg segflg);
int kern_setgroups(struct thread *td, u_int ngrp, gid_t *groups);
More information about the svn-src-user
mailing list