git: 2247f4894174 - main - aio: micro-optimize the lio_opcode assignments
Alan Somers
asomers at FreeBSD.org
Wed Jan 20 16:03:45 UTC 2021
The branch main has been updated by asomers:
URL: https://cgit.FreeBSD.org/src/commit/?id=2247f48941743cab420394b5ff0fc65ec8c69a99
commit 2247f48941743cab420394b5ff0fc65ec8c69a99
Author: Alan Somers <asomers at FreeBSD.org>
AuthorDate: 2021-01-03 04:25:05 +0000
Commit: Alan Somers <asomers at FreeBSD.org>
CommitDate: 2021-01-20 16:02:25 +0000
aio: micro-optimize the lio_opcode assignments
This allows slightly more efficient opcode testing in-kernel. It is
transparent to userland, except to applications that sneakily submit
aio fsync or aio mlock operations via lio_listio, which has never been
documented, requires the use of deliberately undefined constants
(LIO_SYNC and LIO_MLOCK), and is arguably a bug.
Reviewed by: jhb
Differential Revision: https://reviews.freebsd.org/D27942
---
sys/kern/sys_socket.c | 8 +++-----
sys/kern/vfs_aio.c | 48 ++++++++++++++----------------------------------
sys/sys/aio.h | 11 ++++++-----
3 files changed, 23 insertions(+), 44 deletions(-)
diff --git a/sys/kern/sys_socket.c b/sys/kern/sys_socket.c
index 18803b6a5ac0..e53b0367960b 100644
--- a/sys/kern/sys_socket.c
+++ b/sys/kern/sys_socket.c
@@ -775,10 +775,10 @@ soo_aio_cancel(struct kaiocb *job)
so = job->fd_file->f_data;
opcode = job->uaiocb.aio_lio_opcode;
- if (opcode == LIO_READ || opcode == LIO_READV)
+ if (opcode & LIO_READ)
sb = &so->so_rcv;
else {
- MPASS(opcode == LIO_WRITE || opcode == LIO_WRITEV);
+ MPASS(opcode & LIO_WRITE);
sb = &so->so_snd;
}
@@ -808,13 +808,11 @@ soo_aio_queue(struct file *fp, struct kaiocb *job)
if (error == 0)
return (0);
- switch (job->uaiocb.aio_lio_opcode) {
+ switch (job->uaiocb.aio_lio_opcode & (LIO_WRITE | LIO_READ)) {
case LIO_READ:
- case LIO_READV:
sb = &so->so_rcv;
break;
case LIO_WRITE:
- case LIO_WRITEV:
sb = &so->so_snd;
break;
default:
diff --git a/sys/kern/vfs_aio.c b/sys/kern/vfs_aio.c
index bc0d7e04c9d5..9b45a06c5f9f 100644
--- a/sys/kern/vfs_aio.c
+++ b/sys/kern/vfs_aio.c
@@ -817,8 +817,7 @@ aio_process_rw(struct kaiocb *job)
if (error != 0 && job->uiop->uio_resid != cnt) {
if (error == ERESTART || error == EINTR || error == EWOULDBLOCK)
error = 0;
- if (error == EPIPE &&
- (opcode == LIO_WRITE || opcode == LIO_WRITEV)) {
+ if (error == EPIPE && (opcode & LIO_WRITE)) {
PROC_LOCK(job->userproc);
kern_psignal(job->userproc, SIGPIPE);
PROC_UNLOCK(job->userproc);
@@ -841,8 +840,7 @@ aio_process_sync(struct kaiocb *job)
struct file *fp = job->fd_file;
int error = 0;
- KASSERT(job->uaiocb.aio_lio_opcode == LIO_SYNC ||
- job->uaiocb.aio_lio_opcode == LIO_DSYNC,
+ KASSERT(job->uaiocb.aio_lio_opcode & LIO_SYNC,
("%s: opcode %d", __func__, job->uaiocb.aio_lio_opcode));
td->td_ucred = job->cred;
@@ -1239,8 +1237,7 @@ aio_qbio(struct proc *p, struct kaiocb *job)
if (vp->v_bufobj.bo_bsize == 0)
return (-1);
- bio_cmd = opcode == LIO_WRITE || opcode == LIO_WRITEV ? BIO_WRITE :
- BIO_READ;
+ bio_cmd = (opcode & LIO_WRITE) ? BIO_WRITE : BIO_READ;
iovcnt = job->uiop->uio_iovcnt;
if (iovcnt > max_buf_aio)
return (-1);
@@ -1422,7 +1419,7 @@ aiocb_copyin(struct aiocb *ujob, struct kaiocb *kjob, int type)
error = copyin(ujob, kcb, sizeof(struct aiocb));
if (error)
return (error);
- if (type == LIO_READV || type == LIO_WRITEV) {
+ if (type & LIO_VECTORED) {
/* malloc a uio and copy in the iovec */
error = copyinuio(__DEVOLATILE(struct iovec*, kcb->aio_iov),
kcb->aio_iovcnt, &kjob->uiop);
@@ -1609,7 +1606,7 @@ aio_aqueue(struct thread *td, struct aiocb *ujob, struct aioliojob *lj,
if (error)
goto err3;
- if ((opcode == LIO_SYNC || opcode == LIO_DSYNC) && fp->f_vnode == NULL) {
+ if ((opcode & LIO_SYNC) && fp->f_vnode == NULL) {
error = EINVAL;
goto err3;
}
@@ -1669,14 +1666,10 @@ no_kqueue:
job->jobflags = KAIOCB_QUEUEING;
job->lio = lj;
- switch (opcode) {
- case LIO_READV:
- case LIO_WRITEV:
+ if (opcode & LIO_VECTORED) {
/* Use the uio copied in by aio_copyin */
MPASS(job->uiop != &job->uio && job->uiop != NULL);
- break;
- case LIO_READ:
- case LIO_WRITE:
+ } else {
/* Setup the inline uio */
job->iov[0].iov_base = (void *)(uintptr_t)job->uaiocb.aio_buf;
job->iov[0].iov_len = job->uaiocb.aio_nbytes;
@@ -1684,18 +1677,13 @@ no_kqueue:
job->uio.uio_iovcnt = 1;
job->uio.uio_resid = job->uaiocb.aio_nbytes;
job->uio.uio_segflg = UIO_USERSPACE;
- /* FALLTHROUGH */
- default:
job->uiop = &job->uio;
- break;
}
- switch (opcode) {
+ switch (opcode & (LIO_READ | LIO_WRITE)) {
case LIO_READ:
- case LIO_READV:
job->uiop->uio_rw = UIO_READ;
break;
case LIO_WRITE:
- case LIO_WRITEV:
job->uiop->uio_rw = UIO_WRITE;
break;
}
@@ -1813,21 +1801,14 @@ aio_queue_file(struct file *fp, struct kaiocb *job)
return (EOPNOTSUPP);
}
- switch (job->uaiocb.aio_lio_opcode) {
- case LIO_READ:
- case LIO_READV:
- case LIO_WRITE:
- case LIO_WRITEV:
+ if (job->uaiocb.aio_lio_opcode & (LIO_WRITE | LIO_READ)) {
aio_schedule(job, aio_process_rw);
error = 0;
- break;
- case LIO_SYNC:
- case LIO_DSYNC:
+ } else if (job->uaiocb.aio_lio_opcode & LIO_SYNC) {
AIO_LOCK(ki);
TAILQ_FOREACH(job2, &ki->kaio_jobqueue, plist) {
if (job2->fd_file == job->fd_file &&
- job2->uaiocb.aio_lio_opcode != LIO_SYNC &&
- job2->uaiocb.aio_lio_opcode != LIO_DSYNC &&
+ ((job2->uaiocb.aio_lio_opcode & LIO_SYNC) == 0) &&
job2->seqno < job->seqno) {
job2->jobflags |= KAIOCB_CHECKSYNC;
job->pending++;
@@ -1847,8 +1828,7 @@ aio_queue_file(struct file *fp, struct kaiocb *job)
AIO_UNLOCK(ki);
aio_schedule(job, aio_process_sync);
error = 0;
- break;
- default:
+ } else {
error = EINVAL;
}
return (error);
@@ -2498,7 +2478,7 @@ aio_biowakeup(struct bio *bp)
*/
if (flags & BIO_ERROR)
atomic_set_int(&job->error, bio_error);
- if (opcode == LIO_WRITE || opcode == LIO_WRITEV)
+ if (opcode & LIO_WRITE)
atomic_add_int(&job->outblock, nblks);
else
atomic_add_int(&job->inblock, nblks);
@@ -2836,7 +2816,7 @@ aiocb32_copyin(struct aiocb *ujob, struct kaiocb *kjob, int type)
CP(job32, *kcb, aio_fildes);
CP(job32, *kcb, aio_offset);
CP(job32, *kcb, aio_lio_opcode);
- if (type == LIO_READV || type == LIO_WRITEV) {
+ if (type & LIO_VECTORED) {
iov32 = PTRIN(job32.aio_iov);
CP(job32, *kcb, aio_iovcnt);
/* malloc a uio and copy in the iovec */
diff --git a/sys/sys/aio.h b/sys/sys/aio.h
index ee928b8bf846..d3e03efa310e 100644
--- a/sys/sys/aio.h
+++ b/sys/sys/aio.h
@@ -44,11 +44,12 @@
#define LIO_WRITE 0x1
#define LIO_READ 0x2
#if defined(_KERNEL) || defined(_WANT_ALL_LIO_OPCODES)
-#define LIO_SYNC 0x3
-#define LIO_MLOCK 0x4
-#define LIO_WRITEV 0x5
-#define LIO_READV 0x6
-#define LIO_DSYNC 0x7
+#define LIO_VECTORED 0x4
+#define LIO_WRITEV (LIO_WRITE | LIO_VECTORED)
+#define LIO_READV (LIO_READ | LIO_VECTORED)
+#define LIO_SYNC 0x8
+#define LIO_DSYNC (0x10 | LIO_SYNC)
+#define LIO_MLOCK 0x20
#endif
/*
More information about the dev-commits-src-all
mailing list