git: fc9b28879e89 - main - uiomove_*: centralize the copy function selection
- Go to: [ bottom of page ] [ top of archives ] [ this month ]
Date: Sun, 21 Jun 2026 16:14:39 UTC
The branch main has been updated by brooks:
URL: https://cgit.FreeBSD.org/src/commit/?id=fc9b28879e8935cc830b83d8a36d85b9b98b276a
commit fc9b28879e8935cc830b83d8a36d85b9b98b276a
Author: Brooks Davis <brooks@FreeBSD.org>
AuthorDate: 2026-06-21 15:47:53 +0000
Commit: Brooks Davis <brooks@FreeBSD.org>
CommitDate: 2026-06-21 16:13:27 +0000
uiomove_*: centralize the copy function selection
Add a uiomove_step() for the central set of switch statements which
choose between userspace and kernel and if data is going to or from the
iovec.
Refactor uiomove_fromphys loops to unconditionally free per-iteration
resources and drop gotos.
While here, switch from bcopy to memcpy.
Reviewed by: kib
Suggested by: emaste
Sponsored by: Innovate UK
Differential Revision: https://reviews.freebsd.org/D57688
---
sys/amd64/amd64/uio_machdep.c | 34 ++------------------
sys/arm/arm/uio_machdep.c | 33 ++------------------
sys/arm64/arm64/uio_machdep.c | 36 ++--------------------
sys/i386/i386/uio_machdep.c | 33 ++------------------
sys/kern/subr_uio.c | 65 ++++++++++++++++++++++-----------------
sys/powerpc/powerpc/uio_machdep.c | 33 ++------------------
sys/riscv/riscv/uio_machdep.c | 36 ++--------------------
sys/sys/uio.h | 1 +
8 files changed, 56 insertions(+), 215 deletions(-)
diff --git a/sys/amd64/amd64/uio_machdep.c b/sys/amd64/amd64/uio_machdep.c
index 11e6ad2b1da9..9fdfc898172b 100644
--- a/sys/amd64/amd64/uio_machdep.c
+++ b/sys/amd64/amd64/uio_machdep.c
@@ -96,38 +96,14 @@ uiomove_fromphys(vm_page_t ma[], vm_offset_t offset, int n, struct uio *uio)
&ma[offset >> PAGE_SHIFT], &vaddr, 1, true);
cp = (char *)vaddr + page_offset;
}
- switch (uio->uio_segflg) {
- case UIO_USERSPACE:
- maybe_yield();
- switch (uio->uio_rw) {
- case UIO_READ:
- error = copyout(cp, iov->iov_base, cnt);
- break;
- case UIO_WRITE:
- error = copyin(iov->iov_base, cp, cnt);
- break;
- }
- if (error)
- goto out;
- break;
- case UIO_SYSSPACE:
- switch (uio->uio_rw) {
- case UIO_READ:
- bcopy(cp, iov->iov_base, cnt);
- break;
- case UIO_WRITE:
- bcopy(iov->iov_base, cp, cnt);
- break;
- }
- break;
- case UIO_NOCOPY:
- break;
- }
+ error = uiomove_step(cp, iov->iov_base, cnt, uio);
if (__predict_false(mapped)) {
pmap_unmap_io_transient(&ma[offset >> PAGE_SHIFT],
&vaddr, 1, true);
mapped = false;
}
+ if (error != 0)
+ break;
iov->iov_base = (char *)iov->iov_base + cnt;
iov->iov_len -= cnt;
uio->uio_resid -= cnt;
@@ -135,10 +111,6 @@ uiomove_fromphys(vm_page_t ma[], vm_offset_t offset, int n, struct uio *uio)
offset += cnt;
n -= cnt;
}
-out:
- if (__predict_false(mapped))
- pmap_unmap_io_transient(&ma[offset >> PAGE_SHIFT], &vaddr, 1,
- true);
if (save == 0)
td->td_pflags &= ~TDP_DEADLKTREAT;
return (error);
diff --git a/sys/arm/arm/uio_machdep.c b/sys/arm/arm/uio_machdep.c
index 6b8e4352e06f..7df6104b25fc 100644
--- a/sys/arm/arm/uio_machdep.c
+++ b/sys/arm/arm/uio_machdep.c
@@ -91,36 +91,10 @@ uiomove_fromphys(vm_page_t ma[], vm_offset_t offset, int n, struct uio *uio)
cnt = min(cnt, PAGE_SIZE - page_offset);
sf = sf_buf_alloc(ma[offset >> PAGE_SHIFT], 0);
cp = (char*)sf_buf_kva(sf) + page_offset;
- switch (uio->uio_segflg) {
- case UIO_USERSPACE:
- maybe_yield();
- switch (uio->uio_rw) {
- case UIO_READ:
- error = copyout(cp, iov->iov_base, cnt);
- break;
- case UIO_WRITE:
- error = copyin(iov->iov_base, cp, cnt);
- break;
- }
- if (error) {
- sf_buf_free(sf);
- goto out;
- }
- break;
- case UIO_SYSSPACE:
- switch (uio->uio_rw) {
- case UIO_READ:
- bcopy(cp, iov->iov_base, cnt);
- break;
- case UIO_WRITE:
- bcopy(iov->iov_base, cp, cnt);
- break;
- }
- break;
- case UIO_NOCOPY:
- break;
- }
+ error = uiomove_step(cp, iov->iov_base, cnt, uio);
sf_buf_free(sf);
+ if (error != 0)
+ break;
iov->iov_base = (char *)iov->iov_base + cnt;
iov->iov_len -= cnt;
uio->uio_resid -= cnt;
@@ -128,7 +102,6 @@ uiomove_fromphys(vm_page_t ma[], vm_offset_t offset, int n, struct uio *uio)
offset += cnt;
n -= cnt;
}
-out:
if (save == 0)
td->td_pflags &= ~TDP_DEADLKTREAT;
return (error);
diff --git a/sys/arm64/arm64/uio_machdep.c b/sys/arm64/arm64/uio_machdep.c
index 976055a69491..77f596498c45 100644
--- a/sys/arm64/arm64/uio_machdep.c
+++ b/sys/arm64/arm64/uio_machdep.c
@@ -92,38 +92,14 @@ uiomove_fromphys(vm_page_t ma[], vm_offset_t offset, int n, struct uio *uio)
&ma[offset >> PAGE_SHIFT], &vaddr, 1, true);
cp = (char *)vaddr + page_offset;
}
- switch (uio->uio_segflg) {
- case UIO_USERSPACE:
- maybe_yield();
- switch (uio->uio_rw) {
- case UIO_READ:
- error = copyout(cp, iov->iov_base, cnt);
- break;
- case UIO_WRITE:
- error = copyin(iov->iov_base, cp, cnt);
- break;
- }
- if (error)
- goto out;
- break;
- case UIO_SYSSPACE:
- switch (uio->uio_rw) {
- case UIO_READ:
- bcopy(cp, iov->iov_base, cnt);
- break;
- case UIO_WRITE:
- bcopy(iov->iov_base, cp, cnt);
- break;
- }
- break;
- case UIO_NOCOPY:
- break;
- }
+ error = uiomove_step(cp, iov->iov_base, cnt, uio);
if (__predict_false(mapped)) {
pmap_unmap_io_transient(&ma[offset >> PAGE_SHIFT],
&vaddr, 1, true);
mapped = false;
}
+ if (error != 0)
+ break;
iov->iov_base = (char *)iov->iov_base + cnt;
iov->iov_len -= cnt;
uio->uio_resid -= cnt;
@@ -131,12 +107,6 @@ uiomove_fromphys(vm_page_t ma[], vm_offset_t offset, int n, struct uio *uio)
offset += cnt;
n -= cnt;
}
-out:
- if (__predict_false(mapped)) {
- panic("ARM64TODO: uiomove_fromphys");
- pmap_unmap_io_transient(&ma[offset >> PAGE_SHIFT], &vaddr, 1,
- true);
- }
if (save == 0)
td->td_pflags &= ~TDP_DEADLKTREAT;
return (error);
diff --git a/sys/i386/i386/uio_machdep.c b/sys/i386/i386/uio_machdep.c
index b41460aef5b4..674de877d2a1 100644
--- a/sys/i386/i386/uio_machdep.c
+++ b/sys/i386/i386/uio_machdep.c
@@ -91,38 +91,11 @@ uiomove_fromphys(vm_page_t ma[], vm_offset_t offset, int n, struct uio *uio)
sched_pin();
sf = sf_buf_alloc(ma[offset >> PAGE_SHIFT], SFB_CPUPRIVATE);
cp = (char *)sf_buf_kva(sf) + page_offset;
- switch (uio->uio_segflg) {
- case UIO_USERSPACE:
- maybe_yield();
- switch (uio->uio_rw) {
- case UIO_READ:
- error = copyout(cp, iov->iov_base, cnt);
- break;
- case UIO_WRITE:
- error = copyin(iov->iov_base, cp, cnt);
- break;
- }
- if (error) {
- sf_buf_free(sf);
- sched_unpin();
- goto out;
- }
- break;
- case UIO_SYSSPACE:
- switch (uio->uio_rw) {
- case UIO_READ:
- bcopy(cp, iov->iov_base, cnt);
- break;
- case UIO_WRITE:
- bcopy(iov->iov_base, cp, cnt);
- break;
- }
- break;
- case UIO_NOCOPY:
- break;
- }
+ error = uiomove_step(cp, iov->iov_base, cnt, uio);
sf_buf_free(sf);
sched_unpin();
+ if (error != 0)
+ break;
iov->iov_base = (char *)iov->iov_base + cnt;
iov->iov_len -= cnt;
uio->uio_resid -= cnt;
diff --git a/sys/kern/subr_uio.c b/sys/kern/subr_uio.c
index fa5865fbe938..fea1395a7f77 100644
--- a/sys/kern/subr_uio.c
+++ b/sys/kern/subr_uio.c
@@ -200,6 +200,40 @@ uiomove_nofault(void *cp, int n, struct uio *uio)
return (uiomove_faultflag(cp, n, uio, 1));
}
+int
+uiomove_step(void *cp, void *base, size_t len, struct uio *uio)
+{
+ int error = 0;
+
+ switch (uio->uio_segflg) {
+ case UIO_USERSPACE:
+ maybe_yield();
+ switch (uio->uio_rw) {
+ case UIO_READ:
+ error = copyout(cp, base, len);
+ break;
+ case UIO_WRITE:
+ error = copyin(base, cp, len);
+ break;
+ }
+ break;
+ case UIO_SYSSPACE:
+ switch (uio->uio_rw) {
+ case UIO_READ:
+ memcpy(base, cp, len);
+ break;
+ case UIO_WRITE:
+ memcpy(cp, base, len);
+ break;
+ }
+ break;
+ case UIO_NOCOPY:
+ break;
+ }
+
+ return (error);
+}
+
static int
uiomove_faultflag(void *cp, int n, struct uio *uio, int nofault)
{
@@ -246,34 +280,9 @@ uiomove_faultflag(void *cp, int n, struct uio *uio, int nofault)
if (cnt > n)
cnt = n;
- switch (uio->uio_segflg) {
- case UIO_USERSPACE:
- maybe_yield();
- switch (uio->uio_rw) {
- case UIO_READ:
- error = copyout(cp, iov->iov_base, cnt);
- break;
- case UIO_WRITE:
- error = copyin(iov->iov_base, cp, cnt);
- break;
- }
- if (error)
- goto out;
- break;
-
- case UIO_SYSSPACE:
- switch (uio->uio_rw) {
- case UIO_READ:
- bcopy(cp, iov->iov_base, cnt);
- break;
- case UIO_WRITE:
- bcopy(iov->iov_base, cp, cnt);
- break;
- }
- break;
- case UIO_NOCOPY:
- break;
- }
+ error = uiomove_step(cp, iov->iov_base, cnt, uio);
+ if (error != 0)
+ goto out;
iov->iov_base = (char *)iov->iov_base + cnt;
iov->iov_len -= cnt;
uio->uio_resid -= cnt;
diff --git a/sys/powerpc/powerpc/uio_machdep.c b/sys/powerpc/powerpc/uio_machdep.c
index 63911963279a..05d06a95f544 100644
--- a/sys/powerpc/powerpc/uio_machdep.c
+++ b/sys/powerpc/powerpc/uio_machdep.c
@@ -97,36 +97,10 @@ uiomove_fromphys(vm_page_t ma[], vm_offset_t offset, int n, struct uio *uio)
sf = sf_buf_alloc(m, 0);
cp = (char*)sf_buf_kva(sf) + page_offset;
- switch (uio->uio_segflg) {
- case UIO_USERSPACE:
- maybe_yield();
- switch (uio->uio_rw) {
- case UIO_READ:
- error = copyout(cp, iov->iov_base, cnt);
- break;
- case UIO_WRITE:
- error = copyin(iov->iov_base, cp, cnt);
- break;
- }
- if (error) {
- sf_buf_free(sf);
- goto out;
- }
- break;
- case UIO_SYSSPACE:
- switch (uio->uio_rw) {
- case UIO_READ:
- bcopy(cp, iov->iov_base, cnt);
- break;
- case UIO_WRITE:
- bcopy(iov->iov_base, cp, cnt);
- break;
- }
- break;
- case UIO_NOCOPY:
- break;
- }
+ error = uiomove_step(cp, iov->iov_base, cnt, uio);
sf_buf_free(sf);
+ if (error != 0)
+ break;
iov->iov_base = (char *)iov->iov_base + cnt;
iov->iov_len -= cnt;
uio->uio_resid -= cnt;
@@ -134,7 +108,6 @@ uiomove_fromphys(vm_page_t ma[], vm_offset_t offset, int n, struct uio *uio)
offset += cnt;
n -= cnt;
}
-out:
if (save == 0)
td->td_pflags &= ~TDP_DEADLKTREAT;
return (error);
diff --git a/sys/riscv/riscv/uio_machdep.c b/sys/riscv/riscv/uio_machdep.c
index f171feb1a4bd..77f596498c45 100644
--- a/sys/riscv/riscv/uio_machdep.c
+++ b/sys/riscv/riscv/uio_machdep.c
@@ -92,38 +92,14 @@ uiomove_fromphys(vm_page_t ma[], vm_offset_t offset, int n, struct uio *uio)
&ma[offset >> PAGE_SHIFT], &vaddr, 1, true);
cp = (char *)vaddr + page_offset;
}
- switch (uio->uio_segflg) {
- case UIO_USERSPACE:
- maybe_yield();
- switch (uio->uio_rw) {
- case UIO_READ:
- error = copyout(cp, iov->iov_base, cnt);
- break;
- case UIO_WRITE:
- error = copyin(iov->iov_base, cp, cnt);
- break;
- }
- if (error)
- goto out;
- break;
- case UIO_SYSSPACE:
- switch (uio->uio_rw) {
- case UIO_READ:
- bcopy(cp, iov->iov_base, cnt);
- break;
- case UIO_WRITE:
- bcopy(iov->iov_base, cp, cnt);
- break;
- }
- break;
- case UIO_NOCOPY:
- break;
- }
+ error = uiomove_step(cp, iov->iov_base, cnt, uio);
if (__predict_false(mapped)) {
pmap_unmap_io_transient(&ma[offset >> PAGE_SHIFT],
&vaddr, 1, true);
mapped = false;
}
+ if (error != 0)
+ break;
iov->iov_base = (char *)iov->iov_base + cnt;
iov->iov_len -= cnt;
uio->uio_resid -= cnt;
@@ -131,12 +107,6 @@ uiomove_fromphys(vm_page_t ma[], vm_offset_t offset, int n, struct uio *uio)
offset += cnt;
n -= cnt;
}
-out:
- if (__predict_false(mapped)) {
- panic("TODO 3");
- pmap_unmap_io_transient(&ma[offset >> PAGE_SHIFT], &vaddr, 1,
- true);
- }
if (save == 0)
td->td_pflags &= ~TDP_DEADLKTREAT;
return (error);
diff --git a/sys/sys/uio.h b/sys/sys/uio.h
index 58bb7b13b253..217f65365cb0 100644
--- a/sys/sys/uio.h
+++ b/sys/sys/uio.h
@@ -98,6 +98,7 @@ int uiomove_fromphys(struct vm_page *ma[], vm_offset_t offset, int n,
struct uio *uio);
int uiomove_nofault(void *cp, int n, struct uio *uio);
int uiomove_object(struct vm_object *obj, off_t obj_size, struct uio *uio);
+int uiomove_step(void *cp, void *base, size_t cnt, struct uio *uio);
#else /* !_KERNEL */