git: 3bcb7c2a3320 - main - powerpc: initial straight port of busdma_machdep.c -> busdma_bounce.c
- Go to: [ bottom of page ] [ top of archives ] [ this month ]
Date: Fri, 01 May 2026 21:15:16 UTC
The branch main has been updated by adrian:
URL: https://cgit.FreeBSD.org/src/commit/?id=3bcb7c2a33206d52cecb349b77b2a631728bc7d1
commit 3bcb7c2a33206d52cecb349b77b2a631728bc7d1
Author: Adrian Chadd <adrian@FreeBSD.org>
AuthorDate: 2026-02-14 17:03:27 +0000
Commit: Adrian Chadd <adrian@FreeBSD.org>
CommitDate: 2026-05-01 21:14:23 +0000
powerpc: initial straight port of busdma_machdep.c -> busdma_bounce.c
This is a straight port of the code and doesn't yet handle
different implementations (which will be in a subsequent commit.)
Locally tested:
* G5 SMP (2x PPC970mp)
* power8 / power9 pseries QEMU VM
* power8 powernv
Differential Revision: https://reviews.freebsd.org/D55313
---
sys/conf/files.powerpc | 1 +
sys/powerpc/include/bus_dma.h | 145 ++++++-
sys/powerpc/include/bus_dma_impl.h | 82 ++++
sys/powerpc/powerpc/busdma_bounce.c | 787 +++++++++++++++++++++++++++++++++++
sys/powerpc/powerpc/busdma_machdep.c | 711 ++-----------------------------
sys/powerpc/pseries/phyp_vscsi.c | 1 -
6 files changed, 1039 insertions(+), 688 deletions(-)
diff --git a/sys/conf/files.powerpc b/sys/conf/files.powerpc
index 698433422101..164a5d01d4b9 100644
--- a/sys/conf/files.powerpc
+++ b/sys/conf/files.powerpc
@@ -361,6 +361,7 @@ powerpc/powernv/xive.c optional powernv
powerpc/powerpc/altivec.c standard
powerpc/powerpc/autoconf.c standard
powerpc/powerpc/bus_machdep.c standard
+powerpc/powerpc/busdma_bounce.c standard
powerpc/powerpc/busdma_machdep.c standard
powerpc/powerpc/clock.c standard
powerpc/powerpc/copyinout.c optional aim
diff --git a/sys/powerpc/include/bus_dma.h b/sys/powerpc/include/bus_dma.h
index 09bc7b15e94e..c12161011a99 100644
--- a/sys/powerpc/include/bus_dma.h
+++ b/sys/powerpc/include/bus_dma.h
@@ -2,6 +2,7 @@
* SPDX-License-Identifier: BSD-2-Clause
*
* Copyright (c) 2005 Scott Long
+ * Copyright (c) 2026 Raptor Engineering, LLC
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@@ -29,9 +30,149 @@
#ifndef _POWERPC_BUS_DMA_H_
#define _POWERPC_BUS_DMA_H_
+#define WANT_INLINE_DMAMAP
#include <sys/bus_dma.h>
-#include <sys/bus_dma_internal.h>
-int bus_dma_tag_set_iommu(bus_dma_tag_t, device_t iommu, void *cookie);
+#include <machine/bus_dma_impl.h>
+
+/*
+ * Allocate a handle for mapping from kva/uva/physical
+ * address space into bus device space.
+ */
+static inline int
+bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp)
+{
+ struct bus_dma_tag_common *tc;
+
+ tc = (struct bus_dma_tag_common *)dmat;
+ return (tc->impl->map_create(dmat, flags, mapp));
+}
+
+/*
+ * Destroy a handle for mapping from kva/uva/physical
+ * address space into bus device space.
+ */
+static inline int
+bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map)
+{
+ struct bus_dma_tag_common *tc;
+
+ tc = (struct bus_dma_tag_common *)dmat;
+ return (tc->impl->map_destroy(dmat, map));
+}
+
+/*
+ * Allocate a piece of memory that can be efficiently mapped into
+ * bus device space based on the constraints listed in the dma tag.
+ * A dmamap to for use with dmamap_load is also allocated.
+ */
+static inline int
+bus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags,
+ bus_dmamap_t *mapp)
+{
+ struct bus_dma_tag_common *tc;
+
+ tc = (struct bus_dma_tag_common *)dmat;
+ return (tc->impl->mem_alloc(dmat, vaddr, flags, mapp));
+}
+
+/*
+ * Free a piece of memory and it's allociated dmamap, that was allocated
+ * via bus_dmamem_alloc.
+ */
+static inline void
+bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map)
+{
+ struct bus_dma_tag_common *tc;
+
+ tc = (struct bus_dma_tag_common *)dmat;
+ tc->impl->mem_free(dmat, vaddr, map);
+}
+
+/*
+ * Release the mapping held by map.
+ */
+static inline void
+bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map)
+{
+ struct bus_dma_tag_common *tc;
+
+ tc = (struct bus_dma_tag_common *)dmat;
+ tc->impl->map_unload(dmat, map);
+}
+
+static inline void
+bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op)
+{
+ struct bus_dma_tag_common *tc;
+
+ tc = (struct bus_dma_tag_common *)dmat;
+ tc->impl->map_sync(dmat, map, op);
+}
+
+static inline int
+_bus_dmamap_load_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf,
+ bus_size_t buflen, int flags, bus_dma_segment_t *segs, int *segp)
+{
+ struct bus_dma_tag_common *tc;
+
+ tc = (struct bus_dma_tag_common *)dmat;
+ return (tc->impl->load_phys(dmat, map, buf, buflen, flags, segs,
+ segp));
+}
+
+static inline int
+_bus_dmamap_load_ma(bus_dma_tag_t dmat, bus_dmamap_t map, struct vm_page **ma,
+ bus_size_t tlen, int ma_offs, int flags, bus_dma_segment_t *segs,
+ int *segp)
+{
+ struct bus_dma_tag_common *tc;
+
+ tc = (struct bus_dma_tag_common *)dmat;
+ return (tc->impl->load_ma(dmat, map, ma, tlen, ma_offs, flags,
+ segs, segp));
+}
+
+static inline int
+_bus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf,
+ bus_size_t buflen, struct pmap *pmap, int flags, bus_dma_segment_t *segs,
+ int *segp)
+{
+ struct bus_dma_tag_common *tc;
+
+ tc = (struct bus_dma_tag_common *)dmat;
+ return (tc->impl->load_buffer(dmat, map, buf, buflen, pmap, flags, segs,
+ segp));
+}
+
+static inline void
+_bus_dmamap_waitok(bus_dma_tag_t dmat, bus_dmamap_t map,
+ struct memdesc *mem, bus_dmamap_callback_t *callback, void *callback_arg)
+{
+ struct bus_dma_tag_common *tc;
+
+ tc = (struct bus_dma_tag_common *)dmat;
+ tc->impl->map_waitok(dmat, map, mem, callback, callback_arg);
+}
+
+static inline bus_dma_segment_t *
+_bus_dmamap_complete(bus_dma_tag_t dmat, bus_dmamap_t map,
+ bus_dma_segment_t *segs, int nsegs, int error)
+{
+ struct bus_dma_tag_common *tc;
+
+ tc = (struct bus_dma_tag_common *)dmat;
+ return (tc->impl->map_complete(dmat, map, segs, nsegs, error));
+}
+
+static inline int
+bus_dma_tag_set_iommu(bus_dma_tag_t dmat, device_t iommu, void *cookie)
+{
+ struct bus_dma_tag_common *tc;
+
+ tc = (struct bus_dma_tag_common *)dmat;
+ return (tc->impl->set_iommu(dmat, iommu, cookie));
+
+}
#endif /* _POWERPC_BUS_DMA_H_ */
diff --git a/sys/powerpc/include/bus_dma_impl.h b/sys/powerpc/include/bus_dma_impl.h
new file mode 100644
index 000000000000..e24bd6dd8806
--- /dev/null
+++ b/sys/powerpc/include/bus_dma_impl.h
@@ -0,0 +1,82 @@
+/*-
+ * Copyright (c) 2013 The FreeBSD Foundation
+ *
+ * This software was developed by Konstantin Belousov <kib@FreeBSD.org>
+ * under sponsorship from the FreeBSD Foundation.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _MACHINE_BUS_DMA_IMPL_H_
+#define _MACHINE_BUS_DMA_IMPL_H_
+
+/* Note: must be first entry in bus_dma_tag */
+struct bus_dma_tag_common {
+ struct bus_dma_impl *impl;
+ bus_size_t alignment;
+ bus_addr_t boundary;
+ bus_addr_t lowaddr;
+ bus_addr_t highaddr;
+ bus_size_t maxsize;
+ u_int nsegments;
+ bus_size_t maxsegsz;
+ int flags;
+ bus_dma_lock_t *lockfunc;
+ void *lockfuncarg;
+};
+
+struct bus_dma_impl {
+ int (*tag_create)(bus_dma_tag_t parent,
+ bus_size_t alignment, bus_addr_t boundary, bus_addr_t lowaddr,
+ bus_addr_t highaddr, bus_size_t maxsize, int nsegments,
+ bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc,
+ void *lockfuncarg, bus_dma_tag_t *dmat);
+ int (*tag_destroy)(bus_dma_tag_t dmat);
+ int (*tag_set_domain)(bus_dma_tag_t, int domain);
+ int (*map_create)(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp);
+ int (*map_destroy)(bus_dma_tag_t dmat, bus_dmamap_t map);
+ int (*mem_alloc)(bus_dma_tag_t dmat, void** vaddr, int flags,
+ bus_dmamap_t *mapp);
+ void (*mem_free)(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map);
+ int (*load_ma)(bus_dma_tag_t dmat, bus_dmamap_t map,
+ struct vm_page **ma, bus_size_t tlen, int ma_offs, int flags,
+ bus_dma_segment_t *segs, int *segp);
+ int (*load_phys)(bus_dma_tag_t dmat, bus_dmamap_t map,
+ vm_paddr_t buf, bus_size_t buflen, int flags,
+ bus_dma_segment_t *segs, int *segp);
+ int (*load_buffer)(bus_dma_tag_t dmat, bus_dmamap_t map,
+ void *buf, bus_size_t buflen, struct pmap *pmap, int flags,
+ bus_dma_segment_t *segs, int *segp);
+ void (*map_waitok)(bus_dma_tag_t dmat, bus_dmamap_t map,
+ struct memdesc *mem, bus_dmamap_callback_t *callback,
+ void *callback_arg);
+ bus_dma_segment_t *(*map_complete)(bus_dma_tag_t dmat, bus_dmamap_t map,
+ bus_dma_segment_t *segs, int nsegs, int error);
+ void (*map_unload)(bus_dma_tag_t dmat, bus_dmamap_t map);
+ void (*map_sync)(bus_dma_tag_t dmat, bus_dmamap_t map,
+ bus_dmasync_op_t op);
+ int (*set_iommu)(bus_dma_tag_t dmat, device_t iommu, void *cookie);
+};
+
+extern struct bus_dma_impl bus_dma_bounce_impl;
+
+#endif
diff --git a/sys/powerpc/powerpc/busdma_bounce.c b/sys/powerpc/powerpc/busdma_bounce.c
new file mode 100644
index 000000000000..cdc25a5d8fb7
--- /dev/null
+++ b/sys/powerpc/powerpc/busdma_bounce.c
@@ -0,0 +1,787 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 1997, 1998 Justin T. Gibbs.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions, and the following disclaimer,
+ * without modification, immediately at the beginning of the file.
+ * 2. The name of the author may not be used to endorse or promote products
+ * derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/*
+ * From amd64/busdma_machdep.c, r204214
+ */
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/malloc.h>
+#include <sys/bus.h>
+#include <sys/interrupt.h>
+#include <sys/kernel.h>
+#include <sys/ktr.h>
+#include <sys/lock.h>
+#include <sys/proc.h>
+#include <sys/memdesc.h>
+#include <sys/mutex.h>
+#include <sys/sysctl.h>
+#include <sys/uio.h>
+
+#include <vm/vm.h>
+#include <vm/vm_extern.h>
+#include <vm/vm_kern.h>
+#include <vm/vm_page.h>
+#include <vm/vm_map.h>
+
+#include <machine/atomic.h>
+#include <machine/bus.h>
+#include <machine/cpufunc.h>
+#include <machine/md_var.h>
+
+#include "iommu_if.h"
+
+#define MAX_BPAGES MIN(8192, physmem/40)
+
+struct bus_dmamap {
+ STAILQ_HEAD(, bounce_page) bpages;
+ int pagesneeded;
+ int pagesreserved;
+ bus_dma_tag_t dmat;
+ struct memdesc mem;
+ bus_dma_segment_t *segments;
+ int nsegs;
+ bus_dmamap_callback_t *callback;
+ void *callback_arg;
+ __sbintime_t queued_time;
+ STAILQ_ENTRY(bus_dmamap) links;
+ int contigalloc;
+};
+
+static MALLOC_DEFINE(M_BUSDMA, "busdma", "busdma metadata");
+
+#define dmat_alignment(dmat) ((dmat)->common.alignment)
+/* XXX TODO: bounce flags? */
+#define dmat_bounce_flags(dmat) (0)
+#define dmat_boundary(dmat) ((dmat)->common.boundary)
+#define dmat_flags(dmat) ((dmat)->common.flags)
+#define dmat_highaddr(dmat) ((dmat)->common.highaddr)
+#define dmat_lowaddr(dmat) ((dmat)->common.lowaddr)
+#define dmat_lockfunc(dmat) ((dmat)->common.lockfunc)
+#define dmat_lockfuncarg(dmat) ((dmat)->common.lockfuncarg)
+#define dmat_maxsegsz(dmat) ((dmat)->common.maxsegsz)
+#define dmat_nsegments(dmat) ((dmat)->common.nsegments)
+
+/*
+ * Note: at this point the code requires a 'struct bus_dma_tag' to exist
+ * or the included code will not function correctly.
+ */
+
+struct bus_dma_tag {
+ struct bus_dma_tag_common common;
+ int map_count;
+ int bounce_flags;
+ struct bounce_zone *bounce_zone;
+ device_t iommu;
+ void *iommu_cookie;
+};
+
+static SYSCTL_NODE(_hw, OID_AUTO, busdma, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
+ "Busdma parameters");
+
+#include "../../kern/subr_busdma_bounce.c"
+
+/*
+ * Returns true if the address falls within the tag's exclusion window, or
+ * fails to meet its alignment requirements.
+ */
+static __inline bool
+must_bounce(bus_dma_tag_t dmat, bus_addr_t paddr)
+{
+
+ if (dmat->iommu == NULL && paddr > dmat->common.lowaddr &&
+ paddr <= dmat->common.highaddr)
+ return (true);
+ if (!vm_addr_align_ok(paddr, dmat->common.alignment))
+ return (true);
+
+ return (false);
+}
+
+#define BUS_DMA_COULD_BOUNCE BUS_DMA_BUS3
+#define BUS_DMA_MIN_ALLOC_COMP BUS_DMA_BUS4
+
+static int
+bounce_bus_dma_zone_setup(bus_dma_tag_t newtag)
+{
+ struct bounce_zone *bz;
+ const u_long maxsize = newtag->common.maxsize;
+ int error = 0;
+
+ if ((error = alloc_bounce_zone(newtag)) != 0) {
+ return (error);
+ }
+ bz = newtag->bounce_zone;
+
+ if (ptoa(bz->total_bpages) < maxsize) {
+ int pages;
+
+ pages = atop(maxsize) - bz->total_bpages;
+
+ /* Add pages to our bounce pool */
+ if (alloc_bounce_pages(newtag, pages) < pages)
+ error = ENOMEM;
+ }
+ /* Performed initial allocation */
+ newtag->common.flags |= BUS_DMA_MIN_ALLOC_COMP;
+
+ return (error);
+}
+
+/*
+ * Allocate a device specific dma_tag.
+ *
+ * TODO: this does ALL of the work, rather than it being split into
+ * common and bounce specific. That'll need fixing.
+ */
+static int
+bounce_bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment,
+ bus_addr_t boundary, bus_addr_t lowaddr,
+ bus_addr_t highaddr, bus_size_t maxsize, int nsegments,
+ bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc,
+ void *lockfuncarg, bus_dma_tag_t *dmat)
+{
+ bus_dma_tag_t newtag;
+ int error = 0;
+
+ /* Basic sanity checking */
+ if (boundary != 0 && boundary < maxsegsz)
+ maxsegsz = boundary;
+
+ if (maxsegsz == 0) {
+ return (EINVAL);
+ }
+
+ /* Return a NULL tag on failure */
+ *dmat = NULL;
+
+ newtag = (bus_dma_tag_t)malloc(sizeof(*newtag), M_DEVBUF,
+ M_ZERO | M_NOWAIT);
+ if (newtag == NULL) {
+ CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d",
+ __func__, newtag, 0, error);
+ return (ENOMEM);
+ }
+
+ newtag->common.alignment = alignment;
+ newtag->common.boundary = boundary;
+ newtag->common.lowaddr = trunc_page((vm_paddr_t)lowaddr) + (PAGE_SIZE - 1);
+ newtag->common.highaddr = trunc_page((vm_paddr_t)highaddr) + (PAGE_SIZE - 1);
+ newtag->common.maxsize = maxsize;
+ newtag->common.nsegments = nsegments;
+ newtag->common.maxsegsz = maxsegsz;
+ newtag->common.flags = flags;
+ newtag->map_count = 0;
+ newtag->common.impl = &bus_dma_bounce_impl;
+ if (lockfunc != NULL) {
+ newtag->common.lockfunc = lockfunc;
+ newtag->common.lockfuncarg = lockfuncarg;
+ } else {
+ newtag->common.lockfunc = _busdma_dflt_lock;
+ newtag->common.lockfuncarg = NULL;
+ }
+
+ /* Take into account any restrictions imposed by our parent tag */
+ if (parent != NULL) {
+ newtag->common.lowaddr = MIN(parent->common.lowaddr, newtag->common.lowaddr);
+ newtag->common.highaddr = MAX(parent->common.highaddr, newtag->common.highaddr);
+ if (newtag->common.boundary == 0)
+ newtag->common.boundary = parent->common.boundary;
+ else if (parent->common.boundary != 0)
+ newtag->common.boundary = MIN(parent->common.boundary,
+ newtag->common.boundary);
+
+ newtag->iommu = parent->iommu;
+ newtag->iommu_cookie = parent->iommu_cookie;
+ }
+
+ if (newtag->common.lowaddr < ptoa((vm_paddr_t)Maxmem) && newtag->iommu == NULL)
+ newtag->common.flags |= BUS_DMA_COULD_BOUNCE;
+
+ if (newtag->common.alignment > 1)
+ newtag->common.flags |= BUS_DMA_COULD_BOUNCE;
+
+ if (((newtag->common.flags & BUS_DMA_COULD_BOUNCE) != 0) &&
+ (flags & BUS_DMA_ALLOCNOW) != 0) {
+ /* Must bounce */
+ error = bounce_bus_dma_zone_setup(newtag);
+ }
+
+ if (error != 0) {
+ free(newtag, M_DEVBUF);
+ } else {
+ *dmat = newtag;
+ }
+ CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d",
+ __func__, newtag, (newtag != NULL ? newtag->common.flags : 0), error);
+ return (error);
+}
+
+static int
+bounce_bus_dma_tag_set_domain(bus_dma_tag_t dmat, int domain)
+{
+
+ /* TODO */
+ return (0);
+}
+
+static int
+bounce_bus_dma_tag_destroy(bus_dma_tag_t dmat)
+{
+ int error = 0;
+
+ if (dmat != NULL) {
+ if (dmat->map_count != 0) {
+ error = EBUSY;
+ goto out;
+ }
+
+ free(dmat, M_DEVBUF);
+ }
+out:
+ CTR3(KTR_BUSDMA, "%s tag %p error %d", __func__, dmat, error);
+ return (error);
+}
+
+/*
+ * Allocate a handle for mapping from kva/uva/physical
+ * address space into bus device space.
+ */
+static int
+bounce_bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp)
+{
+ int error;
+
+ error = 0;
+
+ *mapp = (bus_dmamap_t)malloc(sizeof(**mapp), M_DEVBUF,
+ M_NOWAIT | M_ZERO);
+ if (*mapp == NULL) {
+ CTR3(KTR_BUSDMA, "%s: tag %p error %d",
+ __func__, dmat, ENOMEM);
+ return (ENOMEM);
+ }
+
+ /* Initialize the new map */
+ STAILQ_INIT(&((*mapp)->bpages));
+
+ /*
+ * Bouncing might be required if the driver asks for an active
+ * exclusion region, a data alignment that is stricter than 1, and/or
+ * an active address boundary.
+ */
+ if (dmat->common.flags & BUS_DMA_COULD_BOUNCE) {
+ /* Must bounce */
+ struct bounce_zone *bz;
+ int maxpages;
+
+ if (dmat->bounce_zone == NULL) {
+ if ((error = alloc_bounce_zone(dmat)) != 0)
+ return (error);
+ }
+ bz = dmat->bounce_zone;
+
+ /*
+ * Attempt to add pages to our pool on a per-instance
+ * basis up to a sane limit.
+ */
+ if (dmat->common.alignment > 1)
+ maxpages = MAX_BPAGES;
+ else
+ maxpages = MIN(MAX_BPAGES, Maxmem -atop(dmat->common.lowaddr));
+ if ((dmat->common.flags & BUS_DMA_MIN_ALLOC_COMP) == 0
+ || (bz->map_count > 0 && bz->total_bpages < maxpages)) {
+ int pages;
+
+ pages = MAX(atop(dmat->common.maxsize), 1);
+ pages = MIN(maxpages - bz->total_bpages, pages);
+ pages = MAX(pages, 1);
+ if (alloc_bounce_pages(dmat, pages) < pages)
+ error = ENOMEM;
+
+ if ((dmat->common.flags & BUS_DMA_MIN_ALLOC_COMP) == 0) {
+ if (error == 0)
+ dmat->common.flags |= BUS_DMA_MIN_ALLOC_COMP;
+ } else {
+ error = 0;
+ }
+ }
+ bz->map_count++;
+ }
+
+ (*mapp)->nsegs = 0;
+ (*mapp)->segments = (bus_dma_segment_t *)malloc(
+ sizeof(bus_dma_segment_t) * dmat->common.nsegments, M_DEVBUF,
+ M_NOWAIT);
+ if ((*mapp)->segments == NULL) {
+ CTR3(KTR_BUSDMA, "%s: tag %p error %d",
+ __func__, dmat, ENOMEM);
+ return (ENOMEM);
+ }
+
+ if (error == 0)
+ dmat->map_count++;
+ CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
+ __func__, dmat, dmat->common.flags, error);
+ return (error);
+}
+
+/*
+ * Destroy a handle for mapping from kva/uva/physical
+ * address space into bus device space.
+ */
+static int
+bounce_bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map)
+{
+ if (dmat->common.flags & BUS_DMA_COULD_BOUNCE) {
+ if (STAILQ_FIRST(&map->bpages) != NULL) {
+ CTR3(KTR_BUSDMA, "%s: tag %p error %d",
+ __func__, dmat, EBUSY);
+ return (EBUSY);
+ }
+ if (dmat->bounce_zone)
+ dmat->bounce_zone->map_count--;
+ }
+ free(map->segments, M_DEVBUF);
+ free(map, M_DEVBUF);
+ dmat->map_count--;
+ CTR2(KTR_BUSDMA, "%s: tag %p error 0", __func__, dmat);
+ return (0);
+}
+
+/*
+ * Allocate a piece of memory that can be efficiently mapped into
+ * bus device space based on the constraints lited in the dma tag.
+ * A dmamap to for use with dmamap_load is also allocated.
+ */
+static int
+bounce_bus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags,
+ bus_dmamap_t *mapp)
+{
+ vm_memattr_t attr;
+ int mflags;
+
+ if (flags & BUS_DMA_NOWAIT)
+ mflags = M_NOWAIT;
+ else
+ mflags = M_WAITOK;
+
+ bus_dmamap_create(dmat, flags, mapp);
+
+ if (flags & BUS_DMA_ZERO)
+ mflags |= M_ZERO;
+ if (flags & BUS_DMA_NOCACHE)
+ attr = VM_MEMATTR_UNCACHEABLE;
+ else
+ attr = VM_MEMATTR_DEFAULT;
+
+ /*
+ * XXX:
+ * (dmat->common.alignment <= dmat->common.maxsize) is just a quick hack; the exact
+ * alignment guarantees of malloc need to be nailed down, and the
+ * code below should be rewritten to take that into account.
+ *
+ * In the meantime, we'll warn the user if malloc gets it wrong.
+ */
+ if ((dmat->common.maxsize <= PAGE_SIZE) &&
+ (dmat->common.alignment <= dmat->common.maxsize) &&
+ dmat->common.lowaddr >= ptoa((vm_paddr_t)Maxmem) &&
+ attr == VM_MEMATTR_DEFAULT) {
+ *vaddr = malloc(dmat->common.maxsize, M_DEVBUF, mflags);
+ } else {
+ /*
+ * XXX Use Contigmalloc until it is merged into this facility
+ * and handles multi-seg allocations. Nobody is doing
+ * multi-seg allocations yet though.
+ * XXX Certain AGP hardware does.
+ */
+ *vaddr = kmem_alloc_contig(dmat->common.maxsize, mflags, 0ul,
+ dmat->common.lowaddr, dmat->common.alignment ? dmat->common.alignment : 1ul,
+ dmat->common.boundary, attr);
+ (*mapp)->contigalloc = 1;
+ }
+ if (*vaddr == NULL) {
+ CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
+ __func__, dmat, dmat->common.flags, ENOMEM);
+ return (ENOMEM);
+ } else if (!vm_addr_align_ok(vtophys(*vaddr), dmat->common.alignment)) {
+ printf("bus_dmamem_alloc failed to align memory properly.\n");
+ }
+ CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
+ __func__, dmat, dmat->common.flags, 0);
+ return (0);
+}
+
+/*
+ * Free a piece of memory and it's allociated dmamap, that was allocated
+ * via bus_dmamem_alloc.
+ */
+static void
+bounce_bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map)
+{
+
+ if (!map->contigalloc)
+ free(vaddr, M_DEVBUF);
+ else
+ kmem_free(vaddr, dmat->common.maxsize);
+ bus_dmamap_destroy(dmat, map);
+ CTR3(KTR_BUSDMA, "%s: tag %p flags 0x%x", __func__, dmat, dmat->common.flags);
+}
+
+static void
+_bus_dmamap_count_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf,
+ bus_size_t buflen, int flags)
+{
+ bus_addr_t curaddr;
+ bus_size_t sgsize;
+
+ if (map->pagesneeded == 0) {
+ CTR4(KTR_BUSDMA, "lowaddr= %d Maxmem= %d, boundary= %d, "
+ "alignment= %d", dmat->common.lowaddr, ptoa((vm_paddr_t)Maxmem),
+ dmat->common.boundary, dmat->common.alignment);
+ CTR2(KTR_BUSDMA, "map= %p, pagesneeded= %d", map, map->pagesneeded);
+ /*
+ * Count the number of bounce pages
+ * needed in order to complete this transfer
+ */
+ curaddr = buf;
+ while (buflen != 0) {
+ sgsize = buflen;
+ if (must_bounce(dmat, curaddr)) {
+ sgsize = MIN(sgsize,
+ PAGE_SIZE - (curaddr & PAGE_MASK));
+ map->pagesneeded++;
+ }
+ curaddr += sgsize;
+ buflen -= sgsize;
+ }
+ CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded);
+ }
+}
+
+static void
+_bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, pmap_t pmap,
+ void *buf, bus_size_t buflen, int flags)
+{
+ vm_offset_t vaddr;
+ vm_offset_t vendaddr;
+ bus_addr_t paddr;
+
+ if (map->pagesneeded == 0) {
+ CTR4(KTR_BUSDMA, "lowaddr= %d Maxmem= %d, boundary= %d, "
+ "alignment= %d", dmat->common.lowaddr, ptoa((vm_paddr_t)Maxmem),
+ dmat->common.boundary, dmat->common.alignment);
+ CTR2(KTR_BUSDMA, "map= %p, pagesneeded= %d", map, map->pagesneeded);
+ /*
+ * Count the number of bounce pages
+ * needed in order to complete this transfer
+ */
+ vaddr = (vm_offset_t)buf;
+ vendaddr = (vm_offset_t)buf + buflen;
+
+ while (vaddr < vendaddr) {
+ bus_size_t sg_len;
+
+ sg_len = MIN(vendaddr - vaddr,
+ PAGE_SIZE - ((vm_offset_t)vaddr & PAGE_MASK));
+ if (pmap == kernel_pmap)
+ paddr = pmap_kextract(vaddr);
+ else
+ paddr = pmap_extract(pmap, vaddr);
+ if (must_bounce(dmat, paddr)) {
+ sg_len = roundup2(sg_len, dmat->common.alignment);
+ map->pagesneeded++;
+ }
+ vaddr += sg_len;
+ }
+ CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded);
+ }
+}
+
+/*
+ * Utility function to load a physical buffer. segp contains
+ * the starting segment on entrace, and the ending segment on exit.
+ */
+static int
+bounce_bus_dmamap_load_phys(bus_dma_tag_t dmat,
+ bus_dmamap_t map,
+ vm_paddr_t buf, bus_size_t buflen,
+ int flags,
+ bus_dma_segment_t *segs,
+ int *segp)
+{
+ bus_addr_t curaddr;
+ bus_size_t sgsize;
+ int error;
+
+ if (segs == NULL)
+ segs = map->segments;
+
+ if ((dmat->common.flags & BUS_DMA_COULD_BOUNCE) != 0) {
+ _bus_dmamap_count_phys(dmat, map, buf, buflen, flags);
+ if (map->pagesneeded != 0) {
+ error = _bus_dmamap_reserve_pages(dmat, map, flags);
+ if (error)
+ return (error);
+ }
+ }
+
+ while (buflen > 0) {
+ curaddr = buf;
+ sgsize = buflen;
+ if (map->pagesneeded != 0 && must_bounce(dmat, curaddr)) {
+ sgsize = MIN(sgsize, PAGE_SIZE - (curaddr & PAGE_MASK));
+ curaddr = add_bounce_page(dmat, map, 0, curaddr,
+ sgsize);
+ }
+ if (!_bus_dmamap_addsegs(dmat, map, curaddr, sgsize, segs,
+ segp))
+ break;
+ buf += sgsize;
+ buflen -= sgsize;
+ }
+
+ /*
+ * Did we fit?
+ */
+ return (buflen != 0 ? EFBIG : 0); /* XXX better return value here? */
+}
+
+static int
+bounce_bus_dmamap_load_ma(bus_dma_tag_t dmat, bus_dmamap_t map,
+ struct vm_page **ma, bus_size_t tlen, int ma_offs, int flags,
+ bus_dma_segment_t *segs, int *segp)
+{
+
+ return (bus_dmamap_load_ma_triv(dmat, map, ma, tlen, ma_offs, flags,
+ segs, segp));
+}
+
+/*
+ * Utility function to load a linear buffer. segp contains
+ * the starting segment on entrance, and the ending segment on exit.
+ */
+static int
+bounce_bus_dmamap_load_buffer(bus_dma_tag_t dmat,
+ bus_dmamap_t map,
+ void *buf, bus_size_t buflen,
+ pmap_t pmap,
+ int flags,
+ bus_dma_segment_t *segs,
+ int *segp)
+{
+ bus_size_t sgsize;
+ bus_addr_t curaddr;
+ char *kvaddr, *vaddr;
+ int error;
+
+ if (segs == NULL)
+ segs = map->segments;
+
+ if ((dmat->common.flags & BUS_DMA_COULD_BOUNCE) != 0) {
+ _bus_dmamap_count_pages(dmat, map, pmap, buf, buflen, flags);
+ if (map->pagesneeded != 0) {
+ error = _bus_dmamap_reserve_pages(dmat, map, flags);
+ if (error)
+ return (error);
+ }
+ }
+
+ vaddr = buf;
+
+ while (buflen > 0) {
+ /*
+ * Get the physical address for this segment.
+ */
+ if (pmap == kernel_pmap) {
+ curaddr = pmap_kextract((vm_offset_t)vaddr);
+ kvaddr = vaddr;
+ } else {
+ curaddr = pmap_extract(pmap, (vm_offset_t)vaddr);
+ kvaddr = NULL;
+ }
+
+ /*
+ * Compute the segment size, and adjust counts.
+ */
+ sgsize = MIN(buflen, PAGE_SIZE - (curaddr & PAGE_MASK));
+ if (map->pagesneeded != 0 && must_bounce(dmat, curaddr)) {
+ sgsize = roundup2(sgsize, dmat->common.alignment);
+ sgsize = MIN(sgsize, buflen);
+ curaddr = add_bounce_page(dmat, map, kvaddr, curaddr,
+ sgsize);
+ }
+
+ if (!_bus_dmamap_addsegs(dmat, map, curaddr, sgsize, segs,
+ segp))
+ break;
+ vaddr += sgsize;
+ buflen -= MIN(sgsize, buflen); /* avoid underflow */
+ }
+
+ /*
+ * Did we fit?
+ */
+ return (buflen != 0 ? EFBIG : 0); /* XXX better return value here? */
+}
+
+static void
+bounce_bus_dmamap_waitok(bus_dma_tag_t dmat, bus_dmamap_t map,
+ struct memdesc *mem, bus_dmamap_callback_t *callback,
+ void *callback_arg)
+{
+
+ if (dmat->common.flags & BUS_DMA_COULD_BOUNCE) {
+ map->dmat = dmat;
+ map->mem = *mem;
+ map->callback = callback;
+ map->callback_arg = callback_arg;
+ }
+}
+
+static bus_dma_segment_t *
+bounce_bus_dmamap_complete(bus_dma_tag_t dmat, bus_dmamap_t map,
+ bus_dma_segment_t *segs, int nsegs, int error)
+{
+
+ map->nsegs = nsegs;
+ if (segs != NULL)
+ memcpy(map->segments, segs, map->nsegs*sizeof(segs[0]));
+ if (dmat->iommu != NULL)
+ IOMMU_MAP(dmat->iommu, map->segments, &map->nsegs,
+ dmat->common.lowaddr, dmat->common.highaddr, dmat->common.alignment,
+ dmat->common.boundary, dmat->iommu_cookie);
+
+ if (segs != NULL)
+ memcpy(segs, map->segments, map->nsegs*sizeof(segs[0]));
+ else
+ segs = map->segments;
+
+ return (segs);
*** 870 LINES SKIPPED ***