svn commit: r347337 - head/sys/dev/xdma

Ruslan Bukin br at FreeBSD.org
Wed May 8 15:22:29 UTC 2019


Author: br
Date: Wed May  8 15:22:27 2019
New Revision: 347337
URL: https://svnweb.freebsd.org/changeset/base/347337

Log:
  o Implement a bounce buffer based on device reserved memory.
    Grab device reserved physical memory regions from FDT using standard
    "memory-region" property and use vmem(9) to allocate buffers from it.
    The same vmem could be used by DMA engine drivers to allocate memory for
    DMA descriptors.
    This is required for platforms that provide uncached memory region
    reserved exclusively for DMA operations.
  o Change sleepable sx(9) lock type to non-sleepable mutex(9) since
    network drivers usually hold mutex during DMA operations. So we don't
    take sleepable lock after non-sleepable.
  
  Tested on U.S. Government Furnished Equipment (GFE) 64-bit RISC-V cores.
  
  Sponsored by:	DARPA, AFRL

Modified:
  head/sys/dev/xdma/xdma.c
  head/sys/dev/xdma/xdma.h
  head/sys/dev/xdma/xdma_bank.c
  head/sys/dev/xdma/xdma_bio.c
  head/sys/dev/xdma/xdma_mbuf.c
  head/sys/dev/xdma/xdma_queue.c
  head/sys/dev/xdma/xdma_sg.c

Modified: head/sys/dev/xdma/xdma.c
==============================================================================
--- head/sys/dev/xdma/xdma.c	Wed May  8 15:20:16 2019	(r347336)
+++ head/sys/dev/xdma/xdma.c	Wed May  8 15:22:27 2019	(r347337)
@@ -1,7 +1,8 @@
 /*-
- * Copyright (c) 2016-2018 Ruslan Bukin <br at bsdpad.com>
- * All rights reserved.
+ * SPDX-License-Identifier: BSD-2-Clause
  *
+ * Copyright (c) 2016-2019 Ruslan Bukin <br at bsdpad.com>
+ *
  * This software was developed by SRI International and the University of
  * Cambridge Computer Laboratory under DARPA/AFRL contract FA8750-10-C-0237
  * ("CTSRD"), as part of the DARPA CRASH research programme.
@@ -43,7 +44,6 @@ __FBSDID("$FreeBSD$");
 #include <sys/lock.h>
 #include <sys/sysctl.h>
 #include <sys/systm.h>
-#include <sys/sx.h>
 
 #include <machine/bus.h>
 
@@ -61,12 +61,14 @@ __FBSDID("$FreeBSD$");
  * Multiple xDMA controllers may work with single DMA device,
  * so we have global lock for physical channel management.
  */
-static struct sx xdma_sx;
+static struct mtx xdma_mtx;
 
-#define	XDMA_LOCK()			sx_xlock(&xdma_sx)
-#define	XDMA_UNLOCK()			sx_xunlock(&xdma_sx)
-#define	XDMA_ASSERT_LOCKED()		sx_xassert(&xdma_sx, MA_OWNED)
+#define	XDMA_LOCK()			mtx_lock(&xdma_mtx)
+#define	XDMA_UNLOCK()			mtx_unlock(&xdma_mtx)
+#define	XDMA_ASSERT_LOCKED()		mtx_assert(&xdma_mtx, MA_OWNED)
 
+#define	FDT_REG_CELLS	4
+
 /*
  * Allocate virtual xDMA channel.
  */
@@ -95,11 +97,11 @@ xdma_channel_alloc(xdma_controller_t *xdma, uint32_t c
 
 	TAILQ_INIT(&xchan->ie_handlers);
 
-	sx_init(&xchan->sx_lock, "xDMA chan");
-	sx_init(&xchan->sx_qin_lock, "xDMA qin");
-	sx_init(&xchan->sx_qout_lock, "xDMA qout");
-	sx_init(&xchan->sx_bank_lock, "xDMA bank");
-	sx_init(&xchan->sx_proc_lock, "xDMA proc");
+	mtx_init(&xchan->mtx_lock, "xDMA chan", NULL, MTX_DEF);
+	mtx_init(&xchan->mtx_qin_lock, "xDMA qin", NULL, MTX_DEF);
+	mtx_init(&xchan->mtx_qout_lock, "xDMA qout", NULL, MTX_DEF);
+	mtx_init(&xchan->mtx_bank_lock, "xDMA bank", NULL, MTX_DEF);
+	mtx_init(&xchan->mtx_proc_lock, "xDMA proc", NULL, MTX_DEF);
 
 	TAILQ_INIT(&xchan->bank);
 	TAILQ_INIT(&xchan->queue_in);
@@ -138,11 +140,11 @@ xdma_channel_free(xdma_channel_t *xchan)
 
 	xdma_teardown_all_intr(xchan);
 
-	sx_destroy(&xchan->sx_lock);
-	sx_destroy(&xchan->sx_qin_lock);
-	sx_destroy(&xchan->sx_qout_lock);
-	sx_destroy(&xchan->sx_bank_lock);
-	sx_destroy(&xchan->sx_proc_lock);
+	mtx_destroy(&xchan->mtx_lock);
+	mtx_destroy(&xchan->mtx_qin_lock);
+	mtx_destroy(&xchan->mtx_qout_lock);
+	mtx_destroy(&xchan->mtx_bank_lock);
+	mtx_destroy(&xchan->mtx_proc_lock);
 
 	TAILQ_REMOVE(&xdma->channels, xchan, xchan_next);
 
@@ -303,6 +305,95 @@ xdma_ofw_md_data(xdma_controller_t *xdma, pcell_t *cel
 	return (ret);
 }
 
+static int
+xdma_handle_mem_node(vmem_t *vmem, phandle_t memory)
+{
+	pcell_t reg[FDT_REG_CELLS * FDT_MEM_REGIONS];
+	pcell_t *regp;
+	int addr_cells, size_cells;
+	int i, reg_len, ret, tuple_size, tuples;
+	vmem_addr_t mem_start;
+	vmem_size_t mem_size;
+
+	if ((ret = fdt_addrsize_cells(OF_parent(memory), &addr_cells,
+	    &size_cells)) != 0)
+		return (ret);
+
+	if (addr_cells > 2)
+		return (ERANGE);
+
+	tuple_size = sizeof(pcell_t) * (addr_cells + size_cells);
+	reg_len = OF_getproplen(memory, "reg");
+	if (reg_len <= 0 || reg_len > sizeof(reg))
+		return (ERANGE);
+
+	if (OF_getprop(memory, "reg", reg, reg_len) <= 0)
+		return (ENXIO);
+
+	tuples = reg_len / tuple_size;
+	regp = (pcell_t *)®
+	for (i = 0; i < tuples; i++) {
+		ret = fdt_data_to_res(regp, addr_cells, size_cells,
+		    &mem_start, &mem_size);
+		if (ret != 0)
+			return (ret);
+
+		vmem_add(vmem, mem_start, mem_size, 0);
+		regp += addr_cells + size_cells;
+	}
+
+	return (0);
+}
+
+vmem_t *
+xdma_get_memory(device_t dev)
+{
+	phandle_t mem_node, node;
+	pcell_t mem_handle;
+	vmem_t *vmem;
+
+	node = ofw_bus_get_node(dev);
+	if (node <= 0) {
+		device_printf(dev,
+		    "%s called on not ofw based device.\n", __func__);
+		return (NULL);
+	}
+
+	if (!OF_hasprop(node, "memory-region"))
+		return (NULL);
+
+	if (OF_getencprop(node, "memory-region", (void *)&mem_handle,
+	    sizeof(mem_handle)) <= 0)
+		return (NULL);
+
+	vmem = vmem_create("xDMA vmem", 0, 0, PAGE_SIZE,
+	    PAGE_SIZE, M_BESTFIT | M_WAITOK);
+	if (vmem == NULL)
+		return (NULL);
+
+	mem_node = OF_node_from_xref(mem_handle);
+	if (xdma_handle_mem_node(vmem, mem_node) != 0) {
+		vmem_destroy(vmem);
+		return (NULL);
+	}
+
+	return (vmem);
+}
+
+void
+xdma_put_memory(vmem_t *vmem)
+{
+
+	vmem_destroy(vmem);
+}
+
+void
+xchan_set_memory(xdma_channel_t *xchan, vmem_t *vmem)
+{
+
+	xchan->vmem = vmem;
+}
+
 /*
  * Allocate xdma controller.
  */
@@ -400,7 +491,7 @@ static void
 xdma_init(void)
 {
 
-	sx_init(&xdma_sx, "xDMA");
+	mtx_init(&xdma_mtx, "xDMA", NULL, MTX_DEF);
 }
 
 SYSINIT(xdma, SI_SUB_DRIVERS, SI_ORDER_FIRST, xdma_init, NULL);

Modified: head/sys/dev/xdma/xdma.h
==============================================================================
--- head/sys/dev/xdma/xdma.h	Wed May  8 15:20:16 2019	(r347336)
+++ head/sys/dev/xdma/xdma.h	Wed May  8 15:22:27 2019	(r347337)
@@ -1,7 +1,8 @@
 /*-
- * Copyright (c) 2016-2018 Ruslan Bukin <br at bsdpad.com>
- * All rights reserved.
+ * SPDX-License-Identifier: BSD-2-Clause
  *
+ * Copyright (c) 2016-2019 Ruslan Bukin <br at bsdpad.com>
+ *
  * This software was developed by SRI International and the University of
  * Cambridge Computer Laboratory under DARPA/AFRL contract FA8750-10-C-0237
  * ("CTSRD"), as part of the DARPA CRASH research programme.
@@ -34,6 +35,7 @@
 #define _DEV_XDMA_XDMA_H_
 
 #include <sys/proc.h>
+#include <sys/vmem.h>
 
 enum xdma_direction {
 	XDMA_MEM_TO_MEM,
@@ -73,6 +75,7 @@ struct xdma_controller {
 	device_t dev;		/* DMA consumer device_t. */
 	device_t dma_dev;	/* A real DMA device_t. */
 	void *data;		/* OFW MD part. */
+	vmem_t *vmem;		/* Bounce memory. */
 
 	/* List of virtual channels allocated. */
 	TAILQ_HEAD(xdma_channel_list, xdma_channel)	channels;
@@ -84,6 +87,9 @@ struct xchan_buf {
 	bus_dmamap_t			map;
 	uint32_t			nsegs;
 	uint32_t			nsegs_left;
+	vm_offset_t			vaddr;
+	vm_offset_t			paddr;
+	vm_size_t			size;
 };
 
 struct xdma_request {
@@ -117,6 +123,7 @@ struct xdma_sglist {
 
 struct xdma_channel {
 	xdma_controller_t		*xdma;
+	vmem_t				*vmem;
 
 	uint32_t			flags;
 #define	XCHAN_BUFS_ALLOCATED		(1 << 0)
@@ -139,11 +146,11 @@ struct xdma_channel {
 	TAILQ_HEAD(, xdma_intr_handler)	ie_handlers;
 	TAILQ_ENTRY(xdma_channel)	xchan_next;
 
-	struct sx			sx_lock;
-	struct sx			sx_qin_lock;
-	struct sx			sx_qout_lock;
-	struct sx			sx_bank_lock;
-	struct sx			sx_proc_lock;
+	struct mtx			mtx_lock;
+	struct mtx			mtx_qin_lock;
+	struct mtx			mtx_qout_lock;
+	struct mtx			mtx_bank_lock;
+	struct mtx			mtx_proc_lock;
 
 	/* Request queue. */
 	bus_dma_tag_t			dma_tag_bufs;
@@ -176,30 +183,30 @@ struct xdma_intr_handler {
 
 static MALLOC_DEFINE(M_XDMA, "xdma", "xDMA framework");
 
-#define	XCHAN_LOCK(xchan)		sx_xlock(&(xchan)->sx_lock)
-#define	XCHAN_UNLOCK(xchan)		sx_xunlock(&(xchan)->sx_lock)
+#define	XCHAN_LOCK(xchan)		mtx_lock(&(xchan)->mtx_lock)
+#define	XCHAN_UNLOCK(xchan)		mtx_unlock(&(xchan)->mtx_lock)
 #define	XCHAN_ASSERT_LOCKED(xchan)	\
-    sx_assert(&(xchan)->sx_lock, SX_XLOCKED)
+    mtx_assert(&(xchan)->mtx_lock, MA_OWNED)
 
-#define	QUEUE_IN_LOCK(xchan)		sx_xlock(&(xchan)->sx_qin_lock)
-#define	QUEUE_IN_UNLOCK(xchan)		sx_xunlock(&(xchan)->sx_qin_lock)
+#define	QUEUE_IN_LOCK(xchan)		mtx_lock(&(xchan)->mtx_qin_lock)
+#define	QUEUE_IN_UNLOCK(xchan)		mtx_unlock(&(xchan)->mtx_qin_lock)
 #define	QUEUE_IN_ASSERT_LOCKED(xchan)	\
-    sx_assert(&(xchan)->sx_qin_lock, SX_XLOCKED)
+    mtx_assert(&(xchan)->mtx_qin_lock, MA_OWNED)
 
-#define	QUEUE_OUT_LOCK(xchan)		sx_xlock(&(xchan)->sx_qout_lock)
-#define	QUEUE_OUT_UNLOCK(xchan)		sx_xunlock(&(xchan)->sx_qout_lock)
+#define	QUEUE_OUT_LOCK(xchan)		mtx_lock(&(xchan)->mtx_qout_lock)
+#define	QUEUE_OUT_UNLOCK(xchan)		mtx_unlock(&(xchan)->mtx_qout_lock)
 #define	QUEUE_OUT_ASSERT_LOCKED(xchan)	\
-    sx_assert(&(xchan)->sx_qout_lock, SX_XLOCKED)
+    mtx_assert(&(xchan)->mtx_qout_lock, MA_OWNED)
 
-#define	QUEUE_BANK_LOCK(xchan)		sx_xlock(&(xchan)->sx_bank_lock)
-#define	QUEUE_BANK_UNLOCK(xchan)	sx_xunlock(&(xchan)->sx_bank_lock)
+#define	QUEUE_BANK_LOCK(xchan)		mtx_lock(&(xchan)->mtx_bank_lock)
+#define	QUEUE_BANK_UNLOCK(xchan)	mtx_unlock(&(xchan)->mtx_bank_lock)
 #define	QUEUE_BANK_ASSERT_LOCKED(xchan)	\
-    sx_assert(&(xchan)->sx_bank_lock, SX_XLOCKED)
+    mtx_assert(&(xchan)->mtx_bank_lock, MA_OWNED)
 
-#define	QUEUE_PROC_LOCK(xchan)		sx_xlock(&(xchan)->sx_proc_lock)
-#define	QUEUE_PROC_UNLOCK(xchan)	sx_xunlock(&(xchan)->sx_proc_lock)
+#define	QUEUE_PROC_LOCK(xchan)		mtx_lock(&(xchan)->mtx_proc_lock)
+#define	QUEUE_PROC_UNLOCK(xchan)	mtx_unlock(&(xchan)->mtx_proc_lock)
 #define	QUEUE_PROC_ASSERT_LOCKED(xchan)	\
-    sx_assert(&(xchan)->sx_proc_lock, SX_XLOCKED)
+    mtx_assert(&(xchan)->mtx_proc_lock, MA_OWNED)
 
 #define	XDMA_SGLIST_MAXLEN	2048
 #define	XDMA_MAX_SEG		128
@@ -207,11 +214,14 @@ static MALLOC_DEFINE(M_XDMA, "xdma", "xDMA framework")
 /* xDMA controller ops */
 xdma_controller_t *xdma_ofw_get(device_t dev, const char *prop);
 int xdma_put(xdma_controller_t *xdma);
+vmem_t * xdma_get_memory(device_t dev);
+void xdma_put_memory(vmem_t *vmem);
 
 /* xDMA channel ops */
 xdma_channel_t * xdma_channel_alloc(xdma_controller_t *, uint32_t caps);
 int xdma_channel_free(xdma_channel_t *);
 int xdma_request(xdma_channel_t *xchan, struct xdma_request *r);
+void xchan_set_memory(xdma_channel_t *xchan, vmem_t *vmem);
 
 /* SG interface */
 int xdma_prep_sg(xdma_channel_t *, uint32_t,

Modified: head/sys/dev/xdma/xdma_bank.c
==============================================================================
--- head/sys/dev/xdma/xdma_bank.c	Wed May  8 15:20:16 2019	(r347336)
+++ head/sys/dev/xdma/xdma_bank.c	Wed May  8 15:22:27 2019	(r347337)
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2018 Ruslan Bukin <br at bsdpad.com>
+ * Copyright (c) 2018-2019 Ruslan Bukin <br at bsdpad.com>
  * All rights reserved.
  *
  * This software was developed by SRI International and the University of
@@ -37,7 +37,6 @@ __FBSDID("$FreeBSD$");
 #include <sys/bus.h>
 #include <sys/kernel.h>
 #include <sys/malloc.h>
-#include <sys/sx.h>
 
 #include <machine/bus.h>
 

Modified: head/sys/dev/xdma/xdma_bio.c
==============================================================================
--- head/sys/dev/xdma/xdma_bio.c	Wed May  8 15:20:16 2019	(r347336)
+++ head/sys/dev/xdma/xdma_bio.c	Wed May  8 15:22:27 2019	(r347337)
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2017-2018 Ruslan Bukin <br at bsdpad.com>
+ * Copyright (c) 2017-2019 Ruslan Bukin <br at bsdpad.com>
  * All rights reserved.
  *
  * This software was developed by SRI International and the University of
@@ -37,7 +37,6 @@ __FBSDID("$FreeBSD$");
 #include <sys/bus.h>
 #include <sys/kernel.h>
 #include <sys/malloc.h>
-#include <sys/sx.h>
 
 #include <machine/bus.h>
 

Modified: head/sys/dev/xdma/xdma_mbuf.c
==============================================================================
--- head/sys/dev/xdma/xdma_mbuf.c	Wed May  8 15:20:16 2019	(r347336)
+++ head/sys/dev/xdma/xdma_mbuf.c	Wed May  8 15:22:27 2019	(r347337)
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2017-2018 Ruslan Bukin <br at bsdpad.com>
+ * Copyright (c) 2017-2019 Ruslan Bukin <br at bsdpad.com>
  * All rights reserved.
  *
  * This software was developed by SRI International and the University of
@@ -36,7 +36,6 @@ __FBSDID("$FreeBSD$");
 #include <sys/conf.h>
 #include <sys/bus.h>
 #include <sys/kernel.h>
-#include <sys/sx.h>
 #include <sys/mbuf.h>
 
 #include <machine/bus.h>

Modified: head/sys/dev/xdma/xdma_queue.c
==============================================================================
--- head/sys/dev/xdma/xdma_queue.c	Wed May  8 15:20:16 2019	(r347336)
+++ head/sys/dev/xdma/xdma_queue.c	Wed May  8 15:22:27 2019	(r347337)
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2018 Ruslan Bukin <br at bsdpad.com>
+ * Copyright (c) 2018-2019 Ruslan Bukin <br at bsdpad.com>
  * All rights reserved.
  *
  * This software was developed by SRI International and the University of
@@ -37,7 +37,6 @@ __FBSDID("$FreeBSD$");
 #include <sys/bus.h>
 #include <sys/kernel.h>
 #include <sys/malloc.h>
-#include <sys/sx.h>
 
 #include <machine/bus.h>
 

Modified: head/sys/dev/xdma/xdma_sg.c
==============================================================================
--- head/sys/dev/xdma/xdma_sg.c	Wed May  8 15:20:16 2019	(r347336)
+++ head/sys/dev/xdma/xdma_sg.c	Wed May  8 15:22:27 2019	(r347337)
@@ -1,7 +1,8 @@
 /*-
- * Copyright (c) 2018 Ruslan Bukin <br at bsdpad.com>
- * All rights reserved.
+ * SPDX-License-Identifier: BSD-2-Clause
  *
+ * Copyright (c) 2018-2019 Ruslan Bukin <br at bsdpad.com>
+ *
  * This software was developed by SRI International and the University of
  * Cambridge Computer Laboratory under DARPA/AFRL contract FA8750-10-C-0237
  * ("CTSRD"), as part of the DARPA CRASH research programme.
@@ -38,10 +39,14 @@ __FBSDID("$FreeBSD$");
 #include <sys/kernel.h>
 #include <sys/malloc.h>
 #include <sys/mbuf.h>
-#include <sys/sx.h>
+#include <sys/rwlock.h>
 
 #include <machine/bus.h>
 
+#include <vm/vm.h>
+#include <vm/vm_extern.h>
+#include <vm/vm_page.h>
+
 #ifdef FDT
 #include <dev/fdt/fdt_common.h>
 #include <dev/ofw/ofw_bus.h>
@@ -58,25 +63,71 @@ struct seg_load_request {
 	uint32_t error;
 };
 
+static void
+xchan_bufs_free_reserved(xdma_channel_t *xchan)
+{
+	struct xdma_request *xr;
+	vm_size_t size;
+	int i;
+
+	for (i = 0; i < xchan->xr_num; i++) {
+		xr = &xchan->xr_mem[i];
+		size = xr->buf.size;
+		if (xr->buf.vaddr) {
+			pmap_kremove_device(xr->buf.vaddr, size);
+			kva_free(xr->buf.vaddr, size);
+			xr->buf.vaddr = 0;
+		}
+		if (xr->buf.paddr) {
+			vmem_free(xchan->vmem, xr->buf.paddr, size);
+			xr->buf.paddr = 0;
+		}
+		xr->buf.size = 0;
+	}
+}
+
 static int
-_xchan_bufs_alloc(xdma_channel_t *xchan)
+xchan_bufs_alloc_reserved(xdma_channel_t *xchan)
 {
 	xdma_controller_t *xdma;
 	struct xdma_request *xr;
+	vmem_addr_t addr;
+	vm_size_t size;
 	int i;
 
 	xdma = xchan->xdma;
 
+	if (xchan->vmem == NULL)
+		return (ENOBUFS);
+
 	for (i = 0; i < xchan->xr_num; i++) {
 		xr = &xchan->xr_mem[i];
-		/* TODO: bounce buffer */
+		size = round_page(xchan->maxsegsize);
+		if (vmem_alloc(xchan->vmem, size,
+		    M_BESTFIT | M_NOWAIT, &addr)) {
+			device_printf(xdma->dev,
+			    "%s: Can't allocate memory\n", __func__);
+			xchan_bufs_free_reserved(xchan);
+			return (ENOMEM);
+		}
+		
+		xr->buf.size = size;
+		xr->buf.paddr = addr;
+		xr->buf.vaddr = kva_alloc(size);
+		if (xr->buf.vaddr == 0) {
+			device_printf(xdma->dev,
+			    "%s: Can't allocate KVA\n", __func__);
+			xchan_bufs_free_reserved(xchan);
+			return (ENOMEM);
+		}
+		pmap_kenter_device(xr->buf.vaddr, size, addr);
 	}
 
 	return (0);
 }
 
 static int
-_xchan_bufs_alloc_busdma(xdma_channel_t *xchan)
+xchan_bufs_alloc_busdma(xdma_channel_t *xchan)
 {
 	xdma_controller_t *xdma;
 	struct xdma_request *xr;
@@ -138,9 +189,10 @@ xchan_bufs_alloc(xdma_channel_t *xchan)
 	}
 
 	if (xchan->caps & XCHAN_CAP_BUSDMA)
-		ret = _xchan_bufs_alloc_busdma(xchan);
-	else
-		ret = _xchan_bufs_alloc(xchan);
+		ret = xchan_bufs_alloc_busdma(xchan);
+	else {
+		ret = xchan_bufs_alloc_reserved(xchan);
+	}
 	if (ret != 0) {
 		device_printf(xdma->dev,
 		    "%s: Can't allocate bufs.\n", __func__);
@@ -169,12 +221,8 @@ xchan_bufs_free(xdma_channel_t *xchan)
 			bus_dmamap_destroy(xchan->dma_tag_bufs, b->map);
 		}
 		bus_dma_tag_destroy(xchan->dma_tag_bufs);
-	} else {
-		for (i = 0; i < xchan->xr_num; i++) {
-			xr = &xchan->xr_mem[i];
-			/* TODO: bounce buffer */
-		}
-	}
+	} else
+		xchan_bufs_free_reserved(xchan);
 
 	xchan->flags &= ~XCHAN_BUFS_ALLOCATED;
 
@@ -296,6 +344,11 @@ xchan_seg_done(xdma_channel_t *xchan,
 				bus_dmamap_sync(xchan->dma_tag_bufs, b->map, 
 				    BUS_DMASYNC_POSTREAD);
 			bus_dmamap_unload(xchan->dma_tag_bufs, b->map);
+		} else {
+			if (xr->req_type == XR_TYPE_MBUF &&
+			    xr->direction == XDMA_DEV_TO_MEM)
+				m_copyback(xr->m, 0, st->transferred,
+				    (void *)xr->buf.vaddr);
 		}
 		xr->status.error = st->error;
 		xr->status.transferred = st->transferred;
@@ -437,7 +490,13 @@ _xdma_load_data(xdma_channel_t *xchan, struct xdma_req
 
 	switch (xr->req_type) {
 	case XR_TYPE_MBUF:
-		seg[0].ds_addr = mtod(m, bus_addr_t);
+		if ((xchan->caps & XCHAN_CAP_NOBUFS) == 0) {
+			if (xr->direction == XDMA_MEM_TO_DEV)
+				m_copydata(m, 0, m->m_pkthdr.len,
+				    (void *)xr->buf.vaddr);
+			seg[0].ds_addr = (bus_addr_t)xr->buf.paddr;
+		} else
+			seg[0].ds_addr = mtod(m, bus_addr_t);
 		seg[0].ds_len = m->m_pkthdr.len;
 		break;
 	case XR_TYPE_BIO:
@@ -494,6 +553,7 @@ xdma_process(xdma_channel_t *xchan,
 	xdma = xchan->xdma;
 
 	n = 0;
+	c = 0;
 
 	ret = XDMA_CHANNEL_CAPACITY(xdma->dma_dev, xchan, &capacity);
 	if (ret != 0) {


More information about the svn-src-all mailing list