svn commit: r289982 - head/sys/dev/ioat
Conrad E. Meyer
cem at FreeBSD.org
Mon Oct 26 03:30:40 UTC 2015
Author: cem
Date: Mon Oct 26 03:30:38 2015
New Revision: 289982
URL: https://svnweb.freebsd.org/changeset/base/289982
Log:
ioat: Allocate memory for ring resize sanely
Add a new flag for DMA operations, DMA_NO_WAIT. It behaves much like
other NOWAIT flags -- if queueing an operation would sleep, abort and
return NULL instead.
When growing the internal descriptor ring, the memory allocation is
performed outside of all locks. A lock-protected flag is used to avoid
duplicated work. Threads that cannot sleep and attempt to queue
operations when the descriptor ring is full allocate a larger ring with
M_NOWAIT, or bail if that fails.
ioat_reserve_space() could become an external API if is important to
callers that they have room for a sequence of operations, or that those
operations succeed each other directly in the hardware ring.
This patch splits the internal head index (->head) from the hardware's
head-of-chain (DMACOUNT) register (->hw_head). In the future, for
simplicity's sake, we could drop the 'ring' array entirely and just use
a linked list (with head and tail pointers rather than indices).
Suggested by: Witness
Sponsored by: EMC / Isilon Storage Division
Modified:
head/sys/dev/ioat/ioat.c
head/sys/dev/ioat/ioat.h
head/sys/dev/ioat/ioat_internal.h
Modified: head/sys/dev/ioat/ioat.c
==============================================================================
--- head/sys/dev/ioat/ioat.c Mon Oct 26 03:15:06 2015 (r289981)
+++ head/sys/dev/ioat/ioat.c Mon Oct 26 03:30:38 2015 (r289982)
@@ -68,13 +68,21 @@ static boolean_t ioat_model_resets_msix(
static void ioat_process_events(struct ioat_softc *ioat);
static inline uint32_t ioat_get_active(struct ioat_softc *ioat);
static inline uint32_t ioat_get_ring_space(struct ioat_softc *ioat);
+static void ioat_free_ring(struct ioat_softc *, uint32_t size,
+ struct ioat_descriptor **);
static void ioat_free_ring_entry(struct ioat_softc *ioat,
struct ioat_descriptor *desc);
-static struct ioat_descriptor *ioat_alloc_ring_entry(struct ioat_softc *ioat);
-static int ioat_reserve_space_and_lock(struct ioat_softc *ioat, int num_descs);
+static struct ioat_descriptor *ioat_alloc_ring_entry(struct ioat_softc *,
+ int mflags);
+static int ioat_reserve_space(struct ioat_softc *, uint32_t, int mflags);
static struct ioat_descriptor *ioat_get_ring_entry(struct ioat_softc *ioat,
uint32_t index);
-static boolean_t resize_ring(struct ioat_softc *ioat, int order);
+static struct ioat_descriptor **ioat_prealloc_ring(struct ioat_softc *,
+ uint32_t size, boolean_t need_dscr, int mflags);
+static int ring_grow(struct ioat_softc *, uint32_t oldorder,
+ struct ioat_descriptor **);
+static int ring_shrink(struct ioat_softc *, uint32_t oldorder,
+ struct ioat_descriptor **);
static void ioat_timer_callback(void *arg);
static void dump_descriptor(void *hw_desc);
static void ioat_submit_single(struct ioat_softc *ioat);
@@ -276,7 +284,6 @@ static int
ioat_detach(device_t device)
{
struct ioat_softc *ioat;
- uint32_t i;
ioat = DEVICE2SOFTC(device);
@@ -292,11 +299,8 @@ ioat_detach(device_t device)
bus_release_resource(device, SYS_RES_MEMORY,
ioat->pci_resource_id, ioat->pci_resource);
- if (ioat->ring != NULL) {
- for (i = 0; i < (1 << ioat->ring_size_order); i++)
- ioat_free_ring_entry(ioat, ioat->ring[i]);
- free(ioat->ring, M_IOAT);
- }
+ if (ioat->ring != NULL)
+ ioat_free_ring(ioat, 1 << ioat->ring_size_order, ioat->ring);
if (ioat->comp_update != NULL) {
bus_dmamap_unload(ioat->comp_update_tag, ioat->comp_update_map);
@@ -409,13 +413,13 @@ ioat3_attach(device_t device)
&ioat->hw_desc_tag);
ioat->ring = malloc(num_descriptors * sizeof(*ring), M_IOAT,
- M_ZERO | M_NOWAIT);
+ M_ZERO | M_WAITOK);
if (ioat->ring == NULL)
return (ENOMEM);
ring = ioat->ring;
for (i = 0; i < num_descriptors; i++) {
- ring[i] = ioat_alloc_ring_entry(ioat);
+ ring[i] = ioat_alloc_ring_entry(ioat, M_WAITOK);
if (ring[i] == NULL)
return (ENOMEM);
@@ -431,7 +435,7 @@ ioat3_attach(device_t device)
ring[i]->u.dma->next = ring[0]->hw_desc_bus_addr;
- ioat->head = 0;
+ ioat->head = ioat->hw_head = 0;
ioat->tail = 0;
ioat->last_seen = 0;
return (0);
@@ -598,6 +602,7 @@ out:
mtx_unlock(&ioat->cleanup_lock);
ioat_putn(ioat, completed, IOAT_ACTIVE_DESCR_REF);
+ wakeup(&ioat->tail);
}
/*
@@ -638,7 +643,7 @@ ioat_release(bus_dmaengine_t dmaengine)
ioat = to_ioat_softc(dmaengine);
CTR0(KTR_IOAT, __func__);
- ioat_write_2(ioat, IOAT_DMACOUNT_OFFSET, (uint16_t)ioat->head);
+ ioat_write_2(ioat, IOAT_DMACOUNT_OFFSET, (uint16_t)ioat->hw_head);
mtx_unlock(&ioat->submit_lock);
}
@@ -649,14 +654,19 @@ ioat_null(bus_dmaengine_t dmaengine, bus
struct ioat_softc *ioat;
struct ioat_descriptor *desc;
struct ioat_dma_hw_descriptor *hw_desc;
+ int mflags;
KASSERT((flags & ~DMA_ALL_FLAGS) == 0, ("Unrecognized flag(s): %#x",
flags & ~DMA_ALL_FLAGS));
+ if ((flags & DMA_NO_WAIT) != 0)
+ mflags = M_NOWAIT;
+ else
+ mflags = M_WAITOK;
ioat = to_ioat_softc(dmaengine);
mtx_assert(&ioat->submit_lock, MA_OWNED);
- if (ioat_reserve_space_and_lock(ioat, 1) != 0)
+ if (ioat_reserve_space(ioat, 1, mflags) != 0)
return (NULL);
CTR0(KTR_IOAT, __func__);
@@ -690,9 +700,14 @@ ioat_copy(bus_dmaengine_t dmaengine, bus
struct ioat_descriptor *desc;
struct ioat_dma_hw_descriptor *hw_desc;
struct ioat_softc *ioat;
+ int mflags;
KASSERT((flags & ~DMA_ALL_FLAGS) == 0, ("Unrecognized flag(s): %#x",
flags & ~DMA_ALL_FLAGS));
+ if ((flags & DMA_NO_WAIT) != 0)
+ mflags = M_NOWAIT;
+ else
+ mflags = M_WAITOK;
ioat = to_ioat_softc(dmaengine);
mtx_assert(&ioat->submit_lock, MA_OWNED);
@@ -703,7 +718,7 @@ ioat_copy(bus_dmaengine_t dmaengine, bus
return (NULL);
}
- if (ioat_reserve_space_and_lock(ioat, 1) != 0)
+ if (ioat_reserve_space(ioat, 1, mflags) != 0)
return (NULL);
CTR0(KTR_IOAT, __func__);
@@ -749,21 +764,26 @@ ioat_get_ring_space(struct ioat_softc *i
}
static struct ioat_descriptor *
-ioat_alloc_ring_entry(struct ioat_softc *ioat)
+ioat_alloc_ring_entry(struct ioat_softc *ioat, int mflags)
{
struct ioat_dma_hw_descriptor *hw_desc;
struct ioat_descriptor *desc;
- int error;
+ int error, busdmaflag;
error = ENOMEM;
hw_desc = NULL;
- desc = malloc(sizeof(*desc), M_IOAT, M_NOWAIT);
+ if ((mflags & M_WAITOK) != 0)
+ busdmaflag = BUS_DMA_WAITOK;
+ else
+ busdmaflag = BUS_DMA_NOWAIT;
+
+ desc = malloc(sizeof(*desc), M_IOAT, mflags);
if (desc == NULL)
goto out;
bus_dmamem_alloc(ioat->hw_desc_tag, (void **)&hw_desc,
- BUS_DMA_ZERO | BUS_DMA_NOWAIT, &ioat->hw_desc_map);
+ BUS_DMA_ZERO | busdmaflag, &ioat->hw_desc_map);
if (hw_desc == NULL)
goto out;
@@ -771,7 +791,7 @@ ioat_alloc_ring_entry(struct ioat_softc
error = bus_dmamap_load(ioat->hw_desc_tag, ioat->hw_desc_map, hw_desc,
sizeof(*hw_desc), ioat_dmamap_cb, &desc->hw_desc_bus_addr,
- BUS_DMA_NOWAIT);
+ busdmaflag);
if (error)
goto out;
@@ -796,22 +816,130 @@ ioat_free_ring_entry(struct ioat_softc *
free(desc, M_IOAT);
}
+/*
+ * Reserves space in this IOAT descriptor ring by ensuring enough slots remain
+ * for 'num_descs'.
+ *
+ * If mflags contains M_WAITOK, blocks until enough space is available.
+ *
+ * Returns zero on success, or an errno on error. If num_descs is beyond the
+ * maximum ring size, returns EINVAl; if allocation would block and mflags
+ * contains M_NOWAIT, returns EAGAIN.
+ *
+ * Must be called with the submit_lock held; returns with the lock held. The
+ * lock may be dropped to allocate the ring.
+ *
+ * (The submit_lock is needed to add any entries to the ring, so callers are
+ * assured enough room is available.)
+ */
static int
-ioat_reserve_space_and_lock(struct ioat_softc *ioat, int num_descs)
+ioat_reserve_space(struct ioat_softc *ioat, uint32_t num_descs, int mflags)
{
- boolean_t retry;
+ struct ioat_descriptor **new_ring;
+ uint32_t order;
+ int error;
- while (1) {
+ mtx_assert(&ioat->submit_lock, MA_OWNED);
+ error = 0;
+
+ if (num_descs < 1 || num_descs > (1 << IOAT_MAX_ORDER)) {
+ error = EINVAL;
+ goto out;
+ }
+
+ for (;;) {
if (ioat_get_ring_space(ioat) >= num_descs)
- return (0);
+ goto out;
- mtx_lock(&ioat->cleanup_lock);
- retry = resize_ring(ioat, ioat->ring_size_order + 1);
- mtx_unlock(&ioat->cleanup_lock);
+ order = ioat->ring_size_order;
+ if (ioat->is_resize_pending || order == IOAT_MAX_ORDER) {
+ if ((mflags & M_WAITOK) != 0) {
+ msleep(&ioat->tail, &ioat->submit_lock, 0,
+ "ioat_rsz", 0);
+ continue;
+ }
- if (!retry)
- return (ENOMEM);
+ error = EAGAIN;
+ break;
+ }
+
+ ioat->is_resize_pending = TRUE;
+ for (;;) {
+ mtx_unlock(&ioat->submit_lock);
+
+ new_ring = ioat_prealloc_ring(ioat, 1 << (order + 1),
+ TRUE, mflags);
+
+ mtx_lock(&ioat->submit_lock);
+ KASSERT(ioat->ring_size_order == order,
+ ("is_resize_pending should protect order"));
+
+ if (new_ring == NULL) {
+ KASSERT((mflags & M_WAITOK) == 0,
+ ("allocation failed"));
+ error = EAGAIN;
+ break;
+ }
+
+ error = ring_grow(ioat, order, new_ring);
+ if (error == 0)
+ break;
+ }
+ ioat->is_resize_pending = FALSE;
+ wakeup(&ioat->tail);
+ if (error)
+ break;
+ }
+
+out:
+ mtx_assert(&ioat->submit_lock, MA_OWNED);
+ return (error);
+}
+
+static struct ioat_descriptor **
+ioat_prealloc_ring(struct ioat_softc *ioat, uint32_t size, boolean_t need_dscr,
+ int mflags)
+{
+ struct ioat_descriptor **ring;
+ uint32_t i;
+ int error;
+
+ KASSERT(size > 0 && powerof2(size), ("bogus size"));
+
+ ring = malloc(size * sizeof(*ring), M_IOAT, M_ZERO | mflags);
+ if (ring == NULL)
+ return (NULL);
+
+ if (need_dscr) {
+ error = ENOMEM;
+ for (i = size / 2; i < size; i++) {
+ ring[i] = ioat_alloc_ring_entry(ioat, mflags);
+ if (ring[i] == NULL)
+ goto out;
+ ring[i]->id = i;
+ }
}
+ error = 0;
+
+out:
+ if (error != 0 && ring != NULL) {
+ ioat_free_ring(ioat, size, ring);
+ ring = NULL;
+ }
+ return (ring);
+}
+
+static void
+ioat_free_ring(struct ioat_softc *ioat, uint32_t size,
+ struct ioat_descriptor **ring)
+{
+ uint32_t i;
+
+ for (i = 0; i < size; i++) {
+ if (ring[i] != NULL)
+ ioat_free_ring_entry(ioat, ring[i]);
+ }
+ free(ring, M_IOAT);
}
static struct ioat_descriptor *
@@ -821,105 +949,155 @@ ioat_get_ring_entry(struct ioat_softc *i
return (ioat->ring[index % (1 << ioat->ring_size_order)]);
}
-static boolean_t
-resize_ring(struct ioat_softc *ioat, int order)
+static int
+ring_grow(struct ioat_softc *ioat, uint32_t oldorder,
+ struct ioat_descriptor **newring)
{
- struct ioat_descriptor **ring;
- struct ioat_descriptor *next;
+ struct ioat_descriptor *tmp, *next;
struct ioat_dma_hw_descriptor *hw;
- struct ioat_descriptor *ent;
- uint32_t current_size, active, new_size, i, new_idx, current_idx;
- uint32_t new_idx2;
-
- current_size = 1 << ioat->ring_size_order;
- active = (ioat->head - ioat->tail) & (current_size - 1);
- new_size = 1 << order;
+ uint32_t oldsize, newsize, head, tail, i, end;
+ int error;
- if (order > IOAT_MAX_ORDER)
- return (FALSE);
+ CTR0(KTR_IOAT, __func__);
- /*
- * when shrinking, verify that we can hold the current active
- * set in the new ring
- */
- if (active >= new_size)
- return (FALSE);
+ mtx_assert(&ioat->submit_lock, MA_OWNED);
- /* allocate the array to hold the software ring */
- ring = malloc(new_size * sizeof(*ring), M_IOAT, M_ZERO | M_NOWAIT);
- if (ring == NULL)
- return (FALSE);
+ if (oldorder != ioat->ring_size_order || oldorder >= IOAT_MAX_ORDER) {
+ error = EINVAL;
+ goto out;
+ }
- ioat_log_message(2, "ring resize: new: %d old: %d\n",
- new_size, current_size);
+ oldsize = (1 << oldorder);
+ newsize = (1 << (oldorder + 1));
- /* allocate/trim descriptors as needed */
- if (new_size > current_size) {
- /* copy current descriptors to the new ring */
- for (i = 0; i < current_size; i++) {
- current_idx = (ioat->tail + i) & (current_size - 1);
- new_idx = (ioat->tail + i) & (new_size - 1);
+ mtx_lock(&ioat->cleanup_lock);
- ring[new_idx] = ioat->ring[current_idx];
- ring[new_idx]->id = new_idx;
- }
+ head = ioat->head & (oldsize - 1);
+ tail = ioat->tail & (oldsize - 1);
- /* add new descriptors to the ring */
- for (i = current_size; i < new_size; i++) {
- new_idx = (ioat->tail + i) & (new_size - 1);
-
- ring[new_idx] = ioat_alloc_ring_entry(ioat);
- if (ring[new_idx] == NULL) {
- while (i--) {
- new_idx2 = (ioat->tail + i) &
- (new_size - 1);
-
- ioat_free_ring_entry(ioat,
- ring[new_idx2]);
- }
- free(ring, M_IOAT);
- return (FALSE);
- }
- ring[new_idx]->id = new_idx;
- }
+ /* Copy old descriptors to new ring */
+ for (i = 0; i < oldsize; i++)
+ newring[i] = ioat->ring[i];
- for (i = current_size - 1; i < new_size; i++) {
- new_idx = (ioat->tail + i) & (new_size - 1);
- next = ring[(new_idx + 1) & (new_size - 1)];
- hw = ring[new_idx]->u.dma;
+ /*
+ * If head has wrapped but tail hasn't, we must swap some descriptors
+ * around so that tail can increment directly to head.
+ */
+ if (head < tail) {
+ for (i = 0; i <= head; i++) {
+ tmp = newring[oldsize + i];
- hw->next = next->hw_desc_bus_addr;
- }
- } else {
- /*
- * copy current descriptors to the new ring, dropping the
- * removed descriptors
- */
- for (i = 0; i < new_size; i++) {
- current_idx = (ioat->tail + i) & (current_size - 1);
- new_idx = (ioat->tail + i) & (new_size - 1);
+ newring[oldsize + i] = newring[i];
+ newring[oldsize + i]->id = oldsize + i;
- ring[new_idx] = ioat->ring[current_idx];
- ring[new_idx]->id = new_idx;
+ newring[i] = tmp;
+ newring[i]->id = i;
}
+ head += oldsize;
+ }
- /* free deleted descriptors */
- for (i = new_size; i < current_size; i++) {
- ent = ioat_get_ring_entry(ioat, ioat->tail + i);
- ioat_free_ring_entry(ioat, ent);
- }
+ KASSERT(head >= tail, ("invariants"));
+
+ /* Head didn't wrap; we only need to link in oldsize..newsize */
+ if (head < oldsize) {
+ i = oldsize - 1;
+ end = newsize;
+ } else {
+ /* Head did wrap; link newhead..newsize and 0..oldhead */
+ i = head;
+ end = newsize + (head - oldsize) + 1;
+ }
+
+ /*
+ * Fix up hardware ring, being careful not to trample the active
+ * section (tail -> head).
+ */
+ for (; i < end; i++) {
+ KASSERT((i & (newsize - 1)) < tail ||
+ (i & (newsize - 1)) >= head, ("trampling snake"));
- /* fix up hardware ring */
- hw = ring[(ioat->tail + new_size - 1) & (new_size - 1)]->u.dma;
- next = ring[(ioat->tail + new_size) & (new_size - 1)];
+ next = newring[(i + 1) & (newsize - 1)];
+ hw = newring[i & (newsize - 1)]->u.dma;
hw->next = next->hw_desc_bus_addr;
}
free(ioat->ring, M_IOAT);
- ioat->ring = ring;
- ioat->ring_size_order = order;
+ ioat->ring = newring;
+ ioat->ring_size_order = oldorder + 1;
+ ioat->tail = tail;
+ ioat->head = head;
+ error = 0;
- return (TRUE);
+ mtx_unlock(&ioat->cleanup_lock);
+out:
+ if (error)
+ ioat_free_ring(ioat, (1 << (oldorder + 1)), newring);
+ return (error);
+}
+
+static int
+ring_shrink(struct ioat_softc *ioat, uint32_t oldorder,
+ struct ioat_descriptor **newring)
+{
+ struct ioat_dma_hw_descriptor *hw;
+ struct ioat_descriptor *ent, *next;
+ uint32_t oldsize, newsize, current_idx, new_idx, i;
+ int error;
+
+ CTR0(KTR_IOAT, __func__);
+
+ mtx_assert(&ioat->submit_lock, MA_OWNED);
+
+ if (oldorder != ioat->ring_size_order || oldorder <= IOAT_MIN_ORDER) {
+ error = EINVAL;
+ goto out_unlocked;
+ }
+
+ oldsize = (1 << oldorder);
+ newsize = (1 << (oldorder - 1));
+
+ mtx_lock(&ioat->cleanup_lock);
+
+ /* Can't shrink below current active set! */
+ if (ioat_get_active(ioat) >= newsize) {
+ error = ENOMEM;
+ goto out;
+ }
+
+ /*
+ * Copy current descriptors to the new ring, dropping the removed
+ * descriptors.
+ */
+ for (i = 0; i < newsize; i++) {
+ current_idx = (ioat->tail + i) & (oldsize - 1);
+ new_idx = (ioat->tail + i) & (newsize - 1);
+
+ newring[new_idx] = ioat->ring[current_idx];
+ newring[new_idx]->id = new_idx;
+ }
+
+ /* Free deleted descriptors */
+ for (i = newsize; i < oldsize; i++) {
+ ent = ioat_get_ring_entry(ioat, ioat->tail + i);
+ ioat_free_ring_entry(ioat, ent);
+ }
+
+ /* Fix up hardware ring. */
+ hw = newring[(ioat->tail + newsize - 1) & (newsize - 1)]->u.dma;
+ next = newring[(ioat->tail + newsize) & (newsize - 1)];
+ hw->next = next->hw_desc_bus_addr;
+
+ free(ioat->ring, M_IOAT);
+ ioat->ring = newring;
+ ioat->ring_size_order = oldorder - 1;
+ error = 0;
+
+out:
+ mtx_unlock(&ioat->cleanup_lock);
+out_unlocked:
+ if (error)
+ ioat_free_ring(ioat, (1 << (oldorder - 1)), newring);
+ return (error);
}
static void
@@ -931,19 +1109,22 @@ ioat_halted_debug(struct ioat_softc *ioa
if (chanerr == 0)
return;
+ mtx_lock(&ioat->submit_lock);
desc = ioat_get_ring_entry(ioat, ioat->tail + 0);
dump_descriptor(desc->u.raw);
desc = ioat_get_ring_entry(ioat, ioat->tail + 1);
dump_descriptor(desc->u.raw);
+ mtx_unlock(&ioat->submit_lock);
}
static void
ioat_timer_callback(void *arg)
{
+ struct ioat_descriptor **newring;
struct ioat_softc *ioat;
uint64_t status;
- uint32_t chanerr;
+ uint32_t chanerr, order;
ioat = arg;
ioat_log_message(1, "%s\n", __func__);
@@ -962,17 +1143,31 @@ ioat_timer_callback(void *arg)
ioat_process_events(ioat);
} else {
mtx_lock(&ioat->submit_lock);
- mtx_lock(&ioat->cleanup_lock);
+ order = ioat->ring_size_order;
+ if (ioat->is_resize_pending || order == IOAT_MIN_ORDER) {
+ mtx_unlock(&ioat->submit_lock);
+ goto out;
+ }
+ ioat->is_resize_pending = TRUE;
+ mtx_unlock(&ioat->submit_lock);
+
+ newring = ioat_prealloc_ring(ioat, 1 << (order - 1), FALSE,
+ M_NOWAIT);
+
+ mtx_lock(&ioat->submit_lock);
+ KASSERT(ioat->ring_size_order == order,
+ ("resize_pending protects order"));
- if (ioat_get_active(ioat) == 0 &&
- ioat->ring_size_order > IOAT_MIN_ORDER)
- resize_ring(ioat, ioat->ring_size_order - 1);
+ if (newring != NULL)
+ ring_shrink(ioat, order, newring);
- mtx_unlock(&ioat->cleanup_lock);
+ ioat->is_resize_pending = FALSE;
mtx_unlock(&ioat->submit_lock);
+out:
+ /* Slowly scale the ring down if idle. */
if (ioat->ring_size_order > IOAT_MIN_ORDER)
- callout_reset(&ioat->timer, IOAT_INTR_TIMO,
+ callout_reset(&ioat->timer, 10 * hz,
ioat_timer_callback, ioat);
}
}
@@ -986,6 +1181,7 @@ ioat_submit_single(struct ioat_softc *io
ioat_get(ioat, IOAT_ACTIVE_DESCR_REF);
atomic_add_rel_int(&ioat->head, 1);
+ atomic_add_rel_int(&ioat->hw_head, 1);
if (!ioat->is_completion_pending) {
ioat->is_completion_pending = TRUE;
@@ -1071,7 +1267,7 @@ ioat_reset_hw(struct ioat_softc *ioat)
* The internal ring counter resets to zero, so we have to start over
* at zero as well.
*/
- ioat->tail = ioat->head = 0;
+ ioat->tail = ioat->head = ioat->hw_head = 0;
ioat->last_seen = 0;
ioat_write_chanctrl(ioat, IOAT_CHANCTRL_RUN);
@@ -1134,11 +1330,13 @@ ioat_setup_sysctl(device_t device)
&ioat->max_xfer_size, 0, "HW maximum transfer size");
SYSCTL_ADD_UINT(ctx, par, OID_AUTO, "ring_size_order", CTLFLAG_RD,
- &ioat->ring_size_order, 0, "HW descriptor ring size order");
+ &ioat->ring_size_order, 0, "SW descriptor ring size order");
SYSCTL_ADD_UINT(ctx, par, OID_AUTO, "head", CTLFLAG_RD, &ioat->head, 0,
- "HW descriptor head pointer index");
+ "SW descriptor head pointer index");
SYSCTL_ADD_UINT(ctx, par, OID_AUTO, "tail", CTLFLAG_RD, &ioat->tail, 0,
- "HW descriptor tail pointer index");
+ "SW descriptor tail pointer index");
+ SYSCTL_ADD_UINT(ctx, par, OID_AUTO, "hw_head", CTLFLAG_RD,
+ &ioat->hw_head, 0, "HW DMACOUNT");
SYSCTL_ADD_UQUAD(ctx, par, OID_AUTO, "last_completion", CTLFLAG_RD,
ioat->comp_update, "HW addr of last completion");
Modified: head/sys/dev/ioat/ioat.h
==============================================================================
--- head/sys/dev/ioat/ioat.h Mon Oct 26 03:15:06 2015 (r289981)
+++ head/sys/dev/ioat/ioat.h Mon Oct 26 03:30:38 2015 (r289982)
@@ -41,7 +41,12 @@ __FBSDID("$FreeBSD$");
* this on the last operation in a group
*/
#define DMA_INT_EN 0x1
-#define DMA_ALL_FLAGS (DMA_INT_EN)
+/*
+ * Like M_NOWAIT. Operations will return NULL if they cannot allocate a
+ * descriptor without blocking.
+ */
+#define DMA_NO_WAIT 0x2
+#define DMA_ALL_FLAGS (DMA_INT_EN | DMA_NO_WAIT)
typedef void *bus_dmaengine_t;
struct bus_dmadesc;
Modified: head/sys/dev/ioat/ioat_internal.h
==============================================================================
--- head/sys/dev/ioat/ioat_internal.h Mon Oct 26 03:15:06 2015 (r289981)
+++ head/sys/dev/ioat/ioat_internal.h Mon Oct 26 03:30:38 2015 (r289982)
@@ -361,6 +361,7 @@ struct ioat_softc {
uint32_t head;
uint32_t tail;
+ uint32_t hw_head;
uint32_t ring_size_order;
bus_addr_t last_seen;
More information about the svn-src-head
mailing list