svn commit: r354223 - head/sys/dev/ena
Marcin Wojtas
mw at FreeBSD.org
Thu Oct 31 15:59:31 UTC 2019
Author: mw
Date: Thu Oct 31 15:59:29 2019
New Revision: 354223
URL: https://svnweb.freebsd.org/changeset/base/354223
Log:
Add support for ENA NETMAP Tx
Two new tables are added to ena_tx_buffer structure:
* netmap_map_seg stores DMA mapping structures,
* netmap_buf_idx stores buff indexes taken from the slots.
When Tx resources are being set, the new mapping structures are created
and netmap Tx rings are being reset.
When Tx resources are being released, used netmap bufs are unmapped from
DMA and then mapping structures are destroyed.
When Tx interrupt occurrs, ena_netmap_tx_irq is called.
ena_netmap_txsync callback signalizes that there are new packets which
should be transmitted.
First, it fills ena_netmap_ctx. Then it performs two actions:
* ena_netmap_tx_frames moves packets from netmap ring to NIC,
* ena_netmap_tx_cleanup restores buffers from NIC and gives them back
to the userspace app.
0 is returned in case of Tx error that could be handled by the driver.
ena_netmap_tx_frames checks if there are packets ready for transmission.
Then, for each of them, ena_netmap_tx_frame is called. If error occurs,
transmitting is stopped, but if the error was cause due to HW ring being
full, information about that is not propagated to the userspace app.
When all packets are ready, doorbell is written to NIC and netmap ring
state is updated.
Parsing of one packet is done by the ena_netmap_tx_frame function.
First, it checks if number of slots does not exceed NIC limit. Invalid
packets are being dropped and the error is propagated to the upper
layer. As each netmap buffer has equal size, which is typically greater
then 2KiB, there shouldn't be any packets which contain too many slots.
Then, the ena_com_tx_ctx structure is being filled. As netmap does not
support any hardware offloads, ena_com_tx_meta structure is set to zero.
After that, ena_netmap_map_slots maps all memory slots for DMA.
If the device works in the LLQ mode, the push header is being determined
by checking if the header fits within the first socket.
If so, the portion of data is being copied directly from the slot.
In other case, the data is copied to the intermediate buffer.
First slots are treated the same as as the others, because DMA mapping
has no impact on LLQ mode. Index of each netmap buffer is taken from
slot and stored in netmap_buf_idx array. In case of mapping error,
memory is unmapped and packets are put back to the netmap ring.
ena_netmap_tx_cleanup performs out of order cleanup of sent buffers.
First, req_id is taken and is validated. As validate_tx_req_id from
ena.c is specific to kernels mbuf, another implementation is provided.
Each req_id is cleaned up by ena_netmap_tx_clean_one function. Buffers
are being unmaped from DMA and put back to netmap ring. In the end,
state of netmap and NIC rings are being updated.
Differential Revision: https://reviews.freebsd.org/D21936
Submitted by: Rafal Kozik <rk at semihalf.com>
Michal Krawczyk <mk at semihalf.com>
Obtained from: Semihalf
Sponsored by: Amazon, Inc.
Modified:
head/sys/dev/ena/ena.c
head/sys/dev/ena/ena.h
head/sys/dev/ena/ena_datapath.c
head/sys/dev/ena/ena_netmap.c
head/sys/dev/ena/ena_netmap.h
Modified: head/sys/dev/ena/ena.c
==============================================================================
--- head/sys/dev/ena/ena.c Thu Oct 31 15:57:44 2019 (r354222)
+++ head/sys/dev/ena/ena.c Thu Oct 31 15:59:29 2019 (r354223)
@@ -103,6 +103,7 @@ static int ena_setup_tx_dma_tag(struct ena_adapter *);
static int ena_free_tx_dma_tag(struct ena_adapter *);
static int ena_setup_rx_dma_tag(struct ena_adapter *);
static int ena_free_rx_dma_tag(struct ena_adapter *);
+static void ena_release_all_tx_dmamap(struct ena_ring *);
static int ena_setup_tx_resources(struct ena_adapter *, int);
static void ena_free_tx_resources(struct ena_adapter *, int);
static int ena_setup_all_tx_resources(struct ena_adapter *);
@@ -531,6 +532,44 @@ ena_free_rx_dma_tag(struct ena_adapter *adapter)
return (ret);
}
+static void
+ena_release_all_tx_dmamap(struct ena_ring *tx_ring)
+{
+ struct ena_adapter *adapter = tx_ring->adapter;
+ struct ena_tx_buffer *tx_info;
+ bus_dma_tag_t tx_tag = adapter->tx_buf_tag;;
+ int i;
+#ifdef DEV_NETMAP
+ struct ena_netmap_tx_info *nm_info;
+ int j;
+#endif /* DEV_NETMAP */
+
+ for (i = 0; i < tx_ring->ring_size; ++i) {
+ tx_info = &tx_ring->tx_buffer_info[i];
+#ifdef DEV_NETMAP
+ if (adapter->ifp->if_capenable & IFCAP_NETMAP) {
+ nm_info = &tx_info->nm_info;
+ for (j = 0; j < ENA_PKT_MAX_BUFS; ++j) {
+ if (nm_info->map_seg[j] != NULL) {
+ bus_dmamap_destroy(tx_tag,
+ nm_info->map_seg[j]);
+ nm_info->map_seg[j] = NULL;
+ }
+ }
+ }
+#endif /* DEV_NETMAP */
+ if (tx_info->map_head != NULL) {
+ bus_dmamap_destroy(tx_tag, tx_info->map_head);
+ tx_info->map_head = NULL;
+ }
+
+ if (tx_info->map_seg != NULL) {
+ bus_dmamap_destroy(tx_tag, tx_info->map_seg);
+ tx_info->map_seg = NULL;
+ }
+ }
+}
+
/**
* ena_setup_tx_resources - allocate Tx resources (Descriptors)
* @adapter: network interface device structure
@@ -544,7 +583,13 @@ ena_setup_tx_resources(struct ena_adapter *adapter, in
struct ena_que *que = &adapter->que[qid];
struct ena_ring *tx_ring = que->tx_ring;
int size, i, err;
+#ifdef DEV_NETMAP
+ bus_dmamap_t *map;
+ int j;
+ ena_netmap_reset_tx_ring(adapter, qid);
+#endif /* DEV_NETMAP */
+
size = sizeof(struct ena_tx_buffer) * tx_ring->ring_size;
tx_ring->tx_buffer_info = malloc(size, M_DEVBUF, M_NOWAIT | M_ZERO);
@@ -587,7 +632,7 @@ ena_setup_tx_resources(struct ena_adapter *adapter, in
ena_trace(ENA_ALERT,
"Unable to create Tx DMA map_head for buffer %d\n",
i);
- goto err_buf_info_unmap;
+ goto err_map_release;
}
tx_ring->tx_buffer_info[i].seg_mapped = false;
@@ -597,9 +642,24 @@ ena_setup_tx_resources(struct ena_adapter *adapter, in
ena_trace(ENA_ALERT,
"Unable to create Tx DMA map_seg for buffer %d\n",
i);
- goto err_buf_info_head_unmap;
+ goto err_map_release;
}
tx_ring->tx_buffer_info[i].head_mapped = false;
+
+#ifdef DEV_NETMAP
+ if (adapter->ifp->if_capenable & IFCAP_NETMAP) {
+ map = tx_ring->tx_buffer_info[i].nm_info.map_seg;
+ for (j = 0; j < ENA_PKT_MAX_BUFS; j++) {
+ err = bus_dmamap_create(adapter->tx_buf_tag, 0,
+ &map[j]);
+ if (unlikely(err != 0)) {
+ ena_trace(ENA_ALERT, "Unable to create "
+ "Tx DMA for buffer %d %d\n", i, j);
+ goto err_map_release;
+ }
+ }
+ }
+#endif /* DEV_NETMAP */
}
/* Allocate taskqueues */
@@ -610,7 +670,7 @@ ena_setup_tx_resources(struct ena_adapter *adapter, in
ena_trace(ENA_ALERT,
"Unable to create taskqueue for enqueue task\n");
i = tx_ring->ring_size;
- goto err_buf_info_unmap;
+ goto err_map_release;
}
tx_ring->running = true;
@@ -620,17 +680,8 @@ ena_setup_tx_resources(struct ena_adapter *adapter, in
return (0);
-err_buf_info_head_unmap:
- bus_dmamap_destroy(adapter->tx_buf_tag,
- tx_ring->tx_buffer_info[i].map_head);
-err_buf_info_unmap:
- while (i--) {
- bus_dmamap_destroy(adapter->tx_buf_tag,
- tx_ring->tx_buffer_info[i].map_head);
- bus_dmamap_destroy(adapter->tx_buf_tag,
- tx_ring->tx_buffer_info[i].map_seg);
- }
- free(tx_ring->push_buf_intermediate_buf, M_DEVBUF);
+err_map_release:
+ ena_release_all_tx_dmamap(tx_ring);
err_tx_ids_free:
free(tx_ring->free_tx_ids, M_DEVBUF);
tx_ring->free_tx_ids = NULL;
@@ -652,6 +703,10 @@ static void
ena_free_tx_resources(struct ena_adapter *adapter, int qid)
{
struct ena_ring *tx_ring = &adapter->tx_ring[qid];
+#ifdef DEV_NETMAP
+ struct ena_netmap_tx_info *nm_info;
+ int j;
+#endif /* DEV_NETMAP */
while (taskqueue_cancel(tx_ring->enqueue_tq, &tx_ring->enqueue_task,
NULL))
@@ -686,6 +741,24 @@ ena_free_tx_resources(struct ena_adapter *adapter, int
}
bus_dmamap_destroy(adapter->tx_buf_tag,
tx_ring->tx_buffer_info[i].map_seg);
+
+#ifdef DEV_NETMAP
+ if (adapter->ifp->if_capenable & IFCAP_NETMAP) {
+ nm_info = &tx_ring->tx_buffer_info[i].nm_info;
+ for (j = 0; j < ENA_PKT_MAX_BUFS; j++) {
+ if (nm_info->socket_buf_idx[j] != 0) {
+ bus_dmamap_sync(adapter->tx_buf_tag,
+ nm_info->map_seg[j],
+ BUS_DMASYNC_POSTWRITE);
+ ena_netmap_unload(adapter,
+ nm_info->map_seg[j]);
+ }
+ bus_dmamap_destroy(adapter->tx_buf_tag,
+ nm_info->map_seg[j]);
+ nm_info->socket_buf_idx[j] = 0;
+ }
+ }
+#endif /* DEV_NETMAP */
m_freem(tx_ring->tx_buffer_info[i].mbuf);
tx_ring->tx_buffer_info[i].mbuf = NULL;
Modified: head/sys/dev/ena/ena.h
==============================================================================
--- head/sys/dev/ena/ena.h Thu Oct 31 15:57:44 2019 (r354222)
+++ head/sys/dev/ena/ena.h Thu Oct 31 15:59:29 2019 (r354223)
@@ -226,6 +226,14 @@ struct ena_calc_queue_size_ctx {
uint16_t max_rx_sgl_size;
};
+#ifdef DEV_NETMAP
+struct ena_netmap_tx_info {
+ uint32_t socket_buf_idx[ENA_PKT_MAX_BUFS];
+ bus_dmamap_t map_seg[ENA_PKT_MAX_BUFS];
+ unsigned int sockets_used;
+};
+#endif
+
struct ena_tx_buffer {
struct mbuf *mbuf;
/* # of ena desc for this specific mbuf
@@ -244,6 +252,10 @@ struct ena_tx_buffer {
/* Used to detect missing tx packets */
struct bintime timestamp;
bool print_once;
+
+#ifdef DEV_NETMAP
+ struct ena_netmap_tx_info nm_info;
+#endif /* DEV_NETMAP */
struct ena_com_buf bufs[ENA_PKT_MAX_BUFS];
} __aligned(CACHE_LINE_SIZE);
Modified: head/sys/dev/ena/ena_datapath.c
==============================================================================
--- head/sys/dev/ena/ena_datapath.c Thu Oct 31 15:57:44 2019 (r354222)
+++ head/sys/dev/ena/ena_datapath.c Thu Oct 31 15:59:29 2019 (r354223)
@@ -239,6 +239,11 @@ ena_tx_cleanup(struct ena_ring *tx_ring)
io_cq = &adapter->ena_dev->io_cq_queues[ena_qid];
next_to_clean = tx_ring->next_to_clean;
+#ifdef DEV_NETMAP
+ if (netmap_tx_irq(adapter->ifp, tx_ring->qid) != NM_IRQ_PASS)
+ return (0);
+#endif /* DEV_NETMAP */
+
do {
struct ena_tx_buffer *tx_info;
struct mbuf *mbuf;
Modified: head/sys/dev/ena/ena_netmap.c
==============================================================================
--- head/sys/dev/ena/ena_netmap.c Thu Oct 31 15:57:44 2019 (r354222)
+++ head/sys/dev/ena/ena_netmap.c Thu Oct 31 15:59:29 2019 (r354223)
@@ -58,6 +58,23 @@ static int ena_netmap_txsync(struct netmap_kring *, in
static int ena_netmap_rxsync(struct netmap_kring *, int);
/* Helper functions */
+static int ena_netmap_tx_frames(struct ena_netmap_ctx *);
+static int ena_netmap_tx_frame(struct ena_netmap_ctx *);
+static inline uint16_t ena_netmap_count_slots(struct ena_netmap_ctx *);
+static inline uint16_t ena_netmap_packet_len(struct netmap_slot *, u_int,
+ uint16_t);
+static int ena_netmap_copy_data(struct netmap_adapter *,
+ struct netmap_slot *, u_int, uint16_t, uint16_t, void *);
+static int ena_netmap_map_single_slot(struct netmap_adapter *,
+ struct netmap_slot *, bus_dma_tag_t, bus_dmamap_t, void **, uint64_t *);
+static int ena_netmap_tx_map_slots(struct ena_netmap_ctx *,
+ struct ena_tx_buffer *, void **, uint16_t *, uint16_t *);
+static void ena_netmap_unmap_last_socket_chain(struct ena_netmap_ctx *,
+ struct ena_tx_buffer *);
+static void ena_netmap_tx_cleanup(struct ena_netmap_ctx *);
+static uint16_t ena_netmap_tx_clean_one(struct ena_netmap_ctx *,
+ uint16_t);
+static inline int validate_tx_req_id(struct ena_ring *, uint16_t);
static int ena_netmap_rx_frames(struct ena_netmap_ctx *);
static int ena_netmap_rx_frame(struct ena_netmap_ctx *);
static int ena_netmap_rx_load_desc(struct ena_netmap_ctx *, uint16_t,
@@ -206,6 +223,13 @@ ena_netmap_reset_rx_ring(struct ena_adapter *adapter,
netmap_reset(NA(adapter->ifp), NR_RX, qid, 0);
}
+void
+ena_netmap_reset_tx_ring(struct ena_adapter *adapter, int qid)
+{
+ if (adapter->ifp->if_capenable & IFCAP_NETMAP)
+ netmap_reset(NA(adapter->ifp), NR_TX, qid, 0);
+}
+
static int
ena_netmap_reg(struct netmap_adapter *na, int onoff)
{
@@ -242,11 +266,578 @@ ena_netmap_reg(struct netmap_adapter *na, int onoff)
static int
ena_netmap_txsync(struct netmap_kring *kring, int flags)
{
- ena_trace(ENA_NETMAP, "netmap txsync\n");
+ struct ena_netmap_ctx ctx;
+ int rc = 0;
+
+ ena_netmap_fill_ctx(kring, &ctx, ENA_IO_TXQ_IDX(kring->ring_id));
+ ctx.ring = &ctx.adapter->tx_ring[kring->ring_id];
+
+ ENA_RING_MTX_LOCK(ctx.ring);
+ if (unlikely(!ENA_FLAG_ISSET(ENA_FLAG_DEV_UP, ctx.adapter)))
+ goto txsync_end;
+
+ if (unlikely(!ENA_FLAG_ISSET(ENA_FLAG_LINK_UP, ctx.adapter)))
+ goto txsync_end;
+
+ rc = ena_netmap_tx_frames(&ctx);
+ ena_netmap_tx_cleanup(&ctx);
+
+txsync_end:
+ ENA_RING_MTX_UNLOCK(ctx.ring);
+ return (rc);
+}
+
+static int
+ena_netmap_tx_frames(struct ena_netmap_ctx *ctx)
+{
+ struct ena_ring *tx_ring = ctx->ring;
+ int rc = 0;
+
+ ctx->nm_i = ctx->kring->nr_hwcur;
+ ctx->nt = ctx->ring->next_to_use;
+
+ __builtin_prefetch(&ctx->slots[ctx->nm_i]);
+
+ while (ctx->nm_i != ctx->kring->rhead) {
+ if ((rc = ena_netmap_tx_frame(ctx)) != 0) {
+ /*
+ * When there is no empty space in Tx ring, error is
+ * still being returned. It should not be passed to the
+ * netmap, as application knows current ring state from
+ * netmap ring pointers. Returning error there could
+ * cause application to exit, but the Tx ring is commonly
+ * being full.
+ */
+ if (rc == ENA_COM_NO_MEM)
+ rc = 0;
+ break;
+ }
+ tx_ring->acum_pkts++;
+ }
+
+ /* If any packet was sent... */
+ if (likely(ctx->nm_i != ctx->kring->nr_hwcur)) {
+ wmb();
+ /* ...send the doorbell to the device. */
+ ena_com_write_sq_doorbell(ctx->io_sq);
+ counter_u64_add(ctx->ring->tx_stats.doorbells, 1);
+ tx_ring->acum_pkts = 0;
+
+ ctx->ring->next_to_use = ctx->nt;
+ ctx->kring->nr_hwcur = ctx->nm_i;
+ }
+
+ return (rc);
+}
+
+static int
+ena_netmap_tx_frame(struct ena_netmap_ctx *ctx)
+{
+ struct ena_com_tx_ctx ena_tx_ctx;
+ struct ena_adapter *adapter;
+ struct ena_ring *tx_ring;
+ struct ena_tx_buffer *tx_info;
+ uint16_t req_id;
+ uint16_t header_len;
+ uint16_t packet_len;
+ int nb_hw_desc;
+ int rc;
+ void *push_hdr;
+
+ adapter = ctx->adapter;
+ if (ena_netmap_count_slots(ctx) > adapter->max_tx_sgl_size) {
+ ena_trace(ENA_WARNING, "Too many slots per packet\n");
+ return (EINVAL);
+ }
+
+ tx_ring = ctx->ring;
+
+ req_id = tx_ring->free_tx_ids[ctx->nt];
+ tx_info = &tx_ring->tx_buffer_info[req_id];
+ tx_info->num_of_bufs = 0;
+ tx_info->nm_info.sockets_used = 0;
+
+ rc = ena_netmap_tx_map_slots(ctx, tx_info, &push_hdr, &header_len,
+ &packet_len);
+ if (unlikely(rc != 0)) {
+ device_printf(adapter->pdev, "Failed to map Tx slot\n");
+ return (rc);
+ }
+
+ bzero(&ena_tx_ctx, sizeof(struct ena_com_tx_ctx));
+ ena_tx_ctx.ena_bufs = tx_info->bufs;
+ ena_tx_ctx.push_header = push_hdr;
+ ena_tx_ctx.num_bufs = tx_info->num_of_bufs;
+ ena_tx_ctx.req_id = req_id;
+ ena_tx_ctx.header_len = header_len;
+
+ /* There are no any offloads, as the netmap doesn't support them */
+
+ if (tx_ring->acum_pkts == DB_THRESHOLD ||
+ ena_com_is_doorbell_needed(ctx->io_sq, &ena_tx_ctx)) {
+ wmb();
+ ena_com_write_sq_doorbell(ctx->io_sq);
+ counter_u64_add(tx_ring->tx_stats.doorbells, 1);
+ tx_ring->acum_pkts = 0;
+ }
+
+ rc = ena_com_prepare_tx(ctx->io_sq, &ena_tx_ctx, &nb_hw_desc);
+ if (unlikely(rc != 0)) {
+ if (likely(rc == ENA_COM_NO_MEM)) {
+ ena_trace(ENA_NETMAP | ENA_DBG | ENA_TXPTH,
+ "Tx ring[%d] is out of space\n", tx_ring->que->id);
+ } else {
+ device_printf(adapter->pdev,
+ "Failed to prepare Tx bufs\n");
+ }
+ counter_u64_add(tx_ring->tx_stats.prepare_ctx_err, 1);
+
+ ena_netmap_unmap_last_socket_chain(ctx, tx_info);
+ return (rc);
+ }
+
+ counter_enter();
+ counter_u64_add_protected(tx_ring->tx_stats.cnt, 1);
+ counter_u64_add_protected(tx_ring->tx_stats.bytes, packet_len);
+ counter_u64_add_protected(adapter->hw_stats.tx_packets, 1);
+ counter_u64_add_protected(adapter->hw_stats.tx_bytes, packet_len);
+ counter_exit();
+
+ tx_info->tx_descs = nb_hw_desc;
+
+ ctx->nt = ENA_TX_RING_IDX_NEXT(ctx->nt, ctx->ring->ring_size);
+
+ for (unsigned int i = 0; i < tx_info->num_of_bufs; i++)
+ bus_dmamap_sync(adapter->tx_buf_tag,
+ tx_info->nm_info.map_seg[i], BUS_DMASYNC_PREWRITE);
+
return (0);
}
+static inline uint16_t
+ena_netmap_count_slots(struct ena_netmap_ctx *ctx)
+{
+ uint16_t slots = 1;
+ uint16_t nm = ctx->nm_i;
+
+ while ((ctx->slots[nm].flags & NS_MOREFRAG) != 0) {
+ slots++;
+ nm = nm_next(nm, ctx->lim);
+ }
+
+ return slots;
+}
+
+static inline uint16_t
+ena_netmap_packet_len(struct netmap_slot *slots, u_int slot_index,
+ uint16_t limit)
+{
+ struct netmap_slot *nm_slot;
+ uint16_t packet_size = 0;
+
+ do {
+ nm_slot = &slots[slot_index];
+ packet_size += nm_slot->len;
+ slot_index = nm_next(slot_index, limit);
+ } while ((nm_slot->flags & NS_MOREFRAG) != 0);
+
+ return packet_size;
+}
+
static int
+ena_netmap_copy_data(struct netmap_adapter *na, struct netmap_slot *slots,
+ u_int slot_index, uint16_t limit, uint16_t bytes_to_copy, void *destination)
+{
+ struct netmap_slot *nm_slot;
+ void *slot_vaddr;
+ uint16_t packet_size;
+ uint16_t data_amount;
+
+ packet_size = 0;
+ do {
+ nm_slot = &slots[slot_index];
+ slot_vaddr = NMB(na, nm_slot);
+ if (unlikely(slot_vaddr == NULL))
+ return (EINVAL);
+
+ data_amount = min_t(uint16_t, bytes_to_copy, nm_slot->len);
+ memcpy(destination, slot_vaddr, data_amount);
+ bytes_to_copy -= data_amount;
+
+ slot_index = nm_next(slot_index, limit);
+ } while ((nm_slot->flags & NS_MOREFRAG) != 0 && bytes_to_copy > 0);
+
+ return (0);
+}
+
+static int
+ena_netmap_map_single_slot(struct netmap_adapter *na, struct netmap_slot *slot,
+ bus_dma_tag_t dmatag, bus_dmamap_t dmamap, void **vaddr, uint64_t *paddr)
+{
+ int rc;
+
+ *vaddr = PNMB(na, slot, paddr);
+ if (unlikely(vaddr == NULL)) {
+ ena_trace(ENA_ALERT, "Slot address is NULL\n");
+ return (EINVAL);
+ }
+
+ rc = netmap_load_map(na, dmatag, dmamap, *vaddr);
+ if (unlikely(rc != 0)) {
+ ena_trace(ENA_ALERT, "Failed to map slot %d for DMA\n",
+ slot->buf_idx);
+ return (EINVAL);
+ }
+
+ return (0);
+}
+
+static int
+ena_netmap_tx_map_slots(struct ena_netmap_ctx *ctx,
+ struct ena_tx_buffer *tx_info, void **push_hdr, uint16_t *header_len,
+ uint16_t *packet_len)
+{
+ struct netmap_slot *slot;
+ struct ena_com_buf *ena_buf;
+ struct ena_adapter *adapter;
+ struct ena_ring *tx_ring;
+ struct ena_netmap_tx_info *nm_info;
+ bus_dmamap_t *nm_maps;
+ void *vaddr;
+ uint64_t paddr;
+ uint32_t *nm_buf_idx;
+ uint32_t slot_head_len;
+ uint32_t frag_len;
+ uint32_t remaining_len;
+ uint16_t push_len;
+ uint16_t delta;
+ int rc;
+
+ adapter = ctx->adapter;
+ tx_ring = ctx->ring;
+ ena_buf = tx_info->bufs;
+ nm_info = &tx_info->nm_info;
+ nm_maps = nm_info->map_seg;
+ nm_buf_idx = nm_info->socket_buf_idx;
+ slot = &ctx->slots[ctx->nm_i];
+
+ slot_head_len = slot->len;
+ *packet_len = ena_netmap_packet_len(ctx->slots, ctx->nm_i, ctx->lim);
+ remaining_len = *packet_len;
+ delta = 0;
+
+ __builtin_prefetch(&ctx->slots[ctx->nm_i + 1]);
+ if (tx_ring->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) {
+ /*
+ * When the device is in LLQ mode, the driver will copy
+ * the header into the device memory space.
+ * The ena_com layer assumes that the header is in a linear
+ * memory space.
+ * This assumption might be wrong since part of the header
+ * can be in the fragmented buffers.
+ * First, check if header fits in the first slot. If not, copy
+ * it to separate buffer that will be holding linearized data.
+ */
+ push_len = min_t(uint32_t, *packet_len,
+ tx_ring->tx_max_header_size);
+ *header_len = push_len;
+ /* If header is in linear space, just point to socket's data. */
+ if (likely(push_len <= slot_head_len)) {
+ *push_hdr = NMB(ctx->na, slot);
+ if (unlikely(push_hdr == NULL)) {
+ device_printf(adapter->pdev,
+ "Slot vaddress is NULL\n");
+ return (EINVAL);
+ }
+ /*
+ * Otherwise, copy whole portion of header from multiple slots
+ * to intermediate buffer.
+ */
+ } else {
+ rc = ena_netmap_copy_data(ctx->na,
+ ctx->slots,
+ ctx->nm_i,
+ ctx->lim,
+ push_len,
+ tx_ring->push_buf_intermediate_buf);
+ if (unlikely(rc)) {
+ device_printf(adapter->pdev,
+ "Failed to copy data from slots to push_buf\n");
+ return (EINVAL);
+ }
+
+ *push_hdr = tx_ring->push_buf_intermediate_buf;
+ counter_u64_add(tx_ring->tx_stats.llq_buffer_copy, 1);
+
+ delta = push_len - slot_head_len;
+ }
+
+ ena_trace(ENA_NETMAP | ENA_DBG | ENA_TXPTH,
+ "slot: %d header_buf->vaddr: %p push_len: %d\n",
+ slot->buf_idx, *push_hdr, push_len);
+
+ /*
+ * If header was in linear memory space, map for the dma rest of the data
+ * in the first mbuf of the mbuf chain.
+ */
+ if (slot_head_len > push_len) {
+ rc = ena_netmap_map_single_slot(ctx->na,
+ slot,
+ adapter->tx_buf_tag,
+ *nm_maps,
+ &vaddr,
+ &paddr);
+ if (unlikely(rc != 0)) {
+ device_printf(adapter->pdev,
+ "DMA mapping error\n");
+ return (rc);
+ }
+ nm_maps++;
+
+ ena_buf->paddr = paddr + push_len;
+ ena_buf->len = slot->len - push_len;
+ ena_buf++;
+
+ tx_info->num_of_bufs++;
+ }
+
+ remaining_len -= slot->len;
+
+ /* Save buf idx before advancing */
+ *nm_buf_idx = slot->buf_idx;
+ nm_buf_idx++;
+ slot->buf_idx = 0;
+
+ /* Advance to the next socket */
+ ctx->nm_i = nm_next(ctx->nm_i, ctx->lim);
+ slot = &ctx->slots[ctx->nm_i];
+ nm_info->sockets_used++;
+
+ /*
+ * If header is in non linear space (delta > 0), then skip mbufs
+ * containing header and map the last one containing both header
+ * and the packet data.
+ * The first segment is already counted in.
+ */
+ while (delta > 0) {
+ __builtin_prefetch(&ctx->slots[ctx->nm_i + 1]);
+ frag_len = slot->len;
+
+ /*
+ * If whole segment contains header just move to the
+ * next one and reduce delta.
+ */
+ if (unlikely(delta >= frag_len)) {
+ delta -= frag_len;
+ } else {
+ /*
+ * Map the data and then assign it with the
+ * offsets
+ */
+ rc = ena_netmap_map_single_slot(ctx->na,
+ slot,
+ adapter->tx_buf_tag,
+ *nm_maps,
+ &vaddr,
+ &paddr);
+ if (unlikely(rc != 0)) {
+ device_printf(adapter->pdev,
+ "DMA mapping error\n");
+ goto error_map;
+ }
+ nm_maps++;
+
+ ena_buf->paddr = paddr + delta;
+ ena_buf->len = slot->len - delta;
+ ena_buf++;
+
+ tx_info->num_of_bufs++;
+ delta = 0;
+ }
+
+ remaining_len -= slot->len;
+
+ /* Save buf idx before advancing */
+ *nm_buf_idx = slot->buf_idx;
+ nm_buf_idx++;
+ slot->buf_idx = 0;
+
+ /* Advance to the next socket */
+ ctx->nm_i = nm_next(ctx->nm_i, ctx->lim);
+ slot = &ctx->slots[ctx->nm_i];
+ nm_info->sockets_used++;
+ }
+ } else {
+ *push_hdr = NULL;
+ /*
+ * header_len is just a hint for the device. Because netmap is
+ * not giving us any information about packet header length and
+ * it is not guaranteed that all packet headers will be in the
+ * 1st slot, setting header_len to 0 is making the device ignore
+ * this value and resolve header on it's own.
+ */
+ *header_len = 0;
+ }
+
+ /* Map all remaining data (regular routine for non-LLQ mode) */
+ while (remaining_len > 0) {
+ __builtin_prefetch(&ctx->slots[ctx->nm_i + 1]);
+
+ rc = ena_netmap_map_single_slot(ctx->na,
+ slot,
+ adapter->tx_buf_tag,
+ *nm_maps,
+ &vaddr,
+ &paddr);
+ if (unlikely(rc != 0)) {
+ device_printf(adapter->pdev,
+ "DMA mapping error\n");
+ goto error_map;
+ }
+ nm_maps++;
+
+ ena_buf->paddr = paddr;
+ ena_buf->len = slot->len;
+ ena_buf++;
+
+ tx_info->num_of_bufs++;
+
+ remaining_len -= slot->len;
+
+ /* Save buf idx before advancing */
+ *nm_buf_idx = slot->buf_idx;
+ nm_buf_idx++;
+ slot->buf_idx = 0;
+
+ /* Advance to the next socket */
+ ctx->nm_i = nm_next(ctx->nm_i, ctx->lim);
+ slot = &ctx->slots[ctx->nm_i];
+ nm_info->sockets_used++;
+ }
+
+ return (0);
+
+error_map:
+ ena_netmap_unmap_last_socket_chain(ctx, tx_info);
+
+ return (rc);
+}
+
+static void
+ena_netmap_unmap_last_socket_chain(struct ena_netmap_ctx *ctx,
+ struct ena_tx_buffer *tx_info)
+{
+ struct ena_netmap_tx_info *nm_info;
+ int n;
+
+ nm_info = &tx_info->nm_info;
+
+ /**
+ * As the used sockets must not be equal to the buffers used in the LLQ
+ * mode, they must be treated separately.
+ * First, unmap the DMA maps.
+ */
+ n = tx_info->num_of_bufs;
+ while (n--) {
+ netmap_unload_map(ctx->na, ctx->adapter->tx_buf_tag,
+ nm_info->map_seg[n]);
+ }
+ tx_info->num_of_bufs = 0;
+
+ /* Next, retain the sockets back to the userspace */
+ n = nm_info->sockets_used;
+ while (n--) {
+ ctx->slots[ctx->nm_i].buf_idx = nm_info->socket_buf_idx[n];
+ ctx->slots[ctx->nm_i].flags = NS_BUF_CHANGED;
+ nm_info->socket_buf_idx[n] = 0;
+ ctx->nm_i = nm_prev(ctx->nm_i, ctx->lim);
+ }
+ nm_info->sockets_used = 0;
+}
+
+static void
+ena_netmap_tx_cleanup(struct ena_netmap_ctx *ctx)
+{
+ uint16_t req_id;
+ uint16_t total_tx_descs = 0;
+
+ ctx->nm_i = ctx->kring->nr_hwtail;
+ ctx->nt = ctx->ring->next_to_clean;
+
+ /* Reclaim buffers for completed transmissions */
+ while (ena_com_tx_comp_req_id_get(ctx->io_cq, &req_id) >= 0) {
+ if (validate_tx_req_id(ctx->ring, req_id) != 0)
+ break;
+ total_tx_descs += ena_netmap_tx_clean_one(ctx, req_id);
+ }
+
+ ctx->kring->nr_hwtail = ctx->nm_i;
+
+ if (total_tx_descs > 0) {
+ /* acknowledge completion of sent packets */
+ ctx->ring->next_to_clean = ctx->nt;
+ ena_com_comp_ack(ctx->ring->ena_com_io_sq, total_tx_descs);
+ ena_com_update_dev_comp_head(ctx->ring->ena_com_io_cq);
+ }
+}
+
+static uint16_t
+ena_netmap_tx_clean_one(struct ena_netmap_ctx *ctx, uint16_t req_id)
+{
+ struct ena_tx_buffer *tx_info;
+ struct ena_netmap_tx_info *nm_info;
+ int n;
+
+ tx_info = &ctx->ring->tx_buffer_info[req_id];
+ nm_info = &tx_info->nm_info;
+
+ /**
+ * As the used sockets must not be equal to the buffers used in the LLQ
+ * mode, they must be treated separately.
+ * First, unmap the DMA maps.
+ */
+ n = tx_info->num_of_bufs;
+ for (n = 0; n < tx_info->num_of_bufs; n++) {
+ netmap_unload_map(ctx->na, ctx->adapter->tx_buf_tag,
+ nm_info->map_seg[n]);
+ }
+ tx_info->num_of_bufs = 0;
+
+ /* Next, retain the sockets back to the userspace */
+ for (n = 0; n < nm_info->sockets_used; n++) {
+ ctx->nm_i = nm_next(ctx->nm_i, ctx->lim);
+ ENA_ASSERT(ctx->slots[ctx->nm_i].buf_idx == 0,
+ "Tx idx is not 0.\n");
+ ctx->slots[ctx->nm_i].buf_idx = nm_info->socket_buf_idx[n];
+ ctx->slots[ctx->nm_i].flags = NS_BUF_CHANGED;
+ nm_info->socket_buf_idx[n] = 0;
+ }
+ nm_info->sockets_used = 0;
+
+ ctx->ring->free_tx_ids[ctx->nt] = req_id;
+ ctx->nt = ENA_TX_RING_IDX_NEXT(ctx->nt, ctx->lim);
+
+ return tx_info->tx_descs;
+}
+
+static inline int
+validate_tx_req_id(struct ena_ring *tx_ring, uint16_t req_id)
+{
+ struct ena_adapter *adapter = tx_ring->adapter;
+
+ if (likely(req_id < tx_ring->ring_size))
+ return (0);
+
+ ena_trace(ENA_WARNING, "Invalid req_id: %hu\n", req_id);
+ counter_u64_add(tx_ring->tx_stats.bad_req_id, 1);
+
+ adapter->reset_reason = ENA_REGS_RESET_INV_TX_REQ_ID;
+ ENA_FLAG_SET_ATOMIC(ENA_FLAG_TRIGGER_RESET, adapter);
+
+ return (EFAULT);
+}
+
+static int
ena_netmap_rxsync(struct netmap_kring *kring, int flags)
{
struct ena_netmap_ctx ctx;
@@ -435,6 +1026,14 @@ ena_netmap_fill_ctx(struct netmap_kring *kring, struct
ctx->io_cq = &ctx->adapter->ena_dev->io_cq_queues[ena_qid];
ctx->io_sq = &ctx->adapter->ena_dev->io_sq_queues[ena_qid];
ctx->slots = kring->ring->slot;
+}
+
+void
+ena_netmap_unload(struct ena_adapter *adapter, bus_dmamap_t map)
+{
+ struct netmap_adapter *na = NA(adapter->ifp);
+
+ netmap_unload_map(na, adapter->tx_buf_tag, map);
}
#endif /* DEV_NETMAP */
Modified: head/sys/dev/ena/ena_netmap.h
==============================================================================
--- head/sys/dev/ena/ena_netmap.h Thu Oct 31 15:57:44 2019 (r354222)
+++ head/sys/dev/ena/ena_netmap.h Thu Oct 31 15:59:29 2019 (r354223)
@@ -52,5 +52,7 @@ int ena_netmap_alloc_rx_slot(struct ena_adapter *, str
void ena_netmap_free_rx_slot(struct ena_adapter *, struct ena_ring *,
struct ena_rx_buffer *);
void ena_netmap_reset_rx_ring(struct ena_adapter *, int);
+void ena_netmap_reset_tx_ring(struct ena_adapter *, int);
+void ena_netmap_unload(struct ena_adapter *, bus_dmamap_t);
#endif /* _ENA_NETMAP_H_ */
More information about the svn-src-head
mailing list