svn commit: r361468 - in stable/11/sys: contrib/ena-com dev/ena modules/ena
Marcin Wojtas
mw at FreeBSD.org
Mon May 25 17:47:32 UTC 2020
Author: mw
Date: Mon May 25 17:47:31 2020
New Revision: 361468
URL: https://svnweb.freebsd.org/changeset/base/361468
Log:
MFC: ENA netmap support and bug fixes
r360777 Optimize ENA Rx refill for low memory conditions
r358289 Rework and simplify Tx DMA mapping in ENA
r354242 Make valdiate_rx_req_id static inline because it uses other static
r354225 Update ENA version to v2.1.0
r354224 Add support for ENA NETMAP partial initialization
r354223 Add support for ENA NETMAP Tx
r354222 Add support for ENA NETMAP Rx
r354221 Introduce NETMAP support in ENA
r354220 Split Rx/Tx from initialization code in ENA driver
r354219 Fix ENA keep-alive timeout due to prolonged reset
Sponsored by: Amazon, Inc.
Added:
stable/11/sys/dev/ena/ena_datapath.c
- copied, changed from r354225, head/sys/dev/ena/ena_datapath.c
stable/11/sys/dev/ena/ena_datapath.h
- copied unchanged from r354225, head/sys/dev/ena/ena_datapath.h
stable/11/sys/dev/ena/ena_netmap.c
- copied unchanged from r354225, head/sys/dev/ena/ena_netmap.c
stable/11/sys/dev/ena/ena_netmap.h
- copied unchanged from r354225, head/sys/dev/ena/ena_netmap.h
Modified:
stable/11/sys/contrib/ena-com/ena_plat.h
stable/11/sys/dev/ena/ena.c
stable/11/sys/dev/ena/ena.h
stable/11/sys/dev/ena/ena_sysctl.c
stable/11/sys/dev/ena/ena_sysctl.h
stable/11/sys/modules/ena/Makefile
Directory Properties:
stable/11/ (props changed)
Modified: stable/11/sys/contrib/ena-com/ena_plat.h
==============================================================================
--- stable/11/sys/contrib/ena-com/ena_plat.h Mon May 25 17:41:20 2020 (r361467)
+++ stable/11/sys/contrib/ena-com/ena_plat.h Mon May 25 17:47:31 2020 (r361468)
@@ -103,6 +103,7 @@ extern struct ena_bus_space ebs;
#define ENA_RSC (1 << 6) /* Goes with TXPTH or RXPTH, free/alloc res. */
#define ENA_IOQ (1 << 7) /* Detailed info about IO queues. */
#define ENA_ADMQ (1 << 8) /* Detailed info about admin queue. */
+#define ENA_NETMAP (1 << 9) /* Detailed info about netmap. */
extern int ena_log_level;
Modified: stable/11/sys/dev/ena/ena.c
==============================================================================
--- stable/11/sys/dev/ena/ena.c Mon May 25 17:41:20 2020 (r361467)
+++ stable/11/sys/dev/ena/ena.c Mon May 25 17:47:31 2020 (r361468)
@@ -76,9 +76,14 @@ __FBSDID("$FreeBSD$");
#include <vm/vm.h>
#include <vm/pmap.h>
+#include "ena_datapath.h"
#include "ena.h"
#include "ena_sysctl.h"
+#ifdef DEV_NETMAP
+#include "ena_netmap.h"
+#endif /* DEV_NETMAP */
+
/*********************************************************
* Function prototypes
*********************************************************/
@@ -98,11 +103,11 @@ static int ena_setup_tx_dma_tag(struct ena_adapter *);
static int ena_free_tx_dma_tag(struct ena_adapter *);
static int ena_setup_rx_dma_tag(struct ena_adapter *);
static int ena_free_rx_dma_tag(struct ena_adapter *);
+static void ena_release_all_tx_dmamap(struct ena_ring *);
static int ena_setup_tx_resources(struct ena_adapter *, int);
static void ena_free_tx_resources(struct ena_adapter *, int);
static int ena_setup_all_tx_resources(struct ena_adapter *);
static void ena_free_all_tx_resources(struct ena_adapter *);
-static inline int validate_rx_req_id(struct ena_ring *, uint16_t);
static int ena_setup_rx_resources(struct ena_adapter *, unsigned int);
static void ena_free_rx_resources(struct ena_adapter *, unsigned int);
static int ena_setup_all_rx_resources(struct ena_adapter *);
@@ -111,7 +116,6 @@ static inline int ena_alloc_rx_mbuf(struct ena_adapter
struct ena_rx_buffer *);
static void ena_free_rx_mbuf(struct ena_adapter *, struct ena_ring *,
struct ena_rx_buffer *);
-static int ena_refill_rx_bufs(struct ena_ring *, uint32_t);
static void ena_free_rx_bufs(struct ena_adapter *, unsigned int);
static void ena_refill_all_rx_bufs(struct ena_adapter *);
static void ena_free_all_rx_bufs(struct ena_adapter *);
@@ -121,16 +125,6 @@ static void ena_destroy_all_tx_queues(struct ena_adapt
static void ena_destroy_all_rx_queues(struct ena_adapter *);
static void ena_destroy_all_io_queues(struct ena_adapter *);
static int ena_create_io_queues(struct ena_adapter *);
-static int ena_tx_cleanup(struct ena_ring *);
-static int ena_rx_cleanup(struct ena_ring *);
-static inline int validate_tx_req_id(struct ena_ring *, uint16_t);
-static void ena_rx_hash_mbuf(struct ena_ring *, struct ena_com_rx_ctx *,
- struct mbuf *);
-static struct mbuf* ena_rx_mbuf(struct ena_ring *, struct ena_com_rx_buf_info *,
- struct ena_com_rx_ctx *, uint16_t *);
-static inline void ena_rx_checksum(struct ena_ring *, struct ena_com_rx_ctx *,
- struct mbuf *);
-static void ena_cleanup(void *arg, int pending);
static int ena_handle_msix(void *);
static int ena_enable_msix(struct ena_adapter *);
static void ena_setup_mgmnt_intr(struct ena_adapter *);
@@ -144,8 +138,6 @@ static void ena_disable_msix(struct ena_adapter *);
static void ena_unmask_all_io_irqs(struct ena_adapter *);
static int ena_rss_configure(struct ena_adapter *);
static int ena_up_complete(struct ena_adapter *);
-static int ena_up(struct ena_adapter *);
-static void ena_down(struct ena_adapter *);
static uint64_t ena_get_counter(if_t, ift_counter);
static int ena_media_change(if_t);
static void ena_media_status(if_t, struct ifmediareq *);
@@ -156,15 +148,6 @@ static void ena_update_host_info(struct ena_admin_host
static void ena_update_hwassist(struct ena_adapter *);
static int ena_setup_ifnet(device_t, struct ena_adapter *,
struct ena_com_dev_get_features_ctx *);
-static void ena_tx_csum(struct ena_com_tx_ctx *, struct mbuf *);
-static int ena_check_and_collapse_mbuf(struct ena_ring *tx_ring,
- struct mbuf **mbuf);
-static void ena_dmamap_llq(void *, bus_dma_segment_t *, int, int);
-static int ena_xmit_mbuf(struct ena_ring *, struct mbuf **);
-static void ena_start_xmit(struct ena_ring *);
-static int ena_mq_start(if_t, struct mbuf *);
-static void ena_deferred_mq_start(void *, int);
-static void ena_qflush(if_t);
static int ena_enable_wc(struct resource *);
static int ena_set_queues_placement_policy(device_t, struct ena_com_dev *,
struct ena_admin_feature_llq_desc *, struct ena_llq_configurations *);
@@ -385,6 +368,7 @@ ena_init_io_rings_common(struct ena_adapter *adapter,
ring->ena_dev = adapter->ena_dev;
ring->first_interrupt = false;
ring->no_interrupt_event_cnt = 0;
+ ring->rx_mbuf_sz = ena_mbuf_sz;
}
static void
@@ -525,9 +509,9 @@ ena_setup_rx_dma_tag(struct ena_adapter *adapter)
ENA_DMA_BIT_MASK(adapter->dma_width), /* lowaddr of excl window */
BUS_SPACE_MAXADDR, /* highaddr of excl window */
NULL, NULL, /* filter, filterarg */
- MJUM16BYTES, /* maxsize */
+ ena_mbuf_sz, /* maxsize */
adapter->max_rx_sgl_size, /* nsegments */
- MJUM16BYTES, /* maxsegsize */
+ ena_mbuf_sz, /* maxsegsize */
0, /* flags */
NULL, /* lockfunc */
NULL, /* lockarg */
@@ -549,6 +533,39 @@ ena_free_rx_dma_tag(struct ena_adapter *adapter)
return (ret);
}
+static void
+ena_release_all_tx_dmamap(struct ena_ring *tx_ring)
+{
+ struct ena_adapter *adapter = tx_ring->adapter;
+ struct ena_tx_buffer *tx_info;
+ bus_dma_tag_t tx_tag = adapter->tx_buf_tag;;
+ int i;
+#ifdef DEV_NETMAP
+ struct ena_netmap_tx_info *nm_info;
+ int j;
+#endif /* DEV_NETMAP */
+
+ for (i = 0; i < tx_ring->ring_size; ++i) {
+ tx_info = &tx_ring->tx_buffer_info[i];
+#ifdef DEV_NETMAP
+ if (adapter->ifp->if_capenable & IFCAP_NETMAP) {
+ nm_info = &tx_info->nm_info;
+ for (j = 0; j < ENA_PKT_MAX_BUFS; ++j) {
+ if (nm_info->map_seg[j] != NULL) {
+ bus_dmamap_destroy(tx_tag,
+ nm_info->map_seg[j]);
+ nm_info->map_seg[j] = NULL;
+ }
+ }
+ }
+#endif /* DEV_NETMAP */
+ if (tx_info->dmamap != NULL) {
+ bus_dmamap_destroy(tx_tag, tx_info->dmamap);
+ tx_info->dmamap = NULL;
+ }
+ }
+}
+
/**
* ena_setup_tx_resources - allocate Tx resources (Descriptors)
* @adapter: network interface device structure
@@ -562,7 +579,13 @@ ena_setup_tx_resources(struct ena_adapter *adapter, in
struct ena_que *que = &adapter->que[qid];
struct ena_ring *tx_ring = que->tx_ring;
int size, i, err;
+#ifdef DEV_NETMAP
+ bus_dmamap_t *map;
+ int j;
+ ena_netmap_reset_tx_ring(adapter, qid);
+#endif /* DEV_NETMAP */
+
size = sizeof(struct ena_tx_buffer) * tx_ring->ring_size;
tx_ring->tx_buffer_info = malloc(size, M_DEVBUF, M_NOWAIT | M_ZERO);
@@ -600,24 +623,28 @@ ena_setup_tx_resources(struct ena_adapter *adapter, in
/* ... and create the buffer DMA maps */
for (i = 0; i < tx_ring->ring_size; i++) {
err = bus_dmamap_create(adapter->tx_buf_tag, 0,
- &tx_ring->tx_buffer_info[i].map_head);
+ &tx_ring->tx_buffer_info[i].dmamap);
if (unlikely(err != 0)) {
ena_trace(ENA_ALERT,
- "Unable to create Tx DMA map_head for buffer %d\n",
+ "Unable to create Tx DMA map for buffer %d\n",
i);
- goto err_buf_info_unmap;
+ goto err_map_release;
}
- tx_ring->tx_buffer_info[i].seg_mapped = false;
- err = bus_dmamap_create(adapter->tx_buf_tag, 0,
- &tx_ring->tx_buffer_info[i].map_seg);
- if (unlikely(err != 0)) {
- ena_trace(ENA_ALERT,
- "Unable to create Tx DMA map_seg for buffer %d\n",
- i);
- goto err_buf_info_head_unmap;
+#ifdef DEV_NETMAP
+ if (adapter->ifp->if_capenable & IFCAP_NETMAP) {
+ map = tx_ring->tx_buffer_info[i].nm_info.map_seg;
+ for (j = 0; j < ENA_PKT_MAX_BUFS; j++) {
+ err = bus_dmamap_create(adapter->tx_buf_tag, 0,
+ &map[j]);
+ if (unlikely(err != 0)) {
+ ena_trace(ENA_ALERT, "Unable to create "
+ "Tx DMA for buffer %d %d\n", i, j);
+ goto err_map_release;
+ }
+ }
}
- tx_ring->tx_buffer_info[i].head_mapped = false;
+#endif /* DEV_NETMAP */
}
/* Allocate taskqueues */
@@ -628,7 +655,7 @@ ena_setup_tx_resources(struct ena_adapter *adapter, in
ena_trace(ENA_ALERT,
"Unable to create taskqueue for enqueue task\n");
i = tx_ring->ring_size;
- goto err_buf_info_unmap;
+ goto err_map_release;
}
tx_ring->running = true;
@@ -638,17 +665,8 @@ ena_setup_tx_resources(struct ena_adapter *adapter, in
return (0);
-err_buf_info_head_unmap:
- bus_dmamap_destroy(adapter->tx_buf_tag,
- tx_ring->tx_buffer_info[i].map_head);
-err_buf_info_unmap:
- while (i--) {
- bus_dmamap_destroy(adapter->tx_buf_tag,
- tx_ring->tx_buffer_info[i].map_head);
- bus_dmamap_destroy(adapter->tx_buf_tag,
- tx_ring->tx_buffer_info[i].map_seg);
- }
- free(tx_ring->push_buf_intermediate_buf, M_DEVBUF);
+err_map_release:
+ ena_release_all_tx_dmamap(tx_ring);
err_tx_ids_free:
free(tx_ring->free_tx_ids, M_DEVBUF);
tx_ring->free_tx_ids = NULL;
@@ -670,6 +688,10 @@ static void
ena_free_tx_resources(struct ena_adapter *adapter, int qid)
{
struct ena_ring *tx_ring = &adapter->tx_ring[qid];
+#ifdef DEV_NETMAP
+ struct ena_netmap_tx_info *nm_info;
+ int j;
+#endif /* DEV_NETMAP */
while (taskqueue_cancel(tx_ring->enqueue_tq, &tx_ring->enqueue_task,
NULL))
@@ -683,27 +705,30 @@ ena_free_tx_resources(struct ena_adapter *adapter, int
/* Free buffer DMA maps, */
for (int i = 0; i < tx_ring->ring_size; i++) {
- if (tx_ring->tx_buffer_info[i].head_mapped == true) {
- bus_dmamap_sync(adapter->tx_buf_tag,
- tx_ring->tx_buffer_info[i].map_head,
- BUS_DMASYNC_POSTWRITE);
- bus_dmamap_unload(adapter->tx_buf_tag,
- tx_ring->tx_buffer_info[i].map_head);
- tx_ring->tx_buffer_info[i].head_mapped = false;
- }
+ bus_dmamap_sync(adapter->tx_buf_tag,
+ tx_ring->tx_buffer_info[i].dmamap, BUS_DMASYNC_POSTWRITE);
+ bus_dmamap_unload(adapter->tx_buf_tag,
+ tx_ring->tx_buffer_info[i].dmamap);
bus_dmamap_destroy(adapter->tx_buf_tag,
- tx_ring->tx_buffer_info[i].map_head);
+ tx_ring->tx_buffer_info[i].dmamap);
- if (tx_ring->tx_buffer_info[i].seg_mapped == true) {
- bus_dmamap_sync(adapter->tx_buf_tag,
- tx_ring->tx_buffer_info[i].map_seg,
- BUS_DMASYNC_POSTWRITE);
- bus_dmamap_unload(adapter->tx_buf_tag,
- tx_ring->tx_buffer_info[i].map_seg);
- tx_ring->tx_buffer_info[i].seg_mapped = false;
+#ifdef DEV_NETMAP
+ if (adapter->ifp->if_capenable & IFCAP_NETMAP) {
+ nm_info = &tx_ring->tx_buffer_info[i].nm_info;
+ for (j = 0; j < ENA_PKT_MAX_BUFS; j++) {
+ if (nm_info->socket_buf_idx[j] != 0) {
+ bus_dmamap_sync(adapter->tx_buf_tag,
+ nm_info->map_seg[j],
+ BUS_DMASYNC_POSTWRITE);
+ ena_netmap_unload(adapter,
+ nm_info->map_seg[j]);
+ }
+ bus_dmamap_destroy(adapter->tx_buf_tag,
+ nm_info->map_seg[j]);
+ nm_info->socket_buf_idx[j] = 0;
+ }
}
- bus_dmamap_destroy(adapter->tx_buf_tag,
- tx_ring->tx_buffer_info[i].map_seg);
+#endif /* DEV_NETMAP */
m_freem(tx_ring->tx_buffer_info[i].mbuf);
tx_ring->tx_buffer_info[i].mbuf = NULL;
@@ -766,25 +791,6 @@ ena_free_all_tx_resources(struct ena_adapter *adapter)
ena_free_tx_resources(adapter, i);
}
-static inline int
-validate_rx_req_id(struct ena_ring *rx_ring, uint16_t req_id)
-{
- if (likely(req_id < rx_ring->ring_size))
- return (0);
-
- device_printf(rx_ring->adapter->pdev, "Invalid rx req_id: %hu\n",
- req_id);
- counter_u64_add(rx_ring->rx_stats.bad_req_id, 1);
-
- /* Trigger device reset */
- if (likely(!ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, rx_ring->adapter))) {
- rx_ring->adapter->reset_reason = ENA_REGS_RESET_INV_RX_REQ_ID;
- ENA_FLAG_SET_ATOMIC(ENA_FLAG_TRIGGER_RESET, rx_ring->adapter);
- }
-
- return (EFAULT);
-}
-
/**
* ena_setup_rx_resources - allocate Rx resources (Descriptors)
* @adapter: network interface device structure
@@ -801,6 +807,11 @@ ena_setup_rx_resources(struct ena_adapter *adapter, un
size = sizeof(struct ena_rx_buffer) * rx_ring->ring_size;
+#ifdef DEV_NETMAP
+ ena_netmap_reset_rx_ring(adapter, qid);
+ rx_ring->initialized = false;
+#endif /* DEV_NETMAP */
+
/*
* Alloc extra element so in rx path
* we can always prefetch rx_info + 1
@@ -953,7 +964,8 @@ ena_alloc_rx_mbuf(struct ena_adapter *adapter,
return (0);
/* Get mbuf using UMA allocator */
- rx_info->mbuf = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, MJUM16BYTES);
+ rx_info->mbuf = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR,
+ rx_ring->rx_mbuf_sz);
if (unlikely(rx_info->mbuf == NULL)) {
counter_u64_add(rx_ring->rx_stats.mjum_alloc_fail, 1);
@@ -964,7 +976,7 @@ ena_alloc_rx_mbuf(struct ena_adapter *adapter,
}
mlen = MCLBYTES;
} else {
- mlen = MJUM16BYTES;
+ mlen = rx_ring->rx_mbuf_sz;
}
/* Set mbuf length*/
rx_info->mbuf->m_pkthdr.len = rx_info->mbuf->m_len = mlen;
@@ -1024,7 +1036,7 @@ ena_free_rx_mbuf(struct ena_adapter *adapter, struct e
* @num: number of descriptors to refill
* Refills the ring with newly allocated DMA-mapped mbufs for receiving
**/
-static int
+int
ena_refill_rx_bufs(struct ena_ring *rx_ring, uint32_t num)
{
struct ena_adapter *adapter = rx_ring->adapter;
@@ -1045,8 +1057,12 @@ ena_refill_rx_bufs(struct ena_ring *rx_ring, uint32_t
req_id = rx_ring->free_rx_ids[next_to_use];
rx_info = &rx_ring->rx_buffer_info[req_id];
-
- rc = ena_alloc_rx_mbuf(adapter, rx_ring, rx_info);
+#ifdef DEV_NETMAP
+ if (ena_rx_ring_in_netmap(adapter, rx_ring->qid))
+ rc = ena_netmap_alloc_rx_slot(adapter, rx_ring, rx_info);
+ else
+#endif /* DEV_NETMAP */
+ rc = ena_alloc_rx_mbuf(adapter, rx_ring, rx_info);
if (unlikely(rc != 0)) {
ena_trace(ENA_WARNING,
"failed to alloc buffer for rx queue %d\n",
@@ -1091,6 +1107,14 @@ ena_free_rx_bufs(struct ena_adapter *adapter, unsigned
if (rx_info->mbuf != NULL)
ena_free_rx_mbuf(adapter, rx_ring, rx_info);
+#ifdef DEV_NETMAP
+ if (((if_getflags(adapter->ifp) & IFF_DYING) == 0) &&
+ (adapter->ifp->if_capenable & IFCAP_NETMAP)) {
+ if (rx_info->netmap_buf_idx != 0)
+ ena_netmap_free_rx_slot(adapter, rx_ring,
+ rx_info);
+ }
+#endif /* DEV_NETMAP */
}
}
@@ -1109,10 +1133,12 @@ ena_refill_all_rx_bufs(struct ena_adapter *adapter)
rx_ring = &adapter->rx_ring[i];
bufs_num = rx_ring->ring_size - 1;
rc = ena_refill_rx_bufs(rx_ring, bufs_num);
-
if (unlikely(rc != bufs_num))
ena_trace(ENA_WARNING, "refilling Queue %d failed. "
"Allocated %d buffers from: %d\n", i, rc, bufs_num);
+#ifdef DEV_NETMAP
+ rx_ring->initialized = true;
+#endif /* DEV_NETMAP */
}
}
@@ -1154,22 +1180,10 @@ ena_free_tx_bufs(struct ena_adapter *adapter, unsigned
qid, i);
}
- if (tx_info->head_mapped == true) {
- bus_dmamap_sync(adapter->tx_buf_tag, tx_info->map_head,
- BUS_DMASYNC_POSTWRITE);
- bus_dmamap_unload(adapter->tx_buf_tag,
- tx_info->map_head);
- tx_info->head_mapped = false;
- }
+ bus_dmamap_sync(adapter->tx_buf_tag, tx_info->dmamap,
+ BUS_DMASYNC_POSTWRITE);
+ bus_dmamap_unload(adapter->tx_buf_tag, tx_info->dmamap);
- if (tx_info->seg_mapped == true) {
- bus_dmamap_sync(adapter->tx_buf_tag, tx_info->map_seg,
- BUS_DMASYNC_POSTWRITE);
- bus_dmamap_unload(adapter->tx_buf_tag,
- tx_info->map_seg);
- tx_info->seg_mapped = false;
- }
-
m_free(tx_info->mbuf);
tx_info->mbuf = NULL;
}
@@ -1227,30 +1241,6 @@ ena_destroy_all_io_queues(struct ena_adapter *adapter)
ena_destroy_all_rx_queues(adapter);
}
-static inline int
-validate_tx_req_id(struct ena_ring *tx_ring, uint16_t req_id)
-{
- struct ena_adapter *adapter = tx_ring->adapter;
- struct ena_tx_buffer *tx_info = NULL;
-
- if (likely(req_id < tx_ring->ring_size)) {
- tx_info = &tx_ring->tx_buffer_info[req_id];
- if (tx_info->mbuf != NULL)
- return (0);
- device_printf(adapter->pdev,
- "tx_info doesn't have valid mbuf\n");
- }
-
- device_printf(adapter->pdev, "Invalid req_id: %hu\n", req_id);
- counter_u64_add(tx_ring->tx_stats.bad_req_id, 1);
-
- /* Trigger device reset */
- adapter->reset_reason = ENA_REGS_RESET_INV_TX_REQ_ID;
- ENA_FLAG_SET_ATOMIC(ENA_FLAG_TRIGGER_RESET, adapter);
-
- return (EFAULT);
-}
-
static int
ena_create_io_queues(struct ena_adapter *adapter)
{
@@ -1344,475 +1334,6 @@ err_tx:
return (ENXIO);
}
-/**
- * ena_tx_cleanup - clear sent packets and corresponding descriptors
- * @tx_ring: ring for which we want to clean packets
- *
- * Once packets are sent, we ask the device in a loop for no longer used
- * descriptors. We find the related mbuf chain in a map (index in an array)
- * and free it, then update ring state.
- * This is performed in "endless" loop, updating ring pointers every
- * TX_COMMIT. The first check of free descriptor is performed before the actual
- * loop, then repeated at the loop end.
- **/
-static int
-ena_tx_cleanup(struct ena_ring *tx_ring)
-{
- struct ena_adapter *adapter;
- struct ena_com_io_cq* io_cq;
- uint16_t next_to_clean;
- uint16_t req_id;
- uint16_t ena_qid;
- unsigned int total_done = 0;
- int rc;
- int commit = TX_COMMIT;
- int budget = TX_BUDGET;
- int work_done;
- bool above_thresh;
-
- adapter = tx_ring->que->adapter;
- ena_qid = ENA_IO_TXQ_IDX(tx_ring->que->id);
- io_cq = &adapter->ena_dev->io_cq_queues[ena_qid];
- next_to_clean = tx_ring->next_to_clean;
-
- do {
- struct ena_tx_buffer *tx_info;
- struct mbuf *mbuf;
-
- rc = ena_com_tx_comp_req_id_get(io_cq, &req_id);
- if (unlikely(rc != 0))
- break;
-
- rc = validate_tx_req_id(tx_ring, req_id);
- if (unlikely(rc != 0))
- break;
-
- tx_info = &tx_ring->tx_buffer_info[req_id];
-
- mbuf = tx_info->mbuf;
-
- tx_info->mbuf = NULL;
- bintime_clear(&tx_info->timestamp);
-
- /* Map is no longer required */
- if (tx_info->head_mapped == true) {
- bus_dmamap_sync(adapter->tx_buf_tag, tx_info->map_head,
- BUS_DMASYNC_POSTWRITE);
- bus_dmamap_unload(adapter->tx_buf_tag,
- tx_info->map_head);
- tx_info->head_mapped = false;
- }
- if (tx_info->seg_mapped == true) {
- bus_dmamap_sync(adapter->tx_buf_tag, tx_info->map_seg,
- BUS_DMASYNC_POSTWRITE);
- bus_dmamap_unload(adapter->tx_buf_tag,
- tx_info->map_seg);
- tx_info->seg_mapped = false;
- }
-
- ena_trace(ENA_DBG | ENA_TXPTH, "tx: q %d mbuf %p completed\n",
- tx_ring->qid, mbuf);
-
- m_freem(mbuf);
-
- total_done += tx_info->tx_descs;
-
- tx_ring->free_tx_ids[next_to_clean] = req_id;
- next_to_clean = ENA_TX_RING_IDX_NEXT(next_to_clean,
- tx_ring->ring_size);
-
- if (unlikely(--commit == 0)) {
- commit = TX_COMMIT;
- /* update ring state every TX_COMMIT descriptor */
- tx_ring->next_to_clean = next_to_clean;
- ena_com_comp_ack(
- &adapter->ena_dev->io_sq_queues[ena_qid],
- total_done);
- ena_com_update_dev_comp_head(io_cq);
- total_done = 0;
- }
- } while (likely(--budget));
-
- work_done = TX_BUDGET - budget;
-
- ena_trace(ENA_DBG | ENA_TXPTH, "tx: q %d done. total pkts: %d\n",
- tx_ring->qid, work_done);
-
- /* If there is still something to commit update ring state */
- if (likely(commit != TX_COMMIT)) {
- tx_ring->next_to_clean = next_to_clean;
- ena_com_comp_ack(&adapter->ena_dev->io_sq_queues[ena_qid],
- total_done);
- ena_com_update_dev_comp_head(io_cq);
- }
-
- /*
- * Need to make the rings circular update visible to
- * ena_xmit_mbuf() before checking for tx_ring->running.
- */
- mb();
-
- above_thresh = ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq,
- ENA_TX_RESUME_THRESH);
- if (unlikely(!tx_ring->running && above_thresh)) {
- ENA_RING_MTX_LOCK(tx_ring);
- above_thresh =
- ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq,
- ENA_TX_RESUME_THRESH);
- if (!tx_ring->running && above_thresh) {
- tx_ring->running = true;
- counter_u64_add(tx_ring->tx_stats.queue_wakeup, 1);
- taskqueue_enqueue(tx_ring->enqueue_tq,
- &tx_ring->enqueue_task);
- }
- ENA_RING_MTX_UNLOCK(tx_ring);
- }
-
- return (work_done);
-}
-
-static void
-ena_rx_hash_mbuf(struct ena_ring *rx_ring, struct ena_com_rx_ctx *ena_rx_ctx,
- struct mbuf *mbuf)
-{
- struct ena_adapter *adapter = rx_ring->adapter;
-
- if (likely(ENA_FLAG_ISSET(ENA_FLAG_RSS_ACTIVE, adapter))) {
- mbuf->m_pkthdr.flowid = ena_rx_ctx->hash;
-
- if (ena_rx_ctx->frag &&
- (ena_rx_ctx->l3_proto != ENA_ETH_IO_L3_PROTO_UNKNOWN)) {
- M_HASHTYPE_SET(mbuf, M_HASHTYPE_OPAQUE_HASH);
- return;
- }
-
- switch (ena_rx_ctx->l3_proto) {
- case ENA_ETH_IO_L3_PROTO_IPV4:
- switch (ena_rx_ctx->l4_proto) {
- case ENA_ETH_IO_L4_PROTO_TCP:
- M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_TCP_IPV4);
- break;
- case ENA_ETH_IO_L4_PROTO_UDP:
- M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_UDP_IPV4);
- break;
- default:
- M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_IPV4);
- }
- break;
- case ENA_ETH_IO_L3_PROTO_IPV6:
- switch (ena_rx_ctx->l4_proto) {
- case ENA_ETH_IO_L4_PROTO_TCP:
- M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_TCP_IPV6);
- break;
- case ENA_ETH_IO_L4_PROTO_UDP:
- M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_UDP_IPV6);
- break;
- default:
- M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_IPV6);
- }
- break;
- case ENA_ETH_IO_L3_PROTO_UNKNOWN:
- M_HASHTYPE_SET(mbuf, M_HASHTYPE_NONE);
- break;
- default:
- M_HASHTYPE_SET(mbuf, M_HASHTYPE_OPAQUE_HASH);
- }
- } else {
- mbuf->m_pkthdr.flowid = rx_ring->qid;
- M_HASHTYPE_SET(mbuf, M_HASHTYPE_NONE);
- }
-}
-
-/**
- * ena_rx_mbuf - assemble mbuf from descriptors
- * @rx_ring: ring for which we want to clean packets
- * @ena_bufs: buffer info
- * @ena_rx_ctx: metadata for this packet(s)
- * @next_to_clean: ring pointer, will be updated only upon success
- *
- **/
-static struct mbuf*
-ena_rx_mbuf(struct ena_ring *rx_ring, struct ena_com_rx_buf_info *ena_bufs,
- struct ena_com_rx_ctx *ena_rx_ctx, uint16_t *next_to_clean)
-{
- struct mbuf *mbuf;
- struct ena_rx_buffer *rx_info;
- struct ena_adapter *adapter;
- unsigned int descs = ena_rx_ctx->descs;
- int rc;
- uint16_t ntc, len, req_id, buf = 0;
-
- ntc = *next_to_clean;
- adapter = rx_ring->adapter;
-
- len = ena_bufs[buf].len;
- req_id = ena_bufs[buf].req_id;
- rc = validate_rx_req_id(rx_ring, req_id);
- if (unlikely(rc != 0))
- return (NULL);
-
- rx_info = &rx_ring->rx_buffer_info[req_id];
- if (unlikely(rx_info->mbuf == NULL)) {
- device_printf(adapter->pdev, "NULL mbuf in rx_info");
- return (NULL);
- }
-
- ena_trace(ENA_DBG | ENA_RXPTH, "rx_info %p, mbuf %p, paddr %jx\n",
- rx_info, rx_info->mbuf, (uintmax_t)rx_info->ena_buf.paddr);
-
- bus_dmamap_sync(adapter->rx_buf_tag, rx_info->map,
- BUS_DMASYNC_POSTREAD);
- mbuf = rx_info->mbuf;
- mbuf->m_flags |= M_PKTHDR;
- mbuf->m_pkthdr.len = len;
- mbuf->m_len = len;
- mbuf->m_pkthdr.rcvif = rx_ring->que->adapter->ifp;
-
- /* Fill mbuf with hash key and it's interpretation for optimization */
- ena_rx_hash_mbuf(rx_ring, ena_rx_ctx, mbuf);
-
- ena_trace(ENA_DBG | ENA_RXPTH, "rx mbuf 0x%p, flags=0x%x, len: %d\n",
- mbuf, mbuf->m_flags, mbuf->m_pkthdr.len);
-
- /* DMA address is not needed anymore, unmap it */
- bus_dmamap_unload(rx_ring->adapter->rx_buf_tag, rx_info->map);
-
- rx_info->mbuf = NULL;
- rx_ring->free_rx_ids[ntc] = req_id;
- ntc = ENA_RX_RING_IDX_NEXT(ntc, rx_ring->ring_size);
-
- /*
- * While we have more than 1 descriptors for one rcvd packet, append
- * other mbufs to the main one
- */
- while (--descs) {
- ++buf;
- len = ena_bufs[buf].len;
- req_id = ena_bufs[buf].req_id;
- rc = validate_rx_req_id(rx_ring, req_id);
- if (unlikely(rc != 0)) {
- /*
- * If the req_id is invalid, then the device will be
- * reset. In that case we must free all mbufs that
- * were already gathered.
- */
- m_freem(mbuf);
- return (NULL);
- }
- rx_info = &rx_ring->rx_buffer_info[req_id];
-
- if (unlikely(rx_info->mbuf == NULL)) {
- device_printf(adapter->pdev, "NULL mbuf in rx_info");
- /*
- * If one of the required mbufs was not allocated yet,
- * we can break there.
- * All earlier used descriptors will be reallocated
- * later and not used mbufs can be reused.
- * The next_to_clean pointer will not be updated in case
- * of an error, so caller should advance it manually
- * in error handling routine to keep it up to date
- * with hw ring.
- */
- m_freem(mbuf);
- return (NULL);
- }
-
- bus_dmamap_sync(adapter->rx_buf_tag, rx_info->map,
- BUS_DMASYNC_POSTREAD);
- if (unlikely(m_append(mbuf, len, rx_info->mbuf->m_data) == 0)) {
- counter_u64_add(rx_ring->rx_stats.mbuf_alloc_fail, 1);
- ena_trace(ENA_WARNING, "Failed to append Rx mbuf %p\n",
- mbuf);
- }
-
- ena_trace(ENA_DBG | ENA_RXPTH,
- "rx mbuf updated. len %d\n", mbuf->m_pkthdr.len);
-
- /* Free already appended mbuf, it won't be useful anymore */
- bus_dmamap_unload(rx_ring->adapter->rx_buf_tag, rx_info->map);
- m_freem(rx_info->mbuf);
- rx_info->mbuf = NULL;
-
- rx_ring->free_rx_ids[ntc] = req_id;
- ntc = ENA_RX_RING_IDX_NEXT(ntc, rx_ring->ring_size);
- }
-
- *next_to_clean = ntc;
-
- return (mbuf);
-}
-
-/**
- * ena_rx_checksum - indicate in mbuf if hw indicated a good cksum
- **/
-static inline void
-ena_rx_checksum(struct ena_ring *rx_ring, struct ena_com_rx_ctx *ena_rx_ctx,
- struct mbuf *mbuf)
-{
-
- /* if IP and error */
- if (unlikely((ena_rx_ctx->l3_proto == ENA_ETH_IO_L3_PROTO_IPV4) &&
- ena_rx_ctx->l3_csum_err)) {
- /* ipv4 checksum error */
- mbuf->m_pkthdr.csum_flags = 0;
- counter_u64_add(rx_ring->rx_stats.bad_csum, 1);
- ena_trace(ENA_DBG, "RX IPv4 header checksum error\n");
- return;
- }
-
- /* if TCP/UDP */
- if ((ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_TCP) ||
- (ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_UDP)) {
- if (ena_rx_ctx->l4_csum_err) {
- /* TCP/UDP checksum error */
- mbuf->m_pkthdr.csum_flags = 0;
- counter_u64_add(rx_ring->rx_stats.bad_csum, 1);
- ena_trace(ENA_DBG, "RX L4 checksum error\n");
- } else {
- mbuf->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
- mbuf->m_pkthdr.csum_flags |= CSUM_IP_VALID;
- }
- }
-}
-
-/**
- * ena_rx_cleanup - handle rx irq
- * @arg: ring for which irq is being handled
- **/
-static int
-ena_rx_cleanup(struct ena_ring *rx_ring)
-{
- struct ena_adapter *adapter;
- struct mbuf *mbuf;
- struct ena_com_rx_ctx ena_rx_ctx;
- struct ena_com_io_cq* io_cq;
- struct ena_com_io_sq* io_sq;
- if_t ifp;
- uint16_t ena_qid;
- uint16_t next_to_clean;
- uint32_t refill_required;
- uint32_t refill_threshold;
- uint32_t do_if_input = 0;
- unsigned int qid;
- int rc, i;
- int budget = RX_BUDGET;
-
- adapter = rx_ring->que->adapter;
- ifp = adapter->ifp;
- qid = rx_ring->que->id;
- ena_qid = ENA_IO_RXQ_IDX(qid);
- io_cq = &adapter->ena_dev->io_cq_queues[ena_qid];
- io_sq = &adapter->ena_dev->io_sq_queues[ena_qid];
- next_to_clean = rx_ring->next_to_clean;
-
- ena_trace(ENA_DBG, "rx: qid %d\n", qid);
-
- do {
- ena_rx_ctx.ena_bufs = rx_ring->ena_bufs;
- ena_rx_ctx.max_bufs = adapter->max_rx_sgl_size;
- ena_rx_ctx.descs = 0;
- bus_dmamap_sync(io_cq->cdesc_addr.mem_handle.tag,
- io_cq->cdesc_addr.mem_handle.map, BUS_DMASYNC_POSTREAD);
- rc = ena_com_rx_pkt(io_cq, io_sq, &ena_rx_ctx);
-
- if (unlikely(rc != 0))
- goto error;
-
- if (unlikely(ena_rx_ctx.descs == 0))
- break;
-
- ena_trace(ENA_DBG | ENA_RXPTH, "rx: q %d got packet from ena. "
- "descs #: %d l3 proto %d l4 proto %d hash: %x\n",
- rx_ring->qid, ena_rx_ctx.descs, ena_rx_ctx.l3_proto,
- ena_rx_ctx.l4_proto, ena_rx_ctx.hash);
-
- /* Receive mbuf from the ring */
- mbuf = ena_rx_mbuf(rx_ring, rx_ring->ena_bufs,
- &ena_rx_ctx, &next_to_clean);
- bus_dmamap_sync(io_cq->cdesc_addr.mem_handle.tag,
- io_cq->cdesc_addr.mem_handle.map, BUS_DMASYNC_PREREAD);
- /* Exit if we failed to retrieve a buffer */
- if (unlikely(mbuf == NULL)) {
- for (i = 0; i < ena_rx_ctx.descs; ++i) {
- rx_ring->free_rx_ids[next_to_clean] =
- rx_ring->ena_bufs[i].req_id;
- next_to_clean =
- ENA_RX_RING_IDX_NEXT(next_to_clean,
- rx_ring->ring_size);
-
- }
- break;
- }
-
- if (((ifp->if_capenable & IFCAP_RXCSUM) != 0) ||
- ((ifp->if_capenable & IFCAP_RXCSUM_IPV6) != 0)) {
- ena_rx_checksum(rx_ring, &ena_rx_ctx, mbuf);
- }
-
- counter_enter();
- counter_u64_add_protected(rx_ring->rx_stats.bytes,
- mbuf->m_pkthdr.len);
- counter_u64_add_protected(adapter->hw_stats.rx_bytes,
- mbuf->m_pkthdr.len);
- counter_exit();
- /*
- * LRO is only for IP/TCP packets and TCP checksum of the packet
- * should be computed by hardware.
- */
- do_if_input = 1;
- if (((ifp->if_capenable & IFCAP_LRO) != 0) &&
- ((mbuf->m_pkthdr.csum_flags & CSUM_IP_VALID) != 0) &&
- (ena_rx_ctx.l4_proto == ENA_ETH_IO_L4_PROTO_TCP)) {
- /*
- * Send to the stack if:
- * - LRO not enabled, or
- * - no LRO resources, or
- * - lro enqueue fails
- */
- if ((rx_ring->lro.lro_cnt != 0) &&
- (tcp_lro_rx(&rx_ring->lro, mbuf, 0) == 0))
- do_if_input = 0;
- }
- if (do_if_input != 0) {
- ena_trace(ENA_DBG | ENA_RXPTH,
- "calling if_input() with mbuf %p\n", mbuf);
- (*ifp->if_input)(ifp, mbuf);
- }
-
- counter_enter();
- counter_u64_add_protected(rx_ring->rx_stats.cnt, 1);
- counter_u64_add_protected(adapter->hw_stats.rx_packets, 1);
- counter_exit();
- } while (--budget);
-
- rx_ring->next_to_clean = next_to_clean;
-
- refill_required = ena_com_free_desc(io_sq);
- refill_threshold = min_t(int,
- rx_ring->ring_size / ENA_RX_REFILL_THRESH_DIVIDER,
- ENA_RX_REFILL_THRESH_PACKET);
-
- if (refill_required > refill_threshold) {
- ena_com_update_dev_comp_head(rx_ring->ena_com_io_cq);
- ena_refill_rx_bufs(rx_ring, refill_required);
- }
-
- tcp_lro_flush_all(&rx_ring->lro);
-
- return (RX_BUDGET - budget);
-
-error:
- counter_u64_add(rx_ring->rx_stats.bad_desc_num, 1);
-
- /* Too many desc from the device. Trigger reset */
- if (likely(!ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter))) {
- adapter->reset_reason = ENA_REGS_RESET_TOO_MANY_RX_DESCS;
- ENA_FLAG_SET_ATOMIC(ENA_FLAG_TRIGGER_RESET, adapter);
- }
-
- return (0);
-}
-
/*********************************************************************
*
* MSIX & Interrupt Service routine
@@ -1833,52 +1354,6 @@ ena_intr_msix_mgmnt(void *arg)
ena_com_aenq_intr_handler(adapter->ena_dev, arg);
}
-static void
-ena_cleanup(void *arg, int pending)
-{
- struct ena_que *que = arg;
- struct ena_adapter *adapter = que->adapter;
- if_t ifp = adapter->ifp;
- struct ena_ring *tx_ring;
- struct ena_ring *rx_ring;
- struct ena_com_io_cq* io_cq;
- struct ena_eth_io_intr_reg intr_reg;
- int qid, ena_qid;
- int txc, rxc, i;
-
- if (unlikely((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0))
- return;
-
- ena_trace(ENA_DBG, "MSI-X TX/RX routine\n");
-
- tx_ring = que->tx_ring;
- rx_ring = que->rx_ring;
- qid = que->id;
- ena_qid = ENA_IO_TXQ_IDX(qid);
- io_cq = &adapter->ena_dev->io_cq_queues[ena_qid];
-
- tx_ring->first_interrupt = true;
- rx_ring->first_interrupt = true;
-
- for (i = 0; i < CLEAN_BUDGET; ++i) {
- rxc = ena_rx_cleanup(rx_ring);
- txc = ena_tx_cleanup(tx_ring);
-
- if (unlikely((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0))
- return;
-
- if ((txc != TX_BUDGET) && (rxc != RX_BUDGET))
- break;
- }
-
- /* Signal that work is done and unmask interrupt */
- ena_com_update_intr_reg(&intr_reg,
- RX_IRQ_INTERVAL,
- TX_IRQ_INTERVAL,
- true);
- ena_com_unmask_intr(io_cq, &intr_reg);
-}
-
/**
* ena_handle_msix - MSIX Interrupt Handler for Tx/Rx
* @arg: queue
@@ -2289,7 +1764,7 @@ ena_up_complete(struct ena_adapter *adapter)
return (0);
}
-static int
+int
ena_up(struct ena_adapter *adapter)
{
int rc = 0;
@@ -2350,8 +1825,14 @@ ena_up(struct ena_adapter *adapter)
*** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
More information about the svn-src-all
mailing list