svn commit: r365380 - in stable/12/sys: conf contrib/ena-com dev/ena modules/ena
Marcin Wojtas
mw at FreeBSD.org
Sun Sep 6 14:23:32 UTC 2020
Author: mw
Date: Sun Sep 6 14:23:31 2020
New Revision: 365380
URL: https://svnweb.freebsd.org/changeset/base/365380
Log:
MFC: ENA netmap support and bug fixes
r363638 Fix ENA build when integrated into kernel
r354242 Make valdiate_rx_req_id static inline because it uses other static
r354225 Update ENA version to v2.1.0
r354224 Add support for ENA NETMAP partial initialization
r354223 Add support for ENA NETMAP Tx
r354222 Add support for ENA NETMAP Rx
r354221 Introduce NETMAP support in ENA
r354220 Split Rx/Tx from initialization code in ENA driver
r354219 Fix ENA keep-alive timeout due to prolonged reset
r354218 Add WC support for arm64 in the ENA driver
Obtained from: Semihalf
Sponsored by: Amazon, Inc.
Added:
stable/12/sys/dev/ena/ena_datapath.c
- copied, changed from r354225, head/sys/dev/ena/ena_datapath.c
stable/12/sys/dev/ena/ena_datapath.h
- copied unchanged from r354225, head/sys/dev/ena/ena_datapath.h
stable/12/sys/dev/ena/ena_netmap.c
- copied unchanged from r354225, head/sys/dev/ena/ena_netmap.c
stable/12/sys/dev/ena/ena_netmap.h
- copied unchanged from r354225, head/sys/dev/ena/ena_netmap.h
Modified:
stable/12/sys/conf/files
stable/12/sys/contrib/ena-com/ena_plat.h
stable/12/sys/dev/ena/ena.c
stable/12/sys/dev/ena/ena.h
stable/12/sys/modules/ena/Makefile
Directory Properties:
stable/12/ (props changed)
Modified: stable/12/sys/conf/files
==============================================================================
--- stable/12/sys/conf/files Sun Sep 6 14:13:51 2020 (r365379)
+++ stable/12/sys/conf/files Sun Sep 6 14:23:31 2020 (r365380)
@@ -1707,6 +1707,10 @@ dev/e1000/e1000_osdep.c optional em \
dev/et/if_et.c optional et
dev/ena/ena.c optional ena \
compile-with "${NORMAL_C} -I$S/contrib"
+dev/ena/ena_datapath.c optional ena \
+ compile-with "${NORMAL_C} -I$S/contrib"
+dev/ena/ena_netmap.c optional ena \
+ compile-with "${NORMAL_C} -I$S/contrib"
dev/ena/ena_sysctl.c optional ena \
compile-with "${NORMAL_C} -I$S/contrib"
contrib/ena-com/ena_com.c optional ena
Modified: stable/12/sys/contrib/ena-com/ena_plat.h
==============================================================================
--- stable/12/sys/contrib/ena-com/ena_plat.h Sun Sep 6 14:13:51 2020 (r365379)
+++ stable/12/sys/contrib/ena-com/ena_plat.h Sun Sep 6 14:23:31 2020 (r365380)
@@ -103,6 +103,7 @@ extern struct ena_bus_space ebs;
#define ENA_RSC (1 << 6) /* Goes with TXPTH or RXPTH, free/alloc res. */
#define ENA_IOQ (1 << 7) /* Detailed info about IO queues. */
#define ENA_ADMQ (1 << 8) /* Detailed info about admin queue. */
+#define ENA_NETMAP (1 << 9) /* Detailed info about netmap. */
extern int ena_log_level;
Modified: stable/12/sys/dev/ena/ena.c
==============================================================================
--- stable/12/sys/dev/ena/ena.c Sun Sep 6 14:13:51 2020 (r365379)
+++ stable/12/sys/dev/ena/ena.c Sun Sep 6 14:23:31 2020 (r365380)
@@ -81,9 +81,14 @@ __FBSDID("$FreeBSD$");
#include <vm/vm.h>
#include <vm/pmap.h>
+#include "ena_datapath.h"
#include "ena.h"
#include "ena_sysctl.h"
+#ifdef DEV_NETMAP
+#include "ena_netmap.h"
+#endif /* DEV_NETMAP */
+
/*********************************************************
* Function prototypes
*********************************************************/
@@ -103,11 +108,11 @@ static int ena_setup_tx_dma_tag(struct ena_adapter *);
static int ena_free_tx_dma_tag(struct ena_adapter *);
static int ena_setup_rx_dma_tag(struct ena_adapter *);
static int ena_free_rx_dma_tag(struct ena_adapter *);
+static void ena_release_all_tx_dmamap(struct ena_ring *);
static int ena_setup_tx_resources(struct ena_adapter *, int);
static void ena_free_tx_resources(struct ena_adapter *, int);
static int ena_setup_all_tx_resources(struct ena_adapter *);
static void ena_free_all_tx_resources(struct ena_adapter *);
-static inline int validate_rx_req_id(struct ena_ring *, uint16_t);
static int ena_setup_rx_resources(struct ena_adapter *, unsigned int);
static void ena_free_rx_resources(struct ena_adapter *, unsigned int);
static int ena_setup_all_rx_resources(struct ena_adapter *);
@@ -116,7 +121,6 @@ static inline int ena_alloc_rx_mbuf(struct ena_adapter
struct ena_rx_buffer *);
static void ena_free_rx_mbuf(struct ena_adapter *, struct ena_ring *,
struct ena_rx_buffer *);
-static int ena_refill_rx_bufs(struct ena_ring *, uint32_t);
static void ena_free_rx_bufs(struct ena_adapter *, unsigned int);
static void ena_refill_all_rx_bufs(struct ena_adapter *);
static void ena_free_all_rx_bufs(struct ena_adapter *);
@@ -126,16 +130,6 @@ static void ena_destroy_all_tx_queues(struct ena_adapt
static void ena_destroy_all_rx_queues(struct ena_adapter *);
static void ena_destroy_all_io_queues(struct ena_adapter *);
static int ena_create_io_queues(struct ena_adapter *);
-static int ena_tx_cleanup(struct ena_ring *);
-static int ena_rx_cleanup(struct ena_ring *);
-static inline int validate_tx_req_id(struct ena_ring *, uint16_t);
-static void ena_rx_hash_mbuf(struct ena_ring *, struct ena_com_rx_ctx *,
- struct mbuf *);
-static struct mbuf* ena_rx_mbuf(struct ena_ring *, struct ena_com_rx_buf_info *,
- struct ena_com_rx_ctx *, uint16_t *);
-static inline void ena_rx_checksum(struct ena_ring *, struct ena_com_rx_ctx *,
- struct mbuf *);
-static void ena_cleanup(void *arg, int pending);
static int ena_handle_msix(void *);
static int ena_enable_msix(struct ena_adapter *);
static void ena_setup_mgmnt_intr(struct ena_adapter *);
@@ -149,8 +143,6 @@ static void ena_disable_msix(struct ena_adapter *);
static void ena_unmask_all_io_irqs(struct ena_adapter *);
static int ena_rss_configure(struct ena_adapter *);
static int ena_up_complete(struct ena_adapter *);
-static int ena_up(struct ena_adapter *);
-static void ena_down(struct ena_adapter *);
static uint64_t ena_get_counter(if_t, ift_counter);
static int ena_media_change(if_t);
static void ena_media_status(if_t, struct ifmediareq *);
@@ -161,15 +153,6 @@ static void ena_update_host_info(struct ena_admin_host
static void ena_update_hwassist(struct ena_adapter *);
static int ena_setup_ifnet(device_t, struct ena_adapter *,
struct ena_com_dev_get_features_ctx *);
-static void ena_tx_csum(struct ena_com_tx_ctx *, struct mbuf *);
-static int ena_check_and_collapse_mbuf(struct ena_ring *tx_ring,
- struct mbuf **mbuf);
-static void ena_dmamap_llq(void *, bus_dma_segment_t *, int, int);
-static int ena_xmit_mbuf(struct ena_ring *, struct mbuf **);
-static void ena_start_xmit(struct ena_ring *);
-static int ena_mq_start(if_t, struct mbuf *);
-static void ena_deferred_mq_start(void *, int);
-static void ena_qflush(if_t);
static int ena_enable_wc(struct resource *);
static int ena_set_queues_placement_policy(device_t, struct ena_com_dev *,
struct ena_admin_feature_llq_desc *, struct ena_llq_configurations *);
@@ -555,6 +538,44 @@ ena_free_rx_dma_tag(struct ena_adapter *adapter)
return (ret);
}
+static void
+ena_release_all_tx_dmamap(struct ena_ring *tx_ring)
+{
+ struct ena_adapter *adapter = tx_ring->adapter;
+ struct ena_tx_buffer *tx_info;
+ bus_dma_tag_t tx_tag = adapter->tx_buf_tag;;
+ int i;
+#ifdef DEV_NETMAP
+ struct ena_netmap_tx_info *nm_info;
+ int j;
+#endif /* DEV_NETMAP */
+
+ for (i = 0; i < tx_ring->ring_size; ++i) {
+ tx_info = &tx_ring->tx_buffer_info[i];
+#ifdef DEV_NETMAP
+ if (adapter->ifp->if_capenable & IFCAP_NETMAP) {
+ nm_info = &tx_info->nm_info;
+ for (j = 0; j < ENA_PKT_MAX_BUFS; ++j) {
+ if (nm_info->map_seg[j] != NULL) {
+ bus_dmamap_destroy(tx_tag,
+ nm_info->map_seg[j]);
+ nm_info->map_seg[j] = NULL;
+ }
+ }
+ }
+#endif /* DEV_NETMAP */
+ if (tx_info->map_head != NULL) {
+ bus_dmamap_destroy(tx_tag, tx_info->map_head);
+ tx_info->map_head = NULL;
+ }
+
+ if (tx_info->map_seg != NULL) {
+ bus_dmamap_destroy(tx_tag, tx_info->map_seg);
+ tx_info->map_seg = NULL;
+ }
+ }
+}
+
/**
* ena_setup_tx_resources - allocate Tx resources (Descriptors)
* @adapter: network interface device structure
@@ -568,7 +589,13 @@ ena_setup_tx_resources(struct ena_adapter *adapter, in
struct ena_que *que = &adapter->que[qid];
struct ena_ring *tx_ring = que->tx_ring;
int size, i, err;
+#ifdef DEV_NETMAP
+ bus_dmamap_t *map;
+ int j;
+ ena_netmap_reset_tx_ring(adapter, qid);
+#endif /* DEV_NETMAP */
+
size = sizeof(struct ena_tx_buffer) * tx_ring->ring_size;
tx_ring->tx_buffer_info = malloc(size, M_DEVBUF, M_NOWAIT | M_ZERO);
@@ -611,7 +638,7 @@ ena_setup_tx_resources(struct ena_adapter *adapter, in
ena_trace(ENA_ALERT,
"Unable to create Tx DMA map_head for buffer %d\n",
i);
- goto err_buf_info_unmap;
+ goto err_map_release;
}
tx_ring->tx_buffer_info[i].seg_mapped = false;
@@ -621,9 +648,24 @@ ena_setup_tx_resources(struct ena_adapter *adapter, in
ena_trace(ENA_ALERT,
"Unable to create Tx DMA map_seg for buffer %d\n",
i);
- goto err_buf_info_head_unmap;
+ goto err_map_release;
}
tx_ring->tx_buffer_info[i].head_mapped = false;
+
+#ifdef DEV_NETMAP
+ if (adapter->ifp->if_capenable & IFCAP_NETMAP) {
+ map = tx_ring->tx_buffer_info[i].nm_info.map_seg;
+ for (j = 0; j < ENA_PKT_MAX_BUFS; j++) {
+ err = bus_dmamap_create(adapter->tx_buf_tag, 0,
+ &map[j]);
+ if (unlikely(err != 0)) {
+ ena_trace(ENA_ALERT, "Unable to create "
+ "Tx DMA for buffer %d %d\n", i, j);
+ goto err_map_release;
+ }
+ }
+ }
+#endif /* DEV_NETMAP */
}
/* Allocate taskqueues */
@@ -634,7 +676,7 @@ ena_setup_tx_resources(struct ena_adapter *adapter, in
ena_trace(ENA_ALERT,
"Unable to create taskqueue for enqueue task\n");
i = tx_ring->ring_size;
- goto err_buf_info_unmap;
+ goto err_map_release;
}
tx_ring->running = true;
@@ -644,17 +686,8 @@ ena_setup_tx_resources(struct ena_adapter *adapter, in
return (0);
-err_buf_info_head_unmap:
- bus_dmamap_destroy(adapter->tx_buf_tag,
- tx_ring->tx_buffer_info[i].map_head);
-err_buf_info_unmap:
- while (i--) {
- bus_dmamap_destroy(adapter->tx_buf_tag,
- tx_ring->tx_buffer_info[i].map_head);
- bus_dmamap_destroy(adapter->tx_buf_tag,
- tx_ring->tx_buffer_info[i].map_seg);
- }
- free(tx_ring->push_buf_intermediate_buf, M_DEVBUF);
+err_map_release:
+ ena_release_all_tx_dmamap(tx_ring);
err_tx_ids_free:
free(tx_ring->free_tx_ids, M_DEVBUF);
tx_ring->free_tx_ids = NULL;
@@ -676,6 +709,10 @@ static void
ena_free_tx_resources(struct ena_adapter *adapter, int qid)
{
struct ena_ring *tx_ring = &adapter->tx_ring[qid];
+#ifdef DEV_NETMAP
+ struct ena_netmap_tx_info *nm_info;
+ int j;
+#endif /* DEV_NETMAP */
while (taskqueue_cancel(tx_ring->enqueue_tq, &tx_ring->enqueue_task,
NULL))
@@ -711,6 +748,24 @@ ena_free_tx_resources(struct ena_adapter *adapter, int
bus_dmamap_destroy(adapter->tx_buf_tag,
tx_ring->tx_buffer_info[i].map_seg);
+#ifdef DEV_NETMAP
+ if (adapter->ifp->if_capenable & IFCAP_NETMAP) {
+ nm_info = &tx_ring->tx_buffer_info[i].nm_info;
+ for (j = 0; j < ENA_PKT_MAX_BUFS; j++) {
+ if (nm_info->socket_buf_idx[j] != 0) {
+ bus_dmamap_sync(adapter->tx_buf_tag,
+ nm_info->map_seg[j],
+ BUS_DMASYNC_POSTWRITE);
+ ena_netmap_unload(adapter,
+ nm_info->map_seg[j]);
+ }
+ bus_dmamap_destroy(adapter->tx_buf_tag,
+ nm_info->map_seg[j]);
+ nm_info->socket_buf_idx[j] = 0;
+ }
+ }
+#endif /* DEV_NETMAP */
+
m_freem(tx_ring->tx_buffer_info[i].mbuf);
tx_ring->tx_buffer_info[i].mbuf = NULL;
}
@@ -772,25 +827,6 @@ ena_free_all_tx_resources(struct ena_adapter *adapter)
ena_free_tx_resources(adapter, i);
}
-static inline int
-validate_rx_req_id(struct ena_ring *rx_ring, uint16_t req_id)
-{
- if (likely(req_id < rx_ring->ring_size))
- return (0);
-
- device_printf(rx_ring->adapter->pdev, "Invalid rx req_id: %hu\n",
- req_id);
- counter_u64_add(rx_ring->rx_stats.bad_req_id, 1);
-
- /* Trigger device reset */
- if (likely(!ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, rx_ring->adapter))) {
- rx_ring->adapter->reset_reason = ENA_REGS_RESET_INV_RX_REQ_ID;
- ENA_FLAG_SET_ATOMIC(ENA_FLAG_TRIGGER_RESET, rx_ring->adapter);
- }
-
- return (EFAULT);
-}
-
/**
* ena_setup_rx_resources - allocate Rx resources (Descriptors)
* @adapter: network interface device structure
@@ -807,6 +843,11 @@ ena_setup_rx_resources(struct ena_adapter *adapter, un
size = sizeof(struct ena_rx_buffer) * rx_ring->ring_size;
+#ifdef DEV_NETMAP
+ ena_netmap_reset_rx_ring(adapter, qid);
+ rx_ring->initialized = false;
+#endif /* DEV_NETMAP */
+
/*
* Alloc extra element so in rx path
* we can always prefetch rx_info + 1
@@ -1031,7 +1072,7 @@ ena_free_rx_mbuf(struct ena_adapter *adapter, struct e
* @num: number of descriptors to refill
* Refills the ring with newly allocated DMA-mapped mbufs for receiving
**/
-static int
+int
ena_refill_rx_bufs(struct ena_ring *rx_ring, uint32_t num)
{
struct ena_adapter *adapter = rx_ring->adapter;
@@ -1052,8 +1093,12 @@ ena_refill_rx_bufs(struct ena_ring *rx_ring, uint32_t
req_id = rx_ring->free_rx_ids[next_to_use];
rx_info = &rx_ring->rx_buffer_info[req_id];
-
- rc = ena_alloc_rx_mbuf(adapter, rx_ring, rx_info);
+#ifdef DEV_NETMAP
+ if (ena_rx_ring_in_netmap(adapter, rx_ring->qid))
+ rc = ena_netmap_alloc_rx_slot(adapter, rx_ring, rx_info);
+ else
+#endif /* DEV_NETMAP */
+ rc = ena_alloc_rx_mbuf(adapter, rx_ring, rx_info);
if (unlikely(rc != 0)) {
ena_trace(ENA_WARNING,
"failed to alloc buffer for rx queue %d\n",
@@ -1098,6 +1143,14 @@ ena_free_rx_bufs(struct ena_adapter *adapter, unsigned
if (rx_info->mbuf != NULL)
ena_free_rx_mbuf(adapter, rx_ring, rx_info);
+#ifdef DEV_NETMAP
+ if (((if_getflags(adapter->ifp) & IFF_DYING) == 0) &&
+ (adapter->ifp->if_capenable & IFCAP_NETMAP)) {
+ if (rx_info->netmap_buf_idx != 0)
+ ena_netmap_free_rx_slot(adapter, rx_ring,
+ rx_info);
+ }
+#endif /* DEV_NETMAP */
}
}
@@ -1116,10 +1169,12 @@ ena_refill_all_rx_bufs(struct ena_adapter *adapter)
rx_ring = &adapter->rx_ring[i];
bufs_num = rx_ring->ring_size - 1;
rc = ena_refill_rx_bufs(rx_ring, bufs_num);
-
if (unlikely(rc != bufs_num))
ena_trace(ENA_WARNING, "refilling Queue %d failed. "
"Allocated %d buffers from: %d\n", i, rc, bufs_num);
+#ifdef DEV_NETMAP
+ rx_ring->initialized = true;
+#endif /* DEV_NETMAP */
}
}
@@ -1234,30 +1289,6 @@ ena_destroy_all_io_queues(struct ena_adapter *adapter)
ena_destroy_all_rx_queues(adapter);
}
-static inline int
-validate_tx_req_id(struct ena_ring *tx_ring, uint16_t req_id)
-{
- struct ena_adapter *adapter = tx_ring->adapter;
- struct ena_tx_buffer *tx_info = NULL;
-
- if (likely(req_id < tx_ring->ring_size)) {
- tx_info = &tx_ring->tx_buffer_info[req_id];
- if (tx_info->mbuf != NULL)
- return (0);
- device_printf(adapter->pdev,
- "tx_info doesn't have valid mbuf\n");
- }
-
- device_printf(adapter->pdev, "Invalid req_id: %hu\n", req_id);
- counter_u64_add(tx_ring->tx_stats.bad_req_id, 1);
-
- /* Trigger device reset */
- adapter->reset_reason = ENA_REGS_RESET_INV_TX_REQ_ID;
- ENA_FLAG_SET_ATOMIC(ENA_FLAG_TRIGGER_RESET, adapter);
-
- return (EFAULT);
-}
-
static int
ena_create_io_queues(struct ena_adapter *adapter)
{
@@ -1351,488 +1382,6 @@ err_tx:
return (ENXIO);
}
-/**
- * ena_tx_cleanup - clear sent packets and corresponding descriptors
- * @tx_ring: ring for which we want to clean packets
- *
- * Once packets are sent, we ask the device in a loop for no longer used
- * descriptors. We find the related mbuf chain in a map (index in an array)
- * and free it, then update ring state.
- * This is performed in "endless" loop, updating ring pointers every
- * TX_COMMIT. The first check of free descriptor is performed before the actual
- * loop, then repeated at the loop end.
- **/
-static int
-ena_tx_cleanup(struct ena_ring *tx_ring)
-{
- struct ena_adapter *adapter;
- struct ena_com_io_cq* io_cq;
- uint16_t next_to_clean;
- uint16_t req_id;
- uint16_t ena_qid;
- unsigned int total_done = 0;
- int rc;
- int commit = TX_COMMIT;
- int budget = TX_BUDGET;
- int work_done;
- bool above_thresh;
-
- adapter = tx_ring->que->adapter;
- ena_qid = ENA_IO_TXQ_IDX(tx_ring->que->id);
- io_cq = &adapter->ena_dev->io_cq_queues[ena_qid];
- next_to_clean = tx_ring->next_to_clean;
-
- do {
- struct ena_tx_buffer *tx_info;
- struct mbuf *mbuf;
-
- rc = ena_com_tx_comp_req_id_get(io_cq, &req_id);
- if (unlikely(rc != 0))
- break;
-
- rc = validate_tx_req_id(tx_ring, req_id);
- if (unlikely(rc != 0))
- break;
-
- tx_info = &tx_ring->tx_buffer_info[req_id];
-
- mbuf = tx_info->mbuf;
-
- tx_info->mbuf = NULL;
- bintime_clear(&tx_info->timestamp);
-
- /* Map is no longer required */
- if (tx_info->head_mapped == true) {
- bus_dmamap_sync(adapter->tx_buf_tag, tx_info->map_head,
- BUS_DMASYNC_POSTWRITE);
- bus_dmamap_unload(adapter->tx_buf_tag,
- tx_info->map_head);
- tx_info->head_mapped = false;
- }
- if (tx_info->seg_mapped == true) {
- bus_dmamap_sync(adapter->tx_buf_tag, tx_info->map_seg,
- BUS_DMASYNC_POSTWRITE);
- bus_dmamap_unload(adapter->tx_buf_tag,
- tx_info->map_seg);
- tx_info->seg_mapped = false;
- }
-
- ena_trace(ENA_DBG | ENA_TXPTH, "tx: q %d mbuf %p completed\n",
- tx_ring->qid, mbuf);
-
- m_freem(mbuf);
-
- total_done += tx_info->tx_descs;
-
- tx_ring->free_tx_ids[next_to_clean] = req_id;
- next_to_clean = ENA_TX_RING_IDX_NEXT(next_to_clean,
- tx_ring->ring_size);
-
- if (unlikely(--commit == 0)) {
- commit = TX_COMMIT;
- /* update ring state every TX_COMMIT descriptor */
- tx_ring->next_to_clean = next_to_clean;
- ena_com_comp_ack(
- &adapter->ena_dev->io_sq_queues[ena_qid],
- total_done);
- ena_com_update_dev_comp_head(io_cq);
- total_done = 0;
- }
- } while (likely(--budget));
-
- work_done = TX_BUDGET - budget;
-
- ena_trace(ENA_DBG | ENA_TXPTH, "tx: q %d done. total pkts: %d\n",
- tx_ring->qid, work_done);
-
- /* If there is still something to commit update ring state */
- if (likely(commit != TX_COMMIT)) {
- tx_ring->next_to_clean = next_to_clean;
- ena_com_comp_ack(&adapter->ena_dev->io_sq_queues[ena_qid],
- total_done);
- ena_com_update_dev_comp_head(io_cq);
- }
-
- /*
- * Need to make the rings circular update visible to
- * ena_xmit_mbuf() before checking for tx_ring->running.
- */
- mb();
-
- above_thresh = ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq,
- ENA_TX_RESUME_THRESH);
- if (unlikely(!tx_ring->running && above_thresh)) {
- ENA_RING_MTX_LOCK(tx_ring);
- above_thresh =
- ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq,
- ENA_TX_RESUME_THRESH);
- if (!tx_ring->running && above_thresh) {
- tx_ring->running = true;
- counter_u64_add(tx_ring->tx_stats.queue_wakeup, 1);
- taskqueue_enqueue(tx_ring->enqueue_tq,
- &tx_ring->enqueue_task);
- }
- ENA_RING_MTX_UNLOCK(tx_ring);
- }
-
- return (work_done);
-}
-
-static void
-ena_rx_hash_mbuf(struct ena_ring *rx_ring, struct ena_com_rx_ctx *ena_rx_ctx,
- struct mbuf *mbuf)
-{
- struct ena_adapter *adapter = rx_ring->adapter;
-
- if (likely(ENA_FLAG_ISSET(ENA_FLAG_RSS_ACTIVE, adapter))) {
- mbuf->m_pkthdr.flowid = ena_rx_ctx->hash;
-
-#ifdef RSS
- /*
- * Hardware and software RSS are in agreement only when both are
- * configured to Toeplitz algorithm. This driver configures
- * that algorithm only when software RSS is enabled and uses it.
- */
- if (adapter->ena_dev->rss.hash_func != ENA_ADMIN_TOEPLITZ &&
- ena_rx_ctx->l3_proto != ENA_ETH_IO_L3_PROTO_UNKNOWN) {
- M_HASHTYPE_SET(mbuf, M_HASHTYPE_OPAQUE_HASH);
- return;
- }
-#endif
-
- if (ena_rx_ctx->frag &&
- (ena_rx_ctx->l3_proto != ENA_ETH_IO_L3_PROTO_UNKNOWN)) {
- M_HASHTYPE_SET(mbuf, M_HASHTYPE_OPAQUE_HASH);
- return;
- }
-
- switch (ena_rx_ctx->l3_proto) {
- case ENA_ETH_IO_L3_PROTO_IPV4:
- switch (ena_rx_ctx->l4_proto) {
- case ENA_ETH_IO_L4_PROTO_TCP:
- M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_TCP_IPV4);
- break;
- case ENA_ETH_IO_L4_PROTO_UDP:
- M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_UDP_IPV4);
- break;
- default:
- M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_IPV4);
- }
- break;
- case ENA_ETH_IO_L3_PROTO_IPV6:
- switch (ena_rx_ctx->l4_proto) {
- case ENA_ETH_IO_L4_PROTO_TCP:
- M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_TCP_IPV6);
- break;
- case ENA_ETH_IO_L4_PROTO_UDP:
- M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_UDP_IPV6);
- break;
- default:
- M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_IPV6);
- }
- break;
- case ENA_ETH_IO_L3_PROTO_UNKNOWN:
- M_HASHTYPE_SET(mbuf, M_HASHTYPE_NONE);
- break;
- default:
- M_HASHTYPE_SET(mbuf, M_HASHTYPE_OPAQUE_HASH);
- }
- } else {
- mbuf->m_pkthdr.flowid = rx_ring->qid;
- M_HASHTYPE_SET(mbuf, M_HASHTYPE_NONE);
- }
-}
-
-/**
- * ena_rx_mbuf - assemble mbuf from descriptors
- * @rx_ring: ring for which we want to clean packets
- * @ena_bufs: buffer info
- * @ena_rx_ctx: metadata for this packet(s)
- * @next_to_clean: ring pointer, will be updated only upon success
- *
- **/
-static struct mbuf*
-ena_rx_mbuf(struct ena_ring *rx_ring, struct ena_com_rx_buf_info *ena_bufs,
- struct ena_com_rx_ctx *ena_rx_ctx, uint16_t *next_to_clean)
-{
- struct mbuf *mbuf;
- struct ena_rx_buffer *rx_info;
- struct ena_adapter *adapter;
- unsigned int descs = ena_rx_ctx->descs;
- int rc;
- uint16_t ntc, len, req_id, buf = 0;
-
- ntc = *next_to_clean;
- adapter = rx_ring->adapter;
-
- len = ena_bufs[buf].len;
- req_id = ena_bufs[buf].req_id;
- rc = validate_rx_req_id(rx_ring, req_id);
- if (unlikely(rc != 0))
- return (NULL);
-
- rx_info = &rx_ring->rx_buffer_info[req_id];
- if (unlikely(rx_info->mbuf == NULL)) {
- device_printf(adapter->pdev, "NULL mbuf in rx_info");
- return (NULL);
- }
-
- ena_trace(ENA_DBG | ENA_RXPTH, "rx_info %p, mbuf %p, paddr %jx\n",
- rx_info, rx_info->mbuf, (uintmax_t)rx_info->ena_buf.paddr);
-
- bus_dmamap_sync(adapter->rx_buf_tag, rx_info->map,
- BUS_DMASYNC_POSTREAD);
- mbuf = rx_info->mbuf;
- mbuf->m_flags |= M_PKTHDR;
- mbuf->m_pkthdr.len = len;
- mbuf->m_len = len;
- mbuf->m_pkthdr.rcvif = rx_ring->que->adapter->ifp;
-
- /* Fill mbuf with hash key and it's interpretation for optimization */
- ena_rx_hash_mbuf(rx_ring, ena_rx_ctx, mbuf);
-
- ena_trace(ENA_DBG | ENA_RXPTH, "rx mbuf 0x%p, flags=0x%x, len: %d\n",
- mbuf, mbuf->m_flags, mbuf->m_pkthdr.len);
-
- /* DMA address is not needed anymore, unmap it */
- bus_dmamap_unload(rx_ring->adapter->rx_buf_tag, rx_info->map);
-
- rx_info->mbuf = NULL;
- rx_ring->free_rx_ids[ntc] = req_id;
- ntc = ENA_RX_RING_IDX_NEXT(ntc, rx_ring->ring_size);
-
- /*
- * While we have more than 1 descriptors for one rcvd packet, append
- * other mbufs to the main one
- */
- while (--descs) {
- ++buf;
- len = ena_bufs[buf].len;
- req_id = ena_bufs[buf].req_id;
- rc = validate_rx_req_id(rx_ring, req_id);
- if (unlikely(rc != 0)) {
- /*
- * If the req_id is invalid, then the device will be
- * reset. In that case we must free all mbufs that
- * were already gathered.
- */
- m_freem(mbuf);
- return (NULL);
- }
- rx_info = &rx_ring->rx_buffer_info[req_id];
-
- if (unlikely(rx_info->mbuf == NULL)) {
- device_printf(adapter->pdev, "NULL mbuf in rx_info");
- /*
- * If one of the required mbufs was not allocated yet,
- * we can break there.
- * All earlier used descriptors will be reallocated
- * later and not used mbufs can be reused.
- * The next_to_clean pointer will not be updated in case
- * of an error, so caller should advance it manually
- * in error handling routine to keep it up to date
- * with hw ring.
- */
- m_freem(mbuf);
- return (NULL);
- }
-
- bus_dmamap_sync(adapter->rx_buf_tag, rx_info->map,
- BUS_DMASYNC_POSTREAD);
- if (unlikely(m_append(mbuf, len, rx_info->mbuf->m_data) == 0)) {
- counter_u64_add(rx_ring->rx_stats.mbuf_alloc_fail, 1);
- ena_trace(ENA_WARNING, "Failed to append Rx mbuf %p\n",
- mbuf);
- }
-
- ena_trace(ENA_DBG | ENA_RXPTH,
- "rx mbuf updated. len %d\n", mbuf->m_pkthdr.len);
-
- /* Free already appended mbuf, it won't be useful anymore */
- bus_dmamap_unload(rx_ring->adapter->rx_buf_tag, rx_info->map);
- m_freem(rx_info->mbuf);
- rx_info->mbuf = NULL;
-
- rx_ring->free_rx_ids[ntc] = req_id;
- ntc = ENA_RX_RING_IDX_NEXT(ntc, rx_ring->ring_size);
- }
-
- *next_to_clean = ntc;
-
- return (mbuf);
-}
-
-/**
- * ena_rx_checksum - indicate in mbuf if hw indicated a good cksum
- **/
-static inline void
-ena_rx_checksum(struct ena_ring *rx_ring, struct ena_com_rx_ctx *ena_rx_ctx,
- struct mbuf *mbuf)
-{
-
- /* if IP and error */
- if (unlikely((ena_rx_ctx->l3_proto == ENA_ETH_IO_L3_PROTO_IPV4) &&
- ena_rx_ctx->l3_csum_err)) {
- /* ipv4 checksum error */
- mbuf->m_pkthdr.csum_flags = 0;
- counter_u64_add(rx_ring->rx_stats.bad_csum, 1);
- ena_trace(ENA_DBG, "RX IPv4 header checksum error\n");
- return;
- }
-
- /* if TCP/UDP */
- if ((ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_TCP) ||
- (ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_UDP)) {
- if (ena_rx_ctx->l4_csum_err) {
- /* TCP/UDP checksum error */
- mbuf->m_pkthdr.csum_flags = 0;
- counter_u64_add(rx_ring->rx_stats.bad_csum, 1);
- ena_trace(ENA_DBG, "RX L4 checksum error\n");
- } else {
- mbuf->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
- mbuf->m_pkthdr.csum_flags |= CSUM_IP_VALID;
- }
- }
-}
-
-/**
- * ena_rx_cleanup - handle rx irq
- * @arg: ring for which irq is being handled
- **/
-static int
-ena_rx_cleanup(struct ena_ring *rx_ring)
-{
- struct ena_adapter *adapter;
- struct mbuf *mbuf;
- struct ena_com_rx_ctx ena_rx_ctx;
- struct ena_com_io_cq* io_cq;
- struct ena_com_io_sq* io_sq;
- if_t ifp;
- uint16_t ena_qid;
- uint16_t next_to_clean;
- uint32_t refill_required;
- uint32_t refill_threshold;
- uint32_t do_if_input = 0;
- unsigned int qid;
- int rc, i;
- int budget = RX_BUDGET;
-
- adapter = rx_ring->que->adapter;
- ifp = adapter->ifp;
- qid = rx_ring->que->id;
- ena_qid = ENA_IO_RXQ_IDX(qid);
- io_cq = &adapter->ena_dev->io_cq_queues[ena_qid];
- io_sq = &adapter->ena_dev->io_sq_queues[ena_qid];
- next_to_clean = rx_ring->next_to_clean;
-
- ena_trace(ENA_DBG, "rx: qid %d\n", qid);
-
- do {
- ena_rx_ctx.ena_bufs = rx_ring->ena_bufs;
- ena_rx_ctx.max_bufs = adapter->max_rx_sgl_size;
- ena_rx_ctx.descs = 0;
- bus_dmamap_sync(io_cq->cdesc_addr.mem_handle.tag,
- io_cq->cdesc_addr.mem_handle.map, BUS_DMASYNC_POSTREAD);
- rc = ena_com_rx_pkt(io_cq, io_sq, &ena_rx_ctx);
-
- if (unlikely(rc != 0))
- goto error;
-
- if (unlikely(ena_rx_ctx.descs == 0))
- break;
-
- ena_trace(ENA_DBG | ENA_RXPTH, "rx: q %d got packet from ena. "
- "descs #: %d l3 proto %d l4 proto %d hash: %x\n",
- rx_ring->qid, ena_rx_ctx.descs, ena_rx_ctx.l3_proto,
- ena_rx_ctx.l4_proto, ena_rx_ctx.hash);
-
- /* Receive mbuf from the ring */
- mbuf = ena_rx_mbuf(rx_ring, rx_ring->ena_bufs,
- &ena_rx_ctx, &next_to_clean);
- bus_dmamap_sync(io_cq->cdesc_addr.mem_handle.tag,
- io_cq->cdesc_addr.mem_handle.map, BUS_DMASYNC_PREREAD);
- /* Exit if we failed to retrieve a buffer */
- if (unlikely(mbuf == NULL)) {
- for (i = 0; i < ena_rx_ctx.descs; ++i) {
- rx_ring->free_rx_ids[next_to_clean] =
- rx_ring->ena_bufs[i].req_id;
- next_to_clean =
- ENA_RX_RING_IDX_NEXT(next_to_clean,
- rx_ring->ring_size);
-
- }
- break;
- }
-
- if (((ifp->if_capenable & IFCAP_RXCSUM) != 0) ||
- ((ifp->if_capenable & IFCAP_RXCSUM_IPV6) != 0)) {
- ena_rx_checksum(rx_ring, &ena_rx_ctx, mbuf);
- }
-
- counter_enter();
- counter_u64_add_protected(rx_ring->rx_stats.bytes,
- mbuf->m_pkthdr.len);
- counter_u64_add_protected(adapter->hw_stats.rx_bytes,
- mbuf->m_pkthdr.len);
- counter_exit();
- /*
- * LRO is only for IP/TCP packets and TCP checksum of the packet
- * should be computed by hardware.
- */
- do_if_input = 1;
- if (((ifp->if_capenable & IFCAP_LRO) != 0) &&
- ((mbuf->m_pkthdr.csum_flags & CSUM_IP_VALID) != 0) &&
- (ena_rx_ctx.l4_proto == ENA_ETH_IO_L4_PROTO_TCP)) {
- /*
- * Send to the stack if:
- * - LRO not enabled, or
- * - no LRO resources, or
- * - lro enqueue fails
- */
- if ((rx_ring->lro.lro_cnt != 0) &&
- (tcp_lro_rx(&rx_ring->lro, mbuf, 0) == 0))
- do_if_input = 0;
- }
- if (do_if_input != 0) {
- ena_trace(ENA_DBG | ENA_RXPTH,
- "calling if_input() with mbuf %p\n", mbuf);
- (*ifp->if_input)(ifp, mbuf);
- }
-
- counter_enter();
- counter_u64_add_protected(rx_ring->rx_stats.cnt, 1);
- counter_u64_add_protected(adapter->hw_stats.rx_packets, 1);
- counter_exit();
- } while (--budget);
-
- rx_ring->next_to_clean = next_to_clean;
-
- refill_required = ena_com_free_desc(io_sq);
- refill_threshold = min_t(int,
- rx_ring->ring_size / ENA_RX_REFILL_THRESH_DIVIDER,
- ENA_RX_REFILL_THRESH_PACKET);
-
- if (refill_required > refill_threshold) {
- ena_com_update_dev_comp_head(rx_ring->ena_com_io_cq);
- ena_refill_rx_bufs(rx_ring, refill_required);
- }
-
- tcp_lro_flush_all(&rx_ring->lro);
-
- return (RX_BUDGET - budget);
-
-error:
- counter_u64_add(rx_ring->rx_stats.bad_desc_num, 1);
-
- /* Too many desc from the device. Trigger reset */
- if (likely(!ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter))) {
- adapter->reset_reason = ENA_REGS_RESET_TOO_MANY_RX_DESCS;
- ENA_FLAG_SET_ATOMIC(ENA_FLAG_TRIGGER_RESET, adapter);
- }
-
- return (0);
-}
-
/*********************************************************************
*
* MSIX & Interrupt Service routine
@@ -1853,52 +1402,6 @@ ena_intr_msix_mgmnt(void *arg)
ena_com_aenq_intr_handler(adapter->ena_dev, arg);
}
-static void
-ena_cleanup(void *arg, int pending)
-{
- struct ena_que *que = arg;
- struct ena_adapter *adapter = que->adapter;
- if_t ifp = adapter->ifp;
- struct ena_ring *tx_ring;
- struct ena_ring *rx_ring;
- struct ena_com_io_cq* io_cq;
- struct ena_eth_io_intr_reg intr_reg;
- int qid, ena_qid;
- int txc, rxc, i;
-
- if (unlikely((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0))
- return;
-
- ena_trace(ENA_DBG, "MSI-X TX/RX routine\n");
-
- tx_ring = que->tx_ring;
- rx_ring = que->rx_ring;
- qid = que->id;
- ena_qid = ENA_IO_TXQ_IDX(qid);
- io_cq = &adapter->ena_dev->io_cq_queues[ena_qid];
-
- tx_ring->first_interrupt = true;
- rx_ring->first_interrupt = true;
-
- for (i = 0; i < CLEAN_BUDGET; ++i) {
- rxc = ena_rx_cleanup(rx_ring);
- txc = ena_tx_cleanup(tx_ring);
-
- if (unlikely((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0))
- return;
-
- if ((txc != TX_BUDGET) && (rxc != RX_BUDGET))
- break;
- }
-
- /* Signal that work is done and unmask interrupt */
- ena_com_update_intr_reg(&intr_reg,
- RX_IRQ_INTERVAL,
- TX_IRQ_INTERVAL,
- true);
- ena_com_unmask_intr(io_cq, &intr_reg);
-}
-
/**
* ena_handle_msix - MSIX Interrupt Handler for Tx/Rx
* @arg: queue
@@ -2309,7 +1812,7 @@ ena_up_complete(struct ena_adapter *adapter)
return (0);
}
-static int
+int
ena_up(struct ena_adapter *adapter)
{
int rc = 0;
@@ -2370,8 +1873,14 @@ ena_up(struct ena_adapter *adapter)
if_setdrvflagbits(adapter->ifp, IFF_DRV_RUNNING,
IFF_DRV_OACTIVE);
- callout_reset_sbt(&adapter->timer_service, SBT_1S, SBT_1S,
- ena_timer_service, (void *)adapter, 0);
+ /* Activate timer service only if the device is running.
+ * If this flag is not set, it means that the driver is being
+ * reset and timer service will be activated afterwards.
+ */
+ if (ENA_FLAG_ISSET(ENA_FLAG_DEVICE_RUNNING, adapter)) {
+ callout_reset_sbt(&adapter->timer_service, SBT_1S,
+ SBT_1S, ena_timer_service, (void *)adapter, 0);
+ }
ENA_FLAG_SET_ATOMIC(ENA_FLAG_DEV_UP, adapter);
@@ -2680,7 +2189,7 @@ ena_setup_ifnet(device_t pdev, struct ena_adapter *ada
return (0);
}
-static void
+void
ena_down(struct ena_adapter *adapter)
{
int rc;
@@ -2715,579 +2224,7 @@ ena_down(struct ena_adapter *adapter)
}
}
-static void
-ena_tx_csum(struct ena_com_tx_ctx *ena_tx_ctx, struct mbuf *mbuf)
-{
- struct ena_com_tx_meta *ena_meta;
- struct ether_vlan_header *eh;
- struct mbuf *mbuf_next;
- u32 mss;
- bool offload;
- uint16_t etype;
- int ehdrlen;
- struct ip *ip;
- int iphlen;
- struct tcphdr *th;
- int offset;
-
- offload = false;
- ena_meta = &ena_tx_ctx->ena_meta;
- mss = mbuf->m_pkthdr.tso_segsz;
-
- if (mss != 0)
- offload = true;
-
- if ((mbuf->m_pkthdr.csum_flags & CSUM_TSO) != 0)
- offload = true;
-
- if ((mbuf->m_pkthdr.csum_flags & CSUM_OFFLOAD) != 0)
- offload = true;
-
- if (!offload) {
- ena_tx_ctx->meta_valid = 0;
- return;
- }
-
- /* Determine where frame payload starts. */
- eh = mtod(mbuf, struct ether_vlan_header *);
*** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
More information about the svn-src-all
mailing list