svn commit: r315217 - in head/sys: dev/bnxt dev/e1000 net
Sean Bruno
sbruno at FreeBSD.org
Mon Mar 13 22:53:08 UTC 2017
Author: sbruno
Date: Mon Mar 13 22:53:06 2017
New Revision: 315217
URL: https://svnweb.freebsd.org/changeset/base/315217
Log:
IFLIB updates
- unconditionally enable BUS_DMA on non-x86 architectures
- speed up rxd zeroing via customized function
- support out of order updates to rxd's
- add prefetching to hardware descriptor rings
- only prefetch on 10G or faster hardware
- add seperate tx queue intr function
- preliminary rework of NETMAP interfaces, WIP
Submitted by: Matt Macy <mmacy at nextbsd.org>
Sponsored by: Limelight Networks
Modified:
head/sys/dev/bnxt/bnxt_txrx.c
head/sys/dev/bnxt/if_bnxt.c
head/sys/dev/e1000/em_txrx.c
head/sys/dev/e1000/if_em.c
head/sys/dev/e1000/if_em.h
head/sys/dev/e1000/igb_txrx.c
head/sys/net/ifdi_if.m
head/sys/net/iflib.c
head/sys/net/iflib.h
Modified: head/sys/dev/bnxt/bnxt_txrx.c
==============================================================================
--- head/sys/dev/bnxt/bnxt_txrx.c Mon Mar 13 22:45:51 2017 (r315216)
+++ head/sys/dev/bnxt/bnxt_txrx.c Mon Mar 13 22:53:06 2017 (r315217)
@@ -48,17 +48,19 @@ __FBSDID("$FreeBSD$");
*/
static int bnxt_isc_txd_encap(void *sc, if_pkt_info_t pi);
-static void bnxt_isc_txd_flush(void *sc, uint16_t txqid, uint32_t pidx);
-static int bnxt_isc_txd_credits_update(void *sc, uint16_t txqid, uint32_t cidx,
- bool clear);
+static void bnxt_isc_txd_flush(void *sc, uint16_t txqid, qidx_t pidx);
+static int bnxt_isc_txd_credits_update(void *sc, uint16_t txqid, bool clear);
-static void bnxt_isc_rxd_refill(void *sc, uint16_t rxqid, uint8_t flid,
+static void bnxt_isc_rxd_refill(void *sc, if_rxd_update_t iru);
+
+/* uint16_t rxqid, uint8_t flid,
uint32_t pidx, uint64_t *paddrs, caddr_t *vaddrs, uint16_t count,
uint16_t buf_size);
+*/
static void bnxt_isc_rxd_flush(void *sc, uint16_t rxqid, uint8_t flid,
- uint32_t pidx);
-static int bnxt_isc_rxd_available(void *sc, uint16_t rxqid, uint32_t idx,
- int budget);
+ qidx_t pidx);
+static int bnxt_isc_rxd_available(void *sc, uint16_t rxqid, qidx_t idx,
+ qidx_t budget);
static int bnxt_isc_rxd_pkt_get(void *sc, if_rxd_info_t ri);
static int bnxt_intr(void *sc);
@@ -172,7 +174,7 @@ bnxt_isc_txd_encap(void *sc, if_pkt_info
}
static void
-bnxt_isc_txd_flush(void *sc, uint16_t txqid, uint32_t pidx)
+bnxt_isc_txd_flush(void *sc, uint16_t txqid, qidx_t pidx)
{
struct bnxt_softc *softc = (struct bnxt_softc *)sc;
struct bnxt_ring *tx_ring = &softc->tx_rings[txqid];
@@ -185,7 +187,7 @@ bnxt_isc_txd_flush(void *sc, uint16_t tx
}
static int
-bnxt_isc_txd_credits_update(void *sc, uint16_t txqid, uint32_t idx, bool clear)
+bnxt_isc_txd_credits_update(void *sc, uint16_t txqid, bool clear)
{
struct bnxt_softc *softc = (struct bnxt_softc *)sc;
struct bnxt_cp_ring *cpr = &softc->tx_cp_rings[txqid];
@@ -249,15 +251,27 @@ done:
}
static void
-bnxt_isc_rxd_refill(void *sc, uint16_t rxqid, uint8_t flid,
- uint32_t pidx, uint64_t *paddrs,
- caddr_t *vaddrs, uint16_t count, uint16_t len)
+bnxt_isc_rxd_refill(void *sc, if_rxd_update_t iru)
{
struct bnxt_softc *softc = (struct bnxt_softc *)sc;
struct bnxt_ring *rx_ring;
struct rx_prod_pkt_bd *rxbd;
uint16_t type;
uint16_t i;
+ uint16_t rxqid;
+ uint16_t count, len;
+ uint32_t pidx;
+ uint8_t flid;
+ uint64_t *paddrs;
+ caddr_t *vaddrs;
+
+ rxqid = iru->iru_qsidx;
+ count = iru->iru_count;
+ len = iru->iru_buf_size;
+ pidx = iru->iru_pidx;
+ flid = iru->iru_flidx;
+ vaddrs = iru->iru_vaddrs;
+ paddrs = iru->iru_paddrs;
if (flid == 0) {
rx_ring = &softc->rx_rings[rxqid];
@@ -284,7 +298,7 @@ bnxt_isc_rxd_refill(void *sc, uint16_t r
static void
bnxt_isc_rxd_flush(void *sc, uint16_t rxqid, uint8_t flid,
- uint32_t pidx)
+ qidx_t pidx)
{
struct bnxt_softc *softc = (struct bnxt_softc *)sc;
struct bnxt_ring *rx_ring;
@@ -310,7 +324,7 @@ bnxt_isc_rxd_flush(void *sc, uint16_t rx
}
static int
-bnxt_isc_rxd_available(void *sc, uint16_t rxqid, uint32_t idx, int budget)
+bnxt_isc_rxd_available(void *sc, uint16_t rxqid, qidx_t idx, qidx_t budget)
{
struct bnxt_softc *softc = (struct bnxt_softc *)sc;
struct bnxt_cp_ring *cpr = &softc->rx_cp_rings[rxqid];
@@ -412,37 +426,6 @@ cmpl_invalid:
return avail;
}
-static void
-bnxt_set_rsstype(if_rxd_info_t ri, uint8_t rss_hash_type)
-{
- uint8_t rss_profile_id;
-
- rss_profile_id = BNXT_GET_RSS_PROFILE_ID(rss_hash_type);
- switch (rss_profile_id) {
- case BNXT_RSS_HASH_TYPE_TCPV4:
- ri->iri_rsstype = M_HASHTYPE_RSS_TCP_IPV4;
- break;
- case BNXT_RSS_HASH_TYPE_UDPV4:
- ri->iri_rsstype = M_HASHTYPE_RSS_UDP_IPV4;
- break;
- case BNXT_RSS_HASH_TYPE_IPV4:
- ri->iri_rsstype = M_HASHTYPE_RSS_IPV4;
- break;
- case BNXT_RSS_HASH_TYPE_TCPV6:
- ri->iri_rsstype = M_HASHTYPE_RSS_TCP_IPV6;
- break;
- case BNXT_RSS_HASH_TYPE_UDPV6:
- ri->iri_rsstype = M_HASHTYPE_RSS_UDP_IPV6;
- break;
- case BNXT_RSS_HASH_TYPE_IPV6:
- ri->iri_rsstype = M_HASHTYPE_RSS_IPV6;
- break;
- default:
- ri->iri_rsstype = M_HASHTYPE_OPAQUE;
- break;
- }
-}
-
static int
bnxt_pkt_get_l2(struct bnxt_softc *softc, if_rxd_info_t ri,
struct bnxt_cp_ring *cpr, uint16_t flags_type)
@@ -460,7 +443,13 @@ bnxt_pkt_get_l2(struct bnxt_softc *softc
/* Extract from the first 16-byte BD */
if (flags_type & RX_PKT_CMPL_FLAGS_RSS_VALID) {
ri->iri_flowid = le32toh(rcp->rss_hash);
- bnxt_set_rsstype(ri, rcp->rss_hash_type);
+ /*
+ * TODO: Extract something useful from rcp->rss_hash_type
+ * (undocumented)
+ * May be documented in the "LSI ES"
+ * also check the firmware code.
+ */
+ ri->iri_rsstype = M_HASHTYPE_OPAQUE;
}
else {
ri->iri_rsstype = M_HASHTYPE_NONE;
@@ -540,7 +529,13 @@ bnxt_pkt_get_tpa(struct bnxt_softc *soft
/* Extract from the first 16-byte BD */
if (le16toh(tpas->low.flags_type) & RX_TPA_START_CMPL_FLAGS_RSS_VALID) {
ri->iri_flowid = le32toh(tpas->low.rss_hash);
- bnxt_set_rsstype(ri, tpas->low.rss_hash_type);
+ /*
+ * TODO: Extract something useful from tpas->low.rss_hash_type
+ * (undocumented)
+ * May be documented in the "LSI ES"
+ * also check the firmware code.
+ */
+ ri->iri_rsstype = M_HASHTYPE_OPAQUE;
}
else {
ri->iri_rsstype = M_HASHTYPE_NONE;
Modified: head/sys/dev/bnxt/if_bnxt.c
==============================================================================
--- head/sys/dev/bnxt/if_bnxt.c Mon Mar 13 22:45:51 2017 (r315216)
+++ head/sys/dev/bnxt/if_bnxt.c Mon Mar 13 22:53:06 2017 (r315217)
@@ -253,7 +253,8 @@ static device_method_t bnxt_iflib_method
DEVMETHOD(ifdi_update_admin_status, bnxt_update_admin_status),
DEVMETHOD(ifdi_intr_enable, bnxt_intr_enable),
- DEVMETHOD(ifdi_queue_intr_enable, bnxt_queue_intr_enable),
+ DEVMETHOD(ifdi_tx_queue_intr_enable, bnxt_queue_intr_enable),
+ DEVMETHOD(ifdi_rx_queue_intr_enable, bnxt_queue_intr_enable),
DEVMETHOD(ifdi_intr_disable, bnxt_disable_intr),
DEVMETHOD(ifdi_msix_intr_assign, bnxt_msix_intr_assign),
@@ -711,6 +712,8 @@ bnxt_attach_pre(if_ctx_t ctx)
scctx->isc_tx_tso_size_max = BNXT_TSO_SIZE;
scctx->isc_tx_tso_segsize_max = BNXT_TSO_SIZE;
scctx->isc_vectors = softc->func.max_cp_rings;
+ scctx->isc_txrx = &bnxt_txrx;
+
if (scctx->isc_nrxd[0] <
((scctx->isc_nrxd[1] * 4) + scctx->isc_nrxd[2]))
device_printf(softc->dev,
@@ -1479,7 +1482,7 @@ bnxt_msix_intr_assign(if_ctx_t ctx, int
for (i=0; i<softc->scctx->isc_nrxqsets; i++) {
rc = iflib_irq_alloc_generic(ctx, &softc->rx_cp_rings[i].irq,
- softc->rx_cp_rings[i].ring.id + 1, IFLIB_INTR_RX,
+ softc->rx_cp_rings[i].ring.id + 1, IFLIB_INTR_RXTX,
bnxt_handle_rx_cp, &softc->rx_cp_rings[i], i, "rx_cp");
if (rc) {
device_printf(iflib_get_dev(ctx),
Modified: head/sys/dev/e1000/em_txrx.c
==============================================================================
--- head/sys/dev/e1000/em_txrx.c Mon Mar 13 22:45:51 2017 (r315216)
+++ head/sys/dev/e1000/em_txrx.c Mon Mar 13 22:53:06 2017 (r315217)
@@ -1,5 +1,5 @@
/*-
- * Copyright (c) 2016 Matt Macy <mmacy at nextbsd.org>
+ * Copyright (c) 2016-2017 Matt Macy <mmacy at nextbsd.org>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@@ -27,7 +27,7 @@
/* $FreeBSD$ */
#include "if_em.h"
-#ifdef RSS
+#ifdef RSS
#include <net/rss_config.h>
#include <netinet/in_rss.h>
#endif
@@ -41,23 +41,24 @@
/*********************************************************************
* Local Function prototypes
*********************************************************************/
-static int em_tso_setup(struct adapter *adapter, if_pkt_info_t pi, u32 *txd_upper, u32 *txd_lower);
-static int em_transmit_checksum_setup(struct adapter *adapter, if_pkt_info_t pi, u32 *txd_upper, u32 *txd_lower);
+static int em_tso_setup(struct adapter *adapter, if_pkt_info_t pi, u32 *txd_upper,
+ u32 *txd_lower);
+static int em_transmit_checksum_setup(struct adapter *adapter, if_pkt_info_t pi,
+ u32 *txd_upper, u32 *txd_lower);
static int em_isc_txd_encap(void *arg, if_pkt_info_t pi);
-static void em_isc_txd_flush(void *arg, uint16_t txqid, uint32_t pidx);
-static int em_isc_txd_credits_update(void *arg, uint16_t txqid, uint32_t cidx_init, bool clear);
-static void em_isc_rxd_refill(void *arg, uint16_t rxqid, uint8_t flid __unused,
- uint32_t pidx, uint64_t *paddrs, caddr_t *vaddrs __unused, uint16_t count, uint16_t buflen __unused);
-static void em_isc_rxd_flush(void *arg, uint16_t rxqid, uint8_t flid __unused, uint32_t pidx);
-static int em_isc_rxd_available(void *arg, uint16_t rxqid, uint32_t idx,
- int budget);
+static void em_isc_txd_flush(void *arg, uint16_t txqid, qidx_t pidx);
+static int em_isc_txd_credits_update(void *arg, uint16_t txqid, bool clear);
+static void em_isc_rxd_refill(void *arg, if_rxd_update_t iru);
+static void em_isc_rxd_flush(void *arg, uint16_t rxqid, uint8_t flid __unused,
+ qidx_t pidx);
+static int em_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx,
+ qidx_t budget);
static int em_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri);
-static void lem_isc_rxd_refill(void *arg, uint16_t rxqid, uint8_t flid __unused,
- uint32_t pidx, uint64_t *paddrs, caddr_t *vaddrs __unused, uint16_t count, uint16_t buflen __unused);
+static void lem_isc_rxd_refill(void *arg, if_rxd_update_t iru);
-static int lem_isc_rxd_available(void *arg, uint16_t rxqid, uint32_t idx,
- int budget);
+static int lem_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx,
+ qidx_t budget);
static int lem_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri);
static void lem_receive_checksum(int status, int errors, if_rxd_info_t ri);
@@ -65,7 +66,7 @@ static void em_receive_checksum(uint32_t
static int em_determine_rsstype(u32 pkt_info);
extern int em_intr(void *arg);
-struct if_txrx em_txrx = {
+struct if_txrx em_txrx = {
em_isc_txd_encap,
em_isc_txd_flush,
em_isc_txd_credits_update,
@@ -76,7 +77,7 @@ struct if_txrx em_txrx = {
em_intr
};
-struct if_txrx lem_txrx = {
+struct if_txrx lem_txrx = {
em_isc_txd_encap,
em_isc_txd_flush,
em_isc_txd_credits_update,
@@ -87,7 +88,42 @@ struct if_txrx lem_txrx = {
em_intr
};
-extern if_shared_ctx_t em_sctx;
+extern if_shared_ctx_t em_sctx;
+
+void
+em_dump_rs(struct adapter *adapter)
+{
+ if_softc_ctx_t scctx = adapter->shared;
+ struct em_tx_queue *que;
+ struct tx_ring *txr;
+ qidx_t i, ntxd, qid, cur;
+ int16_t rs_cidx;
+ uint8_t status;
+
+ printf("\n");
+ ntxd = scctx->isc_ntxd[0];
+ for (qid = 0; qid < adapter->tx_num_queues; qid++) {
+ que = &adapter->tx_queues[qid];
+ txr = &que->txr;
+ rs_cidx = txr->tx_rs_cidx;
+ if (rs_cidx != txr->tx_rs_pidx) {
+ cur = txr->tx_rsq[rs_cidx];
+ status = txr->tx_base[cur].upper.fields.status;
+ if (!(status & E1000_TXD_STAT_DD))
+ printf("qid[%d]->tx_rsq[%d]: %d clear ", qid, rs_cidx, cur);
+ } else {
+ rs_cidx = (rs_cidx-1)&(ntxd-1);
+ cur = txr->tx_rsq[rs_cidx];
+ printf("qid[%d]->tx_rsq[rs_cidx-1=%d]: %d ", qid, rs_cidx, cur);
+ }
+ printf("cidx_prev=%d rs_pidx=%d ",txr->tx_cidx_processed, txr->tx_rs_pidx);
+ for (i = 0; i < ntxd; i++) {
+ if (txr->tx_base[i].upper.fields.status & E1000_TXD_STAT_DD)
+ printf("%d set ", i);
+ }
+ printf("\n");
+ }
+}
/**********************************************************************
*
@@ -99,14 +135,13 @@ static int
em_tso_setup(struct adapter *adapter, if_pkt_info_t pi, u32 *txd_upper, u32 *txd_lower)
{
if_softc_ctx_t scctx = adapter->shared;
- struct em_tx_queue *que = &adapter->tx_queues[pi->ipi_qsidx];
- struct tx_ring *txr = &que->txr;
+ struct em_tx_queue *que = &adapter->tx_queues[pi->ipi_qsidx];
+ struct tx_ring *txr = &que->txr;
struct e1000_context_desc *TXD;
- struct em_txbuffer *tx_buffer;
- int cur, hdr_len;
+ int cur, hdr_len;
hdr_len = pi->ipi_ehdrlen + pi->ipi_ip_hlen + pi->ipi_tcp_hlen;
- *txd_lower = (E1000_TXD_CMD_DEXT | /* Extended descr type */
+ *txd_lower = (E1000_TXD_CMD_DEXT | /* Extended descr type */
E1000_TXD_DTYP_D | /* Data descr type */
E1000_TXD_CMD_TSE); /* Do TSE on this packet */
@@ -114,10 +149,9 @@ em_tso_setup(struct adapter *adapter, if
*txd_upper = (E1000_TXD_POPTS_IXSM | E1000_TXD_POPTS_TXSM) << 8;
cur = pi->ipi_pidx;
- TXD = (struct e1000_context_desc *)&txr->tx_base[cur];
- tx_buffer = &txr->tx_buffers[cur];
-
- /*
+ TXD = (struct e1000_context_desc *)&txr->tx_base[cur];
+
+ /*
* Start offset for header checksum calculation.
* End offset for header checksum calculation.
* Offset of place put the checksum.
@@ -127,7 +161,7 @@ em_tso_setup(struct adapter *adapter, if
htole16(pi->ipi_ehdrlen + pi->ipi_ip_hlen - 1);
TXD->lower_setup.ip_fields.ipcso = pi->ipi_ehdrlen + offsetof(struct ip, ip_sum);
- /*
+ /*
* Start offset for payload checksum calculation.
* End offset for payload checksum calculation.
* Offset of place to put the checksum.
@@ -136,8 +170,8 @@ em_tso_setup(struct adapter *adapter, if
TXD->upper_setup.tcp_fields.tucse = 0;
TXD->upper_setup.tcp_fields.tucso =
pi->ipi_ehdrlen + pi->ipi_ip_hlen + offsetof(struct tcphdr, th_sum);
-
- /*
+
+ /*
* Payload size per packet w/o any headers.
* Length of all headers up to payload.
*/
@@ -150,7 +184,6 @@ em_tso_setup(struct adapter *adapter, if
E1000_TXD_CMD_IP | /* Do IP csum */
E1000_TXD_CMD_TCP | /* Do TCP checksum */
(pi->ipi_len - hdr_len)); /* Total len */
- tx_buffer->eop = -1;
txr->tx_tso = TRUE;
if (++cur == scctx->isc_ntxd[0]) {
@@ -178,21 +211,20 @@ em_tso_setup(struct adapter *adapter, if
* until the previous request completes. This means setting up
* a new context effectively disables pipelined Tx data DMA which
* in turn greatly slow down performance to send small sized
- * frames.
+ * frames.
**********************************************************************/
static int
em_transmit_checksum_setup(struct adapter *adapter, if_pkt_info_t pi, u32 *txd_upper, u32 *txd_lower)
{
- struct e1000_context_desc *TXD = NULL;
- if_softc_ctx_t scctx = adapter->shared;
- struct em_tx_queue *que = &adapter->tx_queues[pi->ipi_qsidx];
- struct tx_ring *txr = &que->txr;
- struct em_txbuffer *tx_buffer;
- int csum_flags = pi->ipi_csum_flags;
- int cur, hdr_len;
- u32 cmd;
-
+ struct e1000_context_desc *TXD = NULL;
+ if_softc_ctx_t scctx = adapter->shared;
+ struct em_tx_queue *que = &adapter->tx_queues[pi->ipi_qsidx];
+ struct tx_ring *txr = &que->txr;
+ int csum_flags = pi->ipi_csum_flags;
+ int cur, hdr_len;
+ u32 cmd;
+
cur = pi->ipi_pidx;
hdr_len = pi->ipi_ehdrlen + pi->ipi_ip_hlen;
cmd = adapter->txd_cmd;
@@ -220,7 +252,7 @@ em_transmit_checksum_setup(struct adapte
TXD = (struct e1000_context_desc *)&txr->tx_base[cur];
if (csum_flags & CSUM_IP) {
- *txd_upper |= E1000_TXD_POPTS_IXSM << 8;
+ *txd_upper |= E1000_TXD_POPTS_IXSM << 8;
/*
* Start offset for header checksum calculation.
* End offset for header checksum calculation.
@@ -235,7 +267,7 @@ em_transmit_checksum_setup(struct adapte
if (csum_flags & (CSUM_TCP|CSUM_UDP)) {
uint8_t tucso;
- *txd_upper |= E1000_TXD_POPTS_TXSM << 8;
+ *txd_upper |= E1000_TXD_POPTS_TXSM << 8;
*txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D;
if (csum_flags & CSUM_TCP) {
@@ -243,9 +275,9 @@ em_transmit_checksum_setup(struct adapte
cmd |= E1000_TXD_CMD_TCP;
} else
tucso = hdr_len + offsetof(struct udphdr, uh_sum);
- TXD->upper_setup.tcp_fields.tucss = hdr_len;
- TXD->upper_setup.tcp_fields.tucse = htole16(0);
- TXD->upper_setup.tcp_fields.tucso = tucso;
+ TXD->upper_setup.tcp_fields.tucss = hdr_len;
+ TXD->upper_setup.tcp_fields.tucse = htole16(0);
+ TXD->upper_setup.tcp_fields.tucso = tucso;
}
txr->csum_lhlen = pi->ipi_ehdrlen;
@@ -258,9 +290,6 @@ em_transmit_checksum_setup(struct adapte
TXD->cmd_and_length =
htole32(E1000_TXD_CMD_IFCS | E1000_TXD_CMD_DEXT | cmd);
- tx_buffer = &txr->tx_buffers[cur];
- tx_buffer->eop = -1;
-
if (++cur == scctx->isc_ntxd[0]) {
cur = 0;
}
@@ -272,24 +301,26 @@ em_transmit_checksum_setup(struct adapte
static int
em_isc_txd_encap(void *arg, if_pkt_info_t pi)
{
- struct adapter *sc = arg;
- if_softc_ctx_t scctx = sc->shared;
- struct em_tx_queue *que = &sc->tx_queues[pi->ipi_qsidx];
- struct tx_ring *txr = &que->txr;
- bus_dma_segment_t *segs = pi->ipi_segs;
- int nsegs = pi->ipi_nsegs;
- int csum_flags = pi->ipi_csum_flags;
- int i, j, first, pidx_last;
- u32 txd_upper = 0, txd_lower = 0;
-
- struct em_txbuffer *tx_buffer;
+ struct adapter *sc = arg;
+ if_softc_ctx_t scctx = sc->shared;
+ struct em_tx_queue *que = &sc->tx_queues[pi->ipi_qsidx];
+ struct tx_ring *txr = &que->txr;
+ bus_dma_segment_t *segs = pi->ipi_segs;
+ int nsegs = pi->ipi_nsegs;
+ int csum_flags = pi->ipi_csum_flags;
+ int i, j, first, pidx_last;
+ u32 txd_flags, txd_upper = 0, txd_lower = 0;
+
struct e1000_tx_desc *ctxd = NULL;
- bool do_tso, tso_desc;
-
- i = first = pi->ipi_pidx;
+ bool do_tso, tso_desc;
+ qidx_t ntxd;
+
+ txd_flags = pi->ipi_flags & IPI_TX_INTR ? E1000_TXD_CMD_RS : 0;
+ i = first = pi->ipi_pidx;
do_tso = (csum_flags & CSUM_TSO);
tso_desc = FALSE;
- /*
+ ntxd = scctx->isc_ntxd[0];
+ /*
* TSO Hardware workaround, if this packet is not
* TSO, and is only a single descriptor long, and
* it follows a TSO burst, then we need to add a
@@ -310,10 +341,10 @@ em_isc_txd_encap(void *arg, if_pkt_info_
}
if (pi->ipi_mflags & M_VLANTAG) {
- /* Set the vlan id. */
+ /* Set the vlan id. */
txd_upper |= htole16(pi->ipi_vtag) << 16;
- /* Tell hardware to add tag */
- txd_lower |= htole32(E1000_TXD_CMD_VLE);
+ /* Tell hardware to add tag */
+ txd_lower |= htole32(E1000_TXD_CMD_VLE);
}
DPRINTF(iflib_get_dev(sc->ctx), "encap: set up tx: nsegs=%d first=%d i=%d\n", nsegs, first, i);
@@ -326,28 +357,26 @@ em_isc_txd_encap(void *arg, if_pkt_info_
uint32_t cmd;
ctxd = &txr->tx_base[i];
- tx_buffer = &txr->tx_buffers[i];
seg_addr = segs[j].ds_addr;
seg_len = segs[j].ds_len;
cmd = E1000_TXD_CMD_IFCS | sc->txd_cmd;
/*
- ** TSO Workaround:
- ** If this is the last descriptor, we want to
- ** split it so we have a small final sentinel
- */
+ * TSO Workaround:
+ * If this is the last descriptor, we want to
+ * split it so we have a small final sentinel
+ */
if (tso_desc && (j == (nsegs - 1)) && (seg_len > 8)) {
seg_len -= TSO_WORKAROUND;
ctxd->buffer_addr = htole64(seg_addr);
ctxd->lower.data = htole32(cmd | txd_lower | seg_len);
ctxd->upper.data = htole32(txd_upper);
- if (++i == scctx->isc_ntxd[0])
+ if (++i == scctx->isc_ntxd[0])
i = 0;
/* Now make the sentinel */
ctxd = &txr->tx_base[i];
- tx_buffer = &txr->tx_buffers[i];
ctxd->buffer_addr = htole64(seg_addr + seg_len);
ctxd->lower.data = htole32(cmd | txd_lower | TSO_WORKAROUND);
ctxd->upper.data = htole32(txd_upper);
@@ -364,27 +393,28 @@ em_isc_txd_encap(void *arg, if_pkt_info_
i = 0;
DPRINTF(iflib_get_dev(sc->ctx), "pidx_last=%d i=%d ntxd[0]=%d\n", pidx_last, i, scctx->isc_ntxd[0]);
}
- tx_buffer->eop = -1;
}
/*
- * Last Descriptor of Packet
+ * Last Descriptor of Packet
* needs End Of Packet (EOP)
* and Report Status (RS)
- */
- ctxd->lower.data |=
- htole32(E1000_TXD_CMD_EOP | E1000_TXD_CMD_RS);
-
- tx_buffer = &txr->tx_buffers[first];
- tx_buffer->eop = pidx_last;
+ */
+ if (txd_flags) {
+ txr->tx_rsq[txr->tx_rs_pidx] = pidx_last;
+ DPRINTF(iflib_get_dev(sc->ctx), "setting to RS on %d rs_pidx %d first: %d\n", pidx_last, txr->tx_rs_pidx, first);
+ txr->tx_rs_pidx = (txr->tx_rs_pidx+1) & (ntxd-1);
+ MPASS(txr->tx_rs_pidx != txr->tx_rs_cidx);
+ }
+ ctxd->lower.data |= htole32(E1000_TXD_CMD_EOP | txd_flags);
DPRINTF(iflib_get_dev(sc->ctx), "tx_buffers[%d]->eop = %d ipi_new_pidx=%d\n", first, pidx_last, i);
pi->ipi_new_pidx = i;
- return (0);
+ return (0);
}
static void
-em_isc_txd_flush(void *arg, uint16_t txqid, uint32_t pidx)
+em_isc_txd_flush(void *arg, uint16_t txqid, qidx_t pidx)
{
struct adapter *adapter = arg;
struct em_tx_queue *que = &adapter->tx_queues[txqid];
@@ -394,86 +424,72 @@ em_isc_txd_flush(void *arg, uint16_t txq
}
static int
-em_isc_txd_credits_update(void *arg, uint16_t txqid, uint32_t cidx_init, bool clear)
+em_isc_txd_credits_update(void *arg, uint16_t txqid, bool clear)
{
struct adapter *adapter = arg;
if_softc_ctx_t scctx = adapter->shared;
struct em_tx_queue *que = &adapter->tx_queues[txqid];
struct tx_ring *txr = &que->txr;
- u32 cidx, processed = 0;
- int last, done;
- struct em_txbuffer *buf;
- struct e1000_tx_desc *tx_desc, *eop_desc;
-
- cidx = cidx_init;
- buf = &txr->tx_buffers[cidx];
- tx_desc = &txr->tx_base[cidx];
- last = buf->eop;
- if (last == -1)
- return (processed);
- eop_desc = &txr->tx_base[last];
-
- DPRINTF(iflib_get_dev(adapter->ctx),
- "credits_update: cidx_init=%d clear=%d last=%d\n",
- cidx_init, clear, last);
- /*
- * What this does is get the index of the
- * first descriptor AFTER the EOP of the
- * first packet, that way we can do the
- * simple comparison on the inner while loop.
- */
- if (++last == scctx->isc_ntxd[0])
- last = 0;
- done = last;
-
-
- while (eop_desc->upper.fields.status & E1000_TXD_STAT_DD) {
- /* We clean the range of the packet */
- while (cidx != done) {
- if (clear) {
- tx_desc->upper.data = 0;
- tx_desc->lower.data = 0;
- tx_desc->buffer_addr = 0;
- buf->eop = -1;
- }
- tx_desc++;
- buf++;
- processed++;
-
- /* wrap the ring ? */
- if (++cidx == scctx->isc_ntxd[0]) {
- cidx = 0;
- }
- buf = &txr->tx_buffers[cidx];
- tx_desc = &txr->tx_base[cidx];
- }
- /* See if we can continue to the next packet */
- last = buf->eop;
- if (last == -1)
+ qidx_t processed = 0;
+ int updated;
+ qidx_t cur, prev, ntxd, rs_cidx;
+ int32_t delta;
+ uint8_t status;
+
+ rs_cidx = txr->tx_rs_cidx;
+ if (rs_cidx == txr->tx_rs_pidx)
+ return (0);
+ cur = txr->tx_rsq[rs_cidx];
+ MPASS(cur != QIDX_INVALID);
+ status = txr->tx_base[cur].upper.fields.status;
+ updated = !!(status & E1000_TXD_STAT_DD);
+
+ if (clear == false || updated == 0)
+ return (updated);
+
+ prev = txr->tx_cidx_processed;
+ ntxd = scctx->isc_ntxd[0];
+ do {
+ delta = (int32_t)cur - (int32_t)prev;
+ MPASS(prev == 0 || delta != 0);
+ if (delta < 0)
+ delta += ntxd;
+ DPRINTF(iflib_get_dev(adapter->ctx),
+ "%s: cidx_processed=%u cur=%u clear=%d delta=%d\n",
+ __FUNCTION__, prev, cur, clear, delta);
+
+ processed += delta;
+ prev = cur;
+ rs_cidx = (rs_cidx + 1) & (ntxd-1);
+ if (rs_cidx == txr->tx_rs_pidx)
break;
- eop_desc = &txr->tx_base[last];
- /* Get new done point */
- if (++last == scctx->isc_ntxd[0])
- last = 0;
- done = last;
- }
+ cur = txr->tx_rsq[rs_cidx];
+ MPASS(cur != QIDX_INVALID);
+ status = txr->tx_base[cur].upper.fields.status;
+ } while ((status & E1000_TXD_STAT_DD));
- DPRINTF(iflib_get_dev(adapter->ctx), "Processed %d credits update\n", processed);
+ txr->tx_rs_cidx = rs_cidx;
+ txr->tx_cidx_processed = prev;
return(processed);
}
static void
-lem_isc_rxd_refill(void *arg, uint16_t rxqid, uint8_t flid __unused,
- uint32_t pidx, uint64_t *paddrs, caddr_t *vaddrs __unused, uint16_t count, uint16_t buflen __unused)
+lem_isc_rxd_refill(void *arg, if_rxd_update_t iru)
{
struct adapter *sc = arg;
if_softc_ctx_t scctx = sc->shared;
- struct em_rx_queue *que = &sc->rx_queues[rxqid];
+ struct em_rx_queue *que = &sc->rx_queues[iru->iru_qsidx];
struct rx_ring *rxr = &que->rxr;
struct e1000_rx_desc *rxd;
+ uint64_t *paddrs;
+ uint32_t next_pidx, pidx;
+ uint16_t count;
int i;
- uint32_t next_pidx;
+
+ paddrs = iru->iru_paddrs;
+ pidx = iru->iru_pidx;
+ count = iru->iru_count;
for (i = 0, next_pidx = pidx; i < count; i++) {
rxd = (struct e1000_rx_desc *)&rxr->rx_base[next_pidx];
@@ -487,48 +503,60 @@ lem_isc_rxd_refill(void *arg, uint16_t r
}
static void
-em_isc_rxd_refill(void *arg, uint16_t rxqid, uint8_t flid __unused,
- uint32_t pidx, uint64_t *paddrs, caddr_t *vaddrs __unused, uint16_t count, uint16_t buflen __unused)
+em_isc_rxd_refill(void *arg, if_rxd_update_t iru)
{
struct adapter *sc = arg;
if_softc_ctx_t scctx = sc->shared;
+ uint16_t rxqid = iru->iru_qsidx;
struct em_rx_queue *que = &sc->rx_queues[rxqid];
struct rx_ring *rxr = &que->rxr;
union e1000_rx_desc_extended *rxd;
+ uint64_t *paddrs;
+ uint32_t next_pidx, pidx;
+ uint16_t count;
int i;
- uint32_t next_pidx;
+
+ paddrs = iru->iru_paddrs;
+ pidx = iru->iru_pidx;
+ count = iru->iru_count;
for (i = 0, next_pidx = pidx; i < count; i++) {
rxd = &rxr->rx_base[next_pidx];
rxd->read.buffer_addr = htole64(paddrs[i]);
/* DD bits must be cleared */
- rxd->wb.upper.status_error = 0;
-
+ rxd->wb.upper.status_error = 0;
+
if (++next_pidx == scctx->isc_nrxd[0])
next_pidx = 0;
}
}
static void
-em_isc_rxd_flush(void *arg, uint16_t rxqid, uint8_t flid __unused, uint32_t pidx)
+em_isc_rxd_flush(void *arg, uint16_t rxqid, uint8_t flid __unused, qidx_t pidx)
{
- struct adapter *sc = arg;
- struct em_rx_queue *que = &sc->rx_queues[rxqid];
- struct rx_ring *rxr = &que->rxr;
+ struct adapter *sc = arg;
+ struct em_rx_queue *que = &sc->rx_queues[rxqid];
+ struct rx_ring *rxr = &que->rxr;
- E1000_WRITE_REG(&sc->hw, E1000_RDT(rxr->me), pidx);
+ E1000_WRITE_REG(&sc->hw, E1000_RDT(rxr->me), pidx);
}
static int
-lem_isc_rxd_available(void *arg, uint16_t rxqid, uint32_t idx, int budget)
+lem_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx, qidx_t budget)
{
- struct adapter *sc = arg;
+ struct adapter *sc = arg;
if_softc_ctx_t scctx = sc->shared;
- struct em_rx_queue *que = &sc->rx_queues[rxqid];
- struct rx_ring *rxr = &que->rxr;
+ struct em_rx_queue *que = &sc->rx_queues[rxqid];
+ struct rx_ring *rxr = &que->rxr;
struct e1000_rx_desc *rxd;
- u32 staterr = 0;
- int cnt, i;
+ u32 staterr = 0;
+ int cnt, i;
+
+ if (budget == 1) {
+ rxd = (struct e1000_rx_desc *)&rxr->rx_base[idx];
+ staterr = rxd->status;
+ return (staterr & E1000_RXD_STAT_DD);
+ }
for (cnt = 0, i = idx; cnt < scctx->isc_nrxd[0] && cnt <= budget;) {
rxd = (struct e1000_rx_desc *)&rxr->rx_base[i];
@@ -547,15 +575,21 @@ lem_isc_rxd_available(void *arg, uint16_
}
static int
-em_isc_rxd_available(void *arg, uint16_t rxqid, uint32_t idx, int budget)
+em_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx, qidx_t budget)
{
- struct adapter *sc = arg;
+ struct adapter *sc = arg;
if_softc_ctx_t scctx = sc->shared;
- struct em_rx_queue *que = &sc->rx_queues[rxqid];
- struct rx_ring *rxr = &que->rxr;
+ struct em_rx_queue *que = &sc->rx_queues[rxqid];
+ struct rx_ring *rxr = &que->rxr;
union e1000_rx_desc_extended *rxd;
- u32 staterr = 0;
- int cnt, i;
+ u32 staterr = 0;
+ int cnt, i;
+
+ if (budget == 1) {
+ rxd = &rxr->rx_base[idx];
+ staterr = le32toh(rxd->wb.upper.status_error);
+ return (staterr & E1000_RXD_STAT_DD);
+ }
for (cnt = 0, i = idx; cnt < scctx->isc_nrxd[0] && cnt <= budget;) {
rxd = &rxr->rx_base[i];
@@ -578,15 +612,15 @@ em_isc_rxd_available(void *arg, uint16_t
static int
lem_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri)
{
- struct adapter *adapter = arg;
- if_softc_ctx_t scctx = adapter->shared;
- struct em_rx_queue *que = &adapter->rx_queues[ri->iri_qsidx];
- struct rx_ring *rxr = &que->rxr;
+ struct adapter *adapter = arg;
+ if_softc_ctx_t scctx = adapter->shared;
+ struct em_rx_queue *que = &adapter->rx_queues[ri->iri_qsidx];
+ struct rx_ring *rxr = &que->rxr;
struct e1000_rx_desc *rxd;
- u16 len;
- u32 status, errors;
- bool eop;
- int i, cidx;
+ u16 len;
+ u32 status, errors;
+ bool eop;
+ int i, cidx;
status = errors = i = 0;
cidx = ri->iri_cidx;
@@ -639,31 +673,31 @@ lem_isc_rxd_pkt_get(void *arg, if_rxd_in
static int
em_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri)
{
- struct adapter *adapter = arg;
- if_softc_ctx_t scctx = adapter->shared;
- struct em_rx_queue *que = &adapter->rx_queues[ri->iri_qsidx];
- struct rx_ring *rxr = &que->rxr;
+ struct adapter *adapter = arg;
+ if_softc_ctx_t scctx = adapter->shared;
+ struct em_rx_queue *que = &adapter->rx_queues[ri->iri_qsidx];
+ struct rx_ring *rxr = &que->rxr;
union e1000_rx_desc_extended *rxd;
- u16 len;
- u32 pkt_info;
- u32 staterr = 0;
- bool eop;
- int i, cidx, vtag;
+ u16 len;
+ u32 pkt_info;
+ u32 staterr = 0;
+ bool eop;
+ int i, cidx, vtag;
i = vtag = 0;
cidx = ri->iri_cidx;
do {
rxd = &rxr->rx_base[cidx];
- staterr = le32toh(rxd->wb.upper.status_error);
+ staterr = le32toh(rxd->wb.upper.status_error);
pkt_info = le32toh(rxd->wb.lower.mrq);
-
+
/* Error Checking then decrement count */
MPASS ((staterr & E1000_RXD_STAT_DD) != 0);
len = le16toh(rxd->wb.upper.length);
- ri->iri_len += len;
+ ri->iri_len += len;
eop = (staterr & E1000_RXD_STAT_EOP) != 0;
@@ -690,15 +724,14 @@ em_isc_rxd_pkt_get(void *arg, if_rxd_inf
if (staterr & E1000_RXD_STAT_VP) {
vtag = le16toh(rxd->wb.upper.vlan);
- }
-
+ }
+
ri->iri_vtag = vtag;
if (vtag)
ri->iri_flags |= M_VLANTAG;
-
- ri->iri_flowid =
- le32toh(rxd->wb.lower.hi_dword.rss);
- ri->iri_rsstype = em_determine_rsstype(pkt_info);
+
+ ri->iri_flowid = le32toh(rxd->wb.lower.hi_dword.rss);
+ ri->iri_rsstype = em_determine_rsstype(pkt_info);
ri->iri_nfrags = i;
return (0);
@@ -736,23 +769,24 @@ lem_receive_checksum(int status, int err
static int
em_determine_rsstype(u32 pkt_info)
{
- switch (pkt_info & E1000_RXDADV_RSSTYPE_MASK) {
- case E1000_RXDADV_RSSTYPE_IPV4_TCP:
- return M_HASHTYPE_RSS_TCP_IPV4;
- case E1000_RXDADV_RSSTYPE_IPV4:
- return M_HASHTYPE_RSS_IPV4;
- case E1000_RXDADV_RSSTYPE_IPV6_TCP:
- return M_HASHTYPE_RSS_TCP_IPV6;
- case E1000_RXDADV_RSSTYPE_IPV6_EX:
- return M_HASHTYPE_RSS_IPV6_EX;
- case E1000_RXDADV_RSSTYPE_IPV6:
- return M_HASHTYPE_RSS_IPV6;
- case E1000_RXDADV_RSSTYPE_IPV6_TCP_EX:
- return M_HASHTYPE_RSS_TCP_IPV6_EX;
- default:
- return M_HASHTYPE_OPAQUE;
- }
+ switch (pkt_info & E1000_RXDADV_RSSTYPE_MASK) {
+ case E1000_RXDADV_RSSTYPE_IPV4_TCP:
+ return M_HASHTYPE_RSS_TCP_IPV4;
+ case E1000_RXDADV_RSSTYPE_IPV4:
+ return M_HASHTYPE_RSS_IPV4;
+ case E1000_RXDADV_RSSTYPE_IPV6_TCP:
+ return M_HASHTYPE_RSS_TCP_IPV6;
+ case E1000_RXDADV_RSSTYPE_IPV6_EX:
+ return M_HASHTYPE_RSS_IPV6_EX;
+ case E1000_RXDADV_RSSTYPE_IPV6:
+ return M_HASHTYPE_RSS_IPV6;
+ case E1000_RXDADV_RSSTYPE_IPV6_TCP_EX:
+ return M_HASHTYPE_RSS_TCP_IPV6_EX;
+ default:
+ return M_HASHTYPE_OPAQUE;
+ }
}
+
static void
em_receive_checksum(uint32_t status, if_rxd_info_t ri)
{
@@ -764,7 +798,7 @@ em_receive_checksum(uint32_t status, if_
/* If the IP checksum exists and there is no IP Checksum error */
if ((status & (E1000_RXD_STAT_IPCS | E1000_RXDEXT_STATERR_IPE)) ==
- E1000_RXD_STAT_IPCS) {
+ E1000_RXD_STAT_IPCS) {
ri->iri_csum_flags = (CSUM_IP_CHECKED | CSUM_IP_VALID);
}
Modified: head/sys/dev/e1000/if_em.c
==============================================================================
--- head/sys/dev/e1000/if_em.c Mon Mar 13 22:45:51 2017 (r315216)
+++ head/sys/dev/e1000/if_em.c Mon Mar 13 22:53:06 2017 (r315217)
@@ -50,113 +50,113 @@ char em_driver_version[] = "7.6.1-k";
static pci_vendor_info_t em_vendor_info_array[] =
{
/* Intel(R) PRO/1000 Network Connection - Legacy em*/
- PVID(0x8086, E1000_DEV_ID_82540EM, "Intel(R) PRO/1000 Network Connection"),
- PVID(0x8086, E1000_DEV_ID_82540EM_LOM, "Intel(R) PRO/1000 Network Connection"),
- PVID(0x8086, E1000_DEV_ID_82540EP, "Intel(R) PRO/1000 Network Connection"),
- PVID(0x8086, E1000_DEV_ID_82540EP_LOM, "Intel(R) PRO/1000 Network Connection"),
- PVID(0x8086, E1000_DEV_ID_82540EP_LP, "Intel(R) PRO/1000 Network Connection"),
-
- PVID(0x8086, E1000_DEV_ID_82541EI, "Intel(R) PRO/1000 Network Connection"),
- PVID(0x8086, E1000_DEV_ID_82541ER, "Intel(R) PRO/1000 Network Connection"),
- PVID(0x8086, E1000_DEV_ID_82541ER_LOM, "Intel(R) PRO/1000 Network Connection"),
- PVID(0x8086, E1000_DEV_ID_82541EI_MOBILE, "Intel(R) PRO/1000 Network Connection"),
- PVID(0x8086, E1000_DEV_ID_82541GI, "Intel(R) PRO/1000 Network Connection"),
- PVID(0x8086, E1000_DEV_ID_82541GI_LF, "Intel(R) PRO/1000 Network Connection"),
- PVID(0x8086, E1000_DEV_ID_82541GI_MOBILE, "Intel(R) PRO/1000 Network Connection"),
-
- PVID(0x8086, E1000_DEV_ID_82542, "Intel(R) PRO/1000 Network Connection"),
-
- PVID(0x8086, E1000_DEV_ID_82543GC_FIBER, "Intel(R) PRO/1000 Network Connection"),
- PVID(0x8086, E1000_DEV_ID_82543GC_COPPER, "Intel(R) PRO/1000 Network Connection"),
-
- PVID(0x8086, E1000_DEV_ID_82544EI_COPPER, "Intel(R) PRO/1000 Network Connection"),
- PVID(0x8086, E1000_DEV_ID_82544EI_FIBER, "Intel(R) PRO/1000 Network Connection"),
- PVID(0x8086, E1000_DEV_ID_82544GC_COPPER, "Intel(R) PRO/1000 Network Connection"),
- PVID(0x8086, E1000_DEV_ID_82544GC_LOM, "Intel(R) PRO/1000 Network Connection"),
-
- PVID(0x8086, E1000_DEV_ID_82545EM_COPPER, "Intel(R) PRO/1000 Network Connection"),
- PVID(0x8086, E1000_DEV_ID_82545EM_FIBER, "Intel(R) PRO/1000 Network Connection"),
- PVID(0x8086, E1000_DEV_ID_82545GM_COPPER, "Intel(R) PRO/1000 Network Connection"),
- PVID(0x8086, E1000_DEV_ID_82545GM_FIBER, "Intel(R) PRO/1000 Network Connection"),
- PVID(0x8086, E1000_DEV_ID_82545GM_SERDES, "Intel(R) PRO/1000 Network Connection"),
-
- PVID(0x8086, E1000_DEV_ID_82546EB_COPPER, "Intel(R) PRO/1000 Network Connection"),
- PVID(0x8086, E1000_DEV_ID_82546EB_FIBER, "Intel(R) PRO/1000 Network Connection"),
- PVID(0x8086, E1000_DEV_ID_82546EB_QUAD_COPPER, "Intel(R) PRO/1000 Network Connection"),
- PVID(0x8086, E1000_DEV_ID_82546GB_COPPER, "Intel(R) PRO/1000 Network Connection"),
- PVID(0x8086, E1000_DEV_ID_82546GB_FIBER, "Intel(R) PRO/1000 Network Connection"),
- PVID(0x8086, E1000_DEV_ID_82546GB_SERDES, "Intel(R) PRO/1000 Network Connection"),
- PVID(0x8086, E1000_DEV_ID_82546GB_PCIE, "Intel(R) PRO/1000 Network Connection"),
- PVID(0x8086, E1000_DEV_ID_82546GB_QUAD_COPPER, "Intel(R) PRO/1000 Network Connection"),
- PVID(0x8086, E1000_DEV_ID_82546GB_QUAD_COPPER_KSP3, "Intel(R) PRO/1000 Network Connection"),
-
- PVID(0x8086, E1000_DEV_ID_82547EI, "Intel(R) PRO/1000 Network Connection"),
*** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
More information about the svn-src-all
mailing list