svn commit: r340766 - head/sys/dev/sfxge/common
Andrew Rybchenko
arybchik at FreeBSD.org
Thu Nov 22 14:31:39 UTC 2018
Author: arybchik
Date: Thu Nov 22 14:31:35 2018
New Revision: 340766
URL: https://svnweb.freebsd.org/changeset/base/340766
Log:
sfxge(4): support packed stream Rx mode in libefx
Submitted by: Artem V. Andreev <Artem.Andreev at oktetlabs.ru>
Sponsored by: Solarflare Communications, Inc.
Differential Revision: https://reviews.freebsd.org/D18022
Modified:
head/sys/dev/sfxge/common/ef10_ev.c
head/sys/dev/sfxge/common/ef10_impl.h
head/sys/dev/sfxge/common/ef10_nic.c
head/sys/dev/sfxge/common/ef10_rx.c
head/sys/dev/sfxge/common/efsys.h
head/sys/dev/sfxge/common/efx.h
head/sys/dev/sfxge/common/efx_check.h
head/sys/dev/sfxge/common/efx_impl.h
head/sys/dev/sfxge/common/efx_nic.c
head/sys/dev/sfxge/common/efx_regs_ef10.h
head/sys/dev/sfxge/common/efx_rx.c
head/sys/dev/sfxge/common/siena_nic.c
Modified: head/sys/dev/sfxge/common/ef10_ev.c
==============================================================================
--- head/sys/dev/sfxge/common/ef10_ev.c Thu Nov 22 14:10:46 2018 (r340765)
+++ head/sys/dev/sfxge/common/ef10_ev.c Thu Nov 22 14:31:35 2018 (r340766)
@@ -762,8 +762,90 @@ ef10_ev_qstats_update(
}
#endif /* EFSYS_OPT_QSTATS */
+#if EFSYS_OPT_RX_PACKED_STREAM
static __checkReturn boolean_t
+ef10_ev_rx_packed_stream(
+ __in efx_evq_t *eep,
+ __in efx_qword_t *eqp,
+ __in const efx_ev_callbacks_t *eecp,
+ __in_opt void *arg)
+{
+ uint32_t label;
+ uint32_t pkt_count_lbits;
+ uint16_t flags;
+ boolean_t should_abort;
+ efx_evq_rxq_state_t *eersp;
+ unsigned int pkt_count;
+ unsigned int current_id;
+ boolean_t new_buffer;
+
+ pkt_count_lbits = EFX_QWORD_FIELD(*eqp, ESF_DZ_RX_DSC_PTR_LBITS);
+ label = EFX_QWORD_FIELD(*eqp, ESF_DZ_RX_QLABEL);
+ new_buffer = EFX_QWORD_FIELD(*eqp, ESF_DZ_RX_EV_ROTATE);
+
+ flags = 0;
+
+ eersp = &eep->ee_rxq_state[label];
+
+ /*
+ * RX_DSC_PTR_LBITS has least significant bits of the global
+ * (not per-buffer) packet counter. It is guaranteed that
+ * maximum number of completed packets fits in lbits-mask.
+ * So, modulo lbits-mask arithmetic should be used to calculate
+ * packet counter increment.
+ */
+ pkt_count = (pkt_count_lbits - eersp->eers_rx_stream_npackets) &
+ EFX_MASK32(ESF_DZ_RX_DSC_PTR_LBITS);
+ eersp->eers_rx_stream_npackets += pkt_count;
+
+ if (new_buffer) {
+ flags |= EFX_PKT_PACKED_STREAM_NEW_BUFFER;
+ eersp->eers_rx_packed_stream_credits++;
+ eersp->eers_rx_read_ptr++;
+ }
+ current_id = eersp->eers_rx_read_ptr & eersp->eers_rx_mask;
+
+ /* Check for errors that invalidate checksum and L3/L4 fields */
+ if (EFX_QWORD_FIELD(*eqp, ESF_DZ_RX_ECC_ERR) != 0) {
+ /* RX frame truncated (error flag is misnamed) */
+ EFX_EV_QSTAT_INCR(eep, EV_RX_FRM_TRUNC);
+ flags |= EFX_DISCARD;
+ goto deliver;
+ }
+ if (EFX_QWORD_FIELD(*eqp, ESF_DZ_RX_ECRC_ERR) != 0) {
+ /* Bad Ethernet frame CRC */
+ EFX_EV_QSTAT_INCR(eep, EV_RX_ETH_CRC_ERR);
+ flags |= EFX_DISCARD;
+ goto deliver;
+ }
+
+ if (EFX_QWORD_FIELD(*eqp, ESF_DZ_RX_PARSE_INCOMPLETE)) {
+ flags |= EFX_PKT_PACKED_STREAM_PARSE_INCOMPLETE;
+ goto deliver;
+ }
+
+ if (EFX_QWORD_FIELD(*eqp, ESF_DZ_RX_IPCKSUM_ERR))
+ EFX_EV_QSTAT_INCR(eep, EV_RX_IPV4_HDR_CHKSUM_ERR);
+
+ if (EFX_QWORD_FIELD(*eqp, ESF_DZ_RX_TCPUDP_CKSUM_ERR))
+ EFX_EV_QSTAT_INCR(eep, EV_RX_TCP_UDP_CHKSUM_ERR);
+
+deliver:
+ /* If we're not discarding the packet then it is ok */
+ if (~flags & EFX_DISCARD)
+ EFX_EV_QSTAT_INCR(eep, EV_RX_OK);
+
+ EFSYS_ASSERT(eecp->eec_rx_ps != NULL);
+ should_abort = eecp->eec_rx_ps(arg, label, current_id, pkt_count,
+ flags);
+
+ return (should_abort);
+}
+
+#endif /* EFSYS_OPT_RX_PACKED_STREAM */
+
+static __checkReturn boolean_t
ef10_ev_rx(
__in efx_evq_t *eep,
__in efx_qword_t *eqp,
@@ -792,9 +874,20 @@ ef10_ev_rx(
return (B_FALSE);
/* Basic packet information */
+ label = EFX_QWORD_FIELD(*eqp, ESF_DZ_RX_QLABEL);
+ eersp = &eep->ee_rxq_state[label];
+
+#if EFSYS_OPT_RX_PACKED_STREAM
+ /*
+ * Packed stream events are very different,
+ * so handle them separately
+ */
+ if (eersp->eers_rx_packed_stream)
+ return (ef10_ev_rx_packed_stream(eep, eqp, eecp, arg));
+#endif
+
size = EFX_QWORD_FIELD(*eqp, ESF_DZ_RX_BYTES);
next_read_lbits = EFX_QWORD_FIELD(*eqp, ESF_DZ_RX_DSC_PTR_LBITS);
- label = EFX_QWORD_FIELD(*eqp, ESF_DZ_RX_QLABEL);
eth_tag_class = EFX_QWORD_FIELD(*eqp, ESF_DZ_RX_ETH_TAG_CLASS);
mac_class = EFX_QWORD_FIELD(*eqp, ESF_DZ_RX_MAC_CLASS);
l3_class = EFX_QWORD_FIELD(*eqp, ESF_DZ_RX_L3_CLASS);
@@ -824,7 +917,6 @@ ef10_ev_rx(
flags |= EFX_PKT_UNICAST;
/* Increment the count of descriptors read */
- eersp = &eep->ee_rxq_state[label];
desc_count = (next_read_lbits - eersp->eers_rx_read_ptr) &
EFX_MASK32(ESF_DZ_RX_DSC_PTR_LBITS);
eersp->eers_rx_read_ptr += desc_count;
@@ -1246,7 +1338,8 @@ ef10_ev_mcdi(
ef10_ev_rxlabel_init(
__in efx_evq_t *eep,
__in efx_rxq_t *erp,
- __in unsigned int label)
+ __in unsigned int label,
+ __in boolean_t packed_stream)
{
efx_evq_rxq_state_t *eersp;
@@ -1255,8 +1348,41 @@ ef10_ev_rxlabel_init(
EFSYS_ASSERT3U(eersp->eers_rx_mask, ==, 0);
+#if EFSYS_OPT_RX_PACKED_STREAM
+ /*
+ * For packed stream modes, the very first event will
+ * have a new buffer flag set, so it will be incremented,
+ * yielding the correct pointer. That results in a simpler
+ * code than trying to detect start-of-the-world condition
+ * in the event handler.
+ */
+ eersp->eers_rx_read_ptr = packed_stream ? ~0 : 0;
+#else
eersp->eers_rx_read_ptr = 0;
+#endif
eersp->eers_rx_mask = erp->er_mask;
+#if EFSYS_OPT_RX_PACKED_STREAM
+ eersp->eers_rx_stream_npackets = 0;
+ eersp->eers_rx_packed_stream = packed_stream;
+ if (packed_stream) {
+ eersp->eers_rx_packed_stream_credits = (eep->ee_mask + 1) /
+ EFX_DIV_ROUND_UP(EFX_RX_PACKED_STREAM_MEM_PER_CREDIT,
+ EFX_RX_PACKED_STREAM_MIN_PACKET_SPACE);
+ EFSYS_ASSERT3U(eersp->eers_rx_packed_stream_credits, !=, 0);
+ /*
+ * A single credit is allocated to the queue when it is started.
+ * It is immediately spent by the first packet which has NEW
+ * BUFFER flag set, though, but still we shall take into
+ * account, as to not wrap around the maximum number of credits
+ * accidentally
+ */
+ eersp->eers_rx_packed_stream_credits--;
+ EFSYS_ASSERT3U(eersp->eers_rx_packed_stream_credits, <=,
+ EFX_RX_PACKED_STREAM_MAX_CREDITS);
+ }
+#else
+ EFSYS_ASSERT(!packed_stream);
+#endif
}
void
@@ -1273,6 +1399,11 @@ ef10_ev_rxlabel_fini(
eersp->eers_rx_read_ptr = 0;
eersp->eers_rx_mask = 0;
+#if EFSYS_OPT_RX_PACKED_STREAM
+ eersp->eers_rx_stream_npackets = 0;
+ eersp->eers_rx_packed_stream = B_FALSE;
+ eersp->eers_rx_packed_stream_credits = 0;
+#endif
}
#endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD */
Modified: head/sys/dev/sfxge/common/ef10_impl.h
==============================================================================
--- head/sys/dev/sfxge/common/ef10_impl.h Thu Nov 22 14:10:46 2018 (r340765)
+++ head/sys/dev/sfxge/common/ef10_impl.h Thu Nov 22 14:31:35 2018 (r340766)
@@ -118,7 +118,8 @@ ef10_ev_qstats_update(
ef10_ev_rxlabel_init(
__in efx_evq_t *eep,
__in efx_rxq_t *erp,
- __in unsigned int label);
+ __in unsigned int label,
+ __in boolean_t packed_stream);
void
ef10_ev_rxlabel_fini(
@@ -685,6 +686,22 @@ ef10_tx_qpush(
__in unsigned int added,
__in unsigned int pushed);
+#if EFSYS_OPT_RX_PACKED_STREAM
+extern void
+ef10_rx_qpush_ps_credits(
+ __in efx_rxq_t *erp);
+
+extern __checkReturn uint8_t *
+ef10_rx_qps_packet_info(
+ __in efx_rxq_t *erp,
+ __in uint8_t *buffer,
+ __in uint32_t buffer_length,
+ __in uint32_t current_offset,
+ __out uint16_t *lengthp,
+ __out uint32_t *next_offsetp,
+ __out uint32_t *timestamp);
+#endif
+
extern __checkReturn efx_rc_t
ef10_tx_qpace(
__in efx_txq_t *etp,
@@ -1131,6 +1148,35 @@ ef10_external_port_mapping(
__in uint32_t port,
__out uint8_t *external_portp);
+#if EFSYS_OPT_RX_PACKED_STREAM
+
+/* Data space per credit in packed stream mode */
+#define EFX_RX_PACKED_STREAM_MEM_PER_CREDIT (1 << 16)
+
+/*
+ * Received packets are always aligned at this boundary. Also there always
+ * exists a gap of this size between packets.
+ * (see SF-112241-TC, 4.5)
+ */
+#define EFX_RX_PACKED_STREAM_ALIGNMENT 64
+
+/*
+ * Size of a pseudo-header prepended to received packets
+ * in packed stream mode
+ */
+#define EFX_RX_PACKED_STREAM_RX_PREFIX_SIZE 8
+
+/* Minimum space for packet in packed stream mode */
+#define EFX_RX_PACKED_STREAM_MIN_PACKET_SPACE \
+ P2ROUNDUP(EFX_RX_PACKED_STREAM_RX_PREFIX_SIZE + \
+ EFX_MAC_PDU_MIN + \
+ EFX_RX_PACKED_STREAM_ALIGNMENT, \
+ EFX_RX_PACKED_STREAM_ALIGNMENT)
+
+/* Maximum number of credits */
+#define EFX_RX_PACKED_STREAM_MAX_CREDITS 127
+
+#endif /* EFSYS_OPT_RX_PACKED_STREAM */
#ifdef __cplusplus
}
Modified: head/sys/dev/sfxge/common/ef10_nic.c
==============================================================================
--- head/sys/dev/sfxge/common/ef10_nic.c Thu Nov 22 14:10:46 2018 (r340765)
+++ head/sys/dev/sfxge/common/ef10_nic.c Thu Nov 22 14:31:35 2018 (r340766)
@@ -1072,6 +1072,17 @@ ef10_get_datapath_caps(
encp->enc_rx_disable_scatter_supported =
CAP_FLAG(flags, RX_DISABLE_SCATTER) ? B_TRUE : B_FALSE;
+ /* Check if the firmware supports packed stream mode */
+ encp->enc_rx_packed_stream_supported =
+ CAP_FLAG(flags, RX_PACKED_STREAM) ? B_TRUE : B_FALSE;
+
+ /*
+ * Check if the firmware supports configurable buffer sizes
+ * for packed stream mode (otherwise buffer size is 1Mbyte)
+ */
+ encp->enc_rx_var_packed_stream_supported =
+ CAP_FLAG(flags, RX_PACKED_STREAM_VAR_BUFFERS) ? B_TRUE : B_FALSE;
+
/* Check if the firmware supports set mac with running filters */
encp->enc_allow_set_mac_with_installed_filters =
CAP_FLAG(flags, VADAPTOR_PERMIT_SET_MAC_WHEN_FILTERS_INSTALLED) ?
Modified: head/sys/dev/sfxge/common/ef10_rx.c
==============================================================================
--- head/sys/dev/sfxge/common/ef10_rx.c Thu Nov 22 14:10:46 2018 (r340765)
+++ head/sys/dev/sfxge/common/ef10_rx.c Thu Nov 22 14:31:35 2018 (r340766)
@@ -46,41 +46,51 @@ efx_mcdi_init_rxq(
__in uint32_t label,
__in uint32_t instance,
__in efsys_mem_t *esmp,
- __in boolean_t disable_scatter)
+ __in boolean_t disable_scatter,
+ __in uint32_t ps_bufsize)
{
efx_mcdi_req_t req;
- uint8_t payload[
- MC_CMD_INIT_RXQ_IN_LEN(EFX_RXQ_NBUFS(EFX_RXQ_MAXNDESCS))];
+ uint8_t payload[MAX(MC_CMD_INIT_RXQ_EXT_IN_LEN,
+ MC_CMD_INIT_RXQ_EXT_OUT_LEN)];
int npages = EFX_RXQ_NBUFS(size);
int i;
efx_qword_t *dma_addr;
uint64_t addr;
efx_rc_t rc;
+ uint32_t dma_mode;
/* If this changes, then the payload size might need to change. */
EFSYS_ASSERT3U(MC_CMD_INIT_RXQ_OUT_LEN, ==, 0);
EFSYS_ASSERT3U(size, <=, EFX_RXQ_MAXNDESCS);
+ if (ps_bufsize > 0)
+ dma_mode = MC_CMD_INIT_RXQ_EXT_IN_PACKED_STREAM;
+ else
+ dma_mode = MC_CMD_INIT_RXQ_EXT_IN_SINGLE_PACKET;
+
(void) memset(payload, 0, sizeof (payload));
req.emr_cmd = MC_CMD_INIT_RXQ;
req.emr_in_buf = payload;
- req.emr_in_length = MC_CMD_INIT_RXQ_IN_LEN(npages);
+ req.emr_in_length = MC_CMD_INIT_RXQ_EXT_IN_LEN;
req.emr_out_buf = payload;
- req.emr_out_length = MC_CMD_INIT_RXQ_OUT_LEN;
+ req.emr_out_length = MC_CMD_INIT_RXQ_EXT_OUT_LEN;
- MCDI_IN_SET_DWORD(req, INIT_RXQ_IN_SIZE, size);
- MCDI_IN_SET_DWORD(req, INIT_RXQ_IN_TARGET_EVQ, target_evq);
- MCDI_IN_SET_DWORD(req, INIT_RXQ_IN_LABEL, label);
- MCDI_IN_SET_DWORD(req, INIT_RXQ_IN_INSTANCE, instance);
- MCDI_IN_POPULATE_DWORD_6(req, INIT_RXQ_IN_FLAGS,
- INIT_RXQ_IN_FLAG_BUFF_MODE, 0,
- INIT_RXQ_IN_FLAG_HDR_SPLIT, 0,
- INIT_RXQ_IN_FLAG_TIMESTAMP, 0,
- INIT_RXQ_IN_CRC_MODE, 0,
- INIT_RXQ_IN_FLAG_PREFIX, 1,
- INIT_RXQ_IN_FLAG_DISABLE_SCATTER, disable_scatter);
- MCDI_IN_SET_DWORD(req, INIT_RXQ_IN_OWNER_ID, 0);
- MCDI_IN_SET_DWORD(req, INIT_RXQ_IN_PORT_ID, EVB_PORT_ID_ASSIGNED);
+ MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_SIZE, size);
+ MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_TARGET_EVQ, target_evq);
+ MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_LABEL, label);
+ MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_INSTANCE, instance);
+ MCDI_IN_POPULATE_DWORD_8(req, INIT_RXQ_EXT_IN_FLAGS,
+ INIT_RXQ_EXT_IN_FLAG_BUFF_MODE, 0,
+ INIT_RXQ_EXT_IN_FLAG_HDR_SPLIT, 0,
+ INIT_RXQ_EXT_IN_FLAG_TIMESTAMP, 0,
+ INIT_RXQ_EXT_IN_CRC_MODE, 0,
+ INIT_RXQ_EXT_IN_FLAG_PREFIX, 1,
+ INIT_RXQ_EXT_IN_FLAG_DISABLE_SCATTER, disable_scatter,
+ INIT_RXQ_EXT_IN_DMA_MODE,
+ dma_mode,
+ INIT_RXQ_EXT_IN_PACKED_STREAM_BUFF_SIZE, ps_bufsize);
+ MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_OWNER_ID, 0);
+ MCDI_IN_SET_DWORD(req, INIT_RXQ_EXT_IN_PORT_ID, EVB_PORT_ID_ASSIGNED);
dma_addr = MCDI_IN2(req, efx_qword_t, INIT_RXQ_IN_DMA_ADDR);
addr = EFSYS_MEM_ADDR(esmp);
@@ -707,6 +717,95 @@ ef10_rx_qpush(
erp->er_index, &dword, B_FALSE);
}
+#if EFSYS_OPT_RX_PACKED_STREAM
+
+ void
+ef10_rx_qpush_ps_credits(
+ __in efx_rxq_t *erp)
+{
+ efx_nic_t *enp = erp->er_enp;
+ efx_dword_t dword;
+ efx_evq_rxq_state_t *rxq_state =
+ &erp->er_eep->ee_rxq_state[erp->er_label];
+ uint32_t credits;
+
+ EFSYS_ASSERT(rxq_state->eers_rx_packed_stream);
+
+ if (rxq_state->eers_rx_packed_stream_credits == 0)
+ return;
+
+ /*
+ * It is a bug if we think that FW has utilized more
+ * credits than it is allowed to have (maximum). However,
+ * make sure that we do not credit more than maximum anyway.
+ */
+ credits = MIN(rxq_state->eers_rx_packed_stream_credits,
+ EFX_RX_PACKED_STREAM_MAX_CREDITS);
+ EFX_POPULATE_DWORD_3(dword,
+ ERF_DZ_RX_DESC_MAGIC_DOORBELL, 1,
+ ERF_DZ_RX_DESC_MAGIC_CMD,
+ ERE_DZ_RX_DESC_MAGIC_CMD_PS_CREDITS,
+ ERF_DZ_RX_DESC_MAGIC_DATA, credits);
+ EFX_BAR_TBL_WRITED(enp, ER_DZ_RX_DESC_UPD_REG,
+ erp->er_index, &dword, B_FALSE);
+
+ rxq_state->eers_rx_packed_stream_credits = 0;
+}
+
+/*
+ * In accordance with SF-112241-TC the received data has the following layout:
+ * - 8 byte pseudo-header which consist of:
+ * - 4 byte little-endian timestamp
+ * - 2 byte little-endian captured length in bytes
+ * - 2 byte little-endian original packet length in bytes
+ * - captured packet bytes
+ * - optional padding to align to 64 bytes boundary
+ * - 64 bytes scratch space for the host software
+ */
+ __checkReturn uint8_t *
+ef10_rx_qps_packet_info(
+ __in efx_rxq_t *erp,
+ __in uint8_t *buffer,
+ __in uint32_t buffer_length,
+ __in uint32_t current_offset,
+ __out uint16_t *lengthp,
+ __out uint32_t *next_offsetp,
+ __out uint32_t *timestamp)
+{
+ uint16_t buf_len;
+ uint8_t *pkt_start;
+ efx_qword_t *qwordp;
+ efx_evq_rxq_state_t *rxq_state =
+ &erp->er_eep->ee_rxq_state[erp->er_label];
+
+ EFSYS_ASSERT(rxq_state->eers_rx_packed_stream);
+
+ buffer += current_offset;
+ pkt_start = buffer + EFX_RX_PACKED_STREAM_RX_PREFIX_SIZE;
+
+ qwordp = (efx_qword_t *)buffer;
+ *timestamp = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_TSTAMP);
+ *lengthp = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_ORIG_LEN);
+ buf_len = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_CAP_LEN);
+
+ buf_len = P2ROUNDUP(buf_len + EFX_RX_PACKED_STREAM_RX_PREFIX_SIZE,
+ EFX_RX_PACKED_STREAM_ALIGNMENT);
+ *next_offsetp =
+ current_offset + buf_len + EFX_RX_PACKED_STREAM_ALIGNMENT;
+
+ EFSYS_ASSERT3U(*next_offsetp, <=, buffer_length);
+ EFSYS_ASSERT3U(current_offset + *lengthp, <, *next_offsetp);
+
+ if ((*next_offsetp ^ current_offset) &
+ EFX_RX_PACKED_STREAM_MEM_PER_CREDIT)
+ rxq_state->eers_rx_packed_stream_credits++;
+
+ return (pkt_start);
+}
+
+
+#endif
+
__checkReturn efx_rc_t
ef10_rx_qflush(
__in efx_rxq_t *erp)
@@ -749,6 +848,7 @@ ef10_rx_qcreate(
efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
efx_rc_t rc;
boolean_t disable_scatter;
+ unsigned int ps_buf_size;
_NOTE(ARGUNUSED(id, erp))
@@ -768,6 +868,51 @@ ef10_rx_qcreate(
goto fail2;
}
+ switch (type) {
+ case EFX_RXQ_TYPE_DEFAULT:
+ case EFX_RXQ_TYPE_SCATTER:
+ ps_buf_size = 0;
+ break;
+#if EFSYS_OPT_RX_PACKED_STREAM
+ case EFX_RXQ_TYPE_PACKED_STREAM_1M:
+ ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_1M;
+ break;
+ case EFX_RXQ_TYPE_PACKED_STREAM_512K:
+ ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_512K;
+ break;
+ case EFX_RXQ_TYPE_PACKED_STREAM_256K:
+ ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_256K;
+ break;
+ case EFX_RXQ_TYPE_PACKED_STREAM_128K:
+ ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_128K;
+ break;
+ case EFX_RXQ_TYPE_PACKED_STREAM_64K:
+ ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_64K;
+ break;
+#endif /* EFSYS_OPT_RX_PACKED_STREAM */
+ default:
+ rc = ENOTSUP;
+ goto fail3;
+ }
+
+#if EFSYS_OPT_RX_PACKED_STREAM
+ if (ps_buf_size != 0) {
+ /* Check if datapath firmware supports packed stream mode */
+ if (encp->enc_rx_packed_stream_supported == B_FALSE) {
+ rc = ENOTSUP;
+ goto fail4;
+ }
+ /* Check if packed stream allows configurable buffer sizes */
+ if ((type != EFX_RXQ_TYPE_PACKED_STREAM_1M) &&
+ (encp->enc_rx_var_packed_stream_supported == B_FALSE)) {
+ rc = ENOTSUP;
+ goto fail5;
+ }
+ }
+#else /* EFSYS_OPT_RX_PACKED_STREAM */
+ EFSYS_ASSERT(ps_buf_size == 0);
+#endif /* EFSYS_OPT_RX_PACKED_STREAM */
+
/* Scatter can only be disabled if the firmware supports doing so */
if (type == EFX_RXQ_TYPE_SCATTER)
disable_scatter = B_FALSE;
@@ -775,16 +920,24 @@ ef10_rx_qcreate(
disable_scatter = encp->enc_rx_disable_scatter_supported;
if ((rc = efx_mcdi_init_rxq(enp, n, eep->ee_index, label, index,
- esmp, disable_scatter)) != 0)
- goto fail3;
+ esmp, disable_scatter, ps_buf_size)) != 0)
+ goto fail6;
erp->er_eep = eep;
erp->er_label = label;
- ef10_ev_rxlabel_init(eep, erp, label);
+ ef10_ev_rxlabel_init(eep, erp, label, ps_buf_size != 0);
return (0);
+fail6:
+ EFSYS_PROBE(fail6);
+#if EFSYS_OPT_RX_PACKED_STREAM
+fail5:
+ EFSYS_PROBE(fail5);
+fail4:
+ EFSYS_PROBE(fail4);
+#endif /* EFSYS_OPT_RX_PACKED_STREAM */
fail3:
EFSYS_PROBE(fail3);
fail2:
Modified: head/sys/dev/sfxge/common/efsys.h
==============================================================================
--- head/sys/dev/sfxge/common/efsys.h Thu Nov 22 14:10:46 2018 (r340765)
+++ head/sys/dev/sfxge/common/efsys.h Thu Nov 22 14:31:35 2018 (r340766)
@@ -281,6 +281,8 @@ sfxge_map_mbuf_fast(bus_dma_tag_t tag, bus_dmamap_t ma
#define EFSYS_OPT_ALLOW_UNCONFIGURED_NIC 0
+#define EFSYS_OPT_RX_PACKED_STREAM 0
+
/* ID */
typedef struct __efsys_identifier_s efsys_identifier_t;
Modified: head/sys/dev/sfxge/common/efx.h
==============================================================================
--- head/sys/dev/sfxge/common/efx.h Thu Nov 22 14:10:46 2018 (r340765)
+++ head/sys/dev/sfxge/common/efx.h Thu Nov 22 14:31:35 2018 (r340766)
@@ -52,6 +52,9 @@ extern "C" {
#define EFX_FIELD_OFFSET(_type, _field) \
((size_t) &(((_type *)0)->_field))
+/* The macro expands divider twice */
+#define EFX_DIV_ROUND_UP(_n, _d) (((_n) + (_d) - 1) / (_d))
+
/* Return codes */
typedef __success(return == 0) int efx_rc_t;
@@ -1086,6 +1089,7 @@ efx_bist_stop(
#define EFX_FEATURE_PIO_BUFFERS 0x00000800
#define EFX_FEATURE_FW_ASSISTED_TSO 0x00001000
#define EFX_FEATURE_FW_ASSISTED_TSO_V2 0x00002000
+#define EFX_FEATURE_PACKED_STREAM 0x00004000
typedef struct efx_nic_cfg_s {
uint32_t enc_board_type;
@@ -1181,6 +1185,8 @@ typedef struct efx_nic_cfg_s {
boolean_t enc_allow_set_mac_with_installed_filters;
boolean_t enc_enhanced_set_mac_supported;
boolean_t enc_init_evq_v2_supported;
+ boolean_t enc_rx_packed_stream_supported;
+ boolean_t enc_rx_var_packed_stream_supported;
boolean_t enc_pm_and_rxdp_counters;
boolean_t enc_mac_stats_40g_tx_size_bins;
/* External port identifier */
@@ -1622,6 +1628,16 @@ typedef __checkReturn boolean_t
#define EFX_ADDR_MISMATCH 0x4000
#define EFX_DISCARD 0x8000
+/*
+ * The following flags are used only for packed stream
+ * mode. The values for the flags are reused to fit into 16 bit,
+ * since EFX_PKT_START and EFX_PKT_CONT are never used in
+ * packed stream mode
+ */
+#define EFX_PKT_PACKED_STREAM_NEW_BUFFER EFX_PKT_START
+#define EFX_PKT_PACKED_STREAM_PARSE_INCOMPLETE EFX_PKT_CONT
+
+
#define EFX_EV_RX_NLABELS 32
#define EFX_EV_TX_NLABELS 32
@@ -1633,7 +1649,29 @@ typedef __checkReturn boolean_t
__in uint32_t size,
__in uint16_t flags);
+#if EFSYS_OPT_RX_PACKED_STREAM
+
+/*
+ * Packed stream mode is documented in SF-112241-TC.
+ * The general idea is that, instead of putting each incoming
+ * packet into a separate buffer which is specified in a RX
+ * descriptor, a large buffer is provided to the hardware and
+ * packets are put there in a continuous stream.
+ * The main advantage of such an approach is that RX queue refilling
+ * happens much less frequently.
+ */
+
typedef __checkReturn boolean_t
+(*efx_rx_ps_ev_t)(
+ __in_opt void *arg,
+ __in uint32_t label,
+ __in uint32_t id,
+ __in uint32_t pkt_count,
+ __in uint16_t flags);
+
+#endif
+
+typedef __checkReturn boolean_t
(*efx_tx_ev_t)(
__in_opt void *arg,
__in uint32_t label,
@@ -1722,6 +1760,9 @@ typedef __checkReturn boolean_t
typedef struct efx_ev_callbacks_s {
efx_initialized_ev_t eec_initialized;
efx_rx_ev_t eec_rx;
+#if EFSYS_OPT_RX_PACKED_STREAM
+ efx_rx_ps_ev_t eec_rx_ps;
+#endif
efx_tx_ev_t eec_tx;
efx_exception_ev_t eec_exception;
efx_rxq_flush_done_ev_t eec_rxq_flush_done;
@@ -1900,6 +1941,11 @@ efx_pseudo_hdr_pkt_length_get(
typedef enum efx_rxq_type_e {
EFX_RXQ_TYPE_DEFAULT,
EFX_RXQ_TYPE_SCATTER,
+ EFX_RXQ_TYPE_PACKED_STREAM_1M,
+ EFX_RXQ_TYPE_PACKED_STREAM_512K,
+ EFX_RXQ_TYPE_PACKED_STREAM_256K,
+ EFX_RXQ_TYPE_PACKED_STREAM_128K,
+ EFX_RXQ_TYPE_PACKED_STREAM_64K,
EFX_RXQ_NTYPES
} efx_rxq_type_t;
@@ -1939,6 +1985,29 @@ efx_rx_qpush(
__in efx_rxq_t *erp,
__in unsigned int added,
__inout unsigned int *pushedp);
+
+#if EFSYS_OPT_RX_PACKED_STREAM
+
+/*
+ * Fake length for RXQ descriptors in packed stream mode
+ * to make hardware happy
+ */
+#define EFX_RXQ_PACKED_STREAM_FAKE_BUF_SIZE 32
+
+extern void
+efx_rx_qpush_ps_credits(
+ __in efx_rxq_t *erp);
+
+extern __checkReturn uint8_t *
+efx_rx_qps_packet_info(
+ __in efx_rxq_t *erp,
+ __in uint8_t *buffer,
+ __in uint32_t buffer_length,
+ __in uint32_t current_offset,
+ __out uint16_t *lengthp,
+ __out uint32_t *next_offsetp,
+ __out uint32_t *timestamp);
+#endif
extern __checkReturn efx_rc_t
efx_rx_qflush(
Modified: head/sys/dev/sfxge/common/efx_check.h
==============================================================================
--- head/sys/dev/sfxge/common/efx_check.h Thu Nov 22 14:10:46 2018 (r340765)
+++ head/sys/dev/sfxge/common/efx_check.h Thu Nov 22 14:31:35 2018 (r340766)
@@ -339,5 +339,11 @@
# endif
#endif /* EFSYS_OPT_ALLOW_UNCONFIGURED_NIC */
+/* Support packed stream mode */
+#if EFSYS_OPT_RX_PACKED_STREAM
+# if !(EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD)
+# error "PACKED_STREAM requires HUNTINGTON or MEDFORD"
+# endif
+#endif
#endif /* _SYS_EFX_CHECK_H */
Modified: head/sys/dev/sfxge/common/efx_impl.h
==============================================================================
--- head/sys/dev/sfxge/common/efx_impl.h Thu Nov 22 14:10:46 2018 (r340765)
+++ head/sys/dev/sfxge/common/efx_impl.h Thu Nov 22 14:31:35 2018 (r340766)
@@ -170,6 +170,12 @@ typedef struct efx_rx_ops_s {
unsigned int, unsigned int,
unsigned int);
void (*erxo_qpush)(efx_rxq_t *, unsigned int, unsigned int *);
+#if EFSYS_OPT_RX_PACKED_STREAM
+ void (*erxo_qpush_ps_credits)(efx_rxq_t *);
+ uint8_t * (*erxo_qps_packet_info)(efx_rxq_t *, uint8_t *,
+ uint32_t, uint32_t,
+ uint16_t *, uint32_t *, uint32_t *);
+#endif
efx_rc_t (*erxo_qflush)(efx_rxq_t *);
void (*erxo_qenable)(efx_rxq_t *);
efx_rc_t (*erxo_qcreate)(efx_nic_t *enp, unsigned int,
@@ -694,6 +700,11 @@ typedef boolean_t (*efx_ev_handler_t)(efx_evq_t *, efx
typedef struct efx_evq_rxq_state_s {
unsigned int eers_rx_read_ptr;
unsigned int eers_rx_mask;
+#if EFSYS_OPT_RX_PACKED_STREAM
+ unsigned int eers_rx_stream_npackets;
+ boolean_t eers_rx_packed_stream;
+ unsigned int eers_rx_packed_stream_credits;
+#endif
} efx_evq_rxq_state_t;
struct efx_evq_s {
Modified: head/sys/dev/sfxge/common/efx_nic.c
==============================================================================
--- head/sys/dev/sfxge/common/efx_nic.c Thu Nov 22 14:10:46 2018 (r340765)
+++ head/sys/dev/sfxge/common/efx_nic.c Thu Nov 22 14:31:35 2018 (r340766)
@@ -289,7 +289,8 @@ efx_nic_create(
EFX_FEATURE_MCDI_DMA |
EFX_FEATURE_PIO_BUFFERS |
EFX_FEATURE_FW_ASSISTED_TSO |
- EFX_FEATURE_FW_ASSISTED_TSO_V2;
+ EFX_FEATURE_FW_ASSISTED_TSO_V2 |
+ EFX_FEATURE_PACKED_STREAM;
break;
#endif /* EFSYS_OPT_HUNTINGTON */
@@ -308,7 +309,8 @@ efx_nic_create(
EFX_FEATURE_MAC_HEADER_FILTERS |
EFX_FEATURE_MCDI_DMA |
EFX_FEATURE_PIO_BUFFERS |
- EFX_FEATURE_FW_ASSISTED_TSO_V2;
+ EFX_FEATURE_FW_ASSISTED_TSO_V2 |
+ EFX_FEATURE_PACKED_STREAM;
break;
#endif /* EFSYS_OPT_MEDFORD */
Modified: head/sys/dev/sfxge/common/efx_regs_ef10.h
==============================================================================
--- head/sys/dev/sfxge/common/efx_regs_ef10.h Thu Nov 22 14:10:46 2018 (r340765)
+++ head/sys/dev/sfxge/common/efx_regs_ef10.h Thu Nov 22 14:31:35 2018 (r340766)
@@ -169,7 +169,6 @@ extern "C" {
#define ERF_DZ_RX_DESC_WPTR_LBN 0
#define ERF_DZ_RX_DESC_WPTR_WIDTH 12
-
/*
* TX_DESC_UPD_REG(96bit):
*
@@ -543,6 +542,31 @@ extern "C" {
#define ERF_DD_EVQ_IND_TIMER_VAL_LBN 0
#define ERF_DD_EVQ_IND_TIMER_VAL_WIDTH 8
+/* Packed stream magic doorbell command */
+#define ERF_DZ_RX_DESC_MAGIC_DOORBELL_LBN 11
+#define ERF_DZ_RX_DESC_MAGIC_DOORBELL_WIDTH 1
+
+#define ERF_DZ_RX_DESC_MAGIC_CMD_LBN 8
+#define ERF_DZ_RX_DESC_MAGIC_CMD_WIDTH 3
+#define ERE_DZ_RX_DESC_MAGIC_CMD_PS_CREDITS 0
+
+#define ERF_DZ_RX_DESC_MAGIC_DATA_LBN 0
+#define ERF_DZ_RX_DESC_MAGIC_DATA_WIDTH 8
+
+/* Packed stream RX packet prefix */
+#define ES_DZ_PS_RX_PREFIX_TSTAMP_LBN 0
+#define ES_DZ_PS_RX_PREFIX_TSTAMP_WIDTH 32
+#define ES_DZ_PS_RX_PREFIX_CAP_LEN_LBN 32
+#define ES_DZ_PS_RX_PREFIX_CAP_LEN_WIDTH 16
+#define ES_DZ_PS_RX_PREFIX_ORIG_LEN_LBN 48
+#define ES_DZ_PS_RX_PREFIX_ORIG_LEN_WIDTH 16
+
+/*
+ * An extra flag for the packed stream mode,
+ * signalling the start of a new buffer
+ */
+#define ESF_DZ_RX_EV_ROTATE_LBN 53
+#define ESF_DZ_RX_EV_ROTATE_WIDTH 1
#ifdef __cplusplus
}
Modified: head/sys/dev/sfxge/common/efx_rx.c
==============================================================================
--- head/sys/dev/sfxge/common/efx_rx.c Thu Nov 22 14:10:46 2018 (r340765)
+++ head/sys/dev/sfxge/common/efx_rx.c Thu Nov 22 14:31:35 2018 (r340766)
@@ -103,6 +103,22 @@ siena_rx_qpush(
__in unsigned int added,
__inout unsigned int *pushedp);
+#if EFSYS_OPT_RX_PACKED_STREAM
+static void
+siena_rx_qpush_ps_credits(
+ __in efx_rxq_t *erp);
+
+static __checkReturn uint8_t *
+siena_rx_qps_packet_info(
+ __in efx_rxq_t *erp,
+ __in uint8_t *buffer,
+ __in uint32_t buffer_length,
+ __in uint32_t current_offset,
+ __out uint16_t *lengthp,
+ __out uint32_t *next_offsetp,
+ __out uint32_t *timestamp);
+#endif
+
static __checkReturn efx_rc_t
siena_rx_qflush(
__in efx_rxq_t *erp);
@@ -146,6 +162,10 @@ static const efx_rx_ops_t __efx_rx_siena_ops = {
siena_rx_prefix_pktlen, /* erxo_prefix_pktlen */
siena_rx_qpost, /* erxo_qpost */
siena_rx_qpush, /* erxo_qpush */
+#if EFSYS_OPT_RX_PACKED_STREAM
+ siena_rx_qpush_ps_credits, /* erxo_qpush_ps_credits */
+ siena_rx_qps_packet_info, /* erxo_qps_packet_info */
+#endif
siena_rx_qflush, /* erxo_qflush */
siena_rx_qenable, /* erxo_qenable */
siena_rx_qcreate, /* erxo_qcreate */
@@ -169,6 +189,10 @@ static const efx_rx_ops_t __efx_rx_ef10_ops = {
ef10_rx_prefix_pktlen, /* erxo_prefix_pktlen */
ef10_rx_qpost, /* erxo_qpost */
ef10_rx_qpush, /* erxo_qpush */
+#if EFSYS_OPT_RX_PACKED_STREAM
+ ef10_rx_qpush_ps_credits, /* erxo_qpush_ps_credits */
+ ef10_rx_qps_packet_info, /* erxo_qps_packet_info */
+#endif
ef10_rx_qflush, /* erxo_qflush */
ef10_rx_qenable, /* erxo_qenable */
ef10_rx_qcreate, /* erxo_qcreate */
@@ -430,7 +454,41 @@ efx_rx_qpost(
erxop->erxo_qpost(erp, addrp, size, n, completed, added);
}
+#if EFSYS_OPT_RX_PACKED_STREAM
+
void
+efx_rx_qpush_ps_credits(
+ __in efx_rxq_t *erp)
+{
+ efx_nic_t *enp = erp->er_enp;
+ const efx_rx_ops_t *erxop = enp->en_erxop;
+
+ EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC);
+
+ erxop->erxo_qpush_ps_credits(erp);
+}
+
+ __checkReturn uint8_t *
+efx_rx_qps_packet_info(
+ __in efx_rxq_t *erp,
+ __in uint8_t *buffer,
+ __in uint32_t buffer_length,
+ __in uint32_t current_offset,
+ __out uint16_t *lengthp,
+ __out uint32_t *next_offsetp,
+ __out uint32_t *timestamp)
+{
+ efx_nic_t *enp = erp->er_enp;
+ const efx_rx_ops_t *erxop = enp->en_erxop;
+
+ return (erxop->erxo_qps_packet_info(erp, buffer,
+ buffer_length, current_offset, lengthp,
+ next_offsetp, timestamp));
+}
+
+#endif /* EFSYS_OPT_RX_PACKED_STREAM */
+
+ void
efx_rx_qpush(
__in efx_rxq_t *erp,
__in unsigned int added,
@@ -1075,6 +1133,32 @@ siena_rx_qpush(
EFX_BAR_TBL_WRITED3(enp, FR_BZ_RX_DESC_UPD_REGP0,
erp->er_index, &dword, B_FALSE);
}
+
+#if EFSYS_OPT_RX_PACKED_STREAM
+static void
+siena_rx_qpush_ps_credits(
+ __in efx_rxq_t *erp)
+{
+ /* Not supported by Siena hardware */
+ EFSYS_ASSERT(0);
+}
+
+static uint8_t *
+siena_rx_qps_packet_info(
+ __in efx_rxq_t *erp,
+ __in uint8_t *buffer,
+ __in uint32_t buffer_length,
+ __in uint32_t current_offset,
+ __out uint16_t *lengthp,
+ __out uint32_t *next_offsetp,
+ __out uint32_t *timestamp)
+{
+ /* Not supported by Siena hardware */
+ EFSYS_ASSERT(0);
+
+ return (NULL);
+}
+#endif /* EFSYS_OPT_RX_PACKED_STREAM */
static __checkReturn efx_rc_t
siena_rx_qflush(
Modified: head/sys/dev/sfxge/common/siena_nic.c
==============================================================================
--- head/sys/dev/sfxge/common/siena_nic.c Thu Nov 22 14:10:46 2018 (r340765)
+++ head/sys/dev/sfxge/common/siena_nic.c Thu Nov 22 14:31:35 2018 (r340766)
@@ -167,6 +167,8 @@ siena_board_cfg(
encp->enc_fw_assisted_tso_v2_enabled = B_FALSE;
encp->enc_fw_assisted_tso_v2_n_contexts = 0;
encp->enc_allow_set_mac_with_installed_filters = B_TRUE;
+ encp->enc_rx_packed_stream_supported = B_FALSE;
+ encp->enc_rx_var_packed_stream_supported = B_FALSE;
/* Siena supports two 10G ports, and 8 lanes of PCIe Gen2 */
encp->enc_required_pcie_bandwidth_mbps = 2 * 10000;
More information about the svn-src-all
mailing list