svn commit: r353900 - head/sys/dev/cxgbe
John Baldwin
jhb at FreeBSD.org
Tue Oct 22 20:41:55 UTC 2019
Author: jhb
Date: Tue Oct 22 20:41:54 2019
New Revision: 353900
URL: https://svnweb.freebsd.org/changeset/base/353900
Log:
Split Chelsio send tags into a generic base tag and a ratelimit tag.
NIC KTLS will add a new TLS send tag type in cxgbe(4) that is a
distinct tag from a ratelimit tag. To support this, refactor
cxgbe_snd_tag to be a simple send tag with a type and convert the
existing ratelimit tag to a new cxgbe_rate_tag structure.
Reviewed by: np
Sponsored by: Chelsio Communications
Differential Revision: https://reviews.freebsd.org/D22072
Modified:
head/sys/dev/cxgbe/adapter.h
head/sys/dev/cxgbe/offload.h
head/sys/dev/cxgbe/t4_main.c
head/sys/dev/cxgbe/t4_sched.c
head/sys/dev/cxgbe/t4_sge.c
Modified: head/sys/dev/cxgbe/adapter.h
==============================================================================
--- head/sys/dev/cxgbe/adapter.h Tue Oct 22 20:01:47 2019 (r353899)
+++ head/sys/dev/cxgbe/adapter.h Tue Oct 22 20:41:54 2019 (r353900)
@@ -1155,6 +1155,7 @@ int update_mac_settings(struct ifnet *, int);
int adapter_full_init(struct adapter *);
int adapter_full_uninit(struct adapter *);
uint64_t cxgbe_get_counter(struct ifnet *, ift_counter);
+void cxgbe_snd_tag_init(struct cxgbe_snd_tag *, struct ifnet *, int);
int vi_full_init(struct vi_info *);
int vi_full_uninit(struct vi_info *);
void vi_sysctls(struct vi_info *);
@@ -1212,7 +1213,7 @@ void t4_register_cpl_handler(int, cpl_handler_t);
void t4_register_shared_cpl_handler(int, cpl_handler_t, int);
#ifdef RATELIMIT
int ethofld_transmit(struct ifnet *, struct mbuf *);
-void send_etid_flush_wr(struct cxgbe_snd_tag *);
+void send_etid_flush_wr(struct cxgbe_rate_tag *);
#endif
/* t4_tracer.c */
@@ -1238,13 +1239,13 @@ int sysctl_tc_params(SYSCTL_HANDLER_ARGS);
#ifdef RATELIMIT
void t4_init_etid_table(struct adapter *);
void t4_free_etid_table(struct adapter *);
-struct cxgbe_snd_tag *lookup_etid(struct adapter *, int);
-int cxgbe_snd_tag_alloc(struct ifnet *, union if_snd_tag_alloc_params *,
+struct cxgbe_rate_tag *lookup_etid(struct adapter *, int);
+int cxgbe_rate_tag_alloc(struct ifnet *, union if_snd_tag_alloc_params *,
struct m_snd_tag **);
-int cxgbe_snd_tag_modify(struct m_snd_tag *, union if_snd_tag_modify_params *);
-int cxgbe_snd_tag_query(struct m_snd_tag *, union if_snd_tag_query_params *);
-void cxgbe_snd_tag_free(struct m_snd_tag *);
-void cxgbe_snd_tag_free_locked(struct cxgbe_snd_tag *);
+int cxgbe_rate_tag_modify(struct m_snd_tag *, union if_snd_tag_modify_params *);
+int cxgbe_rate_tag_query(struct m_snd_tag *, union if_snd_tag_query_params *);
+void cxgbe_rate_tag_free(struct m_snd_tag *);
+void cxgbe_rate_tag_free_locked(struct cxgbe_rate_tag *);
void cxgbe_ratelimit_query(struct ifnet *, struct if_ratelimit_query_results *);
#endif
Modified: head/sys/dev/cxgbe/offload.h
==============================================================================
--- head/sys/dev/cxgbe/offload.h Tue Oct 22 20:01:47 2019 (r353899)
+++ head/sys/dev/cxgbe/offload.h Tue Oct 22 20:41:54 2019 (r353900)
@@ -79,7 +79,7 @@ union aopen_entry {
union aopen_entry *next;
};
-/* cxgbe_snd_tag flags */
+/* cxgbe_rate_tag flags */
enum {
EO_FLOWC_PENDING = (1 << 0), /* flowc needs to be sent */
EO_FLOWC_RPL_PENDING = (1 << 1), /* flowc credits due back */
@@ -89,6 +89,11 @@ enum {
struct cxgbe_snd_tag {
struct m_snd_tag com;
+ int type;
+};
+
+struct cxgbe_rate_tag {
+ struct cxgbe_snd_tag com;
struct adapter *adapter;
u_int flags;
struct mtx lock;
@@ -114,8 +119,14 @@ mst_to_cst(struct m_snd_tag *t)
return (__containerof(t, struct cxgbe_snd_tag, com));
}
+static inline struct cxgbe_rate_tag *
+mst_to_crt(struct m_snd_tag *t)
+{
+ return ((struct cxgbe_rate_tag *)mst_to_cst(t));
+}
+
union etid_entry {
- struct cxgbe_snd_tag *cst;
+ struct cxgbe_rate_tag *cst;
union etid_entry *next;
};
Modified: head/sys/dev/cxgbe/t4_main.c
==============================================================================
--- head/sys/dev/cxgbe/t4_main.c Tue Oct 22 20:01:47 2019 (r353899)
+++ head/sys/dev/cxgbe/t4_main.c Tue Oct 22 20:41:54 2019 (r353900)
@@ -230,6 +230,15 @@ static void cxgbe_init(void *);
static int cxgbe_ioctl(struct ifnet *, unsigned long, caddr_t);
static int cxgbe_transmit(struct ifnet *, struct mbuf *);
static void cxgbe_qflush(struct ifnet *);
+#ifdef RATELIMIT
+static int cxgbe_snd_tag_alloc(struct ifnet *, union if_snd_tag_alloc_params *,
+ struct m_snd_tag **);
+static int cxgbe_snd_tag_modify(struct m_snd_tag *,
+ union if_snd_tag_modify_params *);
+static int cxgbe_snd_tag_query(struct m_snd_tag *,
+ union if_snd_tag_query_params *);
+static void cxgbe_snd_tag_free(struct m_snd_tag *);
+#endif
MALLOC_DEFINE(M_CXGBE, "cxgbe", "Chelsio T4/T5 Ethernet driver and services");
@@ -2044,11 +2053,18 @@ cxgbe_transmit(struct ifnet *ifp, struct mbuf *m)
struct port_info *pi = vi->pi;
struct adapter *sc = pi->adapter;
struct sge_txq *txq;
+#ifdef RATELIMIT
+ struct cxgbe_snd_tag *cst;
+#endif
void *items[1];
int rc;
M_ASSERTPKTHDR(m);
MPASS(m->m_nextpkt == NULL); /* not quite ready for this yet */
+#ifdef RATELIMIT
+ if (m->m_pkthdr.csum_flags & CSUM_SND_TAG)
+ MPASS(m->m_pkthdr.snd_tag->ifp == ifp);
+#endif
if (__predict_false(pi->link_cfg.link_ok == false)) {
m_freem(m);
@@ -2063,8 +2079,9 @@ cxgbe_transmit(struct ifnet *ifp, struct mbuf *m)
}
#ifdef RATELIMIT
if (m->m_pkthdr.csum_flags & CSUM_SND_TAG) {
- MPASS(m->m_pkthdr.snd_tag->ifp == ifp);
- return (ethofld_transmit(ifp, m));
+ cst = mst_to_cst(m->m_pkthdr.snd_tag);
+ if (cst->type == IF_SND_TAG_TYPE_RATE_LIMIT)
+ return (ethofld_transmit(ifp, m));
}
#endif
@@ -2221,6 +2238,87 @@ cxgbe_get_counter(struct ifnet *ifp, ift_counter c)
return (if_get_counter_default(ifp, c));
}
}
+
+#ifdef RATELIMIT
+void
+cxgbe_snd_tag_init(struct cxgbe_snd_tag *cst, struct ifnet *ifp, int type)
+{
+
+ m_snd_tag_init(&cst->com, ifp);
+ cst->type = type;
+}
+
+static int
+cxgbe_snd_tag_alloc(struct ifnet *ifp, union if_snd_tag_alloc_params *params,
+ struct m_snd_tag **pt)
+{
+ int error;
+
+ switch (params->hdr.type) {
+#ifdef RATELIMIT
+ case IF_SND_TAG_TYPE_RATE_LIMIT:
+ error = cxgbe_rate_tag_alloc(ifp, params, pt);
+ break;
+#endif
+ default:
+ error = EOPNOTSUPP;
+ }
+ if (error == 0)
+ MPASS(mst_to_cst(*pt)->type == params->hdr.type);
+ return (error);
+}
+
+static int
+cxgbe_snd_tag_modify(struct m_snd_tag *mst,
+ union if_snd_tag_modify_params *params)
+{
+ struct cxgbe_snd_tag *cst;
+
+ cst = mst_to_cst(mst);
+ switch (cst->type) {
+#ifdef RATELIMIT
+ case IF_SND_TAG_TYPE_RATE_LIMIT:
+ return (cxgbe_rate_tag_modify(mst, params));
+#endif
+ default:
+ return (EOPNOTSUPP);
+ }
+}
+
+static int
+cxgbe_snd_tag_query(struct m_snd_tag *mst,
+ union if_snd_tag_query_params *params)
+{
+ struct cxgbe_snd_tag *cst;
+
+ cst = mst_to_cst(mst);
+ switch (cst->type) {
+#ifdef RATELIMIT
+ case IF_SND_TAG_TYPE_RATE_LIMIT:
+ return (cxgbe_rate_tag_query(mst, params));
+#endif
+ default:
+ return (EOPNOTSUPP);
+ }
+}
+
+static void
+cxgbe_snd_tag_free(struct m_snd_tag *mst)
+{
+ struct cxgbe_snd_tag *cst;
+
+ cst = mst_to_cst(mst);
+ switch (cst->type) {
+#ifdef RATELIMIT
+ case IF_SND_TAG_TYPE_RATE_LIMIT:
+ cxgbe_rate_tag_free(mst);
+ return;
+#endif
+ default:
+ panic("shouldn't get here");
+ }
+}
+#endif
/*
* The kernel picks a media from the list we had provided but we still validate
Modified: head/sys/dev/cxgbe/t4_sched.c
==============================================================================
--- head/sys/dev/cxgbe/t4_sched.c Tue Oct 22 20:01:47 2019 (r353899)
+++ head/sys/dev/cxgbe/t4_sched.c Tue Oct 22 20:41:54 2019 (r353900)
@@ -711,11 +711,11 @@ t4_free_etid_table(struct adapter *sc)
}
/* etid services */
-static int alloc_etid(struct adapter *, struct cxgbe_snd_tag *);
+static int alloc_etid(struct adapter *, struct cxgbe_rate_tag *);
static void free_etid(struct adapter *, int);
static int
-alloc_etid(struct adapter *sc, struct cxgbe_snd_tag *cst)
+alloc_etid(struct adapter *sc, struct cxgbe_rate_tag *cst)
{
struct tid_info *t = &sc->tids;
int etid = -1;
@@ -733,7 +733,7 @@ alloc_etid(struct adapter *sc, struct cxgbe_snd_tag *c
return (etid);
}
-struct cxgbe_snd_tag *
+struct cxgbe_rate_tag *
lookup_etid(struct adapter *sc, int etid)
{
struct tid_info *t = &sc->tids;
@@ -755,17 +755,16 @@ free_etid(struct adapter *sc, int etid)
}
int
-cxgbe_snd_tag_alloc(struct ifnet *ifp, union if_snd_tag_alloc_params *params,
+cxgbe_rate_tag_alloc(struct ifnet *ifp, union if_snd_tag_alloc_params *params,
struct m_snd_tag **pt)
{
int rc, schedcl;
struct vi_info *vi = ifp->if_softc;
struct port_info *pi = vi->pi;
struct adapter *sc = pi->adapter;
- struct cxgbe_snd_tag *cst;
+ struct cxgbe_rate_tag *cst;
- if (params->hdr.type != IF_SND_TAG_TYPE_RATE_LIMIT)
- return (ENOTSUP);
+ MPASS(params->hdr.type == IF_SND_TAG_TYPE_RATE_LIMIT);
rc = t4_reserve_cl_rl_kbps(sc, pi->port_id,
(params->rate_limit.max_rate * 8ULL / 1000), &schedcl);
@@ -789,7 +788,7 @@ failed:
mtx_init(&cst->lock, "cst_lock", NULL, MTX_DEF);
mbufq_init(&cst->pending_tx, INT_MAX);
mbufq_init(&cst->pending_fwack, INT_MAX);
- m_snd_tag_init(&cst->com, ifp);
+ cxgbe_snd_tag_init(&cst->com, ifp, IF_SND_TAG_TYPE_RATE_LIMIT);
cst->flags |= EO_FLOWC_PENDING | EO_SND_TAG_REF;
cst->adapter = sc;
cst->port_id = pi->port_id;
@@ -806,7 +805,7 @@ failed:
* Queues will be selected later when the connection flowid is available.
*/
- *pt = &cst->com;
+ *pt = &cst->com.com;
return (0);
}
@@ -814,11 +813,11 @@ failed:
* Change in parameters, no change in ifp.
*/
int
-cxgbe_snd_tag_modify(struct m_snd_tag *mst,
+cxgbe_rate_tag_modify(struct m_snd_tag *mst,
union if_snd_tag_modify_params *params)
{
int rc, schedcl;
- struct cxgbe_snd_tag *cst = mst_to_cst(mst);
+ struct cxgbe_rate_tag *cst = mst_to_crt(mst);
struct adapter *sc = cst->adapter;
/* XXX: is schedcl -1 ok here? */
@@ -840,10 +839,10 @@ cxgbe_snd_tag_modify(struct m_snd_tag *mst,
}
int
-cxgbe_snd_tag_query(struct m_snd_tag *mst,
+cxgbe_rate_tag_query(struct m_snd_tag *mst,
union if_snd_tag_query_params *params)
{
- struct cxgbe_snd_tag *cst = mst_to_cst(mst);
+ struct cxgbe_rate_tag *cst = mst_to_crt(mst);
params->rate_limit.max_rate = cst->max_rate;
@@ -858,7 +857,7 @@ cxgbe_snd_tag_query(struct m_snd_tag *mst,
* Unlocks cst and frees it.
*/
void
-cxgbe_snd_tag_free_locked(struct cxgbe_snd_tag *cst)
+cxgbe_rate_tag_free_locked(struct cxgbe_rate_tag *cst)
{
struct adapter *sc = cst->adapter;
@@ -879,9 +878,9 @@ cxgbe_snd_tag_free_locked(struct cxgbe_snd_tag *cst)
}
void
-cxgbe_snd_tag_free(struct m_snd_tag *mst)
+cxgbe_rate_tag_free(struct m_snd_tag *mst)
{
- struct cxgbe_snd_tag *cst = mst_to_cst(mst);
+ struct cxgbe_rate_tag *cst = mst_to_crt(mst);
mtx_lock(&cst->lock);
@@ -896,7 +895,7 @@ cxgbe_snd_tag_free(struct m_snd_tag *mst)
* credits for the etid otherwise.
*/
if (cst->tx_credits == cst->tx_total) {
- cxgbe_snd_tag_free_locked(cst);
+ cxgbe_rate_tag_free_locked(cst);
return; /* cst is gone. */
}
send_etid_flush_wr(cst);
Modified: head/sys/dev/cxgbe/t4_sge.c
==============================================================================
--- head/sys/dev/cxgbe/t4_sge.c Tue Oct 22 20:01:47 2019 (r353899)
+++ head/sys/dev/cxgbe/t4_sge.c Tue Oct 22 20:41:54 2019 (r353900)
@@ -2307,10 +2307,10 @@ set_mbuf_eo_tsclk_tsoff(struct mbuf *m, uint8_t tsclk_
}
static inline int
-needs_eo(struct mbuf *m)
+needs_eo(struct cxgbe_snd_tag *cst)
{
- return (m->m_pkthdr.csum_flags & CSUM_SND_TAG);
+ return (cst != NULL && cst->type == IF_SND_TAG_TYPE_RATE_LIMIT);
}
#endif
@@ -2542,6 +2542,9 @@ parse_pkt(struct adapter *sc, struct mbuf **mp)
#if defined(INET) || defined(INET6)
struct tcphdr *tcp;
#endif
+#ifdef RATELIMIT
+ struct cxgbe_snd_tag *cst;
+#endif
uint16_t eh_type;
uint8_t cflags;
@@ -2562,6 +2565,12 @@ restart:
M_ASSERTPKTHDR(m0);
MPASS(m0->m_pkthdr.len > 0);
nsegs = count_mbuf_nsegs(m0, 0, &cflags);
+#ifdef RATELIMIT
+ if (m0->m_pkthdr.csum_flags & CSUM_SND_TAG)
+ cst = mst_to_cst(m0->m_pkthdr.snd_tag);
+ else
+ cst = NULL;
+#endif
if (nsegs > (needs_tso(m0) ? TX_SGL_SEGS_TSO : TX_SGL_SEGS)) {
if (defragged++ > 0 || (m = m_defrag(m0, M_NOWAIT)) == NULL) {
rc = EFBIG;
@@ -2595,16 +2604,17 @@ restart:
* checksumming is enabled. needs_l4_csum happens to check for all the
* right things.
*/
- if (__predict_false(needs_eo(m0) && !needs_l4_csum(m0))) {
+ if (__predict_false(needs_eo(cst) && !needs_l4_csum(m0))) {
m_snd_tag_rele(m0->m_pkthdr.snd_tag);
m0->m_pkthdr.snd_tag = NULL;
m0->m_pkthdr.csum_flags &= ~CSUM_SND_TAG;
+ cst = NULL;
}
#endif
if (!needs_tso(m0) &&
#ifdef RATELIMIT
- !needs_eo(m0) &&
+ !needs_eo(cst) &&
#endif
!(sc->flags & IS_VF && (needs_l3_csum(m0) || needs_l4_csum(m0))))
return (0);
@@ -2666,7 +2676,7 @@ restart:
#endif
}
#ifdef RATELIMIT
- if (needs_eo(m0)) {
+ if (needs_eo(cst)) {
u_int immhdrs;
/* EO WRs have the headers in the WR and not the GL. */
@@ -5723,7 +5733,7 @@ done:
#define ETID_FLOWC_LEN16 (howmany(ETID_FLOWC_LEN, 16))
static int
-send_etid_flowc_wr(struct cxgbe_snd_tag *cst, struct port_info *pi,
+send_etid_flowc_wr(struct cxgbe_rate_tag *cst, struct port_info *pi,
struct vi_info *vi)
{
struct wrq_cookie cookie;
@@ -5769,7 +5779,7 @@ send_etid_flowc_wr(struct cxgbe_snd_tag *cst, struct p
#define ETID_FLUSH_LEN16 (howmany(sizeof (struct fw_flowc_wr), 16))
void
-send_etid_flush_wr(struct cxgbe_snd_tag *cst)
+send_etid_flush_wr(struct cxgbe_rate_tag *cst)
{
struct fw_flowc_wr *flowc;
struct wrq_cookie cookie;
@@ -5795,7 +5805,7 @@ send_etid_flush_wr(struct cxgbe_snd_tag *cst)
}
static void
-write_ethofld_wr(struct cxgbe_snd_tag *cst, struct fw_eth_tx_eo_wr *wr,
+write_ethofld_wr(struct cxgbe_rate_tag *cst, struct fw_eth_tx_eo_wr *wr,
struct mbuf *m0, int compl)
{
struct cpl_tx_pkt_core *cpl;
@@ -5944,7 +5954,7 @@ write_ethofld_wr(struct cxgbe_snd_tag *cst, struct fw_
}
static void
-ethofld_tx(struct cxgbe_snd_tag *cst)
+ethofld_tx(struct cxgbe_rate_tag *cst)
{
struct mbuf *m;
struct wrq_cookie cookie;
@@ -5977,7 +5987,7 @@ ethofld_tx(struct cxgbe_snd_tag *cst)
cst->tx_credits -= next_credits;
cst->tx_nocompl += next_credits;
compl = cst->ncompl == 0 || cst->tx_nocompl >= cst->tx_total / 2;
- ETHER_BPF_MTAP(cst->com.ifp, m);
+ ETHER_BPF_MTAP(cst->com.com.ifp, m);
write_ethofld_wr(cst, wr, m, compl);
commit_wrq_wr(cst->eo_txq, wr, &cookie);
if (compl) {
@@ -5989,7 +5999,7 @@ ethofld_tx(struct cxgbe_snd_tag *cst)
/*
* Drop the mbuf's reference on the tag now rather
* than waiting until m_freem(). This ensures that
- * cxgbe_snd_tag_free gets called when the inp drops
+ * cxgbe_rate_tag_free gets called when the inp drops
* its reference on the tag and there are no more
* mbufs in the pending_tx queue and can flush any
* pending requests. Otherwise if the last mbuf
@@ -5998,7 +6008,7 @@ ethofld_tx(struct cxgbe_snd_tag *cst)
*/
m->m_pkthdr.snd_tag = NULL;
m->m_pkthdr.csum_flags &= ~CSUM_SND_TAG;
- m_snd_tag_rele(&cst->com);
+ m_snd_tag_rele(&cst->com.com);
mbufq_enqueue(&cst->pending_fwack, m);
}
@@ -6007,13 +6017,13 @@ ethofld_tx(struct cxgbe_snd_tag *cst)
int
ethofld_transmit(struct ifnet *ifp, struct mbuf *m0)
{
- struct cxgbe_snd_tag *cst;
+ struct cxgbe_rate_tag *cst;
int rc;
MPASS(m0->m_nextpkt == NULL);
MPASS(m0->m_pkthdr.csum_flags & CSUM_SND_TAG);
MPASS(m0->m_pkthdr.snd_tag != NULL);
- cst = mst_to_cst(m0->m_pkthdr.snd_tag);
+ cst = mst_to_crt(m0->m_pkthdr.snd_tag);
mtx_lock(&cst->lock);
MPASS(cst->flags & EO_SND_TAG_REF);
@@ -6052,10 +6062,10 @@ ethofld_transmit(struct ifnet *ifp, struct mbuf *m0)
* ethofld_tx() in case we are sending the final mbuf after
* the inp was freed.
*/
- m_snd_tag_ref(&cst->com);
+ m_snd_tag_ref(&cst->com.com);
ethofld_tx(cst);
mtx_unlock(&cst->lock);
- m_snd_tag_rele(&cst->com);
+ m_snd_tag_rele(&cst->com.com);
return (0);
done:
@@ -6072,7 +6082,7 @@ ethofld_fw4_ack(struct sge_iq *iq, const struct rss_he
const struct cpl_fw4_ack *cpl = (const void *)(rss + 1);
struct mbuf *m;
u_int etid = G_CPL_FW4_ACK_FLOWID(be32toh(OPCODE_TID(cpl)));
- struct cxgbe_snd_tag *cst;
+ struct cxgbe_rate_tag *cst;
uint8_t credits = cpl->credits;
cst = lookup_etid(sc, etid);
@@ -6104,7 +6114,7 @@ ethofld_fw4_ack(struct sge_iq *iq, const struct rss_he
cst->flags &= ~EO_FLUSH_RPL_PENDING;
cst->tx_credits += cpl->credits;
- cxgbe_snd_tag_free_locked(cst);
+ cxgbe_rate_tag_free_locked(cst);
return (0); /* cst is gone. */
}
KASSERT(m != NULL,
@@ -6126,12 +6136,12 @@ ethofld_fw4_ack(struct sge_iq *iq, const struct rss_he
* As with ethofld_transmit(), hold an extra reference
* so that the tag is stable across ethold_tx().
*/
- m_snd_tag_ref(&cst->com);
+ m_snd_tag_ref(&cst->com.com);
m = mbufq_first(&cst->pending_tx);
if (m != NULL && cst->tx_credits >= mbuf_eo_len16(m))
ethofld_tx(cst);
mtx_unlock(&cst->lock);
- m_snd_tag_rele(&cst->com);
+ m_snd_tag_rele(&cst->com.com);
} else {
/*
* There shouldn't be any pending packets if the tag
More information about the svn-src-all
mailing list