svn commit: r337538 - in head/sys/dev/cxgbe: . common tom
Navdeep Parhar
np at FreeBSD.org
Thu Aug 9 14:19:49 UTC 2018
Author: np
Date: Thu Aug 9 14:19:47 2018
New Revision: 337538
URL: https://svnweb.freebsd.org/changeset/base/337538
Log:
cxgbe(4): Add support for high priority filters on T6+. They have their
own region in the TCAM starting with T6, unlike previous chips where
they were in the same region as normal filters.
These filters "hit" before anything else in the LE's lookup. The exact
order is:
a) High priority filters
b) TOE's active region (TCAM and/or hash)
c) Servers (TOE hw listeners)
d) Normal filters
MFC after: 1 week
Sponsored by: Chelsio Communications
Modified:
head/sys/dev/cxgbe/common/common.h
head/sys/dev/cxgbe/offload.h
head/sys/dev/cxgbe/t4_filter.c
head/sys/dev/cxgbe/t4_main.c
head/sys/dev/cxgbe/t4_sge.c
head/sys/dev/cxgbe/tom/t4_tom.c
Modified: head/sys/dev/cxgbe/common/common.h
==============================================================================
--- head/sys/dev/cxgbe/common/common.h Thu Aug 9 14:04:51 2018 (r337537)
+++ head/sys/dev/cxgbe/common/common.h Thu Aug 9 14:19:47 2018 (r337538)
@@ -441,6 +441,13 @@ static inline int is_ftid(const struct adapter *sc, u_
tid <= sc->tids.ftid_end);
}
+static inline int is_hpftid(const struct adapter *sc, u_int tid)
+{
+
+ return (sc->tids.nhpftids > 0 && tid >= sc->tids.hpftid_base &&
+ tid <= sc->tids.hpftid_end);
+}
+
static inline int is_etid(const struct adapter *sc, u_int tid)
{
Modified: head/sys/dev/cxgbe/offload.h
==============================================================================
--- head/sys/dev/cxgbe/offload.h Thu Aug 9 14:04:51 2018 (r337537)
+++ head/sys/dev/cxgbe/offload.h Thu Aug 9 14:19:47 2018 (r337538)
@@ -133,7 +133,12 @@ struct tid_info {
u_int ftid_base;
u_int ftid_end;
+ u_int nhpftids;
+ u_int hpftid_base;
+ u_int hpftid_end;
+
u_int ntids;
+ u_int tid_base;
u_int netids;
u_int etid_base;
@@ -150,10 +155,13 @@ struct tid_info {
union aopen_entry *afree;
u_int atids_in_use;
+ /* High priority filters and normal filters share the lock and cv. */
struct mtx ftid_lock __aligned(CACHE_LINE_SIZE);
struct cv ftid_cv;
struct filter_entry *ftid_tab;
+ struct filter_entry *hpftid_tab;
u_int ftids_in_use;
+ u_int hpftids_in_use;
/*
* hashfilter and TOE are mutually exclusive and both use ntids and
Modified: head/sys/dev/cxgbe/t4_filter.c
==============================================================================
--- head/sys/dev/cxgbe/t4_filter.c Thu Aug 9 14:04:51 2018 (r337537)
+++ head/sys/dev/cxgbe/t4_filter.c Thu Aug 9 14:19:47 2018 (r337538)
@@ -64,12 +64,20 @@ struct filter_entry {
};
static void free_filter_resources(struct filter_entry *);
+static int get_tcamfilter(struct adapter *, struct t4_filter *);
static int get_hashfilter(struct adapter *, struct t4_filter *);
static int set_hashfilter(struct adapter *, struct t4_filter *, uint64_t,
struct l2t_entry *, struct smt_entry *);
static int del_hashfilter(struct adapter *, struct t4_filter *);
static int configure_hashfilter_tcb(struct adapter *, struct filter_entry *);
+static inline bool
+separate_hpfilter_region(struct adapter *sc)
+{
+
+ return (chip_id(sc) >= CHELSIO_T6);
+}
+
static int
alloc_hftid_tab(struct tid_info *t, int flags)
{
@@ -302,7 +310,7 @@ set_filter_mode(struct adapter *sc, uint32_t mode)
if (rc)
return (rc);
- if (sc->tids.ftids_in_use > 0) {
+ if (sc->tids.ftids_in_use > 0 || sc->tids.hpftids_in_use > 0) {
rc = EBUSY;
goto done;
}
@@ -343,39 +351,10 @@ get_filter_hits(struct adapter *sc, uint32_t tid)
int
get_filter(struct adapter *sc, struct t4_filter *t)
{
- int i, nfilters = sc->tids.nftids;
- struct filter_entry *f;
-
if (t->fs.hash)
return (get_hashfilter(sc, t));
-
- if (sc->tids.ftids_in_use == 0 || sc->tids.ftid_tab == NULL ||
- t->idx >= nfilters) {
- t->idx = 0xffffffff;
- return (0);
- }
-
- mtx_lock(&sc->tids.ftid_lock);
- f = &sc->tids.ftid_tab[t->idx];
- for (i = t->idx; i < nfilters; i++, f++) {
- if (f->valid) {
- MPASS(f->tid == sc->tids.ftid_base + i);
- t->idx = i;
- t->l2tidx = f->l2te ? f->l2te->idx : 0;
- t->smtidx = f->smt ? f->smt->idx : 0;
- if (f->fs.hitcnts)
- t->hits = get_filter_hits(sc, f->tid);
- else
- t->hits = UINT64_MAX;
- t->fs = f->fs;
-
- goto done;
- }
- }
- t->idx = 0xffffffff;
-done:
- mtx_unlock(&sc->tids.ftid_lock);
- return (0);
+ else
+ return (get_tcamfilter(sc, t));
}
static int
@@ -387,15 +366,23 @@ set_tcamfilter(struct adapter *sc, struct t4_filter *t
u_int vnic_vld, vnic_vld_mask;
struct wrq_cookie cookie;
int i, rc, busy, locked;
+ u_int tid;
const int ntids = t->fs.type ? 4 : 1;
MPASS(!t->fs.hash);
- MPASS(t->idx < sc->tids.nftids);
/* Already validated against fconf, iconf */
MPASS((t->fs.val.pfvf_vld & t->fs.val.ovlan_vld) == 0);
MPASS((t->fs.mask.pfvf_vld & t->fs.mask.ovlan_vld) == 0);
- f = &sc->tids.ftid_tab[t->idx];
+ if (separate_hpfilter_region(sc) && t->fs.prio) {
+ MPASS(t->idx < sc->tids.nhpftids);
+ f = &sc->tids.hpftid_tab[t->idx];
+ tid = sc->tids.hpftid_base + t->idx;
+ } else {
+ MPASS(t->idx < sc->tids.nftids);
+ f = &sc->tids.ftid_tab[t->idx];
+ tid = sc->tids.ftid_base + t->idx;
+ }
rc = busy = locked = 0;
mtx_lock(&sc->tids.ftid_lock);
for (i = 0; i < ntids; i++) {
@@ -418,7 +405,10 @@ set_tcamfilter(struct adapter *sc, struct t4_filter *t
rc = ENOMEM;
else {
f->pending = 1;
- sc->tids.ftids_in_use++;
+ if (separate_hpfilter_region(sc) && t->fs.prio)
+ sc->tids.hpftids_in_use++;
+ else
+ sc->tids.ftids_in_use++;
}
}
mtx_unlock(&sc->tids.ftid_lock);
@@ -434,7 +424,7 @@ set_tcamfilter(struct adapter *sc, struct t4_filter *t
* Can't fail now. A set-filter WR will definitely be sent.
*/
- f->tid = sc->tids.ftid_base + t->idx;
+ f->tid = tid;
f->fs = t->fs;
f->l2te = l2te;
f->smt = smt;
@@ -649,10 +639,17 @@ set_filter(struct adapter *sc, struct t4_filter *t)
if (rc != 0)
return (rc);
} else {
- if (ti->nftids == 0)
- return (ENOTSUP);
- if (t->idx >= ti->nftids)
- return (EINVAL);
+ if (separate_hpfilter_region(sc) && t->fs.prio) {
+ if (ti->nhpftids == 0)
+ return (ENOTSUP);
+ if (t->idx >= ti->nhpftids)
+ return (EINVAL);
+ } else {
+ if (ti->nftids == 0)
+ return (ENOTSUP);
+ if (t->idx >= ti->nftids)
+ return (EINVAL);
+ }
/* IPv6 filter idx must be 4 aligned */
if (t->fs.type == 1 &&
((t->idx & 0x3) || t->idx + 4 >= ti->nftids))
@@ -702,17 +699,37 @@ set_filter(struct adapter *sc, struct t4_filter *t)
if (rc != 0)
goto done;
}
+ } else if (separate_hpfilter_region(sc) && t->fs.prio &&
+ __predict_false(ti->hpftid_tab == NULL)) {
+ MPASS(ti->nhpftids != 0);
+ KASSERT(ti->hpftids_in_use == 0,
+ ("%s: no memory allocated but hpftids_in_use is %u",
+ __func__, ti->hpftids_in_use));
+ ti->hpftid_tab = malloc(sizeof(struct filter_entry) *
+ ti->nhpftids, M_CXGBE, M_NOWAIT | M_ZERO);
+ if (ti->hpftid_tab == NULL) {
+ rc = ENOMEM;
+ goto done;
+ }
+ if (!mtx_initialized(&sc->tids.ftid_lock)) {
+ mtx_init(&ti->ftid_lock, "T4 filters", 0, MTX_DEF);
+ cv_init(&ti->ftid_cv, "t4fcv");
+ }
} else if (__predict_false(ti->ftid_tab == NULL)) {
+ MPASS(ti->nftids != 0);
KASSERT(ti->ftids_in_use == 0,
- ("%s: no memory allocated but ftids_in_use > 0", __func__));
+ ("%s: no memory allocated but ftids_in_use is %u",
+ __func__, ti->ftids_in_use));
ti->ftid_tab = malloc(sizeof(struct filter_entry) * ti->nftids,
M_CXGBE, M_NOWAIT | M_ZERO);
if (ti->ftid_tab == NULL) {
rc = ENOMEM;
goto done;
}
- mtx_init(&ti->ftid_lock, "T4 filters", 0, MTX_DEF);
- cv_init(&ti->ftid_cv, "t4fcv");
+ if (!mtx_initialized(&sc->tids.ftid_lock)) {
+ mtx_init(&ti->ftid_lock, "T4 filters", 0, MTX_DEF);
+ cv_init(&ti->ftid_cv, "t4fcv");
+ }
}
done:
end_synchronized_op(sc, 0);
@@ -768,16 +785,32 @@ del_tcamfilter(struct adapter *sc, struct t4_filter *t
struct filter_entry *f;
struct fw_filter_wr *fwr;
struct wrq_cookie cookie;
- int rc;
+ int rc, nfilters;
+#ifdef INVARIANTS
+ u_int tid_base;
+#endif
- MPASS(sc->tids.ftid_tab != NULL);
- MPASS(sc->tids.nftids > 0);
-
- if (t->idx >= sc->tids.nftids)
- return (EINVAL);
-
mtx_lock(&sc->tids.ftid_lock);
- f = &sc->tids.ftid_tab[t->idx];
+ if (separate_hpfilter_region(sc) && t->fs.prio) {
+ nfilters = sc->tids.nhpftids;
+ f = sc->tids.hpftid_tab;
+#ifdef INVARIANTS
+ tid_base = sc->tids.hpftid_base;
+#endif
+ } else {
+ nfilters = sc->tids.nftids;
+ f = sc->tids.ftid_tab;
+#ifdef INVARIANTS
+ tid_base = sc->tids.ftid_base;
+#endif
+ }
+ MPASS(f != NULL); /* Caller checked this. */
+ if (t->idx >= nfilters) {
+ rc = EINVAL;
+ goto done;
+ }
+ f += t->idx;
+
if (f->locked) {
rc = EPERM;
goto done;
@@ -790,7 +823,7 @@ del_tcamfilter(struct adapter *sc, struct t4_filter *t
rc = EINVAL;
goto done;
}
- MPASS(f->tid == sc->tids.ftid_base + t->idx);
+ MPASS(f->tid == tid_base + t->idx);
fwr = start_wrq_wr(&sc->sge.mgmtq, howmany(sizeof(*fwr), 16), &cookie);
if (fwr == NULL) {
rc = ENOMEM;
@@ -833,6 +866,9 @@ del_filter(struct adapter *sc, struct t4_filter *t)
if (t->fs.hash) {
if (sc->tids.hftid_tab != NULL)
return (del_hashfilter(sc, t));
+ } else if (separate_hpfilter_region(sc) && t->fs.prio) {
+ if (sc->tids.hpftid_tab != NULL)
+ return (del_tcamfilter(sc, t));
} else {
if (sc->tids.ftid_tab != NULL)
return (del_tcamfilter(sc, t));
@@ -899,21 +935,28 @@ t4_filter_rpl(struct sge_iq *iq, const struct rss_head
struct adapter *sc = iq->adapter;
const struct cpl_set_tcb_rpl *rpl = (const void *)(rss + 1);
u_int tid = GET_TID(rpl);
- u_int rc, cleanup, idx;
+ u_int rc, idx;
struct filter_entry *f;
KASSERT(m == NULL, ("%s: payload with opcode %02x", __func__,
rss->opcode));
- MPASS(is_ftid(sc, tid));
- cleanup = 0;
- idx = tid - sc->tids.ftid_base;
- f = &sc->tids.ftid_tab[idx];
+
+ if (is_hpftid(sc, tid)) {
+ idx = tid - sc->tids.hpftid_base;
+ f = &sc->tids.hpftid_tab[idx];
+ } else if (is_ftid(sc, tid)) {
+ idx = tid - sc->tids.ftid_base;
+ f = &sc->tids.ftid_tab[idx];
+ } else
+ panic("%s: FW reply for invalid TID %d.", __func__, tid);
+
+ MPASS(f->tid == tid);
rc = G_COOKIE(rpl->cookie);
mtx_lock(&sc->tids.ftid_lock);
KASSERT(f->pending, ("%s: reply %d for filter[%u] that isn't pending.",
- __func__, rc, idx));
+ __func__, rc, tid));
switch(rc) {
case FW_FILTER_WR_FLT_ADDED:
/* set-filter succeeded */
@@ -936,7 +979,10 @@ t4_filter_rpl(struct sge_iq *iq, const struct rss_head
/* set-filter failed due to lack of SMT space. */
MPASS(f->valid == 0);
free_filter_resources(f);
- sc->tids.ftids_in_use--;
+ if (separate_hpfilter_region(sc) && f->fs.prio)
+ sc->tids.hpftids_in_use--;
+ else
+ sc->tids.ftids_in_use--;
break;
case FW_FILTER_WR_SUCCESS:
case FW_FILTER_WR_EINVAL:
@@ -1077,10 +1123,68 @@ t4_del_hashfilter_rpl(struct sge_iq *iq, const struct
}
static int
+get_tcamfilter(struct adapter *sc, struct t4_filter *t)
+{
+ int i, nfilters;
+ struct filter_entry *f;
+ u_int in_use;
+#ifdef INVARIANTS
+ u_int tid_base;
+#endif
+
+ MPASS(!t->fs.hash);
+
+ if (separate_hpfilter_region(sc) && t->fs.prio) {
+ nfilters = sc->tids.nhpftids;
+ f = sc->tids.hpftid_tab;
+ in_use = sc->tids.hpftids_in_use;
+#ifdef INVARIANTS
+ tid_base = sc->tids.hpftid_base;
+#endif
+ } else {
+ nfilters = sc->tids.nftids;
+ f = sc->tids.ftid_tab;
+ in_use = sc->tids.ftids_in_use;
+#ifdef INVARIANTS
+ tid_base = sc->tids.ftid_base;
+#endif
+ }
+
+ if (in_use == 0 || f == NULL || t->idx >= nfilters) {
+ t->idx = 0xffffffff;
+ return (0);
+ }
+
+ f += t->idx;
+ mtx_lock(&sc->tids.ftid_lock);
+ for (i = t->idx; i < nfilters; i++, f++) {
+ if (f->valid) {
+ MPASS(f->tid == tid_base + i);
+ t->idx = i;
+ t->l2tidx = f->l2te ? f->l2te->idx : 0;
+ t->smtidx = f->smt ? f->smt->idx : 0;
+ if (f->fs.hitcnts)
+ t->hits = get_filter_hits(sc, f->tid);
+ else
+ t->hits = UINT64_MAX;
+ t->fs = f->fs;
+
+ goto done;
+ }
+ }
+ t->idx = 0xffffffff;
+done:
+ mtx_unlock(&sc->tids.ftid_lock);
+ return (0);
+}
+
+static int
get_hashfilter(struct adapter *sc, struct t4_filter *t)
{
int i, nfilters = sc->tids.ntids;
struct filter_entry *f;
+
+ MPASS(t->fs.hash);
if (sc->tids.tids_in_use == 0 || sc->tids.hftid_tab == NULL ||
t->idx >= nfilters) {
Modified: head/sys/dev/cxgbe/t4_main.c
==============================================================================
--- head/sys/dev/cxgbe/t4_main.c Thu Aug 9 14:04:51 2018 (r337537)
+++ head/sys/dev/cxgbe/t4_main.c Thu Aug 9 14:19:47 2018 (r337538)
@@ -1404,6 +1404,7 @@ t4_detach_common(device_t dev)
free(sc->sge.iqmap, M_CXGBE);
free(sc->sge.eqmap, M_CXGBE);
free(sc->tids.ftid_tab, M_CXGBE);
+ free(sc->tids.hpftid_tab, M_CXGBE);
if (sc->tids.hftid_tab)
free_hftid_tab(&sc->tids);
free(sc->tids.atid_tab, M_CXGBE);
@@ -3853,6 +3854,47 @@ get_params__post_init(struct adapter *sc)
__func__, sc->vres.l2t.size, L2T_SIZE));
sc->params.core_vdd = val[6];
+ if (chip_id(sc) >= CHELSIO_T6) {
+
+#ifdef INVARIANTS
+ if (sc->params.fw_vers >=
+ (V_FW_HDR_FW_VER_MAJOR(1) | V_FW_HDR_FW_VER_MINOR(20) |
+ V_FW_HDR_FW_VER_MICRO(1) | V_FW_HDR_FW_VER_BUILD(0))) {
+ /*
+ * Note that the code to enable the region should run
+ * before t4_fw_initialize and not here. This is just a
+ * reminder to add said code.
+ */
+ device_printf(sc->dev,
+ "hpfilter region not enabled.\n");
+ }
+#endif
+
+ sc->tids.tid_base = t4_read_reg(sc,
+ A_LE_DB_ACTIVE_TABLE_START_INDEX);
+
+ param[0] = FW_PARAM_PFVF(HPFILTER_START);
+ param[1] = FW_PARAM_PFVF(HPFILTER_END);
+ rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val);
+ if (rc != 0) {
+ device_printf(sc->dev,
+ "failed to query hpfilter parameters: %d.\n", rc);
+ return (rc);
+ }
+ if ((int)val[1] > (int)val[0]) {
+ sc->tids.hpftid_base = val[0];
+ sc->tids.hpftid_end = val[1];
+ sc->tids.nhpftids = val[1] - val[0] + 1;
+
+ /*
+ * These should go off if the layout changes and the
+ * driver needs to catch up.
+ */
+ MPASS(sc->tids.hpftid_base == 0);
+ MPASS(sc->tids.tid_base == sc->tids.nhpftids);
+ }
+ }
+
/*
* MPSBGMAP is queried separately because only recent firmwares support
* it as a parameter and we don't want the compound query above to fail
@@ -3916,6 +3958,12 @@ get_params__post_init(struct adapter *sc)
return (rc);
}
sc->tids.ntids = val[0];
+ if (sc->params.fw_vers <
+ (V_FW_HDR_FW_VER_MAJOR(1) | V_FW_HDR_FW_VER_MINOR(20) |
+ V_FW_HDR_FW_VER_MICRO(5) | V_FW_HDR_FW_VER_BUILD(0))) {
+ MPASS(sc->tids.ntids >= sc->tids.nhpftids);
+ sc->tids.ntids -= sc->tids.nhpftids;
+ }
sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS);
sc->params.hash_filter = 1;
}
@@ -8127,6 +8175,11 @@ sysctl_tids(SYSCTL_HANDLER_ARGS)
t->atids_in_use);
}
+ if (t->nhpftids) {
+ sbuf_printf(sb, "HPFTID range: %u-%u, in use: %u\n",
+ t->hpftid_base, t->hpftid_end, t->hpftids_in_use);
+ }
+
if (t->ntids) {
sbuf_printf(sb, "TID range: ");
if (t4_read_reg(sc, A_LE_DB_CONFIG) & F_HASHEN) {
@@ -8141,10 +8194,10 @@ sysctl_tids(SYSCTL_HANDLER_ARGS)
}
if (b)
- sbuf_printf(sb, "0-%u, ", b - 1);
+ sbuf_printf(sb, "%u-%u, ", t->tid_base, b - 1);
sbuf_printf(sb, "%u-%u", hb, t->ntids - 1);
} else
- sbuf_printf(sb, "0-%u", t->ntids - 1);
+ sbuf_printf(sb, "%u-%u", t->tid_base, t->ntids - 1);
sbuf_printf(sb, ", in use: %u\n",
atomic_load_acq_int(&t->tids_in_use));
}
@@ -8155,8 +8208,8 @@ sysctl_tids(SYSCTL_HANDLER_ARGS)
}
if (t->nftids) {
- sbuf_printf(sb, "FTID range: %u-%u\n", t->ftid_base,
- t->ftid_base + t->nftids - 1);
+ sbuf_printf(sb, "FTID range: %u-%u, in use: %u\n", t->ftid_base,
+ t->ftid_end, t->ftids_in_use);
}
if (t->netids) {
Modified: head/sys/dev/cxgbe/t4_sge.c
==============================================================================
--- head/sys/dev/cxgbe/t4_sge.c Thu Aug 9 14:04:51 2018 (r337537)
+++ head/sys/dev/cxgbe/t4_sge.c Thu Aug 9 14:19:47 2018 (r337538)
@@ -367,7 +367,7 @@ set_tcb_rpl_handler(struct sge_iq *iq, const struct rs
MPASS(m == NULL);
tid = GET_TID(cpl);
- if (is_ftid(iq->adapter, tid)) {
+ if (is_hpftid(iq->adapter, tid) || is_ftid(iq->adapter, tid)) {
/*
* The return code for filter-write is put in the CPL cookie so
* we have to rely on the hardware tid (is_ftid) to determine
Modified: head/sys/dev/cxgbe/tom/t4_tom.c
==============================================================================
--- head/sys/dev/cxgbe/tom/t4_tom.c Thu Aug 9 14:04:51 2018 (r337537)
+++ head/sys/dev/cxgbe/tom/t4_tom.c Thu Aug 9 14:19:47 2018 (r337538)
@@ -510,7 +510,10 @@ insert_tid(struct adapter *sc, int tid, void *ctx, int
{
struct tid_info *t = &sc->tids;
- t->tid_tab[tid] = ctx;
+ MPASS(tid >= t->tid_base);
+ MPASS(tid - t->tid_base < t->ntids);
+
+ t->tid_tab[tid - t->tid_base] = ctx;
atomic_add_int(&t->tids_in_use, ntids);
}
@@ -519,7 +522,7 @@ lookup_tid(struct adapter *sc, int tid)
{
struct tid_info *t = &sc->tids;
- return (t->tid_tab[tid]);
+ return (t->tid_tab[tid - t->tid_base]);
}
void
@@ -527,7 +530,7 @@ update_tid(struct adapter *sc, int tid, void *ctx)
{
struct tid_info *t = &sc->tids;
- t->tid_tab[tid] = ctx;
+ t->tid_tab[tid - t->tid_base] = ctx;
}
void
@@ -535,7 +538,7 @@ remove_tid(struct adapter *sc, int tid, int ntids)
{
struct tid_info *t = &sc->tids;
- t->tid_tab[tid] = NULL;
+ t->tid_tab[tid - t->tid_base] = NULL;
atomic_subtract_int(&t->tids_in_use, ntids);
}
More information about the svn-src-all
mailing list