svn commit: r354581 - in head: share/man/man4 sys/dev/ntb
Alexander Motin
mav at FreeBSD.org
Sun Nov 10 03:37:46 UTC 2019
Author: mav
Date: Sun Nov 10 03:37:45 2019
New Revision: 354581
URL: https://svnweb.freebsd.org/changeset/base/354581
Log:
Add compact scraptchpad protocol for ntb_transport(4).
Previously ntb_transport(4) required at least 6 scratchpad registers,
plus 2 more for each additional memory window. That is too much for some
configurations, where several drivers have to share resources of the same
NTB hardware. This patch introduces new compact version of the protocol,
requiring only 3 scratchpad registers, plus one more for each additional
memory window. The optimization is based on fact that neither of version,
number of windows or number of queue pairs really need more then one byte
each, and window sizes of 4GB are not very useful now. The new protocol
is activated automatically when the configuration is low on scratchpad
registers, or it can be activated explicitly with loader tunable.
MFC after: 2 weeks
Sponsored by: iXsystems, Inc.
Modified:
head/share/man/man4/ntb_transport.4
head/sys/dev/ntb/ntb_transport.c
Modified: head/share/man/man4/ntb_transport.4
==============================================================================
--- head/share/man/man4/ntb_transport.4 Sun Nov 10 03:24:53 2019 (r354580)
+++ head/share/man/man4/ntb_transport.4 Sun Nov 10 03:37:45 2019 (r354581)
@@ -25,7 +25,7 @@
.\"
.\" $FreeBSD$
.\"
-.Dd August 29, 2019
+.Dd November 9, 2019
.Dt NTB_TRANSPORT 4
.Os
.Sh NAME
@@ -64,6 +64,12 @@ is a name of the driver to attach (empty means any),
is a number of queues to allocate (empty means automatic).
The default configuration is empty string, which means single consumer
with one queue per memory window, allowing any driver to attach.
+.It Va hint.ntb_transport. Ns Ar X Ns Va .compact
+Non-zero value enables compact version of sratchpad protocol, using twice
+less registers.
+Enabled automatically if there is not enough registers to negotiate all
+available memory windows.
+The compact version does not support memory windows of 4GB and above.
.El
.Sh DESCRIPTION
The
@@ -85,7 +91,8 @@ instance:
.It
1 or more memory windows;
.It
-6 scratchpads, plus 2 more for each additional memory window;
+6 scratchpads, plus 2 more for each additional memory window,
+or 3 plus 1 in case of compact protocol;
.It
1 doorbell for each memory window or configured queue.
.El
Modified: head/sys/dev/ntb/ntb_transport.c
==============================================================================
--- head/sys/dev/ntb/ntb_transport.c Sun Nov 10 03:24:53 2019 (r354580)
+++ head/sys/dev/ntb/ntb_transport.c Sun Nov 10 03:37:45 2019 (r354581)
@@ -203,6 +203,7 @@ struct ntb_transport_ctx {
struct ntb_transport_child *child;
struct ntb_transport_mw *mw_vec;
struct ntb_transport_qp *qp_vec;
+ int compact;
unsigned mw_count;
unsigned qp_count;
uint64_t qp_bitmap;
@@ -252,6 +253,15 @@ enum {
NTBT_WATCHDOG_SPAD = 15
};
+/*
+ * Compart version of sratchpad protocol, using twice less registers.
+ */
+enum {
+ NTBTC_PARAMS = 0, /* NUM_QPS << 24 + NUM_MWS << 16 + VERSION */
+ NTBTC_QP_LINKS, /* QP links status */
+ NTBTC_MW0_SZ, /* MW size limited to 32 bits. */
+};
+
#define QP_TO_MW(nt, qp) ((qp) % nt->mw_count)
#define NTB_QP_DEF_NUM_ENTRIES 100
#define NTB_LINK_DOWN_TIMEOUT 100
@@ -352,15 +362,31 @@ ntb_transport_attach(device_t dev)
device_printf(dev, "At least 1 memory window required.\n");
return (ENXIO);
}
- if (spad_count < 6) {
- device_printf(dev, "At least 6 scratchpads required.\n");
- return (ENXIO);
+ nt->compact = (spad_count < 4 + 2 * nt->mw_count);
+ snprintf(buf, sizeof(buf), "hint.%s.%d.compact", device_get_name(dev),
+ device_get_unit(dev));
+ TUNABLE_INT_FETCH(buf, &nt->compact);
+ if (nt->compact) {
+ if (spad_count < 3) {
+ device_printf(dev, "At least 3 scratchpads required.\n");
+ return (ENXIO);
+ }
+ if (spad_count < 2 + nt->mw_count) {
+ nt->mw_count = spad_count - 2;
+ device_printf(dev, "Scratchpads enough only for %d "
+ "memory windows.\n", nt->mw_count);
+ }
+ } else {
+ if (spad_count < 6) {
+ device_printf(dev, "At least 6 scratchpads required.\n");
+ return (ENXIO);
+ }
+ if (spad_count < 4 + 2 * nt->mw_count) {
+ nt->mw_count = (spad_count - 4) / 2;
+ device_printf(dev, "Scratchpads enough only for %d "
+ "memory windows.\n", nt->mw_count);
+ }
}
- if (spad_count < 4 + 2 * nt->mw_count) {
- nt->mw_count = (spad_count - 4) / 2;
- device_printf(dev, "Scratchpads enough only for %d "
- "memory windows.\n", nt->mw_count);
- }
if (db_bitmap == 0) {
device_printf(dev, "At least one doorbell required.\n");
return (ENXIO);
@@ -380,10 +406,16 @@ ntb_transport_attach(device_t dev)
mw->tx_size = mw->phys_size;
if (max_mw_size != 0 && mw->tx_size > max_mw_size) {
device_printf(dev, "Memory window %d limited from "
- "%ju to %ju\n", i, (uintmax_t)mw->phys_size,
+ "%ju to %ju\n", i, (uintmax_t)mw->tx_size,
max_mw_size);
mw->tx_size = max_mw_size;
}
+ if (nt->compact && mw->tx_size > UINT32_MAX) {
+ device_printf(dev, "Memory window %d is too big "
+ "(%ju)\n", i, (uintmax_t)mw->tx_size);
+ rc = ENXIO;
+ goto err;
+ }
mw->rx_size = 0;
mw->buff_size = 0;
@@ -1110,37 +1142,61 @@ ntb_transport_link_work(void *arg)
int rc;
/* send the local info, in the opposite order of the way we read it */
- for (i = 0; i < nt->mw_count; i++) {
- size = nt->mw_vec[i].tx_size;
- ntb_peer_spad_write(dev, NTBT_MW0_SZ_HIGH + (i * 2),
- size >> 32);
- ntb_peer_spad_write(dev, NTBT_MW0_SZ_LOW + (i * 2), size);
+ if (nt->compact) {
+ for (i = 0; i < nt->mw_count; i++) {
+ size = nt->mw_vec[i].tx_size;
+ KASSERT(size <= UINT32_MAX, ("size too big (%jx)", size));
+ ntb_peer_spad_write(dev, NTBTC_MW0_SZ + i, size);
+ }
+ ntb_peer_spad_write(dev, NTBTC_QP_LINKS, 0);
+ ntb_peer_spad_write(dev, NTBTC_PARAMS,
+ (nt->qp_count << 24) | (nt->mw_count << 16) |
+ NTB_TRANSPORT_VERSION);
+ } else {
+ for (i = 0; i < nt->mw_count; i++) {
+ size = nt->mw_vec[i].tx_size;
+ ntb_peer_spad_write(dev, NTBT_MW0_SZ_HIGH + (i * 2),
+ size >> 32);
+ ntb_peer_spad_write(dev, NTBT_MW0_SZ_LOW + (i * 2), size);
+ }
+ ntb_peer_spad_write(dev, NTBT_NUM_MWS, nt->mw_count);
+ ntb_peer_spad_write(dev, NTBT_NUM_QPS, nt->qp_count);
+ ntb_peer_spad_write(dev, NTBT_QP_LINKS, 0);
+ ntb_peer_spad_write(dev, NTBT_VERSION, NTB_TRANSPORT_VERSION);
}
- ntb_peer_spad_write(dev, NTBT_NUM_MWS, nt->mw_count);
- ntb_peer_spad_write(dev, NTBT_NUM_QPS, nt->qp_count);
- ntb_peer_spad_write(dev, NTBT_QP_LINKS, 0);
- ntb_peer_spad_write(dev, NTBT_VERSION, NTB_TRANSPORT_VERSION);
/* Query the remote side for its info */
val = 0;
- ntb_spad_read(dev, NTBT_VERSION, &val);
- if (val != NTB_TRANSPORT_VERSION)
- goto out;
+ if (nt->compact) {
+ ntb_spad_read(dev, NTBTC_PARAMS, &val);
+ if (val != ((nt->qp_count << 24) | (nt->mw_count << 16) |
+ NTB_TRANSPORT_VERSION))
+ goto out;
+ } else {
+ ntb_spad_read(dev, NTBT_VERSION, &val);
+ if (val != NTB_TRANSPORT_VERSION)
+ goto out;
- ntb_spad_read(dev, NTBT_NUM_QPS, &val);
- if (val != nt->qp_count)
- goto out;
+ ntb_spad_read(dev, NTBT_NUM_QPS, &val);
+ if (val != nt->qp_count)
+ goto out;
- ntb_spad_read(dev, NTBT_NUM_MWS, &val);
- if (val != nt->mw_count)
- goto out;
+ ntb_spad_read(dev, NTBT_NUM_MWS, &val);
+ if (val != nt->mw_count)
+ goto out;
+ }
for (i = 0; i < nt->mw_count; i++) {
- ntb_spad_read(dev, NTBT_MW0_SZ_HIGH + (i * 2), &val);
- val64 = (uint64_t)val << 32;
+ if (nt->compact) {
+ ntb_spad_read(dev, NTBTC_MW0_SZ + i, &val);
+ val64 = val;
+ } else {
+ ntb_spad_read(dev, NTBT_MW0_SZ_HIGH + (i * 2), &val);
+ val64 = (uint64_t)val << 32;
- ntb_spad_read(dev, NTBT_MW0_SZ_LOW + (i * 2), &val);
- val64 |= val;
+ ntb_spad_read(dev, NTBT_MW0_SZ_LOW + (i * 2), &val);
+ val64 |= val;
+ }
mw = &nt->mw_vec[i];
mw->rx_size = val64;
More information about the svn-src-all
mailing list