svn commit: r289546 - in head/sys/dev/ntb: if_ntb ntb_hw
Conrad E. Meyer
cem at FreeBSD.org
Sun Oct 18 20:20:59 UTC 2015
Author: cem
Date: Sun Oct 18 20:20:57 2015
New Revision: 289546
URL: https://svnweb.freebsd.org/changeset/base/289546
Log:
if_ntb: MFV e26a5843: Move MW/DB management to if_ntb
This is the last e26a5843 patch. The general thrust of the rewrite was
to move more responsibility for Memory Window and Doorbell interrupt
management from the ntb_hw driver to if_ntb.
A number of APIs have been added, removed, or replaced. The old
DB callback mechanism has been excised. Instead, callers (if_ntb) are
responsible for configuring MWs and handling their interrupts more
directly.
This adds a tunable, hw.ntb.max_mw_size, allowing users to limit the
size of memory windows used by if_ntb (identical to the Linux modparam
of the same name).
Despite attempts to keep mechanical name changes to separate commits,
some have snuck in here. At least the driver should be much more
similar to the latest Linux one now -- making porting fixes easier.
Authored by: Allen Hubbe
Obtained from: Linux (Dual BSD/GPL driver)
Sponsored by: EMC / Isilon Storage Division
Modified:
head/sys/dev/ntb/if_ntb/if_ntb.c
head/sys/dev/ntb/ntb_hw/ntb_hw.c
head/sys/dev/ntb/ntb_hw/ntb_hw.h
head/sys/dev/ntb/ntb_hw/ntb_regs.h
Modified: head/sys/dev/ntb/if_ntb/if_ntb.c
==============================================================================
--- head/sys/dev/ntb/if_ntb/if_ntb.c Sun Oct 18 20:20:48 2015 (r289545)
+++ head/sys/dev/ntb/if_ntb/if_ntb.c Sun Oct 18 20:20:57 2015 (r289546)
@@ -81,12 +81,22 @@ BITSET_DEFINE(_qpset, QP_SETSIZE);
#define KTR_NTB KTR_SPARE3
-#define NTB_TRANSPORT_VERSION 3
+#define NTB_TRANSPORT_VERSION 4
#define NTB_RX_MAX_PKTS 64
#define NTB_RXQ_SIZE 300
+enum ntb_link_event {
+ NTB_LINK_DOWN = 0,
+ NTB_LINK_UP,
+};
+
static unsigned int transport_mtu = 0x4000 + ETHER_HDR_LEN + ETHER_CRC_LEN;
+static uint64_t max_mw_size;
+SYSCTL_UQUAD(_hw_ntb, OID_AUTO, max_mw_size, CTLFLAG_RDTUN, &max_mw_size, 0,
+ "If enabled (non-zero), limit the size of large memory windows. "
+ "Both sides of the NTB MUST set the same value here.");
+
static unsigned int max_num_clients;
SYSCTL_UINT(_hw_ntb, OID_AUTO, max_num_clients, CTLFLAG_RDTUN,
&max_num_clients, 0, "Maximum number of NTB transport clients. "
@@ -99,11 +109,15 @@ struct ntb_queue_entry {
/* ntb_queue list reference */
STAILQ_ENTRY(ntb_queue_entry) entry;
- /* info on data to be transfered */
+ /* info on data to be transferred */
void *cb_data;
void *buf;
- uint64_t len;
- uint64_t flags;
+ unsigned len;
+ unsigned flags;
+
+ struct ntb_transport_qp *qp;
+ struct ntb_payload_header *x_hdr;
+ unsigned index;
};
struct ntb_rx_info {
@@ -128,6 +142,7 @@ struct ntb_transport_qp {
struct ntb_queue_list tx_free_q;
struct mtx ntb_tx_free_q_lock;
void *tx_mw;
+ bus_addr_t tx_mw_phys;
uint64_t tx_index;
uint64_t tx_max_entry;
uint64_t tx_max_frame;
@@ -139,6 +154,7 @@ struct ntb_transport_qp {
struct mtx ntb_rx_pend_q_lock;
struct mtx ntb_rx_free_q_lock;
struct task rx_completion_task;
+ struct task rxc_db_work;
void *rx_buff;
uint64_t rx_index;
uint64_t rx_max_entry;
@@ -171,11 +187,18 @@ struct ntb_queue_handlers {
void (*event_handler)(void *data, enum ntb_link_event status);
};
-
struct ntb_transport_mw {
+ vm_paddr_t phys_addr;
+ size_t phys_size;
+ size_t xlat_align;
+ size_t xlat_align_size;
+ /* Tx buff is off vbase / phys_addr */
+ void *vbase;
size_t xlat_size;
+ size_t buff_size;
+ /* Rx buff is off virt_addr / dma_addr */
void *virt_addr;
- vm_paddr_t dma_addr;
+ bus_addr_t dma_addr;
};
struct ntb_transport_ctx {
@@ -184,15 +207,18 @@ struct ntb_transport_ctx {
struct ntb_transport_mw mw_vec[NTB_MAX_NUM_MW];
struct ntb_transport_qp *qp_vec;
struct _qpset qp_bitmap;
+ struct _qpset qp_bitmap_free;
unsigned mw_count;
unsigned qp_count;
enum ntb_link_event link_is_up;
struct callout link_work;
- struct ntb_transport_qp *qp;
uint64_t bufsize;
u_char eaddr[ETHER_ADDR_LEN];
struct mtx tx_lock;
struct mtx rx_lock;
+
+ /* The hardcoded single queuepair in ntb_setup_interface() */
+ struct ntb_transport_qp *qp;
};
static struct ntb_transport_ctx net_softc;
@@ -244,31 +270,32 @@ static void ntb_net_rx_handler(struct nt
void *data, int len);
static void ntb_net_event_handler(void *data, enum ntb_link_event status);
static int ntb_transport_init(struct ntb_softc *ntb);
-static void ntb_transport_free(void *transport);
+static void ntb_transport_free(struct ntb_transport_ctx *);
static void ntb_transport_init_queue(struct ntb_transport_ctx *nt,
unsigned int qp_num);
static void ntb_transport_free_queue(struct ntb_transport_qp *qp);
-static struct ntb_transport_qp * ntb_transport_create_queue(void *data,
+static struct ntb_transport_qp *ntb_transport_create_queue(void *data,
struct ntb_softc *pdev, const struct ntb_queue_handlers *handlers);
static void ntb_transport_link_up(struct ntb_transport_qp *qp);
static int ntb_transport_tx_enqueue(struct ntb_transport_qp *qp, void *cb,
void *data, unsigned int len);
static int ntb_process_tx(struct ntb_transport_qp *qp,
struct ntb_queue_entry *entry);
-static void ntb_tx_copy_task(struct ntb_transport_qp *qp,
+static void ntb_memcpy_tx(struct ntb_transport_qp *qp,
struct ntb_queue_entry *entry, void *offset);
static void ntb_qp_full(void *arg);
-static int ntb_transport_rxc_db(void *arg, int dummy);
+static void ntb_transport_rxc_db(void *arg, int pending);
static void ntb_rx_pendq_full(void *arg);
static int ntb_process_rxc(struct ntb_transport_qp *qp);
static void ntb_rx_copy_task(struct ntb_transport_qp *qp,
struct ntb_queue_entry *entry, void *offset);
-static void ntb_rx_completion_task(void *arg, int pending);
-static void ntb_transport_event_callback(void *data, enum ntb_hw_event event);
+static void ntb_complete_rxc(void *arg, int pending);
+static void ntb_transport_doorbell_callback(void *data, int vector);
+static void ntb_transport_event_callback(void *data);
static void ntb_transport_link_work(void *arg);
static int ntb_set_mw(struct ntb_transport_ctx *, int num_mw, unsigned size);
static void ntb_free_mw(struct ntb_transport_ctx *nt, int num_mw);
-static void ntb_transport_setup_qp_mw(struct ntb_transport_ctx *nt,
+static int ntb_transport_setup_qp_mw(struct ntb_transport_ctx *nt,
unsigned int qp_num);
static void ntb_qp_link_work(void *arg);
static void ntb_transport_link_cleanup(struct ntb_transport_ctx *nt);
@@ -283,6 +310,11 @@ static struct ntb_queue_entry *ntb_list_
static void create_random_local_eui48(u_char *eaddr);
static unsigned int ntb_transport_max_size(struct ntb_transport_qp *qp);
+static const struct ntb_ctx_ops ntb_transport_ops = {
+ .link_event = ntb_transport_event_callback,
+ .db_event = ntb_transport_doorbell_callback,
+};
+
MALLOC_DEFINE(M_NTB_IF, "if_ntb", "ntb network driver");
/* Module setup and teardown */
@@ -320,6 +352,7 @@ ntb_setup_interface(void)
struct ifnet *ifp;
struct ntb_queue_handlers handlers = { ntb_net_rx_handler,
ntb_net_tx_handler, ntb_net_event_handler };
+ int rc;
net_softc.ntb = devclass_get_softc(devclass_find("ntb_hw"), 0);
if (net_softc.ntb == NULL) {
@@ -327,11 +360,16 @@ ntb_setup_interface(void)
return (ENXIO);
}
- ntb_transport_init(net_softc.ntb);
+ rc = ntb_transport_init(net_softc.ntb);
+ if (rc != 0) {
+ printf("ntb: Cannot init transport: %d\n", rc);
+ return (rc);
+ }
ifp = net_softc.ifp = if_alloc(IFT_ETHER);
if (ifp == NULL) {
- printf("ntb: cannot allocate ifnet structure\n");
+ ntb_transport_free(&net_softc);
+ printf("ntb: Cannot allocate ifnet structure\n");
return (ENOMEM);
}
@@ -498,83 +536,107 @@ static int
ntb_transport_init(struct ntb_softc *ntb)
{
struct ntb_transport_ctx *nt = &net_softc;
+ struct ntb_transport_mw *mw;
+ uint64_t qp_bitmap;
int rc;
- uint8_t i;
+ unsigned i;
nt->mw_count = ntb_mw_count(ntb);
- if (max_num_clients == 0)
- nt->qp_count = MIN(ntb_get_max_cbs(ntb), ntb_mw_count(ntb));
- else
- nt->qp_count = MIN(ntb_get_max_cbs(ntb), max_num_clients);
+ for (i = 0; i < nt->mw_count; i++) {
+ mw = &nt->mw_vec[i];
+
+ rc = ntb_mw_get_range(ntb, i, &mw->phys_addr, &mw->vbase,
+ &mw->phys_size, &mw->xlat_align, &mw->xlat_align_size);
+ if (rc != 0)
+ goto err;
+
+ mw->buff_size = 0;
+ mw->xlat_size = 0;
+ mw->virt_addr = 0;
+ mw->dma_addr = 0;
+ }
+
+ qp_bitmap = ntb_db_valid_mask(ntb);
+ nt->qp_count = flsll(qp_bitmap);
+ KASSERT(nt->qp_count != 0, ("bogus db bitmap"));
+ nt->qp_count -= 1;
+
+ if (max_num_clients != 0 && max_num_clients < nt->qp_count)
+ nt->qp_count = max_num_clients;
+ else if (nt->mw_count < nt->qp_count)
+ nt->qp_count = nt->mw_count;
+ KASSERT(nt->qp_count <= QP_SETSIZE, ("invalid qp_count"));
- ntb_register_transport(ntb, nt);
mtx_init(&nt->tx_lock, "ntb transport tx", NULL, MTX_DEF);
mtx_init(&nt->rx_lock, "ntb transport rx", NULL, MTX_DEF);
nt->qp_vec = malloc(nt->qp_count * sizeof(*nt->qp_vec), M_NTB_IF,
M_WAITOK | M_ZERO);
- KASSERT(nt->qp_count <= QP_SETSIZE, ("invalid qp_count"));
-
for (i = 0; i < nt->qp_count; i++) {
set_bit(i, &nt->qp_bitmap);
+ set_bit(i, &nt->qp_bitmap_free);
ntb_transport_init_queue(nt, i);
}
callout_init(&nt->link_work, 0);
- rc = ntb_register_event_callback(ntb, ntb_transport_event_callback);
+ rc = ntb_set_ctx(ntb, nt, &ntb_transport_ops);
if (rc != 0)
goto err;
- if (ntb_link_is_up(ntb)) {
- if (bootverbose)
- device_printf(ntb_get_device(ntb), "link up\n");
- callout_reset(&nt->link_work, 0, ntb_transport_link_work, nt);
- }
-
+ nt->link_is_up = false;
+ ntb_link_enable(ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO);
+ ntb_link_event(ntb);
return (0);
err:
free(nt->qp_vec, M_NTB_IF);
- ntb_unregister_transport(ntb);
+ nt->qp_vec = NULL;
return (rc);
}
static void
-ntb_transport_free(void *transport)
+ntb_transport_free(struct ntb_transport_ctx *nt)
{
- struct ntb_transport_ctx *nt = transport;
struct ntb_softc *ntb = nt->ntb;
+ struct _qpset qp_bitmap_alloc;
uint8_t i;
ntb_transport_link_cleanup(nt);
callout_drain(&nt->link_work);
- /* verify that all the qps are freed */
+ BIT_COPY(QP_SETSIZE, &nt->qp_bitmap, &qp_bitmap_alloc);
+ BIT_NAND(QP_SETSIZE, &qp_bitmap_alloc, &nt->qp_bitmap_free);
+
+ /* Verify that all the QPs are freed */
for (i = 0; i < nt->qp_count; i++)
- if (!test_bit(i, &nt->qp_bitmap))
+ if (test_bit(i, &qp_bitmap_alloc))
ntb_transport_free_queue(&nt->qp_vec[i]);
- ntb_unregister_event_callback(ntb);
+ ntb_link_disable(ntb);
+ ntb_clear_ctx(ntb);
- for (i = 0; i < NTB_MAX_NUM_MW; i++)
+ for (i = 0; i < nt->mw_count; i++)
ntb_free_mw(nt, i);
free(nt->qp_vec, M_NTB_IF);
- ntb_unregister_transport(ntb);
}
static void
ntb_transport_init_queue(struct ntb_transport_ctx *nt, unsigned int qp_num)
{
+ struct ntb_transport_mw *mw;
struct ntb_transport_qp *qp;
- unsigned int num_qps_mw, tx_size;
- uint8_t mw_num, mw_count;
+ vm_paddr_t mw_base;
+ uint64_t mw_size, qp_offset;
+ size_t tx_size;
+ unsigned num_qps_mw, mw_num, mw_count;
- mw_count = ntb_mw_count(nt->ntb);
+ mw_count = nt->mw_count;
mw_num = QP_TO_MW(nt, qp_num);
+ mw = &nt->mw_vec[mw_num];
qp = &nt->qp_vec[qp_num];
qp->qp_num = qp_num;
@@ -589,13 +651,22 @@ ntb_transport_init_queue(struct ntb_tran
else
num_qps_mw = nt->qp_count / mw_count;
- tx_size = ntb_get_mw_size(qp->ntb, mw_num) / num_qps_mw;
- qp->rx_info = (struct ntb_rx_info *)
- ((char *)ntb_get_mw_vbase(qp->ntb, mw_num) +
- (qp_num / mw_count * tx_size));
+ mw_base = mw->phys_addr;
+ mw_size = mw->phys_size;
+
+ tx_size = mw_size / num_qps_mw;
+ qp_offset = tx_size * qp_num / mw_count;
+
+ qp->tx_mw = (char *)mw->vbase + qp_offset;
+ KASSERT(qp->tx_mw != NULL, ("uh oh?"));
+
+ /* XXX Assumes that a vm_paddr_t is equivalent to bus_addr_t */
+ qp->tx_mw_phys = mw_base + qp_offset;
+ KASSERT(qp->tx_mw_phys != 0, ("uh oh?"));
+
tx_size -= sizeof(struct ntb_rx_info);
+ qp->rx_info = (void *)((char *)qp->tx_mw + tx_size);
- qp->tx_mw = qp->rx_info + 1;
/* Due to house-keeping, there must be at least 2 buffs */
qp->tx_max_frame = min(transport_mtu + sizeof(struct ntb_payload_header),
tx_size / 2);
@@ -608,7 +679,8 @@ ntb_transport_init_queue(struct ntb_tran
mtx_init(&qp->ntb_rx_pend_q_lock, "ntb rx pend q", NULL, MTX_SPIN);
mtx_init(&qp->ntb_rx_free_q_lock, "ntb rx free q", NULL, MTX_SPIN);
mtx_init(&qp->ntb_tx_free_q_lock, "ntb tx free q", NULL, MTX_SPIN);
- TASK_INIT(&qp->rx_completion_task, 0, ntb_rx_completion_task, qp);
+ TASK_INIT(&qp->rx_completion_task, 0, ntb_complete_rxc, qp);
+ TASK_INIT(&qp->rxc_db_work, 0, ntb_transport_rxc_db, qp);
STAILQ_INIT(&qp->rx_pend_q);
STAILQ_INIT(&qp->rx_free_q);
@@ -625,7 +697,14 @@ ntb_transport_free_queue(struct ntb_tran
callout_drain(&qp->link_work);
- ntb_unregister_db_callback(qp->ntb, qp->qp_num);
+ ntb_db_set_mask(qp->ntb, 1ull << qp->qp_num);
+ taskqueue_drain(taskqueue_swi, &qp->rxc_db_work);
+ taskqueue_drain(taskqueue_swi, &qp->rx_completion_task);
+
+ qp->cb_data = NULL;
+ qp->rx_handler = NULL;
+ qp->tx_handler = NULL;
+ qp->event_handler = NULL;
while ((entry = ntb_list_rm(&qp->ntb_rx_free_q_lock, &qp->rx_free_q)))
free(entry, M_NTB_IF);
@@ -636,7 +715,7 @@ ntb_transport_free_queue(struct ntb_tran
while ((entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q)))
free(entry, M_NTB_IF);
- set_bit(qp->qp_num, &qp->transport->qp_bitmap);
+ set_bit(qp->qp_num, &qp->transport->qp_bitmap_free);
}
/**
@@ -654,37 +733,34 @@ ntb_transport_free_queue(struct ntb_tran
* RETURNS: pointer to newly created ntb_queue, NULL on error.
*/
static struct ntb_transport_qp *
-ntb_transport_create_queue(void *data, struct ntb_softc *pdev,
+ntb_transport_create_queue(void *data, struct ntb_softc *ntb,
const struct ntb_queue_handlers *handlers)
{
struct ntb_queue_entry *entry;
struct ntb_transport_qp *qp;
struct ntb_transport_ctx *nt;
unsigned int free_queue;
- int rc, i;
+ int i;
- nt = ntb_find_transport(pdev);
- if (nt == NULL)
- goto err;
+ nt = ntb_get_ctx(ntb, NULL);
+ KASSERT(nt != NULL, ("bogus"));
free_queue = ffs_bit(&nt->qp_bitmap);
if (free_queue == 0)
- goto err;
+ return (NULL);
/* decrement free_queue to make it zero based */
free_queue--;
- clear_bit(free_queue, &nt->qp_bitmap);
-
qp = &nt->qp_vec[free_queue];
+ clear_bit(1ull << qp->qp_num, &nt->qp_bitmap_free);
qp->cb_data = data;
qp->rx_handler = handlers->rx_handler;
qp->tx_handler = handlers->tx_handler;
qp->event_handler = handlers->event_handler;
for (i = 0; i < NTB_QP_DEF_NUM_ENTRIES; i++) {
- entry = malloc(sizeof(struct ntb_queue_entry), M_NTB_IF,
- M_WAITOK|M_ZERO);
+ entry = malloc(sizeof(*entry), M_NTB_IF, M_WAITOK | M_ZERO);
entry->cb_data = nt->ifp;
entry->buf = NULL;
entry->len = transport_mtu;
@@ -692,26 +768,13 @@ ntb_transport_create_queue(void *data, s
}
for (i = 0; i < NTB_QP_DEF_NUM_ENTRIES; i++) {
- entry = malloc(sizeof(struct ntb_queue_entry), M_NTB_IF,
- M_WAITOK|M_ZERO);
+ entry = malloc(sizeof(*entry), M_NTB_IF, M_WAITOK | M_ZERO);
ntb_list_add(&qp->ntb_tx_free_q_lock, entry, &qp->tx_free_q);
}
- rc = ntb_register_db_callback(qp->ntb, free_queue, qp,
- ntb_transport_rxc_db);
- if (rc != 0)
- goto err1;
-
+ ntb_db_clear(ntb, 1ull << qp->qp_num);
+ ntb_db_clear_mask(ntb, 1ull << qp->qp_num);
return (qp);
-
-err1:
- while ((entry = ntb_list_rm(&qp->ntb_tx_free_q_lock, &qp->tx_free_q)))
- free(entry, M_NTB_IF);
- while ((entry = ntb_list_rm(&qp->ntb_rx_free_q_lock, &qp->rx_free_q)))
- free(entry, M_NTB_IF);
- set_bit(free_queue, &nt->qp_bitmap);
-err:
- return (NULL);
}
/**
@@ -813,7 +876,7 @@ ntb_process_tx(struct ntb_transport_qp *
return (0);
}
CTR2(KTR_NTB, "TX: copying entry %p to offset %p", entry, offset);
- ntb_tx_copy_task(qp, entry, offset);
+ ntb_memcpy_tx(qp, entry, offset);
qp->tx_index++;
qp->tx_index %= qp->tx_max_entry;
@@ -824,20 +887,31 @@ ntb_process_tx(struct ntb_transport_qp *
}
static void
-ntb_tx_copy_task(struct ntb_transport_qp *qp, struct ntb_queue_entry *entry,
+ntb_memcpy_tx(struct ntb_transport_qp *qp, struct ntb_queue_entry *entry,
void *offset)
{
struct ntb_payload_header *hdr;
- CTR2(KTR_NTB, "TX: copying %d bytes to offset %p", entry->len, offset);
- if (entry->buf != NULL)
- m_copydata((struct mbuf *)entry->buf, 0, entry->len, offset);
-
+ /* This piece is from Linux' ntb_async_tx() */
hdr = (struct ntb_payload_header *)((char *)offset + qp->tx_max_frame -
sizeof(struct ntb_payload_header));
+ entry->x_hdr = hdr;
hdr->len = entry->len; /* TODO: replace with bus_space_write */
hdr->ver = qp->tx_pkts; /* TODO: replace with bus_space_write */
- wmb();
+
+ /* This piece is ntb_memcpy_tx() */
+ CTR2(KTR_NTB, "TX: copying %d bytes to offset %p", entry->len, offset);
+ if (entry->buf != NULL) {
+ m_copydata((struct mbuf *)entry->buf, 0, entry->len, offset);
+
+ /*
+ * Ensure that the data is fully copied before setting the
+ * flags
+ */
+ wmb();
+ }
+
+ /* The rest is ntb_tx_copy_callback() */
/* TODO: replace with bus_space_write */
hdr->flags = entry->flags | IF_NTB_DESC_DONE_FLAG;
@@ -879,8 +953,8 @@ ntb_rx_pendq_full(void *arg)
ntb_transport_rxc_db(arg, 0);
}
-static int
-ntb_transport_rxc_db(void *arg, int dummy __unused)
+static void
+ntb_transport_rxc_db(void *arg, int pending __unused)
{
struct ntb_transport_qp *qp = arg;
uint64_t i;
@@ -890,9 +964,9 @@ ntb_transport_rxc_db(void *arg, int dumm
* Limit the number of packets processed in a single interrupt to
* provide fairness to others
*/
- mtx_lock(&qp->transport->rx_lock);
CTR0(KTR_NTB, "RX: transport_rx");
- for (i = 0; i < MIN(qp->rx_max_entry, INT_MAX); i++) {
+ mtx_lock(&qp->transport->rx_lock);
+ for (i = 0; i < qp->rx_max_entry; i++) {
rc = ntb_process_rxc(qp);
if (rc != 0) {
CTR0(KTR_NTB, "RX: process_rxc failed");
@@ -901,7 +975,20 @@ ntb_transport_rxc_db(void *arg, int dumm
}
mtx_unlock(&qp->transport->rx_lock);
- return ((int)i);
+ if (i == qp->rx_max_entry)
+ taskqueue_enqueue(taskqueue_swi, &qp->rxc_db_work);
+ else if ((ntb_db_read(qp->ntb) & (1ull << qp->qp_num)) != 0) {
+ /* If db is set, clear it and read it back to commit clear. */
+ ntb_db_clear(qp->ntb, 1ull << qp->qp_num);
+ (void)ntb_db_read(qp->ntb);
+
+ /*
+ * An interrupt may have arrived between finishing
+ * ntb_process_rxc and clearing the doorbell bit: there might
+ * be some more work to do.
+ */
+ taskqueue_enqueue(taskqueue_swi, &qp->rxc_db_work);
+ }
}
static int
@@ -918,66 +1005,61 @@ ntb_process_rxc(struct ntb_transport_qp
sizeof(struct ntb_payload_header));
CTR1(KTR_NTB, "RX: process_rxc rx_index = %u", qp->rx_index);
- entry = ntb_list_rm(&qp->ntb_rx_pend_q_lock, &qp->rx_pend_q);
- if (entry == NULL) {
- qp->rx_err_no_buf++;
- CTR0(KTR_NTB, "RX: No entries in rx_pend_q");
- return (ENOMEM);
- }
- callout_stop(&qp->rx_full);
- CTR1(KTR_NTB, "RX: rx entry %p from rx_pend_q", entry);
-
if ((hdr->flags & IF_NTB_DESC_DONE_FLAG) == 0) {
- CTR1(KTR_NTB,
- "RX: hdr not done. Returning entry %p to rx_pend_q", entry);
- ntb_list_add(&qp->ntb_rx_pend_q_lock, entry, &qp->rx_pend_q);
+ CTR0(KTR_NTB, "RX: hdr not done");
qp->rx_ring_empty++;
return (EAGAIN);
}
- if (hdr->ver != (uint32_t) qp->rx_pkts) {
- CTR3(KTR_NTB,"RX: ver != rx_pkts (%x != %lx). "
- "Returning entry %p to rx_pend_q", hdr->ver, qp->rx_pkts,
- entry);
- ntb_list_add(&qp->ntb_rx_pend_q_lock, entry, &qp->rx_pend_q);
+ if ((hdr->flags & IF_NTB_LINK_DOWN_FLAG) != 0) {
+ CTR0(KTR_NTB, "RX: link down");
+ ntb_qp_link_down(qp);
+ hdr->flags = 0;
+ return (EAGAIN);
+ }
+
+ if (hdr->ver != (uint32_t)qp->rx_pkts) {
+ CTR2(KTR_NTB,"RX: ver != rx_pkts (%x != %lx). "
+ "Returning entry %p to rx_pend_q", hdr->ver, qp->rx_pkts);
qp->rx_err_ver++;
return (EIO);
}
- if ((hdr->flags & IF_NTB_LINK_DOWN_FLAG) != 0) {
- ntb_qp_link_down(qp);
- CTR1(KTR_NTB,
- "RX: link down. adding entry %p back to rx_pend_q", entry);
- ntb_list_add(&qp->ntb_rx_pend_q_lock, entry, &qp->rx_pend_q);
- goto out;
+ entry = ntb_list_rm(&qp->ntb_rx_pend_q_lock, &qp->rx_pend_q);
+ if (entry == NULL) {
+ qp->rx_err_no_buf++;
+ CTR0(KTR_NTB, "RX: No entries in rx_pend_q");
+ return (EAGAIN);
}
+ callout_stop(&qp->rx_full);
+ CTR1(KTR_NTB, "RX: rx entry %p from rx_pend_q", entry);
- if (hdr->len <= entry->len) {
- entry->len = hdr->len;
- ntb_rx_copy_task(qp, entry, offset);
- } else {
- CTR1(KTR_NTB,
- "RX: len too long. Returning entry %p to rx_pend_q", entry);
- ntb_list_add(&qp->ntb_rx_pend_q_lock, entry, &qp->rx_pend_q);
+ entry->x_hdr = hdr;
+ entry->index = qp->rx_index;
+ if (hdr->len > entry->len) {
+ CTR2(KTR_NTB, "RX: len too long. Wanted %ju got %ju",
+ (uintmax_t)hdr->len, (uintmax_t)entry->len);
qp->rx_err_oflow++;
- }
- qp->rx_bytes += hdr->len;
- qp->rx_pkts++;
- CTR1(KTR_NTB, "RX: received %ld rx_pkts", qp->rx_pkts);
+ entry->len = -EIO;
+ entry->flags |= IF_NTB_DESC_DONE_FLAG;
+ ntb_list_add(&qp->ntb_rx_free_q_lock, entry, &qp->rx_free_q);
+ taskqueue_enqueue(taskqueue_swi, &qp->rx_completion_task);
+ } else {
+ qp->rx_bytes += hdr->len;
+ qp->rx_pkts++;
-out:
- /* Ensure that the data is globally visible before clearing the flag */
- wmb();
- hdr->flags = 0;
- /* TODO: replace with bus_space_write */
- qp->rx_info->entry = qp->rx_index;
+ CTR1(KTR_NTB, "RX: received %ld rx_pkts", qp->rx_pkts);
+
+ entry->len = hdr->len;
+
+ ntb_rx_copy_task(qp, entry, offset);
+ }
qp->rx_index++;
qp->rx_index %= qp->rx_max_entry;
-
return (0);
}
@@ -995,6 +1077,12 @@ ntb_rx_copy_task(struct ntb_transport_qp
entry->buf = (void *)m;
+ /* Ensure that the data is globally visible before clearing the flag */
+ wmb();
+ entry->x_hdr->flags = 0;
+ /* TODO: replace with bus_space_write */
+ qp->rx_info->entry = qp->rx_index;
+
CTR2(KTR_NTB,
"RX: copied entry %p to mbuf %p. Adding entry to rx_free_q", entry,
m);
@@ -1004,7 +1092,7 @@ ntb_rx_copy_task(struct ntb_transport_qp
}
static void
-ntb_rx_completion_task(void *arg, int pending)
+ntb_complete_rxc(void *arg, int pending)
{
struct ntb_transport_qp *qp = arg;
struct mbuf *m;
@@ -1033,25 +1121,52 @@ ntb_rx_completion_task(void *arg, int pe
}
}
+static void
+ntb_transport_doorbell_callback(void *data, int vector)
+{
+ struct ntb_transport_ctx *nt = data;
+ struct ntb_transport_qp *qp;
+ struct _qpset db_bits;
+ uint64_t vec_mask;
+ unsigned qp_num;
+
+ BIT_COPY(QP_SETSIZE, &nt->qp_bitmap, &db_bits);
+ BIT_NAND(QP_SETSIZE, &db_bits, &nt->qp_bitmap_free);
+
+ vec_mask = ntb_db_vector_mask(nt->ntb, vector);
+ while (vec_mask != 0) {
+ qp_num = ffsl(vec_mask);
+ /* i386 doesn't have ffsll(), fake it */
+ if (qp_num == 0) {
+ qp_num = ffsl(vec_mask >> 32);
+ KASSERT(qp_num != 0, ("ffs"));
+ qp_num += 32;
+ }
+ qp_num--;
+
+ if (test_bit(qp_num, &db_bits)) {
+ qp = &nt->qp_vec[qp_num];
+ taskqueue_enqueue(taskqueue_swi, &qp->rxc_db_work);
+ }
+
+ vec_mask &= ~(1ull << qp_num);
+ }
+}
+
/* Link Event handler */
static void
-ntb_transport_event_callback(void *data, enum ntb_hw_event event)
+ntb_transport_event_callback(void *data)
{
struct ntb_transport_ctx *nt = data;
- switch (event) {
- case NTB_EVENT_HW_LINK_UP:
+ if (ntb_link_is_up(nt->ntb, NULL, NULL)) {
if (bootverbose)
device_printf(ntb_get_device(nt->ntb), "HW link up\n");
callout_reset(&nt->link_work, 0, ntb_transport_link_work, nt);
- break;
- case NTB_EVENT_HW_LINK_DOWN:
+ } else {
if (bootverbose)
device_printf(ntb_get_device(nt->ntb), "HW link down\n");
ntb_transport_link_cleanup(nt);
- break;
- default:
- panic("ntb: Unknown NTB event");
}
}
@@ -1062,72 +1177,48 @@ ntb_transport_link_work(void *arg)
struct ntb_transport_ctx *nt = arg;
struct ntb_softc *ntb = nt->ntb;
struct ntb_transport_qp *qp;
- uint64_t val64;
- uint32_t val, i, num_mw;
+ uint64_t val64, size;
+ uint32_t val;
+ unsigned i;
int rc;
- num_mw = ntb_mw_count(ntb);
-
/* send the local info, in the opposite order of the way we read it */
- for (i = 0; i < num_mw; i++) {
- rc = ntb_peer_spad_write(ntb, IF_NTB_MW0_SZ_HIGH + (i * 2),
- (uint64_t)ntb_get_mw_size(ntb, i) >> 32);
- if (rc != 0)
- goto out;
+ for (i = 0; i < nt->mw_count; i++) {
+ size = nt->mw_vec[i].phys_size;
- rc = ntb_peer_spad_write(ntb, IF_NTB_MW0_SZ_LOW + (i * 2),
- (uint32_t)ntb_get_mw_size(ntb, i));
- if (rc != 0)
- goto out;
+ if (max_mw_size != 0 && size > max_mw_size)
+ size = max_mw_size;
+
+ ntb_peer_spad_write(ntb, IF_NTB_MW0_SZ_HIGH + (i * 2),
+ size >> 32);
+ ntb_peer_spad_write(ntb, IF_NTB_MW0_SZ_LOW + (i * 2), size);
}
- rc = ntb_peer_spad_write(ntb, IF_NTB_NUM_MWS, num_mw);
- if (rc != 0)
- goto out;
+ ntb_peer_spad_write(ntb, IF_NTB_NUM_MWS, nt->mw_count);
- rc = ntb_peer_spad_write(ntb, IF_NTB_NUM_QPS, nt->qp_count);
- if (rc != 0)
- goto out;
+ ntb_peer_spad_write(ntb, IF_NTB_NUM_QPS, nt->qp_count);
- rc = ntb_peer_spad_write(ntb, IF_NTB_VERSION, NTB_TRANSPORT_VERSION);
- if (rc != 0)
- goto out;
+ ntb_peer_spad_write(ntb, IF_NTB_VERSION, NTB_TRANSPORT_VERSION);
/* Query the remote side for its info */
- rc = ntb_spad_read(ntb, IF_NTB_VERSION, &val);
- if (rc != 0)
- goto out;
-
+ val = 0;
+ ntb_spad_read(ntb, IF_NTB_VERSION, &val);
if (val != NTB_TRANSPORT_VERSION)
goto out;
- rc = ntb_spad_read(ntb, IF_NTB_NUM_QPS, &val);
- if (rc != 0)
- goto out;
-
+ ntb_spad_read(ntb, IF_NTB_NUM_QPS, &val);
if (val != nt->qp_count)
goto out;
- rc = ntb_spad_read(ntb, IF_NTB_NUM_MWS, &val);
- if (rc != 0)
- goto out;
-
- if (val != num_mw)
+ ntb_spad_read(ntb, IF_NTB_NUM_MWS, &val);
+ if (val != nt->mw_count)
goto out;
- for (i = 0; i < num_mw; i++) {
- rc = ntb_spad_read(ntb, IF_NTB_MW0_SZ_HIGH + (i * 2),
- &val);
- if (rc != 0)
- goto free_mws;
-
+ for (i = 0; i < nt->mw_count; i++) {
+ ntb_spad_read(ntb, IF_NTB_MW0_SZ_HIGH + (i * 2), &val);
val64 = (uint64_t)val << 32;
- rc = ntb_spad_read(ntb, IF_NTB_MW0_SZ_LOW + (i * 2),
- &val);
- if (rc != 0)
- goto free_mws;
-
+ ntb_spad_read(ntb, IF_NTB_MW0_SZ_LOW + (i * 2), &val);
val64 |= val;
rc = ntb_set_mw(nt, i, val64);
@@ -1151,33 +1242,40 @@ ntb_transport_link_work(void *arg)
return;
free_mws:
- for (i = 0; i < NTB_MAX_NUM_MW; i++)
+ for (i = 0; i < nt->mw_count; i++)
ntb_free_mw(nt, i);
out:
- if (ntb_link_is_up(ntb))
+ if (ntb_link_is_up(ntb, NULL, NULL))
callout_reset(&nt->link_work,
NTB_LINK_DOWN_TIMEOUT * hz / 1000, ntb_transport_link_work, nt);
}
static int
-ntb_set_mw(struct ntb_transport_ctx *nt, int num_mw, unsigned int size)
+ntb_set_mw(struct ntb_transport_ctx *nt, int num_mw, unsigned size)
{
struct ntb_transport_mw *mw = &nt->mw_vec[num_mw];
+ unsigned xlat_size, buff_size;
+ int rc;
+
+ xlat_size = roundup(size, mw->xlat_align_size);
+ buff_size = roundup(size, mw->xlat_align);
/* No need to re-setup */
- if (mw->xlat_size == size)
+ if (mw->xlat_size == xlat_size)
return (0);
- if (mw->xlat_size != 0)
+ if (mw->buff_size != 0)
ntb_free_mw(nt, num_mw);
- /* Alloc memory for receiving data. Must be 4k aligned */
- mw->xlat_size = size;
+ /* Alloc memory for receiving data. Must be aligned */
+ mw->xlat_size = xlat_size;
+ mw->buff_size = buff_size;
- mw->virt_addr = contigmalloc(mw->xlat_size, M_NTB_IF, M_ZERO, 0,
- BUS_SPACE_MAXADDR, mw->xlat_size, 0);
+ mw->virt_addr = contigmalloc(mw->buff_size, M_NTB_IF, M_ZERO, 0,
+ BUS_SPACE_MAXADDR, mw->xlat_align, 0);
if (mw->virt_addr == NULL) {
mw->xlat_size = 0;
+ mw->buff_size = 0;
printf("ntb: Unable to allocate MW buffer of size %zu\n",
mw->xlat_size);
return (ENOMEM);
@@ -1190,7 +1288,7 @@ ntb_set_mw(struct ntb_transport_ctx *nt,
* requested. XXX: This may not be needed -- brought in for parity
* with the Linux driver.
*/
- if (mw->dma_addr % size != 0) {
+ if (mw->dma_addr % mw->xlat_align != 0) {
device_printf(ntb_get_device(nt->ntb),
"DMA memory 0x%jx not aligned to BAR size 0x%x\n",
(uintmax_t)mw->dma_addr, size);
@@ -1199,7 +1297,13 @@ ntb_set_mw(struct ntb_transport_ctx *nt,
}
/* Notify HW the memory location of the receive buffer */
- ntb_set_mw_addr(nt->ntb, num_mw, mw->dma_addr);
+ rc = ntb_mw_set_trans(nt->ntb, num_mw, mw->dma_addr, mw->xlat_size);
+ if (rc) {
+ device_printf(ntb_get_device(nt->ntb),
+ "Unable to set mw%d translation", num_mw);
+ ntb_free_mw(nt, num_mw);
+ return (rc);
+ }
return (0);
}
@@ -1212,33 +1316,41 @@ ntb_free_mw(struct ntb_transport_ctx *nt
if (mw->virt_addr == NULL)
return;
+ ntb_mw_clear_trans(nt->ntb, num_mw);
contigfree(mw->virt_addr, mw->xlat_size, M_NTB_IF);
+ mw->xlat_size = 0;
+ mw->buff_size = 0;
mw->virt_addr = NULL;
}
-static void
+static int
ntb_transport_setup_qp_mw(struct ntb_transport_ctx *nt, unsigned int qp_num)
{
struct ntb_transport_qp *qp = &nt->qp_vec[qp_num];
+ struct ntb_transport_mw *mw;
void *offset;
- unsigned int rx_size, num_qps_mw;
- uint8_t mw_num, mw_count;
- unsigned int i;
+ uint64_t i;
+ size_t rx_size;
+ unsigned num_qps_mw, mw_num, mw_count;
- mw_count = ntb_mw_count(nt->ntb);
+ mw_count = nt->mw_count;
mw_num = QP_TO_MW(nt, qp_num);
+ mw = &nt->mw_vec[mw_num];
+
+ if (mw->virt_addr == NULL)
+ return (ENOMEM);
if (nt->qp_count % mw_count && mw_num + 1 < nt->qp_count / mw_count)
num_qps_mw = nt->qp_count / mw_count + 1;
else
num_qps_mw = nt->qp_count / mw_count;
- rx_size = nt->mw_vec[mw_num].xlat_size / num_qps_mw;
- qp->remote_rx_info = (void *)((uint8_t *)nt->mw_vec[mw_num].virt_addr +
- (qp_num / mw_count * rx_size));
+ rx_size = mw->xlat_size / num_qps_mw;
+ qp->rx_buff = (char *)mw->virt_addr + rx_size * qp_num / mw_count;
rx_size -= sizeof(struct ntb_rx_info);
- qp->rx_buff = qp->remote_rx_info + 1;
+ qp->remote_rx_info = (void*)((char *)qp->rx_buff + rx_size);
+
/* Due to house-keeping, there must be at least 2 buffs */
qp->rx_max_frame = min(transport_mtu + sizeof(struct ntb_payload_header),
rx_size / 2);
@@ -1247,7 +1359,7 @@ ntb_transport_setup_qp_mw(struct ntb_tra
qp->remote_rx_info->entry = qp->rx_max_entry - 1;
- /* setup the hdr offsets with 0's */
+ /* Set up the hdr offsets with 0s */
for (i = 0; i < qp->rx_max_entry; i++) {
offset = (void *)((uint8_t *)qp->rx_buff +
qp->rx_max_frame * (i + 1) -
@@ -1258,6 +1370,8 @@ ntb_transport_setup_qp_mw(struct ntb_tra
qp->rx_pkts = 0;
qp->tx_pkts = 0;
qp->tx_index = 0;
+
+ return (0);
}
static void
@@ -1266,46 +1380,51 @@ ntb_qp_link_work(void *arg)
struct ntb_transport_qp *qp = arg;
struct ntb_softc *ntb = qp->ntb;
struct ntb_transport_ctx *nt = qp->transport;
- int rc, val;
-
+ int val;
- rc = ntb_peer_spad_read(ntb, IF_NTB_QP_LINKS, &val);
- if (rc != 0)
- return;
+ ntb_spad_read(ntb, IF_NTB_QP_LINKS, &val);
- rc = ntb_peer_spad_write(ntb, IF_NTB_QP_LINKS, val | 1 << qp->qp_num);
+ ntb_peer_spad_write(ntb, IF_NTB_QP_LINKS, val | (1ull << qp->qp_num));
/* query remote spad for qp ready bits */
- rc = ntb_spad_read(ntb, IF_NTB_QP_LINKS, &val);
+ ntb_peer_spad_read(ntb, IF_NTB_QP_LINKS, &val);
/* See if the remote side is up */
- if ((1 << qp->qp_num & val) != 0) {
+ if ((val & (1ull << qp->qp_num)) != 0) {
+ if (bootverbose)
+ device_printf(ntb_get_device(ntb), "qp link up\n");
qp->link_is_up = true;
+
if (qp->event_handler != NULL)
qp->event_handler(qp->cb_data, NTB_LINK_UP);
- if (bootverbose)
- device_printf(ntb_get_device(ntb), "qp link up\n");
*** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
More information about the svn-src-all
mailing list