svn commit: r305240 - in head/sys: dev/cxgbe dev/cxgbe/cxgbei modules/cxgbe/cxgbei
Navdeep Parhar
np at FreeBSD.org
Thu Sep 1 20:43:02 UTC 2016
Author: np
Date: Thu Sep 1 20:43:01 2016
New Revision: 305240
URL: https://svnweb.freebsd.org/changeset/base/305240
Log:
cxgbe/cxgbei: Retire all DDP related code from cxgbei and switch to
routines available in t4_tom to manage the iSCSI DDP page pod region.
This adds the ability to use multiple DDP page sizes to the iSCSI
driver, among other improvements.
Sponsored by: Chelsio Communications
Deleted:
head/sys/dev/cxgbe/cxgbei/cxgbei_ulp2_ddp.c
head/sys/dev/cxgbe/cxgbei/cxgbei_ulp2_ddp.h
Modified:
head/sys/dev/cxgbe/cxgbei/cxgbei.c
head/sys/dev/cxgbe/cxgbei/cxgbei.h
head/sys/dev/cxgbe/cxgbei/icl_cxgbei.c
head/sys/dev/cxgbe/offload.h
head/sys/dev/cxgbe/t4_main.c
head/sys/dev/cxgbe/t4_sge.c
head/sys/modules/cxgbe/cxgbei/Makefile
Modified: head/sys/dev/cxgbe/cxgbei/cxgbei.c
==============================================================================
--- head/sys/dev/cxgbe/cxgbei/cxgbei.c Thu Sep 1 20:38:59 2016 (r305239)
+++ head/sys/dev/cxgbe/cxgbei/cxgbei.c Thu Sep 1 20:43:01 2016 (r305240)
@@ -90,7 +90,6 @@ __FBSDID("$FreeBSD$");
#include "common/t4_regs.h" /* for PCIE_MEM_ACCESS */
#include "tom/t4_tom.h"
#include "cxgbei.h"
-#include "cxgbei_ulp2_ddp.h"
static int worker_thread_count;
static struct cxgbei_worker_thread_softc *cwt_softc;
@@ -101,376 +100,6 @@ struct icl_pdu *icl_cxgbei_new_pdu(int);
void icl_cxgbei_new_pdu_set_conn(struct icl_pdu *, struct icl_conn *);
void icl_cxgbei_conn_pdu_free(struct icl_conn *, struct icl_pdu *);
-/*
- * Direct Data Placement -
- * Directly place the iSCSI Data-In or Data-Out PDU's payload into pre-posted
- * final destination host-memory buffers based on the Initiator Task Tag (ITT)
- * in Data-In or Target Task Tag (TTT) in Data-Out PDUs.
- * The host memory address is programmed into h/w in the format of pagepod
- * entries.
- * The location of the pagepod entry is encoded into ddp tag which is used as
- * the base for ITT/TTT.
- */
-
-/*
- * functions to program the pagepod in h/w
- */
-static void inline
-ppod_set(struct pagepod *ppod,
- struct cxgbei_ulp2_pagepod_hdr *hdr,
- struct cxgbei_ulp2_gather_list *gl,
- unsigned int pidx)
-{
- int i;
-
- memcpy(ppod, hdr, sizeof(*hdr));
-
- for (i = 0; i < (PPOD_PAGES + 1); i++, pidx++) {
- ppod->addr[i] = pidx < gl->nelem ?
- cpu_to_be64(gl->dma_sg[pidx].phys_addr) : 0ULL;
- }
-}
-
-static void inline
-ppod_clear(struct pagepod *ppod)
-{
- memset(ppod, 0, sizeof(*ppod));
-}
-
-static inline void
-ulp_mem_io_set_hdr(struct adapter *sc, int tid, struct ulp_mem_io *req,
- unsigned int wr_len, unsigned int dlen,
- unsigned int pm_addr)
-{
- struct ulptx_idata *idata = (struct ulptx_idata *)(req + 1);
-
- INIT_ULPTX_WR(req, wr_len, 0, 0);
- req->cmd = cpu_to_be32(V_ULPTX_CMD(ULP_TX_MEM_WRITE) |
- V_ULP_MEMIO_ORDER(is_t4(sc)) |
- V_T5_ULP_MEMIO_IMM(is_t5(sc)));
- req->dlen = htonl(V_ULP_MEMIO_DATA_LEN(dlen >> 5));
- req->len16 = htonl(DIV_ROUND_UP(wr_len - sizeof(req->wr), 16)
- | V_FW_WR_FLOWID(tid));
- req->lock_addr = htonl(V_ULP_MEMIO_ADDR(pm_addr >> 5));
-
- idata->cmd_more = htonl(V_ULPTX_CMD(ULP_TX_SC_IMM));
- idata->len = htonl(dlen);
-}
-
-#define ULPMEM_IDATA_MAX_NPPODS 1 /* 256/PPOD_SIZE */
-#define PCIE_MEMWIN_MAX_NPPODS 16 /* 1024/PPOD_SIZE */
-
-static int
-ppod_write_idata(struct cxgbei_data *ci,
- struct cxgbei_ulp2_pagepod_hdr *hdr,
- unsigned int idx, unsigned int npods,
- struct cxgbei_ulp2_gather_list *gl,
- unsigned int gl_pidx, struct toepcb *toep)
-{
- u_int dlen = PPOD_SIZE * npods;
- u_int pm_addr = idx * PPOD_SIZE + ci->llimit;
- u_int wr_len = roundup(sizeof(struct ulp_mem_io) +
- sizeof(struct ulptx_idata) + dlen, 16);
- struct ulp_mem_io *req;
- struct ulptx_idata *idata;
- struct pagepod *ppod;
- u_int i;
- struct wrqe *wr;
- struct adapter *sc = toep->vi->pi->adapter;
-
- wr = alloc_wrqe(wr_len, toep->ctrlq);
- if (wr == NULL) {
- CXGBE_UNIMPLEMENTED("ppod_write_idata: alloc_wrqe failure");
- return (ENOMEM);
- }
-
- req = wrtod(wr);
- memset(req, 0, wr_len);
- ulp_mem_io_set_hdr(sc, toep->tid, req, wr_len, dlen, pm_addr);
- idata = (struct ulptx_idata *)(req + 1);
-
- ppod = (struct pagepod *)(idata + 1);
- for (i = 0; i < npods; i++, ppod++, gl_pidx += PPOD_PAGES) {
- if (!hdr) /* clear the pagepod */
- ppod_clear(ppod);
- else /* set the pagepod */
- ppod_set(ppod, hdr, gl, gl_pidx);
- }
-
- t4_wrq_tx(sc, wr);
- return 0;
-}
-
-int
-t4_ddp_set_map(struct cxgbei_data *ci, void *iccp,
- struct cxgbei_ulp2_pagepod_hdr *hdr, u_int idx, u_int npods,
- struct cxgbei_ulp2_gather_list *gl, int reply)
-{
- struct icl_cxgbei_conn *icc = (struct icl_cxgbei_conn *)iccp;
- struct toepcb *toep = icc->toep;
- int err;
- unsigned int pidx = 0, w_npods = 0, cnt;
-
- /*
- * on T4, if we use a mix of IMMD and DSGL with ULP_MEM_WRITE,
- * the order would not be guaranteed, so we will stick with IMMD
- */
- gl->tid = toep->tid;
- gl->port_id = toep->vi->pi->port_id;
- gl->egress_dev = (void *)toep->vi->ifp;
-
- /* send via immediate data */
- for (; w_npods < npods; idx += cnt, w_npods += cnt,
- pidx += PPOD_PAGES) {
- cnt = npods - w_npods;
- if (cnt > ULPMEM_IDATA_MAX_NPPODS)
- cnt = ULPMEM_IDATA_MAX_NPPODS;
- err = ppod_write_idata(ci, hdr, idx, cnt, gl, pidx, toep);
- if (err) {
- printf("%s: ppod_write_idata failed\n", __func__);
- break;
- }
- }
- return err;
-}
-
-void
-t4_ddp_clear_map(struct cxgbei_data *ci, struct cxgbei_ulp2_gather_list *gl,
- u_int tag, u_int idx, u_int npods, struct icl_cxgbei_conn *icc)
-{
- struct toepcb *toep = icc->toep;
- int err = -1;
- u_int pidx = 0;
- u_int w_npods = 0;
- u_int cnt;
-
- for (; w_npods < npods; idx += cnt, w_npods += cnt,
- pidx += PPOD_PAGES) {
- cnt = npods - w_npods;
- if (cnt > ULPMEM_IDATA_MAX_NPPODS)
- cnt = ULPMEM_IDATA_MAX_NPPODS;
- err = ppod_write_idata(ci, NULL, idx, cnt, gl, 0, toep);
- if (err)
- break;
- }
-}
-
-static int
-cxgbei_map_sg(struct cxgbei_sgl *sgl, struct ccb_scsiio *csio)
-{
- unsigned int data_len = csio->dxfer_len;
- unsigned int sgoffset = (uint64_t)csio->data_ptr & PAGE_MASK;
- unsigned int nsge;
- unsigned char *sgaddr = csio->data_ptr;
- unsigned int len = 0;
-
- nsge = (csio->dxfer_len + sgoffset + PAGE_SIZE - 1) >> PAGE_SHIFT;
- sgl->sg_addr = sgaddr;
- sgl->sg_offset = sgoffset;
- if (data_len < (PAGE_SIZE - sgoffset))
- len = data_len;
- else
- len = PAGE_SIZE - sgoffset;
-
- sgl->sg_length = len;
-
- data_len -= len;
- sgaddr += len;
- sgl = sgl+1;
-
- while (data_len > 0) {
- sgl->sg_addr = sgaddr;
- len = (data_len < PAGE_SIZE)? data_len: PAGE_SIZE;
- sgl->sg_length = len;
- sgaddr += len;
- data_len -= len;
- sgl = sgl + 1;
- }
-
- return nsge;
-}
-
-static int
-cxgbei_map_sg_tgt(struct cxgbei_sgl *sgl, union ctl_io *io)
-{
- unsigned int data_len, sgoffset, nsge;
- unsigned char *sgaddr;
- unsigned int len = 0, index = 0, ctl_sg_count, i;
- struct ctl_sg_entry ctl_sg_entry, *ctl_sglist;
-
- if (io->scsiio.kern_sg_entries > 0) {
- ctl_sglist = (struct ctl_sg_entry *)io->scsiio.kern_data_ptr;
- ctl_sg_count = io->scsiio.kern_sg_entries;
- } else {
- ctl_sglist = &ctl_sg_entry;
- ctl_sglist->addr = io->scsiio.kern_data_ptr;
- ctl_sglist->len = io->scsiio.kern_data_len;
- ctl_sg_count = 1;
- }
-
- sgaddr = sgl->sg_addr = ctl_sglist[index].addr;
- sgoffset = sgl->sg_offset = (uint64_t)sgl->sg_addr & PAGE_MASK;
- data_len = ctl_sglist[index].len;
-
- if (data_len < (PAGE_SIZE - sgoffset))
- len = data_len;
- else
- len = PAGE_SIZE - sgoffset;
-
- sgl->sg_length = len;
-
- data_len -= len;
- sgaddr += len;
- sgl = sgl+1;
-
- len = 0;
- for (i = 0; i< ctl_sg_count; i++)
- len += ctl_sglist[i].len;
- nsge = (len + sgoffset + PAGE_SIZE -1) >> PAGE_SHIFT;
- while (data_len > 0) {
- sgl->sg_addr = sgaddr;
- len = (data_len < PAGE_SIZE)? data_len: PAGE_SIZE;
- sgl->sg_length = len;
- sgaddr += len;
- data_len -= len;
- sgl = sgl + 1;
- if (data_len == 0) {
- if (index == ctl_sg_count - 1)
- break;
- index++;
- sgaddr = ctl_sglist[index].addr;
- data_len = ctl_sglist[index].len;
- }
- }
-
- return nsge;
-}
-
-static int
-t4_sk_ddp_tag_reserve(struct cxgbei_data *ci, struct icl_cxgbei_conn *icc,
- u_int xferlen, struct cxgbei_sgl *sgl, u_int sgcnt, u_int *ddp_tag)
-{
- struct cxgbei_ulp2_gather_list *gl;
- int err = -EINVAL;
- struct toepcb *toep = icc->toep;
-
- gl = cxgbei_ulp2_ddp_make_gl_from_iscsi_sgvec(xferlen, sgl, sgcnt, ci, 0);
- if (gl) {
- err = cxgbei_ulp2_ddp_tag_reserve(ci, icc, toep->tid,
- &ci->tag_format, ddp_tag, gl, 0, 0);
- if (err) {
- cxgbei_ulp2_ddp_release_gl(ci, gl);
- }
- }
-
- return err;
-}
-
-static unsigned int
-cxgbei_task_reserve_itt(struct icl_conn *ic, void **prv,
- struct ccb_scsiio *scmd, unsigned int *itt)
-{
- struct icl_cxgbei_conn *icc = ic_to_icc(ic);
- int xferlen = scmd->dxfer_len;
- struct cxgbei_task_data *tdata = NULL;
- struct cxgbei_sgl *sge = NULL;
- struct toepcb *toep = icc->toep;
- struct adapter *sc = td_adapter(toep->td);
- struct cxgbei_data *ci = sc->iscsi_ulp_softc;
- int err = -1;
-
- MPASS(icc->icc_signature == CXGBEI_CONN_SIGNATURE);
-
- tdata = (struct cxgbei_task_data *)*prv;
- if (xferlen == 0 || tdata == NULL)
- goto out;
- if (xferlen < DDP_THRESHOLD)
- goto out;
-
- if ((scmd->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_IN) {
- tdata->nsge = cxgbei_map_sg(tdata->sgl, scmd);
- if (tdata->nsge == 0) {
- CTR1(KTR_CXGBE, "%s: map_sg failed", __func__);
- return 0;
- }
- sge = tdata->sgl;
-
- tdata->sc_ddp_tag = *itt;
-
- CTR3(KTR_CXGBE, "%s: *itt:0x%x sc_ddp_tag:0x%x",
- __func__, *itt, tdata->sc_ddp_tag);
- if (cxgbei_ulp2_sw_tag_usable(&ci->tag_format,
- tdata->sc_ddp_tag)) {
- err = t4_sk_ddp_tag_reserve(ci, icc, scmd->dxfer_len,
- sge, tdata->nsge, &tdata->sc_ddp_tag);
- } else {
- CTR3(KTR_CXGBE,
- "%s: itt:0x%x sc_ddp_tag:0x%x not usable",
- __func__, *itt, tdata->sc_ddp_tag);
- }
- }
-out:
- if (err < 0)
- tdata->sc_ddp_tag =
- cxgbei_ulp2_set_non_ddp_tag(&ci->tag_format, *itt);
-
- return tdata->sc_ddp_tag;
-}
-
-static unsigned int
-cxgbei_task_reserve_ttt(struct icl_conn *ic, void **prv, union ctl_io *io,
- unsigned int *ttt)
-{
- struct icl_cxgbei_conn *icc = ic_to_icc(ic);
- struct toepcb *toep = icc->toep;
- struct adapter *sc = td_adapter(toep->td);
- struct cxgbei_data *ci = sc->iscsi_ulp_softc;
- struct cxgbei_task_data *tdata = NULL;
- int xferlen, err = -1;
- struct cxgbei_sgl *sge = NULL;
-
- MPASS(icc->icc_signature == CXGBEI_CONN_SIGNATURE);
-
- xferlen = (io->scsiio.kern_data_len - io->scsiio.ext_data_filled);
- tdata = (struct cxgbei_task_data *)*prv;
- if ((xferlen == 0) || (tdata == NULL))
- goto out;
- if (xferlen < DDP_THRESHOLD)
- goto out;
- tdata->nsge = cxgbei_map_sg_tgt(tdata->sgl, io);
- if (tdata->nsge == 0) {
- CTR1(KTR_CXGBE, "%s: map_sg failed", __func__);
- return 0;
- }
- sge = tdata->sgl;
-
- tdata->sc_ddp_tag = *ttt;
- if (cxgbei_ulp2_sw_tag_usable(&ci->tag_format, tdata->sc_ddp_tag)) {
- err = t4_sk_ddp_tag_reserve(ci, icc, xferlen, sge,
- tdata->nsge, &tdata->sc_ddp_tag);
- } else {
- CTR2(KTR_CXGBE, "%s: sc_ddp_tag:0x%x not usable",
- __func__, tdata->sc_ddp_tag);
- }
-out:
- if (err < 0)
- tdata->sc_ddp_tag =
- cxgbei_ulp2_set_non_ddp_tag(&ci->tag_format, *ttt);
- return tdata->sc_ddp_tag;
-}
-
-static int
-t4_sk_ddp_tag_release(struct icl_cxgbei_conn *icc, unsigned int ddp_tag)
-{
- struct toepcb *toep = icc->toep;
- struct adapter *sc = td_adapter(toep->td);
- struct cxgbei_data *ci = sc->iscsi_ulp_softc;
-
- cxgbei_ulp2_ddp_tag_release(ci, ddp_tag, icc);
-
- return (0);
-}
-
static void
read_pdu_limits(struct adapter *sc, uint32_t *max_tx_pdu_len,
uint32_t *max_rx_pdu_len)
@@ -504,58 +133,43 @@ read_pdu_limits(struct adapter *sc, uint
static int
cxgbei_init(struct adapter *sc, struct cxgbei_data *ci)
{
- int nppods, bits, rc;
- static const u_int pgsz_order[] = {0, 1, 2, 3};
+ struct ppod_region *pr;
+ uint32_t r;
+ int rc;
MPASS(sc->vres.iscsi.size > 0);
+ MPASS(ci != NULL);
- ci->llimit = sc->vres.iscsi.start;
- ci->ulimit = sc->vres.iscsi.start + sc->vres.iscsi.size - 1;
read_pdu_limits(sc, &ci->max_tx_pdu_len, &ci->max_rx_pdu_len);
- nppods = sc->vres.iscsi.size >> IPPOD_SIZE_SHIFT;
- if (nppods <= 1024)
- return (ENXIO);
-
- bits = fls(nppods);
- if (bits > IPPOD_IDX_MAX_SIZE)
- bits = IPPOD_IDX_MAX_SIZE;
- nppods = (1 << (bits - 1)) - 1;
-
- rc = bus_dma_tag_create(NULL, 1, 0, BUS_SPACE_MAXADDR,
- BUS_SPACE_MAXADDR, NULL, NULL, UINT32_MAX , 8, BUS_SPACE_MAXSIZE,
- BUS_DMA_ALLOCNOW, NULL, NULL, &ci->ulp_ddp_tag);
+ ci->ddp_threshold = 2048;
+ pr = &ci->pr;
+
+ r = t4_read_reg(sc, A_ULP_RX_ISCSI_PSZ);
+ rc = t4_init_ppod_region(pr, &sc->vres.iscsi, r, "iSCSI page pods");
if (rc != 0) {
- device_printf(sc->dev, "%s: failed to create DMA tag: %u.\n",
+ device_printf(sc->dev,
+ "%s: failed to initialize the iSCSI page pod region: %u.\n",
__func__, rc);
return (rc);
}
- ci->colors = malloc(nppods * sizeof(char), M_CXGBE, M_NOWAIT | M_ZERO);
- ci->gl_map = malloc(nppods * sizeof(struct cxgbei_ulp2_gather_list *),
- M_CXGBE, M_NOWAIT | M_ZERO);
- if (ci->colors == NULL || ci->gl_map == NULL) {
- bus_dma_tag_destroy(ci->ulp_ddp_tag);
- free(ci->colors, M_CXGBE);
- free(ci->gl_map, M_CXGBE);
- return (ENOMEM);
+ r = t4_read_reg(sc, A_ULP_RX_ISCSI_TAGMASK);
+ r &= V_ISCSITAGMASK(M_ISCSITAGMASK);
+ if (r != pr->pr_tag_mask) {
+ /*
+ * Recent firmwares are supposed to set up the the iSCSI tagmask
+ * but we'll do it ourselves it the computed value doesn't match
+ * what's in the register.
+ */
+ device_printf(sc->dev,
+ "tagmask 0x%08x does not match computed mask 0x%08x.\n", r,
+ pr->pr_tag_mask);
+ t4_set_reg_field(sc, A_ULP_RX_ISCSI_TAGMASK,
+ V_ISCSITAGMASK(M_ISCSITAGMASK), pr->pr_tag_mask);
}
- mtx_init(&ci->map_lock, "ddp lock", NULL, MTX_DEF | MTX_DUPOK);
- ci->nppods = nppods;
- ci->idx_last = nppods;
- ci->idx_bits = bits;
- ci->idx_mask = (1 << bits) - 1;
- ci->rsvd_tag_mask = (1 << (bits + IPPOD_IDX_SHIFT)) - 1;
-
- ci->tag_format.sw_bits = bits;
- ci->tag_format.rsvd_bits = bits;
- ci->tag_format.rsvd_shift = IPPOD_IDX_SHIFT;
- ci->tag_format.rsvd_mask = ci->idx_mask;
-
- t4_iscsi_init(sc, ci->idx_mask << IPPOD_IDX_SHIFT, pgsz_order);
-
- return (rc);
+ return (0);
}
static int
@@ -772,47 +386,6 @@ do_rx_iscsi_ddp(struct sge_iq *iq, const
return (0);
}
-/* initiator */
-void
-cxgbei_conn_task_reserve_itt(void *conn, void **prv,
- void *scmd, unsigned int *itt)
-{
- unsigned int tag;
- tag = cxgbei_task_reserve_itt(conn, prv, scmd, itt);
- if (tag)
- *itt = htonl(tag);
- return;
-}
-
-/* target */
-void
-cxgbei_conn_transfer_reserve_ttt(void *conn, void **prv,
- void *scmd, unsigned int *ttt)
-{
- unsigned int tag;
- tag = cxgbei_task_reserve_ttt(conn, prv, scmd, ttt);
- if (tag)
- *ttt = htonl(tag);
- return;
-}
-
-void
-cxgbei_cleanup_task(void *conn, void *ofld_priv)
-{
- struct icl_conn *ic = (struct icl_conn *)conn;
- struct icl_cxgbei_conn *icc = ic_to_icc(ic);
- struct cxgbei_task_data *tdata = ofld_priv;
- struct adapter *sc = icc->sc;
- struct cxgbei_data *ci = sc->iscsi_ulp_softc;
-
- MPASS(icc->icc_signature == CXGBEI_CONN_SIGNATURE);
- MPASS(tdata != NULL);
-
- if (cxgbei_ulp2_is_ddp_tag(&ci->tag_format, tdata->sc_ddp_tag))
- t4_sk_ddp_tag_release(icc, tdata->sc_ddp_tag);
- memset(tdata, 0, sizeof(*tdata));
-}
-
static int
cxgbei_activate(struct adapter *sc)
{
@@ -834,7 +407,7 @@ cxgbei_activate(struct adapter *sc)
}
/* per-adapter softc for iSCSI */
- ci = malloc(sizeof(*ci), M_CXGBE, M_ZERO | M_NOWAIT);
+ ci = malloc(sizeof(*ci), M_CXGBE, M_ZERO | M_WAITOK);
if (ci == NULL)
return (ENOMEM);
@@ -852,12 +425,13 @@ cxgbei_activate(struct adapter *sc)
static int
cxgbei_deactivate(struct adapter *sc)
{
+ struct cxgbei_data *ci = sc->iscsi_ulp_softc;
ASSERT_SYNCHRONIZED_OP(sc);
- if (sc->iscsi_ulp_softc != NULL) {
- cxgbei_ddp_cleanup(sc->iscsi_ulp_softc);
- free(sc->iscsi_ulp_softc, M_CXGBE);
+ if (ci != NULL) {
+ t4_free_ppod_region(&ci->pr);
+ free(ci, M_CXGBE);
sc->iscsi_ulp_softc = NULL;
}
Modified: head/sys/dev/cxgbe/cxgbei/cxgbei.h
==============================================================================
--- head/sys/dev/cxgbe/cxgbei/cxgbei.h Thu Sep 1 20:38:59 2016 (r305239)
+++ head/sys/dev/cxgbe/cxgbei/cxgbei.h Thu Sep 1 20:43:01 2016 (r305240)
@@ -105,69 +105,18 @@ ip_to_icp(struct icl_pdu *ip)
return (__containerof(ip, struct icl_cxgbei_pdu, ip));
}
-struct cxgbei_sgl {
- int sg_flag;
- void *sg_addr;
- void *sg_dma_addr;
- size_t sg_offset;
- size_t sg_length;
-};
-
-#define cxgbei_scsi_for_each_sg(_sgl, _sgel, _n, _i) \
- for (_i = 0, _sgel = (cxgbei_sgl*) (_sgl); _i < _n; _i++, \
- _sgel++)
-#define sg_dma_addr(_sgel) _sgel->sg_dma_addr
-#define sg_virt(_sgel) _sgel->sg_addr
-#define sg_len(_sgel) _sgel->sg_length
-#define sg_off(_sgel) _sgel->sg_offset
-#define sg_next(_sgel) _sgel + 1
-
-/* private data for each scsi task */
-struct cxgbei_task_data {
- struct cxgbei_sgl sgl[256];
- u_int nsge;
- u_int sc_ddp_tag;
-};
-
-struct cxgbei_ulp2_tag_format {
- u_char sw_bits;
- u_char rsvd_bits;
- u_char rsvd_shift;
- u_char filler[1];
- uint32_t rsvd_mask;
-};
-
struct cxgbei_data {
- u_int llimit;
- u_int ulimit;
- u_int nppods;
- u_int idx_last;
- u_char idx_bits;
- uint32_t idx_mask;
- uint32_t rsvd_tag_mask;
u_int max_tx_pdu_len;
u_int max_rx_pdu_len;
- struct mtx map_lock;
- bus_dma_tag_t ulp_ddp_tag;
- unsigned char *colors;
- struct cxgbei_ulp2_gather_list **gl_map;
+ u_int ddp_threshold;
+ struct ppod_region pr;
- struct cxgbei_ulp2_tag_format tag_format;
};
-void cxgbei_conn_task_reserve_itt(void *, void **, void *, unsigned int *);
-void cxgbei_conn_transfer_reserve_ttt(void *, void **, void *, unsigned int *);
-void cxgbei_cleanup_task(void *, void *);
+/* cxgbei.c */
u_int cxgbei_select_worker_thread(struct icl_cxgbei_conn *);
-struct cxgbei_ulp2_pagepod_hdr;
-int t4_ddp_set_map(struct cxgbei_data *, void *,
- struct cxgbei_ulp2_pagepod_hdr *, u_int, u_int,
- struct cxgbei_ulp2_gather_list *, int);
-void t4_ddp_clear_map(struct cxgbei_data *, struct cxgbei_ulp2_gather_list *,
- u_int, u_int, u_int, struct icl_cxgbei_conn *);
-
/* icl_cxgbei.c */
int icl_cxgbei_mod_load(void);
int icl_cxgbei_mod_unload(void);
Modified: head/sys/dev/cxgbe/cxgbei/icl_cxgbei.c
==============================================================================
--- head/sys/dev/cxgbe/cxgbei/icl_cxgbei.c Thu Sep 1 20:38:59 2016 (r305239)
+++ head/sys/dev/cxgbe/cxgbei/icl_cxgbei.c Thu Sep 1 20:43:01 2016 (r305240)
@@ -60,6 +60,8 @@ __FBSDID("$FreeBSD$");
#include <sys/uio.h>
#include <machine/bus.h>
#include <vm/uma.h>
+#include <vm/vm.h>
+#include <vm/pmap.h>
#include <netinet/in.h>
#include <netinet/in_pcb.h>
#include <netinet/tcp.h>
@@ -70,6 +72,28 @@ __FBSDID("$FreeBSD$");
#include <dev/iscsi/iscsi_proto.h>
#include <icl_conn_if.h>
+#include <cam/scsi/scsi_all.h>
+#include <cam/scsi/scsi_da.h>
+#include <cam/ctl/ctl_io.h>
+#include <cam/ctl/ctl.h>
+#include <cam/ctl/ctl_backend.h>
+#include <cam/ctl/ctl_error.h>
+#include <cam/ctl/ctl_frontend.h>
+#include <cam/ctl/ctl_debug.h>
+#include <cam/ctl/ctl_ha.h>
+#include <cam/ctl/ctl_ioctl.h>
+
+#include <cam/cam.h>
+#include <cam/cam_ccb.h>
+#include <cam/cam_xpt.h>
+#include <cam/cam_debug.h>
+#include <cam/cam_sim.h>
+#include <cam/cam_xpt_sim.h>
+#include <cam/cam_xpt_periph.h>
+#include <cam/cam_periph.h>
+#include <cam/cam_compat.h>
+#include <cam/scsi/scsi_message.h>
+
#include "common/common.h"
#include "common/t4_tcb.h"
#include "tom/t4_tom.h"
@@ -90,8 +114,7 @@ static int recvspace = 1048576;
SYSCTL_INT(_kern_icl_cxgbei, OID_AUTO, recvspace, CTLFLAG_RWTUN,
&recvspace, 0, "Default receive socket buffer size");
-static uma_zone_t icl_transfer_zone;
-
+static uma_zone_t prsv_zone;
static volatile u_int icl_cxgbei_ncons;
#define ICL_CONN_LOCK(X) mtx_lock(X->ic_lock)
@@ -242,12 +265,6 @@ icl_cxgbei_conn_pdu_data_segment_length(
return (icl_pdu_data_segment_length(request));
}
-static uint32_t
-icl_conn_build_tasktag(struct icl_conn *ic, uint32_t tag)
-{
- return tag;
-}
-
static struct mbuf *
finalize_pdu(struct icl_cxgbei_conn *icc, struct icl_cxgbei_pdu *icp)
{
@@ -776,55 +793,215 @@ icl_cxgbei_conn_close(struct icl_conn *i
int
icl_cxgbei_conn_task_setup(struct icl_conn *ic, struct icl_pdu *ip,
- struct ccb_scsiio *csio, uint32_t *task_tagp, void **prvp)
+ struct ccb_scsiio *csio, uint32_t *ittp, void **arg)
{
- void *prv;
+ struct icl_cxgbei_conn *icc = ic_to_icc(ic);
+ struct toepcb *toep = icc->toep;
+ struct adapter *sc = icc->sc;
+ struct cxgbei_data *ci = sc->iscsi_ulp_softc;
+ struct ppod_region *pr = &ci->pr;
+ struct ppod_reservation *prsv;
+ uint32_t itt;
+ int rc = 0;
+
+ /* This is for the offload driver's state. Must not be set already. */
+ MPASS(arg != NULL);
+ MPASS(*arg == NULL);
+
+ if ((csio->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_IN ||
+ csio->dxfer_len < ci->ddp_threshold) {
+no_ddp:
+ /*
+ * No DDP for this I/O. Allocate an ITT (based on the one
+ * passed in) that cannot be a valid hardware DDP tag in the
+ * iSCSI region.
+ */
+ itt = *ittp & M_PPOD_TAG;
+ itt = V_PPOD_TAG(itt) | pr->pr_invalid_bit;
+ *ittp = htobe32(itt);
+ MPASS(*arg == NULL); /* State is maintained for DDP only. */
+ return (0);
+ }
- *task_tagp = icl_conn_build_tasktag(ic, *task_tagp);
+ /*
+ * Reserve resources for DDP, update the itt that should be used in the
+ * PDU, and save DDP specific state for this I/O in *arg.
+ */
- prv = uma_zalloc(icl_transfer_zone, M_NOWAIT | M_ZERO);
- if (prv == NULL)
- return (ENOMEM);
+ prsv = uma_zalloc(prsv_zone, M_NOWAIT);
+ if (prsv == NULL) {
+ rc = ENOMEM;
+ goto no_ddp;
+ }
- *prvp = prv;
+ /* XXX add support for all CAM_DATA_ types */
+ MPASS((csio->ccb_h.flags & CAM_DATA_MASK) == CAM_DATA_VADDR);
+ rc = t4_alloc_page_pods_for_buf(pr, (vm_offset_t)csio->data_ptr,
+ csio->dxfer_len, prsv);
+ if (rc != 0) {
+ uma_zfree(prsv_zone, prsv);
+ goto no_ddp;
+ }
- cxgbei_conn_task_reserve_itt(ic, prvp, csio, task_tagp);
+ rc = t4_write_page_pods_for_buf(sc, toep->ofld_txq, toep->tid, prsv,
+ (vm_offset_t)csio->data_ptr, csio->dxfer_len);
+ if (rc != 0) {
+ t4_free_page_pods(prsv);
+ uma_zfree(prsv_zone, prsv);
+ goto no_ddp;
+ }
+ *ittp = htobe32(prsv->prsv_tag);
+ *arg = prsv;
return (0);
}
void
-icl_cxgbei_conn_task_done(struct icl_conn *ic, void *prv)
+icl_cxgbei_conn_task_done(struct icl_conn *ic, void *arg)
{
- cxgbei_cleanup_task(ic, prv);
- uma_zfree(icl_transfer_zone, prv);
+ if (arg != NULL) {
+ struct ppod_reservation *prsv = arg;
+
+ t4_free_page_pods(prsv);
+ uma_zfree(prsv_zone, prsv);
+ }
}
+/* XXXNP: PDU should be passed in as parameter, like on the initiator. */
+#define io_to_request_pdu(io) ((io)->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr)
+#define io_to_ppod_reservation(io) ((io)->io_hdr.ctl_private[CTL_PRIV_FRONTEND2].ptr)
+
int
icl_cxgbei_conn_transfer_setup(struct icl_conn *ic, union ctl_io *io,
- uint32_t *transfer_tag, void **prvp)
+ uint32_t *tttp, void **arg)
{
- void *prv;
+ struct icl_cxgbei_conn *icc = ic_to_icc(ic);
+ struct toepcb *toep = icc->toep;
+ struct ctl_scsiio *ctsio = &io->scsiio;
+ struct adapter *sc = icc->sc;
+ struct cxgbei_data *ci = sc->iscsi_ulp_softc;
+ struct ppod_region *pr = &ci->pr;
+ struct ppod_reservation *prsv;
+ uint32_t ttt;
+ int xferlen, rc = 0, alias;
+
+ /* This is for the offload driver's state. Must not be set already. */
+ MPASS(arg != NULL);
+ MPASS(*arg == NULL);
+
+ if (ctsio->ext_data_filled == 0) {
+ int first_burst;
+ struct icl_pdu *ip = io_to_request_pdu(io);
+ vm_offset_t buf;
+#ifdef INVARIANTS
+ struct icl_cxgbei_pdu *icp = ip_to_icp(ip);
+
+ MPASS(icp->icp_signature == CXGBEI_PDU_SIGNATURE);
+ MPASS(ic == ip->ip_conn);
+ MPASS(ip->ip_bhs_mbuf != NULL);
+#endif
+ first_burst = icl_pdu_data_segment_length(ip);
+
+ /*
+ * Note that ICL calls conn_transfer_setup even if the first
+ * burst had everything and there's nothing left to transfer.
+ */
+ MPASS(ctsio->kern_data_len >= first_burst);
+ xferlen = ctsio->kern_data_len;
+ if (xferlen - first_burst < ci->ddp_threshold) {
+no_ddp:
+ /*
+ * No DDP for this transfer. Allocate a TTT (based on
+ * the one passed in) that cannot be a valid hardware
+ * DDP tag in the iSCSI region.
+ */
+ ttt = *tttp & M_PPOD_TAG;
+ ttt = V_PPOD_TAG(ttt) | pr->pr_invalid_bit;
+ *tttp = htobe32(ttt);
+ MPASS(io_to_ppod_reservation(io) == NULL);
+ return (0);
+ }
- *transfer_tag = icl_conn_build_tasktag(ic, *transfer_tag);
+ if (ctsio->kern_sg_entries == 0)
+ buf = (vm_offset_t)ctsio->kern_data_ptr;
+ else if (ctsio->kern_sg_entries == 1) {
+ struct ctl_sg_entry *sgl = (void *)ctsio->kern_data_ptr;
+
+ MPASS(sgl->len == xferlen);
+ buf = (vm_offset_t)sgl->addr;
+ } else {
+ rc = EAGAIN; /* XXX implement */
+ goto no_ddp;
+ }
- prv = uma_zalloc(icl_transfer_zone, M_NOWAIT | M_ZERO);
- if (prv == NULL)
- return (ENOMEM);
- *prvp = prv;
+ /*
+ * Reserve resources for DDP, update the ttt that should be used
+ * in the PDU, and save DDP specific state for this I/O.
+ */
- cxgbei_conn_transfer_reserve_ttt(ic, prvp, io, transfer_tag);
+ MPASS(io_to_ppod_reservation(io) == NULL);
+ prsv = uma_zalloc(prsv_zone, M_NOWAIT);
+ if (prsv == NULL) {
+ rc = ENOMEM;
+ goto no_ddp;
+ }
+
+ rc = t4_alloc_page_pods_for_buf(pr, buf, xferlen, prsv);
+ if (rc != 0) {
+ uma_zfree(prsv_zone, prsv);
+ goto no_ddp;
+ }
+
+ rc = t4_write_page_pods_for_buf(sc, toep->ofld_txq, toep->tid,
+ prsv, buf, xferlen);
+ if (rc != 0) {
+ t4_free_page_pods(prsv);
+ uma_zfree(prsv_zone, prsv);
+ goto no_ddp;
+ }
+
+ *tttp = htobe32(prsv->prsv_tag);
+ io_to_ppod_reservation(io) = prsv;
+ *arg = ctsio;
+ return (0);
+ }
+
+ /*
+ * In the middle of an I/O. A non-NULL page pod reservation indicates
+ * that a DDP buffer is being used for the I/O.
+ */
+
+ prsv = io_to_ppod_reservation(ctsio);
+ if (prsv == NULL)
+ goto no_ddp;
+
+ alias = (prsv->prsv_tag & pr->pr_alias_mask) >> pr->pr_alias_shift;
+ alias++;
+ prsv->prsv_tag &= ~pr->pr_alias_mask;
+ prsv->prsv_tag |= alias << pr->pr_alias_shift & pr->pr_alias_mask;
+
+ *tttp = htobe32(prsv->prsv_tag);
+ *arg = ctsio;
return (0);
}
void
-icl_cxgbei_conn_transfer_done(struct icl_conn *ic, void *prv)
+icl_cxgbei_conn_transfer_done(struct icl_conn *ic, void *arg)
{
- cxgbei_cleanup_task(ic, prv);
- uma_zfree(icl_transfer_zone, prv);
+ struct ctl_scsiio *ctsio = arg;
+
+ if (ctsio != NULL && ctsio->kern_data_len == ctsio->ext_data_filled) {
+ struct ppod_reservation *prsv;
+
+ prsv = io_to_ppod_reservation(ctsio);
+ MPASS(prsv != NULL);
+
+ t4_free_page_pods(prsv);
+ uma_zfree(prsv_zone, prsv);
+ }
}
static void
@@ -882,9 +1059,12 @@ icl_cxgbei_mod_load(void)
{
int rc;
- icl_transfer_zone = uma_zcreate("icl_transfer",
- 16 * 1024, NULL, NULL, NULL, NULL,
- UMA_ALIGN_PTR, 0);
+ /*
+ * Space to track pagepod reservations.
+ */
+ prsv_zone = uma_zcreate("Pagepod reservations",
+ sizeof(struct ppod_reservation), NULL, NULL, NULL, NULL,
+ CACHE_LINE_SIZE, 0);
refcount_init(&icl_cxgbei_ncons, 0);
@@ -903,7 +1083,7 @@ icl_cxgbei_mod_unload(void)
icl_unregister("cxgbei", false);
- uma_zdestroy(icl_transfer_zone);
+ uma_zdestroy(prsv_zone);
return (0);
}
Modified: head/sys/dev/cxgbe/offload.h
==============================================================================
--- head/sys/dev/cxgbe/offload.h Thu Sep 1 20:38:59 2016 (r305239)
+++ head/sys/dev/cxgbe/offload.h Thu Sep 1 20:43:01 2016 (r305240)
@@ -155,7 +155,6 @@ int t4_register_uld(struct uld_info *);
int t4_unregister_uld(struct uld_info *);
int t4_activate_uld(struct adapter *, int);
int t4_deactivate_uld(struct adapter *, int);
-void t4_iscsi_init(struct adapter *, u_int, const u_int *);
int uld_active(struct adapter *, int);
#endif
#endif
Modified: head/sys/dev/cxgbe/t4_main.c
==============================================================================
--- head/sys/dev/cxgbe/t4_main.c Thu Sep 1 20:38:59 2016 (r305239)
+++ head/sys/dev/cxgbe/t4_main.c Thu Sep 1 20:43:01 2016 (r305240)
@@ -8929,16 +8929,6 @@ t4_db_dropped(struct adapter *sc)
}
#ifdef TCP_OFFLOAD
-void
-t4_iscsi_init(struct adapter *sc, u_int tag_mask, const u_int *pgsz_order)
-{
-
- t4_write_reg(sc, A_ULP_RX_ISCSI_TAGMASK, tag_mask);
- t4_write_reg(sc, A_ULP_RX_ISCSI_PSZ, V_HPZ0(pgsz_order[0]) |
- V_HPZ1(pgsz_order[1]) | V_HPZ2(pgsz_order[2]) |
- V_HPZ3(pgsz_order[3]));
-}
-
static int
toe_capability(struct vi_info *vi, int enable)
{
Modified: head/sys/dev/cxgbe/t4_sge.c
==============================================================================
--- head/sys/dev/cxgbe/t4_sge.c Thu Sep 1 20:38:59 2016 (r305239)
+++ head/sys/dev/cxgbe/t4_sge.c Thu Sep 1 20:43:01 2016 (r305240)
@@ -575,11 +575,20 @@ t4_tweak_chip_settings(struct adapter *s
V_TIMERVALUE5(us_to_core_ticks(sc, intr_timer[5]));
t4_write_reg(sc, A_SGE_TIMER_VALUE_4_AND_5, v);
- /* 4K, 16K, 64K, 256K DDP "page sizes" */
+ /* 4K, 16K, 64K, 256K DDP "page sizes" for TDDP */
v = V_HPZ0(0) | V_HPZ1(2) | V_HPZ2(4) | V_HPZ3(6);
t4_write_reg(sc, A_ULP_RX_TDDP_PSZ, v);
- m = v = F_TDDPTAGTCB;
+ /*
+ * 4K, 8K, 16K, 64K DDP "page sizes" for iSCSI DDP. These have been
+ * chosen with MAXPHYS = 128K in mind. The largest DDP buffer that we
+ * may have to deal with is MAXPHYS + 1 page.
+ */
+ v = V_HPZ0(0) | V_HPZ1(1) | V_HPZ2(2) | V_HPZ3(4);
+ t4_write_reg(sc, A_ULP_RX_ISCSI_PSZ, v);
+
+ /* We use multiple DDP page sizes both in plain-TOE and ISCSI modes. */
+ m = v = F_TDDPTAGTCB | F_ISCSITAGTCB;
t4_set_reg_field(sc, A_ULP_RX_CTL, m, v);
m = V_INDICATESIZE(M_INDICATESIZE) | F_REARMDDPOFFSET |
*** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
More information about the svn-src-all
mailing list