PERFORCE change 134046 for review
Kip Macy
kmacy at FreeBSD.org
Thu Jan 24 17:18:13 PST 2008
http://perforce.freebsd.org/chv.cgi?CH=134046
Change 134046 by kmacy at kmacy:storage:toehead on 2008/01/25 01:17:18
import more DDP cpl_io infrastructure
Affected files ...
.. //depot/projects/toehead/sys/dev/cxgb/ulp/tom/cxgb_cpl_io.c#6 edit
.. //depot/projects/toehead/sys/dev/cxgb/ulp/tom/cxgb_cpl_socket.c#5 edit
.. //depot/projects/toehead/sys/dev/cxgb/ulp/tom/cxgb_defs.h#2 edit
.. //depot/projects/toehead/sys/dev/cxgb/ulp/tom/cxgb_t3_ddp.h#5 edit
Differences ...
==== //depot/projects/toehead/sys/dev/cxgb/ulp/tom/cxgb_cpl_io.c#6 (text+ko) ====
@@ -189,36 +189,35 @@
* it is sent directly.
*/
static inline void
-send_or_defer(struct socket *so, struct tcpcb *tp, struct mbuf *m, int through_l2t)
+send_or_defer(struct toepcb *toep, struct mbuf *m, int through_l2t)
{
- struct toepcb *toep = tp->t_toe;
+ struct tcpcb *tp = toep->tp_tp;
-
if (__predict_false(tp->t_state == TCPS_SYN_SENT)) {
INP_LOCK(tp->t_inpcb);
mbufq_tail(&toep->out_of_order_queue, m); // defer
INP_UNLOCK(tp->t_inpcb);
} else if (through_l2t)
- l2t_send(T3C_DEV(so), m, toep->tp_l2t); // send through L2T
+ l2t_send(TOEP_T3C_DEV(toep), m, toep->tp_l2t); // send through L2T
else
- cxgb_ofld_send(T3C_DEV(so), m); // send directly
+ cxgb_ofld_send(TOEP_T3C_DEV(toep), m); // send directly
}
static inline unsigned int
-mkprio(unsigned int cntrl, const struct socket *so)
+mkprio(unsigned int cntrl, const struct toepcb *toep)
{
- return cntrl;
+ return (cntrl);
}
/*
* Populate a TID_RELEASE WR. The skb must be already propely sized.
*/
static inline void
-mk_tid_release(struct mbuf *m, const struct socket *so, unsigned int tid)
+mk_tid_release(struct mbuf *m, const struct toepcb *toep, unsigned int tid)
{
struct cpl_tid_release *req;
- m_set_priority(m, mkprio(CPL_PRIORITY_SETUP, so));
+ m_set_priority(m, mkprio(CPL_PRIORITY_SETUP, toep));
m->m_pkthdr.len = m->m_len = sizeof(*req);
req = mtod(m, struct cpl_tid_release *);
req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
@@ -357,7 +356,7 @@
toep->tp_wr_unacked += mbuf_wrs[count];
make_tx_data_wr(so, m0, bytes, tail);
- m_set_priority(m0, mkprio(CPL_PRIORITY_DATA, so));
+ m_set_priority(m0, mkprio(CPL_PRIORITY_DATA, toep));
m_set_sgl(m0, segs);
m_set_sgllen(m0, count);
/*
@@ -473,12 +472,67 @@
req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RX_DATA_ACK, toep->tp_tid));
req->credit_dack = htonl(dack | V_RX_CREDITS(credits));
- m_set_priority(m, mkprio(CPL_PRIORITY_ACK, toeptoso(toep)));
+ m_set_priority(m, mkprio(CPL_PRIORITY_ACK, toep));
cxgb_ofld_send(TOM_DATA(tdev)->cdev, m);
return (credits);
}
/*
+ * Send RX_DATA_ACK CPL message to request a modulation timer to be scheduled.
+ * This is only used in DDP mode, so we take the opportunity to also set the
+ * DACK mode and flush any Rx credits.
+ */
+void
+t3_send_rx_modulate(struct toepcb *toep)
+{
+ struct mbuf *m;
+ struct cpl_rx_data_ack *req;
+
+ m = m_gethdr_nofail(sizeof(*req));
+
+#ifdef notyet
+ req = (struct cpl_rx_data_ack *)__skb_put(skb, sizeof(*req));
+ req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
+#else
+ req = mtod(m, struct cpl_rx_data_ack *);
+#endif
+ OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RX_DATA_ACK, toep->tp_tid));
+ req->credit_dack = htonl(F_RX_MODULATE | F_RX_DACK_CHANGE |
+ V_RX_DACK_MODE(1) |
+ V_RX_CREDITS(toep->tp_copied_seq - toep->tp_rcv_wup));
+ m_set_priority(m, mkprio(CPL_PRIORITY_CONTROL, toep));
+ cxgb_ofld_send(TOEP_T3C_DEV(toep), m);
+ toep->tp_rcv_wup = toep->tp_copied_seq;
+}
+
+/*
+ * Handle receipt of an urgent pointer.
+ */
+static void
+handle_urg_ptr(struct socket *so, uint32_t urg_seq)
+{
+#ifdef notyet
+ struct tcpcb *tp = sototcpcb(so);
+
+ urg_seq--; /* initially points past the urgent data, per BSD */
+
+ if (tp->urg_data && !after(urg_seq, tp->urg_seq))
+ return; /* duplicate pointer */
+ sk_send_sigurg(sk);
+ if (tp->urg_seq == tp->copied_seq && tp->urg_data &&
+ !sock_flag(sk, SOCK_URGINLINE) && tp->copied_seq != tp->rcv_nxt) {
+ struct sk_buff *skb = skb_peek(&sk->sk_receive_queue);
+
+ tp->copied_seq++;
+ if (skb && tp->copied_seq - TCP_SKB_CB(skb)->seq >= skb->len)
+ tom_eat_skb(sk, skb, 0);
+ }
+ tp->urg_data = TCP_URG_NOTYET;
+ tp->urg_seq = urg_seq;
+#endif
+}
+
+/*
* Returns true if a socket cannot accept new Rx data.
*/
static inline int
@@ -488,6 +542,35 @@
}
/*
+ * Process an urgent data notification.
+ */
+static void
+rx_urg_notify(struct toepcb *toep, struct mbuf *m)
+{
+ struct cpl_rx_urg_notify *hdr = cplhdr(m);
+ struct socket *so = toeptoso(toep);
+
+ VALIDATE_SOCK(so);
+
+ if (!so_no_receive(so))
+ handle_urg_ptr(so, ntohl(hdr->seq));
+
+ m_freem(m);
+}
+
+/*
+ * Handler for RX_URG_NOTIFY CPL messages.
+ */
+static int
+do_rx_urg_notify(struct t3cdev *cdev, struct mbuf *m, void *ctx)
+{
+ struct toepcb *toep = (struct toepcb *)ctx;
+
+ rx_urg_notify(toep, m);
+ return (0);
+}
+
+/*
* Set of states for which we should return RX credits.
*/
#define CREDIT_RETURN_STATE (TCPF_ESTABLISHED | TCPF_FIN_WAIT1 | TCPF_FIN_WAIT2)
@@ -643,12 +726,10 @@
static void
-__set_tcb_field(struct socket *so, struct mbuf *m, uint16_t word,
+__set_tcb_field(struct toepcb *toep, struct mbuf *m, uint16_t word,
uint64_t mask, uint64_t val, int no_reply)
{
struct cpl_set_tcb_field *req;
- struct tcpcb *tp = sototcpcb(so);
- struct toepcb *toep = tp->t_toe;
req = mtod(m, struct cpl_set_tcb_field *);
m->m_pkthdr.len = m->m_len = sizeof(*req);
@@ -660,8 +741,8 @@
req->mask = htobe64(mask);
req->val = htobe64(val);
- m_set_priority(m, mkprio(CPL_PRIORITY_CONTROL, so));
- send_or_defer(so, tp, m, 0);
+ m_set_priority(m, mkprio(CPL_PRIORITY_CONTROL, toep));
+ send_or_defer(toep, m, 0);
}
static void
@@ -679,7 +760,7 @@
m = m_gethdr_nofail(sizeof(struct cpl_set_tcb_field));
- __set_tcb_field(so, m, word, mask, val, 1);
+ __set_tcb_field(toep, m, word, mask, val, 1);
}
/*
@@ -759,7 +840,6 @@
}
-
void
t3_set_ddp_tag(struct socket *so, int buf_idx, unsigned int tag_color)
{
@@ -814,7 +894,7 @@
return (ENOMEM);
INP_LOCK_ASSERT(tp->t_inpcb);
- m_set_priority(m, mkprio(CPL_PRIORITY_CONTROL, so));
+ m_set_priority(m, mkprio(CPL_PRIORITY_CONTROL, toep));
req = mtod(m, struct cpl_get_tcb *);
m->m_pkthdr.len = m->m_len = sizeof(*req);
req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
@@ -1097,7 +1177,7 @@
struct toepcb *toep = tp->t_toe;
struct toedev *tdev = TOE_DEV(so);
- m_set_priority((struct mbuf *)m, mkprio(CPL_PRIORITY_SETUP, so));
+ m_set_priority((struct mbuf *)m, mkprio(CPL_PRIORITY_SETUP, toep));
req = mtod(m, struct cpl_act_open_req *);
m->m_pkthdr.len = m->m_len = sizeof(*req);
@@ -1346,7 +1426,7 @@
mode |= CPL_ABORT_POST_CLOSE_REQ;
m = m_gethdr_nofail(sizeof(*req));
- m_set_priority(m, mkprio(CPL_PRIORITY_DATA, so));
+ m_set_priority(m, mkprio(CPL_PRIORITY_DATA, toep));
set_arp_failure_handler(m, abort_arp_failure);
req = mtod(m, struct cpl_abort_req *);
@@ -1901,10 +1981,9 @@
tp->t_rcvtime = ticks;
sbappendstream_locked(&so->so_rcv, m);
-#ifdef notyet
- if (!sock_flag(sk, SOCK_DEAD))
- sk->sk_data_ready(sk, 0);
-#endif
+
+ if ((so->so_state & SS_NOFDREF) == 0)
+ sorwakeup_locked(so);
}
#define DDP_ERR (F_DDP_PPOD_MISMATCH | F_DDP_LLIMIT_ERR | F_DDP_ULIMIT_ERR |\
@@ -1979,9 +2058,7 @@
tp->rcv_nxt, bsp->cur_offset, ddp_report,
G_DDP_OFFSET(ddp_report));
#endif
-#if 0
- skb->mac.raw = (unsigned char *)bsp->gl;
-#endif
+ m->m_ddp_gl = (unsigned char *)bsp->gl;
m->m_pkthdr.csum_flags = (bsp->flags & DDP_BF_NOCOPY) | 1;
if (bsp->flags & DDP_BF_NOCOPY)
bsp->flags &= ~DDP_BF_NOCOPY;
@@ -1989,8 +2066,8 @@
tp->rcv_nxt += m->m_len;
tp->t_rcvtime = ticks;
- sbappendstream_locked(&so->so_rcv, m)
- ;
+ sbappendstream_locked(&so->so_rcv, m);
+
if ((so->so_state & SS_NOFDREF) == 0)
sorwakeup_locked(so);
@@ -2037,6 +2114,61 @@
}
/*
+ * For TCP DDP a PEER_CLOSE may also be an implicit RX_DDP_COMPLETE. This
+ * function deals with the data that may be reported along with the FIN.
+ * Returns -1 if no further processing of the PEER_CLOSE is needed, >= 0 to
+ * perform normal FIN-related processing. In the latter case 1 indicates that
+ * there was an implicit RX_DDP_COMPLETE and the skb should not be freed, 0 the
+ * skb can be freed.
+ */
+static int
+handle_peer_close_data(struct socket *so, struct mbuf *m)
+{
+ struct tcpcb *tp = sototcpcb(so);
+ struct toepcb *toep = tp->t_toe;
+ struct ddp_state *q;
+ struct ddp_buf_state *bsp;
+ struct cpl_peer_close *req = cplhdr(m);
+ unsigned int rcv_nxt = ntohl(req->rcv_nxt) - 1; /* exclude FIN */
+
+ if (tp->rcv_nxt == rcv_nxt) /* no data */
+ return 0;
+
+ if (__predict_false(so_no_receive(so))) {
+ handle_excess_rx(toep, m);
+
+ /*
+ * Although we discard the data we want to process the FIN so
+ * that PEER_CLOSE + data behaves the same as RX_DATA_DDP +
+ * PEER_CLOSE without data. In particular this PEER_CLOSE
+ * may be what will close the connection. We return 1 because
+ * handle_excess_rx() already freed the packet.
+ */
+ return (1);
+ }
+
+ q = &toep->tp_ddp_state;
+ bsp = &q->buf_state[q->cur_buf];
+ m->m_pkthdr.len = rcv_nxt - tp->rcv_nxt;
+ m->m_ddp_gl = (unsigned char *)bsp->gl;
+ m->m_cur_offset = bsp->cur_offset;
+ m->m_ddp_flags =
+ DDP_BF_PSH | (bsp->flags & DDP_BF_NOCOPY) | 1;
+ m->m_seq = tp->rcv_nxt;
+ tp->rcv_nxt = rcv_nxt;
+ bsp->cur_offset += m->m_pkthdr.len;
+ if (!(bsp->flags & DDP_BF_NOFLIP))
+ q->cur_buf ^= 1;
+ tp->t_rcvtime = ticks;
+#ifdef notyet
+ __skb_queue_tail(&sk->sk_receive_queue, skb);
+#endif
+ if (__predict_true((so->so_state & SS_NOFDREF) == 0))
+ sorwakeup(so);
+ return (1);
+}
+
+/*
* Handle a peer FIN.
*/
static void
@@ -2058,15 +2190,11 @@
goto out;
}
-#ifdef notyet
- if (ULP_MODE(tp) == ULP_MODE_TCPDDP) {
- keep = handle_peer_close_data(so, skb);
+ if (toep->tp_ulp_mode == ULP_MODE_TCPDDP) {
+ keep = handle_peer_close_data(so, m);
if (keep < 0)
return;
}
- sk->sk_shutdown |= RCV_SHUTDOWN;
- sock_set_flag(so, SOCK_DONE);
-#endif
INP_INFO_WLOCK(&tcbinfo);
INP_LOCK(tp->t_inpcb);
if (TCPS_HAVERCVDFIN(tp->t_state) == 0)
@@ -2113,8 +2241,6 @@
sowwakeup(so);
wakeup(&so->so_timeo);
#ifdef notyet
- sk->sk_state_change(sk);
-
/* Do not send POLL_HUP for half duplex close. */
if ((sk->sk_shutdown & SEND_SHUTDOWN) ||
sk->sk_state == TCP_CLOSE)
@@ -2525,10 +2651,8 @@
(is_t3a(TOE_DEV(so)) && (toep->tp_flags & TP_CLOSE_CON_REQUESTED))) {
so->so_error = abort_status_to_errno(so, req->status,
&rst_status);
-#if 0
- if (!sock_flag(sk, SOCK_DEAD))
- sk->sk_error_report(sk);
-#endif
+ if (__predict_true((so->so_state & SS_NOFDREF) == 0))
+ sorwakeup(so);
/*
* SYN_RECV needs special processing. If abort_syn_rcv()
* returns 0 is has taken care of the abort.
@@ -2937,8 +3061,7 @@
DPRINTF("opt0l_status=%08x\n", rpl->opt0l_status);
- m_set_priority(reply_mbuf, mkprio(CPL_PRIORITY_SETUP, so));
-
+ m_set_priority(reply_mbuf, mkprio(CPL_PRIORITY_SETUP, newtoep));
#ifdef DEBUG_PRINT
{
int i;
@@ -2954,24 +3077,22 @@
l2t_send(cdev, reply_mbuf, e);
m_free(m);
-#ifdef notyet
/*
* XXX this call path has to be converted to not depend on sockets
*/
if (newtoep->tp_ulp_mode)
- __set_tcb_field(newso, ddp_mbuf, W_TCB_RX_DDP_FLAGS,
+ __set_tcb_field(newtoep, ddp_mbuf, W_TCB_RX_DDP_FLAGS,
V_TF_DDP_OFF(1) |
TP_DDP_TIMER_WORKAROUND_MASK,
V_TF_DDP_OFF(1) |
TP_DDP_TIMER_WORKAROUND_VAL, 1);
-#endif
return;
reject:
if (tdev->tod_ttid == TOE_ID_CHELSIO_T3)
mk_pass_accept_rpl(reply_mbuf, m);
else
- mk_tid_release(reply_mbuf, NULL, tid);
+ mk_tid_release(reply_mbuf, newtoep, tid);
cxgb_ofld_send(cdev, reply_mbuf);
m_free(m);
out:
@@ -3184,10 +3305,9 @@
tp = sototcpcb(so);
INP_LOCK(tp->t_inpcb);
-#ifdef notyet
- so->so_snd.sb_flags |= SB_TOE;
- so->so_rcv.sb_flags |= SB_TOE;
-#endif
+
+ so->so_snd.sb_flags |= SB_NOCOALESCE;
+ so->so_rcv.sb_flags |= SB_NOCOALESCE;
toep->tp_tp = tp;
toep->tp_flags = 0;
tp->t_toe = toep;
@@ -3495,6 +3615,15 @@
return 0;
}
+/*
+ * Handler for TRACE_PKT CPL messages. Just sink these packets.
+ */
+static int
+do_trace_pkt(struct t3cdev *dev, struct mbuf *m, void *ctx)
+{
+ m_freem(m);
+ return 0;
+}
/*
* Reset a connection that is on a listener's SYN queue or accept queue,
@@ -3574,7 +3703,7 @@
for (i = 0; i < nppods; ++i) {
m = m_gethdr_nofail(sizeof(*req) + PPOD_SIZE);
- m_set_priority(m, mkprio(CPL_PRIORITY_CONTROL, so));
+ m_set_priority(m, mkprio(CPL_PRIORITY_CONTROL, toep));
req = mtod(m, struct ulp_mem_io *);
m->m_pkthdr.len = m->m_len = sizeof(*req) + PPOD_SIZE;
req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_BYPASS));
@@ -3596,14 +3725,281 @@
htobe64(VM_PAGE_TO_PHYS(gl->dgl_pages[pidx])) : 0;
} else
p->pp_vld_tid = 0; /* mark sentinel page pods invalid */
- send_or_defer(so, tp, m, 0);
+ send_or_defer(toep, m, 0);
ppod_addr += PPOD_SIZE;
}
return (0);
}
+/*
+ * Build a CPL_BARRIER message as payload of a ULP_TX_PKT command.
+ */
+static inline void
+mk_cpl_barrier_ulp(struct cpl_barrier *b)
+{
+ struct ulp_txpkt *txpkt = (struct ulp_txpkt *)b;
+
+ txpkt->cmd_dest = htonl(V_ULPTX_CMD(ULP_TXPKT));
+ txpkt->len = htonl(V_ULPTX_NFLITS(sizeof(*b) / 8));
+ b->opcode = CPL_BARRIER;
+}
+
+/*
+ * Build a CPL_GET_TCB message as payload of a ULP_TX_PKT command.
+ */
+static inline void
+mk_get_tcb_ulp(struct cpl_get_tcb *req, unsigned int tid, unsigned int cpuno)
+{
+ struct ulp_txpkt *txpkt = (struct ulp_txpkt *)req;
+
+ txpkt = (struct ulp_txpkt *)req;
+ txpkt->cmd_dest = htonl(V_ULPTX_CMD(ULP_TXPKT));
+ txpkt->len = htonl(V_ULPTX_NFLITS(sizeof(*req) / 8));
+ OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_GET_TCB, tid));
+ req->cpuno = htons(cpuno);
+}
+
+/*
+ * Build a CPL_SET_TCB_FIELD message as payload of a ULP_TX_PKT command.
+ */
+static inline void
+mk_set_tcb_field_ulp(struct cpl_set_tcb_field *req, unsigned int tid,
+ unsigned int word, uint64_t mask, uint64_t val)
+{
+ struct ulp_txpkt *txpkt = (struct ulp_txpkt *)req;
+
+ txpkt->cmd_dest = htonl(V_ULPTX_CMD(ULP_TXPKT));
+ txpkt->len = htonl(V_ULPTX_NFLITS(sizeof(*req) / 8));
+ OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid));
+ req->reply = V_NO_REPLY(1);
+ req->cpu_idx = 0;
+ req->word = htons(word);
+ req->mask = htobe64(mask);
+ req->val = htobe64(val);
+}
+
+/*
+ * Build a CPL_RX_DATA_ACK message as payload of a ULP_TX_PKT command.
+ */
+static void
+mk_rx_data_ack_ulp(struct cpl_rx_data_ack *ack, unsigned int tid, unsigned int credits)
+{
+ struct ulp_txpkt *txpkt = (struct ulp_txpkt *)ack;
+
+ txpkt->cmd_dest = htonl(V_ULPTX_CMD(ULP_TXPKT));
+ txpkt->len = htonl(V_ULPTX_NFLITS(sizeof(*ack) / 8));
+ OPCODE_TID(ack) = htonl(MK_OPCODE_TID(CPL_RX_DATA_ACK, tid));
+ ack->credit_dack = htonl(F_RX_MODULATE | F_RX_DACK_CHANGE |
+ V_RX_DACK_MODE(1) | V_RX_CREDITS(credits));
+}
void
+t3_cancel_ddpbuf(struct toepcb *toep, unsigned int bufidx)
+{
+ unsigned int wrlen;
+ struct mbuf *m;
+ struct work_request_hdr *wr;
+ struct cpl_barrier *lock;
+ struct cpl_set_tcb_field *req;
+ struct cpl_get_tcb *getreq;
+ struct ddp_state *p = &toep->tp_ddp_state;
+
+ wrlen = sizeof(*wr) + sizeof(*req) + 2 * sizeof(*lock) +
+ sizeof(*getreq);
+ m = m_gethdr_nofail(wrlen);
+ m_set_priority(m, mkprio(CPL_PRIORITY_CONTROL, toep));
+#ifdef notyet
+ wr = (struct work_request_hdr *)__skb_put(skb, wrlen);
+ wr->wr_hi = htonl(V_WR_OP(FW_WROPCODE_BYPASS));
+#else
+ wr = mtod(m, struct work_request_hdr *);
+#endif
+ lock = (struct cpl_barrier *)(wr + 1);
+ mk_cpl_barrier_ulp(lock);
+
+ req = (struct cpl_set_tcb_field *)(lock + 1);
+
+ /* Hmmm, not sure if this actually a good thing: reactivating
+ * the other buffer might be an issue if it has been completed
+ * already. However, that is unlikely, since the fact that the UBUF
+ * is not completed indicates that there is no oustanding data.
+ */
+ if (bufidx == 0)
+ mk_set_tcb_field_ulp(req, toep->tp_tid, W_TCB_RX_DDP_FLAGS,
+ V_TF_DDP_ACTIVE_BUF(1) |
+ V_TF_DDP_BUF0_VALID(1),
+ V_TF_DDP_ACTIVE_BUF(1));
+ else
+ mk_set_tcb_field_ulp(req, toep->tp_tid, W_TCB_RX_DDP_FLAGS,
+ V_TF_DDP_ACTIVE_BUF(1) |
+ V_TF_DDP_BUF1_VALID(1), 0);
+
+ getreq = (struct cpl_get_tcb *)(req + 1);
+ mk_get_tcb_ulp(getreq, toep->tp_tid, toep->tp_qset);
+
+ mk_cpl_barrier_ulp((struct cpl_barrier *)(getreq + 1));
+
+ /* Keep track of the number of oustanding CPL_GET_TCB requests
+ */
+ p->get_tcb_count++;
+
+#ifdef T3_TRACE
+ T3_TRACE1(TIDTB(so),
+ "t3_cancel_ddpbuf: bufidx %u", bufidx);
+#endif
+ cxgb_ofld_send(TOEP_T3C_DEV(toep), m);
+}
+
+/**
+ * t3_overlay_ddpbuf - overlay an existing DDP buffer with a new one
+ * @sk: the socket associated with the buffers
+ * @bufidx: index of HW DDP buffer (0 or 1)
+ * @tag0: new tag for HW buffer 0
+ * @tag1: new tag for HW buffer 1
+ * @len: new length for HW buf @bufidx
+ *
+ * Sends a compound WR to overlay a new DDP buffer on top of an existing
+ * buffer by changing the buffer tag and length and setting the valid and
+ * active flag accordingly. The caller must ensure the new buffer is at
+ * least as big as the existing one. Since we typically reprogram both HW
+ * buffers this function sets both tags for convenience. Read the TCB to
+ * determine how made data was written into the buffer before the overlay
+ * took place.
+ */
+void
+t3_overlay_ddpbuf(struct toepcb *toep, unsigned int bufidx, unsigned int tag0,
+ unsigned int tag1, unsigned int len)
+{
+ unsigned int wrlen;
+ struct mbuf *m;
+ struct work_request_hdr *wr;
+ struct cpl_get_tcb *getreq;
+ struct cpl_set_tcb_field *req;
+ struct ddp_state *p = &toep->tp_ddp_state;
+
+ wrlen = sizeof(*wr) + 3 * sizeof(*req) + sizeof(*getreq);
+ m = m_gethdr_nofail(wrlen);
+ m_set_priority(m, mkprio(CPL_PRIORITY_CONTROL, toep));
+#ifdef notyet
+ wr = (struct work_request_hdr *)__skb_put(skb, wrlen);
+#else
+ wr = mtod(m, struct work_request_hdr *);
+#endif
+ /* Set the ATOMIC flag to make sure that TP processes the following
+ * CPLs in an atomic manner and no wire segments can be interleaved.
+ */
+ wr->wr_hi = htonl(V_WR_OP(FW_WROPCODE_BYPASS) | F_WR_ATOMIC);
+
+ req = (struct cpl_set_tcb_field *)(wr + 1);
+ mk_set_tcb_field_ulp(req, toep->tp_tid, W_TCB_RX_DDP_BUF0_TAG,
+ V_TCB_RX_DDP_BUF0_TAG(M_TCB_RX_DDP_BUF0_TAG) |
+ V_TCB_RX_DDP_BUF1_TAG(M_TCB_RX_DDP_BUF1_TAG) << 32,
+ V_TCB_RX_DDP_BUF0_TAG(tag0) |
+ V_TCB_RX_DDP_BUF1_TAG((uint64_t)tag1) << 32);
+ req++;
+ if (bufidx == 0) {
+ mk_set_tcb_field_ulp(req, toep->tp_tid, W_TCB_RX_DDP_BUF0_LEN,
+ V_TCB_RX_DDP_BUF0_LEN(M_TCB_RX_DDP_BUF0_LEN),
+ V_TCB_RX_DDP_BUF0_LEN((uint64_t)len));
+ req++;
+ mk_set_tcb_field_ulp(req, toep->tp_tid, W_TCB_RX_DDP_FLAGS,
+ V_TF_DDP_PUSH_DISABLE_0(1) |
+ V_TF_DDP_BUF0_VALID(1) | V_TF_DDP_ACTIVE_BUF(1),
+ V_TF_DDP_PUSH_DISABLE_0(0) |
+ V_TF_DDP_BUF0_VALID(1));
+ } else {
+ mk_set_tcb_field_ulp(req, toep->tp_tid, W_TCB_RX_DDP_BUF1_LEN,
+ V_TCB_RX_DDP_BUF1_LEN(M_TCB_RX_DDP_BUF1_LEN),
+ V_TCB_RX_DDP_BUF1_LEN((uint64_t)len));
+ req++;
+ mk_set_tcb_field_ulp(req, toep->tp_tid, W_TCB_RX_DDP_FLAGS,
+ V_TF_DDP_PUSH_DISABLE_1(1) |
+ V_TF_DDP_BUF1_VALID(1) | V_TF_DDP_ACTIVE_BUF(1),
+ V_TF_DDP_PUSH_DISABLE_1(0) |
+ V_TF_DDP_BUF1_VALID(1) | V_TF_DDP_ACTIVE_BUF(1));
+ }
+
+ getreq = (struct cpl_get_tcb *)(req + 1);
+ mk_get_tcb_ulp(getreq, toep->tp_tid, toep->tp_qset);
+
+ /* Keep track of the number of oustanding CPL_GET_TCB requests
+ */
+ p->get_tcb_count++;
+
+#ifdef T3_TRACE
+ T3_TRACE4(TIDTB(sk),
+ "t3_overlay_ddpbuf: bufidx %u tag0 %u tag1 %u "
+ "len %d",
+ bufidx, tag0, tag1, len);
+#endif
+ cxgb_ofld_send(TOEP_T3C_DEV(toep), m);
+}
+
+/*
+ * Sends a compound WR containing all the CPL messages needed to program the
+ * two HW DDP buffers, namely optionally setting up the length and offset of
+ * each buffer, programming the DDP flags, and optionally sending RX_DATA_ACK.
+ */
+void
+t3_setup_ddpbufs(struct toepcb *toep, unsigned int len0, unsigned int offset0,
+ unsigned int len1, unsigned int offset1,
+ uint64_t ddp_flags, uint64_t flag_mask, int modulate)
+{
+ unsigned int wrlen;
+ struct mbuf *m;
+ struct work_request_hdr *wr;
+ struct cpl_set_tcb_field *req;
+
+ wrlen = sizeof(*wr) + sizeof(*req) + (len0 ? sizeof(*req) : 0) +
+ (len1 ? sizeof(*req) : 0) +
+ (modulate ? sizeof(struct cpl_rx_data_ack) : 0);
+ m = m_gethdr_nofail(wrlen);
+ m_set_priority(m, mkprio(CPL_PRIORITY_CONTROL, toep));
+#ifdef notyet
+ wr = (struct work_request_hdr *)__skb_put(skb, wrlen);
+ wr->wr_hi = htonl(V_WR_OP(FW_WROPCODE_BYPASS));
+#else
+ wr = mtod(m, struct work_request_hdr *);
+#endif
+ req = (struct cpl_set_tcb_field *)(wr + 1);
+ if (len0) { /* program buffer 0 offset and length */
+ mk_set_tcb_field_ulp(req, toep->tp_tid, W_TCB_RX_DDP_BUF0_OFFSET,
+ V_TCB_RX_DDP_BUF0_OFFSET(M_TCB_RX_DDP_BUF0_OFFSET) |
+ V_TCB_RX_DDP_BUF0_LEN(M_TCB_RX_DDP_BUF0_LEN),
+ V_TCB_RX_DDP_BUF0_OFFSET((uint64_t)offset0) |
+ V_TCB_RX_DDP_BUF0_LEN((uint64_t)len0));
+ req++;
+ }
+ if (len1) { /* program buffer 1 offset and length */
+ mk_set_tcb_field_ulp(req, toep->tp_tid, W_TCB_RX_DDP_BUF1_OFFSET,
+ V_TCB_RX_DDP_BUF1_OFFSET(M_TCB_RX_DDP_BUF1_OFFSET) |
+ V_TCB_RX_DDP_BUF1_LEN(M_TCB_RX_DDP_BUF1_LEN) << 32,
+ V_TCB_RX_DDP_BUF1_OFFSET((uint64_t)offset1) |
+ V_TCB_RX_DDP_BUF1_LEN((uint64_t)len1) << 32);
+ req++;
+ }
+
+ mk_set_tcb_field_ulp(req, toep->tp_tid, W_TCB_RX_DDP_FLAGS, flag_mask,
+ ddp_flags);
+
+ if (modulate) {
+ mk_rx_data_ack_ulp((struct cpl_rx_data_ack *)(req + 1), toep->tp_tid,
+ toep->tp_copied_seq - toep->tp_rcv_wup);
+ toep->tp_rcv_wup = toep->tp_copied_seq;
+ }
+
+#ifdef T3_TRACE
+ T3_TRACE5(TIDTB(sk),
+ "t3_setup_ddpbufs: len0 %u len1 %u ddp_flags 0x%08x%08x "
+ "modulate %d",
+ len0, len1, ddp_flags >> 32, ddp_flags & 0xffffffff,
+ modulate);
+#endif
+
+ cxgb_ofld_send(TOEP_T3C_DEV(toep), m);
+}
+
+void
t3_init_wr_tab(unsigned int wr_len)
{
int i;
@@ -3650,10 +4046,8 @@
t3tom_register_cpl_handler(CPL_ABORT_RPL_RSS, do_abort_rpl);
t3tom_register_cpl_handler(CPL_RX_DATA_DDP, do_rx_data_ddp);
t3tom_register_cpl_handler(CPL_RX_DDP_COMPLETE, do_rx_ddp_complete);
-#ifdef notyet
t3tom_register_cpl_handler(CPL_RX_URG_NOTIFY, do_rx_urg_notify);
t3tom_register_cpl_handler(CPL_TRACE_PKT, do_trace_pkt);
-#endif
t3tom_register_cpl_handler(CPL_GET_TCB_RPL, do_get_tcb_rpl);
return (0);
}
==== //depot/projects/toehead/sys/dev/cxgb/ulp/tom/cxgb_cpl_socket.c#5 (text+ko) ====
@@ -810,14 +810,14 @@
p->cur_buf = bufidx;
p->kbuf_idx = bufidx;
if (!bufidx)
- t3_setup_ddpbufs(so, 0, 0, 0, 0,
+ t3_setup_ddpbufs(toep, 0, 0, 0, 0,
V_TF_DDP_PSH_NO_INVALIDATE(p->kbuf_noinval) |
V_TF_DDP_BUF0_VALID(1),
V_TF_DDP_PSH_NO_INVALIDATE(1) | V_TF_DDP_OFF(1) |
V_TF_DDP_BUF0_VALID(1) |
V_TF_DDP_ACTIVE_BUF(activate), modulate);
else
- t3_setup_ddpbufs(so, 0, 0, 0, 0,
+ t3_setup_ddpbufs(toep, 0, 0, 0, 0,
V_TF_DDP_PSH_NO_INVALIDATE(p->kbuf_noinval) |
V_TF_DDP_BUF1_VALID(1) |
V_TF_DDP_ACTIVE_BUF(activate),
@@ -956,15 +956,15 @@
}
if (ubuf_idx == 0) {
- t3_overlay_ddpbuf(so, 0, p->ubuf_tag << 6, p->kbuf_tag[1] << 6,
+ t3_overlay_ddpbuf(toep, 0, p->ubuf_tag << 6, p->kbuf_tag[1] << 6,
len);
- t3_setup_ddpbufs(so, 0, 0, p->kbuf[1]->dgl_length, 0,
+ t3_setup_ddpbufs(toep, 0, 0, p->kbuf[1]->dgl_length, 0,
flags,
OVERLAY_MASK | flags, 1);
} else {
- t3_overlay_ddpbuf(so, 1, p->kbuf_tag[0] << 6, p->ubuf_tag << 6,
+ t3_overlay_ddpbuf(toep, 1, p->kbuf_tag[0] << 6, p->ubuf_tag << 6,
len);
- t3_setup_ddpbufs(so, p->kbuf[0]->dgl_length, 0, 0, 0,
+ t3_setup_ddpbufs(toep, p->kbuf[0]->dgl_length, 0, 0, 0,
flags,
OVERLAY_MASK | flags, 1);
}
==== //depot/projects/toehead/sys/dev/cxgb/ulp/tom/cxgb_defs.h#2 (text+ko) ====
@@ -40,6 +40,7 @@
#define toeptoso(toep) ((toep)->tp_tp->t_inpcb->inp_socket)
#define sototoep(so) (sototcpcb((so))->t_toe)
+struct toepcb;
struct listen_ctx;
typedef void (*defer_handler_t)(struct toedev *dev, struct mbuf *m);
@@ -54,6 +55,7 @@
int t3_init_cpl_io(void);
void t3_init_wr_tab(unsigned int wr_len);
uint32_t t3_send_rx_credits(struct tcpcb *tp, uint32_t credits, uint32_t dack, int nofail);
+void t3_send_rx_modulate(struct toepcb *toep);
void t3_cleanup_rbuf(struct tcpcb *tp);
void t3_init_socket_ops(void);
==== //depot/projects/toehead/sys/dev/cxgb/ulp/tom/cxgb_t3_ddp.h#5 (text+ko) ====
@@ -165,10 +165,10 @@
int t3_enter_ddp(struct socket *so, unsigned int kbuf_size, unsigned int waitall);
void t3_cleanup_ddp(struct socket *so);
void t3_release_ddp_resources(struct toepcb *toep);
-void t3_cancel_ddpbuf(struct socket *so, unsigned int bufidx);
-void t3_overlay_ddpbuf(struct socket *so, unsigned int bufidx, unsigned int tag0,
+void t3_cancel_ddpbuf(struct toepcb *, unsigned int bufidx);
+void t3_overlay_ddpbuf(struct toepcb *, unsigned int bufidx, unsigned int tag0,
unsigned int tag1, unsigned int len);
-void t3_setup_ddpbufs(struct socket *so, unsigned int len0, unsigned int offset0,
+void t3_setup_ddpbufs(struct toepcb *, unsigned int len0, unsigned int offset0,
unsigned int len1, unsigned int offset1,
uint64_t ddp_flags, uint64_t flag_mask, int modulate);
#endif /* T3_DDP_H */
More information about the p4-projects
mailing list