Deleted Added
sdiff udiff text old ( 342583 ) new ( 345664 )
full compact
t4_cpl_io.c (342583) t4_cpl_io.c (345664)
1/*-
2 * Copyright (c) 2012, 2015 Chelsio Communications, Inc.
3 * All rights reserved.
4 * Written by: Navdeep Parhar <np@FreeBSD.org>
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:

--- 12 unchanged lines hidden (view full) ---

21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 */
27
28#include <sys/cdefs.h>
1/*-
2 * Copyright (c) 2012, 2015 Chelsio Communications, Inc.
3 * All rights reserved.
4 * Written by: Navdeep Parhar <np@FreeBSD.org>
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:

--- 12 unchanged lines hidden (view full) ---

21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 */
27
28#include <sys/cdefs.h>
29__FBSDID("$FreeBSD: stable/11/sys/dev/cxgbe/tom/t4_cpl_io.c 342583 2018-12-29 00:30:17Z jhb $");
29__FBSDID("$FreeBSD: stable/11/sys/dev/cxgbe/tom/t4_cpl_io.c 345664 2019-03-28 23:43:38Z jhb $");
30
31#include "opt_inet.h"
32
33#ifdef TCP_OFFLOAD
34#include <sys/param.h>
35#include <sys/aio.h>
36#include <sys/file.h>
37#include <sys/kernel.h>

--- 39 unchanged lines hidden (view full) ---

77#define V_tcp_autosndbuf_max VNET(tcp_autosndbuf_max)
78VNET_DECLARE(int, tcp_do_autorcvbuf);
79#define V_tcp_do_autorcvbuf VNET(tcp_do_autorcvbuf)
80VNET_DECLARE(int, tcp_autorcvbuf_inc);
81#define V_tcp_autorcvbuf_inc VNET(tcp_autorcvbuf_inc)
82VNET_DECLARE(int, tcp_autorcvbuf_max);
83#define V_tcp_autorcvbuf_max VNET(tcp_autorcvbuf_max)
84
30
31#include "opt_inet.h"
32
33#ifdef TCP_OFFLOAD
34#include <sys/param.h>
35#include <sys/aio.h>
36#include <sys/file.h>
37#include <sys/kernel.h>

--- 39 unchanged lines hidden (view full) ---

77#define V_tcp_autosndbuf_max VNET(tcp_autosndbuf_max)
78VNET_DECLARE(int, tcp_do_autorcvbuf);
79#define V_tcp_do_autorcvbuf VNET(tcp_do_autorcvbuf)
80VNET_DECLARE(int, tcp_autorcvbuf_inc);
81#define V_tcp_autorcvbuf_inc VNET(tcp_autorcvbuf_inc)
82VNET_DECLARE(int, tcp_autorcvbuf_max);
83#define V_tcp_autorcvbuf_max VNET(tcp_autorcvbuf_max)
84
85#define IS_AIOTX_MBUF(m) \
86 ((m)->m_flags & M_EXT && (m)->m_ext.ext_flags & EXT_FLAG_AIOTX)
87
88static void t4_aiotx_cancel(struct kaiocb *job);
89static void t4_aiotx_queue_toep(struct toepcb *toep);
90
91static size_t
92aiotx_mbuf_pgoff(struct mbuf *m)
93{
94 struct aiotx_buffer *ab;
95

--- 14 unchanged lines hidden (view full) ---

110 return (ab->ps.pages + npages);
111}
112
113void
114send_flowc_wr(struct toepcb *toep, struct flowc_tx_params *ftxp)
115{
116 struct wrqe *wr;
117 struct fw_flowc_wr *flowc;
85static void t4_aiotx_cancel(struct kaiocb *job);
86static void t4_aiotx_queue_toep(struct toepcb *toep);
87
88static size_t
89aiotx_mbuf_pgoff(struct mbuf *m)
90{
91 struct aiotx_buffer *ab;
92

--- 14 unchanged lines hidden (view full) ---

107 return (ab->ps.pages + npages);
108}
109
110void
111send_flowc_wr(struct toepcb *toep, struct flowc_tx_params *ftxp)
112{
113 struct wrqe *wr;
114 struct fw_flowc_wr *flowc;
118 unsigned int nparams = ftxp ? 8 : 6, flowclen;
115 unsigned int nparams, flowclen, paramidx;
119 struct vi_info *vi = toep->vi;
120 struct port_info *pi = vi->pi;
121 struct adapter *sc = pi->adapter;
122 unsigned int pfvf = G_FW_VIID_PFN(vi->viid) << S_FW_VIID_PFN;
123 struct ofld_tx_sdesc *txsd = &toep->txsd[toep->txsd_pidx];
124
125 KASSERT(!(toep->flags & TPF_FLOWC_WR_SENT),
126 ("%s: flowc for tid %u sent already", __func__, toep->tid));
127
116 struct vi_info *vi = toep->vi;
117 struct port_info *pi = vi->pi;
118 struct adapter *sc = pi->adapter;
119 unsigned int pfvf = G_FW_VIID_PFN(vi->viid) << S_FW_VIID_PFN;
120 struct ofld_tx_sdesc *txsd = &toep->txsd[toep->txsd_pidx];
121
122 KASSERT(!(toep->flags & TPF_FLOWC_WR_SENT),
123 ("%s: flowc for tid %u sent already", __func__, toep->tid));
124
125 if (ftxp != NULL)
126 nparams = 8;
127 else
128 nparams = 6;
129 if (toep->ulp_mode == ULP_MODE_TLS)
130 nparams++;
131 if (toep->tls.fcplenmax != 0)
132 nparams++;
133
128 flowclen = sizeof(*flowc) + nparams * sizeof(struct fw_flowc_mnemval);
129
130 wr = alloc_wrqe(roundup2(flowclen, 16), toep->ofld_txq);
131 if (wr == NULL) {
132 /* XXX */
133 panic("%s: allocation failure.", __func__);
134 }
135 flowc = wrtod(wr);
136 memset(flowc, 0, wr->wr_len);
137
138 flowc->op_to_nparams = htobe32(V_FW_WR_OP(FW_FLOWC_WR) |
139 V_FW_FLOWC_WR_NPARAMS(nparams));
140 flowc->flowid_len16 = htonl(V_FW_WR_LEN16(howmany(flowclen, 16)) |
141 V_FW_WR_FLOWID(toep->tid));
142
134 flowclen = sizeof(*flowc) + nparams * sizeof(struct fw_flowc_mnemval);
135
136 wr = alloc_wrqe(roundup2(flowclen, 16), toep->ofld_txq);
137 if (wr == NULL) {
138 /* XXX */
139 panic("%s: allocation failure.", __func__);
140 }
141 flowc = wrtod(wr);
142 memset(flowc, 0, wr->wr_len);
143
144 flowc->op_to_nparams = htobe32(V_FW_WR_OP(FW_FLOWC_WR) |
145 V_FW_FLOWC_WR_NPARAMS(nparams));
146 flowc->flowid_len16 = htonl(V_FW_WR_LEN16(howmany(flowclen, 16)) |
147 V_FW_WR_FLOWID(toep->tid));
148
143 flowc->mnemval[0].mnemonic = FW_FLOWC_MNEM_PFNVFN;
144 flowc->mnemval[0].val = htobe32(pfvf);
145 flowc->mnemval[1].mnemonic = FW_FLOWC_MNEM_CH;
146 flowc->mnemval[1].val = htobe32(pi->tx_chan);
147 flowc->mnemval[2].mnemonic = FW_FLOWC_MNEM_PORT;
148 flowc->mnemval[2].val = htobe32(pi->tx_chan);
149 flowc->mnemval[3].mnemonic = FW_FLOWC_MNEM_IQID;
150 flowc->mnemval[3].val = htobe32(toep->ofld_rxq->iq.abs_id);
149#define FLOWC_PARAM(__m, __v) \
150 do { \
151 flowc->mnemval[paramidx].mnemonic = FW_FLOWC_MNEM_##__m; \
152 flowc->mnemval[paramidx].val = htobe32(__v); \
153 paramidx++; \
154 } while (0)
155
156 paramidx = 0;
157
158 FLOWC_PARAM(PFNVFN, pfvf);
159 FLOWC_PARAM(CH, pi->tx_chan);
160 FLOWC_PARAM(PORT, pi->tx_chan);
161 FLOWC_PARAM(IQID, toep->ofld_rxq->iq.abs_id);
151 if (ftxp) {
152 uint32_t sndbuf = min(ftxp->snd_space, sc->tt.sndbuf);
153
162 if (ftxp) {
163 uint32_t sndbuf = min(ftxp->snd_space, sc->tt.sndbuf);
164
154 flowc->mnemval[4].mnemonic = FW_FLOWC_MNEM_SNDNXT;
155 flowc->mnemval[4].val = htobe32(ftxp->snd_nxt);
156 flowc->mnemval[5].mnemonic = FW_FLOWC_MNEM_RCVNXT;
157 flowc->mnemval[5].val = htobe32(ftxp->rcv_nxt);
158 flowc->mnemval[6].mnemonic = FW_FLOWC_MNEM_SNDBUF;
159 flowc->mnemval[6].val = htobe32(sndbuf);
160 flowc->mnemval[7].mnemonic = FW_FLOWC_MNEM_MSS;
161 flowc->mnemval[7].val = htobe32(ftxp->mss);
165 FLOWC_PARAM(SNDNXT, ftxp->snd_nxt);
166 FLOWC_PARAM(RCVNXT, ftxp->rcv_nxt);
167 FLOWC_PARAM(SNDBUF, sndbuf);
168 FLOWC_PARAM(MSS, ftxp->mss);
162
163 CTR6(KTR_CXGBE,
164 "%s: tid %u, mss %u, sndbuf %u, snd_nxt 0x%x, rcv_nxt 0x%x",
165 __func__, toep->tid, ftxp->mss, sndbuf, ftxp->snd_nxt,
166 ftxp->rcv_nxt);
167 } else {
169
170 CTR6(KTR_CXGBE,
171 "%s: tid %u, mss %u, sndbuf %u, snd_nxt 0x%x, rcv_nxt 0x%x",
172 __func__, toep->tid, ftxp->mss, sndbuf, ftxp->snd_nxt,
173 ftxp->rcv_nxt);
174 } else {
168 flowc->mnemval[4].mnemonic = FW_FLOWC_MNEM_SNDBUF;
169 flowc->mnemval[4].val = htobe32(512);
170 flowc->mnemval[5].mnemonic = FW_FLOWC_MNEM_MSS;
171 flowc->mnemval[5].val = htobe32(512);
175 FLOWC_PARAM(SNDBUF, 512);
176 FLOWC_PARAM(MSS, 512);
172
173 CTR2(KTR_CXGBE, "%s: tid %u", __func__, toep->tid);
174 }
177
178 CTR2(KTR_CXGBE, "%s: tid %u", __func__, toep->tid);
179 }
180 if (toep->ulp_mode == ULP_MODE_TLS)
181 FLOWC_PARAM(ULP_MODE, toep->ulp_mode);
182 if (toep->tls.fcplenmax != 0)
183 FLOWC_PARAM(TXDATAPLEN_MAX, toep->tls.fcplenmax);
184#undef FLOWC_PARAM
175
185
186 KASSERT(paramidx == nparams, ("nparams mismatch"));
187
176 txsd->tx_credits = howmany(flowclen, 16);
177 txsd->plen = 0;
178 KASSERT(toep->tx_credits >= txsd->tx_credits && toep->txsd_avail > 0,
179 ("%s: not enough credits (%d)", __func__, toep->tx_credits));
180 toep->tx_credits -= txsd->tx_credits;
181 if (__predict_false(++toep->txsd_pidx == toep->txsd_total))
182 toep->txsd_pidx = 0;
183 toep->txsd_avail--;

--- 171 unchanged lines hidden (view full) ---

355 ftxp.rcv_nxt = tp->rcv_nxt;
356 ftxp.snd_space = bufsize;
357 ftxp.mss = tp->t_maxseg;
358 send_flowc_wr(toep, &ftxp);
359
360 soisconnected(so);
361}
362
188 txsd->tx_credits = howmany(flowclen, 16);
189 txsd->plen = 0;
190 KASSERT(toep->tx_credits >= txsd->tx_credits && toep->txsd_avail > 0,
191 ("%s: not enough credits (%d)", __func__, toep->tx_credits));
192 toep->tx_credits -= txsd->tx_credits;
193 if (__predict_false(++toep->txsd_pidx == toep->txsd_total))
194 toep->txsd_pidx = 0;
195 toep->txsd_avail--;

--- 171 unchanged lines hidden (view full) ---

367 ftxp.rcv_nxt = tp->rcv_nxt;
368 ftxp.snd_space = bufsize;
369 ftxp.mss = tp->t_maxseg;
370 send_flowc_wr(toep, &ftxp);
371
372 soisconnected(so);
373}
374
363static int
375int
364send_rx_credits(struct adapter *sc, struct toepcb *toep, int credits)
365{
366 struct wrqe *wr;
367 struct cpl_rx_data_ack *req;
368 uint32_t dack = F_RX_DACK_CHANGE | V_RX_DACK_MODE(1);
369
370 KASSERT(credits >= 0, ("%s: %d credits", __func__, credits));
371

--- 5 unchanged lines hidden (view full) ---

377 INIT_TP_WR_MIT_CPL(req, CPL_RX_DATA_ACK, toep->tid);
378 req->credit_dack = htobe32(dack | V_RX_CREDITS(credits));
379
380 t4_wrq_tx(sc, wr);
381 return (credits);
382}
383
384void
376send_rx_credits(struct adapter *sc, struct toepcb *toep, int credits)
377{
378 struct wrqe *wr;
379 struct cpl_rx_data_ack *req;
380 uint32_t dack = F_RX_DACK_CHANGE | V_RX_DACK_MODE(1);
381
382 KASSERT(credits >= 0, ("%s: %d credits", __func__, credits));
383

--- 5 unchanged lines hidden (view full) ---

389 INIT_TP_WR_MIT_CPL(req, CPL_RX_DATA_ACK, toep->tid);
390 req->credit_dack = htobe32(dack | V_RX_CREDITS(credits));
391
392 t4_wrq_tx(sc, wr);
393 return (credits);
394}
395
396void
397send_rx_modulate(struct adapter *sc, struct toepcb *toep)
398{
399 struct wrqe *wr;
400 struct cpl_rx_data_ack *req;
401
402 wr = alloc_wrqe(sizeof(*req), toep->ctrlq);
403 if (wr == NULL)
404 return;
405 req = wrtod(wr);
406
407 INIT_TP_WR_MIT_CPL(req, CPL_RX_DATA_ACK, toep->tid);
408 req->credit_dack = htobe32(F_RX_MODULATE_RX);
409
410 t4_wrq_tx(sc, wr);
411}
412
413void
385t4_rcvd_locked(struct toedev *tod, struct tcpcb *tp)
386{
387 struct adapter *sc = tod->tod_softc;
388 struct inpcb *inp = tp->t_inpcb;
389 struct socket *so = inp->inp_socket;
390 struct sockbuf *sb = &so->so_rcv;
391 struct toepcb *toep = tp->t_toe;
392 int credits;
393
394 INP_WLOCK_ASSERT(inp);
395
396 SOCKBUF_LOCK_ASSERT(sb);
397 KASSERT(toep->sb_cc >= sbused(sb),
398 ("%s: sb %p has more data (%d) than last time (%d).",
399 __func__, sb, sbused(sb), toep->sb_cc));
400
414t4_rcvd_locked(struct toedev *tod, struct tcpcb *tp)
415{
416 struct adapter *sc = tod->tod_softc;
417 struct inpcb *inp = tp->t_inpcb;
418 struct socket *so = inp->inp_socket;
419 struct sockbuf *sb = &so->so_rcv;
420 struct toepcb *toep = tp->t_toe;
421 int credits;
422
423 INP_WLOCK_ASSERT(inp);
424
425 SOCKBUF_LOCK_ASSERT(sb);
426 KASSERT(toep->sb_cc >= sbused(sb),
427 ("%s: sb %p has more data (%d) than last time (%d).",
428 __func__, sb, sbused(sb), toep->sb_cc));
429
401 toep->rx_credits += toep->sb_cc - sbused(sb);
430 credits = toep->sb_cc - sbused(sb);
402 toep->sb_cc = sbused(sb);
431 toep->sb_cc = sbused(sb);
432 if (toep->ulp_mode == ULP_MODE_TLS) {
433 if (toep->tls.rcv_over >= credits) {
434 toep->tls.rcv_over -= credits;
435 credits = 0;
436 } else {
437 credits -= toep->tls.rcv_over;
438 toep->tls.rcv_over = 0;
439 }
440 }
441 toep->rx_credits += credits;
403
404 if (toep->rx_credits > 0 &&
405 (tp->rcv_wnd <= 32 * 1024 || toep->rx_credits >= 64 * 1024 ||
406 (toep->rx_credits >= 16 * 1024 && tp->rcv_wnd <= 128 * 1024) ||
407 toep->sb_cc + tp->rcv_wnd < sb->sb_lowat)) {
408
409 credits = send_rx_credits(sc, toep, toep->rx_credits);
410 toep->rx_credits -= credits;
411 tp->rcv_wnd += credits;
412 tp->rcv_adv += credits;
442
443 if (toep->rx_credits > 0 &&
444 (tp->rcv_wnd <= 32 * 1024 || toep->rx_credits >= 64 * 1024 ||
445 (toep->rx_credits >= 16 * 1024 && tp->rcv_wnd <= 128 * 1024) ||
446 toep->sb_cc + tp->rcv_wnd < sb->sb_lowat)) {
447
448 credits = send_rx_credits(sc, toep, toep->rx_credits);
449 toep->rx_credits -= credits;
450 tp->rcv_wnd += credits;
451 tp->rcv_adv += credits;
413 }
452 } else if (toep->flags & TPF_FORCE_CREDITS)
453 send_rx_modulate(sc, toep);
414}
415
416void
417t4_rcvd(struct toedev *tod, struct tcpcb *tp)
418{
419 struct inpcb *inp = tp->t_inpcb;
420 struct socket *so = inp->inp_socket;
421 struct sockbuf *sb = &so->so_rcv;
422
423 SOCKBUF_LOCK(sb);
424 t4_rcvd_locked(tod, tp);
425 SOCKBUF_UNLOCK(sb);
426}
427
428/*
429 * Close a connection by sending a CPL_CLOSE_CON_REQ message.
430 */
454}
455
456void
457t4_rcvd(struct toedev *tod, struct tcpcb *tp)
458{
459 struct inpcb *inp = tp->t_inpcb;
460 struct socket *so = inp->inp_socket;
461 struct sockbuf *sb = &so->so_rcv;
462
463 SOCKBUF_LOCK(sb);
464 t4_rcvd_locked(tod, tp);
465 SOCKBUF_UNLOCK(sb);
466}
467
468/*
469 * Close a connection by sending a CPL_CLOSE_CON_REQ message.
470 */
431static int
432close_conn(struct adapter *sc, struct toepcb *toep)
471int
472t4_close_conn(struct adapter *sc, struct toepcb *toep)
433{
434 struct wrqe *wr;
435 struct cpl_close_con_req *req;
436 unsigned int tid = toep->tid;
437
438 CTR3(KTR_CXGBE, "%s: tid %u%s", __func__, toep->tid,
439 toep->flags & TPF_FIN_SENT ? ", IGNORED" : "");
440

--- 184 unchanged lines hidden (view full) ---

625 bool aiotx_mbuf_seen;
626
627 INP_WLOCK_ASSERT(inp);
628 KASSERT(toep->flags & TPF_FLOWC_WR_SENT,
629 ("%s: flowc_wr not sent for tid %u.", __func__, toep->tid));
630
631 KASSERT(toep->ulp_mode == ULP_MODE_NONE ||
632 toep->ulp_mode == ULP_MODE_TCPDDP ||
473{
474 struct wrqe *wr;
475 struct cpl_close_con_req *req;
476 unsigned int tid = toep->tid;
477
478 CTR3(KTR_CXGBE, "%s: tid %u%s", __func__, toep->tid,
479 toep->flags & TPF_FIN_SENT ? ", IGNORED" : "");
480

--- 184 unchanged lines hidden (view full) ---

665 bool aiotx_mbuf_seen;
666
667 INP_WLOCK_ASSERT(inp);
668 KASSERT(toep->flags & TPF_FLOWC_WR_SENT,
669 ("%s: flowc_wr not sent for tid %u.", __func__, toep->tid));
670
671 KASSERT(toep->ulp_mode == ULP_MODE_NONE ||
672 toep->ulp_mode == ULP_MODE_TCPDDP ||
673 toep->ulp_mode == ULP_MODE_TLS ||
633 toep->ulp_mode == ULP_MODE_RDMA,
634 ("%s: ulp_mode %u for toep %p", __func__, toep->ulp_mode, toep));
635
636#ifdef VERBOSE_TRACES
637 CTR4(KTR_CXGBE, "%s: tid %d toep flags %#x tp flags %#x drop %d",
638 __func__, toep->tid, toep->flags, tp->t_flags);
639#endif
640 if (__predict_false(toep->flags & TPF_ABORT_SHUTDOWN))

--- 190 unchanged lines hidden (view full) ---

831 }
832 toep->txsd_avail--;
833
834 t4_l2t_send(sc, wr, toep->l2te);
835 } while (m != NULL);
836
837 /* Send a FIN if requested, but only if there's no more data to send */
838 if (m == NULL && toep->flags & TPF_SEND_FIN)
674 toep->ulp_mode == ULP_MODE_RDMA,
675 ("%s: ulp_mode %u for toep %p", __func__, toep->ulp_mode, toep));
676
677#ifdef VERBOSE_TRACES
678 CTR4(KTR_CXGBE, "%s: tid %d toep flags %#x tp flags %#x drop %d",
679 __func__, toep->tid, toep->flags, tp->t_flags);
680#endif
681 if (__predict_false(toep->flags & TPF_ABORT_SHUTDOWN))

--- 190 unchanged lines hidden (view full) ---

872 }
873 toep->txsd_avail--;
874
875 t4_l2t_send(sc, wr, toep->l2te);
876 } while (m != NULL);
877
878 /* Send a FIN if requested, but only if there's no more data to send */
879 if (m == NULL && toep->flags & TPF_SEND_FIN)
839 close_conn(sc, toep);
880 t4_close_conn(sc, toep);
840}
841
842static inline void
843rqdrop_locked(struct mbufq *q, int plen)
844{
845 struct mbuf *m;
846
847 while (plen > 0) {

--- 175 unchanged lines hidden (view full) ---

1023 }
1024 toep->txsd_avail--;
1025
1026 t4_l2t_send(sc, wr, toep->l2te);
1027 }
1028
1029 /* Send a FIN if requested, but only if there are no more PDUs to send */
1030 if (mbufq_first(pduq) == NULL && toep->flags & TPF_SEND_FIN)
881}
882
883static inline void
884rqdrop_locked(struct mbufq *q, int plen)
885{
886 struct mbuf *m;
887
888 while (plen > 0) {

--- 175 unchanged lines hidden (view full) ---

1064 }
1065 toep->txsd_avail--;
1066
1067 t4_l2t_send(sc, wr, toep->l2te);
1068 }
1069
1070 /* Send a FIN if requested, but only if there are no more PDUs to send */
1071 if (mbufq_first(pduq) == NULL && toep->flags & TPF_SEND_FIN)
1031 close_conn(sc, toep);
1072 t4_close_conn(sc, toep);
1032}
1033
1034int
1035t4_tod_output(struct toedev *tod, struct tcpcb *tp)
1036{
1037 struct adapter *sc = tod->tod_softc;
1038#ifdef INVARIANTS
1039 struct inpcb *inp = tp->t_inpcb;
1040#endif
1041 struct toepcb *toep = tp->t_toe;
1042
1043 INP_WLOCK_ASSERT(inp);
1044 KASSERT((inp->inp_flags & INP_DROPPED) == 0,
1045 ("%s: inp %p dropped.", __func__, inp));
1046 KASSERT(toep != NULL, ("%s: toep is NULL", __func__));
1047
1048 if (toep->ulp_mode == ULP_MODE_ISCSI)
1049 t4_push_pdus(sc, toep, 0);
1073}
1074
1075int
1076t4_tod_output(struct toedev *tod, struct tcpcb *tp)
1077{
1078 struct adapter *sc = tod->tod_softc;
1079#ifdef INVARIANTS
1080 struct inpcb *inp = tp->t_inpcb;
1081#endif
1082 struct toepcb *toep = tp->t_toe;
1083
1084 INP_WLOCK_ASSERT(inp);
1085 KASSERT((inp->inp_flags & INP_DROPPED) == 0,
1086 ("%s: inp %p dropped.", __func__, inp));
1087 KASSERT(toep != NULL, ("%s: toep is NULL", __func__));
1088
1089 if (toep->ulp_mode == ULP_MODE_ISCSI)
1090 t4_push_pdus(sc, toep, 0);
1091 else if (tls_tx_key(toep))
1092 t4_push_tls_records(sc, toep, 0);
1050 else
1051 t4_push_frames(sc, toep, 0);
1052
1053 return (0);
1054}
1055
1056int
1057t4_send_fin(struct toedev *tod, struct tcpcb *tp)

--- 8 unchanged lines hidden (view full) ---

1066 KASSERT((inp->inp_flags & INP_DROPPED) == 0,
1067 ("%s: inp %p dropped.", __func__, inp));
1068 KASSERT(toep != NULL, ("%s: toep is NULL", __func__));
1069
1070 toep->flags |= TPF_SEND_FIN;
1071 if (tp->t_state >= TCPS_ESTABLISHED) {
1072 if (toep->ulp_mode == ULP_MODE_ISCSI)
1073 t4_push_pdus(sc, toep, 0);
1093 else
1094 t4_push_frames(sc, toep, 0);
1095
1096 return (0);
1097}
1098
1099int
1100t4_send_fin(struct toedev *tod, struct tcpcb *tp)

--- 8 unchanged lines hidden (view full) ---

1109 KASSERT((inp->inp_flags & INP_DROPPED) == 0,
1110 ("%s: inp %p dropped.", __func__, inp));
1111 KASSERT(toep != NULL, ("%s: toep is NULL", __func__));
1112
1113 toep->flags |= TPF_SEND_FIN;
1114 if (tp->t_state >= TCPS_ESTABLISHED) {
1115 if (toep->ulp_mode == ULP_MODE_ISCSI)
1116 t4_push_pdus(sc, toep, 0);
1117 else if (tls_tx_key(toep))
1118 t4_push_tls_records(sc, toep, 0);
1074 else
1075 t4_push_frames(sc, toep, 0);
1076 }
1077
1078 return (0);
1079}
1080
1081int

--- 616 unchanged lines hidden (view full) ---

1698 txsd = &toep->txsd[toep->txsd_cidx];
1699 plen = 0;
1700 while (credits) {
1701 KASSERT(credits >= txsd->tx_credits,
1702 ("%s: too many (or partial) credits", __func__));
1703 credits -= txsd->tx_credits;
1704 toep->tx_credits += txsd->tx_credits;
1705 plen += txsd->plen;
1119 else
1120 t4_push_frames(sc, toep, 0);
1121 }
1122
1123 return (0);
1124}
1125
1126int

--- 616 unchanged lines hidden (view full) ---

1743 txsd = &toep->txsd[toep->txsd_cidx];
1744 plen = 0;
1745 while (credits) {
1746 KASSERT(credits >= txsd->tx_credits,
1747 ("%s: too many (or partial) credits", __func__));
1748 credits -= txsd->tx_credits;
1749 toep->tx_credits += txsd->tx_credits;
1750 plen += txsd->plen;
1751 if (txsd->iv_buffer) {
1752 free(txsd->iv_buffer, M_CXGBE);
1753 txsd->iv_buffer = NULL;
1754 }
1706 txsd++;
1707 toep->txsd_avail++;
1708 KASSERT(toep->txsd_avail <= toep->txsd_total,
1709 ("%s: txsd avail > total", __func__));
1710 if (__predict_false(++toep->txsd_cidx == toep->txsd_total)) {
1711 txsd = &toep->txsd[0];
1712 toep->txsd_cidx = 0;
1713 }

--- 9 unchanged lines hidden (view full) ---

1723#ifdef VERBOSE_TRACES
1724 CTR2(KTR_CXGBE, "%s: tid %d calling t4_push_frames", __func__,
1725 tid);
1726#endif
1727 toep->flags &= ~TPF_TX_SUSPENDED;
1728 CURVNET_SET(toep->vnet);
1729 if (toep->ulp_mode == ULP_MODE_ISCSI)
1730 t4_push_pdus(sc, toep, plen);
1755 txsd++;
1756 toep->txsd_avail++;
1757 KASSERT(toep->txsd_avail <= toep->txsd_total,
1758 ("%s: txsd avail > total", __func__));
1759 if (__predict_false(++toep->txsd_cidx == toep->txsd_total)) {
1760 txsd = &toep->txsd[0];
1761 toep->txsd_cidx = 0;
1762 }

--- 9 unchanged lines hidden (view full) ---

1772#ifdef VERBOSE_TRACES
1773 CTR2(KTR_CXGBE, "%s: tid %d calling t4_push_frames", __func__,
1774 tid);
1775#endif
1776 toep->flags &= ~TPF_TX_SUSPENDED;
1777 CURVNET_SET(toep->vnet);
1778 if (toep->ulp_mode == ULP_MODE_ISCSI)
1779 t4_push_pdus(sc, toep, plen);
1780 else if (tls_tx_key(toep))
1781 t4_push_tls_records(sc, toep, plen);
1731 else
1732 t4_push_frames(sc, toep, plen);
1733 CURVNET_RESTORE();
1734 } else if (plen > 0) {
1735 struct sockbuf *sb = &so->so_snd;
1736 int sbu;
1737
1738 SOCKBUF_LOCK(sb);

--- 13 unchanged lines hidden (view full) ---

1752 sowwakeup_locked(so); /* unlocks so_snd */
1753 rqdrop_locked(&toep->ulp_pdu_reclaimq, plen);
1754 } else {
1755#ifdef VERBOSE_TRACES
1756 CTR3(KTR_CXGBE, "%s: tid %d dropped %d bytes", __func__,
1757 tid, plen);
1758#endif
1759 sbdrop_locked(sb, plen);
1782 else
1783 t4_push_frames(sc, toep, plen);
1784 CURVNET_RESTORE();
1785 } else if (plen > 0) {
1786 struct sockbuf *sb = &so->so_snd;
1787 int sbu;
1788
1789 SOCKBUF_LOCK(sb);

--- 13 unchanged lines hidden (view full) ---

1803 sowwakeup_locked(so); /* unlocks so_snd */
1804 rqdrop_locked(&toep->ulp_pdu_reclaimq, plen);
1805 } else {
1806#ifdef VERBOSE_TRACES
1807 CTR3(KTR_CXGBE, "%s: tid %d dropped %d bytes", __func__,
1808 tid, plen);
1809#endif
1810 sbdrop_locked(sb, plen);
1811 if (tls_tx_key(toep)) {
1812 struct tls_ofld_info *tls_ofld = &toep->tls;
1813
1814 MPASS(tls_ofld->sb_off >= plen);
1815 tls_ofld->sb_off -= plen;
1816 }
1760 if (!TAILQ_EMPTY(&toep->aiotx_jobq))
1761 t4_aiotx_queue_toep(toep);
1762 sowwakeup_locked(so); /* unlocks so_snd */
1763 }
1764 SOCKBUF_UNLOCK_ASSERT(sb);
1765 }
1766
1767 INP_WUNLOCK(inp);

--- 35 unchanged lines hidden (view full) ---

1803 log(LOG_ERR, "%s: Unexpected CPL_SET_TCB_RPL for tid %u on iq %p\n",
1804 __func__, tid, iq);
1805#endif
1806
1807 return (0);
1808}
1809
1810void
1817 if (!TAILQ_EMPTY(&toep->aiotx_jobq))
1818 t4_aiotx_queue_toep(toep);
1819 sowwakeup_locked(so); /* unlocks so_snd */
1820 }
1821 SOCKBUF_UNLOCK_ASSERT(sb);
1822 }
1823
1824 INP_WUNLOCK(inp);

--- 35 unchanged lines hidden (view full) ---

1860 log(LOG_ERR, "%s: Unexpected CPL_SET_TCB_RPL for tid %u on iq %p\n",
1861 __func__, tid, iq);
1862#endif
1863
1864 return (0);
1865}
1866
1867void
1811t4_set_tcb_field(struct adapter *sc, struct sge_wrq *wrq, int tid,
1812 uint16_t word, uint64_t mask, uint64_t val, int reply, int cookie, int iqid)
1868t4_set_tcb_field(struct adapter *sc, struct sge_wrq *wrq, struct toepcb *toep,
1869 uint16_t word, uint64_t mask, uint64_t val, int reply, int cookie)
1813{
1814 struct wrqe *wr;
1815 struct cpl_set_tcb_field *req;
1870{
1871 struct wrqe *wr;
1872 struct cpl_set_tcb_field *req;
1873 struct ofld_tx_sdesc *txsd;
1816
1817 MPASS((cookie & ~M_COOKIE) == 0);
1874
1875 MPASS((cookie & ~M_COOKIE) == 0);
1818 MPASS((iqid & ~M_QUEUENO) == 0);
1819
1820 wr = alloc_wrqe(sizeof(*req), wrq);
1821 if (wr == NULL) {
1822 /* XXX */
1823 panic("%s: allocation failure.", __func__);
1824 }
1825 req = wrtod(wr);
1826
1876
1877 wr = alloc_wrqe(sizeof(*req), wrq);
1878 if (wr == NULL) {
1879 /* XXX */
1880 panic("%s: allocation failure.", __func__);
1881 }
1882 req = wrtod(wr);
1883
1827 INIT_TP_WR_MIT_CPL(req, CPL_SET_TCB_FIELD, tid);
1828 req->reply_ctrl = htobe16(V_QUEUENO(iqid));
1884 INIT_TP_WR_MIT_CPL(req, CPL_SET_TCB_FIELD, toep->tid);
1885 req->reply_ctrl = htobe16(V_QUEUENO(toep->ofld_rxq->iq.abs_id));
1829 if (reply == 0)
1830 req->reply_ctrl |= htobe16(F_NO_REPLY);
1831 req->word_cookie = htobe16(V_WORD(word) | V_COOKIE(cookie));
1832 req->mask = htobe64(mask);
1833 req->val = htobe64(val);
1886 if (reply == 0)
1887 req->reply_ctrl |= htobe16(F_NO_REPLY);
1888 req->word_cookie = htobe16(V_WORD(word) | V_COOKIE(cookie));
1889 req->mask = htobe64(mask);
1890 req->val = htobe64(val);
1891 if ((wrq->eq.flags & EQ_TYPEMASK) == EQ_OFLD) {
1892 txsd = &toep->txsd[toep->txsd_pidx];
1893 txsd->tx_credits = howmany(sizeof(*req), 16);
1894 txsd->plen = 0;
1895 KASSERT(toep->tx_credits >= txsd->tx_credits &&
1896 toep->txsd_avail > 0,
1897 ("%s: not enough credits (%d)", __func__,
1898 toep->tx_credits));
1899 toep->tx_credits -= txsd->tx_credits;
1900 if (__predict_false(++toep->txsd_pidx == toep->txsd_total))
1901 toep->txsd_pidx = 0;
1902 toep->txsd_avail--;
1903 }
1834
1835 t4_wrq_tx(sc, wr);
1836}
1837
1838void
1839t4_init_cpl_io_handlers(void)
1840{
1841

--- 384 unchanged lines hidden (view full) ---

2226
2227 /* This only handles writes. */
2228 if (job->uaiocb.aio_lio_opcode != LIO_WRITE)
2229 return (EOPNOTSUPP);
2230
2231 if (!sc->tt.tx_zcopy)
2232 return (EOPNOTSUPP);
2233
1904
1905 t4_wrq_tx(sc, wr);
1906}
1907
1908void
1909t4_init_cpl_io_handlers(void)
1910{
1911

--- 384 unchanged lines hidden (view full) ---

2296
2297 /* This only handles writes. */
2298 if (job->uaiocb.aio_lio_opcode != LIO_WRITE)
2299 return (EOPNOTSUPP);
2300
2301 if (!sc->tt.tx_zcopy)
2302 return (EOPNOTSUPP);
2303
2304 if (tls_tx_key(toep))
2305 return (EOPNOTSUPP);
2306
2234 SOCKBUF_LOCK(&so->so_snd);
2235#ifdef VERBOSE_TRACES
2236 CTR2(KTR_CXGBE, "%s: queueing %p", __func__, job);
2237#endif
2238 if (!aio_set_cancel_function(job, t4_aiotx_cancel))
2239 panic("new job was cancelled");
2240 TAILQ_INSERT_TAIL(&toep->aiotx_jobq, job, list);
2241 if (sowriteable(so))

--- 13 unchanged lines hidden --- 		2307 SOCKBUF_LOCK(&so->so_snd);
2308#ifdef VERBOSE_TRACES
2309 CTR2(KTR_CXGBE, "%s: queueing %p", __func__, job);
2310#endif
2311 if (!aio_set_cancel_function(job, t4_aiotx_cancel))
2312 panic("new job was cancelled");
2313 TAILQ_INSERT_TAIL(&toep->aiotx_jobq, job, list);
2314 if (sowriteable(so))

--- 13 unchanged lines hidden ---