Lines Matching defs:toep
101 send_flowc_wr(struct toepcb *toep, struct flowc_tx_params *ftxp)
106 struct port_info *pi = toep->port;
109 struct ofld_tx_sdesc *txsd = &toep->txsd[toep->txsd_pidx];
111 KASSERT(!(toep->flags & TPF_FLOWC_WR_SENT),
112 ("%s: flowc for tid %u sent already", __func__, toep->tid));
114 CTR2(KTR_CXGBE, "%s: tid %u", __func__, toep->tid);
118 wr = alloc_wrqe(roundup2(flowclen, 16), toep->ofld_txq);
129 V_FW_WR_FLOWID(toep->tid));
138 flowc->mnemval[3].val = htobe32(toep->ofld_rxq->iq.abs_id);
159 KASSERT(toep->tx_credits >= txsd->tx_credits && toep->txsd_avail > 0,
160 ("%s: not enough credits (%d)", __func__, toep->tx_credits));
161 toep->tx_credits -= txsd->tx_credits;
162 if (__predict_false(++toep->txsd_pidx == toep->txsd_total))
163 toep->txsd_pidx = 0;
164 toep->txsd_avail--;
166 toep->flags |= TPF_FLOWC_WR_SENT;
171 send_reset(struct adapter *sc, struct toepcb *toep, uint32_t snd_nxt)
175 int tid = toep->tid;
176 struct inpcb *inp = toep->inp;
182 __func__, toep->tid,
185 toep->flags, inp->inp_flags,
186 toep->flags & TPF_ABORT_SHUTDOWN ?
189 if (toep->flags & TPF_ABORT_SHUTDOWN)
192 toep->flags |= TPF_ABORT_SHUTDOWN;
194 KASSERT(toep->flags & TPF_FLOWC_WR_SENT,
197 wr = alloc_wrqe(sizeof(*req), toep->ofld_txq);
209 req->rsvd1 = !(toep->flags & TPF_TX_DATA_SENT);
223 t4_l2t_send(sc, wr, toep->l2te);
233 struct toepcb *toep = tp->t_toe;
234 struct adapter *sc = td_adapter(toep->td);
270 make_established(struct toepcb *toep, uint32_t snd_isn, uint32_t rcv_isn,
273 struct inpcb *inp = toep->inp;
287 CTR4(KTR_CXGBE, "%s: tid %d, toep %p, inp %p",
288 __func__, toep->tid, toep, inp);
296 tp->rcv_wnd = toep->rx_credits << 10;
308 toep->rx_credits = bufsize - tp->rcv_wnd;
329 send_flowc_wr(toep, &ftxp);
335 send_rx_credits(struct adapter *sc, struct toepcb *toep, int credits)
343 wr = alloc_wrqe(sizeof(*req), toep->ctrlq);
348 INIT_TP_WR_MIT_CPL(req, CPL_RX_DATA_ACK, toep->tid);
362 struct toepcb *toep = tp->t_toe;
368 KASSERT(toep->sb_cc >= sb->sb_cc,
370 __func__, sb, sb->sb_cc, toep->sb_cc));
371 if (toep->ulp_mode == ULP_MODE_ISCSI) {
372 toep->rx_credits += toep->sb_cc;
373 toep->sb_cc = 0;
375 toep->rx_credits += toep->sb_cc - sb->sb_cc;
376 toep->sb_cc = sb->sb_cc;
378 credits = toep->rx_credits;
384 credits = send_rx_credits(sc, toep, credits);
386 toep->rx_credits -= credits;
397 close_conn(struct adapter *sc, struct toepcb *toep)
401 unsigned int tid = toep->tid;
403 CTR3(KTR_CXGBE, "%s: tid %u%s", __func__, toep->tid,
404 toep->flags & TPF_FIN_SENT ? ", IGNORED" : "");
406 if (toep->flags & TPF_FIN_SENT)
409 KASSERT(toep->flags & TPF_FLOWC_WR_SENT,
412 wr = alloc_wrqe(sizeof(*req), toep->ofld_txq);
427 toep->flags |= TPF_FIN_SENT;
428 toep->flags &= ~TPF_SEND_FIN;
429 t4_l2t_send(sc, wr, toep->l2te);
478 write_tx_wr(void *dst, struct toepcb *toep, unsigned int immdlen,
486 txwr->flowid_len16 = htobe32(V_FW_WR_FLOWID(toep->tid) |
490 if (toep->ulp_mode == ULP_MODE_ISCSI)
494 wr_ulp_mode = V_FW_OFLD_TX_DATA_WR_ULPMODE(toep->ulp_mode);
572 t4_push_frames(struct adapter *sc, struct toepcb *toep, int drop)
578 struct inpcb *inp = toep->inp;
583 struct ofld_tx_sdesc *txsd = &toep->txsd[toep->txsd_pidx];
586 KASSERT(toep->flags & TPF_FLOWC_WR_SENT,
587 ("%s: flowc_wr not sent for tid %u.", __func__, toep->tid));
589 KASSERT(toep->ulp_mode == ULP_MODE_NONE ||
590 toep->ulp_mode == ULP_MODE_TCPDDP ||
591 toep->ulp_mode == ULP_MODE_RDMA,
592 ("%s: ulp_mode %u for toep %p", __func__, toep->ulp_mode, toep));
598 if (__predict_false(toep->flags & TPF_TX_SUSPENDED)) {
605 tx_credits = min(toep->tx_credits, MAX_OFLD_TX_CREDITS);
632 toep->flags |= TPF_TX_SUSPENDED;
658 toep->plen_nocompl + plen >= sb->sb_hiwat / 4)
688 if (__predict_false(toep->flags & TPF_FIN_SENT))
696 toep->ofld_txq);
699 toep->flags |= TPF_TX_SUSPENDED;
704 write_tx_wr(txwr, toep, plen, plen, credits, shove, 0);
714 wr = alloc_wrqe(roundup2(wr_len, 16), toep->ofld_txq);
717 toep->flags |= TPF_TX_SUSPENDED;
722 write_tx_wr(txwr, toep, 0, plen, credits, shove, 0);
732 KASSERT(toep->tx_credits >= credits,
735 toep->tx_credits -= credits;
736 toep->tx_nocompl += credits;
737 toep->plen_nocompl += plen;
738 if (toep->tx_credits <= toep->tx_total * 3 / 8 &&
739 toep->tx_nocompl >= toep->tx_total / 4)
744 toep->tx_nocompl = 0;
745 toep->plen_nocompl = 0;
756 toep->flags |= TPF_TX_DATA_SENT;
757 if (toep->tx_credits < MIN_OFLD_TX_CREDITS)
758 toep->flags |= TPF_TX_SUSPENDED;
760 KASSERT(toep->txsd_avail > 0, ("%s: no txsd", __func__));
764 if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) {
765 toep->txsd_pidx = 0;
766 txsd = &toep->txsd[0];
768 toep->txsd_avail--;
770 t4_l2t_send(sc, wr, toep->l2te);
774 if (m == NULL && toep->flags & TPF_SEND_FIN)
775 close_conn(sc, toep);
780 t4_ulp_push_frames(struct adapter *sc, struct toepcb *toep, int drop)
786 struct inpcb *inp = toep->inp;
795 if (toep->flags & TPF_ABORT_SHUTDOWN)
801 txsd = &toep->txsd[toep->txsd_pidx];
803 KASSERT(toep->flags & TPF_FLOWC_WR_SENT,
804 ("%s: flowc_wr not sent for tid %u.", __func__, toep->tid));
810 if (__predict_false(toep->flags & TPF_TX_SUSPENDED))
813 sndptr = t4_queue_iscsi_callback(so, toep, 1, &qlen);
818 tx_credits = min(toep->tx_credits, MAX_OFLD_TX_CREDITS);
823 t4_cpl_iscsi_callback(toep->td, toep, &drop,
839 toep->flags |= TPF_TX_SUSPENDED;
848 toep->flags |= TPF_TX_SUSPENDED;
861 if (__predict_false(toep->flags & TPF_FIN_SENT))
869 toep->ofld_txq);
872 toep->flags |= TPF_TX_SUSPENDED;
877 write_tx_wr(txwr, toep, plen, ulp_len, credits, shove,
886 wr = alloc_wrqe(roundup(wr_len, 16), toep->ofld_txq);
889 toep->flags |= TPF_TX_SUSPENDED;
894 write_tx_wr(txwr, toep, 0, ulp_len, credits, shove,
905 KASSERT(toep->tx_credits >= credits,
908 toep->tx_credits -= credits;
909 toep->tx_nocompl += credits;
910 toep->plen_nocompl += plen;
911 if (toep->tx_credits <= toep->tx_total * 3 / 8 &&
912 toep->tx_nocompl >= toep->tx_total / 4)
917 toep->tx_nocompl = 0;
918 toep->plen_nocompl = 0;
924 sndptr = m = t4_queue_iscsi_callback(so, toep, 2, &qlen);
926 toep->flags |= TPF_TX_DATA_SENT;
927 if (toep->tx_credits < MIN_OFLD_TX_CREDITS) {
928 toep->flags |= TPF_TX_SUSPENDED;
931 KASSERT(toep->txsd_avail > 0, ("%s: no txsd", __func__));
935 if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) {
936 toep->txsd_pidx = 0;
937 txsd = &toep->txsd[0];
939 toep->txsd_avail--;
941 t4_l2t_send(sc, wr, toep->l2te);
945 if (m == NULL && toep->flags & TPF_SEND_FIN)
946 close_conn(sc, toep);
956 struct toepcb *toep = tp->t_toe;
961 KASSERT(toep != NULL, ("%s: toep is NULL", __func__));
963 t4_push_frames(sc, toep, 0);
975 struct toepcb *toep = tp->t_toe;
980 KASSERT(toep != NULL, ("%s: toep is NULL", __func__));
982 toep->flags |= TPF_SEND_FIN;
984 if (toep->ulp_mode == ULP_MODE_ISCSI)
985 t4_ulp_push_frames(sc, toep, 0);
987 t4_push_frames(sc, toep, 0);
1000 struct toepcb *toep = tp->t_toe;
1005 KASSERT(toep != NULL, ("%s: toep is NULL", __func__));
1008 KASSERT(toep->flags & TPF_FLOWC_WR_SENT,
1010 __func__, toep->tid, tcpstates[tp->t_state]));
1012 send_reset(sc, toep, 0);
1025 struct toepcb *toep = lookup_tid(sc, tid);
1026 struct inpcb *inp = toep->inp;
1038 if (__predict_false(toep->flags & TPF_SYNQE)) {
1040 struct synq_entry *synqe = (void *)toep;
1056 toep, toep->flags);
1060 KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__));
1067 tid, tp ? tcpstates[tp->t_state] : "no tp", toep->flags, inp);
1069 if (toep->flags & TPF_ABORT_SHUTDOWN)
1077 if (__predict_false(toep->ddp_flags & (DDP_BUF0_ACTIVE | DDP_BUF1_ACTIVE))) {
1080 toep->ddp_flags &= ~(DDP_BUF0_ACTIVE | DDP_BUF1_ACTIVE);
1082 KASSERT(toep->sb_cc >= sb->sb_cc,
1084 __func__, sb, sb->sb_cc, toep->sb_cc));
1085 toep->rx_credits += toep->sb_cc - sb->sb_cc;
1087 toep->rx_credits -= m->m_len; /* adjust for F_RX_FC_DDP */
1090 toep->sb_cc = sb->sb_cc;
1094 if (toep->ulp_mode != ULP_MODE_RDMA) {
1119 final_cpl_received(toep);
1142 struct toepcb *toep = lookup_tid(sc, tid);
1143 struct inpcb *inp = toep->inp;
1153 KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__));
1160 __func__, tid, tp ? tcpstates[tp->t_state] : "no tp", toep->flags);
1162 if (toep->flags & TPF_ABORT_SHUTDOWN)
1176 final_cpl_received(toep); /* no more CPLs expected */
1287 t4_cpl_iscsi_callback(struct tom_data *td, struct toepcb *toep, void *m,
1293 so = toep->inp->inp_socket;
1295 INP_WLOCK(toep->inp);
1296 so = toep->inp->inp_socket;
1297 INP_WUNLOCK(toep->inp);
1301 if (toep->ulp_mode == ULP_MODE_ISCSI) {
1311 t4_queue_iscsi_callback(struct socket *so, struct toepcb *toep,
1316 if (toep->ulp_mode == ULP_MODE_ISCSI)
1332 struct toepcb *toep = lookup_tid(sc, tid);
1333 struct sge_wrq *ofld_txq = toep->ofld_txq;
1344 if (toep->flags & TPF_SYNQE)
1347 KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__));
1351 __func__, cpl->status, tid, toep->flags);
1355 inp = toep->inp;
1363 __func__, tid, tp ? tcpstates[tp->t_state] : "no tp", toep->flags,
1371 if (toep->flags & TPF_ABORT_SHUTDOWN) {
1375 toep->flags |= TPF_ABORT_SHUTDOWN;
1388 final_cpl_received(toep);
1404 struct toepcb *toep = lookup_tid(sc, tid);
1405 struct inpcb *inp = toep->inp;
1414 if (toep->flags & TPF_SYNQE)
1417 KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__));
1419 CTR5(KTR_CXGBE, "%s: tid %u, toep %p, inp %p, status %d",
1420 __func__, tid, toep, inp, cpl->status);
1422 KASSERT(toep->flags & TPF_ABORT_SHUTDOWN,
1426 final_cpl_received(toep);
1437 struct toepcb *toep = lookup_tid(sc, tid);
1438 struct inpcb *inp = toep->inp;
1445 if (__predict_false(toep->flags & TPF_SYNQE)) {
1447 struct synq_entry *synqe = (void *)toep;
1463 toep, toep->flags);
1468 KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__));
1526 toep->rx_credits += newsize - hiwat;
1529 if (toep->ulp_mode == ULP_MODE_TCPDDP) {
1530 int changed = !(toep->ddp_flags & DDP_ON) ^ cpl->ddp_off;
1533 if (toep->ddp_flags & DDP_SC_REQ)
1534 toep->ddp_flags ^= DDP_ON | DDP_SC_REQ;
1541 toep->ddp_flags &= ~(DDP_ON | DDP_BUF0_ACTIVE |
1545 insert_ddp_data(toep, ddp_placed);
1549 if ((toep->ddp_flags & DDP_OK) == 0 &&
1550 time_uptime >= toep->ddp_disabled + DDP_RETRY_WAIT) {
1551 toep->ddp_score = DDP_LOW_SCORE;
1552 toep->ddp_flags |= DDP_OK;
1557 if (toep->ddp_flags & DDP_ON) {
1569 __func__, tid, toep->flags, be32toh(cpl->seq), len);
1572 } else if ((toep->ddp_flags & (DDP_OK|DDP_SC_REQ)) == DDP_OK &&
1581 enable_ddp(sc, toep);
1585 KASSERT(toep->sb_cc >= sb->sb_cc,
1587 __func__, sb, sb->sb_cc, toep->sb_cc));
1588 toep->rx_credits += toep->sb_cc - sb->sb_cc;
1590 toep->sb_cc = sb->sb_cc;
1628 struct toepcb *toep = lookup_tid(sc, tid);
1643 if (__predict_false(toep->flags & TPF_SYNQE)) {
1644 KASSERT(toep->flags & TPF_ABORT_SHUTDOWN,
1645 ("%s: credits for a synq entry %p", __func__, toep));
1649 inp = toep->inp;
1654 KASSERT(toep->tid == tid, ("%s: toep tid mismatch", __func__));
1658 if (__predict_false(toep->flags & TPF_ABORT_SHUTDOWN)) {
1675 __func__, snd_una, toep->tid, tp->snd_una);
1686 txsd = &toep->txsd[toep->txsd_cidx];
1692 toep->tx_credits += txsd->tx_credits;
1695 toep->txsd_avail++;
1696 KASSERT(toep->txsd_avail <= toep->txsd_total,
1698 if (__predict_false(++toep->txsd_cidx == toep->txsd_total)) {
1699 txsd = &toep->txsd[0];
1700 toep->txsd_cidx = 0;
1704 if (toep->tx_credits == toep->tx_total) {
1705 toep->tx_nocompl = 0;
1706 toep->plen_nocompl = 0;
1709 if (toep->flags & TPF_TX_SUSPENDED &&
1710 toep->tx_credits >= toep->tx_total / 4) {
1711 toep->flags &= ~TPF_TX_SUSPENDED;
1712 if (toep->ulp_mode == ULP_MODE_ISCSI)
1713 t4_ulp_push_frames(sc, toep, plen);
1715 t4_push_frames(sc, toep, plen);
1719 if (toep->ulp_mode == ULP_MODE_ISCSI)
1720 t4_cpl_iscsi_callback(toep->td, toep, &plen,
1752 struct toepcb *toep = lookup_tid(sc, tid);
1754 t4_cpl_iscsi_callback(toep->td, toep, m, CPL_SET_TCB_RPL);
1762 t4_set_tcb_field(struct adapter *sc, struct toepcb *toep, int ctrl,
1768 wr = alloc_wrqe(sizeof(*req), ctrl ? toep->ctrlq : toep->ofld_txq);
1775 INIT_TP_WR_MIT_CPL(req, CPL_SET_TCB_FIELD, toep->tid);
1777 V_QUEUENO(toep->ofld_rxq->iq.abs_id));