Deleted Added
sdiff udiff text old ( 265426 ) new ( 270297 )
full compact
t4_cpl_io.c (265426) t4_cpl_io.c (270297)
1/*-
2 * Copyright (c) 2012 Chelsio Communications, Inc.
3 * All rights reserved.
4 * Written by: Navdeep Parhar <np@FreeBSD.org>
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:

--- 12 unchanged lines hidden (view full) ---

21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 */
27
28#include <sys/cdefs.h>
1/*-
2 * Copyright (c) 2012 Chelsio Communications, Inc.
3 * All rights reserved.
4 * Written by: Navdeep Parhar <np@FreeBSD.org>
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:

--- 12 unchanged lines hidden (view full) ---

21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 */
27
28#include <sys/cdefs.h>
29__FBSDID("$FreeBSD: stable/10/sys/dev/cxgbe/tom/t4_cpl_io.c 265426 2014-05-06 07:21:50Z np $");
29__FBSDID("$FreeBSD: stable/10/sys/dev/cxgbe/tom/t4_cpl_io.c 270297 2014-08-21 19:54:02Z np $");
30
31#include "opt_inet.h"
32
33#ifdef TCP_OFFLOAD
34#include <sys/param.h>
35#include <sys/types.h>
36#include <sys/kernel.h>
37#include <sys/ktr.h>

--- 27 unchanged lines hidden (view full) ---

65#define V_tcp_autosndbuf_max VNET(tcp_autosndbuf_max)
66VNET_DECLARE(int, tcp_do_autorcvbuf);
67#define V_tcp_do_autorcvbuf VNET(tcp_do_autorcvbuf)
68VNET_DECLARE(int, tcp_autorcvbuf_inc);
69#define V_tcp_autorcvbuf_inc VNET(tcp_autorcvbuf_inc)
70VNET_DECLARE(int, tcp_autorcvbuf_max);
71#define V_tcp_autorcvbuf_max VNET(tcp_autorcvbuf_max)
72
30
31#include "opt_inet.h"
32
33#ifdef TCP_OFFLOAD
34#include <sys/param.h>
35#include <sys/types.h>
36#include <sys/kernel.h>
37#include <sys/ktr.h>

--- 27 unchanged lines hidden (view full) ---

65#define V_tcp_autosndbuf_max VNET(tcp_autosndbuf_max)
66VNET_DECLARE(int, tcp_do_autorcvbuf);
67#define V_tcp_do_autorcvbuf VNET(tcp_do_autorcvbuf)
68VNET_DECLARE(int, tcp_autorcvbuf_inc);
69#define V_tcp_autorcvbuf_inc VNET(tcp_autorcvbuf_inc)
70VNET_DECLARE(int, tcp_autorcvbuf_max);
71#define V_tcp_autorcvbuf_max VNET(tcp_autorcvbuf_max)
72
73/*
74 * For ULP connections HW may add headers, e.g., for digests, that aren't part
75 * of the messages sent by the host but that are part of the TCP payload and
76 * therefore consume TCP sequence space. Tx connection parameters that
77 * operate in TCP sequence space are affected by the HW additions and need to
78 * compensate for them to accurately track TCP sequence numbers. This array
79 * contains the compensating extra lengths for ULP packets. It is indexed by
80 * a packet's ULP submode.
81 */
82const unsigned int t4_ulp_extra_len[] = {0, 4, 4, 8};
83
84/*
85 * Return the length of any HW additions that will be made to a Tx packet.
86 * Such additions can happen for some types of ULP packets.
87 */
88static inline unsigned int
89ulp_extra_len(struct mbuf *m, int *ulp_mode)
90{
91 struct m_tag *mtag;
92
93 if ((mtag = m_tag_find(m, CXGBE_ISCSI_MBUF_TAG, NULL)) == NULL)
94 return (0);
95 *ulp_mode = *((int *)(mtag + 1));
96
97 return (t4_ulp_extra_len[*ulp_mode & 3]);
98}
99
73void
74send_flowc_wr(struct toepcb *toep, struct flowc_tx_params *ftxp)
75{
76 struct wrqe *wr;
77 struct fw_flowc_wr *flowc;
78 unsigned int nparams = ftxp ? 8 : 6, flowclen;
79 struct port_info *pi = toep->port;
80 struct adapter *sc = pi->adapter;

--- 255 unchanged lines hidden (view full) ---

336 int credits;
337
338 INP_WLOCK_ASSERT(inp);
339
340 SOCKBUF_LOCK(sb);
341 KASSERT(toep->sb_cc >= sb->sb_cc,
342 ("%s: sb %p has more data (%d) than last time (%d).",
343 __func__, sb, sb->sb_cc, toep->sb_cc));
100void
101send_flowc_wr(struct toepcb *toep, struct flowc_tx_params *ftxp)
102{
103 struct wrqe *wr;
104 struct fw_flowc_wr *flowc;
105 unsigned int nparams = ftxp ? 8 : 6, flowclen;
106 struct port_info *pi = toep->port;
107 struct adapter *sc = pi->adapter;

--- 255 unchanged lines hidden (view full) ---

363 int credits;
364
365 INP_WLOCK_ASSERT(inp);
366
367 SOCKBUF_LOCK(sb);
368 KASSERT(toep->sb_cc >= sb->sb_cc,
369 ("%s: sb %p has more data (%d) than last time (%d).",
370 __func__, sb, sb->sb_cc, toep->sb_cc));
344 toep->rx_credits += toep->sb_cc - sb->sb_cc;
345 toep->sb_cc = sb->sb_cc;
371 if (toep->ulp_mode == ULP_MODE_ISCSI) {
372 toep->rx_credits += toep->sb_cc;
373 toep->sb_cc = 0;
374 } else {
375 toep->rx_credits += toep->sb_cc - sb->sb_cc;
376 toep->sb_cc = sb->sb_cc;
377 }
346 credits = toep->rx_credits;
347 SOCKBUF_UNLOCK(sb);
348
349 if (credits > 0 &&
350 (credits + 16384 >= tp->rcv_wnd || credits >= 15 * 1024)) {
351
352 credits = send_rx_credits(sc, toep, credits);
353 SOCKBUF_LOCK(sb);

--- 85 unchanged lines hidden (view full) ---

439 if ((sge_pair_credits * 16) % 24 == 16)
440 nseg++;
441
442 return (nseg);
443}
444
445static inline void
446write_tx_wr(void *dst, struct toepcb *toep, unsigned int immdlen,
378 credits = toep->rx_credits;
379 SOCKBUF_UNLOCK(sb);
380
381 if (credits > 0 &&
382 (credits + 16384 >= tp->rcv_wnd || credits >= 15 * 1024)) {
383
384 credits = send_rx_credits(sc, toep, credits);
385 SOCKBUF_LOCK(sb);

--- 85 unchanged lines hidden (view full) ---

471 if ((sge_pair_credits * 16) % 24 == 16)
472 nseg++;
473
474 return (nseg);
475}
476
477static inline void
478write_tx_wr(void *dst, struct toepcb *toep, unsigned int immdlen,
447 unsigned int plen, uint8_t credits, int shove)
479 unsigned int plen, uint8_t credits, int shove, int ulp_mode)
448{
449 struct fw_ofld_tx_data_wr *txwr = dst;
480{
481 struct fw_ofld_tx_data_wr *txwr = dst;
482 unsigned int wr_ulp_mode;
450
451 txwr->op_to_immdlen = htobe32(V_WR_OP(FW_OFLD_TX_DATA_WR) |
452 V_FW_WR_IMMDLEN(immdlen));
453 txwr->flowid_len16 = htobe32(V_FW_WR_FLOWID(toep->tid) |
454 V_FW_WR_LEN16(credits));
483
484 txwr->op_to_immdlen = htobe32(V_WR_OP(FW_OFLD_TX_DATA_WR) |
485 V_FW_WR_IMMDLEN(immdlen));
486 txwr->flowid_len16 = htobe32(V_FW_WR_FLOWID(toep->tid) |
487 V_FW_WR_LEN16(credits));
488
489 /* for iscsi, the mode & submode setting is per-packet */
490 if (toep->ulp_mode == ULP_MODE_ISCSI)
491 wr_ulp_mode = V_FW_OFLD_TX_DATA_WR_ULPMODE(ulp_mode >> 4) |
492 V_FW_OFLD_TX_DATA_WR_ULPSUBMODE(ulp_mode & 3);
493 else
494 wr_ulp_mode = V_FW_OFLD_TX_DATA_WR_ULPMODE(toep->ulp_mode);
495
455 txwr->lsodisable_to_proxy =
496 txwr->lsodisable_to_proxy =
456 htobe32(V_FW_OFLD_TX_DATA_WR_ULPMODE(toep->ulp_mode) |
497 htobe32(wr_ulp_mode |
457 V_FW_OFLD_TX_DATA_WR_URGENT(0) | /* XXX */
458 V_FW_OFLD_TX_DATA_WR_SHOVE(shove));
459 txwr->plen = htobe32(plen);
460}
461
462/*
463 * Generate a DSGL from a starting mbuf. The total number of segments and the
464 * maximum segments in any one mbuf are provided.

--- 57 unchanged lines hidden (view full) ---

522 *
523 * drop indicates the number of bytes that should be dropped from the head of
524 * the send buffer. It is an optimization that lets do_fw4_ack avoid creating
525 * contention on the send buffer lock (before this change it used to do
526 * sowwakeup and then t4_push_frames right after that when recovering from tx
527 * stalls). When drop is set this function MUST drop the bytes and wake up any
528 * writers.
529 */
498 V_FW_OFLD_TX_DATA_WR_URGENT(0) | /* XXX */
499 V_FW_OFLD_TX_DATA_WR_SHOVE(shove));
500 txwr->plen = htobe32(plen);
501}
502
503/*
504 * Generate a DSGL from a starting mbuf. The total number of segments and the
505 * maximum segments in any one mbuf are provided.

--- 57 unchanged lines hidden (view full) ---

563 *
564 * drop indicates the number of bytes that should be dropped from the head of
565 * the send buffer. It is an optimization that lets do_fw4_ack avoid creating
566 * contention on the send buffer lock (before this change it used to do
567 * sowwakeup and then t4_push_frames right after that when recovering from tx
568 * stalls). When drop is set this function MUST drop the bytes and wake up any
569 * writers.
570 */
530static void
571void
531t4_push_frames(struct adapter *sc, struct toepcb *toep, int drop)
532{
533 struct mbuf *sndptr, *m, *sb_sndptr;
534 struct fw_ofld_tx_data_wr *txwr;
535 struct wrqe *wr;
536 u_int plen, nsegs, credits, max_imm, max_nsegs, max_nsegs_1mbuf;
537 struct inpcb *inp = toep->inp;
538 struct tcpcb *tp = intotcpcb(inp);

--- 116 unchanged lines hidden (view full) ---

655 toep->ofld_txq);
656 if (wr == NULL) {
657 /* XXX: how will we recover from this? */
658 toep->flags |= TPF_TX_SUSPENDED;
659 return;
660 }
661 txwr = wrtod(wr);
662 credits = howmany(wr->wr_len, 16);
572t4_push_frames(struct adapter *sc, struct toepcb *toep, int drop)
573{
574 struct mbuf *sndptr, *m, *sb_sndptr;
575 struct fw_ofld_tx_data_wr *txwr;
576 struct wrqe *wr;
577 u_int plen, nsegs, credits, max_imm, max_nsegs, max_nsegs_1mbuf;
578 struct inpcb *inp = toep->inp;
579 struct tcpcb *tp = intotcpcb(inp);

--- 116 unchanged lines hidden (view full) ---

696 toep->ofld_txq);
697 if (wr == NULL) {
698 /* XXX: how will we recover from this? */
699 toep->flags |= TPF_TX_SUSPENDED;
700 return;
701 }
702 txwr = wrtod(wr);
703 credits = howmany(wr->wr_len, 16);
663 write_tx_wr(txwr, toep, plen, plen, credits, shove);
704 write_tx_wr(txwr, toep, plen, plen, credits, shove, 0);
664 m_copydata(sndptr, 0, plen, (void *)(txwr + 1));
665 nsegs = 0;
666 } else {
667 int wr_len;
668
669 /* DSGL tx */
670
671 wr_len = sizeof(*txwr) + sizeof(struct ulptx_sgl) +
672 ((3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1)) * 8;
673 wr = alloc_wrqe(roundup2(wr_len, 16), toep->ofld_txq);
674 if (wr == NULL) {
675 /* XXX: how will we recover from this? */
676 toep->flags |= TPF_TX_SUSPENDED;
677 return;
678 }
679 txwr = wrtod(wr);
680 credits = howmany(wr_len, 16);
705 m_copydata(sndptr, 0, plen, (void *)(txwr + 1));
706 nsegs = 0;
707 } else {
708 int wr_len;
709
710 /* DSGL tx */
711
712 wr_len = sizeof(*txwr) + sizeof(struct ulptx_sgl) +
713 ((3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1)) * 8;
714 wr = alloc_wrqe(roundup2(wr_len, 16), toep->ofld_txq);
715 if (wr == NULL) {
716 /* XXX: how will we recover from this? */
717 toep->flags |= TPF_TX_SUSPENDED;
718 return;
719 }
720 txwr = wrtod(wr);
721 credits = howmany(wr_len, 16);
681 write_tx_wr(txwr, toep, 0, plen, credits, shove);
722 write_tx_wr(txwr, toep, 0, plen, credits, shove, 0);
682 write_tx_sgl(txwr + 1, sndptr, m, nsegs,
683 max_nsegs_1mbuf);
684 if (wr_len & 0xf) {
685 uint64_t *pad = (uint64_t *)
686 ((uintptr_t)txwr + wr_len);
687 *pad = 0;
688 }
689 }

--- 39 unchanged lines hidden (view full) ---

729 t4_l2t_send(sc, wr, toep->l2te);
730 } while (m != NULL);
731
732 /* Send a FIN if requested, but only if there's no more data to send */
733 if (m == NULL && toep->flags & TPF_SEND_FIN)
734 close_conn(sc, toep);
735}
736
723 write_tx_sgl(txwr + 1, sndptr, m, nsegs,
724 max_nsegs_1mbuf);
725 if (wr_len & 0xf) {
726 uint64_t *pad = (uint64_t *)
727 ((uintptr_t)txwr + wr_len);
728 *pad = 0;
729 }
730 }

--- 39 unchanged lines hidden (view full) ---

770 t4_l2t_send(sc, wr, toep->l2te);
771 } while (m != NULL);
772
773 /* Send a FIN if requested, but only if there's no more data to send */
774 if (m == NULL && toep->flags & TPF_SEND_FIN)
775 close_conn(sc, toep);
776}
777
778/* Send ULP data over TOE using TX_DATA_WR. We send whole mbuf at once */
779void
780t4_ulp_push_frames(struct adapter *sc, struct toepcb *toep, int drop)
781{
782 struct mbuf *sndptr, *m = NULL;
783 struct fw_ofld_tx_data_wr *txwr;
784 struct wrqe *wr;
785 unsigned int plen, nsegs, credits, max_imm, max_nsegs, max_nsegs_1mbuf;
786 struct inpcb *inp = toep->inp;
787 struct tcpcb *tp;
788 struct socket *so;
789 struct sockbuf *sb;
790 int tx_credits, ulp_len = 0, ulp_mode = 0, qlen = 0;
791 int shove, compl;
792 struct ofld_tx_sdesc *txsd;
793
794 INP_WLOCK_ASSERT(inp);
795 if (toep->flags & TPF_ABORT_SHUTDOWN)
796 return;
797
798 tp = intotcpcb(inp);
799 so = inp->inp_socket;
800 sb = &so->so_snd;
801 txsd = &toep->txsd[toep->txsd_pidx];
802
803 KASSERT(toep->flags & TPF_FLOWC_WR_SENT,
804 ("%s: flowc_wr not sent for tid %u.", __func__, toep->tid));
805
806 /*
807 * This function doesn't resume by itself. Someone else must clear the
808 * flag and call this function.
809 */
810 if (__predict_false(toep->flags & TPF_TX_SUSPENDED))
811 return;
812
813 sndptr = t4_queue_iscsi_callback(so, toep, 1, &qlen);
814 if (!qlen)
815 return;
816
817 do {
818 tx_credits = min(toep->tx_credits, MAX_OFLD_TX_CREDITS);
819 max_imm = max_imm_payload(tx_credits);
820 max_nsegs = max_dsgl_nsegs(tx_credits);
821
822 if (drop) {
823 t4_cpl_iscsi_callback(toep->td, toep, &drop,
824 CPL_FW4_ACK);
825 drop = 0;
826 }
827
828 plen = 0;
829 nsegs = 0;
830 max_nsegs_1mbuf = 0; /* max # of SGL segments in any one mbuf */
831 for (m = sndptr; m != NULL; m = m->m_next) {
832 int n = sglist_count(mtod(m, void *), m->m_len);
833
834 nsegs += n;
835 plen += m->m_len;
836
837 /* This mbuf sent us _over_ the nsegs limit, return */
838 if (plen > max_imm && nsegs > max_nsegs) {
839 toep->flags |= TPF_TX_SUSPENDED;
840 return;
841 }
842
843 if (max_nsegs_1mbuf < n)
844 max_nsegs_1mbuf = n;
845
846 /* This mbuf put us right at the max_nsegs limit */
847 if (plen > max_imm && nsegs == max_nsegs) {
848 toep->flags |= TPF_TX_SUSPENDED;
849 return;
850 }
851 }
852
853 shove = m == NULL && !(tp->t_flags & TF_MORETOCOME);
854 /* nothing to send */
855 if (plen == 0) {
856 KASSERT(m == NULL,
857 ("%s: nothing to send, but m != NULL", __func__));
858 break;
859 }
860
861 if (__predict_false(toep->flags & TPF_FIN_SENT))
862 panic("%s: excess tx.", __func__);
863
864 ulp_len = plen + ulp_extra_len(sndptr, &ulp_mode);
865 if (plen <= max_imm) {
866
867 /* Immediate data tx */
868 wr = alloc_wrqe(roundup(sizeof(*txwr) + plen, 16),
869 toep->ofld_txq);
870 if (wr == NULL) {
871 /* XXX: how will we recover from this? */
872 toep->flags |= TPF_TX_SUSPENDED;
873 return;
874 }
875 txwr = wrtod(wr);
876 credits = howmany(wr->wr_len, 16);
877 write_tx_wr(txwr, toep, plen, ulp_len, credits, shove,
878 ulp_mode);
879 m_copydata(sndptr, 0, plen, (void *)(txwr + 1));
880 } else {
881 int wr_len;
882
883 /* DSGL tx */
884 wr_len = sizeof(*txwr) + sizeof(struct ulptx_sgl) +
885 ((3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1)) * 8;
886 wr = alloc_wrqe(roundup(wr_len, 16), toep->ofld_txq);
887 if (wr == NULL) {
888 /* XXX: how will we recover from this? */
889 toep->flags |= TPF_TX_SUSPENDED;
890 return;
891 }
892 txwr = wrtod(wr);
893 credits = howmany(wr_len, 16);
894 write_tx_wr(txwr, toep, 0, ulp_len, credits, shove,
895 ulp_mode);
896 write_tx_sgl(txwr + 1, sndptr, m, nsegs,
897 max_nsegs_1mbuf);
898 if (wr_len & 0xf) {
899 uint64_t *pad = (uint64_t *)
900 ((uintptr_t)txwr + wr_len);
901 *pad = 0;
902 }
903 }
904
905 KASSERT(toep->tx_credits >= credits,
906 ("%s: not enough credits", __func__));
907
908 toep->tx_credits -= credits;
909 toep->tx_nocompl += credits;
910 toep->plen_nocompl += plen;
911 if (toep->tx_credits <= toep->tx_total * 3 / 8 &&
912 toep->tx_nocompl >= toep->tx_total / 4)
913 compl = 1;
914
915 if (compl) {
916 txwr->op_to_immdlen |= htobe32(F_FW_WR_COMPL);
917 toep->tx_nocompl = 0;
918 toep->plen_nocompl = 0;
919 }
920 tp->snd_nxt += ulp_len;
921 tp->snd_max += ulp_len;
922
923 /* goto next mbuf */
924 sndptr = m = t4_queue_iscsi_callback(so, toep, 2, &qlen);
925
926 toep->flags |= TPF_TX_DATA_SENT;
927 if (toep->tx_credits < MIN_OFLD_TX_CREDITS) {
928 toep->flags |= TPF_TX_SUSPENDED;
929 }
930
931 KASSERT(toep->txsd_avail > 0, ("%s: no txsd", __func__));
932 txsd->plen = plen;
933 txsd->tx_credits = credits;
934 txsd++;
935 if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) {
936 toep->txsd_pidx = 0;
937 txsd = &toep->txsd[0];
938 }
939 toep->txsd_avail--;
940
941 t4_l2t_send(sc, wr, toep->l2te);
942 } while (m != NULL);
943
944 /* Send a FIN if requested, but only if there's no more data to send */
945 if (m == NULL && toep->flags & TPF_SEND_FIN)
946 close_conn(sc, toep);
947}
948
737int
738t4_tod_output(struct toedev *tod, struct tcpcb *tp)
739{
740 struct adapter *sc = tod->tod_softc;
741#ifdef INVARIANTS
742 struct inpcb *inp = tp->t_inpcb;
743#endif
744 struct toepcb *toep = tp->t_toe;

--- 18 unchanged lines hidden (view full) ---

763 struct toepcb *toep = tp->t_toe;
764
765 INP_WLOCK_ASSERT(inp);
766 KASSERT((inp->inp_flags & INP_DROPPED) == 0,
767 ("%s: inp %p dropped.", __func__, inp));
768 KASSERT(toep != NULL, ("%s: toep is NULL", __func__));
769
770 toep->flags |= TPF_SEND_FIN;
949int
950t4_tod_output(struct toedev *tod, struct tcpcb *tp)
951{
952 struct adapter *sc = tod->tod_softc;
953#ifdef INVARIANTS
954 struct inpcb *inp = tp->t_inpcb;
955#endif
956 struct toepcb *toep = tp->t_toe;

--- 18 unchanged lines hidden (view full) ---

975 struct toepcb *toep = tp->t_toe;
976
977 INP_WLOCK_ASSERT(inp);
978 KASSERT((inp->inp_flags & INP_DROPPED) == 0,
979 ("%s: inp %p dropped.", __func__, inp));
980 KASSERT(toep != NULL, ("%s: toep is NULL", __func__));
981
982 toep->flags |= TPF_SEND_FIN;
771 if (tp->t_state >= TCPS_ESTABLISHED)
772 t4_push_frames(sc, toep, 0);
983 if (tp->t_state >= TCPS_ESTABLISHED) {
984 if (toep->ulp_mode == ULP_MODE_ISCSI)
985 t4_ulp_push_frames(sc, toep, 0);
986 else
987 t4_push_frames(sc, toep, 0);
988 }
773
774 return (0);
775}
776
777int
778t4_send_rst(struct toedev *tod, struct tcpcb *tp)
779{
780 struct adapter *sc = tod->tod_softc;

--- 233 unchanged lines hidden (view full) ---

1014 case CPL_ERR_FINWAIT2_TIMEDOUT:
1015 case CPL_ERR_KEEPALIVE_TIMEDOUT:
1016 return (ETIMEDOUT);
1017 default:
1018 return (EIO);
1019 }
1020}
1021
989
990 return (0);
991}
992
993int
994t4_send_rst(struct toedev *tod, struct tcpcb *tp)
995{
996 struct adapter *sc = tod->tod_softc;

--- 233 unchanged lines hidden (view full) ---

1230 case CPL_ERR_FINWAIT2_TIMEDOUT:
1231 case CPL_ERR_KEEPALIVE_TIMEDOUT:
1232 return (ETIMEDOUT);
1233 default:
1234 return (EIO);
1235 }
1236}
1237
1238int
1239cpl_not_handled(struct sge_iq *, const struct rss_header *, struct mbuf *);
1022/*
1240/*
1241 * tom_cpl_iscsi_callback -
1242 * iscsi and tom would share the following cpl messages, so when any of these
1243 * message is received, after tom is done with processing it, the messages
1244 * needs to be forwarded to iscsi for further processing:
1245 * - CPL_SET_TCB_RPL
1246 * - CPL_RX_DATA_DDP
1247 */
1248void (*tom_cpl_iscsi_callback)(struct tom_data *, struct socket *, void *,
1249 unsigned int);
1250
1251struct mbuf *(*tom_queue_iscsi_callback)(struct socket *, unsigned int, int *);
1252/*
1253 * Check if the handler function is set for a given CPL
1254 * return 0 if the function is NULL or cpl_not_handled, 1 otherwise.
1255 */
1256int
1257t4tom_cpl_handler_registered(struct adapter *sc, unsigned int opcode)
1258{
1259
1260 MPASS(opcode < nitems(sc->cpl_handler));
1261
1262 return (sc->cpl_handler[opcode] &&
1263 sc->cpl_handler[opcode] != cpl_not_handled);
1264}
1265
1266/*
1267 * set the tom_cpl_iscsi_callback function, this function should be used
1268 * whenever both toe and iscsi need to process the same cpl msg.
1269 */
1270void
1271t4tom_register_cpl_iscsi_callback(void (*fp)(struct tom_data *, struct socket *,
1272 void *, unsigned int))
1273{
1274
1275 tom_cpl_iscsi_callback = fp;
1276}
1277
1278void
1279t4tom_register_queue_iscsi_callback(struct mbuf *(*fp)(struct socket *,
1280 unsigned int, int *qlen))
1281{
1282
1283 tom_queue_iscsi_callback = fp;
1284}
1285
1286int
1287t4_cpl_iscsi_callback(struct tom_data *td, struct toepcb *toep, void *m,
1288 unsigned int opcode)
1289{
1290 struct socket *so;
1291
1292 if (opcode == CPL_FW4_ACK)
1293 so = toep->inp->inp_socket;
1294 else {
1295 INP_WLOCK(toep->inp);
1296 so = toep->inp->inp_socket;
1297 INP_WUNLOCK(toep->inp);
1298 }
1299
1300 if (tom_cpl_iscsi_callback && so) {
1301 if (toep->ulp_mode == ULP_MODE_ISCSI) {
1302 tom_cpl_iscsi_callback(td, so, m, opcode);
1303 return (0);
1304 }
1305 }
1306
1307 return (1);
1308}
1309
1310struct mbuf *
1311t4_queue_iscsi_callback(struct socket *so, struct toepcb *toep,
1312 unsigned int cmd, int *qlen)
1313{
1314
1315 if (tom_queue_iscsi_callback && so) {
1316 if (toep->ulp_mode == ULP_MODE_ISCSI)
1317 return (tom_queue_iscsi_callback(so, cmd, qlen));
1318 }
1319
1320 return (NULL);
1321}
1322
1323/*
1023 * TCP RST from the peer, timeout, or some other such critical error.
1024 */
1025static int
1026do_abort_req(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
1027{
1028 struct adapter *sc = iq->adapter;
1029 const struct cpl_abort_req_rss *cpl = (const void *)(rss + 1);
1030 unsigned int tid = GET_TID(cpl);

--- 372 unchanged lines hidden (view full) ---

1403 if (toep->tx_credits == toep->tx_total) {
1404 toep->tx_nocompl = 0;
1405 toep->plen_nocompl = 0;
1406 }
1407
1408 if (toep->flags & TPF_TX_SUSPENDED &&
1409 toep->tx_credits >= toep->tx_total / 4) {
1410 toep->flags &= ~TPF_TX_SUSPENDED;
1324 * TCP RST from the peer, timeout, or some other such critical error.
1325 */
1326static int
1327do_abort_req(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
1328{
1329 struct adapter *sc = iq->adapter;
1330 const struct cpl_abort_req_rss *cpl = (const void *)(rss + 1);
1331 unsigned int tid = GET_TID(cpl);

--- 372 unchanged lines hidden (view full) ---

1704 if (toep->tx_credits == toep->tx_total) {
1705 toep->tx_nocompl = 0;
1706 toep->plen_nocompl = 0;
1707 }
1708
1709 if (toep->flags & TPF_TX_SUSPENDED &&
1710 toep->tx_credits >= toep->tx_total / 4) {
1711 toep->flags &= ~TPF_TX_SUSPENDED;
1411 t4_push_frames(sc, toep, plen);
1712 if (toep->ulp_mode == ULP_MODE_ISCSI)
1713 t4_ulp_push_frames(sc, toep, plen);
1714 else
1715 t4_push_frames(sc, toep, plen);
1412 } else if (plen > 0) {
1413 struct sockbuf *sb = &so->so_snd;
1414
1716 } else if (plen > 0) {
1717 struct sockbuf *sb = &so->so_snd;
1718
1415 SOCKBUF_LOCK(sb);
1416 sbdrop_locked(sb, plen);
1417 sowwakeup_locked(so);
1418 SOCKBUF_UNLOCK_ASSERT(sb);
1719 if (toep->ulp_mode == ULP_MODE_ISCSI)
1720 t4_cpl_iscsi_callback(toep->td, toep, &plen,
1721 CPL_FW4_ACK);
1722 else {
1723 SOCKBUF_LOCK(sb);
1724 sbdrop_locked(sb, plen);
1725 sowwakeup_locked(so);
1726 SOCKBUF_UNLOCK_ASSERT(sb);
1727 }
1419 }
1420
1421 INP_WUNLOCK(inp);
1422
1423 return (0);
1424}
1425
1426static int

--- 7 unchanged lines hidden (view full) ---

1434#endif
1435
1436 KASSERT(opcode == CPL_SET_TCB_RPL,
1437 ("%s: unexpected opcode 0x%x", __func__, opcode));
1438 KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__));
1439
1440 if (is_ftid(sc, tid))
1441 return (t4_filter_rpl(iq, rss, m)); /* TCB is a filter */
1728 }
1729
1730 INP_WUNLOCK(inp);
1731
1732 return (0);
1733}
1734
1735static int

--- 7 unchanged lines hidden (view full) ---

1743#endif
1744
1745 KASSERT(opcode == CPL_SET_TCB_RPL,
1746 ("%s: unexpected opcode 0x%x", __func__, opcode));
1747 KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__));
1748
1749 if (is_ftid(sc, tid))
1750 return (t4_filter_rpl(iq, rss, m)); /* TCB is a filter */
1751 else {
1752 struct toepcb *toep = lookup_tid(sc, tid);
1442
1753
1754 t4_cpl_iscsi_callback(toep->td, toep, m, CPL_SET_TCB_RPL);
1755 return (0);
1756 }
1757
1443 CXGBE_UNIMPLEMENTED(__func__);
1444}
1445
1446void
1447t4_set_tcb_field(struct adapter *sc, struct toepcb *toep, int ctrl,
1448 uint16_t word, uint64_t mask, uint64_t val)
1449{
1450 struct wrqe *wr;

--- 39 unchanged lines hidden --- 		1758 CXGBE_UNIMPLEMENTED(__func__);
1759}
1760
1761void
1762t4_set_tcb_field(struct adapter *sc, struct toepcb *toep, int ctrl,
1763 uint16_t word, uint64_t mask, uint64_t val)
1764{
1765 struct wrqe *wr;

--- 39 unchanged lines hidden ---