t4_cpl_io.c (265426) | t4_cpl_io.c (270297) |
---|---|
1/*- 2 * Copyright (c) 2012 Chelsio Communications, Inc. 3 * All rights reserved. 4 * Written by: Navdeep Parhar <np@FreeBSD.org> 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: --- 12 unchanged lines hidden (view full) --- 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28#include <sys/cdefs.h> | 1/*- 2 * Copyright (c) 2012 Chelsio Communications, Inc. 3 * All rights reserved. 4 * Written by: Navdeep Parhar <np@FreeBSD.org> 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: --- 12 unchanged lines hidden (view full) --- 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28#include <sys/cdefs.h> |
29__FBSDID("$FreeBSD: stable/10/sys/dev/cxgbe/tom/t4_cpl_io.c 265426 2014-05-06 07:21:50Z np $"); | 29__FBSDID("$FreeBSD: stable/10/sys/dev/cxgbe/tom/t4_cpl_io.c 270297 2014-08-21 19:54:02Z np $"); |
30 31#include "opt_inet.h" 32 33#ifdef TCP_OFFLOAD 34#include <sys/param.h> 35#include <sys/types.h> 36#include <sys/kernel.h> 37#include <sys/ktr.h> --- 27 unchanged lines hidden (view full) --- 65#define V_tcp_autosndbuf_max VNET(tcp_autosndbuf_max) 66VNET_DECLARE(int, tcp_do_autorcvbuf); 67#define V_tcp_do_autorcvbuf VNET(tcp_do_autorcvbuf) 68VNET_DECLARE(int, tcp_autorcvbuf_inc); 69#define V_tcp_autorcvbuf_inc VNET(tcp_autorcvbuf_inc) 70VNET_DECLARE(int, tcp_autorcvbuf_max); 71#define V_tcp_autorcvbuf_max VNET(tcp_autorcvbuf_max) 72 | 30 31#include "opt_inet.h" 32 33#ifdef TCP_OFFLOAD 34#include <sys/param.h> 35#include <sys/types.h> 36#include <sys/kernel.h> 37#include <sys/ktr.h> --- 27 unchanged lines hidden (view full) --- 65#define V_tcp_autosndbuf_max VNET(tcp_autosndbuf_max) 66VNET_DECLARE(int, tcp_do_autorcvbuf); 67#define V_tcp_do_autorcvbuf VNET(tcp_do_autorcvbuf) 68VNET_DECLARE(int, tcp_autorcvbuf_inc); 69#define V_tcp_autorcvbuf_inc VNET(tcp_autorcvbuf_inc) 70VNET_DECLARE(int, tcp_autorcvbuf_max); 71#define V_tcp_autorcvbuf_max VNET(tcp_autorcvbuf_max) 72 |
73/* 74 * For ULP connections HW may add headers, e.g., for digests, that aren't part 75 * of the messages sent by the host but that are part of the TCP payload and 76 * therefore consume TCP sequence space. Tx connection parameters that 77 * operate in TCP sequence space are affected by the HW additions and need to 78 * compensate for them to accurately track TCP sequence numbers. This array 79 * contains the compensating extra lengths for ULP packets. It is indexed by 80 * a packet's ULP submode. 81 */ 82const unsigned int t4_ulp_extra_len[] = {0, 4, 4, 8}; 83 84/* 85 * Return the length of any HW additions that will be made to a Tx packet. 86 * Such additions can happen for some types of ULP packets. 87 */ 88static inline unsigned int 89ulp_extra_len(struct mbuf *m, int *ulp_mode) 90{ 91 struct m_tag *mtag; 92 93 if ((mtag = m_tag_find(m, CXGBE_ISCSI_MBUF_TAG, NULL)) == NULL) 94 return (0); 95 *ulp_mode = *((int *)(mtag + 1)); 96 97 return (t4_ulp_extra_len[*ulp_mode & 3]); 98} 99 |
|
73void 74send_flowc_wr(struct toepcb *toep, struct flowc_tx_params *ftxp) 75{ 76 struct wrqe *wr; 77 struct fw_flowc_wr *flowc; 78 unsigned int nparams = ftxp ? 8 : 6, flowclen; 79 struct port_info *pi = toep->port; 80 struct adapter *sc = pi->adapter; --- 255 unchanged lines hidden (view full) --- 336 int credits; 337 338 INP_WLOCK_ASSERT(inp); 339 340 SOCKBUF_LOCK(sb); 341 KASSERT(toep->sb_cc >= sb->sb_cc, 342 ("%s: sb %p has more data (%d) than last time (%d).", 343 __func__, sb, sb->sb_cc, toep->sb_cc)); | 100void 101send_flowc_wr(struct toepcb *toep, struct flowc_tx_params *ftxp) 102{ 103 struct wrqe *wr; 104 struct fw_flowc_wr *flowc; 105 unsigned int nparams = ftxp ? 8 : 6, flowclen; 106 struct port_info *pi = toep->port; 107 struct adapter *sc = pi->adapter; --- 255 unchanged lines hidden (view full) --- 363 int credits; 364 365 INP_WLOCK_ASSERT(inp); 366 367 SOCKBUF_LOCK(sb); 368 KASSERT(toep->sb_cc >= sb->sb_cc, 369 ("%s: sb %p has more data (%d) than last time (%d).", 370 __func__, sb, sb->sb_cc, toep->sb_cc)); |
344 toep->rx_credits += toep->sb_cc - sb->sb_cc; 345 toep->sb_cc = sb->sb_cc; | 371 if (toep->ulp_mode == ULP_MODE_ISCSI) { 372 toep->rx_credits += toep->sb_cc; 373 toep->sb_cc = 0; 374 } else { 375 toep->rx_credits += toep->sb_cc - sb->sb_cc; 376 toep->sb_cc = sb->sb_cc; 377 } |
346 credits = toep->rx_credits; 347 SOCKBUF_UNLOCK(sb); 348 349 if (credits > 0 && 350 (credits + 16384 >= tp->rcv_wnd || credits >= 15 * 1024)) { 351 352 credits = send_rx_credits(sc, toep, credits); 353 SOCKBUF_LOCK(sb); --- 85 unchanged lines hidden (view full) --- 439 if ((sge_pair_credits * 16) % 24 == 16) 440 nseg++; 441 442 return (nseg); 443} 444 445static inline void 446write_tx_wr(void *dst, struct toepcb *toep, unsigned int immdlen, | 378 credits = toep->rx_credits; 379 SOCKBUF_UNLOCK(sb); 380 381 if (credits > 0 && 382 (credits + 16384 >= tp->rcv_wnd || credits >= 15 * 1024)) { 383 384 credits = send_rx_credits(sc, toep, credits); 385 SOCKBUF_LOCK(sb); --- 85 unchanged lines hidden (view full) --- 471 if ((sge_pair_credits * 16) % 24 == 16) 472 nseg++; 473 474 return (nseg); 475} 476 477static inline void 478write_tx_wr(void *dst, struct toepcb *toep, unsigned int immdlen, |
447 unsigned int plen, uint8_t credits, int shove) | 479 unsigned int plen, uint8_t credits, int shove, int ulp_mode) |
448{ 449 struct fw_ofld_tx_data_wr *txwr = dst; | 480{ 481 struct fw_ofld_tx_data_wr *txwr = dst; |
482 unsigned int wr_ulp_mode; |
|
450 451 txwr->op_to_immdlen = htobe32(V_WR_OP(FW_OFLD_TX_DATA_WR) | 452 V_FW_WR_IMMDLEN(immdlen)); 453 txwr->flowid_len16 = htobe32(V_FW_WR_FLOWID(toep->tid) | 454 V_FW_WR_LEN16(credits)); | 483 484 txwr->op_to_immdlen = htobe32(V_WR_OP(FW_OFLD_TX_DATA_WR) | 485 V_FW_WR_IMMDLEN(immdlen)); 486 txwr->flowid_len16 = htobe32(V_FW_WR_FLOWID(toep->tid) | 487 V_FW_WR_LEN16(credits)); |
488 489 /* for iscsi, the mode & submode setting is per-packet */ 490 if (toep->ulp_mode == ULP_MODE_ISCSI) 491 wr_ulp_mode = V_FW_OFLD_TX_DATA_WR_ULPMODE(ulp_mode >> 4) | 492 V_FW_OFLD_TX_DATA_WR_ULPSUBMODE(ulp_mode & 3); 493 else 494 wr_ulp_mode = V_FW_OFLD_TX_DATA_WR_ULPMODE(toep->ulp_mode); 495 |
|
455 txwr->lsodisable_to_proxy = | 496 txwr->lsodisable_to_proxy = |
456 htobe32(V_FW_OFLD_TX_DATA_WR_ULPMODE(toep->ulp_mode) | | 497 htobe32(wr_ulp_mode | |
457 V_FW_OFLD_TX_DATA_WR_URGENT(0) | /* XXX */ 458 V_FW_OFLD_TX_DATA_WR_SHOVE(shove)); 459 txwr->plen = htobe32(plen); 460} 461 462/* 463 * Generate a DSGL from a starting mbuf. The total number of segments and the 464 * maximum segments in any one mbuf are provided. --- 57 unchanged lines hidden (view full) --- 522 * 523 * drop indicates the number of bytes that should be dropped from the head of 524 * the send buffer. It is an optimization that lets do_fw4_ack avoid creating 525 * contention on the send buffer lock (before this change it used to do 526 * sowwakeup and then t4_push_frames right after that when recovering from tx 527 * stalls). When drop is set this function MUST drop the bytes and wake up any 528 * writers. 529 */ | 498 V_FW_OFLD_TX_DATA_WR_URGENT(0) | /* XXX */ 499 V_FW_OFLD_TX_DATA_WR_SHOVE(shove)); 500 txwr->plen = htobe32(plen); 501} 502 503/* 504 * Generate a DSGL from a starting mbuf. The total number of segments and the 505 * maximum segments in any one mbuf are provided. --- 57 unchanged lines hidden (view full) --- 563 * 564 * drop indicates the number of bytes that should be dropped from the head of 565 * the send buffer. It is an optimization that lets do_fw4_ack avoid creating 566 * contention on the send buffer lock (before this change it used to do 567 * sowwakeup and then t4_push_frames right after that when recovering from tx 568 * stalls). When drop is set this function MUST drop the bytes and wake up any 569 * writers. 570 */ |
530static void | 571void |
531t4_push_frames(struct adapter *sc, struct toepcb *toep, int drop) 532{ 533 struct mbuf *sndptr, *m, *sb_sndptr; 534 struct fw_ofld_tx_data_wr *txwr; 535 struct wrqe *wr; 536 u_int plen, nsegs, credits, max_imm, max_nsegs, max_nsegs_1mbuf; 537 struct inpcb *inp = toep->inp; 538 struct tcpcb *tp = intotcpcb(inp); --- 116 unchanged lines hidden (view full) --- 655 toep->ofld_txq); 656 if (wr == NULL) { 657 /* XXX: how will we recover from this? */ 658 toep->flags |= TPF_TX_SUSPENDED; 659 return; 660 } 661 txwr = wrtod(wr); 662 credits = howmany(wr->wr_len, 16); | 572t4_push_frames(struct adapter *sc, struct toepcb *toep, int drop) 573{ 574 struct mbuf *sndptr, *m, *sb_sndptr; 575 struct fw_ofld_tx_data_wr *txwr; 576 struct wrqe *wr; 577 u_int plen, nsegs, credits, max_imm, max_nsegs, max_nsegs_1mbuf; 578 struct inpcb *inp = toep->inp; 579 struct tcpcb *tp = intotcpcb(inp); --- 116 unchanged lines hidden (view full) --- 696 toep->ofld_txq); 697 if (wr == NULL) { 698 /* XXX: how will we recover from this? */ 699 toep->flags |= TPF_TX_SUSPENDED; 700 return; 701 } 702 txwr = wrtod(wr); 703 credits = howmany(wr->wr_len, 16); |
663 write_tx_wr(txwr, toep, plen, plen, credits, shove); | 704 write_tx_wr(txwr, toep, plen, plen, credits, shove, 0); |
664 m_copydata(sndptr, 0, plen, (void *)(txwr + 1)); 665 nsegs = 0; 666 } else { 667 int wr_len; 668 669 /* DSGL tx */ 670 671 wr_len = sizeof(*txwr) + sizeof(struct ulptx_sgl) + 672 ((3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1)) * 8; 673 wr = alloc_wrqe(roundup2(wr_len, 16), toep->ofld_txq); 674 if (wr == NULL) { 675 /* XXX: how will we recover from this? */ 676 toep->flags |= TPF_TX_SUSPENDED; 677 return; 678 } 679 txwr = wrtod(wr); 680 credits = howmany(wr_len, 16); | 705 m_copydata(sndptr, 0, plen, (void *)(txwr + 1)); 706 nsegs = 0; 707 } else { 708 int wr_len; 709 710 /* DSGL tx */ 711 712 wr_len = sizeof(*txwr) + sizeof(struct ulptx_sgl) + 713 ((3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1)) * 8; 714 wr = alloc_wrqe(roundup2(wr_len, 16), toep->ofld_txq); 715 if (wr == NULL) { 716 /* XXX: how will we recover from this? */ 717 toep->flags |= TPF_TX_SUSPENDED; 718 return; 719 } 720 txwr = wrtod(wr); 721 credits = howmany(wr_len, 16); |
681 write_tx_wr(txwr, toep, 0, plen, credits, shove); | 722 write_tx_wr(txwr, toep, 0, plen, credits, shove, 0); |
682 write_tx_sgl(txwr + 1, sndptr, m, nsegs, 683 max_nsegs_1mbuf); 684 if (wr_len & 0xf) { 685 uint64_t *pad = (uint64_t *) 686 ((uintptr_t)txwr + wr_len); 687 *pad = 0; 688 } 689 } --- 39 unchanged lines hidden (view full) --- 729 t4_l2t_send(sc, wr, toep->l2te); 730 } while (m != NULL); 731 732 /* Send a FIN if requested, but only if there's no more data to send */ 733 if (m == NULL && toep->flags & TPF_SEND_FIN) 734 close_conn(sc, toep); 735} 736 | 723 write_tx_sgl(txwr + 1, sndptr, m, nsegs, 724 max_nsegs_1mbuf); 725 if (wr_len & 0xf) { 726 uint64_t *pad = (uint64_t *) 727 ((uintptr_t)txwr + wr_len); 728 *pad = 0; 729 } 730 } --- 39 unchanged lines hidden (view full) --- 770 t4_l2t_send(sc, wr, toep->l2te); 771 } while (m != NULL); 772 773 /* Send a FIN if requested, but only if there's no more data to send */ 774 if (m == NULL && toep->flags & TPF_SEND_FIN) 775 close_conn(sc, toep); 776} 777 |
778/* Send ULP data over TOE using TX_DATA_WR. We send whole mbuf at once */ 779void 780t4_ulp_push_frames(struct adapter *sc, struct toepcb *toep, int drop) 781{ 782 struct mbuf *sndptr, *m = NULL; 783 struct fw_ofld_tx_data_wr *txwr; 784 struct wrqe *wr; 785 unsigned int plen, nsegs, credits, max_imm, max_nsegs, max_nsegs_1mbuf; 786 struct inpcb *inp = toep->inp; 787 struct tcpcb *tp; 788 struct socket *so; 789 struct sockbuf *sb; 790 int tx_credits, ulp_len = 0, ulp_mode = 0, qlen = 0; 791 int shove, compl; 792 struct ofld_tx_sdesc *txsd; 793 794 INP_WLOCK_ASSERT(inp); 795 if (toep->flags & TPF_ABORT_SHUTDOWN) 796 return; 797 798 tp = intotcpcb(inp); 799 so = inp->inp_socket; 800 sb = &so->so_snd; 801 txsd = &toep->txsd[toep->txsd_pidx]; 802 803 KASSERT(toep->flags & TPF_FLOWC_WR_SENT, 804 ("%s: flowc_wr not sent for tid %u.", __func__, toep->tid)); 805 806 /* 807 * This function doesn't resume by itself. Someone else must clear the 808 * flag and call this function. 809 */ 810 if (__predict_false(toep->flags & TPF_TX_SUSPENDED)) 811 return; 812 813 sndptr = t4_queue_iscsi_callback(so, toep, 1, &qlen); 814 if (!qlen) 815 return; 816 817 do { 818 tx_credits = min(toep->tx_credits, MAX_OFLD_TX_CREDITS); 819 max_imm = max_imm_payload(tx_credits); 820 max_nsegs = max_dsgl_nsegs(tx_credits); 821 822 if (drop) { 823 t4_cpl_iscsi_callback(toep->td, toep, &drop, 824 CPL_FW4_ACK); 825 drop = 0; 826 } 827 828 plen = 0; 829 nsegs = 0; 830 max_nsegs_1mbuf = 0; /* max # of SGL segments in any one mbuf */ 831 for (m = sndptr; m != NULL; m = m->m_next) { 832 int n = sglist_count(mtod(m, void *), m->m_len); 833 834 nsegs += n; 835 plen += m->m_len; 836 837 /* This mbuf sent us _over_ the nsegs limit, return */ 838 if (plen > max_imm && nsegs > max_nsegs) { 839 toep->flags |= TPF_TX_SUSPENDED; 840 return; 841 } 842 843 if (max_nsegs_1mbuf < n) 844 max_nsegs_1mbuf = n; 845 846 /* This mbuf put us right at the max_nsegs limit */ 847 if (plen > max_imm && nsegs == max_nsegs) { 848 toep->flags |= TPF_TX_SUSPENDED; 849 return; 850 } 851 } 852 853 shove = m == NULL && !(tp->t_flags & TF_MORETOCOME); 854 /* nothing to send */ 855 if (plen == 0) { 856 KASSERT(m == NULL, 857 ("%s: nothing to send, but m != NULL", __func__)); 858 break; 859 } 860 861 if (__predict_false(toep->flags & TPF_FIN_SENT)) 862 panic("%s: excess tx.", __func__); 863 864 ulp_len = plen + ulp_extra_len(sndptr, &ulp_mode); 865 if (plen <= max_imm) { 866 867 /* Immediate data tx */ 868 wr = alloc_wrqe(roundup(sizeof(*txwr) + plen, 16), 869 toep->ofld_txq); 870 if (wr == NULL) { 871 /* XXX: how will we recover from this? */ 872 toep->flags |= TPF_TX_SUSPENDED; 873 return; 874 } 875 txwr = wrtod(wr); 876 credits = howmany(wr->wr_len, 16); 877 write_tx_wr(txwr, toep, plen, ulp_len, credits, shove, 878 ulp_mode); 879 m_copydata(sndptr, 0, plen, (void *)(txwr + 1)); 880 } else { 881 int wr_len; 882 883 /* DSGL tx */ 884 wr_len = sizeof(*txwr) + sizeof(struct ulptx_sgl) + 885 ((3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1)) * 8; 886 wr = alloc_wrqe(roundup(wr_len, 16), toep->ofld_txq); 887 if (wr == NULL) { 888 /* XXX: how will we recover from this? */ 889 toep->flags |= TPF_TX_SUSPENDED; 890 return; 891 } 892 txwr = wrtod(wr); 893 credits = howmany(wr_len, 16); 894 write_tx_wr(txwr, toep, 0, ulp_len, credits, shove, 895 ulp_mode); 896 write_tx_sgl(txwr + 1, sndptr, m, nsegs, 897 max_nsegs_1mbuf); 898 if (wr_len & 0xf) { 899 uint64_t *pad = (uint64_t *) 900 ((uintptr_t)txwr + wr_len); 901 *pad = 0; 902 } 903 } 904 905 KASSERT(toep->tx_credits >= credits, 906 ("%s: not enough credits", __func__)); 907 908 toep->tx_credits -= credits; 909 toep->tx_nocompl += credits; 910 toep->plen_nocompl += plen; 911 if (toep->tx_credits <= toep->tx_total * 3 / 8 && 912 toep->tx_nocompl >= toep->tx_total / 4) 913 compl = 1; 914 915 if (compl) { 916 txwr->op_to_immdlen |= htobe32(F_FW_WR_COMPL); 917 toep->tx_nocompl = 0; 918 toep->plen_nocompl = 0; 919 } 920 tp->snd_nxt += ulp_len; 921 tp->snd_max += ulp_len; 922 923 /* goto next mbuf */ 924 sndptr = m = t4_queue_iscsi_callback(so, toep, 2, &qlen); 925 926 toep->flags |= TPF_TX_DATA_SENT; 927 if (toep->tx_credits < MIN_OFLD_TX_CREDITS) { 928 toep->flags |= TPF_TX_SUSPENDED; 929 } 930 931 KASSERT(toep->txsd_avail > 0, ("%s: no txsd", __func__)); 932 txsd->plen = plen; 933 txsd->tx_credits = credits; 934 txsd++; 935 if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) { 936 toep->txsd_pidx = 0; 937 txsd = &toep->txsd[0]; 938 } 939 toep->txsd_avail--; 940 941 t4_l2t_send(sc, wr, toep->l2te); 942 } while (m != NULL); 943 944 /* Send a FIN if requested, but only if there's no more data to send */ 945 if (m == NULL && toep->flags & TPF_SEND_FIN) 946 close_conn(sc, toep); 947} 948 |
|
737int 738t4_tod_output(struct toedev *tod, struct tcpcb *tp) 739{ 740 struct adapter *sc = tod->tod_softc; 741#ifdef INVARIANTS 742 struct inpcb *inp = tp->t_inpcb; 743#endif 744 struct toepcb *toep = tp->t_toe; --- 18 unchanged lines hidden (view full) --- 763 struct toepcb *toep = tp->t_toe; 764 765 INP_WLOCK_ASSERT(inp); 766 KASSERT((inp->inp_flags & INP_DROPPED) == 0, 767 ("%s: inp %p dropped.", __func__, inp)); 768 KASSERT(toep != NULL, ("%s: toep is NULL", __func__)); 769 770 toep->flags |= TPF_SEND_FIN; | 949int 950t4_tod_output(struct toedev *tod, struct tcpcb *tp) 951{ 952 struct adapter *sc = tod->tod_softc; 953#ifdef INVARIANTS 954 struct inpcb *inp = tp->t_inpcb; 955#endif 956 struct toepcb *toep = tp->t_toe; --- 18 unchanged lines hidden (view full) --- 975 struct toepcb *toep = tp->t_toe; 976 977 INP_WLOCK_ASSERT(inp); 978 KASSERT((inp->inp_flags & INP_DROPPED) == 0, 979 ("%s: inp %p dropped.", __func__, inp)); 980 KASSERT(toep != NULL, ("%s: toep is NULL", __func__)); 981 982 toep->flags |= TPF_SEND_FIN; |
771 if (tp->t_state >= TCPS_ESTABLISHED) 772 t4_push_frames(sc, toep, 0); | 983 if (tp->t_state >= TCPS_ESTABLISHED) { 984 if (toep->ulp_mode == ULP_MODE_ISCSI) 985 t4_ulp_push_frames(sc, toep, 0); 986 else 987 t4_push_frames(sc, toep, 0); 988 } |
773 774 return (0); 775} 776 777int 778t4_send_rst(struct toedev *tod, struct tcpcb *tp) 779{ 780 struct adapter *sc = tod->tod_softc; --- 233 unchanged lines hidden (view full) --- 1014 case CPL_ERR_FINWAIT2_TIMEDOUT: 1015 case CPL_ERR_KEEPALIVE_TIMEDOUT: 1016 return (ETIMEDOUT); 1017 default: 1018 return (EIO); 1019 } 1020} 1021 | 989 990 return (0); 991} 992 993int 994t4_send_rst(struct toedev *tod, struct tcpcb *tp) 995{ 996 struct adapter *sc = tod->tod_softc; --- 233 unchanged lines hidden (view full) --- 1230 case CPL_ERR_FINWAIT2_TIMEDOUT: 1231 case CPL_ERR_KEEPALIVE_TIMEDOUT: 1232 return (ETIMEDOUT); 1233 default: 1234 return (EIO); 1235 } 1236} 1237 |
1238int 1239cpl_not_handled(struct sge_iq *, const struct rss_header *, struct mbuf *); |
|
1022/* | 1240/* |
1241 * tom_cpl_iscsi_callback - 1242 * iscsi and tom would share the following cpl messages, so when any of these 1243 * message is received, after tom is done with processing it, the messages 1244 * needs to be forwarded to iscsi for further processing: 1245 * - CPL_SET_TCB_RPL 1246 * - CPL_RX_DATA_DDP 1247 */ 1248void (*tom_cpl_iscsi_callback)(struct tom_data *, struct socket *, void *, 1249 unsigned int); 1250 1251struct mbuf *(*tom_queue_iscsi_callback)(struct socket *, unsigned int, int *); 1252/* 1253 * Check if the handler function is set for a given CPL 1254 * return 0 if the function is NULL or cpl_not_handled, 1 otherwise. 1255 */ 1256int 1257t4tom_cpl_handler_registered(struct adapter *sc, unsigned int opcode) 1258{ 1259 1260 MPASS(opcode < nitems(sc->cpl_handler)); 1261 1262 return (sc->cpl_handler[opcode] && 1263 sc->cpl_handler[opcode] != cpl_not_handled); 1264} 1265 1266/* 1267 * set the tom_cpl_iscsi_callback function, this function should be used 1268 * whenever both toe and iscsi need to process the same cpl msg. 1269 */ 1270void 1271t4tom_register_cpl_iscsi_callback(void (*fp)(struct tom_data *, struct socket *, 1272 void *, unsigned int)) 1273{ 1274 1275 tom_cpl_iscsi_callback = fp; 1276} 1277 1278void 1279t4tom_register_queue_iscsi_callback(struct mbuf *(*fp)(struct socket *, 1280 unsigned int, int *qlen)) 1281{ 1282 1283 tom_queue_iscsi_callback = fp; 1284} 1285 1286int 1287t4_cpl_iscsi_callback(struct tom_data *td, struct toepcb *toep, void *m, 1288 unsigned int opcode) 1289{ 1290 struct socket *so; 1291 1292 if (opcode == CPL_FW4_ACK) 1293 so = toep->inp->inp_socket; 1294 else { 1295 INP_WLOCK(toep->inp); 1296 so = toep->inp->inp_socket; 1297 INP_WUNLOCK(toep->inp); 1298 } 1299 1300 if (tom_cpl_iscsi_callback && so) { 1301 if (toep->ulp_mode == ULP_MODE_ISCSI) { 1302 tom_cpl_iscsi_callback(td, so, m, opcode); 1303 return (0); 1304 } 1305 } 1306 1307 return (1); 1308} 1309 1310struct mbuf * 1311t4_queue_iscsi_callback(struct socket *so, struct toepcb *toep, 1312 unsigned int cmd, int *qlen) 1313{ 1314 1315 if (tom_queue_iscsi_callback && so) { 1316 if (toep->ulp_mode == ULP_MODE_ISCSI) 1317 return (tom_queue_iscsi_callback(so, cmd, qlen)); 1318 } 1319 1320 return (NULL); 1321} 1322 1323/* |
|
1023 * TCP RST from the peer, timeout, or some other such critical error. 1024 */ 1025static int 1026do_abort_req(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 1027{ 1028 struct adapter *sc = iq->adapter; 1029 const struct cpl_abort_req_rss *cpl = (const void *)(rss + 1); 1030 unsigned int tid = GET_TID(cpl); --- 372 unchanged lines hidden (view full) --- 1403 if (toep->tx_credits == toep->tx_total) { 1404 toep->tx_nocompl = 0; 1405 toep->plen_nocompl = 0; 1406 } 1407 1408 if (toep->flags & TPF_TX_SUSPENDED && 1409 toep->tx_credits >= toep->tx_total / 4) { 1410 toep->flags &= ~TPF_TX_SUSPENDED; | 1324 * TCP RST from the peer, timeout, or some other such critical error. 1325 */ 1326static int 1327do_abort_req(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m) 1328{ 1329 struct adapter *sc = iq->adapter; 1330 const struct cpl_abort_req_rss *cpl = (const void *)(rss + 1); 1331 unsigned int tid = GET_TID(cpl); --- 372 unchanged lines hidden (view full) --- 1704 if (toep->tx_credits == toep->tx_total) { 1705 toep->tx_nocompl = 0; 1706 toep->plen_nocompl = 0; 1707 } 1708 1709 if (toep->flags & TPF_TX_SUSPENDED && 1710 toep->tx_credits >= toep->tx_total / 4) { 1711 toep->flags &= ~TPF_TX_SUSPENDED; |
1411 t4_push_frames(sc, toep, plen); | 1712 if (toep->ulp_mode == ULP_MODE_ISCSI) 1713 t4_ulp_push_frames(sc, toep, plen); 1714 else 1715 t4_push_frames(sc, toep, plen); |
1412 } else if (plen > 0) { 1413 struct sockbuf *sb = &so->so_snd; 1414 | 1716 } else if (plen > 0) { 1717 struct sockbuf *sb = &so->so_snd; 1718 |
1415 SOCKBUF_LOCK(sb); 1416 sbdrop_locked(sb, plen); 1417 sowwakeup_locked(so); 1418 SOCKBUF_UNLOCK_ASSERT(sb); | 1719 if (toep->ulp_mode == ULP_MODE_ISCSI) 1720 t4_cpl_iscsi_callback(toep->td, toep, &plen, 1721 CPL_FW4_ACK); 1722 else { 1723 SOCKBUF_LOCK(sb); 1724 sbdrop_locked(sb, plen); 1725 sowwakeup_locked(so); 1726 SOCKBUF_UNLOCK_ASSERT(sb); 1727 } |
1419 } 1420 1421 INP_WUNLOCK(inp); 1422 1423 return (0); 1424} 1425 1426static int --- 7 unchanged lines hidden (view full) --- 1434#endif 1435 1436 KASSERT(opcode == CPL_SET_TCB_RPL, 1437 ("%s: unexpected opcode 0x%x", __func__, opcode)); 1438 KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__)); 1439 1440 if (is_ftid(sc, tid)) 1441 return (t4_filter_rpl(iq, rss, m)); /* TCB is a filter */ | 1728 } 1729 1730 INP_WUNLOCK(inp); 1731 1732 return (0); 1733} 1734 1735static int --- 7 unchanged lines hidden (view full) --- 1743#endif 1744 1745 KASSERT(opcode == CPL_SET_TCB_RPL, 1746 ("%s: unexpected opcode 0x%x", __func__, opcode)); 1747 KASSERT(m == NULL, ("%s: wasn't expecting payload", __func__)); 1748 1749 if (is_ftid(sc, tid)) 1750 return (t4_filter_rpl(iq, rss, m)); /* TCB is a filter */ |
1751 else { 1752 struct toepcb *toep = lookup_tid(sc, tid); |
|
1442 | 1753 |
1754 t4_cpl_iscsi_callback(toep->td, toep, m, CPL_SET_TCB_RPL); 1755 return (0); 1756 } 1757 |
|
1443 CXGBE_UNIMPLEMENTED(__func__); 1444} 1445 1446void 1447t4_set_tcb_field(struct adapter *sc, struct toepcb *toep, int ctrl, 1448 uint16_t word, uint64_t mask, uint64_t val) 1449{ 1450 struct wrqe *wr; --- 39 unchanged lines hidden --- | 1758 CXGBE_UNIMPLEMENTED(__func__); 1759} 1760 1761void 1762t4_set_tcb_field(struct adapter *sc, struct toepcb *toep, int ctrl, 1763 uint16_t word, uint64_t mask, uint64_t val) 1764{ 1765 struct wrqe *wr; --- 39 unchanged lines hidden --- |