cxgb_cpl_io.c revision 181011
1174641Skmacy/************************************************************************** 2174641Skmacy 3178302SkmacyCopyright (c) 2007-2008, Chelsio Inc. 4174641SkmacyAll rights reserved. 5174641Skmacy 6174641SkmacyRedistribution and use in source and binary forms, with or without 7174641Skmacymodification, are permitted provided that the following conditions are met: 8174641Skmacy 9174641Skmacy 1. Redistributions of source code must retain the above copyright notice, 10174641Skmacy this list of conditions and the following disclaimer. 11174641Skmacy 12174641Skmacy 2. Neither the name of the Chelsio Corporation nor the names of its 13174641Skmacy contributors may be used to endorse or promote products derived from 14174641Skmacy this software without specific prior written permission. 15174641Skmacy 16174641SkmacyTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 17174641SkmacyAND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18174641SkmacyIMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19174641SkmacyARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 20174641SkmacyLIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21174641SkmacyCONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22174641SkmacySUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23174641SkmacyINTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24174641SkmacyCONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25174641SkmacyARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26174641SkmacyPOSSIBILITY OF SUCH DAMAGE. 27174641Skmacy 28174641Skmacy***************************************************************************/ 29174641Skmacy 30174641Skmacy#include <sys/cdefs.h> 31174641Skmacy__FBSDID("$FreeBSD: head/sys/dev/cxgb/ulp/tom/cxgb_cpl_io.c 181011 2008-07-30 20:08:34Z kmacy $"); 32174641Skmacy 33174641Skmacy#include <sys/param.h> 34174641Skmacy#include <sys/systm.h> 35174641Skmacy#include <sys/fcntl.h> 36174641Skmacy#include <sys/kernel.h> 37174641Skmacy#include <sys/limits.h> 38176472Skmacy#include <sys/ktr.h> 39174641Skmacy#include <sys/lock.h> 40174641Skmacy#include <sys/mbuf.h> 41174641Skmacy#include <sys/mutex.h> 42181011Skmacy#include <sys/sockbuf.h> 43181011Skmacy#include <sys/sockstate.h> 44181011Skmacy#include <sys/sockopt.h> 45174641Skmacy#include <sys/socket.h> 46174641Skmacy#include <sys/sysctl.h> 47174641Skmacy#include <sys/syslog.h> 48174641Skmacy#include <sys/protosw.h> 49174641Skmacy#include <sys/priv.h> 50174641Skmacy 51174641Skmacy#include <net/if.h> 52174641Skmacy#include <net/route.h> 53174641Skmacy 54174641Skmacy#include <netinet/in.h> 55174641Skmacy#include <netinet/in_pcb.h> 56174641Skmacy#include <netinet/in_systm.h> 57174641Skmacy#include <netinet/in_var.h> 58174641Skmacy 59174641Skmacy 60174641Skmacy#include <dev/cxgb/cxgb_osdep.h> 61174641Skmacy#include <dev/cxgb/sys/mbufq.h> 62174641Skmacy 63174641Skmacy#include <netinet/ip.h> 64174641Skmacy#include <netinet/tcp_var.h> 65174641Skmacy#include <netinet/tcp_fsm.h> 66174708Skmacy#include <netinet/tcp_offload.h> 67174641Skmacy#include <netinet/tcp_seq.h> 68174641Skmacy#include <netinet/tcp_syncache.h> 69176472Skmacy#include <netinet/tcp_timer.h> 70174641Skmacy#include <net/route.h> 71174641Skmacy 72174641Skmacy#include <dev/cxgb/t3cdev.h> 73174641Skmacy#include <dev/cxgb/common/cxgb_firmware_exports.h> 74174641Skmacy#include <dev/cxgb/common/cxgb_t3_cpl.h> 75174641Skmacy#include <dev/cxgb/common/cxgb_tcb.h> 76174641Skmacy#include <dev/cxgb/common/cxgb_ctl_defs.h> 77174641Skmacy#include <dev/cxgb/cxgb_offload.h> 78174641Skmacy#include <vm/vm.h> 79174641Skmacy#include <vm/pmap.h> 80174641Skmacy#include <machine/bus.h> 81174641Skmacy#include <dev/cxgb/sys/mvec.h> 82174641Skmacy#include <dev/cxgb/ulp/toecore/cxgb_toedev.h> 83174641Skmacy#include <dev/cxgb/ulp/tom/cxgb_defs.h> 84174641Skmacy#include <dev/cxgb/ulp/tom/cxgb_tom.h> 85174641Skmacy#include <dev/cxgb/ulp/tom/cxgb_t3_ddp.h> 86174641Skmacy#include <dev/cxgb/ulp/tom/cxgb_toepcb.h> 87174708Skmacy#include <dev/cxgb/ulp/tom/cxgb_tcp.h> 88174641Skmacy 89178302Skmacy#include <dev/cxgb/ulp/tom/cxgb_tcp_offload.h> 90178302Skmacy 91174641Skmacy/* 92174641Skmacy * For ULP connections HW may add headers, e.g., for digests, that aren't part 93174641Skmacy * of the messages sent by the host but that are part of the TCP payload and 94174641Skmacy * therefore consume TCP sequence space. Tx connection parameters that 95174641Skmacy * operate in TCP sequence space are affected by the HW additions and need to 96174641Skmacy * compensate for them to accurately track TCP sequence numbers. This array 97174641Skmacy * contains the compensating extra lengths for ULP packets. It is indexed by 98174641Skmacy * a packet's ULP submode. 99174641Skmacy */ 100174641Skmacyconst unsigned int t3_ulp_extra_len[] = {0, 4, 4, 8}; 101174641Skmacy 102174641Skmacy#ifdef notyet 103174641Skmacy/* 104174641Skmacy * This sk_buff holds a fake header-only TCP segment that we use whenever we 105174641Skmacy * need to exploit SW TCP functionality that expects TCP headers, such as 106174641Skmacy * tcp_create_openreq_child(). It's a RO buffer that may be used by multiple 107174641Skmacy * CPUs without locking. 108174641Skmacy */ 109174641Skmacystatic struct mbuf *tcphdr_mbuf __read_mostly; 110174641Skmacy#endif 111174641Skmacy 112174641Skmacy/* 113174641Skmacy * Size of WRs in bytes. Note that we assume all devices we are handling have 114174641Skmacy * the same WR size. 115174641Skmacy */ 116174641Skmacystatic unsigned int wrlen __read_mostly; 117174641Skmacy 118174641Skmacy/* 119174641Skmacy * The number of WRs needed for an skb depends on the number of page fragments 120174641Skmacy * in the skb and whether it has any payload in its main body. This maps the 121174641Skmacy * length of the gather list represented by an skb into the # of necessary WRs. 122174641Skmacy */ 123176472Skmacystatic unsigned int mbuf_wrs[TX_MAX_SEGS + 1] __read_mostly; 124174641Skmacy 125174641Skmacy/* 126174641Skmacy * Max receive window supported by HW in bytes. Only a small part of it can 127174641Skmacy * be set through option0, the rest needs to be set through RX_DATA_ACK. 128174641Skmacy */ 129174641Skmacy#define MAX_RCV_WND ((1U << 27) - 1) 130174641Skmacy 131174641Skmacy/* 132174641Skmacy * Min receive window. We want it to be large enough to accommodate receive 133174641Skmacy * coalescing, handle jumbo frames, and not trigger sender SWS avoidance. 134174641Skmacy */ 135174641Skmacy#define MIN_RCV_WND (24 * 1024U) 136178302Skmacy#define INP_TOS(inp) ((inp_ip_tos_get(inp) >> 2) & M_TOS) 137174641Skmacy 138174641Skmacy#define VALIDATE_SEQ 0 139174641Skmacy#define VALIDATE_SOCK(so) 140174641Skmacy#define DEBUG_WR 0 141174641Skmacy 142178302Skmacy#define TCP_TIMEWAIT 1 143178302Skmacy#define TCP_CLOSE 2 144178302Skmacy#define TCP_DROP 3 145178302Skmacy 146174641Skmacyextern int tcp_do_autorcvbuf; 147174641Skmacyextern int tcp_do_autosndbuf; 148174641Skmacyextern int tcp_autorcvbuf_max; 149174641Skmacyextern int tcp_autosndbuf_max; 150174641Skmacy 151174641Skmacystatic void t3_send_reset(struct toepcb *toep); 152174641Skmacystatic void send_abort_rpl(struct mbuf *m, struct toedev *tdev, int rst_status); 153174641Skmacystatic inline void free_atid(struct t3cdev *cdev, unsigned int tid); 154174641Skmacystatic void handle_syncache_event(int event, void *arg); 155174641Skmacy 156176472Skmacystatic inline void 157176472SkmacySBAPPEND(struct sockbuf *sb, struct mbuf *n) 158176472Skmacy{ 159178302Skmacy struct mbuf *m; 160174641Skmacy 161176472Skmacy m = sb->sb_mb; 162176472Skmacy while (m) { 163176472Skmacy KASSERT(((m->m_flags & M_EXT) && (m->m_ext.ext_type == EXT_EXTREF)) || 164176472Skmacy !(m->m_flags & M_EXT), ("unexpected type M_EXT=%d ext_type=%d m_len=%d\n", 165176472Skmacy !!(m->m_flags & M_EXT), m->m_ext.ext_type, m->m_len)); 166176472Skmacy KASSERT(m->m_next != (struct mbuf *)0xffffffff, ("bad next value m_next=%p m_nextpkt=%p m_flags=0x%x", 167176472Skmacy m->m_next, m->m_nextpkt, m->m_flags)); 168176472Skmacy m = m->m_next; 169176472Skmacy } 170176472Skmacy m = n; 171176472Skmacy while (m) { 172176472Skmacy KASSERT(((m->m_flags & M_EXT) && (m->m_ext.ext_type == EXT_EXTREF)) || 173176472Skmacy !(m->m_flags & M_EXT), ("unexpected type M_EXT=%d ext_type=%d m_len=%d\n", 174176472Skmacy !!(m->m_flags & M_EXT), m->m_ext.ext_type, m->m_len)); 175176472Skmacy KASSERT(m->m_next != (struct mbuf *)0xffffffff, ("bad next value m_next=%p m_nextpkt=%p m_flags=0x%x", 176176472Skmacy m->m_next, m->m_nextpkt, m->m_flags)); 177176472Skmacy m = m->m_next; 178176472Skmacy } 179178767Skmacy KASSERT(sb->sb_flags & SB_NOCOALESCE, ("NOCOALESCE not set")); 180178767Skmacy sbappendstream_locked(sb, n); 181176472Skmacy m = sb->sb_mb; 182178302Skmacy 183176472Skmacy while (m) { 184176472Skmacy KASSERT(m->m_next != (struct mbuf *)0xffffffff, ("bad next value m_next=%p m_nextpkt=%p m_flags=0x%x", 185176472Skmacy m->m_next, m->m_nextpkt, m->m_flags)); 186176472Skmacy m = m->m_next; 187176472Skmacy } 188176472Skmacy} 189176472Skmacy 190174641Skmacystatic inline int 191174641Skmacyis_t3a(const struct toedev *dev) 192174641Skmacy{ 193174641Skmacy return (dev->tod_ttid == TOE_ID_CHELSIO_T3); 194174641Skmacy} 195174641Skmacy 196174641Skmacystatic void 197174641Skmacydump_toepcb(struct toepcb *toep) 198174641Skmacy{ 199174641Skmacy DPRINTF("qset_idx=%d qset=%d ulp_mode=%d mtu_idx=%d tid=%d\n", 200174641Skmacy toep->tp_qset_idx, toep->tp_qset, toep->tp_ulp_mode, 201174641Skmacy toep->tp_mtu_idx, toep->tp_tid); 202174641Skmacy 203174641Skmacy DPRINTF("wr_max=%d wr_avail=%d wr_unacked=%d mss_clamp=%d flags=0x%x\n", 204174641Skmacy toep->tp_wr_max, toep->tp_wr_avail, toep->tp_wr_unacked, 205174641Skmacy toep->tp_mss_clamp, toep->tp_flags); 206174641Skmacy} 207174641Skmacy 208176472Skmacy#ifndef RTALLOC2_DEFINED 209174641Skmacystatic struct rtentry * 210174641Skmacyrtalloc2(struct sockaddr *dst, int report, u_long ignflags) 211174641Skmacy{ 212174641Skmacy struct rtentry *rt = NULL; 213174641Skmacy 214174641Skmacy if ((rt = rtalloc1(dst, report, ignflags)) != NULL) 215174641Skmacy RT_UNLOCK(rt); 216174641Skmacy 217174641Skmacy return (rt); 218174641Skmacy} 219176472Skmacy#endif 220178302Skmacy 221174641Skmacy/* 222174641Skmacy * Determine whether to send a CPL message now or defer it. A message is 223174641Skmacy * deferred if the connection is in SYN_SENT since we don't know the TID yet. 224174641Skmacy * For connections in other states the message is sent immediately. 225174641Skmacy * If through_l2t is set the message is subject to ARP processing, otherwise 226174641Skmacy * it is sent directly. 227174641Skmacy */ 228174641Skmacystatic inline void 229176472Skmacysend_or_defer(struct toepcb *toep, struct mbuf *m, int through_l2t) 230174641Skmacy{ 231176472Skmacy struct tcpcb *tp = toep->tp_tp; 232174641Skmacy 233174641Skmacy if (__predict_false(tp->t_state == TCPS_SYN_SENT)) { 234177530Skmacy inp_wlock(tp->t_inpcb); 235174641Skmacy mbufq_tail(&toep->out_of_order_queue, m); // defer 236177530Skmacy inp_wunlock(tp->t_inpcb); 237174641Skmacy } else if (through_l2t) 238176472Skmacy l2t_send(TOEP_T3C_DEV(toep), m, toep->tp_l2t); // send through L2T 239174641Skmacy else 240176472Skmacy cxgb_ofld_send(TOEP_T3C_DEV(toep), m); // send directly 241174641Skmacy} 242174641Skmacy 243174641Skmacystatic inline unsigned int 244176472Skmacymkprio(unsigned int cntrl, const struct toepcb *toep) 245174641Skmacy{ 246176472Skmacy return (cntrl); 247174641Skmacy} 248174641Skmacy 249174641Skmacy/* 250174641Skmacy * Populate a TID_RELEASE WR. The skb must be already propely sized. 251174641Skmacy */ 252174641Skmacystatic inline void 253176472Skmacymk_tid_release(struct mbuf *m, const struct toepcb *toep, unsigned int tid) 254174641Skmacy{ 255174641Skmacy struct cpl_tid_release *req; 256174641Skmacy 257176472Skmacy m_set_priority(m, mkprio(CPL_PRIORITY_SETUP, toep)); 258174641Skmacy m->m_pkthdr.len = m->m_len = sizeof(*req); 259174641Skmacy req = mtod(m, struct cpl_tid_release *); 260174641Skmacy req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 261176472Skmacy req->wr.wr_lo = 0; 262174641Skmacy OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_TID_RELEASE, tid)); 263174641Skmacy} 264174641Skmacy 265174641Skmacystatic inline void 266174641Skmacymake_tx_data_wr(struct socket *so, struct mbuf *m, int len, struct mbuf *tail) 267174641Skmacy{ 268178302Skmacy struct tcpcb *tp = so_sototcpcb(so); 269174641Skmacy struct toepcb *toep = tp->t_toe; 270174641Skmacy struct tx_data_wr *req; 271178302Skmacy struct sockbuf *snd; 272178302Skmacy 273177575Skmacy inp_lock_assert(tp->t_inpcb); 274178302Skmacy snd = so_sockbuf_snd(so); 275178302Skmacy 276174641Skmacy req = mtod(m, struct tx_data_wr *); 277174641Skmacy m->m_len = sizeof(*req); 278174641Skmacy req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_TX_DATA)); 279174641Skmacy req->wr_lo = htonl(V_WR_TID(toep->tp_tid)); 280174641Skmacy /* len includes the length of any HW ULP additions */ 281174641Skmacy req->len = htonl(len); 282174641Skmacy req->param = htonl(V_TX_PORT(toep->tp_l2t->smt_idx)); 283174641Skmacy /* V_TX_ULP_SUBMODE sets both the mode and submode */ 284174641Skmacy req->flags = htonl(V_TX_ULP_SUBMODE(/*skb_ulp_mode(skb)*/ 0) | 285174641Skmacy V_TX_URG(/* skb_urgent(skb) */ 0 ) | 286174641Skmacy V_TX_SHOVE((!(tp->t_flags & TF_MORETOCOME) && 287174641Skmacy (tail ? 0 : 1)))); 288174641Skmacy req->sndseq = htonl(tp->snd_nxt); 289174641Skmacy if (__predict_false((toep->tp_flags & TP_DATASENT) == 0)) { 290174641Skmacy req->flags |= htonl(V_TX_ACK_PAGES(2) | F_TX_INIT | 291174641Skmacy V_TX_CPU_IDX(toep->tp_qset)); 292174641Skmacy 293174641Skmacy /* Sendbuffer is in units of 32KB. 294174641Skmacy */ 295178302Skmacy if (tcp_do_autosndbuf && snd->sb_flags & SB_AUTOSIZE) 296174641Skmacy req->param |= htonl(V_TX_SNDBUF(tcp_autosndbuf_max >> 15)); 297178302Skmacy else { 298178302Skmacy req->param |= htonl(V_TX_SNDBUF(snd->sb_hiwat >> 15)); 299178302Skmacy } 300178302Skmacy 301174641Skmacy toep->tp_flags |= TP_DATASENT; 302174641Skmacy } 303174641Skmacy} 304174641Skmacy 305176472Skmacy#define IMM_LEN 64 /* XXX - see WR_LEN in the cxgb driver */ 306176472Skmacy 307174641Skmacyint 308174641Skmacyt3_push_frames(struct socket *so, int req_completion) 309174641Skmacy{ 310178302Skmacy struct tcpcb *tp = so_sototcpcb(so); 311174641Skmacy struct toepcb *toep = tp->t_toe; 312174641Skmacy 313174641Skmacy struct mbuf *tail, *m0, *last; 314174641Skmacy struct t3cdev *cdev; 315174641Skmacy struct tom_data *d; 316178302Skmacy int state, bytes, count, total_bytes; 317174641Skmacy bus_dma_segment_t segs[TX_MAX_SEGS], *segp; 318178302Skmacy struct sockbuf *snd; 319178302Skmacy 320174641Skmacy if (tp->t_state == TCPS_SYN_SENT || tp->t_state == TCPS_CLOSED) { 321174641Skmacy DPRINTF("tcp state=%d\n", tp->t_state); 322174641Skmacy return (0); 323174641Skmacy } 324174641Skmacy 325178302Skmacy state = so_state_get(so); 326178302Skmacy 327178302Skmacy if (state & (SS_ISDISCONNECTING|SS_ISDISCONNECTED)) { 328174641Skmacy DPRINTF("disconnecting\n"); 329174641Skmacy 330174641Skmacy return (0); 331174641Skmacy } 332174641Skmacy 333177575Skmacy inp_lock_assert(tp->t_inpcb); 334178302Skmacy 335178302Skmacy snd = so_sockbuf_snd(so); 336178302Skmacy sockbuf_lock(snd); 337178302Skmacy 338178302Skmacy d = TOM_DATA(toep->tp_toedev); 339174641Skmacy cdev = d->cdev; 340178302Skmacy 341178302Skmacy last = tail = snd->sb_sndptr ? snd->sb_sndptr : snd->sb_mb; 342178302Skmacy 343174641Skmacy total_bytes = 0; 344174641Skmacy DPRINTF("wr_avail=%d tail=%p snd.cc=%d tp_last=%p\n", 345178302Skmacy toep->tp_wr_avail, tail, snd->sb_cc, toep->tp_m_last); 346174641Skmacy 347178302Skmacy if (last && toep->tp_m_last == last && snd->sb_sndptroff != 0) { 348174641Skmacy KASSERT(tail, ("sbdrop error")); 349174641Skmacy last = tail = tail->m_next; 350174641Skmacy } 351174641Skmacy 352174641Skmacy if ((toep->tp_wr_avail == 0 ) || (tail == NULL)) { 353174641Skmacy DPRINTF("wr_avail=%d tail=%p\n", toep->tp_wr_avail, tail); 354178302Skmacy sockbuf_unlock(snd); 355178302Skmacy 356174641Skmacy return (0); 357174641Skmacy } 358174641Skmacy 359174641Skmacy toep->tp_m_last = NULL; 360174641Skmacy while (toep->tp_wr_avail && (tail != NULL)) { 361174641Skmacy count = bytes = 0; 362176472Skmacy segp = segs; 363174641Skmacy if ((m0 = m_gethdr(M_NOWAIT, MT_DATA)) == NULL) { 364178302Skmacy sockbuf_unlock(snd); 365174641Skmacy return (0); 366174641Skmacy } 367176472Skmacy /* 368176472Skmacy * If the data in tail fits as in-line, then 369176472Skmacy * make an immediate data wr. 370176472Skmacy */ 371176472Skmacy if (tail->m_len <= IMM_LEN) { 372176472Skmacy count = 1; 373176472Skmacy bytes = tail->m_len; 374174641Skmacy last = tail; 375174641Skmacy tail = tail->m_next; 376176472Skmacy m_set_sgl(m0, NULL); 377176472Skmacy m_set_sgllen(m0, 0); 378176472Skmacy make_tx_data_wr(so, m0, bytes, tail); 379176472Skmacy m_append(m0, bytes, mtod(last, caddr_t)); 380176472Skmacy KASSERT(!m0->m_next, ("bad append")); 381176472Skmacy } else { 382176472Skmacy while ((mbuf_wrs[count + 1] <= toep->tp_wr_avail) 383176472Skmacy && (tail != NULL) && (count < TX_MAX_SEGS-1)) { 384176472Skmacy bytes += tail->m_len; 385176472Skmacy last = tail; 386176472Skmacy count++; 387176472Skmacy /* 388176472Skmacy * technically an abuse to be using this for a VA 389176472Skmacy * but less gross than defining my own structure 390176472Skmacy * or calling pmap_kextract from here :-| 391176472Skmacy */ 392176472Skmacy segp->ds_addr = (bus_addr_t)tail->m_data; 393176472Skmacy segp->ds_len = tail->m_len; 394176472Skmacy DPRINTF("count=%d wr_needed=%d ds_addr=%p ds_len=%d\n", 395176472Skmacy count, mbuf_wrs[count], tail->m_data, tail->m_len); 396176472Skmacy segp++; 397176472Skmacy tail = tail->m_next; 398176472Skmacy } 399176472Skmacy DPRINTF("wr_avail=%d mbuf_wrs[%d]=%d tail=%p\n", 400176472Skmacy toep->tp_wr_avail, count, mbuf_wrs[count], tail); 401176472Skmacy 402176472Skmacy m_set_sgl(m0, segs); 403176472Skmacy m_set_sgllen(m0, count); 404176472Skmacy make_tx_data_wr(so, m0, bytes, tail); 405174641Skmacy } 406176472Skmacy m_set_priority(m0, mkprio(CPL_PRIORITY_DATA, toep)); 407176472Skmacy 408174641Skmacy if (tail) { 409178302Skmacy snd->sb_sndptr = tail; 410174641Skmacy toep->tp_m_last = NULL; 411174641Skmacy } else 412178302Skmacy toep->tp_m_last = snd->sb_sndptr = last; 413174641Skmacy 414176472Skmacy 415174641Skmacy DPRINTF("toep->tp_m_last=%p\n", toep->tp_m_last); 416174641Skmacy 417178302Skmacy snd->sb_sndptroff += bytes; 418174641Skmacy total_bytes += bytes; 419174641Skmacy toep->tp_write_seq += bytes; 420180644Skmacy CTR6(KTR_TOM, "t3_push_frames: wr_avail=%d mbuf_wrs[%d]=%d" 421180644Skmacy " tail=%p sndptr=%p sndptroff=%d", 422180644Skmacy toep->tp_wr_avail, count, mbuf_wrs[count], 423180644Skmacy tail, snd->sb_sndptr, snd->sb_sndptroff); 424176472Skmacy if (tail) 425180644Skmacy CTR4(KTR_TOM, "t3_push_frames: total_bytes=%d" 426180644Skmacy " tp_m_last=%p tailbuf=%p snd_una=0x%08x", 427180644Skmacy total_bytes, toep->tp_m_last, tail->m_data, 428180644Skmacy tp->snd_una); 429176472Skmacy else 430180644Skmacy CTR3(KTR_TOM, "t3_push_frames: total_bytes=%d" 431180644Skmacy " tp_m_last=%p snd_una=0x%08x", 432176472Skmacy total_bytes, toep->tp_m_last, tp->snd_una); 433174641Skmacy 434174641Skmacy 435178302Skmacy#ifdef KTR 436178302Skmacy{ 437178302Skmacy int i; 438178302Skmacy 439176472Skmacy i = 0; 440176472Skmacy while (i < count && m_get_sgllen(m0)) { 441176472Skmacy if ((count - i) >= 3) { 442176472Skmacy CTR6(KTR_TOM, 443180644Skmacy "t3_push_frames: pa=0x%zx len=%d pa=0x%zx" 444180644Skmacy " len=%d pa=0x%zx len=%d", 445180644Skmacy segs[i].ds_addr, segs[i].ds_len, 446180644Skmacy segs[i + 1].ds_addr, segs[i + 1].ds_len, 447176472Skmacy segs[i + 2].ds_addr, segs[i + 2].ds_len); 448176472Skmacy i += 3; 449176472Skmacy } else if ((count - i) == 2) { 450176472Skmacy CTR4(KTR_TOM, 451180644Skmacy "t3_push_frames: pa=0x%zx len=%d pa=0x%zx" 452180644Skmacy " len=%d", 453180644Skmacy segs[i].ds_addr, segs[i].ds_len, 454180644Skmacy segs[i + 1].ds_addr, segs[i + 1].ds_len); 455176472Skmacy i += 2; 456176472Skmacy } else { 457176472Skmacy CTR2(KTR_TOM, "t3_push_frames: pa=0x%zx len=%d", 458176472Skmacy segs[i].ds_addr, segs[i].ds_len); 459176472Skmacy i++; 460176472Skmacy } 461174641Skmacy 462176472Skmacy } 463178302Skmacy} 464178302Skmacy#endif 465176472Skmacy /* 466174641Skmacy * remember credits used 467174641Skmacy */ 468174641Skmacy m0->m_pkthdr.csum_data = mbuf_wrs[count]; 469174641Skmacy m0->m_pkthdr.len = bytes; 470176472Skmacy toep->tp_wr_avail -= mbuf_wrs[count]; 471176472Skmacy toep->tp_wr_unacked += mbuf_wrs[count]; 472176472Skmacy 473174641Skmacy if ((req_completion && toep->tp_wr_unacked == mbuf_wrs[count]) || 474174641Skmacy toep->tp_wr_unacked >= toep->tp_wr_max / 2) { 475174641Skmacy struct work_request_hdr *wr = cplhdr(m0); 476174641Skmacy 477174641Skmacy wr->wr_hi |= htonl(F_WR_COMPL); 478174641Skmacy toep->tp_wr_unacked = 0; 479174641Skmacy } 480176472Skmacy KASSERT((m0->m_pkthdr.csum_data > 0) && 481176472Skmacy (m0->m_pkthdr.csum_data <= 4), ("bad credit count %d", 482176472Skmacy m0->m_pkthdr.csum_data)); 483174641Skmacy m0->m_type = MT_DONTFREE; 484174641Skmacy enqueue_wr(toep, m0); 485174641Skmacy DPRINTF("sending offload tx with %d bytes in %d segments\n", 486174641Skmacy bytes, count); 487174641Skmacy l2t_send(cdev, m0, toep->tp_l2t); 488174641Skmacy } 489178302Skmacy sockbuf_unlock(snd); 490174641Skmacy return (total_bytes); 491174641Skmacy} 492174641Skmacy 493174641Skmacy/* 494174641Skmacy * Close a connection by sending a CPL_CLOSE_CON_REQ message. Cannot fail 495174641Skmacy * under any circumstances. We take the easy way out and always queue the 496174641Skmacy * message to the write_queue. We can optimize the case where the queue is 497174641Skmacy * already empty though the optimization is probably not worth it. 498174641Skmacy */ 499174641Skmacystatic void 500174641Skmacyclose_conn(struct socket *so) 501174641Skmacy{ 502174641Skmacy struct mbuf *m; 503174641Skmacy struct cpl_close_con_req *req; 504174641Skmacy struct tom_data *d; 505178302Skmacy struct inpcb *inp = so_sotoinpcb(so); 506174641Skmacy struct tcpcb *tp; 507174641Skmacy struct toepcb *toep; 508174641Skmacy unsigned int tid; 509174641Skmacy 510174641Skmacy 511177530Skmacy inp_wlock(inp); 512178302Skmacy tp = so_sototcpcb(so); 513174641Skmacy toep = tp->t_toe; 514174641Skmacy 515174641Skmacy if (tp->t_state != TCPS_SYN_SENT) 516174641Skmacy t3_push_frames(so, 1); 517174641Skmacy 518174641Skmacy if (toep->tp_flags & TP_FIN_SENT) { 519177530Skmacy inp_wunlock(inp); 520174641Skmacy return; 521174641Skmacy } 522174641Skmacy 523174641Skmacy tid = toep->tp_tid; 524174641Skmacy 525174641Skmacy d = TOM_DATA(toep->tp_toedev); 526174641Skmacy 527174641Skmacy m = m_gethdr_nofail(sizeof(*req)); 528178302Skmacy m_set_priority(m, CPL_PRIORITY_DATA); 529178302Skmacy m_set_sgl(m, NULL); 530178302Skmacy m_set_sgllen(m, 0); 531174641Skmacy 532174641Skmacy toep->tp_flags |= TP_FIN_SENT; 533174641Skmacy req = mtod(m, struct cpl_close_con_req *); 534174641Skmacy 535174641Skmacy req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_CLOSE_CON)); 536174641Skmacy req->wr.wr_lo = htonl(V_WR_TID(tid)); 537174641Skmacy OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_CON_REQ, tid)); 538178302Skmacy req->rsvd = 0; 539177530Skmacy inp_wunlock(inp); 540174641Skmacy /* 541174641Skmacy * XXX - need to defer shutdown while there is still data in the queue 542174641Skmacy * 543174641Skmacy */ 544178302Skmacy CTR4(KTR_TOM, "%s CLOSE_CON_REQ so %p tp %p tid=%u", __FUNCTION__, so, tp, tid); 545174641Skmacy cxgb_ofld_send(d->cdev, m); 546174641Skmacy 547174641Skmacy} 548174641Skmacy 549174641Skmacy/* 550174641Skmacy * Handle an ARP failure for a CPL_ABORT_REQ. Change it into a no RST variant 551174641Skmacy * and send it along. 552174641Skmacy */ 553174641Skmacystatic void 554174641Skmacyabort_arp_failure(struct t3cdev *cdev, struct mbuf *m) 555174641Skmacy{ 556174641Skmacy struct cpl_abort_req *req = cplhdr(m); 557174641Skmacy 558174641Skmacy req->cmd = CPL_ABORT_NO_RST; 559174641Skmacy cxgb_ofld_send(cdev, m); 560174641Skmacy} 561174641Skmacy 562174641Skmacy/* 563174641Skmacy * Send RX credits through an RX_DATA_ACK CPL message. If nofail is 0 we are 564174641Skmacy * permitted to return without sending the message in case we cannot allocate 565174641Skmacy * an sk_buff. Returns the number of credits sent. 566174641Skmacy */ 567174641Skmacyuint32_t 568174641Skmacyt3_send_rx_credits(struct tcpcb *tp, uint32_t credits, uint32_t dack, int nofail) 569174641Skmacy{ 570174641Skmacy struct mbuf *m; 571174641Skmacy struct cpl_rx_data_ack *req; 572174641Skmacy struct toepcb *toep = tp->t_toe; 573174641Skmacy struct toedev *tdev = toep->tp_toedev; 574174641Skmacy 575174641Skmacy m = m_gethdr_nofail(sizeof(*req)); 576174641Skmacy 577174641Skmacy DPRINTF("returning %u credits to HW\n", credits); 578174641Skmacy 579174641Skmacy req = mtod(m, struct cpl_rx_data_ack *); 580174641Skmacy req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 581176472Skmacy req->wr.wr_lo = 0; 582174641Skmacy OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RX_DATA_ACK, toep->tp_tid)); 583174641Skmacy req->credit_dack = htonl(dack | V_RX_CREDITS(credits)); 584176472Skmacy m_set_priority(m, mkprio(CPL_PRIORITY_ACK, toep)); 585174641Skmacy cxgb_ofld_send(TOM_DATA(tdev)->cdev, m); 586174641Skmacy return (credits); 587174641Skmacy} 588174641Skmacy 589176472Skmacy/* 590176472Skmacy * Send RX_DATA_ACK CPL message to request a modulation timer to be scheduled. 591176472Skmacy * This is only used in DDP mode, so we take the opportunity to also set the 592176472Skmacy * DACK mode and flush any Rx credits. 593176472Skmacy */ 594176472Skmacyvoid 595176472Skmacyt3_send_rx_modulate(struct toepcb *toep) 596176472Skmacy{ 597176472Skmacy struct mbuf *m; 598176472Skmacy struct cpl_rx_data_ack *req; 599174641Skmacy 600176472Skmacy m = m_gethdr_nofail(sizeof(*req)); 601176472Skmacy 602176472Skmacy req = mtod(m, struct cpl_rx_data_ack *); 603176472Skmacy req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 604176472Skmacy req->wr.wr_lo = 0; 605176472Skmacy m->m_pkthdr.len = m->m_len = sizeof(*req); 606176472Skmacy 607176472Skmacy OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RX_DATA_ACK, toep->tp_tid)); 608176472Skmacy req->credit_dack = htonl(F_RX_MODULATE | F_RX_DACK_CHANGE | 609176472Skmacy V_RX_DACK_MODE(1) | 610176472Skmacy V_RX_CREDITS(toep->tp_copied_seq - toep->tp_rcv_wup)); 611176472Skmacy m_set_priority(m, mkprio(CPL_PRIORITY_CONTROL, toep)); 612176472Skmacy cxgb_ofld_send(TOEP_T3C_DEV(toep), m); 613176472Skmacy toep->tp_rcv_wup = toep->tp_copied_seq; 614176472Skmacy} 615176472Skmacy 616174641Skmacy/* 617176472Skmacy * Handle receipt of an urgent pointer. 618176472Skmacy */ 619176472Skmacystatic void 620176472Skmacyhandle_urg_ptr(struct socket *so, uint32_t urg_seq) 621176472Skmacy{ 622176472Skmacy#ifdef URGENT_DATA_SUPPORTED 623178302Skmacy struct tcpcb *tp = so_sototcpcb(so); 624176472Skmacy 625176472Skmacy urg_seq--; /* initially points past the urgent data, per BSD */ 626176472Skmacy 627176472Skmacy if (tp->urg_data && !after(urg_seq, tp->urg_seq)) 628176472Skmacy return; /* duplicate pointer */ 629176472Skmacy sk_send_sigurg(sk); 630176472Skmacy if (tp->urg_seq == tp->copied_seq && tp->urg_data && 631176472Skmacy !sock_flag(sk, SOCK_URGINLINE) && tp->copied_seq != tp->rcv_nxt) { 632176472Skmacy struct sk_buff *skb = skb_peek(&sk->sk_receive_queue); 633176472Skmacy 634176472Skmacy tp->copied_seq++; 635176472Skmacy if (skb && tp->copied_seq - TCP_SKB_CB(skb)->seq >= skb->len) 636176472Skmacy tom_eat_skb(sk, skb, 0); 637176472Skmacy } 638176472Skmacy tp->urg_data = TCP_URG_NOTYET; 639176472Skmacy tp->urg_seq = urg_seq; 640176472Skmacy#endif 641176472Skmacy} 642176472Skmacy 643176472Skmacy/* 644176472Skmacy * Returns true if a socket cannot accept new Rx data. 645176472Skmacy */ 646176472Skmacystatic inline int 647176472Skmacyso_no_receive(const struct socket *so) 648176472Skmacy{ 649178302Skmacy return (so_state_get(so) & (SS_ISDISCONNECTED|SS_ISDISCONNECTING)); 650176472Skmacy} 651176472Skmacy 652176472Skmacy/* 653176472Skmacy * Process an urgent data notification. 654176472Skmacy */ 655176472Skmacystatic void 656176472Skmacyrx_urg_notify(struct toepcb *toep, struct mbuf *m) 657176472Skmacy{ 658176472Skmacy struct cpl_rx_urg_notify *hdr = cplhdr(m); 659178302Skmacy struct socket *so = inp_inpcbtosocket(toep->tp_tp->t_inpcb); 660176472Skmacy 661176472Skmacy VALIDATE_SOCK(so); 662176472Skmacy 663176472Skmacy if (!so_no_receive(so)) 664176472Skmacy handle_urg_ptr(so, ntohl(hdr->seq)); 665176472Skmacy 666176472Skmacy m_freem(m); 667176472Skmacy} 668176472Skmacy 669176472Skmacy/* 670176472Skmacy * Handler for RX_URG_NOTIFY CPL messages. 671176472Skmacy */ 672176472Skmacystatic int 673176472Skmacydo_rx_urg_notify(struct t3cdev *cdev, struct mbuf *m, void *ctx) 674176472Skmacy{ 675176472Skmacy struct toepcb *toep = (struct toepcb *)ctx; 676176472Skmacy 677176472Skmacy rx_urg_notify(toep, m); 678176472Skmacy return (0); 679176472Skmacy} 680176472Skmacy 681177340Skmacystatic __inline int 682177340Skmacyis_delack_mode_valid(struct toedev *dev, struct toepcb *toep) 683177340Skmacy{ 684177340Skmacy return (toep->tp_ulp_mode || 685177340Skmacy (toep->tp_ulp_mode == ULP_MODE_TCPDDP && 686177340Skmacy dev->tod_ttid >= TOE_ID_CHELSIO_T3)); 687177340Skmacy} 688177340Skmacy 689176472Skmacy/* 690174641Skmacy * Set of states for which we should return RX credits. 691174641Skmacy */ 692174641Skmacy#define CREDIT_RETURN_STATE (TCPF_ESTABLISHED | TCPF_FIN_WAIT1 | TCPF_FIN_WAIT2) 693174641Skmacy 694174641Skmacy/* 695174641Skmacy * Called after some received data has been read. It returns RX credits 696174641Skmacy * to the HW for the amount of data processed. 697174641Skmacy */ 698174641Skmacyvoid 699176472Skmacyt3_cleanup_rbuf(struct tcpcb *tp, int copied) 700174641Skmacy{ 701174641Skmacy struct toepcb *toep = tp->t_toe; 702174641Skmacy struct socket *so; 703174641Skmacy struct toedev *dev; 704174641Skmacy int dack_mode, must_send, read; 705174641Skmacy u32 thres, credits, dack = 0; 706178302Skmacy struct sockbuf *rcv; 707178302Skmacy 708178302Skmacy so = inp_inpcbtosocket(tp->t_inpcb); 709178302Skmacy rcv = so_sockbuf_rcv(so); 710174641Skmacy 711174641Skmacy if (!((tp->t_state == TCPS_ESTABLISHED) || (tp->t_state == TCPS_FIN_WAIT_1) || 712176472Skmacy (tp->t_state == TCPS_FIN_WAIT_2))) { 713176472Skmacy if (copied) { 714178302Skmacy sockbuf_lock(rcv); 715176472Skmacy toep->tp_copied_seq += copied; 716178302Skmacy sockbuf_unlock(rcv); 717176472Skmacy } 718176472Skmacy 719174641Skmacy return; 720176472Skmacy } 721174641Skmacy 722178302Skmacy inp_lock_assert(tp->t_inpcb); 723178302Skmacy 724178302Skmacy sockbuf_lock(rcv); 725176472Skmacy if (copied) 726176472Skmacy toep->tp_copied_seq += copied; 727176472Skmacy else { 728178302Skmacy read = toep->tp_enqueued_bytes - rcv->sb_cc; 729176472Skmacy toep->tp_copied_seq += read; 730176472Skmacy } 731174641Skmacy credits = toep->tp_copied_seq - toep->tp_rcv_wup; 732178302Skmacy toep->tp_enqueued_bytes = rcv->sb_cc; 733178302Skmacy sockbuf_unlock(rcv); 734174641Skmacy 735178302Skmacy if (credits > rcv->sb_mbmax) { 736178302Skmacy log(LOG_ERR, "copied_seq=%u rcv_wup=%u credits=%u\n", 737178302Skmacy toep->tp_copied_seq, toep->tp_rcv_wup, credits); 738178302Skmacy credits = rcv->sb_mbmax; 739176472Skmacy } 740176472Skmacy 741176472Skmacy 742178302Skmacy /* 743174641Skmacy * XXX this won't accurately reflect credit return - we need 744174641Skmacy * to look at the difference between the amount that has been 745174641Skmacy * put in the recv sockbuf and what is there now 746174641Skmacy */ 747174641Skmacy 748174641Skmacy if (__predict_false(!credits)) 749174641Skmacy return; 750174641Skmacy 751174641Skmacy dev = toep->tp_toedev; 752174641Skmacy thres = TOM_TUNABLE(dev, rx_credit_thres); 753174641Skmacy 754174641Skmacy if (__predict_false(thres == 0)) 755174641Skmacy return; 756174641Skmacy 757177340Skmacy if (is_delack_mode_valid(dev, toep)) { 758174641Skmacy dack_mode = TOM_TUNABLE(dev, delack); 759174641Skmacy if (__predict_false(dack_mode != toep->tp_delack_mode)) { 760174641Skmacy u32 r = tp->rcv_nxt - toep->tp_delack_seq; 761174641Skmacy 762174641Skmacy if (r >= tp->rcv_wnd || r >= 16 * toep->tp_mss_clamp) 763174641Skmacy dack = F_RX_DACK_CHANGE | 764174641Skmacy V_RX_DACK_MODE(dack_mode); 765174641Skmacy } 766177340Skmacy } else 767177340Skmacy dack = F_RX_DACK_CHANGE | V_RX_DACK_MODE(1); 768177340Skmacy 769174641Skmacy /* 770174641Skmacy * For coalescing to work effectively ensure the receive window has 771174641Skmacy * at least 16KB left. 772174641Skmacy */ 773174641Skmacy must_send = credits + 16384 >= tp->rcv_wnd; 774174641Skmacy 775174641Skmacy if (must_send || credits >= thres) 776174641Skmacy toep->tp_rcv_wup += t3_send_rx_credits(tp, credits, dack, must_send); 777174641Skmacy} 778174641Skmacy 779174641Skmacystatic int 780174641Skmacycxgb_toe_disconnect(struct tcpcb *tp) 781174641Skmacy{ 782174641Skmacy struct socket *so; 783174641Skmacy 784174641Skmacy DPRINTF("cxgb_toe_disconnect\n"); 785174641Skmacy 786178302Skmacy so = inp_inpcbtosocket(tp->t_inpcb); 787174641Skmacy close_conn(so); 788174641Skmacy return (0); 789174641Skmacy} 790174641Skmacy 791174641Skmacystatic int 792174708Skmacycxgb_toe_reset(struct tcpcb *tp) 793174641Skmacy{ 794174641Skmacy struct toepcb *toep = tp->t_toe; 795177530Skmacy 796174641Skmacy t3_send_reset(toep); 797174641Skmacy 798174641Skmacy /* 799174641Skmacy * unhook from socket 800174641Skmacy */ 801174641Skmacy tp->t_flags &= ~TF_TOE; 802174641Skmacy toep->tp_tp = NULL; 803174641Skmacy tp->t_toe = NULL; 804174641Skmacy return (0); 805174641Skmacy} 806174641Skmacy 807174641Skmacystatic int 808174641Skmacycxgb_toe_send(struct tcpcb *tp) 809174641Skmacy{ 810174641Skmacy struct socket *so; 811174641Skmacy 812174641Skmacy DPRINTF("cxgb_toe_send\n"); 813174641Skmacy dump_toepcb(tp->t_toe); 814174641Skmacy 815178302Skmacy so = inp_inpcbtosocket(tp->t_inpcb); 816174641Skmacy t3_push_frames(so, 1); 817174641Skmacy return (0); 818174641Skmacy} 819174641Skmacy 820174641Skmacystatic int 821174641Skmacycxgb_toe_rcvd(struct tcpcb *tp) 822174641Skmacy{ 823177530Skmacy 824177575Skmacy inp_lock_assert(tp->t_inpcb); 825178302Skmacy 826176472Skmacy t3_cleanup_rbuf(tp, 0); 827174641Skmacy 828174641Skmacy return (0); 829174641Skmacy} 830174641Skmacy 831174641Skmacystatic void 832174641Skmacycxgb_toe_detach(struct tcpcb *tp) 833174641Skmacy{ 834174641Skmacy struct toepcb *toep; 835177530Skmacy 836177530Skmacy /* 837174641Skmacy * XXX how do we handle teardown in the SYN_SENT state? 838174641Skmacy * 839174641Skmacy */ 840177575Skmacy inp_lock_assert(tp->t_inpcb); 841174641Skmacy toep = tp->t_toe; 842174641Skmacy toep->tp_tp = NULL; 843174641Skmacy 844174641Skmacy /* 845174641Skmacy * unhook from socket 846174641Skmacy */ 847174641Skmacy tp->t_flags &= ~TF_TOE; 848174641Skmacy tp->t_toe = NULL; 849174641Skmacy} 850174641Skmacy 851174641Skmacy 852174641Skmacystatic struct toe_usrreqs cxgb_toe_usrreqs = { 853174641Skmacy .tu_disconnect = cxgb_toe_disconnect, 854174708Skmacy .tu_reset = cxgb_toe_reset, 855174641Skmacy .tu_send = cxgb_toe_send, 856174641Skmacy .tu_rcvd = cxgb_toe_rcvd, 857174641Skmacy .tu_detach = cxgb_toe_detach, 858174641Skmacy .tu_detach = cxgb_toe_detach, 859174641Skmacy .tu_syncache_event = handle_syncache_event, 860174641Skmacy}; 861174641Skmacy 862174641Skmacy 863174641Skmacystatic void 864176472Skmacy__set_tcb_field(struct toepcb *toep, struct mbuf *m, uint16_t word, 865174641Skmacy uint64_t mask, uint64_t val, int no_reply) 866174641Skmacy{ 867174641Skmacy struct cpl_set_tcb_field *req; 868174641Skmacy 869176472Skmacy CTR4(KTR_TCB, "__set_tcb_field_ulp(tid=%u word=0x%x mask=%jx val=%jx", 870176472Skmacy toep->tp_tid, word, mask, val); 871176472Skmacy 872174641Skmacy req = mtod(m, struct cpl_set_tcb_field *); 873174641Skmacy m->m_pkthdr.len = m->m_len = sizeof(*req); 874174641Skmacy req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 875176472Skmacy req->wr.wr_lo = 0; 876174641Skmacy OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, toep->tp_tid)); 877174641Skmacy req->reply = V_NO_REPLY(no_reply); 878174641Skmacy req->cpu_idx = 0; 879174641Skmacy req->word = htons(word); 880174641Skmacy req->mask = htobe64(mask); 881174641Skmacy req->val = htobe64(val); 882174641Skmacy 883176472Skmacy m_set_priority(m, mkprio(CPL_PRIORITY_CONTROL, toep)); 884176472Skmacy send_or_defer(toep, m, 0); 885174641Skmacy} 886174641Skmacy 887174641Skmacystatic void 888178302Skmacyt3_set_tcb_field(struct toepcb *toep, uint16_t word, uint64_t mask, uint64_t val) 889174641Skmacy{ 890174641Skmacy struct mbuf *m; 891178302Skmacy struct tcpcb *tp = toep->tp_tp; 892174641Skmacy 893174641Skmacy if (toep == NULL) 894174641Skmacy return; 895176472Skmacy 896176472Skmacy if (tp->t_state == TCPS_CLOSED || (toep->tp_flags & TP_ABORT_SHUTDOWN)) { 897176472Skmacy printf("not seting field\n"); 898174641Skmacy return; 899176472Skmacy } 900176472Skmacy 901174641Skmacy m = m_gethdr_nofail(sizeof(struct cpl_set_tcb_field)); 902174641Skmacy 903176472Skmacy __set_tcb_field(toep, m, word, mask, val, 1); 904174641Skmacy} 905174641Skmacy 906174641Skmacy/* 907174641Skmacy * Set one of the t_flags bits in the TCB. 908174641Skmacy */ 909174641Skmacystatic void 910178302Skmacyset_tcb_tflag(struct toepcb *toep, unsigned int bit_pos, int val) 911174641Skmacy{ 912178302Skmacy 913178302Skmacy t3_set_tcb_field(toep, W_TCB_T_FLAGS1, 1ULL << bit_pos, val << bit_pos); 914174641Skmacy} 915174641Skmacy 916174641Skmacy/* 917174641Skmacy * Send a SET_TCB_FIELD CPL message to change a connection's Nagle setting. 918174641Skmacy */ 919174641Skmacystatic void 920178302Skmacyt3_set_nagle(struct toepcb *toep) 921174641Skmacy{ 922178302Skmacy struct tcpcb *tp = toep->tp_tp; 923174641Skmacy 924178302Skmacy set_tcb_tflag(toep, S_TF_NAGLE, !(tp->t_flags & TF_NODELAY)); 925174641Skmacy} 926174641Skmacy 927174641Skmacy/* 928174641Skmacy * Send a SET_TCB_FIELD CPL message to change a connection's keepalive setting. 929174641Skmacy */ 930174641Skmacyvoid 931178302Skmacyt3_set_keepalive(struct toepcb *toep, int on_off) 932174641Skmacy{ 933178302Skmacy 934178302Skmacy set_tcb_tflag(toep, S_TF_KEEPALIVE, on_off); 935174641Skmacy} 936174641Skmacy 937174641Skmacyvoid 938178302Skmacyt3_set_rcv_coalesce_enable(struct toepcb *toep, int on_off) 939174641Skmacy{ 940178302Skmacy set_tcb_tflag(toep, S_TF_RCV_COALESCE_ENABLE, on_off); 941174641Skmacy} 942174641Skmacy 943177340Skmacyvoid 944178302Skmacyt3_set_dack_mss(struct toepcb *toep, int on_off) 945177340Skmacy{ 946178302Skmacy 947178302Skmacy set_tcb_tflag(toep, S_TF_DACK_MSS, on_off); 948177340Skmacy} 949177340Skmacy 950174641Skmacy/* 951174641Skmacy * Send a SET_TCB_FIELD CPL message to change a connection's TOS setting. 952174641Skmacy */ 953174641Skmacystatic void 954178302Skmacyt3_set_tos(struct toepcb *toep) 955174641Skmacy{ 956178302Skmacy int tos = inp_ip_tos_get(toep->tp_tp->t_inpcb); 957178302Skmacy 958178302Skmacy t3_set_tcb_field(toep, W_TCB_TOS, V_TCB_TOS(M_TCB_TOS), 959178302Skmacy V_TCB_TOS(tos)); 960174641Skmacy} 961174641Skmacy 962174641Skmacy 963174641Skmacy/* 964174641Skmacy * In DDP mode, TP fails to schedule a timer to push RX data to the host when 965174641Skmacy * DDP is disabled (data is delivered to freelist). [Note that, the peer should 966174641Skmacy * set the PSH bit in the last segment, which would trigger delivery.] 967174641Skmacy * We work around the issue by setting a DDP buffer in a partial placed state, 968174641Skmacy * which guarantees that TP will schedule a timer. 969174641Skmacy */ 970174641Skmacy#define TP_DDP_TIMER_WORKAROUND_MASK\ 971174641Skmacy (V_TF_DDP_BUF0_VALID(1) | V_TF_DDP_ACTIVE_BUF(1) |\ 972174641Skmacy ((V_TCB_RX_DDP_BUF0_OFFSET(M_TCB_RX_DDP_BUF0_OFFSET) |\ 973174641Skmacy V_TCB_RX_DDP_BUF0_LEN(3)) << 32)) 974174641Skmacy#define TP_DDP_TIMER_WORKAROUND_VAL\ 975174641Skmacy (V_TF_DDP_BUF0_VALID(1) | V_TF_DDP_ACTIVE_BUF(0) |\ 976174641Skmacy ((V_TCB_RX_DDP_BUF0_OFFSET((uint64_t)1) | V_TCB_RX_DDP_BUF0_LEN((uint64_t)2)) <<\ 977174641Skmacy 32)) 978174641Skmacy 979174641Skmacystatic void 980178302Skmacyt3_enable_ddp(struct toepcb *toep, int on) 981174641Skmacy{ 982176472Skmacy if (on) { 983176472Skmacy 984178302Skmacy t3_set_tcb_field(toep, W_TCB_RX_DDP_FLAGS, V_TF_DDP_OFF(1), 985174641Skmacy V_TF_DDP_OFF(0)); 986176472Skmacy } else 987178302Skmacy t3_set_tcb_field(toep, W_TCB_RX_DDP_FLAGS, 988174641Skmacy V_TF_DDP_OFF(1) | 989174641Skmacy TP_DDP_TIMER_WORKAROUND_MASK, 990174641Skmacy V_TF_DDP_OFF(1) | 991174641Skmacy TP_DDP_TIMER_WORKAROUND_VAL); 992174641Skmacy 993174641Skmacy} 994174641Skmacy 995174641Skmacyvoid 996178302Skmacyt3_set_ddp_tag(struct toepcb *toep, int buf_idx, unsigned int tag_color) 997174641Skmacy{ 998178302Skmacy t3_set_tcb_field(toep, W_TCB_RX_DDP_BUF0_TAG + buf_idx, 999174641Skmacy V_TCB_RX_DDP_BUF0_TAG(M_TCB_RX_DDP_BUF0_TAG), 1000174641Skmacy tag_color); 1001174641Skmacy} 1002174641Skmacy 1003174641Skmacyvoid 1004178302Skmacyt3_set_ddp_buf(struct toepcb *toep, int buf_idx, unsigned int offset, 1005174641Skmacy unsigned int len) 1006174641Skmacy{ 1007174641Skmacy if (buf_idx == 0) 1008178302Skmacy t3_set_tcb_field(toep, W_TCB_RX_DDP_BUF0_OFFSET, 1009174641Skmacy V_TCB_RX_DDP_BUF0_OFFSET(M_TCB_RX_DDP_BUF0_OFFSET) | 1010174641Skmacy V_TCB_RX_DDP_BUF0_LEN(M_TCB_RX_DDP_BUF0_LEN), 1011174641Skmacy V_TCB_RX_DDP_BUF0_OFFSET((uint64_t)offset) | 1012174641Skmacy V_TCB_RX_DDP_BUF0_LEN((uint64_t)len)); 1013174641Skmacy else 1014178302Skmacy t3_set_tcb_field(toep, W_TCB_RX_DDP_BUF1_OFFSET, 1015174641Skmacy V_TCB_RX_DDP_BUF1_OFFSET(M_TCB_RX_DDP_BUF1_OFFSET) | 1016174641Skmacy V_TCB_RX_DDP_BUF1_LEN(M_TCB_RX_DDP_BUF1_LEN << 32), 1017174641Skmacy V_TCB_RX_DDP_BUF1_OFFSET((uint64_t)offset) | 1018174641Skmacy V_TCB_RX_DDP_BUF1_LEN(((uint64_t)len) << 32)); 1019174641Skmacy} 1020174641Skmacy 1021174641Skmacystatic int 1022174641Skmacyt3_set_cong_control(struct socket *so, const char *name) 1023174641Skmacy{ 1024176472Skmacy#ifdef CONGESTION_CONTROL_SUPPORTED 1025174641Skmacy int cong_algo; 1026174641Skmacy 1027174641Skmacy for (cong_algo = 0; cong_algo < ARRAY_SIZE(t3_cong_ops); cong_algo++) 1028174641Skmacy if (!strcmp(name, t3_cong_ops[cong_algo].name)) 1029174641Skmacy break; 1030174641Skmacy 1031174641Skmacy if (cong_algo >= ARRAY_SIZE(t3_cong_ops)) 1032174641Skmacy return -EINVAL; 1033174641Skmacy#endif 1034174641Skmacy return 0; 1035174641Skmacy} 1036174641Skmacy 1037174641Skmacyint 1038178302Skmacyt3_get_tcb(struct toepcb *toep) 1039174641Skmacy{ 1040174641Skmacy struct cpl_get_tcb *req; 1041178302Skmacy struct tcpcb *tp = toep->tp_tp; 1042174641Skmacy struct mbuf *m = m_gethdr(M_NOWAIT, MT_DATA); 1043174641Skmacy 1044174641Skmacy if (!m) 1045174641Skmacy return (ENOMEM); 1046174641Skmacy 1047177575Skmacy inp_lock_assert(tp->t_inpcb); 1048176472Skmacy m_set_priority(m, mkprio(CPL_PRIORITY_CONTROL, toep)); 1049174641Skmacy req = mtod(m, struct cpl_get_tcb *); 1050174641Skmacy m->m_pkthdr.len = m->m_len = sizeof(*req); 1051174641Skmacy req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 1052176472Skmacy req->wr.wr_lo = 0; 1053174641Skmacy OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_GET_TCB, toep->tp_tid)); 1054174641Skmacy req->cpuno = htons(toep->tp_qset); 1055176472Skmacy req->rsvd = 0; 1056178302Skmacy if (tp->t_state == TCPS_SYN_SENT) 1057174641Skmacy mbufq_tail(&toep->out_of_order_queue, m); // defer 1058174641Skmacy else 1059178302Skmacy cxgb_ofld_send(TOEP_T3C_DEV(toep), m); 1060174641Skmacy return 0; 1061174641Skmacy} 1062174641Skmacy 1063174641Skmacystatic inline void 1064178302Skmacyso_insert_tid(struct tom_data *d, struct toepcb *toep, unsigned int tid) 1065174641Skmacy{ 1066178302Skmacy 1067174641Skmacy toepcb_hold(toep); 1068174641Skmacy 1069174641Skmacy cxgb_insert_tid(d->cdev, d->client, toep, tid); 1070174641Skmacy} 1071174641Skmacy 1072174641Skmacy/** 1073174641Skmacy * find_best_mtu - find the entry in the MTU table closest to an MTU 1074174641Skmacy * @d: TOM state 1075174641Skmacy * @mtu: the target MTU 1076174641Skmacy * 1077174641Skmacy * Returns the index of the value in the MTU table that is closest to but 1078174641Skmacy * does not exceed the target MTU. 1079174641Skmacy */ 1080174641Skmacystatic unsigned int 1081174641Skmacyfind_best_mtu(const struct t3c_data *d, unsigned short mtu) 1082174641Skmacy{ 1083174641Skmacy int i = 0; 1084174641Skmacy 1085174641Skmacy while (i < d->nmtus - 1 && d->mtus[i + 1] <= mtu) 1086174641Skmacy ++i; 1087174641Skmacy return (i); 1088174641Skmacy} 1089174641Skmacy 1090174641Skmacystatic unsigned int 1091174641Skmacyselect_mss(struct t3c_data *td, struct tcpcb *tp, unsigned int pmtu) 1092174641Skmacy{ 1093174641Skmacy unsigned int idx; 1094174641Skmacy 1095174641Skmacy#ifdef notyet 1096178302Skmacy struct rtentry *dst = so_sotoinpcb(so)->inp_route.ro_rt; 1097174641Skmacy#endif 1098174641Skmacy if (tp) { 1099174641Skmacy tp->t_maxseg = pmtu - 40; 1100174641Skmacy if (tp->t_maxseg < td->mtus[0] - 40) 1101174641Skmacy tp->t_maxseg = td->mtus[0] - 40; 1102174641Skmacy idx = find_best_mtu(td, tp->t_maxseg + 40); 1103174641Skmacy 1104174641Skmacy tp->t_maxseg = td->mtus[idx] - 40; 1105174641Skmacy } else 1106174641Skmacy idx = find_best_mtu(td, pmtu); 1107174641Skmacy 1108174641Skmacy return (idx); 1109174641Skmacy} 1110174641Skmacy 1111174641Skmacystatic inline void 1112174641Skmacyfree_atid(struct t3cdev *cdev, unsigned int tid) 1113174641Skmacy{ 1114174641Skmacy struct toepcb *toep = cxgb_free_atid(cdev, tid); 1115174641Skmacy 1116174641Skmacy if (toep) 1117174641Skmacy toepcb_release(toep); 1118174641Skmacy} 1119174641Skmacy 1120174641Skmacy/* 1121174641Skmacy * Release resources held by an offload connection (TID, L2T entry, etc.) 1122174641Skmacy */ 1123174641Skmacystatic void 1124174641Skmacyt3_release_offload_resources(struct toepcb *toep) 1125174641Skmacy{ 1126174641Skmacy struct tcpcb *tp = toep->tp_tp; 1127174641Skmacy struct toedev *tdev = toep->tp_toedev; 1128174641Skmacy struct t3cdev *cdev; 1129178302Skmacy struct socket *so; 1130174641Skmacy unsigned int tid = toep->tp_tid; 1131178302Skmacy struct sockbuf *rcv; 1132178302Skmacy 1133178302Skmacy CTR0(KTR_TOM, "t3_release_offload_resources"); 1134174641Skmacy 1135174641Skmacy if (!tdev) 1136174641Skmacy return; 1137174641Skmacy 1138174641Skmacy cdev = TOEP_T3C_DEV(toep); 1139174641Skmacy if (!cdev) 1140174641Skmacy return; 1141174641Skmacy 1142174641Skmacy toep->tp_qset = 0; 1143174641Skmacy t3_release_ddp_resources(toep); 1144174641Skmacy 1145174641Skmacy#ifdef CTRL_SKB_CACHE 1146174641Skmacy kfree_skb(CTRL_SKB_CACHE(tp)); 1147174641Skmacy CTRL_SKB_CACHE(tp) = NULL; 1148174641Skmacy#endif 1149174641Skmacy 1150174641Skmacy if (toep->tp_wr_avail != toep->tp_wr_max) { 1151174641Skmacy purge_wr_queue(toep); 1152174641Skmacy reset_wr_list(toep); 1153174641Skmacy } 1154174641Skmacy 1155174641Skmacy if (toep->tp_l2t) { 1156174641Skmacy l2t_release(L2DATA(cdev), toep->tp_l2t); 1157174641Skmacy toep->tp_l2t = NULL; 1158174641Skmacy } 1159174641Skmacy toep->tp_tp = NULL; 1160174641Skmacy if (tp) { 1161177575Skmacy inp_lock_assert(tp->t_inpcb); 1162178302Skmacy so = inp_inpcbtosocket(tp->t_inpcb); 1163178302Skmacy rcv = so_sockbuf_rcv(so); 1164178302Skmacy /* 1165178302Skmacy * cancel any offloaded reads 1166178302Skmacy * 1167178302Skmacy */ 1168178302Skmacy sockbuf_lock(rcv); 1169174641Skmacy tp->t_toe = NULL; 1170174641Skmacy tp->t_flags &= ~TF_TOE; 1171178302Skmacy if (toep->tp_ddp_state.user_ddp_pending) { 1172178302Skmacy t3_cancel_ubuf(toep, rcv); 1173178302Skmacy toep->tp_ddp_state.user_ddp_pending = 0; 1174178302Skmacy } 1175178302Skmacy so_sorwakeup_locked(so); 1176178302Skmacy 1177174641Skmacy } 1178174641Skmacy 1179174641Skmacy if (toep->tp_state == TCPS_SYN_SENT) { 1180174641Skmacy free_atid(cdev, tid); 1181174641Skmacy#ifdef notyet 1182174641Skmacy __skb_queue_purge(&tp->out_of_order_queue); 1183174641Skmacy#endif 1184174641Skmacy } else { // we have TID 1185174641Skmacy cxgb_remove_tid(cdev, toep, tid); 1186174641Skmacy toepcb_release(toep); 1187174641Skmacy } 1188174641Skmacy#if 0 1189174641Skmacy log(LOG_INFO, "closing TID %u, state %u\n", tid, tp->t_state); 1190174641Skmacy#endif 1191174641Skmacy} 1192174641Skmacy 1193174641Skmacystatic void 1194174641Skmacyinstall_offload_ops(struct socket *so) 1195174641Skmacy{ 1196178302Skmacy struct tcpcb *tp = so_sototcpcb(so); 1197174641Skmacy 1198174641Skmacy KASSERT(tp->t_toe != NULL, ("toepcb not set")); 1199174641Skmacy 1200174641Skmacy t3_install_socket_ops(so); 1201174641Skmacy tp->t_flags |= TF_TOE; 1202174641Skmacy tp->t_tu = &cxgb_toe_usrreqs; 1203174641Skmacy} 1204174641Skmacy 1205174641Skmacy/* 1206174641Skmacy * Determine the receive window scaling factor given a target max 1207174641Skmacy * receive window. 1208174641Skmacy */ 1209174641Skmacystatic __inline int 1210174641Skmacyselect_rcv_wscale(int space) 1211174641Skmacy{ 1212174641Skmacy int wscale = 0; 1213174641Skmacy 1214174641Skmacy if (space > MAX_RCV_WND) 1215174641Skmacy space = MAX_RCV_WND; 1216174641Skmacy 1217174641Skmacy if (tcp_do_rfc1323) 1218174641Skmacy for (; space > 65535 && wscale < 14; space >>= 1, ++wscale) ; 1219176472Skmacy 1220176472Skmacy return (wscale); 1221174641Skmacy} 1222174641Skmacy 1223174641Skmacy/* 1224174641Skmacy * Determine the receive window size for a socket. 1225174641Skmacy */ 1226176472Skmacystatic unsigned long 1227176472Skmacyselect_rcv_wnd(struct toedev *dev, struct socket *so) 1228174641Skmacy{ 1229174641Skmacy struct tom_data *d = TOM_DATA(dev); 1230174641Skmacy unsigned int wnd; 1231174641Skmacy unsigned int max_rcv_wnd; 1232178302Skmacy struct sockbuf *rcv; 1233174641Skmacy 1234178302Skmacy rcv = so_sockbuf_rcv(so); 1235178302Skmacy 1236174641Skmacy if (tcp_do_autorcvbuf) 1237174641Skmacy wnd = tcp_autorcvbuf_max; 1238174641Skmacy else 1239178302Skmacy wnd = rcv->sb_hiwat; 1240176472Skmacy 1241174641Skmacy 1242176472Skmacy 1243174641Skmacy /* XXX 1244174641Skmacy * For receive coalescing to work effectively we need a receive window 1245174641Skmacy * that can accomodate a coalesced segment. 1246174641Skmacy */ 1247174641Skmacy if (wnd < MIN_RCV_WND) 1248174641Skmacy wnd = MIN_RCV_WND; 1249174641Skmacy 1250174641Skmacy /* PR 5138 */ 1251176472Skmacy max_rcv_wnd = (dev->tod_ttid < TOE_ID_CHELSIO_T3C ? 1252174641Skmacy (uint32_t)d->rx_page_size * 23 : 1253174641Skmacy MAX_RCV_WND); 1254174641Skmacy 1255174641Skmacy return min(wnd, max_rcv_wnd); 1256174641Skmacy} 1257174641Skmacy 1258174641Skmacy/* 1259174641Skmacy * Assign offload parameters to some socket fields. This code is used by 1260174641Skmacy * both active and passive opens. 1261174641Skmacy */ 1262174641Skmacystatic inline void 1263174641Skmacyinit_offload_socket(struct socket *so, struct toedev *dev, unsigned int tid, 1264174641Skmacy struct l2t_entry *e, struct rtentry *dst, struct toepcb *toep) 1265174641Skmacy{ 1266178302Skmacy struct tcpcb *tp = so_sototcpcb(so); 1267174641Skmacy struct t3c_data *td = T3C_DATA(TOM_DATA(dev)->cdev); 1268178302Skmacy struct sockbuf *snd, *rcv; 1269178302Skmacy 1270178302Skmacy#ifdef notyet 1271174641Skmacy SOCK_LOCK_ASSERT(so); 1272178302Skmacy#endif 1273174641Skmacy 1274178302Skmacy snd = so_sockbuf_snd(so); 1275178302Skmacy rcv = so_sockbuf_rcv(so); 1276178302Skmacy 1277178302Skmacy log(LOG_INFO, "initializing offload socket\n"); 1278174641Skmacy /* 1279174641Skmacy * We either need to fix push frames to work with sbcompress 1280174641Skmacy * or we need to add this 1281174641Skmacy */ 1282178302Skmacy snd->sb_flags |= SB_NOCOALESCE; 1283178302Skmacy rcv->sb_flags |= SB_NOCOALESCE; 1284176472Skmacy 1285174641Skmacy tp->t_toe = toep; 1286174641Skmacy toep->tp_tp = tp; 1287174641Skmacy toep->tp_toedev = dev; 1288174641Skmacy 1289174641Skmacy toep->tp_tid = tid; 1290174641Skmacy toep->tp_l2t = e; 1291174641Skmacy toep->tp_wr_max = toep->tp_wr_avail = TOM_TUNABLE(dev, max_wrs); 1292174641Skmacy toep->tp_wr_unacked = 0; 1293174641Skmacy toep->tp_delack_mode = 0; 1294174641Skmacy 1295174641Skmacy toep->tp_mtu_idx = select_mss(td, tp, dst->rt_ifp->if_mtu); 1296174641Skmacy /* 1297174641Skmacy * XXX broken 1298174641Skmacy * 1299174641Skmacy */ 1300176472Skmacy tp->rcv_wnd = select_rcv_wnd(dev, so); 1301176472Skmacy 1302178302Skmacy toep->tp_ulp_mode = TOM_TUNABLE(dev, ddp) && !(so_options_get(so) & SO_NO_DDP) && 1303174641Skmacy tp->rcv_wnd >= MIN_DDP_RCV_WIN ? ULP_MODE_TCPDDP : 0; 1304174641Skmacy toep->tp_qset_idx = 0; 1305174641Skmacy 1306174641Skmacy reset_wr_list(toep); 1307174641Skmacy DPRINTF("initialization done\n"); 1308174641Skmacy} 1309174641Skmacy 1310174641Skmacy/* 1311174641Skmacy * The next two functions calculate the option 0 value for a socket. 1312174641Skmacy */ 1313174641Skmacystatic inline unsigned int 1314174641Skmacycalc_opt0h(struct socket *so, int mtu_idx) 1315174641Skmacy{ 1316178302Skmacy struct tcpcb *tp = so_sototcpcb(so); 1317174641Skmacy int wscale = select_rcv_wscale(tp->rcv_wnd); 1318178302Skmacy 1319174641Skmacy return V_NAGLE((tp->t_flags & TF_NODELAY) == 0) | 1320178302Skmacy V_KEEP_ALIVE((so_options_get(so) & SO_KEEPALIVE) != 0) | F_TCAM_BYPASS | 1321174641Skmacy V_WND_SCALE(wscale) | V_MSS_IDX(mtu_idx); 1322174641Skmacy} 1323174641Skmacy 1324174641Skmacystatic inline unsigned int 1325174641Skmacycalc_opt0l(struct socket *so, int ulp_mode) 1326174641Skmacy{ 1327178302Skmacy struct tcpcb *tp = so_sototcpcb(so); 1328174641Skmacy unsigned int val; 1329174641Skmacy 1330178302Skmacy val = V_TOS(INP_TOS(tp->t_inpcb)) | V_ULP_MODE(ulp_mode) | 1331174641Skmacy V_RCV_BUFSIZ(min(tp->rcv_wnd >> 10, (u32)M_RCV_BUFSIZ)); 1332174641Skmacy 1333178302Skmacy DPRINTF("opt0l tos=%08x rcv_wnd=%ld opt0l=%08x\n", INP_TOS(tp->t_inpcb), tp->rcv_wnd, val); 1334174641Skmacy return (val); 1335174641Skmacy} 1336174641Skmacy 1337174641Skmacystatic inline unsigned int 1338174641Skmacycalc_opt2(const struct socket *so, struct toedev *dev) 1339174641Skmacy{ 1340174641Skmacy int flv_valid; 1341174641Skmacy 1342174641Skmacy flv_valid = (TOM_TUNABLE(dev, cong_alg) != -1); 1343174641Skmacy 1344176472Skmacy return (V_FLAVORS_VALID(flv_valid) | 1345176472Skmacy V_CONG_CONTROL_FLAVOR(flv_valid ? TOM_TUNABLE(dev, cong_alg) : 0)); 1346174641Skmacy} 1347176472Skmacy 1348176472Skmacy#if DEBUG_WR > 1 1349176472Skmacystatic int 1350176472Skmacycount_pending_wrs(const struct toepcb *toep) 1351176472Skmacy{ 1352176472Skmacy const struct mbuf *m; 1353176472Skmacy int n = 0; 1354176472Skmacy 1355176472Skmacy wr_queue_walk(toep, m) 1356176472Skmacy n += m->m_pkthdr.csum_data; 1357176472Skmacy return (n); 1358176472Skmacy} 1359176472Skmacy#endif 1360176472Skmacy 1361174641Skmacy#if 0 1362174641Skmacy(((*(struct tom_data **)&(dev)->l4opt)->conf.cong_alg) != -1) 1363174641Skmacy#endif 1364174641Skmacy 1365174641Skmacystatic void 1366174641Skmacymk_act_open_req(struct socket *so, struct mbuf *m, 1367174641Skmacy unsigned int atid, const struct l2t_entry *e) 1368174641Skmacy{ 1369174641Skmacy struct cpl_act_open_req *req; 1370178302Skmacy struct inpcb *inp = so_sotoinpcb(so); 1371178302Skmacy struct tcpcb *tp = inp_inpcbtotcpcb(inp); 1372174641Skmacy struct toepcb *toep = tp->t_toe; 1373178302Skmacy struct toedev *tdev = toep->tp_toedev; 1374174641Skmacy 1375176472Skmacy m_set_priority((struct mbuf *)m, mkprio(CPL_PRIORITY_SETUP, toep)); 1376174641Skmacy 1377174641Skmacy req = mtod(m, struct cpl_act_open_req *); 1378174641Skmacy m->m_pkthdr.len = m->m_len = sizeof(*req); 1379176472Skmacy 1380174641Skmacy req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 1381176472Skmacy req->wr.wr_lo = 0; 1382174641Skmacy OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_ACT_OPEN_REQ, atid)); 1383178302Skmacy inp_4tuple_get(inp, &req->local_ip, &req->local_port, &req->peer_ip, &req->peer_port); 1384178302Skmacy#if 0 1385174641Skmacy req->local_port = inp->inp_lport; 1386174641Skmacy req->peer_port = inp->inp_fport; 1387174641Skmacy memcpy(&req->local_ip, &inp->inp_laddr, 4); 1388174641Skmacy memcpy(&req->peer_ip, &inp->inp_faddr, 4); 1389178302Skmacy#endif 1390174641Skmacy req->opt0h = htonl(calc_opt0h(so, toep->tp_mtu_idx) | V_L2T_IDX(e->idx) | 1391174641Skmacy V_TX_CHANNEL(e->smt_idx)); 1392174641Skmacy req->opt0l = htonl(calc_opt0l(so, toep->tp_ulp_mode)); 1393174641Skmacy req->params = 0; 1394174641Skmacy req->opt2 = htonl(calc_opt2(so, tdev)); 1395174641Skmacy} 1396174641Skmacy 1397174641Skmacy 1398174641Skmacy/* 1399174641Skmacy * Convert an ACT_OPEN_RPL status to an errno. 1400174641Skmacy */ 1401174641Skmacystatic int 1402174641Skmacyact_open_rpl_status_to_errno(int status) 1403174641Skmacy{ 1404174641Skmacy switch (status) { 1405174641Skmacy case CPL_ERR_CONN_RESET: 1406174641Skmacy return (ECONNREFUSED); 1407174641Skmacy case CPL_ERR_ARP_MISS: 1408174641Skmacy return (EHOSTUNREACH); 1409174641Skmacy case CPL_ERR_CONN_TIMEDOUT: 1410174641Skmacy return (ETIMEDOUT); 1411174641Skmacy case CPL_ERR_TCAM_FULL: 1412174641Skmacy return (ENOMEM); 1413174641Skmacy case CPL_ERR_CONN_EXIST: 1414174641Skmacy log(LOG_ERR, "ACTIVE_OPEN_RPL: 4-tuple in use\n"); 1415174641Skmacy return (EADDRINUSE); 1416174641Skmacy default: 1417174641Skmacy return (EIO); 1418174641Skmacy } 1419174641Skmacy} 1420174641Skmacy 1421174641Skmacystatic void 1422174641Skmacyfail_act_open(struct toepcb *toep, int errno) 1423174641Skmacy{ 1424174641Skmacy struct tcpcb *tp = toep->tp_tp; 1425174641Skmacy 1426174641Skmacy t3_release_offload_resources(toep); 1427174641Skmacy if (tp) { 1428178302Skmacy inp_wunlock(tp->t_inpcb); 1429178302Skmacy tcp_offload_drop(tp, errno); 1430174641Skmacy } 1431174641Skmacy 1432174641Skmacy#ifdef notyet 1433174641Skmacy TCP_INC_STATS_BH(TCP_MIB_ATTEMPTFAILS); 1434174641Skmacy#endif 1435174641Skmacy} 1436174641Skmacy 1437174641Skmacy/* 1438174641Skmacy * Handle active open failures. 1439174641Skmacy */ 1440174641Skmacystatic void 1441174641Skmacyactive_open_failed(struct toepcb *toep, struct mbuf *m) 1442174641Skmacy{ 1443174641Skmacy struct cpl_act_open_rpl *rpl = cplhdr(m); 1444174641Skmacy struct inpcb *inp; 1445174641Skmacy 1446174641Skmacy if (toep->tp_tp == NULL) 1447174641Skmacy goto done; 1448174641Skmacy 1449174641Skmacy inp = toep->tp_tp->t_inpcb; 1450174641Skmacy 1451174641Skmacy/* 1452174641Skmacy * Don't handle connection retry for now 1453174641Skmacy */ 1454174641Skmacy#ifdef notyet 1455174641Skmacy struct inet_connection_sock *icsk = inet_csk(sk); 1456174641Skmacy 1457174641Skmacy if (rpl->status == CPL_ERR_CONN_EXIST && 1458174641Skmacy icsk->icsk_retransmit_timer.function != act_open_retry_timer) { 1459174641Skmacy icsk->icsk_retransmit_timer.function = act_open_retry_timer; 1460174641Skmacy sk_reset_timer(so, &icsk->icsk_retransmit_timer, 1461174641Skmacy jiffies + HZ / 2); 1462174641Skmacy } else 1463178302Skmacy#endif 1464178302Skmacy { 1465178767Skmacy inp_wlock(inp); 1466178767Skmacy /* 1467178767Skmacy * drops the inpcb lock 1468178767Skmacy */ 1469174641Skmacy fail_act_open(toep, act_open_rpl_status_to_errno(rpl->status)); 1470178302Skmacy } 1471178302Skmacy 1472178302Skmacy done: 1473174641Skmacy m_free(m); 1474174641Skmacy} 1475174641Skmacy 1476174641Skmacy/* 1477174641Skmacy * Return whether a failed active open has allocated a TID 1478174641Skmacy */ 1479174641Skmacystatic inline int 1480174641Skmacyact_open_has_tid(int status) 1481174641Skmacy{ 1482174641Skmacy return status != CPL_ERR_TCAM_FULL && status != CPL_ERR_CONN_EXIST && 1483174641Skmacy status != CPL_ERR_ARP_MISS; 1484174641Skmacy} 1485174641Skmacy 1486174641Skmacy/* 1487174641Skmacy * Process an ACT_OPEN_RPL CPL message. 1488174641Skmacy */ 1489174641Skmacystatic int 1490174641Skmacydo_act_open_rpl(struct t3cdev *cdev, struct mbuf *m, void *ctx) 1491174641Skmacy{ 1492174641Skmacy struct toepcb *toep = (struct toepcb *)ctx; 1493174641Skmacy struct cpl_act_open_rpl *rpl = cplhdr(m); 1494174641Skmacy 1495174641Skmacy if (cdev->type != T3A && act_open_has_tid(rpl->status)) 1496174641Skmacy cxgb_queue_tid_release(cdev, GET_TID(rpl)); 1497174641Skmacy 1498174641Skmacy active_open_failed(toep, m); 1499174641Skmacy return (0); 1500174641Skmacy} 1501174641Skmacy 1502174641Skmacy/* 1503174641Skmacy * Handle an ARP failure for an active open. XXX purge ofo queue 1504174641Skmacy * 1505174641Skmacy * XXX badly broken for crossed SYNs as the ATID is no longer valid. 1506174641Skmacy * XXX crossed SYN errors should be generated by PASS_ACCEPT_RPL which should 1507174641Skmacy * check SOCK_DEAD or sk->sk_sock. Or maybe generate the error here but don't 1508174641Skmacy * free the atid. Hmm. 1509174641Skmacy */ 1510174641Skmacy#ifdef notyet 1511174641Skmacystatic void 1512174641Skmacyact_open_req_arp_failure(struct t3cdev *dev, struct mbuf *m) 1513174641Skmacy{ 1514174641Skmacy struct toepcb *toep = m_get_toep(m); 1515174641Skmacy struct tcpcb *tp = toep->tp_tp; 1516174641Skmacy struct inpcb *inp = tp->t_inpcb; 1517178302Skmacy struct socket *so; 1518174641Skmacy 1519177530Skmacy inp_wlock(inp); 1520174641Skmacy if (tp->t_state == TCPS_SYN_SENT || tp->t_state == TCPS_SYN_RECEIVED) { 1521178767Skmacy /* 1522178767Skmacy * drops the inpcb lock 1523178767Skmacy */ 1524174641Skmacy fail_act_open(so, EHOSTUNREACH); 1525174641Skmacy printf("freeing %p\n", m); 1526174641Skmacy 1527174641Skmacy m_free(m); 1528178302Skmacy } else 1529178302Skmacy inp_wunlock(inp); 1530174641Skmacy} 1531174641Skmacy#endif 1532174641Skmacy/* 1533174641Skmacy * Send an active open request. 1534174641Skmacy */ 1535174641Skmacyint 1536174641Skmacyt3_connect(struct toedev *tdev, struct socket *so, 1537174641Skmacy struct rtentry *rt, struct sockaddr *nam) 1538174641Skmacy{ 1539174641Skmacy struct mbuf *m; 1540174641Skmacy struct l2t_entry *e; 1541174641Skmacy struct tom_data *d = TOM_DATA(tdev); 1542178302Skmacy struct inpcb *inp = so_sotoinpcb(so); 1543174641Skmacy struct tcpcb *tp = intotcpcb(inp); 1544174641Skmacy struct toepcb *toep; /* allocated by init_offload_socket */ 1545174641Skmacy 1546174641Skmacy int atid; 1547174641Skmacy 1548174641Skmacy toep = toepcb_alloc(); 1549174641Skmacy if (toep == NULL) 1550174641Skmacy goto out_err; 1551174641Skmacy 1552174641Skmacy if ((atid = cxgb_alloc_atid(d->cdev, d->client, toep)) < 0) 1553174641Skmacy goto out_err; 1554174641Skmacy 1555174641Skmacy e = t3_l2t_get(d->cdev, rt, rt->rt_ifp, nam); 1556174641Skmacy if (!e) 1557174641Skmacy goto free_tid; 1558174641Skmacy 1559177575Skmacy inp_lock_assert(inp); 1560174641Skmacy m = m_gethdr(MT_DATA, M_WAITOK); 1561174641Skmacy 1562174641Skmacy#if 0 1563174641Skmacy m->m_toe.mt_toepcb = tp->t_toe; 1564174641Skmacy set_arp_failure_handler((struct mbuf *)m, act_open_req_arp_failure); 1565174641Skmacy#endif 1566178302Skmacy so_lock(so); 1567174641Skmacy 1568174641Skmacy init_offload_socket(so, tdev, atid, e, rt, toep); 1569174641Skmacy 1570174641Skmacy install_offload_ops(so); 1571174641Skmacy 1572174641Skmacy mk_act_open_req(so, m, atid, e); 1573178302Skmacy so_unlock(so); 1574174641Skmacy 1575174641Skmacy soisconnecting(so); 1576174641Skmacy toep = tp->t_toe; 1577174641Skmacy m_set_toep(m, tp->t_toe); 1578174641Skmacy 1579174641Skmacy toep->tp_state = TCPS_SYN_SENT; 1580174641Skmacy l2t_send(d->cdev, (struct mbuf *)m, e); 1581174641Skmacy 1582174641Skmacy if (toep->tp_ulp_mode) 1583178302Skmacy t3_enable_ddp(toep, 0); 1584174641Skmacy return (0); 1585174641Skmacy 1586174641Skmacyfree_tid: 1587174641Skmacy printf("failing connect - free atid\n"); 1588174641Skmacy 1589174641Skmacy free_atid(d->cdev, atid); 1590174641Skmacyout_err: 1591174641Skmacy printf("return ENOMEM\n"); 1592174641Skmacy return (ENOMEM); 1593174641Skmacy} 1594174641Skmacy 1595174641Skmacy/* 1596174641Skmacy * Send an ABORT_REQ message. Cannot fail. This routine makes sure we do 1597174641Skmacy * not send multiple ABORT_REQs for the same connection and also that we do 1598174641Skmacy * not try to send a message after the connection has closed. Returns 1 if 1599174641Skmacy * an ABORT_REQ wasn't generated after all, 0 otherwise. 1600174641Skmacy */ 1601174641Skmacystatic void 1602174641Skmacyt3_send_reset(struct toepcb *toep) 1603174641Skmacy{ 1604174641Skmacy 1605174641Skmacy struct cpl_abort_req *req; 1606174641Skmacy unsigned int tid = toep->tp_tid; 1607174641Skmacy int mode = CPL_ABORT_SEND_RST; 1608174641Skmacy struct tcpcb *tp = toep->tp_tp; 1609174641Skmacy struct toedev *tdev = toep->tp_toedev; 1610174641Skmacy struct socket *so = NULL; 1611174641Skmacy struct mbuf *m; 1612178302Skmacy struct sockbuf *snd; 1613174641Skmacy 1614174641Skmacy if (tp) { 1615177575Skmacy inp_lock_assert(tp->t_inpcb); 1616178302Skmacy so = inp_inpcbtosocket(tp->t_inpcb); 1617174641Skmacy } 1618174641Skmacy 1619174641Skmacy if (__predict_false((toep->tp_flags & TP_ABORT_SHUTDOWN) || 1620174641Skmacy tdev == NULL)) 1621174641Skmacy return; 1622174641Skmacy toep->tp_flags |= (TP_ABORT_RPL_PENDING|TP_ABORT_SHUTDOWN); 1623178302Skmacy 1624178302Skmacy snd = so_sockbuf_snd(so); 1625174641Skmacy /* Purge the send queue so we don't send anything after an abort. */ 1626174641Skmacy if (so) 1627178302Skmacy sbflush(snd); 1628174641Skmacy if ((toep->tp_flags & TP_CLOSE_CON_REQUESTED) && is_t3a(tdev)) 1629174641Skmacy mode |= CPL_ABORT_POST_CLOSE_REQ; 1630174641Skmacy 1631174641Skmacy m = m_gethdr_nofail(sizeof(*req)); 1632176472Skmacy m_set_priority(m, mkprio(CPL_PRIORITY_DATA, toep)); 1633174641Skmacy set_arp_failure_handler(m, abort_arp_failure); 1634174641Skmacy 1635174641Skmacy req = mtod(m, struct cpl_abort_req *); 1636174641Skmacy req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_HOST_ABORT_CON_REQ)); 1637174641Skmacy req->wr.wr_lo = htonl(V_WR_TID(tid)); 1638174641Skmacy OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_ABORT_REQ, tid)); 1639174641Skmacy req->rsvd0 = tp ? htonl(tp->snd_nxt) : 0; 1640174641Skmacy req->rsvd1 = !(toep->tp_flags & TP_DATASENT); 1641174641Skmacy req->cmd = mode; 1642174641Skmacy if (tp && (tp->t_state == TCPS_SYN_SENT)) 1643174641Skmacy mbufq_tail(&toep->out_of_order_queue, m); // defer 1644174641Skmacy else 1645174641Skmacy l2t_send(TOEP_T3C_DEV(toep), m, toep->tp_l2t); 1646174641Skmacy} 1647174641Skmacy 1648174641Skmacystatic int 1649174641Skmacyt3_ip_ctloutput(struct socket *so, struct sockopt *sopt) 1650174641Skmacy{ 1651174641Skmacy struct inpcb *inp; 1652174641Skmacy int error, optval; 1653174641Skmacy 1654174641Skmacy if (sopt->sopt_name == IP_OPTIONS) 1655174641Skmacy return (ENOPROTOOPT); 1656174641Skmacy 1657174641Skmacy if (sopt->sopt_name != IP_TOS) 1658174641Skmacy return (EOPNOTSUPP); 1659174641Skmacy 1660174641Skmacy error = sooptcopyin(sopt, &optval, sizeof optval, sizeof optval); 1661174641Skmacy 1662174641Skmacy if (error) 1663174641Skmacy return (error); 1664174641Skmacy 1665174641Skmacy if (optval > IPTOS_PREC_CRITIC_ECP && !suser(curthread)) 1666174641Skmacy return (EPERM); 1667174641Skmacy 1668178302Skmacy inp = so_sotoinpcb(so); 1669178767Skmacy inp_wlock(inp); 1670178302Skmacy inp_ip_tos_set(inp, optval); 1671178302Skmacy#if 0 1672174641Skmacy inp->inp_ip_tos = optval; 1673178302Skmacy#endif 1674178302Skmacy t3_set_tos(inp_inpcbtotcpcb(inp)->t_toe); 1675178767Skmacy inp_wunlock(inp); 1676178767Skmacy 1677174641Skmacy return (0); 1678174641Skmacy} 1679174641Skmacy 1680174641Skmacystatic int 1681174641Skmacyt3_tcp_ctloutput(struct socket *so, struct sockopt *sopt) 1682174641Skmacy{ 1683174641Skmacy int err = 0; 1684174641Skmacy size_t copied; 1685174641Skmacy 1686174641Skmacy if (sopt->sopt_name != TCP_CONGESTION && 1687174641Skmacy sopt->sopt_name != TCP_NODELAY) 1688174641Skmacy return (EOPNOTSUPP); 1689180583Skmacy 1690174641Skmacy if (sopt->sopt_name == TCP_CONGESTION) { 1691174641Skmacy char name[TCP_CA_NAME_MAX]; 1692174641Skmacy int optlen = sopt->sopt_valsize; 1693174641Skmacy struct tcpcb *tp; 1694174641Skmacy 1695180583Skmacy if (sopt->sopt_dir == SOPT_GET) { 1696180583Skmacy KASSERT(0, ("unimplemented")); 1697180583Skmacy return (EOPNOTSUPP); 1698180583Skmacy } 1699180583Skmacy 1700174641Skmacy if (optlen < 1) 1701174641Skmacy return (EINVAL); 1702174641Skmacy 1703174641Skmacy err = copyinstr(sopt->sopt_val, name, 1704174641Skmacy min(TCP_CA_NAME_MAX - 1, optlen), &copied); 1705174641Skmacy if (err) 1706174641Skmacy return (err); 1707174641Skmacy if (copied < 1) 1708174641Skmacy return (EINVAL); 1709174641Skmacy 1710178302Skmacy tp = so_sototcpcb(so); 1711174641Skmacy /* 1712174641Skmacy * XXX I need to revisit this 1713174641Skmacy */ 1714174641Skmacy if ((err = t3_set_cong_control(so, name)) == 0) { 1715176472Skmacy#ifdef CONGESTION_CONTROL_SUPPORTED 1716174641Skmacy tp->t_cong_control = strdup(name, M_CXGB); 1717174641Skmacy#endif 1718174641Skmacy } else 1719174641Skmacy return (err); 1720174641Skmacy } else { 1721174641Skmacy int optval, oldval; 1722174641Skmacy struct inpcb *inp; 1723174641Skmacy struct tcpcb *tp; 1724178302Skmacy 1725180583Skmacy if (sopt->sopt_dir == SOPT_GET) 1726180583Skmacy return (EOPNOTSUPP); 1727180583Skmacy 1728174641Skmacy err = sooptcopyin(sopt, &optval, sizeof optval, 1729174641Skmacy sizeof optval); 1730174641Skmacy 1731174641Skmacy if (err) 1732174641Skmacy return (err); 1733174641Skmacy 1734178302Skmacy inp = so_sotoinpcb(so); 1735178302Skmacy tp = inp_inpcbtotcpcb(inp); 1736174641Skmacy 1737177530Skmacy inp_wlock(inp); 1738174641Skmacy 1739174641Skmacy oldval = tp->t_flags; 1740174641Skmacy if (optval) 1741174641Skmacy tp->t_flags |= TF_NODELAY; 1742174641Skmacy else 1743174641Skmacy tp->t_flags &= ~TF_NODELAY; 1744177530Skmacy inp_wunlock(inp); 1745178302Skmacy 1746178302Skmacy 1747178767Skmacy if (oldval != tp->t_flags && (tp->t_toe != NULL)) 1748178302Skmacy t3_set_nagle(tp->t_toe); 1749174641Skmacy 1750174641Skmacy } 1751174641Skmacy 1752174641Skmacy return (0); 1753174641Skmacy} 1754174641Skmacy 1755178302Skmacyint 1756174641Skmacyt3_ctloutput(struct socket *so, struct sockopt *sopt) 1757174641Skmacy{ 1758174641Skmacy int err; 1759174641Skmacy 1760174641Skmacy if (sopt->sopt_level != IPPROTO_TCP) 1761174641Skmacy err = t3_ip_ctloutput(so, sopt); 1762174641Skmacy else 1763174641Skmacy err = t3_tcp_ctloutput(so, sopt); 1764174641Skmacy 1765174641Skmacy if (err != EOPNOTSUPP) 1766174641Skmacy return (err); 1767174641Skmacy 1768176472Skmacy return (tcp_ctloutput(so, sopt)); 1769174641Skmacy} 1770174641Skmacy 1771174641Skmacy/* 1772176472Skmacy * Returns true if we need to explicitly request RST when we receive new data 1773176472Skmacy * on an RX-closed connection. 1774176472Skmacy */ 1775176472Skmacystatic inline int 1776176472Skmacyneed_rst_on_excess_rx(const struct toepcb *toep) 1777176472Skmacy{ 1778176472Skmacy return (1); 1779176472Skmacy} 1780176472Skmacy 1781176472Skmacy/* 1782176472Skmacy * Handles Rx data that arrives in a state where the socket isn't accepting 1783176472Skmacy * new data. 1784176472Skmacy */ 1785176472Skmacystatic void 1786176472Skmacyhandle_excess_rx(struct toepcb *toep, struct mbuf *m) 1787176472Skmacy{ 1788176472Skmacy 1789178302Skmacy if (need_rst_on_excess_rx(toep) && 1790178302Skmacy !(toep->tp_flags & TP_ABORT_SHUTDOWN)) 1791176472Skmacy t3_send_reset(toep); 1792176472Skmacy m_freem(m); 1793176472Skmacy} 1794176472Skmacy 1795176472Skmacy/* 1796176472Skmacy * Process a get_tcb_rpl as a DDP completion (similar to RX_DDP_COMPLETE) 1797176472Skmacy * by getting the DDP offset from the TCB. 1798176472Skmacy */ 1799176472Skmacystatic void 1800176472Skmacytcb_rpl_as_ddp_complete(struct toepcb *toep, struct mbuf *m) 1801176472Skmacy{ 1802176472Skmacy struct ddp_state *q = &toep->tp_ddp_state; 1803176472Skmacy struct ddp_buf_state *bsp; 1804176472Skmacy struct cpl_get_tcb_rpl *hdr; 1805176472Skmacy unsigned int ddp_offset; 1806176472Skmacy struct socket *so; 1807176472Skmacy struct tcpcb *tp; 1808178302Skmacy struct sockbuf *rcv; 1809178302Skmacy int state; 1810176472Skmacy 1811176472Skmacy uint64_t t; 1812176472Skmacy __be64 *tcb; 1813176472Skmacy 1814176472Skmacy tp = toep->tp_tp; 1815178302Skmacy so = inp_inpcbtosocket(tp->t_inpcb); 1816176472Skmacy 1817177575Skmacy inp_lock_assert(tp->t_inpcb); 1818178302Skmacy rcv = so_sockbuf_rcv(so); 1819178302Skmacy sockbuf_lock(rcv); 1820176472Skmacy 1821178302Skmacy /* Note that we only accout for CPL_GET_TCB issued by the DDP code. 1822178302Skmacy * We really need a cookie in order to dispatch the RPLs. 1823176472Skmacy */ 1824176472Skmacy q->get_tcb_count--; 1825176472Skmacy 1826176472Skmacy /* It is a possible that a previous CPL already invalidated UBUF DDP 1827176472Skmacy * and moved the cur_buf idx and hence no further processing of this 1828176472Skmacy * skb is required. However, the app might be sleeping on 1829176472Skmacy * !q->get_tcb_count and we need to wake it up. 1830176472Skmacy */ 1831176472Skmacy if (q->cancel_ubuf && !t3_ddp_ubuf_pending(toep)) { 1832178302Skmacy int state = so_state_get(so); 1833178302Skmacy 1834176472Skmacy m_freem(m); 1835178302Skmacy if (__predict_true((state & SS_NOFDREF) == 0)) 1836178302Skmacy so_sorwakeup_locked(so); 1837176472Skmacy else 1838178302Skmacy sockbuf_unlock(rcv); 1839178302Skmacy 1840176472Skmacy return; 1841176472Skmacy } 1842176472Skmacy 1843176472Skmacy bsp = &q->buf_state[q->cur_buf]; 1844176472Skmacy hdr = cplhdr(m); 1845176472Skmacy tcb = (__be64 *)(hdr + 1); 1846176472Skmacy if (q->cur_buf == 0) { 1847176472Skmacy t = be64toh(tcb[(31 - W_TCB_RX_DDP_BUF0_OFFSET) / 2]); 1848176472Skmacy ddp_offset = t >> (32 + S_TCB_RX_DDP_BUF0_OFFSET); 1849176472Skmacy } else { 1850176472Skmacy t = be64toh(tcb[(31 - W_TCB_RX_DDP_BUF1_OFFSET) / 2]); 1851176472Skmacy ddp_offset = t >> S_TCB_RX_DDP_BUF1_OFFSET; 1852176472Skmacy } 1853176472Skmacy ddp_offset &= M_TCB_RX_DDP_BUF0_OFFSET; 1854176472Skmacy m->m_cur_offset = bsp->cur_offset; 1855176472Skmacy bsp->cur_offset = ddp_offset; 1856176472Skmacy m->m_len = m->m_pkthdr.len = ddp_offset - m->m_cur_offset; 1857176472Skmacy 1858176472Skmacy CTR5(KTR_TOM, 1859176472Skmacy "tcb_rpl_as_ddp_complete: idx=%d seq=0x%x hwbuf=%u ddp_offset=%u cur_offset=%u", 1860176472Skmacy q->cur_buf, tp->rcv_nxt, q->cur_buf, ddp_offset, m->m_cur_offset); 1861178302Skmacy KASSERT(ddp_offset >= m->m_cur_offset, 1862178302Skmacy ("ddp_offset=%u less than cur_offset=%u", 1863176472Skmacy ddp_offset, m->m_cur_offset)); 1864176472Skmacy 1865176472Skmacy#if 0 1866176472Skmacy{ 1867176472Skmacy unsigned int ddp_flags, rcv_nxt, rx_hdr_offset, buf_idx; 1868176472Skmacy 1869176472Skmacy t = be64toh(tcb[(31 - W_TCB_RX_DDP_FLAGS) / 2]); 1870176472Skmacy ddp_flags = (t >> S_TCB_RX_DDP_FLAGS) & M_TCB_RX_DDP_FLAGS; 1871176472Skmacy 1872176472Skmacy t = be64toh(tcb[(31 - W_TCB_RCV_NXT) / 2]); 1873176472Skmacy rcv_nxt = t >> S_TCB_RCV_NXT; 1874176472Skmacy rcv_nxt &= M_TCB_RCV_NXT; 1875176472Skmacy 1876176472Skmacy t = be64toh(tcb[(31 - W_TCB_RX_HDR_OFFSET) / 2]); 1877176472Skmacy rx_hdr_offset = t >> (32 + S_TCB_RX_HDR_OFFSET); 1878176472Skmacy rx_hdr_offset &= M_TCB_RX_HDR_OFFSET; 1879176472Skmacy 1880176472Skmacy T3_TRACE2(TIDTB(sk), 1881176472Skmacy "tcb_rpl_as_ddp_complete: DDP FLAGS 0x%x dma up to 0x%x", 1882176472Skmacy ddp_flags, rcv_nxt - rx_hdr_offset); 1883176472Skmacy T3_TRACE4(TB(q), 1884176472Skmacy "tcb_rpl_as_ddp_complete: rcvnxt 0x%x hwbuf %u cur_offset %u cancel %u", 1885176472Skmacy tp->rcv_nxt, q->cur_buf, bsp->cur_offset, q->cancel_ubuf); 1886176472Skmacy T3_TRACE3(TB(q), 1887176472Skmacy "tcb_rpl_as_ddp_complete: TCB rcvnxt 0x%x hwbuf 0x%x ddp_offset %u", 1888176472Skmacy rcv_nxt - rx_hdr_offset, ddp_flags, ddp_offset); 1889176472Skmacy T3_TRACE2(TB(q), 1890176472Skmacy "tcb_rpl_as_ddp_complete: flags0 0x%x flags1 0x%x", 1891176472Skmacy q->buf_state[0].flags, q->buf_state[1].flags); 1892176472Skmacy 1893176472Skmacy} 1894176472Skmacy#endif 1895176472Skmacy if (__predict_false(so_no_receive(so) && m->m_pkthdr.len)) { 1896176472Skmacy handle_excess_rx(toep, m); 1897176472Skmacy return; 1898176472Skmacy } 1899176472Skmacy 1900176472Skmacy#ifdef T3_TRACE 1901176472Skmacy if ((int)m->m_pkthdr.len < 0) { 1902176472Skmacy t3_ddp_error(so, "tcb_rpl_as_ddp_complete: neg len"); 1903176472Skmacy } 1904176472Skmacy#endif 1905176472Skmacy if (bsp->flags & DDP_BF_NOCOPY) { 1906176472Skmacy#ifdef T3_TRACE 1907176472Skmacy T3_TRACE0(TB(q), 1908176472Skmacy "tcb_rpl_as_ddp_complete: CANCEL UBUF"); 1909176472Skmacy 1910176472Skmacy if (!q->cancel_ubuf && !(sk->sk_shutdown & RCV_SHUTDOWN)) { 1911176472Skmacy printk("!cancel_ubuf"); 1912176472Skmacy t3_ddp_error(sk, "tcb_rpl_as_ddp_complete: !cancel_ubuf"); 1913176472Skmacy } 1914176472Skmacy#endif 1915176472Skmacy m->m_ddp_flags = DDP_BF_PSH | DDP_BF_NOCOPY | 1; 1916176472Skmacy bsp->flags &= ~(DDP_BF_NOCOPY|DDP_BF_NODATA); 1917176472Skmacy q->cur_buf ^= 1; 1918176472Skmacy } else if (bsp->flags & DDP_BF_NOFLIP) { 1919176472Skmacy 1920176472Skmacy m->m_ddp_flags = 1; /* always a kernel buffer */ 1921176472Skmacy 1922176472Skmacy /* now HW buffer carries a user buffer */ 1923176472Skmacy bsp->flags &= ~DDP_BF_NOFLIP; 1924176472Skmacy bsp->flags |= DDP_BF_NOCOPY; 1925176472Skmacy 1926176472Skmacy /* It is possible that the CPL_GET_TCB_RPL doesn't indicate 1927176472Skmacy * any new data in which case we're done. If in addition the 1928176472Skmacy * offset is 0, then there wasn't a completion for the kbuf 1929176472Skmacy * and we need to decrement the posted count. 1930176472Skmacy */ 1931176472Skmacy if (m->m_pkthdr.len == 0) { 1932176472Skmacy if (ddp_offset == 0) { 1933176472Skmacy q->kbuf_posted--; 1934176472Skmacy bsp->flags |= DDP_BF_NODATA; 1935176472Skmacy } 1936178302Skmacy sockbuf_unlock(rcv); 1937176472Skmacy m_free(m); 1938176472Skmacy return; 1939176472Skmacy } 1940176472Skmacy } else { 1941178302Skmacy sockbuf_unlock(rcv); 1942178302Skmacy 1943176472Skmacy /* This reply is for a CPL_GET_TCB_RPL to cancel the UBUF DDP, 1944176472Skmacy * but it got here way late and nobody cares anymore. 1945176472Skmacy */ 1946176472Skmacy m_free(m); 1947176472Skmacy return; 1948176472Skmacy } 1949176472Skmacy 1950176472Skmacy m->m_ddp_gl = (unsigned char *)bsp->gl; 1951176472Skmacy m->m_flags |= M_DDP; 1952176472Skmacy m->m_seq = tp->rcv_nxt; 1953176472Skmacy tp->rcv_nxt += m->m_pkthdr.len; 1954176472Skmacy tp->t_rcvtime = ticks; 1955176472Skmacy CTR3(KTR_TOM, "tcb_rpl_as_ddp_complete: seq 0x%x hwbuf %u m->m_pktlen %u", 1956176472Skmacy m->m_seq, q->cur_buf, m->m_pkthdr.len); 1957178302Skmacy if (m->m_pkthdr.len == 0) { 1958176472Skmacy q->user_ddp_pending = 0; 1959178302Skmacy m_free(m); 1960178302Skmacy } else 1961178302Skmacy SBAPPEND(rcv, m); 1962178302Skmacy 1963178302Skmacy state = so_state_get(so); 1964178302Skmacy if (__predict_true((state & SS_NOFDREF) == 0)) 1965178302Skmacy so_sorwakeup_locked(so); 1966176472Skmacy else 1967178302Skmacy sockbuf_unlock(rcv); 1968176472Skmacy} 1969176472Skmacy 1970176472Skmacy/* 1971176472Skmacy * Process a CPL_GET_TCB_RPL. These can also be generated by the DDP code, 1972176472Skmacy * in that case they are similar to DDP completions. 1973176472Skmacy */ 1974176472Skmacystatic int 1975176472Skmacydo_get_tcb_rpl(struct t3cdev *cdev, struct mbuf *m, void *ctx) 1976176472Skmacy{ 1977176472Skmacy struct toepcb *toep = (struct toepcb *)ctx; 1978176472Skmacy 1979176472Skmacy /* OK if socket doesn't exist */ 1980176472Skmacy if (toep == NULL) { 1981176472Skmacy printf("null toep in do_get_tcb_rpl\n"); 1982176472Skmacy return (CPL_RET_BUF_DONE); 1983176472Skmacy } 1984176472Skmacy 1985177530Skmacy inp_wlock(toep->tp_tp->t_inpcb); 1986176472Skmacy tcb_rpl_as_ddp_complete(toep, m); 1987177530Skmacy inp_wunlock(toep->tp_tp->t_inpcb); 1988176472Skmacy 1989176472Skmacy return (0); 1990176472Skmacy} 1991176472Skmacy 1992176472Skmacystatic void 1993176472Skmacyhandle_ddp_data(struct toepcb *toep, struct mbuf *m) 1994176472Skmacy{ 1995176472Skmacy struct tcpcb *tp = toep->tp_tp; 1996178302Skmacy struct socket *so; 1997176472Skmacy struct ddp_state *q; 1998176472Skmacy struct ddp_buf_state *bsp; 1999176472Skmacy struct cpl_rx_data *hdr = cplhdr(m); 2000176472Skmacy unsigned int rcv_nxt = ntohl(hdr->seq); 2001178302Skmacy struct sockbuf *rcv; 2002178302Skmacy 2003176472Skmacy if (tp->rcv_nxt == rcv_nxt) 2004176472Skmacy return; 2005176472Skmacy 2006177575Skmacy inp_lock_assert(tp->t_inpcb); 2007178302Skmacy so = inp_inpcbtosocket(tp->t_inpcb); 2008178302Skmacy rcv = so_sockbuf_rcv(so); 2009178302Skmacy sockbuf_lock(rcv); 2010178302Skmacy 2011176472Skmacy q = &toep->tp_ddp_state; 2012176472Skmacy bsp = &q->buf_state[q->cur_buf]; 2013176472Skmacy KASSERT(SEQ_GT(rcv_nxt, tp->rcv_nxt), ("tp->rcv_nxt=0x%08x decreased rcv_nxt=0x08%x", 2014176472Skmacy rcv_nxt, tp->rcv_nxt)); 2015176472Skmacy m->m_len = m->m_pkthdr.len = rcv_nxt - tp->rcv_nxt; 2016176472Skmacy KASSERT(m->m_len > 0, ("%s m_len=%d", __FUNCTION__, m->m_len)); 2017176472Skmacy CTR3(KTR_TOM, "rcv_nxt=0x%x tp->rcv_nxt=0x%x len=%d", 2018176472Skmacy rcv_nxt, tp->rcv_nxt, m->m_pkthdr.len); 2019176472Skmacy 2020176472Skmacy#ifdef T3_TRACE 2021176472Skmacy if ((int)m->m_pkthdr.len < 0) { 2022176472Skmacy t3_ddp_error(so, "handle_ddp_data: neg len"); 2023176472Skmacy } 2024176472Skmacy#endif 2025176472Skmacy m->m_ddp_gl = (unsigned char *)bsp->gl; 2026176472Skmacy m->m_flags |= M_DDP; 2027176472Skmacy m->m_cur_offset = bsp->cur_offset; 2028176472Skmacy m->m_ddp_flags = DDP_BF_PSH | (bsp->flags & DDP_BF_NOCOPY) | 1; 2029176472Skmacy if (bsp->flags & DDP_BF_NOCOPY) 2030176472Skmacy bsp->flags &= ~DDP_BF_NOCOPY; 2031176472Skmacy 2032176472Skmacy m->m_seq = tp->rcv_nxt; 2033176472Skmacy tp->rcv_nxt = rcv_nxt; 2034176472Skmacy bsp->cur_offset += m->m_pkthdr.len; 2035176472Skmacy if (!(bsp->flags & DDP_BF_NOFLIP)) 2036176472Skmacy q->cur_buf ^= 1; 2037176472Skmacy /* 2038176472Skmacy * For now, don't re-enable DDP after a connection fell out of DDP 2039176472Skmacy * mode. 2040176472Skmacy */ 2041176472Skmacy q->ubuf_ddp_ready = 0; 2042178302Skmacy sockbuf_unlock(rcv); 2043176472Skmacy} 2044176472Skmacy 2045176472Skmacy/* 2046174641Skmacy * Process new data received for a connection. 2047174641Skmacy */ 2048174641Skmacystatic void 2049174641Skmacynew_rx_data(struct toepcb *toep, struct mbuf *m) 2050174641Skmacy{ 2051174641Skmacy struct cpl_rx_data *hdr = cplhdr(m); 2052174641Skmacy struct tcpcb *tp = toep->tp_tp; 2053178302Skmacy struct socket *so; 2054178302Skmacy struct sockbuf *rcv; 2055178302Skmacy int state; 2056174641Skmacy int len = be16toh(hdr->len); 2057174641Skmacy 2058177530Skmacy inp_wlock(tp->t_inpcb); 2059178302Skmacy 2060178302Skmacy so = inp_inpcbtosocket(tp->t_inpcb); 2061174641Skmacy 2062176472Skmacy if (__predict_false(so_no_receive(so))) { 2063176472Skmacy handle_excess_rx(toep, m); 2064177530Skmacy inp_wunlock(tp->t_inpcb); 2065176472Skmacy TRACE_EXIT; 2066174641Skmacy return; 2067174641Skmacy } 2068174641Skmacy 2069176472Skmacy if (toep->tp_ulp_mode == ULP_MODE_TCPDDP) 2070176472Skmacy handle_ddp_data(toep, m); 2071176472Skmacy 2072176472Skmacy m->m_seq = ntohl(hdr->seq); 2073176472Skmacy m->m_ulp_mode = 0; /* for iSCSI */ 2074174641Skmacy 2075174641Skmacy#if VALIDATE_SEQ 2076176472Skmacy if (__predict_false(m->m_seq != tp->rcv_nxt)) { 2077176472Skmacy log(LOG_ERR, 2078174641Skmacy "%s: TID %u: Bad sequence number %u, expected %u\n", 2079178302Skmacy toep->tp_toedev->name, toep->tp_tid, m->m_seq, 2080174641Skmacy tp->rcv_nxt); 2081176472Skmacy m_freem(m); 2082177530Skmacy inp_wunlock(tp->t_inpcb); 2083174641Skmacy return; 2084174641Skmacy } 2085174641Skmacy#endif 2086174641Skmacy m_adj(m, sizeof(*hdr)); 2087174641Skmacy 2088176472Skmacy#ifdef URGENT_DATA_SUPPORTED 2089174641Skmacy /* 2090174641Skmacy * We don't handle urgent data yet 2091174641Skmacy */ 2092174641Skmacy if (__predict_false(hdr->urg)) 2093174641Skmacy handle_urg_ptr(so, tp->rcv_nxt + ntohs(hdr->urg)); 2094174641Skmacy if (__predict_false(tp->urg_data == TCP_URG_NOTYET && 2095174641Skmacy tp->urg_seq - tp->rcv_nxt < skb->len)) 2096174641Skmacy tp->urg_data = TCP_URG_VALID | skb->data[tp->urg_seq - 2097174641Skmacy tp->rcv_nxt]; 2098174641Skmacy#endif 2099174641Skmacy if (__predict_false(hdr->dack_mode != toep->tp_delack_mode)) { 2100174641Skmacy toep->tp_delack_mode = hdr->dack_mode; 2101174641Skmacy toep->tp_delack_seq = tp->rcv_nxt; 2102174641Skmacy } 2103176472Skmacy CTR6(KTR_TOM, "appending mbuf=%p pktlen=%d m_len=%d len=%d rcv_nxt=0x%x enqueued_bytes=%d", 2104176472Skmacy m, m->m_pkthdr.len, m->m_len, len, tp->rcv_nxt, toep->tp_enqueued_bytes); 2105174641Skmacy 2106174641Skmacy if (len < m->m_pkthdr.len) 2107174641Skmacy m->m_pkthdr.len = m->m_len = len; 2108174641Skmacy 2109174641Skmacy tp->rcv_nxt += m->m_pkthdr.len; 2110174641Skmacy tp->t_rcvtime = ticks; 2111174641Skmacy toep->tp_enqueued_bytes += m->m_pkthdr.len; 2112178302Skmacy CTR2(KTR_TOM, 2113176472Skmacy "new_rx_data: seq 0x%x len %u", 2114176472Skmacy m->m_seq, m->m_pkthdr.len); 2115178302Skmacy inp_wunlock(tp->t_inpcb); 2116178302Skmacy rcv = so_sockbuf_rcv(so); 2117178302Skmacy sockbuf_lock(rcv); 2118178302Skmacy#if 0 2119178302Skmacy if (sb_notify(rcv)) 2120178302Skmacy DPRINTF("rx_data so=%p flags=0x%x len=%d\n", so, rcv->sb_flags, m->m_pkthdr.len); 2121174641Skmacy#endif 2122178302Skmacy SBAPPEND(rcv, m); 2123174641Skmacy 2124176472Skmacy#ifdef notyet 2125176472Skmacy /* 2126176472Skmacy * We're giving too many credits to the card - but disable this check so we can keep on moving :-| 2127176472Skmacy * 2128176472Skmacy */ 2129178302Skmacy KASSERT(rcv->sb_cc < (rcv->sb_mbmax << 1), 2130176472Skmacy 2131174641Skmacy ("so=%p, data contents exceed mbmax, sb_cc=%d sb_mbmax=%d", 2132178302Skmacy so, rcv->sb_cc, rcv->sb_mbmax)); 2133176472Skmacy#endif 2134174641Skmacy 2135176472Skmacy 2136176472Skmacy CTR2(KTR_TOM, "sb_cc=%d sb_mbcnt=%d", 2137178302Skmacy rcv->sb_cc, rcv->sb_mbcnt); 2138178302Skmacy 2139178302Skmacy state = so_state_get(so); 2140178302Skmacy if (__predict_true((state & SS_NOFDREF) == 0)) 2141178302Skmacy so_sorwakeup_locked(so); 2142174641Skmacy else 2143178302Skmacy sockbuf_unlock(rcv); 2144174641Skmacy} 2145174641Skmacy 2146174641Skmacy/* 2147174641Skmacy * Handler for RX_DATA CPL messages. 2148174641Skmacy */ 2149174641Skmacystatic int 2150174641Skmacydo_rx_data(struct t3cdev *cdev, struct mbuf *m, void *ctx) 2151174641Skmacy{ 2152174641Skmacy struct toepcb *toep = (struct toepcb *)ctx; 2153174641Skmacy 2154174641Skmacy DPRINTF("rx_data len=%d\n", m->m_pkthdr.len); 2155174641Skmacy 2156174641Skmacy new_rx_data(toep, m); 2157174641Skmacy 2158174641Skmacy return (0); 2159174641Skmacy} 2160174641Skmacy 2161174641Skmacystatic void 2162176472Skmacynew_rx_data_ddp(struct toepcb *toep, struct mbuf *m) 2163174641Skmacy{ 2164176472Skmacy struct tcpcb *tp; 2165174641Skmacy struct ddp_state *q; 2166174641Skmacy struct ddp_buf_state *bsp; 2167174641Skmacy struct cpl_rx_data_ddp *hdr; 2168178302Skmacy struct socket *so; 2169174641Skmacy unsigned int ddp_len, rcv_nxt, ddp_report, end_offset, buf_idx; 2170176472Skmacy int nomoredata = 0; 2171177340Skmacy unsigned int delack_mode; 2172178302Skmacy struct sockbuf *rcv; 2173176472Skmacy 2174178302Skmacy tp = toep->tp_tp; 2175177530Skmacy inp_wlock(tp->t_inpcb); 2176178302Skmacy so = inp_inpcbtosocket(tp->t_inpcb); 2177178302Skmacy 2178176472Skmacy if (__predict_false(so_no_receive(so))) { 2179176472Skmacy 2180176472Skmacy handle_excess_rx(toep, m); 2181177530Skmacy inp_wunlock(tp->t_inpcb); 2182174641Skmacy return; 2183174641Skmacy } 2184176472Skmacy 2185174641Skmacy q = &toep->tp_ddp_state; 2186174641Skmacy hdr = cplhdr(m); 2187174641Skmacy ddp_report = ntohl(hdr->u.ddp_report); 2188174641Skmacy buf_idx = (ddp_report >> S_DDP_BUF_IDX) & 1; 2189174641Skmacy bsp = &q->buf_state[buf_idx]; 2190174641Skmacy 2191176472Skmacy CTR4(KTR_TOM, 2192176472Skmacy "new_rx_data_ddp: tp->rcv_nxt 0x%x cur_offset %u " 2193176472Skmacy "hdr seq 0x%x len %u", 2194176472Skmacy tp->rcv_nxt, bsp->cur_offset, ntohl(hdr->seq), 2195176472Skmacy ntohs(hdr->len)); 2196176472Skmacy CTR3(KTR_TOM, 2197176472Skmacy "new_rx_data_ddp: offset %u ddp_report 0x%x buf_idx=%d", 2198176472Skmacy G_DDP_OFFSET(ddp_report), ddp_report, buf_idx); 2199176472Skmacy 2200174641Skmacy ddp_len = ntohs(hdr->len); 2201174641Skmacy rcv_nxt = ntohl(hdr->seq) + ddp_len; 2202174641Skmacy 2203177340Skmacy delack_mode = G_DDP_DACK_MODE(ddp_report); 2204177340Skmacy if (__predict_false(G_DDP_DACK_MODE(ddp_report) != toep->tp_delack_mode)) { 2205177340Skmacy toep->tp_delack_mode = delack_mode; 2206177340Skmacy toep->tp_delack_seq = tp->rcv_nxt; 2207177340Skmacy } 2208177340Skmacy 2209176472Skmacy m->m_seq = tp->rcv_nxt; 2210174641Skmacy tp->rcv_nxt = rcv_nxt; 2211174641Skmacy 2212176472Skmacy tp->t_rcvtime = ticks; 2213174641Skmacy /* 2214174641Skmacy * Store the length in m->m_len. We are changing the meaning of 2215174641Skmacy * m->m_len here, we need to be very careful that nothing from now on 2216174641Skmacy * interprets ->len of this packet the usual way. 2217174641Skmacy */ 2218176472Skmacy m->m_len = m->m_pkthdr.len = rcv_nxt - m->m_seq; 2219177530Skmacy inp_wunlock(tp->t_inpcb); 2220176472Skmacy CTR3(KTR_TOM, 2221176472Skmacy "new_rx_data_ddp: m_len=%u rcv_next 0x%08x rcv_nxt_prev=0x%08x ", 2222176472Skmacy m->m_len, rcv_nxt, m->m_seq); 2223174641Skmacy /* 2224174641Skmacy * Figure out where the new data was placed in the buffer and store it 2225174641Skmacy * in when. Assumes the buffer offset starts at 0, consumer needs to 2226174641Skmacy * account for page pod's pg_offset. 2227174641Skmacy */ 2228174641Skmacy end_offset = G_DDP_OFFSET(ddp_report) + ddp_len; 2229176472Skmacy m->m_cur_offset = end_offset - m->m_pkthdr.len; 2230174641Skmacy 2231178302Skmacy rcv = so_sockbuf_rcv(so); 2232178302Skmacy sockbuf_lock(rcv); 2233178302Skmacy 2234176472Skmacy m->m_ddp_gl = (unsigned char *)bsp->gl; 2235176472Skmacy m->m_flags |= M_DDP; 2236176472Skmacy bsp->cur_offset = end_offset; 2237176472Skmacy toep->tp_enqueued_bytes += m->m_pkthdr.len; 2238176472Skmacy 2239174641Skmacy /* 2240176472Skmacy * Length is only meaningful for kbuf 2241174641Skmacy */ 2242176472Skmacy if (!(bsp->flags & DDP_BF_NOCOPY)) 2243176472Skmacy KASSERT(m->m_len <= bsp->gl->dgl_length, 2244176472Skmacy ("length received exceeds ddp pages: len=%d dgl_length=%d", 2245176472Skmacy m->m_len, bsp->gl->dgl_length)); 2246174641Skmacy 2247176472Skmacy KASSERT(m->m_len > 0, ("%s m_len=%d", __FUNCTION__, m->m_len)); 2248176472Skmacy KASSERT(m->m_next == NULL, ("m_len=%p", m->m_next)); 2249176472Skmacy /* 2250174641Skmacy * Bit 0 of flags stores whether the DDP buffer is completed. 2251174641Skmacy * Note that other parts of the code depend on this being in bit 0. 2252174641Skmacy */ 2253174641Skmacy if ((bsp->flags & DDP_BF_NOINVAL) && end_offset != bsp->gl->dgl_length) { 2254174641Skmacy panic("spurious ddp completion"); 2255174641Skmacy } else { 2256176472Skmacy m->m_ddp_flags = !!(ddp_report & F_DDP_BUF_COMPLETE); 2257176472Skmacy if (m->m_ddp_flags && !(bsp->flags & DDP_BF_NOFLIP)) 2258174641Skmacy q->cur_buf ^= 1; /* flip buffers */ 2259174641Skmacy } 2260174641Skmacy 2261174641Skmacy if (bsp->flags & DDP_BF_NOCOPY) { 2262176472Skmacy m->m_ddp_flags |= (bsp->flags & DDP_BF_NOCOPY); 2263174641Skmacy bsp->flags &= ~DDP_BF_NOCOPY; 2264174641Skmacy } 2265174641Skmacy 2266174641Skmacy if (ddp_report & F_DDP_PSH) 2267176472Skmacy m->m_ddp_flags |= DDP_BF_PSH; 2268176472Skmacy if (nomoredata) 2269176472Skmacy m->m_ddp_flags |= DDP_BF_NODATA; 2270176472Skmacy 2271177340Skmacy#ifdef notyet 2272177340Skmacy skb_reset_transport_header(skb); 2273177340Skmacy tcp_hdr(skb)->fin = 0; /* changes original hdr->ddp_report */ 2274177340Skmacy#endif 2275178302Skmacy SBAPPEND(rcv, m); 2276178302Skmacy 2277178302Skmacy if ((so_state_get(so) & SS_NOFDREF) == 0 && ((ddp_report & F_DDP_PSH) || 2278178302Skmacy (((m->m_ddp_flags & (DDP_BF_NOCOPY|1)) == (DDP_BF_NOCOPY|1)) 2279178302Skmacy || !(m->m_ddp_flags & DDP_BF_NOCOPY)))) 2280178302Skmacy so_sorwakeup_locked(so); 2281176472Skmacy else 2282178302Skmacy sockbuf_unlock(rcv); 2283174641Skmacy} 2284174641Skmacy 2285174641Skmacy#define DDP_ERR (F_DDP_PPOD_MISMATCH | F_DDP_LLIMIT_ERR | F_DDP_ULIMIT_ERR |\ 2286174641Skmacy F_DDP_PPOD_PARITY_ERR | F_DDP_PADDING_ERR | F_DDP_OFFSET_ERR |\ 2287174641Skmacy F_DDP_INVALID_TAG | F_DDP_COLOR_ERR | F_DDP_TID_MISMATCH |\ 2288174641Skmacy F_DDP_INVALID_PPOD) 2289174641Skmacy 2290174641Skmacy/* 2291174641Skmacy * Handler for RX_DATA_DDP CPL messages. 2292174641Skmacy */ 2293174641Skmacystatic int 2294174641Skmacydo_rx_data_ddp(struct t3cdev *cdev, struct mbuf *m, void *ctx) 2295174641Skmacy{ 2296174641Skmacy struct toepcb *toep = ctx; 2297174641Skmacy const struct cpl_rx_data_ddp *hdr = cplhdr(m); 2298174641Skmacy 2299174641Skmacy VALIDATE_SOCK(so); 2300174641Skmacy 2301174641Skmacy if (__predict_false(ntohl(hdr->ddpvld_status) & DDP_ERR)) { 2302174641Skmacy log(LOG_ERR, "RX_DATA_DDP for TID %u reported error 0x%x\n", 2303174641Skmacy GET_TID(hdr), G_DDP_VALID(ntohl(hdr->ddpvld_status))); 2304176472Skmacy return (CPL_RET_BUF_DONE); 2305174641Skmacy } 2306174641Skmacy#if 0 2307174641Skmacy skb->h.th = tcphdr_skb->h.th; 2308174641Skmacy#endif 2309176472Skmacy new_rx_data_ddp(toep, m); 2310174641Skmacy return (0); 2311174641Skmacy} 2312174641Skmacy 2313174641Skmacystatic void 2314176472Skmacyprocess_ddp_complete(struct toepcb *toep, struct mbuf *m) 2315174641Skmacy{ 2316176472Skmacy struct tcpcb *tp = toep->tp_tp; 2317178302Skmacy struct socket *so; 2318174641Skmacy struct ddp_state *q; 2319174641Skmacy struct ddp_buf_state *bsp; 2320174641Skmacy struct cpl_rx_ddp_complete *hdr; 2321177340Skmacy unsigned int ddp_report, buf_idx, when, delack_mode; 2322176472Skmacy int nomoredata = 0; 2323178302Skmacy struct sockbuf *rcv; 2324178302Skmacy 2325178302Skmacy inp_wlock(tp->t_inpcb); 2326178302Skmacy so = inp_inpcbtosocket(tp->t_inpcb); 2327174641Skmacy 2328176472Skmacy if (__predict_false(so_no_receive(so))) { 2329178302Skmacy struct inpcb *inp = so_sotoinpcb(so); 2330176472Skmacy 2331176472Skmacy handle_excess_rx(toep, m); 2332177530Skmacy inp_wunlock(inp); 2333174641Skmacy return; 2334174641Skmacy } 2335174641Skmacy q = &toep->tp_ddp_state; 2336174641Skmacy hdr = cplhdr(m); 2337174641Skmacy ddp_report = ntohl(hdr->ddp_report); 2338174641Skmacy buf_idx = (ddp_report >> S_DDP_BUF_IDX) & 1; 2339176472Skmacy m->m_pkthdr.csum_data = tp->rcv_nxt; 2340176472Skmacy 2341178302Skmacy rcv = so_sockbuf_rcv(so); 2342178302Skmacy sockbuf_lock(rcv); 2343178302Skmacy 2344174641Skmacy bsp = &q->buf_state[buf_idx]; 2345174641Skmacy when = bsp->cur_offset; 2346176472Skmacy m->m_len = m->m_pkthdr.len = G_DDP_OFFSET(ddp_report) - when; 2347176472Skmacy tp->rcv_nxt += m->m_len; 2348176472Skmacy tp->t_rcvtime = ticks; 2349177340Skmacy 2350177340Skmacy delack_mode = G_DDP_DACK_MODE(ddp_report); 2351177340Skmacy if (__predict_false(G_DDP_DACK_MODE(ddp_report) != toep->tp_delack_mode)) { 2352177340Skmacy toep->tp_delack_mode = delack_mode; 2353177340Skmacy toep->tp_delack_seq = tp->rcv_nxt; 2354177340Skmacy } 2355177340Skmacy#ifdef notyet 2356177340Skmacy skb_reset_transport_header(skb); 2357177340Skmacy tcp_hdr(skb)->fin = 0; /* changes valid memory past CPL */ 2358177340Skmacy#endif 2359177530Skmacy inp_wunlock(tp->t_inpcb); 2360174641Skmacy 2361178767Skmacy KASSERT(m->m_len >= 0, ("%s m_len=%d", __FUNCTION__, m->m_len)); 2362176472Skmacy CTR5(KTR_TOM, 2363176472Skmacy "process_ddp_complete: tp->rcv_nxt 0x%x cur_offset %u " 2364176472Skmacy "ddp_report 0x%x offset %u, len %u", 2365176472Skmacy tp->rcv_nxt, bsp->cur_offset, ddp_report, 2366176472Skmacy G_DDP_OFFSET(ddp_report), m->m_len); 2367178767Skmacy 2368178767Skmacy m->m_cur_offset = bsp->cur_offset; 2369174641Skmacy bsp->cur_offset += m->m_len; 2370174641Skmacy 2371176472Skmacy if (!(bsp->flags & DDP_BF_NOFLIP)) { 2372174641Skmacy q->cur_buf ^= 1; /* flip buffers */ 2373176472Skmacy if (G_DDP_OFFSET(ddp_report) < q->kbuf[0]->dgl_length) 2374176472Skmacy nomoredata=1; 2375176472Skmacy } 2376176472Skmacy 2377176472Skmacy CTR4(KTR_TOM, 2378176472Skmacy "process_ddp_complete: tp->rcv_nxt 0x%x cur_offset %u " 2379176472Skmacy "ddp_report %u offset %u", 2380176472Skmacy tp->rcv_nxt, bsp->cur_offset, ddp_report, 2381176472Skmacy G_DDP_OFFSET(ddp_report)); 2382176472Skmacy 2383176472Skmacy m->m_ddp_gl = (unsigned char *)bsp->gl; 2384176472Skmacy m->m_flags |= M_DDP; 2385176472Skmacy m->m_ddp_flags = (bsp->flags & DDP_BF_NOCOPY) | 1; 2386174641Skmacy if (bsp->flags & DDP_BF_NOCOPY) 2387174641Skmacy bsp->flags &= ~DDP_BF_NOCOPY; 2388176472Skmacy if (nomoredata) 2389176472Skmacy m->m_ddp_flags |= DDP_BF_NODATA; 2390174641Skmacy 2391178302Skmacy SBAPPEND(rcv, m); 2392178302Skmacy if ((so_state_get(so) & SS_NOFDREF) == 0) 2393178302Skmacy so_sorwakeup_locked(so); 2394176472Skmacy else 2395178302Skmacy sockbuf_unlock(rcv); 2396174641Skmacy} 2397174641Skmacy 2398174641Skmacy/* 2399174641Skmacy * Handler for RX_DDP_COMPLETE CPL messages. 2400174641Skmacy */ 2401174641Skmacystatic int 2402174641Skmacydo_rx_ddp_complete(struct t3cdev *cdev, struct mbuf *m, void *ctx) 2403174641Skmacy{ 2404174641Skmacy struct toepcb *toep = ctx; 2405174641Skmacy 2406174641Skmacy VALIDATE_SOCK(so); 2407174641Skmacy#if 0 2408174641Skmacy skb->h.th = tcphdr_skb->h.th; 2409174641Skmacy#endif 2410176472Skmacy process_ddp_complete(toep, m); 2411174641Skmacy return (0); 2412174641Skmacy} 2413174641Skmacy 2414174641Skmacy/* 2415174641Skmacy * Move a socket to TIME_WAIT state. We need to make some adjustments to the 2416174641Skmacy * socket state before calling tcp_time_wait to comply with its expectations. 2417174641Skmacy */ 2418174641Skmacystatic void 2419178302Skmacyenter_timewait(struct tcpcb *tp) 2420174641Skmacy{ 2421178302Skmacy /* 2422178302Skmacy * Bump rcv_nxt for the peer FIN. We don't do this at the time we 2423178302Skmacy * process peer_close because we don't want to carry the peer FIN in 2424178302Skmacy * the socket's receive queue and if we increment rcv_nxt without 2425178302Skmacy * having the FIN in the receive queue we'll confuse facilities such 2426178302Skmacy * as SIOCINQ. 2427178302Skmacy */ 2428178302Skmacy inp_wlock(tp->t_inpcb); 2429178302Skmacy tp->rcv_nxt++; 2430174641Skmacy 2431178302Skmacy tp->ts_recent_age = 0; /* defeat recycling */ 2432178302Skmacy tp->t_srtt = 0; /* defeat tcp_update_metrics */ 2433178302Skmacy inp_wunlock(tp->t_inpcb); 2434178302Skmacy tcp_offload_twstart(tp); 2435178302Skmacy} 2436178302Skmacy 2437174641Skmacy/* 2438176472Skmacy * For TCP DDP a PEER_CLOSE may also be an implicit RX_DDP_COMPLETE. This 2439176472Skmacy * function deals with the data that may be reported along with the FIN. 2440176472Skmacy * Returns -1 if no further processing of the PEER_CLOSE is needed, >= 0 to 2441176472Skmacy * perform normal FIN-related processing. In the latter case 1 indicates that 2442176472Skmacy * there was an implicit RX_DDP_COMPLETE and the skb should not be freed, 0 the 2443176472Skmacy * skb can be freed. 2444176472Skmacy */ 2445176472Skmacystatic int 2446176472Skmacyhandle_peer_close_data(struct socket *so, struct mbuf *m) 2447176472Skmacy{ 2448178302Skmacy struct tcpcb *tp = so_sototcpcb(so); 2449176472Skmacy struct toepcb *toep = tp->t_toe; 2450176472Skmacy struct ddp_state *q; 2451176472Skmacy struct ddp_buf_state *bsp; 2452176472Skmacy struct cpl_peer_close *req = cplhdr(m); 2453176472Skmacy unsigned int rcv_nxt = ntohl(req->rcv_nxt) - 1; /* exclude FIN */ 2454178302Skmacy struct sockbuf *rcv; 2455178302Skmacy 2456176472Skmacy if (tp->rcv_nxt == rcv_nxt) /* no data */ 2457176472Skmacy return (0); 2458176472Skmacy 2459178302Skmacy CTR0(KTR_TOM, "handle_peer_close_data"); 2460176472Skmacy if (__predict_false(so_no_receive(so))) { 2461176472Skmacy handle_excess_rx(toep, m); 2462176472Skmacy 2463176472Skmacy /* 2464176472Skmacy * Although we discard the data we want to process the FIN so 2465176472Skmacy * that PEER_CLOSE + data behaves the same as RX_DATA_DDP + 2466176472Skmacy * PEER_CLOSE without data. In particular this PEER_CLOSE 2467176472Skmacy * may be what will close the connection. We return 1 because 2468176472Skmacy * handle_excess_rx() already freed the packet. 2469176472Skmacy */ 2470176472Skmacy return (1); 2471176472Skmacy } 2472176472Skmacy 2473177575Skmacy inp_lock_assert(tp->t_inpcb); 2474176472Skmacy q = &toep->tp_ddp_state; 2475178302Skmacy rcv = so_sockbuf_rcv(so); 2476178302Skmacy sockbuf_lock(rcv); 2477178302Skmacy 2478176472Skmacy bsp = &q->buf_state[q->cur_buf]; 2479176472Skmacy m->m_len = m->m_pkthdr.len = rcv_nxt - tp->rcv_nxt; 2480176472Skmacy KASSERT(m->m_len > 0, ("%s m_len=%d", __FUNCTION__, m->m_len)); 2481176472Skmacy m->m_ddp_gl = (unsigned char *)bsp->gl; 2482176472Skmacy m->m_flags |= M_DDP; 2483176472Skmacy m->m_cur_offset = bsp->cur_offset; 2484176472Skmacy m->m_ddp_flags = 2485176472Skmacy DDP_BF_PSH | (bsp->flags & DDP_BF_NOCOPY) | 1; 2486176472Skmacy m->m_seq = tp->rcv_nxt; 2487176472Skmacy tp->rcv_nxt = rcv_nxt; 2488176472Skmacy bsp->cur_offset += m->m_pkthdr.len; 2489176472Skmacy if (!(bsp->flags & DDP_BF_NOFLIP)) 2490176472Skmacy q->cur_buf ^= 1; 2491177340Skmacy#ifdef notyet 2492177340Skmacy skb_reset_transport_header(skb); 2493177340Skmacy tcp_hdr(skb)->fin = 0; /* changes valid memory past CPL */ 2494177340Skmacy#endif 2495176472Skmacy tp->t_rcvtime = ticks; 2496178302Skmacy SBAPPEND(rcv, m); 2497178302Skmacy if (__predict_true((so_state_get(so) & SS_NOFDREF) == 0)) 2498178302Skmacy so_sorwakeup_locked(so); 2499176472Skmacy else 2500178302Skmacy sockbuf_unlock(rcv); 2501178302Skmacy 2502176472Skmacy return (1); 2503176472Skmacy} 2504176472Skmacy 2505176472Skmacy/* 2506174641Skmacy * Handle a peer FIN. 2507174641Skmacy */ 2508174641Skmacystatic void 2509178302Skmacydo_peer_fin(struct toepcb *toep, struct mbuf *m) 2510174641Skmacy{ 2511178302Skmacy struct socket *so; 2512178302Skmacy struct tcpcb *tp = toep->tp_tp; 2513178302Skmacy int keep, action; 2514174641Skmacy 2515178302Skmacy action = keep = 0; 2516178302Skmacy CTR1(KTR_TOM, "do_peer_fin state=%d", tp->t_state); 2517178302Skmacy if (!is_t3a(toep->tp_toedev) && (toep->tp_flags & TP_ABORT_RPL_PENDING)) { 2518174641Skmacy printf("abort_pending set\n"); 2519174641Skmacy 2520174641Skmacy goto out; 2521174641Skmacy } 2522177530Skmacy inp_wlock(tp->t_inpcb); 2523178302Skmacy so = inp_inpcbtosocket(toep->tp_tp->t_inpcb); 2524176472Skmacy if (toep->tp_ulp_mode == ULP_MODE_TCPDDP) { 2525176472Skmacy keep = handle_peer_close_data(so, m); 2526176472Skmacy if (keep < 0) { 2527177530Skmacy inp_wunlock(tp->t_inpcb); 2528174641Skmacy return; 2529176472Skmacy } 2530174641Skmacy } 2531176472Skmacy if (TCPS_HAVERCVDFIN(tp->t_state) == 0) { 2532178767Skmacy CTR1(KTR_TOM, 2533178767Skmacy "waking up waiters for cantrcvmore on %p ", so); 2534174641Skmacy socantrcvmore(so); 2535178767Skmacy 2536176472Skmacy /* 2537176472Skmacy * If connection is half-synchronized 2538176472Skmacy * (ie NEEDSYN flag on) then delay ACK, 2539176472Skmacy * so it may be piggybacked when SYN is sent. 2540176472Skmacy * Otherwise, since we received a FIN then no 2541176472Skmacy * more input can be expected, send ACK now. 2542176472Skmacy */ 2543176472Skmacy if (tp->t_flags & TF_NEEDSYN) 2544176472Skmacy tp->t_flags |= TF_DELACK; 2545176472Skmacy else 2546176472Skmacy tp->t_flags |= TF_ACKNOW; 2547176472Skmacy tp->rcv_nxt++; 2548176472Skmacy } 2549176472Skmacy 2550174641Skmacy switch (tp->t_state) { 2551174641Skmacy case TCPS_SYN_RECEIVED: 2552174641Skmacy tp->t_starttime = ticks; 2553174641Skmacy /* FALLTHROUGH */ 2554174641Skmacy case TCPS_ESTABLISHED: 2555174641Skmacy tp->t_state = TCPS_CLOSE_WAIT; 2556174641Skmacy break; 2557174641Skmacy case TCPS_FIN_WAIT_1: 2558174641Skmacy tp->t_state = TCPS_CLOSING; 2559174641Skmacy break; 2560174641Skmacy case TCPS_FIN_WAIT_2: 2561174641Skmacy /* 2562174641Skmacy * If we've sent an abort_req we must have sent it too late, 2563174641Skmacy * HW will send us a reply telling us so, and this peer_close 2564174641Skmacy * is really the last message for this connection and needs to 2565174641Skmacy * be treated as an abort_rpl, i.e., transition the connection 2566174641Skmacy * to TCP_CLOSE (note that the host stack does this at the 2567174641Skmacy * time of generating the RST but we must wait for HW). 2568174641Skmacy * Otherwise we enter TIME_WAIT. 2569174641Skmacy */ 2570174641Skmacy t3_release_offload_resources(toep); 2571174641Skmacy if (toep->tp_flags & TP_ABORT_RPL_PENDING) { 2572178302Skmacy action = TCP_CLOSE; 2573176472Skmacy } else { 2574178302Skmacy action = TCP_TIMEWAIT; 2575176472Skmacy } 2576174641Skmacy break; 2577174641Skmacy default: 2578174641Skmacy log(LOG_ERR, 2579174641Skmacy "%s: TID %u received PEER_CLOSE in bad state %d\n", 2580178302Skmacy toep->tp_toedev->tod_name, toep->tp_tid, tp->t_state); 2581174641Skmacy } 2582178302Skmacy inp_wunlock(tp->t_inpcb); 2583174641Skmacy 2584178302Skmacy if (action == TCP_TIMEWAIT) { 2585178302Skmacy enter_timewait(tp); 2586178302Skmacy } else if (action == TCP_DROP) { 2587178302Skmacy tcp_offload_drop(tp, 0); 2588178302Skmacy } else if (action == TCP_CLOSE) { 2589178302Skmacy tcp_offload_close(tp); 2590178302Skmacy } 2591178767Skmacy 2592174641Skmacy#ifdef notyet 2593176472Skmacy /* Do not send POLL_HUP for half duplex close. */ 2594176472Skmacy if ((sk->sk_shutdown & SEND_SHUTDOWN) || 2595176472Skmacy sk->sk_state == TCP_CLOSE) 2596176472Skmacy sk_wake_async(so, 1, POLL_HUP); 2597176472Skmacy else 2598176472Skmacy sk_wake_async(so, 1, POLL_IN); 2599176472Skmacy#endif 2600174641Skmacy 2601174641Skmacyout: 2602174641Skmacy if (!keep) 2603174641Skmacy m_free(m); 2604174641Skmacy} 2605174641Skmacy 2606174641Skmacy/* 2607174641Skmacy * Handler for PEER_CLOSE CPL messages. 2608174641Skmacy */ 2609174641Skmacystatic int 2610174641Skmacydo_peer_close(struct t3cdev *cdev, struct mbuf *m, void *ctx) 2611174641Skmacy{ 2612174641Skmacy struct toepcb *toep = (struct toepcb *)ctx; 2613174641Skmacy 2614174641Skmacy VALIDATE_SOCK(so); 2615174641Skmacy 2616178302Skmacy do_peer_fin(toep, m); 2617174641Skmacy return (0); 2618174641Skmacy} 2619174641Skmacy 2620174641Skmacystatic void 2621178302Skmacyprocess_close_con_rpl(struct toepcb *toep, struct mbuf *m) 2622174641Skmacy{ 2623174641Skmacy struct cpl_close_con_rpl *rpl = cplhdr(m); 2624178302Skmacy struct tcpcb *tp = toep->tp_tp; 2625178302Skmacy struct socket *so; 2626178302Skmacy int action = 0; 2627178302Skmacy struct sockbuf *rcv; 2628178302Skmacy 2629178302Skmacy inp_wlock(tp->t_inpcb); 2630178302Skmacy so = inp_inpcbtosocket(tp->t_inpcb); 2631178302Skmacy 2632174641Skmacy tp->snd_una = ntohl(rpl->snd_nxt) - 1; /* exclude FIN */ 2633174641Skmacy 2634178302Skmacy if (!is_t3a(toep->tp_toedev) && (toep->tp_flags & TP_ABORT_RPL_PENDING)) { 2635178302Skmacy inp_wunlock(tp->t_inpcb); 2636174641Skmacy goto out; 2637178302Skmacy } 2638174641Skmacy 2639178302Skmacy CTR3(KTR_TOM, "process_close_con_rpl(%p) state=%d dead=%d", toep, 2640178302Skmacy tp->t_state, !!(so_state_get(so) & SS_NOFDREF)); 2641178302Skmacy 2642174641Skmacy switch (tp->t_state) { 2643174641Skmacy case TCPS_CLOSING: /* see FIN_WAIT2 case in do_peer_fin */ 2644174641Skmacy t3_release_offload_resources(toep); 2645174641Skmacy if (toep->tp_flags & TP_ABORT_RPL_PENDING) { 2646178302Skmacy action = TCP_CLOSE; 2647174641Skmacy 2648176472Skmacy } else { 2649178302Skmacy action = TCP_TIMEWAIT; 2650176472Skmacy } 2651174641Skmacy break; 2652174641Skmacy case TCPS_LAST_ACK: 2653174641Skmacy /* 2654174641Skmacy * In this state we don't care about pending abort_rpl. 2655174641Skmacy * If we've sent abort_req it was post-close and was sent too 2656174641Skmacy * late, this close_con_rpl is the actual last message. 2657174641Skmacy */ 2658174641Skmacy t3_release_offload_resources(toep); 2659178302Skmacy action = TCP_CLOSE; 2660174641Skmacy break; 2661174641Skmacy case TCPS_FIN_WAIT_1: 2662176472Skmacy /* 2663176472Skmacy * If we can't receive any more 2664176472Skmacy * data, then closing user can proceed. 2665176472Skmacy * Starting the timer is contrary to the 2666176472Skmacy * specification, but if we don't get a FIN 2667176472Skmacy * we'll hang forever. 2668176472Skmacy * 2669176472Skmacy * XXXjl: 2670176472Skmacy * we should release the tp also, and use a 2671176472Skmacy * compressed state. 2672176472Skmacy */ 2673178302Skmacy if (so) 2674178302Skmacy rcv = so_sockbuf_rcv(so); 2675178302Skmacy else 2676178302Skmacy break; 2677178302Skmacy 2678178302Skmacy if (rcv->sb_state & SBS_CANTRCVMORE) { 2679176472Skmacy int timeout; 2680178302Skmacy 2681178302Skmacy if (so) 2682178302Skmacy soisdisconnected(so); 2683176472Skmacy timeout = (tcp_fast_finwait2_recycle) ? 2684176472Skmacy tcp_finwait2_timeout : tcp_maxidle; 2685176472Skmacy tcp_timer_activate(tp, TT_2MSL, timeout); 2686176472Skmacy } 2687176472Skmacy tp->t_state = TCPS_FIN_WAIT_2; 2688178302Skmacy if ((so_options_get(so) & SO_LINGER) && so_linger_get(so) == 0 && 2689174641Skmacy (toep->tp_flags & TP_ABORT_SHUTDOWN) == 0) { 2690178302Skmacy action = TCP_DROP; 2691174641Skmacy } 2692174641Skmacy 2693174641Skmacy break; 2694174641Skmacy default: 2695174641Skmacy log(LOG_ERR, 2696174641Skmacy "%s: TID %u received CLOSE_CON_RPL in bad state %d\n", 2697178302Skmacy toep->tp_toedev->tod_name, toep->tp_tid, 2698174641Skmacy tp->t_state); 2699174641Skmacy } 2700178302Skmacy inp_wunlock(tp->t_inpcb); 2701178302Skmacy 2702178302Skmacy 2703178302Skmacy if (action == TCP_TIMEWAIT) { 2704180675Skmacy enter_timewait(tp); 2705178302Skmacy } else if (action == TCP_DROP) { 2706178302Skmacy tcp_offload_drop(tp, 0); 2707178302Skmacy } else if (action == TCP_CLOSE) { 2708178302Skmacy tcp_offload_close(tp); 2709178302Skmacy } 2710174641Skmacyout: 2711176472Skmacy m_freem(m); 2712174641Skmacy} 2713174641Skmacy 2714174641Skmacy/* 2715174641Skmacy * Handler for CLOSE_CON_RPL CPL messages. 2716174641Skmacy */ 2717174641Skmacystatic int 2718174641Skmacydo_close_con_rpl(struct t3cdev *cdev, struct mbuf *m, 2719174641Skmacy void *ctx) 2720174641Skmacy{ 2721174641Skmacy struct toepcb *toep = (struct toepcb *)ctx; 2722174641Skmacy 2723178302Skmacy process_close_con_rpl(toep, m); 2724174641Skmacy return (0); 2725174641Skmacy} 2726174641Skmacy 2727174641Skmacy/* 2728174641Skmacy * Process abort replies. We only process these messages if we anticipate 2729174641Skmacy * them as the coordination between SW and HW in this area is somewhat lacking 2730174641Skmacy * and sometimes we get ABORT_RPLs after we are done with the connection that 2731174641Skmacy * originated the ABORT_REQ. 2732174641Skmacy */ 2733174641Skmacystatic void 2734178302Skmacyprocess_abort_rpl(struct toepcb *toep, struct mbuf *m) 2735174641Skmacy{ 2736178302Skmacy struct tcpcb *tp = toep->tp_tp; 2737178302Skmacy struct socket *so; 2738178302Skmacy int needclose = 0; 2739174641Skmacy 2740174641Skmacy#ifdef T3_TRACE 2741174641Skmacy T3_TRACE1(TIDTB(sk), 2742174641Skmacy "process_abort_rpl: GTS rpl pending %d", 2743174641Skmacy sock_flag(sk, ABORT_RPL_PENDING)); 2744174641Skmacy#endif 2745176472Skmacy 2746177530Skmacy inp_wlock(tp->t_inpcb); 2747178302Skmacy so = inp_inpcbtosocket(tp->t_inpcb); 2748174641Skmacy 2749174641Skmacy if (toep->tp_flags & TP_ABORT_RPL_PENDING) { 2750174641Skmacy /* 2751174641Skmacy * XXX panic on tcpdrop 2752174641Skmacy */ 2753178302Skmacy if (!(toep->tp_flags & TP_ABORT_RPL_RCVD) && !is_t3a(toep->tp_toedev)) 2754174641Skmacy toep->tp_flags |= TP_ABORT_RPL_RCVD; 2755174641Skmacy else { 2756174641Skmacy toep->tp_flags &= ~(TP_ABORT_RPL_RCVD|TP_ABORT_RPL_PENDING); 2757174641Skmacy if (!(toep->tp_flags & TP_ABORT_REQ_RCVD) || 2758178302Skmacy !is_t3a(toep->tp_toedev)) { 2759174641Skmacy if (toep->tp_flags & TP_ABORT_REQ_RCVD) 2760174641Skmacy panic("TP_ABORT_REQ_RCVD set"); 2761174641Skmacy t3_release_offload_resources(toep); 2762178302Skmacy needclose = 1; 2763174641Skmacy } 2764174641Skmacy } 2765174641Skmacy } 2766178302Skmacy inp_wunlock(tp->t_inpcb); 2767174641Skmacy 2768178302Skmacy if (needclose) 2769178302Skmacy tcp_offload_close(tp); 2770178302Skmacy 2771174641Skmacy m_free(m); 2772174641Skmacy} 2773174641Skmacy 2774174641Skmacy/* 2775174641Skmacy * Handle an ABORT_RPL_RSS CPL message. 2776174641Skmacy */ 2777174641Skmacystatic int 2778174641Skmacydo_abort_rpl(struct t3cdev *cdev, struct mbuf *m, void *ctx) 2779174641Skmacy{ 2780174641Skmacy struct cpl_abort_rpl_rss *rpl = cplhdr(m); 2781174641Skmacy struct toepcb *toep; 2782174641Skmacy 2783174641Skmacy /* 2784174641Skmacy * Ignore replies to post-close aborts indicating that the abort was 2785174641Skmacy * requested too late. These connections are terminated when we get 2786174641Skmacy * PEER_CLOSE or CLOSE_CON_RPL and by the time the abort_rpl_rss 2787174641Skmacy * arrives the TID is either no longer used or it has been recycled. 2788174641Skmacy */ 2789174641Skmacy if (rpl->status == CPL_ERR_ABORT_FAILED) { 2790174641Skmacydiscard: 2791174641Skmacy m_free(m); 2792174641Skmacy return (0); 2793174641Skmacy } 2794174641Skmacy 2795174641Skmacy toep = (struct toepcb *)ctx; 2796174641Skmacy 2797174641Skmacy /* 2798174641Skmacy * Sometimes we've already closed the socket, e.g., a post-close 2799174641Skmacy * abort races with ABORT_REQ_RSS, the latter frees the socket 2800174641Skmacy * expecting the ABORT_REQ will fail with CPL_ERR_ABORT_FAILED, 2801174641Skmacy * but FW turns the ABORT_REQ into a regular one and so we get 2802174641Skmacy * ABORT_RPL_RSS with status 0 and no socket. Only on T3A. 2803174641Skmacy */ 2804174641Skmacy if (!toep) 2805174641Skmacy goto discard; 2806174641Skmacy 2807174641Skmacy if (toep->tp_tp == NULL) { 2808178302Skmacy log(LOG_NOTICE, "removing tid for abort\n"); 2809174641Skmacy cxgb_remove_tid(cdev, toep, toep->tp_tid); 2810174641Skmacy if (toep->tp_l2t) 2811174641Skmacy l2t_release(L2DATA(cdev), toep->tp_l2t); 2812174641Skmacy 2813174641Skmacy toepcb_release(toep); 2814174641Skmacy goto discard; 2815174641Skmacy } 2816174641Skmacy 2817178302Skmacy log(LOG_NOTICE, "toep=%p\n", toep); 2818178302Skmacy log(LOG_NOTICE, "tp=%p\n", toep->tp_tp); 2819174641Skmacy 2820174641Skmacy toepcb_hold(toep); 2821178302Skmacy process_abort_rpl(toep, m); 2822174641Skmacy toepcb_release(toep); 2823174641Skmacy return (0); 2824174641Skmacy} 2825174641Skmacy 2826174641Skmacy/* 2827176472Skmacy * Convert the status code of an ABORT_REQ into a FreeBSD error code. Also 2828174641Skmacy * indicate whether RST should be sent in response. 2829174641Skmacy */ 2830174641Skmacystatic int 2831174641Skmacyabort_status_to_errno(struct socket *so, int abort_reason, int *need_rst) 2832174641Skmacy{ 2833178302Skmacy struct tcpcb *tp = so_sototcpcb(so); 2834174641Skmacy 2835174641Skmacy switch (abort_reason) { 2836174641Skmacy case CPL_ERR_BAD_SYN: 2837174641Skmacy#if 0 2838174641Skmacy NET_INC_STATS_BH(LINUX_MIB_TCPABORTONSYN); // fall through 2839174641Skmacy#endif 2840174641Skmacy case CPL_ERR_CONN_RESET: 2841174641Skmacy // XXX need to handle SYN_RECV due to crossed SYNs 2842174641Skmacy return (tp->t_state == TCPS_CLOSE_WAIT ? EPIPE : ECONNRESET); 2843174641Skmacy case CPL_ERR_XMIT_TIMEDOUT: 2844174641Skmacy case CPL_ERR_PERSIST_TIMEDOUT: 2845174641Skmacy case CPL_ERR_FINWAIT2_TIMEDOUT: 2846174641Skmacy case CPL_ERR_KEEPALIVE_TIMEDOUT: 2847174641Skmacy#if 0 2848174641Skmacy NET_INC_STATS_BH(LINUX_MIB_TCPABORTONTIMEOUT); 2849174641Skmacy#endif 2850174641Skmacy return (ETIMEDOUT); 2851174641Skmacy default: 2852174641Skmacy return (EIO); 2853174641Skmacy } 2854174641Skmacy} 2855174641Skmacy 2856174641Skmacystatic inline void 2857174641Skmacyset_abort_rpl_wr(struct mbuf *m, unsigned int tid, int cmd) 2858174641Skmacy{ 2859174641Skmacy struct cpl_abort_rpl *rpl = cplhdr(m); 2860174641Skmacy 2861174641Skmacy rpl->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_HOST_ABORT_CON_RPL)); 2862174641Skmacy rpl->wr.wr_lo = htonl(V_WR_TID(tid)); 2863174641Skmacy m->m_len = m->m_pkthdr.len = sizeof(*rpl); 2864174641Skmacy 2865174641Skmacy OPCODE_TID(rpl) = htonl(MK_OPCODE_TID(CPL_ABORT_RPL, tid)); 2866174641Skmacy rpl->cmd = cmd; 2867174641Skmacy} 2868174641Skmacy 2869174641Skmacystatic void 2870174641Skmacysend_deferred_abort_rpl(struct toedev *tdev, struct mbuf *m) 2871174641Skmacy{ 2872174641Skmacy struct mbuf *reply_mbuf; 2873174641Skmacy struct cpl_abort_req_rss *req = cplhdr(m); 2874174641Skmacy 2875174641Skmacy reply_mbuf = m_gethdr_nofail(sizeof(struct cpl_abort_rpl)); 2876174641Skmacy m_set_priority(m, CPL_PRIORITY_DATA); 2877174641Skmacy m->m_len = m->m_pkthdr.len = sizeof(struct cpl_abort_rpl); 2878174641Skmacy set_abort_rpl_wr(reply_mbuf, GET_TID(req), req->status); 2879174641Skmacy cxgb_ofld_send(TOM_DATA(tdev)->cdev, reply_mbuf); 2880174641Skmacy m_free(m); 2881174641Skmacy} 2882174641Skmacy 2883174641Skmacy/* 2884174641Skmacy * Returns whether an ABORT_REQ_RSS message is a negative advice. 2885174641Skmacy */ 2886174641Skmacystatic inline int 2887174641Skmacyis_neg_adv_abort(unsigned int status) 2888174641Skmacy{ 2889174641Skmacy return status == CPL_ERR_RTX_NEG_ADVICE || 2890174641Skmacy status == CPL_ERR_PERSIST_NEG_ADVICE; 2891174641Skmacy} 2892174641Skmacy 2893174641Skmacystatic void 2894174641Skmacysend_abort_rpl(struct mbuf *m, struct toedev *tdev, int rst_status) 2895174641Skmacy{ 2896174641Skmacy struct mbuf *reply_mbuf; 2897174641Skmacy struct cpl_abort_req_rss *req = cplhdr(m); 2898174641Skmacy 2899174641Skmacy reply_mbuf = m_gethdr(M_NOWAIT, MT_DATA); 2900174641Skmacy 2901174641Skmacy if (!reply_mbuf) { 2902174641Skmacy /* Defer the reply. Stick rst_status into req->cmd. */ 2903174641Skmacy req->status = rst_status; 2904174641Skmacy t3_defer_reply(m, tdev, send_deferred_abort_rpl); 2905174641Skmacy return; 2906174641Skmacy } 2907174641Skmacy 2908174641Skmacy m_set_priority(reply_mbuf, CPL_PRIORITY_DATA); 2909174641Skmacy set_abort_rpl_wr(reply_mbuf, GET_TID(req), rst_status); 2910174641Skmacy m_free(m); 2911174641Skmacy 2912174641Skmacy /* 2913174641Skmacy * XXX need to sync with ARP as for SYN_RECV connections we can send 2914174641Skmacy * these messages while ARP is pending. For other connection states 2915174641Skmacy * it's not a problem. 2916174641Skmacy */ 2917174641Skmacy cxgb_ofld_send(TOM_DATA(tdev)->cdev, reply_mbuf); 2918174641Skmacy} 2919174641Skmacy 2920174641Skmacy#ifdef notyet 2921174641Skmacystatic void 2922174641Skmacycleanup_syn_rcv_conn(struct socket *child, struct socket *parent) 2923174641Skmacy{ 2924176507Skmacy CXGB_UNIMPLEMENTED(); 2925174641Skmacy#ifdef notyet 2926174641Skmacy struct request_sock *req = child->sk_user_data; 2927174641Skmacy 2928174641Skmacy inet_csk_reqsk_queue_removed(parent, req); 2929174641Skmacy synq_remove(tcp_sk(child)); 2930174641Skmacy __reqsk_free(req); 2931174641Skmacy child->sk_user_data = NULL; 2932174641Skmacy#endif 2933174641Skmacy} 2934174641Skmacy 2935174641Skmacy 2936174641Skmacy/* 2937174641Skmacy * Performs the actual work to abort a SYN_RECV connection. 2938174641Skmacy */ 2939174641Skmacystatic void 2940174641Skmacydo_abort_syn_rcv(struct socket *child, struct socket *parent) 2941174641Skmacy{ 2942178302Skmacy struct tcpcb *parenttp = so_sototcpcb(parent); 2943178302Skmacy struct tcpcb *childtp = so_sototcpcb(child); 2944174641Skmacy 2945174641Skmacy /* 2946174641Skmacy * If the server is still open we clean up the child connection, 2947174641Skmacy * otherwise the server already did the clean up as it was purging 2948174641Skmacy * its SYN queue and the skb was just sitting in its backlog. 2949174641Skmacy */ 2950174641Skmacy if (__predict_false(parenttp->t_state == TCPS_LISTEN)) { 2951174641Skmacy cleanup_syn_rcv_conn(child, parent); 2952177530Skmacy inp_wlock(childtp->t_inpcb); 2953174641Skmacy t3_release_offload_resources(childtp->t_toe); 2954178302Skmacy inp_wunlock(childtp->t_inpcb); 2955178302Skmacy tcp_offload_close(childtp); 2956174641Skmacy } 2957174641Skmacy} 2958174641Skmacy#endif 2959174641Skmacy 2960174641Skmacy/* 2961174641Skmacy * Handle abort requests for a SYN_RECV connection. These need extra work 2962174641Skmacy * because the socket is on its parent's SYN queue. 2963174641Skmacy */ 2964174641Skmacystatic int 2965174641Skmacyabort_syn_rcv(struct socket *so, struct mbuf *m) 2966174641Skmacy{ 2967176507Skmacy CXGB_UNIMPLEMENTED(); 2968174641Skmacy#ifdef notyet 2969174641Skmacy struct socket *parent; 2970178302Skmacy struct toedev *tdev = toep->tp_toedev; 2971174641Skmacy struct t3cdev *cdev = TOM_DATA(tdev)->cdev; 2972174641Skmacy struct socket *oreq = so->so_incomp; 2973174641Skmacy struct t3c_tid_entry *t3c_stid; 2974174641Skmacy struct tid_info *t; 2975174641Skmacy 2976174641Skmacy if (!oreq) 2977174641Skmacy return -1; /* somehow we are not on the SYN queue */ 2978174641Skmacy 2979174641Skmacy t = &(T3C_DATA(cdev))->tid_maps; 2980174641Skmacy t3c_stid = lookup_stid(t, oreq->ts_recent); 2981174641Skmacy parent = ((struct listen_ctx *)t3c_stid->ctx)->lso; 2982174641Skmacy 2983178302Skmacy so_lock(parent); 2984174641Skmacy do_abort_syn_rcv(so, parent); 2985174641Skmacy send_abort_rpl(m, tdev, CPL_ABORT_NO_RST); 2986178302Skmacy so_unlock(parent); 2987174641Skmacy#endif 2988174641Skmacy return (0); 2989174641Skmacy} 2990174641Skmacy 2991174641Skmacy/* 2992174641Skmacy * Process abort requests. If we are waiting for an ABORT_RPL we ignore this 2993174641Skmacy * request except that we need to reply to it. 2994174641Skmacy */ 2995174641Skmacystatic void 2996178302Skmacyprocess_abort_req(struct toepcb *toep, struct mbuf *m, struct toedev *tdev) 2997174641Skmacy{ 2998174641Skmacy int rst_status = CPL_ABORT_NO_RST; 2999174641Skmacy const struct cpl_abort_req_rss *req = cplhdr(m); 3000178302Skmacy struct tcpcb *tp = toep->tp_tp; 3001178302Skmacy struct socket *so; 3002178302Skmacy int needclose = 0; 3003178302Skmacy 3004177530Skmacy inp_wlock(tp->t_inpcb); 3005178302Skmacy so = inp_inpcbtosocket(toep->tp_tp->t_inpcb); 3006174641Skmacy if ((toep->tp_flags & TP_ABORT_REQ_RCVD) == 0) { 3007174641Skmacy toep->tp_flags |= (TP_ABORT_REQ_RCVD|TP_ABORT_SHUTDOWN); 3008174641Skmacy m_free(m); 3009174641Skmacy goto skip; 3010174641Skmacy } 3011174641Skmacy 3012174641Skmacy toep->tp_flags &= ~TP_ABORT_REQ_RCVD; 3013174641Skmacy /* 3014174641Skmacy * Three cases to consider: 3015174641Skmacy * a) We haven't sent an abort_req; close the connection. 3016174641Skmacy * b) We have sent a post-close abort_req that will get to TP too late 3017174641Skmacy * and will generate a CPL_ERR_ABORT_FAILED reply. The reply will 3018174641Skmacy * be ignored and the connection should be closed now. 3019174641Skmacy * c) We have sent a regular abort_req that will get to TP too late. 3020174641Skmacy * That will generate an abort_rpl with status 0, wait for it. 3021174641Skmacy */ 3022174641Skmacy if (((toep->tp_flags & TP_ABORT_RPL_PENDING) == 0) || 3023178302Skmacy (is_t3a(toep->tp_toedev) && (toep->tp_flags & TP_CLOSE_CON_REQUESTED))) { 3024178302Skmacy int error; 3025178302Skmacy 3026178302Skmacy error = abort_status_to_errno(so, req->status, 3027174641Skmacy &rst_status); 3028178302Skmacy so_error_set(so, error); 3029178302Skmacy 3030178302Skmacy if (__predict_true((so_state_get(so) & SS_NOFDREF) == 0)) 3031178302Skmacy so_sorwakeup(so); 3032174641Skmacy /* 3033174641Skmacy * SYN_RECV needs special processing. If abort_syn_rcv() 3034174641Skmacy * returns 0 is has taken care of the abort. 3035174641Skmacy */ 3036174641Skmacy if ((tp->t_state == TCPS_SYN_RECEIVED) && !abort_syn_rcv(so, m)) 3037174641Skmacy goto skip; 3038174641Skmacy 3039174641Skmacy t3_release_offload_resources(toep); 3040178302Skmacy needclose = 1; 3041174641Skmacy } 3042178302Skmacy inp_wunlock(tp->t_inpcb); 3043178302Skmacy 3044178302Skmacy if (needclose) 3045178302Skmacy tcp_offload_close(tp); 3046178302Skmacy 3047174641Skmacy send_abort_rpl(m, tdev, rst_status); 3048174641Skmacy return; 3049174641Skmacyskip: 3050178302Skmacy inp_wunlock(tp->t_inpcb); 3051174641Skmacy} 3052174641Skmacy 3053174641Skmacy/* 3054174641Skmacy * Handle an ABORT_REQ_RSS CPL message. 3055174641Skmacy */ 3056174641Skmacystatic int 3057174641Skmacydo_abort_req(struct t3cdev *cdev, struct mbuf *m, void *ctx) 3058174641Skmacy{ 3059174641Skmacy const struct cpl_abort_req_rss *req = cplhdr(m); 3060174641Skmacy struct toepcb *toep = (struct toepcb *)ctx; 3061174641Skmacy 3062174641Skmacy if (is_neg_adv_abort(req->status)) { 3063174641Skmacy m_free(m); 3064174641Skmacy return (0); 3065174641Skmacy } 3066174641Skmacy 3067178302Skmacy log(LOG_NOTICE, "aborting tid=%d\n", toep->tp_tid); 3068174641Skmacy 3069174641Skmacy if ((toep->tp_flags & (TP_SYN_RCVD|TP_ABORT_REQ_RCVD)) == TP_SYN_RCVD) { 3070174641Skmacy cxgb_remove_tid(cdev, toep, toep->tp_tid); 3071174641Skmacy toep->tp_flags |= TP_ABORT_REQ_RCVD; 3072174641Skmacy 3073174641Skmacy send_abort_rpl(m, toep->tp_toedev, CPL_ABORT_NO_RST); 3074174641Skmacy if (toep->tp_l2t) 3075174641Skmacy l2t_release(L2DATA(cdev), toep->tp_l2t); 3076174641Skmacy 3077174641Skmacy /* 3078174641Skmacy * Unhook 3079174641Skmacy */ 3080174641Skmacy toep->tp_tp->t_toe = NULL; 3081174641Skmacy toep->tp_tp->t_flags &= ~TF_TOE; 3082174641Skmacy toep->tp_tp = NULL; 3083174641Skmacy /* 3084174641Skmacy * XXX need to call syncache_chkrst - but we don't 3085174641Skmacy * have a way of doing that yet 3086174641Skmacy */ 3087174641Skmacy toepcb_release(toep); 3088178302Skmacy log(LOG_ERR, "abort for unestablished connection :-(\n"); 3089174641Skmacy return (0); 3090174641Skmacy } 3091174641Skmacy if (toep->tp_tp == NULL) { 3092178302Skmacy log(LOG_NOTICE, "disconnected toepcb\n"); 3093174641Skmacy /* should be freed momentarily */ 3094174641Skmacy return (0); 3095174641Skmacy } 3096174641Skmacy 3097178302Skmacy 3098174641Skmacy toepcb_hold(toep); 3099178302Skmacy process_abort_req(toep, m, toep->tp_toedev); 3100174641Skmacy toepcb_release(toep); 3101174641Skmacy return (0); 3102174641Skmacy} 3103174641Skmacy#ifdef notyet 3104174641Skmacystatic void 3105174641Skmacypass_open_abort(struct socket *child, struct socket *parent, struct mbuf *m) 3106174641Skmacy{ 3107174641Skmacy struct toedev *tdev = TOE_DEV(parent); 3108174641Skmacy 3109174641Skmacy do_abort_syn_rcv(child, parent); 3110174641Skmacy if (tdev->tod_ttid == TOE_ID_CHELSIO_T3) { 3111174641Skmacy struct cpl_pass_accept_rpl *rpl = cplhdr(m); 3112174641Skmacy 3113174641Skmacy rpl->opt0h = htonl(F_TCAM_BYPASS); 3114174641Skmacy rpl->opt0l_status = htonl(CPL_PASS_OPEN_REJECT); 3115174641Skmacy cxgb_ofld_send(TOM_DATA(tdev)->cdev, m); 3116174641Skmacy } else 3117174641Skmacy m_free(m); 3118174641Skmacy} 3119174641Skmacy#endif 3120174641Skmacystatic void 3121174641Skmacyhandle_pass_open_arp_failure(struct socket *so, struct mbuf *m) 3122174641Skmacy{ 3123176507Skmacy CXGB_UNIMPLEMENTED(); 3124174641Skmacy 3125174641Skmacy#ifdef notyet 3126174641Skmacy struct t3cdev *cdev; 3127174641Skmacy struct socket *parent; 3128174641Skmacy struct socket *oreq; 3129174641Skmacy struct t3c_tid_entry *t3c_stid; 3130174641Skmacy struct tid_info *t; 3131178302Skmacy struct tcpcb *otp, *tp = so_sototcpcb(so); 3132174641Skmacy struct toepcb *toep = tp->t_toe; 3133174641Skmacy 3134174641Skmacy /* 3135174641Skmacy * If the connection is being aborted due to the parent listening 3136174641Skmacy * socket going away there's nothing to do, the ABORT_REQ will close 3137174641Skmacy * the connection. 3138174641Skmacy */ 3139174641Skmacy if (toep->tp_flags & TP_ABORT_RPL_PENDING) { 3140174641Skmacy m_free(m); 3141174641Skmacy return; 3142174641Skmacy } 3143174641Skmacy 3144174641Skmacy oreq = so->so_incomp; 3145178302Skmacy otp = so_sototcpcb(oreq); 3146174641Skmacy 3147174641Skmacy cdev = T3C_DEV(so); 3148174641Skmacy t = &(T3C_DATA(cdev))->tid_maps; 3149174641Skmacy t3c_stid = lookup_stid(t, otp->ts_recent); 3150174641Skmacy parent = ((struct listen_ctx *)t3c_stid->ctx)->lso; 3151174641Skmacy 3152178302Skmacy so_lock(parent); 3153174641Skmacy pass_open_abort(so, parent, m); 3154178302Skmacy so_unlock(parent); 3155174641Skmacy#endif 3156174641Skmacy} 3157174641Skmacy 3158174641Skmacy/* 3159174641Skmacy * Handle an ARP failure for a CPL_PASS_ACCEPT_RPL. This is treated similarly 3160174641Skmacy * to an ABORT_REQ_RSS in SYN_RECV as both events need to tear down a SYN_RECV 3161174641Skmacy * connection. 3162174641Skmacy */ 3163174641Skmacystatic void 3164174641Skmacypass_accept_rpl_arp_failure(struct t3cdev *cdev, struct mbuf *m) 3165174641Skmacy{ 3166174641Skmacy 3167174641Skmacy#ifdef notyet 3168174641Skmacy TCP_INC_STATS_BH(TCP_MIB_ATTEMPTFAILS); 3169174641Skmacy BLOG_SKB_CB(skb)->dev = TOE_DEV(skb->sk); 3170174641Skmacy#endif 3171174641Skmacy handle_pass_open_arp_failure(m_get_socket(m), m); 3172174641Skmacy} 3173174641Skmacy 3174174641Skmacy/* 3175174641Skmacy * Populate a reject CPL_PASS_ACCEPT_RPL WR. 3176174641Skmacy */ 3177174641Skmacystatic void 3178174641Skmacymk_pass_accept_rpl(struct mbuf *reply_mbuf, struct mbuf *req_mbuf) 3179174641Skmacy{ 3180174641Skmacy struct cpl_pass_accept_req *req = cplhdr(req_mbuf); 3181174641Skmacy struct cpl_pass_accept_rpl *rpl = cplhdr(reply_mbuf); 3182174641Skmacy unsigned int tid = GET_TID(req); 3183174641Skmacy 3184174641Skmacy m_set_priority(reply_mbuf, CPL_PRIORITY_SETUP); 3185174641Skmacy rpl->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 3186174641Skmacy OPCODE_TID(rpl) = htonl(MK_OPCODE_TID(CPL_PASS_ACCEPT_RPL, tid)); 3187174641Skmacy rpl->peer_ip = req->peer_ip; // req->peer_ip not overwritten yet 3188174641Skmacy rpl->opt0h = htonl(F_TCAM_BYPASS); 3189174641Skmacy rpl->opt0l_status = htonl(CPL_PASS_OPEN_REJECT); 3190174641Skmacy rpl->opt2 = 0; 3191174641Skmacy rpl->rsvd = rpl->opt2; /* workaround for HW bug */ 3192174641Skmacy} 3193174641Skmacy 3194174641Skmacy/* 3195174641Skmacy * Send a deferred reject to an accept request. 3196174641Skmacy */ 3197174641Skmacystatic void 3198174641Skmacyreject_pass_request(struct toedev *tdev, struct mbuf *m) 3199174641Skmacy{ 3200174641Skmacy struct mbuf *reply_mbuf; 3201174641Skmacy 3202174641Skmacy reply_mbuf = m_gethdr_nofail(sizeof(struct cpl_pass_accept_rpl)); 3203174641Skmacy mk_pass_accept_rpl(reply_mbuf, m); 3204174641Skmacy cxgb_ofld_send(TOM_DATA(tdev)->cdev, reply_mbuf); 3205174641Skmacy m_free(m); 3206174641Skmacy} 3207174641Skmacy 3208174641Skmacystatic void 3209174641Skmacyhandle_syncache_event(int event, void *arg) 3210174641Skmacy{ 3211174641Skmacy struct toepcb *toep = arg; 3212174641Skmacy 3213174641Skmacy switch (event) { 3214174708Skmacy case TOE_SC_ENTRY_PRESENT: 3215174641Skmacy /* 3216174641Skmacy * entry already exists - free toepcb 3217174641Skmacy * and l2t 3218174641Skmacy */ 3219174641Skmacy printf("syncache entry present\n"); 3220174641Skmacy toepcb_release(toep); 3221174641Skmacy break; 3222174708Skmacy case TOE_SC_DROP: 3223174641Skmacy /* 3224174641Skmacy * The syncache has given up on this entry 3225174641Skmacy * either it timed out, or it was evicted 3226174641Skmacy * we need to explicitly release the tid 3227174641Skmacy */ 3228174641Skmacy printf("syncache entry dropped\n"); 3229174641Skmacy toepcb_release(toep); 3230174641Skmacy break; 3231174641Skmacy default: 3232174641Skmacy log(LOG_ERR, "unknown syncache event %d\n", event); 3233174641Skmacy break; 3234174641Skmacy } 3235174641Skmacy} 3236174641Skmacy 3237174641Skmacystatic void 3238174641Skmacysyncache_add_accept_req(struct cpl_pass_accept_req *req, struct socket *lso, struct toepcb *toep) 3239174641Skmacy{ 3240174641Skmacy struct in_conninfo inc; 3241174641Skmacy struct tcpopt to; 3242174641Skmacy struct tcphdr th; 3243174641Skmacy struct inpcb *inp; 3244174641Skmacy int mss, wsf, sack, ts; 3245176472Skmacy uint32_t rcv_isn = ntohl(req->rcv_isn); 3246176472Skmacy 3247174641Skmacy bzero(&to, sizeof(struct tcpopt)); 3248178302Skmacy inp = so_sotoinpcb(lso); 3249174641Skmacy 3250174641Skmacy /* 3251174641Skmacy * Fill out information for entering us into the syncache 3252174641Skmacy */ 3253174641Skmacy inc.inc_fport = th.th_sport = req->peer_port; 3254174641Skmacy inc.inc_lport = th.th_dport = req->local_port; 3255176472Skmacy th.th_seq = req->rcv_isn; 3256174641Skmacy th.th_flags = TH_SYN; 3257174641Skmacy 3258176472Skmacy toep->tp_iss = toep->tp_delack_seq = toep->tp_rcv_wup = toep->tp_copied_seq = rcv_isn + 1; 3259176472Skmacy 3260174641Skmacy 3261174641Skmacy inc.inc_isipv6 = 0; 3262174641Skmacy inc.inc_len = 0; 3263174641Skmacy inc.inc_faddr.s_addr = req->peer_ip; 3264174641Skmacy inc.inc_laddr.s_addr = req->local_ip; 3265174641Skmacy 3266174641Skmacy DPRINTF("syncache add of %d:%d %d:%d\n", 3267174641Skmacy ntohl(req->local_ip), ntohs(req->local_port), 3268174641Skmacy ntohl(req->peer_ip), ntohs(req->peer_port)); 3269174641Skmacy 3270174641Skmacy mss = req->tcp_options.mss; 3271174641Skmacy wsf = req->tcp_options.wsf; 3272174641Skmacy ts = req->tcp_options.tstamp; 3273174641Skmacy sack = req->tcp_options.sack; 3274174641Skmacy to.to_mss = mss; 3275174641Skmacy to.to_wscale = wsf; 3276174641Skmacy to.to_flags = (mss ? TOF_MSS : 0) | (wsf ? TOF_SCALE : 0) | (ts ? TOF_TS : 0) | (sack ? TOF_SACKPERM : 0); 3277180649Skmacy tcp_offload_syncache_add(&inc, &to, &th, inp, &lso, &cxgb_toe_usrreqs, toep); 3278174641Skmacy} 3279174641Skmacy 3280174641Skmacy 3281174641Skmacy/* 3282174641Skmacy * Process a CPL_PASS_ACCEPT_REQ message. Does the part that needs the socket 3283174641Skmacy * lock held. Note that the sock here is a listening socket that is not owned 3284174641Skmacy * by the TOE. 3285174641Skmacy */ 3286174641Skmacystatic void 3287174641Skmacyprocess_pass_accept_req(struct socket *so, struct mbuf *m, struct toedev *tdev, 3288174641Skmacy struct listen_ctx *lctx) 3289174641Skmacy{ 3290174641Skmacy int rt_flags; 3291174641Skmacy struct l2t_entry *e; 3292174641Skmacy struct iff_mac tim; 3293174641Skmacy struct mbuf *reply_mbuf, *ddp_mbuf = NULL; 3294174641Skmacy struct cpl_pass_accept_rpl *rpl; 3295174641Skmacy struct cpl_pass_accept_req *req = cplhdr(m); 3296174641Skmacy unsigned int tid = GET_TID(req); 3297174641Skmacy struct tom_data *d = TOM_DATA(tdev); 3298174641Skmacy struct t3cdev *cdev = d->cdev; 3299178302Skmacy struct tcpcb *tp = so_sototcpcb(so); 3300174641Skmacy struct toepcb *newtoep; 3301174641Skmacy struct rtentry *dst; 3302174641Skmacy struct sockaddr_in nam; 3303174641Skmacy struct t3c_data *td = T3C_DATA(cdev); 3304174641Skmacy 3305174641Skmacy reply_mbuf = m_gethdr(M_NOWAIT, MT_DATA); 3306174641Skmacy if (__predict_false(reply_mbuf == NULL)) { 3307174641Skmacy if (tdev->tod_ttid == TOE_ID_CHELSIO_T3) 3308174641Skmacy t3_defer_reply(m, tdev, reject_pass_request); 3309174641Skmacy else { 3310174641Skmacy cxgb_queue_tid_release(cdev, tid); 3311174641Skmacy m_free(m); 3312174641Skmacy } 3313174641Skmacy DPRINTF("failed to get reply_mbuf\n"); 3314174641Skmacy 3315174641Skmacy goto out; 3316174641Skmacy } 3317174641Skmacy 3318174641Skmacy if (tp->t_state != TCPS_LISTEN) { 3319174641Skmacy DPRINTF("socket not in listen state\n"); 3320174641Skmacy 3321174641Skmacy goto reject; 3322174641Skmacy } 3323174641Skmacy 3324174641Skmacy tim.mac_addr = req->dst_mac; 3325174641Skmacy tim.vlan_tag = ntohs(req->vlan_tag); 3326174641Skmacy if (cdev->ctl(cdev, GET_IFF_FROM_MAC, &tim) < 0 || !tim.dev) { 3327174641Skmacy DPRINTF("rejecting from failed GET_IFF_FROM_MAC\n"); 3328174641Skmacy goto reject; 3329174641Skmacy } 3330174641Skmacy 3331174641Skmacy#ifdef notyet 3332174641Skmacy /* 3333174641Skmacy * XXX do route lookup to confirm that we're still listening on this 3334174641Skmacy * address 3335174641Skmacy */ 3336174641Skmacy if (ip_route_input(skb, req->local_ip, req->peer_ip, 3337174641Skmacy G_PASS_OPEN_TOS(ntohl(req->tos_tid)), tim.dev)) 3338174641Skmacy goto reject; 3339174641Skmacy rt_flags = ((struct rtable *)skb->dst)->rt_flags & 3340174641Skmacy (RTCF_BROADCAST | RTCF_MULTICAST | RTCF_LOCAL); 3341174641Skmacy dst_release(skb->dst); // done with the input route, release it 3342174641Skmacy skb->dst = NULL; 3343174641Skmacy 3344174641Skmacy if ((rt_flags & RTF_LOCAL) == 0) 3345174641Skmacy goto reject; 3346174641Skmacy#endif 3347174641Skmacy /* 3348174641Skmacy * XXX 3349174641Skmacy */ 3350174641Skmacy rt_flags = RTF_LOCAL; 3351174641Skmacy if ((rt_flags & RTF_LOCAL) == 0) 3352174641Skmacy goto reject; 3353174641Skmacy 3354174641Skmacy /* 3355174641Skmacy * Calculate values and add to syncache 3356174641Skmacy */ 3357174641Skmacy 3358174641Skmacy newtoep = toepcb_alloc(); 3359174641Skmacy if (newtoep == NULL) 3360174641Skmacy goto reject; 3361174641Skmacy 3362174641Skmacy bzero(&nam, sizeof(struct sockaddr_in)); 3363174641Skmacy 3364174641Skmacy nam.sin_len = sizeof(struct sockaddr_in); 3365174641Skmacy nam.sin_family = AF_INET; 3366174641Skmacy nam.sin_addr.s_addr =req->peer_ip; 3367174641Skmacy dst = rtalloc2((struct sockaddr *)&nam, 1, 0); 3368174641Skmacy 3369174641Skmacy if (dst == NULL) { 3370174641Skmacy printf("failed to find route\n"); 3371174641Skmacy goto reject; 3372174641Skmacy } 3373174641Skmacy e = newtoep->tp_l2t = t3_l2t_get(d->cdev, dst, tim.dev, 3374174641Skmacy (struct sockaddr *)&nam); 3375174641Skmacy if (e == NULL) { 3376174641Skmacy DPRINTF("failed to get l2t\n"); 3377174641Skmacy } 3378174641Skmacy /* 3379174641Skmacy * Point to our listen socket until accept 3380174641Skmacy */ 3381174641Skmacy newtoep->tp_tp = tp; 3382174641Skmacy newtoep->tp_flags = TP_SYN_RCVD; 3383174641Skmacy newtoep->tp_tid = tid; 3384174641Skmacy newtoep->tp_toedev = tdev; 3385176472Skmacy tp->rcv_wnd = select_rcv_wnd(tdev, so); 3386174641Skmacy 3387174641Skmacy cxgb_insert_tid(cdev, d->client, newtoep, tid); 3388178302Skmacy so_lock(so); 3389174641Skmacy LIST_INSERT_HEAD(&lctx->synq_head, newtoep, synq_entry); 3390178302Skmacy so_unlock(so); 3391178302Skmacy 3392178302Skmacy newtoep->tp_ulp_mode = TOM_TUNABLE(tdev, ddp) && !(so_options_get(so) & SO_NO_DDP) && 3393176472Skmacy tp->rcv_wnd >= MIN_DDP_RCV_WIN ? ULP_MODE_TCPDDP : 0; 3394176472Skmacy 3395176472Skmacy if (newtoep->tp_ulp_mode) { 3396174641Skmacy ddp_mbuf = m_gethdr(M_NOWAIT, MT_DATA); 3397174641Skmacy 3398176472Skmacy if (ddp_mbuf == NULL) 3399174641Skmacy newtoep->tp_ulp_mode = 0; 3400174641Skmacy } 3401176472Skmacy 3402176472Skmacy CTR4(KTR_TOM, "ddp=%d rcv_wnd=%ld min_win=%d ulp_mode=%d", 3403176472Skmacy TOM_TUNABLE(tdev, ddp), tp->rcv_wnd, MIN_DDP_RCV_WIN, newtoep->tp_ulp_mode); 3404174641Skmacy set_arp_failure_handler(reply_mbuf, pass_accept_rpl_arp_failure); 3405174641Skmacy /* 3406174641Skmacy * XXX workaround for lack of syncache drop 3407174641Skmacy */ 3408174641Skmacy toepcb_hold(newtoep); 3409174641Skmacy syncache_add_accept_req(req, so, newtoep); 3410174641Skmacy 3411174641Skmacy rpl = cplhdr(reply_mbuf); 3412174641Skmacy reply_mbuf->m_pkthdr.len = reply_mbuf->m_len = sizeof(*rpl); 3413174641Skmacy rpl->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 3414174641Skmacy rpl->wr.wr_lo = 0; 3415174641Skmacy OPCODE_TID(rpl) = htonl(MK_OPCODE_TID(CPL_PASS_ACCEPT_RPL, tid)); 3416174641Skmacy rpl->opt2 = htonl(calc_opt2(so, tdev)); 3417174641Skmacy rpl->rsvd = rpl->opt2; /* workaround for HW bug */ 3418174641Skmacy rpl->peer_ip = req->peer_ip; // req->peer_ip is not overwritten 3419174641Skmacy 3420174641Skmacy rpl->opt0h = htonl(calc_opt0h(so, select_mss(td, NULL, dst->rt_ifp->if_mtu)) | 3421174641Skmacy V_L2T_IDX(e->idx) | V_TX_CHANNEL(e->smt_idx)); 3422176472Skmacy rpl->opt0l_status = htonl(calc_opt0l(so, newtoep->tp_ulp_mode) | 3423174641Skmacy CPL_PASS_OPEN_ACCEPT); 3424174641Skmacy 3425174641Skmacy DPRINTF("opt0l_status=%08x\n", rpl->opt0l_status); 3426174641Skmacy 3427176472Skmacy m_set_priority(reply_mbuf, mkprio(CPL_PRIORITY_SETUP, newtoep)); 3428174641Skmacy 3429174641Skmacy l2t_send(cdev, reply_mbuf, e); 3430174641Skmacy m_free(m); 3431176472Skmacy if (newtoep->tp_ulp_mode) { 3432176472Skmacy __set_tcb_field(newtoep, ddp_mbuf, W_TCB_RX_DDP_FLAGS, 3433174641Skmacy V_TF_DDP_OFF(1) | 3434174641Skmacy TP_DDP_TIMER_WORKAROUND_MASK, 3435174641Skmacy V_TF_DDP_OFF(1) | 3436176472Skmacy TP_DDP_TIMER_WORKAROUND_VAL, 1); 3437176472Skmacy } else 3438176472Skmacy printf("not offloading\n"); 3439176472Skmacy 3440176472Skmacy 3441174641Skmacy 3442174641Skmacy return; 3443174641Skmacyreject: 3444174641Skmacy if (tdev->tod_ttid == TOE_ID_CHELSIO_T3) 3445174641Skmacy mk_pass_accept_rpl(reply_mbuf, m); 3446174641Skmacy else 3447176472Skmacy mk_tid_release(reply_mbuf, newtoep, tid); 3448174641Skmacy cxgb_ofld_send(cdev, reply_mbuf); 3449174641Skmacy m_free(m); 3450174641Skmacyout: 3451174641Skmacy#if 0 3452174641Skmacy TCP_INC_STATS_BH(TCP_MIB_ATTEMPTFAILS); 3453174641Skmacy#else 3454174641Skmacy return; 3455174641Skmacy#endif 3456174641Skmacy} 3457174641Skmacy 3458174641Skmacy/* 3459174641Skmacy * Handle a CPL_PASS_ACCEPT_REQ message. 3460174641Skmacy */ 3461174641Skmacystatic int 3462174641Skmacydo_pass_accept_req(struct t3cdev *cdev, struct mbuf *m, void *ctx) 3463174641Skmacy{ 3464174641Skmacy struct listen_ctx *listen_ctx = (struct listen_ctx *)ctx; 3465178302Skmacy struct socket *lso = listen_ctx->lso; /* XXX need an interlock against the listen socket going away */ 3466174641Skmacy struct tom_data *d = listen_ctx->tom_data; 3467174641Skmacy 3468174641Skmacy#if VALIDATE_TID 3469174641Skmacy struct cpl_pass_accept_req *req = cplhdr(m); 3470174641Skmacy unsigned int tid = GET_TID(req); 3471174641Skmacy struct tid_info *t = &(T3C_DATA(cdev))->tid_maps; 3472174641Skmacy 3473174641Skmacy if (unlikely(!lsk)) { 3474174641Skmacy printk(KERN_ERR "%s: PASS_ACCEPT_REQ had unknown STID %lu\n", 3475174641Skmacy cdev->name, 3476174641Skmacy (unsigned long)((union listen_entry *)ctx - 3477174641Skmacy t->stid_tab)); 3478174641Skmacy return CPL_RET_BUF_DONE; 3479174641Skmacy } 3480174641Skmacy if (unlikely(tid >= t->ntids)) { 3481174641Skmacy printk(KERN_ERR "%s: passive open TID %u too large\n", 3482174641Skmacy cdev->name, tid); 3483174641Skmacy return CPL_RET_BUF_DONE; 3484174641Skmacy } 3485174641Skmacy /* 3486174641Skmacy * For T3A the current user of the TID may have closed but its last 3487174641Skmacy * message(s) may have been backlogged so the TID appears to be still 3488174641Skmacy * in use. Just take the TID away, the connection can close at its 3489174641Skmacy * own leisure. For T3B this situation is a bug. 3490174641Skmacy */ 3491174641Skmacy if (!valid_new_tid(t, tid) && 3492174641Skmacy cdev->type != T3A) { 3493174641Skmacy printk(KERN_ERR "%s: passive open uses existing TID %u\n", 3494174641Skmacy cdev->name, tid); 3495174641Skmacy return CPL_RET_BUF_DONE; 3496174641Skmacy } 3497174641Skmacy#endif 3498174641Skmacy 3499174641Skmacy process_pass_accept_req(lso, m, &d->tdev, listen_ctx); 3500174641Skmacy return (0); 3501174641Skmacy} 3502174641Skmacy 3503174641Skmacy/* 3504174641Skmacy * Called when a connection is established to translate the TCP options 3505176472Skmacy * reported by HW to FreeBSD's native format. 3506174641Skmacy */ 3507174641Skmacystatic void 3508174641Skmacyassign_rxopt(struct socket *so, unsigned int opt) 3509174641Skmacy{ 3510178302Skmacy struct tcpcb *tp = so_sototcpcb(so); 3511174641Skmacy struct toepcb *toep = tp->t_toe; 3512178302Skmacy const struct t3c_data *td = T3C_DATA(TOEP_T3C_DEV(toep)); 3513174641Skmacy 3514177575Skmacy inp_lock_assert(tp->t_inpcb); 3515174641Skmacy 3516174641Skmacy toep->tp_mss_clamp = td->mtus[G_TCPOPT_MSS(opt)] - 40; 3517174641Skmacy tp->t_flags |= G_TCPOPT_TSTAMP(opt) ? TF_RCVD_TSTMP : 0; 3518174641Skmacy tp->t_flags |= G_TCPOPT_SACK(opt) ? TF_SACK_PERMIT : 0; 3519174641Skmacy tp->t_flags |= G_TCPOPT_WSCALE_OK(opt) ? TF_RCVD_SCALE : 0; 3520176472Skmacy if ((tp->t_flags & (TF_RCVD_SCALE|TF_REQ_SCALE)) == 3521176472Skmacy (TF_RCVD_SCALE|TF_REQ_SCALE)) 3522176472Skmacy tp->rcv_scale = tp->request_r_scale; 3523174641Skmacy} 3524174641Skmacy 3525174641Skmacy/* 3526174641Skmacy * Completes some final bits of initialization for just established connections 3527174641Skmacy * and changes their state to TCP_ESTABLISHED. 3528174641Skmacy * 3529174641Skmacy * snd_isn here is the ISN after the SYN, i.e., the true ISN + 1. 3530174641Skmacy */ 3531174641Skmacystatic void 3532174641Skmacymake_established(struct socket *so, u32 snd_isn, unsigned int opt) 3533174641Skmacy{ 3534178302Skmacy struct tcpcb *tp = so_sototcpcb(so); 3535174641Skmacy struct toepcb *toep = tp->t_toe; 3536174641Skmacy 3537174641Skmacy toep->tp_write_seq = tp->iss = tp->snd_max = tp->snd_nxt = tp->snd_una = snd_isn; 3538174641Skmacy assign_rxopt(so, opt); 3539178302Skmacy 3540178302Skmacy /* 3541178302Skmacy *XXXXXXXXXXX 3542178302Skmacy * 3543178302Skmacy */ 3544178302Skmacy#ifdef notyet 3545174641Skmacy so->so_proto->pr_ctloutput = t3_ctloutput; 3546178302Skmacy#endif 3547174641Skmacy 3548174641Skmacy#if 0 3549174641Skmacy inet_sk(sk)->id = tp->write_seq ^ jiffies; 3550174641Skmacy#endif 3551174641Skmacy /* 3552174641Skmacy * XXX not clear what rcv_wup maps to 3553174641Skmacy */ 3554174641Skmacy /* 3555174641Skmacy * Causes the first RX_DATA_ACK to supply any Rx credits we couldn't 3556174641Skmacy * pass through opt0. 3557174641Skmacy */ 3558174641Skmacy if (tp->rcv_wnd > (M_RCV_BUFSIZ << 10)) 3559174641Skmacy toep->tp_rcv_wup -= tp->rcv_wnd - (M_RCV_BUFSIZ << 10); 3560174641Skmacy 3561174641Skmacy dump_toepcb(toep); 3562174641Skmacy 3563174641Skmacy#ifdef notyet 3564174641Skmacy/* 3565174641Skmacy * no clean interface for marking ARP up to date 3566174641Skmacy */ 3567174641Skmacy dst_confirm(sk->sk_dst_cache); 3568174641Skmacy#endif 3569176472Skmacy tp->t_starttime = ticks; 3570174641Skmacy tp->t_state = TCPS_ESTABLISHED; 3571176472Skmacy soisconnected(so); 3572174641Skmacy} 3573174641Skmacy 3574174641Skmacystatic int 3575174641Skmacysyncache_expand_establish_req(struct cpl_pass_establish *req, struct socket **so, struct toepcb *toep) 3576174641Skmacy{ 3577174641Skmacy 3578174641Skmacy struct in_conninfo inc; 3579174641Skmacy struct tcpopt to; 3580174641Skmacy struct tcphdr th; 3581174641Skmacy int mss, wsf, sack, ts; 3582174641Skmacy struct mbuf *m = NULL; 3583174641Skmacy const struct t3c_data *td = T3C_DATA(TOM_DATA(toep->tp_toedev)->cdev); 3584174641Skmacy unsigned int opt; 3585174641Skmacy 3586174641Skmacy#ifdef MAC 3587174641Skmacy#error "no MAC support" 3588174641Skmacy#endif 3589174641Skmacy 3590174641Skmacy opt = ntohs(req->tcp_opt); 3591174641Skmacy 3592174641Skmacy bzero(&to, sizeof(struct tcpopt)); 3593174641Skmacy 3594174641Skmacy /* 3595174641Skmacy * Fill out information for entering us into the syncache 3596174641Skmacy */ 3597174641Skmacy inc.inc_fport = th.th_sport = req->peer_port; 3598174641Skmacy inc.inc_lport = th.th_dport = req->local_port; 3599174641Skmacy th.th_seq = req->rcv_isn; 3600174641Skmacy th.th_flags = TH_ACK; 3601174641Skmacy 3602174641Skmacy inc.inc_isipv6 = 0; 3603174641Skmacy inc.inc_len = 0; 3604174641Skmacy inc.inc_faddr.s_addr = req->peer_ip; 3605174641Skmacy inc.inc_laddr.s_addr = req->local_ip; 3606174641Skmacy 3607174641Skmacy mss = td->mtus[G_TCPOPT_MSS(opt)] - 40; 3608174641Skmacy wsf = G_TCPOPT_WSCALE_OK(opt); 3609174641Skmacy ts = G_TCPOPT_TSTAMP(opt); 3610174641Skmacy sack = G_TCPOPT_SACK(opt); 3611174641Skmacy 3612174641Skmacy to.to_mss = mss; 3613174641Skmacy to.to_wscale = G_TCPOPT_SND_WSCALE(opt); 3614174641Skmacy to.to_flags = (mss ? TOF_MSS : 0) | (wsf ? TOF_SCALE : 0) | (ts ? TOF_TS : 0) | (sack ? TOF_SACKPERM : 0); 3615174641Skmacy 3616174641Skmacy DPRINTF("syncache expand of %d:%d %d:%d mss:%d wsf:%d ts:%d sack:%d\n", 3617174641Skmacy ntohl(req->local_ip), ntohs(req->local_port), 3618174641Skmacy ntohl(req->peer_ip), ntohs(req->peer_port), 3619174641Skmacy mss, wsf, ts, sack); 3620180649Skmacy return tcp_offload_syncache_expand(&inc, &to, &th, so, m); 3621174641Skmacy} 3622174641Skmacy 3623174641Skmacy 3624174641Skmacy/* 3625174641Skmacy * Process a CPL_PASS_ESTABLISH message. XXX a lot of the locking doesn't work 3626174641Skmacy * if we are in TCP_SYN_RECV due to crossed SYNs 3627174641Skmacy */ 3628174641Skmacystatic int 3629174641Skmacydo_pass_establish(struct t3cdev *cdev, struct mbuf *m, void *ctx) 3630174641Skmacy{ 3631174641Skmacy struct cpl_pass_establish *req = cplhdr(m); 3632174641Skmacy struct toepcb *toep = (struct toepcb *)ctx; 3633178302Skmacy struct tcpcb *tp = toep->tp_tp; 3634174641Skmacy struct socket *so, *lso; 3635174641Skmacy struct t3c_data *td = T3C_DATA(cdev); 3636178302Skmacy struct sockbuf *snd, *rcv; 3637178302Skmacy 3638174641Skmacy // Complete socket initialization now that we have the SND_ISN 3639174641Skmacy 3640174641Skmacy struct toedev *tdev; 3641174641Skmacy 3642178302Skmacy 3643174641Skmacy tdev = toep->tp_toedev; 3644174641Skmacy 3645178302Skmacy inp_wlock(tp->t_inpcb); 3646178302Skmacy 3647178302Skmacy /* 3648178302Skmacy * 3649178302Skmacy * XXX need to add reference while we're manipulating 3650178302Skmacy */ 3651178302Skmacy so = lso = inp_inpcbtosocket(tp->t_inpcb); 3652178302Skmacy 3653178302Skmacy inp_wunlock(tp->t_inpcb); 3654178302Skmacy 3655178302Skmacy so_lock(so); 3656174641Skmacy LIST_REMOVE(toep, synq_entry); 3657178302Skmacy so_unlock(so); 3658174641Skmacy 3659174641Skmacy if (!syncache_expand_establish_req(req, &so, toep)) { 3660174641Skmacy /* 3661174641Skmacy * No entry 3662174641Skmacy */ 3663176507Skmacy CXGB_UNIMPLEMENTED(); 3664174641Skmacy } 3665174641Skmacy if (so == NULL) { 3666174641Skmacy /* 3667174641Skmacy * Couldn't create the socket 3668174641Skmacy */ 3669176507Skmacy CXGB_UNIMPLEMENTED(); 3670174641Skmacy } 3671174641Skmacy 3672178302Skmacy tp = so_sototcpcb(so); 3673177530Skmacy inp_wlock(tp->t_inpcb); 3674176472Skmacy 3675178767Skmacy snd = so_sockbuf_snd(so); 3676178767Skmacy rcv = so_sockbuf_rcv(so); 3677178767Skmacy 3678178302Skmacy snd->sb_flags |= SB_NOCOALESCE; 3679178302Skmacy rcv->sb_flags |= SB_NOCOALESCE; 3680176472Skmacy 3681174641Skmacy toep->tp_tp = tp; 3682174641Skmacy toep->tp_flags = 0; 3683174641Skmacy tp->t_toe = toep; 3684174641Skmacy reset_wr_list(toep); 3685176472Skmacy tp->rcv_wnd = select_rcv_wnd(tdev, so); 3686176472Skmacy tp->rcv_nxt = toep->tp_copied_seq; 3687174641Skmacy install_offload_ops(so); 3688174641Skmacy 3689174641Skmacy toep->tp_wr_max = toep->tp_wr_avail = TOM_TUNABLE(tdev, max_wrs); 3690174641Skmacy toep->tp_wr_unacked = 0; 3691174641Skmacy toep->tp_qset = G_QNUM(ntohl(m->m_pkthdr.csum_data)); 3692174641Skmacy toep->tp_qset_idx = 0; 3693174641Skmacy toep->tp_mtu_idx = select_mss(td, tp, toep->tp_l2t->neigh->rt_ifp->if_mtu); 3694174641Skmacy 3695174641Skmacy /* 3696174641Skmacy * XXX Cancel any keep alive timer 3697174641Skmacy */ 3698174641Skmacy 3699174641Skmacy make_established(so, ntohl(req->snd_isn), ntohs(req->tcp_opt)); 3700178302Skmacy 3701178302Skmacy /* 3702178302Skmacy * XXX workaround for lack of syncache drop 3703178302Skmacy */ 3704178302Skmacy toepcb_release(toep); 3705177530Skmacy inp_wunlock(tp->t_inpcb); 3706174641Skmacy 3707176472Skmacy CTR1(KTR_TOM, "do_pass_establish tid=%u", toep->tp_tid); 3708176472Skmacy cxgb_log_tcb(cdev->adapter, toep->tp_tid); 3709174641Skmacy#ifdef notyet 3710174641Skmacy /* 3711174641Skmacy * XXX not sure how these checks map to us 3712174641Skmacy */ 3713174641Skmacy if (unlikely(sk->sk_socket)) { // simultaneous opens only 3714174641Skmacy sk->sk_state_change(sk); 3715174641Skmacy sk_wake_async(so, 0, POLL_OUT); 3716174641Skmacy } 3717174641Skmacy /* 3718174641Skmacy * The state for the new connection is now up to date. 3719174641Skmacy * Next check if we should add the connection to the parent's 3720174641Skmacy * accept queue. When the parent closes it resets connections 3721174641Skmacy * on its SYN queue, so check if we are being reset. If so we 3722174641Skmacy * don't need to do anything more, the coming ABORT_RPL will 3723174641Skmacy * destroy this socket. Otherwise move the connection to the 3724174641Skmacy * accept queue. 3725174641Skmacy * 3726174641Skmacy * Note that we reset the synq before closing the server so if 3727174641Skmacy * we are not being reset the stid is still open. 3728174641Skmacy */ 3729174641Skmacy if (unlikely(!tp->forward_skb_hint)) { // removed from synq 3730174641Skmacy __kfree_skb(skb); 3731174641Skmacy goto unlock; 3732174641Skmacy } 3733174641Skmacy#endif 3734174641Skmacy m_free(m); 3735174641Skmacy 3736174641Skmacy return (0); 3737174641Skmacy} 3738174641Skmacy 3739174641Skmacy/* 3740174641Skmacy * Fill in the right TID for CPL messages waiting in the out-of-order queue 3741174641Skmacy * and send them to the TOE. 3742174641Skmacy */ 3743174641Skmacystatic void 3744178302Skmacyfixup_and_send_ofo(struct toepcb *toep) 3745174641Skmacy{ 3746174641Skmacy struct mbuf *m; 3747178302Skmacy struct toedev *tdev = toep->tp_toedev; 3748178302Skmacy struct tcpcb *tp = toep->tp_tp; 3749174641Skmacy unsigned int tid = toep->tp_tid; 3750174641Skmacy 3751178302Skmacy log(LOG_NOTICE, "fixup_and_send_ofo\n"); 3752174641Skmacy 3753177575Skmacy inp_lock_assert(tp->t_inpcb); 3754174641Skmacy while ((m = mbufq_dequeue(&toep->out_of_order_queue)) != NULL) { 3755174641Skmacy /* 3756174641Skmacy * A variety of messages can be waiting but the fields we'll 3757174641Skmacy * be touching are common to all so any message type will do. 3758174641Skmacy */ 3759174641Skmacy struct cpl_close_con_req *p = cplhdr(m); 3760174641Skmacy 3761174641Skmacy p->wr.wr_lo = htonl(V_WR_TID(tid)); 3762174641Skmacy OPCODE_TID(p) = htonl(MK_OPCODE_TID(p->ot.opcode, tid)); 3763174641Skmacy cxgb_ofld_send(TOM_DATA(tdev)->cdev, m); 3764174641Skmacy } 3765174641Skmacy} 3766174641Skmacy 3767174641Skmacy/* 3768174641Skmacy * Updates socket state from an active establish CPL message. Runs with the 3769174641Skmacy * socket lock held. 3770174641Skmacy */ 3771174641Skmacystatic void 3772174641Skmacysocket_act_establish(struct socket *so, struct mbuf *m) 3773174641Skmacy{ 3774174641Skmacy struct cpl_act_establish *req = cplhdr(m); 3775174641Skmacy u32 rcv_isn = ntohl(req->rcv_isn); /* real RCV_ISN + 1 */ 3776178302Skmacy struct tcpcb *tp = so_sototcpcb(so); 3777174641Skmacy struct toepcb *toep = tp->t_toe; 3778174641Skmacy 3779174641Skmacy if (__predict_false(tp->t_state != TCPS_SYN_SENT)) 3780174641Skmacy log(LOG_ERR, "TID %u expected SYN_SENT, found %d\n", 3781174641Skmacy toep->tp_tid, tp->t_state); 3782174641Skmacy 3783174641Skmacy tp->ts_recent_age = ticks; 3784174641Skmacy tp->irs = tp->rcv_wnd = tp->rcv_nxt = rcv_isn; 3785174641Skmacy toep->tp_delack_seq = toep->tp_rcv_wup = toep->tp_copied_seq = tp->irs; 3786174641Skmacy 3787174641Skmacy make_established(so, ntohl(req->snd_isn), ntohs(req->tcp_opt)); 3788174641Skmacy 3789174641Skmacy /* 3790174641Skmacy * Now that we finally have a TID send any CPL messages that we had to 3791174641Skmacy * defer for lack of a TID. 3792174641Skmacy */ 3793174641Skmacy if (mbufq_len(&toep->out_of_order_queue)) 3794178302Skmacy fixup_and_send_ofo(toep); 3795174641Skmacy 3796178302Skmacy if (__predict_false(so_state_get(so) & SS_NOFDREF)) { 3797176472Skmacy /* 3798176472Skmacy * XXX does this even make sense? 3799174641Skmacy */ 3800178302Skmacy so_sorwakeup(so); 3801174641Skmacy } 3802174641Skmacy m_free(m); 3803174641Skmacy#ifdef notyet 3804174641Skmacy/* 3805174641Skmacy * XXX assume no write requests permitted while socket connection is 3806174641Skmacy * incomplete 3807174641Skmacy */ 3808174641Skmacy /* 3809174641Skmacy * Currently the send queue must be empty at this point because the 3810174641Skmacy * socket layer does not send anything before a connection is 3811174641Skmacy * established. To be future proof though we handle the possibility 3812174641Skmacy * that there are pending buffers to send (either TX_DATA or 3813174641Skmacy * CLOSE_CON_REQ). First we need to adjust the sequence number of the 3814174641Skmacy * buffers according to the just learned write_seq, and then we send 3815174641Skmacy * them on their way. 3816174641Skmacy */ 3817174641Skmacy fixup_pending_writeq_buffers(sk); 3818174641Skmacy if (t3_push_frames(so, 1)) 3819174641Skmacy sk->sk_write_space(sk); 3820174641Skmacy#endif 3821174641Skmacy 3822176472Skmacy toep->tp_state = tp->t_state; 3823174641Skmacy tcpstat.tcps_connects++; 3824174641Skmacy 3825174641Skmacy} 3826174641Skmacy 3827174641Skmacy/* 3828174641Skmacy * Process a CPL_ACT_ESTABLISH message. 3829174641Skmacy */ 3830174641Skmacystatic int 3831174641Skmacydo_act_establish(struct t3cdev *cdev, struct mbuf *m, void *ctx) 3832174641Skmacy{ 3833174641Skmacy struct cpl_act_establish *req = cplhdr(m); 3834174641Skmacy unsigned int tid = GET_TID(req); 3835174641Skmacy unsigned int atid = G_PASS_OPEN_TID(ntohl(req->tos_tid)); 3836174641Skmacy struct toepcb *toep = (struct toepcb *)ctx; 3837174641Skmacy struct tcpcb *tp = toep->tp_tp; 3838174641Skmacy struct socket *so; 3839174641Skmacy struct toedev *tdev; 3840174641Skmacy struct tom_data *d; 3841174641Skmacy 3842174641Skmacy if (tp == NULL) { 3843174641Skmacy free_atid(cdev, atid); 3844174641Skmacy return (0); 3845174641Skmacy } 3846177530Skmacy inp_wlock(tp->t_inpcb); 3847178302Skmacy 3848174641Skmacy /* 3849178302Skmacy * XXX 3850178302Skmacy */ 3851178302Skmacy so = inp_inpcbtosocket(tp->t_inpcb); 3852178302Skmacy tdev = toep->tp_toedev; /* blow up here if link was down */ 3853178302Skmacy d = TOM_DATA(tdev); 3854178302Skmacy 3855178302Skmacy /* 3856174641Skmacy * It's OK if the TID is currently in use, the owning socket may have 3857174641Skmacy * backlogged its last CPL message(s). Just take it away. 3858174641Skmacy */ 3859174641Skmacy toep->tp_tid = tid; 3860174641Skmacy toep->tp_tp = tp; 3861178302Skmacy so_insert_tid(d, toep, tid); 3862174641Skmacy free_atid(cdev, atid); 3863174641Skmacy toep->tp_qset = G_QNUM(ntohl(m->m_pkthdr.csum_data)); 3864174641Skmacy 3865174641Skmacy socket_act_establish(so, m); 3866177530Skmacy inp_wunlock(tp->t_inpcb); 3867176472Skmacy CTR1(KTR_TOM, "do_act_establish tid=%u", toep->tp_tid); 3868176472Skmacy cxgb_log_tcb(cdev->adapter, toep->tp_tid); 3869176472Skmacy 3870174641Skmacy return (0); 3871174641Skmacy} 3872174641Skmacy 3873174641Skmacy/* 3874174641Skmacy * Process an acknowledgment of WR completion. Advance snd_una and send the 3875174641Skmacy * next batch of work requests from the write queue. 3876174641Skmacy */ 3877174641Skmacystatic void 3878174641Skmacywr_ack(struct toepcb *toep, struct mbuf *m) 3879174641Skmacy{ 3880174641Skmacy struct tcpcb *tp = toep->tp_tp; 3881174641Skmacy struct cpl_wr_ack *hdr = cplhdr(m); 3882178302Skmacy struct socket *so; 3883174641Skmacy unsigned int credits = ntohs(hdr->credits); 3884174641Skmacy u32 snd_una = ntohl(hdr->snd_una); 3885174641Skmacy int bytes = 0; 3886178302Skmacy struct sockbuf *snd; 3887174641Skmacy 3888176472Skmacy CTR2(KTR_SPARE2, "wr_ack: snd_una=%u credits=%d", snd_una, credits); 3889174641Skmacy 3890177530Skmacy inp_wlock(tp->t_inpcb); 3891178302Skmacy so = inp_inpcbtosocket(tp->t_inpcb); 3892174641Skmacy toep->tp_wr_avail += credits; 3893174641Skmacy if (toep->tp_wr_unacked > toep->tp_wr_max - toep->tp_wr_avail) 3894174641Skmacy toep->tp_wr_unacked = toep->tp_wr_max - toep->tp_wr_avail; 3895174641Skmacy 3896174641Skmacy while (credits) { 3897174641Skmacy struct mbuf *p = peek_wr(toep); 3898174641Skmacy 3899174641Skmacy if (__predict_false(!p)) { 3900174641Skmacy log(LOG_ERR, "%u WR_ACK credits for TID %u with " 3901176472Skmacy "nothing pending, state %u wr_avail=%u\n", 3902176472Skmacy credits, toep->tp_tid, tp->t_state, toep->tp_wr_avail); 3903174641Skmacy break; 3904174641Skmacy } 3905176472Skmacy CTR2(KTR_TOM, 3906178302Skmacy "wr_ack: p->credits=%d p->bytes=%d", 3907178302Skmacy p->m_pkthdr.csum_data, p->m_pkthdr.len); 3908178302Skmacy KASSERT(p->m_pkthdr.csum_data != 0, 3909178302Skmacy ("empty request still on list")); 3910176472Skmacy 3911174641Skmacy if (__predict_false(credits < p->m_pkthdr.csum_data)) { 3912176472Skmacy 3913174641Skmacy#if DEBUG_WR > 1 3914174641Skmacy struct tx_data_wr *w = cplhdr(p); 3915174641Skmacy log(LOG_ERR, 3916174641Skmacy "TID %u got %u WR credits, need %u, len %u, " 3917174641Skmacy "main body %u, frags %u, seq # %u, ACK una %u," 3918174641Skmacy " ACK nxt %u, WR_AVAIL %u, WRs pending %u\n", 3919174641Skmacy toep->tp_tid, credits, p->csum, p->len, 3920174641Skmacy p->len - p->data_len, skb_shinfo(p)->nr_frags, 3921174641Skmacy ntohl(w->sndseq), snd_una, ntohl(hdr->snd_nxt), 3922176472Skmacy toep->tp_wr_avail, count_pending_wrs(tp) - credits); 3923174641Skmacy#endif 3924174641Skmacy p->m_pkthdr.csum_data -= credits; 3925174641Skmacy break; 3926174641Skmacy } else { 3927174641Skmacy dequeue_wr(toep); 3928174641Skmacy credits -= p->m_pkthdr.csum_data; 3929174641Skmacy bytes += p->m_pkthdr.len; 3930176472Skmacy CTR3(KTR_TOM, 3931176472Skmacy "wr_ack: done with wr of %d bytes remain credits=%d wr credits=%d", 3932176472Skmacy p->m_pkthdr.len, credits, p->m_pkthdr.csum_data); 3933174641Skmacy 3934174641Skmacy m_free(p); 3935174641Skmacy } 3936174641Skmacy } 3937174641Skmacy 3938174641Skmacy#if DEBUG_WR 3939174641Skmacy check_wr_invariants(tp); 3940174641Skmacy#endif 3941174641Skmacy 3942174641Skmacy if (__predict_false(SEQ_LT(snd_una, tp->snd_una))) { 3943174641Skmacy#if VALIDATE_SEQ 3944174641Skmacy struct tom_data *d = TOM_DATA(TOE_DEV(so)); 3945174641Skmacy 3946174641Skmacy log(LOG_ERR "%s: unexpected sequence # %u in WR_ACK " 3947174641Skmacy "for TID %u, snd_una %u\n", (&d->tdev)->name, snd_una, 3948174641Skmacy toep->tp_tid, tp->snd_una); 3949174641Skmacy#endif 3950174641Skmacy goto out_free; 3951174641Skmacy } 3952174641Skmacy 3953174641Skmacy if (tp->snd_una != snd_una) { 3954174641Skmacy tp->snd_una = snd_una; 3955174641Skmacy tp->ts_recent_age = ticks; 3956174641Skmacy#ifdef notyet 3957174641Skmacy /* 3958174641Skmacy * Keep ARP entry "minty fresh" 3959174641Skmacy */ 3960174641Skmacy dst_confirm(sk->sk_dst_cache); 3961174641Skmacy#endif 3962174641Skmacy if (tp->snd_una == tp->snd_nxt) 3963174641Skmacy toep->tp_flags &= ~TP_TX_WAIT_IDLE; 3964174641Skmacy } 3965178302Skmacy 3966178302Skmacy snd = so_sockbuf_snd(so); 3967174641Skmacy if (bytes) { 3968176472Skmacy CTR1(KTR_SPARE2, "wr_ack: sbdrop(%d)", bytes); 3969178302Skmacy snd = so_sockbuf_snd(so); 3970178302Skmacy sockbuf_lock(snd); 3971178302Skmacy sbdrop_locked(snd, bytes); 3972178302Skmacy so_sowwakeup_locked(so); 3973174641Skmacy } 3974178302Skmacy 3975178302Skmacy if (snd->sb_sndptroff < snd->sb_cc) 3976174641Skmacy t3_push_frames(so, 0); 3977174641Skmacy 3978174641Skmacyout_free: 3979177530Skmacy inp_wunlock(tp->t_inpcb); 3980174641Skmacy m_free(m); 3981174641Skmacy} 3982174641Skmacy 3983174641Skmacy/* 3984174641Skmacy * Handler for TX_DATA_ACK CPL messages. 3985174641Skmacy */ 3986174641Skmacystatic int 3987174641Skmacydo_wr_ack(struct t3cdev *dev, struct mbuf *m, void *ctx) 3988174641Skmacy{ 3989174641Skmacy struct toepcb *toep = (struct toepcb *)ctx; 3990174641Skmacy 3991174641Skmacy VALIDATE_SOCK(so); 3992174641Skmacy 3993174641Skmacy wr_ack(toep, m); 3994174641Skmacy return 0; 3995174641Skmacy} 3996174641Skmacy 3997176472Skmacy/* 3998176472Skmacy * Handler for TRACE_PKT CPL messages. Just sink these packets. 3999176472Skmacy */ 4000176472Skmacystatic int 4001176472Skmacydo_trace_pkt(struct t3cdev *dev, struct mbuf *m, void *ctx) 4002176472Skmacy{ 4003176472Skmacy m_freem(m); 4004176472Skmacy return 0; 4005176472Skmacy} 4006174641Skmacy 4007174641Skmacy/* 4008174641Skmacy * Reset a connection that is on a listener's SYN queue or accept queue, 4009174641Skmacy * i.e., one that has not had a struct socket associated with it. 4010174641Skmacy * Must be called from process context. 4011174641Skmacy * 4012174641Skmacy * Modeled after code in inet_csk_listen_stop(). 4013174641Skmacy */ 4014174641Skmacystatic void 4015174641Skmacyt3_reset_listen_child(struct socket *child) 4016174641Skmacy{ 4017178302Skmacy struct tcpcb *tp = so_sototcpcb(child); 4018174641Skmacy 4019174641Skmacy t3_send_reset(tp->t_toe); 4020174641Skmacy} 4021174641Skmacy 4022178302Skmacy 4023178302Skmacystatic void 4024178302Skmacyt3_child_disconnect(struct socket *so, void *arg) 4025178302Skmacy{ 4026178302Skmacy struct tcpcb *tp = so_sototcpcb(so); 4027178302Skmacy 4028178302Skmacy if (tp->t_flags & TF_TOE) { 4029178302Skmacy inp_wlock(tp->t_inpcb); 4030178302Skmacy t3_reset_listen_child(so); 4031178302Skmacy inp_wunlock(tp->t_inpcb); 4032178302Skmacy } 4033178302Skmacy} 4034178302Skmacy 4035174641Skmacy/* 4036174641Skmacy * Disconnect offloaded established but not yet accepted connections sitting 4037174641Skmacy * on a server's accept_queue. We just send an ABORT_REQ at this point and 4038174641Skmacy * finish off the disconnect later as we may need to wait for the ABORT_RPL. 4039174641Skmacy */ 4040174641Skmacyvoid 4041174641Skmacyt3_disconnect_acceptq(struct socket *listen_so) 4042174641Skmacy{ 4043174641Skmacy 4044178302Skmacy so_lock(listen_so); 4045178302Skmacy so_listeners_apply_all(listen_so, t3_child_disconnect, NULL); 4046178302Skmacy so_unlock(listen_so); 4047174641Skmacy} 4048174641Skmacy 4049174641Skmacy/* 4050174641Skmacy * Reset offloaded connections sitting on a server's syn queue. As above 4051174641Skmacy * we send ABORT_REQ and finish off when we get ABORT_RPL. 4052174641Skmacy */ 4053174641Skmacy 4054174641Skmacyvoid 4055174641Skmacyt3_reset_synq(struct listen_ctx *lctx) 4056174641Skmacy{ 4057174641Skmacy struct toepcb *toep; 4058174641Skmacy 4059178302Skmacy so_lock(lctx->lso); 4060174641Skmacy while (!LIST_EMPTY(&lctx->synq_head)) { 4061174641Skmacy toep = LIST_FIRST(&lctx->synq_head); 4062174641Skmacy LIST_REMOVE(toep, synq_entry); 4063174641Skmacy toep->tp_tp = NULL; 4064174641Skmacy t3_send_reset(toep); 4065174641Skmacy cxgb_remove_tid(TOEP_T3C_DEV(toep), toep, toep->tp_tid); 4066174641Skmacy toepcb_release(toep); 4067174641Skmacy } 4068178302Skmacy so_unlock(lctx->lso); 4069174641Skmacy} 4070174641Skmacy 4071176472Skmacy 4072176472Skmacyint 4073178302Skmacyt3_setup_ppods(struct toepcb *toep, const struct ddp_gather_list *gl, 4074176472Skmacy unsigned int nppods, unsigned int tag, unsigned int maxoff, 4075176472Skmacy unsigned int pg_off, unsigned int color) 4076176472Skmacy{ 4077176472Skmacy unsigned int i, j, pidx; 4078176472Skmacy struct pagepod *p; 4079176472Skmacy struct mbuf *m; 4080176472Skmacy struct ulp_mem_io *req; 4081176472Skmacy unsigned int tid = toep->tp_tid; 4082178302Skmacy const struct tom_data *td = TOM_DATA(toep->tp_toedev); 4083176472Skmacy unsigned int ppod_addr = tag * PPOD_SIZE + td->ddp_llimit; 4084176472Skmacy 4085176472Skmacy CTR6(KTR_TOM, "t3_setup_ppods(gl=%p nppods=%u tag=%u maxoff=%u pg_off=%u color=%u)", 4086176472Skmacy gl, nppods, tag, maxoff, pg_off, color); 4087176472Skmacy 4088176472Skmacy for (i = 0; i < nppods; ++i) { 4089176472Skmacy m = m_gethdr_nofail(sizeof(*req) + PPOD_SIZE); 4090176472Skmacy m_set_priority(m, mkprio(CPL_PRIORITY_CONTROL, toep)); 4091176472Skmacy req = mtod(m, struct ulp_mem_io *); 4092176472Skmacy m->m_pkthdr.len = m->m_len = sizeof(*req) + PPOD_SIZE; 4093176472Skmacy req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_BYPASS)); 4094176472Skmacy req->wr.wr_lo = 0; 4095176472Skmacy req->cmd_lock_addr = htonl(V_ULP_MEMIO_ADDR(ppod_addr >> 5) | 4096176472Skmacy V_ULPTX_CMD(ULP_MEM_WRITE)); 4097176472Skmacy req->len = htonl(V_ULP_MEMIO_DATA_LEN(PPOD_SIZE / 32) | 4098176472Skmacy V_ULPTX_NFLITS(PPOD_SIZE / 8 + 1)); 4099176472Skmacy 4100176472Skmacy p = (struct pagepod *)(req + 1); 4101176472Skmacy if (__predict_false(i < nppods - NUM_SENTINEL_PPODS)) { 4102176472Skmacy p->pp_vld_tid = htonl(F_PPOD_VALID | V_PPOD_TID(tid)); 4103176472Skmacy p->pp_pgsz_tag_color = htonl(V_PPOD_TAG(tag) | 4104176472Skmacy V_PPOD_COLOR(color)); 4105176472Skmacy p->pp_max_offset = htonl(maxoff); 4106176472Skmacy p->pp_page_offset = htonl(pg_off); 4107176472Skmacy p->pp_rsvd = 0; 4108176472Skmacy for (pidx = 4 * i, j = 0; j < 5; ++j, ++pidx) 4109176472Skmacy p->pp_addr[j] = pidx < gl->dgl_nelem ? 4110176472Skmacy htobe64(VM_PAGE_TO_PHYS(gl->dgl_pages[pidx])) : 0; 4111176472Skmacy } else 4112176472Skmacy p->pp_vld_tid = 0; /* mark sentinel page pods invalid */ 4113176472Skmacy send_or_defer(toep, m, 0); 4114176472Skmacy ppod_addr += PPOD_SIZE; 4115176472Skmacy } 4116176472Skmacy return (0); 4117176472Skmacy} 4118176472Skmacy 4119176472Skmacy/* 4120176472Skmacy * Build a CPL_BARRIER message as payload of a ULP_TX_PKT command. 4121176472Skmacy */ 4122176472Skmacystatic inline void 4123176472Skmacymk_cpl_barrier_ulp(struct cpl_barrier *b) 4124176472Skmacy{ 4125176472Skmacy struct ulp_txpkt *txpkt = (struct ulp_txpkt *)b; 4126176472Skmacy 4127176472Skmacy txpkt->cmd_dest = htonl(V_ULPTX_CMD(ULP_TXPKT)); 4128176472Skmacy txpkt->len = htonl(V_ULPTX_NFLITS(sizeof(*b) / 8)); 4129176472Skmacy b->opcode = CPL_BARRIER; 4130176472Skmacy} 4131176472Skmacy 4132176472Skmacy/* 4133176472Skmacy * Build a CPL_GET_TCB message as payload of a ULP_TX_PKT command. 4134176472Skmacy */ 4135176472Skmacystatic inline void 4136176472Skmacymk_get_tcb_ulp(struct cpl_get_tcb *req, unsigned int tid, unsigned int cpuno) 4137176472Skmacy{ 4138176472Skmacy struct ulp_txpkt *txpkt = (struct ulp_txpkt *)req; 4139176472Skmacy 4140176472Skmacy txpkt = (struct ulp_txpkt *)req; 4141176472Skmacy txpkt->cmd_dest = htonl(V_ULPTX_CMD(ULP_TXPKT)); 4142176472Skmacy txpkt->len = htonl(V_ULPTX_NFLITS(sizeof(*req) / 8)); 4143176472Skmacy OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_GET_TCB, tid)); 4144176472Skmacy req->cpuno = htons(cpuno); 4145176472Skmacy} 4146176472Skmacy 4147176472Skmacy/* 4148176472Skmacy * Build a CPL_SET_TCB_FIELD message as payload of a ULP_TX_PKT command. 4149176472Skmacy */ 4150176472Skmacystatic inline void 4151176472Skmacymk_set_tcb_field_ulp(struct cpl_set_tcb_field *req, unsigned int tid, 4152176472Skmacy unsigned int word, uint64_t mask, uint64_t val) 4153176472Skmacy{ 4154176472Skmacy struct ulp_txpkt *txpkt = (struct ulp_txpkt *)req; 4155176472Skmacy 4156176472Skmacy CTR4(KTR_TCB, "mk_set_tcb_field_ulp(tid=%u word=0x%x mask=%jx val=%jx", 4157176472Skmacy tid, word, mask, val); 4158176472Skmacy 4159176472Skmacy txpkt->cmd_dest = htonl(V_ULPTX_CMD(ULP_TXPKT)); 4160176472Skmacy txpkt->len = htonl(V_ULPTX_NFLITS(sizeof(*req) / 8)); 4161176472Skmacy OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid)); 4162176472Skmacy req->reply = V_NO_REPLY(1); 4163176472Skmacy req->cpu_idx = 0; 4164176472Skmacy req->word = htons(word); 4165176472Skmacy req->mask = htobe64(mask); 4166176472Skmacy req->val = htobe64(val); 4167176472Skmacy} 4168176472Skmacy 4169176472Skmacy/* 4170176472Skmacy * Build a CPL_RX_DATA_ACK message as payload of a ULP_TX_PKT command. 4171176472Skmacy */ 4172176472Skmacystatic void 4173178302Skmacymk_rx_data_ack_ulp(struct toepcb *toep, struct cpl_rx_data_ack *ack, 4174177340Skmacy unsigned int tid, unsigned int credits) 4175176472Skmacy{ 4176176472Skmacy struct ulp_txpkt *txpkt = (struct ulp_txpkt *)ack; 4177176472Skmacy 4178176472Skmacy txpkt->cmd_dest = htonl(V_ULPTX_CMD(ULP_TXPKT)); 4179176472Skmacy txpkt->len = htonl(V_ULPTX_NFLITS(sizeof(*ack) / 8)); 4180176472Skmacy OPCODE_TID(ack) = htonl(MK_OPCODE_TID(CPL_RX_DATA_ACK, tid)); 4181176472Skmacy ack->credit_dack = htonl(F_RX_MODULATE | F_RX_DACK_CHANGE | 4182178302Skmacy V_RX_DACK_MODE(TOM_TUNABLE(toep->tp_toedev, delack)) | 4183177340Skmacy V_RX_CREDITS(credits)); 4184176472Skmacy} 4185176472Skmacy 4186174641Skmacyvoid 4187176472Skmacyt3_cancel_ddpbuf(struct toepcb *toep, unsigned int bufidx) 4188176472Skmacy{ 4189176472Skmacy unsigned int wrlen; 4190176472Skmacy struct mbuf *m; 4191176472Skmacy struct work_request_hdr *wr; 4192176472Skmacy struct cpl_barrier *lock; 4193176472Skmacy struct cpl_set_tcb_field *req; 4194176472Skmacy struct cpl_get_tcb *getreq; 4195176472Skmacy struct ddp_state *p = &toep->tp_ddp_state; 4196176472Skmacy 4197178302Skmacy#if 0 4198176472Skmacy SOCKBUF_LOCK_ASSERT(&toeptoso(toep)->so_rcv); 4199178302Skmacy#endif 4200176472Skmacy wrlen = sizeof(*wr) + sizeof(*req) + 2 * sizeof(*lock) + 4201176472Skmacy sizeof(*getreq); 4202176472Skmacy m = m_gethdr_nofail(wrlen); 4203176472Skmacy m_set_priority(m, mkprio(CPL_PRIORITY_CONTROL, toep)); 4204176472Skmacy wr = mtod(m, struct work_request_hdr *); 4205176472Skmacy bzero(wr, wrlen); 4206176472Skmacy 4207176472Skmacy wr->wr_hi = htonl(V_WR_OP(FW_WROPCODE_BYPASS)); 4208176472Skmacy m->m_pkthdr.len = m->m_len = wrlen; 4209176472Skmacy 4210176472Skmacy lock = (struct cpl_barrier *)(wr + 1); 4211176472Skmacy mk_cpl_barrier_ulp(lock); 4212176472Skmacy 4213176472Skmacy req = (struct cpl_set_tcb_field *)(lock + 1); 4214176472Skmacy 4215176472Skmacy CTR1(KTR_TCB, "t3_cancel_ddpbuf(bufidx=%u)", bufidx); 4216176472Skmacy 4217176472Skmacy /* Hmmm, not sure if this actually a good thing: reactivating 4218176472Skmacy * the other buffer might be an issue if it has been completed 4219176472Skmacy * already. However, that is unlikely, since the fact that the UBUF 4220176472Skmacy * is not completed indicates that there is no oustanding data. 4221176472Skmacy */ 4222176472Skmacy if (bufidx == 0) 4223176472Skmacy mk_set_tcb_field_ulp(req, toep->tp_tid, W_TCB_RX_DDP_FLAGS, 4224176472Skmacy V_TF_DDP_ACTIVE_BUF(1) | 4225176472Skmacy V_TF_DDP_BUF0_VALID(1), 4226176472Skmacy V_TF_DDP_ACTIVE_BUF(1)); 4227176472Skmacy else 4228176472Skmacy mk_set_tcb_field_ulp(req, toep->tp_tid, W_TCB_RX_DDP_FLAGS, 4229176472Skmacy V_TF_DDP_ACTIVE_BUF(1) | 4230176472Skmacy V_TF_DDP_BUF1_VALID(1), 0); 4231176472Skmacy 4232176472Skmacy getreq = (struct cpl_get_tcb *)(req + 1); 4233176472Skmacy mk_get_tcb_ulp(getreq, toep->tp_tid, toep->tp_qset); 4234176472Skmacy 4235176472Skmacy mk_cpl_barrier_ulp((struct cpl_barrier *)(getreq + 1)); 4236176472Skmacy 4237176472Skmacy /* Keep track of the number of oustanding CPL_GET_TCB requests 4238176472Skmacy */ 4239176472Skmacy p->get_tcb_count++; 4240176472Skmacy 4241176472Skmacy#ifdef T3_TRACE 4242176472Skmacy T3_TRACE1(TIDTB(so), 4243176472Skmacy "t3_cancel_ddpbuf: bufidx %u", bufidx); 4244176472Skmacy#endif 4245176472Skmacy cxgb_ofld_send(TOEP_T3C_DEV(toep), m); 4246176472Skmacy} 4247176472Skmacy 4248176472Skmacy/** 4249176472Skmacy * t3_overlay_ddpbuf - overlay an existing DDP buffer with a new one 4250176472Skmacy * @sk: the socket associated with the buffers 4251176472Skmacy * @bufidx: index of HW DDP buffer (0 or 1) 4252176472Skmacy * @tag0: new tag for HW buffer 0 4253176472Skmacy * @tag1: new tag for HW buffer 1 4254176472Skmacy * @len: new length for HW buf @bufidx 4255176472Skmacy * 4256176472Skmacy * Sends a compound WR to overlay a new DDP buffer on top of an existing 4257176472Skmacy * buffer by changing the buffer tag and length and setting the valid and 4258176472Skmacy * active flag accordingly. The caller must ensure the new buffer is at 4259176472Skmacy * least as big as the existing one. Since we typically reprogram both HW 4260176472Skmacy * buffers this function sets both tags for convenience. Read the TCB to 4261176472Skmacy * determine how made data was written into the buffer before the overlay 4262176472Skmacy * took place. 4263176472Skmacy */ 4264176472Skmacyvoid 4265176472Skmacyt3_overlay_ddpbuf(struct toepcb *toep, unsigned int bufidx, unsigned int tag0, 4266176472Skmacy unsigned int tag1, unsigned int len) 4267176472Skmacy{ 4268176472Skmacy unsigned int wrlen; 4269176472Skmacy struct mbuf *m; 4270176472Skmacy struct work_request_hdr *wr; 4271176472Skmacy struct cpl_get_tcb *getreq; 4272176472Skmacy struct cpl_set_tcb_field *req; 4273176472Skmacy struct ddp_state *p = &toep->tp_ddp_state; 4274176472Skmacy 4275176472Skmacy CTR4(KTR_TCB, "t3_setup_ppods(bufidx=%u tag0=%u tag1=%u len=%u)", 4276176472Skmacy bufidx, tag0, tag1, len); 4277178302Skmacy#if 0 4278176472Skmacy SOCKBUF_LOCK_ASSERT(&toeptoso(toep)->so_rcv); 4279178302Skmacy#endif 4280176472Skmacy wrlen = sizeof(*wr) + 3 * sizeof(*req) + sizeof(*getreq); 4281176472Skmacy m = m_gethdr_nofail(wrlen); 4282176472Skmacy m_set_priority(m, mkprio(CPL_PRIORITY_CONTROL, toep)); 4283176472Skmacy wr = mtod(m, struct work_request_hdr *); 4284176472Skmacy m->m_pkthdr.len = m->m_len = wrlen; 4285176472Skmacy bzero(wr, wrlen); 4286176472Skmacy 4287176472Skmacy 4288176472Skmacy /* Set the ATOMIC flag to make sure that TP processes the following 4289176472Skmacy * CPLs in an atomic manner and no wire segments can be interleaved. 4290176472Skmacy */ 4291176472Skmacy wr->wr_hi = htonl(V_WR_OP(FW_WROPCODE_BYPASS) | F_WR_ATOMIC); 4292176472Skmacy req = (struct cpl_set_tcb_field *)(wr + 1); 4293176472Skmacy mk_set_tcb_field_ulp(req, toep->tp_tid, W_TCB_RX_DDP_BUF0_TAG, 4294176472Skmacy V_TCB_RX_DDP_BUF0_TAG(M_TCB_RX_DDP_BUF0_TAG) | 4295176472Skmacy V_TCB_RX_DDP_BUF1_TAG(M_TCB_RX_DDP_BUF1_TAG) << 32, 4296176472Skmacy V_TCB_RX_DDP_BUF0_TAG(tag0) | 4297176472Skmacy V_TCB_RX_DDP_BUF1_TAG((uint64_t)tag1) << 32); 4298176472Skmacy req++; 4299176472Skmacy if (bufidx == 0) { 4300176472Skmacy mk_set_tcb_field_ulp(req, toep->tp_tid, W_TCB_RX_DDP_BUF0_LEN, 4301176472Skmacy V_TCB_RX_DDP_BUF0_LEN(M_TCB_RX_DDP_BUF0_LEN), 4302176472Skmacy V_TCB_RX_DDP_BUF0_LEN((uint64_t)len)); 4303176472Skmacy req++; 4304176472Skmacy mk_set_tcb_field_ulp(req, toep->tp_tid, W_TCB_RX_DDP_FLAGS, 4305176472Skmacy V_TF_DDP_PUSH_DISABLE_0(1) | 4306176472Skmacy V_TF_DDP_BUF0_VALID(1) | V_TF_DDP_ACTIVE_BUF(1), 4307176472Skmacy V_TF_DDP_PUSH_DISABLE_0(0) | 4308176472Skmacy V_TF_DDP_BUF0_VALID(1)); 4309176472Skmacy } else { 4310176472Skmacy mk_set_tcb_field_ulp(req, toep->tp_tid, W_TCB_RX_DDP_BUF1_LEN, 4311176472Skmacy V_TCB_RX_DDP_BUF1_LEN(M_TCB_RX_DDP_BUF1_LEN), 4312176472Skmacy V_TCB_RX_DDP_BUF1_LEN((uint64_t)len)); 4313176472Skmacy req++; 4314176472Skmacy mk_set_tcb_field_ulp(req, toep->tp_tid, W_TCB_RX_DDP_FLAGS, 4315176472Skmacy V_TF_DDP_PUSH_DISABLE_1(1) | 4316176472Skmacy V_TF_DDP_BUF1_VALID(1) | V_TF_DDP_ACTIVE_BUF(1), 4317176472Skmacy V_TF_DDP_PUSH_DISABLE_1(0) | 4318176472Skmacy V_TF_DDP_BUF1_VALID(1) | V_TF_DDP_ACTIVE_BUF(1)); 4319176472Skmacy } 4320176472Skmacy 4321176472Skmacy getreq = (struct cpl_get_tcb *)(req + 1); 4322176472Skmacy mk_get_tcb_ulp(getreq, toep->tp_tid, toep->tp_qset); 4323176472Skmacy 4324176472Skmacy /* Keep track of the number of oustanding CPL_GET_TCB requests 4325176472Skmacy */ 4326176472Skmacy p->get_tcb_count++; 4327176472Skmacy 4328176472Skmacy#ifdef T3_TRACE 4329176472Skmacy T3_TRACE4(TIDTB(sk), 4330176472Skmacy "t3_overlay_ddpbuf: bufidx %u tag0 %u tag1 %u " 4331176472Skmacy "len %d", 4332176472Skmacy bufidx, tag0, tag1, len); 4333176472Skmacy#endif 4334176472Skmacy cxgb_ofld_send(TOEP_T3C_DEV(toep), m); 4335176472Skmacy} 4336176472Skmacy 4337176472Skmacy/* 4338176472Skmacy * Sends a compound WR containing all the CPL messages needed to program the 4339176472Skmacy * two HW DDP buffers, namely optionally setting up the length and offset of 4340176472Skmacy * each buffer, programming the DDP flags, and optionally sending RX_DATA_ACK. 4341176472Skmacy */ 4342176472Skmacyvoid 4343176472Skmacyt3_setup_ddpbufs(struct toepcb *toep, unsigned int len0, unsigned int offset0, 4344176472Skmacy unsigned int len1, unsigned int offset1, 4345176472Skmacy uint64_t ddp_flags, uint64_t flag_mask, int modulate) 4346176472Skmacy{ 4347176472Skmacy unsigned int wrlen; 4348176472Skmacy struct mbuf *m; 4349176472Skmacy struct work_request_hdr *wr; 4350176472Skmacy struct cpl_set_tcb_field *req; 4351176472Skmacy 4352176472Skmacy CTR6(KTR_TCB, "t3_setup_ddpbufs(len0=%u offset0=%u len1=%u offset1=%u ddp_flags=0x%08x%08x ", 4353176472Skmacy len0, offset0, len1, offset1, ddp_flags >> 32, ddp_flags & 0xffffffff); 4354176472Skmacy 4355178302Skmacy#if 0 4356176472Skmacy SOCKBUF_LOCK_ASSERT(&toeptoso(toep)->so_rcv); 4357178302Skmacy#endif 4358176472Skmacy wrlen = sizeof(*wr) + sizeof(*req) + (len0 ? sizeof(*req) : 0) + 4359176472Skmacy (len1 ? sizeof(*req) : 0) + 4360176472Skmacy (modulate ? sizeof(struct cpl_rx_data_ack) : 0); 4361176472Skmacy m = m_gethdr_nofail(wrlen); 4362176472Skmacy m_set_priority(m, mkprio(CPL_PRIORITY_CONTROL, toep)); 4363176472Skmacy wr = mtod(m, struct work_request_hdr *); 4364176472Skmacy bzero(wr, wrlen); 4365176472Skmacy 4366176472Skmacy wr->wr_hi = htonl(V_WR_OP(FW_WROPCODE_BYPASS)); 4367176472Skmacy m->m_pkthdr.len = m->m_len = wrlen; 4368176472Skmacy 4369176472Skmacy req = (struct cpl_set_tcb_field *)(wr + 1); 4370176472Skmacy if (len0) { /* program buffer 0 offset and length */ 4371176472Skmacy mk_set_tcb_field_ulp(req, toep->tp_tid, W_TCB_RX_DDP_BUF0_OFFSET, 4372176472Skmacy V_TCB_RX_DDP_BUF0_OFFSET(M_TCB_RX_DDP_BUF0_OFFSET) | 4373176472Skmacy V_TCB_RX_DDP_BUF0_LEN(M_TCB_RX_DDP_BUF0_LEN), 4374176472Skmacy V_TCB_RX_DDP_BUF0_OFFSET((uint64_t)offset0) | 4375176472Skmacy V_TCB_RX_DDP_BUF0_LEN((uint64_t)len0)); 4376176472Skmacy req++; 4377176472Skmacy } 4378176472Skmacy if (len1) { /* program buffer 1 offset and length */ 4379176472Skmacy mk_set_tcb_field_ulp(req, toep->tp_tid, W_TCB_RX_DDP_BUF1_OFFSET, 4380176472Skmacy V_TCB_RX_DDP_BUF1_OFFSET(M_TCB_RX_DDP_BUF1_OFFSET) | 4381176472Skmacy V_TCB_RX_DDP_BUF1_LEN(M_TCB_RX_DDP_BUF1_LEN) << 32, 4382176472Skmacy V_TCB_RX_DDP_BUF1_OFFSET((uint64_t)offset1) | 4383176472Skmacy V_TCB_RX_DDP_BUF1_LEN((uint64_t)len1) << 32); 4384176472Skmacy req++; 4385176472Skmacy } 4386176472Skmacy 4387176472Skmacy mk_set_tcb_field_ulp(req, toep->tp_tid, W_TCB_RX_DDP_FLAGS, flag_mask, 4388176472Skmacy ddp_flags); 4389176472Skmacy 4390176472Skmacy if (modulate) { 4391178302Skmacy mk_rx_data_ack_ulp(toep, 4392177340Skmacy (struct cpl_rx_data_ack *)(req + 1), toep->tp_tid, 4393177340Skmacy toep->tp_copied_seq - toep->tp_rcv_wup); 4394176472Skmacy toep->tp_rcv_wup = toep->tp_copied_seq; 4395176472Skmacy } 4396176472Skmacy 4397176472Skmacy#ifdef T3_TRACE 4398176472Skmacy T3_TRACE5(TIDTB(sk), 4399176472Skmacy "t3_setup_ddpbufs: len0 %u len1 %u ddp_flags 0x%08x%08x " 4400176472Skmacy "modulate %d", 4401176472Skmacy len0, len1, ddp_flags >> 32, ddp_flags & 0xffffffff, 4402176472Skmacy modulate); 4403176472Skmacy#endif 4404176472Skmacy 4405176472Skmacy cxgb_ofld_send(TOEP_T3C_DEV(toep), m); 4406176472Skmacy} 4407176472Skmacy 4408176472Skmacyvoid 4409174641Skmacyt3_init_wr_tab(unsigned int wr_len) 4410174641Skmacy{ 4411174641Skmacy int i; 4412174641Skmacy 4413174641Skmacy if (mbuf_wrs[1]) /* already initialized */ 4414174641Skmacy return; 4415174641Skmacy 4416174641Skmacy for (i = 1; i < ARRAY_SIZE(mbuf_wrs); i++) { 4417174641Skmacy int sgl_len = (3 * i) / 2 + (i & 1); 4418174641Skmacy 4419174641Skmacy sgl_len += 3; 4420174641Skmacy mbuf_wrs[i] = sgl_len <= wr_len ? 4421174641Skmacy 1 : 1 + (sgl_len - 2) / (wr_len - 1); 4422174641Skmacy } 4423174641Skmacy 4424174641Skmacy wrlen = wr_len * 8; 4425174641Skmacy} 4426174641Skmacy 4427174641Skmacyint 4428174641Skmacyt3_init_cpl_io(void) 4429174641Skmacy{ 4430174641Skmacy#ifdef notyet 4431174641Skmacy tcphdr_skb = alloc_skb(sizeof(struct tcphdr), GFP_KERNEL); 4432174641Skmacy if (!tcphdr_skb) { 4433174641Skmacy log(LOG_ERR, 4434174641Skmacy "Chelsio TCP offload: can't allocate sk_buff\n"); 4435174641Skmacy return -1; 4436174641Skmacy } 4437174641Skmacy skb_put(tcphdr_skb, sizeof(struct tcphdr)); 4438174641Skmacy tcphdr_skb->h.raw = tcphdr_skb->data; 4439174641Skmacy memset(tcphdr_skb->data, 0, tcphdr_skb->len); 4440174641Skmacy#endif 4441174641Skmacy 4442174641Skmacy t3tom_register_cpl_handler(CPL_ACT_ESTABLISH, do_act_establish); 4443174641Skmacy t3tom_register_cpl_handler(CPL_ACT_OPEN_RPL, do_act_open_rpl); 4444174641Skmacy t3tom_register_cpl_handler(CPL_TX_DMA_ACK, do_wr_ack); 4445174641Skmacy t3tom_register_cpl_handler(CPL_RX_DATA, do_rx_data); 4446174641Skmacy t3tom_register_cpl_handler(CPL_CLOSE_CON_RPL, do_close_con_rpl); 4447174641Skmacy t3tom_register_cpl_handler(CPL_PEER_CLOSE, do_peer_close); 4448174641Skmacy t3tom_register_cpl_handler(CPL_PASS_ESTABLISH, do_pass_establish); 4449174641Skmacy t3tom_register_cpl_handler(CPL_PASS_ACCEPT_REQ, do_pass_accept_req); 4450174641Skmacy t3tom_register_cpl_handler(CPL_ABORT_REQ_RSS, do_abort_req); 4451174641Skmacy t3tom_register_cpl_handler(CPL_ABORT_RPL_RSS, do_abort_rpl); 4452174641Skmacy t3tom_register_cpl_handler(CPL_RX_DATA_DDP, do_rx_data_ddp); 4453174641Skmacy t3tom_register_cpl_handler(CPL_RX_DDP_COMPLETE, do_rx_ddp_complete); 4454174641Skmacy t3tom_register_cpl_handler(CPL_RX_URG_NOTIFY, do_rx_urg_notify); 4455174641Skmacy t3tom_register_cpl_handler(CPL_TRACE_PKT, do_trace_pkt); 4456174641Skmacy t3tom_register_cpl_handler(CPL_GET_TCB_RPL, do_get_tcb_rpl); 4457174641Skmacy return (0); 4458174641Skmacy} 4459174641Skmacy 4460