cxgb_cpl_io.c revision 180583
1174641Skmacy/************************************************************************** 2174641Skmacy 3178302SkmacyCopyright (c) 2007-2008, Chelsio Inc. 4174641SkmacyAll rights reserved. 5174641Skmacy 6174641SkmacyRedistribution and use in source and binary forms, with or without 7174641Skmacymodification, are permitted provided that the following conditions are met: 8174641Skmacy 9174641Skmacy 1. Redistributions of source code must retain the above copyright notice, 10174641Skmacy this list of conditions and the following disclaimer. 11174641Skmacy 12174641Skmacy 2. Neither the name of the Chelsio Corporation nor the names of its 13174641Skmacy contributors may be used to endorse or promote products derived from 14174641Skmacy this software without specific prior written permission. 15174641Skmacy 16174641SkmacyTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 17174641SkmacyAND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18174641SkmacyIMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19174641SkmacyARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 20174641SkmacyLIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21174641SkmacyCONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22174641SkmacySUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23174641SkmacyINTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24174641SkmacyCONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25174641SkmacyARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26174641SkmacyPOSSIBILITY OF SUCH DAMAGE. 27174641Skmacy 28174641Skmacy***************************************************************************/ 29174641Skmacy 30174641Skmacy#include <sys/cdefs.h> 31174641Skmacy__FBSDID("$FreeBSD: head/sys/dev/cxgb/ulp/tom/cxgb_cpl_io.c 180583 2008-07-18 06:12:31Z kmacy $"); 32174641Skmacy 33174641Skmacy#include <sys/param.h> 34174641Skmacy#include <sys/systm.h> 35174641Skmacy#include <sys/fcntl.h> 36174641Skmacy#include <sys/kernel.h> 37174641Skmacy#include <sys/limits.h> 38176472Skmacy#include <sys/ktr.h> 39174641Skmacy#include <sys/lock.h> 40174641Skmacy#include <sys/mbuf.h> 41174641Skmacy#include <sys/mutex.h> 42174641Skmacy#include <sys/socket.h> 43174641Skmacy#include <sys/sysctl.h> 44174641Skmacy#include <sys/syslog.h> 45174641Skmacy#include <sys/protosw.h> 46174641Skmacy#include <sys/priv.h> 47174641Skmacy 48174641Skmacy#include <net/if.h> 49174641Skmacy#include <net/route.h> 50174641Skmacy 51174641Skmacy#include <netinet/in.h> 52174641Skmacy#include <netinet/in_pcb.h> 53174641Skmacy#include <netinet/in_systm.h> 54174641Skmacy#include <netinet/in_var.h> 55174641Skmacy 56174641Skmacy 57174641Skmacy#include <dev/cxgb/cxgb_osdep.h> 58174641Skmacy#include <dev/cxgb/sys/mbufq.h> 59174641Skmacy 60174641Skmacy#include <netinet/ip.h> 61174641Skmacy#include <netinet/tcp_var.h> 62174641Skmacy#include <netinet/tcp_fsm.h> 63174708Skmacy#include <netinet/tcp_offload.h> 64174641Skmacy#include <netinet/tcp_seq.h> 65174641Skmacy#include <netinet/tcp_syncache.h> 66176472Skmacy#include <netinet/tcp_timer.h> 67174641Skmacy#include <net/route.h> 68174641Skmacy 69174641Skmacy#include <dev/cxgb/t3cdev.h> 70174641Skmacy#include <dev/cxgb/common/cxgb_firmware_exports.h> 71174641Skmacy#include <dev/cxgb/common/cxgb_t3_cpl.h> 72174641Skmacy#include <dev/cxgb/common/cxgb_tcb.h> 73174641Skmacy#include <dev/cxgb/common/cxgb_ctl_defs.h> 74174641Skmacy#include <dev/cxgb/cxgb_offload.h> 75174641Skmacy#include <vm/vm.h> 76174641Skmacy#include <vm/pmap.h> 77174641Skmacy#include <machine/bus.h> 78174641Skmacy#include <dev/cxgb/sys/mvec.h> 79174641Skmacy#include <dev/cxgb/ulp/toecore/cxgb_toedev.h> 80174641Skmacy#include <dev/cxgb/ulp/tom/cxgb_defs.h> 81174641Skmacy#include <dev/cxgb/ulp/tom/cxgb_tom.h> 82174641Skmacy#include <dev/cxgb/ulp/tom/cxgb_t3_ddp.h> 83174641Skmacy#include <dev/cxgb/ulp/tom/cxgb_toepcb.h> 84174708Skmacy#include <dev/cxgb/ulp/tom/cxgb_tcp.h> 85174641Skmacy 86178302Skmacy#include <dev/cxgb/ulp/tom/cxgb_tcp_offload.h> 87178302Skmacy 88174641Skmacy/* 89174641Skmacy * For ULP connections HW may add headers, e.g., for digests, that aren't part 90174641Skmacy * of the messages sent by the host but that are part of the TCP payload and 91174641Skmacy * therefore consume TCP sequence space. Tx connection parameters that 92174641Skmacy * operate in TCP sequence space are affected by the HW additions and need to 93174641Skmacy * compensate for them to accurately track TCP sequence numbers. This array 94174641Skmacy * contains the compensating extra lengths for ULP packets. It is indexed by 95174641Skmacy * a packet's ULP submode. 96174641Skmacy */ 97174641Skmacyconst unsigned int t3_ulp_extra_len[] = {0, 4, 4, 8}; 98174641Skmacy 99174641Skmacy#ifdef notyet 100174641Skmacy/* 101174641Skmacy * This sk_buff holds a fake header-only TCP segment that we use whenever we 102174641Skmacy * need to exploit SW TCP functionality that expects TCP headers, such as 103174641Skmacy * tcp_create_openreq_child(). It's a RO buffer that may be used by multiple 104174641Skmacy * CPUs without locking. 105174641Skmacy */ 106174641Skmacystatic struct mbuf *tcphdr_mbuf __read_mostly; 107174641Skmacy#endif 108174641Skmacy 109174641Skmacy/* 110174641Skmacy * Size of WRs in bytes. Note that we assume all devices we are handling have 111174641Skmacy * the same WR size. 112174641Skmacy */ 113174641Skmacystatic unsigned int wrlen __read_mostly; 114174641Skmacy 115174641Skmacy/* 116174641Skmacy * The number of WRs needed for an skb depends on the number of page fragments 117174641Skmacy * in the skb and whether it has any payload in its main body. This maps the 118174641Skmacy * length of the gather list represented by an skb into the # of necessary WRs. 119174641Skmacy */ 120176472Skmacystatic unsigned int mbuf_wrs[TX_MAX_SEGS + 1] __read_mostly; 121174641Skmacy 122174641Skmacy/* 123174641Skmacy * Max receive window supported by HW in bytes. Only a small part of it can 124174641Skmacy * be set through option0, the rest needs to be set through RX_DATA_ACK. 125174641Skmacy */ 126174641Skmacy#define MAX_RCV_WND ((1U << 27) - 1) 127174641Skmacy 128174641Skmacy/* 129174641Skmacy * Min receive window. We want it to be large enough to accommodate receive 130174641Skmacy * coalescing, handle jumbo frames, and not trigger sender SWS avoidance. 131174641Skmacy */ 132174641Skmacy#define MIN_RCV_WND (24 * 1024U) 133178302Skmacy#define INP_TOS(inp) ((inp_ip_tos_get(inp) >> 2) & M_TOS) 134174641Skmacy 135174641Skmacy#define VALIDATE_SEQ 0 136174641Skmacy#define VALIDATE_SOCK(so) 137174641Skmacy#define DEBUG_WR 0 138174641Skmacy 139178302Skmacy#define TCP_TIMEWAIT 1 140178302Skmacy#define TCP_CLOSE 2 141178302Skmacy#define TCP_DROP 3 142178302Skmacy 143174641Skmacyextern int tcp_do_autorcvbuf; 144174641Skmacyextern int tcp_do_autosndbuf; 145174641Skmacyextern int tcp_autorcvbuf_max; 146174641Skmacyextern int tcp_autosndbuf_max; 147174641Skmacy 148174641Skmacystatic void t3_send_reset(struct toepcb *toep); 149174641Skmacystatic void send_abort_rpl(struct mbuf *m, struct toedev *tdev, int rst_status); 150174641Skmacystatic inline void free_atid(struct t3cdev *cdev, unsigned int tid); 151174641Skmacystatic void handle_syncache_event(int event, void *arg); 152174641Skmacy 153176472Skmacystatic inline void 154176472SkmacySBAPPEND(struct sockbuf *sb, struct mbuf *n) 155176472Skmacy{ 156178302Skmacy struct mbuf *m; 157174641Skmacy 158176472Skmacy m = sb->sb_mb; 159176472Skmacy while (m) { 160176472Skmacy KASSERT(((m->m_flags & M_EXT) && (m->m_ext.ext_type == EXT_EXTREF)) || 161176472Skmacy !(m->m_flags & M_EXT), ("unexpected type M_EXT=%d ext_type=%d m_len=%d\n", 162176472Skmacy !!(m->m_flags & M_EXT), m->m_ext.ext_type, m->m_len)); 163176472Skmacy KASSERT(m->m_next != (struct mbuf *)0xffffffff, ("bad next value m_next=%p m_nextpkt=%p m_flags=0x%x", 164176472Skmacy m->m_next, m->m_nextpkt, m->m_flags)); 165176472Skmacy m = m->m_next; 166176472Skmacy } 167176472Skmacy m = n; 168176472Skmacy while (m) { 169176472Skmacy KASSERT(((m->m_flags & M_EXT) && (m->m_ext.ext_type == EXT_EXTREF)) || 170176472Skmacy !(m->m_flags & M_EXT), ("unexpected type M_EXT=%d ext_type=%d m_len=%d\n", 171176472Skmacy !!(m->m_flags & M_EXT), m->m_ext.ext_type, m->m_len)); 172176472Skmacy KASSERT(m->m_next != (struct mbuf *)0xffffffff, ("bad next value m_next=%p m_nextpkt=%p m_flags=0x%x", 173176472Skmacy m->m_next, m->m_nextpkt, m->m_flags)); 174176472Skmacy m = m->m_next; 175176472Skmacy } 176178767Skmacy KASSERT(sb->sb_flags & SB_NOCOALESCE, ("NOCOALESCE not set")); 177178767Skmacy sbappendstream_locked(sb, n); 178176472Skmacy m = sb->sb_mb; 179178302Skmacy 180176472Skmacy while (m) { 181176472Skmacy KASSERT(m->m_next != (struct mbuf *)0xffffffff, ("bad next value m_next=%p m_nextpkt=%p m_flags=0x%x", 182176472Skmacy m->m_next, m->m_nextpkt, m->m_flags)); 183176472Skmacy m = m->m_next; 184176472Skmacy } 185176472Skmacy} 186176472Skmacy 187174641Skmacystatic inline int 188174641Skmacyis_t3a(const struct toedev *dev) 189174641Skmacy{ 190174641Skmacy return (dev->tod_ttid == TOE_ID_CHELSIO_T3); 191174641Skmacy} 192174641Skmacy 193174641Skmacystatic void 194174641Skmacydump_toepcb(struct toepcb *toep) 195174641Skmacy{ 196174641Skmacy DPRINTF("qset_idx=%d qset=%d ulp_mode=%d mtu_idx=%d tid=%d\n", 197174641Skmacy toep->tp_qset_idx, toep->tp_qset, toep->tp_ulp_mode, 198174641Skmacy toep->tp_mtu_idx, toep->tp_tid); 199174641Skmacy 200174641Skmacy DPRINTF("wr_max=%d wr_avail=%d wr_unacked=%d mss_clamp=%d flags=0x%x\n", 201174641Skmacy toep->tp_wr_max, toep->tp_wr_avail, toep->tp_wr_unacked, 202174641Skmacy toep->tp_mss_clamp, toep->tp_flags); 203174641Skmacy} 204174641Skmacy 205176472Skmacy#ifndef RTALLOC2_DEFINED 206174641Skmacystatic struct rtentry * 207174641Skmacyrtalloc2(struct sockaddr *dst, int report, u_long ignflags) 208174641Skmacy{ 209174641Skmacy struct rtentry *rt = NULL; 210174641Skmacy 211174641Skmacy if ((rt = rtalloc1(dst, report, ignflags)) != NULL) 212174641Skmacy RT_UNLOCK(rt); 213174641Skmacy 214174641Skmacy return (rt); 215174641Skmacy} 216176472Skmacy#endif 217178302Skmacy 218174641Skmacy/* 219174641Skmacy * Determine whether to send a CPL message now or defer it. A message is 220174641Skmacy * deferred if the connection is in SYN_SENT since we don't know the TID yet. 221174641Skmacy * For connections in other states the message is sent immediately. 222174641Skmacy * If through_l2t is set the message is subject to ARP processing, otherwise 223174641Skmacy * it is sent directly. 224174641Skmacy */ 225174641Skmacystatic inline void 226176472Skmacysend_or_defer(struct toepcb *toep, struct mbuf *m, int through_l2t) 227174641Skmacy{ 228176472Skmacy struct tcpcb *tp = toep->tp_tp; 229174641Skmacy 230174641Skmacy if (__predict_false(tp->t_state == TCPS_SYN_SENT)) { 231177530Skmacy inp_wlock(tp->t_inpcb); 232174641Skmacy mbufq_tail(&toep->out_of_order_queue, m); // defer 233177530Skmacy inp_wunlock(tp->t_inpcb); 234174641Skmacy } else if (through_l2t) 235176472Skmacy l2t_send(TOEP_T3C_DEV(toep), m, toep->tp_l2t); // send through L2T 236174641Skmacy else 237176472Skmacy cxgb_ofld_send(TOEP_T3C_DEV(toep), m); // send directly 238174641Skmacy} 239174641Skmacy 240174641Skmacystatic inline unsigned int 241176472Skmacymkprio(unsigned int cntrl, const struct toepcb *toep) 242174641Skmacy{ 243176472Skmacy return (cntrl); 244174641Skmacy} 245174641Skmacy 246174641Skmacy/* 247174641Skmacy * Populate a TID_RELEASE WR. The skb must be already propely sized. 248174641Skmacy */ 249174641Skmacystatic inline void 250176472Skmacymk_tid_release(struct mbuf *m, const struct toepcb *toep, unsigned int tid) 251174641Skmacy{ 252174641Skmacy struct cpl_tid_release *req; 253174641Skmacy 254176472Skmacy m_set_priority(m, mkprio(CPL_PRIORITY_SETUP, toep)); 255174641Skmacy m->m_pkthdr.len = m->m_len = sizeof(*req); 256174641Skmacy req = mtod(m, struct cpl_tid_release *); 257174641Skmacy req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 258176472Skmacy req->wr.wr_lo = 0; 259174641Skmacy OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_TID_RELEASE, tid)); 260174641Skmacy} 261174641Skmacy 262174641Skmacystatic inline void 263174641Skmacymake_tx_data_wr(struct socket *so, struct mbuf *m, int len, struct mbuf *tail) 264174641Skmacy{ 265178302Skmacy struct tcpcb *tp = so_sototcpcb(so); 266174641Skmacy struct toepcb *toep = tp->t_toe; 267174641Skmacy struct tx_data_wr *req; 268178302Skmacy struct sockbuf *snd; 269178302Skmacy 270177575Skmacy inp_lock_assert(tp->t_inpcb); 271178302Skmacy snd = so_sockbuf_snd(so); 272178302Skmacy 273174641Skmacy req = mtod(m, struct tx_data_wr *); 274174641Skmacy m->m_len = sizeof(*req); 275174641Skmacy req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_TX_DATA)); 276174641Skmacy req->wr_lo = htonl(V_WR_TID(toep->tp_tid)); 277174641Skmacy /* len includes the length of any HW ULP additions */ 278174641Skmacy req->len = htonl(len); 279174641Skmacy req->param = htonl(V_TX_PORT(toep->tp_l2t->smt_idx)); 280174641Skmacy /* V_TX_ULP_SUBMODE sets both the mode and submode */ 281174641Skmacy req->flags = htonl(V_TX_ULP_SUBMODE(/*skb_ulp_mode(skb)*/ 0) | 282174641Skmacy V_TX_URG(/* skb_urgent(skb) */ 0 ) | 283174641Skmacy V_TX_SHOVE((!(tp->t_flags & TF_MORETOCOME) && 284174641Skmacy (tail ? 0 : 1)))); 285174641Skmacy req->sndseq = htonl(tp->snd_nxt); 286174641Skmacy if (__predict_false((toep->tp_flags & TP_DATASENT) == 0)) { 287174641Skmacy req->flags |= htonl(V_TX_ACK_PAGES(2) | F_TX_INIT | 288174641Skmacy V_TX_CPU_IDX(toep->tp_qset)); 289174641Skmacy 290174641Skmacy /* Sendbuffer is in units of 32KB. 291174641Skmacy */ 292178302Skmacy if (tcp_do_autosndbuf && snd->sb_flags & SB_AUTOSIZE) 293174641Skmacy req->param |= htonl(V_TX_SNDBUF(tcp_autosndbuf_max >> 15)); 294178302Skmacy else { 295178302Skmacy req->param |= htonl(V_TX_SNDBUF(snd->sb_hiwat >> 15)); 296178302Skmacy } 297178302Skmacy 298174641Skmacy toep->tp_flags |= TP_DATASENT; 299174641Skmacy } 300174641Skmacy} 301174641Skmacy 302176472Skmacy#define IMM_LEN 64 /* XXX - see WR_LEN in the cxgb driver */ 303176472Skmacy 304174641Skmacyint 305174641Skmacyt3_push_frames(struct socket *so, int req_completion) 306174641Skmacy{ 307178302Skmacy struct tcpcb *tp = so_sototcpcb(so); 308174641Skmacy struct toepcb *toep = tp->t_toe; 309174641Skmacy 310174641Skmacy struct mbuf *tail, *m0, *last; 311174641Skmacy struct t3cdev *cdev; 312174641Skmacy struct tom_data *d; 313178302Skmacy int state, bytes, count, total_bytes; 314174641Skmacy bus_dma_segment_t segs[TX_MAX_SEGS], *segp; 315178302Skmacy struct sockbuf *snd; 316178302Skmacy 317174641Skmacy if (tp->t_state == TCPS_SYN_SENT || tp->t_state == TCPS_CLOSED) { 318174641Skmacy DPRINTF("tcp state=%d\n", tp->t_state); 319174641Skmacy return (0); 320174641Skmacy } 321174641Skmacy 322178302Skmacy state = so_state_get(so); 323178302Skmacy 324178302Skmacy if (state & (SS_ISDISCONNECTING|SS_ISDISCONNECTED)) { 325174641Skmacy DPRINTF("disconnecting\n"); 326174641Skmacy 327174641Skmacy return (0); 328174641Skmacy } 329174641Skmacy 330177575Skmacy inp_lock_assert(tp->t_inpcb); 331178302Skmacy 332178302Skmacy snd = so_sockbuf_snd(so); 333178302Skmacy sockbuf_lock(snd); 334178302Skmacy 335178302Skmacy d = TOM_DATA(toep->tp_toedev); 336174641Skmacy cdev = d->cdev; 337178302Skmacy 338178302Skmacy last = tail = snd->sb_sndptr ? snd->sb_sndptr : snd->sb_mb; 339178302Skmacy 340174641Skmacy total_bytes = 0; 341174641Skmacy DPRINTF("wr_avail=%d tail=%p snd.cc=%d tp_last=%p\n", 342178302Skmacy toep->tp_wr_avail, tail, snd->sb_cc, toep->tp_m_last); 343174641Skmacy 344178302Skmacy if (last && toep->tp_m_last == last && snd->sb_sndptroff != 0) { 345174641Skmacy KASSERT(tail, ("sbdrop error")); 346174641Skmacy last = tail = tail->m_next; 347174641Skmacy } 348174641Skmacy 349174641Skmacy if ((toep->tp_wr_avail == 0 ) || (tail == NULL)) { 350174641Skmacy DPRINTF("wr_avail=%d tail=%p\n", toep->tp_wr_avail, tail); 351178302Skmacy sockbuf_unlock(snd); 352178302Skmacy 353174641Skmacy return (0); 354174641Skmacy } 355174641Skmacy 356174641Skmacy toep->tp_m_last = NULL; 357174641Skmacy while (toep->tp_wr_avail && (tail != NULL)) { 358174641Skmacy count = bytes = 0; 359176472Skmacy segp = segs; 360174641Skmacy if ((m0 = m_gethdr(M_NOWAIT, MT_DATA)) == NULL) { 361178302Skmacy sockbuf_unlock(snd); 362174641Skmacy return (0); 363174641Skmacy } 364176472Skmacy /* 365176472Skmacy * If the data in tail fits as in-line, then 366176472Skmacy * make an immediate data wr. 367176472Skmacy */ 368176472Skmacy if (tail->m_len <= IMM_LEN) { 369176472Skmacy count = 1; 370176472Skmacy bytes = tail->m_len; 371174641Skmacy last = tail; 372174641Skmacy tail = tail->m_next; 373176472Skmacy m_set_sgl(m0, NULL); 374176472Skmacy m_set_sgllen(m0, 0); 375176472Skmacy make_tx_data_wr(so, m0, bytes, tail); 376176472Skmacy m_append(m0, bytes, mtod(last, caddr_t)); 377176472Skmacy KASSERT(!m0->m_next, ("bad append")); 378176472Skmacy } else { 379176472Skmacy while ((mbuf_wrs[count + 1] <= toep->tp_wr_avail) 380176472Skmacy && (tail != NULL) && (count < TX_MAX_SEGS-1)) { 381176472Skmacy bytes += tail->m_len; 382176472Skmacy last = tail; 383176472Skmacy count++; 384176472Skmacy /* 385176472Skmacy * technically an abuse to be using this for a VA 386176472Skmacy * but less gross than defining my own structure 387176472Skmacy * or calling pmap_kextract from here :-| 388176472Skmacy */ 389176472Skmacy segp->ds_addr = (bus_addr_t)tail->m_data; 390176472Skmacy segp->ds_len = tail->m_len; 391176472Skmacy DPRINTF("count=%d wr_needed=%d ds_addr=%p ds_len=%d\n", 392176472Skmacy count, mbuf_wrs[count], tail->m_data, tail->m_len); 393176472Skmacy segp++; 394176472Skmacy tail = tail->m_next; 395176472Skmacy } 396176472Skmacy DPRINTF("wr_avail=%d mbuf_wrs[%d]=%d tail=%p\n", 397176472Skmacy toep->tp_wr_avail, count, mbuf_wrs[count], tail); 398176472Skmacy 399176472Skmacy m_set_sgl(m0, segs); 400176472Skmacy m_set_sgllen(m0, count); 401176472Skmacy make_tx_data_wr(so, m0, bytes, tail); 402174641Skmacy } 403176472Skmacy m_set_priority(m0, mkprio(CPL_PRIORITY_DATA, toep)); 404176472Skmacy 405174641Skmacy if (tail) { 406178302Skmacy snd->sb_sndptr = tail; 407174641Skmacy toep->tp_m_last = NULL; 408174641Skmacy } else 409178302Skmacy toep->tp_m_last = snd->sb_sndptr = last; 410174641Skmacy 411176472Skmacy 412174641Skmacy DPRINTF("toep->tp_m_last=%p\n", toep->tp_m_last); 413174641Skmacy 414178302Skmacy snd->sb_sndptroff += bytes; 415174641Skmacy total_bytes += bytes; 416174641Skmacy toep->tp_write_seq += bytes; 417176472Skmacy CTR6(KTR_TOM, "t3_push_frames: wr_avail=%d mbuf_wrs[%d]=%d tail=%p sndptr=%p sndptroff=%d", 418178302Skmacy toep->tp_wr_avail, count, mbuf_wrs[count], tail, snd->sb_sndptr, snd->sb_sndptroff); 419176472Skmacy if (tail) 420176472Skmacy CTR4(KTR_TOM, "t3_push_frames: total_bytes=%d tp_m_last=%p tailbuf=%p snd_una=0x%08x", 421176472Skmacy total_bytes, toep->tp_m_last, tail->m_data, tp->snd_una); 422176472Skmacy else 423176472Skmacy CTR3(KTR_TOM, "t3_push_frames: total_bytes=%d tp_m_last=%p snd_una=0x%08x", 424176472Skmacy total_bytes, toep->tp_m_last, tp->snd_una); 425174641Skmacy 426174641Skmacy 427178302Skmacy#ifdef KTR 428178302Skmacy{ 429178302Skmacy int i; 430178302Skmacy 431176472Skmacy i = 0; 432176472Skmacy while (i < count && m_get_sgllen(m0)) { 433176472Skmacy if ((count - i) >= 3) { 434176472Skmacy CTR6(KTR_TOM, 435176472Skmacy "t3_push_frames: pa=0x%zx len=%d pa=0x%zx len=%d pa=0x%zx len=%d", 436176472Skmacy segs[i].ds_addr, segs[i].ds_len, segs[i + 1].ds_addr, segs[i + 1].ds_len, 437176472Skmacy segs[i + 2].ds_addr, segs[i + 2].ds_len); 438176472Skmacy i += 3; 439176472Skmacy } else if ((count - i) == 2) { 440176472Skmacy CTR4(KTR_TOM, 441176472Skmacy "t3_push_frames: pa=0x%zx len=%d pa=0x%zx len=%d", 442176472Skmacy segs[i].ds_addr, segs[i].ds_len, segs[i + 1].ds_addr, segs[i + 1].ds_len); 443176472Skmacy i += 2; 444176472Skmacy } else { 445176472Skmacy CTR2(KTR_TOM, "t3_push_frames: pa=0x%zx len=%d", 446176472Skmacy segs[i].ds_addr, segs[i].ds_len); 447176472Skmacy i++; 448176472Skmacy } 449174641Skmacy 450176472Skmacy } 451178302Skmacy} 452178302Skmacy#endif 453176472Skmacy /* 454174641Skmacy * remember credits used 455174641Skmacy */ 456174641Skmacy m0->m_pkthdr.csum_data = mbuf_wrs[count]; 457174641Skmacy m0->m_pkthdr.len = bytes; 458176472Skmacy toep->tp_wr_avail -= mbuf_wrs[count]; 459176472Skmacy toep->tp_wr_unacked += mbuf_wrs[count]; 460176472Skmacy 461174641Skmacy if ((req_completion && toep->tp_wr_unacked == mbuf_wrs[count]) || 462174641Skmacy toep->tp_wr_unacked >= toep->tp_wr_max / 2) { 463174641Skmacy struct work_request_hdr *wr = cplhdr(m0); 464174641Skmacy 465174641Skmacy wr->wr_hi |= htonl(F_WR_COMPL); 466174641Skmacy toep->tp_wr_unacked = 0; 467174641Skmacy } 468176472Skmacy KASSERT((m0->m_pkthdr.csum_data > 0) && 469176472Skmacy (m0->m_pkthdr.csum_data <= 4), ("bad credit count %d", 470176472Skmacy m0->m_pkthdr.csum_data)); 471174641Skmacy m0->m_type = MT_DONTFREE; 472174641Skmacy enqueue_wr(toep, m0); 473174641Skmacy DPRINTF("sending offload tx with %d bytes in %d segments\n", 474174641Skmacy bytes, count); 475174641Skmacy l2t_send(cdev, m0, toep->tp_l2t); 476174641Skmacy } 477178302Skmacy sockbuf_unlock(snd); 478174641Skmacy return (total_bytes); 479174641Skmacy} 480174641Skmacy 481174641Skmacy/* 482174641Skmacy * Close a connection by sending a CPL_CLOSE_CON_REQ message. Cannot fail 483174641Skmacy * under any circumstances. We take the easy way out and always queue the 484174641Skmacy * message to the write_queue. We can optimize the case where the queue is 485174641Skmacy * already empty though the optimization is probably not worth it. 486174641Skmacy */ 487174641Skmacystatic void 488174641Skmacyclose_conn(struct socket *so) 489174641Skmacy{ 490174641Skmacy struct mbuf *m; 491174641Skmacy struct cpl_close_con_req *req; 492174641Skmacy struct tom_data *d; 493178302Skmacy struct inpcb *inp = so_sotoinpcb(so); 494174641Skmacy struct tcpcb *tp; 495174641Skmacy struct toepcb *toep; 496174641Skmacy unsigned int tid; 497174641Skmacy 498174641Skmacy 499177530Skmacy inp_wlock(inp); 500178302Skmacy tp = so_sototcpcb(so); 501174641Skmacy toep = tp->t_toe; 502174641Skmacy 503174641Skmacy if (tp->t_state != TCPS_SYN_SENT) 504174641Skmacy t3_push_frames(so, 1); 505174641Skmacy 506174641Skmacy if (toep->tp_flags & TP_FIN_SENT) { 507177530Skmacy inp_wunlock(inp); 508174641Skmacy return; 509174641Skmacy } 510174641Skmacy 511174641Skmacy tid = toep->tp_tid; 512174641Skmacy 513174641Skmacy d = TOM_DATA(toep->tp_toedev); 514174641Skmacy 515174641Skmacy m = m_gethdr_nofail(sizeof(*req)); 516178302Skmacy m_set_priority(m, CPL_PRIORITY_DATA); 517178302Skmacy m_set_sgl(m, NULL); 518178302Skmacy m_set_sgllen(m, 0); 519174641Skmacy 520174641Skmacy toep->tp_flags |= TP_FIN_SENT; 521174641Skmacy req = mtod(m, struct cpl_close_con_req *); 522174641Skmacy 523174641Skmacy req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_CLOSE_CON)); 524174641Skmacy req->wr.wr_lo = htonl(V_WR_TID(tid)); 525174641Skmacy OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_CON_REQ, tid)); 526178302Skmacy req->rsvd = 0; 527177530Skmacy inp_wunlock(inp); 528174641Skmacy /* 529174641Skmacy * XXX - need to defer shutdown while there is still data in the queue 530174641Skmacy * 531174641Skmacy */ 532178302Skmacy CTR4(KTR_TOM, "%s CLOSE_CON_REQ so %p tp %p tid=%u", __FUNCTION__, so, tp, tid); 533174641Skmacy cxgb_ofld_send(d->cdev, m); 534174641Skmacy 535174641Skmacy} 536174641Skmacy 537174641Skmacy/* 538174641Skmacy * Handle an ARP failure for a CPL_ABORT_REQ. Change it into a no RST variant 539174641Skmacy * and send it along. 540174641Skmacy */ 541174641Skmacystatic void 542174641Skmacyabort_arp_failure(struct t3cdev *cdev, struct mbuf *m) 543174641Skmacy{ 544174641Skmacy struct cpl_abort_req *req = cplhdr(m); 545174641Skmacy 546174641Skmacy req->cmd = CPL_ABORT_NO_RST; 547174641Skmacy cxgb_ofld_send(cdev, m); 548174641Skmacy} 549174641Skmacy 550174641Skmacy/* 551174641Skmacy * Send RX credits through an RX_DATA_ACK CPL message. If nofail is 0 we are 552174641Skmacy * permitted to return without sending the message in case we cannot allocate 553174641Skmacy * an sk_buff. Returns the number of credits sent. 554174641Skmacy */ 555174641Skmacyuint32_t 556174641Skmacyt3_send_rx_credits(struct tcpcb *tp, uint32_t credits, uint32_t dack, int nofail) 557174641Skmacy{ 558174641Skmacy struct mbuf *m; 559174641Skmacy struct cpl_rx_data_ack *req; 560174641Skmacy struct toepcb *toep = tp->t_toe; 561174641Skmacy struct toedev *tdev = toep->tp_toedev; 562174641Skmacy 563174641Skmacy m = m_gethdr_nofail(sizeof(*req)); 564174641Skmacy 565174641Skmacy DPRINTF("returning %u credits to HW\n", credits); 566174641Skmacy 567174641Skmacy req = mtod(m, struct cpl_rx_data_ack *); 568174641Skmacy req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 569176472Skmacy req->wr.wr_lo = 0; 570174641Skmacy OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RX_DATA_ACK, toep->tp_tid)); 571174641Skmacy req->credit_dack = htonl(dack | V_RX_CREDITS(credits)); 572176472Skmacy m_set_priority(m, mkprio(CPL_PRIORITY_ACK, toep)); 573174641Skmacy cxgb_ofld_send(TOM_DATA(tdev)->cdev, m); 574174641Skmacy return (credits); 575174641Skmacy} 576174641Skmacy 577176472Skmacy/* 578176472Skmacy * Send RX_DATA_ACK CPL message to request a modulation timer to be scheduled. 579176472Skmacy * This is only used in DDP mode, so we take the opportunity to also set the 580176472Skmacy * DACK mode and flush any Rx credits. 581176472Skmacy */ 582176472Skmacyvoid 583176472Skmacyt3_send_rx_modulate(struct toepcb *toep) 584176472Skmacy{ 585176472Skmacy struct mbuf *m; 586176472Skmacy struct cpl_rx_data_ack *req; 587174641Skmacy 588176472Skmacy m = m_gethdr_nofail(sizeof(*req)); 589176472Skmacy 590176472Skmacy req = mtod(m, struct cpl_rx_data_ack *); 591176472Skmacy req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 592176472Skmacy req->wr.wr_lo = 0; 593176472Skmacy m->m_pkthdr.len = m->m_len = sizeof(*req); 594176472Skmacy 595176472Skmacy OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RX_DATA_ACK, toep->tp_tid)); 596176472Skmacy req->credit_dack = htonl(F_RX_MODULATE | F_RX_DACK_CHANGE | 597176472Skmacy V_RX_DACK_MODE(1) | 598176472Skmacy V_RX_CREDITS(toep->tp_copied_seq - toep->tp_rcv_wup)); 599176472Skmacy m_set_priority(m, mkprio(CPL_PRIORITY_CONTROL, toep)); 600176472Skmacy cxgb_ofld_send(TOEP_T3C_DEV(toep), m); 601176472Skmacy toep->tp_rcv_wup = toep->tp_copied_seq; 602176472Skmacy} 603176472Skmacy 604174641Skmacy/* 605176472Skmacy * Handle receipt of an urgent pointer. 606176472Skmacy */ 607176472Skmacystatic void 608176472Skmacyhandle_urg_ptr(struct socket *so, uint32_t urg_seq) 609176472Skmacy{ 610176472Skmacy#ifdef URGENT_DATA_SUPPORTED 611178302Skmacy struct tcpcb *tp = so_sototcpcb(so); 612176472Skmacy 613176472Skmacy urg_seq--; /* initially points past the urgent data, per BSD */ 614176472Skmacy 615176472Skmacy if (tp->urg_data && !after(urg_seq, tp->urg_seq)) 616176472Skmacy return; /* duplicate pointer */ 617176472Skmacy sk_send_sigurg(sk); 618176472Skmacy if (tp->urg_seq == tp->copied_seq && tp->urg_data && 619176472Skmacy !sock_flag(sk, SOCK_URGINLINE) && tp->copied_seq != tp->rcv_nxt) { 620176472Skmacy struct sk_buff *skb = skb_peek(&sk->sk_receive_queue); 621176472Skmacy 622176472Skmacy tp->copied_seq++; 623176472Skmacy if (skb && tp->copied_seq - TCP_SKB_CB(skb)->seq >= skb->len) 624176472Skmacy tom_eat_skb(sk, skb, 0); 625176472Skmacy } 626176472Skmacy tp->urg_data = TCP_URG_NOTYET; 627176472Skmacy tp->urg_seq = urg_seq; 628176472Skmacy#endif 629176472Skmacy} 630176472Skmacy 631176472Skmacy/* 632176472Skmacy * Returns true if a socket cannot accept new Rx data. 633176472Skmacy */ 634176472Skmacystatic inline int 635176472Skmacyso_no_receive(const struct socket *so) 636176472Skmacy{ 637178302Skmacy return (so_state_get(so) & (SS_ISDISCONNECTED|SS_ISDISCONNECTING)); 638176472Skmacy} 639176472Skmacy 640176472Skmacy/* 641176472Skmacy * Process an urgent data notification. 642176472Skmacy */ 643176472Skmacystatic void 644176472Skmacyrx_urg_notify(struct toepcb *toep, struct mbuf *m) 645176472Skmacy{ 646176472Skmacy struct cpl_rx_urg_notify *hdr = cplhdr(m); 647178302Skmacy struct socket *so = inp_inpcbtosocket(toep->tp_tp->t_inpcb); 648176472Skmacy 649176472Skmacy VALIDATE_SOCK(so); 650176472Skmacy 651176472Skmacy if (!so_no_receive(so)) 652176472Skmacy handle_urg_ptr(so, ntohl(hdr->seq)); 653176472Skmacy 654176472Skmacy m_freem(m); 655176472Skmacy} 656176472Skmacy 657176472Skmacy/* 658176472Skmacy * Handler for RX_URG_NOTIFY CPL messages. 659176472Skmacy */ 660176472Skmacystatic int 661176472Skmacydo_rx_urg_notify(struct t3cdev *cdev, struct mbuf *m, void *ctx) 662176472Skmacy{ 663176472Skmacy struct toepcb *toep = (struct toepcb *)ctx; 664176472Skmacy 665176472Skmacy rx_urg_notify(toep, m); 666176472Skmacy return (0); 667176472Skmacy} 668176472Skmacy 669177340Skmacystatic __inline int 670177340Skmacyis_delack_mode_valid(struct toedev *dev, struct toepcb *toep) 671177340Skmacy{ 672177340Skmacy return (toep->tp_ulp_mode || 673177340Skmacy (toep->tp_ulp_mode == ULP_MODE_TCPDDP && 674177340Skmacy dev->tod_ttid >= TOE_ID_CHELSIO_T3)); 675177340Skmacy} 676177340Skmacy 677176472Skmacy/* 678174641Skmacy * Set of states for which we should return RX credits. 679174641Skmacy */ 680174641Skmacy#define CREDIT_RETURN_STATE (TCPF_ESTABLISHED | TCPF_FIN_WAIT1 | TCPF_FIN_WAIT2) 681174641Skmacy 682174641Skmacy/* 683174641Skmacy * Called after some received data has been read. It returns RX credits 684174641Skmacy * to the HW for the amount of data processed. 685174641Skmacy */ 686174641Skmacyvoid 687176472Skmacyt3_cleanup_rbuf(struct tcpcb *tp, int copied) 688174641Skmacy{ 689174641Skmacy struct toepcb *toep = tp->t_toe; 690174641Skmacy struct socket *so; 691174641Skmacy struct toedev *dev; 692174641Skmacy int dack_mode, must_send, read; 693174641Skmacy u32 thres, credits, dack = 0; 694178302Skmacy struct sockbuf *rcv; 695178302Skmacy 696178302Skmacy so = inp_inpcbtosocket(tp->t_inpcb); 697178302Skmacy rcv = so_sockbuf_rcv(so); 698174641Skmacy 699174641Skmacy if (!((tp->t_state == TCPS_ESTABLISHED) || (tp->t_state == TCPS_FIN_WAIT_1) || 700176472Skmacy (tp->t_state == TCPS_FIN_WAIT_2))) { 701176472Skmacy if (copied) { 702178302Skmacy sockbuf_lock(rcv); 703176472Skmacy toep->tp_copied_seq += copied; 704178302Skmacy sockbuf_unlock(rcv); 705176472Skmacy } 706176472Skmacy 707174641Skmacy return; 708176472Skmacy } 709174641Skmacy 710178302Skmacy inp_lock_assert(tp->t_inpcb); 711178302Skmacy 712178302Skmacy sockbuf_lock(rcv); 713176472Skmacy if (copied) 714176472Skmacy toep->tp_copied_seq += copied; 715176472Skmacy else { 716178302Skmacy read = toep->tp_enqueued_bytes - rcv->sb_cc; 717176472Skmacy toep->tp_copied_seq += read; 718176472Skmacy } 719174641Skmacy credits = toep->tp_copied_seq - toep->tp_rcv_wup; 720178302Skmacy toep->tp_enqueued_bytes = rcv->sb_cc; 721178302Skmacy sockbuf_unlock(rcv); 722174641Skmacy 723178302Skmacy if (credits > rcv->sb_mbmax) { 724178302Skmacy log(LOG_ERR, "copied_seq=%u rcv_wup=%u credits=%u\n", 725178302Skmacy toep->tp_copied_seq, toep->tp_rcv_wup, credits); 726178302Skmacy credits = rcv->sb_mbmax; 727176472Skmacy } 728176472Skmacy 729176472Skmacy 730178302Skmacy /* 731174641Skmacy * XXX this won't accurately reflect credit return - we need 732174641Skmacy * to look at the difference between the amount that has been 733174641Skmacy * put in the recv sockbuf and what is there now 734174641Skmacy */ 735174641Skmacy 736174641Skmacy if (__predict_false(!credits)) 737174641Skmacy return; 738174641Skmacy 739174641Skmacy dev = toep->tp_toedev; 740174641Skmacy thres = TOM_TUNABLE(dev, rx_credit_thres); 741174641Skmacy 742174641Skmacy if (__predict_false(thres == 0)) 743174641Skmacy return; 744174641Skmacy 745177340Skmacy if (is_delack_mode_valid(dev, toep)) { 746174641Skmacy dack_mode = TOM_TUNABLE(dev, delack); 747174641Skmacy if (__predict_false(dack_mode != toep->tp_delack_mode)) { 748174641Skmacy u32 r = tp->rcv_nxt - toep->tp_delack_seq; 749174641Skmacy 750174641Skmacy if (r >= tp->rcv_wnd || r >= 16 * toep->tp_mss_clamp) 751174641Skmacy dack = F_RX_DACK_CHANGE | 752174641Skmacy V_RX_DACK_MODE(dack_mode); 753174641Skmacy } 754177340Skmacy } else 755177340Skmacy dack = F_RX_DACK_CHANGE | V_RX_DACK_MODE(1); 756177340Skmacy 757174641Skmacy /* 758174641Skmacy * For coalescing to work effectively ensure the receive window has 759174641Skmacy * at least 16KB left. 760174641Skmacy */ 761174641Skmacy must_send = credits + 16384 >= tp->rcv_wnd; 762174641Skmacy 763174641Skmacy if (must_send || credits >= thres) 764174641Skmacy toep->tp_rcv_wup += t3_send_rx_credits(tp, credits, dack, must_send); 765174641Skmacy} 766174641Skmacy 767174641Skmacystatic int 768174641Skmacycxgb_toe_disconnect(struct tcpcb *tp) 769174641Skmacy{ 770174641Skmacy struct socket *so; 771174641Skmacy 772174641Skmacy DPRINTF("cxgb_toe_disconnect\n"); 773174641Skmacy 774178302Skmacy so = inp_inpcbtosocket(tp->t_inpcb); 775174641Skmacy close_conn(so); 776174641Skmacy return (0); 777174641Skmacy} 778174641Skmacy 779174641Skmacystatic int 780174708Skmacycxgb_toe_reset(struct tcpcb *tp) 781174641Skmacy{ 782174641Skmacy struct toepcb *toep = tp->t_toe; 783177530Skmacy 784174641Skmacy t3_send_reset(toep); 785174641Skmacy 786174641Skmacy /* 787174641Skmacy * unhook from socket 788174641Skmacy */ 789174641Skmacy tp->t_flags &= ~TF_TOE; 790174641Skmacy toep->tp_tp = NULL; 791174641Skmacy tp->t_toe = NULL; 792174641Skmacy return (0); 793174641Skmacy} 794174641Skmacy 795174641Skmacystatic int 796174641Skmacycxgb_toe_send(struct tcpcb *tp) 797174641Skmacy{ 798174641Skmacy struct socket *so; 799174641Skmacy 800174641Skmacy DPRINTF("cxgb_toe_send\n"); 801174641Skmacy dump_toepcb(tp->t_toe); 802174641Skmacy 803178302Skmacy so = inp_inpcbtosocket(tp->t_inpcb); 804174641Skmacy t3_push_frames(so, 1); 805174641Skmacy return (0); 806174641Skmacy} 807174641Skmacy 808174641Skmacystatic int 809174641Skmacycxgb_toe_rcvd(struct tcpcb *tp) 810174641Skmacy{ 811177530Skmacy 812177575Skmacy inp_lock_assert(tp->t_inpcb); 813178302Skmacy 814176472Skmacy t3_cleanup_rbuf(tp, 0); 815174641Skmacy 816174641Skmacy return (0); 817174641Skmacy} 818174641Skmacy 819174641Skmacystatic void 820174641Skmacycxgb_toe_detach(struct tcpcb *tp) 821174641Skmacy{ 822174641Skmacy struct toepcb *toep; 823177530Skmacy 824177530Skmacy /* 825174641Skmacy * XXX how do we handle teardown in the SYN_SENT state? 826174641Skmacy * 827174641Skmacy */ 828177575Skmacy inp_lock_assert(tp->t_inpcb); 829174641Skmacy toep = tp->t_toe; 830174641Skmacy toep->tp_tp = NULL; 831174641Skmacy 832174641Skmacy /* 833174641Skmacy * unhook from socket 834174641Skmacy */ 835174641Skmacy tp->t_flags &= ~TF_TOE; 836174641Skmacy tp->t_toe = NULL; 837174641Skmacy} 838174641Skmacy 839174641Skmacy 840174641Skmacystatic struct toe_usrreqs cxgb_toe_usrreqs = { 841174641Skmacy .tu_disconnect = cxgb_toe_disconnect, 842174708Skmacy .tu_reset = cxgb_toe_reset, 843174641Skmacy .tu_send = cxgb_toe_send, 844174641Skmacy .tu_rcvd = cxgb_toe_rcvd, 845174641Skmacy .tu_detach = cxgb_toe_detach, 846174641Skmacy .tu_detach = cxgb_toe_detach, 847174641Skmacy .tu_syncache_event = handle_syncache_event, 848174641Skmacy}; 849174641Skmacy 850174641Skmacy 851174641Skmacystatic void 852176472Skmacy__set_tcb_field(struct toepcb *toep, struct mbuf *m, uint16_t word, 853174641Skmacy uint64_t mask, uint64_t val, int no_reply) 854174641Skmacy{ 855174641Skmacy struct cpl_set_tcb_field *req; 856174641Skmacy 857176472Skmacy CTR4(KTR_TCB, "__set_tcb_field_ulp(tid=%u word=0x%x mask=%jx val=%jx", 858176472Skmacy toep->tp_tid, word, mask, val); 859176472Skmacy 860174641Skmacy req = mtod(m, struct cpl_set_tcb_field *); 861174641Skmacy m->m_pkthdr.len = m->m_len = sizeof(*req); 862174641Skmacy req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 863176472Skmacy req->wr.wr_lo = 0; 864174641Skmacy OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, toep->tp_tid)); 865174641Skmacy req->reply = V_NO_REPLY(no_reply); 866174641Skmacy req->cpu_idx = 0; 867174641Skmacy req->word = htons(word); 868174641Skmacy req->mask = htobe64(mask); 869174641Skmacy req->val = htobe64(val); 870174641Skmacy 871176472Skmacy m_set_priority(m, mkprio(CPL_PRIORITY_CONTROL, toep)); 872176472Skmacy send_or_defer(toep, m, 0); 873174641Skmacy} 874174641Skmacy 875174641Skmacystatic void 876178302Skmacyt3_set_tcb_field(struct toepcb *toep, uint16_t word, uint64_t mask, uint64_t val) 877174641Skmacy{ 878174641Skmacy struct mbuf *m; 879178302Skmacy struct tcpcb *tp = toep->tp_tp; 880174641Skmacy 881174641Skmacy if (toep == NULL) 882174641Skmacy return; 883176472Skmacy 884176472Skmacy if (tp->t_state == TCPS_CLOSED || (toep->tp_flags & TP_ABORT_SHUTDOWN)) { 885176472Skmacy printf("not seting field\n"); 886174641Skmacy return; 887176472Skmacy } 888176472Skmacy 889174641Skmacy m = m_gethdr_nofail(sizeof(struct cpl_set_tcb_field)); 890174641Skmacy 891176472Skmacy __set_tcb_field(toep, m, word, mask, val, 1); 892174641Skmacy} 893174641Skmacy 894174641Skmacy/* 895174641Skmacy * Set one of the t_flags bits in the TCB. 896174641Skmacy */ 897174641Skmacystatic void 898178302Skmacyset_tcb_tflag(struct toepcb *toep, unsigned int bit_pos, int val) 899174641Skmacy{ 900178302Skmacy 901178302Skmacy t3_set_tcb_field(toep, W_TCB_T_FLAGS1, 1ULL << bit_pos, val << bit_pos); 902174641Skmacy} 903174641Skmacy 904174641Skmacy/* 905174641Skmacy * Send a SET_TCB_FIELD CPL message to change a connection's Nagle setting. 906174641Skmacy */ 907174641Skmacystatic void 908178302Skmacyt3_set_nagle(struct toepcb *toep) 909174641Skmacy{ 910178302Skmacy struct tcpcb *tp = toep->tp_tp; 911174641Skmacy 912178302Skmacy set_tcb_tflag(toep, S_TF_NAGLE, !(tp->t_flags & TF_NODELAY)); 913174641Skmacy} 914174641Skmacy 915174641Skmacy/* 916174641Skmacy * Send a SET_TCB_FIELD CPL message to change a connection's keepalive setting. 917174641Skmacy */ 918174641Skmacyvoid 919178302Skmacyt3_set_keepalive(struct toepcb *toep, int on_off) 920174641Skmacy{ 921178302Skmacy 922178302Skmacy set_tcb_tflag(toep, S_TF_KEEPALIVE, on_off); 923174641Skmacy} 924174641Skmacy 925174641Skmacyvoid 926178302Skmacyt3_set_rcv_coalesce_enable(struct toepcb *toep, int on_off) 927174641Skmacy{ 928178302Skmacy set_tcb_tflag(toep, S_TF_RCV_COALESCE_ENABLE, on_off); 929174641Skmacy} 930174641Skmacy 931177340Skmacyvoid 932178302Skmacyt3_set_dack_mss(struct toepcb *toep, int on_off) 933177340Skmacy{ 934178302Skmacy 935178302Skmacy set_tcb_tflag(toep, S_TF_DACK_MSS, on_off); 936177340Skmacy} 937177340Skmacy 938174641Skmacy/* 939174641Skmacy * Send a SET_TCB_FIELD CPL message to change a connection's TOS setting. 940174641Skmacy */ 941174641Skmacystatic void 942178302Skmacyt3_set_tos(struct toepcb *toep) 943174641Skmacy{ 944178302Skmacy int tos = inp_ip_tos_get(toep->tp_tp->t_inpcb); 945178302Skmacy 946178302Skmacy t3_set_tcb_field(toep, W_TCB_TOS, V_TCB_TOS(M_TCB_TOS), 947178302Skmacy V_TCB_TOS(tos)); 948174641Skmacy} 949174641Skmacy 950174641Skmacy 951174641Skmacy/* 952174641Skmacy * In DDP mode, TP fails to schedule a timer to push RX data to the host when 953174641Skmacy * DDP is disabled (data is delivered to freelist). [Note that, the peer should 954174641Skmacy * set the PSH bit in the last segment, which would trigger delivery.] 955174641Skmacy * We work around the issue by setting a DDP buffer in a partial placed state, 956174641Skmacy * which guarantees that TP will schedule a timer. 957174641Skmacy */ 958174641Skmacy#define TP_DDP_TIMER_WORKAROUND_MASK\ 959174641Skmacy (V_TF_DDP_BUF0_VALID(1) | V_TF_DDP_ACTIVE_BUF(1) |\ 960174641Skmacy ((V_TCB_RX_DDP_BUF0_OFFSET(M_TCB_RX_DDP_BUF0_OFFSET) |\ 961174641Skmacy V_TCB_RX_DDP_BUF0_LEN(3)) << 32)) 962174641Skmacy#define TP_DDP_TIMER_WORKAROUND_VAL\ 963174641Skmacy (V_TF_DDP_BUF0_VALID(1) | V_TF_DDP_ACTIVE_BUF(0) |\ 964174641Skmacy ((V_TCB_RX_DDP_BUF0_OFFSET((uint64_t)1) | V_TCB_RX_DDP_BUF0_LEN((uint64_t)2)) <<\ 965174641Skmacy 32)) 966174641Skmacy 967174641Skmacystatic void 968178302Skmacyt3_enable_ddp(struct toepcb *toep, int on) 969174641Skmacy{ 970176472Skmacy if (on) { 971176472Skmacy 972178302Skmacy t3_set_tcb_field(toep, W_TCB_RX_DDP_FLAGS, V_TF_DDP_OFF(1), 973174641Skmacy V_TF_DDP_OFF(0)); 974176472Skmacy } else 975178302Skmacy t3_set_tcb_field(toep, W_TCB_RX_DDP_FLAGS, 976174641Skmacy V_TF_DDP_OFF(1) | 977174641Skmacy TP_DDP_TIMER_WORKAROUND_MASK, 978174641Skmacy V_TF_DDP_OFF(1) | 979174641Skmacy TP_DDP_TIMER_WORKAROUND_VAL); 980174641Skmacy 981174641Skmacy} 982174641Skmacy 983174641Skmacyvoid 984178302Skmacyt3_set_ddp_tag(struct toepcb *toep, int buf_idx, unsigned int tag_color) 985174641Skmacy{ 986178302Skmacy t3_set_tcb_field(toep, W_TCB_RX_DDP_BUF0_TAG + buf_idx, 987174641Skmacy V_TCB_RX_DDP_BUF0_TAG(M_TCB_RX_DDP_BUF0_TAG), 988174641Skmacy tag_color); 989174641Skmacy} 990174641Skmacy 991174641Skmacyvoid 992178302Skmacyt3_set_ddp_buf(struct toepcb *toep, int buf_idx, unsigned int offset, 993174641Skmacy unsigned int len) 994174641Skmacy{ 995174641Skmacy if (buf_idx == 0) 996178302Skmacy t3_set_tcb_field(toep, W_TCB_RX_DDP_BUF0_OFFSET, 997174641Skmacy V_TCB_RX_DDP_BUF0_OFFSET(M_TCB_RX_DDP_BUF0_OFFSET) | 998174641Skmacy V_TCB_RX_DDP_BUF0_LEN(M_TCB_RX_DDP_BUF0_LEN), 999174641Skmacy V_TCB_RX_DDP_BUF0_OFFSET((uint64_t)offset) | 1000174641Skmacy V_TCB_RX_DDP_BUF0_LEN((uint64_t)len)); 1001174641Skmacy else 1002178302Skmacy t3_set_tcb_field(toep, W_TCB_RX_DDP_BUF1_OFFSET, 1003174641Skmacy V_TCB_RX_DDP_BUF1_OFFSET(M_TCB_RX_DDP_BUF1_OFFSET) | 1004174641Skmacy V_TCB_RX_DDP_BUF1_LEN(M_TCB_RX_DDP_BUF1_LEN << 32), 1005174641Skmacy V_TCB_RX_DDP_BUF1_OFFSET((uint64_t)offset) | 1006174641Skmacy V_TCB_RX_DDP_BUF1_LEN(((uint64_t)len) << 32)); 1007174641Skmacy} 1008174641Skmacy 1009174641Skmacystatic int 1010174641Skmacyt3_set_cong_control(struct socket *so, const char *name) 1011174641Skmacy{ 1012176472Skmacy#ifdef CONGESTION_CONTROL_SUPPORTED 1013174641Skmacy int cong_algo; 1014174641Skmacy 1015174641Skmacy for (cong_algo = 0; cong_algo < ARRAY_SIZE(t3_cong_ops); cong_algo++) 1016174641Skmacy if (!strcmp(name, t3_cong_ops[cong_algo].name)) 1017174641Skmacy break; 1018174641Skmacy 1019174641Skmacy if (cong_algo >= ARRAY_SIZE(t3_cong_ops)) 1020174641Skmacy return -EINVAL; 1021174641Skmacy#endif 1022174641Skmacy return 0; 1023174641Skmacy} 1024174641Skmacy 1025174641Skmacyint 1026178302Skmacyt3_get_tcb(struct toepcb *toep) 1027174641Skmacy{ 1028174641Skmacy struct cpl_get_tcb *req; 1029178302Skmacy struct tcpcb *tp = toep->tp_tp; 1030174641Skmacy struct mbuf *m = m_gethdr(M_NOWAIT, MT_DATA); 1031174641Skmacy 1032174641Skmacy if (!m) 1033174641Skmacy return (ENOMEM); 1034174641Skmacy 1035177575Skmacy inp_lock_assert(tp->t_inpcb); 1036176472Skmacy m_set_priority(m, mkprio(CPL_PRIORITY_CONTROL, toep)); 1037174641Skmacy req = mtod(m, struct cpl_get_tcb *); 1038174641Skmacy m->m_pkthdr.len = m->m_len = sizeof(*req); 1039174641Skmacy req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 1040176472Skmacy req->wr.wr_lo = 0; 1041174641Skmacy OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_GET_TCB, toep->tp_tid)); 1042174641Skmacy req->cpuno = htons(toep->tp_qset); 1043176472Skmacy req->rsvd = 0; 1044178302Skmacy if (tp->t_state == TCPS_SYN_SENT) 1045174641Skmacy mbufq_tail(&toep->out_of_order_queue, m); // defer 1046174641Skmacy else 1047178302Skmacy cxgb_ofld_send(TOEP_T3C_DEV(toep), m); 1048174641Skmacy return 0; 1049174641Skmacy} 1050174641Skmacy 1051174641Skmacystatic inline void 1052178302Skmacyso_insert_tid(struct tom_data *d, struct toepcb *toep, unsigned int tid) 1053174641Skmacy{ 1054178302Skmacy 1055174641Skmacy toepcb_hold(toep); 1056174641Skmacy 1057174641Skmacy cxgb_insert_tid(d->cdev, d->client, toep, tid); 1058174641Skmacy} 1059174641Skmacy 1060174641Skmacy/** 1061174641Skmacy * find_best_mtu - find the entry in the MTU table closest to an MTU 1062174641Skmacy * @d: TOM state 1063174641Skmacy * @mtu: the target MTU 1064174641Skmacy * 1065174641Skmacy * Returns the index of the value in the MTU table that is closest to but 1066174641Skmacy * does not exceed the target MTU. 1067174641Skmacy */ 1068174641Skmacystatic unsigned int 1069174641Skmacyfind_best_mtu(const struct t3c_data *d, unsigned short mtu) 1070174641Skmacy{ 1071174641Skmacy int i = 0; 1072174641Skmacy 1073174641Skmacy while (i < d->nmtus - 1 && d->mtus[i + 1] <= mtu) 1074174641Skmacy ++i; 1075174641Skmacy return (i); 1076174641Skmacy} 1077174641Skmacy 1078174641Skmacystatic unsigned int 1079174641Skmacyselect_mss(struct t3c_data *td, struct tcpcb *tp, unsigned int pmtu) 1080174641Skmacy{ 1081174641Skmacy unsigned int idx; 1082174641Skmacy 1083174641Skmacy#ifdef notyet 1084178302Skmacy struct rtentry *dst = so_sotoinpcb(so)->inp_route.ro_rt; 1085174641Skmacy#endif 1086174641Skmacy if (tp) { 1087174641Skmacy tp->t_maxseg = pmtu - 40; 1088174641Skmacy if (tp->t_maxseg < td->mtus[0] - 40) 1089174641Skmacy tp->t_maxseg = td->mtus[0] - 40; 1090174641Skmacy idx = find_best_mtu(td, tp->t_maxseg + 40); 1091174641Skmacy 1092174641Skmacy tp->t_maxseg = td->mtus[idx] - 40; 1093174641Skmacy } else 1094174641Skmacy idx = find_best_mtu(td, pmtu); 1095174641Skmacy 1096174641Skmacy return (idx); 1097174641Skmacy} 1098174641Skmacy 1099174641Skmacystatic inline void 1100174641Skmacyfree_atid(struct t3cdev *cdev, unsigned int tid) 1101174641Skmacy{ 1102174641Skmacy struct toepcb *toep = cxgb_free_atid(cdev, tid); 1103174641Skmacy 1104174641Skmacy if (toep) 1105174641Skmacy toepcb_release(toep); 1106174641Skmacy} 1107174641Skmacy 1108174641Skmacy/* 1109174641Skmacy * Release resources held by an offload connection (TID, L2T entry, etc.) 1110174641Skmacy */ 1111174641Skmacystatic void 1112174641Skmacyt3_release_offload_resources(struct toepcb *toep) 1113174641Skmacy{ 1114174641Skmacy struct tcpcb *tp = toep->tp_tp; 1115174641Skmacy struct toedev *tdev = toep->tp_toedev; 1116174641Skmacy struct t3cdev *cdev; 1117178302Skmacy struct socket *so; 1118174641Skmacy unsigned int tid = toep->tp_tid; 1119178302Skmacy struct sockbuf *rcv; 1120178302Skmacy 1121178302Skmacy CTR0(KTR_TOM, "t3_release_offload_resources"); 1122174641Skmacy 1123174641Skmacy if (!tdev) 1124174641Skmacy return; 1125174641Skmacy 1126174641Skmacy cdev = TOEP_T3C_DEV(toep); 1127174641Skmacy if (!cdev) 1128174641Skmacy return; 1129174641Skmacy 1130174641Skmacy toep->tp_qset = 0; 1131174641Skmacy t3_release_ddp_resources(toep); 1132174641Skmacy 1133174641Skmacy#ifdef CTRL_SKB_CACHE 1134174641Skmacy kfree_skb(CTRL_SKB_CACHE(tp)); 1135174641Skmacy CTRL_SKB_CACHE(tp) = NULL; 1136174641Skmacy#endif 1137174641Skmacy 1138174641Skmacy if (toep->tp_wr_avail != toep->tp_wr_max) { 1139174641Skmacy purge_wr_queue(toep); 1140174641Skmacy reset_wr_list(toep); 1141174641Skmacy } 1142174641Skmacy 1143174641Skmacy if (toep->tp_l2t) { 1144174641Skmacy l2t_release(L2DATA(cdev), toep->tp_l2t); 1145174641Skmacy toep->tp_l2t = NULL; 1146174641Skmacy } 1147174641Skmacy toep->tp_tp = NULL; 1148174641Skmacy if (tp) { 1149177575Skmacy inp_lock_assert(tp->t_inpcb); 1150178302Skmacy so = inp_inpcbtosocket(tp->t_inpcb); 1151178302Skmacy rcv = so_sockbuf_rcv(so); 1152178302Skmacy /* 1153178302Skmacy * cancel any offloaded reads 1154178302Skmacy * 1155178302Skmacy */ 1156178302Skmacy sockbuf_lock(rcv); 1157174641Skmacy tp->t_toe = NULL; 1158174641Skmacy tp->t_flags &= ~TF_TOE; 1159178302Skmacy if (toep->tp_ddp_state.user_ddp_pending) { 1160178302Skmacy t3_cancel_ubuf(toep, rcv); 1161178302Skmacy toep->tp_ddp_state.user_ddp_pending = 0; 1162178302Skmacy } 1163178302Skmacy so_sorwakeup_locked(so); 1164178302Skmacy 1165174641Skmacy } 1166174641Skmacy 1167174641Skmacy if (toep->tp_state == TCPS_SYN_SENT) { 1168174641Skmacy free_atid(cdev, tid); 1169174641Skmacy#ifdef notyet 1170174641Skmacy __skb_queue_purge(&tp->out_of_order_queue); 1171174641Skmacy#endif 1172174641Skmacy } else { // we have TID 1173174641Skmacy cxgb_remove_tid(cdev, toep, tid); 1174174641Skmacy toepcb_release(toep); 1175174641Skmacy } 1176174641Skmacy#if 0 1177174641Skmacy log(LOG_INFO, "closing TID %u, state %u\n", tid, tp->t_state); 1178174641Skmacy#endif 1179174641Skmacy} 1180174641Skmacy 1181174641Skmacystatic void 1182174641Skmacyinstall_offload_ops(struct socket *so) 1183174641Skmacy{ 1184178302Skmacy struct tcpcb *tp = so_sototcpcb(so); 1185174641Skmacy 1186174641Skmacy KASSERT(tp->t_toe != NULL, ("toepcb not set")); 1187174641Skmacy 1188174641Skmacy t3_install_socket_ops(so); 1189174641Skmacy tp->t_flags |= TF_TOE; 1190174641Skmacy tp->t_tu = &cxgb_toe_usrreqs; 1191174641Skmacy} 1192174641Skmacy 1193174641Skmacy/* 1194174641Skmacy * Determine the receive window scaling factor given a target max 1195174641Skmacy * receive window. 1196174641Skmacy */ 1197174641Skmacystatic __inline int 1198174641Skmacyselect_rcv_wscale(int space) 1199174641Skmacy{ 1200174641Skmacy int wscale = 0; 1201174641Skmacy 1202174641Skmacy if (space > MAX_RCV_WND) 1203174641Skmacy space = MAX_RCV_WND; 1204174641Skmacy 1205174641Skmacy if (tcp_do_rfc1323) 1206174641Skmacy for (; space > 65535 && wscale < 14; space >>= 1, ++wscale) ; 1207176472Skmacy 1208176472Skmacy return (wscale); 1209174641Skmacy} 1210174641Skmacy 1211174641Skmacy/* 1212174641Skmacy * Determine the receive window size for a socket. 1213174641Skmacy */ 1214176472Skmacystatic unsigned long 1215176472Skmacyselect_rcv_wnd(struct toedev *dev, struct socket *so) 1216174641Skmacy{ 1217174641Skmacy struct tom_data *d = TOM_DATA(dev); 1218174641Skmacy unsigned int wnd; 1219174641Skmacy unsigned int max_rcv_wnd; 1220178302Skmacy struct sockbuf *rcv; 1221174641Skmacy 1222178302Skmacy rcv = so_sockbuf_rcv(so); 1223178302Skmacy 1224174641Skmacy if (tcp_do_autorcvbuf) 1225174641Skmacy wnd = tcp_autorcvbuf_max; 1226174641Skmacy else 1227178302Skmacy wnd = rcv->sb_hiwat; 1228176472Skmacy 1229174641Skmacy 1230176472Skmacy 1231174641Skmacy /* XXX 1232174641Skmacy * For receive coalescing to work effectively we need a receive window 1233174641Skmacy * that can accomodate a coalesced segment. 1234174641Skmacy */ 1235174641Skmacy if (wnd < MIN_RCV_WND) 1236174641Skmacy wnd = MIN_RCV_WND; 1237174641Skmacy 1238174641Skmacy /* PR 5138 */ 1239176472Skmacy max_rcv_wnd = (dev->tod_ttid < TOE_ID_CHELSIO_T3C ? 1240174641Skmacy (uint32_t)d->rx_page_size * 23 : 1241174641Skmacy MAX_RCV_WND); 1242174641Skmacy 1243174641Skmacy return min(wnd, max_rcv_wnd); 1244174641Skmacy} 1245174641Skmacy 1246174641Skmacy/* 1247174641Skmacy * Assign offload parameters to some socket fields. This code is used by 1248174641Skmacy * both active and passive opens. 1249174641Skmacy */ 1250174641Skmacystatic inline void 1251174641Skmacyinit_offload_socket(struct socket *so, struct toedev *dev, unsigned int tid, 1252174641Skmacy struct l2t_entry *e, struct rtentry *dst, struct toepcb *toep) 1253174641Skmacy{ 1254178302Skmacy struct tcpcb *tp = so_sototcpcb(so); 1255174641Skmacy struct t3c_data *td = T3C_DATA(TOM_DATA(dev)->cdev); 1256178302Skmacy struct sockbuf *snd, *rcv; 1257178302Skmacy 1258178302Skmacy#ifdef notyet 1259174641Skmacy SOCK_LOCK_ASSERT(so); 1260178302Skmacy#endif 1261174641Skmacy 1262178302Skmacy snd = so_sockbuf_snd(so); 1263178302Skmacy rcv = so_sockbuf_rcv(so); 1264178302Skmacy 1265178302Skmacy log(LOG_INFO, "initializing offload socket\n"); 1266174641Skmacy /* 1267174641Skmacy * We either need to fix push frames to work with sbcompress 1268174641Skmacy * or we need to add this 1269174641Skmacy */ 1270178302Skmacy snd->sb_flags |= SB_NOCOALESCE; 1271178302Skmacy rcv->sb_flags |= SB_NOCOALESCE; 1272176472Skmacy 1273174641Skmacy tp->t_toe = toep; 1274174641Skmacy toep->tp_tp = tp; 1275174641Skmacy toep->tp_toedev = dev; 1276174641Skmacy 1277174641Skmacy toep->tp_tid = tid; 1278174641Skmacy toep->tp_l2t = e; 1279174641Skmacy toep->tp_wr_max = toep->tp_wr_avail = TOM_TUNABLE(dev, max_wrs); 1280174641Skmacy toep->tp_wr_unacked = 0; 1281174641Skmacy toep->tp_delack_mode = 0; 1282174641Skmacy 1283174641Skmacy toep->tp_mtu_idx = select_mss(td, tp, dst->rt_ifp->if_mtu); 1284174641Skmacy /* 1285174641Skmacy * XXX broken 1286174641Skmacy * 1287174641Skmacy */ 1288176472Skmacy tp->rcv_wnd = select_rcv_wnd(dev, so); 1289176472Skmacy 1290178302Skmacy toep->tp_ulp_mode = TOM_TUNABLE(dev, ddp) && !(so_options_get(so) & SO_NO_DDP) && 1291174641Skmacy tp->rcv_wnd >= MIN_DDP_RCV_WIN ? ULP_MODE_TCPDDP : 0; 1292174641Skmacy toep->tp_qset_idx = 0; 1293174641Skmacy 1294174641Skmacy reset_wr_list(toep); 1295174641Skmacy DPRINTF("initialization done\n"); 1296174641Skmacy} 1297174641Skmacy 1298174641Skmacy/* 1299174641Skmacy * The next two functions calculate the option 0 value for a socket. 1300174641Skmacy */ 1301174641Skmacystatic inline unsigned int 1302174641Skmacycalc_opt0h(struct socket *so, int mtu_idx) 1303174641Skmacy{ 1304178302Skmacy struct tcpcb *tp = so_sototcpcb(so); 1305174641Skmacy int wscale = select_rcv_wscale(tp->rcv_wnd); 1306178302Skmacy 1307174641Skmacy return V_NAGLE((tp->t_flags & TF_NODELAY) == 0) | 1308178302Skmacy V_KEEP_ALIVE((so_options_get(so) & SO_KEEPALIVE) != 0) | F_TCAM_BYPASS | 1309174641Skmacy V_WND_SCALE(wscale) | V_MSS_IDX(mtu_idx); 1310174641Skmacy} 1311174641Skmacy 1312174641Skmacystatic inline unsigned int 1313174641Skmacycalc_opt0l(struct socket *so, int ulp_mode) 1314174641Skmacy{ 1315178302Skmacy struct tcpcb *tp = so_sototcpcb(so); 1316174641Skmacy unsigned int val; 1317174641Skmacy 1318178302Skmacy val = V_TOS(INP_TOS(tp->t_inpcb)) | V_ULP_MODE(ulp_mode) | 1319174641Skmacy V_RCV_BUFSIZ(min(tp->rcv_wnd >> 10, (u32)M_RCV_BUFSIZ)); 1320174641Skmacy 1321178302Skmacy DPRINTF("opt0l tos=%08x rcv_wnd=%ld opt0l=%08x\n", INP_TOS(tp->t_inpcb), tp->rcv_wnd, val); 1322174641Skmacy return (val); 1323174641Skmacy} 1324174641Skmacy 1325174641Skmacystatic inline unsigned int 1326174641Skmacycalc_opt2(const struct socket *so, struct toedev *dev) 1327174641Skmacy{ 1328174641Skmacy int flv_valid; 1329174641Skmacy 1330174641Skmacy flv_valid = (TOM_TUNABLE(dev, cong_alg) != -1); 1331174641Skmacy 1332176472Skmacy return (V_FLAVORS_VALID(flv_valid) | 1333176472Skmacy V_CONG_CONTROL_FLAVOR(flv_valid ? TOM_TUNABLE(dev, cong_alg) : 0)); 1334174641Skmacy} 1335176472Skmacy 1336176472Skmacy#if DEBUG_WR > 1 1337176472Skmacystatic int 1338176472Skmacycount_pending_wrs(const struct toepcb *toep) 1339176472Skmacy{ 1340176472Skmacy const struct mbuf *m; 1341176472Skmacy int n = 0; 1342176472Skmacy 1343176472Skmacy wr_queue_walk(toep, m) 1344176472Skmacy n += m->m_pkthdr.csum_data; 1345176472Skmacy return (n); 1346176472Skmacy} 1347176472Skmacy#endif 1348176472Skmacy 1349174641Skmacy#if 0 1350174641Skmacy(((*(struct tom_data **)&(dev)->l4opt)->conf.cong_alg) != -1) 1351174641Skmacy#endif 1352174641Skmacy 1353174641Skmacystatic void 1354174641Skmacymk_act_open_req(struct socket *so, struct mbuf *m, 1355174641Skmacy unsigned int atid, const struct l2t_entry *e) 1356174641Skmacy{ 1357174641Skmacy struct cpl_act_open_req *req; 1358178302Skmacy struct inpcb *inp = so_sotoinpcb(so); 1359178302Skmacy struct tcpcb *tp = inp_inpcbtotcpcb(inp); 1360174641Skmacy struct toepcb *toep = tp->t_toe; 1361178302Skmacy struct toedev *tdev = toep->tp_toedev; 1362174641Skmacy 1363176472Skmacy m_set_priority((struct mbuf *)m, mkprio(CPL_PRIORITY_SETUP, toep)); 1364174641Skmacy 1365174641Skmacy req = mtod(m, struct cpl_act_open_req *); 1366174641Skmacy m->m_pkthdr.len = m->m_len = sizeof(*req); 1367176472Skmacy 1368174641Skmacy req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 1369176472Skmacy req->wr.wr_lo = 0; 1370174641Skmacy OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_ACT_OPEN_REQ, atid)); 1371178302Skmacy inp_4tuple_get(inp, &req->local_ip, &req->local_port, &req->peer_ip, &req->peer_port); 1372178302Skmacy#if 0 1373174641Skmacy req->local_port = inp->inp_lport; 1374174641Skmacy req->peer_port = inp->inp_fport; 1375174641Skmacy memcpy(&req->local_ip, &inp->inp_laddr, 4); 1376174641Skmacy memcpy(&req->peer_ip, &inp->inp_faddr, 4); 1377178302Skmacy#endif 1378174641Skmacy req->opt0h = htonl(calc_opt0h(so, toep->tp_mtu_idx) | V_L2T_IDX(e->idx) | 1379174641Skmacy V_TX_CHANNEL(e->smt_idx)); 1380174641Skmacy req->opt0l = htonl(calc_opt0l(so, toep->tp_ulp_mode)); 1381174641Skmacy req->params = 0; 1382174641Skmacy req->opt2 = htonl(calc_opt2(so, tdev)); 1383174641Skmacy} 1384174641Skmacy 1385174641Skmacy 1386174641Skmacy/* 1387174641Skmacy * Convert an ACT_OPEN_RPL status to an errno. 1388174641Skmacy */ 1389174641Skmacystatic int 1390174641Skmacyact_open_rpl_status_to_errno(int status) 1391174641Skmacy{ 1392174641Skmacy switch (status) { 1393174641Skmacy case CPL_ERR_CONN_RESET: 1394174641Skmacy return (ECONNREFUSED); 1395174641Skmacy case CPL_ERR_ARP_MISS: 1396174641Skmacy return (EHOSTUNREACH); 1397174641Skmacy case CPL_ERR_CONN_TIMEDOUT: 1398174641Skmacy return (ETIMEDOUT); 1399174641Skmacy case CPL_ERR_TCAM_FULL: 1400174641Skmacy return (ENOMEM); 1401174641Skmacy case CPL_ERR_CONN_EXIST: 1402174641Skmacy log(LOG_ERR, "ACTIVE_OPEN_RPL: 4-tuple in use\n"); 1403174641Skmacy return (EADDRINUSE); 1404174641Skmacy default: 1405174641Skmacy return (EIO); 1406174641Skmacy } 1407174641Skmacy} 1408174641Skmacy 1409174641Skmacystatic void 1410174641Skmacyfail_act_open(struct toepcb *toep, int errno) 1411174641Skmacy{ 1412174641Skmacy struct tcpcb *tp = toep->tp_tp; 1413174641Skmacy 1414174641Skmacy t3_release_offload_resources(toep); 1415174641Skmacy if (tp) { 1416178302Skmacy inp_wunlock(tp->t_inpcb); 1417178302Skmacy tcp_offload_drop(tp, errno); 1418174641Skmacy } 1419174641Skmacy 1420174641Skmacy#ifdef notyet 1421174641Skmacy TCP_INC_STATS_BH(TCP_MIB_ATTEMPTFAILS); 1422174641Skmacy#endif 1423174641Skmacy} 1424174641Skmacy 1425174641Skmacy/* 1426174641Skmacy * Handle active open failures. 1427174641Skmacy */ 1428174641Skmacystatic void 1429174641Skmacyactive_open_failed(struct toepcb *toep, struct mbuf *m) 1430174641Skmacy{ 1431174641Skmacy struct cpl_act_open_rpl *rpl = cplhdr(m); 1432174641Skmacy struct inpcb *inp; 1433174641Skmacy 1434174641Skmacy if (toep->tp_tp == NULL) 1435174641Skmacy goto done; 1436174641Skmacy 1437174641Skmacy inp = toep->tp_tp->t_inpcb; 1438174641Skmacy 1439174641Skmacy/* 1440174641Skmacy * Don't handle connection retry for now 1441174641Skmacy */ 1442174641Skmacy#ifdef notyet 1443174641Skmacy struct inet_connection_sock *icsk = inet_csk(sk); 1444174641Skmacy 1445174641Skmacy if (rpl->status == CPL_ERR_CONN_EXIST && 1446174641Skmacy icsk->icsk_retransmit_timer.function != act_open_retry_timer) { 1447174641Skmacy icsk->icsk_retransmit_timer.function = act_open_retry_timer; 1448174641Skmacy sk_reset_timer(so, &icsk->icsk_retransmit_timer, 1449174641Skmacy jiffies + HZ / 2); 1450174641Skmacy } else 1451178302Skmacy#endif 1452178302Skmacy { 1453178767Skmacy inp_wlock(inp); 1454178767Skmacy /* 1455178767Skmacy * drops the inpcb lock 1456178767Skmacy */ 1457174641Skmacy fail_act_open(toep, act_open_rpl_status_to_errno(rpl->status)); 1458178302Skmacy } 1459178302Skmacy 1460178302Skmacy done: 1461174641Skmacy m_free(m); 1462174641Skmacy} 1463174641Skmacy 1464174641Skmacy/* 1465174641Skmacy * Return whether a failed active open has allocated a TID 1466174641Skmacy */ 1467174641Skmacystatic inline int 1468174641Skmacyact_open_has_tid(int status) 1469174641Skmacy{ 1470174641Skmacy return status != CPL_ERR_TCAM_FULL && status != CPL_ERR_CONN_EXIST && 1471174641Skmacy status != CPL_ERR_ARP_MISS; 1472174641Skmacy} 1473174641Skmacy 1474174641Skmacy/* 1475174641Skmacy * Process an ACT_OPEN_RPL CPL message. 1476174641Skmacy */ 1477174641Skmacystatic int 1478174641Skmacydo_act_open_rpl(struct t3cdev *cdev, struct mbuf *m, void *ctx) 1479174641Skmacy{ 1480174641Skmacy struct toepcb *toep = (struct toepcb *)ctx; 1481174641Skmacy struct cpl_act_open_rpl *rpl = cplhdr(m); 1482174641Skmacy 1483174641Skmacy if (cdev->type != T3A && act_open_has_tid(rpl->status)) 1484174641Skmacy cxgb_queue_tid_release(cdev, GET_TID(rpl)); 1485174641Skmacy 1486174641Skmacy active_open_failed(toep, m); 1487174641Skmacy return (0); 1488174641Skmacy} 1489174641Skmacy 1490174641Skmacy/* 1491174641Skmacy * Handle an ARP failure for an active open. XXX purge ofo queue 1492174641Skmacy * 1493174641Skmacy * XXX badly broken for crossed SYNs as the ATID is no longer valid. 1494174641Skmacy * XXX crossed SYN errors should be generated by PASS_ACCEPT_RPL which should 1495174641Skmacy * check SOCK_DEAD or sk->sk_sock. Or maybe generate the error here but don't 1496174641Skmacy * free the atid. Hmm. 1497174641Skmacy */ 1498174641Skmacy#ifdef notyet 1499174641Skmacystatic void 1500174641Skmacyact_open_req_arp_failure(struct t3cdev *dev, struct mbuf *m) 1501174641Skmacy{ 1502174641Skmacy struct toepcb *toep = m_get_toep(m); 1503174641Skmacy struct tcpcb *tp = toep->tp_tp; 1504174641Skmacy struct inpcb *inp = tp->t_inpcb; 1505178302Skmacy struct socket *so; 1506174641Skmacy 1507177530Skmacy inp_wlock(inp); 1508174641Skmacy if (tp->t_state == TCPS_SYN_SENT || tp->t_state == TCPS_SYN_RECEIVED) { 1509178767Skmacy /* 1510178767Skmacy * drops the inpcb lock 1511178767Skmacy */ 1512174641Skmacy fail_act_open(so, EHOSTUNREACH); 1513174641Skmacy printf("freeing %p\n", m); 1514174641Skmacy 1515174641Skmacy m_free(m); 1516178302Skmacy } else 1517178302Skmacy inp_wunlock(inp); 1518174641Skmacy} 1519174641Skmacy#endif 1520174641Skmacy/* 1521174641Skmacy * Send an active open request. 1522174641Skmacy */ 1523174641Skmacyint 1524174641Skmacyt3_connect(struct toedev *tdev, struct socket *so, 1525174641Skmacy struct rtentry *rt, struct sockaddr *nam) 1526174641Skmacy{ 1527174641Skmacy struct mbuf *m; 1528174641Skmacy struct l2t_entry *e; 1529174641Skmacy struct tom_data *d = TOM_DATA(tdev); 1530178302Skmacy struct inpcb *inp = so_sotoinpcb(so); 1531174641Skmacy struct tcpcb *tp = intotcpcb(inp); 1532174641Skmacy struct toepcb *toep; /* allocated by init_offload_socket */ 1533174641Skmacy 1534174641Skmacy int atid; 1535174641Skmacy 1536174641Skmacy toep = toepcb_alloc(); 1537174641Skmacy if (toep == NULL) 1538174641Skmacy goto out_err; 1539174641Skmacy 1540174641Skmacy if ((atid = cxgb_alloc_atid(d->cdev, d->client, toep)) < 0) 1541174641Skmacy goto out_err; 1542174641Skmacy 1543174641Skmacy e = t3_l2t_get(d->cdev, rt, rt->rt_ifp, nam); 1544174641Skmacy if (!e) 1545174641Skmacy goto free_tid; 1546174641Skmacy 1547177575Skmacy inp_lock_assert(inp); 1548174641Skmacy m = m_gethdr(MT_DATA, M_WAITOK); 1549174641Skmacy 1550174641Skmacy#if 0 1551174641Skmacy m->m_toe.mt_toepcb = tp->t_toe; 1552174641Skmacy set_arp_failure_handler((struct mbuf *)m, act_open_req_arp_failure); 1553174641Skmacy#endif 1554178302Skmacy so_lock(so); 1555174641Skmacy 1556174641Skmacy init_offload_socket(so, tdev, atid, e, rt, toep); 1557174641Skmacy 1558174641Skmacy install_offload_ops(so); 1559174641Skmacy 1560174641Skmacy mk_act_open_req(so, m, atid, e); 1561178302Skmacy so_unlock(so); 1562174641Skmacy 1563174641Skmacy soisconnecting(so); 1564174641Skmacy toep = tp->t_toe; 1565174641Skmacy m_set_toep(m, tp->t_toe); 1566174641Skmacy 1567174641Skmacy toep->tp_state = TCPS_SYN_SENT; 1568174641Skmacy l2t_send(d->cdev, (struct mbuf *)m, e); 1569174641Skmacy 1570174641Skmacy if (toep->tp_ulp_mode) 1571178302Skmacy t3_enable_ddp(toep, 0); 1572174641Skmacy return (0); 1573174641Skmacy 1574174641Skmacyfree_tid: 1575174641Skmacy printf("failing connect - free atid\n"); 1576174641Skmacy 1577174641Skmacy free_atid(d->cdev, atid); 1578174641Skmacyout_err: 1579174641Skmacy printf("return ENOMEM\n"); 1580174641Skmacy return (ENOMEM); 1581174641Skmacy} 1582174641Skmacy 1583174641Skmacy/* 1584174641Skmacy * Send an ABORT_REQ message. Cannot fail. This routine makes sure we do 1585174641Skmacy * not send multiple ABORT_REQs for the same connection and also that we do 1586174641Skmacy * not try to send a message after the connection has closed. Returns 1 if 1587174641Skmacy * an ABORT_REQ wasn't generated after all, 0 otherwise. 1588174641Skmacy */ 1589174641Skmacystatic void 1590174641Skmacyt3_send_reset(struct toepcb *toep) 1591174641Skmacy{ 1592174641Skmacy 1593174641Skmacy struct cpl_abort_req *req; 1594174641Skmacy unsigned int tid = toep->tp_tid; 1595174641Skmacy int mode = CPL_ABORT_SEND_RST; 1596174641Skmacy struct tcpcb *tp = toep->tp_tp; 1597174641Skmacy struct toedev *tdev = toep->tp_toedev; 1598174641Skmacy struct socket *so = NULL; 1599174641Skmacy struct mbuf *m; 1600178302Skmacy struct sockbuf *snd; 1601174641Skmacy 1602174641Skmacy if (tp) { 1603177575Skmacy inp_lock_assert(tp->t_inpcb); 1604178302Skmacy so = inp_inpcbtosocket(tp->t_inpcb); 1605174641Skmacy } 1606174641Skmacy 1607174641Skmacy if (__predict_false((toep->tp_flags & TP_ABORT_SHUTDOWN) || 1608174641Skmacy tdev == NULL)) 1609174641Skmacy return; 1610174641Skmacy toep->tp_flags |= (TP_ABORT_RPL_PENDING|TP_ABORT_SHUTDOWN); 1611178302Skmacy 1612178302Skmacy snd = so_sockbuf_snd(so); 1613174641Skmacy /* Purge the send queue so we don't send anything after an abort. */ 1614174641Skmacy if (so) 1615178302Skmacy sbflush(snd); 1616174641Skmacy if ((toep->tp_flags & TP_CLOSE_CON_REQUESTED) && is_t3a(tdev)) 1617174641Skmacy mode |= CPL_ABORT_POST_CLOSE_REQ; 1618174641Skmacy 1619174641Skmacy m = m_gethdr_nofail(sizeof(*req)); 1620176472Skmacy m_set_priority(m, mkprio(CPL_PRIORITY_DATA, toep)); 1621174641Skmacy set_arp_failure_handler(m, abort_arp_failure); 1622174641Skmacy 1623174641Skmacy req = mtod(m, struct cpl_abort_req *); 1624174641Skmacy req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_HOST_ABORT_CON_REQ)); 1625174641Skmacy req->wr.wr_lo = htonl(V_WR_TID(tid)); 1626174641Skmacy OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_ABORT_REQ, tid)); 1627174641Skmacy req->rsvd0 = tp ? htonl(tp->snd_nxt) : 0; 1628174641Skmacy req->rsvd1 = !(toep->tp_flags & TP_DATASENT); 1629174641Skmacy req->cmd = mode; 1630174641Skmacy if (tp && (tp->t_state == TCPS_SYN_SENT)) 1631174641Skmacy mbufq_tail(&toep->out_of_order_queue, m); // defer 1632174641Skmacy else 1633174641Skmacy l2t_send(TOEP_T3C_DEV(toep), m, toep->tp_l2t); 1634174641Skmacy} 1635174641Skmacy 1636174641Skmacystatic int 1637174641Skmacyt3_ip_ctloutput(struct socket *so, struct sockopt *sopt) 1638174641Skmacy{ 1639174641Skmacy struct inpcb *inp; 1640174641Skmacy int error, optval; 1641174641Skmacy 1642174641Skmacy if (sopt->sopt_name == IP_OPTIONS) 1643174641Skmacy return (ENOPROTOOPT); 1644174641Skmacy 1645174641Skmacy if (sopt->sopt_name != IP_TOS) 1646174641Skmacy return (EOPNOTSUPP); 1647174641Skmacy 1648174641Skmacy error = sooptcopyin(sopt, &optval, sizeof optval, sizeof optval); 1649174641Skmacy 1650174641Skmacy if (error) 1651174641Skmacy return (error); 1652174641Skmacy 1653174641Skmacy if (optval > IPTOS_PREC_CRITIC_ECP && !suser(curthread)) 1654174641Skmacy return (EPERM); 1655174641Skmacy 1656178302Skmacy inp = so_sotoinpcb(so); 1657178767Skmacy inp_wlock(inp); 1658178302Skmacy inp_ip_tos_set(inp, optval); 1659178302Skmacy#if 0 1660174641Skmacy inp->inp_ip_tos = optval; 1661178302Skmacy#endif 1662178302Skmacy t3_set_tos(inp_inpcbtotcpcb(inp)->t_toe); 1663178767Skmacy inp_wunlock(inp); 1664178767Skmacy 1665174641Skmacy return (0); 1666174641Skmacy} 1667174641Skmacy 1668174641Skmacystatic int 1669174641Skmacyt3_tcp_ctloutput(struct socket *so, struct sockopt *sopt) 1670174641Skmacy{ 1671174641Skmacy int err = 0; 1672174641Skmacy size_t copied; 1673174641Skmacy 1674174641Skmacy if (sopt->sopt_name != TCP_CONGESTION && 1675174641Skmacy sopt->sopt_name != TCP_NODELAY) 1676174641Skmacy return (EOPNOTSUPP); 1677180583Skmacy 1678174641Skmacy if (sopt->sopt_name == TCP_CONGESTION) { 1679174641Skmacy char name[TCP_CA_NAME_MAX]; 1680174641Skmacy int optlen = sopt->sopt_valsize; 1681174641Skmacy struct tcpcb *tp; 1682174641Skmacy 1683180583Skmacy if (sopt->sopt_dir == SOPT_GET) { 1684180583Skmacy KASSERT(0, ("unimplemented")); 1685180583Skmacy return (EOPNOTSUPP); 1686180583Skmacy } 1687180583Skmacy 1688174641Skmacy if (optlen < 1) 1689174641Skmacy return (EINVAL); 1690174641Skmacy 1691174641Skmacy err = copyinstr(sopt->sopt_val, name, 1692174641Skmacy min(TCP_CA_NAME_MAX - 1, optlen), &copied); 1693174641Skmacy if (err) 1694174641Skmacy return (err); 1695174641Skmacy if (copied < 1) 1696174641Skmacy return (EINVAL); 1697174641Skmacy 1698178302Skmacy tp = so_sototcpcb(so); 1699174641Skmacy /* 1700174641Skmacy * XXX I need to revisit this 1701174641Skmacy */ 1702174641Skmacy if ((err = t3_set_cong_control(so, name)) == 0) { 1703176472Skmacy#ifdef CONGESTION_CONTROL_SUPPORTED 1704174641Skmacy tp->t_cong_control = strdup(name, M_CXGB); 1705174641Skmacy#endif 1706174641Skmacy } else 1707174641Skmacy return (err); 1708174641Skmacy } else { 1709174641Skmacy int optval, oldval; 1710174641Skmacy struct inpcb *inp; 1711174641Skmacy struct tcpcb *tp; 1712178302Skmacy 1713180583Skmacy if (sopt->sopt_dir == SOPT_GET) 1714180583Skmacy return (EOPNOTSUPP); 1715180583Skmacy 1716174641Skmacy err = sooptcopyin(sopt, &optval, sizeof optval, 1717174641Skmacy sizeof optval); 1718174641Skmacy 1719174641Skmacy if (err) 1720174641Skmacy return (err); 1721174641Skmacy 1722178302Skmacy inp = so_sotoinpcb(so); 1723178302Skmacy tp = inp_inpcbtotcpcb(inp); 1724174641Skmacy 1725177530Skmacy inp_wlock(inp); 1726174641Skmacy 1727174641Skmacy oldval = tp->t_flags; 1728174641Skmacy if (optval) 1729174641Skmacy tp->t_flags |= TF_NODELAY; 1730174641Skmacy else 1731174641Skmacy tp->t_flags &= ~TF_NODELAY; 1732177530Skmacy inp_wunlock(inp); 1733178302Skmacy 1734178302Skmacy 1735178767Skmacy if (oldval != tp->t_flags && (tp->t_toe != NULL)) 1736178302Skmacy t3_set_nagle(tp->t_toe); 1737174641Skmacy 1738174641Skmacy } 1739174641Skmacy 1740174641Skmacy return (0); 1741174641Skmacy} 1742174641Skmacy 1743178302Skmacyint 1744174641Skmacyt3_ctloutput(struct socket *so, struct sockopt *sopt) 1745174641Skmacy{ 1746174641Skmacy int err; 1747174641Skmacy 1748174641Skmacy if (sopt->sopt_level != IPPROTO_TCP) 1749174641Skmacy err = t3_ip_ctloutput(so, sopt); 1750174641Skmacy else 1751174641Skmacy err = t3_tcp_ctloutput(so, sopt); 1752174641Skmacy 1753174641Skmacy if (err != EOPNOTSUPP) 1754174641Skmacy return (err); 1755174641Skmacy 1756176472Skmacy return (tcp_ctloutput(so, sopt)); 1757174641Skmacy} 1758174641Skmacy 1759174641Skmacy/* 1760176472Skmacy * Returns true if we need to explicitly request RST when we receive new data 1761176472Skmacy * on an RX-closed connection. 1762176472Skmacy */ 1763176472Skmacystatic inline int 1764176472Skmacyneed_rst_on_excess_rx(const struct toepcb *toep) 1765176472Skmacy{ 1766176472Skmacy return (1); 1767176472Skmacy} 1768176472Skmacy 1769176472Skmacy/* 1770176472Skmacy * Handles Rx data that arrives in a state where the socket isn't accepting 1771176472Skmacy * new data. 1772176472Skmacy */ 1773176472Skmacystatic void 1774176472Skmacyhandle_excess_rx(struct toepcb *toep, struct mbuf *m) 1775176472Skmacy{ 1776176472Skmacy 1777178302Skmacy if (need_rst_on_excess_rx(toep) && 1778178302Skmacy !(toep->tp_flags & TP_ABORT_SHUTDOWN)) 1779176472Skmacy t3_send_reset(toep); 1780176472Skmacy m_freem(m); 1781176472Skmacy} 1782176472Skmacy 1783176472Skmacy/* 1784176472Skmacy * Process a get_tcb_rpl as a DDP completion (similar to RX_DDP_COMPLETE) 1785176472Skmacy * by getting the DDP offset from the TCB. 1786176472Skmacy */ 1787176472Skmacystatic void 1788176472Skmacytcb_rpl_as_ddp_complete(struct toepcb *toep, struct mbuf *m) 1789176472Skmacy{ 1790176472Skmacy struct ddp_state *q = &toep->tp_ddp_state; 1791176472Skmacy struct ddp_buf_state *bsp; 1792176472Skmacy struct cpl_get_tcb_rpl *hdr; 1793176472Skmacy unsigned int ddp_offset; 1794176472Skmacy struct socket *so; 1795176472Skmacy struct tcpcb *tp; 1796178302Skmacy struct sockbuf *rcv; 1797178302Skmacy int state; 1798176472Skmacy 1799176472Skmacy uint64_t t; 1800176472Skmacy __be64 *tcb; 1801176472Skmacy 1802176472Skmacy tp = toep->tp_tp; 1803178302Skmacy so = inp_inpcbtosocket(tp->t_inpcb); 1804176472Skmacy 1805177575Skmacy inp_lock_assert(tp->t_inpcb); 1806178302Skmacy rcv = so_sockbuf_rcv(so); 1807178302Skmacy sockbuf_lock(rcv); 1808176472Skmacy 1809178302Skmacy /* Note that we only accout for CPL_GET_TCB issued by the DDP code. 1810178302Skmacy * We really need a cookie in order to dispatch the RPLs. 1811176472Skmacy */ 1812176472Skmacy q->get_tcb_count--; 1813176472Skmacy 1814176472Skmacy /* It is a possible that a previous CPL already invalidated UBUF DDP 1815176472Skmacy * and moved the cur_buf idx and hence no further processing of this 1816176472Skmacy * skb is required. However, the app might be sleeping on 1817176472Skmacy * !q->get_tcb_count and we need to wake it up. 1818176472Skmacy */ 1819176472Skmacy if (q->cancel_ubuf && !t3_ddp_ubuf_pending(toep)) { 1820178302Skmacy int state = so_state_get(so); 1821178302Skmacy 1822176472Skmacy m_freem(m); 1823178302Skmacy if (__predict_true((state & SS_NOFDREF) == 0)) 1824178302Skmacy so_sorwakeup_locked(so); 1825176472Skmacy else 1826178302Skmacy sockbuf_unlock(rcv); 1827178302Skmacy 1828176472Skmacy return; 1829176472Skmacy } 1830176472Skmacy 1831176472Skmacy bsp = &q->buf_state[q->cur_buf]; 1832176472Skmacy hdr = cplhdr(m); 1833176472Skmacy tcb = (__be64 *)(hdr + 1); 1834176472Skmacy if (q->cur_buf == 0) { 1835176472Skmacy t = be64toh(tcb[(31 - W_TCB_RX_DDP_BUF0_OFFSET) / 2]); 1836176472Skmacy ddp_offset = t >> (32 + S_TCB_RX_DDP_BUF0_OFFSET); 1837176472Skmacy } else { 1838176472Skmacy t = be64toh(tcb[(31 - W_TCB_RX_DDP_BUF1_OFFSET) / 2]); 1839176472Skmacy ddp_offset = t >> S_TCB_RX_DDP_BUF1_OFFSET; 1840176472Skmacy } 1841176472Skmacy ddp_offset &= M_TCB_RX_DDP_BUF0_OFFSET; 1842176472Skmacy m->m_cur_offset = bsp->cur_offset; 1843176472Skmacy bsp->cur_offset = ddp_offset; 1844176472Skmacy m->m_len = m->m_pkthdr.len = ddp_offset - m->m_cur_offset; 1845176472Skmacy 1846176472Skmacy CTR5(KTR_TOM, 1847176472Skmacy "tcb_rpl_as_ddp_complete: idx=%d seq=0x%x hwbuf=%u ddp_offset=%u cur_offset=%u", 1848176472Skmacy q->cur_buf, tp->rcv_nxt, q->cur_buf, ddp_offset, m->m_cur_offset); 1849178302Skmacy KASSERT(ddp_offset >= m->m_cur_offset, 1850178302Skmacy ("ddp_offset=%u less than cur_offset=%u", 1851176472Skmacy ddp_offset, m->m_cur_offset)); 1852176472Skmacy 1853176472Skmacy#if 0 1854176472Skmacy{ 1855176472Skmacy unsigned int ddp_flags, rcv_nxt, rx_hdr_offset, buf_idx; 1856176472Skmacy 1857176472Skmacy t = be64toh(tcb[(31 - W_TCB_RX_DDP_FLAGS) / 2]); 1858176472Skmacy ddp_flags = (t >> S_TCB_RX_DDP_FLAGS) & M_TCB_RX_DDP_FLAGS; 1859176472Skmacy 1860176472Skmacy t = be64toh(tcb[(31 - W_TCB_RCV_NXT) / 2]); 1861176472Skmacy rcv_nxt = t >> S_TCB_RCV_NXT; 1862176472Skmacy rcv_nxt &= M_TCB_RCV_NXT; 1863176472Skmacy 1864176472Skmacy t = be64toh(tcb[(31 - W_TCB_RX_HDR_OFFSET) / 2]); 1865176472Skmacy rx_hdr_offset = t >> (32 + S_TCB_RX_HDR_OFFSET); 1866176472Skmacy rx_hdr_offset &= M_TCB_RX_HDR_OFFSET; 1867176472Skmacy 1868176472Skmacy T3_TRACE2(TIDTB(sk), 1869176472Skmacy "tcb_rpl_as_ddp_complete: DDP FLAGS 0x%x dma up to 0x%x", 1870176472Skmacy ddp_flags, rcv_nxt - rx_hdr_offset); 1871176472Skmacy T3_TRACE4(TB(q), 1872176472Skmacy "tcb_rpl_as_ddp_complete: rcvnxt 0x%x hwbuf %u cur_offset %u cancel %u", 1873176472Skmacy tp->rcv_nxt, q->cur_buf, bsp->cur_offset, q->cancel_ubuf); 1874176472Skmacy T3_TRACE3(TB(q), 1875176472Skmacy "tcb_rpl_as_ddp_complete: TCB rcvnxt 0x%x hwbuf 0x%x ddp_offset %u", 1876176472Skmacy rcv_nxt - rx_hdr_offset, ddp_flags, ddp_offset); 1877176472Skmacy T3_TRACE2(TB(q), 1878176472Skmacy "tcb_rpl_as_ddp_complete: flags0 0x%x flags1 0x%x", 1879176472Skmacy q->buf_state[0].flags, q->buf_state[1].flags); 1880176472Skmacy 1881176472Skmacy} 1882176472Skmacy#endif 1883176472Skmacy if (__predict_false(so_no_receive(so) && m->m_pkthdr.len)) { 1884176472Skmacy handle_excess_rx(toep, m); 1885176472Skmacy return; 1886176472Skmacy } 1887176472Skmacy 1888176472Skmacy#ifdef T3_TRACE 1889176472Skmacy if ((int)m->m_pkthdr.len < 0) { 1890176472Skmacy t3_ddp_error(so, "tcb_rpl_as_ddp_complete: neg len"); 1891176472Skmacy } 1892176472Skmacy#endif 1893176472Skmacy if (bsp->flags & DDP_BF_NOCOPY) { 1894176472Skmacy#ifdef T3_TRACE 1895176472Skmacy T3_TRACE0(TB(q), 1896176472Skmacy "tcb_rpl_as_ddp_complete: CANCEL UBUF"); 1897176472Skmacy 1898176472Skmacy if (!q->cancel_ubuf && !(sk->sk_shutdown & RCV_SHUTDOWN)) { 1899176472Skmacy printk("!cancel_ubuf"); 1900176472Skmacy t3_ddp_error(sk, "tcb_rpl_as_ddp_complete: !cancel_ubuf"); 1901176472Skmacy } 1902176472Skmacy#endif 1903176472Skmacy m->m_ddp_flags = DDP_BF_PSH | DDP_BF_NOCOPY | 1; 1904176472Skmacy bsp->flags &= ~(DDP_BF_NOCOPY|DDP_BF_NODATA); 1905176472Skmacy q->cur_buf ^= 1; 1906176472Skmacy } else if (bsp->flags & DDP_BF_NOFLIP) { 1907176472Skmacy 1908176472Skmacy m->m_ddp_flags = 1; /* always a kernel buffer */ 1909176472Skmacy 1910176472Skmacy /* now HW buffer carries a user buffer */ 1911176472Skmacy bsp->flags &= ~DDP_BF_NOFLIP; 1912176472Skmacy bsp->flags |= DDP_BF_NOCOPY; 1913176472Skmacy 1914176472Skmacy /* It is possible that the CPL_GET_TCB_RPL doesn't indicate 1915176472Skmacy * any new data in which case we're done. If in addition the 1916176472Skmacy * offset is 0, then there wasn't a completion for the kbuf 1917176472Skmacy * and we need to decrement the posted count. 1918176472Skmacy */ 1919176472Skmacy if (m->m_pkthdr.len == 0) { 1920176472Skmacy if (ddp_offset == 0) { 1921176472Skmacy q->kbuf_posted--; 1922176472Skmacy bsp->flags |= DDP_BF_NODATA; 1923176472Skmacy } 1924178302Skmacy sockbuf_unlock(rcv); 1925176472Skmacy m_free(m); 1926176472Skmacy return; 1927176472Skmacy } 1928176472Skmacy } else { 1929178302Skmacy sockbuf_unlock(rcv); 1930178302Skmacy 1931176472Skmacy /* This reply is for a CPL_GET_TCB_RPL to cancel the UBUF DDP, 1932176472Skmacy * but it got here way late and nobody cares anymore. 1933176472Skmacy */ 1934176472Skmacy m_free(m); 1935176472Skmacy return; 1936176472Skmacy } 1937176472Skmacy 1938176472Skmacy m->m_ddp_gl = (unsigned char *)bsp->gl; 1939176472Skmacy m->m_flags |= M_DDP; 1940176472Skmacy m->m_seq = tp->rcv_nxt; 1941176472Skmacy tp->rcv_nxt += m->m_pkthdr.len; 1942176472Skmacy tp->t_rcvtime = ticks; 1943176472Skmacy CTR3(KTR_TOM, "tcb_rpl_as_ddp_complete: seq 0x%x hwbuf %u m->m_pktlen %u", 1944176472Skmacy m->m_seq, q->cur_buf, m->m_pkthdr.len); 1945178302Skmacy if (m->m_pkthdr.len == 0) { 1946176472Skmacy q->user_ddp_pending = 0; 1947178302Skmacy m_free(m); 1948178302Skmacy } else 1949178302Skmacy SBAPPEND(rcv, m); 1950178302Skmacy 1951178302Skmacy state = so_state_get(so); 1952178302Skmacy if (__predict_true((state & SS_NOFDREF) == 0)) 1953178302Skmacy so_sorwakeup_locked(so); 1954176472Skmacy else 1955178302Skmacy sockbuf_unlock(rcv); 1956176472Skmacy} 1957176472Skmacy 1958176472Skmacy/* 1959176472Skmacy * Process a CPL_GET_TCB_RPL. These can also be generated by the DDP code, 1960176472Skmacy * in that case they are similar to DDP completions. 1961176472Skmacy */ 1962176472Skmacystatic int 1963176472Skmacydo_get_tcb_rpl(struct t3cdev *cdev, struct mbuf *m, void *ctx) 1964176472Skmacy{ 1965176472Skmacy struct toepcb *toep = (struct toepcb *)ctx; 1966176472Skmacy 1967176472Skmacy /* OK if socket doesn't exist */ 1968176472Skmacy if (toep == NULL) { 1969176472Skmacy printf("null toep in do_get_tcb_rpl\n"); 1970176472Skmacy return (CPL_RET_BUF_DONE); 1971176472Skmacy } 1972176472Skmacy 1973177530Skmacy inp_wlock(toep->tp_tp->t_inpcb); 1974176472Skmacy tcb_rpl_as_ddp_complete(toep, m); 1975177530Skmacy inp_wunlock(toep->tp_tp->t_inpcb); 1976176472Skmacy 1977176472Skmacy return (0); 1978176472Skmacy} 1979176472Skmacy 1980176472Skmacystatic void 1981176472Skmacyhandle_ddp_data(struct toepcb *toep, struct mbuf *m) 1982176472Skmacy{ 1983176472Skmacy struct tcpcb *tp = toep->tp_tp; 1984178302Skmacy struct socket *so; 1985176472Skmacy struct ddp_state *q; 1986176472Skmacy struct ddp_buf_state *bsp; 1987176472Skmacy struct cpl_rx_data *hdr = cplhdr(m); 1988176472Skmacy unsigned int rcv_nxt = ntohl(hdr->seq); 1989178302Skmacy struct sockbuf *rcv; 1990178302Skmacy 1991176472Skmacy if (tp->rcv_nxt == rcv_nxt) 1992176472Skmacy return; 1993176472Skmacy 1994177575Skmacy inp_lock_assert(tp->t_inpcb); 1995178302Skmacy so = inp_inpcbtosocket(tp->t_inpcb); 1996178302Skmacy rcv = so_sockbuf_rcv(so); 1997178302Skmacy sockbuf_lock(rcv); 1998178302Skmacy 1999176472Skmacy q = &toep->tp_ddp_state; 2000176472Skmacy bsp = &q->buf_state[q->cur_buf]; 2001176472Skmacy KASSERT(SEQ_GT(rcv_nxt, tp->rcv_nxt), ("tp->rcv_nxt=0x%08x decreased rcv_nxt=0x08%x", 2002176472Skmacy rcv_nxt, tp->rcv_nxt)); 2003176472Skmacy m->m_len = m->m_pkthdr.len = rcv_nxt - tp->rcv_nxt; 2004176472Skmacy KASSERT(m->m_len > 0, ("%s m_len=%d", __FUNCTION__, m->m_len)); 2005176472Skmacy CTR3(KTR_TOM, "rcv_nxt=0x%x tp->rcv_nxt=0x%x len=%d", 2006176472Skmacy rcv_nxt, tp->rcv_nxt, m->m_pkthdr.len); 2007176472Skmacy 2008176472Skmacy#ifdef T3_TRACE 2009176472Skmacy if ((int)m->m_pkthdr.len < 0) { 2010176472Skmacy t3_ddp_error(so, "handle_ddp_data: neg len"); 2011176472Skmacy } 2012176472Skmacy#endif 2013176472Skmacy m->m_ddp_gl = (unsigned char *)bsp->gl; 2014176472Skmacy m->m_flags |= M_DDP; 2015176472Skmacy m->m_cur_offset = bsp->cur_offset; 2016176472Skmacy m->m_ddp_flags = DDP_BF_PSH | (bsp->flags & DDP_BF_NOCOPY) | 1; 2017176472Skmacy if (bsp->flags & DDP_BF_NOCOPY) 2018176472Skmacy bsp->flags &= ~DDP_BF_NOCOPY; 2019176472Skmacy 2020176472Skmacy m->m_seq = tp->rcv_nxt; 2021176472Skmacy tp->rcv_nxt = rcv_nxt; 2022176472Skmacy bsp->cur_offset += m->m_pkthdr.len; 2023176472Skmacy if (!(bsp->flags & DDP_BF_NOFLIP)) 2024176472Skmacy q->cur_buf ^= 1; 2025176472Skmacy /* 2026176472Skmacy * For now, don't re-enable DDP after a connection fell out of DDP 2027176472Skmacy * mode. 2028176472Skmacy */ 2029176472Skmacy q->ubuf_ddp_ready = 0; 2030178302Skmacy sockbuf_unlock(rcv); 2031176472Skmacy} 2032176472Skmacy 2033176472Skmacy/* 2034174641Skmacy * Process new data received for a connection. 2035174641Skmacy */ 2036174641Skmacystatic void 2037174641Skmacynew_rx_data(struct toepcb *toep, struct mbuf *m) 2038174641Skmacy{ 2039174641Skmacy struct cpl_rx_data *hdr = cplhdr(m); 2040174641Skmacy struct tcpcb *tp = toep->tp_tp; 2041178302Skmacy struct socket *so; 2042178302Skmacy struct sockbuf *rcv; 2043178302Skmacy int state; 2044174641Skmacy int len = be16toh(hdr->len); 2045174641Skmacy 2046177530Skmacy inp_wlock(tp->t_inpcb); 2047178302Skmacy 2048178302Skmacy so = inp_inpcbtosocket(tp->t_inpcb); 2049174641Skmacy 2050176472Skmacy if (__predict_false(so_no_receive(so))) { 2051176472Skmacy handle_excess_rx(toep, m); 2052177530Skmacy inp_wunlock(tp->t_inpcb); 2053176472Skmacy TRACE_EXIT; 2054174641Skmacy return; 2055174641Skmacy } 2056174641Skmacy 2057176472Skmacy if (toep->tp_ulp_mode == ULP_MODE_TCPDDP) 2058176472Skmacy handle_ddp_data(toep, m); 2059176472Skmacy 2060176472Skmacy m->m_seq = ntohl(hdr->seq); 2061176472Skmacy m->m_ulp_mode = 0; /* for iSCSI */ 2062174641Skmacy 2063174641Skmacy#if VALIDATE_SEQ 2064176472Skmacy if (__predict_false(m->m_seq != tp->rcv_nxt)) { 2065176472Skmacy log(LOG_ERR, 2066174641Skmacy "%s: TID %u: Bad sequence number %u, expected %u\n", 2067178302Skmacy toep->tp_toedev->name, toep->tp_tid, m->m_seq, 2068174641Skmacy tp->rcv_nxt); 2069176472Skmacy m_freem(m); 2070177530Skmacy inp_wunlock(tp->t_inpcb); 2071174641Skmacy return; 2072174641Skmacy } 2073174641Skmacy#endif 2074174641Skmacy m_adj(m, sizeof(*hdr)); 2075174641Skmacy 2076176472Skmacy#ifdef URGENT_DATA_SUPPORTED 2077174641Skmacy /* 2078174641Skmacy * We don't handle urgent data yet 2079174641Skmacy */ 2080174641Skmacy if (__predict_false(hdr->urg)) 2081174641Skmacy handle_urg_ptr(so, tp->rcv_nxt + ntohs(hdr->urg)); 2082174641Skmacy if (__predict_false(tp->urg_data == TCP_URG_NOTYET && 2083174641Skmacy tp->urg_seq - tp->rcv_nxt < skb->len)) 2084174641Skmacy tp->urg_data = TCP_URG_VALID | skb->data[tp->urg_seq - 2085174641Skmacy tp->rcv_nxt]; 2086174641Skmacy#endif 2087174641Skmacy if (__predict_false(hdr->dack_mode != toep->tp_delack_mode)) { 2088174641Skmacy toep->tp_delack_mode = hdr->dack_mode; 2089174641Skmacy toep->tp_delack_seq = tp->rcv_nxt; 2090174641Skmacy } 2091176472Skmacy CTR6(KTR_TOM, "appending mbuf=%p pktlen=%d m_len=%d len=%d rcv_nxt=0x%x enqueued_bytes=%d", 2092176472Skmacy m, m->m_pkthdr.len, m->m_len, len, tp->rcv_nxt, toep->tp_enqueued_bytes); 2093174641Skmacy 2094174641Skmacy if (len < m->m_pkthdr.len) 2095174641Skmacy m->m_pkthdr.len = m->m_len = len; 2096174641Skmacy 2097174641Skmacy tp->rcv_nxt += m->m_pkthdr.len; 2098174641Skmacy tp->t_rcvtime = ticks; 2099174641Skmacy toep->tp_enqueued_bytes += m->m_pkthdr.len; 2100178302Skmacy CTR2(KTR_TOM, 2101176472Skmacy "new_rx_data: seq 0x%x len %u", 2102176472Skmacy m->m_seq, m->m_pkthdr.len); 2103178302Skmacy inp_wunlock(tp->t_inpcb); 2104178302Skmacy rcv = so_sockbuf_rcv(so); 2105178302Skmacy sockbuf_lock(rcv); 2106178302Skmacy#if 0 2107178302Skmacy if (sb_notify(rcv)) 2108178302Skmacy DPRINTF("rx_data so=%p flags=0x%x len=%d\n", so, rcv->sb_flags, m->m_pkthdr.len); 2109174641Skmacy#endif 2110178302Skmacy SBAPPEND(rcv, m); 2111174641Skmacy 2112176472Skmacy#ifdef notyet 2113176472Skmacy /* 2114176472Skmacy * We're giving too many credits to the card - but disable this check so we can keep on moving :-| 2115176472Skmacy * 2116176472Skmacy */ 2117178302Skmacy KASSERT(rcv->sb_cc < (rcv->sb_mbmax << 1), 2118176472Skmacy 2119174641Skmacy ("so=%p, data contents exceed mbmax, sb_cc=%d sb_mbmax=%d", 2120178302Skmacy so, rcv->sb_cc, rcv->sb_mbmax)); 2121176472Skmacy#endif 2122174641Skmacy 2123176472Skmacy 2124176472Skmacy CTR2(KTR_TOM, "sb_cc=%d sb_mbcnt=%d", 2125178302Skmacy rcv->sb_cc, rcv->sb_mbcnt); 2126178302Skmacy 2127178302Skmacy state = so_state_get(so); 2128178302Skmacy if (__predict_true((state & SS_NOFDREF) == 0)) 2129178302Skmacy so_sorwakeup_locked(so); 2130174641Skmacy else 2131178302Skmacy sockbuf_unlock(rcv); 2132174641Skmacy} 2133174641Skmacy 2134174641Skmacy/* 2135174641Skmacy * Handler for RX_DATA CPL messages. 2136174641Skmacy */ 2137174641Skmacystatic int 2138174641Skmacydo_rx_data(struct t3cdev *cdev, struct mbuf *m, void *ctx) 2139174641Skmacy{ 2140174641Skmacy struct toepcb *toep = (struct toepcb *)ctx; 2141174641Skmacy 2142174641Skmacy DPRINTF("rx_data len=%d\n", m->m_pkthdr.len); 2143174641Skmacy 2144174641Skmacy new_rx_data(toep, m); 2145174641Skmacy 2146174641Skmacy return (0); 2147174641Skmacy} 2148174641Skmacy 2149174641Skmacystatic void 2150176472Skmacynew_rx_data_ddp(struct toepcb *toep, struct mbuf *m) 2151174641Skmacy{ 2152176472Skmacy struct tcpcb *tp; 2153174641Skmacy struct ddp_state *q; 2154174641Skmacy struct ddp_buf_state *bsp; 2155174641Skmacy struct cpl_rx_data_ddp *hdr; 2156178302Skmacy struct socket *so; 2157174641Skmacy unsigned int ddp_len, rcv_nxt, ddp_report, end_offset, buf_idx; 2158176472Skmacy int nomoredata = 0; 2159177340Skmacy unsigned int delack_mode; 2160178302Skmacy struct sockbuf *rcv; 2161176472Skmacy 2162178302Skmacy tp = toep->tp_tp; 2163177530Skmacy inp_wlock(tp->t_inpcb); 2164178302Skmacy so = inp_inpcbtosocket(tp->t_inpcb); 2165178302Skmacy 2166176472Skmacy if (__predict_false(so_no_receive(so))) { 2167176472Skmacy 2168176472Skmacy handle_excess_rx(toep, m); 2169177530Skmacy inp_wunlock(tp->t_inpcb); 2170174641Skmacy return; 2171174641Skmacy } 2172176472Skmacy 2173174641Skmacy q = &toep->tp_ddp_state; 2174174641Skmacy hdr = cplhdr(m); 2175174641Skmacy ddp_report = ntohl(hdr->u.ddp_report); 2176174641Skmacy buf_idx = (ddp_report >> S_DDP_BUF_IDX) & 1; 2177174641Skmacy bsp = &q->buf_state[buf_idx]; 2178174641Skmacy 2179176472Skmacy CTR4(KTR_TOM, 2180176472Skmacy "new_rx_data_ddp: tp->rcv_nxt 0x%x cur_offset %u " 2181176472Skmacy "hdr seq 0x%x len %u", 2182176472Skmacy tp->rcv_nxt, bsp->cur_offset, ntohl(hdr->seq), 2183176472Skmacy ntohs(hdr->len)); 2184176472Skmacy CTR3(KTR_TOM, 2185176472Skmacy "new_rx_data_ddp: offset %u ddp_report 0x%x buf_idx=%d", 2186176472Skmacy G_DDP_OFFSET(ddp_report), ddp_report, buf_idx); 2187176472Skmacy 2188174641Skmacy ddp_len = ntohs(hdr->len); 2189174641Skmacy rcv_nxt = ntohl(hdr->seq) + ddp_len; 2190174641Skmacy 2191177340Skmacy delack_mode = G_DDP_DACK_MODE(ddp_report); 2192177340Skmacy if (__predict_false(G_DDP_DACK_MODE(ddp_report) != toep->tp_delack_mode)) { 2193177340Skmacy toep->tp_delack_mode = delack_mode; 2194177340Skmacy toep->tp_delack_seq = tp->rcv_nxt; 2195177340Skmacy } 2196177340Skmacy 2197176472Skmacy m->m_seq = tp->rcv_nxt; 2198174641Skmacy tp->rcv_nxt = rcv_nxt; 2199174641Skmacy 2200176472Skmacy tp->t_rcvtime = ticks; 2201174641Skmacy /* 2202174641Skmacy * Store the length in m->m_len. We are changing the meaning of 2203174641Skmacy * m->m_len here, we need to be very careful that nothing from now on 2204174641Skmacy * interprets ->len of this packet the usual way. 2205174641Skmacy */ 2206176472Skmacy m->m_len = m->m_pkthdr.len = rcv_nxt - m->m_seq; 2207177530Skmacy inp_wunlock(tp->t_inpcb); 2208176472Skmacy CTR3(KTR_TOM, 2209176472Skmacy "new_rx_data_ddp: m_len=%u rcv_next 0x%08x rcv_nxt_prev=0x%08x ", 2210176472Skmacy m->m_len, rcv_nxt, m->m_seq); 2211174641Skmacy /* 2212174641Skmacy * Figure out where the new data was placed in the buffer and store it 2213174641Skmacy * in when. Assumes the buffer offset starts at 0, consumer needs to 2214174641Skmacy * account for page pod's pg_offset. 2215174641Skmacy */ 2216174641Skmacy end_offset = G_DDP_OFFSET(ddp_report) + ddp_len; 2217176472Skmacy m->m_cur_offset = end_offset - m->m_pkthdr.len; 2218174641Skmacy 2219178302Skmacy rcv = so_sockbuf_rcv(so); 2220178302Skmacy sockbuf_lock(rcv); 2221178302Skmacy 2222176472Skmacy m->m_ddp_gl = (unsigned char *)bsp->gl; 2223176472Skmacy m->m_flags |= M_DDP; 2224176472Skmacy bsp->cur_offset = end_offset; 2225176472Skmacy toep->tp_enqueued_bytes += m->m_pkthdr.len; 2226176472Skmacy 2227174641Skmacy /* 2228176472Skmacy * Length is only meaningful for kbuf 2229174641Skmacy */ 2230176472Skmacy if (!(bsp->flags & DDP_BF_NOCOPY)) 2231176472Skmacy KASSERT(m->m_len <= bsp->gl->dgl_length, 2232176472Skmacy ("length received exceeds ddp pages: len=%d dgl_length=%d", 2233176472Skmacy m->m_len, bsp->gl->dgl_length)); 2234174641Skmacy 2235176472Skmacy KASSERT(m->m_len > 0, ("%s m_len=%d", __FUNCTION__, m->m_len)); 2236176472Skmacy KASSERT(m->m_next == NULL, ("m_len=%p", m->m_next)); 2237176472Skmacy /* 2238174641Skmacy * Bit 0 of flags stores whether the DDP buffer is completed. 2239174641Skmacy * Note that other parts of the code depend on this being in bit 0. 2240174641Skmacy */ 2241174641Skmacy if ((bsp->flags & DDP_BF_NOINVAL) && end_offset != bsp->gl->dgl_length) { 2242174641Skmacy panic("spurious ddp completion"); 2243174641Skmacy } else { 2244176472Skmacy m->m_ddp_flags = !!(ddp_report & F_DDP_BUF_COMPLETE); 2245176472Skmacy if (m->m_ddp_flags && !(bsp->flags & DDP_BF_NOFLIP)) 2246174641Skmacy q->cur_buf ^= 1; /* flip buffers */ 2247174641Skmacy } 2248174641Skmacy 2249174641Skmacy if (bsp->flags & DDP_BF_NOCOPY) { 2250176472Skmacy m->m_ddp_flags |= (bsp->flags & DDP_BF_NOCOPY); 2251174641Skmacy bsp->flags &= ~DDP_BF_NOCOPY; 2252174641Skmacy } 2253174641Skmacy 2254174641Skmacy if (ddp_report & F_DDP_PSH) 2255176472Skmacy m->m_ddp_flags |= DDP_BF_PSH; 2256176472Skmacy if (nomoredata) 2257176472Skmacy m->m_ddp_flags |= DDP_BF_NODATA; 2258176472Skmacy 2259177340Skmacy#ifdef notyet 2260177340Skmacy skb_reset_transport_header(skb); 2261177340Skmacy tcp_hdr(skb)->fin = 0; /* changes original hdr->ddp_report */ 2262177340Skmacy#endif 2263178302Skmacy SBAPPEND(rcv, m); 2264178302Skmacy 2265178302Skmacy if ((so_state_get(so) & SS_NOFDREF) == 0 && ((ddp_report & F_DDP_PSH) || 2266178302Skmacy (((m->m_ddp_flags & (DDP_BF_NOCOPY|1)) == (DDP_BF_NOCOPY|1)) 2267178302Skmacy || !(m->m_ddp_flags & DDP_BF_NOCOPY)))) 2268178302Skmacy so_sorwakeup_locked(so); 2269176472Skmacy else 2270178302Skmacy sockbuf_unlock(rcv); 2271174641Skmacy} 2272174641Skmacy 2273174641Skmacy#define DDP_ERR (F_DDP_PPOD_MISMATCH | F_DDP_LLIMIT_ERR | F_DDP_ULIMIT_ERR |\ 2274174641Skmacy F_DDP_PPOD_PARITY_ERR | F_DDP_PADDING_ERR | F_DDP_OFFSET_ERR |\ 2275174641Skmacy F_DDP_INVALID_TAG | F_DDP_COLOR_ERR | F_DDP_TID_MISMATCH |\ 2276174641Skmacy F_DDP_INVALID_PPOD) 2277174641Skmacy 2278174641Skmacy/* 2279174641Skmacy * Handler for RX_DATA_DDP CPL messages. 2280174641Skmacy */ 2281174641Skmacystatic int 2282174641Skmacydo_rx_data_ddp(struct t3cdev *cdev, struct mbuf *m, void *ctx) 2283174641Skmacy{ 2284174641Skmacy struct toepcb *toep = ctx; 2285174641Skmacy const struct cpl_rx_data_ddp *hdr = cplhdr(m); 2286174641Skmacy 2287174641Skmacy VALIDATE_SOCK(so); 2288174641Skmacy 2289174641Skmacy if (__predict_false(ntohl(hdr->ddpvld_status) & DDP_ERR)) { 2290174641Skmacy log(LOG_ERR, "RX_DATA_DDP for TID %u reported error 0x%x\n", 2291174641Skmacy GET_TID(hdr), G_DDP_VALID(ntohl(hdr->ddpvld_status))); 2292176472Skmacy return (CPL_RET_BUF_DONE); 2293174641Skmacy } 2294174641Skmacy#if 0 2295174641Skmacy skb->h.th = tcphdr_skb->h.th; 2296174641Skmacy#endif 2297176472Skmacy new_rx_data_ddp(toep, m); 2298174641Skmacy return (0); 2299174641Skmacy} 2300174641Skmacy 2301174641Skmacystatic void 2302176472Skmacyprocess_ddp_complete(struct toepcb *toep, struct mbuf *m) 2303174641Skmacy{ 2304176472Skmacy struct tcpcb *tp = toep->tp_tp; 2305178302Skmacy struct socket *so; 2306174641Skmacy struct ddp_state *q; 2307174641Skmacy struct ddp_buf_state *bsp; 2308174641Skmacy struct cpl_rx_ddp_complete *hdr; 2309177340Skmacy unsigned int ddp_report, buf_idx, when, delack_mode; 2310176472Skmacy int nomoredata = 0; 2311178302Skmacy struct sockbuf *rcv; 2312178302Skmacy 2313178302Skmacy inp_wlock(tp->t_inpcb); 2314178302Skmacy so = inp_inpcbtosocket(tp->t_inpcb); 2315174641Skmacy 2316176472Skmacy if (__predict_false(so_no_receive(so))) { 2317178302Skmacy struct inpcb *inp = so_sotoinpcb(so); 2318176472Skmacy 2319176472Skmacy handle_excess_rx(toep, m); 2320177530Skmacy inp_wunlock(inp); 2321174641Skmacy return; 2322174641Skmacy } 2323174641Skmacy q = &toep->tp_ddp_state; 2324174641Skmacy hdr = cplhdr(m); 2325174641Skmacy ddp_report = ntohl(hdr->ddp_report); 2326174641Skmacy buf_idx = (ddp_report >> S_DDP_BUF_IDX) & 1; 2327176472Skmacy m->m_pkthdr.csum_data = tp->rcv_nxt; 2328176472Skmacy 2329178302Skmacy rcv = so_sockbuf_rcv(so); 2330178302Skmacy sockbuf_lock(rcv); 2331178302Skmacy 2332174641Skmacy bsp = &q->buf_state[buf_idx]; 2333174641Skmacy when = bsp->cur_offset; 2334176472Skmacy m->m_len = m->m_pkthdr.len = G_DDP_OFFSET(ddp_report) - when; 2335176472Skmacy tp->rcv_nxt += m->m_len; 2336176472Skmacy tp->t_rcvtime = ticks; 2337177340Skmacy 2338177340Skmacy delack_mode = G_DDP_DACK_MODE(ddp_report); 2339177340Skmacy if (__predict_false(G_DDP_DACK_MODE(ddp_report) != toep->tp_delack_mode)) { 2340177340Skmacy toep->tp_delack_mode = delack_mode; 2341177340Skmacy toep->tp_delack_seq = tp->rcv_nxt; 2342177340Skmacy } 2343177340Skmacy#ifdef notyet 2344177340Skmacy skb_reset_transport_header(skb); 2345177340Skmacy tcp_hdr(skb)->fin = 0; /* changes valid memory past CPL */ 2346177340Skmacy#endif 2347177530Skmacy inp_wunlock(tp->t_inpcb); 2348174641Skmacy 2349178767Skmacy KASSERT(m->m_len >= 0, ("%s m_len=%d", __FUNCTION__, m->m_len)); 2350176472Skmacy CTR5(KTR_TOM, 2351176472Skmacy "process_ddp_complete: tp->rcv_nxt 0x%x cur_offset %u " 2352176472Skmacy "ddp_report 0x%x offset %u, len %u", 2353176472Skmacy tp->rcv_nxt, bsp->cur_offset, ddp_report, 2354176472Skmacy G_DDP_OFFSET(ddp_report), m->m_len); 2355178767Skmacy 2356178767Skmacy m->m_cur_offset = bsp->cur_offset; 2357174641Skmacy bsp->cur_offset += m->m_len; 2358174641Skmacy 2359176472Skmacy if (!(bsp->flags & DDP_BF_NOFLIP)) { 2360174641Skmacy q->cur_buf ^= 1; /* flip buffers */ 2361176472Skmacy if (G_DDP_OFFSET(ddp_report) < q->kbuf[0]->dgl_length) 2362176472Skmacy nomoredata=1; 2363176472Skmacy } 2364176472Skmacy 2365176472Skmacy CTR4(KTR_TOM, 2366176472Skmacy "process_ddp_complete: tp->rcv_nxt 0x%x cur_offset %u " 2367176472Skmacy "ddp_report %u offset %u", 2368176472Skmacy tp->rcv_nxt, bsp->cur_offset, ddp_report, 2369176472Skmacy G_DDP_OFFSET(ddp_report)); 2370176472Skmacy 2371176472Skmacy m->m_ddp_gl = (unsigned char *)bsp->gl; 2372176472Skmacy m->m_flags |= M_DDP; 2373176472Skmacy m->m_ddp_flags = (bsp->flags & DDP_BF_NOCOPY) | 1; 2374174641Skmacy if (bsp->flags & DDP_BF_NOCOPY) 2375174641Skmacy bsp->flags &= ~DDP_BF_NOCOPY; 2376176472Skmacy if (nomoredata) 2377176472Skmacy m->m_ddp_flags |= DDP_BF_NODATA; 2378174641Skmacy 2379178302Skmacy SBAPPEND(rcv, m); 2380178302Skmacy if ((so_state_get(so) & SS_NOFDREF) == 0) 2381178302Skmacy so_sorwakeup_locked(so); 2382176472Skmacy else 2383178302Skmacy sockbuf_unlock(rcv); 2384174641Skmacy} 2385174641Skmacy 2386174641Skmacy/* 2387174641Skmacy * Handler for RX_DDP_COMPLETE CPL messages. 2388174641Skmacy */ 2389174641Skmacystatic int 2390174641Skmacydo_rx_ddp_complete(struct t3cdev *cdev, struct mbuf *m, void *ctx) 2391174641Skmacy{ 2392174641Skmacy struct toepcb *toep = ctx; 2393174641Skmacy 2394174641Skmacy VALIDATE_SOCK(so); 2395174641Skmacy#if 0 2396174641Skmacy skb->h.th = tcphdr_skb->h.th; 2397174641Skmacy#endif 2398176472Skmacy process_ddp_complete(toep, m); 2399174641Skmacy return (0); 2400174641Skmacy} 2401174641Skmacy 2402174641Skmacy/* 2403174641Skmacy * Move a socket to TIME_WAIT state. We need to make some adjustments to the 2404174641Skmacy * socket state before calling tcp_time_wait to comply with its expectations. 2405174641Skmacy */ 2406174641Skmacystatic void 2407178302Skmacyenter_timewait(struct tcpcb *tp) 2408174641Skmacy{ 2409178302Skmacy /* 2410178302Skmacy * Bump rcv_nxt for the peer FIN. We don't do this at the time we 2411178302Skmacy * process peer_close because we don't want to carry the peer FIN in 2412178302Skmacy * the socket's receive queue and if we increment rcv_nxt without 2413178302Skmacy * having the FIN in the receive queue we'll confuse facilities such 2414178302Skmacy * as SIOCINQ. 2415178302Skmacy */ 2416178302Skmacy inp_wlock(tp->t_inpcb); 2417178302Skmacy tp->rcv_nxt++; 2418174641Skmacy 2419178302Skmacy tp->ts_recent_age = 0; /* defeat recycling */ 2420178302Skmacy tp->t_srtt = 0; /* defeat tcp_update_metrics */ 2421178302Skmacy inp_wunlock(tp->t_inpcb); 2422178302Skmacy tcp_offload_twstart(tp); 2423178302Skmacy} 2424178302Skmacy 2425178302Skmacystatic void 2426178302Skmacyenter_timewait_disconnect(struct tcpcb *tp) 2427178302Skmacy{ 2428174641Skmacy /* 2429174641Skmacy * Bump rcv_nxt for the peer FIN. We don't do this at the time we 2430174641Skmacy * process peer_close because we don't want to carry the peer FIN in 2431174641Skmacy * the socket's receive queue and if we increment rcv_nxt without 2432174641Skmacy * having the FIN in the receive queue we'll confuse facilities such 2433174641Skmacy * as SIOCINQ. 2434174641Skmacy */ 2435178302Skmacy inp_wlock(tp->t_inpcb); 2436174641Skmacy tp->rcv_nxt++; 2437174641Skmacy 2438174641Skmacy tp->ts_recent_age = 0; /* defeat recycling */ 2439174641Skmacy tp->t_srtt = 0; /* defeat tcp_update_metrics */ 2440178302Skmacy inp_wunlock(tp->t_inpcb); 2441178302Skmacy tcp_offload_twstart_disconnect(tp); 2442174641Skmacy} 2443174641Skmacy 2444174641Skmacy/* 2445176472Skmacy * For TCP DDP a PEER_CLOSE may also be an implicit RX_DDP_COMPLETE. This 2446176472Skmacy * function deals with the data that may be reported along with the FIN. 2447176472Skmacy * Returns -1 if no further processing of the PEER_CLOSE is needed, >= 0 to 2448176472Skmacy * perform normal FIN-related processing. In the latter case 1 indicates that 2449176472Skmacy * there was an implicit RX_DDP_COMPLETE and the skb should not be freed, 0 the 2450176472Skmacy * skb can be freed. 2451176472Skmacy */ 2452176472Skmacystatic int 2453176472Skmacyhandle_peer_close_data(struct socket *so, struct mbuf *m) 2454176472Skmacy{ 2455178302Skmacy struct tcpcb *tp = so_sototcpcb(so); 2456176472Skmacy struct toepcb *toep = tp->t_toe; 2457176472Skmacy struct ddp_state *q; 2458176472Skmacy struct ddp_buf_state *bsp; 2459176472Skmacy struct cpl_peer_close *req = cplhdr(m); 2460176472Skmacy unsigned int rcv_nxt = ntohl(req->rcv_nxt) - 1; /* exclude FIN */ 2461178302Skmacy struct sockbuf *rcv; 2462178302Skmacy 2463176472Skmacy if (tp->rcv_nxt == rcv_nxt) /* no data */ 2464176472Skmacy return (0); 2465176472Skmacy 2466178302Skmacy CTR0(KTR_TOM, "handle_peer_close_data"); 2467176472Skmacy if (__predict_false(so_no_receive(so))) { 2468176472Skmacy handle_excess_rx(toep, m); 2469176472Skmacy 2470176472Skmacy /* 2471176472Skmacy * Although we discard the data we want to process the FIN so 2472176472Skmacy * that PEER_CLOSE + data behaves the same as RX_DATA_DDP + 2473176472Skmacy * PEER_CLOSE without data. In particular this PEER_CLOSE 2474176472Skmacy * may be what will close the connection. We return 1 because 2475176472Skmacy * handle_excess_rx() already freed the packet. 2476176472Skmacy */ 2477176472Skmacy return (1); 2478176472Skmacy } 2479176472Skmacy 2480177575Skmacy inp_lock_assert(tp->t_inpcb); 2481176472Skmacy q = &toep->tp_ddp_state; 2482178302Skmacy rcv = so_sockbuf_rcv(so); 2483178302Skmacy sockbuf_lock(rcv); 2484178302Skmacy 2485176472Skmacy bsp = &q->buf_state[q->cur_buf]; 2486176472Skmacy m->m_len = m->m_pkthdr.len = rcv_nxt - tp->rcv_nxt; 2487176472Skmacy KASSERT(m->m_len > 0, ("%s m_len=%d", __FUNCTION__, m->m_len)); 2488176472Skmacy m->m_ddp_gl = (unsigned char *)bsp->gl; 2489176472Skmacy m->m_flags |= M_DDP; 2490176472Skmacy m->m_cur_offset = bsp->cur_offset; 2491176472Skmacy m->m_ddp_flags = 2492176472Skmacy DDP_BF_PSH | (bsp->flags & DDP_BF_NOCOPY) | 1; 2493176472Skmacy m->m_seq = tp->rcv_nxt; 2494176472Skmacy tp->rcv_nxt = rcv_nxt; 2495176472Skmacy bsp->cur_offset += m->m_pkthdr.len; 2496176472Skmacy if (!(bsp->flags & DDP_BF_NOFLIP)) 2497176472Skmacy q->cur_buf ^= 1; 2498177340Skmacy#ifdef notyet 2499177340Skmacy skb_reset_transport_header(skb); 2500177340Skmacy tcp_hdr(skb)->fin = 0; /* changes valid memory past CPL */ 2501177340Skmacy#endif 2502176472Skmacy tp->t_rcvtime = ticks; 2503178302Skmacy SBAPPEND(rcv, m); 2504178302Skmacy if (__predict_true((so_state_get(so) & SS_NOFDREF) == 0)) 2505178302Skmacy so_sorwakeup_locked(so); 2506176472Skmacy else 2507178302Skmacy sockbuf_unlock(rcv); 2508178302Skmacy 2509176472Skmacy return (1); 2510176472Skmacy} 2511176472Skmacy 2512176472Skmacy/* 2513174641Skmacy * Handle a peer FIN. 2514174641Skmacy */ 2515174641Skmacystatic void 2516178302Skmacydo_peer_fin(struct toepcb *toep, struct mbuf *m) 2517174641Skmacy{ 2518178302Skmacy struct socket *so; 2519178302Skmacy struct tcpcb *tp = toep->tp_tp; 2520178302Skmacy int keep, action; 2521174641Skmacy 2522178302Skmacy action = keep = 0; 2523178302Skmacy CTR1(KTR_TOM, "do_peer_fin state=%d", tp->t_state); 2524178302Skmacy if (!is_t3a(toep->tp_toedev) && (toep->tp_flags & TP_ABORT_RPL_PENDING)) { 2525174641Skmacy printf("abort_pending set\n"); 2526174641Skmacy 2527174641Skmacy goto out; 2528174641Skmacy } 2529177530Skmacy inp_wlock(tp->t_inpcb); 2530178302Skmacy so = inp_inpcbtosocket(toep->tp_tp->t_inpcb); 2531176472Skmacy if (toep->tp_ulp_mode == ULP_MODE_TCPDDP) { 2532176472Skmacy keep = handle_peer_close_data(so, m); 2533176472Skmacy if (keep < 0) { 2534177530Skmacy inp_wunlock(tp->t_inpcb); 2535174641Skmacy return; 2536176472Skmacy } 2537174641Skmacy } 2538176472Skmacy if (TCPS_HAVERCVDFIN(tp->t_state) == 0) { 2539178767Skmacy CTR1(KTR_TOM, 2540178767Skmacy "waking up waiters for cantrcvmore on %p ", so); 2541174641Skmacy socantrcvmore(so); 2542178767Skmacy 2543176472Skmacy /* 2544176472Skmacy * If connection is half-synchronized 2545176472Skmacy * (ie NEEDSYN flag on) then delay ACK, 2546176472Skmacy * so it may be piggybacked when SYN is sent. 2547176472Skmacy * Otherwise, since we received a FIN then no 2548176472Skmacy * more input can be expected, send ACK now. 2549176472Skmacy */ 2550176472Skmacy if (tp->t_flags & TF_NEEDSYN) 2551176472Skmacy tp->t_flags |= TF_DELACK; 2552176472Skmacy else 2553176472Skmacy tp->t_flags |= TF_ACKNOW; 2554176472Skmacy tp->rcv_nxt++; 2555176472Skmacy } 2556176472Skmacy 2557174641Skmacy switch (tp->t_state) { 2558174641Skmacy case TCPS_SYN_RECEIVED: 2559174641Skmacy tp->t_starttime = ticks; 2560174641Skmacy /* FALLTHROUGH */ 2561174641Skmacy case TCPS_ESTABLISHED: 2562174641Skmacy tp->t_state = TCPS_CLOSE_WAIT; 2563174641Skmacy break; 2564174641Skmacy case TCPS_FIN_WAIT_1: 2565174641Skmacy tp->t_state = TCPS_CLOSING; 2566174641Skmacy break; 2567174641Skmacy case TCPS_FIN_WAIT_2: 2568174641Skmacy /* 2569174641Skmacy * If we've sent an abort_req we must have sent it too late, 2570174641Skmacy * HW will send us a reply telling us so, and this peer_close 2571174641Skmacy * is really the last message for this connection and needs to 2572174641Skmacy * be treated as an abort_rpl, i.e., transition the connection 2573174641Skmacy * to TCP_CLOSE (note that the host stack does this at the 2574174641Skmacy * time of generating the RST but we must wait for HW). 2575174641Skmacy * Otherwise we enter TIME_WAIT. 2576174641Skmacy */ 2577174641Skmacy t3_release_offload_resources(toep); 2578174641Skmacy if (toep->tp_flags & TP_ABORT_RPL_PENDING) { 2579178302Skmacy action = TCP_CLOSE; 2580176472Skmacy } else { 2581178302Skmacy action = TCP_TIMEWAIT; 2582176472Skmacy } 2583174641Skmacy break; 2584174641Skmacy default: 2585174641Skmacy log(LOG_ERR, 2586174641Skmacy "%s: TID %u received PEER_CLOSE in bad state %d\n", 2587178302Skmacy toep->tp_toedev->tod_name, toep->tp_tid, tp->t_state); 2588174641Skmacy } 2589178302Skmacy inp_wunlock(tp->t_inpcb); 2590174641Skmacy 2591178302Skmacy if (action == TCP_TIMEWAIT) { 2592178302Skmacy enter_timewait(tp); 2593178302Skmacy } else if (action == TCP_DROP) { 2594178302Skmacy tcp_offload_drop(tp, 0); 2595178302Skmacy } else if (action == TCP_CLOSE) { 2596178302Skmacy tcp_offload_close(tp); 2597178302Skmacy } 2598178767Skmacy 2599174641Skmacy#ifdef notyet 2600176472Skmacy /* Do not send POLL_HUP for half duplex close. */ 2601176472Skmacy if ((sk->sk_shutdown & SEND_SHUTDOWN) || 2602176472Skmacy sk->sk_state == TCP_CLOSE) 2603176472Skmacy sk_wake_async(so, 1, POLL_HUP); 2604176472Skmacy else 2605176472Skmacy sk_wake_async(so, 1, POLL_IN); 2606176472Skmacy#endif 2607174641Skmacy 2608174641Skmacyout: 2609174641Skmacy if (!keep) 2610174641Skmacy m_free(m); 2611174641Skmacy} 2612174641Skmacy 2613174641Skmacy/* 2614174641Skmacy * Handler for PEER_CLOSE CPL messages. 2615174641Skmacy */ 2616174641Skmacystatic int 2617174641Skmacydo_peer_close(struct t3cdev *cdev, struct mbuf *m, void *ctx) 2618174641Skmacy{ 2619174641Skmacy struct toepcb *toep = (struct toepcb *)ctx; 2620174641Skmacy 2621174641Skmacy VALIDATE_SOCK(so); 2622174641Skmacy 2623178302Skmacy do_peer_fin(toep, m); 2624174641Skmacy return (0); 2625174641Skmacy} 2626174641Skmacy 2627174641Skmacystatic void 2628178302Skmacyprocess_close_con_rpl(struct toepcb *toep, struct mbuf *m) 2629174641Skmacy{ 2630174641Skmacy struct cpl_close_con_rpl *rpl = cplhdr(m); 2631178302Skmacy struct tcpcb *tp = toep->tp_tp; 2632178302Skmacy struct socket *so; 2633178302Skmacy int action = 0; 2634178302Skmacy struct sockbuf *rcv; 2635178302Skmacy 2636178302Skmacy inp_wlock(tp->t_inpcb); 2637178302Skmacy so = inp_inpcbtosocket(tp->t_inpcb); 2638178302Skmacy 2639174641Skmacy tp->snd_una = ntohl(rpl->snd_nxt) - 1; /* exclude FIN */ 2640174641Skmacy 2641178302Skmacy if (!is_t3a(toep->tp_toedev) && (toep->tp_flags & TP_ABORT_RPL_PENDING)) { 2642178302Skmacy inp_wunlock(tp->t_inpcb); 2643174641Skmacy goto out; 2644178302Skmacy } 2645174641Skmacy 2646178302Skmacy CTR3(KTR_TOM, "process_close_con_rpl(%p) state=%d dead=%d", toep, 2647178302Skmacy tp->t_state, !!(so_state_get(so) & SS_NOFDREF)); 2648178302Skmacy 2649174641Skmacy switch (tp->t_state) { 2650174641Skmacy case TCPS_CLOSING: /* see FIN_WAIT2 case in do_peer_fin */ 2651174641Skmacy t3_release_offload_resources(toep); 2652174641Skmacy if (toep->tp_flags & TP_ABORT_RPL_PENDING) { 2653178302Skmacy action = TCP_CLOSE; 2654174641Skmacy 2655176472Skmacy } else { 2656178302Skmacy action = TCP_TIMEWAIT; 2657176472Skmacy } 2658174641Skmacy break; 2659174641Skmacy case TCPS_LAST_ACK: 2660174641Skmacy /* 2661174641Skmacy * In this state we don't care about pending abort_rpl. 2662174641Skmacy * If we've sent abort_req it was post-close and was sent too 2663174641Skmacy * late, this close_con_rpl is the actual last message. 2664174641Skmacy */ 2665174641Skmacy t3_release_offload_resources(toep); 2666178302Skmacy action = TCP_CLOSE; 2667174641Skmacy break; 2668174641Skmacy case TCPS_FIN_WAIT_1: 2669176472Skmacy /* 2670176472Skmacy * If we can't receive any more 2671176472Skmacy * data, then closing user can proceed. 2672176472Skmacy * Starting the timer is contrary to the 2673176472Skmacy * specification, but if we don't get a FIN 2674176472Skmacy * we'll hang forever. 2675176472Skmacy * 2676176472Skmacy * XXXjl: 2677176472Skmacy * we should release the tp also, and use a 2678176472Skmacy * compressed state. 2679176472Skmacy */ 2680178302Skmacy if (so) 2681178302Skmacy rcv = so_sockbuf_rcv(so); 2682178302Skmacy else 2683178302Skmacy break; 2684178302Skmacy 2685178302Skmacy if (rcv->sb_state & SBS_CANTRCVMORE) { 2686176472Skmacy int timeout; 2687178302Skmacy 2688178302Skmacy if (so) 2689178302Skmacy soisdisconnected(so); 2690176472Skmacy timeout = (tcp_fast_finwait2_recycle) ? 2691176472Skmacy tcp_finwait2_timeout : tcp_maxidle; 2692176472Skmacy tcp_timer_activate(tp, TT_2MSL, timeout); 2693176472Skmacy } 2694176472Skmacy tp->t_state = TCPS_FIN_WAIT_2; 2695178302Skmacy if ((so_options_get(so) & SO_LINGER) && so_linger_get(so) == 0 && 2696174641Skmacy (toep->tp_flags & TP_ABORT_SHUTDOWN) == 0) { 2697178302Skmacy action = TCP_DROP; 2698174641Skmacy } 2699174641Skmacy 2700174641Skmacy break; 2701174641Skmacy default: 2702174641Skmacy log(LOG_ERR, 2703174641Skmacy "%s: TID %u received CLOSE_CON_RPL in bad state %d\n", 2704178302Skmacy toep->tp_toedev->tod_name, toep->tp_tid, 2705174641Skmacy tp->t_state); 2706174641Skmacy } 2707178302Skmacy inp_wunlock(tp->t_inpcb); 2708178302Skmacy 2709178302Skmacy 2710178302Skmacy if (action == TCP_TIMEWAIT) { 2711178302Skmacy enter_timewait_disconnect(tp); 2712178302Skmacy } else if (action == TCP_DROP) { 2713178302Skmacy tcp_offload_drop(tp, 0); 2714178302Skmacy } else if (action == TCP_CLOSE) { 2715178302Skmacy tcp_offload_close(tp); 2716178302Skmacy } 2717174641Skmacyout: 2718176472Skmacy m_freem(m); 2719174641Skmacy} 2720174641Skmacy 2721174641Skmacy/* 2722174641Skmacy * Handler for CLOSE_CON_RPL CPL messages. 2723174641Skmacy */ 2724174641Skmacystatic int 2725174641Skmacydo_close_con_rpl(struct t3cdev *cdev, struct mbuf *m, 2726174641Skmacy void *ctx) 2727174641Skmacy{ 2728174641Skmacy struct toepcb *toep = (struct toepcb *)ctx; 2729174641Skmacy 2730178302Skmacy process_close_con_rpl(toep, m); 2731174641Skmacy return (0); 2732174641Skmacy} 2733174641Skmacy 2734174641Skmacy/* 2735174641Skmacy * Process abort replies. We only process these messages if we anticipate 2736174641Skmacy * them as the coordination between SW and HW in this area is somewhat lacking 2737174641Skmacy * and sometimes we get ABORT_RPLs after we are done with the connection that 2738174641Skmacy * originated the ABORT_REQ. 2739174641Skmacy */ 2740174641Skmacystatic void 2741178302Skmacyprocess_abort_rpl(struct toepcb *toep, struct mbuf *m) 2742174641Skmacy{ 2743178302Skmacy struct tcpcb *tp = toep->tp_tp; 2744178302Skmacy struct socket *so; 2745178302Skmacy int needclose = 0; 2746174641Skmacy 2747174641Skmacy#ifdef T3_TRACE 2748174641Skmacy T3_TRACE1(TIDTB(sk), 2749174641Skmacy "process_abort_rpl: GTS rpl pending %d", 2750174641Skmacy sock_flag(sk, ABORT_RPL_PENDING)); 2751174641Skmacy#endif 2752176472Skmacy 2753177530Skmacy inp_wlock(tp->t_inpcb); 2754178302Skmacy so = inp_inpcbtosocket(tp->t_inpcb); 2755174641Skmacy 2756174641Skmacy if (toep->tp_flags & TP_ABORT_RPL_PENDING) { 2757174641Skmacy /* 2758174641Skmacy * XXX panic on tcpdrop 2759174641Skmacy */ 2760178302Skmacy if (!(toep->tp_flags & TP_ABORT_RPL_RCVD) && !is_t3a(toep->tp_toedev)) 2761174641Skmacy toep->tp_flags |= TP_ABORT_RPL_RCVD; 2762174641Skmacy else { 2763174641Skmacy toep->tp_flags &= ~(TP_ABORT_RPL_RCVD|TP_ABORT_RPL_PENDING); 2764174641Skmacy if (!(toep->tp_flags & TP_ABORT_REQ_RCVD) || 2765178302Skmacy !is_t3a(toep->tp_toedev)) { 2766174641Skmacy if (toep->tp_flags & TP_ABORT_REQ_RCVD) 2767174641Skmacy panic("TP_ABORT_REQ_RCVD set"); 2768174641Skmacy t3_release_offload_resources(toep); 2769178302Skmacy needclose = 1; 2770174641Skmacy } 2771174641Skmacy } 2772174641Skmacy } 2773178302Skmacy inp_wunlock(tp->t_inpcb); 2774174641Skmacy 2775178302Skmacy if (needclose) 2776178302Skmacy tcp_offload_close(tp); 2777178302Skmacy 2778174641Skmacy m_free(m); 2779174641Skmacy} 2780174641Skmacy 2781174641Skmacy/* 2782174641Skmacy * Handle an ABORT_RPL_RSS CPL message. 2783174641Skmacy */ 2784174641Skmacystatic int 2785174641Skmacydo_abort_rpl(struct t3cdev *cdev, struct mbuf *m, void *ctx) 2786174641Skmacy{ 2787174641Skmacy struct cpl_abort_rpl_rss *rpl = cplhdr(m); 2788174641Skmacy struct toepcb *toep; 2789174641Skmacy 2790174641Skmacy /* 2791174641Skmacy * Ignore replies to post-close aborts indicating that the abort was 2792174641Skmacy * requested too late. These connections are terminated when we get 2793174641Skmacy * PEER_CLOSE or CLOSE_CON_RPL and by the time the abort_rpl_rss 2794174641Skmacy * arrives the TID is either no longer used or it has been recycled. 2795174641Skmacy */ 2796174641Skmacy if (rpl->status == CPL_ERR_ABORT_FAILED) { 2797174641Skmacydiscard: 2798174641Skmacy m_free(m); 2799174641Skmacy return (0); 2800174641Skmacy } 2801174641Skmacy 2802174641Skmacy toep = (struct toepcb *)ctx; 2803174641Skmacy 2804174641Skmacy /* 2805174641Skmacy * Sometimes we've already closed the socket, e.g., a post-close 2806174641Skmacy * abort races with ABORT_REQ_RSS, the latter frees the socket 2807174641Skmacy * expecting the ABORT_REQ will fail with CPL_ERR_ABORT_FAILED, 2808174641Skmacy * but FW turns the ABORT_REQ into a regular one and so we get 2809174641Skmacy * ABORT_RPL_RSS with status 0 and no socket. Only on T3A. 2810174641Skmacy */ 2811174641Skmacy if (!toep) 2812174641Skmacy goto discard; 2813174641Skmacy 2814174641Skmacy if (toep->tp_tp == NULL) { 2815178302Skmacy log(LOG_NOTICE, "removing tid for abort\n"); 2816174641Skmacy cxgb_remove_tid(cdev, toep, toep->tp_tid); 2817174641Skmacy if (toep->tp_l2t) 2818174641Skmacy l2t_release(L2DATA(cdev), toep->tp_l2t); 2819174641Skmacy 2820174641Skmacy toepcb_release(toep); 2821174641Skmacy goto discard; 2822174641Skmacy } 2823174641Skmacy 2824178302Skmacy log(LOG_NOTICE, "toep=%p\n", toep); 2825178302Skmacy log(LOG_NOTICE, "tp=%p\n", toep->tp_tp); 2826174641Skmacy 2827174641Skmacy toepcb_hold(toep); 2828178302Skmacy process_abort_rpl(toep, m); 2829174641Skmacy toepcb_release(toep); 2830174641Skmacy return (0); 2831174641Skmacy} 2832174641Skmacy 2833174641Skmacy/* 2834176472Skmacy * Convert the status code of an ABORT_REQ into a FreeBSD error code. Also 2835174641Skmacy * indicate whether RST should be sent in response. 2836174641Skmacy */ 2837174641Skmacystatic int 2838174641Skmacyabort_status_to_errno(struct socket *so, int abort_reason, int *need_rst) 2839174641Skmacy{ 2840178302Skmacy struct tcpcb *tp = so_sototcpcb(so); 2841174641Skmacy 2842174641Skmacy switch (abort_reason) { 2843174641Skmacy case CPL_ERR_BAD_SYN: 2844174641Skmacy#if 0 2845174641Skmacy NET_INC_STATS_BH(LINUX_MIB_TCPABORTONSYN); // fall through 2846174641Skmacy#endif 2847174641Skmacy case CPL_ERR_CONN_RESET: 2848174641Skmacy // XXX need to handle SYN_RECV due to crossed SYNs 2849174641Skmacy return (tp->t_state == TCPS_CLOSE_WAIT ? EPIPE : ECONNRESET); 2850174641Skmacy case CPL_ERR_XMIT_TIMEDOUT: 2851174641Skmacy case CPL_ERR_PERSIST_TIMEDOUT: 2852174641Skmacy case CPL_ERR_FINWAIT2_TIMEDOUT: 2853174641Skmacy case CPL_ERR_KEEPALIVE_TIMEDOUT: 2854174641Skmacy#if 0 2855174641Skmacy NET_INC_STATS_BH(LINUX_MIB_TCPABORTONTIMEOUT); 2856174641Skmacy#endif 2857174641Skmacy return (ETIMEDOUT); 2858174641Skmacy default: 2859174641Skmacy return (EIO); 2860174641Skmacy } 2861174641Skmacy} 2862174641Skmacy 2863174641Skmacystatic inline void 2864174641Skmacyset_abort_rpl_wr(struct mbuf *m, unsigned int tid, int cmd) 2865174641Skmacy{ 2866174641Skmacy struct cpl_abort_rpl *rpl = cplhdr(m); 2867174641Skmacy 2868174641Skmacy rpl->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_HOST_ABORT_CON_RPL)); 2869174641Skmacy rpl->wr.wr_lo = htonl(V_WR_TID(tid)); 2870174641Skmacy m->m_len = m->m_pkthdr.len = sizeof(*rpl); 2871174641Skmacy 2872174641Skmacy OPCODE_TID(rpl) = htonl(MK_OPCODE_TID(CPL_ABORT_RPL, tid)); 2873174641Skmacy rpl->cmd = cmd; 2874174641Skmacy} 2875174641Skmacy 2876174641Skmacystatic void 2877174641Skmacysend_deferred_abort_rpl(struct toedev *tdev, struct mbuf *m) 2878174641Skmacy{ 2879174641Skmacy struct mbuf *reply_mbuf; 2880174641Skmacy struct cpl_abort_req_rss *req = cplhdr(m); 2881174641Skmacy 2882174641Skmacy reply_mbuf = m_gethdr_nofail(sizeof(struct cpl_abort_rpl)); 2883174641Skmacy m_set_priority(m, CPL_PRIORITY_DATA); 2884174641Skmacy m->m_len = m->m_pkthdr.len = sizeof(struct cpl_abort_rpl); 2885174641Skmacy set_abort_rpl_wr(reply_mbuf, GET_TID(req), req->status); 2886174641Skmacy cxgb_ofld_send(TOM_DATA(tdev)->cdev, reply_mbuf); 2887174641Skmacy m_free(m); 2888174641Skmacy} 2889174641Skmacy 2890174641Skmacy/* 2891174641Skmacy * Returns whether an ABORT_REQ_RSS message is a negative advice. 2892174641Skmacy */ 2893174641Skmacystatic inline int 2894174641Skmacyis_neg_adv_abort(unsigned int status) 2895174641Skmacy{ 2896174641Skmacy return status == CPL_ERR_RTX_NEG_ADVICE || 2897174641Skmacy status == CPL_ERR_PERSIST_NEG_ADVICE; 2898174641Skmacy} 2899174641Skmacy 2900174641Skmacystatic void 2901174641Skmacysend_abort_rpl(struct mbuf *m, struct toedev *tdev, int rst_status) 2902174641Skmacy{ 2903174641Skmacy struct mbuf *reply_mbuf; 2904174641Skmacy struct cpl_abort_req_rss *req = cplhdr(m); 2905174641Skmacy 2906174641Skmacy reply_mbuf = m_gethdr(M_NOWAIT, MT_DATA); 2907174641Skmacy 2908174641Skmacy if (!reply_mbuf) { 2909174641Skmacy /* Defer the reply. Stick rst_status into req->cmd. */ 2910174641Skmacy req->status = rst_status; 2911174641Skmacy t3_defer_reply(m, tdev, send_deferred_abort_rpl); 2912174641Skmacy return; 2913174641Skmacy } 2914174641Skmacy 2915174641Skmacy m_set_priority(reply_mbuf, CPL_PRIORITY_DATA); 2916174641Skmacy set_abort_rpl_wr(reply_mbuf, GET_TID(req), rst_status); 2917174641Skmacy m_free(m); 2918174641Skmacy 2919174641Skmacy /* 2920174641Skmacy * XXX need to sync with ARP as for SYN_RECV connections we can send 2921174641Skmacy * these messages while ARP is pending. For other connection states 2922174641Skmacy * it's not a problem. 2923174641Skmacy */ 2924174641Skmacy cxgb_ofld_send(TOM_DATA(tdev)->cdev, reply_mbuf); 2925174641Skmacy} 2926174641Skmacy 2927174641Skmacy#ifdef notyet 2928174641Skmacystatic void 2929174641Skmacycleanup_syn_rcv_conn(struct socket *child, struct socket *parent) 2930174641Skmacy{ 2931176507Skmacy CXGB_UNIMPLEMENTED(); 2932174641Skmacy#ifdef notyet 2933174641Skmacy struct request_sock *req = child->sk_user_data; 2934174641Skmacy 2935174641Skmacy inet_csk_reqsk_queue_removed(parent, req); 2936174641Skmacy synq_remove(tcp_sk(child)); 2937174641Skmacy __reqsk_free(req); 2938174641Skmacy child->sk_user_data = NULL; 2939174641Skmacy#endif 2940174641Skmacy} 2941174641Skmacy 2942174641Skmacy 2943174641Skmacy/* 2944174641Skmacy * Performs the actual work to abort a SYN_RECV connection. 2945174641Skmacy */ 2946174641Skmacystatic void 2947174641Skmacydo_abort_syn_rcv(struct socket *child, struct socket *parent) 2948174641Skmacy{ 2949178302Skmacy struct tcpcb *parenttp = so_sototcpcb(parent); 2950178302Skmacy struct tcpcb *childtp = so_sototcpcb(child); 2951174641Skmacy 2952174641Skmacy /* 2953174641Skmacy * If the server is still open we clean up the child connection, 2954174641Skmacy * otherwise the server already did the clean up as it was purging 2955174641Skmacy * its SYN queue and the skb was just sitting in its backlog. 2956174641Skmacy */ 2957174641Skmacy if (__predict_false(parenttp->t_state == TCPS_LISTEN)) { 2958174641Skmacy cleanup_syn_rcv_conn(child, parent); 2959177530Skmacy inp_wlock(childtp->t_inpcb); 2960174641Skmacy t3_release_offload_resources(childtp->t_toe); 2961178302Skmacy inp_wunlock(childtp->t_inpcb); 2962178302Skmacy tcp_offload_close(childtp); 2963174641Skmacy } 2964174641Skmacy} 2965174641Skmacy#endif 2966174641Skmacy 2967174641Skmacy/* 2968174641Skmacy * Handle abort requests for a SYN_RECV connection. These need extra work 2969174641Skmacy * because the socket is on its parent's SYN queue. 2970174641Skmacy */ 2971174641Skmacystatic int 2972174641Skmacyabort_syn_rcv(struct socket *so, struct mbuf *m) 2973174641Skmacy{ 2974176507Skmacy CXGB_UNIMPLEMENTED(); 2975174641Skmacy#ifdef notyet 2976174641Skmacy struct socket *parent; 2977178302Skmacy struct toedev *tdev = toep->tp_toedev; 2978174641Skmacy struct t3cdev *cdev = TOM_DATA(tdev)->cdev; 2979174641Skmacy struct socket *oreq = so->so_incomp; 2980174641Skmacy struct t3c_tid_entry *t3c_stid; 2981174641Skmacy struct tid_info *t; 2982174641Skmacy 2983174641Skmacy if (!oreq) 2984174641Skmacy return -1; /* somehow we are not on the SYN queue */ 2985174641Skmacy 2986174641Skmacy t = &(T3C_DATA(cdev))->tid_maps; 2987174641Skmacy t3c_stid = lookup_stid(t, oreq->ts_recent); 2988174641Skmacy parent = ((struct listen_ctx *)t3c_stid->ctx)->lso; 2989174641Skmacy 2990178302Skmacy so_lock(parent); 2991174641Skmacy do_abort_syn_rcv(so, parent); 2992174641Skmacy send_abort_rpl(m, tdev, CPL_ABORT_NO_RST); 2993178302Skmacy so_unlock(parent); 2994174641Skmacy#endif 2995174641Skmacy return (0); 2996174641Skmacy} 2997174641Skmacy 2998174641Skmacy/* 2999174641Skmacy * Process abort requests. If we are waiting for an ABORT_RPL we ignore this 3000174641Skmacy * request except that we need to reply to it. 3001174641Skmacy */ 3002174641Skmacystatic void 3003178302Skmacyprocess_abort_req(struct toepcb *toep, struct mbuf *m, struct toedev *tdev) 3004174641Skmacy{ 3005174641Skmacy int rst_status = CPL_ABORT_NO_RST; 3006174641Skmacy const struct cpl_abort_req_rss *req = cplhdr(m); 3007178302Skmacy struct tcpcb *tp = toep->tp_tp; 3008178302Skmacy struct socket *so; 3009178302Skmacy int needclose = 0; 3010178302Skmacy 3011177530Skmacy inp_wlock(tp->t_inpcb); 3012178302Skmacy so = inp_inpcbtosocket(toep->tp_tp->t_inpcb); 3013174641Skmacy if ((toep->tp_flags & TP_ABORT_REQ_RCVD) == 0) { 3014174641Skmacy toep->tp_flags |= (TP_ABORT_REQ_RCVD|TP_ABORT_SHUTDOWN); 3015174641Skmacy m_free(m); 3016174641Skmacy goto skip; 3017174641Skmacy } 3018174641Skmacy 3019174641Skmacy toep->tp_flags &= ~TP_ABORT_REQ_RCVD; 3020174641Skmacy /* 3021174641Skmacy * Three cases to consider: 3022174641Skmacy * a) We haven't sent an abort_req; close the connection. 3023174641Skmacy * b) We have sent a post-close abort_req that will get to TP too late 3024174641Skmacy * and will generate a CPL_ERR_ABORT_FAILED reply. The reply will 3025174641Skmacy * be ignored and the connection should be closed now. 3026174641Skmacy * c) We have sent a regular abort_req that will get to TP too late. 3027174641Skmacy * That will generate an abort_rpl with status 0, wait for it. 3028174641Skmacy */ 3029174641Skmacy if (((toep->tp_flags & TP_ABORT_RPL_PENDING) == 0) || 3030178302Skmacy (is_t3a(toep->tp_toedev) && (toep->tp_flags & TP_CLOSE_CON_REQUESTED))) { 3031178302Skmacy int error; 3032178302Skmacy 3033178302Skmacy error = abort_status_to_errno(so, req->status, 3034174641Skmacy &rst_status); 3035178302Skmacy so_error_set(so, error); 3036178302Skmacy 3037178302Skmacy if (__predict_true((so_state_get(so) & SS_NOFDREF) == 0)) 3038178302Skmacy so_sorwakeup(so); 3039174641Skmacy /* 3040174641Skmacy * SYN_RECV needs special processing. If abort_syn_rcv() 3041174641Skmacy * returns 0 is has taken care of the abort. 3042174641Skmacy */ 3043174641Skmacy if ((tp->t_state == TCPS_SYN_RECEIVED) && !abort_syn_rcv(so, m)) 3044174641Skmacy goto skip; 3045174641Skmacy 3046174641Skmacy t3_release_offload_resources(toep); 3047178302Skmacy needclose = 1; 3048174641Skmacy } 3049178302Skmacy inp_wunlock(tp->t_inpcb); 3050178302Skmacy 3051178302Skmacy if (needclose) 3052178302Skmacy tcp_offload_close(tp); 3053178302Skmacy 3054174641Skmacy send_abort_rpl(m, tdev, rst_status); 3055174641Skmacy return; 3056174641Skmacyskip: 3057178302Skmacy inp_wunlock(tp->t_inpcb); 3058174641Skmacy} 3059174641Skmacy 3060174641Skmacy/* 3061174641Skmacy * Handle an ABORT_REQ_RSS CPL message. 3062174641Skmacy */ 3063174641Skmacystatic int 3064174641Skmacydo_abort_req(struct t3cdev *cdev, struct mbuf *m, void *ctx) 3065174641Skmacy{ 3066174641Skmacy const struct cpl_abort_req_rss *req = cplhdr(m); 3067174641Skmacy struct toepcb *toep = (struct toepcb *)ctx; 3068174641Skmacy 3069174641Skmacy if (is_neg_adv_abort(req->status)) { 3070174641Skmacy m_free(m); 3071174641Skmacy return (0); 3072174641Skmacy } 3073174641Skmacy 3074178302Skmacy log(LOG_NOTICE, "aborting tid=%d\n", toep->tp_tid); 3075174641Skmacy 3076174641Skmacy if ((toep->tp_flags & (TP_SYN_RCVD|TP_ABORT_REQ_RCVD)) == TP_SYN_RCVD) { 3077174641Skmacy cxgb_remove_tid(cdev, toep, toep->tp_tid); 3078174641Skmacy toep->tp_flags |= TP_ABORT_REQ_RCVD; 3079174641Skmacy 3080174641Skmacy send_abort_rpl(m, toep->tp_toedev, CPL_ABORT_NO_RST); 3081174641Skmacy if (toep->tp_l2t) 3082174641Skmacy l2t_release(L2DATA(cdev), toep->tp_l2t); 3083174641Skmacy 3084174641Skmacy /* 3085174641Skmacy * Unhook 3086174641Skmacy */ 3087174641Skmacy toep->tp_tp->t_toe = NULL; 3088174641Skmacy toep->tp_tp->t_flags &= ~TF_TOE; 3089174641Skmacy toep->tp_tp = NULL; 3090174641Skmacy /* 3091174641Skmacy * XXX need to call syncache_chkrst - but we don't 3092174641Skmacy * have a way of doing that yet 3093174641Skmacy */ 3094174641Skmacy toepcb_release(toep); 3095178302Skmacy log(LOG_ERR, "abort for unestablished connection :-(\n"); 3096174641Skmacy return (0); 3097174641Skmacy } 3098174641Skmacy if (toep->tp_tp == NULL) { 3099178302Skmacy log(LOG_NOTICE, "disconnected toepcb\n"); 3100174641Skmacy /* should be freed momentarily */ 3101174641Skmacy return (0); 3102174641Skmacy } 3103174641Skmacy 3104178302Skmacy 3105174641Skmacy toepcb_hold(toep); 3106178302Skmacy process_abort_req(toep, m, toep->tp_toedev); 3107174641Skmacy toepcb_release(toep); 3108174641Skmacy return (0); 3109174641Skmacy} 3110174641Skmacy#ifdef notyet 3111174641Skmacystatic void 3112174641Skmacypass_open_abort(struct socket *child, struct socket *parent, struct mbuf *m) 3113174641Skmacy{ 3114174641Skmacy struct toedev *tdev = TOE_DEV(parent); 3115174641Skmacy 3116174641Skmacy do_abort_syn_rcv(child, parent); 3117174641Skmacy if (tdev->tod_ttid == TOE_ID_CHELSIO_T3) { 3118174641Skmacy struct cpl_pass_accept_rpl *rpl = cplhdr(m); 3119174641Skmacy 3120174641Skmacy rpl->opt0h = htonl(F_TCAM_BYPASS); 3121174641Skmacy rpl->opt0l_status = htonl(CPL_PASS_OPEN_REJECT); 3122174641Skmacy cxgb_ofld_send(TOM_DATA(tdev)->cdev, m); 3123174641Skmacy } else 3124174641Skmacy m_free(m); 3125174641Skmacy} 3126174641Skmacy#endif 3127174641Skmacystatic void 3128174641Skmacyhandle_pass_open_arp_failure(struct socket *so, struct mbuf *m) 3129174641Skmacy{ 3130176507Skmacy CXGB_UNIMPLEMENTED(); 3131174641Skmacy 3132174641Skmacy#ifdef notyet 3133174641Skmacy struct t3cdev *cdev; 3134174641Skmacy struct socket *parent; 3135174641Skmacy struct socket *oreq; 3136174641Skmacy struct t3c_tid_entry *t3c_stid; 3137174641Skmacy struct tid_info *t; 3138178302Skmacy struct tcpcb *otp, *tp = so_sototcpcb(so); 3139174641Skmacy struct toepcb *toep = tp->t_toe; 3140174641Skmacy 3141174641Skmacy /* 3142174641Skmacy * If the connection is being aborted due to the parent listening 3143174641Skmacy * socket going away there's nothing to do, the ABORT_REQ will close 3144174641Skmacy * the connection. 3145174641Skmacy */ 3146174641Skmacy if (toep->tp_flags & TP_ABORT_RPL_PENDING) { 3147174641Skmacy m_free(m); 3148174641Skmacy return; 3149174641Skmacy } 3150174641Skmacy 3151174641Skmacy oreq = so->so_incomp; 3152178302Skmacy otp = so_sototcpcb(oreq); 3153174641Skmacy 3154174641Skmacy cdev = T3C_DEV(so); 3155174641Skmacy t = &(T3C_DATA(cdev))->tid_maps; 3156174641Skmacy t3c_stid = lookup_stid(t, otp->ts_recent); 3157174641Skmacy parent = ((struct listen_ctx *)t3c_stid->ctx)->lso; 3158174641Skmacy 3159178302Skmacy so_lock(parent); 3160174641Skmacy pass_open_abort(so, parent, m); 3161178302Skmacy so_unlock(parent); 3162174641Skmacy#endif 3163174641Skmacy} 3164174641Skmacy 3165174641Skmacy/* 3166174641Skmacy * Handle an ARP failure for a CPL_PASS_ACCEPT_RPL. This is treated similarly 3167174641Skmacy * to an ABORT_REQ_RSS in SYN_RECV as both events need to tear down a SYN_RECV 3168174641Skmacy * connection. 3169174641Skmacy */ 3170174641Skmacystatic void 3171174641Skmacypass_accept_rpl_arp_failure(struct t3cdev *cdev, struct mbuf *m) 3172174641Skmacy{ 3173174641Skmacy 3174174641Skmacy#ifdef notyet 3175174641Skmacy TCP_INC_STATS_BH(TCP_MIB_ATTEMPTFAILS); 3176174641Skmacy BLOG_SKB_CB(skb)->dev = TOE_DEV(skb->sk); 3177174641Skmacy#endif 3178174641Skmacy handle_pass_open_arp_failure(m_get_socket(m), m); 3179174641Skmacy} 3180174641Skmacy 3181174641Skmacy/* 3182174641Skmacy * Populate a reject CPL_PASS_ACCEPT_RPL WR. 3183174641Skmacy */ 3184174641Skmacystatic void 3185174641Skmacymk_pass_accept_rpl(struct mbuf *reply_mbuf, struct mbuf *req_mbuf) 3186174641Skmacy{ 3187174641Skmacy struct cpl_pass_accept_req *req = cplhdr(req_mbuf); 3188174641Skmacy struct cpl_pass_accept_rpl *rpl = cplhdr(reply_mbuf); 3189174641Skmacy unsigned int tid = GET_TID(req); 3190174641Skmacy 3191174641Skmacy m_set_priority(reply_mbuf, CPL_PRIORITY_SETUP); 3192174641Skmacy rpl->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 3193174641Skmacy OPCODE_TID(rpl) = htonl(MK_OPCODE_TID(CPL_PASS_ACCEPT_RPL, tid)); 3194174641Skmacy rpl->peer_ip = req->peer_ip; // req->peer_ip not overwritten yet 3195174641Skmacy rpl->opt0h = htonl(F_TCAM_BYPASS); 3196174641Skmacy rpl->opt0l_status = htonl(CPL_PASS_OPEN_REJECT); 3197174641Skmacy rpl->opt2 = 0; 3198174641Skmacy rpl->rsvd = rpl->opt2; /* workaround for HW bug */ 3199174641Skmacy} 3200174641Skmacy 3201174641Skmacy/* 3202174641Skmacy * Send a deferred reject to an accept request. 3203174641Skmacy */ 3204174641Skmacystatic void 3205174641Skmacyreject_pass_request(struct toedev *tdev, struct mbuf *m) 3206174641Skmacy{ 3207174641Skmacy struct mbuf *reply_mbuf; 3208174641Skmacy 3209174641Skmacy reply_mbuf = m_gethdr_nofail(sizeof(struct cpl_pass_accept_rpl)); 3210174641Skmacy mk_pass_accept_rpl(reply_mbuf, m); 3211174641Skmacy cxgb_ofld_send(TOM_DATA(tdev)->cdev, reply_mbuf); 3212174641Skmacy m_free(m); 3213174641Skmacy} 3214174641Skmacy 3215174641Skmacystatic void 3216174641Skmacyhandle_syncache_event(int event, void *arg) 3217174641Skmacy{ 3218174641Skmacy struct toepcb *toep = arg; 3219174641Skmacy 3220174641Skmacy switch (event) { 3221174708Skmacy case TOE_SC_ENTRY_PRESENT: 3222174641Skmacy /* 3223174641Skmacy * entry already exists - free toepcb 3224174641Skmacy * and l2t 3225174641Skmacy */ 3226174641Skmacy printf("syncache entry present\n"); 3227174641Skmacy toepcb_release(toep); 3228174641Skmacy break; 3229174708Skmacy case TOE_SC_DROP: 3230174641Skmacy /* 3231174641Skmacy * The syncache has given up on this entry 3232174641Skmacy * either it timed out, or it was evicted 3233174641Skmacy * we need to explicitly release the tid 3234174641Skmacy */ 3235174641Skmacy printf("syncache entry dropped\n"); 3236174641Skmacy toepcb_release(toep); 3237174641Skmacy break; 3238174641Skmacy default: 3239174641Skmacy log(LOG_ERR, "unknown syncache event %d\n", event); 3240174641Skmacy break; 3241174641Skmacy } 3242174641Skmacy} 3243174641Skmacy 3244174641Skmacystatic void 3245174641Skmacysyncache_add_accept_req(struct cpl_pass_accept_req *req, struct socket *lso, struct toepcb *toep) 3246174641Skmacy{ 3247174641Skmacy struct in_conninfo inc; 3248174641Skmacy struct tcpopt to; 3249174641Skmacy struct tcphdr th; 3250174641Skmacy struct inpcb *inp; 3251174641Skmacy int mss, wsf, sack, ts; 3252176472Skmacy uint32_t rcv_isn = ntohl(req->rcv_isn); 3253176472Skmacy 3254174641Skmacy bzero(&to, sizeof(struct tcpopt)); 3255178302Skmacy inp = so_sotoinpcb(lso); 3256174641Skmacy 3257174641Skmacy /* 3258174641Skmacy * Fill out information for entering us into the syncache 3259174641Skmacy */ 3260174641Skmacy inc.inc_fport = th.th_sport = req->peer_port; 3261174641Skmacy inc.inc_lport = th.th_dport = req->local_port; 3262176472Skmacy th.th_seq = req->rcv_isn; 3263174641Skmacy th.th_flags = TH_SYN; 3264174641Skmacy 3265176472Skmacy toep->tp_iss = toep->tp_delack_seq = toep->tp_rcv_wup = toep->tp_copied_seq = rcv_isn + 1; 3266176472Skmacy 3267174641Skmacy 3268174641Skmacy inc.inc_isipv6 = 0; 3269174641Skmacy inc.inc_len = 0; 3270174641Skmacy inc.inc_faddr.s_addr = req->peer_ip; 3271174641Skmacy inc.inc_laddr.s_addr = req->local_ip; 3272174641Skmacy 3273174641Skmacy DPRINTF("syncache add of %d:%d %d:%d\n", 3274174641Skmacy ntohl(req->local_ip), ntohs(req->local_port), 3275174641Skmacy ntohl(req->peer_ip), ntohs(req->peer_port)); 3276174641Skmacy 3277174641Skmacy mss = req->tcp_options.mss; 3278174641Skmacy wsf = req->tcp_options.wsf; 3279174641Skmacy ts = req->tcp_options.tstamp; 3280174641Skmacy sack = req->tcp_options.sack; 3281174641Skmacy to.to_mss = mss; 3282174641Skmacy to.to_wscale = wsf; 3283174641Skmacy to.to_flags = (mss ? TOF_MSS : 0) | (wsf ? TOF_SCALE : 0) | (ts ? TOF_TS : 0) | (sack ? TOF_SACKPERM : 0); 3284174641Skmacy syncache_offload_add(&inc, &to, &th, inp, &lso, &cxgb_toe_usrreqs, toep); 3285174641Skmacy} 3286174641Skmacy 3287174641Skmacy 3288174641Skmacy/* 3289174641Skmacy * Process a CPL_PASS_ACCEPT_REQ message. Does the part that needs the socket 3290174641Skmacy * lock held. Note that the sock here is a listening socket that is not owned 3291174641Skmacy * by the TOE. 3292174641Skmacy */ 3293174641Skmacystatic void 3294174641Skmacyprocess_pass_accept_req(struct socket *so, struct mbuf *m, struct toedev *tdev, 3295174641Skmacy struct listen_ctx *lctx) 3296174641Skmacy{ 3297174641Skmacy int rt_flags; 3298174641Skmacy struct l2t_entry *e; 3299174641Skmacy struct iff_mac tim; 3300174641Skmacy struct mbuf *reply_mbuf, *ddp_mbuf = NULL; 3301174641Skmacy struct cpl_pass_accept_rpl *rpl; 3302174641Skmacy struct cpl_pass_accept_req *req = cplhdr(m); 3303174641Skmacy unsigned int tid = GET_TID(req); 3304174641Skmacy struct tom_data *d = TOM_DATA(tdev); 3305174641Skmacy struct t3cdev *cdev = d->cdev; 3306178302Skmacy struct tcpcb *tp = so_sototcpcb(so); 3307174641Skmacy struct toepcb *newtoep; 3308174641Skmacy struct rtentry *dst; 3309174641Skmacy struct sockaddr_in nam; 3310174641Skmacy struct t3c_data *td = T3C_DATA(cdev); 3311174641Skmacy 3312174641Skmacy reply_mbuf = m_gethdr(M_NOWAIT, MT_DATA); 3313174641Skmacy if (__predict_false(reply_mbuf == NULL)) { 3314174641Skmacy if (tdev->tod_ttid == TOE_ID_CHELSIO_T3) 3315174641Skmacy t3_defer_reply(m, tdev, reject_pass_request); 3316174641Skmacy else { 3317174641Skmacy cxgb_queue_tid_release(cdev, tid); 3318174641Skmacy m_free(m); 3319174641Skmacy } 3320174641Skmacy DPRINTF("failed to get reply_mbuf\n"); 3321174641Skmacy 3322174641Skmacy goto out; 3323174641Skmacy } 3324174641Skmacy 3325174641Skmacy if (tp->t_state != TCPS_LISTEN) { 3326174641Skmacy DPRINTF("socket not in listen state\n"); 3327174641Skmacy 3328174641Skmacy goto reject; 3329174641Skmacy } 3330174641Skmacy 3331174641Skmacy tim.mac_addr = req->dst_mac; 3332174641Skmacy tim.vlan_tag = ntohs(req->vlan_tag); 3333174641Skmacy if (cdev->ctl(cdev, GET_IFF_FROM_MAC, &tim) < 0 || !tim.dev) { 3334174641Skmacy DPRINTF("rejecting from failed GET_IFF_FROM_MAC\n"); 3335174641Skmacy goto reject; 3336174641Skmacy } 3337174641Skmacy 3338174641Skmacy#ifdef notyet 3339174641Skmacy /* 3340174641Skmacy * XXX do route lookup to confirm that we're still listening on this 3341174641Skmacy * address 3342174641Skmacy */ 3343174641Skmacy if (ip_route_input(skb, req->local_ip, req->peer_ip, 3344174641Skmacy G_PASS_OPEN_TOS(ntohl(req->tos_tid)), tim.dev)) 3345174641Skmacy goto reject; 3346174641Skmacy rt_flags = ((struct rtable *)skb->dst)->rt_flags & 3347174641Skmacy (RTCF_BROADCAST | RTCF_MULTICAST | RTCF_LOCAL); 3348174641Skmacy dst_release(skb->dst); // done with the input route, release it 3349174641Skmacy skb->dst = NULL; 3350174641Skmacy 3351174641Skmacy if ((rt_flags & RTF_LOCAL) == 0) 3352174641Skmacy goto reject; 3353174641Skmacy#endif 3354174641Skmacy /* 3355174641Skmacy * XXX 3356174641Skmacy */ 3357174641Skmacy rt_flags = RTF_LOCAL; 3358174641Skmacy if ((rt_flags & RTF_LOCAL) == 0) 3359174641Skmacy goto reject; 3360174641Skmacy 3361174641Skmacy /* 3362174641Skmacy * Calculate values and add to syncache 3363174641Skmacy */ 3364174641Skmacy 3365174641Skmacy newtoep = toepcb_alloc(); 3366174641Skmacy if (newtoep == NULL) 3367174641Skmacy goto reject; 3368174641Skmacy 3369174641Skmacy bzero(&nam, sizeof(struct sockaddr_in)); 3370174641Skmacy 3371174641Skmacy nam.sin_len = sizeof(struct sockaddr_in); 3372174641Skmacy nam.sin_family = AF_INET; 3373174641Skmacy nam.sin_addr.s_addr =req->peer_ip; 3374174641Skmacy dst = rtalloc2((struct sockaddr *)&nam, 1, 0); 3375174641Skmacy 3376174641Skmacy if (dst == NULL) { 3377174641Skmacy printf("failed to find route\n"); 3378174641Skmacy goto reject; 3379174641Skmacy } 3380174641Skmacy e = newtoep->tp_l2t = t3_l2t_get(d->cdev, dst, tim.dev, 3381174641Skmacy (struct sockaddr *)&nam); 3382174641Skmacy if (e == NULL) { 3383174641Skmacy DPRINTF("failed to get l2t\n"); 3384174641Skmacy } 3385174641Skmacy /* 3386174641Skmacy * Point to our listen socket until accept 3387174641Skmacy */ 3388174641Skmacy newtoep->tp_tp = tp; 3389174641Skmacy newtoep->tp_flags = TP_SYN_RCVD; 3390174641Skmacy newtoep->tp_tid = tid; 3391174641Skmacy newtoep->tp_toedev = tdev; 3392176472Skmacy tp->rcv_wnd = select_rcv_wnd(tdev, so); 3393174641Skmacy 3394174641Skmacy cxgb_insert_tid(cdev, d->client, newtoep, tid); 3395178302Skmacy so_lock(so); 3396174641Skmacy LIST_INSERT_HEAD(&lctx->synq_head, newtoep, synq_entry); 3397178302Skmacy so_unlock(so); 3398178302Skmacy 3399178302Skmacy newtoep->tp_ulp_mode = TOM_TUNABLE(tdev, ddp) && !(so_options_get(so) & SO_NO_DDP) && 3400176472Skmacy tp->rcv_wnd >= MIN_DDP_RCV_WIN ? ULP_MODE_TCPDDP : 0; 3401176472Skmacy 3402176472Skmacy if (newtoep->tp_ulp_mode) { 3403174641Skmacy ddp_mbuf = m_gethdr(M_NOWAIT, MT_DATA); 3404174641Skmacy 3405176472Skmacy if (ddp_mbuf == NULL) 3406174641Skmacy newtoep->tp_ulp_mode = 0; 3407174641Skmacy } 3408176472Skmacy 3409176472Skmacy CTR4(KTR_TOM, "ddp=%d rcv_wnd=%ld min_win=%d ulp_mode=%d", 3410176472Skmacy TOM_TUNABLE(tdev, ddp), tp->rcv_wnd, MIN_DDP_RCV_WIN, newtoep->tp_ulp_mode); 3411174641Skmacy set_arp_failure_handler(reply_mbuf, pass_accept_rpl_arp_failure); 3412174641Skmacy /* 3413174641Skmacy * XXX workaround for lack of syncache drop 3414174641Skmacy */ 3415174641Skmacy toepcb_hold(newtoep); 3416174641Skmacy syncache_add_accept_req(req, so, newtoep); 3417174641Skmacy 3418174641Skmacy rpl = cplhdr(reply_mbuf); 3419174641Skmacy reply_mbuf->m_pkthdr.len = reply_mbuf->m_len = sizeof(*rpl); 3420174641Skmacy rpl->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 3421174641Skmacy rpl->wr.wr_lo = 0; 3422174641Skmacy OPCODE_TID(rpl) = htonl(MK_OPCODE_TID(CPL_PASS_ACCEPT_RPL, tid)); 3423174641Skmacy rpl->opt2 = htonl(calc_opt2(so, tdev)); 3424174641Skmacy rpl->rsvd = rpl->opt2; /* workaround for HW bug */ 3425174641Skmacy rpl->peer_ip = req->peer_ip; // req->peer_ip is not overwritten 3426174641Skmacy 3427174641Skmacy rpl->opt0h = htonl(calc_opt0h(so, select_mss(td, NULL, dst->rt_ifp->if_mtu)) | 3428174641Skmacy V_L2T_IDX(e->idx) | V_TX_CHANNEL(e->smt_idx)); 3429176472Skmacy rpl->opt0l_status = htonl(calc_opt0l(so, newtoep->tp_ulp_mode) | 3430174641Skmacy CPL_PASS_OPEN_ACCEPT); 3431174641Skmacy 3432174641Skmacy DPRINTF("opt0l_status=%08x\n", rpl->opt0l_status); 3433174641Skmacy 3434176472Skmacy m_set_priority(reply_mbuf, mkprio(CPL_PRIORITY_SETUP, newtoep)); 3435174641Skmacy 3436174641Skmacy l2t_send(cdev, reply_mbuf, e); 3437174641Skmacy m_free(m); 3438176472Skmacy if (newtoep->tp_ulp_mode) { 3439176472Skmacy __set_tcb_field(newtoep, ddp_mbuf, W_TCB_RX_DDP_FLAGS, 3440174641Skmacy V_TF_DDP_OFF(1) | 3441174641Skmacy TP_DDP_TIMER_WORKAROUND_MASK, 3442174641Skmacy V_TF_DDP_OFF(1) | 3443176472Skmacy TP_DDP_TIMER_WORKAROUND_VAL, 1); 3444176472Skmacy } else 3445176472Skmacy printf("not offloading\n"); 3446176472Skmacy 3447176472Skmacy 3448174641Skmacy 3449174641Skmacy return; 3450174641Skmacyreject: 3451174641Skmacy if (tdev->tod_ttid == TOE_ID_CHELSIO_T3) 3452174641Skmacy mk_pass_accept_rpl(reply_mbuf, m); 3453174641Skmacy else 3454176472Skmacy mk_tid_release(reply_mbuf, newtoep, tid); 3455174641Skmacy cxgb_ofld_send(cdev, reply_mbuf); 3456174641Skmacy m_free(m); 3457174641Skmacyout: 3458174641Skmacy#if 0 3459174641Skmacy TCP_INC_STATS_BH(TCP_MIB_ATTEMPTFAILS); 3460174641Skmacy#else 3461174641Skmacy return; 3462174641Skmacy#endif 3463174641Skmacy} 3464174641Skmacy 3465174641Skmacy/* 3466174641Skmacy * Handle a CPL_PASS_ACCEPT_REQ message. 3467174641Skmacy */ 3468174641Skmacystatic int 3469174641Skmacydo_pass_accept_req(struct t3cdev *cdev, struct mbuf *m, void *ctx) 3470174641Skmacy{ 3471174641Skmacy struct listen_ctx *listen_ctx = (struct listen_ctx *)ctx; 3472178302Skmacy struct socket *lso = listen_ctx->lso; /* XXX need an interlock against the listen socket going away */ 3473174641Skmacy struct tom_data *d = listen_ctx->tom_data; 3474174641Skmacy 3475174641Skmacy#if VALIDATE_TID 3476174641Skmacy struct cpl_pass_accept_req *req = cplhdr(m); 3477174641Skmacy unsigned int tid = GET_TID(req); 3478174641Skmacy struct tid_info *t = &(T3C_DATA(cdev))->tid_maps; 3479174641Skmacy 3480174641Skmacy if (unlikely(!lsk)) { 3481174641Skmacy printk(KERN_ERR "%s: PASS_ACCEPT_REQ had unknown STID %lu\n", 3482174641Skmacy cdev->name, 3483174641Skmacy (unsigned long)((union listen_entry *)ctx - 3484174641Skmacy t->stid_tab)); 3485174641Skmacy return CPL_RET_BUF_DONE; 3486174641Skmacy } 3487174641Skmacy if (unlikely(tid >= t->ntids)) { 3488174641Skmacy printk(KERN_ERR "%s: passive open TID %u too large\n", 3489174641Skmacy cdev->name, tid); 3490174641Skmacy return CPL_RET_BUF_DONE; 3491174641Skmacy } 3492174641Skmacy /* 3493174641Skmacy * For T3A the current user of the TID may have closed but its last 3494174641Skmacy * message(s) may have been backlogged so the TID appears to be still 3495174641Skmacy * in use. Just take the TID away, the connection can close at its 3496174641Skmacy * own leisure. For T3B this situation is a bug. 3497174641Skmacy */ 3498174641Skmacy if (!valid_new_tid(t, tid) && 3499174641Skmacy cdev->type != T3A) { 3500174641Skmacy printk(KERN_ERR "%s: passive open uses existing TID %u\n", 3501174641Skmacy cdev->name, tid); 3502174641Skmacy return CPL_RET_BUF_DONE; 3503174641Skmacy } 3504174641Skmacy#endif 3505174641Skmacy 3506174641Skmacy process_pass_accept_req(lso, m, &d->tdev, listen_ctx); 3507174641Skmacy return (0); 3508174641Skmacy} 3509174641Skmacy 3510174641Skmacy/* 3511174641Skmacy * Called when a connection is established to translate the TCP options 3512176472Skmacy * reported by HW to FreeBSD's native format. 3513174641Skmacy */ 3514174641Skmacystatic void 3515174641Skmacyassign_rxopt(struct socket *so, unsigned int opt) 3516174641Skmacy{ 3517178302Skmacy struct tcpcb *tp = so_sototcpcb(so); 3518174641Skmacy struct toepcb *toep = tp->t_toe; 3519178302Skmacy const struct t3c_data *td = T3C_DATA(TOEP_T3C_DEV(toep)); 3520174641Skmacy 3521177575Skmacy inp_lock_assert(tp->t_inpcb); 3522174641Skmacy 3523174641Skmacy toep->tp_mss_clamp = td->mtus[G_TCPOPT_MSS(opt)] - 40; 3524174641Skmacy tp->t_flags |= G_TCPOPT_TSTAMP(opt) ? TF_RCVD_TSTMP : 0; 3525174641Skmacy tp->t_flags |= G_TCPOPT_SACK(opt) ? TF_SACK_PERMIT : 0; 3526174641Skmacy tp->t_flags |= G_TCPOPT_WSCALE_OK(opt) ? TF_RCVD_SCALE : 0; 3527176472Skmacy if ((tp->t_flags & (TF_RCVD_SCALE|TF_REQ_SCALE)) == 3528176472Skmacy (TF_RCVD_SCALE|TF_REQ_SCALE)) 3529176472Skmacy tp->rcv_scale = tp->request_r_scale; 3530174641Skmacy} 3531174641Skmacy 3532174641Skmacy/* 3533174641Skmacy * Completes some final bits of initialization for just established connections 3534174641Skmacy * and changes their state to TCP_ESTABLISHED. 3535174641Skmacy * 3536174641Skmacy * snd_isn here is the ISN after the SYN, i.e., the true ISN + 1. 3537174641Skmacy */ 3538174641Skmacystatic void 3539174641Skmacymake_established(struct socket *so, u32 snd_isn, unsigned int opt) 3540174641Skmacy{ 3541178302Skmacy struct tcpcb *tp = so_sototcpcb(so); 3542174641Skmacy struct toepcb *toep = tp->t_toe; 3543174641Skmacy 3544174641Skmacy toep->tp_write_seq = tp->iss = tp->snd_max = tp->snd_nxt = tp->snd_una = snd_isn; 3545174641Skmacy assign_rxopt(so, opt); 3546178302Skmacy 3547178302Skmacy /* 3548178302Skmacy *XXXXXXXXXXX 3549178302Skmacy * 3550178302Skmacy */ 3551178302Skmacy#ifdef notyet 3552174641Skmacy so->so_proto->pr_ctloutput = t3_ctloutput; 3553178302Skmacy#endif 3554174641Skmacy 3555174641Skmacy#if 0 3556174641Skmacy inet_sk(sk)->id = tp->write_seq ^ jiffies; 3557174641Skmacy#endif 3558174641Skmacy /* 3559174641Skmacy * XXX not clear what rcv_wup maps to 3560174641Skmacy */ 3561174641Skmacy /* 3562174641Skmacy * Causes the first RX_DATA_ACK to supply any Rx credits we couldn't 3563174641Skmacy * pass through opt0. 3564174641Skmacy */ 3565174641Skmacy if (tp->rcv_wnd > (M_RCV_BUFSIZ << 10)) 3566174641Skmacy toep->tp_rcv_wup -= tp->rcv_wnd - (M_RCV_BUFSIZ << 10); 3567174641Skmacy 3568174641Skmacy dump_toepcb(toep); 3569174641Skmacy 3570174641Skmacy#ifdef notyet 3571174641Skmacy/* 3572174641Skmacy * no clean interface for marking ARP up to date 3573174641Skmacy */ 3574174641Skmacy dst_confirm(sk->sk_dst_cache); 3575174641Skmacy#endif 3576176472Skmacy tp->t_starttime = ticks; 3577174641Skmacy tp->t_state = TCPS_ESTABLISHED; 3578176472Skmacy soisconnected(so); 3579174641Skmacy} 3580174641Skmacy 3581174641Skmacystatic int 3582174641Skmacysyncache_expand_establish_req(struct cpl_pass_establish *req, struct socket **so, struct toepcb *toep) 3583174641Skmacy{ 3584174641Skmacy 3585174641Skmacy struct in_conninfo inc; 3586174641Skmacy struct tcpopt to; 3587174641Skmacy struct tcphdr th; 3588174641Skmacy int mss, wsf, sack, ts; 3589174641Skmacy struct mbuf *m = NULL; 3590174641Skmacy const struct t3c_data *td = T3C_DATA(TOM_DATA(toep->tp_toedev)->cdev); 3591174641Skmacy unsigned int opt; 3592174641Skmacy 3593174641Skmacy#ifdef MAC 3594174641Skmacy#error "no MAC support" 3595174641Skmacy#endif 3596174641Skmacy 3597174641Skmacy opt = ntohs(req->tcp_opt); 3598174641Skmacy 3599174641Skmacy bzero(&to, sizeof(struct tcpopt)); 3600174641Skmacy 3601174641Skmacy /* 3602174641Skmacy * Fill out information for entering us into the syncache 3603174641Skmacy */ 3604174641Skmacy inc.inc_fport = th.th_sport = req->peer_port; 3605174641Skmacy inc.inc_lport = th.th_dport = req->local_port; 3606174641Skmacy th.th_seq = req->rcv_isn; 3607174641Skmacy th.th_flags = TH_ACK; 3608174641Skmacy 3609174641Skmacy inc.inc_isipv6 = 0; 3610174641Skmacy inc.inc_len = 0; 3611174641Skmacy inc.inc_faddr.s_addr = req->peer_ip; 3612174641Skmacy inc.inc_laddr.s_addr = req->local_ip; 3613174641Skmacy 3614174641Skmacy mss = td->mtus[G_TCPOPT_MSS(opt)] - 40; 3615174641Skmacy wsf = G_TCPOPT_WSCALE_OK(opt); 3616174641Skmacy ts = G_TCPOPT_TSTAMP(opt); 3617174641Skmacy sack = G_TCPOPT_SACK(opt); 3618174641Skmacy 3619174641Skmacy to.to_mss = mss; 3620174641Skmacy to.to_wscale = G_TCPOPT_SND_WSCALE(opt); 3621174641Skmacy to.to_flags = (mss ? TOF_MSS : 0) | (wsf ? TOF_SCALE : 0) | (ts ? TOF_TS : 0) | (sack ? TOF_SACKPERM : 0); 3622174641Skmacy 3623174641Skmacy DPRINTF("syncache expand of %d:%d %d:%d mss:%d wsf:%d ts:%d sack:%d\n", 3624174641Skmacy ntohl(req->local_ip), ntohs(req->local_port), 3625174641Skmacy ntohl(req->peer_ip), ntohs(req->peer_port), 3626174641Skmacy mss, wsf, ts, sack); 3627178302Skmacy return syncache_offload_expand(&inc, &to, &th, so, m); 3628174641Skmacy} 3629174641Skmacy 3630174641Skmacy 3631174641Skmacy/* 3632174641Skmacy * Process a CPL_PASS_ESTABLISH message. XXX a lot of the locking doesn't work 3633174641Skmacy * if we are in TCP_SYN_RECV due to crossed SYNs 3634174641Skmacy */ 3635174641Skmacystatic int 3636174641Skmacydo_pass_establish(struct t3cdev *cdev, struct mbuf *m, void *ctx) 3637174641Skmacy{ 3638174641Skmacy struct cpl_pass_establish *req = cplhdr(m); 3639174641Skmacy struct toepcb *toep = (struct toepcb *)ctx; 3640178302Skmacy struct tcpcb *tp = toep->tp_tp; 3641174641Skmacy struct socket *so, *lso; 3642174641Skmacy struct t3c_data *td = T3C_DATA(cdev); 3643178302Skmacy struct sockbuf *snd, *rcv; 3644178302Skmacy 3645174641Skmacy // Complete socket initialization now that we have the SND_ISN 3646174641Skmacy 3647174641Skmacy struct toedev *tdev; 3648174641Skmacy 3649178302Skmacy 3650174641Skmacy tdev = toep->tp_toedev; 3651174641Skmacy 3652178302Skmacy inp_wlock(tp->t_inpcb); 3653178302Skmacy 3654178302Skmacy /* 3655178302Skmacy * 3656178302Skmacy * XXX need to add reference while we're manipulating 3657178302Skmacy */ 3658178302Skmacy so = lso = inp_inpcbtosocket(tp->t_inpcb); 3659178302Skmacy 3660178302Skmacy inp_wunlock(tp->t_inpcb); 3661178302Skmacy 3662178302Skmacy so_lock(so); 3663174641Skmacy LIST_REMOVE(toep, synq_entry); 3664178302Skmacy so_unlock(so); 3665174641Skmacy 3666174641Skmacy if (!syncache_expand_establish_req(req, &so, toep)) { 3667174641Skmacy /* 3668174641Skmacy * No entry 3669174641Skmacy */ 3670176507Skmacy CXGB_UNIMPLEMENTED(); 3671174641Skmacy } 3672174641Skmacy if (so == NULL) { 3673174641Skmacy /* 3674174641Skmacy * Couldn't create the socket 3675174641Skmacy */ 3676176507Skmacy CXGB_UNIMPLEMENTED(); 3677174641Skmacy } 3678174641Skmacy 3679178302Skmacy tp = so_sototcpcb(so); 3680177530Skmacy inp_wlock(tp->t_inpcb); 3681176472Skmacy 3682178767Skmacy snd = so_sockbuf_snd(so); 3683178767Skmacy rcv = so_sockbuf_rcv(so); 3684178767Skmacy 3685178302Skmacy snd->sb_flags |= SB_NOCOALESCE; 3686178302Skmacy rcv->sb_flags |= SB_NOCOALESCE; 3687176472Skmacy 3688174641Skmacy toep->tp_tp = tp; 3689174641Skmacy toep->tp_flags = 0; 3690174641Skmacy tp->t_toe = toep; 3691174641Skmacy reset_wr_list(toep); 3692176472Skmacy tp->rcv_wnd = select_rcv_wnd(tdev, so); 3693176472Skmacy tp->rcv_nxt = toep->tp_copied_seq; 3694174641Skmacy install_offload_ops(so); 3695174641Skmacy 3696174641Skmacy toep->tp_wr_max = toep->tp_wr_avail = TOM_TUNABLE(tdev, max_wrs); 3697174641Skmacy toep->tp_wr_unacked = 0; 3698174641Skmacy toep->tp_qset = G_QNUM(ntohl(m->m_pkthdr.csum_data)); 3699174641Skmacy toep->tp_qset_idx = 0; 3700174641Skmacy toep->tp_mtu_idx = select_mss(td, tp, toep->tp_l2t->neigh->rt_ifp->if_mtu); 3701174641Skmacy 3702174641Skmacy /* 3703174641Skmacy * XXX Cancel any keep alive timer 3704174641Skmacy */ 3705174641Skmacy 3706174641Skmacy make_established(so, ntohl(req->snd_isn), ntohs(req->tcp_opt)); 3707178302Skmacy 3708178302Skmacy /* 3709178302Skmacy * XXX workaround for lack of syncache drop 3710178302Skmacy */ 3711178302Skmacy toepcb_release(toep); 3712177530Skmacy inp_wunlock(tp->t_inpcb); 3713174641Skmacy 3714176472Skmacy CTR1(KTR_TOM, "do_pass_establish tid=%u", toep->tp_tid); 3715176472Skmacy cxgb_log_tcb(cdev->adapter, toep->tp_tid); 3716174641Skmacy#ifdef notyet 3717174641Skmacy /* 3718174641Skmacy * XXX not sure how these checks map to us 3719174641Skmacy */ 3720174641Skmacy if (unlikely(sk->sk_socket)) { // simultaneous opens only 3721174641Skmacy sk->sk_state_change(sk); 3722174641Skmacy sk_wake_async(so, 0, POLL_OUT); 3723174641Skmacy } 3724174641Skmacy /* 3725174641Skmacy * The state for the new connection is now up to date. 3726174641Skmacy * Next check if we should add the connection to the parent's 3727174641Skmacy * accept queue. When the parent closes it resets connections 3728174641Skmacy * on its SYN queue, so check if we are being reset. If so we 3729174641Skmacy * don't need to do anything more, the coming ABORT_RPL will 3730174641Skmacy * destroy this socket. Otherwise move the connection to the 3731174641Skmacy * accept queue. 3732174641Skmacy * 3733174641Skmacy * Note that we reset the synq before closing the server so if 3734174641Skmacy * we are not being reset the stid is still open. 3735174641Skmacy */ 3736174641Skmacy if (unlikely(!tp->forward_skb_hint)) { // removed from synq 3737174641Skmacy __kfree_skb(skb); 3738174641Skmacy goto unlock; 3739174641Skmacy } 3740174641Skmacy#endif 3741174641Skmacy m_free(m); 3742174641Skmacy 3743174641Skmacy return (0); 3744174641Skmacy} 3745174641Skmacy 3746174641Skmacy/* 3747174641Skmacy * Fill in the right TID for CPL messages waiting in the out-of-order queue 3748174641Skmacy * and send them to the TOE. 3749174641Skmacy */ 3750174641Skmacystatic void 3751178302Skmacyfixup_and_send_ofo(struct toepcb *toep) 3752174641Skmacy{ 3753174641Skmacy struct mbuf *m; 3754178302Skmacy struct toedev *tdev = toep->tp_toedev; 3755178302Skmacy struct tcpcb *tp = toep->tp_tp; 3756174641Skmacy unsigned int tid = toep->tp_tid; 3757174641Skmacy 3758178302Skmacy log(LOG_NOTICE, "fixup_and_send_ofo\n"); 3759174641Skmacy 3760177575Skmacy inp_lock_assert(tp->t_inpcb); 3761174641Skmacy while ((m = mbufq_dequeue(&toep->out_of_order_queue)) != NULL) { 3762174641Skmacy /* 3763174641Skmacy * A variety of messages can be waiting but the fields we'll 3764174641Skmacy * be touching are common to all so any message type will do. 3765174641Skmacy */ 3766174641Skmacy struct cpl_close_con_req *p = cplhdr(m); 3767174641Skmacy 3768174641Skmacy p->wr.wr_lo = htonl(V_WR_TID(tid)); 3769174641Skmacy OPCODE_TID(p) = htonl(MK_OPCODE_TID(p->ot.opcode, tid)); 3770174641Skmacy cxgb_ofld_send(TOM_DATA(tdev)->cdev, m); 3771174641Skmacy } 3772174641Skmacy} 3773174641Skmacy 3774174641Skmacy/* 3775174641Skmacy * Updates socket state from an active establish CPL message. Runs with the 3776174641Skmacy * socket lock held. 3777174641Skmacy */ 3778174641Skmacystatic void 3779174641Skmacysocket_act_establish(struct socket *so, struct mbuf *m) 3780174641Skmacy{ 3781174641Skmacy struct cpl_act_establish *req = cplhdr(m); 3782174641Skmacy u32 rcv_isn = ntohl(req->rcv_isn); /* real RCV_ISN + 1 */ 3783178302Skmacy struct tcpcb *tp = so_sototcpcb(so); 3784174641Skmacy struct toepcb *toep = tp->t_toe; 3785174641Skmacy 3786174641Skmacy if (__predict_false(tp->t_state != TCPS_SYN_SENT)) 3787174641Skmacy log(LOG_ERR, "TID %u expected SYN_SENT, found %d\n", 3788174641Skmacy toep->tp_tid, tp->t_state); 3789174641Skmacy 3790174641Skmacy tp->ts_recent_age = ticks; 3791174641Skmacy tp->irs = tp->rcv_wnd = tp->rcv_nxt = rcv_isn; 3792174641Skmacy toep->tp_delack_seq = toep->tp_rcv_wup = toep->tp_copied_seq = tp->irs; 3793174641Skmacy 3794174641Skmacy make_established(so, ntohl(req->snd_isn), ntohs(req->tcp_opt)); 3795174641Skmacy 3796174641Skmacy /* 3797174641Skmacy * Now that we finally have a TID send any CPL messages that we had to 3798174641Skmacy * defer for lack of a TID. 3799174641Skmacy */ 3800174641Skmacy if (mbufq_len(&toep->out_of_order_queue)) 3801178302Skmacy fixup_and_send_ofo(toep); 3802174641Skmacy 3803178302Skmacy if (__predict_false(so_state_get(so) & SS_NOFDREF)) { 3804176472Skmacy /* 3805176472Skmacy * XXX does this even make sense? 3806174641Skmacy */ 3807178302Skmacy so_sorwakeup(so); 3808174641Skmacy } 3809174641Skmacy m_free(m); 3810174641Skmacy#ifdef notyet 3811174641Skmacy/* 3812174641Skmacy * XXX assume no write requests permitted while socket connection is 3813174641Skmacy * incomplete 3814174641Skmacy */ 3815174641Skmacy /* 3816174641Skmacy * Currently the send queue must be empty at this point because the 3817174641Skmacy * socket layer does not send anything before a connection is 3818174641Skmacy * established. To be future proof though we handle the possibility 3819174641Skmacy * that there are pending buffers to send (either TX_DATA or 3820174641Skmacy * CLOSE_CON_REQ). First we need to adjust the sequence number of the 3821174641Skmacy * buffers according to the just learned write_seq, and then we send 3822174641Skmacy * them on their way. 3823174641Skmacy */ 3824174641Skmacy fixup_pending_writeq_buffers(sk); 3825174641Skmacy if (t3_push_frames(so, 1)) 3826174641Skmacy sk->sk_write_space(sk); 3827174641Skmacy#endif 3828174641Skmacy 3829176472Skmacy toep->tp_state = tp->t_state; 3830174641Skmacy tcpstat.tcps_connects++; 3831174641Skmacy 3832174641Skmacy} 3833174641Skmacy 3834174641Skmacy/* 3835174641Skmacy * Process a CPL_ACT_ESTABLISH message. 3836174641Skmacy */ 3837174641Skmacystatic int 3838174641Skmacydo_act_establish(struct t3cdev *cdev, struct mbuf *m, void *ctx) 3839174641Skmacy{ 3840174641Skmacy struct cpl_act_establish *req = cplhdr(m); 3841174641Skmacy unsigned int tid = GET_TID(req); 3842174641Skmacy unsigned int atid = G_PASS_OPEN_TID(ntohl(req->tos_tid)); 3843174641Skmacy struct toepcb *toep = (struct toepcb *)ctx; 3844174641Skmacy struct tcpcb *tp = toep->tp_tp; 3845174641Skmacy struct socket *so; 3846174641Skmacy struct toedev *tdev; 3847174641Skmacy struct tom_data *d; 3848174641Skmacy 3849174641Skmacy if (tp == NULL) { 3850174641Skmacy free_atid(cdev, atid); 3851174641Skmacy return (0); 3852174641Skmacy } 3853177530Skmacy inp_wlock(tp->t_inpcb); 3854178302Skmacy 3855174641Skmacy /* 3856178302Skmacy * XXX 3857178302Skmacy */ 3858178302Skmacy so = inp_inpcbtosocket(tp->t_inpcb); 3859178302Skmacy tdev = toep->tp_toedev; /* blow up here if link was down */ 3860178302Skmacy d = TOM_DATA(tdev); 3861178302Skmacy 3862178302Skmacy /* 3863174641Skmacy * It's OK if the TID is currently in use, the owning socket may have 3864174641Skmacy * backlogged its last CPL message(s). Just take it away. 3865174641Skmacy */ 3866174641Skmacy toep->tp_tid = tid; 3867174641Skmacy toep->tp_tp = tp; 3868178302Skmacy so_insert_tid(d, toep, tid); 3869174641Skmacy free_atid(cdev, atid); 3870174641Skmacy toep->tp_qset = G_QNUM(ntohl(m->m_pkthdr.csum_data)); 3871174641Skmacy 3872174641Skmacy socket_act_establish(so, m); 3873177530Skmacy inp_wunlock(tp->t_inpcb); 3874176472Skmacy CTR1(KTR_TOM, "do_act_establish tid=%u", toep->tp_tid); 3875176472Skmacy cxgb_log_tcb(cdev->adapter, toep->tp_tid); 3876176472Skmacy 3877174641Skmacy return (0); 3878174641Skmacy} 3879174641Skmacy 3880174641Skmacy/* 3881174641Skmacy * Process an acknowledgment of WR completion. Advance snd_una and send the 3882174641Skmacy * next batch of work requests from the write queue. 3883174641Skmacy */ 3884174641Skmacystatic void 3885174641Skmacywr_ack(struct toepcb *toep, struct mbuf *m) 3886174641Skmacy{ 3887174641Skmacy struct tcpcb *tp = toep->tp_tp; 3888174641Skmacy struct cpl_wr_ack *hdr = cplhdr(m); 3889178302Skmacy struct socket *so; 3890174641Skmacy unsigned int credits = ntohs(hdr->credits); 3891174641Skmacy u32 snd_una = ntohl(hdr->snd_una); 3892174641Skmacy int bytes = 0; 3893178302Skmacy struct sockbuf *snd; 3894174641Skmacy 3895176472Skmacy CTR2(KTR_SPARE2, "wr_ack: snd_una=%u credits=%d", snd_una, credits); 3896174641Skmacy 3897177530Skmacy inp_wlock(tp->t_inpcb); 3898178302Skmacy so = inp_inpcbtosocket(tp->t_inpcb); 3899174641Skmacy toep->tp_wr_avail += credits; 3900174641Skmacy if (toep->tp_wr_unacked > toep->tp_wr_max - toep->tp_wr_avail) 3901174641Skmacy toep->tp_wr_unacked = toep->tp_wr_max - toep->tp_wr_avail; 3902174641Skmacy 3903174641Skmacy while (credits) { 3904174641Skmacy struct mbuf *p = peek_wr(toep); 3905174641Skmacy 3906174641Skmacy if (__predict_false(!p)) { 3907174641Skmacy log(LOG_ERR, "%u WR_ACK credits for TID %u with " 3908176472Skmacy "nothing pending, state %u wr_avail=%u\n", 3909176472Skmacy credits, toep->tp_tid, tp->t_state, toep->tp_wr_avail); 3910174641Skmacy break; 3911174641Skmacy } 3912176472Skmacy CTR2(KTR_TOM, 3913178302Skmacy "wr_ack: p->credits=%d p->bytes=%d", 3914178302Skmacy p->m_pkthdr.csum_data, p->m_pkthdr.len); 3915178302Skmacy KASSERT(p->m_pkthdr.csum_data != 0, 3916178302Skmacy ("empty request still on list")); 3917176472Skmacy 3918174641Skmacy if (__predict_false(credits < p->m_pkthdr.csum_data)) { 3919176472Skmacy 3920174641Skmacy#if DEBUG_WR > 1 3921174641Skmacy struct tx_data_wr *w = cplhdr(p); 3922174641Skmacy log(LOG_ERR, 3923174641Skmacy "TID %u got %u WR credits, need %u, len %u, " 3924174641Skmacy "main body %u, frags %u, seq # %u, ACK una %u," 3925174641Skmacy " ACK nxt %u, WR_AVAIL %u, WRs pending %u\n", 3926174641Skmacy toep->tp_tid, credits, p->csum, p->len, 3927174641Skmacy p->len - p->data_len, skb_shinfo(p)->nr_frags, 3928174641Skmacy ntohl(w->sndseq), snd_una, ntohl(hdr->snd_nxt), 3929176472Skmacy toep->tp_wr_avail, count_pending_wrs(tp) - credits); 3930174641Skmacy#endif 3931174641Skmacy p->m_pkthdr.csum_data -= credits; 3932174641Skmacy break; 3933174641Skmacy } else { 3934174641Skmacy dequeue_wr(toep); 3935174641Skmacy credits -= p->m_pkthdr.csum_data; 3936174641Skmacy bytes += p->m_pkthdr.len; 3937176472Skmacy CTR3(KTR_TOM, 3938176472Skmacy "wr_ack: done with wr of %d bytes remain credits=%d wr credits=%d", 3939176472Skmacy p->m_pkthdr.len, credits, p->m_pkthdr.csum_data); 3940174641Skmacy 3941174641Skmacy m_free(p); 3942174641Skmacy } 3943174641Skmacy } 3944174641Skmacy 3945174641Skmacy#if DEBUG_WR 3946174641Skmacy check_wr_invariants(tp); 3947174641Skmacy#endif 3948174641Skmacy 3949174641Skmacy if (__predict_false(SEQ_LT(snd_una, tp->snd_una))) { 3950174641Skmacy#if VALIDATE_SEQ 3951174641Skmacy struct tom_data *d = TOM_DATA(TOE_DEV(so)); 3952174641Skmacy 3953174641Skmacy log(LOG_ERR "%s: unexpected sequence # %u in WR_ACK " 3954174641Skmacy "for TID %u, snd_una %u\n", (&d->tdev)->name, snd_una, 3955174641Skmacy toep->tp_tid, tp->snd_una); 3956174641Skmacy#endif 3957174641Skmacy goto out_free; 3958174641Skmacy } 3959174641Skmacy 3960174641Skmacy if (tp->snd_una != snd_una) { 3961174641Skmacy tp->snd_una = snd_una; 3962174641Skmacy tp->ts_recent_age = ticks; 3963174641Skmacy#ifdef notyet 3964174641Skmacy /* 3965174641Skmacy * Keep ARP entry "minty fresh" 3966174641Skmacy */ 3967174641Skmacy dst_confirm(sk->sk_dst_cache); 3968174641Skmacy#endif 3969174641Skmacy if (tp->snd_una == tp->snd_nxt) 3970174641Skmacy toep->tp_flags &= ~TP_TX_WAIT_IDLE; 3971174641Skmacy } 3972178302Skmacy 3973178302Skmacy snd = so_sockbuf_snd(so); 3974174641Skmacy if (bytes) { 3975176472Skmacy CTR1(KTR_SPARE2, "wr_ack: sbdrop(%d)", bytes); 3976178302Skmacy snd = so_sockbuf_snd(so); 3977178302Skmacy sockbuf_lock(snd); 3978178302Skmacy sbdrop_locked(snd, bytes); 3979178302Skmacy so_sowwakeup_locked(so); 3980174641Skmacy } 3981178302Skmacy 3982178302Skmacy if (snd->sb_sndptroff < snd->sb_cc) 3983174641Skmacy t3_push_frames(so, 0); 3984174641Skmacy 3985174641Skmacyout_free: 3986177530Skmacy inp_wunlock(tp->t_inpcb); 3987174641Skmacy m_free(m); 3988174641Skmacy} 3989174641Skmacy 3990174641Skmacy/* 3991174641Skmacy * Handler for TX_DATA_ACK CPL messages. 3992174641Skmacy */ 3993174641Skmacystatic int 3994174641Skmacydo_wr_ack(struct t3cdev *dev, struct mbuf *m, void *ctx) 3995174641Skmacy{ 3996174641Skmacy struct toepcb *toep = (struct toepcb *)ctx; 3997174641Skmacy 3998174641Skmacy VALIDATE_SOCK(so); 3999174641Skmacy 4000174641Skmacy wr_ack(toep, m); 4001174641Skmacy return 0; 4002174641Skmacy} 4003174641Skmacy 4004176472Skmacy/* 4005176472Skmacy * Handler for TRACE_PKT CPL messages. Just sink these packets. 4006176472Skmacy */ 4007176472Skmacystatic int 4008176472Skmacydo_trace_pkt(struct t3cdev *dev, struct mbuf *m, void *ctx) 4009176472Skmacy{ 4010176472Skmacy m_freem(m); 4011176472Skmacy return 0; 4012176472Skmacy} 4013174641Skmacy 4014174641Skmacy/* 4015174641Skmacy * Reset a connection that is on a listener's SYN queue or accept queue, 4016174641Skmacy * i.e., one that has not had a struct socket associated with it. 4017174641Skmacy * Must be called from process context. 4018174641Skmacy * 4019174641Skmacy * Modeled after code in inet_csk_listen_stop(). 4020174641Skmacy */ 4021174641Skmacystatic void 4022174641Skmacyt3_reset_listen_child(struct socket *child) 4023174641Skmacy{ 4024178302Skmacy struct tcpcb *tp = so_sototcpcb(child); 4025174641Skmacy 4026174641Skmacy t3_send_reset(tp->t_toe); 4027174641Skmacy} 4028174641Skmacy 4029178302Skmacy 4030178302Skmacystatic void 4031178302Skmacyt3_child_disconnect(struct socket *so, void *arg) 4032178302Skmacy{ 4033178302Skmacy struct tcpcb *tp = so_sototcpcb(so); 4034178302Skmacy 4035178302Skmacy if (tp->t_flags & TF_TOE) { 4036178302Skmacy inp_wlock(tp->t_inpcb); 4037178302Skmacy t3_reset_listen_child(so); 4038178302Skmacy inp_wunlock(tp->t_inpcb); 4039178302Skmacy } 4040178302Skmacy} 4041178302Skmacy 4042174641Skmacy/* 4043174641Skmacy * Disconnect offloaded established but not yet accepted connections sitting 4044174641Skmacy * on a server's accept_queue. We just send an ABORT_REQ at this point and 4045174641Skmacy * finish off the disconnect later as we may need to wait for the ABORT_RPL. 4046174641Skmacy */ 4047174641Skmacyvoid 4048174641Skmacyt3_disconnect_acceptq(struct socket *listen_so) 4049174641Skmacy{ 4050174641Skmacy 4051178302Skmacy so_lock(listen_so); 4052178302Skmacy so_listeners_apply_all(listen_so, t3_child_disconnect, NULL); 4053178302Skmacy so_unlock(listen_so); 4054174641Skmacy} 4055174641Skmacy 4056174641Skmacy/* 4057174641Skmacy * Reset offloaded connections sitting on a server's syn queue. As above 4058174641Skmacy * we send ABORT_REQ and finish off when we get ABORT_RPL. 4059174641Skmacy */ 4060174641Skmacy 4061174641Skmacyvoid 4062174641Skmacyt3_reset_synq(struct listen_ctx *lctx) 4063174641Skmacy{ 4064174641Skmacy struct toepcb *toep; 4065174641Skmacy 4066178302Skmacy so_lock(lctx->lso); 4067174641Skmacy while (!LIST_EMPTY(&lctx->synq_head)) { 4068174641Skmacy toep = LIST_FIRST(&lctx->synq_head); 4069174641Skmacy LIST_REMOVE(toep, synq_entry); 4070174641Skmacy toep->tp_tp = NULL; 4071174641Skmacy t3_send_reset(toep); 4072174641Skmacy cxgb_remove_tid(TOEP_T3C_DEV(toep), toep, toep->tp_tid); 4073174641Skmacy toepcb_release(toep); 4074174641Skmacy } 4075178302Skmacy so_unlock(lctx->lso); 4076174641Skmacy} 4077174641Skmacy 4078176472Skmacy 4079176472Skmacyint 4080178302Skmacyt3_setup_ppods(struct toepcb *toep, const struct ddp_gather_list *gl, 4081176472Skmacy unsigned int nppods, unsigned int tag, unsigned int maxoff, 4082176472Skmacy unsigned int pg_off, unsigned int color) 4083176472Skmacy{ 4084176472Skmacy unsigned int i, j, pidx; 4085176472Skmacy struct pagepod *p; 4086176472Skmacy struct mbuf *m; 4087176472Skmacy struct ulp_mem_io *req; 4088176472Skmacy unsigned int tid = toep->tp_tid; 4089178302Skmacy const struct tom_data *td = TOM_DATA(toep->tp_toedev); 4090176472Skmacy unsigned int ppod_addr = tag * PPOD_SIZE + td->ddp_llimit; 4091176472Skmacy 4092176472Skmacy CTR6(KTR_TOM, "t3_setup_ppods(gl=%p nppods=%u tag=%u maxoff=%u pg_off=%u color=%u)", 4093176472Skmacy gl, nppods, tag, maxoff, pg_off, color); 4094176472Skmacy 4095176472Skmacy for (i = 0; i < nppods; ++i) { 4096176472Skmacy m = m_gethdr_nofail(sizeof(*req) + PPOD_SIZE); 4097176472Skmacy m_set_priority(m, mkprio(CPL_PRIORITY_CONTROL, toep)); 4098176472Skmacy req = mtod(m, struct ulp_mem_io *); 4099176472Skmacy m->m_pkthdr.len = m->m_len = sizeof(*req) + PPOD_SIZE; 4100176472Skmacy req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_BYPASS)); 4101176472Skmacy req->wr.wr_lo = 0; 4102176472Skmacy req->cmd_lock_addr = htonl(V_ULP_MEMIO_ADDR(ppod_addr >> 5) | 4103176472Skmacy V_ULPTX_CMD(ULP_MEM_WRITE)); 4104176472Skmacy req->len = htonl(V_ULP_MEMIO_DATA_LEN(PPOD_SIZE / 32) | 4105176472Skmacy V_ULPTX_NFLITS(PPOD_SIZE / 8 + 1)); 4106176472Skmacy 4107176472Skmacy p = (struct pagepod *)(req + 1); 4108176472Skmacy if (__predict_false(i < nppods - NUM_SENTINEL_PPODS)) { 4109176472Skmacy p->pp_vld_tid = htonl(F_PPOD_VALID | V_PPOD_TID(tid)); 4110176472Skmacy p->pp_pgsz_tag_color = htonl(V_PPOD_TAG(tag) | 4111176472Skmacy V_PPOD_COLOR(color)); 4112176472Skmacy p->pp_max_offset = htonl(maxoff); 4113176472Skmacy p->pp_page_offset = htonl(pg_off); 4114176472Skmacy p->pp_rsvd = 0; 4115176472Skmacy for (pidx = 4 * i, j = 0; j < 5; ++j, ++pidx) 4116176472Skmacy p->pp_addr[j] = pidx < gl->dgl_nelem ? 4117176472Skmacy htobe64(VM_PAGE_TO_PHYS(gl->dgl_pages[pidx])) : 0; 4118176472Skmacy } else 4119176472Skmacy p->pp_vld_tid = 0; /* mark sentinel page pods invalid */ 4120176472Skmacy send_or_defer(toep, m, 0); 4121176472Skmacy ppod_addr += PPOD_SIZE; 4122176472Skmacy } 4123176472Skmacy return (0); 4124176472Skmacy} 4125176472Skmacy 4126176472Skmacy/* 4127176472Skmacy * Build a CPL_BARRIER message as payload of a ULP_TX_PKT command. 4128176472Skmacy */ 4129176472Skmacystatic inline void 4130176472Skmacymk_cpl_barrier_ulp(struct cpl_barrier *b) 4131176472Skmacy{ 4132176472Skmacy struct ulp_txpkt *txpkt = (struct ulp_txpkt *)b; 4133176472Skmacy 4134176472Skmacy txpkt->cmd_dest = htonl(V_ULPTX_CMD(ULP_TXPKT)); 4135176472Skmacy txpkt->len = htonl(V_ULPTX_NFLITS(sizeof(*b) / 8)); 4136176472Skmacy b->opcode = CPL_BARRIER; 4137176472Skmacy} 4138176472Skmacy 4139176472Skmacy/* 4140176472Skmacy * Build a CPL_GET_TCB message as payload of a ULP_TX_PKT command. 4141176472Skmacy */ 4142176472Skmacystatic inline void 4143176472Skmacymk_get_tcb_ulp(struct cpl_get_tcb *req, unsigned int tid, unsigned int cpuno) 4144176472Skmacy{ 4145176472Skmacy struct ulp_txpkt *txpkt = (struct ulp_txpkt *)req; 4146176472Skmacy 4147176472Skmacy txpkt = (struct ulp_txpkt *)req; 4148176472Skmacy txpkt->cmd_dest = htonl(V_ULPTX_CMD(ULP_TXPKT)); 4149176472Skmacy txpkt->len = htonl(V_ULPTX_NFLITS(sizeof(*req) / 8)); 4150176472Skmacy OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_GET_TCB, tid)); 4151176472Skmacy req->cpuno = htons(cpuno); 4152176472Skmacy} 4153176472Skmacy 4154176472Skmacy/* 4155176472Skmacy * Build a CPL_SET_TCB_FIELD message as payload of a ULP_TX_PKT command. 4156176472Skmacy */ 4157176472Skmacystatic inline void 4158176472Skmacymk_set_tcb_field_ulp(struct cpl_set_tcb_field *req, unsigned int tid, 4159176472Skmacy unsigned int word, uint64_t mask, uint64_t val) 4160176472Skmacy{ 4161176472Skmacy struct ulp_txpkt *txpkt = (struct ulp_txpkt *)req; 4162176472Skmacy 4163176472Skmacy CTR4(KTR_TCB, "mk_set_tcb_field_ulp(tid=%u word=0x%x mask=%jx val=%jx", 4164176472Skmacy tid, word, mask, val); 4165176472Skmacy 4166176472Skmacy txpkt->cmd_dest = htonl(V_ULPTX_CMD(ULP_TXPKT)); 4167176472Skmacy txpkt->len = htonl(V_ULPTX_NFLITS(sizeof(*req) / 8)); 4168176472Skmacy OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid)); 4169176472Skmacy req->reply = V_NO_REPLY(1); 4170176472Skmacy req->cpu_idx = 0; 4171176472Skmacy req->word = htons(word); 4172176472Skmacy req->mask = htobe64(mask); 4173176472Skmacy req->val = htobe64(val); 4174176472Skmacy} 4175176472Skmacy 4176176472Skmacy/* 4177176472Skmacy * Build a CPL_RX_DATA_ACK message as payload of a ULP_TX_PKT command. 4178176472Skmacy */ 4179176472Skmacystatic void 4180178302Skmacymk_rx_data_ack_ulp(struct toepcb *toep, struct cpl_rx_data_ack *ack, 4181177340Skmacy unsigned int tid, unsigned int credits) 4182176472Skmacy{ 4183176472Skmacy struct ulp_txpkt *txpkt = (struct ulp_txpkt *)ack; 4184176472Skmacy 4185176472Skmacy txpkt->cmd_dest = htonl(V_ULPTX_CMD(ULP_TXPKT)); 4186176472Skmacy txpkt->len = htonl(V_ULPTX_NFLITS(sizeof(*ack) / 8)); 4187176472Skmacy OPCODE_TID(ack) = htonl(MK_OPCODE_TID(CPL_RX_DATA_ACK, tid)); 4188176472Skmacy ack->credit_dack = htonl(F_RX_MODULATE | F_RX_DACK_CHANGE | 4189178302Skmacy V_RX_DACK_MODE(TOM_TUNABLE(toep->tp_toedev, delack)) | 4190177340Skmacy V_RX_CREDITS(credits)); 4191176472Skmacy} 4192176472Skmacy 4193174641Skmacyvoid 4194176472Skmacyt3_cancel_ddpbuf(struct toepcb *toep, unsigned int bufidx) 4195176472Skmacy{ 4196176472Skmacy unsigned int wrlen; 4197176472Skmacy struct mbuf *m; 4198176472Skmacy struct work_request_hdr *wr; 4199176472Skmacy struct cpl_barrier *lock; 4200176472Skmacy struct cpl_set_tcb_field *req; 4201176472Skmacy struct cpl_get_tcb *getreq; 4202176472Skmacy struct ddp_state *p = &toep->tp_ddp_state; 4203176472Skmacy 4204178302Skmacy#if 0 4205176472Skmacy SOCKBUF_LOCK_ASSERT(&toeptoso(toep)->so_rcv); 4206178302Skmacy#endif 4207176472Skmacy wrlen = sizeof(*wr) + sizeof(*req) + 2 * sizeof(*lock) + 4208176472Skmacy sizeof(*getreq); 4209176472Skmacy m = m_gethdr_nofail(wrlen); 4210176472Skmacy m_set_priority(m, mkprio(CPL_PRIORITY_CONTROL, toep)); 4211176472Skmacy wr = mtod(m, struct work_request_hdr *); 4212176472Skmacy bzero(wr, wrlen); 4213176472Skmacy 4214176472Skmacy wr->wr_hi = htonl(V_WR_OP(FW_WROPCODE_BYPASS)); 4215176472Skmacy m->m_pkthdr.len = m->m_len = wrlen; 4216176472Skmacy 4217176472Skmacy lock = (struct cpl_barrier *)(wr + 1); 4218176472Skmacy mk_cpl_barrier_ulp(lock); 4219176472Skmacy 4220176472Skmacy req = (struct cpl_set_tcb_field *)(lock + 1); 4221176472Skmacy 4222176472Skmacy CTR1(KTR_TCB, "t3_cancel_ddpbuf(bufidx=%u)", bufidx); 4223176472Skmacy 4224176472Skmacy /* Hmmm, not sure if this actually a good thing: reactivating 4225176472Skmacy * the other buffer might be an issue if it has been completed 4226176472Skmacy * already. However, that is unlikely, since the fact that the UBUF 4227176472Skmacy * is not completed indicates that there is no oustanding data. 4228176472Skmacy */ 4229176472Skmacy if (bufidx == 0) 4230176472Skmacy mk_set_tcb_field_ulp(req, toep->tp_tid, W_TCB_RX_DDP_FLAGS, 4231176472Skmacy V_TF_DDP_ACTIVE_BUF(1) | 4232176472Skmacy V_TF_DDP_BUF0_VALID(1), 4233176472Skmacy V_TF_DDP_ACTIVE_BUF(1)); 4234176472Skmacy else 4235176472Skmacy mk_set_tcb_field_ulp(req, toep->tp_tid, W_TCB_RX_DDP_FLAGS, 4236176472Skmacy V_TF_DDP_ACTIVE_BUF(1) | 4237176472Skmacy V_TF_DDP_BUF1_VALID(1), 0); 4238176472Skmacy 4239176472Skmacy getreq = (struct cpl_get_tcb *)(req + 1); 4240176472Skmacy mk_get_tcb_ulp(getreq, toep->tp_tid, toep->tp_qset); 4241176472Skmacy 4242176472Skmacy mk_cpl_barrier_ulp((struct cpl_barrier *)(getreq + 1)); 4243176472Skmacy 4244176472Skmacy /* Keep track of the number of oustanding CPL_GET_TCB requests 4245176472Skmacy */ 4246176472Skmacy p->get_tcb_count++; 4247176472Skmacy 4248176472Skmacy#ifdef T3_TRACE 4249176472Skmacy T3_TRACE1(TIDTB(so), 4250176472Skmacy "t3_cancel_ddpbuf: bufidx %u", bufidx); 4251176472Skmacy#endif 4252176472Skmacy cxgb_ofld_send(TOEP_T3C_DEV(toep), m); 4253176472Skmacy} 4254176472Skmacy 4255176472Skmacy/** 4256176472Skmacy * t3_overlay_ddpbuf - overlay an existing DDP buffer with a new one 4257176472Skmacy * @sk: the socket associated with the buffers 4258176472Skmacy * @bufidx: index of HW DDP buffer (0 or 1) 4259176472Skmacy * @tag0: new tag for HW buffer 0 4260176472Skmacy * @tag1: new tag for HW buffer 1 4261176472Skmacy * @len: new length for HW buf @bufidx 4262176472Skmacy * 4263176472Skmacy * Sends a compound WR to overlay a new DDP buffer on top of an existing 4264176472Skmacy * buffer by changing the buffer tag and length and setting the valid and 4265176472Skmacy * active flag accordingly. The caller must ensure the new buffer is at 4266176472Skmacy * least as big as the existing one. Since we typically reprogram both HW 4267176472Skmacy * buffers this function sets both tags for convenience. Read the TCB to 4268176472Skmacy * determine how made data was written into the buffer before the overlay 4269176472Skmacy * took place. 4270176472Skmacy */ 4271176472Skmacyvoid 4272176472Skmacyt3_overlay_ddpbuf(struct toepcb *toep, unsigned int bufidx, unsigned int tag0, 4273176472Skmacy unsigned int tag1, unsigned int len) 4274176472Skmacy{ 4275176472Skmacy unsigned int wrlen; 4276176472Skmacy struct mbuf *m; 4277176472Skmacy struct work_request_hdr *wr; 4278176472Skmacy struct cpl_get_tcb *getreq; 4279176472Skmacy struct cpl_set_tcb_field *req; 4280176472Skmacy struct ddp_state *p = &toep->tp_ddp_state; 4281176472Skmacy 4282176472Skmacy CTR4(KTR_TCB, "t3_setup_ppods(bufidx=%u tag0=%u tag1=%u len=%u)", 4283176472Skmacy bufidx, tag0, tag1, len); 4284178302Skmacy#if 0 4285176472Skmacy SOCKBUF_LOCK_ASSERT(&toeptoso(toep)->so_rcv); 4286178302Skmacy#endif 4287176472Skmacy wrlen = sizeof(*wr) + 3 * sizeof(*req) + sizeof(*getreq); 4288176472Skmacy m = m_gethdr_nofail(wrlen); 4289176472Skmacy m_set_priority(m, mkprio(CPL_PRIORITY_CONTROL, toep)); 4290176472Skmacy wr = mtod(m, struct work_request_hdr *); 4291176472Skmacy m->m_pkthdr.len = m->m_len = wrlen; 4292176472Skmacy bzero(wr, wrlen); 4293176472Skmacy 4294176472Skmacy 4295176472Skmacy /* Set the ATOMIC flag to make sure that TP processes the following 4296176472Skmacy * CPLs in an atomic manner and no wire segments can be interleaved. 4297176472Skmacy */ 4298176472Skmacy wr->wr_hi = htonl(V_WR_OP(FW_WROPCODE_BYPASS) | F_WR_ATOMIC); 4299176472Skmacy req = (struct cpl_set_tcb_field *)(wr + 1); 4300176472Skmacy mk_set_tcb_field_ulp(req, toep->tp_tid, W_TCB_RX_DDP_BUF0_TAG, 4301176472Skmacy V_TCB_RX_DDP_BUF0_TAG(M_TCB_RX_DDP_BUF0_TAG) | 4302176472Skmacy V_TCB_RX_DDP_BUF1_TAG(M_TCB_RX_DDP_BUF1_TAG) << 32, 4303176472Skmacy V_TCB_RX_DDP_BUF0_TAG(tag0) | 4304176472Skmacy V_TCB_RX_DDP_BUF1_TAG((uint64_t)tag1) << 32); 4305176472Skmacy req++; 4306176472Skmacy if (bufidx == 0) { 4307176472Skmacy mk_set_tcb_field_ulp(req, toep->tp_tid, W_TCB_RX_DDP_BUF0_LEN, 4308176472Skmacy V_TCB_RX_DDP_BUF0_LEN(M_TCB_RX_DDP_BUF0_LEN), 4309176472Skmacy V_TCB_RX_DDP_BUF0_LEN((uint64_t)len)); 4310176472Skmacy req++; 4311176472Skmacy mk_set_tcb_field_ulp(req, toep->tp_tid, W_TCB_RX_DDP_FLAGS, 4312176472Skmacy V_TF_DDP_PUSH_DISABLE_0(1) | 4313176472Skmacy V_TF_DDP_BUF0_VALID(1) | V_TF_DDP_ACTIVE_BUF(1), 4314176472Skmacy V_TF_DDP_PUSH_DISABLE_0(0) | 4315176472Skmacy V_TF_DDP_BUF0_VALID(1)); 4316176472Skmacy } else { 4317176472Skmacy mk_set_tcb_field_ulp(req, toep->tp_tid, W_TCB_RX_DDP_BUF1_LEN, 4318176472Skmacy V_TCB_RX_DDP_BUF1_LEN(M_TCB_RX_DDP_BUF1_LEN), 4319176472Skmacy V_TCB_RX_DDP_BUF1_LEN((uint64_t)len)); 4320176472Skmacy req++; 4321176472Skmacy mk_set_tcb_field_ulp(req, toep->tp_tid, W_TCB_RX_DDP_FLAGS, 4322176472Skmacy V_TF_DDP_PUSH_DISABLE_1(1) | 4323176472Skmacy V_TF_DDP_BUF1_VALID(1) | V_TF_DDP_ACTIVE_BUF(1), 4324176472Skmacy V_TF_DDP_PUSH_DISABLE_1(0) | 4325176472Skmacy V_TF_DDP_BUF1_VALID(1) | V_TF_DDP_ACTIVE_BUF(1)); 4326176472Skmacy } 4327176472Skmacy 4328176472Skmacy getreq = (struct cpl_get_tcb *)(req + 1); 4329176472Skmacy mk_get_tcb_ulp(getreq, toep->tp_tid, toep->tp_qset); 4330176472Skmacy 4331176472Skmacy /* Keep track of the number of oustanding CPL_GET_TCB requests 4332176472Skmacy */ 4333176472Skmacy p->get_tcb_count++; 4334176472Skmacy 4335176472Skmacy#ifdef T3_TRACE 4336176472Skmacy T3_TRACE4(TIDTB(sk), 4337176472Skmacy "t3_overlay_ddpbuf: bufidx %u tag0 %u tag1 %u " 4338176472Skmacy "len %d", 4339176472Skmacy bufidx, tag0, tag1, len); 4340176472Skmacy#endif 4341176472Skmacy cxgb_ofld_send(TOEP_T3C_DEV(toep), m); 4342176472Skmacy} 4343176472Skmacy 4344176472Skmacy/* 4345176472Skmacy * Sends a compound WR containing all the CPL messages needed to program the 4346176472Skmacy * two HW DDP buffers, namely optionally setting up the length and offset of 4347176472Skmacy * each buffer, programming the DDP flags, and optionally sending RX_DATA_ACK. 4348176472Skmacy */ 4349176472Skmacyvoid 4350176472Skmacyt3_setup_ddpbufs(struct toepcb *toep, unsigned int len0, unsigned int offset0, 4351176472Skmacy unsigned int len1, unsigned int offset1, 4352176472Skmacy uint64_t ddp_flags, uint64_t flag_mask, int modulate) 4353176472Skmacy{ 4354176472Skmacy unsigned int wrlen; 4355176472Skmacy struct mbuf *m; 4356176472Skmacy struct work_request_hdr *wr; 4357176472Skmacy struct cpl_set_tcb_field *req; 4358176472Skmacy 4359176472Skmacy CTR6(KTR_TCB, "t3_setup_ddpbufs(len0=%u offset0=%u len1=%u offset1=%u ddp_flags=0x%08x%08x ", 4360176472Skmacy len0, offset0, len1, offset1, ddp_flags >> 32, ddp_flags & 0xffffffff); 4361176472Skmacy 4362178302Skmacy#if 0 4363176472Skmacy SOCKBUF_LOCK_ASSERT(&toeptoso(toep)->so_rcv); 4364178302Skmacy#endif 4365176472Skmacy wrlen = sizeof(*wr) + sizeof(*req) + (len0 ? sizeof(*req) : 0) + 4366176472Skmacy (len1 ? sizeof(*req) : 0) + 4367176472Skmacy (modulate ? sizeof(struct cpl_rx_data_ack) : 0); 4368176472Skmacy m = m_gethdr_nofail(wrlen); 4369176472Skmacy m_set_priority(m, mkprio(CPL_PRIORITY_CONTROL, toep)); 4370176472Skmacy wr = mtod(m, struct work_request_hdr *); 4371176472Skmacy bzero(wr, wrlen); 4372176472Skmacy 4373176472Skmacy wr->wr_hi = htonl(V_WR_OP(FW_WROPCODE_BYPASS)); 4374176472Skmacy m->m_pkthdr.len = m->m_len = wrlen; 4375176472Skmacy 4376176472Skmacy req = (struct cpl_set_tcb_field *)(wr + 1); 4377176472Skmacy if (len0) { /* program buffer 0 offset and length */ 4378176472Skmacy mk_set_tcb_field_ulp(req, toep->tp_tid, W_TCB_RX_DDP_BUF0_OFFSET, 4379176472Skmacy V_TCB_RX_DDP_BUF0_OFFSET(M_TCB_RX_DDP_BUF0_OFFSET) | 4380176472Skmacy V_TCB_RX_DDP_BUF0_LEN(M_TCB_RX_DDP_BUF0_LEN), 4381176472Skmacy V_TCB_RX_DDP_BUF0_OFFSET((uint64_t)offset0) | 4382176472Skmacy V_TCB_RX_DDP_BUF0_LEN((uint64_t)len0)); 4383176472Skmacy req++; 4384176472Skmacy } 4385176472Skmacy if (len1) { /* program buffer 1 offset and length */ 4386176472Skmacy mk_set_tcb_field_ulp(req, toep->tp_tid, W_TCB_RX_DDP_BUF1_OFFSET, 4387176472Skmacy V_TCB_RX_DDP_BUF1_OFFSET(M_TCB_RX_DDP_BUF1_OFFSET) | 4388176472Skmacy V_TCB_RX_DDP_BUF1_LEN(M_TCB_RX_DDP_BUF1_LEN) << 32, 4389176472Skmacy V_TCB_RX_DDP_BUF1_OFFSET((uint64_t)offset1) | 4390176472Skmacy V_TCB_RX_DDP_BUF1_LEN((uint64_t)len1) << 32); 4391176472Skmacy req++; 4392176472Skmacy } 4393176472Skmacy 4394176472Skmacy mk_set_tcb_field_ulp(req, toep->tp_tid, W_TCB_RX_DDP_FLAGS, flag_mask, 4395176472Skmacy ddp_flags); 4396176472Skmacy 4397176472Skmacy if (modulate) { 4398178302Skmacy mk_rx_data_ack_ulp(toep, 4399177340Skmacy (struct cpl_rx_data_ack *)(req + 1), toep->tp_tid, 4400177340Skmacy toep->tp_copied_seq - toep->tp_rcv_wup); 4401176472Skmacy toep->tp_rcv_wup = toep->tp_copied_seq; 4402176472Skmacy } 4403176472Skmacy 4404176472Skmacy#ifdef T3_TRACE 4405176472Skmacy T3_TRACE5(TIDTB(sk), 4406176472Skmacy "t3_setup_ddpbufs: len0 %u len1 %u ddp_flags 0x%08x%08x " 4407176472Skmacy "modulate %d", 4408176472Skmacy len0, len1, ddp_flags >> 32, ddp_flags & 0xffffffff, 4409176472Skmacy modulate); 4410176472Skmacy#endif 4411176472Skmacy 4412176472Skmacy cxgb_ofld_send(TOEP_T3C_DEV(toep), m); 4413176472Skmacy} 4414176472Skmacy 4415176472Skmacyvoid 4416174641Skmacyt3_init_wr_tab(unsigned int wr_len) 4417174641Skmacy{ 4418174641Skmacy int i; 4419174641Skmacy 4420174641Skmacy if (mbuf_wrs[1]) /* already initialized */ 4421174641Skmacy return; 4422174641Skmacy 4423174641Skmacy for (i = 1; i < ARRAY_SIZE(mbuf_wrs); i++) { 4424174641Skmacy int sgl_len = (3 * i) / 2 + (i & 1); 4425174641Skmacy 4426174641Skmacy sgl_len += 3; 4427174641Skmacy mbuf_wrs[i] = sgl_len <= wr_len ? 4428174641Skmacy 1 : 1 + (sgl_len - 2) / (wr_len - 1); 4429174641Skmacy } 4430174641Skmacy 4431174641Skmacy wrlen = wr_len * 8; 4432174641Skmacy} 4433174641Skmacy 4434174641Skmacyint 4435174641Skmacyt3_init_cpl_io(void) 4436174641Skmacy{ 4437174641Skmacy#ifdef notyet 4438174641Skmacy tcphdr_skb = alloc_skb(sizeof(struct tcphdr), GFP_KERNEL); 4439174641Skmacy if (!tcphdr_skb) { 4440174641Skmacy log(LOG_ERR, 4441174641Skmacy "Chelsio TCP offload: can't allocate sk_buff\n"); 4442174641Skmacy return -1; 4443174641Skmacy } 4444174641Skmacy skb_put(tcphdr_skb, sizeof(struct tcphdr)); 4445174641Skmacy tcphdr_skb->h.raw = tcphdr_skb->data; 4446174641Skmacy memset(tcphdr_skb->data, 0, tcphdr_skb->len); 4447174641Skmacy#endif 4448174641Skmacy 4449174641Skmacy t3tom_register_cpl_handler(CPL_ACT_ESTABLISH, do_act_establish); 4450174641Skmacy t3tom_register_cpl_handler(CPL_ACT_OPEN_RPL, do_act_open_rpl); 4451174641Skmacy t3tom_register_cpl_handler(CPL_TX_DMA_ACK, do_wr_ack); 4452174641Skmacy t3tom_register_cpl_handler(CPL_RX_DATA, do_rx_data); 4453174641Skmacy t3tom_register_cpl_handler(CPL_CLOSE_CON_RPL, do_close_con_rpl); 4454174641Skmacy t3tom_register_cpl_handler(CPL_PEER_CLOSE, do_peer_close); 4455174641Skmacy t3tom_register_cpl_handler(CPL_PASS_ESTABLISH, do_pass_establish); 4456174641Skmacy t3tom_register_cpl_handler(CPL_PASS_ACCEPT_REQ, do_pass_accept_req); 4457174641Skmacy t3tom_register_cpl_handler(CPL_ABORT_REQ_RSS, do_abort_req); 4458174641Skmacy t3tom_register_cpl_handler(CPL_ABORT_RPL_RSS, do_abort_rpl); 4459174641Skmacy t3tom_register_cpl_handler(CPL_RX_DATA_DDP, do_rx_data_ddp); 4460174641Skmacy t3tom_register_cpl_handler(CPL_RX_DDP_COMPLETE, do_rx_ddp_complete); 4461174641Skmacy t3tom_register_cpl_handler(CPL_RX_URG_NOTIFY, do_rx_urg_notify); 4462174641Skmacy t3tom_register_cpl_handler(CPL_TRACE_PKT, do_trace_pkt); 4463174641Skmacy t3tom_register_cpl_handler(CPL_GET_TCB_RPL, do_get_tcb_rpl); 4464174641Skmacy return (0); 4465174641Skmacy} 4466174641Skmacy 4467