1238423Sjhb/*- 2238423Sjhb * Copyright (c) 2012, 2015 Chelsio Communications, Inc. 3260650Sjhb * All rights reserved. 4238423Sjhb * Written by: Navdeep Parhar <np@FreeBSD.org> 5238423Sjhb * 6238423Sjhb * Redistribution and use in source and binary forms, with or without 7238423Sjhb * modification, are permitted provided that the following conditions 8238423Sjhb * are met: 9238423Sjhb * 1. Redistributions of source code must retain the above copyright 10238423Sjhb * notice, this list of conditions and the following disclaimer. 11238423Sjhb * 2. Redistributions in binary form must reproduce the above copyright 12238423Sjhb * notice, this list of conditions and the following disclaimer in the 13238423Sjhb * documentation and/or other materials provided with the distribution. 14238423Sjhb * 15238423Sjhb * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16238423Sjhb * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17238423Sjhb * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18238423Sjhb * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19238423Sjhb * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20238423Sjhb * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21238423Sjhb * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22238423Sjhb * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23238423Sjhb * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24238423Sjhb * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25238423Sjhb * SUCH DAMAGE. 26238423Sjhb * 27238423Sjhb * $FreeBSD: stable/10/sys/dev/cxgbe/tom/t4_tom.h 318804 2017-05-24 20:01:12Z np $ 28238423Sjhb * 29238423Sjhb */ 30238423Sjhb 31238423Sjhb#ifndef __T4_TOM_H__ 32238423Sjhb#define __T4_TOM_H__ 33238423Sjhb#include <sys/vmem.h> 34238423Sjhb 35238423Sjhb/* 36238423Sjhb * Inline version of mbufq for use on 10.x. Borrowed from 37238423Sjhb * sys/cam/ctl/ctl_ha.c. 38238423Sjhb */ 39238423Sjhbstruct mbufq { 40238423Sjhb struct mbuf *head; 41238423Sjhb struct mbuf *tail; 42238423Sjhb}; 43238423Sjhb 44238423Sjhb#define LISTEN_HASH_SIZE 32 45238423Sjhb 46238423Sjhb/* 47238423Sjhb * Min receive window. We want it to be large enough to accommodate receive 48238423Sjhb * coalescing, handle jumbo frames, and not trigger sender SWS avoidance. 49238423Sjhb */ 50238423Sjhb#define MIN_RCV_WND (24 * 1024U) 51238423Sjhb 52238423Sjhb/* 53238423Sjhb * Max receive window supported by HW in bytes. Only a small part of it can 54238423Sjhb * be set through option0, the rest needs to be set through RX_DATA_ACK. 55238423Sjhb */ 56238423Sjhb#define MAX_RCV_WND ((1U << 27) - 1) 57238423Sjhb 58238423Sjhb#define DDP_RSVD_WIN (16 * 1024U) 59238423Sjhb#define SB_DDP_INDICATE SB_IN_TOE /* soreceive must respond to indicate */ 60238423Sjhb 61238423Sjhb#define USE_DDP_RX_FLOW_CONTROL 62238423Sjhb 63238423Sjhb#define PPOD_SZ(n) ((n) * sizeof(struct pagepod)) 64260650Sjhb#define PPOD_SIZE (PPOD_SZ(1)) 65260650Sjhb 66260650Sjhb/* TOE PCB flags */ 67238423Sjhbenum { 68238423Sjhb TPF_ATTACHED = (1 << 0), /* a tcpcb refers to this toepcb */ 69238423Sjhb TPF_FLOWC_WR_SENT = (1 << 1), /* firmware flow context WR sent */ 70238423Sjhb TPF_TX_DATA_SENT = (1 << 2), /* some data sent */ 71238423Sjhb TPF_TX_SUSPENDED = (1 << 3), /* tx suspended for lack of resources */ 72260650Sjhb TPF_SEND_FIN = (1 << 4), /* send FIN after all pending data */ 73238423Sjhb TPF_FIN_SENT = (1 << 5), /* FIN has been sent */ 74238423Sjhb TPF_ABORT_SHUTDOWN = (1 << 6), /* connection abort is in progress */ 75238423Sjhb TPF_CPL_PENDING = (1 << 7), /* haven't received the last CPL */ 76238423Sjhb TPF_SYNQE = (1 << 8), /* synq_entry, not really a toepcb */ 77238423Sjhb TPF_SYNQE_NEEDFREE = (1 << 9), /* synq_entry was malloc'd separately */ 78238423Sjhb TPF_SYNQE_TCPDDP = (1 << 10), /* ulp_mode TCPDDP in toepcb */ 79238423Sjhb TPF_SYNQE_EXPANDED = (1 << 11), /* toepcb ready, tid context updated */ 80238423Sjhb TPF_SYNQE_HAS_L2TE = (1 << 12), /* we've replied to PASS_ACCEPT_REQ */ 81238423Sjhb}; 82238423Sjhb 83238423Sjhbenum { 84238423Sjhb DDP_OK = (1 << 0), /* OK to turn on DDP */ 85238423Sjhb DDP_SC_REQ = (1 << 1), /* state change (on/off) requested */ 86238423Sjhb DDP_ON = (1 << 2), /* DDP is turned on */ 87238423Sjhb DDP_BUF0_ACTIVE = (1 << 3), /* buffer 0 in use (not invalidated) */ 88238423Sjhb DDP_BUF1_ACTIVE = (1 << 4), /* buffer 1 in use (not invalidated) */ 89238423Sjhb}; 90238423Sjhb 91238423Sjhbstruct ofld_tx_sdesc { 92238423Sjhb uint32_t plen; /* payload length */ 93238423Sjhb uint8_t tx_credits; /* firmware tx credits (unit is 16B) */ 94238423Sjhb}; 95238423Sjhb 96238423Sjhbstruct ppod_region { 97238423Sjhb u_int pr_start; 98238423Sjhb u_int pr_len; 99238423Sjhb u_int pr_page_shift[4]; 100238423Sjhb uint32_t pr_tag_mask; /* hardware tagmask for this region. */ 101238423Sjhb uint32_t pr_invalid_bit; /* OR with this to invalidate tag. */ 102238423Sjhb uint32_t pr_alias_mask; /* AND with tag to get alias bits. */ 103238423Sjhb u_int pr_alias_shift; /* shift this much for first alias bit. */ 104238423Sjhb vmem_t *pr_arena; 105238423Sjhb}; 106238423Sjhb 107238423Sjhbstruct ppod_reservation { 108238423Sjhb struct ppod_region *prsv_pr; 109238423Sjhb uint32_t prsv_tag; /* Full tag: pgsz, alias, tag, color */ 110238423Sjhb u_int prsv_nppods; 111238423Sjhb}; 112238423Sjhb 113238423Sjhbstruct ddp_buffer { 114238423Sjhb int offset; 115238423Sjhb int len; 116238423Sjhb int npages; 117238423Sjhb vm_page_t *pages; 118238423Sjhb struct ppod_reservation prsv; 119238423Sjhb}; 120238423Sjhb 121238423Sjhbstruct toepcb { 122238423Sjhb TAILQ_ENTRY(toepcb) link; /* toep_list */ 123238423Sjhb u_int flags; /* miscellaneous flags */ 124238423Sjhb struct tom_data *td; 125238423Sjhb struct inpcb *inp; /* backpointer to host stack's PCB */ 126238423Sjhb struct vnet *vnet; 127238423Sjhb struct vi_info *vi; /* virtual interface */ 128238423Sjhb struct sge_wrq *ofld_txq; 129238423Sjhb struct sge_ofld_rxq *ofld_rxq; 130238423Sjhb struct sge_wrq *ctrlq; 131238423Sjhb struct l2t_entry *l2te; /* L2 table entry used by this connection */ 132238423Sjhb struct clip_entry *ce; /* CLIP table entry used by this tid */ 133238423Sjhb int tid; /* Connection identifier */ 134238423Sjhb 135238423Sjhb /* tx credit handling */ 136238423Sjhb u_int tx_total; /* total tx WR credits (in 16B units) */ 137238423Sjhb u_int tx_credits; /* tx WR credits (in 16B units) available */ 138238423Sjhb u_int tx_nocompl; /* tx WR credits since last compl request */ 139238423Sjhb u_int plen_nocompl; /* payload since last compl request */ 140238423Sjhb 141238423Sjhb /* rx credit handling */ 142238423Sjhb u_int sb_cc; /* last noted value of so_rcv->sb_cc */ 143238423Sjhb int rx_credits; /* rx credits (in bytes) to be returned to hw */ 144238423Sjhb 145238423Sjhb u_int ulp_mode; /* ULP mode */ 146238423Sjhb void *ulpcb; 147238423Sjhb void *ulpcb2; 148238423Sjhb struct mbufq ulp_pduq; /* PDUs waiting to be sent out. */ 149238423Sjhb struct mbufq ulp_pdu_reclaimq; 150238423Sjhb 151238423Sjhb u_int ddp_flags; 152238423Sjhb struct ddp_buffer *db[2]; 153238423Sjhb time_t ddp_disabled; 154238423Sjhb uint8_t ddp_score; 155238423Sjhb 156238423Sjhb /* Tx software descriptor */ 157238423Sjhb uint8_t txsd_total; 158238423Sjhb uint8_t txsd_pidx; 159238423Sjhb uint8_t txsd_cidx; 160238423Sjhb uint8_t txsd_avail; 161238423Sjhb struct ofld_tx_sdesc txsd[]; 162238423Sjhb}; 163238423Sjhb 164238423Sjhbstruct flowc_tx_params { 165238423Sjhb uint32_t snd_nxt; 166238423Sjhb uint32_t rcv_nxt; 167238423Sjhb unsigned int snd_space; 168238423Sjhb unsigned int mss; 169238423Sjhb}; 170238423Sjhb 171238423Sjhb#define DDP_RETRY_WAIT 5 /* seconds to wait before re-enabling DDP */ 172238423Sjhb#define DDP_LOW_SCORE 1 173238423Sjhb#define DDP_HIGH_SCORE 3 174238423Sjhb 175238423Sjhb/* 176238423Sjhb * Compressed state for embryonic connections for a listener. Barely fits in 177238423Sjhb * 64B, try not to grow it further. 178238423Sjhb */ 179238423Sjhbstruct synq_entry { 180238423Sjhb TAILQ_ENTRY(synq_entry) link; /* listen_ctx's synq link */ 181238423Sjhb int flags; /* same as toepcb's tp_flags */ 182238423Sjhb int tid; 183238423Sjhb struct listen_ctx *lctx; /* backpointer to listen ctx */ 184238423Sjhb struct mbuf *syn; 185260650Sjhb uint32_t iss; 186238423Sjhb uint32_t ts; 187238423Sjhb volatile uintptr_t wr; 188238423Sjhb volatile u_int refcnt; 189238423Sjhb uint16_t l2e_idx; 190238423Sjhb uint16_t rcv_bufsize; 191260650Sjhb}; 192238423Sjhb 193260650Sjhb/* listen_ctx flags */ 194260650Sjhb#define LCTX_RPL_PENDING 1 /* waiting for a CPL_PASS_OPEN_RPL */ 195260650Sjhb 196260650Sjhbstruct listen_ctx { 197260650Sjhb LIST_ENTRY(listen_ctx) link; /* listen hash linkage */ 198260650Sjhb volatile int refcount; 199260650Sjhb int stid; 200260650Sjhb struct stid_region stid_region; 201260650Sjhb int flags; 202260650Sjhb struct inpcb *inp; /* listening socket's inp */ 203260650Sjhb struct vnet *vnet; 204260650Sjhb struct sge_wrq *ctrlq; 205260650Sjhb struct sge_ofld_rxq *ofld_rxq; 206238423Sjhb struct clip_entry *ce; 207238423Sjhb TAILQ_HEAD(, synq_entry) synq; 208260650Sjhb}; 209260650Sjhb 210238423Sjhbstruct clip_entry { 211238423Sjhb TAILQ_ENTRY(clip_entry) link; 212238423Sjhb struct in6_addr lip; /* local IPv6 address */ 213238423Sjhb u_int refcount; 214238423Sjhb}; 215238423Sjhb 216238423SjhbTAILQ_HEAD(clip_head, clip_entry); 217238423Sjhbstruct tom_data { 218238423Sjhb struct toedev tod; 219238423Sjhb 220238423Sjhb /* toepcb's associated with this TOE device */ 221238423Sjhb struct mtx toep_list_lock; 222238423Sjhb TAILQ_HEAD(, toepcb) toep_list; 223238423Sjhb 224238423Sjhb struct mtx lctx_hash_lock; 225238423Sjhb LIST_HEAD(, listen_ctx) *listen_hash; 226238423Sjhb u_long listen_mask; 227238423Sjhb int lctx_count; /* # of lctx in the hash table */ 228238423Sjhb 229238423Sjhb struct ppod_region pr; 230238423Sjhb 231260650Sjhb struct mtx clip_table_lock; 232260650Sjhb struct clip_head clip_table; 233238423Sjhb int clip_gen; 234238423Sjhb 235260650Sjhb /* WRs that will not be sent to the chip because L2 resolution failed */ 236260650Sjhb struct mtx unsent_wr_lock; 237260650Sjhb STAILQ_HEAD(, wrqe) unsent_wr_list; 238260650Sjhb struct task reclaim_wr_resources; 239260650Sjhb}; 240238423Sjhb 241238423Sjhbstatic inline struct tom_data * 242238423Sjhbtod_td(struct toedev *tod) 243238423Sjhb{ 244238423Sjhb 245238423Sjhb return (__containerof(tod, struct tom_data, tod)); 246238423Sjhb} 247238423Sjhb 248238423Sjhbstatic inline struct adapter * 249238423Sjhbtd_adapter(struct tom_data *td) 250238423Sjhb{ 251238423Sjhb 252238423Sjhb return (td->tod.tod_softc); 253238423Sjhb} 254238423Sjhb 255238423Sjhb/* 256238423Sjhb * XXX: Don't define these for the iWARP driver on 10 due to differences 257238423Sjhb * in LinuxKPI. 258238423Sjhb */ 259238423Sjhb#ifndef _LINUX_TYPES_H_ 260238423Sjhbstatic inline void 261238423Sjhbset_mbuf_ulp_submode(struct mbuf *m, uint8_t ulp_submode) 262238423Sjhb{ 263238423Sjhb 264238423Sjhb M_ASSERTPKTHDR(m); 265238423Sjhb m->m_pkthdr.PH_per.eigth[0] = ulp_submode; 266238423Sjhb} 267238423Sjhb 268238423Sjhbstatic inline uint8_t 269238423Sjhbmbuf_ulp_submode(struct mbuf *m) 270238423Sjhb{ 271238423Sjhb 272238423Sjhb M_ASSERTPKTHDR(m); 273238423Sjhb return (m->m_pkthdr.PH_per.eigth[0]); 274238423Sjhb} 275238423Sjhb#endif 276238423Sjhb 277238423Sjhb/* t4_tom.c */ 278238423Sjhbstruct toepcb *alloc_toepcb(struct vi_info *, int, int, int); 279238423Sjhbvoid free_toepcb(struct toepcb *); 280238423Sjhbvoid offload_socket(struct socket *, struct toepcb *); 281238423Sjhbvoid undo_offload_socket(struct socket *); 282238423Sjhbvoid final_cpl_received(struct toepcb *); 283238423Sjhbvoid insert_tid(struct adapter *, int, void *, int); 284238423Sjhbvoid *lookup_tid(struct adapter *, int); 285238423Sjhbvoid update_tid(struct adapter *, int, void *); 286238423Sjhbvoid remove_tid(struct adapter *, int, int); 287238423Sjhbvoid release_tid(struct adapter *, int, struct sge_wrq *); 288238423Sjhbint find_best_mtu_idx(struct adapter *, struct in_conninfo *, int); 289238423Sjhbu_long select_rcv_wnd(struct socket *); 290238423Sjhbint select_rcv_wscale(void); 291238423Sjhbuint64_t calc_opt0(struct socket *, struct vi_info *, struct l2t_entry *, 292238423Sjhb int, int, int, int); 293238423Sjhbuint64_t select_ntuple(struct vi_info *, struct l2t_entry *); 294238423Sjhbvoid set_tcpddp_ulp_mode(struct toepcb *); 295238423Sjhbint negative_advice(int); 296238423Sjhbstruct clip_entry *hold_lip(struct tom_data *, struct in6_addr *, 297238423Sjhb struct clip_entry *); 298238423Sjhbvoid release_lip(struct tom_data *, struct clip_entry *); 299238423Sjhb 300238423Sjhb/* t4_connect.c */ 301238423Sjhbvoid t4_init_connect_cpl_handlers(void); 302238423Sjhbvoid t4_uninit_connect_cpl_handlers(void); 303238423Sjhbint t4_connect(struct toedev *, struct socket *, struct rtentry *, 304238423Sjhb struct sockaddr *); 305238423Sjhbvoid act_open_failure_cleanup(struct adapter *, u_int, u_int); 306238423Sjhb 307238423Sjhb/* t4_listen.c */ 308238423Sjhbvoid t4_init_listen_cpl_handlers(void); 309238423Sjhbvoid t4_uninit_listen_cpl_handlers(void); 310238423Sjhbint t4_listen_start(struct toedev *, struct tcpcb *); 311238423Sjhbint t4_listen_stop(struct toedev *, struct tcpcb *); 312238423Sjhbvoid t4_syncache_added(struct toedev *, void *); 313238423Sjhbvoid t4_syncache_removed(struct toedev *, void *); 314238423Sjhbint t4_syncache_respond(struct toedev *, void *, struct mbuf *); 315238423Sjhbint do_abort_req_synqe(struct sge_iq *, const struct rss_header *, 316238423Sjhb struct mbuf *); 317238423Sjhbint do_abort_rpl_synqe(struct sge_iq *, const struct rss_header *, 318238423Sjhb struct mbuf *); 319238423Sjhbvoid t4_offload_socket(struct toedev *, void *, struct socket *); 320238423Sjhb 321238423Sjhb/* t4_cpl_io.c */ 322238423Sjhbvoid t4_init_cpl_io_handlers(void); 323238423Sjhbvoid t4_uninit_cpl_io_handlers(void); 324238423Sjhbvoid send_abort_rpl(struct adapter *, struct sge_wrq *, int , int); 325238423Sjhbvoid send_flowc_wr(struct toepcb *, struct flowc_tx_params *); 326238423Sjhbvoid send_reset(struct adapter *, struct toepcb *, uint32_t); 327238423Sjhbvoid make_established(struct toepcb *, uint32_t, uint32_t, uint16_t); 328238423Sjhbvoid t4_rcvd(struct toedev *, struct tcpcb *); 329238423Sjhbint t4_tod_output(struct toedev *, struct tcpcb *); 330238423Sjhbint t4_send_fin(struct toedev *, struct tcpcb *); 331238423Sjhbint t4_send_rst(struct toedev *, struct tcpcb *); 332238423Sjhbvoid t4_set_tcb_field(struct adapter *, struct sge_wrq *, int, uint16_t, 333238423Sjhb uint64_t, uint64_t, int, int, int); 334238423Sjhbvoid t4_push_frames(struct adapter *sc, struct toepcb *toep, int drop); 335238423Sjhbvoid t4_push_pdus(struct adapter *sc, struct toepcb *toep, int drop); 336238423Sjhbint do_set_tcb_rpl(struct sge_iq *, const struct rss_header *, struct mbuf *); 337238423Sjhb 338238423Sjhb/* t4_ddp.c */ 339238423Sjhbint t4_init_ppod_region(struct ppod_region *, struct t4_range *, u_int, 340238423Sjhb const char *); 341238423Sjhbvoid t4_free_ppod_region(struct ppod_region *); 342238423Sjhbint t4_alloc_page_pods_for_db(struct ppod_region *, struct ddp_buffer *); 343238423Sjhbint t4_alloc_page_pods_for_buf(struct ppod_region *, vm_offset_t, int, 344238423Sjhb struct ppod_reservation *); 345238423Sjhbint t4_write_page_pods_for_db(struct adapter *, struct sge_wrq *, int, 346238423Sjhb struct ddp_buffer *); 347238423Sjhbint t4_write_page_pods_for_buf(struct adapter *, struct sge_wrq *, int tid, 348238423Sjhb struct ppod_reservation *, vm_offset_t, int); 349238423Sjhbvoid t4_free_page_pods(struct ppod_reservation *); 350238423Sjhbint t4_soreceive_ddp(struct socket *, struct sockaddr **, struct uio *, 351238423Sjhb struct mbuf **, struct mbuf **, int *); 352238423Sjhbint t4_ddp_mod_load(void); 353238423Sjhbvoid t4_ddp_mod_unload(void); 354238423Sjhbvoid enable_ddp(struct adapter *, struct toepcb *toep); 355238423Sjhbvoid release_ddp_resources(struct toepcb *toep); 356238423Sjhbvoid handle_ddp_close(struct toepcb *, struct tcpcb *, struct sockbuf *, 357238423Sjhb uint32_t); 358238423Sjhbvoid insert_ddp_data(struct toepcb *, uint32_t); 359238423Sjhb 360238423Sjhb#endif 361238423Sjhb