1178786Skmacy/************************************************************************** 2178786Skmacy 3178786SkmacyCopyright (c) 2007, Chelsio Inc. 4178786SkmacyAll rights reserved. 5178786Skmacy 6178786SkmacyRedistribution and use in source and binary forms, with or without 7178786Skmacymodification, are permitted provided that the following conditions are met: 8178786Skmacy 9178786Skmacy 1. Redistributions of source code must retain the above copyright notice, 10178786Skmacy this list of conditions and the following disclaimer. 11178786Skmacy 12178786Skmacy 2. Neither the name of the Chelsio Corporation nor the names of its 13178786Skmacy contributors may be used to endorse or promote products derived from 14178786Skmacy this software without specific prior written permission. 15178786Skmacy 16178786SkmacyTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 17178786SkmacyAND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18178786SkmacyIMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19178786SkmacyARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 20178786SkmacyLIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21178786SkmacyCONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22178786SkmacySUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23178786SkmacyINTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24178786SkmacyCONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25178786SkmacyARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26178786SkmacyPOSSIBILITY OF SUCH DAMAGE. 27178786Skmacy 28178786Skmacy***************************************************************************/ 29178786Skmacy#include <sys/cdefs.h> 30178786Skmacy__FBSDID("$FreeBSD$"); 31178786Skmacy 32237263Snp#include "opt_inet.h" 33237263Snp 34237263Snp#ifdef TCP_OFFLOAD 35178786Skmacy#include <sys/param.h> 36178786Skmacy#include <sys/systm.h> 37178786Skmacy#include <sys/kernel.h> 38178786Skmacy#include <sys/bus.h> 39178786Skmacy#include <sys/pciio.h> 40178786Skmacy#include <sys/conf.h> 41178786Skmacy#include <machine/bus.h> 42178786Skmacy#include <machine/resource.h> 43178786Skmacy#include <sys/bus_dma.h> 44178786Skmacy#include <sys/rman.h> 45178786Skmacy#include <sys/ioccom.h> 46178786Skmacy#include <sys/mbuf.h> 47178786Skmacy#include <sys/mutex.h> 48178786Skmacy#include <sys/rwlock.h> 49178786Skmacy#include <sys/linker.h> 50178786Skmacy#include <sys/firmware.h> 51178786Skmacy#include <sys/socket.h> 52237263Snp#include <sys/socketvar.h> 53178786Skmacy#include <sys/sockio.h> 54178786Skmacy#include <sys/smp.h> 55178786Skmacy#include <sys/sysctl.h> 56178786Skmacy#include <sys/syslog.h> 57178786Skmacy#include <sys/queue.h> 58178786Skmacy#include <sys/taskqueue.h> 59178786Skmacy#include <sys/proc.h> 60178786Skmacy#include <sys/queue.h> 61178786Skmacy 62237263Snp#include <net/route.h> 63237263Snp#include <netinet/in_systm.h> 64178786Skmacy#include <netinet/in.h> 65237263Snp#include <netinet/toecore.h> 66237263Snp#include <netinet/in_pcb.h> 67237263Snp#include <netinet/ip.h> 68237263Snp#include <netinet/ip_var.h> 69237263Snp#include <netinet/tcp_var.h> 70237263Snp#include <netinet/tcp.h> 71237263Snp#include <netinet/tcpip.h> 72178786Skmacy 73237263Snp#include <rdma/ib_verbs.h> 74237263Snp#include <rdma/ib_umem.h> 75237263Snp#include <rdma/ib_user_verbs.h> 76237263Snp#include <linux/idr.h> 77237263Snp#include <ulp/iw_cxgb/iw_cxgb_ib_intfc.h> 78178786Skmacy 79178786Skmacy#include <cxgb_include.h> 80183321Skmacy#include <ulp/tom/cxgb_l2t.h> 81237263Snp#include <ulp/tom/cxgb_toepcb.h> 82178786Skmacy#include <ulp/iw_cxgb/iw_cxgb_wr.h> 83178786Skmacy#include <ulp/iw_cxgb/iw_cxgb_hal.h> 84178786Skmacy#include <ulp/iw_cxgb/iw_cxgb_provider.h> 85178786Skmacy#include <ulp/iw_cxgb/iw_cxgb_cm.h> 86178786Skmacy#include <ulp/iw_cxgb/iw_cxgb.h> 87178786Skmacy#include <ulp/iw_cxgb/iw_cxgb_resource.h> 88178786Skmacy#include <ulp/iw_cxgb/iw_cxgb_user.h> 89178786Skmacy 90178786Skmacy#define NO_SUPPORT -1 91178786Skmacy 92237263Snpstatic int build_rdma_send(union t3_wr *wqe, struct ib_send_wr *wr, 93178786Skmacy u8 * flit_cnt) 94178786Skmacy{ 95178786Skmacy int i; 96178786Skmacy u32 plen; 97178786Skmacy 98178786Skmacy switch (wr->opcode) { 99178786Skmacy case IB_WR_SEND: 100178786Skmacy if (wr->send_flags & IB_SEND_SOLICITED) 101178786Skmacy wqe->send.rdmaop = T3_SEND_WITH_SE; 102178786Skmacy else 103178786Skmacy wqe->send.rdmaop = T3_SEND; 104178786Skmacy wqe->send.rem_stag = 0; 105178786Skmacy break; 106237263Snp case IB_WR_SEND_WITH_IMM: 107237263Snp if (wr->send_flags & IB_SEND_SOLICITED) 108237263Snp wqe->send.rdmaop = T3_SEND_WITH_SE_INV; 109237263Snp else 110237263Snp wqe->send.rdmaop = T3_SEND_WITH_INV; 111237263Snp wqe->send.rem_stag = 0; 112178786Skmacy break; 113178786Skmacy default: 114237263Snp return -EINVAL; 115178786Skmacy } 116178786Skmacy if (wr->num_sge > T3_MAX_SGE) 117178786Skmacy return (-EINVAL); 118178786Skmacy wqe->send.reserved[0] = 0; 119178786Skmacy wqe->send.reserved[1] = 0; 120178786Skmacy wqe->send.reserved[2] = 0; 121237263Snp plen = 0; 122237263Snp for (i = 0; i < wr->num_sge; i++) { 123237263Snp if ((plen + wr->sg_list[i].length) < plen) { 124237263Snp return (-EMSGSIZE); 125178786Skmacy } 126237263Snp plen += wr->sg_list[i].length; 127237263Snp wqe->send.sgl[i].stag = 128237263Snp htobe32(wr->sg_list[i].lkey); 129237263Snp wqe->send.sgl[i].len = 130237263Snp htobe32(wr->sg_list[i].length); 131237263Snp wqe->send.sgl[i].to = htobe64(wr->sg_list[i].addr); 132178786Skmacy } 133237263Snp wqe->send.num_sgle = htobe32(wr->num_sge); 134237263Snp *flit_cnt = 4 + ((wr->num_sge) << 1); 135178786Skmacy wqe->send.plen = htobe32(plen); 136178786Skmacy return 0; 137178786Skmacy} 138178786Skmacy 139237263Snpstatic int build_rdma_write(union t3_wr *wqe, struct ib_send_wr *wr, 140178786Skmacy u8 *flit_cnt) 141178786Skmacy{ 142178786Skmacy int i; 143178786Skmacy u32 plen; 144178786Skmacy 145178786Skmacy if (wr->num_sge > T3_MAX_SGE) 146178786Skmacy return (-EINVAL); 147178786Skmacy wqe->write.rdmaop = T3_RDMA_WRITE; 148178786Skmacy wqe->write.reserved[0] = 0; 149178786Skmacy wqe->write.reserved[1] = 0; 150178786Skmacy wqe->write.reserved[2] = 0; 151178786Skmacy wqe->write.stag_sink = htobe32(wr->wr.rdma.rkey); 152178786Skmacy wqe->write.to_sink = htobe64(wr->wr.rdma.remote_addr); 153178786Skmacy 154178786Skmacy if (wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM) { 155178786Skmacy plen = 4; 156237263Snp wqe->write.sgl[0].stag = wr->ex.imm_data; 157178786Skmacy wqe->write.sgl[0].len = 0; 158178786Skmacy wqe->write.num_sgle = 0; 159178786Skmacy *flit_cnt = 6; 160178786Skmacy } else { 161178786Skmacy plen = 0; 162178786Skmacy for (i = 0; i < wr->num_sge; i++) { 163178786Skmacy if ((plen + wr->sg_list[i].length) < plen) { 164178786Skmacy return (-EMSGSIZE); 165178786Skmacy } 166178786Skmacy plen += wr->sg_list[i].length; 167178786Skmacy wqe->write.sgl[i].stag = 168178786Skmacy htobe32(wr->sg_list[i].lkey); 169178786Skmacy wqe->write.sgl[i].len = 170178786Skmacy htobe32(wr->sg_list[i].length); 171178786Skmacy wqe->write.sgl[i].to = 172178786Skmacy htobe64(wr->sg_list[i].addr); 173178786Skmacy } 174178786Skmacy wqe->write.num_sgle = htobe32(wr->num_sge); 175178786Skmacy *flit_cnt = 5 + ((wr->num_sge) << 1); 176178786Skmacy } 177178786Skmacy wqe->write.plen = htobe32(plen); 178178786Skmacy return 0; 179178786Skmacy} 180178786Skmacy 181237263Snpstatic int build_rdma_read(union t3_wr *wqe, struct ib_send_wr *wr, 182178786Skmacy u8 *flit_cnt) 183178786Skmacy{ 184178786Skmacy if (wr->num_sge > 1) 185178786Skmacy return (-EINVAL); 186178786Skmacy wqe->read.rdmaop = T3_READ_REQ; 187178786Skmacy wqe->read.reserved[0] = 0; 188178786Skmacy wqe->read.reserved[1] = 0; 189178786Skmacy wqe->read.reserved[2] = 0; 190178786Skmacy wqe->read.rem_stag = htobe32(wr->wr.rdma.rkey); 191178786Skmacy wqe->read.rem_to = htobe64(wr->wr.rdma.remote_addr); 192178786Skmacy wqe->read.local_stag = htobe32(wr->sg_list[0].lkey); 193178786Skmacy wqe->read.local_len = htobe32(wr->sg_list[0].length); 194178786Skmacy wqe->read.local_to = htobe64(wr->sg_list[0].addr); 195178786Skmacy *flit_cnt = sizeof(struct t3_rdma_read_wr) >> 3; 196178786Skmacy return 0; 197178786Skmacy} 198178786Skmacy 199178786Skmacystatic int iwch_sgl2pbl_map(struct iwch_dev *rhp, struct ib_sge *sg_list, 200178786Skmacy u32 num_sgle, u32 * pbl_addr, u8 * page_size) 201178786Skmacy{ 202178786Skmacy int i; 203178786Skmacy struct iwch_mr *mhp; 204237263Snp u64 offset; 205178786Skmacy for (i = 0; i < num_sgle; i++) { 206178786Skmacy 207178786Skmacy mhp = get_mhp(rhp, (sg_list[i].lkey) >> 8); 208178786Skmacy if (!mhp) { 209178786Skmacy CTR2(KTR_IW_CXGB, "%s %d", __FUNCTION__, __LINE__); 210178786Skmacy return (-EIO); 211178786Skmacy } 212178786Skmacy if (!mhp->attr.state) { 213178786Skmacy CTR2(KTR_IW_CXGB, "%s %d", __FUNCTION__, __LINE__); 214178786Skmacy return (-EIO); 215178786Skmacy } 216178786Skmacy if (mhp->attr.zbva) { 217178786Skmacy CTR2(KTR_IW_CXGB, "%s %d", __FUNCTION__, __LINE__); 218178786Skmacy return (-EIO); 219178786Skmacy } 220178786Skmacy 221178786Skmacy if (sg_list[i].addr < mhp->attr.va_fbo) { 222178786Skmacy CTR2(KTR_IW_CXGB, "%s %d", __FUNCTION__, __LINE__); 223178786Skmacy return (-EINVAL); 224178786Skmacy } 225178786Skmacy if (sg_list[i].addr + ((u64) sg_list[i].length) < 226178786Skmacy sg_list[i].addr) { 227178786Skmacy CTR2(KTR_IW_CXGB, "%s %d", __FUNCTION__, __LINE__); 228178786Skmacy return (-EINVAL); 229178786Skmacy } 230178786Skmacy if (sg_list[i].addr + ((u64) sg_list[i].length) > 231178786Skmacy mhp->attr.va_fbo + ((u64) mhp->attr.len)) { 232178786Skmacy CTR2(KTR_IW_CXGB, "%s %d", __FUNCTION__, __LINE__); 233178786Skmacy return (-EINVAL); 234178786Skmacy } 235178786Skmacy offset = sg_list[i].addr - mhp->attr.va_fbo; 236237263Snp offset += mhp->attr.va_fbo & 237237263Snp ((1UL << (12 + mhp->attr.page_size)) - 1); 238178786Skmacy pbl_addr[i] = ((mhp->attr.pbl_addr - 239178786Skmacy rhp->rdev.rnic_info.pbl_base) >> 3) + 240178786Skmacy (offset >> (12 + mhp->attr.page_size)); 241178786Skmacy page_size[i] = mhp->attr.page_size; 242178786Skmacy } 243178786Skmacy return 0; 244178786Skmacy} 245178786Skmacy 246237263Snpstatic int build_rdma_recv(struct iwch_qp *qhp, union t3_wr *wqe, 247178786Skmacy struct ib_recv_wr *wr) 248178786Skmacy{ 249237263Snp int i, err = 0; 250237263Snp u32 pbl_addr[T3_MAX_SGE]; 251237263Snp u8 page_size[T3_MAX_SGE]; 252237263Snp 253237263Snp if (wr->num_sge > T3_MAX_SGE) 254178786Skmacy return (-EINVAL); 255237263Snp 256237263Snp 257237263Snp err = iwch_sgl2pbl_map(qhp->rhp, wr->sg_list, wr->num_sge, pbl_addr, 258237263Snp page_size); 259237263Snp if (err) 260237263Snp return err; 261237263Snp wqe->recv.pagesz[0] = page_size[0]; 262237263Snp wqe->recv.pagesz[1] = page_size[1]; 263237263Snp wqe->recv.pagesz[2] = page_size[2]; 264237263Snp wqe->recv.pagesz[3] = page_size[3]; 265178786Skmacy wqe->recv.num_sgle = htobe32(wr->num_sge); 266237263Snp 267178786Skmacy for (i = 0; i < wr->num_sge; i++) { 268178786Skmacy wqe->recv.sgl[i].stag = htobe32(wr->sg_list[i].lkey); 269178786Skmacy wqe->recv.sgl[i].len = htobe32(wr->sg_list[i].length); 270237263Snp wqe->recv.sgl[i].to = htobe64(((u32)wr->sg_list[i].addr) & 271237263Snp ((1UL << (12 + page_size[i])) - 1)); 272237263Snp /* pbl_addr is the adapters address in the PBL */ 273237263Snp wqe->recv.pbl_addr[i] = cpu_to_be32(pbl_addr[i]); 274178786Skmacy } 275178786Skmacy for (; i < T3_MAX_SGE; i++) { 276178786Skmacy wqe->recv.sgl[i].stag = 0; 277178786Skmacy wqe->recv.sgl[i].len = 0; 278178786Skmacy wqe->recv.sgl[i].to = 0; 279237263Snp wqe->recv.pbl_addr[i] = 0; 280178786Skmacy } 281237263Snp 282237263Snp qhp->wq.rq[Q_PTR2IDX(qhp->wq.rq_wptr, 283237263Snp qhp->wq.rq_size_log2)].wr_id = wr->wr_id; 284237263Snp qhp->wq.rq[Q_PTR2IDX(qhp->wq.rq_wptr, 285237263Snp qhp->wq.rq_size_log2)].pbl_addr = 0; 286237263Snp 287178786Skmacy return 0; 288178786Skmacy} 289178786Skmacy 290237263Snpstatic int build_zero_stag_recv(struct iwch_qp *qhp, union t3_wr *wqe, 291237263Snp struct ib_recv_wr *wr) 292237263Snp{ 293237263Snp int i; 294237263Snp u32 pbl_addr; 295237263Snp u32 pbl_offset; 296237263Snp 297237263Snp 298237263Snp /* 299237263Snp * The T3 HW requires the PBL in the HW recv descriptor to reference 300237263Snp * a PBL entry. So we allocate the max needed PBL memory here and pass 301237263Snp * it to the uP in the recv WR. The uP will build the PBL and setup 302237263Snp * the HW recv descriptor. 303237263Snp */ 304237263Snp pbl_addr = cxio_hal_pblpool_alloc(&qhp->rhp->rdev, T3_STAG0_PBL_SIZE); 305237263Snp if (!pbl_addr) 306237263Snp return -ENOMEM; 307237263Snp 308237263Snp /* 309237263Snp * Compute the 8B aligned offset. 310237263Snp */ 311237263Snp pbl_offset = (pbl_addr - qhp->rhp->rdev.rnic_info.pbl_base) >> 3; 312237263Snp 313237263Snp wqe->recv.num_sgle = cpu_to_be32(wr->num_sge); 314237263Snp 315237263Snp for (i = 0; i < wr->num_sge; i++) { 316237263Snp 317237263Snp /* 318237263Snp * Use a 128MB page size. This and an imposed 128MB 319237263Snp * sge length limit allows us to require only a 2-entry HW 320237263Snp * PBL for each SGE. This restriction is acceptable since 321237263Snp * since it is not possible to allocate 128MB of contiguous 322237263Snp * DMA coherent memory! 323237263Snp */ 324237263Snp if (wr->sg_list[i].length > T3_STAG0_MAX_PBE_LEN) 325237263Snp return -EINVAL; 326237263Snp wqe->recv.pagesz[i] = T3_STAG0_PAGE_SHIFT; 327237263Snp 328237263Snp /* 329237263Snp * T3 restricts a recv to all zero-stag or all non-zero-stag. 330237263Snp */ 331237263Snp if (wr->sg_list[i].lkey != 0) 332237263Snp return -EINVAL; 333237263Snp wqe->recv.sgl[i].stag = 0; 334237263Snp wqe->recv.sgl[i].len = htobe32(wr->sg_list[i].length); 335237263Snp wqe->recv.sgl[i].to = htobe64(wr->sg_list[i].addr); 336237263Snp wqe->recv.pbl_addr[i] = htobe32(pbl_offset); 337237263Snp pbl_offset += 2; 338237263Snp } 339237263Snp for (; i < T3_MAX_SGE; i++) { 340237263Snp wqe->recv.pagesz[i] = 0; 341237263Snp wqe->recv.sgl[i].stag = 0; 342237263Snp wqe->recv.sgl[i].len = 0; 343237263Snp wqe->recv.sgl[i].to = 0; 344237263Snp wqe->recv.pbl_addr[i] = 0; 345237263Snp } 346237263Snp qhp->wq.rq[Q_PTR2IDX(qhp->wq.rq_wptr, 347237263Snp qhp->wq.rq_size_log2)].wr_id = wr->wr_id; 348237263Snp qhp->wq.rq[Q_PTR2IDX(qhp->wq.rq_wptr, 349237263Snp qhp->wq.rq_size_log2)].pbl_addr = pbl_addr; 350237263Snp return 0; 351237263Snp} 352237263Snp 353178786Skmacyint iwch_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, 354178786Skmacy struct ib_send_wr **bad_wr) 355178786Skmacy{ 356178786Skmacy int err = 0; 357178786Skmacy u8 t3_wr_flit_cnt = 0; 358178786Skmacy enum t3_wr_opcode t3_wr_opcode = 0; 359178786Skmacy enum t3_wr_flags t3_wr_flags; 360178786Skmacy struct iwch_qp *qhp; 361178786Skmacy u32 idx; 362178786Skmacy union t3_wr *wqe; 363178786Skmacy u32 num_wrs; 364178786Skmacy struct t3_swsq *sqp; 365178786Skmacy 366178786Skmacy qhp = to_iwch_qp(ibqp); 367178786Skmacy mtx_lock(&qhp->lock); 368178786Skmacy if (qhp->attr.state > IWCH_QP_STATE_RTS) { 369178786Skmacy mtx_unlock(&qhp->lock); 370237263Snp err = -EINVAL; 371237263Snp goto out; 372178786Skmacy } 373178786Skmacy num_wrs = Q_FREECNT(qhp->wq.sq_rptr, qhp->wq.sq_wptr, 374178786Skmacy qhp->wq.sq_size_log2); 375237263Snp if (num_wrs == 0) { 376178786Skmacy mtx_unlock(&qhp->lock); 377237263Snp err = -EINVAL; 378237263Snp goto out; 379178786Skmacy } 380178786Skmacy while (wr) { 381178786Skmacy if (num_wrs == 0) { 382178786Skmacy err = -ENOMEM; 383178786Skmacy break; 384178786Skmacy } 385178786Skmacy idx = Q_PTR2IDX(qhp->wq.wptr, qhp->wq.size_log2); 386178786Skmacy wqe = (union t3_wr *) (qhp->wq.queue + idx); 387178786Skmacy t3_wr_flags = 0; 388178786Skmacy if (wr->send_flags & IB_SEND_SOLICITED) 389178786Skmacy t3_wr_flags |= T3_SOLICITED_EVENT_FLAG; 390178786Skmacy if (wr->send_flags & IB_SEND_FENCE) 391178786Skmacy t3_wr_flags |= T3_READ_FENCE_FLAG; 392178786Skmacy if (wr->send_flags & IB_SEND_SIGNALED) 393178786Skmacy t3_wr_flags |= T3_COMPLETION_FLAG; 394178786Skmacy sqp = qhp->wq.sq + 395178786Skmacy Q_PTR2IDX(qhp->wq.sq_wptr, qhp->wq.sq_size_log2); 396178786Skmacy switch (wr->opcode) { 397178786Skmacy case IB_WR_SEND: 398178786Skmacy case IB_WR_SEND_WITH_IMM: 399178786Skmacy t3_wr_opcode = T3_WR_SEND; 400237263Snp err = build_rdma_send(wqe, wr, &t3_wr_flit_cnt); 401178786Skmacy break; 402178786Skmacy case IB_WR_RDMA_WRITE: 403178786Skmacy case IB_WR_RDMA_WRITE_WITH_IMM: 404178786Skmacy t3_wr_opcode = T3_WR_WRITE; 405237263Snp err = build_rdma_write(wqe, wr, &t3_wr_flit_cnt); 406178786Skmacy break; 407178786Skmacy case IB_WR_RDMA_READ: 408178786Skmacy t3_wr_opcode = T3_WR_READ; 409178786Skmacy t3_wr_flags = 0; /* T3 reads are always signaled */ 410237263Snp err = build_rdma_read(wqe, wr, &t3_wr_flit_cnt); 411178786Skmacy if (err) 412178786Skmacy break; 413178786Skmacy sqp->read_len = wqe->read.local_len; 414178786Skmacy if (!qhp->wq.oldest_read) 415178786Skmacy qhp->wq.oldest_read = sqp; 416178786Skmacy break; 417178786Skmacy default: 418178786Skmacy CTR2(KTR_IW_CXGB, "%s post of type=%d TBD!", __FUNCTION__, 419178786Skmacy wr->opcode); 420178786Skmacy err = -EINVAL; 421178786Skmacy } 422237263Snp if (err) 423178786Skmacy break; 424237263Snp 425178786Skmacy wqe->send.wrid.id0.hi = qhp->wq.sq_wptr; 426178786Skmacy sqp->wr_id = wr->wr_id; 427178786Skmacy sqp->opcode = wr2opcode(t3_wr_opcode); 428178786Skmacy sqp->sq_wptr = qhp->wq.sq_wptr; 429178786Skmacy sqp->complete = 0; 430178786Skmacy sqp->signaled = (wr->send_flags & IB_SEND_SIGNALED); 431178786Skmacy 432178786Skmacy build_fw_riwrh((void *) wqe, t3_wr_opcode, t3_wr_flags, 433178786Skmacy Q_GENBIT(qhp->wq.wptr, qhp->wq.size_log2), 434178786Skmacy 0, t3_wr_flit_cnt); 435178786Skmacy CTR5(KTR_IW_CXGB, "%s cookie 0x%llx wq idx 0x%x swsq idx %ld opcode %d", 436178786Skmacy __FUNCTION__, (unsigned long long) wr->wr_id, idx, 437178786Skmacy Q_PTR2IDX(qhp->wq.sq_wptr, qhp->wq.sq_size_log2), 438178786Skmacy sqp->opcode); 439178786Skmacy wr = wr->next; 440178786Skmacy num_wrs--; 441178786Skmacy ++(qhp->wq.wptr); 442178786Skmacy ++(qhp->wq.sq_wptr); 443178786Skmacy } 444178786Skmacy mtx_unlock(&qhp->lock); 445178786Skmacy ring_doorbell(qhp->wq.doorbell, qhp->wq.qpid); 446237263Snpout: 447237263Snp if (err) 448237263Snp *bad_wr = wr; 449178786Skmacy return err; 450178786Skmacy} 451178786Skmacy 452178786Skmacyint iwch_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *wr, 453178786Skmacy struct ib_recv_wr **bad_wr) 454178786Skmacy{ 455178786Skmacy int err = 0; 456178786Skmacy struct iwch_qp *qhp; 457178786Skmacy u32 idx; 458178786Skmacy union t3_wr *wqe; 459178786Skmacy u32 num_wrs; 460178786Skmacy 461178786Skmacy qhp = to_iwch_qp(ibqp); 462178786Skmacy mtx_lock(&qhp->lock); 463178786Skmacy if (qhp->attr.state > IWCH_QP_STATE_RTS) { 464178786Skmacy mtx_unlock(&qhp->lock); 465237263Snp err = -EINVAL; 466237263Snp goto out; 467178786Skmacy } 468178786Skmacy num_wrs = Q_FREECNT(qhp->wq.rq_rptr, qhp->wq.rq_wptr, 469178786Skmacy qhp->wq.rq_size_log2) - 1; 470178786Skmacy if (!wr) { 471178786Skmacy mtx_unlock(&qhp->lock); 472237263Snp err = -EINVAL; 473237263Snp goto out; 474178786Skmacy } 475237263Snp 476178786Skmacy while (wr) { 477237263Snp if (wr->num_sge > T3_MAX_SGE) { 478237263Snp err = -EINVAL; 479237263Snp break; 480237263Snp } 481237263Snp 482178786Skmacy idx = Q_PTR2IDX(qhp->wq.wptr, qhp->wq.size_log2); 483178786Skmacy wqe = (union t3_wr *) (qhp->wq.queue + idx); 484237263Snp if (num_wrs) { 485237263Snp if (wr->sg_list[0].lkey) 486237263Snp err = build_rdma_recv(qhp, wqe, wr); 487237263Snp else 488237263Snp err = build_zero_stag_recv(qhp, wqe, wr); 489237263Snp } else 490178786Skmacy err = -ENOMEM; 491237263Snp if (err) 492178786Skmacy break; 493237263Snp 494178786Skmacy build_fw_riwrh((void *) wqe, T3_WR_RCV, T3_COMPLETION_FLAG, 495178786Skmacy Q_GENBIT(qhp->wq.wptr, qhp->wq.size_log2), 496178786Skmacy 0, sizeof(struct t3_receive_wr) >> 3); 497178786Skmacy CTR6(KTR_IW_CXGB, "%s cookie 0x%llx idx 0x%x rq_wptr 0x%x rw_rptr 0x%x " 498178786Skmacy "wqe %p ", __FUNCTION__, (unsigned long long) wr->wr_id, 499178786Skmacy idx, qhp->wq.rq_wptr, qhp->wq.rq_rptr, wqe); 500178786Skmacy ++(qhp->wq.rq_wptr); 501178786Skmacy ++(qhp->wq.wptr); 502178786Skmacy wr = wr->next; 503178786Skmacy num_wrs--; 504178786Skmacy } 505178786Skmacy mtx_unlock(&qhp->lock); 506178786Skmacy ring_doorbell(qhp->wq.doorbell, qhp->wq.qpid); 507237263Snpout: 508237263Snp if (err) 509237263Snp *bad_wr = wr; 510178786Skmacy return err; 511178786Skmacy} 512178786Skmacy 513178786Skmacyint iwch_bind_mw(struct ib_qp *qp, 514178786Skmacy struct ib_mw *mw, 515178786Skmacy struct ib_mw_bind *mw_bind) 516178786Skmacy{ 517178786Skmacy struct iwch_dev *rhp; 518178786Skmacy struct iwch_mw *mhp; 519178786Skmacy struct iwch_qp *qhp; 520178786Skmacy union t3_wr *wqe; 521178786Skmacy u32 pbl_addr; 522178786Skmacy u8 page_size; 523178786Skmacy u32 num_wrs; 524178786Skmacy struct ib_sge sgl; 525178786Skmacy int err=0; 526178786Skmacy enum t3_wr_flags t3_wr_flags; 527178786Skmacy u32 idx; 528178786Skmacy struct t3_swsq *sqp; 529178786Skmacy 530178786Skmacy qhp = to_iwch_qp(qp); 531178786Skmacy mhp = to_iwch_mw(mw); 532178786Skmacy rhp = qhp->rhp; 533178786Skmacy 534178786Skmacy mtx_lock(&qhp->lock); 535178786Skmacy if (qhp->attr.state > IWCH_QP_STATE_RTS) { 536178786Skmacy mtx_unlock(&qhp->lock); 537178786Skmacy return (-EINVAL); 538178786Skmacy } 539178786Skmacy num_wrs = Q_FREECNT(qhp->wq.sq_rptr, qhp->wq.sq_wptr, 540178786Skmacy qhp->wq.sq_size_log2); 541237263Snp if ((num_wrs) == 0) { 542178786Skmacy mtx_unlock(&qhp->lock); 543178786Skmacy return (-ENOMEM); 544178786Skmacy } 545178786Skmacy idx = Q_PTR2IDX(qhp->wq.wptr, qhp->wq.size_log2); 546178786Skmacy CTR4(KTR_IW_CXGB, "%s: idx 0x%0x, mw 0x%p, mw_bind 0x%p", __FUNCTION__, idx, 547178786Skmacy mw, mw_bind); 548178786Skmacy wqe = (union t3_wr *) (qhp->wq.queue + idx); 549178786Skmacy 550178786Skmacy t3_wr_flags = 0; 551178786Skmacy if (mw_bind->send_flags & IB_SEND_SIGNALED) 552178786Skmacy t3_wr_flags = T3_COMPLETION_FLAG; 553178786Skmacy 554178786Skmacy sgl.addr = mw_bind->addr; 555178786Skmacy sgl.lkey = mw_bind->mr->lkey; 556178786Skmacy sgl.length = mw_bind->length; 557178786Skmacy wqe->bind.reserved = 0; 558178786Skmacy wqe->bind.type = T3_VA_BASED_TO; 559178786Skmacy 560178786Skmacy /* TBD: check perms */ 561178786Skmacy wqe->bind.perms = iwch_ib_to_mwbind_access(mw_bind->mw_access_flags); 562178786Skmacy wqe->bind.mr_stag = htobe32(mw_bind->mr->lkey); 563178786Skmacy wqe->bind.mw_stag = htobe32(mw->rkey); 564178786Skmacy wqe->bind.mw_len = htobe32(mw_bind->length); 565178786Skmacy wqe->bind.mw_va = htobe64(mw_bind->addr); 566178786Skmacy err = iwch_sgl2pbl_map(rhp, &sgl, 1, &pbl_addr, &page_size); 567178786Skmacy if (err) { 568178786Skmacy mtx_unlock(&qhp->lock); 569178786Skmacy return (err); 570178786Skmacy } 571178786Skmacy wqe->send.wrid.id0.hi = qhp->wq.sq_wptr; 572178786Skmacy sqp = qhp->wq.sq + Q_PTR2IDX(qhp->wq.sq_wptr, qhp->wq.sq_size_log2); 573178786Skmacy sqp->wr_id = mw_bind->wr_id; 574178786Skmacy sqp->opcode = T3_BIND_MW; 575178786Skmacy sqp->sq_wptr = qhp->wq.sq_wptr; 576178786Skmacy sqp->complete = 0; 577178786Skmacy sqp->signaled = (mw_bind->send_flags & IB_SEND_SIGNALED); 578178786Skmacy wqe->bind.mr_pbl_addr = htobe32(pbl_addr); 579178786Skmacy wqe->bind.mr_pagesz = page_size; 580178786Skmacy wqe->flit[T3_SQ_COOKIE_FLIT] = mw_bind->wr_id; 581178786Skmacy build_fw_riwrh((void *)wqe, T3_WR_BIND, t3_wr_flags, 582178786Skmacy Q_GENBIT(qhp->wq.wptr, qhp->wq.size_log2), 0, 583178786Skmacy sizeof(struct t3_bind_mw_wr) >> 3); 584178786Skmacy ++(qhp->wq.wptr); 585178786Skmacy ++(qhp->wq.sq_wptr); 586178786Skmacy mtx_unlock(&qhp->lock); 587178786Skmacy 588178786Skmacy ring_doorbell(qhp->wq.doorbell, qhp->wq.qpid); 589178786Skmacy 590178786Skmacy return err; 591178786Skmacy} 592178786Skmacy 593237263Snpstatic void build_term_codes(struct respQ_msg_t *rsp_msg, 594178786Skmacy u8 *layer_type, u8 *ecode) 595178786Skmacy{ 596178786Skmacy int status = TPT_ERR_INTERNAL_ERR; 597178786Skmacy int tagged = 0; 598178786Skmacy int opcode = -1; 599178786Skmacy int rqtype = 0; 600178786Skmacy int send_inv = 0; 601178786Skmacy 602178786Skmacy if (rsp_msg) { 603178786Skmacy status = CQE_STATUS(rsp_msg->cqe); 604178786Skmacy opcode = CQE_OPCODE(rsp_msg->cqe); 605178786Skmacy rqtype = RQ_TYPE(rsp_msg->cqe); 606178786Skmacy send_inv = (opcode == T3_SEND_WITH_INV) || 607178786Skmacy (opcode == T3_SEND_WITH_SE_INV); 608178786Skmacy tagged = (opcode == T3_RDMA_WRITE) || 609178786Skmacy (rqtype && (opcode == T3_READ_RESP)); 610178786Skmacy } 611178786Skmacy 612178786Skmacy switch (status) { 613178786Skmacy case TPT_ERR_STAG: 614178786Skmacy if (send_inv) { 615178786Skmacy *layer_type = LAYER_RDMAP|RDMAP_REMOTE_OP; 616178786Skmacy *ecode = RDMAP_CANT_INV_STAG; 617178786Skmacy } else { 618178786Skmacy *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT; 619178786Skmacy *ecode = RDMAP_INV_STAG; 620178786Skmacy } 621178786Skmacy break; 622178786Skmacy case TPT_ERR_PDID: 623178786Skmacy *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT; 624178786Skmacy if ((opcode == T3_SEND_WITH_INV) || 625178786Skmacy (opcode == T3_SEND_WITH_SE_INV)) 626178786Skmacy *ecode = RDMAP_CANT_INV_STAG; 627178786Skmacy else 628178786Skmacy *ecode = RDMAP_STAG_NOT_ASSOC; 629178786Skmacy break; 630178786Skmacy case TPT_ERR_QPID: 631178786Skmacy *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT; 632178786Skmacy *ecode = RDMAP_STAG_NOT_ASSOC; 633178786Skmacy break; 634178786Skmacy case TPT_ERR_ACCESS: 635178786Skmacy *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT; 636178786Skmacy *ecode = RDMAP_ACC_VIOL; 637178786Skmacy break; 638178786Skmacy case TPT_ERR_WRAP: 639178786Skmacy *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT; 640178786Skmacy *ecode = RDMAP_TO_WRAP; 641178786Skmacy break; 642178786Skmacy case TPT_ERR_BOUND: 643178786Skmacy if (tagged) { 644178786Skmacy *layer_type = LAYER_DDP|DDP_TAGGED_ERR; 645178786Skmacy *ecode = DDPT_BASE_BOUNDS; 646178786Skmacy } else { 647178786Skmacy *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT; 648178786Skmacy *ecode = RDMAP_BASE_BOUNDS; 649178786Skmacy } 650178786Skmacy break; 651178786Skmacy case TPT_ERR_INVALIDATE_SHARED_MR: 652178786Skmacy case TPT_ERR_INVALIDATE_MR_WITH_MW_BOUND: 653178786Skmacy *layer_type = LAYER_RDMAP|RDMAP_REMOTE_OP; 654178786Skmacy *ecode = RDMAP_CANT_INV_STAG; 655178786Skmacy break; 656178786Skmacy case TPT_ERR_ECC: 657178786Skmacy case TPT_ERR_ECC_PSTAG: 658178786Skmacy case TPT_ERR_INTERNAL_ERR: 659178786Skmacy *layer_type = LAYER_RDMAP|RDMAP_LOCAL_CATA; 660178786Skmacy *ecode = 0; 661178786Skmacy break; 662178786Skmacy case TPT_ERR_OUT_OF_RQE: 663178786Skmacy *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR; 664178786Skmacy *ecode = DDPU_INV_MSN_NOBUF; 665178786Skmacy break; 666178786Skmacy case TPT_ERR_PBL_ADDR_BOUND: 667178786Skmacy *layer_type = LAYER_DDP|DDP_TAGGED_ERR; 668178786Skmacy *ecode = DDPT_BASE_BOUNDS; 669178786Skmacy break; 670178786Skmacy case TPT_ERR_CRC: 671178786Skmacy *layer_type = LAYER_MPA|DDP_LLP; 672178786Skmacy *ecode = MPA_CRC_ERR; 673178786Skmacy break; 674178786Skmacy case TPT_ERR_MARKER: 675178786Skmacy *layer_type = LAYER_MPA|DDP_LLP; 676178786Skmacy *ecode = MPA_MARKER_ERR; 677178786Skmacy break; 678178786Skmacy case TPT_ERR_PDU_LEN_ERR: 679178786Skmacy *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR; 680178786Skmacy *ecode = DDPU_MSG_TOOBIG; 681178786Skmacy break; 682178786Skmacy case TPT_ERR_DDP_VERSION: 683178786Skmacy if (tagged) { 684178786Skmacy *layer_type = LAYER_DDP|DDP_TAGGED_ERR; 685178786Skmacy *ecode = DDPT_INV_VERS; 686178786Skmacy } else { 687178786Skmacy *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR; 688178786Skmacy *ecode = DDPU_INV_VERS; 689178786Skmacy } 690178786Skmacy break; 691178786Skmacy case TPT_ERR_RDMA_VERSION: 692178786Skmacy *layer_type = LAYER_RDMAP|RDMAP_REMOTE_OP; 693178786Skmacy *ecode = RDMAP_INV_VERS; 694178786Skmacy break; 695178786Skmacy case TPT_ERR_OPCODE: 696178786Skmacy *layer_type = LAYER_RDMAP|RDMAP_REMOTE_OP; 697178786Skmacy *ecode = RDMAP_INV_OPCODE; 698178786Skmacy break; 699178786Skmacy case TPT_ERR_DDP_QUEUE_NUM: 700178786Skmacy *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR; 701178786Skmacy *ecode = DDPU_INV_QN; 702178786Skmacy break; 703178786Skmacy case TPT_ERR_MSN: 704178786Skmacy case TPT_ERR_MSN_GAP: 705178786Skmacy case TPT_ERR_MSN_RANGE: 706178786Skmacy case TPT_ERR_IRD_OVERFLOW: 707178786Skmacy *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR; 708178786Skmacy *ecode = DDPU_INV_MSN_RANGE; 709178786Skmacy break; 710178786Skmacy case TPT_ERR_TBIT: 711178786Skmacy *layer_type = LAYER_DDP|DDP_LOCAL_CATA; 712178786Skmacy *ecode = 0; 713178786Skmacy break; 714178786Skmacy case TPT_ERR_MO: 715178786Skmacy *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR; 716178786Skmacy *ecode = DDPU_INV_MO; 717178786Skmacy break; 718178786Skmacy default: 719178786Skmacy *layer_type = LAYER_RDMAP|DDP_LOCAL_CATA; 720178786Skmacy *ecode = 0; 721178786Skmacy break; 722178786Skmacy } 723178786Skmacy} 724178786Skmacy 725178786Skmacy/* 726178786Skmacy * This posts a TERMINATE with layer=RDMA, type=catastrophic. 727178786Skmacy */ 728178786Skmacyint iwch_post_terminate(struct iwch_qp *qhp, struct respQ_msg_t *rsp_msg) 729178786Skmacy{ 730178786Skmacy union t3_wr *wqe; 731178786Skmacy struct terminate_message *term; 732178786Skmacy struct mbuf *m; 733237263Snp struct ofld_hdr *oh; 734178786Skmacy 735237263Snp CTR3(KTR_IW_CXGB, "%s: tid %u, %p", __func__, qhp->ep->hwtid, rsp_msg); 736237263Snp m = m_gethdr(M_NOWAIT, MT_DATA); 737237263Snp if (m == NULL) { 738178786Skmacy log(LOG_ERR, "%s cannot send TERMINATE!\n", __FUNCTION__); 739178786Skmacy return (-ENOMEM); 740178786Skmacy } 741237263Snp oh = mtod(m, struct ofld_hdr *); 742237263Snp m->m_pkthdr.len = m->m_len = sizeof(*oh) + 40; 743237263Snp oh->flags = V_HDR_NDESC(1) | V_HDR_CTRL(CPL_PRIORITY_DATA) | V_HDR_QSET(0); 744237263Snp wqe = (void *)(oh + 1); 745178786Skmacy memset(wqe, 0, 40); 746178786Skmacy wqe->send.rdmaop = T3_TERMINATE; 747178786Skmacy 748178786Skmacy /* immediate data length */ 749178786Skmacy wqe->send.plen = htonl(4); 750178786Skmacy 751178786Skmacy /* immediate data starts here. */ 752178786Skmacy term = (struct terminate_message *)wqe->send.sgl; 753178786Skmacy build_term_codes(rsp_msg, &term->layer_etype, &term->ecode); 754178786Skmacy wqe->send.wrh.op_seop_flags = htobe32(V_FW_RIWR_OP(T3_WR_SEND) | 755178786Skmacy V_FW_RIWR_FLAGS(T3_COMPLETION_FLAG | T3_NOTIFY_FLAG)); 756178786Skmacy wqe->send.wrh.gen_tid_len = htobe32(V_FW_RIWR_TID(qhp->ep->hwtid)); 757178786Skmacy 758237263Snp return t3_offload_tx(qhp->rhp->rdev.adap, m); 759178786Skmacy} 760178786Skmacy 761178786Skmacy/* 762178786Skmacy * Assumes qhp lock is held. 763178786Skmacy */ 764237263Snpstatic void __flush_qp(struct iwch_qp *qhp, struct iwch_cq *rchp, 765237263Snp struct iwch_cq *schp) 766178786Skmacy{ 767178786Skmacy int count; 768237263Snp int flushed; 769178786Skmacy 770178786Skmacy CTR4(KTR_IW_CXGB, "%s qhp %p rchp %p schp %p", __FUNCTION__, qhp, rchp, schp); 771178786Skmacy /* take a ref on the qhp since we must release the lock */ 772178786Skmacy qhp->refcnt++; 773178786Skmacy mtx_unlock(&qhp->lock); 774178786Skmacy 775204111Suqs /* locking hierarchy: cq lock first, then qp lock. */ 776178786Skmacy mtx_lock(&rchp->lock); 777178786Skmacy mtx_lock(&qhp->lock); 778178786Skmacy cxio_flush_hw_cq(&rchp->cq); 779178786Skmacy cxio_count_rcqes(&rchp->cq, &qhp->wq, &count); 780237263Snp flushed = cxio_flush_rq(&qhp->wq, &rchp->cq, count); 781178786Skmacy mtx_unlock(&qhp->lock); 782178786Skmacy mtx_unlock(&rchp->lock); 783237263Snp if (flushed) 784237263Snp (*rchp->ibcq.comp_handler)(&rchp->ibcq, rchp->ibcq.cq_context); 785178786Skmacy 786204111Suqs /* locking hierarchy: cq lock first, then qp lock. */ 787178786Skmacy mtx_lock(&schp->lock); 788178786Skmacy mtx_lock(&qhp->lock); 789178786Skmacy cxio_flush_hw_cq(&schp->cq); 790178786Skmacy cxio_count_scqes(&schp->cq, &qhp->wq, &count); 791237263Snp flushed = cxio_flush_sq(&qhp->wq, &schp->cq, count); 792178786Skmacy mtx_unlock(&qhp->lock); 793178786Skmacy mtx_unlock(&schp->lock); 794237263Snp if (flushed) 795237263Snp (*schp->ibcq.comp_handler)(&schp->ibcq, schp->ibcq.cq_context); 796178786Skmacy 797178786Skmacy /* deref */ 798178786Skmacy mtx_lock(&qhp->lock); 799178786Skmacy if (--qhp->refcnt == 0) 800178786Skmacy wakeup(qhp); 801178786Skmacy} 802178786Skmacy 803178786Skmacystatic void flush_qp(struct iwch_qp *qhp) 804178786Skmacy{ 805237263Snp struct iwch_cq *rchp, *schp; 806237263Snp 807237263Snp rchp = get_chp(qhp->rhp, qhp->attr.rcq); 808237263Snp schp = get_chp(qhp->rhp, qhp->attr.scq); 809237263Snp 810237263Snp if (qhp->ibqp.uobject) { 811178786Skmacy cxio_set_wq_in_error(&qhp->wq); 812237263Snp cxio_set_cq_in_error(&rchp->cq); 813237263Snp (*rchp->ibcq.comp_handler)(&rchp->ibcq, rchp->ibcq.cq_context); 814237263Snp if (schp != rchp) { 815237263Snp cxio_set_cq_in_error(&schp->cq); 816237263Snp (*schp->ibcq.comp_handler)(&schp->ibcq, 817237263Snp schp->ibcq.cq_context); 818237263Snp } 819237263Snp return; 820237263Snp } 821237263Snp __flush_qp(qhp, rchp, schp); 822178786Skmacy} 823178786Skmacy 824178786Skmacy 825178786Skmacy/* 826178786Skmacy * Return non zero if at least one RECV was pre-posted. 827178786Skmacy */ 828178786Skmacystatic int rqes_posted(struct iwch_qp *qhp) 829178786Skmacy{ 830237263Snp union t3_wr *wqe = qhp->wq.queue; 831237263Snp u16 count = 0; 832237263Snp while ((count+1) != 0 && fw_riwrh_opcode((struct fw_riwrh *)wqe) == T3_WR_RCV) { 833237263Snp count++; 834237263Snp wqe++; 835237263Snp } 836237263Snp return count; 837178786Skmacy} 838178786Skmacy 839178786Skmacystatic int rdma_init(struct iwch_dev *rhp, struct iwch_qp *qhp, 840178786Skmacy enum iwch_qp_attr_mask mask, 841178786Skmacy struct iwch_qp_attributes *attrs) 842178786Skmacy{ 843178786Skmacy struct t3_rdma_init_attr init_attr; 844178786Skmacy int ret; 845237263Snp struct socket *so = qhp->ep->com.so; 846237263Snp struct inpcb *inp = sotoinpcb(so); 847237263Snp struct tcpcb *tp; 848237263Snp struct toepcb *toep; 849178786Skmacy 850178786Skmacy init_attr.tid = qhp->ep->hwtid; 851178786Skmacy init_attr.qpid = qhp->wq.qpid; 852178786Skmacy init_attr.pdid = qhp->attr.pd; 853178786Skmacy init_attr.scqid = qhp->attr.scq; 854178786Skmacy init_attr.rcqid = qhp->attr.rcq; 855178786Skmacy init_attr.rq_addr = qhp->wq.rq_addr; 856178786Skmacy init_attr.rq_size = 1 << qhp->wq.rq_size_log2; 857178786Skmacy init_attr.mpaattrs = uP_RI_MPA_IETF_ENABLE | 858178786Skmacy qhp->attr.mpa_attr.recv_marker_enabled | 859178786Skmacy (qhp->attr.mpa_attr.xmit_marker_enabled << 1) | 860178786Skmacy (qhp->attr.mpa_attr.crc_enabled << 2); 861178786Skmacy 862237263Snp init_attr.qpcaps = uP_RI_QP_RDMA_READ_ENABLE | 863237263Snp uP_RI_QP_RDMA_WRITE_ENABLE | 864237263Snp uP_RI_QP_BIND_ENABLE; 865237263Snp if (!qhp->ibqp.uobject) 866237263Snp init_attr.qpcaps |= uP_RI_QP_STAG0_ENABLE; 867178786Skmacy init_attr.tcp_emss = qhp->ep->emss; 868178786Skmacy init_attr.ord = qhp->attr.max_ord; 869178786Skmacy init_attr.ird = qhp->attr.max_ird; 870178786Skmacy init_attr.qp_dma_addr = qhp->wq.dma_addr; 871178786Skmacy init_attr.qp_dma_size = (1UL << qhp->wq.size_log2); 872237263Snp init_attr.rqe_count = rqes_posted(qhp); 873237263Snp init_attr.flags = qhp->attr.mpa_attr.initiator ? MPA_INITIATOR : 0; 874237263Snp init_attr.rtr_type = 0; 875237263Snp tp = intotcpcb(inp); 876237263Snp toep = tp->t_toe; 877237263Snp init_attr.chan = toep->tp_l2t->smt_idx; 878178786Skmacy init_attr.irs = qhp->ep->rcv_seq; 879178786Skmacy CTR5(KTR_IW_CXGB, "%s init_attr.rq_addr 0x%x init_attr.rq_size = %d " 880178786Skmacy "flags 0x%x qpcaps 0x%x", __FUNCTION__, 881178786Skmacy init_attr.rq_addr, init_attr.rq_size, 882178786Skmacy init_attr.flags, init_attr.qpcaps); 883237263Snp ret = cxio_rdma_init(&rhp->rdev, &init_attr, qhp->ep->com.so); 884178786Skmacy CTR2(KTR_IW_CXGB, "%s ret %d", __FUNCTION__, ret); 885178786Skmacy return ret; 886178786Skmacy} 887178786Skmacy 888178786Skmacyint iwch_modify_qp(struct iwch_dev *rhp, struct iwch_qp *qhp, 889178786Skmacy enum iwch_qp_attr_mask mask, 890178786Skmacy struct iwch_qp_attributes *attrs, 891178786Skmacy int internal) 892178786Skmacy{ 893178786Skmacy int ret = 0; 894178786Skmacy struct iwch_qp_attributes newattr = qhp->attr; 895178786Skmacy int disconnect = 0; 896178786Skmacy int terminate = 0; 897178786Skmacy int abort = 0; 898178786Skmacy int free = 0; 899178786Skmacy struct iwch_ep *ep = NULL; 900178786Skmacy 901178786Skmacy CTR6(KTR_IW_CXGB, "%s qhp %p qpid 0x%x ep %p state %d -> %d", __FUNCTION__, 902178786Skmacy qhp, qhp->wq.qpid, qhp->ep, qhp->attr.state, 903178786Skmacy (mask & IWCH_QP_ATTR_NEXT_STATE) ? attrs->next_state : -1); 904178786Skmacy 905178786Skmacy mtx_lock(&qhp->lock); 906178786Skmacy 907178786Skmacy /* Process attr changes if in IDLE */ 908178786Skmacy if (mask & IWCH_QP_ATTR_VALID_MODIFY) { 909178786Skmacy if (qhp->attr.state != IWCH_QP_STATE_IDLE) { 910178786Skmacy ret = -EIO; 911178786Skmacy goto out; 912178786Skmacy } 913178786Skmacy if (mask & IWCH_QP_ATTR_ENABLE_RDMA_READ) 914178786Skmacy newattr.enable_rdma_read = attrs->enable_rdma_read; 915178786Skmacy if (mask & IWCH_QP_ATTR_ENABLE_RDMA_WRITE) 916178786Skmacy newattr.enable_rdma_write = attrs->enable_rdma_write; 917178786Skmacy if (mask & IWCH_QP_ATTR_ENABLE_RDMA_BIND) 918178786Skmacy newattr.enable_bind = attrs->enable_bind; 919178786Skmacy if (mask & IWCH_QP_ATTR_MAX_ORD) { 920178786Skmacy if (attrs->max_ord > 921178786Skmacy rhp->attr.max_rdma_read_qp_depth) { 922178786Skmacy ret = -EINVAL; 923178786Skmacy goto out; 924178786Skmacy } 925178786Skmacy newattr.max_ord = attrs->max_ord; 926178786Skmacy } 927178786Skmacy if (mask & IWCH_QP_ATTR_MAX_IRD) { 928178786Skmacy if (attrs->max_ird > 929178786Skmacy rhp->attr.max_rdma_reads_per_qp) { 930178786Skmacy ret = -EINVAL; 931178786Skmacy goto out; 932178786Skmacy } 933178786Skmacy newattr.max_ird = attrs->max_ird; 934178786Skmacy } 935178786Skmacy qhp->attr = newattr; 936178786Skmacy } 937178786Skmacy 938178786Skmacy if (!(mask & IWCH_QP_ATTR_NEXT_STATE)) 939178786Skmacy goto out; 940178786Skmacy if (qhp->attr.state == attrs->next_state) 941178786Skmacy goto out; 942178786Skmacy 943178786Skmacy switch (qhp->attr.state) { 944178786Skmacy case IWCH_QP_STATE_IDLE: 945178786Skmacy switch (attrs->next_state) { 946178786Skmacy case IWCH_QP_STATE_RTS: 947178786Skmacy if (!(mask & IWCH_QP_ATTR_LLP_STREAM_HANDLE)) { 948178786Skmacy ret = -EINVAL; 949178786Skmacy goto out; 950178786Skmacy } 951178786Skmacy if (!(mask & IWCH_QP_ATTR_MPA_ATTR)) { 952178786Skmacy ret = -EINVAL; 953178786Skmacy goto out; 954178786Skmacy } 955178786Skmacy qhp->attr.mpa_attr = attrs->mpa_attr; 956178786Skmacy qhp->attr.llp_stream_handle = attrs->llp_stream_handle; 957178786Skmacy qhp->ep = qhp->attr.llp_stream_handle; 958178786Skmacy qhp->attr.state = IWCH_QP_STATE_RTS; 959178786Skmacy 960178786Skmacy /* 961178786Skmacy * Ref the endpoint here and deref when we 962178786Skmacy * disassociate the endpoint from the QP. This 963178786Skmacy * happens in CLOSING->IDLE transition or *->ERROR 964178786Skmacy * transition. 965178786Skmacy */ 966178786Skmacy get_ep(&qhp->ep->com); 967178786Skmacy mtx_unlock(&qhp->lock); 968178786Skmacy ret = rdma_init(rhp, qhp, mask, attrs); 969178786Skmacy mtx_lock(&qhp->lock); 970178786Skmacy if (ret) 971178786Skmacy goto err; 972178786Skmacy break; 973178786Skmacy case IWCH_QP_STATE_ERROR: 974178786Skmacy qhp->attr.state = IWCH_QP_STATE_ERROR; 975178786Skmacy flush_qp(qhp); 976178786Skmacy break; 977178786Skmacy default: 978178786Skmacy ret = -EINVAL; 979178786Skmacy goto out; 980178786Skmacy } 981178786Skmacy break; 982178786Skmacy case IWCH_QP_STATE_RTS: 983178786Skmacy switch (attrs->next_state) { 984178786Skmacy case IWCH_QP_STATE_CLOSING: 985178786Skmacy PANIC_IF(atomic_load_acq_int(&qhp->ep->com.refcount) < 2); 986178786Skmacy qhp->attr.state = IWCH_QP_STATE_CLOSING; 987178786Skmacy if (!internal) { 988178786Skmacy abort=0; 989178786Skmacy disconnect = 1; 990178786Skmacy ep = qhp->ep; 991237263Snp get_ep(&ep->com); 992178786Skmacy } 993178786Skmacy break; 994178786Skmacy case IWCH_QP_STATE_TERMINATE: 995178786Skmacy qhp->attr.state = IWCH_QP_STATE_TERMINATE; 996178786Skmacy if (qhp->ibqp.uobject) 997178786Skmacy cxio_set_wq_in_error(&qhp->wq); 998178786Skmacy if (!internal) 999178786Skmacy terminate = 1; 1000178786Skmacy break; 1001178786Skmacy case IWCH_QP_STATE_ERROR: 1002178786Skmacy qhp->attr.state = IWCH_QP_STATE_ERROR; 1003178786Skmacy if (!internal) { 1004178786Skmacy abort=1; 1005178786Skmacy disconnect = 1; 1006178786Skmacy ep = qhp->ep; 1007237263Snp get_ep(&ep->com); 1008178786Skmacy } 1009178786Skmacy goto err; 1010178786Skmacy break; 1011178786Skmacy default: 1012178786Skmacy ret = -EINVAL; 1013178786Skmacy goto out; 1014178786Skmacy } 1015178786Skmacy break; 1016178786Skmacy case IWCH_QP_STATE_CLOSING: 1017178786Skmacy if (!internal) { 1018178786Skmacy ret = -EINVAL; 1019178786Skmacy goto out; 1020178786Skmacy } 1021178786Skmacy switch (attrs->next_state) { 1022178786Skmacy case IWCH_QP_STATE_IDLE: 1023237263Snp flush_qp(qhp); 1024178786Skmacy qhp->attr.state = IWCH_QP_STATE_IDLE; 1025178786Skmacy qhp->attr.llp_stream_handle = NULL; 1026178786Skmacy put_ep(&qhp->ep->com); 1027178786Skmacy qhp->ep = NULL; 1028178786Skmacy wakeup(qhp); 1029178786Skmacy break; 1030178786Skmacy case IWCH_QP_STATE_ERROR: 1031178786Skmacy goto err; 1032178786Skmacy default: 1033178786Skmacy ret = -EINVAL; 1034178786Skmacy goto err; 1035178786Skmacy } 1036178786Skmacy break; 1037178786Skmacy case IWCH_QP_STATE_ERROR: 1038178786Skmacy if (attrs->next_state != IWCH_QP_STATE_IDLE) { 1039178786Skmacy ret = -EINVAL; 1040178786Skmacy goto out; 1041178786Skmacy } 1042178786Skmacy 1043178786Skmacy if (!Q_EMPTY(qhp->wq.sq_rptr, qhp->wq.sq_wptr) || 1044178786Skmacy !Q_EMPTY(qhp->wq.rq_rptr, qhp->wq.rq_wptr)) { 1045178786Skmacy ret = -EINVAL; 1046178786Skmacy goto out; 1047178786Skmacy } 1048178786Skmacy qhp->attr.state = IWCH_QP_STATE_IDLE; 1049178786Skmacy memset(&qhp->attr, 0, sizeof(qhp->attr)); 1050178786Skmacy break; 1051178786Skmacy case IWCH_QP_STATE_TERMINATE: 1052178786Skmacy if (!internal) { 1053178786Skmacy ret = -EINVAL; 1054178786Skmacy goto out; 1055178786Skmacy } 1056178786Skmacy goto err; 1057178786Skmacy break; 1058178786Skmacy default: 1059178786Skmacy log(LOG_ERR, "%s in a bad state %d\n", 1060178786Skmacy __FUNCTION__, qhp->attr.state); 1061178786Skmacy ret = -EINVAL; 1062178786Skmacy goto err; 1063178786Skmacy break; 1064178786Skmacy } 1065178786Skmacy goto out; 1066178786Skmacyerr: 1067178786Skmacy CTR3(KTR_IW_CXGB, "%s disassociating ep %p qpid 0x%x", __FUNCTION__, qhp->ep, 1068178786Skmacy qhp->wq.qpid); 1069178786Skmacy 1070178786Skmacy /* disassociate the LLP connection */ 1071178786Skmacy qhp->attr.llp_stream_handle = NULL; 1072178786Skmacy ep = qhp->ep; 1073178786Skmacy qhp->ep = NULL; 1074178786Skmacy qhp->attr.state = IWCH_QP_STATE_ERROR; 1075178786Skmacy free=1; 1076178786Skmacy wakeup(qhp); 1077178786Skmacy PANIC_IF(!ep); 1078178786Skmacy flush_qp(qhp); 1079178786Skmacyout: 1080178786Skmacy mtx_unlock(&qhp->lock); 1081178786Skmacy 1082237263Snp if (terminate) 1083178786Skmacy iwch_post_terminate(qhp, NULL); 1084237263Snp 1085178786Skmacy 1086178786Skmacy /* 1087178786Skmacy * If disconnect is 1, then we need to initiate a disconnect 1088178786Skmacy * on the EP. This can be a normal close (RTS->CLOSING) or 1089178786Skmacy * an abnormal close (RTS/CLOSING->ERROR). 1090178786Skmacy */ 1091237263Snp if (disconnect) { 1092178786Skmacy iwch_ep_disconnect(ep, abort, M_NOWAIT); 1093237263Snp put_ep(&ep->com); 1094237263Snp } 1095237263Snp 1096178786Skmacy /* 1097178786Skmacy * If free is 1, then we've disassociated the EP from the QP 1098178786Skmacy * and we need to dereference the EP. 1099178786Skmacy */ 1100254731Snp if (free) 1101178786Skmacy put_ep(&ep->com); 1102178786Skmacy 1103178786Skmacy CTR2(KTR_IW_CXGB, "%s exit state %d", __FUNCTION__, qhp->attr.state); 1104178786Skmacy return ret; 1105178786Skmacy} 1106237263Snp#endif 1107