1321936Shselasky/* 2321936Shselasky * Copyright (c) 2006-2016 Chelsio, Inc. All rights reserved. 3321936Shselasky * 4321936Shselasky * This software is available to you under a choice of one of two 5321936Shselasky * licenses. You may choose to be licensed under the terms of the GNU 6321936Shselasky * General Public License (GPL) Version 2, available from the file 7321936Shselasky * COPYING in the main directory of this source tree, or the 8321936Shselasky * OpenIB.org BSD license below: 9321936Shselasky * 10321936Shselasky * Redistribution and use in source and binary forms, with or 11321936Shselasky * without modification, are permitted provided that the following 12321936Shselasky * conditions are met: 13321936Shselasky * 14321936Shselasky * - Redistributions of source code must retain the above 15321936Shselasky * copyright notice, this list of conditions and the following 16321936Shselasky * disclaimer. 17321936Shselasky * 18321936Shselasky * - Redistributions in binary form must reproduce the above 19321936Shselasky * copyright notice, this list of conditions and the following 20321936Shselasky * disclaimer in the documentation and/or other materials 21321936Shselasky * provided with the distribution. 22321936Shselasky * 23321936Shselasky * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24321936Shselasky * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25321936Shselasky * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26321936Shselasky * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27321936Shselasky * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28321936Shselasky * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29321936Shselasky * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30321936Shselasky * SOFTWARE. 31321936Shselasky */ 32321936Shselasky#include <config.h> 33321936Shselasky 34321936Shselasky#include <assert.h> 35321936Shselasky#include <stdlib.h> 36321936Shselasky#include <pthread.h> 37321936Shselasky#include <string.h> 38321936Shselasky#include <stdio.h> 39321936Shselasky#include "libcxgb4.h" 40321936Shselasky 41321936Shselasky#ifdef STATS 42321936Shselaskystruct c4iw_stats c4iw_stats; 43321936Shselasky#endif 44321936Shselasky 45321936Shselaskystatic void copy_wr_to_sq(struct t4_wq *wq, union t4_wr *wqe, u8 len16) 46321936Shselasky{ 47325522Snp void *src, *dst; 48325522Snp uintptr_t end; 49325522Snp int total, len; 50321936Shselasky 51325522Snp src = &wqe->flits[0]; 52325522Snp dst = &wq->sq.queue->flits[wq->sq.wq_pidx * 53325522Snp (T4_EQ_ENTRY_SIZE / sizeof(__be64))]; 54321936Shselasky if (t4_sq_onchip(wq)) { 55321936Shselasky len16 = align(len16, 4); 56321936Shselasky 57321936Shselasky /* In onchip mode the copy below will be made to WC memory and 58321936Shselasky * could trigger DMA. In offchip mode the copy below only 59321936Shselasky * queues the WQE, DMA cannot start until t4_ring_sq_db 60321936Shselasky * happens */ 61321936Shselasky mmio_wc_start(); 62321936Shselasky } 63321936Shselasky 64325522Snp /* NOTE len16 cannot be large enough to write to the 65325522Snp same sq.queue memory twice in this loop */ 66325522Snp total = len16 * 16; 67325522Snp end = (uintptr_t)&wq->sq.queue[wq->sq.size]; 68325522Snp if (__predict_true((uintptr_t)dst + total <= end)) { 69325522Snp /* Won't wrap around. */ 70325522Snp memcpy(dst, src, total); 71325522Snp } else { 72325522Snp len = end - (uintptr_t)dst; 73325522Snp memcpy(dst, src, len); 74325522Snp memcpy(wq->sq.queue, src + len, total - len); 75321936Shselasky } 76321936Shselasky 77321936Shselasky if (t4_sq_onchip(wq)) 78321936Shselasky mmio_flush_writes(); 79321936Shselasky} 80321936Shselasky 81321936Shselaskystatic void copy_wr_to_rq(struct t4_wq *wq, union t4_recv_wr *wqe, u8 len16) 82321936Shselasky{ 83325522Snp void *src, *dst; 84325522Snp uintptr_t end; 85325522Snp int total, len; 86321936Shselasky 87325522Snp src = &wqe->flits[0]; 88325522Snp dst = &wq->rq.queue->flits[wq->rq.wq_pidx * 89325522Snp (T4_EQ_ENTRY_SIZE / sizeof(__be64))]; 90325522Snp 91325522Snp total = len16 * 16; 92325522Snp end = (uintptr_t)&wq->rq.queue[wq->rq.size]; 93325522Snp if (__predict_true((uintptr_t)dst + total <= end)) { 94325522Snp /* Won't wrap around. */ 95325522Snp memcpy(dst, src, total); 96325522Snp } else { 97325522Snp len = end - (uintptr_t)dst; 98325522Snp memcpy(dst, src, len); 99325522Snp memcpy(wq->rq.queue, src + len, total - len); 100321936Shselasky } 101321936Shselasky} 102321936Shselasky 103321936Shselaskystatic int build_immd(struct t4_sq *sq, struct fw_ri_immd *immdp, 104321936Shselasky struct ibv_send_wr *wr, int max, u32 *plenp) 105321936Shselasky{ 106321936Shselasky u8 *dstp, *srcp; 107321936Shselasky u32 plen = 0; 108321936Shselasky int i; 109321936Shselasky int len; 110321936Shselasky 111321936Shselasky dstp = (u8 *)immdp->data; 112321936Shselasky for (i = 0; i < wr->num_sge; i++) { 113321936Shselasky if ((plen + wr->sg_list[i].length) > max) 114321936Shselasky return -EMSGSIZE; 115321936Shselasky srcp = (u8 *)(unsigned long)wr->sg_list[i].addr; 116321936Shselasky plen += wr->sg_list[i].length; 117321936Shselasky len = wr->sg_list[i].length; 118321936Shselasky memcpy(dstp, srcp, len); 119321936Shselasky dstp += len; 120321936Shselasky srcp += len; 121321936Shselasky } 122321936Shselasky len = ROUND_UP(plen + 8, 16) - (plen + 8); 123321936Shselasky if (len) 124321936Shselasky memset(dstp, 0, len); 125321936Shselasky immdp->op = FW_RI_DATA_IMMD; 126321936Shselasky immdp->r1 = 0; 127321936Shselasky immdp->r2 = 0; 128321936Shselasky immdp->immdlen = htobe32(plen); 129321936Shselasky *plenp = plen; 130321936Shselasky return 0; 131321936Shselasky} 132321936Shselasky 133321936Shselaskystatic int build_isgl(struct fw_ri_isgl *isglp, struct ibv_sge *sg_list, 134321936Shselasky int num_sge, u32 *plenp) 135321936Shselasky{ 136321936Shselasky int i; 137321936Shselasky u32 plen = 0; 138321936Shselasky __be64 *flitp = (__be64 *)isglp->sge; 139321936Shselasky 140321936Shselasky for (i = 0; i < num_sge; i++) { 141321936Shselasky if ((plen + sg_list[i].length) < plen) 142321936Shselasky return -EMSGSIZE; 143321936Shselasky plen += sg_list[i].length; 144321936Shselasky *flitp++ = htobe64(((u64)sg_list[i].lkey << 32) | 145321936Shselasky sg_list[i].length); 146321936Shselasky *flitp++ = htobe64(sg_list[i].addr); 147321936Shselasky } 148321936Shselasky *flitp = 0; 149321936Shselasky isglp->op = FW_RI_DATA_ISGL; 150321936Shselasky isglp->r1 = 0; 151321936Shselasky isglp->nsge = htobe16(num_sge); 152321936Shselasky isglp->r2 = 0; 153321936Shselasky if (plenp) 154321936Shselasky *plenp = plen; 155321936Shselasky return 0; 156321936Shselasky} 157321936Shselasky 158321936Shselaskystatic int build_rdma_send(struct t4_sq *sq, union t4_wr *wqe, 159321936Shselasky struct ibv_send_wr *wr, u8 *len16) 160321936Shselasky{ 161321936Shselasky u32 plen; 162321936Shselasky int size; 163321936Shselasky int ret; 164321936Shselasky 165321936Shselasky if (wr->num_sge > T4_MAX_SEND_SGE) 166321936Shselasky return -EINVAL; 167321936Shselasky if (wr->send_flags & IBV_SEND_SOLICITED) 168321936Shselasky wqe->send.sendop_pkd = htobe32( 169321936Shselasky FW_RI_SEND_WR_SENDOP_V(FW_RI_SEND_WITH_SE)); 170321936Shselasky else 171321936Shselasky wqe->send.sendop_pkd = htobe32( 172321936Shselasky FW_RI_SEND_WR_SENDOP_V(FW_RI_SEND)); 173321936Shselasky wqe->send.stag_inv = 0; 174321936Shselasky wqe->send.r3 = 0; 175321936Shselasky wqe->send.r4 = 0; 176321936Shselasky 177321936Shselasky plen = 0; 178321936Shselasky if (wr->num_sge) { 179321936Shselasky if (wr->send_flags & IBV_SEND_INLINE) { 180321936Shselasky ret = build_immd(sq, wqe->send.u.immd_src, wr, 181321936Shselasky T4_MAX_SEND_INLINE, &plen); 182321936Shselasky if (ret) 183321936Shselasky return ret; 184321936Shselasky size = sizeof wqe->send + sizeof(struct fw_ri_immd) + 185321936Shselasky plen; 186321936Shselasky } else { 187321936Shselasky ret = build_isgl(wqe->send.u.isgl_src, 188321936Shselasky wr->sg_list, wr->num_sge, &plen); 189321936Shselasky if (ret) 190321936Shselasky return ret; 191321936Shselasky size = sizeof wqe->send + sizeof(struct fw_ri_isgl) + 192321936Shselasky wr->num_sge * sizeof (struct fw_ri_sge); 193321936Shselasky } 194321936Shselasky } else { 195321936Shselasky wqe->send.u.immd_src[0].op = FW_RI_DATA_IMMD; 196321936Shselasky wqe->send.u.immd_src[0].r1 = 0; 197321936Shselasky wqe->send.u.immd_src[0].r2 = 0; 198321936Shselasky wqe->send.u.immd_src[0].immdlen = 0; 199321936Shselasky size = sizeof wqe->send + sizeof(struct fw_ri_immd); 200321936Shselasky plen = 0; 201321936Shselasky } 202321936Shselasky *len16 = DIV_ROUND_UP(size, 16); 203321936Shselasky wqe->send.plen = htobe32(plen); 204321936Shselasky return 0; 205321936Shselasky} 206321936Shselasky 207321936Shselaskystatic int build_rdma_write(struct t4_sq *sq, union t4_wr *wqe, 208321936Shselasky struct ibv_send_wr *wr, u8 *len16) 209321936Shselasky{ 210321936Shselasky u32 plen; 211321936Shselasky int size; 212321936Shselasky int ret; 213321936Shselasky 214321936Shselasky if (wr->num_sge > T4_MAX_SEND_SGE) 215321936Shselasky return -EINVAL; 216321936Shselasky wqe->write.r2 = 0; 217321936Shselasky wqe->write.stag_sink = htobe32(wr->wr.rdma.rkey); 218321936Shselasky wqe->write.to_sink = htobe64(wr->wr.rdma.remote_addr); 219321936Shselasky if (wr->num_sge) { 220321936Shselasky if (wr->send_flags & IBV_SEND_INLINE) { 221321936Shselasky ret = build_immd(sq, wqe->write.u.immd_src, wr, 222321936Shselasky T4_MAX_WRITE_INLINE, &plen); 223321936Shselasky if (ret) 224321936Shselasky return ret; 225321936Shselasky size = sizeof wqe->write + sizeof(struct fw_ri_immd) + 226321936Shselasky plen; 227321936Shselasky } else { 228321936Shselasky ret = build_isgl(wqe->write.u.isgl_src, 229321936Shselasky wr->sg_list, wr->num_sge, &plen); 230321936Shselasky if (ret) 231321936Shselasky return ret; 232321936Shselasky size = sizeof wqe->write + sizeof(struct fw_ri_isgl) + 233321936Shselasky wr->num_sge * sizeof (struct fw_ri_sge); 234321936Shselasky } 235321936Shselasky } else { 236321936Shselasky wqe->write.u.immd_src[0].op = FW_RI_DATA_IMMD; 237321936Shselasky wqe->write.u.immd_src[0].r1 = 0; 238321936Shselasky wqe->write.u.immd_src[0].r2 = 0; 239321936Shselasky wqe->write.u.immd_src[0].immdlen = 0; 240321936Shselasky size = sizeof wqe->write + sizeof(struct fw_ri_immd); 241321936Shselasky plen = 0; 242321936Shselasky } 243321936Shselasky *len16 = DIV_ROUND_UP(size, 16); 244321936Shselasky wqe->write.plen = htobe32(plen); 245321936Shselasky return 0; 246321936Shselasky} 247321936Shselasky 248321936Shselaskystatic int build_rdma_read(union t4_wr *wqe, struct ibv_send_wr *wr, u8 *len16) 249321936Shselasky{ 250321936Shselasky if (wr->num_sge > 1) 251321936Shselasky return -EINVAL; 252321936Shselasky if (wr->num_sge) { 253321936Shselasky wqe->read.stag_src = htobe32(wr->wr.rdma.rkey); 254321936Shselasky wqe->read.to_src_hi = htobe32((u32)(wr->wr.rdma.remote_addr >>32)); 255321936Shselasky wqe->read.to_src_lo = htobe32((u32)wr->wr.rdma.remote_addr); 256321936Shselasky wqe->read.stag_sink = htobe32(wr->sg_list[0].lkey); 257321936Shselasky wqe->read.plen = htobe32(wr->sg_list[0].length); 258321936Shselasky wqe->read.to_sink_hi = htobe32((u32)(wr->sg_list[0].addr >> 32)); 259321936Shselasky wqe->read.to_sink_lo = htobe32((u32)(wr->sg_list[0].addr)); 260321936Shselasky } else { 261321936Shselasky wqe->read.stag_src = htobe32(2); 262321936Shselasky wqe->read.to_src_hi = 0; 263321936Shselasky wqe->read.to_src_lo = 0; 264321936Shselasky wqe->read.stag_sink = htobe32(2); 265321936Shselasky wqe->read.plen = 0; 266321936Shselasky wqe->read.to_sink_hi = 0; 267321936Shselasky wqe->read.to_sink_lo = 0; 268321936Shselasky } 269321936Shselasky wqe->read.r2 = 0; 270321936Shselasky wqe->read.r5 = 0; 271321936Shselasky *len16 = DIV_ROUND_UP(sizeof wqe->read, 16); 272321936Shselasky return 0; 273321936Shselasky} 274321936Shselasky 275321936Shselaskystatic int build_rdma_recv(struct c4iw_qp *qhp, union t4_recv_wr *wqe, 276321936Shselasky struct ibv_recv_wr *wr, u8 *len16) 277321936Shselasky{ 278321936Shselasky int ret; 279321936Shselasky 280321936Shselasky ret = build_isgl(&wqe->recv.isgl, wr->sg_list, wr->num_sge, NULL); 281321936Shselasky if (ret) 282321936Shselasky return ret; 283321936Shselasky *len16 = DIV_ROUND_UP(sizeof wqe->recv + 284321936Shselasky wr->num_sge * sizeof(struct fw_ri_sge), 16); 285321936Shselasky return 0; 286321936Shselasky} 287321936Shselasky 288321936Shselaskystatic void ring_kernel_db(struct c4iw_qp *qhp, u32 qid, u16 idx) 289321936Shselasky{ 290321936Shselasky struct ibv_modify_qp cmd = {}; 291321936Shselasky struct ibv_qp_attr attr; 292321936Shselasky int mask; 293321936Shselasky int __attribute__((unused)) ret; 294321936Shselasky 295321936Shselasky /* FIXME: Why do we need this barrier if the kernel is going to 296321936Shselasky trigger the DMA? */ 297321936Shselasky udma_to_device_barrier(); 298321936Shselasky if (qid == qhp->wq.sq.qid) { 299321936Shselasky attr.sq_psn = idx; 300321936Shselasky mask = IBV_QP_SQ_PSN; 301321936Shselasky } else { 302321936Shselasky attr.rq_psn = idx; 303321936Shselasky mask = IBV_QP_RQ_PSN; 304321936Shselasky } 305321936Shselasky ret = ibv_cmd_modify_qp(&qhp->ibv_qp, &attr, mask, &cmd, sizeof cmd); 306321936Shselasky assert(!ret); 307321936Shselasky} 308321936Shselasky 309321936Shselaskyint c4iw_post_send(struct ibv_qp *ibqp, struct ibv_send_wr *wr, 310321936Shselasky struct ibv_send_wr **bad_wr) 311321936Shselasky{ 312321936Shselasky int err = 0; 313321936Shselasky u8 len16 = 0; 314321936Shselasky enum fw_wr_opcodes fw_opcode; 315321936Shselasky enum fw_ri_wr_flags fw_flags; 316321936Shselasky struct c4iw_qp *qhp; 317321936Shselasky union t4_wr *wqe, lwqe; 318321936Shselasky u32 num_wrs; 319321936Shselasky struct t4_swsqe *swsqe; 320321936Shselasky u16 idx = 0; 321321936Shselasky 322321936Shselasky qhp = to_c4iw_qp(ibqp); 323321936Shselasky pthread_spin_lock(&qhp->lock); 324321936Shselasky if (t4_wq_in_error(&qhp->wq)) { 325321936Shselasky pthread_spin_unlock(&qhp->lock); 326321936Shselasky *bad_wr = wr; 327321936Shselasky return -EINVAL; 328321936Shselasky } 329321936Shselasky num_wrs = t4_sq_avail(&qhp->wq); 330321936Shselasky if (num_wrs == 0) { 331321936Shselasky pthread_spin_unlock(&qhp->lock); 332321936Shselasky *bad_wr = wr; 333321936Shselasky return -ENOMEM; 334321936Shselasky } 335321936Shselasky while (wr) { 336321936Shselasky if (num_wrs == 0) { 337321936Shselasky err = -ENOMEM; 338321936Shselasky *bad_wr = wr; 339321936Shselasky break; 340321936Shselasky } 341321936Shselasky 342321936Shselasky wqe = &lwqe; 343321936Shselasky fw_flags = 0; 344321936Shselasky if (wr->send_flags & IBV_SEND_SOLICITED) 345321936Shselasky fw_flags |= FW_RI_SOLICITED_EVENT_FLAG; 346321936Shselasky if (wr->send_flags & IBV_SEND_SIGNALED || qhp->sq_sig_all) 347321936Shselasky fw_flags |= FW_RI_COMPLETION_FLAG; 348321936Shselasky swsqe = &qhp->wq.sq.sw_sq[qhp->wq.sq.pidx]; 349321936Shselasky switch (wr->opcode) { 350321936Shselasky case IBV_WR_SEND: 351321936Shselasky INC_STAT(send); 352321936Shselasky if (wr->send_flags & IBV_SEND_FENCE) 353321936Shselasky fw_flags |= FW_RI_READ_FENCE_FLAG; 354321936Shselasky fw_opcode = FW_RI_SEND_WR; 355321936Shselasky swsqe->opcode = FW_RI_SEND; 356321936Shselasky err = build_rdma_send(&qhp->wq.sq, wqe, wr, &len16); 357321936Shselasky break; 358321936Shselasky case IBV_WR_RDMA_WRITE: 359321936Shselasky INC_STAT(write); 360321936Shselasky fw_opcode = FW_RI_RDMA_WRITE_WR; 361321936Shselasky swsqe->opcode = FW_RI_RDMA_WRITE; 362321936Shselasky err = build_rdma_write(&qhp->wq.sq, wqe, wr, &len16); 363321936Shselasky break; 364321936Shselasky case IBV_WR_RDMA_READ: 365321936Shselasky INC_STAT(read); 366321936Shselasky fw_opcode = FW_RI_RDMA_READ_WR; 367321936Shselasky swsqe->opcode = FW_RI_READ_REQ; 368321936Shselasky fw_flags = 0; 369321936Shselasky err = build_rdma_read(wqe, wr, &len16); 370321936Shselasky if (err) 371321936Shselasky break; 372321936Shselasky swsqe->read_len = wr->sg_list ? wr->sg_list[0].length : 373321936Shselasky 0; 374321936Shselasky if (!qhp->wq.sq.oldest_read) 375321936Shselasky qhp->wq.sq.oldest_read = swsqe; 376321936Shselasky break; 377321936Shselasky default: 378321936Shselasky PDBG("%s post of type=%d TBD!\n", __func__, 379321936Shselasky wr->opcode); 380321936Shselasky err = -EINVAL; 381321936Shselasky } 382321936Shselasky if (err) { 383321936Shselasky *bad_wr = wr; 384321936Shselasky break; 385321936Shselasky } 386321936Shselasky swsqe->idx = qhp->wq.sq.pidx; 387321936Shselasky swsqe->complete = 0; 388321936Shselasky swsqe->signaled = (wr->send_flags & IBV_SEND_SIGNALED) || 389321936Shselasky qhp->sq_sig_all; 390321936Shselasky swsqe->flushed = 0; 391321936Shselasky swsqe->wr_id = wr->wr_id; 392321936Shselasky 393321936Shselasky init_wr_hdr(wqe, qhp->wq.sq.pidx, fw_opcode, fw_flags, len16); 394321936Shselasky PDBG("%s cookie 0x%llx pidx 0x%x opcode 0x%x\n", 395321936Shselasky __func__, (unsigned long long)wr->wr_id, qhp->wq.sq.pidx, 396321936Shselasky swsqe->opcode); 397321936Shselasky wr = wr->next; 398321936Shselasky num_wrs--; 399321936Shselasky copy_wr_to_sq(&qhp->wq, wqe, len16); 400321936Shselasky t4_sq_produce(&qhp->wq, len16); 401321936Shselasky idx += DIV_ROUND_UP(len16*16, T4_EQ_ENTRY_SIZE); 402321936Shselasky } 403321936Shselasky if (t4_wq_db_enabled(&qhp->wq)) { 404321936Shselasky t4_ring_sq_db(&qhp->wq, idx, dev_is_t4(qhp->rhp), 405321936Shselasky len16, wqe); 406321936Shselasky } else 407321936Shselasky ring_kernel_db(qhp, qhp->wq.sq.qid, idx); 408321936Shselasky /* This write is only for debugging, the value does not matter for DMA 409321936Shselasky */ 410321936Shselasky qhp->wq.sq.queue[qhp->wq.sq.size].status.host_wq_pidx = \ 411321936Shselasky (qhp->wq.sq.wq_pidx); 412321936Shselasky 413321936Shselasky pthread_spin_unlock(&qhp->lock); 414321936Shselasky return err; 415321936Shselasky} 416321936Shselasky 417321936Shselaskyint c4iw_post_receive(struct ibv_qp *ibqp, struct ibv_recv_wr *wr, 418321936Shselasky struct ibv_recv_wr **bad_wr) 419321936Shselasky{ 420321936Shselasky int err = 0; 421321936Shselasky struct c4iw_qp *qhp; 422321936Shselasky union t4_recv_wr *wqe, lwqe; 423321936Shselasky u32 num_wrs; 424321936Shselasky u8 len16 = 0; 425321936Shselasky u16 idx = 0; 426321936Shselasky 427321936Shselasky qhp = to_c4iw_qp(ibqp); 428321936Shselasky pthread_spin_lock(&qhp->lock); 429321936Shselasky if (t4_wq_in_error(&qhp->wq)) { 430321936Shselasky pthread_spin_unlock(&qhp->lock); 431321936Shselasky *bad_wr = wr; 432321936Shselasky return -EINVAL; 433321936Shselasky } 434321936Shselasky INC_STAT(recv); 435321936Shselasky num_wrs = t4_rq_avail(&qhp->wq); 436321936Shselasky if (num_wrs == 0) { 437321936Shselasky pthread_spin_unlock(&qhp->lock); 438321936Shselasky *bad_wr = wr; 439321936Shselasky return -ENOMEM; 440321936Shselasky } 441321936Shselasky while (wr) { 442321936Shselasky if (wr->num_sge > T4_MAX_RECV_SGE) { 443321936Shselasky err = -EINVAL; 444321936Shselasky *bad_wr = wr; 445321936Shselasky break; 446321936Shselasky } 447321936Shselasky wqe = &lwqe; 448321936Shselasky if (num_wrs) 449321936Shselasky err = build_rdma_recv(qhp, wqe, wr, &len16); 450321936Shselasky else 451321936Shselasky err = -ENOMEM; 452321936Shselasky if (err) { 453321936Shselasky *bad_wr = wr; 454321936Shselasky break; 455321936Shselasky } 456321936Shselasky 457321936Shselasky qhp->wq.rq.sw_rq[qhp->wq.rq.pidx].wr_id = wr->wr_id; 458321936Shselasky 459321936Shselasky wqe->recv.opcode = FW_RI_RECV_WR; 460321936Shselasky wqe->recv.r1 = 0; 461321936Shselasky wqe->recv.wrid = qhp->wq.rq.pidx; 462321936Shselasky wqe->recv.r2[0] = 0; 463321936Shselasky wqe->recv.r2[1] = 0; 464321936Shselasky wqe->recv.r2[2] = 0; 465321936Shselasky wqe->recv.len16 = len16; 466321936Shselasky PDBG("%s cookie 0x%llx pidx %u\n", __func__, 467321936Shselasky (unsigned long long) wr->wr_id, qhp->wq.rq.pidx); 468321936Shselasky copy_wr_to_rq(&qhp->wq, wqe, len16); 469321936Shselasky t4_rq_produce(&qhp->wq, len16); 470321936Shselasky idx += DIV_ROUND_UP(len16*16, T4_EQ_ENTRY_SIZE); 471321936Shselasky wr = wr->next; 472321936Shselasky num_wrs--; 473321936Shselasky } 474321936Shselasky if (t4_wq_db_enabled(&qhp->wq)) 475321936Shselasky t4_ring_rq_db(&qhp->wq, idx, dev_is_t4(qhp->rhp), 476321936Shselasky len16, wqe); 477321936Shselasky else 478321936Shselasky ring_kernel_db(qhp, qhp->wq.rq.qid, idx); 479321936Shselasky qhp->wq.rq.queue[qhp->wq.rq.size].status.host_wq_pidx = \ 480321936Shselasky (qhp->wq.rq.wq_pidx); 481321936Shselasky pthread_spin_unlock(&qhp->lock); 482321936Shselasky return err; 483321936Shselasky} 484321936Shselasky 485321936Shselaskystatic void update_qp_state(struct c4iw_qp *qhp) 486321936Shselasky{ 487321936Shselasky struct ibv_query_qp cmd; 488321936Shselasky struct ibv_qp_attr attr; 489321936Shselasky struct ibv_qp_init_attr iattr; 490321936Shselasky int ret; 491321936Shselasky 492321936Shselasky ret = ibv_cmd_query_qp(&qhp->ibv_qp, &attr, IBV_QP_STATE, &iattr, 493321936Shselasky &cmd, sizeof cmd); 494321936Shselasky assert(!ret); 495321936Shselasky if (!ret) 496321936Shselasky qhp->ibv_qp.state = attr.qp_state; 497321936Shselasky} 498321936Shselasky 499321936Shselasky/* 500321936Shselasky * Assumes qhp lock is held. 501321936Shselasky */ 502321936Shselaskyvoid c4iw_flush_qp(struct c4iw_qp *qhp) 503321936Shselasky{ 504321936Shselasky struct c4iw_cq *rchp, *schp; 505321936Shselasky int count; 506321936Shselasky 507321936Shselasky if (qhp->wq.flushed) 508321936Shselasky return; 509321936Shselasky 510321936Shselasky update_qp_state(qhp); 511321936Shselasky 512321936Shselasky rchp = to_c4iw_cq(qhp->ibv_qp.recv_cq); 513321936Shselasky schp = to_c4iw_cq(qhp->ibv_qp.send_cq); 514321936Shselasky 515321936Shselasky PDBG("%s qhp %p rchp %p schp %p\n", __func__, qhp, rchp, schp); 516321936Shselasky qhp->wq.flushed = 1; 517321936Shselasky pthread_spin_unlock(&qhp->lock); 518321936Shselasky 519321936Shselasky /* locking heirarchy: cq lock first, then qp lock. */ 520321936Shselasky pthread_spin_lock(&rchp->lock); 521321936Shselasky pthread_spin_lock(&qhp->lock); 522321936Shselasky c4iw_flush_hw_cq(rchp); 523321936Shselasky c4iw_count_rcqes(&rchp->cq, &qhp->wq, &count); 524321936Shselasky c4iw_flush_rq(&qhp->wq, &rchp->cq, count); 525321936Shselasky pthread_spin_unlock(&qhp->lock); 526321936Shselasky pthread_spin_unlock(&rchp->lock); 527321936Shselasky 528321936Shselasky /* locking heirarchy: cq lock first, then qp lock. */ 529321936Shselasky pthread_spin_lock(&schp->lock); 530321936Shselasky pthread_spin_lock(&qhp->lock); 531321936Shselasky if (schp != rchp) 532321936Shselasky c4iw_flush_hw_cq(schp); 533321936Shselasky c4iw_flush_sq(qhp); 534321936Shselasky pthread_spin_unlock(&qhp->lock); 535321936Shselasky pthread_spin_unlock(&schp->lock); 536321936Shselasky pthread_spin_lock(&qhp->lock); 537321936Shselasky} 538321936Shselasky 539321936Shselaskyvoid c4iw_flush_qps(struct c4iw_dev *dev) 540321936Shselasky{ 541321936Shselasky int i; 542321936Shselasky 543321936Shselasky pthread_spin_lock(&dev->lock); 544321936Shselasky for (i=0; i < dev->max_qp; i++) { 545321936Shselasky struct c4iw_qp *qhp = dev->qpid2ptr[i]; 546321936Shselasky if (qhp) { 547321936Shselasky if (!qhp->wq.flushed && t4_wq_in_error(&qhp->wq)) { 548321936Shselasky pthread_spin_lock(&qhp->lock); 549321936Shselasky c4iw_flush_qp(qhp); 550321936Shselasky pthread_spin_unlock(&qhp->lock); 551321936Shselasky } 552321936Shselasky } 553321936Shselasky } 554321936Shselasky pthread_spin_unlock(&dev->lock); 555321936Shselasky} 556