1178786Skmacy/************************************************************************** 2178786Skmacy 3178786SkmacyCopyright (c) 2007, Chelsio Inc. 4178786SkmacyAll rights reserved. 5178786Skmacy 6178786SkmacyRedistribution and use in source and binary forms, with or without 7178786Skmacymodification, are permitted provided that the following conditions are met: 8178786Skmacy 9178786Skmacy 1. Redistributions of source code must retain the above copyright notice, 10178786Skmacy this list of conditions and the following disclaimer. 11178786Skmacy 12178786Skmacy 2. Neither the name of the Chelsio Corporation nor the names of its 13178786Skmacy contributors may be used to endorse or promote products derived from 14178786Skmacy this software without specific prior written permission. 15178786Skmacy 16178786SkmacyTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 17178786SkmacyAND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18178786SkmacyIMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19178786SkmacyARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 20178786SkmacyLIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21178786SkmacyCONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22178786SkmacySUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23178786SkmacyINTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24178786SkmacyCONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25178786SkmacyARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26178786SkmacyPOSSIBILITY OF SUCH DAMAGE. 27178786Skmacy 28178786Skmacy***************************************************************************/ 29178786Skmacy#include <sys/cdefs.h> 30178786Skmacy__FBSDID("$FreeBSD$"); 31178786Skmacy 32237263Snp#include "opt_inet.h" 33237263Snp 34237263Snp#ifdef TCP_OFFLOAD 35178786Skmacy#include <sys/param.h> 36178786Skmacy#include <sys/systm.h> 37178786Skmacy#include <sys/kernel.h> 38178786Skmacy#include <sys/bus.h> 39178786Skmacy#include <sys/pciio.h> 40178786Skmacy#include <sys/conf.h> 41178786Skmacy#include <machine/bus.h> 42178786Skmacy#include <machine/resource.h> 43178786Skmacy#include <sys/bus_dma.h> 44178786Skmacy#include <sys/rman.h> 45178786Skmacy#include <sys/ioccom.h> 46178786Skmacy#include <sys/mbuf.h> 47178786Skmacy#include <sys/rwlock.h> 48178786Skmacy#include <sys/linker.h> 49178786Skmacy#include <sys/firmware.h> 50178786Skmacy#include <sys/socket.h> 51237263Snp#include <sys/socketvar.h> 52237263Snp#include <sys/sockopt.h> 53178786Skmacy#include <sys/sockio.h> 54178786Skmacy#include <sys/smp.h> 55178786Skmacy#include <sys/sysctl.h> 56178786Skmacy#include <sys/syslog.h> 57178786Skmacy#include <sys/queue.h> 58178786Skmacy#include <sys/taskqueue.h> 59178786Skmacy#include <sys/proc.h> 60178786Skmacy#include <sys/queue.h> 61178786Skmacy 62178786Skmacy#include <vm/vm.h> 63178786Skmacy#include <vm/pmap.h> 64178786Skmacy 65237263Snp#include <net/route.h> 66237263Snp#include <netinet/in_systm.h> 67178786Skmacy#include <netinet/in.h> 68237263Snp#include <netinet/in_pcb.h> 69237263Snp#include <netinet/ip.h> 70237263Snp#include <netinet/ip_var.h> 71237263Snp#include <netinet/tcp_var.h> 72237263Snp#include <netinet/toecore.h> 73237263Snp#include <netinet/tcp.h> 74237263Snp#include <netinet/tcpip.h> 75178786Skmacy 76237263Snp#include <rdma/ib_verbs.h> 77237263Snp#include <linux/idr.h> 78237263Snp#include <ulp/iw_cxgb/iw_cxgb_ib_intfc.h> 79178786Skmacy 80178786Skmacy#include <cxgb_include.h> 81183321Skmacy#include <ulp/tom/cxgb_l2t.h> 82237263Snp#include <ulp/tom/cxgb_tom.h> 83237263Snp#include <ulp/tom/cxgb_toepcb.h> 84178786Skmacy#include <ulp/iw_cxgb/iw_cxgb_wr.h> 85178786Skmacy#include <ulp/iw_cxgb/iw_cxgb_hal.h> 86178786Skmacy#include <ulp/iw_cxgb/iw_cxgb_provider.h> 87178786Skmacy#include <ulp/iw_cxgb/iw_cxgb_cm.h> 88178786Skmacy#include <ulp/iw_cxgb/iw_cxgb.h> 89178786Skmacy#include <ulp/iw_cxgb/iw_cxgb_resource.h> 90178786Skmacy 91237263Snp/* Response queue used for RDMA events. */ 92237263Snp#define ASYNC_NOTIF_RSPQ 0 93237263Snpstatic inline int 94237263Snpcxio_rdma_cq_setup(struct cxio_rdev *rdev_p, unsigned id, uint64_t base_addr, 95237263Snp unsigned size, unsigned ovfl_mode, unsigned credits, unsigned credit_thres) 96178786Skmacy{ 97237263Snp struct adapter *sc = rdev_p->adap; 98237263Snp int rc; 99178786Skmacy 100237263Snp mtx_lock_spin(&sc->sge.reg_lock); 101237263Snp rc = -t3_sge_init_cqcntxt(sc, id, base_addr, size, ASYNC_NOTIF_RSPQ, 102237263Snp ovfl_mode, credits, credit_thres); 103237263Snp mtx_unlock_spin(&sc->sge.reg_lock); 104178786Skmacy 105237263Snp return (rc); 106178786Skmacy} 107178786Skmacy 108178786Skmacyint 109178786Skmacycxio_hal_cq_op(struct cxio_rdev *rdev_p, struct t3_cq *cq, 110178786Skmacy enum t3_cq_opcode op, u32 credit) 111178786Skmacy{ 112178786Skmacy int ret; 113178786Skmacy struct t3_cqe *cqe; 114178786Skmacy u32 rptr; 115237263Snp struct adapter *sc = rdev_p->adap; 116178786Skmacy 117237263Snp if (op != CQ_CREDIT_UPDATE) 118237263Snp credit = 0; 119178786Skmacy 120237263Snp mtx_lock_spin(&sc->sge.reg_lock); 121237263Snp ret = t3_sge_cqcntxt_op(sc, cq->cqid, op, credit); 122237263Snp mtx_unlock_spin(&sc->sge.reg_lock); 123237263Snp 124178786Skmacy if ((ret < 0) || (op == CQ_CREDIT_UPDATE)) 125178786Skmacy return (ret); 126178786Skmacy 127178786Skmacy /* 128178786Skmacy * If the rearm returned an index other than our current index, 129178786Skmacy * then there might be CQE's in flight (being DMA'd). We must wait 130178786Skmacy * here for them to complete or the consumer can miss a notification. 131178786Skmacy */ 132178786Skmacy if (Q_PTR2IDX((cq->rptr), cq->size_log2) != ret) { 133178786Skmacy int i=0; 134178786Skmacy 135178786Skmacy rptr = cq->rptr; 136178786Skmacy 137178786Skmacy /* 138178786Skmacy * Keep the generation correct by bumping rptr until it 139178786Skmacy * matches the index returned by the rearm - 1. 140178786Skmacy */ 141178786Skmacy while (Q_PTR2IDX((rptr+1), cq->size_log2) != ret) 142178786Skmacy rptr++; 143178786Skmacy 144178786Skmacy /* 145178786Skmacy * Now rptr is the index for the (last) cqe that was 146178786Skmacy * in-flight at the time the HW rearmed the CQ. We 147178786Skmacy * spin until that CQE is valid. 148178786Skmacy */ 149178786Skmacy cqe = cq->queue + Q_PTR2IDX(rptr, cq->size_log2); 150178786Skmacy while (!CQ_VLD_ENTRY(rptr, cq->size_log2, cqe)) { 151178786Skmacy DELAY(1); 152178786Skmacy if (i++ > 1000000) { 153237263Snp struct adapter *sc = rdev_p->adap; 154237263Snp 155237263Snp log(LOG_ERR, "%s: stalled rnic\n", 156237263Snp device_get_nameunit(sc->dev)); 157178786Skmacy PANIC_IF(1); 158178786Skmacy return (-EIO); 159178786Skmacy } 160178786Skmacy } 161178786Skmacy 162237263Snp return (1); 163178786Skmacy } 164178786Skmacy 165237263Snp return (0); 166178786Skmacy} 167178786Skmacy 168178786Skmacystatic int 169178786Skmacycxio_hal_clear_cq_ctx(struct cxio_rdev *rdev_p, u32 cqid) 170178786Skmacy{ 171237263Snp 172237263Snp return (cxio_rdma_cq_setup(rdev_p, cqid, 0, 0, 0, 0, 0)); 173178786Skmacy} 174178786Skmacy 175178786Skmacystatic int 176178786Skmacycxio_hal_clear_qp_ctx(struct cxio_rdev *rdev_p, u32 qpid) 177178786Skmacy{ 178178786Skmacy u64 sge_cmd; 179178786Skmacy struct t3_modify_qp_wr *wqe; 180237263Snp struct mbuf *m; 181237263Snp 182237263Snp m = M_GETHDR_OFLD(0, CPL_PRIORITY_CONTROL, wqe); 183178786Skmacy if (m == NULL) { 184178786Skmacy CTR1(KTR_IW_CXGB, "%s m_gethdr failed", __FUNCTION__); 185178786Skmacy return (-ENOMEM); 186178786Skmacy } 187178786Skmacy wqe = mtod(m, struct t3_modify_qp_wr *); 188178786Skmacy memset(wqe, 0, sizeof(*wqe)); 189178786Skmacy build_fw_riwrh((struct fw_riwrh *) wqe, T3_WR_QP_MOD, 3, 0, qpid, 7); 190178786Skmacy wqe->flags = htobe32(MODQP_WRITE_EC); 191178786Skmacy sge_cmd = qpid << 8 | 3; 192178786Skmacy wqe->sge_cmd = htobe64(sge_cmd); 193237263Snp return t3_offload_tx(rdev_p->adap, m); 194178786Skmacy} 195178786Skmacy 196178786Skmacyint 197237263Snpcxio_create_cq(struct cxio_rdev *rdev_p, struct t3_cq *cq, int kernel) 198178786Skmacy{ 199178786Skmacy int size = (1UL << (cq->size_log2)) * sizeof(struct t3_cqe); 200178786Skmacy 201237263Snp size += 1; /* one extra page for storing cq-in-err state */ 202178786Skmacy cq->cqid = cxio_hal_get_cqid(rdev_p->rscp); 203178786Skmacy if (!cq->cqid) 204178786Skmacy return (-ENOMEM); 205237263Snp if (kernel) { 206237263Snp cq->sw_queue = malloc(size, M_DEVBUF, M_NOWAIT|M_ZERO); 207237263Snp if (!cq->sw_queue) 208237263Snp return (-ENOMEM); 209237263Snp } 210237263Snp 211237263Snp cq->queue = contigmalloc(size, 212178786Skmacy M_DEVBUF, M_NOWAIT, 0ul, ~0ul, 4096, 0); 213178786Skmacy if (cq->queue) 214178786Skmacy cq->dma_addr = vtophys(cq->queue); 215178786Skmacy else { 216178786Skmacy free(cq->sw_queue, M_DEVBUF); 217178786Skmacy return (-ENOMEM); 218178786Skmacy } 219178786Skmacy memset(cq->queue, 0, size); 220178786Skmacy 221237263Snp return (cxio_rdma_cq_setup(rdev_p, cq->cqid, cq->dma_addr, 222237263Snp 1UL << cq->size_log2, 0, 65535, 1)); 223178786Skmacy} 224178786Skmacy 225178786Skmacystatic u32 226178786Skmacyget_qpid(struct cxio_rdev *rdev_p, struct cxio_ucontext *uctx) 227178786Skmacy{ 228178786Skmacy struct cxio_qpid *entry; 229178786Skmacy u32 qpid; 230178786Skmacy int i; 231178786Skmacy 232178786Skmacy mtx_lock(&uctx->lock); 233178786Skmacy if (!TAILQ_EMPTY(&uctx->qpids)) { 234178786Skmacy 235178786Skmacy entry = TAILQ_FIRST(&uctx->qpids); 236178786Skmacy TAILQ_REMOVE(&uctx->qpids, entry, entry); 237178786Skmacy qpid = entry->qpid; 238178786Skmacy free(entry, M_DEVBUF); 239178786Skmacy } else { 240178786Skmacy qpid = cxio_hal_get_qpid(rdev_p->rscp); 241178786Skmacy if (!qpid) 242178786Skmacy goto out; 243178786Skmacy for (i = qpid+1; i & rdev_p->qpmask; i++) { 244178786Skmacy entry = malloc(sizeof *entry, M_DEVBUF, M_NOWAIT); 245178786Skmacy if (!entry) 246178786Skmacy break; 247178786Skmacy entry->qpid = i; 248178786Skmacy TAILQ_INSERT_TAIL(&uctx->qpids, entry, entry); 249178786Skmacy } 250178786Skmacy } 251178786Skmacyout: 252178786Skmacy mtx_unlock(&uctx->lock); 253178786Skmacy CTR2(KTR_IW_CXGB, "%s qpid 0x%x", __FUNCTION__, qpid); 254178786Skmacy return qpid; 255178786Skmacy} 256178786Skmacy 257178786Skmacystatic void 258178786Skmacyput_qpid(struct cxio_rdev *rdev_p, u32 qpid, 259178786Skmacy struct cxio_ucontext *uctx) 260178786Skmacy{ 261178786Skmacy struct cxio_qpid *entry; 262178786Skmacy 263178786Skmacy entry = malloc(sizeof *entry, M_DEVBUF, M_NOWAIT); 264178786Skmacy CTR2(KTR_IW_CXGB, "%s qpid 0x%x", __FUNCTION__, qpid); 265178786Skmacy entry->qpid = qpid; 266178786Skmacy mtx_lock(&uctx->lock); 267178786Skmacy TAILQ_INSERT_TAIL(&uctx->qpids, entry, entry); 268178786Skmacy mtx_unlock(&uctx->lock); 269178786Skmacy} 270178786Skmacy 271178786Skmacyvoid 272178786Skmacycxio_release_ucontext(struct cxio_rdev *rdev_p, struct cxio_ucontext *uctx) 273178786Skmacy{ 274178786Skmacy struct cxio_qpid *pos, *tmp; 275178786Skmacy 276178786Skmacy mtx_lock(&uctx->lock); 277178786Skmacy TAILQ_FOREACH_SAFE(pos, &uctx->qpids, entry, tmp) { 278178786Skmacy TAILQ_REMOVE(&uctx->qpids, pos, entry); 279178786Skmacy if (!(pos->qpid & rdev_p->qpmask)) 280178786Skmacy cxio_hal_put_qpid(rdev_p->rscp, pos->qpid); 281178786Skmacy free(pos, M_DEVBUF); 282178786Skmacy } 283178786Skmacy mtx_unlock(&uctx->lock); 284178786Skmacy} 285178786Skmacy 286178786Skmacyvoid 287178786Skmacycxio_init_ucontext(struct cxio_rdev *rdev_p, struct cxio_ucontext *uctx) 288178786Skmacy{ 289178786Skmacy TAILQ_INIT(&uctx->qpids); 290178786Skmacy mtx_init(&uctx->lock, "cxio uctx", NULL, MTX_DEF|MTX_DUPOK); 291178786Skmacy} 292178786Skmacy 293178786Skmacyint 294178786Skmacycxio_create_qp(struct cxio_rdev *rdev_p, u32 kernel_domain, 295178786Skmacy struct t3_wq *wq, struct cxio_ucontext *uctx) 296178786Skmacy{ 297178786Skmacy int depth = 1UL << wq->size_log2; 298178786Skmacy int rqsize = 1UL << wq->rq_size_log2; 299178786Skmacy 300178786Skmacy wq->qpid = get_qpid(rdev_p, uctx); 301178786Skmacy if (!wq->qpid) 302178786Skmacy return (-ENOMEM); 303178786Skmacy 304237263Snp wq->rq = malloc(depth * sizeof(struct t3_swrq), M_DEVBUF, M_NOWAIT|M_ZERO); 305178786Skmacy if (!wq->rq) 306178786Skmacy goto err1; 307178786Skmacy 308178786Skmacy wq->rq_addr = cxio_hal_rqtpool_alloc(rdev_p, rqsize); 309178786Skmacy if (!wq->rq_addr) 310178786Skmacy goto err2; 311178786Skmacy 312178786Skmacy wq->sq = malloc(depth * sizeof(struct t3_swsq), M_DEVBUF, M_NOWAIT|M_ZERO); 313178786Skmacy if (!wq->sq) 314178786Skmacy goto err3; 315178786Skmacy wq->queue = contigmalloc(depth *sizeof(union t3_wr), 316178786Skmacy M_DEVBUF, M_NOWAIT, 0ul, ~0ul, 4096, 0); 317178786Skmacy if (wq->queue) 318178786Skmacy wq->dma_addr = vtophys(wq->queue); 319237263Snp else 320178786Skmacy goto err4; 321178786Skmacy 322178786Skmacy memset(wq->queue, 0, depth * sizeof(union t3_wr)); 323178786Skmacy wq->doorbell = rdev_p->rnic_info.kdb_addr; 324178786Skmacy if (!kernel_domain) 325178786Skmacy wq->udb = (u64)rdev_p->rnic_info.udbell_physbase + 326178786Skmacy (wq->qpid << rdev_p->qpshift); 327237263Snp wq->rdev = rdev_p; 328178786Skmacy CTR4(KTR_IW_CXGB, "%s qpid 0x%x doorbell 0x%p udb 0x%llx", __FUNCTION__, 329178786Skmacy wq->qpid, wq->doorbell, (unsigned long long) wq->udb); 330178786Skmacy return 0; 331178786Skmacyerr4: 332178786Skmacy free(wq->sq, M_DEVBUF); 333178786Skmacyerr3: 334178786Skmacy cxio_hal_rqtpool_free(rdev_p, wq->rq_addr, rqsize); 335178786Skmacyerr2: 336178786Skmacy free(wq->rq, M_DEVBUF); 337178786Skmacyerr1: 338178786Skmacy put_qpid(rdev_p, wq->qpid, uctx); 339178786Skmacy return (-ENOMEM); 340178786Skmacy} 341178786Skmacy 342178786Skmacyint 343178786Skmacycxio_destroy_cq(struct cxio_rdev *rdev_p, struct t3_cq *cq) 344178786Skmacy{ 345178786Skmacy int err; 346178786Skmacy err = cxio_hal_clear_cq_ctx(rdev_p, cq->cqid); 347178786Skmacy free(cq->sw_queue, M_DEVBUF); 348178786Skmacy#if 0 349178786Skmacy dma_free_coherent(&(rdev_p->rnic_info.pdev), 350178786Skmacy (1UL << (cq->size_log2)) 351178786Skmacy * sizeof(struct t3_cqe), cq->queue, 352178786Skmacy /* pci_unmap_addr(cq, mapping)*/ 0); 353178786Skmacy#else 354178786Skmacy contigfree(cq->queue,(1UL << (cq->size_log2)) 355178786Skmacy * sizeof(struct t3_cqe), M_DEVBUF); 356178786Skmacy#endif 357178786Skmacy cxio_hal_put_cqid(rdev_p->rscp, cq->cqid); 358178786Skmacy return err; 359178786Skmacy} 360178786Skmacy 361178786Skmacyint 362178786Skmacycxio_destroy_qp(struct cxio_rdev *rdev_p, struct t3_wq *wq, 363178786Skmacy struct cxio_ucontext *uctx) 364178786Skmacy{ 365178786Skmacy 366178786Skmacy#if 0 367178786Skmacy dma_free_coherent(&(rdev_p->rnic_info.pdev), 368178786Skmacy (1UL << (wq->size_log2)) 369178786Skmacy * sizeof(union t3_wr), wq->queue, 370178786Skmacy /* pci_unmap_addr(wq, mapping)*/ 0); 371178786Skmacy#else 372178786Skmacy contigfree(wq->queue, (1UL << (wq->size_log2)) 373178786Skmacy * sizeof(union t3_wr), M_DEVBUF); 374178786Skmacy#endif 375178786Skmacy free(wq->sq, M_DEVBUF); 376178786Skmacy cxio_hal_rqtpool_free(rdev_p, wq->rq_addr, (1UL << wq->rq_size_log2)); 377178786Skmacy free(wq->rq, M_DEVBUF); 378178786Skmacy put_qpid(rdev_p, wq->qpid, uctx); 379178786Skmacy return 0; 380178786Skmacy} 381178786Skmacy 382178786Skmacystatic void 383178786Skmacyinsert_recv_cqe(struct t3_wq *wq, struct t3_cq *cq) 384178786Skmacy{ 385178786Skmacy struct t3_cqe cqe; 386178786Skmacy 387178786Skmacy CTR5(KTR_IW_CXGB, "%s wq %p cq %p sw_rptr 0x%x sw_wptr 0x%x", __FUNCTION__, 388178786Skmacy wq, cq, cq->sw_rptr, cq->sw_wptr); 389178786Skmacy memset(&cqe, 0, sizeof(cqe)); 390178786Skmacy cqe.header = htobe32(V_CQE_STATUS(TPT_ERR_SWFLUSH) | 391178786Skmacy V_CQE_OPCODE(T3_SEND) | 392178786Skmacy V_CQE_TYPE(0) | 393178786Skmacy V_CQE_SWCQE(1) | 394178786Skmacy V_CQE_QPID(wq->qpid) | 395178786Skmacy V_CQE_GENBIT(Q_GENBIT(cq->sw_wptr, 396178786Skmacy cq->size_log2))); 397178786Skmacy *(cq->sw_queue + Q_PTR2IDX(cq->sw_wptr, cq->size_log2)) = cqe; 398178786Skmacy cq->sw_wptr++; 399178786Skmacy} 400178786Skmacy 401237263Snpint 402178786Skmacycxio_flush_rq(struct t3_wq *wq, struct t3_cq *cq, int count) 403178786Skmacy{ 404178786Skmacy u32 ptr; 405237263Snp int flushed = 0; 406178786Skmacy 407178786Skmacy CTR3(KTR_IW_CXGB, "%s wq %p cq %p", __FUNCTION__, wq, cq); 408178786Skmacy 409178786Skmacy /* flush RQ */ 410178786Skmacy CTR4(KTR_IW_CXGB, "%s rq_rptr %u rq_wptr %u skip count %u", __FUNCTION__, 411178786Skmacy wq->rq_rptr, wq->rq_wptr, count); 412178786Skmacy ptr = wq->rq_rptr + count; 413237263Snp while (ptr++ != wq->rq_wptr) { 414178786Skmacy insert_recv_cqe(wq, cq); 415237263Snp flushed++; 416237263Snp } 417237263Snp return flushed; 418178786Skmacy} 419178786Skmacy 420178786Skmacystatic void 421178786Skmacyinsert_sq_cqe(struct t3_wq *wq, struct t3_cq *cq, 422178786Skmacy struct t3_swsq *sqp) 423178786Skmacy{ 424178786Skmacy struct t3_cqe cqe; 425178786Skmacy 426178786Skmacy CTR5(KTR_IW_CXGB, "%s wq %p cq %p sw_rptr 0x%x sw_wptr 0x%x", __FUNCTION__, 427178786Skmacy wq, cq, cq->sw_rptr, cq->sw_wptr); 428178786Skmacy memset(&cqe, 0, sizeof(cqe)); 429178786Skmacy cqe.header = htobe32(V_CQE_STATUS(TPT_ERR_SWFLUSH) | 430178786Skmacy V_CQE_OPCODE(sqp->opcode) | 431178786Skmacy V_CQE_TYPE(1) | 432178786Skmacy V_CQE_SWCQE(1) | 433178786Skmacy V_CQE_QPID(wq->qpid) | 434178786Skmacy V_CQE_GENBIT(Q_GENBIT(cq->sw_wptr, 435178786Skmacy cq->size_log2))); 436178786Skmacy cqe.u.scqe.wrid_hi = sqp->sq_wptr; 437178786Skmacy 438178786Skmacy *(cq->sw_queue + Q_PTR2IDX(cq->sw_wptr, cq->size_log2)) = cqe; 439178786Skmacy cq->sw_wptr++; 440178786Skmacy} 441178786Skmacy 442237263Snpint 443178786Skmacycxio_flush_sq(struct t3_wq *wq, struct t3_cq *cq, int count) 444178786Skmacy{ 445178786Skmacy __u32 ptr; 446237263Snp int flushed = 0; 447178786Skmacy struct t3_swsq *sqp = wq->sq + Q_PTR2IDX(wq->sq_rptr, wq->sq_size_log2); 448178786Skmacy 449178786Skmacy ptr = wq->sq_rptr + count; 450237263Snp sqp = wq->sq + Q_PTR2IDX(ptr, wq->sq_size_log2); 451178786Skmacy while (ptr != wq->sq_wptr) { 452178786Skmacy insert_sq_cqe(wq, cq, sqp); 453178786Skmacy ptr++; 454237263Snp sqp = wq->sq + Q_PTR2IDX(ptr, wq->sq_size_log2); 455237263Snp flushed++; 456178786Skmacy } 457237263Snp return flushed; 458178786Skmacy} 459178786Skmacy 460178786Skmacy/* 461178786Skmacy * Move all CQEs from the HWCQ into the SWCQ. 462178786Skmacy */ 463178786Skmacyvoid 464178786Skmacycxio_flush_hw_cq(struct t3_cq *cq) 465178786Skmacy{ 466178786Skmacy struct t3_cqe *cqe, *swcqe; 467178786Skmacy 468178786Skmacy CTR3(KTR_IW_CXGB, "%s cq %p cqid 0x%x", __FUNCTION__, cq, cq->cqid); 469178786Skmacy cqe = cxio_next_hw_cqe(cq); 470178786Skmacy while (cqe) { 471178786Skmacy CTR3(KTR_IW_CXGB, "%s flushing hwcq rptr 0x%x to swcq wptr 0x%x", 472178786Skmacy __FUNCTION__, cq->rptr, cq->sw_wptr); 473178786Skmacy swcqe = cq->sw_queue + Q_PTR2IDX(cq->sw_wptr, cq->size_log2); 474178786Skmacy *swcqe = *cqe; 475178786Skmacy swcqe->header |= htobe32(V_CQE_SWCQE(1)); 476178786Skmacy cq->sw_wptr++; 477178786Skmacy cq->rptr++; 478178786Skmacy cqe = cxio_next_hw_cqe(cq); 479178786Skmacy } 480178786Skmacy} 481178786Skmacy 482178786Skmacystatic int cqe_completes_wr(struct t3_cqe *cqe, struct t3_wq *wq) 483178786Skmacy{ 484178786Skmacy if (CQE_OPCODE(*cqe) == T3_TERMINATE) 485178786Skmacy return 0; 486178786Skmacy 487178786Skmacy if ((CQE_OPCODE(*cqe) == T3_RDMA_WRITE) && RQ_TYPE(*cqe)) 488178786Skmacy return 0; 489178786Skmacy 490178786Skmacy if ((CQE_OPCODE(*cqe) == T3_READ_RESP) && SQ_TYPE(*cqe)) 491178786Skmacy return 0; 492178786Skmacy 493237263Snp if (CQE_OPCODE(*cqe) && RQ_TYPE(*cqe) && 494178786Skmacy Q_EMPTY(wq->rq_rptr, wq->rq_wptr)) 495178786Skmacy return 0; 496178786Skmacy 497178786Skmacy return 1; 498178786Skmacy} 499178786Skmacy 500178786Skmacyvoid 501178786Skmacycxio_count_scqes(struct t3_cq *cq, struct t3_wq *wq, int *count) 502178786Skmacy{ 503178786Skmacy struct t3_cqe *cqe; 504178786Skmacy u32 ptr; 505178786Skmacy 506178786Skmacy *count = 0; 507178786Skmacy ptr = cq->sw_rptr; 508178786Skmacy while (!Q_EMPTY(ptr, cq->sw_wptr)) { 509178786Skmacy cqe = cq->sw_queue + (Q_PTR2IDX(ptr, cq->size_log2)); 510178786Skmacy if ((SQ_TYPE(*cqe) || (CQE_OPCODE(*cqe) == T3_READ_RESP)) && 511178786Skmacy (CQE_QPID(*cqe) == wq->qpid)) 512178786Skmacy (*count)++; 513178786Skmacy ptr++; 514178786Skmacy } 515178786Skmacy CTR3(KTR_IW_CXGB, "%s cq %p count %d", __FUNCTION__, cq, *count); 516178786Skmacy} 517178786Skmacy 518178786Skmacyvoid 519178786Skmacycxio_count_rcqes(struct t3_cq *cq, struct t3_wq *wq, int *count) 520178786Skmacy{ 521178786Skmacy struct t3_cqe *cqe; 522178786Skmacy u32 ptr; 523178786Skmacy 524178786Skmacy *count = 0; 525178786Skmacy CTR2(KTR_IW_CXGB, "%s count zero %d", __FUNCTION__, *count); 526178786Skmacy ptr = cq->sw_rptr; 527178786Skmacy while (!Q_EMPTY(ptr, cq->sw_wptr)) { 528178786Skmacy cqe = cq->sw_queue + (Q_PTR2IDX(ptr, cq->size_log2)); 529178786Skmacy if (RQ_TYPE(*cqe) && (CQE_OPCODE(*cqe) != T3_READ_RESP) && 530178786Skmacy (CQE_QPID(*cqe) == wq->qpid) && cqe_completes_wr(cqe, wq)) 531178786Skmacy (*count)++; 532178786Skmacy ptr++; 533178786Skmacy } 534178786Skmacy CTR3(KTR_IW_CXGB, "%s cq %p count %d", __FUNCTION__, cq, *count); 535178786Skmacy} 536178786Skmacy 537178786Skmacystatic int 538178786Skmacycxio_hal_init_ctrl_cq(struct cxio_rdev *rdev_p) 539178786Skmacy{ 540178786Skmacy 541237263Snp return (cxio_rdma_cq_setup(rdev_p, 0, 0, 1, 1, 0, 0)); 542178786Skmacy} 543178786Skmacy 544178786Skmacystatic int 545178786Skmacycxio_hal_init_ctrl_qp(struct cxio_rdev *rdev_p) 546178786Skmacy{ 547178786Skmacy int err; 548178786Skmacy u64 sge_cmd, ctx0, ctx1; 549178786Skmacy u64 base_addr; 550178786Skmacy struct t3_modify_qp_wr *wqe; 551178786Skmacy struct mbuf *m; 552178786Skmacy 553237263Snp m = M_GETHDR_OFLD(0, CPL_PRIORITY_CONTROL, wqe); 554178786Skmacy if (m == NULL) { 555178786Skmacy CTR1(KTR_IW_CXGB, "%s m_gethdr failed", __FUNCTION__); 556237263Snp return (ENOMEM); 557178786Skmacy } 558178786Skmacy err = cxio_hal_init_ctrl_cq(rdev_p); 559178786Skmacy if (err) { 560178786Skmacy CTR2(KTR_IW_CXGB, "%s err %d initializing ctrl_cq", __FUNCTION__, err); 561178786Skmacy goto err; 562178786Skmacy } 563237263Snp 564178786Skmacy rdev_p->ctrl_qp.workq = contigmalloc((1 << T3_CTRL_QP_SIZE_LOG2) 565178786Skmacy *sizeof(union t3_wr), M_DEVBUF, M_NOWAIT, 0ul, ~0ul, 4096, 0); 566178786Skmacy if (rdev_p->ctrl_qp.workq) 567178786Skmacy rdev_p->ctrl_qp.dma_addr = vtophys(rdev_p->ctrl_qp.workq); 568237263Snp else { 569178786Skmacy CTR1(KTR_IW_CXGB, "%s dma_alloc_coherent failed", __FUNCTION__); 570237263Snp err = ENOMEM; 571178786Skmacy goto err; 572178786Skmacy } 573237263Snp 574237263Snp rdev_p->ctrl_qp.doorbell = rdev_p->rnic_info.kdb_addr; 575178786Skmacy memset(rdev_p->ctrl_qp.workq, 0, 576178786Skmacy (1 << T3_CTRL_QP_SIZE_LOG2) * sizeof(union t3_wr)); 577178786Skmacy 578178786Skmacy mtx_init(&rdev_p->ctrl_qp.lock, "ctl-qp lock", NULL, MTX_DEF|MTX_DUPOK); 579178786Skmacy 580178786Skmacy /* update HW Ctrl QP context */ 581178786Skmacy base_addr = rdev_p->ctrl_qp.dma_addr; 582178786Skmacy base_addr >>= 12; 583178786Skmacy ctx0 = (V_EC_SIZE((1 << T3_CTRL_QP_SIZE_LOG2)) | 584178786Skmacy V_EC_BASE_LO((u32) base_addr & 0xffff)); 585178786Skmacy ctx0 <<= 32; 586178786Skmacy ctx0 |= V_EC_CREDITS(FW_WR_NUM); 587178786Skmacy base_addr >>= 16; 588178786Skmacy ctx1 = (u32) base_addr; 589178786Skmacy base_addr >>= 32; 590178786Skmacy ctx1 |= ((u64) (V_EC_BASE_HI((u32) base_addr & 0xf) | V_EC_RESPQ(0) | 591178786Skmacy V_EC_TYPE(0) | V_EC_GEN(1) | 592178786Skmacy V_EC_UP_TOKEN(T3_CTL_QP_TID) | F_EC_VALID)) << 32; 593178786Skmacy memset(wqe, 0, sizeof(*wqe)); 594237263Snp build_fw_riwrh((struct fw_riwrh *) wqe, T3_WR_QP_MOD, 0, 0, 595178786Skmacy T3_CTL_QP_TID, 7); 596178786Skmacy wqe->flags = htobe32(MODQP_WRITE_EC); 597178786Skmacy sge_cmd = (3ULL << 56) | FW_RI_SGEEC_START << 8 | 3; 598178786Skmacy wqe->sge_cmd = htobe64(sge_cmd); 599178786Skmacy wqe->ctx1 = htobe64(ctx1); 600178786Skmacy wqe->ctx0 = htobe64(ctx0); 601178786Skmacy CTR3(KTR_IW_CXGB, "CtrlQP dma_addr 0x%llx workq %p size %d", 602178786Skmacy (unsigned long long) rdev_p->ctrl_qp.dma_addr, 603178786Skmacy rdev_p->ctrl_qp.workq, 1 << T3_CTRL_QP_SIZE_LOG2); 604237263Snp return t3_offload_tx(rdev_p->adap, m); 605178786Skmacyerr: 606237263Snp m_freem(m); 607178786Skmacy return err; 608178786Skmacy} 609178786Skmacy 610178786Skmacystatic int 611178786Skmacycxio_hal_destroy_ctrl_qp(struct cxio_rdev *rdev_p) 612178786Skmacy{ 613178786Skmacy#if 0 614178786Skmacy 615178786Skmacy dma_free_coherent(&(rdev_p->rnic_info.pdev), 616178786Skmacy (1UL << T3_CTRL_QP_SIZE_LOG2) 617178786Skmacy * sizeof(union t3_wr), rdev_p->ctrl_qp.workq, 618178786Skmacy /* pci_unmap_addr(&rdev_p->ctrl_qp, mapping)*/ 0); 619178786Skmacy#else 620178786Skmacy contigfree(rdev_p->ctrl_qp.workq,(1UL << T3_CTRL_QP_SIZE_LOG2) 621178786Skmacy * sizeof(union t3_wr), M_DEVBUF); 622178786Skmacy#endif 623178786Skmacy return cxio_hal_clear_qp_ctx(rdev_p, T3_CTRL_QP_ID); 624178786Skmacy} 625178786Skmacy 626178786Skmacy/* write len bytes of data into addr (32B aligned address) 627178786Skmacy * If data is NULL, clear len byte of memory to zero. 628178786Skmacy * caller aquires the ctrl_qp lock before the call 629178786Skmacy */ 630178786Skmacystatic int 631178786Skmacycxio_hal_ctrl_qp_write_mem(struct cxio_rdev *rdev_p, u32 addr, 632237263Snp u32 len, void *data) 633178786Skmacy{ 634178786Skmacy u32 i, nr_wqe, copy_len; 635178786Skmacy u8 *copy_data; 636178786Skmacy u8 wr_len, utx_len; /* lenght in 8 byte flit */ 637178786Skmacy enum t3_wr_flags flag; 638178786Skmacy __be64 *wqe; 639178786Skmacy u64 utx_cmd; 640178786Skmacy addr &= 0x7FFFFFF; 641178786Skmacy nr_wqe = len % 96 ? len / 96 + 1 : len / 96; /* 96B max per WQE */ 642178786Skmacy CTR6(KTR_IW_CXGB, "cxio_hal_ctrl_qp_write_mem wptr 0x%x rptr 0x%x len %d, nr_wqe %d data %p addr 0x%0x", 643178786Skmacy rdev_p->ctrl_qp.wptr, rdev_p->ctrl_qp.rptr, len, 644178786Skmacy nr_wqe, data, addr); 645178786Skmacy utx_len = 3; /* in 32B unit */ 646178786Skmacy for (i = 0; i < nr_wqe; i++) { 647178786Skmacy if (Q_FULL(rdev_p->ctrl_qp.rptr, rdev_p->ctrl_qp.wptr, 648178786Skmacy T3_CTRL_QP_SIZE_LOG2)) { 649178786Skmacy CTR4(KTR_IW_CXGB, "%s ctrl_qp full wtpr 0x%0x rptr 0x%0x, " 650178786Skmacy "wait for more space i %d", __FUNCTION__, 651178786Skmacy rdev_p->ctrl_qp.wptr, rdev_p->ctrl_qp.rptr, i); 652178786Skmacy if (cxio_wait(&rdev_p->ctrl_qp, 653178786Skmacy &rdev_p->ctrl_qp.lock, 654178786Skmacy !Q_FULL(rdev_p->ctrl_qp.rptr, 655178786Skmacy rdev_p->ctrl_qp.wptr, 656178786Skmacy T3_CTRL_QP_SIZE_LOG2))) { 657178786Skmacy CTR1(KTR_IW_CXGB, "%s ctrl_qp workq interrupted", 658178786Skmacy __FUNCTION__); 659178786Skmacy return (-ERESTART); 660178786Skmacy } 661178786Skmacy CTR2(KTR_IW_CXGB, "%s ctrl_qp wakeup, continue posting work request " 662178786Skmacy "i %d", __FUNCTION__, i); 663178786Skmacy } 664178786Skmacy wqe = (__be64 *)(rdev_p->ctrl_qp.workq + (rdev_p->ctrl_qp.wptr % 665178786Skmacy (1 << T3_CTRL_QP_SIZE_LOG2))); 666178786Skmacy flag = 0; 667178786Skmacy if (i == (nr_wqe - 1)) { 668178786Skmacy /* last WQE */ 669237263Snp flag = T3_COMPLETION_FLAG; 670178786Skmacy if (len % 32) 671178786Skmacy utx_len = len / 32 + 1; 672178786Skmacy else 673178786Skmacy utx_len = len / 32; 674178786Skmacy } 675178786Skmacy 676178786Skmacy /* 677178786Skmacy * Force a CQE to return the credit to the workq in case 678178786Skmacy * we posted more than half the max QP size of WRs 679178786Skmacy */ 680178786Skmacy if ((i != 0) && 681178786Skmacy (i % (((1 << T3_CTRL_QP_SIZE_LOG2)) >> 1) == 0)) { 682178786Skmacy flag = T3_COMPLETION_FLAG; 683178786Skmacy CTR2(KTR_IW_CXGB, "%s force completion at i %d", __FUNCTION__, i); 684178786Skmacy } 685178786Skmacy 686178786Skmacy /* build the utx mem command */ 687178786Skmacy wqe += (sizeof(struct t3_bypass_wr) >> 3); 688178786Skmacy utx_cmd = (T3_UTX_MEM_WRITE << 28) | (addr + i * 3); 689178786Skmacy utx_cmd <<= 32; 690178786Skmacy utx_cmd |= (utx_len << 28) | ((utx_len << 2) + 1); 691178786Skmacy *wqe = htobe64(utx_cmd); 692178786Skmacy wqe++; 693178786Skmacy copy_data = (u8 *) data + i * 96; 694178786Skmacy copy_len = len > 96 ? 96 : len; 695178786Skmacy 696178786Skmacy /* clear memory content if data is NULL */ 697178786Skmacy if (data) 698178786Skmacy memcpy(wqe, copy_data, copy_len); 699178786Skmacy else 700178786Skmacy memset(wqe, 0, copy_len); 701178786Skmacy if (copy_len % 32) 702178786Skmacy memset(((u8 *) wqe) + copy_len, 0, 703178786Skmacy 32 - (copy_len % 32)); 704178786Skmacy wr_len = ((sizeof(struct t3_bypass_wr)) >> 3) + 1 + 705178786Skmacy (utx_len << 2); 706178786Skmacy wqe = (__be64 *)(rdev_p->ctrl_qp.workq + (rdev_p->ctrl_qp.wptr % 707178786Skmacy (1 << T3_CTRL_QP_SIZE_LOG2))); 708178786Skmacy 709178786Skmacy /* wptr in the WRID[31:0] */ 710178786Skmacy ((union t3_wrid *)(wqe+1))->id0.low = rdev_p->ctrl_qp.wptr; 711178786Skmacy 712178786Skmacy /* 713178786Skmacy * This must be the last write with a memory barrier 714178786Skmacy * for the genbit 715178786Skmacy */ 716178786Skmacy build_fw_riwrh((struct fw_riwrh *) wqe, T3_WR_BP, flag, 717178786Skmacy Q_GENBIT(rdev_p->ctrl_qp.wptr, 718178786Skmacy T3_CTRL_QP_SIZE_LOG2), T3_CTRL_QP_ID, 719178786Skmacy wr_len); 720178786Skmacy if (flag == T3_COMPLETION_FLAG) 721178786Skmacy ring_doorbell(rdev_p->ctrl_qp.doorbell, T3_CTRL_QP_ID); 722178786Skmacy 723178786Skmacy len -= 96; 724178786Skmacy rdev_p->ctrl_qp.wptr++; 725178786Skmacy } 726178786Skmacy return 0; 727178786Skmacy} 728178786Skmacy 729178786Skmacy/* IN: stag key, pdid, perm, zbva, to, len, page_size, pbl, and pbl_size 730178786Skmacy * OUT: stag index, actual pbl_size, pbl_addr allocated. 731178786Skmacy * TBD: shared memory region support 732178786Skmacy */ 733178786Skmacystatic int 734178786Skmacy__cxio_tpt_op(struct cxio_rdev *rdev_p, u32 reset_tpt_entry, 735178786Skmacy u32 *stag, u8 stag_state, u32 pdid, 736178786Skmacy enum tpt_mem_type type, enum tpt_mem_perm perm, 737237263Snp u32 zbva, u64 to, u32 len, u8 page_size, 738237263Snp u32 pbl_size, u32 pbl_addr) 739178786Skmacy{ 740178786Skmacy int err; 741178786Skmacy struct tpt_entry tpt; 742178786Skmacy u32 stag_idx; 743178786Skmacy u32 wptr; 744178786Skmacy 745178786Skmacy stag_state = stag_state > 0; 746178786Skmacy stag_idx = (*stag) >> 8; 747178786Skmacy 748178786Skmacy if ((!reset_tpt_entry) && !(*stag != T3_STAG_UNSET)) { 749178786Skmacy stag_idx = cxio_hal_get_stag(rdev_p->rscp); 750178786Skmacy if (!stag_idx) 751178786Skmacy return (-ENOMEM); 752178786Skmacy *stag = (stag_idx << 8) | ((*stag) & 0xFF); 753178786Skmacy } 754178786Skmacy CTR5(KTR_IW_CXGB, "%s stag_state 0x%0x type 0x%0x pdid 0x%0x, stag_idx 0x%x", 755178786Skmacy __FUNCTION__, stag_state, type, pdid, stag_idx); 756178786Skmacy 757178786Skmacy mtx_lock(&rdev_p->ctrl_qp.lock); 758178786Skmacy 759178786Skmacy /* write TPT entry */ 760178786Skmacy if (reset_tpt_entry) 761178786Skmacy memset(&tpt, 0, sizeof(tpt)); 762178786Skmacy else { 763178786Skmacy tpt.valid_stag_pdid = htobe32(F_TPT_VALID | 764178786Skmacy V_TPT_STAG_KEY((*stag) & M_TPT_STAG_KEY) | 765178786Skmacy V_TPT_STAG_STATE(stag_state) | 766178786Skmacy V_TPT_STAG_TYPE(type) | V_TPT_PDID(pdid)); 767178786Skmacy PANIC_IF(page_size >= 28); 768178786Skmacy tpt.flags_pagesize_qpid = htobe32(V_TPT_PERM(perm) | 769178786Skmacy F_TPT_MW_BIND_ENABLE | 770178786Skmacy V_TPT_ADDR_TYPE((zbva ? TPT_ZBTO : TPT_VATO)) | 771178786Skmacy V_TPT_PAGE_SIZE(page_size)); 772178786Skmacy tpt.rsvd_pbl_addr = reset_tpt_entry ? 0 : 773237263Snp htobe32(V_TPT_PBL_ADDR(PBL_OFF(rdev_p, pbl_addr)>>3)); 774178786Skmacy tpt.len = htobe32(len); 775178786Skmacy tpt.va_hi = htobe32((u32) (to >> 32)); 776178786Skmacy tpt.va_low_or_fbo = htobe32((u32) (to & 0xFFFFFFFFULL)); 777178786Skmacy tpt.rsvd_bind_cnt_or_pstag = 0; 778178786Skmacy tpt.rsvd_pbl_size = reset_tpt_entry ? 0 : 779237263Snp htobe32(V_TPT_PBL_SIZE((pbl_size) >> 2)); 780178786Skmacy } 781178786Skmacy err = cxio_hal_ctrl_qp_write_mem(rdev_p, 782178786Skmacy stag_idx + 783178786Skmacy (rdev_p->rnic_info.tpt_base >> 5), 784237263Snp sizeof(tpt), &tpt); 785178786Skmacy 786178786Skmacy /* release the stag index to free pool */ 787178786Skmacy if (reset_tpt_entry) 788178786Skmacy cxio_hal_put_stag(rdev_p->rscp, stag_idx); 789237263Snp 790178786Skmacy wptr = rdev_p->ctrl_qp.wptr; 791178786Skmacy mtx_unlock(&rdev_p->ctrl_qp.lock); 792178786Skmacy if (!err) 793178786Skmacy if (cxio_wait(&rdev_p->ctrl_qp, 794178786Skmacy &rdev_p->ctrl_qp.lock, 795178786Skmacy SEQ32_GE(rdev_p->ctrl_qp.rptr, wptr))) 796178786Skmacy return (-ERESTART); 797178786Skmacy return err; 798178786Skmacy} 799178786Skmacy 800237263Snpint cxio_write_pbl(struct cxio_rdev *rdev_p, __be64 *pbl, 801237263Snp u32 pbl_addr, u32 pbl_size) 802237263Snp{ 803237263Snp u32 wptr; 804237263Snp int err; 805237263Snp 806237263Snp CTR4(KTR_IW_CXGB, "%s *pdb_addr 0x%x, pbl_base 0x%x, pbl_size %d", 807237263Snp __func__, pbl_addr, rdev_p->rnic_info.pbl_base, 808237263Snp pbl_size); 809237263Snp 810237263Snp mtx_lock(&rdev_p->ctrl_qp.lock); 811237263Snp err = cxio_hal_ctrl_qp_write_mem(rdev_p, pbl_addr >> 5, pbl_size << 3, 812237263Snp pbl); 813237263Snp wptr = rdev_p->ctrl_qp.wptr; 814237263Snp mtx_unlock(&rdev_p->ctrl_qp.lock); 815237263Snp if (err) 816237263Snp return err; 817237263Snp 818237263Snp if (cxio_wait(&rdev_p->ctrl_qp, 819237263Snp &rdev_p->ctrl_qp.lock, 820237263Snp SEQ32_GE(rdev_p->ctrl_qp.rptr, wptr))) 821237263Snp return ERESTART; 822237263Snp 823237263Snp return 0; 824237263Snp} 825237263Snp 826178786Skmacyint 827178786Skmacycxio_register_phys_mem(struct cxio_rdev *rdev_p, u32 *stag, u32 pdid, 828178786Skmacy enum tpt_mem_perm perm, u32 zbva, u64 to, u32 len, 829237263Snp u8 page_size, u32 pbl_size, u32 pbl_addr) 830178786Skmacy{ 831178786Skmacy *stag = T3_STAG_UNSET; 832178786Skmacy return __cxio_tpt_op(rdev_p, 0, stag, 1, pdid, TPT_NON_SHARED_MR, perm, 833237263Snp zbva, to, len, page_size, pbl_size, pbl_addr); 834178786Skmacy} 835178786Skmacy 836178786Skmacyint 837178786Skmacycxio_reregister_phys_mem(struct cxio_rdev *rdev_p, u32 *stag, u32 pdid, 838178786Skmacy enum tpt_mem_perm perm, u32 zbva, u64 to, u32 len, 839237263Snp u8 page_size, u32 pbl_size, u32 pbl_addr) 840178786Skmacy{ 841178786Skmacy return __cxio_tpt_op(rdev_p, 0, stag, 1, pdid, TPT_NON_SHARED_MR, perm, 842237263Snp zbva, to, len, page_size, pbl_size, pbl_addr); 843178786Skmacy} 844178786Skmacy 845178786Skmacyint 846178786Skmacycxio_dereg_mem(struct cxio_rdev *rdev_p, u32 stag, u32 pbl_size, 847178786Skmacy u32 pbl_addr) 848178786Skmacy{ 849237263Snp return __cxio_tpt_op(rdev_p, 1, &stag, 0, 0, 0, 0, 0, 0ULL, 0, 0, 850237263Snp pbl_size, pbl_addr); 851178786Skmacy} 852178786Skmacy 853178786Skmacyint 854178786Skmacycxio_allocate_window(struct cxio_rdev *rdev_p, u32 * stag, u32 pdid) 855178786Skmacy{ 856178786Skmacy *stag = T3_STAG_UNSET; 857178786Skmacy return __cxio_tpt_op(rdev_p, 0, stag, 0, pdid, TPT_MW, 0, 0, 0ULL, 0, 0, 858237263Snp 0, 0); 859178786Skmacy} 860178786Skmacy 861178786Skmacyint 862178786Skmacycxio_deallocate_window(struct cxio_rdev *rdev_p, u32 stag) 863178786Skmacy{ 864237263Snp return __cxio_tpt_op(rdev_p, 1, &stag, 0, 0, 0, 0, 0, 0ULL, 0, 0, 865237263Snp 0, 0); 866178786Skmacy} 867178786Skmacy 868178786Skmacyint 869237263Snpcxio_rdma_init(struct cxio_rdev *rdev_p, struct t3_rdma_init_attr *attr, 870237263Snp struct socket *so) 871178786Skmacy{ 872178786Skmacy struct t3_rdma_init_wr *wqe; 873237263Snp struct mbuf *m; 874237263Snp struct ofld_hdr *oh; 875237263Snp int rc; 876237263Snp struct tcpcb *tp; 877237263Snp struct inpcb *inp; 878237263Snp struct toepcb *toep; 879237263Snp 880237263Snp m = M_GETHDR_OFLD(0, CPL_PRIORITY_DATA, wqe); 881178786Skmacy if (m == NULL) 882178786Skmacy return (-ENOMEM); 883178786Skmacy CTR2(KTR_IW_CXGB, "%s rdev_p %p", __FUNCTION__, rdev_p); 884178786Skmacy wqe->wrh.op_seop_flags = htobe32(V_FW_RIWR_OP(T3_WR_INIT)); 885178786Skmacy wqe->wrh.gen_tid_len = htobe32(V_FW_RIWR_TID(attr->tid) | 886178786Skmacy V_FW_RIWR_LEN(sizeof(*wqe) >> 3)); 887178786Skmacy wqe->wrid.id1 = 0; 888178786Skmacy wqe->qpid = htobe32(attr->qpid); 889178786Skmacy wqe->pdid = htobe32(attr->pdid); 890178786Skmacy wqe->scqid = htobe32(attr->scqid); 891178786Skmacy wqe->rcqid = htobe32(attr->rcqid); 892178786Skmacy wqe->rq_addr = htobe32(attr->rq_addr - rdev_p->rnic_info.rqt_base); 893178786Skmacy wqe->rq_size = htobe32(attr->rq_size); 894178786Skmacy wqe->mpaattrs = attr->mpaattrs; 895178786Skmacy wqe->qpcaps = attr->qpcaps; 896178786Skmacy wqe->ulpdu_size = htobe16(attr->tcp_emss); 897237263Snp wqe->rqe_count = htobe16(attr->rqe_count); 898237263Snp wqe->flags_rtr_type = htobe16(attr->flags | 899237263Snp V_RTR_TYPE(attr->rtr_type) | 900237263Snp V_CHAN(attr->chan)); 901178786Skmacy wqe->ord = htobe32(attr->ord); 902178786Skmacy wqe->ird = htobe32(attr->ird); 903178786Skmacy wqe->qp_dma_addr = htobe64(attr->qp_dma_addr); 904178786Skmacy wqe->qp_dma_size = htobe32(attr->qp_dma_size); 905178786Skmacy wqe->irs = htobe32(attr->irs); 906178786Skmacy 907237263Snp /* XXX: bad form, fix later */ 908237263Snp inp = sotoinpcb(so); 909237263Snp INP_WLOCK(inp); 910237263Snp tp = intotcpcb(inp); 911237263Snp toep = tp->t_toe; 912237263Snp oh = mtod(m, struct ofld_hdr *); 913237263Snp oh->plen = 0; 914237263Snp oh->flags |= F_HDR_DF; 915237263Snp enqueue_wr(toep, m); 916237263Snp toep->tp_wr_avail--; 917237263Snp toep->tp_wr_unacked++; 918237263Snp rc = t3_offload_tx(rdev_p->adap, m); 919237263Snp INP_WUNLOCK(inp); 920178786Skmacy 921237263Snp return (rc); 922178786Skmacy} 923178786Skmacy 924178786Skmacystatic int 925237263Snpcxio_hal_ev_handler(struct sge_qset *qs, struct rsp_desc *r, struct mbuf *m) 926178786Skmacy{ 927237263Snp struct adapter *sc = qs->adap; 928237263Snp struct iwch_dev *rnicp = sc->iwarp_softc; 929237263Snp struct cxio_rdev *rdev_p = &rnicp->rdev; 930178786Skmacy struct respQ_msg_t *rsp_msg = (struct respQ_msg_t *) m->m_data; 931237263Snp int qpid = CQE_QPID(rsp_msg->cqe); 932178786Skmacy 933178786Skmacy CTR6(KTR_IW_CXGB, "%s cq_id 0x%x cq_ptr 0x%x genbit %0x overflow %0x an %0x", 934178786Skmacy __FUNCTION__, RSPQ_CQID(rsp_msg), RSPQ_CQPTR(rsp_msg), 935178786Skmacy RSPQ_GENBIT(rsp_msg), RSPQ_OVERFLOW(rsp_msg), RSPQ_AN(rsp_msg)); 936178786Skmacy CTR4(KTR_IW_CXGB, "se %0x notify %0x cqbranch %0x creditth %0x", 937178786Skmacy RSPQ_SE(rsp_msg), RSPQ_NOTIFY(rsp_msg), RSPQ_CQBRANCH(rsp_msg), 938178786Skmacy RSPQ_CREDIT_THRESH(rsp_msg)); 939178786Skmacy CTR4(KTR_IW_CXGB, "CQE: QPID 0x%0x type 0x%0x status 0x%0x opcode %d", 940237263Snp qpid, CQE_TYPE(rsp_msg->cqe), CQE_STATUS(rsp_msg->cqe), 941237263Snp CQE_OPCODE(rsp_msg->cqe)); 942178786Skmacy CTR3(KTR_IW_CXGB, "len 0x%0x wrid_hi_stag 0x%x wrid_low_msn 0x%x", 943178786Skmacy CQE_LEN(rsp_msg->cqe), CQE_WRID_HI(rsp_msg->cqe), CQE_WRID_LOW(rsp_msg->cqe)); 944237263Snp 945237263Snp switch(qpid) { 946237263Snp case T3_CTRL_QP_ID: 947178786Skmacy mtx_lock(&rdev_p->ctrl_qp.lock); 948178786Skmacy rdev_p->ctrl_qp.rptr = CQE_WRID_LOW(rsp_msg->cqe) + 1; 949178786Skmacy wakeup(&rdev_p->ctrl_qp); 950178786Skmacy mtx_unlock(&rdev_p->ctrl_qp.lock); 951237263Snp break; 952237263Snp case 0xfff8: 953237263Snp break; 954237263Snp default: 955237263Snp iwch_ev_dispatch(rnicp, m); 956237263Snp } 957237263Snp 958237263Snp m_freem(m); 959237263Snp return (0); 960178786Skmacy} 961178786Skmacy 962178786Skmacy/* Caller takes care of locking if needed */ 963178786Skmacyint 964178786Skmacycxio_rdev_open(struct cxio_rdev *rdev_p) 965178786Skmacy{ 966178786Skmacy int err = 0; 967237263Snp struct rdma_info *ri = &rdev_p->rnic_info; 968237263Snp struct adapter *sc = rdev_p->adap; 969178786Skmacy 970237263Snp KASSERT(rdev_p->adap, ("%s: adap is NULL", __func__)); 971178786Skmacy 972178786Skmacy memset(&rdev_p->ctrl_qp, 0, sizeof(rdev_p->ctrl_qp)); 973178786Skmacy 974237263Snp ri->udbell_physbase = rman_get_start(sc->udbs_res); 975237263Snp ri->udbell_len = rman_get_size(sc->udbs_res); 976237263Snp ri->tpt_base = t3_read_reg(sc, A_ULPTX_TPT_LLIMIT); 977237263Snp ri->tpt_top = t3_read_reg(sc, A_ULPTX_TPT_ULIMIT); 978237263Snp ri->pbl_base = t3_read_reg(sc, A_ULPTX_PBL_LLIMIT); 979237263Snp ri->pbl_top = t3_read_reg(sc, A_ULPTX_PBL_ULIMIT); 980237263Snp ri->rqt_base = t3_read_reg(sc, A_ULPRX_RQ_LLIMIT); 981237263Snp ri->rqt_top = t3_read_reg(sc, A_ULPRX_RQ_ULIMIT); 982237263Snp ri->kdb_addr = (void *)((unsigned long) 983237263Snp rman_get_virtual(sc->regs_res) + A_SG_KDOORBELL); 984237263Snp 985178786Skmacy /* 986178786Skmacy * qpshift is the number of bits to shift the qpid left in order 987178786Skmacy * to get the correct address of the doorbell for that qp. 988178786Skmacy */ 989178786Skmacy cxio_init_ucontext(rdev_p, &rdev_p->uctx); 990178786Skmacy rdev_p->qpshift = PAGE_SHIFT - 991178786Skmacy ilog2(65536 >> 992178786Skmacy ilog2(rdev_p->rnic_info.udbell_len >> 993178786Skmacy PAGE_SHIFT)); 994178786Skmacy rdev_p->qpnr = rdev_p->rnic_info.udbell_len >> PAGE_SHIFT; 995178786Skmacy rdev_p->qpmask = (65536 >> ilog2(rdev_p->qpnr)) - 1; 996237263Snp CTR4(KTR_IW_CXGB, "cxio_rdev_open rnic %p info: tpt_base 0x%0x tpt_top 0x%0x num stags %d", 997237263Snp rdev_p->adap, rdev_p->rnic_info.tpt_base, 998178786Skmacy rdev_p->rnic_info.tpt_top, cxio_num_stags(rdev_p)); 999178786Skmacy CTR4(KTR_IW_CXGB, "pbl_base 0x%0x pbl_top 0x%0x rqt_base 0x%0x, rqt_top 0x%0x", 1000178786Skmacy rdev_p->rnic_info.pbl_base, 1001178786Skmacy rdev_p->rnic_info.pbl_top, rdev_p->rnic_info.rqt_base, 1002178786Skmacy rdev_p->rnic_info.rqt_top); 1003178786Skmacy CTR6(KTR_IW_CXGB, "udbell_len 0x%0x udbell_physbase 0x%lx kdb_addr %p qpshift %lu " 1004178786Skmacy "qpnr %d qpmask 0x%x", 1005178786Skmacy rdev_p->rnic_info.udbell_len, 1006178786Skmacy rdev_p->rnic_info.udbell_physbase, rdev_p->rnic_info.kdb_addr, 1007178786Skmacy rdev_p->qpshift, rdev_p->qpnr, rdev_p->qpmask); 1008178786Skmacy 1009178786Skmacy err = cxio_hal_init_ctrl_qp(rdev_p); 1010178786Skmacy if (err) { 1011178786Skmacy log(LOG_ERR, "%s error %d initializing ctrl_qp.\n", 1012178786Skmacy __FUNCTION__, err); 1013178786Skmacy goto err1; 1014178786Skmacy } 1015178786Skmacy err = cxio_hal_init_resource(rdev_p, cxio_num_stags(rdev_p), 0, 1016178786Skmacy 0, T3_MAX_NUM_QP, T3_MAX_NUM_CQ, 1017178786Skmacy T3_MAX_NUM_PD); 1018178786Skmacy if (err) { 1019178786Skmacy log(LOG_ERR, "%s error %d initializing hal resources.\n", 1020178786Skmacy __FUNCTION__, err); 1021178786Skmacy goto err2; 1022178786Skmacy } 1023178786Skmacy err = cxio_hal_pblpool_create(rdev_p); 1024178786Skmacy if (err) { 1025178786Skmacy log(LOG_ERR, "%s error %d initializing pbl mem pool.\n", 1026178786Skmacy __FUNCTION__, err); 1027178786Skmacy goto err3; 1028178786Skmacy } 1029178786Skmacy err = cxio_hal_rqtpool_create(rdev_p); 1030178786Skmacy if (err) { 1031178786Skmacy log(LOG_ERR, "%s error %d initializing rqt mem pool.\n", 1032178786Skmacy __FUNCTION__, err); 1033178786Skmacy goto err4; 1034178786Skmacy } 1035178786Skmacy return 0; 1036178786Skmacyerr4: 1037178786Skmacy cxio_hal_pblpool_destroy(rdev_p); 1038178786Skmacyerr3: 1039178786Skmacy cxio_hal_destroy_resource(rdev_p->rscp); 1040178786Skmacyerr2: 1041178786Skmacy cxio_hal_destroy_ctrl_qp(rdev_p); 1042178786Skmacyerr1: 1043178786Skmacy return err; 1044178786Skmacy} 1045178786Skmacy 1046178786Skmacyvoid 1047178786Skmacycxio_rdev_close(struct cxio_rdev *rdev_p) 1048178786Skmacy{ 1049237263Snp cxio_hal_pblpool_destroy(rdev_p); 1050237263Snp cxio_hal_rqtpool_destroy(rdev_p); 1051237263Snp cxio_hal_destroy_ctrl_qp(rdev_p); 1052237263Snp cxio_hal_destroy_resource(rdev_p->rscp); 1053178786Skmacy} 1054178786Skmacy 1055178786Skmacyint 1056237263Snpcxio_hal_init(struct adapter *sc) 1057178786Skmacy{ 1058178786Skmacy#ifdef needed 1059178786Skmacy if (cxio_hal_init_rhdl_resource(T3_MAX_NUM_RI)) 1060237263Snp return (ENOMEM); 1061178786Skmacy#endif 1062237263Snp t3_register_cpl_handler(sc, CPL_ASYNC_NOTIF, cxio_hal_ev_handler); 1063237263Snp 1064237263Snp return (0); 1065178786Skmacy} 1066178786Skmacy 1067178786Skmacyvoid 1068237263Snpcxio_hal_uninit(struct adapter *sc) 1069178786Skmacy{ 1070237263Snp t3_register_cpl_handler(sc, CPL_ASYNC_NOTIF, NULL); 1071178786Skmacy#ifdef needed 1072178786Skmacy cxio_hal_destroy_rhdl_resource(); 1073178786Skmacy#endif 1074178786Skmacy} 1075178786Skmacy 1076178786Skmacystatic void 1077178786Skmacyflush_completed_wrs(struct t3_wq *wq, struct t3_cq *cq) 1078178786Skmacy{ 1079178786Skmacy struct t3_swsq *sqp; 1080178786Skmacy __u32 ptr = wq->sq_rptr; 1081178786Skmacy int count = Q_COUNT(wq->sq_rptr, wq->sq_wptr); 1082178786Skmacy 1083178786Skmacy sqp = wq->sq + Q_PTR2IDX(ptr, wq->sq_size_log2); 1084178786Skmacy while (count--) 1085178786Skmacy if (!sqp->signaled) { 1086178786Skmacy ptr++; 1087178786Skmacy sqp = wq->sq + Q_PTR2IDX(ptr, wq->sq_size_log2); 1088178786Skmacy } else if (sqp->complete) { 1089178786Skmacy 1090178786Skmacy /* 1091178786Skmacy * Insert this completed cqe into the swcq. 1092178786Skmacy */ 1093178786Skmacy CTR3(KTR_IW_CXGB, "%s moving cqe into swcq sq idx %ld cq idx %ld", 1094178786Skmacy __FUNCTION__, Q_PTR2IDX(ptr, wq->sq_size_log2), 1095178786Skmacy Q_PTR2IDX(cq->sw_wptr, cq->size_log2)); 1096178786Skmacy sqp->cqe.header |= htonl(V_CQE_SWCQE(1)); 1097178786Skmacy *(cq->sw_queue + Q_PTR2IDX(cq->sw_wptr, cq->size_log2)) 1098178786Skmacy = sqp->cqe; 1099178786Skmacy cq->sw_wptr++; 1100178786Skmacy sqp->signaled = 0; 1101178786Skmacy break; 1102178786Skmacy } else 1103178786Skmacy break; 1104178786Skmacy} 1105178786Skmacy 1106178786Skmacystatic void 1107178786Skmacycreate_read_req_cqe(struct t3_wq *wq, struct t3_cqe *hw_cqe, 1108178786Skmacy struct t3_cqe *read_cqe) 1109178786Skmacy{ 1110178786Skmacy read_cqe->u.scqe.wrid_hi = wq->oldest_read->sq_wptr; 1111178786Skmacy read_cqe->len = wq->oldest_read->read_len; 1112178786Skmacy read_cqe->header = htonl(V_CQE_QPID(CQE_QPID(*hw_cqe)) | 1113178786Skmacy V_CQE_SWCQE(SW_CQE(*hw_cqe)) | 1114178786Skmacy V_CQE_OPCODE(T3_READ_REQ) | 1115178786Skmacy V_CQE_TYPE(1)); 1116178786Skmacy} 1117178786Skmacy 1118178786Skmacy/* 1119178786Skmacy * Return a ptr to the next read wr in the SWSQ or NULL. 1120178786Skmacy */ 1121178786Skmacystatic void 1122178786Skmacyadvance_oldest_read(struct t3_wq *wq) 1123178786Skmacy{ 1124178786Skmacy 1125178786Skmacy u32 rptr = wq->oldest_read - wq->sq + 1; 1126178786Skmacy u32 wptr = Q_PTR2IDX(wq->sq_wptr, wq->sq_size_log2); 1127178786Skmacy 1128178786Skmacy while (Q_PTR2IDX(rptr, wq->sq_size_log2) != wptr) { 1129178786Skmacy wq->oldest_read = wq->sq + Q_PTR2IDX(rptr, wq->sq_size_log2); 1130178786Skmacy 1131178786Skmacy if (wq->oldest_read->opcode == T3_READ_REQ) 1132178786Skmacy return; 1133178786Skmacy rptr++; 1134178786Skmacy } 1135178786Skmacy wq->oldest_read = NULL; 1136178786Skmacy} 1137178786Skmacy 1138178786Skmacy/* 1139178786Skmacy * cxio_poll_cq 1140178786Skmacy * 1141178786Skmacy * Caller must: 1142178786Skmacy * check the validity of the first CQE, 1143178786Skmacy * supply the wq assicated with the qpid. 1144178786Skmacy * 1145178786Skmacy * credit: cq credit to return to sge. 1146178786Skmacy * cqe_flushed: 1 iff the CQE is flushed. 1147178786Skmacy * cqe: copy of the polled CQE. 1148178786Skmacy * 1149178786Skmacy * return value: 1150178786Skmacy * 0 CQE returned, 1151178786Skmacy * -1 CQE skipped, try again. 1152178786Skmacy */ 1153178786Skmacyint 1154178786Skmacycxio_poll_cq(struct t3_wq *wq, struct t3_cq *cq, struct t3_cqe *cqe, 1155178786Skmacy u8 *cqe_flushed, u64 *cookie, u32 *credit) 1156178786Skmacy{ 1157178786Skmacy int ret = 0; 1158178786Skmacy struct t3_cqe *hw_cqe, read_cqe; 1159178786Skmacy 1160178786Skmacy *cqe_flushed = 0; 1161178786Skmacy *credit = 0; 1162178786Skmacy hw_cqe = cxio_next_cqe(cq); 1163178786Skmacy 1164178786Skmacy CTR5(KTR_IW_CXGB, "cxio_poll_cq CQE OOO %d qpid 0x%0x genbit %d type %d status 0x%0x", 1165178786Skmacy CQE_OOO(*hw_cqe), CQE_QPID(*hw_cqe), 1166178786Skmacy CQE_GENBIT(*hw_cqe), CQE_TYPE(*hw_cqe), CQE_STATUS(*hw_cqe)); 1167178786Skmacy CTR4(KTR_IW_CXGB, "opcode 0x%0x len 0x%0x wrid_hi_stag 0x%x wrid_low_msn 0x%x", 1168178786Skmacy CQE_OPCODE(*hw_cqe), CQE_LEN(*hw_cqe), CQE_WRID_HI(*hw_cqe), 1169178786Skmacy CQE_WRID_LOW(*hw_cqe)); 1170178786Skmacy 1171178786Skmacy /* 1172178786Skmacy * skip cqe's not affiliated with a QP. 1173178786Skmacy */ 1174178786Skmacy if (wq == NULL) { 1175178786Skmacy ret = -1; 1176178786Skmacy goto skip_cqe; 1177178786Skmacy } 1178178786Skmacy 1179178786Skmacy /* 1180178786Skmacy * Gotta tweak READ completions: 1181178786Skmacy * 1) the cqe doesn't contain the sq_wptr from the wr. 1182178786Skmacy * 2) opcode not reflected from the wr. 1183178786Skmacy * 3) read_len not reflected from the wr. 1184178786Skmacy * 4) cq_type is RQ_TYPE not SQ_TYPE. 1185178786Skmacy */ 1186178786Skmacy if (RQ_TYPE(*hw_cqe) && (CQE_OPCODE(*hw_cqe) == T3_READ_RESP)) { 1187178786Skmacy 1188178786Skmacy /* 1189178786Skmacy * Don't write to the HWCQ, so create a new read req CQE 1190178786Skmacy * in local memory. 1191178786Skmacy */ 1192178786Skmacy create_read_req_cqe(wq, hw_cqe, &read_cqe); 1193178786Skmacy hw_cqe = &read_cqe; 1194178786Skmacy advance_oldest_read(wq); 1195178786Skmacy } 1196178786Skmacy 1197178786Skmacy /* 1198178786Skmacy * T3A: Discard TERMINATE CQEs. 1199178786Skmacy */ 1200178786Skmacy if (CQE_OPCODE(*hw_cqe) == T3_TERMINATE) { 1201178786Skmacy ret = -1; 1202178786Skmacy wq->error = 1; 1203178786Skmacy goto skip_cqe; 1204178786Skmacy } 1205178786Skmacy 1206178786Skmacy if (CQE_STATUS(*hw_cqe) || wq->error) { 1207178786Skmacy *cqe_flushed = wq->error; 1208178786Skmacy wq->error = 1; 1209178786Skmacy 1210178786Skmacy /* 1211178786Skmacy * T3A inserts errors into the CQE. We cannot return 1212178786Skmacy * these as work completions. 1213178786Skmacy */ 1214178786Skmacy /* incoming write failures */ 1215178786Skmacy if ((CQE_OPCODE(*hw_cqe) == T3_RDMA_WRITE) 1216178786Skmacy && RQ_TYPE(*hw_cqe)) { 1217178786Skmacy ret = -1; 1218178786Skmacy goto skip_cqe; 1219178786Skmacy } 1220178786Skmacy /* incoming read request failures */ 1221178786Skmacy if ((CQE_OPCODE(*hw_cqe) == T3_READ_RESP) && SQ_TYPE(*hw_cqe)) { 1222178786Skmacy ret = -1; 1223178786Skmacy goto skip_cqe; 1224178786Skmacy } 1225178786Skmacy 1226178786Skmacy /* incoming SEND with no receive posted failures */ 1227237263Snp if (CQE_OPCODE(*hw_cqe) && RQ_TYPE(*hw_cqe) && 1228178786Skmacy Q_EMPTY(wq->rq_rptr, wq->rq_wptr)) { 1229178786Skmacy ret = -1; 1230178786Skmacy goto skip_cqe; 1231178786Skmacy } 1232237263Snp PANIC_IF((*cqe_flushed == 0) && !SW_CQE(*hw_cqe)); 1233178786Skmacy goto proc_cqe; 1234178786Skmacy } 1235178786Skmacy 1236178786Skmacy /* 1237178786Skmacy * RECV completion. 1238178786Skmacy */ 1239178786Skmacy if (RQ_TYPE(*hw_cqe)) { 1240178786Skmacy 1241178786Skmacy /* 1242178786Skmacy * HW only validates 4 bits of MSN. So we must validate that 1243178786Skmacy * the MSN in the SEND is the next expected MSN. If its not, 1244178786Skmacy * then we complete this with TPT_ERR_MSN and mark the wq in 1245178786Skmacy * error. 1246178786Skmacy */ 1247237263Snp 1248237263Snp if (Q_EMPTY(wq->rq_rptr, wq->rq_wptr)) { 1249237263Snp wq->error = 1; 1250237263Snp ret = -1; 1251237263Snp goto skip_cqe; 1252237263Snp } 1253237263Snp 1254178786Skmacy if (__predict_false((CQE_WRID_MSN(*hw_cqe) != (wq->rq_rptr + 1)))) { 1255178786Skmacy wq->error = 1; 1256178786Skmacy hw_cqe->header |= htonl(V_CQE_STATUS(TPT_ERR_MSN)); 1257178786Skmacy goto proc_cqe; 1258178786Skmacy } 1259178786Skmacy goto proc_cqe; 1260178786Skmacy } 1261178786Skmacy 1262178786Skmacy /* 1263178786Skmacy * If we get here its a send completion. 1264178786Skmacy * 1265178786Skmacy * Handle out of order completion. These get stuffed 1266178786Skmacy * in the SW SQ. Then the SW SQ is walked to move any 1267178786Skmacy * now in-order completions into the SW CQ. This handles 1268178786Skmacy * 2 cases: 1269178786Skmacy * 1) reaping unsignaled WRs when the first subsequent 1270178786Skmacy * signaled WR is completed. 1271178786Skmacy * 2) out of order read completions. 1272178786Skmacy */ 1273178786Skmacy if (!SW_CQE(*hw_cqe) && (CQE_WRID_SQ_WPTR(*hw_cqe) != wq->sq_rptr)) { 1274178786Skmacy struct t3_swsq *sqp; 1275178786Skmacy 1276178786Skmacy CTR2(KTR_IW_CXGB, "%s out of order completion going in swsq at idx %ld", 1277178786Skmacy __FUNCTION__, 1278178786Skmacy Q_PTR2IDX(CQE_WRID_SQ_WPTR(*hw_cqe), wq->sq_size_log2)); 1279178786Skmacy sqp = wq->sq + 1280178786Skmacy Q_PTR2IDX(CQE_WRID_SQ_WPTR(*hw_cqe), wq->sq_size_log2); 1281178786Skmacy sqp->cqe = *hw_cqe; 1282178786Skmacy sqp->complete = 1; 1283178786Skmacy ret = -1; 1284178786Skmacy goto flush_wq; 1285178786Skmacy } 1286178786Skmacy 1287178786Skmacyproc_cqe: 1288178786Skmacy *cqe = *hw_cqe; 1289178786Skmacy 1290178786Skmacy /* 1291178786Skmacy * Reap the associated WR(s) that are freed up with this 1292178786Skmacy * completion. 1293178786Skmacy */ 1294178786Skmacy if (SQ_TYPE(*hw_cqe)) { 1295178786Skmacy wq->sq_rptr = CQE_WRID_SQ_WPTR(*hw_cqe); 1296178786Skmacy CTR2(KTR_IW_CXGB, "%s completing sq idx %ld", __FUNCTION__, 1297178786Skmacy Q_PTR2IDX(wq->sq_rptr, wq->sq_size_log2)); 1298237263Snp *cookie = wq->sq[Q_PTR2IDX(wq->sq_rptr, wq->sq_size_log2)].wr_id; 1299178786Skmacy wq->sq_rptr++; 1300178786Skmacy } else { 1301178786Skmacy CTR2(KTR_IW_CXGB, "%s completing rq idx %ld", __FUNCTION__, 1302178786Skmacy Q_PTR2IDX(wq->rq_rptr, wq->rq_size_log2)); 1303237263Snp *cookie = wq->rq[Q_PTR2IDX(wq->rq_rptr, wq->rq_size_log2)].wr_id; 1304237263Snp if (wq->rq[Q_PTR2IDX(wq->rq_rptr, wq->rq_size_log2)].pbl_addr) 1305237263Snp cxio_hal_pblpool_free(wq->rdev, 1306237263Snp wq->rq[Q_PTR2IDX(wq->rq_rptr, 1307237263Snp wq->rq_size_log2)].pbl_addr, T3_STAG0_PBL_SIZE); 1308237263Snp PANIC_IF(Q_EMPTY(wq->rq_rptr, wq->rq_wptr)); 1309178786Skmacy wq->rq_rptr++; 1310178786Skmacy } 1311178786Skmacy 1312178786Skmacyflush_wq: 1313178786Skmacy /* 1314178786Skmacy * Flush any completed cqes that are now in-order. 1315178786Skmacy */ 1316178786Skmacy flush_completed_wrs(wq, cq); 1317178786Skmacy 1318178786Skmacyskip_cqe: 1319178786Skmacy if (SW_CQE(*hw_cqe)) { 1320178786Skmacy CTR4(KTR_IW_CXGB, "%s cq %p cqid 0x%x skip sw cqe sw_rptr 0x%x", 1321178786Skmacy __FUNCTION__, cq, cq->cqid, cq->sw_rptr); 1322178786Skmacy ++cq->sw_rptr; 1323178786Skmacy } else { 1324178786Skmacy CTR4(KTR_IW_CXGB, "%s cq %p cqid 0x%x skip hw cqe rptr 0x%x", 1325178786Skmacy __FUNCTION__, cq, cq->cqid, cq->rptr); 1326178786Skmacy ++cq->rptr; 1327178786Skmacy 1328178786Skmacy /* 1329178786Skmacy * T3A: compute credits. 1330178786Skmacy */ 1331178786Skmacy if (((cq->rptr - cq->wptr) > (1 << (cq->size_log2 - 1))) 1332178786Skmacy || ((cq->rptr - cq->wptr) >= 128)) { 1333178786Skmacy *credit = cq->rptr - cq->wptr; 1334178786Skmacy cq->wptr = cq->rptr; 1335178786Skmacy } 1336178786Skmacy } 1337178786Skmacy return ret; 1338178786Skmacy} 1339237263Snp#endif 1340