iw_cxgb_cq.c revision 178786
167754Smsmith 267754Smsmith/************************************************************************** 367754Smsmith 467754SmsmithCopyright (c) 2007, Chelsio Inc. 567754SmsmithAll rights reserved. 667754Smsmith 7217365SjkimRedistribution and use in source and binary forms, with or without 8245582Sjkimmodification, are permitted provided that the following conditions are met: 970243Smsmith 1067754Smsmith 1. Redistributions of source code must retain the above copyright notice, 11217365Sjkim this list of conditions and the following disclaimer. 12217365Sjkim 13217365Sjkim 2. Neither the name of the Chelsio Corporation nor the names of its 14217365Sjkim contributors may be used to endorse or promote products derived from 15217365Sjkim this software without specific prior written permission. 16217365Sjkim 17217365SjkimTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18217365SjkimAND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19217365SjkimIMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20217365SjkimARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 21217365SjkimLIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 22217365SjkimCONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 23217365SjkimSUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 24217365SjkimINTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 2567754SmsmithCONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 26217365SjkimARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 27217365SjkimPOSSIBILITY OF SUCH DAMAGE. 28217365Sjkim 2967754Smsmith***************************************************************************/ 30217365Sjkim#include <sys/cdefs.h> 31217365Sjkim__FBSDID("$FreeBSD: head/sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb_cq.c 178786 2008-05-05 18:46:18Z kmacy $"); 32217365Sjkim 33217365Sjkim#include <sys/param.h> 34217365Sjkim#include <sys/systm.h> 35217365Sjkim#include <sys/kernel.h> 36217365Sjkim#include <sys/bus.h> 37217365Sjkim#include <sys/module.h> 38217365Sjkim#include <sys/pciio.h> 39217365Sjkim#include <sys/conf.h> 40217365Sjkim#include <machine/bus.h> 41217365Sjkim#include <machine/resource.h> 42217365Sjkim#include <sys/bus_dma.h> 4367754Smsmith#include <sys/rman.h> 4467754Smsmith#include <sys/ioccom.h> 4567754Smsmith#include <sys/mbuf.h> 4667754Smsmith#include <sys/mutex.h> 4767754Smsmith#include <sys/rwlock.h> 48167802Sjkim#include <sys/linker.h> 4967754Smsmith#include <sys/firmware.h> 50167802Sjkim#include <sys/socket.h> 5167754Smsmith#include <sys/sockio.h> 52167802Sjkim#include <sys/smp.h> 53167802Sjkim#include <sys/sysctl.h> 54167802Sjkim#include <sys/syslog.h> 55167802Sjkim#include <sys/queue.h> 5667754Smsmith#include <sys/taskqueue.h> 57167802Sjkim#include <sys/proc.h> 58167802Sjkim#include <sys/queue.h> 59167802Sjkim#include <sys/libkern.h> 60167802Sjkim 61167802Sjkim#include <netinet/in.h> 62167802Sjkim#include <contrib/rdma/ib_verbs.h> 63167802Sjkim#include <contrib/rdma/ib_umem.h> 64167802Sjkim#include <contrib/rdma/ib_user_verbs.h> 65167802Sjkim 66167802Sjkim 67167802Sjkim#ifdef CONFIG_DEFINED 68167802Sjkim#include <cxgb_include.h> 69167802Sjkim#include <ulp/iw_cxgb/iw_cxgb_wr.h> 70167802Sjkim#include <ulp/iw_cxgb/iw_cxgb_hal.h> 71167802Sjkim#include <ulp/iw_cxgb/iw_cxgb_provider.h> 72167802Sjkim#include <ulp/iw_cxgb/iw_cxgb_cm.h> 73167802Sjkim#include <ulp/iw_cxgb/iw_cxgb.h> 74167802Sjkim#include <ulp/iw_cxgb/iw_cxgb_resource.h> 75167802Sjkim#include <ulp/iw_cxgb/iw_cxgb_user.h> 76167802Sjkim#else 77167802Sjkim#include <dev/cxgb/cxgb_include.h> 78167802Sjkim#include <dev/cxgb/ulp/iw_cxgb/iw_cxgb_wr.h> 79167802Sjkim#include <dev/cxgb/ulp/iw_cxgb/iw_cxgb_hal.h> 80167802Sjkim#include <dev/cxgb/ulp/iw_cxgb/iw_cxgb_provider.h> 81167802Sjkim#include <dev/cxgb/ulp/iw_cxgb/iw_cxgb_cm.h> 82167802Sjkim#include <dev/cxgb/ulp/iw_cxgb/iw_cxgb.h> 83167802Sjkim#include <dev/cxgb/ulp/iw_cxgb/iw_cxgb_resource.h> 84167802Sjkim#include <dev/cxgb/ulp/iw_cxgb/iw_cxgb_user.h> 85167802Sjkim#endif 86167802Sjkim 87167802Sjkim/* 88167802Sjkim * Get one cq entry from cxio and map it to openib. 89167802Sjkim * 90167802Sjkim * Returns: 91167802Sjkim * 0 cqe returned 92167802Sjkim * -ENOBUFS EMPTY; 93167802Sjkim * -EAGAIN caller must try again 94167802Sjkim * any other neg errno fatal error 95167802Sjkim */ 96167802Sjkimstatic int iwch_poll_cq_one(struct iwch_dev *rhp, struct iwch_cq *chp, 97167802Sjkim struct ib_wc *wc) 98167802Sjkim{ 9991116Smsmith struct iwch_qp *qhp = NULL; 10067754Smsmith struct t3_cqe cqe, *rd_cqe; 10171867Smsmith struct t3_wq *wq; 10277424Smsmith u32 credit = 0; 10380062Smsmith u8 cqe_flushed; 10480062Smsmith u64 cookie; 105138287Smarks int ret = 1; 10667754Smsmith 10767754Smsmith rd_cqe = cxio_next_cqe(&chp->cq); 10877424Smsmith 10980062Smsmith if (!rd_cqe) 110138287Smarks return 0; 11171867Smsmith 11271867Smsmith qhp = get_qhp(rhp, CQE_QPID(*rd_cqe)); 11377424Smsmith if (!qhp) 11480062Smsmith wq = NULL; 11580062Smsmith else { 116138287Smarks mtx_lock(&qhp->lock); 11771867Smsmith wq = &(qhp->wq); 118138287Smarks } 119138287Smarks ret = cxio_poll_cq(wq, &(chp->cq), &cqe, &cqe_flushed, &cookie, 120138287Smarks &credit); 121138287Smarks if (t3a_device(chp->rhp) && credit) { 122138287Smarks CTR3(KTR_IW_CXGB, "%s updating %d cq credits on id %d", __FUNCTION__, 123138287Smarks credit, chp->cq.cqid); 124138287Smarks cxio_hal_cq_op(&rhp->rdev, &chp->cq, CQ_CREDIT_UPDATE, credit); 12571867Smsmith } 12677424Smsmith 12791116Smsmith if (ret) { 12891116Smsmith ret = -EAGAIN; 12991116Smsmith goto out; 13071867Smsmith } 13171867Smsmith ret = 1; 13271867Smsmith 13371867Smsmith wc->wr_id = cookie; 134204773Sjkim wc->qp = &qhp->ibqp; 135204773Sjkim wc->vendor_err = CQE_STATUS(cqe); 136204773Sjkim 137204773Sjkim CTR4(KTR_IW_CXGB, "iwch_poll_cq_one qpid 0x%x type %d opcode %d status 0x%x", 138204773Sjkim CQE_QPID(cqe), CQE_TYPE(cqe), 139204773Sjkim CQE_OPCODE(cqe), CQE_STATUS(cqe)); 140204773Sjkim CTR3(KTR_IW_CXGB, "wrid hi 0x%x lo 0x%x cookie 0x%llx", 141204773Sjkim CQE_WRID_HI(cqe), CQE_WRID_LOW(cqe), (unsigned long long) cookie); 142204773Sjkim 143204773Sjkim if (CQE_TYPE(cqe) == 0) { 14491116Smsmith if (!CQE_STATUS(cqe)) 14571867Smsmith wc->byte_len = CQE_LEN(cqe); 14671867Smsmith else 147131440Smarks wc->byte_len = 0; 148131440Smarks wc->opcode = IB_WC_RECV; 149131440Smarks } else { 150131440Smarks switch (CQE_OPCODE(cqe)) { 151131440Smarks case T3_RDMA_WRITE: 152131440Smarks wc->opcode = IB_WC_RDMA_WRITE; 15387031Smsmith break; 15477424Smsmith case T3_READ_REQ: 155202771Sjkim wc->opcode = IB_WC_RDMA_READ; 156202771Sjkim wc->byte_len = CQE_LEN(cqe); 15787031Smsmith break; 15877424Smsmith case T3_SEND: 15987031Smsmith case T3_SEND_WITH_SE: 16087031Smsmith wc->opcode = IB_WC_SEND; 161202771Sjkim break; 16277424Smsmith case T3_BIND_MW: 163117521Snjl wc->opcode = IB_WC_BIND_MW; 16487031Smsmith break; 165117521Snjl 16677424Smsmith /* these aren't supported yet */ 16787031Smsmith case T3_SEND_WITH_INV: 16887031Smsmith case T3_SEND_WITH_SE_INV: 169202771Sjkim case T3_LOCAL_INV: 17077424Smsmith case T3_FAST_REGISTER: 171117521Snjl default: 17287031Smsmith log(LOG_ERR, "Unexpected opcode %d " 173117521Snjl "in the CQE received for QPID=0x%0x\n", 17477424Smsmith CQE_OPCODE(cqe), CQE_QPID(cqe)); 17577424Smsmith ret = -EINVAL; 17677424Smsmith goto out; 17799146Siwasaki } 17877424Smsmith } 17977424Smsmith 18077424Smsmith if (cqe_flushed) 18177424Smsmith wc->status = IB_WC_WR_FLUSH_ERR; 18277424Smsmith else { 18377424Smsmith 184107325Siwasaki switch (CQE_STATUS(cqe)) { 185107325Siwasaki case TPT_ERR_SUCCESS: 18677424Smsmith wc->status = IB_WC_SUCCESS; 187151937Sjkim break; 18867754Smsmith case TPT_ERR_STAG: 189151937Sjkim wc->status = IB_WC_LOC_ACCESS_ERR; 19067754Smsmith break; 19167754Smsmith case TPT_ERR_PDID: 192151937Sjkim wc->status = IB_WC_LOC_PROT_ERR; 193151937Sjkim break; 194151937Sjkim case TPT_ERR_QPID: 195151937Sjkim case TPT_ERR_ACCESS: 19680062Smsmith wc->status = IB_WC_LOC_ACCESS_ERR; 19780062Smsmith break; 198151937Sjkim case TPT_ERR_WRAP: 199151937Sjkim wc->status = IB_WC_GENERAL_ERR; 200151937Sjkim break; 201151937Sjkim case TPT_ERR_BOUND: 20280062Smsmith wc->status = IB_WC_LOC_LEN_ERR; 20380062Smsmith break; 204151937Sjkim case TPT_ERR_INVALIDATE_SHARED_MR: 205151937Sjkim case TPT_ERR_INVALIDATE_MR_WITH_MW_BOUND: 206151937Sjkim wc->status = IB_WC_MW_BIND_ERR; 207202771Sjkim break; 208151937Sjkim case TPT_ERR_CRC: 20967754Smsmith case TPT_ERR_MARKER: 21099679Siwasaki case TPT_ERR_PDU_LEN_ERR: 211151937Sjkim case TPT_ERR_OUT_OF_RQE: 212151937Sjkim case TPT_ERR_DDP_VERSION: 213151937Sjkim case TPT_ERR_RDMA_VERSION: 21467754Smsmith case TPT_ERR_DDP_QUEUE_NUM: 21585756Smsmith case TPT_ERR_MSN: 21685756Smsmith case TPT_ERR_TBIT: 21785756Smsmith case TPT_ERR_MO: 21867754Smsmith case TPT_ERR_MSN_RANGE: 21967754Smsmith case TPT_ERR_IRD_OVERFLOW: 22067754Smsmith case TPT_ERR_OPCODE: 22191116Smsmith wc->status = IB_WC_FATAL_ERR; 22291116Smsmith break; 22391116Smsmith case TPT_ERR_SWFLUSH: 22491116Smsmith wc->status = IB_WC_WR_FLUSH_ERR; 22591116Smsmith break; 22691116Smsmith default: 22791116Smsmith log(LOG_ERR, "Unexpected cqe_status 0x%x for " 22885756Smsmith "QPID=0x%0x\n", CQE_STATUS(cqe), CQE_QPID(cqe)); 22985756Smsmith ret = -EINVAL; 23085756Smsmith } 23185756Smsmith } 23285756Smsmithout: 23385756Smsmith if (wq) 234138287Smarks mtx_unlock(&qhp->lock); 235138287Smarks return ret; 236138287Smarks} 237202771Sjkim 238202771Sjkimint iwch_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc) 239138287Smarks{ 240138287Smarks struct iwch_dev *rhp; 241138287Smarks struct iwch_cq *chp; 24285756Smsmith int npolled; 24385756Smsmith int err = 0; 244138287Smarks 245138287Smarks chp = to_iwch_cq(ibcq); 246138287Smarks rhp = chp->rhp; 24785756Smsmith 248202771Sjkim mtx_lock(&chp->lock); 24985756Smsmith for (npolled = 0; npolled < num_entries; ++npolled) { 25085756Smsmith#ifdef DEBUG 251202771Sjkim int i=0; 252202771Sjkim#endif 25385756Smsmith 25485756Smsmith /* 25577424Smsmith * Because T3 can post CQEs that are _not_ associated 25667754Smsmith * with a WR, we might have to poll again after removing 25767754Smsmith * one of these. 25867754Smsmith */ 25977424Smsmith do { 26085756Smsmith err = iwch_poll_cq_one(rhp, chp, wc + npolled); 26167754Smsmith#ifdef DEBUG 26267754Smsmith BUG_ON(++i > 1000); 26377424Smsmith#endif 26485756Smsmith } while (err == -EAGAIN); 26567754Smsmith if (err <= 0) 26667754Smsmith break; 26777424Smsmith } 26884491Smsmith mtx_unlock(&chp->lock); 26977424Smsmith 27069746Smsmith if (err < 0) { 27167754Smsmith return err; 27267754Smsmith } else { 27367754Smsmith return npolled; 27477424Smsmith } 27567754Smsmith} 27667754Smsmith 27767754Smsmith