1178786Skmacy/************************************************************************** 2178786Skmacy 3178786SkmacyCopyright (c) 2007, Chelsio Inc. 4178786SkmacyAll rights reserved. 5178786Skmacy 6178786SkmacyRedistribution and use in source and binary forms, with or without 7178786Skmacymodification, are permitted provided that the following conditions are met: 8178786Skmacy 9178786Skmacy 1. Redistributions of source code must retain the above copyright notice, 10178786Skmacy this list of conditions and the following disclaimer. 11178786Skmacy 12178786Skmacy 2. Neither the name of the Chelsio Corporation nor the names of its 13178786Skmacy contributors may be used to endorse or promote products derived from 14178786Skmacy this software without specific prior written permission. 15178786Skmacy 16178786SkmacyTHIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 17178786SkmacyAND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18178786SkmacyIMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19178786SkmacyARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 20178786SkmacyLIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21178786SkmacyCONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22178786SkmacySUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23178786SkmacyINTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24178786SkmacyCONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25178786SkmacyARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26178786SkmacyPOSSIBILITY OF SUCH DAMAGE. 27178786Skmacy 28178786Skmacy***************************************************************************/ 29178786Skmacy#include <sys/cdefs.h> 30178786Skmacy__FBSDID("$FreeBSD$"); 31178786Skmacy 32237920Snp#include "opt_inet.h" 33237920Snp 34237920Snp#ifdef TCP_OFFLOAD 35178786Skmacy#include <sys/param.h> 36178786Skmacy#include <sys/systm.h> 37178786Skmacy#include <sys/kernel.h> 38178786Skmacy#include <sys/bus.h> 39178786Skmacy#include <sys/pciio.h> 40178786Skmacy#include <sys/conf.h> 41178786Skmacy#include <machine/bus.h> 42178786Skmacy#include <machine/resource.h> 43178786Skmacy#include <sys/bus_dma.h> 44178786Skmacy#include <sys/rman.h> 45178786Skmacy#include <sys/ioccom.h> 46178786Skmacy#include <sys/mbuf.h> 47178786Skmacy#include <sys/mutex.h> 48178786Skmacy#include <sys/rwlock.h> 49178786Skmacy#include <sys/linker.h> 50178786Skmacy#include <sys/firmware.h> 51178786Skmacy#include <sys/socket.h> 52178786Skmacy#include <sys/sockio.h> 53178786Skmacy#include <sys/smp.h> 54178786Skmacy#include <sys/sysctl.h> 55178786Skmacy#include <sys/syslog.h> 56178786Skmacy#include <sys/queue.h> 57178786Skmacy#include <sys/taskqueue.h> 58178786Skmacy#include <sys/proc.h> 59178786Skmacy#include <sys/queue.h> 60178786Skmacy#include <sys/libkern.h> 61178786Skmacy 62178786Skmacy#include <netinet/in.h> 63178786Skmacy 64237920Snp#include <rdma/ib_verbs.h> 65237920Snp#include <rdma/ib_umem.h> 66237920Snp#include <rdma/ib_user_verbs.h> 67237920Snp#include <linux/idr.h> 68237920Snp#include <ulp/iw_cxgb/iw_cxgb_ib_intfc.h> 69178786Skmacy 70178786Skmacy#include <cxgb_include.h> 71178786Skmacy#include <ulp/iw_cxgb/iw_cxgb_wr.h> 72178786Skmacy#include <ulp/iw_cxgb/iw_cxgb_hal.h> 73178786Skmacy#include <ulp/iw_cxgb/iw_cxgb_provider.h> 74178786Skmacy#include <ulp/iw_cxgb/iw_cxgb_cm.h> 75178786Skmacy#include <ulp/iw_cxgb/iw_cxgb.h> 76178786Skmacy#include <ulp/iw_cxgb/iw_cxgb_resource.h> 77178786Skmacy#include <ulp/iw_cxgb/iw_cxgb_user.h> 78178786Skmacy 79178786Skmacystatic void 80178786Skmacypost_qp_event(struct iwch_dev *rnicp, struct iwch_qp *qhp, struct iwch_cq *chp, 81178786Skmacy struct respQ_msg_t *rsp_msg, 82178786Skmacy enum ib_event_type ib_event, 83178786Skmacy int send_term) 84178786Skmacy{ 85178786Skmacy struct ib_event event; 86178786Skmacy struct iwch_qp_attributes attrs; 87178786Skmacy 88237920Snp mtx_lock(&rnicp->lock); 89237920Snp 90237920Snp if (!qhp) { 91237920Snp CTR3(KTR_IW_CXGB, "%s unaffiliated error 0x%x qpid 0x%x\n", 92237920Snp __func__, CQE_STATUS(rsp_msg->cqe), 93237920Snp CQE_QPID(rsp_msg->cqe)); 94237920Snp mtx_unlock(&rnicp->lock); 95237920Snp return; 96237920Snp } 97237920Snp 98178786Skmacy if ((qhp->attr.state == IWCH_QP_STATE_ERROR) || 99178786Skmacy (qhp->attr.state == IWCH_QP_STATE_TERMINATE)) { 100178786Skmacy CTR4(KTR_IW_CXGB, "%s AE received after RTS - " 101178786Skmacy "qp state %d qpid 0x%x status 0x%x", __FUNCTION__, 102178786Skmacy qhp->attr.state, qhp->wq.qpid, CQE_STATUS(rsp_msg->cqe)); 103237920Snp mtx_unlock(&rnicp->lock); 104178786Skmacy return; 105178786Skmacy } 106178786Skmacy 107178786Skmacy log(LOG_ERR, "%s - AE qpid 0x%x opcode %d status 0x%x " 108178786Skmacy "type %d wrid.hi 0x%x wrid.lo 0x%x \n", __FUNCTION__, 109178786Skmacy CQE_QPID(rsp_msg->cqe), CQE_OPCODE(rsp_msg->cqe), 110178786Skmacy CQE_STATUS(rsp_msg->cqe), CQE_TYPE(rsp_msg->cqe), 111178786Skmacy CQE_WRID_HI(rsp_msg->cqe), CQE_WRID_LOW(rsp_msg->cqe)); 112178786Skmacy 113237920Snp mtx_unlock(&rnicp->lock); 114178786Skmacy 115237920Snp if (qhp->attr.state == IWCH_QP_STATE_RTS) { 116237920Snp attrs.next_state = IWCH_QP_STATE_TERMINATE; 117237920Snp iwch_modify_qp(qhp->rhp, qhp, IWCH_QP_ATTR_NEXT_STATE, 118237920Snp &attrs, 1); 119237920Snp if (send_term) 120237920Snp iwch_post_terminate(qhp, rsp_msg); 121237920Snp } 122237920Snp 123178786Skmacy event.event = ib_event; 124178786Skmacy event.device = chp->ibcq.device; 125178786Skmacy if (ib_event == IB_EVENT_CQ_ERR) 126178786Skmacy event.element.cq = &chp->ibcq; 127178786Skmacy else 128178786Skmacy event.element.qp = &qhp->ibqp; 129178786Skmacy 130178786Skmacy if (qhp->ibqp.event_handler) 131178786Skmacy (*qhp->ibqp.event_handler)(&event, qhp->ibqp.qp_context); 132178786Skmacy 133237920Snp (*chp->ibcq.comp_handler)(&chp->ibcq, chp->ibcq.cq_context); 134178786Skmacy} 135178786Skmacy 136178786Skmacyvoid 137237920Snpiwch_ev_dispatch(struct iwch_dev *rnicp, struct mbuf *m) 138178786Skmacy{ 139178786Skmacy struct respQ_msg_t *rsp_msg = (struct respQ_msg_t *) m->m_data; 140178786Skmacy struct iwch_cq *chp; 141178786Skmacy struct iwch_qp *qhp; 142178786Skmacy u32 cqid = RSPQ_CQID(rsp_msg); 143178786Skmacy 144178786Skmacy mtx_lock(&rnicp->lock); 145178786Skmacy chp = get_chp(rnicp, cqid); 146178786Skmacy qhp = get_qhp(rnicp, CQE_QPID(rsp_msg->cqe)); 147178786Skmacy if (!chp || !qhp) { 148178786Skmacy log(LOG_ERR,"BAD AE cqid 0x%x qpid 0x%x opcode %d " 149178786Skmacy "status 0x%x type %d wrid.hi 0x%x wrid.lo 0x%x \n", 150178786Skmacy cqid, CQE_QPID(rsp_msg->cqe), 151178786Skmacy CQE_OPCODE(rsp_msg->cqe), CQE_STATUS(rsp_msg->cqe), 152178786Skmacy CQE_TYPE(rsp_msg->cqe), CQE_WRID_HI(rsp_msg->cqe), 153178786Skmacy CQE_WRID_LOW(rsp_msg->cqe)); 154178786Skmacy mtx_unlock(&rnicp->lock); 155237920Snp return; 156178786Skmacy } 157178786Skmacy iwch_qp_add_ref(&qhp->ibqp); 158178786Skmacy mtx_lock(&chp->lock); 159178786Skmacy ++chp->refcnt; 160178786Skmacy mtx_unlock(&chp->lock); 161178786Skmacy mtx_unlock(&rnicp->lock); 162178786Skmacy 163178786Skmacy /* 164178786Skmacy * 1) completion of our sending a TERMINATE. 165178786Skmacy * 2) incoming TERMINATE message. 166178786Skmacy */ 167178786Skmacy if ((CQE_OPCODE(rsp_msg->cqe) == T3_TERMINATE) && 168178786Skmacy (CQE_STATUS(rsp_msg->cqe) == 0)) { 169178786Skmacy if (SQ_TYPE(rsp_msg->cqe)) { 170178786Skmacy CTR3(KTR_IW_CXGB, "%s QPID 0x%x ep %p disconnecting", 171178786Skmacy __FUNCTION__, qhp->wq.qpid, qhp->ep); 172178786Skmacy iwch_ep_disconnect(qhp->ep, 0, M_NOWAIT); 173178786Skmacy } else { 174178786Skmacy CTR2(KTR_IW_CXGB, "%s post REQ_ERR AE QPID 0x%x", __FUNCTION__, 175178786Skmacy qhp->wq.qpid); 176178786Skmacy post_qp_event(rnicp, qhp, chp, rsp_msg, 177178786Skmacy IB_EVENT_QP_REQ_ERR, 0); 178178786Skmacy iwch_ep_disconnect(qhp->ep, 0, M_NOWAIT); 179178786Skmacy } 180178786Skmacy goto done; 181178786Skmacy } 182178786Skmacy 183178786Skmacy /* Bad incoming Read request */ 184178786Skmacy if (SQ_TYPE(rsp_msg->cqe) && 185178786Skmacy (CQE_OPCODE(rsp_msg->cqe) == T3_READ_RESP)) { 186178786Skmacy post_qp_event(rnicp, qhp, chp, rsp_msg, IB_EVENT_QP_REQ_ERR, 1); 187178786Skmacy goto done; 188178786Skmacy } 189178786Skmacy 190178786Skmacy /* Bad incoming write */ 191178786Skmacy if (RQ_TYPE(rsp_msg->cqe) && 192178786Skmacy (CQE_OPCODE(rsp_msg->cqe) == T3_RDMA_WRITE)) { 193178786Skmacy post_qp_event(rnicp, qhp, chp, rsp_msg, IB_EVENT_QP_REQ_ERR, 1); 194178786Skmacy goto done; 195178786Skmacy } 196178786Skmacy 197178786Skmacy switch (CQE_STATUS(rsp_msg->cqe)) { 198178786Skmacy 199178786Skmacy /* Completion Events */ 200178786Skmacy case TPT_ERR_SUCCESS: 201178786Skmacy#if 0 202178786Skmacy /* 203178786Skmacy * Confirm the destination entry if this is a RECV completion. 204178786Skmacy */ 205178786Skmacy if (qhp->ep && SQ_TYPE(rsp_msg->cqe)) 206178786Skmacy dst_confirm(qhp->ep->dst); 207178786Skmacy#endif 208178786Skmacy (*chp->ibcq.comp_handler)(&chp->ibcq, chp->ibcq.cq_context); 209178786Skmacy break; 210178786Skmacy 211178786Skmacy case TPT_ERR_STAG: 212178786Skmacy case TPT_ERR_PDID: 213178786Skmacy case TPT_ERR_QPID: 214178786Skmacy case TPT_ERR_ACCESS: 215178786Skmacy case TPT_ERR_WRAP: 216178786Skmacy case TPT_ERR_BOUND: 217178786Skmacy case TPT_ERR_INVALIDATE_SHARED_MR: 218178786Skmacy case TPT_ERR_INVALIDATE_MR_WITH_MW_BOUND: 219178786Skmacy post_qp_event(rnicp, qhp, chp, rsp_msg, IB_EVENT_QP_ACCESS_ERR, 1); 220178786Skmacy break; 221178786Skmacy 222178786Skmacy /* Device Fatal Errors */ 223178786Skmacy case TPT_ERR_ECC: 224178786Skmacy case TPT_ERR_ECC_PSTAG: 225178786Skmacy case TPT_ERR_INTERNAL_ERR: 226178786Skmacy post_qp_event(rnicp, qhp, chp, rsp_msg, IB_EVENT_DEVICE_FATAL, 1); 227178786Skmacy break; 228178786Skmacy 229178786Skmacy /* QP Fatal Errors */ 230178786Skmacy case TPT_ERR_OUT_OF_RQE: 231178786Skmacy case TPT_ERR_PBL_ADDR_BOUND: 232178786Skmacy case TPT_ERR_CRC: 233178786Skmacy case TPT_ERR_MARKER: 234178786Skmacy case TPT_ERR_PDU_LEN_ERR: 235178786Skmacy case TPT_ERR_DDP_VERSION: 236178786Skmacy case TPT_ERR_RDMA_VERSION: 237178786Skmacy case TPT_ERR_OPCODE: 238178786Skmacy case TPT_ERR_DDP_QUEUE_NUM: 239178786Skmacy case TPT_ERR_MSN: 240178786Skmacy case TPT_ERR_TBIT: 241178786Skmacy case TPT_ERR_MO: 242178786Skmacy case TPT_ERR_MSN_GAP: 243178786Skmacy case TPT_ERR_MSN_RANGE: 244178786Skmacy case TPT_ERR_RQE_ADDR_BOUND: 245178786Skmacy case TPT_ERR_IRD_OVERFLOW: 246178786Skmacy post_qp_event(rnicp, qhp, chp, rsp_msg, IB_EVENT_QP_FATAL, 1); 247178786Skmacy break; 248178786Skmacy 249178786Skmacy default: 250178786Skmacy log(LOG_ERR,"Unknown T3 status 0x%x QPID 0x%x\n", 251178786Skmacy CQE_STATUS(rsp_msg->cqe), qhp->wq.qpid); 252178786Skmacy post_qp_event(rnicp, qhp, chp, rsp_msg, IB_EVENT_QP_FATAL, 1); 253178786Skmacy break; 254178786Skmacy } 255178786Skmacydone: 256178786Skmacy mtx_lock(&chp->lock); 257178786Skmacy if (--chp->refcnt == 0) 258178786Skmacy wakeup(chp); 259178786Skmacy mtx_unlock(&chp->lock); 260178786Skmacy iwch_qp_rem_ref(&qhp->ibqp); 261178786Skmacy} 262237920Snp#endif 263