Lines Matching refs:qp
1081 static int ocrdma_add_qpn_map(struct ocrdma_dev *dev, struct ocrdma_qp *qp)
1085 if (qp->id < OCRDMA_MAX_QP && dev->qp_tbl[qp->id] == NULL) {
1086 dev->qp_tbl[qp->id] = qp;
1092 static void ocrdma_del_qpn_map(struct ocrdma_dev *dev, struct ocrdma_qp *qp)
1094 dev->qp_tbl[qp->id] = NULL;
1105 pr_err("%s(%d) unsupported qp type=0x%x requested\n",
1171 static int ocrdma_copy_qp_uresp(struct ocrdma_qp *qp,
1178 struct ocrdma_pd *pd = qp->pd;
1184 uresp.qp_id = qp->id;
1185 uresp.sq_dbid = qp->sq.dbid;
1187 uresp.sq_page_size = PAGE_ALIGN(qp->sq.len);
1188 uresp.sq_page_addr[0] = virt_to_phys(qp->sq.va);
1189 uresp.num_wqe_allocated = qp->sq.max_cnt;
1191 uresp.rq_dbid = qp->rq.dbid;
1193 uresp.rq_page_size = PAGE_ALIGN(qp->rq.len);
1194 uresp.rq_page_addr[0] = virt_to_phys(qp->rq.va);
1195 uresp.num_rqe_allocated = qp->rq.max_cnt;
1203 if (qp->dpp_enabled) {
1230 static void ocrdma_set_qp_db(struct ocrdma_dev *dev, struct ocrdma_qp *qp,
1234 qp->sq_db = dev->nic_info.db +
1237 qp->rq_db = dev->nic_info.db +
1241 qp->sq_db = dev->nic_info.db +
1244 qp->rq_db = dev->nic_info.db +
1250 static int ocrdma_alloc_wr_id_tbl(struct ocrdma_qp *qp)
1252 qp->wqe_wr_id_tbl =
1253 kcalloc(qp->sq.max_cnt, sizeof(*(qp->wqe_wr_id_tbl)),
1255 if (qp->wqe_wr_id_tbl == NULL)
1257 qp->rqe_wr_id_tbl =
1258 kcalloc(qp->rq.max_cnt, sizeof(u64), GFP_KERNEL);
1259 if (qp->rqe_wr_id_tbl == NULL)
1265 static void ocrdma_set_qp_init_params(struct ocrdma_qp *qp,
1269 qp->pd = pd;
1270 spin_lock_init(&qp->q_lock);
1271 INIT_LIST_HEAD(&qp->sq_entry);
1272 INIT_LIST_HEAD(&qp->rq_entry);
1274 qp->qp_type = attrs->qp_type;
1275 qp->cap_flags = OCRDMA_QP_INB_RD | OCRDMA_QP_INB_WR;
1276 qp->max_inline_data = attrs->cap.max_inline_data;
1277 qp->sq.max_sges = attrs->cap.max_send_sge;
1278 qp->rq.max_sges = attrs->cap.max_recv_sge;
1279 qp->state = OCRDMA_QPS_RST;
1280 qp->signaled = attrs->sq_sig_type == IB_SIGNAL_ALL_WR;
1299 struct ocrdma_qp *qp = get_ocrdma_qp(ibqp);
1316 ocrdma_set_qp_init_params(qp, pd, attrs);
1318 qp->cap_flags |= (OCRDMA_QP_MW_BIND | OCRDMA_QP_LKEY0 |
1322 status = ocrdma_mbx_create_qp(qp, attrs, ureq.enable_dpp_cq,
1330 status = ocrdma_alloc_wr_id_tbl(qp);
1335 status = ocrdma_add_qpn_map(dev, qp);
1338 ocrdma_set_qp_db(dev, qp, pd);
1340 status = ocrdma_copy_qp_uresp(qp, udata, dpp_offset,
1347 qp->ibqp.qp_num = qp->id;
1352 ocrdma_del_qpn_map(dev, qp);
1354 ocrdma_mbx_destroy_qp(dev, qp);
1357 kfree(qp->wqe_wr_id_tbl);
1358 kfree(qp->rqe_wr_id_tbl);
1368 struct ocrdma_qp *qp;
1372 qp = get_ocrdma_qp(ibqp);
1375 status = ocrdma_qp_state_change(qp, attr->qp_state, &old_qps);
1381 return ocrdma_mbx_modify_qp(dev, qp, attr, attr_mask);
1389 struct ocrdma_qp *qp;
1396 qp = get_ocrdma_qp(ibqp);
1402 spin_lock_irqsave(&qp->q_lock, flags);
1403 old_qps = get_ibqp_state(qp->state);
1408 spin_unlock_irqrestore(&qp->q_lock, flags);
1413 __func__, dev->id, attr_mask, qp->id, ibqp->qp_type,
1462 struct ocrdma_qp *qp = get_ocrdma_qp(ibqp);
1467 status = ocrdma_mbx_query_qp(dev, qp, ¶ms);
1471 if (qp->qp_type == IB_QPT_UD)
1483 qp_attr->qp_access_flags = ocrdma_to_ib_qp_acc_flags(qp->cap_flags);
1484 qp_attr->cap.max_send_wr = qp->sq.max_cnt - 1;
1485 qp_attr->cap.max_recv_wr = qp->rq.max_cnt - 1;
1486 qp_attr->cap.max_send_sge = qp->sq.max_sges;
1487 qp_attr->cap.max_recv_sge = qp->rq.max_sges;
1488 qp_attr->cap.max_inline_data = qp->max_inline_data;
1495 qp->sgid_idx,
1538 ocrdma_qp_state_change(qp, qp_attr->qp_state, NULL);
1556 static int is_hw_sq_empty(struct ocrdma_qp *qp)
1558 return (qp->sq.tail == qp->sq.head);
1561 static int is_hw_rq_empty(struct ocrdma_qp *qp)
1563 return (qp->rq.tail == qp->rq.head);
1588 static void ocrdma_discard_cqes(struct ocrdma_qp *qp, struct ocrdma_cq *cq)
1599 * find the matching CQE for a given qp,
1609 if (is_hw_sq_empty(qp) && (!qp->srq && is_hw_rq_empty(qp)))
1619 /* check for matching qp */
1620 if (qpn == 0 || qpn != qp->id)
1624 ocrdma_hwq_inc_tail(&qp->sq);
1626 if (qp->srq) {
1629 qp->srq->rq.max_wqe_idx;
1631 spin_lock_irqsave(&qp->srq->q_lock, flags);
1632 ocrdma_hwq_inc_tail(&qp->srq->rq);
1633 ocrdma_srq_toggle_bit(qp->srq, wqe_idx - 1);
1634 spin_unlock_irqrestore(&qp->srq->q_lock, flags);
1637 ocrdma_hwq_inc_tail(&qp->rq);
1650 void ocrdma_del_flush_qp(struct ocrdma_qp *qp)
1654 struct ocrdma_dev *dev = get_ocrdma_dev(qp->ibqp.device);
1658 found = ocrdma_is_qp_in_sq_flushlist(qp->sq_cq, qp);
1660 list_del(&qp->sq_entry);
1661 if (!qp->srq) {
1662 found = ocrdma_is_qp_in_rq_flushlist(qp->rq_cq, qp);
1664 list_del(&qp->rq_entry);
1672 struct ocrdma_qp *qp;
1678 qp = get_ocrdma_qp(ibqp);
1681 pd = qp->pd;
1684 if (qp->state != OCRDMA_QPS_RST) {
1694 (void) ocrdma_mbx_destroy_qp(dev, qp);
1700 spin_lock_irqsave(&qp->sq_cq->cq_lock, flags);
1701 if (qp->rq_cq && (qp->rq_cq != qp->sq_cq)) {
1702 spin_lock(&qp->rq_cq->cq_lock);
1703 ocrdma_del_qpn_map(dev, qp);
1704 spin_unlock(&qp->rq_cq->cq_lock);
1706 ocrdma_del_qpn_map(dev, qp);
1708 spin_unlock_irqrestore(&qp->sq_cq->cq_lock, flags);
1711 ocrdma_discard_cqes(qp, qp->sq_cq);
1712 ocrdma_discard_cqes(qp, qp->rq_cq);
1717 ocrdma_del_mmap(pd->uctx, (u64) qp->sq.pa,
1718 PAGE_ALIGN(qp->sq.len));
1719 if (!qp->srq)
1720 ocrdma_del_mmap(pd->uctx, (u64) qp->rq.pa,
1721 PAGE_ALIGN(qp->rq.len));
1724 ocrdma_del_flush_qp(qp);
1726 kfree(qp->wqe_wr_id_tbl);
1727 kfree(qp->rqe_wr_id_tbl);
1872 static void ocrdma_build_ud_hdr(struct ocrdma_qp *qp,
1881 if (qp->qp_type == IB_QPT_GSI)
1882 ud_hdr->qkey = qp->qkey;
1918 static int ocrdma_build_inline_sges(struct ocrdma_qp *qp,
1926 if (wr->send_flags & IB_SEND_INLINE && qp->qp_type != IB_QPT_UD) {
1928 if (unlikely(hdr->total_len > qp->max_inline_data)) {
1931 qp->max_inline_data, hdr->total_len);
1958 static int ocrdma_build_send(struct ocrdma_qp *qp, struct ocrdma_hdr_wqe *hdr,
1964 if (qp->qp_type == IB_QPT_UD || qp->qp_type == IB_QPT_GSI) {
1965 ocrdma_build_ud_hdr(qp, hdr, wr);
1972 return ocrdma_build_inline_sges(qp, hdr, sge, wr, wqe_size);
1975 static int ocrdma_build_write(struct ocrdma_qp *qp, struct ocrdma_hdr_wqe *hdr,
1983 status = ocrdma_build_inline_sges(qp, hdr, sge, wr, wqe_size);
1993 static void ocrdma_build_read(struct ocrdma_qp *qp, struct ocrdma_hdr_wqe *hdr,
2022 static int ocrdma_build_reg(struct ocrdma_qp *qp,
2078 static void ocrdma_ring_sq_db(struct ocrdma_qp *qp)
2080 u32 val = qp->sq.dbid | (1 << OCRDMA_DB_SQ_SHIFT);
2082 iowrite32(val, qp->sq_db);
2089 struct ocrdma_qp *qp = get_ocrdma_qp(ibqp);
2093 spin_lock_irqsave(&qp->q_lock, flags);
2094 if (qp->state != OCRDMA_QPS_RTS && qp->state != OCRDMA_QPS_SQD) {
2095 spin_unlock_irqrestore(&qp->q_lock, flags);
2101 if (qp->qp_type == IB_QPT_UD &&
2108 if (ocrdma_hwq_free_cnt(&qp->sq) == 0 ||
2109 wr->num_sge > qp->sq.max_sges) {
2114 hdr = ocrdma_hwq_head(&qp->sq);
2116 if (wr->send_flags & IB_SEND_SIGNALED || qp->signaled)
2132 ocrdma_build_send(qp, hdr, wr);
2138 status = ocrdma_build_send(qp, hdr, wr);
2146 status = ocrdma_build_write(qp, hdr, wr);
2149 ocrdma_build_read(qp, hdr, wr);
2160 status = ocrdma_build_reg(qp, hdr, reg_wr(wr));
2170 if (wr->send_flags & IB_SEND_SIGNALED || qp->signaled)
2171 qp->wqe_wr_id_tbl[qp->sq.head].signaled = 1;
2173 qp->wqe_wr_id_tbl[qp->sq.head].signaled = 0;
2174 qp->wqe_wr_id_tbl[qp->sq.head].wrid = wr->wr_id;
2180 ocrdma_ring_sq_db(qp);
2183 ocrdma_hwq_inc_head(&qp->sq);
2186 spin_unlock_irqrestore(&qp->q_lock, flags);
2190 static void ocrdma_ring_rq_db(struct ocrdma_qp *qp)
2192 u32 val = qp->rq.dbid | (1 << OCRDMA_DB_RQ_SHIFT);
2194 iowrite32(val, qp->rq_db);
2223 struct ocrdma_qp *qp = get_ocrdma_qp(ibqp);
2226 spin_lock_irqsave(&qp->q_lock, flags);
2227 if (qp->state == OCRDMA_QPS_RST || qp->state == OCRDMA_QPS_ERR) {
2228 spin_unlock_irqrestore(&qp->q_lock, flags);
2233 if (ocrdma_hwq_free_cnt(&qp->rq) == 0 ||
2234 wr->num_sge > qp->rq.max_sges) {
2239 rqe = ocrdma_hwq_head(&qp->rq);
2242 qp->rqe_wr_id_tbl[qp->rq.head] = wr->wr_id;
2247 ocrdma_ring_rq_db(qp);
2250 ocrdma_hwq_inc_head(&qp->rq);
2253 spin_unlock_irqrestore(&qp->q_lock, flags);
2399 static void ocrdma_update_wc(struct ocrdma_qp *qp, struct ib_wc *ibwc,
2406 hdr = ocrdma_hwq_head_from_idx(&qp->sq, wqe_idx);
2408 ibwc->wr_id = qp->wqe_wr_id_tbl[wqe_idx].wrid;
2437 static void ocrdma_set_cqe_status_flushed(struct ocrdma_qp *qp,
2449 if (qp->qp_type == IB_QPT_UD || qp->qp_type == IB_QPT_GSI) {
2470 struct ocrdma_qp *qp, int status)
2475 ibwc->qp = &qp->ibqp;
2478 ocrdma_flush_qp(qp);
2479 ocrdma_qp_state_change(qp, IB_QPS_ERR, NULL);
2484 if (!is_hw_rq_empty(qp) || !is_hw_sq_empty(qp)) {
2486 ocrdma_set_cqe_status_flushed(qp, cqe);
2492 struct ocrdma_qp *qp, int status)
2495 ibwc->wr_id = qp->rqe_wr_id_tbl[qp->rq.tail];
2496 ocrdma_hwq_inc_tail(&qp->rq);
2498 return ocrdma_update_err_cqe(ibwc, cqe, qp, status);
2502 struct ocrdma_qp *qp, int status)
2504 ocrdma_update_wc(qp, ibwc, qp->sq.tail);
2505 ocrdma_hwq_inc_tail(&qp->sq);
2507 return ocrdma_update_err_cqe(ibwc, cqe, qp, status);
2511 static bool ocrdma_poll_err_scqe(struct ocrdma_qp *qp,
2516 struct ocrdma_dev *dev = get_ocrdma_dev(qp->ibqp.device);
2525 if (is_hw_sq_empty(qp) && !is_hw_rq_empty(qp)) {
2529 if (!qp->srq && (qp->sq_cq == qp->rq_cq)) {
2532 expand = ocrdma_update_err_rcqe(ibwc, cqe, qp, status);
2543 } else if (is_hw_sq_empty(qp)) {
2550 expand = ocrdma_update_err_scqe(ibwc, cqe, qp, status);
2555 static bool ocrdma_poll_success_scqe(struct ocrdma_qp *qp,
2560 int tail = qp->sq.tail;
2563 if (!qp->wqe_wr_id_tbl[tail].signaled) {
2568 ibwc->qp = &qp->ibqp;
2569 ocrdma_update_wc(qp, ibwc, tail);
2573 OCRDMA_CQE_WQEIDX_MASK) & qp->sq.max_wqe_idx;
2577 ocrdma_hwq_inc_tail(&qp->sq);
2581 static bool ocrdma_poll_scqe(struct ocrdma_qp *qp, struct ocrdma_cqe *cqe,
2591 expand = ocrdma_poll_success_scqe(qp, cqe, ibwc, polled);
2593 expand = ocrdma_poll_err_scqe(qp, cqe, ibwc, polled, stop);
2626 struct ocrdma_qp *qp)
2632 srq = get_ocrdma_srq(qp->ibqp.srq);
2644 static bool ocrdma_poll_err_rcqe(struct ocrdma_qp *qp, struct ocrdma_cqe *cqe,
2649 struct ocrdma_dev *dev = get_ocrdma_dev(qp->ibqp.device);
2657 if (is_hw_rq_empty(qp) && !is_hw_sq_empty(qp)) {
2658 if (!qp->srq && (qp->sq_cq == qp->rq_cq)) {
2661 expand = ocrdma_update_err_scqe(ibwc, cqe, qp, status);
2667 } else if (is_hw_rq_empty(qp)) {
2674 expand = ocrdma_update_err_rcqe(ibwc, cqe, qp, status);
2679 static void ocrdma_poll_success_rcqe(struct ocrdma_qp *qp,
2684 dev = get_ocrdma_dev(qp->ibqp.device);
2686 ibwc->qp = &qp->ibqp;
2689 if (qp->qp_type == IB_QPT_UD || qp->qp_type == IB_QPT_GSI)
2705 if (qp->ibqp.srq) {
2706 ocrdma_update_free_srq_cqe(ibwc, cqe, qp);
2708 ibwc->wr_id = qp->rqe_wr_id_tbl[qp->rq.tail];
2709 ocrdma_hwq_inc_tail(&qp->rq);
2713 static bool ocrdma_poll_rcqe(struct ocrdma_qp *qp, struct ocrdma_cqe *cqe,
2720 if (qp->qp_type == IB_QPT_UD || qp->qp_type == IB_QPT_GSI) {
2731 ocrdma_poll_success_rcqe(qp, cqe, ibwc);
2733 expand = ocrdma_poll_err_rcqe(qp, cqe, ibwc, polled, stop,
2758 struct ocrdma_qp *qp = NULL;
2773 qp = dev->qp_tbl[qpn];
2774 BUG_ON(qp == NULL);
2777 expand = ocrdma_poll_scqe(qp, cqe, ibwc, &polled,
2780 expand = ocrdma_poll_rcqe(qp, cqe, ibwc, &polled,
2812 struct ocrdma_qp *qp, struct ib_wc *ibwc)
2817 if (is_hw_sq_empty(qp) && is_hw_rq_empty(qp))
2819 if (!is_hw_sq_empty(qp) && qp->sq_cq == cq) {
2820 ocrdma_update_wc(qp, ibwc, qp->sq.tail);
2821 ocrdma_hwq_inc_tail(&qp->sq);
2822 } else if (!is_hw_rq_empty(qp) && qp->rq_cq == cq) {
2823 ibwc->wr_id = qp->rqe_wr_id_tbl[qp->rq.tail];
2824 ocrdma_hwq_inc_tail(&qp->rq);
2843 struct ocrdma_qp *qp;
2854 /* adapter returns single error cqe when qp moves to
2860 list_for_each_entry(qp, &cq->sq_head, sq_entry) {
2863 err_cqes = ocrdma_add_err_cqe(cq, cqes_to_poll, qp, wc);