Lines Matching refs:qp
1202 "create qp: unsupported qp type=0x%x requested\n",
1209 "create qp: cannot create a SQ with %d elements (max_send_wr=0x%x)\n",
1216 "create qp: unsupported inline data size=0x%x requested (max_inline=0x%x)\n",
1223 "create qp: unsupported send_sge=0x%x requested (max_send_sge=0x%x)\n",
1230 "create qp: unsupported recv_sge=0x%x requested (max_recv_sge=0x%x)\n",
1247 "create qp: consumer QP cannot use GSI CQs.\n");
1272 struct qedr_qp *qp)
1284 uresp->rq_icid = qp->icid;
1285 if (qp->urq.db_mmap_entry)
1287 rdma_user_mmap_get_offset(qp->urq.db_mmap_entry);
1292 struct qedr_qp *qp)
1298 uresp->sq_icid = qp->icid;
1300 uresp->sq_icid = qp->icid + 1;
1302 if (qp->usq.db_mmap_entry)
1304 rdma_user_mmap_get_offset(qp->usq.db_mmap_entry);
1308 struct qedr_qp *qp, struct ib_udata *udata,
1315 if (qedr_qp_has_sq(qp))
1316 qedr_copy_sq_uresp(dev, uresp, qp);
1318 if (qedr_qp_has_rq(qp))
1319 qedr_copy_rq_uresp(dev, uresp, qp);
1322 uresp->qp_id = qp->qp_id;
1327 "create qp: failed a copy to user space with qp icid=0x%x.\n",
1328 qp->icid);
1343 struct qedr_qp *qp,
1347 spin_lock_init(&qp->q_lock);
1349 kref_init(&qp->refcnt);
1350 init_completion(&qp->iwarp_cm_comp);
1351 init_completion(&qp->qp_rel_comp);
1354 qp->pd = pd;
1355 qp->qp_type = attrs->qp_type;
1356 qp->max_inline_data = attrs->cap.max_inline_data;
1357 qp->state = QED_ROCE_QP_STATE_RESET;
1359 qp->prev_wqe_size = 0;
1361 qp->signaled = attrs->sq_sig_type == IB_SIGNAL_ALL_WR;
1362 qp->dev = dev;
1363 if (qedr_qp_has_sq(qp)) {
1364 qedr_reset_qp_hwq_info(&qp->sq);
1365 qp->sq.max_sges = attrs->cap.max_send_sge;
1366 qp->sq_cq = get_qedr_cq(attrs->send_cq);
1369 qp->sq.max_sges, qp->sq_cq->icid);
1373 qp->srq = get_qedr_srq(attrs->srq);
1375 if (qedr_qp_has_rq(qp)) {
1376 qedr_reset_qp_hwq_info(&qp->rq);
1377 qp->rq_cq = get_qedr_cq(attrs->recv_cq);
1378 qp->rq.max_sges = attrs->cap.max_recv_sge;
1381 qp->rq.max_sges, qp->rq_cq->icid);
1386 pd->pd_id, qp->qp_type, qp->max_inline_data,
1387 qp->state, qp->signaled, (attrs->srq) ? 1 : 0);
1390 qp->sq.max_sges, qp->sq_cq->icid);
1393 static int qedr_set_roce_db_info(struct qedr_dev *dev, struct qedr_qp *qp)
1397 if (qedr_qp_has_sq(qp)) {
1398 qp->sq.db = dev->db_addr +
1400 qp->sq.db_data.data.icid = qp->icid + 1;
1401 rc = qedr_db_recovery_add(dev, qp->sq.db, &qp->sq.db_data,
1407 if (qedr_qp_has_rq(qp)) {
1408 qp->rq.db = dev->db_addr +
1410 qp->rq.db_data.data.icid = qp->icid;
1411 rc = qedr_db_recovery_add(dev, qp->rq.db, &qp->rq.db_data,
1413 if (rc && qedr_qp_has_sq(qp))
1414 qedr_db_recovery_del(dev, qp->sq.db, &qp->sq.db_data);
1730 struct qedr_qp *qp,
1736 params->qp_handle_async_lo = lower_32_bits((uintptr_t) qp);
1737 params->qp_handle_async_hi = upper_32_bits((uintptr_t) qp);
1749 if (qedr_qp_has_sq(qp))
1752 if (qedr_qp_has_rq(qp))
1755 if (qedr_qp_has_srq(qp)) {
1757 params->srq_id = qp->srq->srq_id;
1765 static inline void qedr_qp_user_print(struct qedr_dev *dev, struct qedr_qp *qp)
1767 DP_DEBUG(dev, QEDR_MSG_QP, "create qp: successfully created user QP. "
1768 "qp=%p. "
1774 qp,
1775 qedr_qp_has_sq(qp) ? qp->usq.buf_addr : 0x0,
1776 qedr_qp_has_sq(qp) ? qp->usq.buf_len : 0,
1777 qedr_qp_has_rq(qp) ? qp->urq.buf_addr : 0x0,
1778 qedr_qp_has_sq(qp) ? qp->urq.buf_len : 0);
1783 struct qedr_qp *qp,
1786 qp->usq.pbl_tbl->va = out_params->sq_pbl_virt;
1787 qp->usq.pbl_tbl->pa = out_params->sq_pbl_phys;
1789 qedr_populate_pbls(dev, qp->usq.umem, qp->usq.pbl_tbl,
1790 &qp->usq.pbl_info, FW_PAGE_SHIFT);
1791 if (!qp->srq) {
1792 qp->urq.pbl_tbl->va = out_params->rq_pbl_virt;
1793 qp->urq.pbl_tbl->pa = out_params->rq_pbl_phys;
1796 qedr_populate_pbls(dev, qp->urq.umem, qp->urq.pbl_tbl,
1797 &qp->urq.pbl_info, FW_PAGE_SHIFT);
1802 struct qedr_qp *qp)
1804 if (qedr_qp_has_sq(qp)) {
1805 ib_umem_release(qp->usq.umem);
1806 qp->usq.umem = NULL;
1809 if (qedr_qp_has_rq(qp)) {
1810 ib_umem_release(qp->urq.umem);
1811 qp->urq.umem = NULL;
1815 qedr_free_pbl(dev, &qp->usq.pbl_info, qp->usq.pbl_tbl);
1816 qedr_free_pbl(dev, &qp->urq.pbl_info, qp->urq.pbl_tbl);
1818 kfree(qp->usq.pbl_tbl);
1819 kfree(qp->urq.pbl_tbl);
1822 if (qp->usq.db_rec_data) {
1823 qedr_db_recovery_del(dev, qp->usq.db_addr,
1824 &qp->usq.db_rec_data->db_data);
1825 rdma_user_mmap_entry_remove(qp->usq.db_mmap_entry);
1828 if (qp->urq.db_rec_data) {
1829 qedr_db_recovery_del(dev, qp->urq.db_addr,
1830 &qp->urq.db_rec_data->db_data);
1831 rdma_user_mmap_entry_remove(qp->urq.db_mmap_entry);
1835 qedr_db_recovery_del(dev, qp->urq.db_rec_db2_addr,
1836 &qp->urq.db_rec_db2_data);
1840 struct qedr_qp *qp,
1854 qp->create_type = QEDR_QP_CREATE_USER;
1870 if (qedr_qp_has_sq(qp)) {
1872 rc = qedr_init_user_queue(udata, dev, &qp->usq, ureq.sq_addr,
1878 if (qedr_qp_has_rq(qp)) {
1880 rc = qedr_init_user_queue(udata, dev, &qp->urq, ureq.rq_addr,
1883 ib_umem_release(qp->usq.umem);
1884 qp->usq.umem = NULL;
1886 qedr_free_pbl(dev, &qp->usq.pbl_info,
1887 qp->usq.pbl_tbl);
1889 kfree(qp->usq.pbl_tbl);
1896 qedr_init_common_qp_in_params(dev, pd, qp, attrs, false, &in_params);
1900 if (qp->qp_type == IB_QPT_XRC_TGT) {
1904 in_params.qp_handle_lo = qp->qp_id;
1908 if (qedr_qp_has_sq(qp)) {
1909 in_params.sq_num_pages = qp->usq.pbl_info.num_pbes;
1910 in_params.sq_pbl_ptr = qp->usq.pbl_tbl->pa;
1913 if (qedr_qp_has_rq(qp)) {
1914 in_params.rq_num_pages = qp->urq.pbl_info.num_pbes;
1915 in_params.rq_pbl_ptr = qp->urq.pbl_tbl->pa;
1921 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1924 if (!qp->qed_qp) {
1930 qedr_iwarp_populate_user_qp(dev, qp, &out_params);
1932 qp->qp_id = out_params.qp_id;
1933 qp->icid = out_params.icid;
1936 rc = qedr_copy_qp_uresp(dev, qp, udata, &uresp);
1942 if (qedr_qp_has_sq(qp)) {
1943 qp->usq.db_addr = ctx->dpi_addr + uresp.sq_db_offset;
1944 qp->sq.max_wr = attrs->cap.max_send_wr;
1945 rc = qedr_db_recovery_add(dev, qp->usq.db_addr,
1946 &qp->usq.db_rec_data->db_data,
1953 if (qedr_qp_has_rq(qp)) {
1954 qp->urq.db_addr = ctx->dpi_addr + uresp.rq_db_offset;
1955 qp->rq.max_wr = attrs->cap.max_recv_wr;
1956 rc = qedr_db_recovery_add(dev, qp->urq.db_addr,
1957 &qp->urq.db_rec_data->db_data,
1965 qp->urq.db_rec_db2_addr = ctx->dpi_addr + uresp.rq_db2_offset;
1970 qp->urq.db_rec_db2_data.data.icid = cpu_to_le16(qp->icid);
1971 qp->urq.db_rec_db2_data.data.value =
1974 rc = qedr_db_recovery_add(dev, qp->urq.db_rec_db2_addr,
1975 &qp->urq.db_rec_db2_data,
1981 qedr_qp_user_print(dev, qp);
1984 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
1986 DP_ERR(dev, "create qp: fatal fault. rc=%d", rc);
1989 qedr_cleanup_user(dev, ctx, qp);
1993 static int qedr_set_iwarp_db_info(struct qedr_dev *dev, struct qedr_qp *qp)
1997 qp->sq.db = dev->db_addr +
1999 qp->sq.db_data.data.icid = qp->icid;
2001 rc = qedr_db_recovery_add(dev, qp->sq.db,
2002 &qp->sq.db_data,
2008 qp->rq.db = dev->db_addr +
2010 qp->rq.db_data.data.icid = qp->icid;
2011 qp->rq.iwarp_db2 = dev->db_addr +
2013 qp->rq.iwarp_db2_data.data.icid = qp->icid;
2014 qp->rq.iwarp_db2_data.data.value = DQ_TCM_IWARP_POST_RQ_CF_CMD;
2016 rc = qedr_db_recovery_add(dev, qp->rq.db,
2017 &qp->rq.db_data,
2023 rc = qedr_db_recovery_add(dev, qp->rq.iwarp_db2,
2024 &qp->rq.iwarp_db2_data,
2032 struct qedr_qp *qp,
2047 rc = dev->ops->common->chain_alloc(dev->cdev, &qp->sq.pbl, ¶ms);
2051 in_params->sq_num_pages = qed_chain_get_page_cnt(&qp->sq.pbl);
2052 in_params->sq_pbl_ptr = qed_chain_get_pbl_phys(&qp->sq.pbl);
2058 rc = dev->ops->common->chain_alloc(dev->cdev, &qp->rq.pbl, ¶ms);
2062 in_params->rq_num_pages = qed_chain_get_page_cnt(&qp->rq.pbl);
2063 in_params->rq_pbl_ptr = qed_chain_get_pbl_phys(&qp->rq.pbl);
2065 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
2068 if (!qp->qed_qp)
2071 qp->qp_id = out_params.qp_id;
2072 qp->icid = out_params.icid;
2074 return qedr_set_roce_db_info(dev, qp);
2079 struct qedr_qp *qp,
2099 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
2102 if (!qp->qed_qp)
2113 rc = dev->ops->common->chain_alloc(dev->cdev, &qp->sq.pbl, ¶ms);
2123 rc = dev->ops->common->chain_alloc(dev->cdev, &qp->rq.pbl, ¶ms);
2127 qp->qp_id = out_params.qp_id;
2128 qp->icid = out_params.icid;
2130 return qedr_set_iwarp_db_info(dev, qp);
2133 dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
2138 static void qedr_cleanup_kernel(struct qedr_dev *dev, struct qedr_qp *qp)
2140 dev->ops->common->chain_free(dev->cdev, &qp->sq.pbl);
2141 kfree(qp->wqe_wr_id);
2143 dev->ops->common->chain_free(dev->cdev, &qp->rq.pbl);
2144 kfree(qp->rqe_wr_id);
2146 /* GSI qp is not registered to db mechanism so no need to delete */
2147 if (qp->qp_type == IB_QPT_GSI)
2150 qedr_db_recovery_del(dev, qp->sq.db, &qp->sq.db_data);
2152 if (!qp->srq) {
2153 qedr_db_recovery_del(dev, qp->rq.db, &qp->rq.db_data);
2156 qedr_db_recovery_del(dev, qp->rq.iwarp_db2,
2157 &qp->rq.iwarp_db2_data);
2162 struct qedr_qp *qp,
2174 qp->create_type = QEDR_QP_CREATE_KERNEL;
2186 qp->sq.max_wr = min_t(u32, attrs->cap.max_send_wr * dev->wq_multiplier,
2189 qp->wqe_wr_id = kcalloc(qp->sq.max_wr, sizeof(*qp->wqe_wr_id),
2191 if (!qp->wqe_wr_id) {
2192 DP_ERR(dev, "create qp: failed SQ shadow memory allocation\n");
2197 in_params.qp_handle_lo = lower_32_bits((uintptr_t) qp);
2198 in_params.qp_handle_hi = upper_32_bits((uintptr_t) qp);
2204 qp->rq.max_wr = (u16) max_t(u32, attrs->cap.max_recv_wr, 1);
2207 qp->rqe_wr_id = kcalloc(qp->rq.max_wr, sizeof(*qp->rqe_wr_id),
2209 if (!qp->rqe_wr_id) {
2211 "create qp: failed RQ shadow memory allocation\n");
2212 kfree(qp->wqe_wr_id);
2216 qedr_init_common_qp_in_params(dev, pd, qp, attrs, true, &in_params);
2223 n_rq_elems = qp->rq.max_wr * QEDR_MAX_RQE_ELEMENTS_PER_RQE;
2226 rc = qedr_iwarp_create_kernel_qp(dev, qp, &in_params,
2229 rc = qedr_roce_create_kernel_qp(dev, qp, &in_params,
2232 qedr_cleanup_kernel(dev, qp);
2237 static int qedr_free_qp_resources(struct qedr_dev *dev, struct qedr_qp *qp,
2245 if (qp->qp_type != IB_QPT_GSI) {
2246 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
2251 if (qp->create_type == QEDR_QP_CREATE_USER)
2252 qedr_cleanup_user(dev, ctx, qp);
2254 qedr_cleanup_kernel(dev, qp);
2266 struct qedr_qp *qp = get_qedr_qp(ibqp);
2277 DP_DEBUG(dev, QEDR_MSG_QP, "create qp: called from %s, pd=%p\n",
2285 "create qp: called from %s, event_handler=%p, eepd=%p sq_cq=%p, sq_icid=%d, rq_cq=%p, rq_icid=%d\n",
2292 qedr_set_common_qp_params(dev, qp, pd, attrs);
2295 return qedr_create_gsi_qp(dev, attrs, qp);
2298 rc = qedr_create_user_qp(dev, qp, ibpd, udata, attrs);
2300 rc = qedr_create_kernel_qp(dev, qp, ibpd, attrs);
2305 qp->ibqp.qp_num = qp->qp_id;
2308 rc = xa_insert(&dev->qps, qp->qp_id, qp, GFP_KERNEL);
2316 qedr_free_qp_resources(dev, qp, udata);
2363 struct qedr_qp *qp,
2390 writel(qp->rq.db_data.raw, qp->rq.db);
2443 if ((qp->rq.prod != qp->rq.cons) ||
2444 (qp->sq.prod != qp->sq.cons)) {
2447 qp->rq.prod, qp->rq.cons, qp->sq.prod,
2448 qp->sq.cons);
2468 struct qedr_qp *qp = get_qedr_qp(ibqp);
2470 struct qedr_dev *dev = get_qedr_dev(&qp->dev->ibdev);
2477 "modify qp: qp %p attr_mask=0x%x, state=%d", qp, attr_mask,
2483 old_qp_state = qedr_get_ibqp_state(qp->state);
2493 "modify qp: invalid attribute mask=0x%x specified for\n"
2495 attr_mask, qp->qp_id, ibqp->qp_type,
2524 qp->qkey = attr->qkey;
2548 qp->mtu = min(ib_mtu_enum_to_int(attr->path_mtu),
2553 if (!qp->mtu) {
2554 qp->mtu =
2556 pr_err("Fixing zeroed MTU to qp->mtu = %d\n", qp->mtu);
2566 qp->sgid_idx = grh->sgid_index;
2571 "modify qp: problems with GID index %d (rc=%d)\n",
2593 qp_params.mtu = qp->mtu;
2599 if (qp->mtu)
2600 qp_params.mtu = qp->mtu;
2626 qp->timeout = attr->timeout;
2645 qp->rq_psn = attr->rq_psn;
2673 qp->sq_psn = attr->sq_psn;
2698 qp->dest_qp_num = attr->dest_qp_num;
2701 cur_state = qp->state;
2708 if ((attr_mask & IB_QP_STATE) && qp->qp_type != IB_QPT_GSI &&
2710 qp->state = QED_ROCE_QP_STATE_ERR;
2712 if (qp->qp_type != IB_QPT_GSI)
2714 qp->qed_qp, &qp_params);
2717 if ((qp->qp_type != IB_QPT_GSI) && (!udata))
2718 rc = qedr_update_qp_state(dev, qp, cur_state,
2720 qp->state = qp_params.new_state;
2746 struct qedr_qp *qp = get_qedr_qp(ibqp);
2747 struct qedr_dev *dev = qp->dev;
2754 if (qp->qp_type != IB_QPT_GSI) {
2755 rc = dev->ops->rdma_query_qp(dev->rdma_ctx, qp->qed_qp, ¶ms);
2772 qp_attr->cap.max_send_wr = qp->sq.max_wr;
2773 qp_attr->cap.max_recv_wr = qp->rq.max_wr;
2774 qp_attr->cap.max_send_sge = qp->sq.max_sges;
2775 qp_attr->cap.max_recv_sge = qp->rq.max_sges;
2781 params.flow_label, qp->sgid_idx,
2786 qp_attr->timeout = qp->timeout;
2813 struct qedr_qp *qp = get_qedr_qp(ibqp);
2814 struct qedr_dev *dev = qp->dev;
2818 DP_DEBUG(dev, QEDR_MSG_QP, "destroy qp: destroying %p, qp type=%d\n",
2819 qp, qp->qp_type);
2822 if ((qp->state != QED_ROCE_QP_STATE_RESET) &&
2823 (qp->state != QED_ROCE_QP_STATE_ERR) &&
2824 (qp->state != QED_ROCE_QP_STATE_INIT)) {
2835 * to complete before destroying the qp.
2838 &qp->iwarp_cm_flags))
2839 wait_for_completion(&qp->iwarp_cm_comp);
2852 &qp->iwarp_cm_flags))
2853 wait_for_completion(&qp->iwarp_cm_comp);
2856 if (qp->qp_type == IB_QPT_GSI)
2864 xa_erase(&dev->qps, qp->qp_id);
2866 qedr_free_qp_resources(dev, qp, udata);
2869 qedr_iw_qp_rem_ref(&qp->ibqp);
2870 wait_for_completion(&qp->qp_rel_comp);
3290 struct qedr_qp *qp, u8 *wqe_size,
3324 wqe = (char *)qed_chain_produce(&qp->sq.pbl);
3375 static u32 qedr_prepare_sq_sges(struct qedr_qp *qp, u8 *wqe_size,
3382 struct rdma_sq_sge *sge = qed_chain_produce(&qp->sq.pbl);
3397 struct qedr_qp *qp,
3412 return qedr_prepare_sq_inline_data(dev, qp, &rwqe->wqe_size, wr,
3416 return qedr_prepare_sq_sges(qp, &rwqe->wqe_size, wr);
3420 struct qedr_qp *qp,
3431 return qedr_prepare_sq_inline_data(dev, qp, &swqe->wqe_size, wr,
3435 return qedr_prepare_sq_sges(qp, &swqe->wqe_size, wr);
3438 static int qedr_prepare_reg(struct qedr_qp *qp,
3445 fwqe2 = (struct rdma_sq_fmr_wqe_2nd *)qed_chain_produce(&qp->sq.pbl);
3471 qp->wqe_wr_id[qp->sq.prod].mr = mr;
3502 static inline bool qedr_can_post_send(struct qedr_qp *qp,
3506 struct qedr_dev *dev = qp->dev;
3509 err_wr = wr->num_sge > qp->sq.max_sges;
3510 wq_is_full = qedr_wq_is_full(&qp->sq);
3511 pbl_is_full = qed_chain_get_elem_left_u32(&qp->sq.pbl) <
3514 if (wq_is_full && !(qp->err_bitmap & QEDR_QP_ERR_SQ_FULL)) {
3517 qp);
3518 qp->err_bitmap |= QEDR_QP_ERR_SQ_FULL;
3521 if (err_wr && !(qp->err_bitmap & QEDR_QP_ERR_BAD_SR)) {
3524 qp);
3525 qp->err_bitmap |= QEDR_QP_ERR_BAD_SR;
3529 !(qp->err_bitmap & QEDR_QP_ERR_SQ_PBL_FULL)) {
3532 qp);
3533 qp->err_bitmap |= QEDR_QP_ERR_SQ_PBL_FULL;
3544 struct qedr_qp *qp = get_qedr_qp(ibqp);
3559 if (!qedr_can_post_send(qp, wr)) {
3564 wqe = qed_chain_produce(&qp->sq.pbl);
3565 qp->wqe_wr_id[qp->sq.prod].signaled =
3566 !!(wr->send_flags & IB_SEND_SIGNALED) || qp->signaled;
3571 comp = (!!(wr->send_flags & IB_SEND_SIGNALED)) || qp->signaled;
3575 wqe->prev_wqe_size = qp->prev_wqe_size;
3577 qp->wqe_wr_id[qp->sq.prod].opcode = qedr_ib_to_wc_opcode(wr->opcode);
3589 swqe2 = qed_chain_produce(&qp->sq.pbl);
3592 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3595 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3596 qp->prev_wqe_size = swqe->wqe_size;
3597 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3604 swqe2 = qed_chain_produce(&qp->sq.pbl);
3605 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3608 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3609 qp->prev_wqe_size = swqe->wqe_size;
3610 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3615 swqe2 = qed_chain_produce(&qp->sq.pbl);
3618 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3621 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3622 qp->prev_wqe_size = swqe->wqe_size;
3623 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3637 rwqe2 = qed_chain_produce(&qp->sq.pbl);
3638 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3641 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3642 qp->prev_wqe_size = rwqe->wqe_size;
3643 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3650 rwqe2 = qed_chain_produce(&qp->sq.pbl);
3651 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3654 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3655 qp->prev_wqe_size = rwqe->wqe_size;
3656 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3667 rwqe2 = qed_chain_produce(&qp->sq.pbl);
3668 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3671 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3672 qp->prev_wqe_size = rwqe->wqe_size;
3673 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3681 awqe2 = qed_chain_produce(&qp->sq.pbl);
3685 awqe3 = qed_chain_produce(&qp->sq.pbl);
3699 qedr_prepare_sq_sges(qp, NULL, wr);
3701 qp->wqe_wr_id[qp->sq.prod].wqe_size = awqe1->wqe_size;
3702 qp->prev_wqe_size = awqe1->wqe_size;
3711 qp->wqe_wr_id[qp->sq.prod].wqe_size = iwqe->wqe_size;
3712 qp->prev_wqe_size = iwqe->wqe_size;
3720 rc = qedr_prepare_reg(qp, fwqe1, reg_wr(wr));
3727 qp->wqe_wr_id[qp->sq.prod].wqe_size = fwqe1->wqe_size;
3728 qp->prev_wqe_size = fwqe1->wqe_size;
3743 value = le16_to_cpu(qp->sq.db_data.data.value);
3744 qed_chain_set_prod(&qp->sq.pbl, value, wqe);
3747 qp->prev_wqe_size = wqe->prev_wqe_size;
3759 struct qedr_qp *qp = get_qedr_qp(ibqp);
3765 if (qp->qp_type == IB_QPT_GSI)
3768 spin_lock_irqsave(&qp->q_lock, flags);
3771 if ((qp->state != QED_ROCE_QP_STATE_RTS) &&
3772 (qp->state != QED_ROCE_QP_STATE_ERR) &&
3773 (qp->state != QED_ROCE_QP_STATE_SQD)) {
3774 spin_unlock_irqrestore(&qp->q_lock, flags);
3778 qp->icid, qp->state);
3788 qp->wqe_wr_id[qp->sq.prod].wr_id = wr->wr_id;
3790 qedr_inc_sw_prod(&qp->sq);
3792 qp->sq.db_data.data.value++;
3803 * qp->wqe_wr_id is accessed during qedr_poll_cq, as
3811 writel(qp->sq.db_data.raw, qp->sq.db);
3813 spin_unlock_irqrestore(&qp->q_lock, flags);
3915 struct qedr_qp *qp = get_qedr_qp(ibqp);
3916 struct qedr_dev *dev = qp->dev;
3920 if (qp->qp_type == IB_QPT_GSI)
3923 spin_lock_irqsave(&qp->q_lock, flags);
3928 if (qed_chain_get_elem_left_u32(&qp->rq.pbl) <
3930 wr->num_sge > qp->rq.max_sges) {
3932 qed_chain_get_elem_left_u32(&qp->rq.pbl),
3934 qp->rq.max_sges);
3942 qed_chain_produce(&qp->rq.pbl);
3965 qed_chain_produce(&qp->rq.pbl);
3977 qp->rqe_wr_id[qp->rq.prod].wr_id = wr->wr_id;
3978 qp->rqe_wr_id[qp->rq.prod].wqe_size = i;
3980 qedr_inc_sw_prod(&qp->rq);
3982 /* qp->rqe_wr_id is accessed during qedr_poll_cq, as
3991 qp->rq.db_data.data.value++;
3993 writel(qp->rq.db_data.raw, qp->rq.db);
3996 writel(qp->rq.iwarp_db2_data.raw, qp->rq.iwarp_db2);
4002 spin_unlock_irqrestore(&qp->q_lock, flags);
4018 struct qedr_qp *qp;
4020 qp = (struct qedr_qp *)(uintptr_t)HILO_GEN(resp_cqe->qp_handle.hi,
4023 return qp;
4046 static inline void qedr_chk_if_fmr(struct qedr_qp *qp)
4048 if (qp->wqe_wr_id[qp->sq.cons].opcode == IB_WC_REG_MR)
4049 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
4052 static int process_req(struct qedr_dev *dev, struct qedr_qp *qp,
4059 while (num_entries && qp->sq.wqe_cons != hw_cons) {
4060 if (!qp->wqe_wr_id[qp->sq.cons].signaled && !force) {
4061 qedr_chk_if_fmr(qp);
4070 wc->src_qp = qp->id;
4071 wc->qp = &qp->ibqp;
4073 wc->wr_id = qp->wqe_wr_id[qp->sq.cons].wr_id;
4074 wc->opcode = qp->wqe_wr_id[qp->sq.cons].opcode;
4078 wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
4085 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
4089 wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
4099 while (qp->wqe_wr_id[qp->sq.cons].wqe_size--)
4100 qed_chain_consume(&qp->sq.pbl);
4101 qedr_inc_sw_cons(&qp->sq);
4108 struct qedr_qp *qp, struct qedr_cq *cq,
4116 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
4120 if (qp->state != QED_ROCE_QP_STATE_ERR)
4123 cq->icid, qp->icid);
4124 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
4129 qp->state = QED_ROCE_QP_STATE_ERR;
4130 cnt = process_req(dev, qp, cq, num_entries, wc,
4141 cq->icid, qp->icid);
4147 cq->icid, qp->icid);
4153 cq->icid, qp->icid);
4159 cq->icid, qp->icid);
4165 cq->icid, qp->icid);
4171 cq->icid, qp->icid);
4177 cq->icid, qp->icid);
4183 cq->icid, qp->icid);
4189 cq->icid, qp->icid);
4195 cq->icid, qp->icid);
4201 cq->icid, qp->icid);
4204 cnt += process_req(dev, qp, cq, 1, wc, req->sq_cons,
4264 static void __process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
4288 wc->src_qp = qp->id;
4289 wc->qp = &qp->ibqp;
4293 static int process_resp_one_srq(struct qedr_dev *dev, struct qedr_qp *qp,
4297 struct qedr_srq *srq = qp->srq;
4308 wc->src_qp = qp->id;
4309 wc->qp = &qp->ibqp;
4312 __process_resp_one(dev, qp, cq, wc, resp, wr_id);
4318 static int process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
4322 u64 wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
4324 __process_resp_one(dev, qp, cq, wc, resp, wr_id);
4326 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
4327 qed_chain_consume(&qp->rq.pbl);
4328 qedr_inc_sw_cons(&qp->rq);
4333 static int process_resp_flush(struct qedr_qp *qp, struct qedr_cq *cq,
4338 while (num_entries && qp->rq.wqe_cons != hw_cons) {
4343 wc->src_qp = qp->id;
4345 wc->wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
4346 wc->qp = &qp->ibqp;
4350 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
4351 qed_chain_consume(&qp->rq.pbl);
4352 qedr_inc_sw_cons(&qp->rq);
4358 static void try_consume_resp_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
4361 if (le16_to_cpu(resp->rq_cons_or_srq_id) == qp->rq.wqe_cons) {
4367 static int qedr_poll_cq_resp_srq(struct qedr_dev *dev, struct qedr_qp *qp,
4374 cnt = process_resp_one_srq(dev, qp, cq, wc, resp);
4380 static int qedr_poll_cq_resp(struct qedr_dev *dev, struct qedr_qp *qp,
4388 cnt = process_resp_flush(qp, cq, num_entries, wc,
4390 try_consume_resp_cqe(cq, qp, resp, update);
4392 cnt = process_resp_one(dev, qp, cq, wc, resp);
4400 static void try_consume_req_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
4403 if (le16_to_cpu(req->sq_cons) == qp->sq.wqe_cons) {
4433 struct qedr_qp *qp;
4439 qp = cqe_get_qp(cqe);
4440 if (!qp) {
4445 wc->qp = &qp->ibqp;
4449 cnt = qedr_poll_cq_req(dev, qp, cq, num_entries, wc,
4451 try_consume_req_cqe(cq, qp, &cqe->req, &update);
4454 cnt = qedr_poll_cq_resp(dev, qp, cq, num_entries, wc,
4458 cnt = qedr_poll_cq_resp_srq(dev, qp, cq, num_entries,