Lines Matching refs:qp
90 * @qp: hw qp ptr
92 static inline u64 irdma_nop_hdr(struct irdma_qp_uk *qp){
95 FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity);
100 * @qp: hw qp ptr
103 irdma_nop_1(struct irdma_qp_uk *qp)
108 if (!qp->sq_ring.head)
111 wqe_idx = IRDMA_RING_CURRENT_HEAD(qp->sq_ring);
112 wqe = qp->sq_base[wqe_idx].elem;
114 qp->sq_wrtrk_array[wqe_idx].quanta = IRDMA_QP_WQE_MIN_QUANTA;
123 set_64bit_val(wqe, IRDMA_BYTE_24, irdma_nop_hdr(qp));
130 * @qp: hw qp ptr
134 irdma_clr_wqes(struct irdma_qp_uk *qp, u32 qp_wqe_idx)
140 wqe_idx = (qp_wqe_idx + 128) % qp->sq_ring.size;
141 wqe = qp->sq_base[wqe_idx].elem;
143 memset(wqe, qp->swqe_polarity ? 0 : 0xFF, 0x1000);
145 memset(wqe, qp->swqe_polarity ? 0xFF : 0, 0x1000);
151 * @qp: hw qp ptr
154 irdma_uk_qp_post_wr(struct irdma_qp_uk *qp)
164 get_64bit_val(qp->shadow_area, IRDMA_BYTE_0, &temp);
167 sw_sq_head = IRDMA_RING_CURRENT_HEAD(qp->sq_ring);
168 if (sw_sq_head != qp->initial_ring.head) {
169 if (qp->push_dropped) {
170 db_wr32(qp->qp_id, qp->wqe_alloc_db);
171 qp->push_dropped = false;
173 if (sw_sq_head > qp->initial_ring.head) {
174 if (hw_sq_tail >= qp->initial_ring.head &&
176 db_wr32(qp->qp_id, qp->wqe_alloc_db);
178 if (hw_sq_tail >= qp->initial_ring.head ||
180 db_wr32(qp->qp_id, qp->wqe_alloc_db);
185 qp->initial_ring.head = qp->sq_ring.head;
189 * irdma_qp_ring_push_db - ring qp doorbell
190 * @qp: hw qp ptr
194 irdma_qp_ring_push_db(struct irdma_qp_uk *qp, u32 wqe_idx)
196 set_32bit_val(qp->push_db, 0,
197 FIELD_PREP(IRDMA_WQEALLOC_WQE_DESC_INDEX, wqe_idx >> 3) | qp->qp_id);
198 qp->initial_ring.head = qp->sq_ring.head;
199 qp->push_mode = true;
200 qp->push_dropped = false;
204 irdma_qp_push_wqe(struct irdma_qp_uk *qp, __le64 * wqe, u16 quanta,
209 if (IRDMA_RING_CURRENT_HEAD(qp->initial_ring) !=
210 IRDMA_RING_CURRENT_TAIL(qp->sq_ring) &&
211 !qp->push_mode) {
212 irdma_uk_qp_post_wr(qp);
214 push = (__le64 *) ((uintptr_t)qp->push_wqe +
217 irdma_qp_ring_push_db(qp, wqe_idx);
223 * @qp: hw qp ptr
230 irdma_qp_get_next_send_wqe(struct irdma_qp_uk *qp, u32 *wqe_idx,
240 avail_quanta = qp->uk_attrs->max_hw_sq_chunk -
241 (IRDMA_RING_CURRENT_HEAD(qp->sq_ring) %
242 qp->uk_attrs->max_hw_sq_chunk);
246 if (*quanta > IRDMA_SQ_RING_FREE_QUANTA(qp->sq_ring))
251 IRDMA_SQ_RING_FREE_QUANTA(qp->sq_ring))
254 nop_wqe_idx = IRDMA_RING_CURRENT_HEAD(qp->sq_ring);
256 irdma_nop_1(qp);
257 IRDMA_RING_MOVE_HEAD_NOCHECK(qp->sq_ring);
259 if (qp->push_db && info->push_wqe)
260 irdma_qp_push_wqe(qp, qp->sq_base[nop_wqe_idx].elem,
264 *wqe_idx = IRDMA_RING_CURRENT_HEAD(qp->sq_ring);
266 qp->swqe_polarity = !qp->swqe_polarity;
268 IRDMA_RING_MOVE_HEAD_BY_COUNT_NOCHECK(qp->sq_ring, *quanta);
270 irdma_clr_wqes(qp, *wqe_idx);
272 wqe = qp->sq_base[*wqe_idx].elem;
273 if (qp->uk_attrs->hw_rev == IRDMA_GEN_1 && wqe_quanta == 1 &&
274 (IRDMA_RING_CURRENT_HEAD(qp->sq_ring) & 1)) {
275 wqe_0 = qp->sq_base[IRDMA_RING_CURRENT_HEAD(qp->sq_ring)].elem;
277 qp->swqe_polarity ? 0 : 1));
279 qp->sq_wrtrk_array[*wqe_idx].wrid = info->wr_id;
280 qp->sq_wrtrk_array[*wqe_idx].wr_len = total_size;
281 qp->sq_wrtrk_array[*wqe_idx].quanta = wqe_quanta;
282 qp->sq_wrtrk_array[*wqe_idx].signaled = info->signaled;
288 * irdma_qp_get_next_recv_wqe - get next qp's rcv wqe
289 * @qp: hw qp ptr
293 irdma_qp_get_next_recv_wqe(struct irdma_qp_uk *qp, u32 *wqe_idx)
298 if (IRDMA_RING_FULL_ERR(qp->rq_ring))
301 IRDMA_ATOMIC_RING_MOVE_HEAD(qp->rq_ring, *wqe_idx, ret_code);
306 qp->rwqe_polarity = !qp->rwqe_polarity;
308 wqe = qp->rq_base[*wqe_idx * qp->rq_wqe_size_multiplier].elem;
315 * @qp: hw qp ptr
320 irdma_uk_rdma_write(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info,
333 info->push_wqe = qp->push_db ? true : false;
336 if (op_info->num_lo_sges > qp->max_sq_frag_cnt)
353 wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, &quanta, total_size, info);
357 qp->sq_wrtrk_array[wqe_idx].signaled = info->signaled;
366 qp->wqe_ops.iw_set_fragment(wqe, IRDMA_BYTE_0,
368 qp->swqe_polarity);
373 qp->wqe_ops.iw_set_fragment(wqe, byte_off,
375 qp->swqe_polarity);
380 if (qp->uk_attrs->hw_rev >= IRDMA_GEN_2 && !(frag_cnt & 0x01) &&
382 qp->wqe_ops.iw_set_fragment(wqe, byte_off, NULL,
383 qp->swqe_polarity);
384 if (qp->uk_attrs->hw_rev == IRDMA_GEN_2)
397 FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity);
403 irdma_qp_push_wqe(qp, wqe, quanta, wqe_idx, post_sq);
405 irdma_uk_qp_post_wr(qp);
412 * @qp: hw qp ptr
418 irdma_uk_rdma_read(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info,
432 info->push_wqe = qp->push_db ? true : false;
435 if (qp->max_sq_frag_cnt < op_info->num_lo_sges)
445 wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, &quanta, total_size, info);
449 if (qp->rd_fence_rate && (qp->ord_cnt++ == qp->rd_fence_rate)) {
451 qp->ord_cnt = 0;
454 qp->sq_wrtrk_array[wqe_idx].signaled = info->signaled;
459 qp->wqe_ops.iw_set_fragment(wqe, IRDMA_BYTE_0, op_info->lo_sg_list,
460 qp->swqe_polarity);
462 qp->wqe_ops.iw_set_fragment(wqe, byte_off,
464 qp->swqe_polarity);
469 if (qp->uk_attrs->hw_rev >= IRDMA_GEN_2 &&
471 qp->wqe_ops.iw_set_fragment(wqe, byte_off, NULL,
472 qp->swqe_polarity);
473 if (qp->uk_attrs->hw_rev == IRDMA_GEN_2)
488 FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity);
494 irdma_qp_push_wqe(qp, wqe, quanta, wqe_idx, post_sq);
496 irdma_uk_qp_post_wr(qp);
503 * @qp: hw qp ptr
508 irdma_uk_send(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info,
520 info->push_wqe = qp->push_db ? true : false;
523 if (qp->max_sq_frag_cnt < op_info->num_sges)
537 wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, &quanta, total_size, info);
548 qp->wqe_ops.iw_set_fragment(wqe, IRDMA_BYTE_0,
550 qp->swqe_polarity);
555 qp->wqe_ops.iw_set_fragment(wqe, byte_off, &op_info->sg_list[i],
556 qp->swqe_polarity);
561 if (qp->uk_attrs->hw_rev >= IRDMA_GEN_2 && !(frag_cnt & 0x01) &&
563 qp->wqe_ops.iw_set_fragment(wqe, byte_off, NULL,
564 qp->swqe_polarity);
565 if (qp->uk_attrs->hw_rev == IRDMA_GEN_2)
585 FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity);
591 irdma_qp_push_wqe(qp, wqe, quanta, wqe_idx, post_sq);
593 irdma_uk_qp_post_wr(qp);
754 * @qp: hw qp ptr
759 irdma_uk_inline_rdma_write(struct irdma_qp_uk *qp,
770 info->push_wqe = qp->push_db ? true : false;
773 if (unlikely(qp->max_sq_frag_cnt < op_info->num_lo_sges))
779 if (unlikely(total_size > qp->max_inline_data))
782 quanta = qp->wqe_ops.iw_inline_data_size_to_quanta(total_size);
783 wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, &quanta, total_size, info);
787 qp->sq_wrtrk_array[wqe_idx].signaled = info->signaled;
802 FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity);
808 qp->wqe_ops.iw_copy_inline_data((u8 *)wqe, op_info->lo_sg_list,
809 op_info->num_lo_sges, qp->swqe_polarity);
816 irdma_qp_push_wqe(qp, wqe, quanta, wqe_idx, post_sq);
818 irdma_uk_qp_post_wr(qp);
825 * @qp: hw qp ptr
830 irdma_uk_inline_send(struct irdma_qp_uk *qp,
841 info->push_wqe = qp->push_db ? true : false;
844 if (unlikely(qp->max_sq_frag_cnt < op_info->num_sges))
850 if (unlikely(total_size > qp->max_inline_data))
853 quanta = qp->wqe_ops.iw_inline_data_size_to_quanta(total_size);
854 wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, &quanta, total_size, info);
877 FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity);
882 qp->wqe_ops.iw_copy_inline_data((u8 *)wqe, op_info->sg_list,
883 op_info->num_sges, qp->swqe_polarity);
890 irdma_qp_push_wqe(qp, wqe, quanta, wqe_idx, post_sq);
892 irdma_uk_qp_post_wr(qp);
899 * @qp: hw qp ptr
904 irdma_uk_stag_local_invalidate(struct irdma_qp_uk *qp,
916 info->push_wqe = qp->push_db ? true : false;
920 wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, &quanta, 0, info);
925 qp->wqe_ops.iw_set_fragment(wqe, IRDMA_BYTE_0, &sge, 0);
934 FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity);
941 irdma_qp_push_wqe(qp, wqe, quanta, wqe_idx, post_sq);
943 irdma_uk_qp_post_wr(qp);
950 * @qp: hw qp ptr
955 irdma_uk_mw_bind(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info,
965 info->push_wqe = qp->push_db ? true : false;
969 wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, &quanta, 0, info);
973 qp->wqe_ops.iw_set_mw_bind_wqe(wqe, op_info);
986 FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity);
993 irdma_qp_push_wqe(qp, wqe, quanta, wqe_idx, post_sq);
995 irdma_uk_qp_post_wr(qp);
1002 * @qp: hw qp ptr
1006 irdma_uk_post_receive(struct irdma_qp_uk *qp,
1014 if (qp->max_rq_frag_cnt < info->num_sges)
1017 wqe = irdma_qp_get_next_recv_wqe(qp, &wqe_idx);
1021 qp->rq_wrid_array[wqe_idx] = info->wr_id;
1023 qp->wqe_ops.iw_set_fragment(wqe, IRDMA_BYTE_0, info->sg_list,
1024 qp->rwqe_polarity);
1027 qp->wqe_ops.iw_set_fragment(wqe, byte_off, &info->sg_list[i],
1028 qp->rwqe_polarity);
1033 if (qp->uk_attrs->hw_rev >= IRDMA_GEN_2 && !(info->num_sges & 0x01) &&
1035 qp->wqe_ops.iw_set_fragment(wqe, byte_off, NULL,
1036 qp->rwqe_polarity);
1037 if (qp->uk_attrs->hw_rev == IRDMA_GEN_2)
1043 FIELD_PREP(IRDMAQPSQ_VALID, qp->rwqe_polarity);
1171 irdma_repost_rq_wqes(struct irdma_qp_uk *qp, u32 start_idx,
1176 u8 wqe_quanta = qp->rq_wqe_size_multiplier;
1181 if (pthread_spin_lock(qp->lock))
1184 IRDMA_RING_SET_TAIL(qp->rq_ring, start_idx + 1);
1185 src_wqe = qp->rq_base[start_idx * qp->rq_wqe_size_multiplier].elem;
1186 dst_wqe = irdma_qp_get_next_recv_wqe(qp, &wqe_idx);
1190 if (FIELD_GET(IRDMAQPSQ_VALID, val) != qp->rwqe_polarity)
1195 qp->rq_wrid_array[wqe_idx] = qp->rq_wrid_array[start_idx];
1198 start_idx = (start_idx + 1) % qp->rq_size;
1201 pthread_spin_unlock(qp->lock);
1205 irdma_check_rq_cqe(struct irdma_qp_uk *qp, u32 *array_idx)
1207 u32 exp_idx = (qp->last_rx_cmpl_idx + 1) % qp->rq_size;
1210 if (qp->uk_attrs->feature_flags & IRDMA_FEATURE_RELAX_RQ_ORDER) {
1211 irdma_repost_rq_wqes(qp, exp_idx, *array_idx);
1212 qp->last_rx_cmpl_idx = *array_idx;
1218 qp->last_rx_cmpl_idx = exp_idx;
1223 qp->last_rx_cmpl_idx = *array_idx;
1262 struct irdma_qp_uk *qp;
1340 qp = (struct irdma_qp_uk *)(irdma_uintptr) comp_ctx;
1370 if (!qp || qp->destroy_pending) {
1375 info->qp_handle = (irdma_qp_handle) (irdma_uintptr) qp;
1381 ret_code = irdma_skip_duplicate_flush_cmpl(qp->rq_ring,
1382 qp->rq_flush_seen,
1388 array_idx = wqe_idx / qp->rq_wqe_size_multiplier;
1392 if (!IRDMA_RING_MORE_WORK(qp->rq_ring)) {
1397 info->wr_id = qp->rq_wrid_array[qp->rq_ring.tail];
1399 array_idx = qp->rq_ring.tail;
1401 info->wr_id = qp->rq_wrid_array[array_idx];
1403 if (irdma_check_rq_cqe(qp, &array_idx)) {
1404 info->wr_id = qp->rq_wrid_array[array_idx];
1406 IRDMA_RING_SET_TAIL(qp->rq_ring, array_idx + 1);
1419 IRDMA_RING_SET_TAIL(qp->rq_ring, array_idx + 1);
1421 qp->rq_flush_seen = true;
1422 if (!IRDMA_RING_MORE_WORK(qp->rq_ring))
1423 qp->rq_flush_complete = true;
1427 pring = &qp->rq_ring;
1429 if (qp->first_sq_wq) {
1430 if (wqe_idx + 1 >= qp->conn_wqes)
1431 qp->first_sq_wq = false;
1433 if (wqe_idx < qp->conn_wqes && qp->sq_ring.head == qp->sq_ring.tail) {
1444 qp->push_mode = false;
1445 qp->push_dropped = true;
1447 ret_code = irdma_skip_duplicate_flush_cmpl(qp->sq_ring,
1448 qp->sq_flush_seen,
1454 info->wr_id = qp->sq_wrtrk_array[wqe_idx].wrid;
1455 info->signaled = qp->sq_wrtrk_array[wqe_idx].signaled;
1457 info->bytes_xfered = qp->sq_wrtrk_array[wqe_idx].wr_len;
1459 IRDMA_RING_SET_TAIL(qp->sq_ring,
1460 wqe_idx + qp->sq_wrtrk_array[wqe_idx].quanta);
1462 if (pthread_spin_lock(qp->lock)) {
1466 if (!IRDMA_RING_MORE_WORK(qp->sq_ring)) {
1467 pthread_spin_unlock(qp->lock);
1477 tail = qp->sq_ring.tail;
1478 sw_wqe = qp->sq_base[tail].elem;
1483 IRDMA_RING_SET_TAIL(qp->sq_ring,
1484 tail + qp->sq_wrtrk_array[tail].quanta);
1486 info->wr_id = qp->sq_wrtrk_array[tail].wrid;
1487 info->signaled = qp->sq_wrtrk_array[tail].signaled;
1488 info->bytes_xfered = qp->sq_wrtrk_array[tail].wr_len;
1496 qp->sq_flush_seen = true;
1497 if (!IRDMA_RING_MORE_WORK(qp->sq_ring))
1498 qp->sq_flush_complete = true;
1499 pthread_spin_unlock(qp->lock);
1501 pring = &qp->sq_ring;
1537 * irdma_round_up_wq - return round up qp wq depth
1553 * @uk_attrs: qp HW attributes
1585 * irdma_get_sqdepth - get SQ depth (quanta) @uk_attrs: qp HW attributes @sq_size: SQ size @shift: shift which
1604 * irdma_get_rqdepth - get RQ depth (quanta) @uk_attrs: qp HW attributes @rq_size: SRQ size @shift: shift which
1639 * @qp: hw qp (user and kernel)
1640 * @info: qp initialization info
1643 irdma_setup_connection_wqes(struct irdma_qp_uk *qp,
1650 else if (qp->uk_attrs->feature_flags & IRDMA_FEATURE_RTS_AE)
1652 qp->conn_wqes = move_cnt;
1653 IRDMA_RING_MOVE_HEAD_BY_COUNT_NOCHECK(qp->sq_ring, move_cnt);
1654 IRDMA_RING_MOVE_TAIL_BY_COUNT(qp->sq_ring, move_cnt);
1655 IRDMA_RING_MOVE_HEAD_BY_COUNT_NOCHECK(qp->initial_ring, move_cnt);
1660 * @ukinfo: qp initialization info
1683 * @ukinfo: qp initialization info
1708 * irdma_uk_qp_init - initialize shared qp
1709 * @qp: hw qp (user and kernel)
1710 * @info: qp initialization info
1718 irdma_uk_qp_init(struct irdma_qp_uk *qp, struct irdma_qp_uk_init_info *info)
1723 qp->uk_attrs = info->uk_attrs;
1724 if (info->max_sq_frag_cnt > qp->uk_attrs->max_hw_wq_frags ||
1725 info->max_rq_frag_cnt > qp->uk_attrs->max_hw_wq_frags)
1728 qp->qp_caps = info->qp_caps;
1729 qp->sq_base = info->sq;
1730 qp->rq_base = info->rq;
1731 qp->qp_type = info->type ? info->type : IRDMA_QP_TYPE_IWARP;
1732 qp->shadow_area = info->shadow_area;
1733 qp->sq_wrtrk_array = info->sq_wrtrk_array;
1735 qp->rq_wrid_array = info->rq_wrid_array;
1736 qp->wqe_alloc_db = info->wqe_alloc_db;
1737 qp->last_rx_cmpl_idx = 0xffffffff;
1738 qp->rd_fence_rate = info->rd_fence_rate;
1739 qp->qp_id = info->qp_id;
1740 qp->sq_size = info->sq_size;
1741 qp->push_mode = false;
1742 qp->max_sq_frag_cnt = info->max_sq_frag_cnt;
1743 sq_ring_size = qp->sq_size << info->sq_shift;
1744 IRDMA_RING_INIT(qp->sq_ring, sq_ring_size);
1745 IRDMA_RING_INIT(qp->initial_ring, sq_ring_size);
1747 irdma_setup_connection_wqes(qp, info);
1748 qp->swqe_polarity = 1;
1749 qp->first_sq_wq = true;
1751 qp->swqe_polarity = 0;
1753 qp->swqe_polarity_deferred = 1;
1754 qp->rwqe_polarity = 0;
1755 qp->rq_size = info->rq_size;
1756 qp->max_rq_frag_cnt = info->max_rq_frag_cnt;
1757 qp->max_inline_data = info->max_inline_data;
1758 qp->rq_wqe_size = info->rq_shift;
1759 IRDMA_RING_INIT(qp->rq_ring, qp->rq_size);
1760 qp->rq_wqe_size_multiplier = 1 << info->rq_shift;
1761 if (qp->uk_attrs->hw_rev == IRDMA_GEN_1)
1762 qp->wqe_ops = iw_wqe_uk_ops_gen_1;
1764 qp->wqe_ops = iw_wqe_uk_ops;
1765 qp->start_wqe_idx = info->start_wqe_idx;