Lines Matching refs:qp
56 * @qp: hw qp ptr
58 static int irdma_nop_1(struct irdma_qp_uk *qp)
65 if (!qp->sq_ring.head)
68 wqe_idx = IRDMA_RING_CURRENT_HEAD(qp->sq_ring);
69 wqe = qp->sq_base[wqe_idx].elem;
71 qp->sq_wrtrk_array[wqe_idx].quanta = IRDMA_QP_WQE_MIN_QUANTA;
79 FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity);
91 * @qp: hw qp ptr
94 void irdma_clr_wqes(struct irdma_qp_uk *qp, u32 qp_wqe_idx)
100 wqe_idx = (qp_wqe_idx + 128) % qp->sq_ring.size;
101 sq = qp->sq_base + wqe_idx;
103 memset(sq, qp->swqe_polarity ? 0 : 0xFF,
106 memset(sq, qp->swqe_polarity ? 0xFF : 0,
113 * @qp: hw qp ptr
115 void irdma_uk_qp_post_wr(struct irdma_qp_uk *qp)
125 get_64bit_val(qp->shadow_area, 0, &temp);
128 sw_sq_head = IRDMA_RING_CURRENT_HEAD(qp->sq_ring);
129 if (sw_sq_head != qp->initial_ring.head) {
131 if (sw_sq_head > qp->initial_ring.head) {
132 if (hw_sq_tail >= qp->initial_ring.head &&
134 writel(qp->qp_id, qp->wqe_alloc_db);
136 if (hw_sq_tail >= qp->initial_ring.head ||
138 writel(qp->qp_id, qp->wqe_alloc_db);
143 qp->initial_ring.head = qp->sq_ring.head;
148 * @qp: hw qp ptr
154 __le64 *irdma_qp_get_next_send_wqe(struct irdma_qp_uk *qp, u32 *wqe_idx,
163 avail_quanta = qp->uk_attrs->max_hw_sq_chunk -
164 (IRDMA_RING_CURRENT_HEAD(qp->sq_ring) %
165 qp->uk_attrs->max_hw_sq_chunk);
168 if (quanta > IRDMA_SQ_RING_FREE_QUANTA(qp->sq_ring))
173 IRDMA_SQ_RING_FREE_QUANTA(qp->sq_ring))
177 irdma_nop_1(qp);
178 IRDMA_RING_MOVE_HEAD_NOCHECK(qp->sq_ring);
182 *wqe_idx = IRDMA_RING_CURRENT_HEAD(qp->sq_ring);
184 qp->swqe_polarity = !qp->swqe_polarity;
186 IRDMA_RING_MOVE_HEAD_BY_COUNT_NOCHECK(qp->sq_ring, quanta);
188 wqe = qp->sq_base[*wqe_idx].elem;
189 if (qp->uk_attrs->hw_rev == IRDMA_GEN_1 && quanta == 1 &&
190 (IRDMA_RING_CURRENT_HEAD(qp->sq_ring) & 1)) {
191 wqe_0 = qp->sq_base[IRDMA_RING_CURRENT_HEAD(qp->sq_ring)].elem;
192 wqe_0[3] = cpu_to_le64(FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity ? 0 : 1));
194 qp->sq_wrtrk_array[*wqe_idx].wrid = info->wr_id;
195 qp->sq_wrtrk_array[*wqe_idx].wr_len = total_size;
196 qp->sq_wrtrk_array[*wqe_idx].quanta = quanta;
202 * irdma_qp_get_next_recv_wqe - get next qp's rcv wqe
203 * @qp: hw qp ptr
206 __le64 *irdma_qp_get_next_recv_wqe(struct irdma_qp_uk *qp, u32 *wqe_idx)
211 if (IRDMA_RING_FULL_ERR(qp->rq_ring))
214 IRDMA_ATOMIC_RING_MOVE_HEAD(qp->rq_ring, *wqe_idx, ret_code);
219 qp->rwqe_polarity = !qp->rwqe_polarity;
221 wqe = qp->rq_base[*wqe_idx * qp->rq_wqe_size_multiplier].elem;
228 * @qp: hw qp ptr
232 int irdma_uk_rdma_write(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info,
246 if (op_info->num_lo_sges > qp->max_sq_frag_cnt)
263 wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, quanta, total_size,
268 irdma_clr_wqes(qp, wqe_idx);
278 qp->wqe_ops.iw_set_fragment(wqe, 0,
280 qp->swqe_polarity);
285 qp->wqe_ops.iw_set_fragment(wqe, byte_off,
287 qp->swqe_polarity);
292 if (qp->uk_attrs->hw_rev >= IRDMA_GEN_2 && !(frag_cnt & 0x01) &&
294 qp->wqe_ops.iw_set_fragment(wqe, byte_off, NULL,
295 qp->swqe_polarity);
296 if (qp->uk_attrs->hw_rev == IRDMA_GEN_2)
308 FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity);
315 irdma_uk_qp_post_wr(qp);
322 * @qp: hw qp ptr
327 int irdma_uk_rdma_read(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info,
341 if (qp->max_sq_frag_cnt < op_info->num_lo_sges)
351 wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, quanta, total_size,
356 irdma_clr_wqes(qp, wqe_idx);
362 qp->wqe_ops.iw_set_fragment(wqe, 0, op_info->lo_sg_list,
363 qp->swqe_polarity);
365 qp->wqe_ops.iw_set_fragment(wqe, byte_off,
367 qp->swqe_polarity);
372 if (qp->uk_attrs->hw_rev >= IRDMA_GEN_2 &&
374 qp->wqe_ops.iw_set_fragment(wqe, byte_off, NULL,
375 qp->swqe_polarity);
376 if (qp->uk_attrs->hw_rev == IRDMA_GEN_2)
389 FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity);
396 irdma_uk_qp_post_wr(qp);
403 * @qp: hw qp ptr
407 int irdma_uk_send(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info,
420 if (qp->max_sq_frag_cnt < op_info->num_sges)
434 wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, quanta, total_size,
439 irdma_clr_wqes(qp, wqe_idx);
448 qp->wqe_ops.iw_set_fragment(wqe, 0,
450 qp->swqe_polarity);
455 qp->wqe_ops.iw_set_fragment(wqe, byte_off, &op_info->sg_list[i],
456 qp->swqe_polarity);
461 if (qp->uk_attrs->hw_rev >= IRDMA_GEN_2 && !(frag_cnt & 0x01) &&
463 qp->wqe_ops.iw_set_fragment(wqe, byte_off, NULL,
464 qp->swqe_polarity);
465 if (qp->uk_attrs->hw_rev == IRDMA_GEN_2)
484 FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity);
491 irdma_uk_qp_post_wr(qp);
650 * @qp: hw qp ptr
654 int irdma_uk_inline_rdma_write(struct irdma_qp_uk *qp,
667 if (unlikely(qp->max_sq_frag_cnt < op_info->num_lo_sges))
673 if (unlikely(total_size > qp->max_inline_data))
676 quanta = qp->wqe_ops.iw_inline_data_size_to_quanta(total_size);
677 wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, quanta, total_size,
682 irdma_clr_wqes(qp, wqe_idx);
697 FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity);
703 qp->wqe_ops.iw_copy_inline_data((u8 *)wqe, op_info->lo_sg_list,
705 qp->swqe_polarity);
711 irdma_uk_qp_post_wr(qp);
718 * @qp: hw qp ptr
722 int irdma_uk_inline_send(struct irdma_qp_uk *qp,
735 if (unlikely(qp->max_sq_frag_cnt < op_info->num_sges))
741 if (unlikely(total_size > qp->max_inline_data))
744 quanta = qp->wqe_ops.iw_inline_data_size_to_quanta(total_size);
745 wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, quanta, total_size,
750 irdma_clr_wqes(qp, wqe_idx);
770 FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity);
775 qp->wqe_ops.iw_copy_inline_data((u8 *)wqe, op_info->sg_list,
776 op_info->num_sges, qp->swqe_polarity);
783 irdma_uk_qp_post_wr(qp);
790 * @qp: hw qp ptr
794 int irdma_uk_stag_local_invalidate(struct irdma_qp_uk *qp,
808 wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, IRDMA_QP_WQE_MIN_QUANTA,
813 irdma_clr_wqes(qp, wqe_idx);
816 qp->wqe_ops.iw_set_fragment(wqe, 0, &sge, 0);
824 FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity);
831 irdma_uk_qp_post_wr(qp);
838 * @qp: hw qp ptr
841 int irdma_uk_post_receive(struct irdma_qp_uk *qp,
849 if (qp->max_rq_frag_cnt < info->num_sges)
852 wqe = irdma_qp_get_next_recv_wqe(qp, &wqe_idx);
856 qp->rq_wrid_array[wqe_idx] = info->wr_id;
858 qp->wqe_ops.iw_set_fragment(wqe, 0, info->sg_list,
859 qp->rwqe_polarity);
862 qp->wqe_ops.iw_set_fragment(wqe, byte_off, &info->sg_list[i],
863 qp->rwqe_polarity);
868 if (qp->uk_attrs->hw_rev >= IRDMA_GEN_2 && !(info->num_sges & 0x01) &&
870 qp->wqe_ops.iw_set_fragment(wqe, byte_off, NULL,
871 qp->rwqe_polarity);
872 if (qp->uk_attrs->hw_rev == IRDMA_GEN_2)
878 FIELD_PREP(IRDMAQPSQ_VALID, qp->rwqe_polarity);
975 struct irdma_qp_uk *qp;
1079 qp = (struct irdma_qp_uk *)(unsigned long)comp_ctx;
1080 if (!qp || qp->destroy_pending) {
1085 info->qp_handle = (irdma_qp_handle)(unsigned long)qp;
1091 array_idx = wqe_idx / qp->rq_wqe_size_multiplier;
1095 if (!IRDMA_RING_MORE_WORK(qp->rq_ring)) {
1100 info->wr_id = qp->rq_wrid_array[qp->rq_ring.tail];
1101 array_idx = qp->rq_ring.tail;
1103 info->wr_id = qp->rq_wrid_array[array_idx];
1114 IRDMA_RING_SET_TAIL(qp->rq_ring, array_idx + 1);
1116 qp->rq_flush_seen = true;
1117 if (!IRDMA_RING_MORE_WORK(qp->rq_ring))
1118 qp->rq_flush_complete = true;
1122 pring = &qp->rq_ring;
1124 if (qp->first_sq_wq) {
1125 if (wqe_idx + 1 >= qp->conn_wqes)
1126 qp->first_sq_wq = false;
1128 if (wqe_idx < qp->conn_wqes && qp->sq_ring.head == qp->sq_ring.tail) {
1139 info->wr_id = qp->sq_wrtrk_array[wqe_idx].wrid;
1141 info->bytes_xfered = qp->sq_wrtrk_array[wqe_idx].wr_len;
1143 IRDMA_RING_SET_TAIL(qp->sq_ring,
1144 wqe_idx + qp->sq_wrtrk_array[wqe_idx].quanta);
1146 if (!IRDMA_RING_MORE_WORK(qp->sq_ring)) {
1156 tail = qp->sq_ring.tail;
1157 sw_wqe = qp->sq_base[tail].elem;
1162 IRDMA_RING_SET_TAIL(qp->sq_ring,
1163 tail + qp->sq_wrtrk_array[tail].quanta);
1165 info->wr_id = qp->sq_wrtrk_array[tail].wrid;
1166 info->bytes_xfered = qp->sq_wrtrk_array[tail].wr_len;
1173 qp->sq_flush_seen = true;
1174 if (!IRDMA_RING_MORE_WORK(qp->sq_ring))
1175 qp->sq_flush_complete = true;
1177 pring = &qp->sq_ring;
1213 * irdma_qp_round_up - return round up qp wq depth
1228 * @uk_attrs: qp HW attributes
1260 * @uk_attrs: qp HW attributes
1283 * @uk_attrs: qp HW attributes
1320 * @qp: hw qp (user and kernel)
1321 * @info: qp initialization info
1323 static void irdma_setup_connection_wqes(struct irdma_qp_uk *qp,
1329 (qp->uk_attrs->feature_flags & IRDMA_FEATURE_RTS_AE))
1332 qp->conn_wqes = move_cnt;
1333 IRDMA_RING_MOVE_HEAD_BY_COUNT_NOCHECK(qp->sq_ring, move_cnt);
1334 IRDMA_RING_MOVE_TAIL_BY_COUNT(qp->sq_ring, move_cnt);
1335 IRDMA_RING_MOVE_HEAD_BY_COUNT_NOCHECK(qp->initial_ring, move_cnt);
1340 * @ukinfo: qp initialization info
1365 * @ukinfo: qp initialization info
1387 * @ukinfo: qp initialization info
1411 * irdma_uk_qp_init - initialize shared qp
1412 * @qp: hw qp (user and kernel)
1413 * @info: qp initialization info
1420 int irdma_uk_qp_init(struct irdma_qp_uk *qp, struct irdma_qp_uk_init_info *info)
1425 qp->uk_attrs = info->uk_attrs;
1426 if (info->max_sq_frag_cnt > qp->uk_attrs->max_hw_wq_frags ||
1427 info->max_rq_frag_cnt > qp->uk_attrs->max_hw_wq_frags)
1430 qp->qp_caps = info->qp_caps;
1431 qp->sq_base = info->sq;
1432 qp->rq_base = info->rq;
1433 qp->qp_type = info->type ? info->type : IRDMA_QP_TYPE_IWARP;
1434 qp->shadow_area = info->shadow_area;
1435 qp->sq_wrtrk_array = info->sq_wrtrk_array;
1437 qp->rq_wrid_array = info->rq_wrid_array;
1438 qp->wqe_alloc_db = info->wqe_alloc_db;
1439 qp->qp_id = info->qp_id;
1440 qp->sq_size = info->sq_size;
1441 qp->max_sq_frag_cnt = info->max_sq_frag_cnt;
1442 sq_ring_size = qp->sq_size << info->sq_shift;
1443 IRDMA_RING_INIT(qp->sq_ring, sq_ring_size);
1444 IRDMA_RING_INIT(qp->initial_ring, sq_ring_size);
1446 irdma_setup_connection_wqes(qp, info);
1447 qp->swqe_polarity = 1;
1448 qp->first_sq_wq = true;
1450 qp->swqe_polarity = 0;
1452 qp->swqe_polarity_deferred = 1;
1453 qp->rwqe_polarity = 0;
1454 qp->rq_size = info->rq_size;
1455 qp->max_rq_frag_cnt = info->max_rq_frag_cnt;
1456 qp->max_inline_data = info->max_inline_data;
1457 qp->rq_wqe_size = info->rq_shift;
1458 IRDMA_RING_INIT(qp->rq_ring, qp->rq_size);
1459 qp->rq_wqe_size_multiplier = 1 << info->rq_shift;
1460 if (qp->uk_attrs->hw_rev == IRDMA_GEN_1)
1461 qp->wqe_ops = iw_wqe_uk_ops_gen_1;
1463 qp->wqe_ops = iw_wqe_uk_ops;
1526 * @qp: hw qp ptr
1531 int irdma_nop(struct irdma_qp_uk *qp, u64 wr_id, bool signaled, bool post_sq)
1539 wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, IRDMA_QP_WQE_MIN_QUANTA,
1544 irdma_clr_wqes(qp, wqe_idx);
1552 FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity);
1558 irdma_uk_qp_post_wr(qp);