Lines Matching refs:conn
40 #include "fpga/conn.h"
48 static int mlx5_fpga_conn_map_buf(struct mlx5_fpga_conn *conn,
57 dma_device = mlx5_core_dma_dev(conn->fdev->mdev);
62 mlx5_fpga_warn(conn->fdev, "DMA error on sg 0: %d\n", err);
74 mlx5_fpga_warn(conn->fdev, "DMA error on sg 1: %d\n", err);
84 static void mlx5_fpga_conn_unmap_buf(struct mlx5_fpga_conn *conn,
89 dma_device = mlx5_core_dma_dev(conn->fdev->mdev);
99 static int mlx5_fpga_conn_post_recv(struct mlx5_fpga_conn *conn,
106 err = mlx5_fpga_conn_map_buf(conn, buf);
110 if (unlikely(conn->qp.rq.pc - conn->qp.rq.cc >= conn->qp.rq.size)) {
111 mlx5_fpga_conn_unmap_buf(conn, buf);
115 ix = conn->qp.rq.pc & (conn->qp.rq.size - 1);
116 data = mlx5_wq_cyc_get_wqe(&conn->qp.wq.rq, ix);
118 data->lkey = cpu_to_be32(conn->fdev->conn_res.mkey);
121 conn->qp.rq.pc++;
122 conn->qp.rq.bufs[ix] = buf;
126 *conn->qp.wq.rq.db = cpu_to_be32(conn->qp.rq.pc & 0xffff);
131 static void mlx5_fpga_conn_notify_hw(struct mlx5_fpga_conn *conn, void *wqe)
135 *conn->qp.wq.sq.db = cpu_to_be32(conn->qp.sq.pc);
138 mlx5_write64(wqe, conn->fdev->conn_res.uar->map + MLX5_BF_OFFSET);
141 static void mlx5_fpga_conn_post_send(struct mlx5_fpga_conn *conn,
149 ix = conn->qp.sq.pc & (conn->qp.sq.size - 1);
151 ctrl = mlx5_wq_cyc_get_wqe(&conn->qp.wq.sq, ix);
158 data->lkey = cpu_to_be32(conn->fdev->conn_res.mkey);
166 ctrl->opmod_idx_opcode = cpu_to_be32(((conn->qp.sq.pc & 0xffff) << 8) |
168 ctrl->qpn_ds = cpu_to_be32(size | (conn->qp.qpn << 8));
170 conn->qp.sq.pc++;
171 conn->qp.sq.bufs[ix] = buf;
172 mlx5_fpga_conn_notify_hw(conn, ctrl);
175 int mlx5_fpga_conn_send(struct mlx5_fpga_conn *conn,
181 if (!conn->qp.active)
185 err = mlx5_fpga_conn_map_buf(conn, buf);
189 spin_lock_irqsave(&conn->qp.sq.lock, flags);
191 if (conn->qp.sq.pc - conn->qp.sq.cc >= conn->qp.sq.size) {
192 list_add_tail(&buf->list, &conn->qp.sq.backlog);
196 mlx5_fpga_conn_post_send(conn, buf);
199 spin_unlock_irqrestore(&conn->qp.sq.lock, flags);
203 static int mlx5_fpga_conn_post_recv_buf(struct mlx5_fpga_conn *conn)
216 err = mlx5_fpga_conn_post_recv(conn, buf);
250 static void mlx5_fpga_conn_rq_cqe(struct mlx5_fpga_conn *conn,
256 ix = be16_to_cpu(cqe->wqe_counter) & (conn->qp.rq.size - 1);
257 buf = conn->qp.rq.bufs[ix];
258 conn->qp.rq.bufs[ix] = NULL;
259 conn->qp.rq.cc++;
262 mlx5_fpga_warn(conn->fdev, "RQ buf %p on FPGA QP %u completion status %d\n",
263 buf, conn->fpga_qpn, status);
265 mlx5_fpga_dbg(conn->fdev, "RQ buf %p on FPGA QP %u completion status %d\n",
266 buf, conn->fpga_qpn, status);
268 mlx5_fpga_conn_unmap_buf(conn, buf);
270 if (unlikely(status || !conn->qp.active)) {
271 conn->qp.active = false;
277 mlx5_fpga_dbg(conn->fdev, "Message with %u bytes received successfully\n",
279 conn->recv_cb(conn->cb_arg, buf);
282 err = mlx5_fpga_conn_post_recv(conn, buf);
284 mlx5_fpga_warn(conn->fdev,
290 static void mlx5_fpga_conn_sq_cqe(struct mlx5_fpga_conn *conn,
297 spin_lock_irqsave(&conn->qp.sq.lock, flags);
299 ix = be16_to_cpu(cqe->wqe_counter) & (conn->qp.sq.size - 1);
300 buf = conn->qp.sq.bufs[ix];
301 conn->qp.sq.bufs[ix] = NULL;
302 conn->qp.sq.cc++;
305 if (unlikely(!list_empty(&conn->qp.sq.backlog))) {
306 if (likely(conn->qp.active)) {
307 nextbuf = list_first_entry(&conn->qp.sq.backlog,
310 mlx5_fpga_conn_post_send(conn, nextbuf);
314 spin_unlock_irqrestore(&conn->qp.sq.lock, flags);
317 mlx5_fpga_warn(conn->fdev, "SQ buf %p on FPGA QP %u completion status %d\n",
318 buf, conn->fpga_qpn, status);
320 mlx5_fpga_dbg(conn->fdev, "SQ buf %p on FPGA QP %u completion status %d\n",
321 buf, conn->fpga_qpn, status);
323 mlx5_fpga_conn_unmap_buf(conn, buf);
326 buf->complete(conn, conn->fdev, buf, status);
329 conn->qp.active = false;
332 static void mlx5_fpga_conn_handle_cqe(struct mlx5_fpga_conn *conn,
344 mlx5_fpga_conn_sq_cqe(conn, cqe, status);
351 mlx5_fpga_conn_rq_cqe(conn, cqe, status);
354 mlx5_fpga_warn(conn->fdev, "Unexpected cqe opcode %u\n",
359 static void mlx5_fpga_conn_arm_cq(struct mlx5_fpga_conn *conn)
361 mlx5_cq_arm(&conn->cq.mcq, MLX5_CQ_DB_REQ_NOT,
362 conn->fdev->conn_res.uar->map, conn->cq.wq.cc);
365 static inline void mlx5_fpga_conn_cqes(struct mlx5_fpga_conn *conn,
371 cqe = mlx5_cqwq_get_cqe(&conn->cq.wq);
376 mlx5_cqwq_pop(&conn->cq.wq);
377 mlx5_fpga_conn_handle_cqe(conn, cqe);
378 mlx5_cqwq_update_db_record(&conn->cq.wq);
381 tasklet_schedule(&conn->cq.tasklet);
385 mlx5_fpga_dbg(conn->fdev, "Re-arming CQ with cc# %u\n", conn->cq.wq.cc);
388 mlx5_fpga_conn_arm_cq(conn);
393 struct mlx5_fpga_conn *conn = from_tasklet(conn, t, cq.tasklet);
395 if (unlikely(!conn->qp.active))
397 mlx5_fpga_conn_cqes(conn, MLX5_FPGA_CQ_BUDGET);
403 struct mlx5_fpga_conn *conn;
405 conn = container_of(mcq, struct mlx5_fpga_conn, cq.mcq);
406 if (unlikely(!conn->qp.active))
408 mlx5_fpga_conn_cqes(conn, MLX5_FPGA_CQ_BUDGET);
411 static int mlx5_fpga_conn_create_cq(struct mlx5_fpga_conn *conn, int cq_size)
413 struct mlx5_fpga_device *fdev = conn->fdev;
430 err = mlx5_cqwq_create(mdev, &wqp, temp_cqc, &conn->cq.wq,
431 &conn->cq.wq_ctrl);
435 for (i = 0; i < mlx5_cqwq_get_size(&conn->cq.wq); i++) {
436 cqe = mlx5_cqwq_get_wqe(&conn->cq.wq, i);
441 sizeof(u64) * conn->cq.wq_ctrl.buf.npages;
458 MLX5_SET(cqc, cqc, log_page_size, conn->cq.wq_ctrl.buf.page_shift -
460 MLX5_SET64(cqc, cqc, dbr_addr, conn->cq.wq_ctrl.db.dma);
463 mlx5_fill_page_frag_array(&conn->cq.wq_ctrl.buf, pas);
465 err = mlx5_core_create_cq(mdev, &conn->cq.mcq, in, inlen, out, sizeof(out));
471 conn->cq.mcq.cqe_sz = 64;
472 conn->cq.mcq.set_ci_db = conn->cq.wq_ctrl.db.db;
473 conn->cq.mcq.arm_db = conn->cq.wq_ctrl.db.db + 1;
474 *conn->cq.mcq.set_ci_db = 0;
475 *conn->cq.mcq.arm_db = 0;
476 conn->cq.mcq.vector = 0;
477 conn->cq.mcq.comp = mlx5_fpga_conn_cq_complete;
478 conn->cq.mcq.uar = fdev->conn_res.uar;
479 tasklet_setup(&conn->cq.tasklet, mlx5_fpga_conn_cq_tasklet);
481 mlx5_fpga_dbg(fdev, "Created CQ #0x%x\n", conn->cq.mcq.cqn);
486 mlx5_wq_destroy(&conn->cq.wq_ctrl);
491 static void mlx5_fpga_conn_destroy_cq(struct mlx5_fpga_conn *conn)
493 tasklet_disable(&conn->cq.tasklet);
494 tasklet_kill(&conn->cq.tasklet);
495 mlx5_core_destroy_cq(conn->fdev->mdev, &conn->cq.mcq);
496 mlx5_wq_destroy(&conn->cq.wq_ctrl);
499 static int mlx5_fpga_conn_create_wq(struct mlx5_fpga_conn *conn, void *qpc)
501 struct mlx5_fpga_device *fdev = conn->fdev;
508 return mlx5_wq_qp_create(mdev, &wqp, qpc, &conn->qp.wq,
509 &conn->qp.wq_ctrl);
512 static int mlx5_fpga_conn_create_qp(struct mlx5_fpga_conn *conn,
515 struct mlx5_fpga_device *fdev = conn->fdev;
522 conn->qp.rq.pc = 0;
523 conn->qp.rq.cc = 0;
524 conn->qp.rq.size = roundup_pow_of_two(rx_size);
525 conn->qp.sq.pc = 0;
526 conn->qp.sq.cc = 0;
527 conn->qp.sq.size = roundup_pow_of_two(tx_size);
530 MLX5_SET(qpc, temp_qpc, log_rq_size, ilog2(conn->qp.rq.size));
531 MLX5_SET(qpc, temp_qpc, log_sq_size, ilog2(conn->qp.sq.size));
532 err = mlx5_fpga_conn_create_wq(conn, temp_qpc);
536 conn->qp.rq.bufs = kvcalloc(conn->qp.rq.size,
537 sizeof(conn->qp.rq.bufs[0]),
539 if (!conn->qp.rq.bufs) {
544 conn->qp.sq.bufs = kvcalloc(conn->qp.sq.size,
545 sizeof(conn->qp.sq.bufs[0]),
547 if (!conn->qp.sq.bufs) {
554 conn->qp.wq_ctrl.buf.npages;
564 conn->qp.wq_ctrl.buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT);
571 MLX5_SET(qpc, qpc, log_rq_size, ilog2(conn->qp.rq.size));
573 MLX5_SET(qpc, qpc, log_sq_size, ilog2(conn->qp.sq.size));
574 MLX5_SET(qpc, qpc, cqn_snd, conn->cq.mcq.cqn);
575 MLX5_SET(qpc, qpc, cqn_rcv, conn->cq.mcq.cqn);
577 MLX5_SET64(qpc, qpc, dbr_addr, conn->qp.wq_ctrl.db.dma);
581 mlx5_fill_page_frag_array(&conn->qp.wq_ctrl.buf,
589 conn->qp.qpn = MLX5_GET(create_qp_out, out, qpn);
590 mlx5_fpga_dbg(fdev, "Created QP #0x%x\n", conn->qp.qpn);
595 kvfree(conn->qp.sq.bufs);
597 kvfree(conn->qp.rq.bufs);
599 mlx5_wq_destroy(&conn->qp.wq_ctrl);
605 static void mlx5_fpga_conn_free_recv_bufs(struct mlx5_fpga_conn *conn)
609 for (ix = 0; ix < conn->qp.rq.size; ix++) {
610 if (!conn->qp.rq.bufs[ix])
612 mlx5_fpga_conn_unmap_buf(conn, conn->qp.rq.bufs[ix]);
613 kfree(conn->qp.rq.bufs[ix]);
614 conn->qp.rq.bufs[ix] = NULL;
618 static void mlx5_fpga_conn_flush_send_bufs(struct mlx5_fpga_conn *conn)
623 for (ix = 0; ix < conn->qp.sq.size; ix++) {
624 buf = conn->qp.sq.bufs[ix];
627 conn->qp.sq.bufs[ix] = NULL;
628 mlx5_fpga_conn_unmap_buf(conn, buf);
631 buf->complete(conn, conn->fdev, buf, MLX5_CQE_SYNDROME_WR_FLUSH_ERR);
633 list_for_each_entry_safe(buf, temp, &conn->qp.sq.backlog, list) {
634 mlx5_fpga_conn_unmap_buf(conn, buf);
637 buf->complete(conn, conn->fdev, buf, MLX5_CQE_SYNDROME_WR_FLUSH_ERR);
641 static void mlx5_fpga_conn_destroy_qp(struct mlx5_fpga_conn *conn)
643 struct mlx5_core_dev *dev = conn->fdev->mdev;
647 MLX5_SET(destroy_qp_in, in, qpn, conn->qp.qpn);
650 mlx5_fpga_conn_free_recv_bufs(conn);
651 mlx5_fpga_conn_flush_send_bufs(conn);
652 kvfree(conn->qp.sq.bufs);
653 kvfree(conn->qp.rq.bufs);
654 mlx5_wq_destroy(&conn->qp.wq_ctrl);
657 static int mlx5_fpga_conn_reset_qp(struct mlx5_fpga_conn *conn)
659 struct mlx5_core_dev *mdev = conn->fdev->mdev;
662 mlx5_fpga_dbg(conn->fdev, "Modifying QP %u to RST\n", conn->qp.qpn);
665 MLX5_SET(qp_2rst_in, in, qpn, conn->qp.qpn);
670 static int mlx5_fpga_conn_init_qp(struct mlx5_fpga_conn *conn)
673 struct mlx5_fpga_device *fdev = conn->fdev;
677 mlx5_fpga_dbg(conn->fdev, "Modifying QP %u to INIT\n", conn->qp.qpn);
685 MLX5_SET(qpc, qpc, pd, conn->fdev->conn_res.pdn);
686 MLX5_SET(qpc, qpc, cqn_snd, conn->cq.mcq.cqn);
687 MLX5_SET(qpc, qpc, cqn_rcv, conn->cq.mcq.cqn);
688 MLX5_SET64(qpc, qpc, dbr_addr, conn->qp.wq_ctrl.db.dma);
691 MLX5_SET(rst2init_qp_in, in, qpn, conn->qp.qpn);
696 static int mlx5_fpga_conn_rtr_qp(struct mlx5_fpga_conn *conn)
699 struct mlx5_fpga_device *fdev = conn->fdev;
703 mlx5_fpga_dbg(conn->fdev, "QP RTR\n");
709 MLX5_SET(qpc, qpc, remote_qpn, conn->fpga_qpn);
711 MLX5_GET(fpga_qpc, conn->fpga_qpc, next_send_psn));
715 MLX5_ADDR_OF(fpga_qpc, conn->fpga_qpc, fpga_mac_47_32));
719 conn->qp.sgid_index);
722 MLX5_ADDR_OF(fpga_qpc, conn->fpga_qpc, fpga_ip),
726 MLX5_SET(init2rtr_qp_in, in, qpn, conn->qp.qpn);
731 static int mlx5_fpga_conn_rts_qp(struct mlx5_fpga_conn *conn)
733 struct mlx5_fpga_device *fdev = conn->fdev;
738 mlx5_fpga_dbg(conn->fdev, "QP RTS\n");
746 MLX5_GET(fpga_qpc, conn->fpga_qpc, next_rcv_psn));
751 MLX5_SET(rtr2rts_qp_in, in, qpn, conn->qp.qpn);
757 static int mlx5_fpga_conn_connect(struct mlx5_fpga_conn *conn)
759 struct mlx5_fpga_device *fdev = conn->fdev;
762 MLX5_SET(fpga_qpc, conn->fpga_qpc, state, MLX5_FPGA_QPC_STATE_ACTIVE);
763 err = mlx5_fpga_modify_qp(conn->fdev->mdev, conn->fpga_qpn,
764 MLX5_FPGA_QPC_STATE, &conn->fpga_qpc);
770 err = mlx5_fpga_conn_reset_qp(conn);
776 err = mlx5_fpga_conn_init_qp(conn);
781 conn->qp.active = true;
783 while (!mlx5_fpga_conn_post_recv_buf(conn))
786 err = mlx5_fpga_conn_rtr_qp(conn);
792 err = mlx5_fpga_conn_rts_qp(conn);
800 mlx5_fpga_conn_free_recv_bufs(conn);
802 MLX5_SET(fpga_qpc, conn->fpga_qpc, state, MLX5_FPGA_QPC_STATE_INIT);
803 if (mlx5_fpga_modify_qp(conn->fdev->mdev, conn->fpga_qpn,
804 MLX5_FPGA_QPC_STATE, &conn->fpga_qpc))
814 struct mlx5_fpga_conn *ret, *conn;
821 conn = kzalloc(sizeof(*conn), GFP_KERNEL);
822 if (!conn)
825 conn->fdev = fdev;
826 INIT_LIST_HEAD(&conn->qp.sq.backlog);
828 spin_lock_init(&conn->qp.sq.lock);
830 conn->recv_cb = attr->recv_cb;
831 conn->cb_arg = attr->cb_arg;
833 remote_mac = MLX5_ADDR_OF(fpga_qpc, conn->fpga_qpc, remote_mac_47_32);
842 remote_ip = MLX5_ADDR_OF(fpga_qpc, conn->fpga_qpc, remote_ip);
847 err = mlx5_core_reserved_gid_alloc(fdev->mdev, &conn->qp.sgid_index);
854 err = mlx5_core_roce_gid_set(fdev->mdev, conn->qp.sgid_index,
864 mlx5_fpga_dbg(fdev, "Reserved SGID index %u\n", conn->qp.sgid_index);
869 err = mlx5_fpga_conn_create_cq(conn,
877 mlx5_fpga_conn_arm_cq(conn);
879 err = mlx5_fpga_conn_create_qp(conn, attr->tx_size, attr->rx_size);
886 MLX5_SET(fpga_qpc, conn->fpga_qpc, state, MLX5_FPGA_QPC_STATE_INIT);
887 MLX5_SET(fpga_qpc, conn->fpga_qpc, qp_type, qp_type);
888 MLX5_SET(fpga_qpc, conn->fpga_qpc, st, MLX5_FPGA_QPC_ST_RC);
889 MLX5_SET(fpga_qpc, conn->fpga_qpc, ether_type, ETH_P_8021Q);
890 MLX5_SET(fpga_qpc, conn->fpga_qpc, vid, 0);
891 MLX5_SET(fpga_qpc, conn->fpga_qpc, next_rcv_psn, 1);
892 MLX5_SET(fpga_qpc, conn->fpga_qpc, next_send_psn, 0);
893 MLX5_SET(fpga_qpc, conn->fpga_qpc, pkey, MLX5_FPGA_PKEY);
894 MLX5_SET(fpga_qpc, conn->fpga_qpc, remote_qpn, conn->qp.qpn);
895 MLX5_SET(fpga_qpc, conn->fpga_qpc, rnr_retry, 7);
896 MLX5_SET(fpga_qpc, conn->fpga_qpc, retry_count, 7);
898 err = mlx5_fpga_create_qp(fdev->mdev, &conn->fpga_qpc,
899 &conn->fpga_qpn);
906 err = mlx5_fpga_conn_connect(conn);
912 mlx5_fpga_dbg(fdev, "FPGA QPN is %u\n", conn->fpga_qpn);
913 ret = conn;
917 mlx5_fpga_destroy_qp(conn->fdev->mdev, conn->fpga_qpn);
919 mlx5_fpga_conn_destroy_qp(conn);
921 mlx5_fpga_conn_destroy_cq(conn);
923 mlx5_core_roce_gid_set(fdev->mdev, conn->qp.sgid_index, 0, 0, NULL,
926 mlx5_core_reserved_gid_free(fdev->mdev, conn->qp.sgid_index);
928 kfree(conn);
933 void mlx5_fpga_conn_destroy(struct mlx5_fpga_conn *conn)
935 conn->qp.active = false;
936 tasklet_disable(&conn->cq.tasklet);
937 synchronize_irq(conn->cq.mcq.irqn);
939 mlx5_fpga_destroy_qp(conn->fdev->mdev, conn->fpga_qpn);
940 mlx5_fpga_conn_destroy_qp(conn);
941 mlx5_fpga_conn_destroy_cq(conn);
943 mlx5_core_roce_gid_set(conn->fdev->mdev, conn->qp.sgid_index, 0, 0,
945 mlx5_core_reserved_gid_free(conn->fdev->mdev, conn->qp.sgid_index);
946 kfree(conn);