1/*- 2 * Copyright (c) 2017 Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33#include <linux/etherdevice.h> 34#include <dev/mlx5/vport.h> 35#include <dev/mlx5/mlx5_core/mlx5_core.h> 36#include <dev/mlx5/mlx5_lib/mlx5.h> 37#include <dev/mlx5/mlx5_fpga/core.h> 38#include <dev/mlx5/mlx5_fpga/conn.h> 39 40#define MLX5_FPGA_PKEY 0xFFFF 41#define MLX5_FPGA_PKEY_INDEX 0 /* RoCE PKEY 0xFFFF is always at index 0 */ 42#define MLX5_FPGA_RECV_SIZE 2048 43#define MLX5_FPGA_PORT_NUM 1 44#define MLX5_FPGA_CQ_BUDGET 64 45 46static int mlx5_fpga_conn_map_buf(struct mlx5_fpga_conn *conn, 47 struct mlx5_fpga_dma_buf *buf) 48{ 49 struct device *dma_device; 50 int err = 0; 51 52 if (unlikely(!buf->sg[0].data)) 53 goto out; 54 55 dma_device = &conn->fdev->mdev->pdev->dev; 56 buf->sg[0].dma_addr = dma_map_single(dma_device, buf->sg[0].data, 57 buf->sg[0].size, buf->dma_dir); 58 err = dma_mapping_error(dma_device, buf->sg[0].dma_addr); 59 if (unlikely(err)) { 60 mlx5_fpga_warn(conn->fdev, "DMA error on sg 0: %d\n", err); 61 err = -ENOMEM; 62 goto out; 63 } 64 65 if (!buf->sg[1].data) 66 goto out; 67 68 buf->sg[1].dma_addr = dma_map_single(dma_device, buf->sg[1].data, 69 buf->sg[1].size, buf->dma_dir); 70 err = dma_mapping_error(dma_device, buf->sg[1].dma_addr); 71 if (unlikely(err)) { 72 mlx5_fpga_warn(conn->fdev, "DMA error on sg 1: %d\n", err); 73 dma_unmap_single(dma_device, buf->sg[0].dma_addr, 74 buf->sg[0].size, buf->dma_dir); 75 err = -ENOMEM; 76 } 77 78out: 79 return err; 80} 81 82static void mlx5_fpga_conn_unmap_buf(struct mlx5_fpga_conn *conn, 83 struct mlx5_fpga_dma_buf *buf) 84{ 85 struct device *dma_device; 86 87 dma_device = &conn->fdev->mdev->pdev->dev; 88 if (buf->sg[1].data) 89 dma_unmap_single(dma_device, buf->sg[1].dma_addr, 90 buf->sg[1].size, buf->dma_dir); 91 92 if (likely(buf->sg[0].data)) 93 dma_unmap_single(dma_device, buf->sg[0].dma_addr, 94 buf->sg[0].size, buf->dma_dir); 95} 96 97static int mlx5_fpga_conn_post_recv(struct mlx5_fpga_conn *conn, 98 struct mlx5_fpga_dma_buf *buf) 99{ 100 struct mlx5_wqe_data_seg *data; 101 unsigned int ix; 102 int err = 0; 103 104 err = mlx5_fpga_conn_map_buf(conn, buf); 105 if (unlikely(err)) 106 goto out; 107 108 if (unlikely(conn->qp.rq.pc - conn->qp.rq.cc >= conn->qp.rq.size)) { 109 mlx5_fpga_conn_unmap_buf(conn, buf); 110 return -EBUSY; 111 } 112 113 ix = conn->qp.rq.pc & (conn->qp.rq.size - 1); 114 data = mlx5_wq_cyc_get_wqe(&conn->qp.wq.rq, ix); 115 data->byte_count = cpu_to_be32(buf->sg[0].size); 116 data->lkey = cpu_to_be32(conn->fdev->conn_res.mkey.key); 117 data->addr = cpu_to_be64(buf->sg[0].dma_addr); 118 119 conn->qp.rq.pc++; 120 conn->qp.rq.bufs[ix] = buf; 121 122 /* Make sure that descriptors are written before doorbell record. */ 123 dma_wmb(); 124 *conn->qp.wq.rq.db = cpu_to_be32(conn->qp.rq.pc & 0xffff); 125out: 126 return err; 127} 128 129static void mlx5_fpga_conn_notify_hw(struct mlx5_fpga_conn *conn, void *wqe) 130{ 131 /* ensure wqe is visible to device before updating doorbell record */ 132 dma_wmb(); 133 *conn->qp.wq.sq.db = cpu_to_be32(conn->qp.sq.pc); 134 /* Make sure that doorbell record is visible before ringing */ 135 wmb(); 136 mlx5_write64(wqe, conn->fdev->conn_res.uar->map + MLX5_BF_OFFSET, NULL); 137} 138 139static void mlx5_fpga_conn_post_send(struct mlx5_fpga_conn *conn, 140 struct mlx5_fpga_dma_buf *buf) 141{ 142 struct mlx5_wqe_ctrl_seg *ctrl; 143 struct mlx5_wqe_data_seg *data; 144 unsigned int ix, sgi; 145 int size = 1; 146 147 ix = conn->qp.sq.pc & (conn->qp.sq.size - 1); 148 149 ctrl = mlx5_wq_cyc_get_wqe(&conn->qp.wq.sq, ix); 150 data = (void *)(ctrl + 1); 151 152 for (sgi = 0; sgi < ARRAY_SIZE(buf->sg); sgi++) { 153 if (!buf->sg[sgi].data) 154 break; 155 data->byte_count = cpu_to_be32(buf->sg[sgi].size); 156 data->lkey = cpu_to_be32(conn->fdev->conn_res.mkey.key); 157 data->addr = cpu_to_be64(buf->sg[sgi].dma_addr); 158 data++; 159 size++; 160 } 161 162 ctrl->imm = 0; 163 ctrl->fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE; 164 ctrl->opmod_idx_opcode = cpu_to_be32(((conn->qp.sq.pc & 0xffff) << 8) | 165 MLX5_OPCODE_SEND); 166 ctrl->qpn_ds = cpu_to_be32(size | (conn->qp.mqp.qpn << 8)); 167 168 conn->qp.sq.pc++; 169 conn->qp.sq.bufs[ix] = buf; 170 mlx5_fpga_conn_notify_hw(conn, ctrl); 171} 172 173int mlx5_fpga_conn_send(struct mlx5_fpga_conn *conn, 174 struct mlx5_fpga_dma_buf *buf) 175{ 176 unsigned long flags; 177 int err; 178 179 if (!conn->qp.active) 180 return -ENOTCONN; 181 182 err = mlx5_fpga_conn_map_buf(conn, buf); 183 if (err) 184 return err; 185 186 spin_lock_irqsave(&conn->qp.sq.lock, flags); 187 188 if (conn->qp.sq.pc - conn->qp.sq.cc >= conn->qp.sq.size) { 189 list_add_tail(&buf->list, &conn->qp.sq.backlog); 190 goto out_unlock; 191 } 192 193 mlx5_fpga_conn_post_send(conn, buf); 194 195out_unlock: 196 spin_unlock_irqrestore(&conn->qp.sq.lock, flags); 197 return err; 198} 199 200static int mlx5_fpga_conn_post_recv_buf(struct mlx5_fpga_conn *conn) 201{ 202 struct mlx5_fpga_dma_buf *buf; 203 int err; 204 205 buf = kzalloc(sizeof(*buf) + MLX5_FPGA_RECV_SIZE, 0); 206 if (!buf) 207 return -ENOMEM; 208 209 buf->sg[0].data = (void *)(buf + 1); 210 buf->sg[0].size = MLX5_FPGA_RECV_SIZE; 211 buf->dma_dir = DMA_FROM_DEVICE; 212 213 err = mlx5_fpga_conn_post_recv(conn, buf); 214 if (err) 215 kfree(buf); 216 217 return err; 218} 219 220static int mlx5_fpga_conn_create_mkey(struct mlx5_core_dev *mdev, u32 pdn, 221 struct mlx5_core_mkey *mkey) 222{ 223 int inlen = MLX5_ST_SZ_BYTES(create_mkey_in); 224 void *mkc; 225 u32 *in; 226 int err; 227 228 in = kvzalloc(inlen, GFP_KERNEL); 229 if (!in) 230 return -ENOMEM; 231 232 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 233 MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_PA); 234 MLX5_SET(mkc, mkc, lw, 1); 235 MLX5_SET(mkc, mkc, lr, 1); 236 237 MLX5_SET(mkc, mkc, pd, pdn); 238 MLX5_SET(mkc, mkc, length64, 1); 239 MLX5_SET(mkc, mkc, qpn, 0xffffff); 240 241 err = mlx5_core_create_mkey(mdev, mkey, in, inlen); 242 243 kvfree(in); 244 return err; 245} 246 247static void mlx5_fpga_conn_rq_cqe(struct mlx5_fpga_conn *conn, 248 struct mlx5_cqe64 *cqe, u8 status) 249{ 250 struct mlx5_fpga_dma_buf *buf; 251 int ix, err; 252 253 ix = be16_to_cpu(cqe->wqe_counter) & (conn->qp.rq.size - 1); 254 buf = conn->qp.rq.bufs[ix]; 255 conn->qp.rq.bufs[ix] = NULL; 256 if (!status) 257 buf->sg[0].size = be32_to_cpu(cqe->byte_cnt); 258 conn->qp.rq.cc++; 259 260 if (unlikely(status && (status != MLX5_CQE_SYNDROME_WR_FLUSH_ERR))) 261 mlx5_fpga_warn(conn->fdev, "RQ buf %p on FPGA QP %u completion status %d\n", 262 buf, conn->fpga_qpn, status); 263 else 264 mlx5_fpga_dbg(conn->fdev, "RQ buf %p on FPGA QP %u completion status %d\n", 265 buf, conn->fpga_qpn, status); 266 267 mlx5_fpga_conn_unmap_buf(conn, buf); 268 269 if (unlikely(status || !conn->qp.active)) { 270 conn->qp.active = false; 271 kfree(buf); 272 return; 273 } 274 275 mlx5_fpga_dbg(conn->fdev, "Message with %u bytes received successfully\n", 276 buf->sg[0].size); 277 conn->recv_cb(conn->cb_arg, buf); 278 279 buf->sg[0].size = MLX5_FPGA_RECV_SIZE; 280 err = mlx5_fpga_conn_post_recv(conn, buf); 281 if (unlikely(err)) { 282 mlx5_fpga_warn(conn->fdev, 283 "Failed to re-post recv buf: %d\n", err); 284 kfree(buf); 285 } 286} 287 288static void mlx5_fpga_conn_sq_cqe(struct mlx5_fpga_conn *conn, 289 struct mlx5_cqe64 *cqe, u8 status) 290{ 291 struct mlx5_fpga_dma_buf *buf, *nextbuf; 292 unsigned long flags; 293 int ix; 294 295 spin_lock_irqsave(&conn->qp.sq.lock, flags); 296 297 ix = be16_to_cpu(cqe->wqe_counter) & (conn->qp.sq.size - 1); 298 buf = conn->qp.sq.bufs[ix]; 299 conn->qp.sq.bufs[ix] = NULL; 300 conn->qp.sq.cc++; 301 302 /* Handle backlog still under the spinlock to ensure message post order */ 303 if (unlikely(!list_empty(&conn->qp.sq.backlog))) { 304 if (likely(conn->qp.active)) { 305 nextbuf = list_first_entry(&conn->qp.sq.backlog, 306 struct mlx5_fpga_dma_buf, list); 307 list_del(&nextbuf->list); 308 mlx5_fpga_conn_post_send(conn, nextbuf); 309 } 310 } 311 312 spin_unlock_irqrestore(&conn->qp.sq.lock, flags); 313 314 if (unlikely(status && (status != MLX5_CQE_SYNDROME_WR_FLUSH_ERR))) 315 mlx5_fpga_warn(conn->fdev, "SQ buf %p on FPGA QP %u completion status %d\n", 316 buf, conn->fpga_qpn, status); 317 else 318 mlx5_fpga_dbg(conn->fdev, "SQ buf %p on FPGA QP %u completion status %d\n", 319 buf, conn->fpga_qpn, status); 320 321 mlx5_fpga_conn_unmap_buf(conn, buf); 322 323 if (likely(buf->complete)) 324 buf->complete(conn, conn->fdev, buf, status); 325 326 if (unlikely(status)) 327 conn->qp.active = false; 328} 329 330static void mlx5_fpga_conn_handle_cqe(struct mlx5_fpga_conn *conn, 331 struct mlx5_cqe64 *cqe) 332{ 333 u8 opcode, status = 0; 334 335 opcode = cqe->op_own >> 4; 336 337 switch (opcode) { 338 case MLX5_CQE_REQ_ERR: 339 status = ((struct mlx5_err_cqe *)cqe)->syndrome; 340 /* Fall through */ 341 case MLX5_CQE_REQ: 342 mlx5_fpga_conn_sq_cqe(conn, cqe, status); 343 break; 344 345 case MLX5_CQE_RESP_ERR: 346 status = ((struct mlx5_err_cqe *)cqe)->syndrome; 347 /* Fall through */ 348 case MLX5_CQE_RESP_SEND: 349 mlx5_fpga_conn_rq_cqe(conn, cqe, status); 350 break; 351 default: 352 mlx5_fpga_warn(conn->fdev, "Unexpected cqe opcode %u\n", 353 opcode); 354 } 355} 356 357static void mlx5_fpga_conn_arm_cq(struct mlx5_fpga_conn *conn) 358{ 359 mlx5_cq_arm(&conn->cq.mcq, MLX5_CQ_DB_REQ_NOT, 360 conn->fdev->conn_res.uar->map, conn->cq.wq.cc); 361} 362 363static void mlx5_fpga_conn_cq_event(struct mlx5_core_cq *mcq, 364 enum mlx5_event event) 365{ 366 struct mlx5_fpga_conn *conn; 367 368 conn = container_of(mcq, struct mlx5_fpga_conn, cq.mcq); 369 mlx5_fpga_warn(conn->fdev, "CQ event %u on CQ #%u\n", event, mcq->cqn); 370} 371 372static void mlx5_fpga_conn_event(struct mlx5_core_qp *mqp, int event) 373{ 374 struct mlx5_fpga_conn *conn; 375 376 conn = container_of(mqp, struct mlx5_fpga_conn, qp.mqp); 377 mlx5_fpga_warn(conn->fdev, "QP event %u on QP #%u\n", event, mqp->qpn); 378} 379 380static inline void mlx5_fpga_conn_cqes(struct mlx5_fpga_conn *conn, 381 unsigned int budget) 382{ 383 struct mlx5_cqe64 *cqe; 384 385 while (budget) { 386 cqe = mlx5_cqwq_get_cqe(&conn->cq.wq); 387 if (!cqe) 388 break; 389 390 budget--; 391 mlx5_cqwq_pop(&conn->cq.wq); 392 mlx5_fpga_conn_handle_cqe(conn, cqe); 393 mlx5_cqwq_update_db_record(&conn->cq.wq); 394 } 395 if (!budget) { 396 tasklet_schedule(&conn->cq.tasklet); 397 return; 398 } 399 400 mlx5_fpga_dbg(conn->fdev, "Re-arming CQ with cc# %u\n", conn->cq.wq.cc); 401 /* ensure cq space is freed before enabling more cqes */ 402 wmb(); 403 mlx5_fpga_conn_arm_cq(conn); 404} 405 406static void mlx5_fpga_conn_cq_tasklet(unsigned long data) 407{ 408 struct mlx5_fpga_conn *conn = (void *)data; 409 410 if (unlikely(!conn->qp.active)) 411 return; 412 mlx5_fpga_conn_cqes(conn, MLX5_FPGA_CQ_BUDGET); 413} 414 415static void mlx5_fpga_conn_cq_complete(struct mlx5_core_cq *mcq) 416{ 417 struct mlx5_fpga_conn *conn; 418 419 conn = container_of(mcq, struct mlx5_fpga_conn, cq.mcq); 420 if (unlikely(!conn->qp.active)) 421 return; 422 mlx5_fpga_conn_cqes(conn, MLX5_FPGA_CQ_BUDGET); 423} 424 425static int mlx5_fpga_conn_create_cq(struct mlx5_fpga_conn *conn, int cq_size) 426{ 427 struct mlx5_fpga_device *fdev = conn->fdev; 428 struct mlx5_core_dev *mdev = fdev->mdev; 429 u32 temp_cqc[MLX5_ST_SZ_DW(cqc)] = {0}; 430 struct mlx5_wq_param wqp; 431 struct mlx5_cqe64 *cqe; 432 int inlen, err, eqn; 433 unsigned int irqn; 434 void *cqc, *in; 435 __be64 *pas; 436 u32 i; 437 438 cq_size = roundup_pow_of_two(cq_size); 439 MLX5_SET(cqc, temp_cqc, log_cq_size, ilog2(cq_size)); 440 441 wqp.buf_numa_node = mdev->priv.numa_node; 442 wqp.db_numa_node = mdev->priv.numa_node; 443 444 err = mlx5_cqwq_create(mdev, &wqp, temp_cqc, &conn->cq.wq, 445 &conn->cq.wq_ctrl); 446 if (err) 447 return err; 448 449 for (i = 0; i < mlx5_cqwq_get_size(&conn->cq.wq); i++) { 450 cqe = mlx5_cqwq_get_wqe(&conn->cq.wq, i); 451 cqe->op_own = MLX5_CQE_INVALID << 4 | MLX5_CQE_OWNER_MASK; 452 } 453 454 inlen = MLX5_ST_SZ_BYTES(create_cq_in) + 455 sizeof(u64) * conn->cq.wq_ctrl.frag_buf.npages; 456 in = kvzalloc(inlen, GFP_KERNEL); 457 if (!in) { 458 err = -ENOMEM; 459 goto err_cqwq; 460 } 461 462 err = mlx5_vector2eqn(mdev, smp_processor_id(), &eqn, &irqn); 463 if (err) 464 goto err_cqwq; 465 466 cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context); 467 MLX5_SET(cqc, cqc, log_cq_size, ilog2(cq_size)); 468 MLX5_SET(cqc, cqc, c_eqn, eqn); 469 MLX5_SET(cqc, cqc, uar_page, fdev->conn_res.uar->index); 470 MLX5_SET(cqc, cqc, log_page_size, conn->cq.wq_ctrl.frag_buf.page_shift - 471 MLX5_ADAPTER_PAGE_SHIFT); 472 MLX5_SET64(cqc, cqc, dbr_addr, conn->cq.wq_ctrl.db.dma); 473 474 pas = (__be64 *)MLX5_ADDR_OF(create_cq_in, in, pas); 475 mlx5_fill_page_frag_array(&conn->cq.wq_ctrl.frag_buf, pas); 476 477 err = mlx5_core_create_cq(mdev, &conn->cq.mcq, in, inlen); 478 kvfree(in); 479 480 if (err) 481 goto err_cqwq; 482 483 conn->cq.mcq.cqe_sz = 64; 484 conn->cq.mcq.set_ci_db = conn->cq.wq_ctrl.db.db; 485 conn->cq.mcq.arm_db = conn->cq.wq_ctrl.db.db + 1; 486 *conn->cq.mcq.set_ci_db = 0; 487 *conn->cq.mcq.arm_db = 0; 488 conn->cq.mcq.vector = 0; 489 conn->cq.mcq.comp = mlx5_fpga_conn_cq_complete; 490 conn->cq.mcq.event = mlx5_fpga_conn_cq_event; 491 conn->cq.mcq.irqn = irqn; 492 conn->cq.mcq.uar = fdev->conn_res.uar; 493 tasklet_init(&conn->cq.tasklet, mlx5_fpga_conn_cq_tasklet, 494 (unsigned long)conn); 495 496 mlx5_fpga_dbg(fdev, "Created CQ #0x%x\n", conn->cq.mcq.cqn); 497 498 goto out; 499 500err_cqwq: 501 mlx5_cqwq_destroy(&conn->cq.wq_ctrl); 502out: 503 return err; 504} 505 506static void mlx5_fpga_conn_destroy_cq(struct mlx5_fpga_conn *conn) 507{ 508 tasklet_disable(&conn->cq.tasklet); 509 tasklet_kill(&conn->cq.tasklet); 510 mlx5_core_destroy_cq(conn->fdev->mdev, &conn->cq.mcq); 511 mlx5_cqwq_destroy(&conn->cq.wq_ctrl); 512} 513 514static int mlx5_fpga_conn_create_wq(struct mlx5_fpga_conn *conn, void *qpc) 515{ 516 struct mlx5_fpga_device *fdev = conn->fdev; 517 struct mlx5_core_dev *mdev = fdev->mdev; 518 struct mlx5_wq_param wqp; 519 520 wqp.buf_numa_node = mdev->priv.numa_node; 521 wqp.db_numa_node = mdev->priv.numa_node; 522 523 return mlx5_wq_qp_create(mdev, &wqp, qpc, &conn->qp.wq, 524 &conn->qp.wq_ctrl); 525} 526 527static int mlx5_fpga_conn_create_qp(struct mlx5_fpga_conn *conn, 528 unsigned int tx_size, unsigned int rx_size) 529{ 530 struct mlx5_fpga_device *fdev = conn->fdev; 531 struct mlx5_core_dev *mdev = fdev->mdev; 532 u32 temp_qpc[MLX5_ST_SZ_DW(qpc)] = {0}; 533 void *in = NULL, *qpc; 534 int err, inlen; 535 536 conn->qp.rq.pc = 0; 537 conn->qp.rq.cc = 0; 538 conn->qp.rq.size = roundup_pow_of_two(rx_size); 539 conn->qp.sq.pc = 0; 540 conn->qp.sq.cc = 0; 541 conn->qp.sq.size = roundup_pow_of_two(tx_size); 542 543 MLX5_SET(qpc, temp_qpc, log_rq_stride, ilog2(MLX5_SEND_WQE_DS) - 4); 544 MLX5_SET(qpc, temp_qpc, log_rq_size, ilog2(conn->qp.rq.size)); 545 MLX5_SET(qpc, temp_qpc, log_sq_size, ilog2(conn->qp.sq.size)); 546 err = mlx5_fpga_conn_create_wq(conn, temp_qpc); 547 if (err) 548 goto out; 549 550 conn->qp.rq.bufs = kvzalloc(sizeof(conn->qp.rq.bufs[0]) * 551 conn->qp.rq.size, GFP_KERNEL); 552 if (!conn->qp.rq.bufs) { 553 err = -ENOMEM; 554 goto err_wq; 555 } 556 557 conn->qp.sq.bufs = kvzalloc(sizeof(conn->qp.sq.bufs[0]) * 558 conn->qp.sq.size, GFP_KERNEL); 559 if (!conn->qp.sq.bufs) { 560 err = -ENOMEM; 561 goto err_rq_bufs; 562 } 563 564 inlen = MLX5_ST_SZ_BYTES(create_qp_in) + 565 MLX5_FLD_SZ_BYTES(create_qp_in, pas[0]) * 566 conn->qp.wq_ctrl.buf.npages; 567 in = kvzalloc(inlen, GFP_KERNEL); 568 if (!in) { 569 err = -ENOMEM; 570 goto err_sq_bufs; 571 } 572 573 qpc = MLX5_ADDR_OF(create_qp_in, in, qpc); 574 MLX5_SET(qpc, qpc, uar_page, fdev->conn_res.uar->index); 575 MLX5_SET(qpc, qpc, log_page_size, 576 conn->qp.wq_ctrl.buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT); 577 MLX5_SET(qpc, qpc, fre, 1); 578 MLX5_SET(qpc, qpc, rlky, 1); 579 MLX5_SET(qpc, qpc, st, MLX5_QP_ST_RC); 580 MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED); 581 MLX5_SET(qpc, qpc, pd, fdev->conn_res.pdn); 582 MLX5_SET(qpc, qpc, log_rq_stride, ilog2(MLX5_SEND_WQE_DS) - 4); 583 MLX5_SET(qpc, qpc, log_rq_size, ilog2(conn->qp.rq.size)); 584 MLX5_SET(qpc, qpc, rq_type, MLX5_NON_ZERO_RQ); 585 MLX5_SET(qpc, qpc, log_sq_size, ilog2(conn->qp.sq.size)); 586 MLX5_SET(qpc, qpc, cqn_snd, conn->cq.mcq.cqn); 587 MLX5_SET(qpc, qpc, cqn_rcv, conn->cq.mcq.cqn); 588 MLX5_SET64(qpc, qpc, dbr_addr, conn->qp.wq_ctrl.db.dma); 589 if (MLX5_CAP_GEN(mdev, cqe_version) == 1) 590 MLX5_SET(qpc, qpc, user_index, 0xFFFFFF); 591 592 mlx5_fill_page_array(&conn->qp.wq_ctrl.buf, 593 (__be64 *)MLX5_ADDR_OF(create_qp_in, in, pas)); 594 595 err = mlx5_core_create_qp(mdev, &conn->qp.mqp, in, inlen); 596 if (err) 597 goto err_sq_bufs; 598 599 conn->qp.mqp.event = mlx5_fpga_conn_event; 600 mlx5_fpga_dbg(fdev, "Created QP #0x%x\n", conn->qp.mqp.qpn); 601 602 goto out; 603 604err_sq_bufs: 605 kvfree(conn->qp.sq.bufs); 606err_rq_bufs: 607 kvfree(conn->qp.rq.bufs); 608err_wq: 609 mlx5_wq_destroy(&conn->qp.wq_ctrl); 610out: 611 kvfree(in); 612 return err; 613} 614 615static void mlx5_fpga_conn_free_recv_bufs(struct mlx5_fpga_conn *conn) 616{ 617 int ix; 618 619 for (ix = 0; ix < conn->qp.rq.size; ix++) { 620 if (!conn->qp.rq.bufs[ix]) 621 continue; 622 mlx5_fpga_conn_unmap_buf(conn, conn->qp.rq.bufs[ix]); 623 kfree(conn->qp.rq.bufs[ix]); 624 conn->qp.rq.bufs[ix] = NULL; 625 } 626} 627 628static void mlx5_fpga_conn_flush_send_bufs(struct mlx5_fpga_conn *conn) 629{ 630 struct mlx5_fpga_dma_buf *buf, *temp; 631 int ix; 632 633 for (ix = 0; ix < conn->qp.sq.size; ix++) { 634 buf = conn->qp.sq.bufs[ix]; 635 if (!buf) 636 continue; 637 conn->qp.sq.bufs[ix] = NULL; 638 mlx5_fpga_conn_unmap_buf(conn, buf); 639 if (!buf->complete) 640 continue; 641 buf->complete(conn, conn->fdev, buf, MLX5_CQE_SYNDROME_WR_FLUSH_ERR); 642 } 643 list_for_each_entry_safe(buf, temp, &conn->qp.sq.backlog, list) { 644 mlx5_fpga_conn_unmap_buf(conn, buf); 645 if (!buf->complete) 646 continue; 647 buf->complete(conn, conn->fdev, buf, MLX5_CQE_SYNDROME_WR_FLUSH_ERR); 648 } 649} 650 651static void mlx5_fpga_conn_destroy_qp(struct mlx5_fpga_conn *conn) 652{ 653 mlx5_core_destroy_qp(conn->fdev->mdev, &conn->qp.mqp); 654 mlx5_fpga_conn_free_recv_bufs(conn); 655 mlx5_fpga_conn_flush_send_bufs(conn); 656 kvfree(conn->qp.sq.bufs); 657 kvfree(conn->qp.rq.bufs); 658 mlx5_wq_destroy(&conn->qp.wq_ctrl); 659} 660 661static inline int mlx5_fpga_conn_reset_qp(struct mlx5_fpga_conn *conn) 662{ 663 struct mlx5_core_dev *mdev = conn->fdev->mdev; 664 665 mlx5_fpga_dbg(conn->fdev, "Modifying QP %u to RST\n", conn->qp.mqp.qpn); 666 667 return mlx5_core_qp_modify(mdev, MLX5_CMD_OP_2RST_QP, 0, NULL, 668 &conn->qp.mqp); 669} 670 671static inline int mlx5_fpga_conn_init_qp(struct mlx5_fpga_conn *conn) 672{ 673 struct mlx5_fpga_device *fdev = conn->fdev; 674 struct mlx5_core_dev *mdev = fdev->mdev; 675 u32 *qpc = NULL; 676 int err; 677 678 mlx5_fpga_dbg(conn->fdev, "Modifying QP %u to INIT\n", conn->qp.mqp.qpn); 679 680 qpc = kzalloc(MLX5_ST_SZ_BYTES(qpc), GFP_KERNEL); 681 if (!qpc) { 682 err = -ENOMEM; 683 goto out; 684 } 685 686 MLX5_SET(qpc, qpc, st, MLX5_QP_ST_RC); 687 MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED); 688 MLX5_SET(qpc, qpc, primary_address_path.pkey_index, MLX5_FPGA_PKEY_INDEX); 689 MLX5_SET(qpc, qpc, primary_address_path.port, MLX5_FPGA_PORT_NUM); 690 MLX5_SET(qpc, qpc, pd, conn->fdev->conn_res.pdn); 691 MLX5_SET(qpc, qpc, cqn_snd, conn->cq.mcq.cqn); 692 MLX5_SET(qpc, qpc, cqn_rcv, conn->cq.mcq.cqn); 693 MLX5_SET64(qpc, qpc, dbr_addr, conn->qp.wq_ctrl.db.dma); 694 695 err = mlx5_core_qp_modify(mdev, MLX5_CMD_OP_RST2INIT_QP, 0, qpc, 696 &conn->qp.mqp); 697 if (err) { 698 mlx5_fpga_warn(fdev, "qp_modify RST2INIT failed: %d\n", err); 699 goto out; 700 } 701 702out: 703 kfree(qpc); 704 return err; 705} 706 707static inline int mlx5_fpga_conn_rtr_qp(struct mlx5_fpga_conn *conn) 708{ 709 struct mlx5_fpga_device *fdev = conn->fdev; 710 struct mlx5_core_dev *mdev = fdev->mdev; 711 u32 *qpc = NULL; 712 int err; 713 714 mlx5_fpga_dbg(conn->fdev, "QP RTR\n"); 715 716 qpc = kzalloc(MLX5_ST_SZ_BYTES(qpc), GFP_KERNEL); 717 if (!qpc) { 718 err = -ENOMEM; 719 goto out; 720 } 721 722 MLX5_SET(qpc, qpc, mtu, MLX5_QPC_MTU_1K_BYTES); 723 MLX5_SET(qpc, qpc, log_msg_max, (u8)MLX5_CAP_GEN(mdev, log_max_msg)); 724 MLX5_SET(qpc, qpc, remote_qpn, conn->fpga_qpn); 725 MLX5_SET(qpc, qpc, next_rcv_psn, 726 MLX5_GET(fpga_qpc, conn->fpga_qpc, next_send_psn)); 727 MLX5_SET(qpc, qpc, primary_address_path.pkey_index, MLX5_FPGA_PKEY_INDEX); 728 MLX5_SET(qpc, qpc, primary_address_path.port, MLX5_FPGA_PORT_NUM); 729 ether_addr_copy(MLX5_ADDR_OF(qpc, qpc, primary_address_path.rmac_47_32), 730 MLX5_ADDR_OF(fpga_qpc, conn->fpga_qpc, fpga_mac_47_32)); 731 MLX5_SET(qpc, qpc, primary_address_path.udp_sport, 732 MLX5_CAP_ROCE(mdev, r_roce_min_src_udp_port)); 733 MLX5_SET(qpc, qpc, primary_address_path.src_addr_index, 734 conn->qp.sgid_index); 735 MLX5_SET(qpc, qpc, primary_address_path.hop_limit, 0); 736 memcpy(MLX5_ADDR_OF(qpc, qpc, primary_address_path.rgid_rip), 737 MLX5_ADDR_OF(fpga_qpc, conn->fpga_qpc, fpga_ip), 738 MLX5_FLD_SZ_BYTES(qpc, primary_address_path.rgid_rip)); 739 740 err = mlx5_core_qp_modify(mdev, MLX5_CMD_OP_INIT2RTR_QP, 0, qpc, 741 &conn->qp.mqp); 742 if (err) { 743 mlx5_fpga_warn(fdev, "qp_modify RST2INIT failed: %d\n", err); 744 goto out; 745 } 746 747out: 748 kfree(qpc); 749 return err; 750} 751 752static inline int mlx5_fpga_conn_rts_qp(struct mlx5_fpga_conn *conn) 753{ 754 struct mlx5_fpga_device *fdev = conn->fdev; 755 struct mlx5_core_dev *mdev = fdev->mdev; 756 u32 *qpc = NULL; 757 u32 opt_mask; 758 int err; 759 760 mlx5_fpga_dbg(conn->fdev, "QP RTS\n"); 761 762 qpc = kzalloc(MLX5_ST_SZ_BYTES(qpc), GFP_KERNEL); 763 if (!qpc) { 764 err = -ENOMEM; 765 goto out; 766 } 767 768 MLX5_SET(qpc, qpc, log_ack_req_freq, 8); 769 MLX5_SET(qpc, qpc, min_rnr_nak, 0x12); 770 MLX5_SET(qpc, qpc, primary_address_path.ack_timeout, 0x12); /* ~1.07s */ 771 MLX5_SET(qpc, qpc, next_send_psn, 772 MLX5_GET(fpga_qpc, conn->fpga_qpc, next_rcv_psn)); 773 MLX5_SET(qpc, qpc, retry_count, 7); 774 MLX5_SET(qpc, qpc, rnr_retry, 7); /* Infinite retry if RNR NACK */ 775 776 opt_mask = MLX5_QP_OPTPAR_RNR_TIMEOUT; 777 err = mlx5_core_qp_modify(mdev, MLX5_CMD_OP_RTR2RTS_QP, opt_mask, qpc, 778 &conn->qp.mqp); 779 if (err) { 780 mlx5_fpga_warn(fdev, "qp_modify RST2INIT failed: %d\n", err); 781 goto out; 782 } 783 784out: 785 kfree(qpc); 786 return err; 787} 788 789static int mlx5_fpga_conn_connect(struct mlx5_fpga_conn *conn) 790{ 791 struct mlx5_fpga_device *fdev = conn->fdev; 792 int err; 793 794 MLX5_SET(fpga_qpc, conn->fpga_qpc, state, MLX5_FPGA_QPC_STATE_ACTIVE); 795 err = mlx5_fpga_modify_qp(conn->fdev->mdev, conn->fpga_qpn, 796 MLX5_FPGA_QPC_STATE, &conn->fpga_qpc); 797 if (err) { 798 mlx5_fpga_err(fdev, "Failed to activate FPGA RC QP: %d\n", err); 799 goto out; 800 } 801 802 err = mlx5_fpga_conn_reset_qp(conn); 803 if (err) { 804 mlx5_fpga_err(fdev, "Failed to change QP state to reset\n"); 805 goto err_fpga_qp; 806 } 807 808 err = mlx5_fpga_conn_init_qp(conn); 809 if (err) { 810 mlx5_fpga_err(fdev, "Failed to modify QP from RESET to INIT\n"); 811 goto err_fpga_qp; 812 } 813 conn->qp.active = true; 814 815 while (!mlx5_fpga_conn_post_recv_buf(conn)) 816 ; 817 818 err = mlx5_fpga_conn_rtr_qp(conn); 819 if (err) { 820 mlx5_fpga_err(fdev, "Failed to change QP state from INIT to RTR\n"); 821 goto err_recv_bufs; 822 } 823 824 err = mlx5_fpga_conn_rts_qp(conn); 825 if (err) { 826 mlx5_fpga_err(fdev, "Failed to change QP state from RTR to RTS\n"); 827 goto err_recv_bufs; 828 } 829 goto out; 830 831err_recv_bufs: 832 mlx5_fpga_conn_free_recv_bufs(conn); 833err_fpga_qp: 834 MLX5_SET(fpga_qpc, conn->fpga_qpc, state, MLX5_FPGA_QPC_STATE_INIT); 835 if (mlx5_fpga_modify_qp(conn->fdev->mdev, conn->fpga_qpn, 836 MLX5_FPGA_QPC_STATE, &conn->fpga_qpc)) 837 mlx5_fpga_err(fdev, "Failed to revert FPGA QP to INIT\n"); 838out: 839 return err; 840} 841 842struct mlx5_fpga_conn *mlx5_fpga_conn_create(struct mlx5_fpga_device *fdev, 843 struct mlx5_fpga_conn_attr *attr, 844 enum mlx5_ifc_fpga_qp_type qp_type) 845{ 846 struct mlx5_fpga_conn *ret, *conn; 847 u8 *remote_mac, *remote_ip; 848 int err; 849 850 if (!attr->recv_cb) 851 return ERR_PTR(-EINVAL); 852 853 conn = kzalloc(sizeof(*conn), GFP_KERNEL); 854 if (!conn) 855 return ERR_PTR(-ENOMEM); 856 857 conn->fdev = fdev; 858 INIT_LIST_HEAD(&conn->qp.sq.backlog); 859 860 spin_lock_init(&conn->qp.sq.lock); 861 862 conn->recv_cb = attr->recv_cb; 863 conn->cb_arg = attr->cb_arg; 864 865 remote_mac = MLX5_ADDR_OF(fpga_qpc, conn->fpga_qpc, remote_mac_47_32); 866 err = mlx5_query_nic_vport_mac_address(fdev->mdev, 0, remote_mac); 867 if (err) { 868 mlx5_fpga_err(fdev, "Failed to query local MAC: %d\n", err); 869 ret = ERR_PTR(err); 870 goto err; 871 } 872 873 /* Build Modified EUI-64 IPv6 address from the MAC address */ 874 remote_ip = MLX5_ADDR_OF(fpga_qpc, conn->fpga_qpc, remote_ip); 875 remote_ip[0] = 0xfe; 876 remote_ip[1] = 0x80; 877 addrconf_addr_eui48(&remote_ip[8], remote_mac); 878 879 err = mlx5_core_reserved_gid_alloc(fdev->mdev, &conn->qp.sgid_index); 880 if (err) { 881 mlx5_fpga_err(fdev, "Failed to allocate SGID: %d\n", err); 882 ret = ERR_PTR(err); 883 goto err; 884 } 885 886 err = mlx5_core_roce_gid_set(fdev->mdev, conn->qp.sgid_index, 887 MLX5_ROCE_VERSION_2, 888 MLX5_ROCE_L3_TYPE_IPV6, 889 remote_ip, remote_mac, true, 0); 890 if (err) { 891 mlx5_fpga_err(fdev, "Failed to set SGID: %d\n", err); 892 ret = ERR_PTR(err); 893 goto err_rsvd_gid; 894 } 895 mlx5_fpga_dbg(fdev, "Reserved SGID index %u\n", conn->qp.sgid_index); 896 897 /* Allow for one cqe per rx/tx wqe, plus one cqe for the next wqe, 898 * created during processing of the cqe 899 */ 900 err = mlx5_fpga_conn_create_cq(conn, 901 (attr->tx_size + attr->rx_size) * 2); 902 if (err) { 903 mlx5_fpga_err(fdev, "Failed to create CQ: %d\n", err); 904 ret = ERR_PTR(err); 905 goto err_gid; 906 } 907 908 mlx5_fpga_conn_arm_cq(conn); 909 910 err = mlx5_fpga_conn_create_qp(conn, attr->tx_size, attr->rx_size); 911 if (err) { 912 mlx5_fpga_err(fdev, "Failed to create QP: %d\n", err); 913 ret = ERR_PTR(err); 914 goto err_cq; 915 } 916 917 MLX5_SET(fpga_qpc, conn->fpga_qpc, state, MLX5_FPGA_QPC_STATE_INIT); 918 MLX5_SET(fpga_qpc, conn->fpga_qpc, qp_type, qp_type); 919 MLX5_SET(fpga_qpc, conn->fpga_qpc, st, MLX5_FPGA_QPC_ST_RC); 920 MLX5_SET(fpga_qpc, conn->fpga_qpc, ether_type, ETH_P_8021Q); 921 MLX5_SET(fpga_qpc, conn->fpga_qpc, vid, 0); 922 MLX5_SET(fpga_qpc, conn->fpga_qpc, next_rcv_psn, 1); 923 MLX5_SET(fpga_qpc, conn->fpga_qpc, next_send_psn, 0); 924 MLX5_SET(fpga_qpc, conn->fpga_qpc, pkey, MLX5_FPGA_PKEY); 925 MLX5_SET(fpga_qpc, conn->fpga_qpc, remote_qpn, conn->qp.mqp.qpn); 926 MLX5_SET(fpga_qpc, conn->fpga_qpc, rnr_retry, 7); 927 MLX5_SET(fpga_qpc, conn->fpga_qpc, retry_count, 7); 928 929 err = mlx5_fpga_create_qp(fdev->mdev, &conn->fpga_qpc, 930 &conn->fpga_qpn); 931 if (err) { 932 mlx5_fpga_err(fdev, "Failed to create FPGA RC QP: %d\n", err); 933 ret = ERR_PTR(err); 934 goto err_qp; 935 } 936 937 err = mlx5_fpga_conn_connect(conn); 938 if (err) { 939 ret = ERR_PTR(err); 940 goto err_conn; 941 } 942 943 mlx5_fpga_dbg(fdev, "FPGA QPN is %u\n", conn->fpga_qpn); 944 ret = conn; 945 goto out; 946 947err_conn: 948 mlx5_fpga_destroy_qp(conn->fdev->mdev, conn->fpga_qpn); 949err_qp: 950 mlx5_fpga_conn_destroy_qp(conn); 951err_cq: 952 mlx5_fpga_conn_destroy_cq(conn); 953err_gid: 954 mlx5_core_roce_gid_set(fdev->mdev, conn->qp.sgid_index, 0, 0, NULL, 955 NULL, false, 0); 956err_rsvd_gid: 957 mlx5_core_reserved_gid_free(fdev->mdev, conn->qp.sgid_index); 958err: 959 kfree(conn); 960out: 961 return ret; 962} 963 964void mlx5_fpga_conn_destroy(struct mlx5_fpga_conn *conn) 965{ 966 struct mlx5_fpga_device *fdev = conn->fdev; 967 struct mlx5_core_dev *mdev = fdev->mdev; 968 int err = 0; 969 970 conn->qp.active = false; 971 tasklet_disable(&conn->cq.tasklet); 972 synchronize_irq(conn->cq.mcq.irqn); 973 974 mlx5_fpga_destroy_qp(conn->fdev->mdev, conn->fpga_qpn); 975 err = mlx5_core_qp_modify(mdev, MLX5_CMD_OP_2ERR_QP, 0, NULL, 976 &conn->qp.mqp); 977 if (err) 978 mlx5_fpga_warn(fdev, "qp_modify 2ERR failed: %d\n", err); 979 mlx5_fpga_conn_destroy_qp(conn); 980 mlx5_fpga_conn_destroy_cq(conn); 981 982 mlx5_core_roce_gid_set(conn->fdev->mdev, conn->qp.sgid_index, 0, 0, 983 NULL, NULL, false, 0); 984 mlx5_core_reserved_gid_free(conn->fdev->mdev, conn->qp.sgid_index); 985 kfree(conn); 986} 987 988int mlx5_fpga_conn_device_init(struct mlx5_fpga_device *fdev) 989{ 990 int err; 991 992 err = mlx5_nic_vport_enable_roce(fdev->mdev); 993 if (err) { 994 mlx5_fpga_err(fdev, "Failed to enable RoCE: %d\n", err); 995 goto out; 996 } 997 998 fdev->conn_res.uar = mlx5_get_uars_page(fdev->mdev); 999 if (IS_ERR(fdev->conn_res.uar)) { 1000 err = PTR_ERR(fdev->conn_res.uar); 1001 mlx5_fpga_err(fdev, "get_uars_page failed, %d\n", err); 1002 goto err_roce; 1003 } 1004 mlx5_fpga_dbg(fdev, "Allocated UAR index %u\n", 1005 fdev->conn_res.uar->index); 1006 1007 err = mlx5_core_alloc_pd(fdev->mdev, &fdev->conn_res.pdn); 1008 if (err) { 1009 mlx5_fpga_err(fdev, "alloc pd failed, %d\n", err); 1010 goto err_uar; 1011 } 1012 mlx5_fpga_dbg(fdev, "Allocated PD %u\n", fdev->conn_res.pdn); 1013 1014 err = mlx5_fpga_conn_create_mkey(fdev->mdev, fdev->conn_res.pdn, 1015 &fdev->conn_res.mkey); 1016 if (err) { 1017 mlx5_fpga_err(fdev, "create mkey failed, %d\n", err); 1018 goto err_dealloc_pd; 1019 } 1020 mlx5_fpga_dbg(fdev, "Created mkey 0x%x\n", fdev->conn_res.mkey.key); 1021 1022 return 0; 1023 1024err_dealloc_pd: 1025 mlx5_core_dealloc_pd(fdev->mdev, fdev->conn_res.pdn); 1026err_uar: 1027 mlx5_put_uars_page(fdev->mdev, fdev->conn_res.uar); 1028err_roce: 1029 mlx5_nic_vport_disable_roce(fdev->mdev); 1030out: 1031 return err; 1032} 1033 1034void mlx5_fpga_conn_device_cleanup(struct mlx5_fpga_device *fdev) 1035{ 1036 mlx5_core_destroy_mkey(fdev->mdev, &fdev->conn_res.mkey); 1037 mlx5_core_dealloc_pd(fdev->mdev, fdev->conn_res.pdn); 1038 mlx5_put_uars_page(fdev->mdev, fdev->conn_res.uar); 1039 mlx5_nic_vport_disable_roce(fdev->mdev); 1040} 1041