1325665Shselasky/*- 2325665Shselasky * Copyright (c) 2016, Mellanox Technologies, Ltd. All rights reserved. 3325665Shselasky * 4325665Shselasky * Redistribution and use in source and binary forms, with or without 5325665Shselasky * modification, are permitted provided that the following conditions 6325665Shselasky * are met: 7325665Shselasky * 1. Redistributions of source code must retain the above copyright 8325665Shselasky * notice, this list of conditions and the following disclaimer. 9325665Shselasky * 2. Redistributions in binary form must reproduce the above copyright 10325665Shselasky * notice, this list of conditions and the following disclaimer in the 11325665Shselasky * documentation and/or other materials provided with the distribution. 12325665Shselasky * 13325665Shselasky * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND 14325665Shselasky * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15325665Shselasky * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16325665Shselasky * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE 17325665Shselasky * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18325665Shselasky * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19325665Shselasky * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20325665Shselasky * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21325665Shselasky * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22325665Shselasky * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23325665Shselasky * SUCH DAMAGE. 24325665Shselasky * 25325665Shselasky * $FreeBSD: stable/11/sys/dev/mlx5/mlx5_ib/mlx5_ib_gsi.c 331769 2018-03-30 18:06:29Z hselasky $ 26325665Shselasky */ 27325665Shselasky 28325665Shselasky#include "mlx5_ib.h" 29325665Shselasky 30325665Shselaskystruct mlx5_ib_gsi_wr { 31325665Shselasky struct ib_cqe cqe; 32325665Shselasky struct ib_wc wc; 33325665Shselasky int send_flags; 34325665Shselasky bool completed:1; 35325665Shselasky}; 36325665Shselasky 37325665Shselaskystruct mlx5_ib_gsi_qp { 38325665Shselasky struct ib_qp ibqp; 39325665Shselasky struct ib_qp *rx_qp; 40325665Shselasky u8 port_num; 41325665Shselasky struct ib_qp_cap cap; 42325665Shselasky enum ib_sig_type sq_sig_type; 43325665Shselasky /* Serialize qp state modifications */ 44325665Shselasky struct mutex mutex; 45325665Shselasky struct ib_cq *cq; 46325665Shselasky struct mlx5_ib_gsi_wr *outstanding_wrs; 47325665Shselasky u32 outstanding_pi, outstanding_ci; 48325665Shselasky int num_qps; 49325665Shselasky /* Protects access to the tx_qps. Post send operations synchronize 50325665Shselasky * with tx_qp creation in setup_qp(). Also protects the 51325665Shselasky * outstanding_wrs array and indices. 52325665Shselasky */ 53325665Shselasky spinlock_t lock; 54325665Shselasky struct ib_qp **tx_qps; 55325665Shselasky}; 56325665Shselasky 57325665Shselaskystatic struct mlx5_ib_gsi_qp *gsi_qp(struct ib_qp *qp) 58325665Shselasky{ 59325665Shselasky return container_of(qp, struct mlx5_ib_gsi_qp, ibqp); 60325665Shselasky} 61325665Shselasky 62325665Shselaskystatic bool mlx5_ib_deth_sqpn_cap(struct mlx5_ib_dev *dev) 63325665Shselasky{ 64325665Shselasky return MLX5_CAP_GEN(dev->mdev, set_deth_sqpn); 65325665Shselasky} 66325665Shselasky 67325665Shselasky/* Call with gsi->lock locked */ 68325665Shselaskystatic void generate_completions(struct mlx5_ib_gsi_qp *gsi) 69325665Shselasky{ 70325665Shselasky struct ib_cq *gsi_cq = gsi->ibqp.send_cq; 71325665Shselasky struct mlx5_ib_gsi_wr *wr; 72325665Shselasky u32 index; 73325665Shselasky 74325665Shselasky for (index = gsi->outstanding_ci; index != gsi->outstanding_pi; 75325665Shselasky index++) { 76325665Shselasky wr = &gsi->outstanding_wrs[index % gsi->cap.max_send_wr]; 77325665Shselasky 78325665Shselasky if (!wr->completed) 79325665Shselasky break; 80325665Shselasky 81325665Shselasky if (gsi->sq_sig_type == IB_SIGNAL_ALL_WR || 82325665Shselasky wr->send_flags & IB_SEND_SIGNALED) 83325665Shselasky WARN_ON_ONCE(mlx5_ib_generate_wc(gsi_cq, &wr->wc)); 84325665Shselasky 85325665Shselasky wr->completed = false; 86325665Shselasky } 87325665Shselasky 88325665Shselasky gsi->outstanding_ci = index; 89325665Shselasky} 90325665Shselasky 91325665Shselaskystatic void handle_single_completion(struct ib_cq *cq, struct ib_wc *wc) 92325665Shselasky{ 93325665Shselasky struct mlx5_ib_gsi_qp *gsi = cq->cq_context; 94325665Shselasky struct mlx5_ib_gsi_wr *wr = 95325665Shselasky container_of(wc->wr_cqe, struct mlx5_ib_gsi_wr, cqe); 96325665Shselasky u64 wr_id; 97325665Shselasky unsigned long flags; 98325665Shselasky 99325665Shselasky spin_lock_irqsave(&gsi->lock, flags); 100325665Shselasky wr->completed = true; 101325665Shselasky wr_id = wr->wc.wr_id; 102325665Shselasky wr->wc = *wc; 103325665Shselasky wr->wc.wr_id = wr_id; 104325665Shselasky wr->wc.qp = &gsi->ibqp; 105325665Shselasky 106325665Shselasky generate_completions(gsi); 107325665Shselasky spin_unlock_irqrestore(&gsi->lock, flags); 108325665Shselasky} 109325665Shselasky 110325665Shselaskystruct ib_qp *mlx5_ib_gsi_create_qp(struct ib_pd *pd, 111325665Shselasky struct ib_qp_init_attr *init_attr) 112325665Shselasky{ 113325665Shselasky struct mlx5_ib_dev *dev = to_mdev(pd->device); 114325665Shselasky struct mlx5_ib_gsi_qp *gsi; 115325665Shselasky struct ib_qp_init_attr hw_init_attr = *init_attr; 116325665Shselasky const u8 port_num = init_attr->port_num; 117325665Shselasky const int num_pkeys = pd->device->attrs.max_pkeys; 118325665Shselasky const int num_qps = mlx5_ib_deth_sqpn_cap(dev) ? num_pkeys : 0; 119325665Shselasky int ret; 120325665Shselasky 121325665Shselasky mlx5_ib_dbg(dev, "creating GSI QP\n"); 122325665Shselasky 123325665Shselasky if (port_num > ARRAY_SIZE(dev->devr.ports) || port_num < 1) { 124325665Shselasky mlx5_ib_warn(dev, 125325665Shselasky "invalid port number %d during GSI QP creation\n", 126325665Shselasky port_num); 127325665Shselasky return ERR_PTR(-EINVAL); 128325665Shselasky } 129325665Shselasky 130325665Shselasky gsi = kzalloc(sizeof(*gsi), GFP_KERNEL); 131325665Shselasky if (!gsi) 132325665Shselasky return ERR_PTR(-ENOMEM); 133325665Shselasky 134325665Shselasky gsi->tx_qps = kcalloc(num_qps, sizeof(*gsi->tx_qps), GFP_KERNEL); 135325665Shselasky if (!gsi->tx_qps) { 136325665Shselasky ret = -ENOMEM; 137325665Shselasky goto err_free; 138325665Shselasky } 139325665Shselasky 140325665Shselasky gsi->outstanding_wrs = kcalloc(init_attr->cap.max_send_wr, 141325665Shselasky sizeof(*gsi->outstanding_wrs), 142325665Shselasky GFP_KERNEL); 143325665Shselasky if (!gsi->outstanding_wrs) { 144325665Shselasky ret = -ENOMEM; 145325665Shselasky goto err_free_tx; 146325665Shselasky } 147325665Shselasky 148325665Shselasky mutex_init(&gsi->mutex); 149325665Shselasky 150325665Shselasky mutex_lock(&dev->devr.mutex); 151325665Shselasky 152325665Shselasky if (dev->devr.ports[port_num - 1].gsi) { 153325665Shselasky mlx5_ib_warn(dev, "GSI QP already exists on port %d\n", 154325665Shselasky port_num); 155325665Shselasky ret = -EBUSY; 156325665Shselasky goto err_free_wrs; 157325665Shselasky } 158325665Shselasky gsi->num_qps = num_qps; 159325665Shselasky spin_lock_init(&gsi->lock); 160325665Shselasky 161325665Shselasky gsi->cap = init_attr->cap; 162325665Shselasky gsi->sq_sig_type = init_attr->sq_sig_type; 163325665Shselasky gsi->ibqp.qp_num = 1; 164325665Shselasky gsi->port_num = port_num; 165325665Shselasky 166325665Shselasky gsi->cq = ib_alloc_cq(pd->device, gsi, init_attr->cap.max_send_wr, 0, 167325665Shselasky IB_POLL_SOFTIRQ); 168325665Shselasky if (IS_ERR(gsi->cq)) { 169325665Shselasky mlx5_ib_warn(dev, "unable to create send CQ for GSI QP. error %ld\n", 170325665Shselasky PTR_ERR(gsi->cq)); 171325665Shselasky ret = PTR_ERR(gsi->cq); 172325665Shselasky goto err_free_wrs; 173325665Shselasky } 174325665Shselasky 175325665Shselasky hw_init_attr.qp_type = MLX5_IB_QPT_HW_GSI; 176325665Shselasky hw_init_attr.send_cq = gsi->cq; 177325665Shselasky if (num_qps) { 178325665Shselasky hw_init_attr.cap.max_send_wr = 0; 179325665Shselasky hw_init_attr.cap.max_send_sge = 0; 180325665Shselasky hw_init_attr.cap.max_inline_data = 0; 181325665Shselasky } 182325665Shselasky gsi->rx_qp = ib_create_qp(pd, &hw_init_attr); 183325665Shselasky if (IS_ERR(gsi->rx_qp)) { 184325665Shselasky mlx5_ib_warn(dev, "unable to create hardware GSI QP. error %ld\n", 185325665Shselasky PTR_ERR(gsi->rx_qp)); 186325665Shselasky ret = PTR_ERR(gsi->rx_qp); 187325665Shselasky goto err_destroy_cq; 188325665Shselasky } 189325665Shselasky 190325665Shselasky dev->devr.ports[init_attr->port_num - 1].gsi = gsi; 191325665Shselasky 192325665Shselasky mutex_unlock(&dev->devr.mutex); 193325665Shselasky 194325665Shselasky return &gsi->ibqp; 195325665Shselasky 196325665Shselaskyerr_destroy_cq: 197325665Shselasky ib_free_cq(gsi->cq); 198325665Shselaskyerr_free_wrs: 199325665Shselasky mutex_unlock(&dev->devr.mutex); 200325665Shselasky kfree(gsi->outstanding_wrs); 201325665Shselaskyerr_free_tx: 202325665Shselasky kfree(gsi->tx_qps); 203325665Shselaskyerr_free: 204325665Shselasky kfree(gsi); 205325665Shselasky return ERR_PTR(ret); 206325665Shselasky} 207325665Shselasky 208325665Shselaskyint mlx5_ib_gsi_destroy_qp(struct ib_qp *qp) 209325665Shselasky{ 210325665Shselasky struct mlx5_ib_dev *dev = to_mdev(qp->device); 211325665Shselasky struct mlx5_ib_gsi_qp *gsi = gsi_qp(qp); 212325665Shselasky const int port_num = gsi->port_num; 213325665Shselasky int qp_index; 214325665Shselasky int ret; 215325665Shselasky 216325665Shselasky mlx5_ib_dbg(dev, "destroying GSI QP\n"); 217325665Shselasky 218325665Shselasky mutex_lock(&dev->devr.mutex); 219325665Shselasky ret = ib_destroy_qp(gsi->rx_qp); 220325665Shselasky if (ret) { 221325665Shselasky mlx5_ib_warn(dev, "unable to destroy hardware GSI QP. error %d\n", 222325665Shselasky ret); 223325665Shselasky mutex_unlock(&dev->devr.mutex); 224325665Shselasky return ret; 225325665Shselasky } 226325665Shselasky dev->devr.ports[port_num - 1].gsi = NULL; 227325665Shselasky mutex_unlock(&dev->devr.mutex); 228325665Shselasky gsi->rx_qp = NULL; 229325665Shselasky 230325665Shselasky for (qp_index = 0; qp_index < gsi->num_qps; ++qp_index) { 231325665Shselasky if (!gsi->tx_qps[qp_index]) 232325665Shselasky continue; 233325665Shselasky WARN_ON_ONCE(ib_destroy_qp(gsi->tx_qps[qp_index])); 234325665Shselasky gsi->tx_qps[qp_index] = NULL; 235325665Shselasky } 236325665Shselasky 237325665Shselasky ib_free_cq(gsi->cq); 238325665Shselasky 239325665Shselasky kfree(gsi->outstanding_wrs); 240325665Shselasky kfree(gsi->tx_qps); 241325665Shselasky kfree(gsi); 242325665Shselasky 243325665Shselasky return 0; 244325665Shselasky} 245325665Shselasky 246325665Shselaskystatic struct ib_qp *create_gsi_ud_qp(struct mlx5_ib_gsi_qp *gsi) 247325665Shselasky{ 248325665Shselasky struct ib_pd *pd = gsi->rx_qp->pd; 249325665Shselasky struct ib_qp_init_attr init_attr = { 250325665Shselasky .event_handler = gsi->rx_qp->event_handler, 251325665Shselasky .qp_context = gsi->rx_qp->qp_context, 252325665Shselasky .send_cq = gsi->cq, 253325665Shselasky .recv_cq = gsi->rx_qp->recv_cq, 254325665Shselasky .cap = { 255325665Shselasky .max_send_wr = gsi->cap.max_send_wr, 256325665Shselasky .max_send_sge = gsi->cap.max_send_sge, 257325665Shselasky .max_inline_data = gsi->cap.max_inline_data, 258325665Shselasky }, 259325665Shselasky .sq_sig_type = gsi->sq_sig_type, 260325665Shselasky .qp_type = IB_QPT_UD, 261325665Shselasky .create_flags = mlx5_ib_create_qp_sqpn_qp1(), 262325665Shselasky }; 263325665Shselasky 264325665Shselasky return ib_create_qp(pd, &init_attr); 265325665Shselasky} 266325665Shselasky 267325665Shselaskystatic int modify_to_rts(struct mlx5_ib_gsi_qp *gsi, struct ib_qp *qp, 268325665Shselasky u16 qp_index) 269325665Shselasky{ 270325665Shselasky struct mlx5_ib_dev *dev = to_mdev(qp->device); 271325665Shselasky struct ib_qp_attr attr; 272325665Shselasky int mask; 273325665Shselasky int ret; 274325665Shselasky 275325665Shselasky mask = IB_QP_STATE | IB_QP_PKEY_INDEX | IB_QP_QKEY | IB_QP_PORT; 276325665Shselasky attr.qp_state = IB_QPS_INIT; 277325665Shselasky attr.pkey_index = qp_index; 278325665Shselasky attr.qkey = IB_QP1_QKEY; 279325665Shselasky attr.port_num = gsi->port_num; 280325665Shselasky ret = ib_modify_qp(qp, &attr, mask); 281325665Shselasky if (ret) { 282325665Shselasky mlx5_ib_err(dev, "could not change QP%d state to INIT: %d\n", 283325665Shselasky qp->qp_num, ret); 284325665Shselasky return ret; 285325665Shselasky } 286325665Shselasky 287325665Shselasky attr.qp_state = IB_QPS_RTR; 288325665Shselasky ret = ib_modify_qp(qp, &attr, IB_QP_STATE); 289325665Shselasky if (ret) { 290325665Shselasky mlx5_ib_err(dev, "could not change QP%d state to RTR: %d\n", 291325665Shselasky qp->qp_num, ret); 292325665Shselasky return ret; 293325665Shselasky } 294325665Shselasky 295325665Shselasky attr.qp_state = IB_QPS_RTS; 296325665Shselasky attr.sq_psn = 0; 297325665Shselasky ret = ib_modify_qp(qp, &attr, IB_QP_STATE | IB_QP_SQ_PSN); 298325665Shselasky if (ret) { 299325665Shselasky mlx5_ib_err(dev, "could not change QP%d state to RTS: %d\n", 300325665Shselasky qp->qp_num, ret); 301325665Shselasky return ret; 302325665Shselasky } 303325665Shselasky 304325665Shselasky return 0; 305325665Shselasky} 306325665Shselasky 307325665Shselaskystatic void setup_qp(struct mlx5_ib_gsi_qp *gsi, u16 qp_index) 308325665Shselasky{ 309325665Shselasky struct ib_device *device = gsi->rx_qp->device; 310325665Shselasky struct mlx5_ib_dev *dev = to_mdev(device); 311325665Shselasky struct ib_qp *qp; 312325665Shselasky unsigned long flags; 313325665Shselasky u16 pkey; 314325665Shselasky int ret; 315325665Shselasky 316325665Shselasky ret = ib_query_pkey(device, gsi->port_num, qp_index, &pkey); 317325665Shselasky if (ret) { 318325665Shselasky mlx5_ib_warn(dev, "unable to read P_Key at port %d, index %d\n", 319325665Shselasky gsi->port_num, qp_index); 320325665Shselasky return; 321325665Shselasky } 322325665Shselasky 323325665Shselasky if (!pkey) { 324325665Shselasky mlx5_ib_dbg(dev, "invalid P_Key at port %d, index %d. Skipping.\n", 325325665Shselasky gsi->port_num, qp_index); 326325665Shselasky return; 327325665Shselasky } 328325665Shselasky 329325665Shselasky spin_lock_irqsave(&gsi->lock, flags); 330325665Shselasky qp = gsi->tx_qps[qp_index]; 331325665Shselasky spin_unlock_irqrestore(&gsi->lock, flags); 332325665Shselasky if (qp) { 333325665Shselasky mlx5_ib_dbg(dev, "already existing GSI TX QP at port %d, index %d. Skipping\n", 334325665Shselasky gsi->port_num, qp_index); 335325665Shselasky return; 336325665Shselasky } 337325665Shselasky 338325665Shselasky qp = create_gsi_ud_qp(gsi); 339325665Shselasky if (IS_ERR(qp)) { 340325665Shselasky mlx5_ib_warn(dev, "unable to create hardware UD QP for GSI: %ld\n", 341325665Shselasky PTR_ERR(qp)); 342325665Shselasky return; 343325665Shselasky } 344325665Shselasky 345325665Shselasky ret = modify_to_rts(gsi, qp, qp_index); 346325665Shselasky if (ret) 347325665Shselasky goto err_destroy_qp; 348325665Shselasky 349325665Shselasky spin_lock_irqsave(&gsi->lock, flags); 350325665Shselasky WARN_ON_ONCE(gsi->tx_qps[qp_index]); 351325665Shselasky gsi->tx_qps[qp_index] = qp; 352325665Shselasky spin_unlock_irqrestore(&gsi->lock, flags); 353325665Shselasky 354325665Shselasky return; 355325665Shselasky 356325665Shselaskyerr_destroy_qp: 357325665Shselasky WARN_ON_ONCE(qp); 358325665Shselasky} 359325665Shselasky 360325665Shselaskystatic void setup_qps(struct mlx5_ib_gsi_qp *gsi) 361325665Shselasky{ 362325665Shselasky u16 qp_index; 363325665Shselasky 364325665Shselasky for (qp_index = 0; qp_index < gsi->num_qps; ++qp_index) 365325665Shselasky setup_qp(gsi, qp_index); 366325665Shselasky} 367325665Shselasky 368325665Shselaskyint mlx5_ib_gsi_modify_qp(struct ib_qp *qp, struct ib_qp_attr *attr, 369325665Shselasky int attr_mask) 370325665Shselasky{ 371325665Shselasky struct mlx5_ib_dev *dev = to_mdev(qp->device); 372325665Shselasky struct mlx5_ib_gsi_qp *gsi = gsi_qp(qp); 373325665Shselasky int ret; 374325665Shselasky 375325665Shselasky mlx5_ib_dbg(dev, "modifying GSI QP to state %d\n", attr->qp_state); 376325665Shselasky 377325665Shselasky mutex_lock(&gsi->mutex); 378325665Shselasky ret = ib_modify_qp(gsi->rx_qp, attr, attr_mask); 379325665Shselasky if (ret) { 380325665Shselasky mlx5_ib_warn(dev, "unable to modify GSI rx QP: %d\n", ret); 381325665Shselasky goto unlock; 382325665Shselasky } 383325665Shselasky 384325665Shselasky if (to_mqp(gsi->rx_qp)->state == IB_QPS_RTS) 385325665Shselasky setup_qps(gsi); 386325665Shselasky 387325665Shselaskyunlock: 388325665Shselasky mutex_unlock(&gsi->mutex); 389325665Shselasky 390325665Shselasky return ret; 391325665Shselasky} 392325665Shselasky 393325665Shselaskyint mlx5_ib_gsi_query_qp(struct ib_qp *qp, struct ib_qp_attr *qp_attr, 394325665Shselasky int qp_attr_mask, 395325665Shselasky struct ib_qp_init_attr *qp_init_attr) 396325665Shselasky{ 397325665Shselasky struct mlx5_ib_gsi_qp *gsi = gsi_qp(qp); 398325665Shselasky int ret; 399325665Shselasky 400325665Shselasky mutex_lock(&gsi->mutex); 401325665Shselasky ret = ib_query_qp(gsi->rx_qp, qp_attr, qp_attr_mask, qp_init_attr); 402325665Shselasky qp_init_attr->cap = gsi->cap; 403325665Shselasky mutex_unlock(&gsi->mutex); 404325665Shselasky 405325665Shselasky return ret; 406325665Shselasky} 407325665Shselasky 408325665Shselasky/* Call with gsi->lock locked */ 409325665Shselaskystatic int mlx5_ib_add_outstanding_wr(struct mlx5_ib_gsi_qp *gsi, 410325665Shselasky struct ib_ud_wr *wr, struct ib_wc *wc) 411325665Shselasky{ 412325665Shselasky struct mlx5_ib_dev *dev = to_mdev(gsi->rx_qp->device); 413325665Shselasky struct mlx5_ib_gsi_wr *gsi_wr; 414325665Shselasky 415325665Shselasky if (gsi->outstanding_pi == gsi->outstanding_ci + gsi->cap.max_send_wr) { 416325665Shselasky mlx5_ib_warn(dev, "no available GSI work request.\n"); 417325665Shselasky return -ENOMEM; 418325665Shselasky } 419325665Shselasky 420325665Shselasky gsi_wr = &gsi->outstanding_wrs[gsi->outstanding_pi % 421325665Shselasky gsi->cap.max_send_wr]; 422325665Shselasky gsi->outstanding_pi++; 423325665Shselasky 424325665Shselasky if (!wc) { 425325665Shselasky memset(&gsi_wr->wc, 0, sizeof(gsi_wr->wc)); 426325665Shselasky gsi_wr->wc.pkey_index = wr->pkey_index; 427325665Shselasky gsi_wr->wc.wr_id = wr->wr.wr_id; 428325665Shselasky } else { 429325665Shselasky gsi_wr->wc = *wc; 430325665Shselasky gsi_wr->completed = true; 431325665Shselasky } 432325665Shselasky 433325665Shselasky gsi_wr->cqe.done = &handle_single_completion; 434325665Shselasky wr->wr.wr_cqe = &gsi_wr->cqe; 435325665Shselasky 436325665Shselasky return 0; 437325665Shselasky} 438325665Shselasky 439325665Shselasky/* Call with gsi->lock locked */ 440325665Shselaskystatic int mlx5_ib_gsi_silent_drop(struct mlx5_ib_gsi_qp *gsi, 441325665Shselasky struct ib_ud_wr *wr) 442325665Shselasky{ 443325665Shselasky struct ib_wc wc = { 444325665Shselasky { .wr_id = wr->wr.wr_id }, 445325665Shselasky .status = IB_WC_SUCCESS, 446325665Shselasky .opcode = IB_WC_SEND, 447325665Shselasky .qp = &gsi->ibqp, 448325665Shselasky }; 449325665Shselasky int ret; 450325665Shselasky 451325665Shselasky ret = mlx5_ib_add_outstanding_wr(gsi, wr, &wc); 452325665Shselasky if (ret) 453325665Shselasky return ret; 454325665Shselasky 455325665Shselasky generate_completions(gsi); 456325665Shselasky 457325665Shselasky return 0; 458325665Shselasky} 459325665Shselasky 460325665Shselasky/* Call with gsi->lock locked */ 461325665Shselaskystatic struct ib_qp *get_tx_qp(struct mlx5_ib_gsi_qp *gsi, struct ib_ud_wr *wr) 462325665Shselasky{ 463325665Shselasky struct mlx5_ib_dev *dev = to_mdev(gsi->rx_qp->device); 464325665Shselasky int qp_index = wr->pkey_index; 465325665Shselasky 466325665Shselasky if (!mlx5_ib_deth_sqpn_cap(dev)) 467325665Shselasky return gsi->rx_qp; 468325665Shselasky 469325665Shselasky if (qp_index >= gsi->num_qps) 470325665Shselasky return NULL; 471325665Shselasky 472325665Shselasky return gsi->tx_qps[qp_index]; 473325665Shselasky} 474325665Shselasky 475325665Shselaskyint mlx5_ib_gsi_post_send(struct ib_qp *qp, struct ib_send_wr *wr, 476325665Shselasky struct ib_send_wr **bad_wr) 477325665Shselasky{ 478325665Shselasky struct mlx5_ib_gsi_qp *gsi = gsi_qp(qp); 479325665Shselasky struct ib_qp *tx_qp; 480325665Shselasky unsigned long flags; 481325665Shselasky int ret; 482325665Shselasky 483325665Shselasky for (; wr; wr = wr->next) { 484325665Shselasky struct ib_ud_wr cur_wr = *ud_wr(wr); 485325665Shselasky 486325665Shselasky cur_wr.wr.next = NULL; 487325665Shselasky 488325665Shselasky spin_lock_irqsave(&gsi->lock, flags); 489325665Shselasky tx_qp = get_tx_qp(gsi, &cur_wr); 490325665Shselasky if (!tx_qp) { 491325665Shselasky ret = mlx5_ib_gsi_silent_drop(gsi, &cur_wr); 492325665Shselasky if (ret) 493325665Shselasky goto err; 494325665Shselasky spin_unlock_irqrestore(&gsi->lock, flags); 495325665Shselasky continue; 496325665Shselasky } 497325665Shselasky 498325665Shselasky ret = mlx5_ib_add_outstanding_wr(gsi, &cur_wr, NULL); 499325665Shselasky if (ret) 500325665Shselasky goto err; 501325665Shselasky 502325665Shselasky ret = ib_post_send(tx_qp, &cur_wr.wr, bad_wr); 503325665Shselasky if (ret) { 504325665Shselasky /* Undo the effect of adding the outstanding wr */ 505325665Shselasky gsi->outstanding_pi = (gsi->outstanding_pi - 1) % 506325665Shselasky gsi->cap.max_send_wr; 507325665Shselasky goto err; 508325665Shselasky } 509325665Shselasky spin_unlock_irqrestore(&gsi->lock, flags); 510325665Shselasky } 511325665Shselasky 512325665Shselasky return 0; 513325665Shselasky 514325665Shselaskyerr: 515325665Shselasky spin_unlock_irqrestore(&gsi->lock, flags); 516325665Shselasky *bad_wr = wr; 517325665Shselasky return ret; 518325665Shselasky} 519325665Shselasky 520325665Shselaskyint mlx5_ib_gsi_post_recv(struct ib_qp *qp, struct ib_recv_wr *wr, 521325665Shselasky struct ib_recv_wr **bad_wr) 522325665Shselasky{ 523325665Shselasky struct mlx5_ib_gsi_qp *gsi = gsi_qp(qp); 524325665Shselasky 525325665Shselasky return ib_post_recv(gsi->rx_qp, wr, bad_wr); 526325665Shselasky} 527325665Shselasky 528325665Shselaskyvoid mlx5_ib_gsi_pkey_change(struct mlx5_ib_gsi_qp *gsi) 529325665Shselasky{ 530325665Shselasky if (!gsi) 531325665Shselasky return; 532325665Shselasky 533325665Shselasky mutex_lock(&gsi->mutex); 534325665Shselasky setup_qps(gsi); 535325665Shselasky mutex_unlock(&gsi->mutex); 536325665Shselasky} 537