1178784Skmacy/* 2178784Skmacy * Copyright (c) 2004 Mellanox Technologies Ltd. All rights reserved. 3178784Skmacy * Copyright (c) 2004 Infinicon Corporation. All rights reserved. 4178784Skmacy * Copyright (c) 2004 Intel Corporation. All rights reserved. 5178784Skmacy * Copyright (c) 2004 Topspin Corporation. All rights reserved. 6178784Skmacy * Copyright (c) 2004 Voltaire Corporation. All rights reserved. 7178784Skmacy * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved. 8178784Skmacy * Copyright (c) 2005, 2006 Cisco Systems. All rights reserved. 9178784Skmacy * 10178784Skmacy * This software is available to you under a choice of one of two 11178784Skmacy * licenses. You may choose to be licensed under the terms of the GNU 12178784Skmacy * General Public License (GPL) Version 2, available from the file 13178784Skmacy * COPYING in the main directory of this source tree, or the 14178784Skmacy * OpenIB.org BSD license below: 15178784Skmacy * 16178784Skmacy * Redistribution and use in source and binary forms, with or 17178784Skmacy * without modification, are permitted provided that the following 18178784Skmacy * conditions are met: 19178784Skmacy * 20178784Skmacy * - Redistributions of source code must retain the above 21178784Skmacy * copyright notice, this list of conditions and the following 22178784Skmacy * disclaimer. 23178784Skmacy * 24178784Skmacy * - Redistributions in binary form must reproduce the above 25178784Skmacy * copyright notice, this list of conditions and the following 26178784Skmacy * disclaimer in the documentation and/or other materials 27178784Skmacy * provided with the distribution. 28178784Skmacy * 29178784Skmacy * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 30178784Skmacy * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 31178784Skmacy * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 32178784Skmacy * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 33178784Skmacy * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 34178784Skmacy * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 35178784Skmacy * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 36178784Skmacy * SOFTWARE. 37178784Skmacy * 38178784Skmacy * $Id: verbs.c 1349 2004-12-16 21:09:43Z roland $ 39178784Skmacy */ 40178784Skmacy 41178784Skmacy#include <sys/cdefs.h> 42178784Skmacy__FBSDID("$FreeBSD$"); 43178784Skmacy 44178784Skmacy#include <sys/param.h> 45178784Skmacy#include <sys/systm.h> 46178784Skmacy#include <sys/kernel.h> 47178784Skmacy#include <sys/libkern.h> 48178784Skmacy#include <sys/module.h> 49178784Skmacy#include <sys/endian.h> 50178784Skmacy 51178784Skmacy#include <contrib/rdma/ib_verbs.h> 52178784Skmacy#include <contrib/rdma/ib_cache.h> 53178784Skmacy 54178784Skmacyint ib_rate_to_mult(enum ib_rate rate) 55178784Skmacy{ 56178784Skmacy switch (rate) { 57178784Skmacy case IB_RATE_2_5_GBPS: return 1; 58178784Skmacy case IB_RATE_5_GBPS: return 2; 59178784Skmacy case IB_RATE_10_GBPS: return 4; 60178784Skmacy case IB_RATE_20_GBPS: return 8; 61178784Skmacy case IB_RATE_30_GBPS: return 12; 62178784Skmacy case IB_RATE_40_GBPS: return 16; 63178784Skmacy case IB_RATE_60_GBPS: return 24; 64178784Skmacy case IB_RATE_80_GBPS: return 32; 65178784Skmacy case IB_RATE_120_GBPS: return 48; 66178784Skmacy default: return -1; 67178784Skmacy } 68178784Skmacy} 69178784Skmacy 70178784Skmacyenum ib_rate mult_to_ib_rate(int mult) 71178784Skmacy{ 72178784Skmacy switch (mult) { 73178784Skmacy case 1: return IB_RATE_2_5_GBPS; 74178784Skmacy case 2: return IB_RATE_5_GBPS; 75178784Skmacy case 4: return IB_RATE_10_GBPS; 76178784Skmacy case 8: return IB_RATE_20_GBPS; 77178784Skmacy case 12: return IB_RATE_30_GBPS; 78178784Skmacy case 16: return IB_RATE_40_GBPS; 79178784Skmacy case 24: return IB_RATE_60_GBPS; 80178784Skmacy case 32: return IB_RATE_80_GBPS; 81178784Skmacy case 48: return IB_RATE_120_GBPS; 82178784Skmacy default: return IB_RATE_PORT_CURRENT; 83178784Skmacy } 84178784Skmacy} 85178784Skmacy 86178784Skmacyenum rdma_transport_type 87178784Skmacyrdma_node_get_transport(enum rdma_node_type node_type) 88178784Skmacy{ 89178784Skmacy switch (node_type) { 90178784Skmacy case RDMA_NODE_IB_CA: 91178784Skmacy case RDMA_NODE_IB_SWITCH: 92178784Skmacy case RDMA_NODE_IB_ROUTER: 93178784Skmacy return RDMA_TRANSPORT_IB; 94178784Skmacy case RDMA_NODE_RNIC: 95178784Skmacy return RDMA_TRANSPORT_IWARP; 96178784Skmacy default: 97178784Skmacy panic("bad condition"); 98178784Skmacy return 0; 99178784Skmacy } 100178784Skmacy} 101178784Skmacy 102178784Skmacy/* Protection domains */ 103178784Skmacy 104178784Skmacystruct ib_pd *ib_alloc_pd(struct ib_device *device) 105178784Skmacy{ 106178784Skmacy struct ib_pd *pd; 107178784Skmacy 108178784Skmacy pd = device->alloc_pd(device, NULL, NULL); 109178784Skmacy 110178784Skmacy if (!IS_ERR(pd)) { 111178784Skmacy pd->device = device; 112178784Skmacy pd->uobject = NULL; 113178784Skmacy atomic_store_rel_int(&pd->usecnt, 0); 114178784Skmacy } 115178784Skmacy 116178784Skmacy return pd; 117178784Skmacy} 118178784Skmacy 119178784Skmacyint ib_dealloc_pd(struct ib_pd *pd) 120178784Skmacy{ 121178784Skmacy if (atomic_load_acq_int(&pd->usecnt)) 122178784Skmacy return (EBUSY); 123178784Skmacy 124178784Skmacy return pd->device->dealloc_pd(pd); 125178784Skmacy} 126178784Skmacy 127178784Skmacy/* Address handles */ 128178784Skmacy 129178784Skmacystruct ib_ah *ib_create_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr) 130178784Skmacy{ 131178784Skmacy struct ib_ah *ah; 132178784Skmacy 133178784Skmacy ah = pd->device->create_ah(pd, ah_attr); 134178784Skmacy 135178784Skmacy if (!IS_ERR(ah)) { 136178784Skmacy ah->device = pd->device; 137178784Skmacy ah->pd = pd; 138178784Skmacy ah->uobject = NULL; 139178784Skmacy atomic_add_acq_int(&pd->usecnt, 1); 140178784Skmacy } 141178784Skmacy 142178784Skmacy return ah; 143178784Skmacy} 144178784Skmacy 145178784Skmacyint ib_init_ah_from_wc(struct ib_device *device, u8 port_num, struct ib_wc *wc, 146178784Skmacy struct ib_grh *grh, struct ib_ah_attr *ah_attr) 147178784Skmacy{ 148178784Skmacy u32 flow_class; 149178784Skmacy u16 gid_index; 150178784Skmacy int ret; 151178784Skmacy 152178784Skmacy memset(ah_attr, 0, sizeof *ah_attr); 153178784Skmacy ah_attr->dlid = wc->slid; 154178784Skmacy ah_attr->sl = wc->sl; 155178784Skmacy ah_attr->src_path_bits = wc->dlid_path_bits; 156178784Skmacy ah_attr->port_num = port_num; 157178784Skmacy 158178784Skmacy if (wc->wc_flags & IB_WC_GRH) { 159178784Skmacy ah_attr->ah_flags = IB_AH_GRH; 160178784Skmacy ah_attr->grh.dgid = grh->sgid; 161178784Skmacy 162178784Skmacy ret = ib_find_cached_gid(device, &grh->dgid, &port_num, 163178784Skmacy &gid_index); 164178784Skmacy if (ret) 165178784Skmacy return ret; 166178784Skmacy 167178784Skmacy ah_attr->grh.sgid_index = (u8) gid_index; 168178784Skmacy flow_class = be32toh(grh->version_tclass_flow); 169178784Skmacy ah_attr->grh.flow_label = flow_class & 0xFFFFF; 170178784Skmacy ah_attr->grh.hop_limit = 0xFF; 171178784Skmacy ah_attr->grh.traffic_class = (flow_class >> 20) & 0xFF; 172178784Skmacy } 173178784Skmacy return 0; 174178784Skmacy} 175178784Skmacy 176178784Skmacystruct ib_ah *ib_create_ah_from_wc(struct ib_pd *pd, struct ib_wc *wc, 177178784Skmacy struct ib_grh *grh, u8 port_num) 178178784Skmacy{ 179178784Skmacy struct ib_ah_attr ah_attr; 180178784Skmacy int ret; 181178784Skmacy 182178784Skmacy ret = ib_init_ah_from_wc(pd->device, port_num, wc, grh, &ah_attr); 183178784Skmacy if (ret) 184178784Skmacy return ERR_PTR(ret); 185178784Skmacy 186178784Skmacy return ib_create_ah(pd, &ah_attr); 187178784Skmacy} 188178784Skmacy 189178784Skmacyint ib_modify_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr) 190178784Skmacy{ 191178784Skmacy return ah->device->modify_ah ? 192178784Skmacy ah->device->modify_ah(ah, ah_attr) : 193178784Skmacy ENOSYS; 194178784Skmacy} 195178784Skmacy 196178784Skmacyint ib_query_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr) 197178784Skmacy{ 198178784Skmacy return ah->device->query_ah ? 199178784Skmacy ah->device->query_ah(ah, ah_attr) : 200178784Skmacy ENOSYS; 201178784Skmacy} 202178784Skmacy 203178784Skmacyint ib_destroy_ah(struct ib_ah *ah) 204178784Skmacy{ 205178784Skmacy struct ib_pd *pd; 206178784Skmacy int ret; 207178784Skmacy 208178784Skmacy pd = ah->pd; 209178784Skmacy ret = ah->device->destroy_ah(ah); 210178784Skmacy if (!ret) 211178784Skmacy atomic_subtract_acq_int(&pd->usecnt, 1); 212178784Skmacy 213178784Skmacy return ret; 214178784Skmacy} 215178784Skmacy 216178784Skmacy/* Shared receive queues */ 217178784Skmacy 218178784Skmacystruct ib_srq *ib_create_srq(struct ib_pd *pd, 219178784Skmacy struct ib_srq_init_attr *srq_init_attr) 220178784Skmacy{ 221178784Skmacy struct ib_srq *srq; 222178784Skmacy 223178784Skmacy if (!pd->device->create_srq) 224178784Skmacy return ERR_PTR(ENOSYS); 225178784Skmacy 226178784Skmacy srq = pd->device->create_srq(pd, srq_init_attr, NULL); 227178784Skmacy 228178784Skmacy if (!IS_ERR(srq)) { 229178784Skmacy srq->device = pd->device; 230178784Skmacy srq->pd = pd; 231178784Skmacy srq->uobject = NULL; 232178784Skmacy srq->event_handler = srq_init_attr->event_handler; 233178784Skmacy srq->srq_context = srq_init_attr->srq_context; 234178784Skmacy atomic_add_acq_int(&pd->usecnt, 1); 235178784Skmacy atomic_store_rel_int(&srq->usecnt, 0); 236178784Skmacy } 237178784Skmacy 238178784Skmacy return srq; 239178784Skmacy} 240178784Skmacy 241178784Skmacyint ib_modify_srq(struct ib_srq *srq, 242178784Skmacy struct ib_srq_attr *srq_attr, 243178784Skmacy enum ib_srq_attr_mask srq_attr_mask) 244178784Skmacy{ 245178784Skmacy return srq->device->modify_srq(srq, srq_attr, srq_attr_mask, NULL); 246178784Skmacy} 247178784Skmacy 248178784Skmacyint ib_query_srq(struct ib_srq *srq, 249178784Skmacy struct ib_srq_attr *srq_attr) 250178784Skmacy{ 251178784Skmacy return srq->device->query_srq ? 252178784Skmacy srq->device->query_srq(srq, srq_attr) : ENOSYS; 253178784Skmacy} 254178784Skmacy 255178784Skmacyint ib_destroy_srq(struct ib_srq *srq) 256178784Skmacy{ 257178784Skmacy struct ib_pd *pd; 258178784Skmacy int ret; 259178784Skmacy 260178784Skmacy if (atomic_load_acq_int(&srq->usecnt)) 261178784Skmacy return (EBUSY); 262178784Skmacy 263178784Skmacy pd = srq->pd; 264178784Skmacy 265178784Skmacy ret = srq->device->destroy_srq(srq); 266178784Skmacy if (!ret) 267178784Skmacy atomic_subtract_acq_int(&pd->usecnt, 1); 268178784Skmacy 269178784Skmacy return ret; 270178784Skmacy} 271178784Skmacy 272178784Skmacy/* Queue pairs */ 273178784Skmacy 274178784Skmacystruct ib_qp *ib_create_qp(struct ib_pd *pd, 275178784Skmacy struct ib_qp_init_attr *qp_init_attr) 276178784Skmacy{ 277178784Skmacy struct ib_qp *qp; 278178784Skmacy 279178784Skmacy qp = pd->device->create_qp(pd, qp_init_attr, NULL); 280178784Skmacy 281178784Skmacy if (!IS_ERR(qp)) { 282178784Skmacy qp->device = pd->device; 283178784Skmacy qp->pd = pd; 284178784Skmacy qp->send_cq = qp_init_attr->send_cq; 285178784Skmacy qp->recv_cq = qp_init_attr->recv_cq; 286178784Skmacy qp->srq = qp_init_attr->srq; 287178784Skmacy qp->uobject = NULL; 288178784Skmacy qp->event_handler = qp_init_attr->event_handler; 289178784Skmacy qp->qp_context = qp_init_attr->qp_context; 290178784Skmacy qp->qp_type = qp_init_attr->qp_type; 291178784Skmacy atomic_add_acq_int(&pd->usecnt, 1); 292178784Skmacy atomic_add_acq_int(&qp_init_attr->send_cq->usecnt, 1); 293178784Skmacy atomic_add_acq_int(&qp_init_attr->recv_cq->usecnt, 1); 294178784Skmacy if (qp_init_attr->srq) 295178784Skmacy atomic_add_acq_int(&qp_init_attr->srq->usecnt, 1); 296178784Skmacy } 297178784Skmacy 298178784Skmacy return qp; 299178784Skmacy} 300178784Skmacy 301178784Skmacystatic const struct { 302178784Skmacy int valid; 303178784Skmacy enum ib_qp_attr_mask req_param[IB_QPT_RAW_ETY + 1]; 304178784Skmacy enum ib_qp_attr_mask opt_param[IB_QPT_RAW_ETY + 1]; 305178784Skmacy} qp_state_table[IB_QPS_ERR + 1][IB_QPS_ERR + 1] = { 306178784Skmacy [IB_QPS_RESET] = { 307178784Skmacy [IB_QPS_RESET] = { .valid = 1 }, 308178784Skmacy [IB_QPS_ERR] = { .valid = 1 }, 309178784Skmacy [IB_QPS_INIT] = { 310178784Skmacy .valid = 1, 311178784Skmacy .req_param = { 312178784Skmacy [IB_QPT_UD] = (IB_QP_PKEY_INDEX | 313178784Skmacy IB_QP_PORT | 314178784Skmacy IB_QP_QKEY), 315178784Skmacy [IB_QPT_UC] = (IB_QP_PKEY_INDEX | 316178784Skmacy IB_QP_PORT | 317178784Skmacy IB_QP_ACCESS_FLAGS), 318178784Skmacy [IB_QPT_RC] = (IB_QP_PKEY_INDEX | 319178784Skmacy IB_QP_PORT | 320178784Skmacy IB_QP_ACCESS_FLAGS), 321178784Skmacy [IB_QPT_SMI] = (IB_QP_PKEY_INDEX | 322178784Skmacy IB_QP_QKEY), 323178784Skmacy [IB_QPT_GSI] = (IB_QP_PKEY_INDEX | 324178784Skmacy IB_QP_QKEY), 325178784Skmacy } 326178784Skmacy }, 327178784Skmacy }, 328178784Skmacy [IB_QPS_INIT] = { 329178784Skmacy [IB_QPS_RESET] = { .valid = 1 }, 330178784Skmacy [IB_QPS_ERR] = { .valid = 1 }, 331178784Skmacy [IB_QPS_INIT] = { 332178784Skmacy .valid = 1, 333178784Skmacy .opt_param = { 334178784Skmacy [IB_QPT_UD] = (IB_QP_PKEY_INDEX | 335178784Skmacy IB_QP_PORT | 336178784Skmacy IB_QP_QKEY), 337178784Skmacy [IB_QPT_UC] = (IB_QP_PKEY_INDEX | 338178784Skmacy IB_QP_PORT | 339178784Skmacy IB_QP_ACCESS_FLAGS), 340178784Skmacy [IB_QPT_RC] = (IB_QP_PKEY_INDEX | 341178784Skmacy IB_QP_PORT | 342178784Skmacy IB_QP_ACCESS_FLAGS), 343178784Skmacy [IB_QPT_SMI] = (IB_QP_PKEY_INDEX | 344178784Skmacy IB_QP_QKEY), 345178784Skmacy [IB_QPT_GSI] = (IB_QP_PKEY_INDEX | 346178784Skmacy IB_QP_QKEY), 347178784Skmacy } 348178784Skmacy }, 349178784Skmacy [IB_QPS_RTR] = { 350178784Skmacy .valid = 1, 351178784Skmacy .req_param = { 352178784Skmacy [IB_QPT_UC] = (IB_QP_AV | 353178784Skmacy IB_QP_PATH_MTU | 354178784Skmacy IB_QP_DEST_QPN | 355178784Skmacy IB_QP_RQ_PSN), 356178784Skmacy [IB_QPT_RC] = (IB_QP_AV | 357178784Skmacy IB_QP_PATH_MTU | 358178784Skmacy IB_QP_DEST_QPN | 359178784Skmacy IB_QP_RQ_PSN | 360178784Skmacy IB_QP_MAX_DEST_RD_ATOMIC | 361178784Skmacy IB_QP_MIN_RNR_TIMER), 362178784Skmacy }, 363178784Skmacy .opt_param = { 364178784Skmacy [IB_QPT_UD] = (IB_QP_PKEY_INDEX | 365178784Skmacy IB_QP_QKEY), 366178784Skmacy [IB_QPT_UC] = (IB_QP_ALT_PATH | 367178784Skmacy IB_QP_ACCESS_FLAGS | 368178784Skmacy IB_QP_PKEY_INDEX), 369178784Skmacy [IB_QPT_RC] = (IB_QP_ALT_PATH | 370178784Skmacy IB_QP_ACCESS_FLAGS | 371178784Skmacy IB_QP_PKEY_INDEX), 372178784Skmacy [IB_QPT_SMI] = (IB_QP_PKEY_INDEX | 373178784Skmacy IB_QP_QKEY), 374178784Skmacy [IB_QPT_GSI] = (IB_QP_PKEY_INDEX | 375178784Skmacy IB_QP_QKEY), 376178784Skmacy } 377178784Skmacy } 378178784Skmacy }, 379178784Skmacy [IB_QPS_RTR] = { 380178784Skmacy [IB_QPS_RESET] = { .valid = 1 }, 381178784Skmacy [IB_QPS_ERR] = { .valid = 1 }, 382178784Skmacy [IB_QPS_RTS] = { 383178784Skmacy .valid = 1, 384178784Skmacy .req_param = { 385178784Skmacy [IB_QPT_UD] = IB_QP_SQ_PSN, 386178784Skmacy [IB_QPT_UC] = IB_QP_SQ_PSN, 387178784Skmacy [IB_QPT_RC] = (IB_QP_TIMEOUT | 388178784Skmacy IB_QP_RETRY_CNT | 389178784Skmacy IB_QP_RNR_RETRY | 390178784Skmacy IB_QP_SQ_PSN | 391178784Skmacy IB_QP_MAX_QP_RD_ATOMIC), 392178784Skmacy [IB_QPT_SMI] = IB_QP_SQ_PSN, 393178784Skmacy [IB_QPT_GSI] = IB_QP_SQ_PSN, 394178784Skmacy }, 395178784Skmacy .opt_param = { 396178784Skmacy [IB_QPT_UD] = (IB_QP_CUR_STATE | 397178784Skmacy IB_QP_QKEY), 398178784Skmacy [IB_QPT_UC] = (IB_QP_CUR_STATE | 399178784Skmacy IB_QP_ALT_PATH | 400178784Skmacy IB_QP_ACCESS_FLAGS | 401178784Skmacy IB_QP_PATH_MIG_STATE), 402178784Skmacy [IB_QPT_RC] = (IB_QP_CUR_STATE | 403178784Skmacy IB_QP_ALT_PATH | 404178784Skmacy IB_QP_ACCESS_FLAGS | 405178784Skmacy IB_QP_MIN_RNR_TIMER | 406178784Skmacy IB_QP_PATH_MIG_STATE), 407178784Skmacy [IB_QPT_SMI] = (IB_QP_CUR_STATE | 408178784Skmacy IB_QP_QKEY), 409178784Skmacy [IB_QPT_GSI] = (IB_QP_CUR_STATE | 410178784Skmacy IB_QP_QKEY), 411178784Skmacy } 412178784Skmacy } 413178784Skmacy }, 414178784Skmacy [IB_QPS_RTS] = { 415178784Skmacy [IB_QPS_RESET] = { .valid = 1 }, 416178784Skmacy [IB_QPS_ERR] = { .valid = 1 }, 417178784Skmacy [IB_QPS_RTS] = { 418178784Skmacy .valid = 1, 419178784Skmacy .opt_param = { 420178784Skmacy [IB_QPT_UD] = (IB_QP_CUR_STATE | 421178784Skmacy IB_QP_QKEY), 422178784Skmacy [IB_QPT_UC] = (IB_QP_CUR_STATE | 423178784Skmacy IB_QP_ACCESS_FLAGS | 424178784Skmacy IB_QP_ALT_PATH | 425178784Skmacy IB_QP_PATH_MIG_STATE), 426178784Skmacy [IB_QPT_RC] = (IB_QP_CUR_STATE | 427178784Skmacy IB_QP_ACCESS_FLAGS | 428178784Skmacy IB_QP_ALT_PATH | 429178784Skmacy IB_QP_PATH_MIG_STATE | 430178784Skmacy IB_QP_MIN_RNR_TIMER), 431178784Skmacy [IB_QPT_SMI] = (IB_QP_CUR_STATE | 432178784Skmacy IB_QP_QKEY), 433178784Skmacy [IB_QPT_GSI] = (IB_QP_CUR_STATE | 434178784Skmacy IB_QP_QKEY), 435178784Skmacy } 436178784Skmacy }, 437178784Skmacy [IB_QPS_SQD] = { 438178784Skmacy .valid = 1, 439178784Skmacy .opt_param = { 440178784Skmacy [IB_QPT_UD] = IB_QP_EN_SQD_ASYNC_NOTIFY, 441178784Skmacy [IB_QPT_UC] = IB_QP_EN_SQD_ASYNC_NOTIFY, 442178784Skmacy [IB_QPT_RC] = IB_QP_EN_SQD_ASYNC_NOTIFY, 443178784Skmacy [IB_QPT_SMI] = IB_QP_EN_SQD_ASYNC_NOTIFY, 444178784Skmacy [IB_QPT_GSI] = IB_QP_EN_SQD_ASYNC_NOTIFY 445178784Skmacy } 446178784Skmacy }, 447178784Skmacy }, 448178784Skmacy [IB_QPS_SQD] = { 449178784Skmacy [IB_QPS_RESET] = { .valid = 1 }, 450178784Skmacy [IB_QPS_ERR] = { .valid = 1 }, 451178784Skmacy [IB_QPS_RTS] = { 452178784Skmacy .valid = 1, 453178784Skmacy .opt_param = { 454178784Skmacy [IB_QPT_UD] = (IB_QP_CUR_STATE | 455178784Skmacy IB_QP_QKEY), 456178784Skmacy [IB_QPT_UC] = (IB_QP_CUR_STATE | 457178784Skmacy IB_QP_ALT_PATH | 458178784Skmacy IB_QP_ACCESS_FLAGS | 459178784Skmacy IB_QP_PATH_MIG_STATE), 460178784Skmacy [IB_QPT_RC] = (IB_QP_CUR_STATE | 461178784Skmacy IB_QP_ALT_PATH | 462178784Skmacy IB_QP_ACCESS_FLAGS | 463178784Skmacy IB_QP_MIN_RNR_TIMER | 464178784Skmacy IB_QP_PATH_MIG_STATE), 465178784Skmacy [IB_QPT_SMI] = (IB_QP_CUR_STATE | 466178784Skmacy IB_QP_QKEY), 467178784Skmacy [IB_QPT_GSI] = (IB_QP_CUR_STATE | 468178784Skmacy IB_QP_QKEY), 469178784Skmacy } 470178784Skmacy }, 471178784Skmacy [IB_QPS_SQD] = { 472178784Skmacy .valid = 1, 473178784Skmacy .opt_param = { 474178784Skmacy [IB_QPT_UD] = (IB_QP_PKEY_INDEX | 475178784Skmacy IB_QP_QKEY), 476178784Skmacy [IB_QPT_UC] = (IB_QP_AV | 477178784Skmacy IB_QP_ALT_PATH | 478178784Skmacy IB_QP_ACCESS_FLAGS | 479178784Skmacy IB_QP_PKEY_INDEX | 480178784Skmacy IB_QP_PATH_MIG_STATE), 481178784Skmacy [IB_QPT_RC] = (IB_QP_PORT | 482178784Skmacy IB_QP_AV | 483178784Skmacy IB_QP_TIMEOUT | 484178784Skmacy IB_QP_RETRY_CNT | 485178784Skmacy IB_QP_RNR_RETRY | 486178784Skmacy IB_QP_MAX_QP_RD_ATOMIC | 487178784Skmacy IB_QP_MAX_DEST_RD_ATOMIC | 488178784Skmacy IB_QP_ALT_PATH | 489178784Skmacy IB_QP_ACCESS_FLAGS | 490178784Skmacy IB_QP_PKEY_INDEX | 491178784Skmacy IB_QP_MIN_RNR_TIMER | 492178784Skmacy IB_QP_PATH_MIG_STATE), 493178784Skmacy [IB_QPT_SMI] = (IB_QP_PKEY_INDEX | 494178784Skmacy IB_QP_QKEY), 495178784Skmacy [IB_QPT_GSI] = (IB_QP_PKEY_INDEX | 496178784Skmacy IB_QP_QKEY), 497178784Skmacy } 498178784Skmacy } 499178784Skmacy }, 500178784Skmacy [IB_QPS_SQE] = { 501178784Skmacy [IB_QPS_RESET] = { .valid = 1 }, 502178784Skmacy [IB_QPS_ERR] = { .valid = 1 }, 503178784Skmacy [IB_QPS_RTS] = { 504178784Skmacy .valid = 1, 505178784Skmacy .opt_param = { 506178784Skmacy [IB_QPT_UD] = (IB_QP_CUR_STATE | 507178784Skmacy IB_QP_QKEY), 508178784Skmacy [IB_QPT_UC] = (IB_QP_CUR_STATE | 509178784Skmacy IB_QP_ACCESS_FLAGS), 510178784Skmacy [IB_QPT_SMI] = (IB_QP_CUR_STATE | 511178784Skmacy IB_QP_QKEY), 512178784Skmacy [IB_QPT_GSI] = (IB_QP_CUR_STATE | 513178784Skmacy IB_QP_QKEY), 514178784Skmacy } 515178784Skmacy } 516178784Skmacy }, 517178784Skmacy [IB_QPS_ERR] = { 518178784Skmacy [IB_QPS_RESET] = { .valid = 1 }, 519178784Skmacy [IB_QPS_ERR] = { .valid = 1 } 520178784Skmacy } 521178784Skmacy}; 522178784Skmacy 523178784Skmacyint ib_modify_qp_is_ok(enum ib_qp_state cur_state, enum ib_qp_state next_state, 524178784Skmacy enum ib_qp_type type, enum ib_qp_attr_mask mask) 525178784Skmacy{ 526178784Skmacy enum ib_qp_attr_mask req_param, opt_param; 527178784Skmacy 528178784Skmacy if (cur_state < 0 || cur_state > IB_QPS_ERR || 529178784Skmacy next_state < 0 || next_state > IB_QPS_ERR) 530178784Skmacy return 0; 531178784Skmacy 532178784Skmacy if (mask & IB_QP_CUR_STATE && 533178784Skmacy cur_state != IB_QPS_RTR && cur_state != IB_QPS_RTS && 534178784Skmacy cur_state != IB_QPS_SQD && cur_state != IB_QPS_SQE) 535178784Skmacy return 0; 536178784Skmacy 537178784Skmacy if (!qp_state_table[cur_state][next_state].valid) 538178784Skmacy return 0; 539178784Skmacy 540178784Skmacy req_param = qp_state_table[cur_state][next_state].req_param[type]; 541178784Skmacy opt_param = qp_state_table[cur_state][next_state].opt_param[type]; 542178784Skmacy 543178784Skmacy if ((mask & req_param) != req_param) 544178784Skmacy return 0; 545178784Skmacy 546178784Skmacy if (mask & ~(req_param | opt_param | IB_QP_STATE)) 547178784Skmacy return 0; 548178784Skmacy 549178784Skmacy return 1; 550178784Skmacy} 551178784Skmacy 552178784Skmacyint ib_modify_qp(struct ib_qp *qp, 553178784Skmacy struct ib_qp_attr *qp_attr, 554178784Skmacy int qp_attr_mask) 555178784Skmacy{ 556178784Skmacy return qp->device->modify_qp(qp, qp_attr, qp_attr_mask, NULL); 557178784Skmacy} 558178784Skmacy 559178784Skmacyint ib_query_qp(struct ib_qp *qp, 560178784Skmacy struct ib_qp_attr *qp_attr, 561178784Skmacy int qp_attr_mask, 562178784Skmacy struct ib_qp_init_attr *qp_init_attr) 563178784Skmacy{ 564178784Skmacy return qp->device->query_qp ? 565178784Skmacy qp->device->query_qp(qp, qp_attr, qp_attr_mask, qp_init_attr) : 566178784Skmacy ENOSYS; 567178784Skmacy} 568178784Skmacy 569178784Skmacyint ib_destroy_qp(struct ib_qp *qp) 570178784Skmacy{ 571178784Skmacy struct ib_pd *pd; 572178784Skmacy struct ib_cq *scq, *rcq; 573178784Skmacy struct ib_srq *srq; 574178784Skmacy int ret; 575178784Skmacy 576178784Skmacy pd = qp->pd; 577178784Skmacy scq = qp->send_cq; 578178784Skmacy rcq = qp->recv_cq; 579178784Skmacy srq = qp->srq; 580178784Skmacy 581178784Skmacy ret = qp->device->destroy_qp(qp); 582178784Skmacy if (!ret) { 583178784Skmacy atomic_subtract_acq_int(&pd->usecnt, 1); 584178784Skmacy atomic_subtract_acq_int(&scq->usecnt, 1); 585178784Skmacy atomic_subtract_acq_int(&rcq->usecnt, 1); 586178784Skmacy if (srq) 587178784Skmacy atomic_subtract_acq_int(&srq->usecnt, 1); 588178784Skmacy } 589178784Skmacy 590178784Skmacy return ret; 591178784Skmacy} 592178784Skmacy 593178784Skmacy/* Completion queues */ 594178784Skmacy 595178784Skmacystruct ib_cq *ib_create_cq(struct ib_device *device, 596178784Skmacy ib_comp_handler comp_handler, 597178784Skmacy void (*event_handler)(struct ib_event *, void *), 598178784Skmacy void *cq_context, int cqe, int comp_vector) 599178784Skmacy{ 600178784Skmacy struct ib_cq *cq; 601178784Skmacy 602178784Skmacy cq = device->create_cq(device, cqe, comp_vector, NULL, NULL); 603178784Skmacy 604178784Skmacy if (!IS_ERR(cq)) { 605178784Skmacy cq->device = device; 606178784Skmacy cq->uobject = NULL; 607178784Skmacy cq->comp_handler = comp_handler; 608178784Skmacy cq->event_handler = event_handler; 609178784Skmacy cq->cq_context = cq_context; 610178784Skmacy atomic_store_rel_int(&cq->usecnt, 0); 611178784Skmacy } 612178784Skmacy 613178784Skmacy return cq; 614178784Skmacy} 615178784Skmacy 616178784Skmacyint ib_destroy_cq(struct ib_cq *cq) 617178784Skmacy{ 618178784Skmacy if (atomic_load_acq_int(&cq->usecnt)) 619178784Skmacy return (EBUSY); 620178784Skmacy 621178784Skmacy return cq->device->destroy_cq(cq); 622178784Skmacy} 623178784Skmacy 624178784Skmacyint ib_resize_cq(struct ib_cq *cq, int cqe) 625178784Skmacy{ 626178784Skmacy return cq->device->resize_cq ? 627178784Skmacy cq->device->resize_cq(cq, cqe, NULL) : ENOSYS; 628178784Skmacy} 629178784Skmacy 630178784Skmacy/* Memory regions */ 631178784Skmacy 632178784Skmacystruct ib_mr *ib_get_dma_mr(struct ib_pd *pd, int mr_access_flags) 633178784Skmacy{ 634178784Skmacy struct ib_mr *mr; 635178784Skmacy 636178784Skmacy mr = pd->device->get_dma_mr(pd, mr_access_flags); 637178784Skmacy 638178784Skmacy if (!IS_ERR(mr)) { 639178784Skmacy mr->device = pd->device; 640178784Skmacy mr->pd = pd; 641178784Skmacy mr->uobject = NULL; 642178784Skmacy atomic_add_acq_int(&pd->usecnt, 1); 643178784Skmacy atomic_store_rel_int(&mr->usecnt, 0); 644178784Skmacy } 645178784Skmacy 646178784Skmacy return mr; 647178784Skmacy} 648178784Skmacy 649178784Skmacystruct ib_mr *ib_reg_phys_mr(struct ib_pd *pd, 650178784Skmacy struct ib_phys_buf *phys_buf_array, 651178784Skmacy int num_phys_buf, 652178784Skmacy int mr_access_flags, 653178784Skmacy u64 *iova_start) 654178784Skmacy{ 655178784Skmacy struct ib_mr *mr; 656178784Skmacy 657178784Skmacy mr = pd->device->reg_phys_mr(pd, phys_buf_array, num_phys_buf, 658178784Skmacy mr_access_flags, iova_start); 659178784Skmacy 660178784Skmacy if (!IS_ERR(mr)) { 661178784Skmacy mr->device = pd->device; 662178784Skmacy mr->pd = pd; 663178784Skmacy mr->uobject = NULL; 664178784Skmacy atomic_add_acq_int(&pd->usecnt, 1); 665178784Skmacy atomic_store_rel_int(&mr->usecnt, 0); 666178784Skmacy } 667178784Skmacy 668178784Skmacy return mr; 669178784Skmacy} 670178784Skmacy 671178784Skmacyint ib_rereg_phys_mr(struct ib_mr *mr, 672178784Skmacy int mr_rereg_mask, 673178784Skmacy struct ib_pd *pd, 674178784Skmacy struct ib_phys_buf *phys_buf_array, 675178784Skmacy int num_phys_buf, 676178784Skmacy int mr_access_flags, 677178784Skmacy u64 *iova_start) 678178784Skmacy{ 679178784Skmacy struct ib_pd *old_pd; 680178784Skmacy int ret; 681178784Skmacy 682178784Skmacy if (!mr->device->rereg_phys_mr) 683178784Skmacy return (ENOSYS); 684178784Skmacy 685178784Skmacy if (atomic_load_acq_int(&mr->usecnt)) 686178784Skmacy return (EBUSY); 687178784Skmacy 688178784Skmacy old_pd = mr->pd; 689178784Skmacy 690178784Skmacy ret = mr->device->rereg_phys_mr(mr, mr_rereg_mask, pd, 691178784Skmacy phys_buf_array, num_phys_buf, 692178784Skmacy mr_access_flags, iova_start); 693178784Skmacy 694178784Skmacy if (!ret && (mr_rereg_mask & IB_MR_REREG_PD)) { 695178784Skmacy atomic_subtract_acq_int(&old_pd->usecnt, 1); 696178784Skmacy atomic_add_acq_int(&pd->usecnt, 1); 697178784Skmacy } 698178784Skmacy 699178784Skmacy return ret; 700178784Skmacy} 701178784Skmacy 702178784Skmacyint ib_query_mr(struct ib_mr *mr, struct ib_mr_attr *mr_attr) 703178784Skmacy{ 704178784Skmacy return mr->device->query_mr ? 705178784Skmacy mr->device->query_mr(mr, mr_attr) : ENOSYS; 706178784Skmacy} 707178784Skmacy 708178784Skmacyint ib_dereg_mr(struct ib_mr *mr) 709178784Skmacy{ 710178784Skmacy struct ib_pd *pd; 711178784Skmacy int ret; 712178784Skmacy 713178784Skmacy if (atomic_load_acq_int(&mr->usecnt)) 714178784Skmacy return (EBUSY); 715178784Skmacy 716178784Skmacy pd = mr->pd; 717178784Skmacy ret = mr->device->dereg_mr(mr); 718178784Skmacy if (!ret) 719178784Skmacy atomic_subtract_acq_int(&pd->usecnt, 1); 720178784Skmacy 721178784Skmacy return ret; 722178784Skmacy} 723178784Skmacy 724178784Skmacy/* Memory windows */ 725178784Skmacy 726178784Skmacystruct ib_mw *ib_alloc_mw(struct ib_pd *pd) 727178784Skmacy{ 728178784Skmacy struct ib_mw *mw; 729178784Skmacy 730178784Skmacy if (!pd->device->alloc_mw) 731178784Skmacy return ERR_PTR(ENOSYS); 732178784Skmacy 733178784Skmacy mw = pd->device->alloc_mw(pd); 734178784Skmacy if (!IS_ERR(mw)) { 735178784Skmacy mw->device = pd->device; 736178784Skmacy mw->pd = pd; 737178784Skmacy mw->uobject = NULL; 738178784Skmacy atomic_add_acq_int(&pd->usecnt, 1); 739178784Skmacy } 740178784Skmacy 741178784Skmacy return mw; 742178784Skmacy} 743178784Skmacy 744178784Skmacyint ib_dealloc_mw(struct ib_mw *mw) 745178784Skmacy{ 746178784Skmacy struct ib_pd *pd; 747178784Skmacy int ret; 748178784Skmacy 749178784Skmacy pd = mw->pd; 750178784Skmacy ret = mw->device->dealloc_mw(mw); 751178784Skmacy if (!ret) 752178784Skmacy atomic_subtract_acq_int(&pd->usecnt, 1); 753178784Skmacy 754178784Skmacy return ret; 755178784Skmacy} 756178784Skmacy 757178784Skmacy/* "Fast" memory regions */ 758178784Skmacy 759178784Skmacystruct ib_fmr *ib_alloc_fmr(struct ib_pd *pd, 760178784Skmacy int mr_access_flags, 761178784Skmacy struct ib_fmr_attr *fmr_attr) 762178784Skmacy{ 763178784Skmacy struct ib_fmr *fmr; 764178784Skmacy 765178784Skmacy if (!pd->device->alloc_fmr) 766178784Skmacy return ERR_PTR(ENOSYS); 767178784Skmacy 768178784Skmacy fmr = pd->device->alloc_fmr(pd, mr_access_flags, fmr_attr); 769178784Skmacy if (!IS_ERR(fmr)) { 770178784Skmacy fmr->device = pd->device; 771178784Skmacy fmr->pd = pd; 772178784Skmacy atomic_add_acq_int(&pd->usecnt, 1); 773178784Skmacy } 774178784Skmacy 775178784Skmacy return fmr; 776178784Skmacy} 777178784Skmacy 778178784Skmacyint ib_unmap_fmr(struct ib_fmr_list_head *fmr_list) 779178784Skmacy{ 780178784Skmacy struct ib_fmr *fmr; 781178784Skmacy 782178784Skmacy if (TAILQ_EMPTY(fmr_list)) 783178784Skmacy return 0; 784178784Skmacy 785178784Skmacy fmr = TAILQ_FIRST(fmr_list); 786178784Skmacy return fmr->device->unmap_fmr(fmr_list); 787178784Skmacy} 788178784Skmacy 789178784Skmacyint ib_dealloc_fmr(struct ib_fmr *fmr) 790178784Skmacy{ 791178784Skmacy struct ib_pd *pd; 792178784Skmacy int ret; 793178784Skmacy 794178784Skmacy pd = fmr->pd; 795178784Skmacy ret = fmr->device->dealloc_fmr(fmr); 796178784Skmacy if (!ret) 797178784Skmacy atomic_subtract_acq_int(&pd->usecnt, 1); 798178784Skmacy 799178784Skmacy return ret; 800178784Skmacy} 801178784Skmacy 802178784Skmacy/* Multicast groups */ 803178784Skmacy 804178784Skmacyint ib_attach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid) 805178784Skmacy{ 806178784Skmacy if (!qp->device->attach_mcast) 807178784Skmacy return (ENOSYS); 808178784Skmacy if (gid->raw[0] != 0xff || qp->qp_type != IB_QPT_UD) 809178784Skmacy return (EINVAL); 810178784Skmacy 811178784Skmacy return qp->device->attach_mcast(qp, gid, lid); 812178784Skmacy} 813178784Skmacy 814178784Skmacyint ib_detach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid) 815178784Skmacy{ 816178784Skmacy if (!qp->device->detach_mcast) 817178784Skmacy return (ENOSYS); 818178784Skmacy if (gid->raw[0] != 0xff || qp->qp_type != IB_QPT_UD) 819178784Skmacy return (EINVAL); 820178784Skmacy 821178784Skmacy return qp->device->detach_mcast(qp, gid, lid); 822178784Skmacy} 823