1/* 2 * Copyright (c) 2004 Mellanox Technologies Ltd. All rights reserved. 3 * Copyright (c) 2004 Infinicon Corporation. All rights reserved. 4 * Copyright (c) 2004 Intel Corporation. All rights reserved. 5 * Copyright (c) 2004 Topspin Corporation. All rights reserved. 6 * Copyright (c) 2004 Voltaire Corporation. All rights reserved. 7 * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved. 8 * Copyright (c) 2005, 2006 Cisco Systems. All rights reserved. 9 * 10 * This software is available to you under a choice of one of two 11 * licenses. You may choose to be licensed under the terms of the GNU 12 * General Public License (GPL) Version 2, available from the file 13 * COPYING in the main directory of this source tree, or the 14 * OpenIB.org BSD license below: 15 * 16 * Redistribution and use in source and binary forms, with or 17 * without modification, are permitted provided that the following 18 * conditions are met: 19 * 20 * - Redistributions of source code must retain the above 21 * copyright notice, this list of conditions and the following 22 * disclaimer. 23 * 24 * - Redistributions in binary form must reproduce the above 25 * copyright notice, this list of conditions and the following 26 * disclaimer in the documentation and/or other materials 27 * provided with the distribution. 28 * 29 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 30 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 31 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 32 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 33 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 34 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 35 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 36 * SOFTWARE. 37 * 38 * $Id: verbs.c 1349 2004-12-16 21:09:43Z roland $ 39 */ 40 41#include <sys/cdefs.h> 42__FBSDID("$FreeBSD$"); 43 44#include <sys/param.h> 45#include <sys/systm.h> 46#include <sys/kernel.h> 47#include <sys/libkern.h> 48#include <sys/module.h> 49#include <sys/endian.h> 50 51#include <contrib/rdma/ib_verbs.h> 52#include <contrib/rdma/ib_cache.h> 53 54int ib_rate_to_mult(enum ib_rate rate) 55{ 56 switch (rate) { 57 case IB_RATE_2_5_GBPS: return 1; 58 case IB_RATE_5_GBPS: return 2; 59 case IB_RATE_10_GBPS: return 4; 60 case IB_RATE_20_GBPS: return 8; 61 case IB_RATE_30_GBPS: return 12; 62 case IB_RATE_40_GBPS: return 16; 63 case IB_RATE_60_GBPS: return 24; 64 case IB_RATE_80_GBPS: return 32; 65 case IB_RATE_120_GBPS: return 48; 66 default: return -1; 67 } 68} 69 70enum ib_rate mult_to_ib_rate(int mult) 71{ 72 switch (mult) { 73 case 1: return IB_RATE_2_5_GBPS; 74 case 2: return IB_RATE_5_GBPS; 75 case 4: return IB_RATE_10_GBPS; 76 case 8: return IB_RATE_20_GBPS; 77 case 12: return IB_RATE_30_GBPS; 78 case 16: return IB_RATE_40_GBPS; 79 case 24: return IB_RATE_60_GBPS; 80 case 32: return IB_RATE_80_GBPS; 81 case 48: return IB_RATE_120_GBPS; 82 default: return IB_RATE_PORT_CURRENT; 83 } 84} 85 86enum rdma_transport_type 87rdma_node_get_transport(enum rdma_node_type node_type) 88{ 89 switch (node_type) { 90 case RDMA_NODE_IB_CA: 91 case RDMA_NODE_IB_SWITCH: 92 case RDMA_NODE_IB_ROUTER: 93 return RDMA_TRANSPORT_IB; 94 case RDMA_NODE_RNIC: 95 return RDMA_TRANSPORT_IWARP; 96 default: 97 panic("bad condition"); 98 return 0; 99 } 100} 101 102/* Protection domains */ 103 104struct ib_pd *ib_alloc_pd(struct ib_device *device) 105{ 106 struct ib_pd *pd; 107 108 pd = device->alloc_pd(device, NULL, NULL); 109 110 if (!IS_ERR(pd)) { 111 pd->device = device; 112 pd->uobject = NULL; 113 atomic_store_rel_int(&pd->usecnt, 0); 114 } 115 116 return pd; 117} 118 119int ib_dealloc_pd(struct ib_pd *pd) 120{ 121 if (atomic_load_acq_int(&pd->usecnt)) 122 return (EBUSY); 123 124 return pd->device->dealloc_pd(pd); 125} 126 127/* Address handles */ 128 129struct ib_ah *ib_create_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr) 130{ 131 struct ib_ah *ah; 132 133 ah = pd->device->create_ah(pd, ah_attr); 134 135 if (!IS_ERR(ah)) { 136 ah->device = pd->device; 137 ah->pd = pd; 138 ah->uobject = NULL; 139 atomic_add_acq_int(&pd->usecnt, 1); 140 } 141 142 return ah; 143} 144 145int ib_init_ah_from_wc(struct ib_device *device, u8 port_num, struct ib_wc *wc, 146 struct ib_grh *grh, struct ib_ah_attr *ah_attr) 147{ 148 u32 flow_class; 149 u16 gid_index; 150 int ret; 151 152 memset(ah_attr, 0, sizeof *ah_attr); 153 ah_attr->dlid = wc->slid; 154 ah_attr->sl = wc->sl; 155 ah_attr->src_path_bits = wc->dlid_path_bits; 156 ah_attr->port_num = port_num; 157 158 if (wc->wc_flags & IB_WC_GRH) { 159 ah_attr->ah_flags = IB_AH_GRH; 160 ah_attr->grh.dgid = grh->sgid; 161 162 ret = ib_find_cached_gid(device, &grh->dgid, &port_num, 163 &gid_index); 164 if (ret) 165 return ret; 166 167 ah_attr->grh.sgid_index = (u8) gid_index; 168 flow_class = be32toh(grh->version_tclass_flow); 169 ah_attr->grh.flow_label = flow_class & 0xFFFFF; 170 ah_attr->grh.hop_limit = 0xFF; 171 ah_attr->grh.traffic_class = (flow_class >> 20) & 0xFF; 172 } 173 return 0; 174} 175 176struct ib_ah *ib_create_ah_from_wc(struct ib_pd *pd, struct ib_wc *wc, 177 struct ib_grh *grh, u8 port_num) 178{ 179 struct ib_ah_attr ah_attr; 180 int ret; 181 182 ret = ib_init_ah_from_wc(pd->device, port_num, wc, grh, &ah_attr); 183 if (ret) 184 return ERR_PTR(ret); 185 186 return ib_create_ah(pd, &ah_attr); 187} 188 189int ib_modify_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr) 190{ 191 return ah->device->modify_ah ? 192 ah->device->modify_ah(ah, ah_attr) : 193 ENOSYS; 194} 195 196int ib_query_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr) 197{ 198 return ah->device->query_ah ? 199 ah->device->query_ah(ah, ah_attr) : 200 ENOSYS; 201} 202 203int ib_destroy_ah(struct ib_ah *ah) 204{ 205 struct ib_pd *pd; 206 int ret; 207 208 pd = ah->pd; 209 ret = ah->device->destroy_ah(ah); 210 if (!ret) 211 atomic_subtract_acq_int(&pd->usecnt, 1); 212 213 return ret; 214} 215 216/* Shared receive queues */ 217 218struct ib_srq *ib_create_srq(struct ib_pd *pd, 219 struct ib_srq_init_attr *srq_init_attr) 220{ 221 struct ib_srq *srq; 222 223 if (!pd->device->create_srq) 224 return ERR_PTR(ENOSYS); 225 226 srq = pd->device->create_srq(pd, srq_init_attr, NULL); 227 228 if (!IS_ERR(srq)) { 229 srq->device = pd->device; 230 srq->pd = pd; 231 srq->uobject = NULL; 232 srq->event_handler = srq_init_attr->event_handler; 233 srq->srq_context = srq_init_attr->srq_context; 234 atomic_add_acq_int(&pd->usecnt, 1); 235 atomic_store_rel_int(&srq->usecnt, 0); 236 } 237 238 return srq; 239} 240 241int ib_modify_srq(struct ib_srq *srq, 242 struct ib_srq_attr *srq_attr, 243 enum ib_srq_attr_mask srq_attr_mask) 244{ 245 return srq->device->modify_srq(srq, srq_attr, srq_attr_mask, NULL); 246} 247 248int ib_query_srq(struct ib_srq *srq, 249 struct ib_srq_attr *srq_attr) 250{ 251 return srq->device->query_srq ? 252 srq->device->query_srq(srq, srq_attr) : ENOSYS; 253} 254 255int ib_destroy_srq(struct ib_srq *srq) 256{ 257 struct ib_pd *pd; 258 int ret; 259 260 if (atomic_load_acq_int(&srq->usecnt)) 261 return (EBUSY); 262 263 pd = srq->pd; 264 265 ret = srq->device->destroy_srq(srq); 266 if (!ret) 267 atomic_subtract_acq_int(&pd->usecnt, 1); 268 269 return ret; 270} 271 272/* Queue pairs */ 273 274struct ib_qp *ib_create_qp(struct ib_pd *pd, 275 struct ib_qp_init_attr *qp_init_attr) 276{ 277 struct ib_qp *qp; 278 279 qp = pd->device->create_qp(pd, qp_init_attr, NULL); 280 281 if (!IS_ERR(qp)) { 282 qp->device = pd->device; 283 qp->pd = pd; 284 qp->send_cq = qp_init_attr->send_cq; 285 qp->recv_cq = qp_init_attr->recv_cq; 286 qp->srq = qp_init_attr->srq; 287 qp->uobject = NULL; 288 qp->event_handler = qp_init_attr->event_handler; 289 qp->qp_context = qp_init_attr->qp_context; 290 qp->qp_type = qp_init_attr->qp_type; 291 atomic_add_acq_int(&pd->usecnt, 1); 292 atomic_add_acq_int(&qp_init_attr->send_cq->usecnt, 1); 293 atomic_add_acq_int(&qp_init_attr->recv_cq->usecnt, 1); 294 if (qp_init_attr->srq) 295 atomic_add_acq_int(&qp_init_attr->srq->usecnt, 1); 296 } 297 298 return qp; 299} 300 301static const struct { 302 int valid; 303 enum ib_qp_attr_mask req_param[IB_QPT_RAW_ETY + 1]; 304 enum ib_qp_attr_mask opt_param[IB_QPT_RAW_ETY + 1]; 305} qp_state_table[IB_QPS_ERR + 1][IB_QPS_ERR + 1] = { 306 [IB_QPS_RESET] = { 307 [IB_QPS_RESET] = { .valid = 1 }, 308 [IB_QPS_ERR] = { .valid = 1 }, 309 [IB_QPS_INIT] = { 310 .valid = 1, 311 .req_param = { 312 [IB_QPT_UD] = (IB_QP_PKEY_INDEX | 313 IB_QP_PORT | 314 IB_QP_QKEY), 315 [IB_QPT_UC] = (IB_QP_PKEY_INDEX | 316 IB_QP_PORT | 317 IB_QP_ACCESS_FLAGS), 318 [IB_QPT_RC] = (IB_QP_PKEY_INDEX | 319 IB_QP_PORT | 320 IB_QP_ACCESS_FLAGS), 321 [IB_QPT_SMI] = (IB_QP_PKEY_INDEX | 322 IB_QP_QKEY), 323 [IB_QPT_GSI] = (IB_QP_PKEY_INDEX | 324 IB_QP_QKEY), 325 } 326 }, 327 }, 328 [IB_QPS_INIT] = { 329 [IB_QPS_RESET] = { .valid = 1 }, 330 [IB_QPS_ERR] = { .valid = 1 }, 331 [IB_QPS_INIT] = { 332 .valid = 1, 333 .opt_param = { 334 [IB_QPT_UD] = (IB_QP_PKEY_INDEX | 335 IB_QP_PORT | 336 IB_QP_QKEY), 337 [IB_QPT_UC] = (IB_QP_PKEY_INDEX | 338 IB_QP_PORT | 339 IB_QP_ACCESS_FLAGS), 340 [IB_QPT_RC] = (IB_QP_PKEY_INDEX | 341 IB_QP_PORT | 342 IB_QP_ACCESS_FLAGS), 343 [IB_QPT_SMI] = (IB_QP_PKEY_INDEX | 344 IB_QP_QKEY), 345 [IB_QPT_GSI] = (IB_QP_PKEY_INDEX | 346 IB_QP_QKEY), 347 } 348 }, 349 [IB_QPS_RTR] = { 350 .valid = 1, 351 .req_param = { 352 [IB_QPT_UC] = (IB_QP_AV | 353 IB_QP_PATH_MTU | 354 IB_QP_DEST_QPN | 355 IB_QP_RQ_PSN), 356 [IB_QPT_RC] = (IB_QP_AV | 357 IB_QP_PATH_MTU | 358 IB_QP_DEST_QPN | 359 IB_QP_RQ_PSN | 360 IB_QP_MAX_DEST_RD_ATOMIC | 361 IB_QP_MIN_RNR_TIMER), 362 }, 363 .opt_param = { 364 [IB_QPT_UD] = (IB_QP_PKEY_INDEX | 365 IB_QP_QKEY), 366 [IB_QPT_UC] = (IB_QP_ALT_PATH | 367 IB_QP_ACCESS_FLAGS | 368 IB_QP_PKEY_INDEX), 369 [IB_QPT_RC] = (IB_QP_ALT_PATH | 370 IB_QP_ACCESS_FLAGS | 371 IB_QP_PKEY_INDEX), 372 [IB_QPT_SMI] = (IB_QP_PKEY_INDEX | 373 IB_QP_QKEY), 374 [IB_QPT_GSI] = (IB_QP_PKEY_INDEX | 375 IB_QP_QKEY), 376 } 377 } 378 }, 379 [IB_QPS_RTR] = { 380 [IB_QPS_RESET] = { .valid = 1 }, 381 [IB_QPS_ERR] = { .valid = 1 }, 382 [IB_QPS_RTS] = { 383 .valid = 1, 384 .req_param = { 385 [IB_QPT_UD] = IB_QP_SQ_PSN, 386 [IB_QPT_UC] = IB_QP_SQ_PSN, 387 [IB_QPT_RC] = (IB_QP_TIMEOUT | 388 IB_QP_RETRY_CNT | 389 IB_QP_RNR_RETRY | 390 IB_QP_SQ_PSN | 391 IB_QP_MAX_QP_RD_ATOMIC), 392 [IB_QPT_SMI] = IB_QP_SQ_PSN, 393 [IB_QPT_GSI] = IB_QP_SQ_PSN, 394 }, 395 .opt_param = { 396 [IB_QPT_UD] = (IB_QP_CUR_STATE | 397 IB_QP_QKEY), 398 [IB_QPT_UC] = (IB_QP_CUR_STATE | 399 IB_QP_ALT_PATH | 400 IB_QP_ACCESS_FLAGS | 401 IB_QP_PATH_MIG_STATE), 402 [IB_QPT_RC] = (IB_QP_CUR_STATE | 403 IB_QP_ALT_PATH | 404 IB_QP_ACCESS_FLAGS | 405 IB_QP_MIN_RNR_TIMER | 406 IB_QP_PATH_MIG_STATE), 407 [IB_QPT_SMI] = (IB_QP_CUR_STATE | 408 IB_QP_QKEY), 409 [IB_QPT_GSI] = (IB_QP_CUR_STATE | 410 IB_QP_QKEY), 411 } 412 } 413 }, 414 [IB_QPS_RTS] = { 415 [IB_QPS_RESET] = { .valid = 1 }, 416 [IB_QPS_ERR] = { .valid = 1 }, 417 [IB_QPS_RTS] = { 418 .valid = 1, 419 .opt_param = { 420 [IB_QPT_UD] = (IB_QP_CUR_STATE | 421 IB_QP_QKEY), 422 [IB_QPT_UC] = (IB_QP_CUR_STATE | 423 IB_QP_ACCESS_FLAGS | 424 IB_QP_ALT_PATH | 425 IB_QP_PATH_MIG_STATE), 426 [IB_QPT_RC] = (IB_QP_CUR_STATE | 427 IB_QP_ACCESS_FLAGS | 428 IB_QP_ALT_PATH | 429 IB_QP_PATH_MIG_STATE | 430 IB_QP_MIN_RNR_TIMER), 431 [IB_QPT_SMI] = (IB_QP_CUR_STATE | 432 IB_QP_QKEY), 433 [IB_QPT_GSI] = (IB_QP_CUR_STATE | 434 IB_QP_QKEY), 435 } 436 }, 437 [IB_QPS_SQD] = { 438 .valid = 1, 439 .opt_param = { 440 [IB_QPT_UD] = IB_QP_EN_SQD_ASYNC_NOTIFY, 441 [IB_QPT_UC] = IB_QP_EN_SQD_ASYNC_NOTIFY, 442 [IB_QPT_RC] = IB_QP_EN_SQD_ASYNC_NOTIFY, 443 [IB_QPT_SMI] = IB_QP_EN_SQD_ASYNC_NOTIFY, 444 [IB_QPT_GSI] = IB_QP_EN_SQD_ASYNC_NOTIFY 445 } 446 }, 447 }, 448 [IB_QPS_SQD] = { 449 [IB_QPS_RESET] = { .valid = 1 }, 450 [IB_QPS_ERR] = { .valid = 1 }, 451 [IB_QPS_RTS] = { 452 .valid = 1, 453 .opt_param = { 454 [IB_QPT_UD] = (IB_QP_CUR_STATE | 455 IB_QP_QKEY), 456 [IB_QPT_UC] = (IB_QP_CUR_STATE | 457 IB_QP_ALT_PATH | 458 IB_QP_ACCESS_FLAGS | 459 IB_QP_PATH_MIG_STATE), 460 [IB_QPT_RC] = (IB_QP_CUR_STATE | 461 IB_QP_ALT_PATH | 462 IB_QP_ACCESS_FLAGS | 463 IB_QP_MIN_RNR_TIMER | 464 IB_QP_PATH_MIG_STATE), 465 [IB_QPT_SMI] = (IB_QP_CUR_STATE | 466 IB_QP_QKEY), 467 [IB_QPT_GSI] = (IB_QP_CUR_STATE | 468 IB_QP_QKEY), 469 } 470 }, 471 [IB_QPS_SQD] = { 472 .valid = 1, 473 .opt_param = { 474 [IB_QPT_UD] = (IB_QP_PKEY_INDEX | 475 IB_QP_QKEY), 476 [IB_QPT_UC] = (IB_QP_AV | 477 IB_QP_ALT_PATH | 478 IB_QP_ACCESS_FLAGS | 479 IB_QP_PKEY_INDEX | 480 IB_QP_PATH_MIG_STATE), 481 [IB_QPT_RC] = (IB_QP_PORT | 482 IB_QP_AV | 483 IB_QP_TIMEOUT | 484 IB_QP_RETRY_CNT | 485 IB_QP_RNR_RETRY | 486 IB_QP_MAX_QP_RD_ATOMIC | 487 IB_QP_MAX_DEST_RD_ATOMIC | 488 IB_QP_ALT_PATH | 489 IB_QP_ACCESS_FLAGS | 490 IB_QP_PKEY_INDEX | 491 IB_QP_MIN_RNR_TIMER | 492 IB_QP_PATH_MIG_STATE), 493 [IB_QPT_SMI] = (IB_QP_PKEY_INDEX | 494 IB_QP_QKEY), 495 [IB_QPT_GSI] = (IB_QP_PKEY_INDEX | 496 IB_QP_QKEY), 497 } 498 } 499 }, 500 [IB_QPS_SQE] = { 501 [IB_QPS_RESET] = { .valid = 1 }, 502 [IB_QPS_ERR] = { .valid = 1 }, 503 [IB_QPS_RTS] = { 504 .valid = 1, 505 .opt_param = { 506 [IB_QPT_UD] = (IB_QP_CUR_STATE | 507 IB_QP_QKEY), 508 [IB_QPT_UC] = (IB_QP_CUR_STATE | 509 IB_QP_ACCESS_FLAGS), 510 [IB_QPT_SMI] = (IB_QP_CUR_STATE | 511 IB_QP_QKEY), 512 [IB_QPT_GSI] = (IB_QP_CUR_STATE | 513 IB_QP_QKEY), 514 } 515 } 516 }, 517 [IB_QPS_ERR] = { 518 [IB_QPS_RESET] = { .valid = 1 }, 519 [IB_QPS_ERR] = { .valid = 1 } 520 } 521}; 522 523int ib_modify_qp_is_ok(enum ib_qp_state cur_state, enum ib_qp_state next_state, 524 enum ib_qp_type type, enum ib_qp_attr_mask mask) 525{ 526 enum ib_qp_attr_mask req_param, opt_param; 527 528 if (cur_state < 0 || cur_state > IB_QPS_ERR || 529 next_state < 0 || next_state > IB_QPS_ERR) 530 return 0; 531 532 if (mask & IB_QP_CUR_STATE && 533 cur_state != IB_QPS_RTR && cur_state != IB_QPS_RTS && 534 cur_state != IB_QPS_SQD && cur_state != IB_QPS_SQE) 535 return 0; 536 537 if (!qp_state_table[cur_state][next_state].valid) 538 return 0; 539 540 req_param = qp_state_table[cur_state][next_state].req_param[type]; 541 opt_param = qp_state_table[cur_state][next_state].opt_param[type]; 542 543 if ((mask & req_param) != req_param) 544 return 0; 545 546 if (mask & ~(req_param | opt_param | IB_QP_STATE)) 547 return 0; 548 549 return 1; 550} 551 552int ib_modify_qp(struct ib_qp *qp, 553 struct ib_qp_attr *qp_attr, 554 int qp_attr_mask) 555{ 556 return qp->device->modify_qp(qp, qp_attr, qp_attr_mask, NULL); 557} 558 559int ib_query_qp(struct ib_qp *qp, 560 struct ib_qp_attr *qp_attr, 561 int qp_attr_mask, 562 struct ib_qp_init_attr *qp_init_attr) 563{ 564 return qp->device->query_qp ? 565 qp->device->query_qp(qp, qp_attr, qp_attr_mask, qp_init_attr) : 566 ENOSYS; 567} 568 569int ib_destroy_qp(struct ib_qp *qp) 570{ 571 struct ib_pd *pd; 572 struct ib_cq *scq, *rcq; 573 struct ib_srq *srq; 574 int ret; 575 576 pd = qp->pd; 577 scq = qp->send_cq; 578 rcq = qp->recv_cq; 579 srq = qp->srq; 580 581 ret = qp->device->destroy_qp(qp); 582 if (!ret) { 583 atomic_subtract_acq_int(&pd->usecnt, 1); 584 atomic_subtract_acq_int(&scq->usecnt, 1); 585 atomic_subtract_acq_int(&rcq->usecnt, 1); 586 if (srq) 587 atomic_subtract_acq_int(&srq->usecnt, 1); 588 } 589 590 return ret; 591} 592 593/* Completion queues */ 594 595struct ib_cq *ib_create_cq(struct ib_device *device, 596 ib_comp_handler comp_handler, 597 void (*event_handler)(struct ib_event *, void *), 598 void *cq_context, int cqe, int comp_vector) 599{ 600 struct ib_cq *cq; 601 602 cq = device->create_cq(device, cqe, comp_vector, NULL, NULL); 603 604 if (!IS_ERR(cq)) { 605 cq->device = device; 606 cq->uobject = NULL; 607 cq->comp_handler = comp_handler; 608 cq->event_handler = event_handler; 609 cq->cq_context = cq_context; 610 atomic_store_rel_int(&cq->usecnt, 0); 611 } 612 613 return cq; 614} 615 616int ib_destroy_cq(struct ib_cq *cq) 617{ 618 if (atomic_load_acq_int(&cq->usecnt)) 619 return (EBUSY); 620 621 return cq->device->destroy_cq(cq); 622} 623 624int ib_resize_cq(struct ib_cq *cq, int cqe) 625{ 626 return cq->device->resize_cq ? 627 cq->device->resize_cq(cq, cqe, NULL) : ENOSYS; 628} 629 630/* Memory regions */ 631 632struct ib_mr *ib_get_dma_mr(struct ib_pd *pd, int mr_access_flags) 633{ 634 struct ib_mr *mr; 635 636 mr = pd->device->get_dma_mr(pd, mr_access_flags); 637 638 if (!IS_ERR(mr)) { 639 mr->device = pd->device; 640 mr->pd = pd; 641 mr->uobject = NULL; 642 atomic_add_acq_int(&pd->usecnt, 1); 643 atomic_store_rel_int(&mr->usecnt, 0); 644 } 645 646 return mr; 647} 648 649struct ib_mr *ib_reg_phys_mr(struct ib_pd *pd, 650 struct ib_phys_buf *phys_buf_array, 651 int num_phys_buf, 652 int mr_access_flags, 653 u64 *iova_start) 654{ 655 struct ib_mr *mr; 656 657 mr = pd->device->reg_phys_mr(pd, phys_buf_array, num_phys_buf, 658 mr_access_flags, iova_start); 659 660 if (!IS_ERR(mr)) { 661 mr->device = pd->device; 662 mr->pd = pd; 663 mr->uobject = NULL; 664 atomic_add_acq_int(&pd->usecnt, 1); 665 atomic_store_rel_int(&mr->usecnt, 0); 666 } 667 668 return mr; 669} 670 671int ib_rereg_phys_mr(struct ib_mr *mr, 672 int mr_rereg_mask, 673 struct ib_pd *pd, 674 struct ib_phys_buf *phys_buf_array, 675 int num_phys_buf, 676 int mr_access_flags, 677 u64 *iova_start) 678{ 679 struct ib_pd *old_pd; 680 int ret; 681 682 if (!mr->device->rereg_phys_mr) 683 return (ENOSYS); 684 685 if (atomic_load_acq_int(&mr->usecnt)) 686 return (EBUSY); 687 688 old_pd = mr->pd; 689 690 ret = mr->device->rereg_phys_mr(mr, mr_rereg_mask, pd, 691 phys_buf_array, num_phys_buf, 692 mr_access_flags, iova_start); 693 694 if (!ret && (mr_rereg_mask & IB_MR_REREG_PD)) { 695 atomic_subtract_acq_int(&old_pd->usecnt, 1); 696 atomic_add_acq_int(&pd->usecnt, 1); 697 } 698 699 return ret; 700} 701 702int ib_query_mr(struct ib_mr *mr, struct ib_mr_attr *mr_attr) 703{ 704 return mr->device->query_mr ? 705 mr->device->query_mr(mr, mr_attr) : ENOSYS; 706} 707 708int ib_dereg_mr(struct ib_mr *mr) 709{ 710 struct ib_pd *pd; 711 int ret; 712 713 if (atomic_load_acq_int(&mr->usecnt)) 714 return (EBUSY); 715 716 pd = mr->pd; 717 ret = mr->device->dereg_mr(mr); 718 if (!ret) 719 atomic_subtract_acq_int(&pd->usecnt, 1); 720 721 return ret; 722} 723 724/* Memory windows */ 725 726struct ib_mw *ib_alloc_mw(struct ib_pd *pd) 727{ 728 struct ib_mw *mw; 729 730 if (!pd->device->alloc_mw) 731 return ERR_PTR(ENOSYS); 732 733 mw = pd->device->alloc_mw(pd); 734 if (!IS_ERR(mw)) { 735 mw->device = pd->device; 736 mw->pd = pd; 737 mw->uobject = NULL; 738 atomic_add_acq_int(&pd->usecnt, 1); 739 } 740 741 return mw; 742} 743 744int ib_dealloc_mw(struct ib_mw *mw) 745{ 746 struct ib_pd *pd; 747 int ret; 748 749 pd = mw->pd; 750 ret = mw->device->dealloc_mw(mw); 751 if (!ret) 752 atomic_subtract_acq_int(&pd->usecnt, 1); 753 754 return ret; 755} 756 757/* "Fast" memory regions */ 758 759struct ib_fmr *ib_alloc_fmr(struct ib_pd *pd, 760 int mr_access_flags, 761 struct ib_fmr_attr *fmr_attr) 762{ 763 struct ib_fmr *fmr; 764 765 if (!pd->device->alloc_fmr) 766 return ERR_PTR(ENOSYS); 767 768 fmr = pd->device->alloc_fmr(pd, mr_access_flags, fmr_attr); 769 if (!IS_ERR(fmr)) { 770 fmr->device = pd->device; 771 fmr->pd = pd; 772 atomic_add_acq_int(&pd->usecnt, 1); 773 } 774 775 return fmr; 776} 777 778int ib_unmap_fmr(struct ib_fmr_list_head *fmr_list) 779{ 780 struct ib_fmr *fmr; 781 782 if (TAILQ_EMPTY(fmr_list)) 783 return 0; 784 785 fmr = TAILQ_FIRST(fmr_list); 786 return fmr->device->unmap_fmr(fmr_list); 787} 788 789int ib_dealloc_fmr(struct ib_fmr *fmr) 790{ 791 struct ib_pd *pd; 792 int ret; 793 794 pd = fmr->pd; 795 ret = fmr->device->dealloc_fmr(fmr); 796 if (!ret) 797 atomic_subtract_acq_int(&pd->usecnt, 1); 798 799 return ret; 800} 801 802/* Multicast groups */ 803 804int ib_attach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid) 805{ 806 if (!qp->device->attach_mcast) 807 return (ENOSYS); 808 if (gid->raw[0] != 0xff || qp->qp_type != IB_QPT_UD) 809 return (EINVAL); 810 811 return qp->device->attach_mcast(qp, gid, lid); 812} 813 814int ib_detach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid) 815{ 816 if (!qp->device->detach_mcast) 817 return (ENOSYS); 818 if (gid->raw[0] != 0xff || qp->qp_type != IB_QPT_UD) 819 return (EINVAL); 820 821 return qp->device->detach_mcast(qp, gid, lid); 822} 823