uverbs_cmd.c revision 324685
1/* 2 * Copyright (c) 2005 Topspin Communications. All rights reserved. 3 * Copyright (c) 2005, 2006, 2007 Cisco Systems. All rights reserved. 4 * Copyright (c) 2005 PathScale, Inc. All rights reserved. 5 * Copyright (c) 2006 Mellanox Technologies. All rights reserved. 6 * 7 * This software is available to you under a choice of one of two 8 * licenses. You may choose to be licensed under the terms of the GNU 9 * General Public License (GPL) Version 2, available from the file 10 * COPYING in the main directory of this source tree, or the 11 * OpenIB.org BSD license below: 12 * 13 * Redistribution and use in source and binary forms, with or 14 * without modification, are permitted provided that the following 15 * conditions are met: 16 * 17 * - Redistributions of source code must retain the above 18 * copyright notice, this list of conditions and the following 19 * disclaimer. 20 * 21 * - Redistributions in binary form must reproduce the above 22 * copyright notice, this list of conditions and the following 23 * disclaimer in the documentation and/or other materials 24 * provided with the distribution. 25 * 26 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 27 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 28 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 29 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 30 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 31 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 32 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 33 * SOFTWARE. 34 */ 35 36#define LINUXKPI_PARAM_PREFIX ibcore_ 37 38#include <linux/file.h> 39#include <linux/fs.h> 40#include <linux/lockdep.h> 41 42#include <asm/uaccess.h> 43#include <asm/fcntl.h> 44 45#include "uverbs.h" 46 47static struct lock_class_key pd_lock_key; 48static struct lock_class_key mr_lock_key; 49static struct lock_class_key cq_lock_key; 50static struct lock_class_key qp_lock_key; 51static struct lock_class_key ah_lock_key; 52static struct lock_class_key srq_lock_key; 53 54#define INIT_UDATA(udata, ibuf, obuf, ilen, olen) \ 55 do { \ 56 (udata)->inbuf = (void __user *) (ibuf); \ 57 (udata)->outbuf = (void __user *) (obuf); \ 58 (udata)->inlen = (ilen); \ 59 (udata)->outlen = (olen); \ 60 } while (0) 61 62/* 63 * The ib_uobject locking scheme is as follows: 64 * 65 * - ib_uverbs_idr_lock protects the uverbs idrs themselves, so it 66 * needs to be held during all idr operations. When an object is 67 * looked up, a reference must be taken on the object's kref before 68 * dropping this lock. 69 * 70 * - Each object also has an rwsem. This rwsem must be held for 71 * reading while an operation that uses the object is performed. 72 * For example, while registering an MR, the associated PD's 73 * uobject.mutex must be held for reading. The rwsem must be held 74 * for writing while initializing or destroying an object. 75 * 76 * - In addition, each object has a "live" flag. If this flag is not 77 * set, then lookups of the object will fail even if it is found in 78 * the idr. This handles a reader that blocks and does not acquire 79 * the rwsem until after the object is destroyed. The destroy 80 * operation will set the live flag to 0 and then drop the rwsem; 81 * this will allow the reader to acquire the rwsem, see that the 82 * live flag is 0, and then drop the rwsem and its reference to 83 * object. The underlying storage will not be freed until the last 84 * reference to the object is dropped. 85 */ 86 87static void init_uobj(struct ib_uobject *uobj, u64 user_handle, 88 struct ib_ucontext *context, struct lock_class_key *key) 89{ 90 uobj->user_handle = user_handle; 91 uobj->context = context; 92 kref_init(&uobj->ref); 93 init_rwsem(&uobj->mutex); 94 lockdep_set_class(&uobj->mutex, key); 95 uobj->live = 0; 96} 97 98static void release_uobj(struct kref *kref) 99{ 100 kfree(container_of(kref, struct ib_uobject, ref)); 101} 102 103static void put_uobj(struct ib_uobject *uobj) 104{ 105 kref_put(&uobj->ref, release_uobj); 106} 107 108static void put_uobj_read(struct ib_uobject *uobj) 109{ 110 up_read(&uobj->mutex); 111 put_uobj(uobj); 112} 113 114static void put_uobj_write(struct ib_uobject *uobj) 115{ 116 up_write(&uobj->mutex); 117 put_uobj(uobj); 118} 119 120static int idr_add_uobj(struct idr *idr, struct ib_uobject *uobj) 121{ 122 int ret; 123 124retry: 125 if (!idr_pre_get(idr, GFP_KERNEL)) 126 return -ENOMEM; 127 128 spin_lock(&ib_uverbs_idr_lock); 129 ret = idr_get_new(idr, uobj, &uobj->id); 130 spin_unlock(&ib_uverbs_idr_lock); 131 132 if (ret == -EAGAIN) 133 goto retry; 134 135 return ret; 136} 137 138void idr_remove_uobj(struct idr *idr, struct ib_uobject *uobj) 139{ 140 spin_lock(&ib_uverbs_idr_lock); 141 idr_remove(idr, uobj->id); 142 spin_unlock(&ib_uverbs_idr_lock); 143} 144 145static struct ib_uobject *__idr_get_uobj(struct idr *idr, int id, 146 struct ib_ucontext *context) 147{ 148 struct ib_uobject *uobj; 149 150 spin_lock(&ib_uverbs_idr_lock); 151 uobj = idr_find(idr, id); 152 if (uobj) { 153 if (uobj->context == context) 154 kref_get(&uobj->ref); 155 else 156 uobj = NULL; 157 } 158 spin_unlock(&ib_uverbs_idr_lock); 159 160 return uobj; 161} 162 163static struct ib_uobject *idr_read_uobj(struct idr *idr, int id, 164 struct ib_ucontext *context, int nested) 165{ 166 struct ib_uobject *uobj; 167 168 uobj = __idr_get_uobj(idr, id, context); 169 if (!uobj) 170 return NULL; 171 172 if (nested) 173 down_read_nested(&uobj->mutex, SINGLE_DEPTH_NESTING); 174 else 175 down_read(&uobj->mutex); 176 if (!uobj->live) { 177 put_uobj_read(uobj); 178 return NULL; 179 } 180 181 return uobj; 182} 183 184static struct ib_uobject *idr_write_uobj(struct idr *idr, int id, 185 struct ib_ucontext *context) 186{ 187 struct ib_uobject *uobj; 188 189 uobj = __idr_get_uobj(idr, id, context); 190 if (!uobj) 191 return NULL; 192 193 down_write(&uobj->mutex); 194 if (!uobj->live) { 195 put_uobj_write(uobj); 196 return NULL; 197 } 198 199 return uobj; 200} 201 202static void *idr_read_obj(struct idr *idr, int id, struct ib_ucontext *context, 203 int nested) 204{ 205 struct ib_uobject *uobj; 206 207 uobj = idr_read_uobj(idr, id, context, nested); 208 return uobj ? uobj->object : NULL; 209} 210 211static struct ib_pd *idr_read_pd(int pd_handle, struct ib_ucontext *context) 212{ 213 return idr_read_obj(&ib_uverbs_pd_idr, pd_handle, context, 0); 214} 215 216static void put_pd_read(struct ib_pd *pd) 217{ 218 put_uobj_read(pd->uobject); 219} 220 221static struct ib_cq *idr_read_cq(int cq_handle, struct ib_ucontext *context, int nested) 222{ 223 return idr_read_obj(&ib_uverbs_cq_idr, cq_handle, context, nested); 224} 225 226static void put_cq_read(struct ib_cq *cq) 227{ 228 put_uobj_read(cq->uobject); 229} 230 231static struct ib_ah *idr_read_ah(int ah_handle, struct ib_ucontext *context) 232{ 233 return idr_read_obj(&ib_uverbs_ah_idr, ah_handle, context, 0); 234} 235 236static void put_ah_read(struct ib_ah *ah) 237{ 238 put_uobj_read(ah->uobject); 239} 240 241static struct ib_qp *idr_read_qp(int qp_handle, struct ib_ucontext *context) 242{ 243 return idr_read_obj(&ib_uverbs_qp_idr, qp_handle, context, 0); 244} 245 246static void put_qp_read(struct ib_qp *qp) 247{ 248 put_uobj_read(qp->uobject); 249} 250 251static struct ib_srq *idr_read_srq(int srq_handle, struct ib_ucontext *context) 252{ 253 return idr_read_obj(&ib_uverbs_srq_idr, srq_handle, context, 0); 254} 255 256static void put_srq_read(struct ib_srq *srq) 257{ 258 put_uobj_read(srq->uobject); 259} 260 261static struct ib_xrcd *idr_read_xrcd(int xrcd_handle, 262 struct ib_ucontext *context, 263 struct ib_uobject **uobj) 264{ 265 *uobj = idr_read_uobj(&ib_uverbs_xrc_domain_idr, xrcd_handle, 266 context, 0); 267 return *uobj ? (*uobj)->object : NULL; 268} 269 270static void put_xrcd_read(struct ib_uobject *uobj) 271{ 272 put_uobj_read(uobj); 273} 274 275ssize_t ib_uverbs_get_context(struct ib_uverbs_file *file, 276 const char __user *buf, 277 int in_len, int out_len) 278{ 279 struct ib_uverbs_get_context cmd; 280 struct ib_uverbs_get_context_resp resp; 281 struct ib_udata udata; 282 struct ib_device *ibdev = file->device->ib_dev; 283 struct ib_ucontext *ucontext; 284 struct file *filp; 285 int ret; 286 287 if (out_len < sizeof resp) 288 return -ENOSPC; 289 290 if (copy_from_user(&cmd, buf, sizeof cmd)) 291 return -EFAULT; 292 293 mutex_lock(&file->mutex); 294 295 if (file->ucontext) { 296 ret = -EINVAL; 297 goto err; 298 } 299 300 INIT_UDATA(&udata, buf + sizeof cmd, 301 (unsigned long) cmd.response + sizeof resp, 302 in_len - sizeof cmd, out_len - sizeof resp); 303 304 ucontext = ibdev->alloc_ucontext(ibdev, &udata); 305 if (IS_ERR(ucontext)) { 306 ret = PTR_ERR(file->ucontext); 307 goto err; 308 } 309 310 ucontext->device = ibdev; 311 INIT_LIST_HEAD(&ucontext->pd_list); 312 INIT_LIST_HEAD(&ucontext->mr_list); 313 INIT_LIST_HEAD(&ucontext->mw_list); 314 INIT_LIST_HEAD(&ucontext->cq_list); 315 INIT_LIST_HEAD(&ucontext->qp_list); 316 INIT_LIST_HEAD(&ucontext->srq_list); 317 INIT_LIST_HEAD(&ucontext->ah_list); 318 INIT_LIST_HEAD(&ucontext->xrcd_list); 319 ucontext->closing = 0; 320 321 resp.num_comp_vectors = file->device->num_comp_vectors; 322 323 filp = ib_uverbs_alloc_event_file(file, 1, &resp.async_fd); 324 if (IS_ERR(filp)) { 325 ret = PTR_ERR(filp); 326 goto err_free; 327 } 328 329 if (copy_to_user((void __user *) (unsigned long) cmd.response, 330 &resp, sizeof resp)) { 331 ret = -EFAULT; 332 goto err_file; 333 } 334 335 file->async_file = filp->private_data; 336 337 INIT_IB_EVENT_HANDLER(&file->event_handler, file->device->ib_dev, 338 ib_uverbs_event_handler); 339 ret = ib_register_event_handler(&file->event_handler); 340 if (ret) 341 goto err_file; 342 343 kref_get(&file->async_file->ref); 344 kref_get(&file->ref); 345 file->ucontext = ucontext; 346 347 fd_install(resp.async_fd, filp); 348 349 mutex_unlock(&file->mutex); 350 351 return in_len; 352 353err_file: 354 put_unused_fd(resp.async_fd); 355 fput(filp); 356 357err_free: 358 ibdev->dealloc_ucontext(ucontext); 359 360err: 361 mutex_unlock(&file->mutex); 362 return ret; 363} 364 365ssize_t ib_uverbs_query_device(struct ib_uverbs_file *file, 366 const char __user *buf, 367 int in_len, int out_len) 368{ 369 struct ib_uverbs_query_device cmd; 370 struct ib_uverbs_query_device_resp resp; 371 struct ib_device_attr attr; 372 int ret; 373 374 if (out_len < sizeof resp) 375 return -ENOSPC; 376 377 if (copy_from_user(&cmd, buf, sizeof cmd)) 378 return -EFAULT; 379 380 ret = ib_query_device(file->device->ib_dev, &attr); 381 if (ret) 382 return ret; 383 384 memset(&resp, 0, sizeof resp); 385 386 resp.fw_ver = attr.fw_ver; 387 resp.node_guid = file->device->ib_dev->node_guid; 388 resp.sys_image_guid = attr.sys_image_guid; 389 resp.max_mr_size = attr.max_mr_size; 390 resp.page_size_cap = attr.page_size_cap; 391 resp.vendor_id = attr.vendor_id; 392 resp.vendor_part_id = attr.vendor_part_id; 393 resp.hw_ver = attr.hw_ver; 394 resp.max_qp = attr.max_qp; 395 resp.max_qp_wr = attr.max_qp_wr; 396 resp.device_cap_flags = attr.device_cap_flags; 397 resp.max_sge = attr.max_sge; 398 resp.max_sge_rd = attr.max_sge_rd; 399 resp.max_cq = attr.max_cq; 400 resp.max_cqe = attr.max_cqe; 401 resp.max_mr = attr.max_mr; 402 resp.max_pd = attr.max_pd; 403 resp.max_qp_rd_atom = attr.max_qp_rd_atom; 404 resp.max_ee_rd_atom = attr.max_ee_rd_atom; 405 resp.max_res_rd_atom = attr.max_res_rd_atom; 406 resp.max_qp_init_rd_atom = attr.max_qp_init_rd_atom; 407 resp.max_ee_init_rd_atom = attr.max_ee_init_rd_atom; 408 resp.atomic_cap = attr.atomic_cap; 409 resp.max_ee = attr.max_ee; 410 resp.max_rdd = attr.max_rdd; 411 resp.max_mw = attr.max_mw; 412 resp.max_raw_ipv6_qp = attr.max_raw_ipv6_qp; 413 resp.max_raw_ethy_qp = attr.max_raw_ethy_qp; 414 resp.max_mcast_grp = attr.max_mcast_grp; 415 resp.max_mcast_qp_attach = attr.max_mcast_qp_attach; 416 resp.max_total_mcast_qp_attach = attr.max_total_mcast_qp_attach; 417 resp.max_ah = attr.max_ah; 418 resp.max_fmr = attr.max_fmr; 419 resp.max_map_per_fmr = attr.max_map_per_fmr; 420 resp.max_srq = attr.max_srq; 421 resp.max_srq_wr = attr.max_srq_wr; 422 resp.max_srq_sge = attr.max_srq_sge; 423 resp.max_pkeys = attr.max_pkeys; 424 resp.local_ca_ack_delay = attr.local_ca_ack_delay; 425 resp.phys_port_cnt = file->device->ib_dev->phys_port_cnt; 426 427 if (copy_to_user((void __user *) (unsigned long) cmd.response, 428 &resp, sizeof resp)) 429 return -EFAULT; 430 431 return in_len; 432} 433 434ssize_t ib_uverbs_query_port(struct ib_uverbs_file *file, 435 const char __user *buf, 436 int in_len, int out_len) 437{ 438 struct ib_uverbs_query_port cmd; 439 struct ib_uverbs_query_port_resp resp; 440 struct ib_port_attr attr; 441 int ret; 442 443 if (out_len < sizeof resp) 444 return -ENOSPC; 445 446 if (copy_from_user(&cmd, buf, sizeof cmd)) 447 return -EFAULT; 448 449 ret = ib_query_port(file->device->ib_dev, cmd.port_num, &attr); 450 if (ret) 451 return ret; 452 453 memset(&resp, 0, sizeof resp); 454 455 resp.state = attr.state; 456 resp.max_mtu = attr.max_mtu; 457 resp.active_mtu = attr.active_mtu; 458 resp.gid_tbl_len = attr.gid_tbl_len; 459 resp.port_cap_flags = attr.port_cap_flags; 460 resp.max_msg_sz = attr.max_msg_sz; 461 resp.bad_pkey_cntr = attr.bad_pkey_cntr; 462 resp.qkey_viol_cntr = attr.qkey_viol_cntr; 463 resp.pkey_tbl_len = attr.pkey_tbl_len; 464 resp.lid = attr.lid; 465 resp.sm_lid = attr.sm_lid; 466 resp.lmc = attr.lmc; 467 resp.max_vl_num = attr.max_vl_num; 468 resp.sm_sl = attr.sm_sl; 469 resp.subnet_timeout = attr.subnet_timeout; 470 resp.init_type_reply = attr.init_type_reply; 471 resp.active_width = attr.active_width; 472 resp.active_speed = attr.active_speed; 473 resp.phys_state = attr.phys_state; 474 resp.link_layer = attr.link_layer; 475 476 if (copy_to_user((void __user *) (unsigned long) cmd.response, 477 &resp, sizeof resp)) 478 return -EFAULT; 479 480 return in_len; 481} 482 483ssize_t ib_uverbs_alloc_pd(struct ib_uverbs_file *file, 484 const char __user *buf, 485 int in_len, int out_len) 486{ 487 struct ib_uverbs_alloc_pd cmd; 488 struct ib_uverbs_alloc_pd_resp resp; 489 struct ib_udata udata; 490 struct ib_uobject *uobj; 491 struct ib_pd *pd; 492 int ret; 493 494 if (out_len < sizeof resp) 495 return -ENOSPC; 496 497 if (copy_from_user(&cmd, buf, sizeof cmd)) 498 return -EFAULT; 499 500 INIT_UDATA(&udata, buf + sizeof cmd, 501 (unsigned long) cmd.response + sizeof resp, 502 in_len - sizeof cmd, out_len - sizeof resp); 503 504 uobj = kmalloc(sizeof *uobj, GFP_KERNEL); 505 if (!uobj) 506 return -ENOMEM; 507 508 init_uobj(uobj, 0, file->ucontext, &pd_lock_key); 509 down_write(&uobj->mutex); 510 511 pd = file->device->ib_dev->alloc_pd(file->device->ib_dev, 512 file->ucontext, &udata); 513 if (IS_ERR(pd)) { 514 ret = PTR_ERR(pd); 515 goto err; 516 } 517 518 pd->device = file->device->ib_dev; 519 pd->uobject = uobj; 520 atomic_set(&pd->usecnt, 0); 521 522 uobj->object = pd; 523 ret = idr_add_uobj(&ib_uverbs_pd_idr, uobj); 524 if (ret) 525 goto err_idr; 526 527 memset(&resp, 0, sizeof resp); 528 resp.pd_handle = uobj->id; 529 530 if (copy_to_user((void __user *) (unsigned long) cmd.response, 531 &resp, sizeof resp)) { 532 ret = -EFAULT; 533 goto err_copy; 534 } 535 536 mutex_lock(&file->mutex); 537 list_add_tail(&uobj->list, &file->ucontext->pd_list); 538 mutex_unlock(&file->mutex); 539 540 uobj->live = 1; 541 542 up_write(&uobj->mutex); 543 544 return in_len; 545 546err_copy: 547 idr_remove_uobj(&ib_uverbs_pd_idr, uobj); 548 549err_idr: 550 ib_dealloc_pd(pd); 551 552err: 553 put_uobj_write(uobj); 554 return ret; 555} 556 557ssize_t ib_uverbs_dealloc_pd(struct ib_uverbs_file *file, 558 const char __user *buf, 559 int in_len, int out_len) 560{ 561 struct ib_uverbs_dealloc_pd cmd; 562 struct ib_uobject *uobj; 563 int ret; 564 565 if (copy_from_user(&cmd, buf, sizeof cmd)) 566 return -EFAULT; 567 568 uobj = idr_write_uobj(&ib_uverbs_pd_idr, cmd.pd_handle, file->ucontext); 569 if (!uobj) 570 return -EINVAL; 571 572 ret = ib_dealloc_pd(uobj->object); 573 if (!ret) 574 uobj->live = 0; 575 576 put_uobj_write(uobj); 577 578 if (ret) 579 return ret; 580 581 idr_remove_uobj(&ib_uverbs_pd_idr, uobj); 582 583 mutex_lock(&file->mutex); 584 list_del(&uobj->list); 585 mutex_unlock(&file->mutex); 586 587 put_uobj(uobj); 588 589 return in_len; 590} 591 592ssize_t ib_uverbs_reg_mr(struct ib_uverbs_file *file, 593 const char __user *buf, int in_len, 594 int out_len) 595{ 596 struct ib_uverbs_reg_mr cmd; 597 struct ib_uverbs_reg_mr_resp resp; 598 struct ib_udata udata; 599 struct ib_uobject *uobj; 600 struct ib_pd *pd; 601 struct ib_mr *mr; 602 int ret; 603 604 if (out_len < sizeof resp) 605 return -ENOSPC; 606 607 if (copy_from_user(&cmd, buf, sizeof cmd)) 608 return -EFAULT; 609 610 INIT_UDATA(&udata, buf + sizeof cmd, 611 (unsigned long) cmd.response + sizeof resp, 612 in_len - sizeof cmd, out_len - sizeof resp); 613 614 if ((cmd.start & ~PAGE_MASK) != (cmd.hca_va & ~PAGE_MASK)) 615 return -EINVAL; 616 617 /* 618 * Local write permission is required if remote write or 619 * remote atomic permission is also requested. 620 */ 621 if (cmd.access_flags & (IB_ACCESS_REMOTE_ATOMIC | IB_ACCESS_REMOTE_WRITE) && 622 !(cmd.access_flags & IB_ACCESS_LOCAL_WRITE)) 623 return -EINVAL; 624 625 uobj = kmalloc(sizeof *uobj, GFP_KERNEL); 626 if (!uobj) 627 return -ENOMEM; 628 629 init_uobj(uobj, 0, file->ucontext, &mr_lock_key); 630 down_write(&uobj->mutex); 631 632 pd = idr_read_pd(cmd.pd_handle, file->ucontext); 633 if (!pd) { 634 ret = -EINVAL; 635 goto err_free; 636 } 637 638 mr = pd->device->reg_user_mr(pd, cmd.start, cmd.length, cmd.hca_va, 639 cmd.access_flags, &udata, 0); 640 if (IS_ERR(mr)) { 641 ret = PTR_ERR(mr); 642 goto err_put; 643 } 644 645 mr->device = pd->device; 646 mr->pd = pd; 647 mr->uobject = uobj; 648 atomic_inc(&pd->usecnt); 649 atomic_set(&mr->usecnt, 0); 650 651 uobj->object = mr; 652 ret = idr_add_uobj(&ib_uverbs_mr_idr, uobj); 653 if (ret) 654 goto err_unreg; 655 656 memset(&resp, 0, sizeof resp); 657 resp.lkey = mr->lkey; 658 resp.rkey = mr->rkey; 659 resp.mr_handle = uobj->id; 660 661 if (copy_to_user((void __user *) (unsigned long) cmd.response, 662 &resp, sizeof resp)) { 663 ret = -EFAULT; 664 goto err_copy; 665 } 666 667 put_pd_read(pd); 668 669 mutex_lock(&file->mutex); 670 list_add_tail(&uobj->list, &file->ucontext->mr_list); 671 mutex_unlock(&file->mutex); 672 673 uobj->live = 1; 674 675 up_write(&uobj->mutex); 676 677 return in_len; 678 679err_copy: 680 idr_remove_uobj(&ib_uverbs_mr_idr, uobj); 681 682err_unreg: 683 ib_dereg_mr(mr); 684 685err_put: 686 put_pd_read(pd); 687 688err_free: 689 put_uobj_write(uobj); 690 return ret; 691} 692 693ssize_t ib_uverbs_dereg_mr(struct ib_uverbs_file *file, 694 const char __user *buf, int in_len, 695 int out_len) 696{ 697 struct ib_uverbs_dereg_mr cmd; 698 struct ib_mr *mr; 699 struct ib_uobject *uobj; 700 int ret = -EINVAL; 701 702 if (copy_from_user(&cmd, buf, sizeof cmd)) 703 return -EFAULT; 704 705 uobj = idr_write_uobj(&ib_uverbs_mr_idr, cmd.mr_handle, file->ucontext); 706 if (!uobj) 707 return -EINVAL; 708 709 mr = uobj->object; 710 711 ret = ib_dereg_mr(mr); 712 if (!ret) 713 uobj->live = 0; 714 715 put_uobj_write(uobj); 716 717 if (ret) 718 return ret; 719 720 idr_remove_uobj(&ib_uverbs_mr_idr, uobj); 721 722 mutex_lock(&file->mutex); 723 list_del(&uobj->list); 724 mutex_unlock(&file->mutex); 725 726 put_uobj(uobj); 727 728 return in_len; 729} 730 731ssize_t ib_uverbs_create_comp_channel(struct ib_uverbs_file *file, 732 const char __user *buf, int in_len, 733 int out_len) 734{ 735 struct ib_uverbs_create_comp_channel cmd; 736 struct ib_uverbs_create_comp_channel_resp resp; 737 struct file *filp; 738 739 if (out_len < sizeof resp) 740 return -ENOSPC; 741 742 if (copy_from_user(&cmd, buf, sizeof cmd)) 743 return -EFAULT; 744 745 filp = ib_uverbs_alloc_event_file(file, 0, &resp.fd); 746 if (IS_ERR(filp)) 747 return PTR_ERR(filp); 748 749 if (copy_to_user((void __user *) (unsigned long) cmd.response, 750 &resp, sizeof resp)) { 751 put_unused_fd(resp.fd); 752 fput(filp); 753 return -EFAULT; 754 } 755 756 fd_install(resp.fd, filp); 757 return in_len; 758} 759 760ssize_t ib_uverbs_create_cq(struct ib_uverbs_file *file, 761 const char __user *buf, int in_len, 762 int out_len) 763{ 764 struct ib_uverbs_create_cq cmd; 765 struct ib_uverbs_create_cq_resp resp; 766 struct ib_udata udata; 767 struct ib_ucq_object *obj; 768 struct ib_uverbs_event_file *ev_file = NULL; 769 struct ib_cq *cq; 770 int ret; 771 772 if (out_len < sizeof resp) 773 return -ENOSPC; 774 775 if (copy_from_user(&cmd, buf, sizeof cmd)) 776 return -EFAULT; 777 778 INIT_UDATA(&udata, buf + sizeof cmd, 779 (unsigned long) cmd.response + sizeof resp, 780 in_len - sizeof cmd, out_len - sizeof resp); 781 782 if (cmd.comp_vector >= file->device->num_comp_vectors) 783 return -EINVAL; 784 785 obj = kmalloc(sizeof *obj, GFP_KERNEL); 786 if (!obj) 787 return -ENOMEM; 788 789 init_uobj(&obj->uobject, cmd.user_handle, file->ucontext, &cq_lock_key); 790 down_write(&obj->uobject.mutex); 791 792 if (cmd.comp_channel >= 0) { 793 ev_file = ib_uverbs_lookup_comp_file(cmd.comp_channel); 794 if (!ev_file) { 795 ret = -EINVAL; 796 goto err; 797 } 798 } 799 800 obj->uverbs_file = file; 801 obj->comp_events_reported = 0; 802 obj->async_events_reported = 0; 803 INIT_LIST_HEAD(&obj->comp_list); 804 INIT_LIST_HEAD(&obj->async_list); 805 806 cq = file->device->ib_dev->create_cq(file->device->ib_dev, cmd.cqe, 807 cmd.comp_vector, 808 file->ucontext, &udata); 809 if (IS_ERR(cq)) { 810 ret = PTR_ERR(cq); 811 goto err_file; 812 } 813 814 cq->device = file->device->ib_dev; 815 cq->uobject = &obj->uobject; 816 cq->comp_handler = ib_uverbs_comp_handler; 817 cq->event_handler = ib_uverbs_cq_event_handler; 818 cq->cq_context = ev_file; 819 atomic_set(&cq->usecnt, 0); 820 821 obj->uobject.object = cq; 822 ret = idr_add_uobj(&ib_uverbs_cq_idr, &obj->uobject); 823 if (ret) 824 goto err_free; 825 826 memset(&resp, 0, sizeof resp); 827 resp.cq_handle = obj->uobject.id; 828 resp.cqe = cq->cqe; 829 830 if (copy_to_user((void __user *) (unsigned long) cmd.response, 831 &resp, sizeof resp)) { 832 ret = -EFAULT; 833 goto err_copy; 834 } 835 836 mutex_lock(&file->mutex); 837 list_add_tail(&obj->uobject.list, &file->ucontext->cq_list); 838 mutex_unlock(&file->mutex); 839 840 obj->uobject.live = 1; 841 842 up_write(&obj->uobject.mutex); 843 844 return in_len; 845 846err_copy: 847 idr_remove_uobj(&ib_uverbs_cq_idr, &obj->uobject); 848 849err_free: 850 ib_destroy_cq(cq); 851 852err_file: 853 if (ev_file) 854 ib_uverbs_release_ucq(file, ev_file, obj); 855 856err: 857 put_uobj_write(&obj->uobject); 858 return ret; 859} 860 861ssize_t ib_uverbs_resize_cq(struct ib_uverbs_file *file, 862 const char __user *buf, int in_len, 863 int out_len) 864{ 865 struct ib_uverbs_resize_cq cmd; 866 struct ib_uverbs_resize_cq_resp resp; 867 struct ib_udata udata; 868 struct ib_cq *cq; 869 int ret = -EINVAL; 870 871 if (copy_from_user(&cmd, buf, sizeof cmd)) 872 return -EFAULT; 873 874 INIT_UDATA(&udata, buf + sizeof cmd, 875 (unsigned long) cmd.response + sizeof resp, 876 in_len - sizeof cmd, out_len - sizeof resp); 877 878 cq = idr_read_cq(cmd.cq_handle, file->ucontext, 0); 879 if (!cq) 880 return -EINVAL; 881 882 ret = cq->device->resize_cq(cq, cmd.cqe, &udata); 883 if (ret) 884 goto out; 885 886 resp.cqe = cq->cqe; 887 888 if (copy_to_user((void __user *) (unsigned long) cmd.response, 889 &resp, sizeof resp.cqe)) 890 ret = -EFAULT; 891 892out: 893 put_cq_read(cq); 894 895 return ret ? ret : in_len; 896} 897 898ssize_t ib_uverbs_poll_cq(struct ib_uverbs_file *file, 899 const char __user *buf, int in_len, 900 int out_len) 901{ 902 struct ib_uverbs_poll_cq cmd; 903 struct ib_uverbs_poll_cq_resp *resp; 904 struct ib_cq *cq; 905 struct ib_wc *wc; 906 int ret = 0; 907 int i; 908 int rsize; 909 910 if (copy_from_user(&cmd, buf, sizeof cmd)) 911 return -EFAULT; 912 913 wc = kmalloc(cmd.ne * sizeof *wc, GFP_KERNEL); 914 if (!wc) 915 return -ENOMEM; 916 917 rsize = sizeof *resp + cmd.ne * sizeof(struct ib_uverbs_wc); 918 resp = kmalloc(rsize, GFP_KERNEL); 919 if (!resp) { 920 ret = -ENOMEM; 921 goto out_wc; 922 } 923 924 cq = idr_read_cq(cmd.cq_handle, file->ucontext, 0); 925 if (!cq) { 926 ret = -EINVAL; 927 goto out; 928 } 929 930 resp->count = ib_poll_cq(cq, cmd.ne, wc); 931 932 put_cq_read(cq); 933 934 for (i = 0; i < resp->count; i++) { 935 resp->wc[i].wr_id = wc[i].wr_id; 936 resp->wc[i].status = wc[i].status; 937 resp->wc[i].opcode = wc[i].opcode; 938 resp->wc[i].vendor_err = wc[i].vendor_err; 939 resp->wc[i].byte_len = wc[i].byte_len; 940 resp->wc[i].ex.imm_data = (__u32 __force) wc[i].ex.imm_data; 941 resp->wc[i].qp_num = wc[i].qp->qp_num; 942 resp->wc[i].src_qp = wc[i].src_qp; 943 resp->wc[i].wc_flags = wc[i].wc_flags; 944 resp->wc[i].pkey_index = wc[i].pkey_index; 945 resp->wc[i].slid = wc[i].slid; 946 resp->wc[i].sl = wc[i].sl; 947 resp->wc[i].dlid_path_bits = wc[i].dlid_path_bits; 948 resp->wc[i].port_num = wc[i].port_num; 949 } 950 951 if (copy_to_user((void __user *) (unsigned long) cmd.response, resp, rsize)) 952 ret = -EFAULT; 953 954out: 955 kfree(resp); 956 957out_wc: 958 kfree(wc); 959 return ret ? ret : in_len; 960} 961 962ssize_t ib_uverbs_req_notify_cq(struct ib_uverbs_file *file, 963 const char __user *buf, int in_len, 964 int out_len) 965{ 966 struct ib_uverbs_req_notify_cq cmd; 967 struct ib_cq *cq; 968 969 if (copy_from_user(&cmd, buf, sizeof cmd)) 970 return -EFAULT; 971 972 cq = idr_read_cq(cmd.cq_handle, file->ucontext, 0); 973 if (!cq) 974 return -EINVAL; 975 976 ib_req_notify_cq(cq, cmd.solicited_only ? 977 IB_CQ_SOLICITED : IB_CQ_NEXT_COMP); 978 979 put_cq_read(cq); 980 981 return in_len; 982} 983 984ssize_t ib_uverbs_destroy_cq(struct ib_uverbs_file *file, 985 const char __user *buf, int in_len, 986 int out_len) 987{ 988 struct ib_uverbs_destroy_cq cmd; 989 struct ib_uverbs_destroy_cq_resp resp; 990 struct ib_uobject *uobj; 991 struct ib_cq *cq; 992 struct ib_ucq_object *obj; 993 struct ib_uverbs_event_file *ev_file; 994 int ret = -EINVAL; 995 996 if (copy_from_user(&cmd, buf, sizeof cmd)) 997 return -EFAULT; 998 999 uobj = idr_write_uobj(&ib_uverbs_cq_idr, cmd.cq_handle, file->ucontext); 1000 if (!uobj) 1001 return -EINVAL; 1002 cq = uobj->object; 1003 ev_file = cq->cq_context; 1004 obj = container_of(cq->uobject, struct ib_ucq_object, uobject); 1005 1006 ret = ib_destroy_cq(cq); 1007 if (!ret) 1008 uobj->live = 0; 1009 1010 put_uobj_write(uobj); 1011 1012 if (ret) 1013 return ret; 1014 1015 idr_remove_uobj(&ib_uverbs_cq_idr, uobj); 1016 1017 mutex_lock(&file->mutex); 1018 list_del(&uobj->list); 1019 mutex_unlock(&file->mutex); 1020 1021 ib_uverbs_release_ucq(file, ev_file, obj); 1022 1023 memset(&resp, 0, sizeof resp); 1024 resp.comp_events_reported = obj->comp_events_reported; 1025 resp.async_events_reported = obj->async_events_reported; 1026 1027 put_uobj(uobj); 1028 1029 if (copy_to_user((void __user *) (unsigned long) cmd.response, 1030 &resp, sizeof resp)) 1031 return -EFAULT; 1032 1033 return in_len; 1034} 1035 1036ssize_t ib_uverbs_create_qp(struct ib_uverbs_file *file, 1037 const char __user *buf, int in_len, 1038 int out_len) 1039{ 1040 struct ib_uverbs_create_qp cmd; 1041 struct ib_uverbs_create_qp_resp resp; 1042 struct ib_udata udata; 1043 struct ib_uqp_object *obj; 1044 struct ib_pd *pd; 1045 struct ib_cq *scq, *rcq; 1046 struct ib_srq *srq; 1047 struct ib_qp *qp; 1048 struct ib_qp_init_attr attr; 1049 struct ib_xrcd *xrcd; 1050 struct ib_uobject *xrcd_uobj; 1051 int ret; 1052 1053 if (out_len < sizeof resp) 1054 return -ENOSPC; 1055 1056 if (copy_from_user(&cmd, buf, sizeof cmd)) 1057 return -EFAULT; 1058 1059 INIT_UDATA(&udata, buf + sizeof cmd, 1060 (unsigned long) cmd.response + sizeof resp, 1061 in_len - sizeof cmd, out_len - sizeof resp); 1062 1063 obj = kmalloc(sizeof *obj, GFP_KERNEL); 1064 if (!obj) 1065 return -ENOMEM; 1066 1067 init_uobj(&obj->uevent.uobject, cmd.user_handle, file->ucontext, &qp_lock_key); 1068 down_write(&obj->uevent.uobject.mutex); 1069 1070 srq = (cmd.is_srq && cmd.qp_type != IB_QPT_XRC) ? 1071 idr_read_srq(cmd.srq_handle, file->ucontext) : NULL; 1072 xrcd = cmd.qp_type == IB_QPT_XRC ? 1073 idr_read_xrcd(cmd.srq_handle, file->ucontext, &xrcd_uobj) : NULL; 1074 pd = idr_read_pd(cmd.pd_handle, file->ucontext); 1075 scq = idr_read_cq(cmd.send_cq_handle, file->ucontext, 0); 1076 rcq = cmd.recv_cq_handle == cmd.send_cq_handle ? 1077 scq : idr_read_cq(cmd.recv_cq_handle, file->ucontext, 1); 1078 1079 if (!pd || !scq || !rcq || (cmd.is_srq && !srq) || 1080 (cmd.qp_type == IB_QPT_XRC && !xrcd)) { 1081 ret = -EINVAL; 1082 goto err_put; 1083 } 1084 1085 attr.create_flags = 0; 1086 attr.event_handler = ib_uverbs_qp_event_handler; 1087 attr.qp_context = file; 1088 attr.send_cq = scq; 1089 attr.recv_cq = rcq; 1090 attr.srq = srq; 1091 attr.sq_sig_type = cmd.sq_sig_all ? IB_SIGNAL_ALL_WR : IB_SIGNAL_REQ_WR; 1092 attr.qp_type = cmd.qp_type; 1093 attr.xrcd = xrcd; 1094 attr.create_flags = 0; 1095 1096 attr.cap.max_send_wr = cmd.max_send_wr; 1097 attr.cap.max_recv_wr = cmd.max_recv_wr; 1098 attr.cap.max_send_sge = cmd.max_send_sge; 1099 attr.cap.max_recv_sge = cmd.max_recv_sge; 1100 attr.cap.max_inline_data = cmd.max_inline_data; 1101 1102 obj->uevent.events_reported = 0; 1103 INIT_LIST_HEAD(&obj->uevent.event_list); 1104 INIT_LIST_HEAD(&obj->mcast_list); 1105 1106 qp = pd->device->create_qp(pd, &attr, &udata); 1107 if (IS_ERR(qp)) { 1108 ret = PTR_ERR(qp); 1109 goto err_put; 1110 } 1111 1112 qp->device = pd->device; 1113 qp->pd = pd; 1114 qp->send_cq = attr.send_cq; 1115 qp->recv_cq = attr.recv_cq; 1116 qp->srq = attr.srq; 1117 qp->uobject = &obj->uevent.uobject; 1118 qp->event_handler = attr.event_handler; 1119 qp->qp_context = attr.qp_context; 1120 qp->qp_type = attr.qp_type; 1121 qp->xrcd = attr.xrcd; 1122 atomic_inc(&pd->usecnt); 1123 atomic_inc(&attr.send_cq->usecnt); 1124 atomic_inc(&attr.recv_cq->usecnt); 1125 if (attr.srq) 1126 atomic_inc(&attr.srq->usecnt); 1127 else if (attr.xrcd) 1128 atomic_inc(&attr.xrcd->usecnt); 1129 1130 obj->uevent.uobject.object = qp; 1131 ret = idr_add_uobj(&ib_uverbs_qp_idr, &obj->uevent.uobject); 1132 if (ret) 1133 goto err_destroy; 1134 1135 memset(&resp, 0, sizeof resp); 1136 resp.qpn = qp->qp_num; 1137 resp.qp_handle = obj->uevent.uobject.id; 1138 resp.max_recv_sge = attr.cap.max_recv_sge; 1139 resp.max_send_sge = attr.cap.max_send_sge; 1140 resp.max_recv_wr = attr.cap.max_recv_wr; 1141 resp.max_send_wr = attr.cap.max_send_wr; 1142 resp.max_inline_data = attr.cap.max_inline_data; 1143 1144 if (copy_to_user((void __user *) (unsigned long) cmd.response, 1145 &resp, sizeof resp)) { 1146 ret = -EFAULT; 1147 goto err_copy; 1148 } 1149 1150 put_pd_read(pd); 1151 put_cq_read(scq); 1152 if (rcq != scq) 1153 put_cq_read(rcq); 1154 if (srq) 1155 put_srq_read(srq); 1156 if (xrcd) 1157 put_xrcd_read(xrcd_uobj); 1158 1159 mutex_lock(&file->mutex); 1160 list_add_tail(&obj->uevent.uobject.list, &file->ucontext->qp_list); 1161 mutex_unlock(&file->mutex); 1162 1163 obj->uevent.uobject.live = 1; 1164 1165 up_write(&obj->uevent.uobject.mutex); 1166 1167 return in_len; 1168 1169err_copy: 1170 idr_remove_uobj(&ib_uverbs_qp_idr, &obj->uevent.uobject); 1171 1172err_destroy: 1173 ib_destroy_qp(qp); 1174 1175err_put: 1176 if (pd) 1177 put_pd_read(pd); 1178 if (scq) 1179 put_cq_read(scq); 1180 if (rcq && rcq != scq) 1181 put_cq_read(rcq); 1182 if (srq) 1183 put_srq_read(srq); 1184 if (xrcd) 1185 put_xrcd_read(xrcd_uobj); 1186 1187 put_uobj_write(&obj->uevent.uobject); 1188 return ret; 1189} 1190 1191ssize_t ib_uverbs_query_qp(struct ib_uverbs_file *file, 1192 const char __user *buf, int in_len, 1193 int out_len) 1194{ 1195 struct ib_uverbs_query_qp cmd; 1196 struct ib_uverbs_query_qp_resp resp; 1197 struct ib_qp *qp; 1198 struct ib_qp_attr *attr; 1199 struct ib_qp_init_attr *init_attr; 1200 int ret; 1201 1202 if (copy_from_user(&cmd, buf, sizeof cmd)) 1203 return -EFAULT; 1204 1205 attr = kmalloc(sizeof *attr, GFP_KERNEL); 1206 init_attr = kmalloc(sizeof *init_attr, GFP_KERNEL); 1207 if (!attr || !init_attr) { 1208 ret = -ENOMEM; 1209 goto out; 1210 } 1211 1212 qp = idr_read_qp(cmd.qp_handle, file->ucontext); 1213 if (!qp) { 1214 ret = -EINVAL; 1215 goto out; 1216 } 1217 1218 ret = ib_query_qp(qp, attr, cmd.attr_mask, init_attr); 1219 1220 put_qp_read(qp); 1221 1222 if (ret) 1223 goto out; 1224 1225 memset(&resp, 0, sizeof resp); 1226 1227 resp.qp_state = attr->qp_state; 1228 resp.cur_qp_state = attr->cur_qp_state; 1229 resp.path_mtu = attr->path_mtu; 1230 resp.path_mig_state = attr->path_mig_state; 1231 resp.qkey = attr->qkey; 1232 resp.rq_psn = attr->rq_psn; 1233 resp.sq_psn = attr->sq_psn; 1234 resp.dest_qp_num = attr->dest_qp_num; 1235 resp.qp_access_flags = attr->qp_access_flags; 1236 resp.pkey_index = attr->pkey_index; 1237 resp.alt_pkey_index = attr->alt_pkey_index; 1238 resp.sq_draining = attr->sq_draining; 1239 resp.max_rd_atomic = attr->max_rd_atomic; 1240 resp.max_dest_rd_atomic = attr->max_dest_rd_atomic; 1241 resp.min_rnr_timer = attr->min_rnr_timer; 1242 resp.port_num = attr->port_num; 1243 resp.timeout = attr->timeout; 1244 resp.retry_cnt = attr->retry_cnt; 1245 resp.rnr_retry = attr->rnr_retry; 1246 resp.alt_port_num = attr->alt_port_num; 1247 resp.alt_timeout = attr->alt_timeout; 1248 1249 memcpy(resp.dest.dgid, attr->ah_attr.grh.dgid.raw, 16); 1250 resp.dest.flow_label = attr->ah_attr.grh.flow_label; 1251 resp.dest.sgid_index = attr->ah_attr.grh.sgid_index; 1252 resp.dest.hop_limit = attr->ah_attr.grh.hop_limit; 1253 resp.dest.traffic_class = attr->ah_attr.grh.traffic_class; 1254 resp.dest.dlid = attr->ah_attr.dlid; 1255 resp.dest.sl = attr->ah_attr.sl; 1256 resp.dest.src_path_bits = attr->ah_attr.src_path_bits; 1257 resp.dest.static_rate = attr->ah_attr.static_rate; 1258 resp.dest.is_global = !!(attr->ah_attr.ah_flags & IB_AH_GRH); 1259 resp.dest.port_num = attr->ah_attr.port_num; 1260 1261 memcpy(resp.alt_dest.dgid, attr->alt_ah_attr.grh.dgid.raw, 16); 1262 resp.alt_dest.flow_label = attr->alt_ah_attr.grh.flow_label; 1263 resp.alt_dest.sgid_index = attr->alt_ah_attr.grh.sgid_index; 1264 resp.alt_dest.hop_limit = attr->alt_ah_attr.grh.hop_limit; 1265 resp.alt_dest.traffic_class = attr->alt_ah_attr.grh.traffic_class; 1266 resp.alt_dest.dlid = attr->alt_ah_attr.dlid; 1267 resp.alt_dest.sl = attr->alt_ah_attr.sl; 1268 resp.alt_dest.src_path_bits = attr->alt_ah_attr.src_path_bits; 1269 resp.alt_dest.static_rate = attr->alt_ah_attr.static_rate; 1270 resp.alt_dest.is_global = !!(attr->alt_ah_attr.ah_flags & IB_AH_GRH); 1271 resp.alt_dest.port_num = attr->alt_ah_attr.port_num; 1272 1273 resp.max_send_wr = init_attr->cap.max_send_wr; 1274 resp.max_recv_wr = init_attr->cap.max_recv_wr; 1275 resp.max_send_sge = init_attr->cap.max_send_sge; 1276 resp.max_recv_sge = init_attr->cap.max_recv_sge; 1277 resp.max_inline_data = init_attr->cap.max_inline_data; 1278 resp.sq_sig_all = init_attr->sq_sig_type == IB_SIGNAL_ALL_WR; 1279 1280 if (copy_to_user((void __user *) (unsigned long) cmd.response, 1281 &resp, sizeof resp)) 1282 ret = -EFAULT; 1283 1284out: 1285 kfree(attr); 1286 kfree(init_attr); 1287 1288 return ret ? ret : in_len; 1289} 1290 1291ssize_t ib_uverbs_modify_qp(struct ib_uverbs_file *file, 1292 const char __user *buf, int in_len, 1293 int out_len) 1294{ 1295 struct ib_uverbs_modify_qp cmd; 1296 struct ib_udata udata; 1297 struct ib_qp *qp; 1298 struct ib_qp_attr *attr; 1299 int ret; 1300 1301 if (copy_from_user(&cmd, buf, sizeof cmd)) 1302 return -EFAULT; 1303 1304 INIT_UDATA(&udata, buf + sizeof cmd, NULL, in_len - sizeof cmd, 1305 out_len); 1306 1307 attr = kmalloc(sizeof *attr, GFP_KERNEL); 1308 if (!attr) 1309 return -ENOMEM; 1310 1311 qp = idr_read_qp(cmd.qp_handle, file->ucontext); 1312 if (!qp) { 1313 ret = -EINVAL; 1314 goto out; 1315 } 1316 1317 attr->qp_state = cmd.qp_state; 1318 attr->cur_qp_state = cmd.cur_qp_state; 1319 attr->path_mtu = cmd.path_mtu; 1320 attr->path_mig_state = cmd.path_mig_state; 1321 attr->qkey = cmd.qkey; 1322 attr->rq_psn = cmd.rq_psn; 1323 attr->sq_psn = cmd.sq_psn; 1324 attr->dest_qp_num = cmd.dest_qp_num; 1325 attr->qp_access_flags = cmd.qp_access_flags; 1326 attr->pkey_index = cmd.pkey_index; 1327 attr->alt_pkey_index = cmd.alt_pkey_index; 1328 attr->en_sqd_async_notify = cmd.en_sqd_async_notify; 1329 attr->max_rd_atomic = cmd.max_rd_atomic; 1330 attr->max_dest_rd_atomic = cmd.max_dest_rd_atomic; 1331 attr->min_rnr_timer = cmd.min_rnr_timer; 1332 attr->port_num = cmd.port_num; 1333 attr->timeout = cmd.timeout; 1334 attr->retry_cnt = cmd.retry_cnt; 1335 attr->rnr_retry = cmd.rnr_retry; 1336 attr->alt_port_num = cmd.alt_port_num; 1337 attr->alt_timeout = cmd.alt_timeout; 1338 1339 memcpy(attr->ah_attr.grh.dgid.raw, cmd.dest.dgid, 16); 1340 attr->ah_attr.grh.flow_label = cmd.dest.flow_label; 1341 attr->ah_attr.grh.sgid_index = cmd.dest.sgid_index; 1342 attr->ah_attr.grh.hop_limit = cmd.dest.hop_limit; 1343 attr->ah_attr.grh.traffic_class = cmd.dest.traffic_class; 1344 attr->ah_attr.dlid = cmd.dest.dlid; 1345 attr->ah_attr.sl = cmd.dest.sl; 1346 attr->ah_attr.src_path_bits = cmd.dest.src_path_bits; 1347 attr->ah_attr.static_rate = cmd.dest.static_rate; 1348 attr->ah_attr.ah_flags = cmd.dest.is_global ? IB_AH_GRH : 0; 1349 attr->ah_attr.port_num = cmd.dest.port_num; 1350 1351 memcpy(attr->alt_ah_attr.grh.dgid.raw, cmd.alt_dest.dgid, 16); 1352 attr->alt_ah_attr.grh.flow_label = cmd.alt_dest.flow_label; 1353 attr->alt_ah_attr.grh.sgid_index = cmd.alt_dest.sgid_index; 1354 attr->alt_ah_attr.grh.hop_limit = cmd.alt_dest.hop_limit; 1355 attr->alt_ah_attr.grh.traffic_class = cmd.alt_dest.traffic_class; 1356 attr->alt_ah_attr.dlid = cmd.alt_dest.dlid; 1357 attr->alt_ah_attr.sl = cmd.alt_dest.sl; 1358 attr->alt_ah_attr.src_path_bits = cmd.alt_dest.src_path_bits; 1359 attr->alt_ah_attr.static_rate = cmd.alt_dest.static_rate; 1360 attr->alt_ah_attr.ah_flags = cmd.alt_dest.is_global ? IB_AH_GRH : 0; 1361 attr->alt_ah_attr.port_num = cmd.alt_dest.port_num; 1362 1363 ret = qp->device->modify_qp(qp, attr, cmd.attr_mask, &udata); 1364 1365 put_qp_read(qp); 1366 1367 if (ret) 1368 goto out; 1369 1370 ret = in_len; 1371 1372out: 1373 kfree(attr); 1374 1375 return ret; 1376} 1377 1378ssize_t ib_uverbs_destroy_qp(struct ib_uverbs_file *file, 1379 const char __user *buf, int in_len, 1380 int out_len) 1381{ 1382 struct ib_uverbs_destroy_qp cmd; 1383 struct ib_uverbs_destroy_qp_resp resp; 1384 struct ib_uobject *uobj; 1385 struct ib_qp *qp; 1386 struct ib_uqp_object *obj; 1387 int ret = -EINVAL; 1388 1389 if (copy_from_user(&cmd, buf, sizeof cmd)) 1390 return -EFAULT; 1391 1392 memset(&resp, 0, sizeof resp); 1393 1394 uobj = idr_write_uobj(&ib_uverbs_qp_idr, cmd.qp_handle, file->ucontext); 1395 if (!uobj) 1396 return -EINVAL; 1397 qp = uobj->object; 1398 obj = container_of(uobj, struct ib_uqp_object, uevent.uobject); 1399 1400 if (!list_empty(&obj->mcast_list)) { 1401 put_uobj_write(uobj); 1402 return -EBUSY; 1403 } 1404 1405 ret = ib_destroy_qp(qp); 1406 if (!ret) 1407 uobj->live = 0; 1408 1409 put_uobj_write(uobj); 1410 1411 if (ret) 1412 return ret; 1413 1414 idr_remove_uobj(&ib_uverbs_qp_idr, uobj); 1415 1416 mutex_lock(&file->mutex); 1417 list_del(&uobj->list); 1418 mutex_unlock(&file->mutex); 1419 1420 ib_uverbs_release_uevent(file, &obj->uevent); 1421 1422 resp.events_reported = obj->uevent.events_reported; 1423 1424 put_uobj(uobj); 1425 1426 if (copy_to_user((void __user *) (unsigned long) cmd.response, 1427 &resp, sizeof resp)) 1428 return -EFAULT; 1429 1430 return in_len; 1431} 1432 1433ssize_t ib_uverbs_post_send(struct ib_uverbs_file *file, 1434 const char __user *buf, int in_len, 1435 int out_len) 1436{ 1437 struct ib_uverbs_post_send cmd; 1438 struct ib_uverbs_post_send_resp resp; 1439 struct ib_uverbs_send_wr *user_wr; 1440 struct ib_send_wr *wr = NULL, *last, *next, *bad_wr; 1441 struct ib_qp *qp; 1442 int i, sg_ind; 1443 int is_ud; 1444 ssize_t ret = -EINVAL; 1445 1446 if (copy_from_user(&cmd, buf, sizeof cmd)) 1447 return -EFAULT; 1448 1449 if (in_len < sizeof cmd + cmd.wqe_size * cmd.wr_count + 1450 cmd.sge_count * sizeof (struct ib_uverbs_sge)) 1451 return -EINVAL; 1452 1453 if (cmd.wqe_size < sizeof (struct ib_uverbs_send_wr)) 1454 return -EINVAL; 1455 1456 user_wr = kmalloc(cmd.wqe_size, GFP_KERNEL); 1457 if (!user_wr) 1458 return -ENOMEM; 1459 1460 qp = idr_read_qp(cmd.qp_handle, file->ucontext); 1461 if (!qp) 1462 goto out; 1463 1464 is_ud = qp->qp_type == IB_QPT_UD; 1465 sg_ind = 0; 1466 last = NULL; 1467 for (i = 0; i < cmd.wr_count; ++i) { 1468 if (copy_from_user(user_wr, 1469 buf + sizeof cmd + i * cmd.wqe_size, 1470 cmd.wqe_size)) { 1471 ret = -EFAULT; 1472 goto out_put; 1473 } 1474 1475 if (user_wr->num_sge + sg_ind > cmd.sge_count) { 1476 ret = -EINVAL; 1477 goto out_put; 1478 } 1479 1480 next = kmalloc(ALIGN(sizeof *next, sizeof (struct ib_sge)) + 1481 user_wr->num_sge * sizeof (struct ib_sge), 1482 GFP_KERNEL); 1483 if (!next) { 1484 ret = -ENOMEM; 1485 goto out_put; 1486 } 1487 1488 if (!last) 1489 wr = next; 1490 else 1491 last->next = next; 1492 last = next; 1493 1494 next->next = NULL; 1495 next->wr_id = user_wr->wr_id; 1496 next->num_sge = user_wr->num_sge; 1497 next->opcode = user_wr->opcode; 1498 next->send_flags = user_wr->send_flags; 1499 1500 if (is_ud) { 1501 next->wr.ud.ah = idr_read_ah(user_wr->wr.ud.ah, 1502 file->ucontext); 1503 if (!next->wr.ud.ah) { 1504 ret = -EINVAL; 1505 goto out_put; 1506 } 1507 next->wr.ud.remote_qpn = user_wr->wr.ud.remote_qpn; 1508 next->wr.ud.remote_qkey = user_wr->wr.ud.remote_qkey; 1509 } else { 1510 switch (next->opcode) { 1511 case IB_WR_RDMA_WRITE_WITH_IMM: 1512 next->ex.imm_data = 1513 (__be32 __force) user_wr->ex.imm_data; 1514 case IB_WR_RDMA_WRITE: 1515 case IB_WR_RDMA_READ: 1516 next->wr.rdma.remote_addr = 1517 user_wr->wr.rdma.remote_addr; 1518 next->wr.rdma.rkey = 1519 user_wr->wr.rdma.rkey; 1520 break; 1521 case IB_WR_SEND_WITH_IMM: 1522 next->ex.imm_data = 1523 (__be32 __force) user_wr->ex.imm_data; 1524 break; 1525 case IB_WR_SEND_WITH_INV: 1526 next->ex.invalidate_rkey = 1527 user_wr->ex.invalidate_rkey; 1528 break; 1529 case IB_WR_ATOMIC_CMP_AND_SWP: 1530 case IB_WR_ATOMIC_FETCH_AND_ADD: 1531 next->wr.atomic.remote_addr = 1532 user_wr->wr.atomic.remote_addr; 1533 next->wr.atomic.compare_add = 1534 user_wr->wr.atomic.compare_add; 1535 next->wr.atomic.swap = user_wr->wr.atomic.swap; 1536 next->wr.atomic.rkey = user_wr->wr.atomic.rkey; 1537 break; 1538 default: 1539 break; 1540 } 1541 } 1542 1543 if (next->num_sge) { 1544 next->sg_list = (void *) next + 1545 ALIGN(sizeof *next, sizeof (struct ib_sge)); 1546 if (copy_from_user(next->sg_list, 1547 buf + sizeof cmd + 1548 cmd.wr_count * cmd.wqe_size + 1549 sg_ind * sizeof (struct ib_sge), 1550 next->num_sge * sizeof (struct ib_sge))) { 1551 ret = -EFAULT; 1552 goto out_put; 1553 } 1554 sg_ind += next->num_sge; 1555 } else 1556 next->sg_list = NULL; 1557 } 1558 1559 resp.bad_wr = 0; 1560 ret = qp->device->post_send(qp, wr, &bad_wr); 1561 if (ret) 1562 for (next = wr; next; next = next->next) { 1563 ++resp.bad_wr; 1564 if (next == bad_wr) 1565 break; 1566 } 1567 1568 if (copy_to_user((void __user *) (unsigned long) cmd.response, 1569 &resp, sizeof resp)) 1570 ret = -EFAULT; 1571 1572out_put: 1573 put_qp_read(qp); 1574 1575 while (wr) { 1576 if (is_ud && wr->wr.ud.ah) 1577 put_ah_read(wr->wr.ud.ah); 1578 next = wr->next; 1579 kfree(wr); 1580 wr = next; 1581 } 1582 1583out: 1584 kfree(user_wr); 1585 1586 return ret ? ret : in_len; 1587} 1588 1589static struct ib_recv_wr *ib_uverbs_unmarshall_recv(const char __user *buf, 1590 int in_len, 1591 u32 wr_count, 1592 u32 sge_count, 1593 u32 wqe_size) 1594{ 1595 struct ib_uverbs_recv_wr *user_wr; 1596 struct ib_recv_wr *wr = NULL, *last, *next; 1597 int sg_ind; 1598 int i; 1599 int ret; 1600 1601 if (in_len < wqe_size * wr_count + 1602 sge_count * sizeof (struct ib_uverbs_sge)) 1603 return ERR_PTR(-EINVAL); 1604 1605 if (wqe_size < sizeof (struct ib_uverbs_recv_wr)) 1606 return ERR_PTR(-EINVAL); 1607 1608 user_wr = kmalloc(wqe_size, GFP_KERNEL); 1609 if (!user_wr) 1610 return ERR_PTR(-ENOMEM); 1611 1612 sg_ind = 0; 1613 last = NULL; 1614 for (i = 0; i < wr_count; ++i) { 1615 if (copy_from_user(user_wr, buf + i * wqe_size, 1616 wqe_size)) { 1617 ret = -EFAULT; 1618 goto err; 1619 } 1620 1621 if (user_wr->num_sge + sg_ind > sge_count) { 1622 ret = -EINVAL; 1623 goto err; 1624 } 1625 1626 next = kmalloc(ALIGN(sizeof *next, sizeof (struct ib_sge)) + 1627 user_wr->num_sge * sizeof (struct ib_sge), 1628 GFP_KERNEL); 1629 if (!next) { 1630 ret = -ENOMEM; 1631 goto err; 1632 } 1633 1634 if (!last) 1635 wr = next; 1636 else 1637 last->next = next; 1638 last = next; 1639 1640 next->next = NULL; 1641 next->wr_id = user_wr->wr_id; 1642 next->num_sge = user_wr->num_sge; 1643 1644 if (next->num_sge) { 1645 next->sg_list = (void *) next + 1646 ALIGN(sizeof *next, sizeof (struct ib_sge)); 1647 if (copy_from_user(next->sg_list, 1648 buf + wr_count * wqe_size + 1649 sg_ind * sizeof (struct ib_sge), 1650 next->num_sge * sizeof (struct ib_sge))) { 1651 ret = -EFAULT; 1652 goto err; 1653 } 1654 sg_ind += next->num_sge; 1655 } else 1656 next->sg_list = NULL; 1657 } 1658 1659 kfree(user_wr); 1660 return wr; 1661 1662err: 1663 kfree(user_wr); 1664 1665 while (wr) { 1666 next = wr->next; 1667 kfree(wr); 1668 wr = next; 1669 } 1670 1671 return ERR_PTR(ret); 1672} 1673 1674ssize_t ib_uverbs_post_recv(struct ib_uverbs_file *file, 1675 const char __user *buf, int in_len, 1676 int out_len) 1677{ 1678 struct ib_uverbs_post_recv cmd; 1679 struct ib_uverbs_post_recv_resp resp; 1680 struct ib_recv_wr *wr, *next, *bad_wr; 1681 struct ib_qp *qp; 1682 ssize_t ret = -EINVAL; 1683 1684 if (copy_from_user(&cmd, buf, sizeof cmd)) 1685 return -EFAULT; 1686 1687 wr = ib_uverbs_unmarshall_recv(buf + sizeof cmd, 1688 in_len - sizeof cmd, cmd.wr_count, 1689 cmd.sge_count, cmd.wqe_size); 1690 if (IS_ERR(wr)) 1691 return PTR_ERR(wr); 1692 1693 qp = idr_read_qp(cmd.qp_handle, file->ucontext); 1694 if (!qp) 1695 goto out; 1696 1697 resp.bad_wr = 0; 1698 ret = qp->device->post_recv(qp, wr, &bad_wr); 1699 1700 put_qp_read(qp); 1701 1702 if (ret) 1703 for (next = wr; next; next = next->next) { 1704 ++resp.bad_wr; 1705 if (next == bad_wr) 1706 break; 1707 } 1708 1709 if (copy_to_user((void __user *) (unsigned long) cmd.response, 1710 &resp, sizeof resp)) 1711 ret = -EFAULT; 1712 1713out: 1714 while (wr) { 1715 next = wr->next; 1716 kfree(wr); 1717 wr = next; 1718 } 1719 1720 return ret ? ret : in_len; 1721} 1722 1723ssize_t ib_uverbs_post_srq_recv(struct ib_uverbs_file *file, 1724 const char __user *buf, int in_len, 1725 int out_len) 1726{ 1727 struct ib_uverbs_post_srq_recv cmd; 1728 struct ib_uverbs_post_srq_recv_resp resp; 1729 struct ib_recv_wr *wr, *next, *bad_wr; 1730 struct ib_srq *srq; 1731 ssize_t ret = -EINVAL; 1732 1733 if (copy_from_user(&cmd, buf, sizeof cmd)) 1734 return -EFAULT; 1735 1736 wr = ib_uverbs_unmarshall_recv(buf + sizeof cmd, 1737 in_len - sizeof cmd, cmd.wr_count, 1738 cmd.sge_count, cmd.wqe_size); 1739 if (IS_ERR(wr)) 1740 return PTR_ERR(wr); 1741 1742 srq = idr_read_srq(cmd.srq_handle, file->ucontext); 1743 if (!srq) 1744 goto out; 1745 1746 resp.bad_wr = 0; 1747 ret = srq->device->post_srq_recv(srq, wr, &bad_wr); 1748 1749 put_srq_read(srq); 1750 1751 if (ret) 1752 for (next = wr; next; next = next->next) { 1753 ++resp.bad_wr; 1754 if (next == bad_wr) 1755 break; 1756 } 1757 1758 if (copy_to_user((void __user *) (unsigned long) cmd.response, 1759 &resp, sizeof resp)) 1760 ret = -EFAULT; 1761 1762out: 1763 while (wr) { 1764 next = wr->next; 1765 kfree(wr); 1766 wr = next; 1767 } 1768 1769 return ret ? ret : in_len; 1770} 1771 1772ssize_t ib_uverbs_create_ah(struct ib_uverbs_file *file, 1773 const char __user *buf, int in_len, 1774 int out_len) 1775{ 1776 struct ib_uverbs_create_ah cmd; 1777 struct ib_uverbs_create_ah_resp resp; 1778 struct ib_uobject *uobj; 1779 struct ib_pd *pd; 1780 struct ib_ah *ah; 1781 struct ib_ah_attr attr; 1782 int ret; 1783 1784 if (out_len < sizeof resp) 1785 return -ENOSPC; 1786 1787 if (copy_from_user(&cmd, buf, sizeof cmd)) 1788 return -EFAULT; 1789 1790 uobj = kmalloc(sizeof *uobj, GFP_KERNEL); 1791 if (!uobj) 1792 return -ENOMEM; 1793 1794 init_uobj(uobj, cmd.user_handle, file->ucontext, &ah_lock_key); 1795 down_write(&uobj->mutex); 1796 1797 pd = idr_read_pd(cmd.pd_handle, file->ucontext); 1798 if (!pd) { 1799 ret = -EINVAL; 1800 goto err; 1801 } 1802 1803 attr.dlid = cmd.attr.dlid; 1804 attr.sl = cmd.attr.sl; 1805 attr.src_path_bits = cmd.attr.src_path_bits; 1806 attr.static_rate = cmd.attr.static_rate; 1807 attr.ah_flags = cmd.attr.is_global ? IB_AH_GRH : 0; 1808 attr.port_num = cmd.attr.port_num; 1809 attr.grh.flow_label = cmd.attr.grh.flow_label; 1810 attr.grh.sgid_index = cmd.attr.grh.sgid_index; 1811 attr.grh.hop_limit = cmd.attr.grh.hop_limit; 1812 attr.grh.traffic_class = cmd.attr.grh.traffic_class; 1813 memcpy(attr.grh.dgid.raw, cmd.attr.grh.dgid, 16); 1814 1815 ah = ib_create_ah(pd, &attr); 1816 if (IS_ERR(ah)) { 1817 ret = PTR_ERR(ah); 1818 goto err_put; 1819 } 1820 1821 ah->uobject = uobj; 1822 uobj->object = ah; 1823 1824 ret = idr_add_uobj(&ib_uverbs_ah_idr, uobj); 1825 if (ret) 1826 goto err_destroy; 1827 1828 resp.ah_handle = uobj->id; 1829 1830 if (copy_to_user((void __user *) (unsigned long) cmd.response, 1831 &resp, sizeof resp)) { 1832 ret = -EFAULT; 1833 goto err_copy; 1834 } 1835 1836 put_pd_read(pd); 1837 1838 mutex_lock(&file->mutex); 1839 list_add_tail(&uobj->list, &file->ucontext->ah_list); 1840 mutex_unlock(&file->mutex); 1841 1842 uobj->live = 1; 1843 1844 up_write(&uobj->mutex); 1845 1846 return in_len; 1847 1848err_copy: 1849 idr_remove_uobj(&ib_uverbs_ah_idr, uobj); 1850 1851err_destroy: 1852 ib_destroy_ah(ah); 1853 1854err_put: 1855 put_pd_read(pd); 1856 1857err: 1858 put_uobj_write(uobj); 1859 return ret; 1860} 1861 1862ssize_t ib_uverbs_destroy_ah(struct ib_uverbs_file *file, 1863 const char __user *buf, int in_len, int out_len) 1864{ 1865 struct ib_uverbs_destroy_ah cmd; 1866 struct ib_ah *ah; 1867 struct ib_uobject *uobj; 1868 int ret; 1869 1870 if (copy_from_user(&cmd, buf, sizeof cmd)) 1871 return -EFAULT; 1872 1873 uobj = idr_write_uobj(&ib_uverbs_ah_idr, cmd.ah_handle, file->ucontext); 1874 if (!uobj) 1875 return -EINVAL; 1876 ah = uobj->object; 1877 1878 ret = ib_destroy_ah(ah); 1879 if (!ret) 1880 uobj->live = 0; 1881 1882 put_uobj_write(uobj); 1883 1884 if (ret) 1885 return ret; 1886 1887 idr_remove_uobj(&ib_uverbs_ah_idr, uobj); 1888 1889 mutex_lock(&file->mutex); 1890 list_del(&uobj->list); 1891 mutex_unlock(&file->mutex); 1892 1893 put_uobj(uobj); 1894 1895 return in_len; 1896} 1897 1898ssize_t ib_uverbs_attach_mcast(struct ib_uverbs_file *file, 1899 const char __user *buf, int in_len, 1900 int out_len) 1901{ 1902 struct ib_uverbs_attach_mcast cmd; 1903 struct ib_qp *qp; 1904 struct ib_uqp_object *obj; 1905 struct ib_uverbs_mcast_entry *mcast; 1906 int ret; 1907 1908 if (copy_from_user(&cmd, buf, sizeof cmd)) 1909 return -EFAULT; 1910 1911 qp = idr_read_qp(cmd.qp_handle, file->ucontext); 1912 if (!qp) 1913 return -EINVAL; 1914 1915 obj = container_of(qp->uobject, struct ib_uqp_object, uevent.uobject); 1916 1917 list_for_each_entry(mcast, &obj->mcast_list, list) 1918 if (cmd.mlid == mcast->lid && 1919 !memcmp(cmd.gid, mcast->gid.raw, sizeof mcast->gid.raw)) { 1920 ret = 0; 1921 goto out_put; 1922 } 1923 1924 mcast = kmalloc(sizeof *mcast, GFP_KERNEL); 1925 if (!mcast) { 1926 ret = -ENOMEM; 1927 goto out_put; 1928 } 1929 1930 mcast->lid = cmd.mlid; 1931 memcpy(mcast->gid.raw, cmd.gid, sizeof mcast->gid.raw); 1932 1933 ret = ib_attach_mcast(qp, &mcast->gid, cmd.mlid); 1934 if (!ret) 1935 list_add_tail(&mcast->list, &obj->mcast_list); 1936 else 1937 kfree(mcast); 1938 1939out_put: 1940 put_qp_read(qp); 1941 1942 return ret ? ret : in_len; 1943} 1944 1945ssize_t ib_uverbs_detach_mcast(struct ib_uverbs_file *file, 1946 const char __user *buf, int in_len, 1947 int out_len) 1948{ 1949 struct ib_uverbs_detach_mcast cmd; 1950 struct ib_uqp_object *obj; 1951 struct ib_qp *qp; 1952 struct ib_uverbs_mcast_entry *mcast; 1953 int ret = -EINVAL; 1954 1955 if (copy_from_user(&cmd, buf, sizeof cmd)) 1956 return -EFAULT; 1957 1958 qp = idr_read_qp(cmd.qp_handle, file->ucontext); 1959 if (!qp) 1960 return -EINVAL; 1961 1962 ret = ib_detach_mcast(qp, (union ib_gid *) cmd.gid, cmd.mlid); 1963 if (ret) 1964 goto out_put; 1965 1966 obj = container_of(qp->uobject, struct ib_uqp_object, uevent.uobject); 1967 1968 list_for_each_entry(mcast, &obj->mcast_list, list) 1969 if (cmd.mlid == mcast->lid && 1970 !memcmp(cmd.gid, mcast->gid.raw, sizeof mcast->gid.raw)) { 1971 list_del(&mcast->list); 1972 kfree(mcast); 1973 break; 1974 } 1975 1976out_put: 1977 put_qp_read(qp); 1978 1979 return ret ? ret : in_len; 1980} 1981 1982ssize_t ib_uverbs_create_srq(struct ib_uverbs_file *file, 1983 const char __user *buf, int in_len, 1984 int out_len) 1985{ 1986 struct ib_uverbs_create_srq cmd; 1987 struct ib_uverbs_create_srq_resp resp; 1988 struct ib_udata udata; 1989 struct ib_uevent_object *obj; 1990 struct ib_pd *pd; 1991 struct ib_srq *srq; 1992 struct ib_srq_init_attr attr; 1993 int ret; 1994 1995 if (out_len < sizeof resp) 1996 return -ENOSPC; 1997 1998 if (copy_from_user(&cmd, buf, sizeof cmd)) 1999 return -EFAULT; 2000 2001 INIT_UDATA(&udata, buf + sizeof cmd, 2002 (unsigned long) cmd.response + sizeof resp, 2003 in_len - sizeof cmd, out_len - sizeof resp); 2004 2005 obj = kmalloc(sizeof *obj, GFP_KERNEL); 2006 if (!obj) 2007 return -ENOMEM; 2008 2009 init_uobj(&obj->uobject, cmd.user_handle, file->ucontext, &srq_lock_key); 2010 down_write(&obj->uobject.mutex); 2011 2012 pd = idr_read_pd(cmd.pd_handle, file->ucontext); 2013 if (!pd) { 2014 ret = -EINVAL; 2015 goto err; 2016 } 2017 2018 attr.event_handler = ib_uverbs_srq_event_handler; 2019 attr.srq_context = file; 2020 attr.attr.max_wr = cmd.max_wr; 2021 attr.attr.max_sge = cmd.max_sge; 2022 attr.attr.srq_limit = cmd.srq_limit; 2023 2024 obj->events_reported = 0; 2025 INIT_LIST_HEAD(&obj->event_list); 2026 2027 srq = pd->device->create_srq(pd, &attr, &udata); 2028 if (IS_ERR(srq)) { 2029 ret = PTR_ERR(srq); 2030 goto err_put; 2031 } 2032 2033 srq->device = pd->device; 2034 srq->pd = pd; 2035 srq->uobject = &obj->uobject; 2036 srq->event_handler = attr.event_handler; 2037 srq->srq_context = attr.srq_context; 2038 srq->ext.xrc.cq = NULL; 2039 srq->ext.xrc.xrcd = NULL; 2040 atomic_inc(&pd->usecnt); 2041 atomic_set(&srq->usecnt, 0); 2042 2043 obj->uobject.object = srq; 2044 ret = idr_add_uobj(&ib_uverbs_srq_idr, &obj->uobject); 2045 if (ret) 2046 goto err_destroy; 2047 2048 memset(&resp, 0, sizeof resp); 2049 resp.srq_handle = obj->uobject.id; 2050 resp.max_wr = attr.attr.max_wr; 2051 resp.max_sge = attr.attr.max_sge; 2052 2053 if (copy_to_user((void __user *) (unsigned long) cmd.response, 2054 &resp, sizeof resp)) { 2055 ret = -EFAULT; 2056 goto err_copy; 2057 } 2058 2059 put_pd_read(pd); 2060 2061 mutex_lock(&file->mutex); 2062 list_add_tail(&obj->uobject.list, &file->ucontext->srq_list); 2063 mutex_unlock(&file->mutex); 2064 2065 obj->uobject.live = 1; 2066 2067 up_write(&obj->uobject.mutex); 2068 2069 return in_len; 2070 2071err_copy: 2072 idr_remove_uobj(&ib_uverbs_srq_idr, &obj->uobject); 2073 2074err_destroy: 2075 ib_destroy_srq(srq); 2076 2077err_put: 2078 put_pd_read(pd); 2079 2080err: 2081 put_uobj_write(&obj->uobject); 2082 return ret; 2083} 2084 2085ssize_t ib_uverbs_create_xrc_srq(struct ib_uverbs_file *file, 2086 const char __user *buf, int in_len, 2087 int out_len) 2088{ 2089 struct ib_uverbs_create_xsrq cmd; 2090 struct ib_uverbs_create_srq_resp resp; 2091 struct ib_udata udata; 2092 struct ib_uevent_object *obj; 2093 struct ib_pd *pd; 2094 struct ib_srq *srq; 2095 struct ib_cq *xrc_cq; 2096 struct ib_xrcd *xrcd; 2097 struct ib_srq_init_attr attr; 2098 struct ib_uobject *xrcd_uobj; 2099 int ret; 2100 2101 if (out_len < sizeof resp) 2102 return -ENOSPC; 2103 2104 if (copy_from_user(&cmd, buf, sizeof cmd)) 2105 return -EFAULT; 2106 2107 INIT_UDATA(&udata, buf + sizeof cmd, 2108 (unsigned long) cmd.response + sizeof resp, 2109 in_len - sizeof cmd, out_len - sizeof resp); 2110 2111 obj = kmalloc(sizeof *obj, GFP_KERNEL); 2112 if (!obj) 2113 return -ENOMEM; 2114 2115 init_uobj(&obj->uobject, cmd.user_handle, file->ucontext, 2116 &srq_lock_key); 2117 down_write(&obj->uobject.mutex); 2118 2119 pd = idr_read_pd(cmd.pd_handle, file->ucontext); 2120 if (!pd) { 2121 ret = -EINVAL; 2122 goto err; 2123 } 2124 2125 xrc_cq = idr_read_cq(cmd.cq_handle, file->ucontext, 0); 2126 if (!xrc_cq) { 2127 ret = -EINVAL; 2128 goto err_put_pd; 2129 } 2130 2131 xrcd = idr_read_xrcd(cmd.xrcd_handle, file->ucontext, &xrcd_uobj); 2132 if (!xrcd) { 2133 ret = -EINVAL; 2134 goto err_put_cq; 2135 } 2136 2137 2138 attr.event_handler = ib_uverbs_srq_event_handler; 2139 attr.srq_context = file; 2140 attr.attr.max_wr = cmd.max_wr; 2141 attr.attr.max_sge = cmd.max_sge; 2142 attr.attr.srq_limit = cmd.srq_limit; 2143 2144 obj->events_reported = 0; 2145 INIT_LIST_HEAD(&obj->event_list); 2146 2147 srq = pd->device->create_xrc_srq(pd, xrc_cq, xrcd, &attr, &udata); 2148 if (IS_ERR(srq)) { 2149 ret = PTR_ERR(srq); 2150 goto err_put; 2151 } 2152 2153 srq->device = pd->device; 2154 srq->pd = pd; 2155 srq->uobject = &obj->uobject; 2156 srq->event_handler = attr.event_handler; 2157 srq->srq_context = attr.srq_context; 2158 srq->ext.xrc.cq = xrc_cq; 2159 srq->ext.xrc.xrcd = xrcd; 2160 atomic_inc(&pd->usecnt); 2161 atomic_inc(&xrc_cq->usecnt); 2162 atomic_inc(&xrcd->usecnt); 2163 2164 atomic_set(&srq->usecnt, 0); 2165 2166 obj->uobject.object = srq; 2167 ret = idr_add_uobj(&ib_uverbs_srq_idr, &obj->uobject); 2168 if (ret) 2169 goto err_destroy; 2170 2171 memset(&resp, 0, sizeof resp); 2172 resp.srq_handle = obj->uobject.id; 2173 resp.max_wr = attr.attr.max_wr; 2174 resp.max_sge = attr.attr.max_sge; 2175 2176 if (copy_to_user((void __user *) (unsigned long) cmd.response, 2177 &resp, sizeof resp)) { 2178 ret = -EFAULT; 2179 goto err_copy; 2180 } 2181 2182 put_xrcd_read(xrcd_uobj); 2183 put_cq_read(xrc_cq); 2184 put_pd_read(pd); 2185 2186 mutex_lock(&file->mutex); 2187 list_add_tail(&obj->uobject.list, &file->ucontext->srq_list); 2188 mutex_unlock(&file->mutex); 2189 2190 obj->uobject.live = 1; 2191 2192 up_write(&obj->uobject.mutex); 2193 2194 return in_len; 2195 2196err_copy: 2197 idr_remove_uobj(&ib_uverbs_srq_idr, &obj->uobject); 2198 2199err_destroy: 2200 ib_destroy_srq(srq); 2201 2202err_put: 2203 put_xrcd_read(xrcd_uobj); 2204 2205err_put_cq: 2206 put_cq_read(xrc_cq); 2207 2208err_put_pd: 2209 put_pd_read(pd); 2210 2211err: 2212 put_uobj_write(&obj->uobject); 2213 return ret; 2214} 2215 2216ssize_t ib_uverbs_modify_srq(struct ib_uverbs_file *file, 2217 const char __user *buf, int in_len, 2218 int out_len) 2219{ 2220 struct ib_uverbs_modify_srq cmd; 2221 struct ib_udata udata; 2222 struct ib_srq *srq; 2223 struct ib_srq_attr attr; 2224 int ret; 2225 2226 if (copy_from_user(&cmd, buf, sizeof cmd)) 2227 return -EFAULT; 2228 2229 INIT_UDATA(&udata, buf + sizeof cmd, NULL, in_len - sizeof cmd, 2230 out_len); 2231 2232 srq = idr_read_srq(cmd.srq_handle, file->ucontext); 2233 if (!srq) 2234 return -EINVAL; 2235 2236 attr.max_wr = cmd.max_wr; 2237 attr.srq_limit = cmd.srq_limit; 2238 2239 ret = srq->device->modify_srq(srq, &attr, cmd.attr_mask, &udata); 2240 2241 put_srq_read(srq); 2242 2243 return ret ? ret : in_len; 2244} 2245 2246ssize_t ib_uverbs_query_srq(struct ib_uverbs_file *file, 2247 const char __user *buf, 2248 int in_len, int out_len) 2249{ 2250 struct ib_uverbs_query_srq cmd; 2251 struct ib_uverbs_query_srq_resp resp; 2252 struct ib_srq_attr attr; 2253 struct ib_srq *srq; 2254 int ret; 2255 2256 if (out_len < sizeof resp) 2257 return -ENOSPC; 2258 2259 if (copy_from_user(&cmd, buf, sizeof cmd)) 2260 return -EFAULT; 2261 2262 srq = idr_read_srq(cmd.srq_handle, file->ucontext); 2263 if (!srq) 2264 return -EINVAL; 2265 2266 ret = ib_query_srq(srq, &attr); 2267 2268 put_srq_read(srq); 2269 2270 if (ret) 2271 return ret; 2272 2273 memset(&resp, 0, sizeof resp); 2274 2275 resp.max_wr = attr.max_wr; 2276 resp.max_sge = attr.max_sge; 2277 resp.srq_limit = attr.srq_limit; 2278 2279 if (copy_to_user((void __user *) (unsigned long) cmd.response, 2280 &resp, sizeof resp)) 2281 return -EFAULT; 2282 2283 return in_len; 2284} 2285 2286ssize_t ib_uverbs_destroy_srq(struct ib_uverbs_file *file, 2287 const char __user *buf, int in_len, 2288 int out_len) 2289{ 2290 struct ib_uverbs_destroy_srq cmd; 2291 struct ib_uverbs_destroy_srq_resp resp; 2292 struct ib_uobject *uobj; 2293 struct ib_srq *srq; 2294 struct ib_uevent_object *obj; 2295 int ret = -EINVAL; 2296 2297 if (copy_from_user(&cmd, buf, sizeof cmd)) 2298 return -EFAULT; 2299 2300 uobj = idr_write_uobj(&ib_uverbs_srq_idr, cmd.srq_handle, file->ucontext); 2301 if (!uobj) 2302 return -EINVAL; 2303 srq = uobj->object; 2304 obj = container_of(uobj, struct ib_uevent_object, uobject); 2305 2306 ret = ib_destroy_srq(srq); 2307 if (!ret) 2308 uobj->live = 0; 2309 2310 put_uobj_write(uobj); 2311 2312 if (ret) 2313 return ret; 2314 2315 idr_remove_uobj(&ib_uverbs_srq_idr, uobj); 2316 2317 mutex_lock(&file->mutex); 2318 list_del(&uobj->list); 2319 mutex_unlock(&file->mutex); 2320 2321 ib_uverbs_release_uevent(file, obj); 2322 2323 memset(&resp, 0, sizeof resp); 2324 resp.events_reported = obj->events_reported; 2325 2326 put_uobj(uobj); 2327 2328 if (copy_to_user((void __user *) (unsigned long) cmd.response, 2329 &resp, sizeof resp)) 2330 ret = -EFAULT; 2331 2332 return ret ? ret : in_len; 2333} 2334 2335static struct inode *xrc_file2inode(struct file *f) 2336{ 2337 return f->f_dentry->d_inode; 2338} 2339 2340struct xrcd_table_entry { 2341 struct rb_node node; 2342 struct inode *inode; 2343 struct ib_xrcd *xrcd; 2344}; 2345 2346static int xrcd_table_insert(struct ib_device *dev, 2347 struct inode *i_n, 2348 struct ib_xrcd *xrcd) 2349{ 2350 struct xrcd_table_entry *entry, *scan; 2351 struct rb_node **p = &dev->ib_uverbs_xrcd_table.rb_node; 2352 struct rb_node *parent = NULL; 2353 2354 entry = kmalloc(sizeof(struct xrcd_table_entry), GFP_KERNEL); 2355 if (!entry) 2356 return -ENOMEM; 2357 2358 entry->inode = i_n; 2359 entry->xrcd = xrcd; 2360 2361 while (*p) { 2362 parent = *p; 2363 scan = rb_entry(parent, struct xrcd_table_entry, node); 2364 2365 if (i_n < scan->inode) 2366 p = &(*p)->rb_left; 2367 else if (i_n > scan->inode) 2368 p = &(*p)->rb_right; 2369 else { 2370 kfree(entry); 2371 return -EEXIST; 2372 } 2373 } 2374 2375 rb_link_node(&entry->node, parent, p); 2376 rb_insert_color(&entry->node, &dev->ib_uverbs_xrcd_table); 2377 igrab(i_n); 2378 return 0; 2379} 2380 2381static struct xrcd_table_entry *xrcd_table_search(struct ib_device *dev, 2382 struct inode *i_n) 2383{ 2384 struct xrcd_table_entry *scan; 2385 struct rb_node **p = &dev->ib_uverbs_xrcd_table.rb_node; 2386 struct rb_node *parent = NULL; 2387 2388 while (*p) { 2389 parent = *p; 2390 scan = rb_entry(parent, struct xrcd_table_entry, node); 2391 2392 if (i_n < scan->inode) 2393 p = &(*p)->rb_left; 2394 else if (i_n > scan->inode) 2395 p = &(*p)->rb_right; 2396 else 2397 return scan; 2398 } 2399 return NULL; 2400} 2401 2402static int find_xrcd(struct ib_device *dev, struct inode *i_n, 2403 struct ib_xrcd **xrcd) 2404{ 2405 struct xrcd_table_entry *entry; 2406 2407 entry = xrcd_table_search(dev, i_n); 2408 if (!entry) 2409 return -EINVAL; 2410 2411 *xrcd = entry->xrcd; 2412 return 0; 2413} 2414 2415 2416static void xrcd_table_delete(struct ib_device *dev, 2417 struct inode *i_n) 2418{ 2419 struct xrcd_table_entry *entry = xrcd_table_search(dev, i_n); 2420 2421 if (entry) { 2422 iput(i_n); 2423 rb_erase(&entry->node, &dev->ib_uverbs_xrcd_table); 2424 kfree(entry); 2425 } 2426} 2427 2428ssize_t ib_uverbs_open_xrc_domain(struct ib_uverbs_file *file, 2429 const char __user *buf, int in_len, 2430 int out_len) 2431{ 2432 struct ib_uverbs_open_xrc_domain cmd; 2433 struct ib_uverbs_open_xrc_domain_resp resp; 2434 struct ib_udata udata; 2435 struct ib_uobject *uobj; 2436 struct ib_uxrcd_object *xrcd_uobj; 2437 struct ib_xrcd *xrcd = NULL; 2438 struct file *f = NULL; 2439 struct inode *inode = NULL; 2440 int ret = 0; 2441 int new_xrcd = 0; 2442 2443 if (out_len < sizeof resp) 2444 return -ENOSPC; 2445 2446 if (copy_from_user(&cmd, buf, sizeof cmd)) 2447 return -EFAULT; 2448 2449 INIT_UDATA(&udata, buf + sizeof cmd, 2450 (unsigned long) cmd.response + sizeof resp, 2451 in_len - sizeof cmd, out_len - sizeof resp); 2452 2453 mutex_lock(&file->device->ib_dev->xrcd_table_mutex); 2454 if (cmd.fd != (u32) (-1)) { 2455 /* search for file descriptor */ 2456 f = fget(cmd.fd); 2457 if (!f) { 2458 ret = -EBADF; 2459 goto err_table_mutex_unlock; 2460 } 2461 2462 inode = xrc_file2inode(f); 2463 if (!inode) { 2464 ret = -EBADF; 2465 goto err_table_mutex_unlock; 2466 } 2467 2468 ret = find_xrcd(file->device->ib_dev, inode, &xrcd); 2469 if (ret && !(cmd.oflags & O_CREAT)) { 2470 /* no file descriptor. Need CREATE flag */ 2471 ret = -EAGAIN; 2472 goto err_table_mutex_unlock; 2473 } 2474 2475 if (xrcd && cmd.oflags & O_EXCL) { 2476 ret = -EINVAL; 2477 goto err_table_mutex_unlock; 2478 } 2479 } 2480 2481 xrcd_uobj = kmalloc(sizeof *xrcd_uobj, GFP_KERNEL); 2482 if (!xrcd_uobj) { 2483 ret = -ENOMEM; 2484 goto err_table_mutex_unlock; 2485 } 2486 2487 uobj = &xrcd_uobj->uobject; 2488 init_uobj(uobj, 0, file->ucontext, &pd_lock_key); 2489 down_write(&uobj->mutex); 2490 2491 if (!xrcd) { 2492 xrcd = file->device->ib_dev->alloc_xrcd(file->device->ib_dev, 2493 file->ucontext, &udata); 2494 if (IS_ERR(xrcd)) { 2495 ret = PTR_ERR(xrcd); 2496 goto err; 2497 } 2498 xrcd->uobject = (cmd.fd == -1) ? uobj : NULL; 2499 xrcd->inode = inode; 2500 xrcd->device = file->device->ib_dev; 2501 atomic_set(&xrcd->usecnt, 0); 2502 new_xrcd = 1; 2503 } 2504 2505 uobj->object = xrcd; 2506 ret = idr_add_uobj(&ib_uverbs_xrc_domain_idr, uobj); 2507 if (ret) 2508 goto err_idr; 2509 2510 memset(&resp, 0, sizeof resp); 2511 resp.xrcd_handle = uobj->id; 2512 2513 if (inode) { 2514 if (new_xrcd) { 2515 /* create new inode/xrcd table entry */ 2516 ret = xrcd_table_insert(file->device->ib_dev, inode, xrcd); 2517 if (ret) 2518 goto err_insert_xrcd; 2519 } 2520 atomic_inc(&xrcd->usecnt); 2521 } 2522 if (f) 2523 fput(f); 2524 2525 if (copy_to_user((void __user *) (unsigned long) cmd.response, 2526 &resp, sizeof resp)) { 2527 ret = -EFAULT; 2528 goto err_copy; 2529 } 2530 2531 INIT_LIST_HEAD(&xrcd_uobj->xrc_reg_qp_list); 2532 2533 mutex_lock(&file->mutex); 2534 list_add_tail(&uobj->list, &file->ucontext->xrcd_list); 2535 mutex_unlock(&file->mutex); 2536 2537 uobj->live = 1; 2538 2539 up_write(&uobj->mutex); 2540 2541 mutex_unlock(&file->device->ib_dev->xrcd_table_mutex); 2542 return in_len; 2543 2544err_copy: 2545 2546 if (inode) { 2547 if (new_xrcd) 2548 xrcd_table_delete(file->device->ib_dev, inode); 2549 atomic_dec(&xrcd->usecnt); 2550 } 2551 2552err_insert_xrcd: 2553 idr_remove_uobj(&ib_uverbs_xrc_domain_idr, uobj); 2554 2555err_idr: 2556 ib_dealloc_xrcd(xrcd); 2557 2558err: 2559 put_uobj_write(uobj); 2560 2561err_table_mutex_unlock: 2562 2563 if (f) 2564 fput(f); 2565 mutex_unlock(&file->device->ib_dev->xrcd_table_mutex); 2566 return ret; 2567} 2568 2569ssize_t ib_uverbs_close_xrc_domain(struct ib_uverbs_file *file, 2570 const char __user *buf, int in_len, 2571 int out_len) 2572{ 2573 struct ib_uverbs_close_xrc_domain cmd; 2574 struct ib_uobject *uobj, *t_uobj; 2575 struct ib_uxrcd_object *xrcd_uobj; 2576 struct ib_xrcd *xrcd = NULL; 2577 struct inode *inode = NULL; 2578 int ret = 0; 2579 2580 if (copy_from_user(&cmd, buf, sizeof cmd)) 2581 return -EFAULT; 2582 2583 mutex_lock(&file->device->ib_dev->xrcd_table_mutex); 2584 uobj = idr_write_uobj(&ib_uverbs_xrc_domain_idr, cmd.xrcd_handle, 2585 file->ucontext); 2586 if (!uobj) { 2587 ret = -EINVAL; 2588 goto err_unlock_mutex; 2589 } 2590 2591 mutex_lock(&file->mutex); 2592 if (!ret) { 2593 list_for_each_entry(t_uobj, &file->ucontext->qp_list, list) { 2594 struct ib_qp *qp = t_uobj->object; 2595 if (qp->xrcd && qp->xrcd == uobj->object) { 2596 ret = -EBUSY; 2597 break; 2598 } 2599 } 2600 } 2601 if (!ret) { 2602 list_for_each_entry(t_uobj, &file->ucontext->srq_list, list) { 2603 struct ib_srq *srq = t_uobj->object; 2604 if (srq->ext.xrc.xrcd && srq->ext.xrc.xrcd == uobj->object) { 2605 ret = -EBUSY; 2606 break; 2607 } 2608 } 2609 } 2610 mutex_unlock(&file->mutex); 2611 if (ret) { 2612 put_uobj_write(uobj); 2613 goto err_unlock_mutex; 2614 } 2615 2616 xrcd_uobj = container_of(uobj, struct ib_uxrcd_object, uobject); 2617 if (!list_empty(&xrcd_uobj->xrc_reg_qp_list)) { 2618 ret = -EBUSY; 2619 put_uobj_write(uobj); 2620 goto err_unlock_mutex; 2621 } 2622 2623 xrcd = (struct ib_xrcd *) (uobj->object); 2624 inode = xrcd->inode; 2625 2626 if (inode) 2627 atomic_dec(&xrcd->usecnt); 2628 2629 ret = ib_dealloc_xrcd(uobj->object); 2630 if (!ret) 2631 uobj->live = 0; 2632 2633 put_uobj_write(uobj); 2634 2635 if (ret && !inode) 2636 goto err_unlock_mutex; 2637 2638 if (!ret && inode) 2639 xrcd_table_delete(file->device->ib_dev, inode); 2640 2641 idr_remove_uobj(&ib_uverbs_xrc_domain_idr, uobj); 2642 2643 mutex_lock(&file->mutex); 2644 list_del(&uobj->list); 2645 mutex_unlock(&file->mutex); 2646 2647 put_uobj(uobj); 2648 2649 mutex_unlock(&file->device->ib_dev->xrcd_table_mutex); 2650 return in_len; 2651 2652err_unlock_mutex: 2653 mutex_unlock(&file->device->ib_dev->xrcd_table_mutex); 2654 return ret; 2655} 2656 2657void ib_uverbs_dealloc_xrcd(struct ib_device *ib_dev, 2658 struct ib_xrcd *xrcd) 2659{ 2660 struct inode *inode = NULL; 2661 int ret = 0; 2662 2663 inode = xrcd->inode; 2664 if (inode) 2665 atomic_dec(&xrcd->usecnt); 2666 2667 ret = ib_dealloc_xrcd(xrcd); 2668 if (!ret && inode) 2669 xrcd_table_delete(ib_dev, inode); 2670} 2671 2672ssize_t ib_uverbs_create_xrc_rcv_qp(struct ib_uverbs_file *file, 2673 const char __user *buf, int in_len, 2674 int out_len) 2675{ 2676 struct ib_uverbs_create_xrc_rcv_qp cmd; 2677 struct ib_uverbs_create_xrc_rcv_qp_resp resp; 2678 struct ib_uxrc_rcv_object *obj; 2679 struct ib_qp_init_attr init_attr; 2680 struct ib_xrcd *xrcd; 2681 struct ib_uobject *uobj; 2682 struct ib_uxrcd_object *xrcd_uobj; 2683 u32 qp_num; 2684 int err; 2685 2686 if (out_len < sizeof resp) 2687 return -ENOSPC; 2688 2689 if (copy_from_user(&cmd, buf, sizeof cmd)) 2690 return -EFAULT; 2691 2692 obj = kzalloc(sizeof *obj, GFP_KERNEL); 2693 if (!obj) 2694 return -ENOMEM; 2695 2696 xrcd = idr_read_xrcd(cmd.xrc_domain_handle, file->ucontext, &uobj); 2697 if (!xrcd) { 2698 err = -EINVAL; 2699 goto err_out; 2700 } 2701 2702 init_attr.event_handler = ib_uverbs_xrc_rcv_qp_event_handler; 2703 init_attr.qp_context = file; 2704 init_attr.srq = NULL; 2705 init_attr.sq_sig_type = 2706 cmd.sq_sig_all ? IB_SIGNAL_ALL_WR : IB_SIGNAL_REQ_WR; 2707 init_attr.qp_type = IB_QPT_XRC; 2708 init_attr.xrcd = xrcd; 2709 2710 init_attr.cap.max_send_wr = 1; 2711 init_attr.cap.max_recv_wr = 0; 2712 init_attr.cap.max_send_sge = 1; 2713 init_attr.cap.max_recv_sge = 0; 2714 init_attr.cap.max_inline_data = 0; 2715 2716 err = xrcd->device->create_xrc_rcv_qp(&init_attr, &qp_num); 2717 if (err) 2718 goto err_put; 2719 2720 memset(&resp, 0, sizeof resp); 2721 resp.qpn = qp_num; 2722 2723 if (copy_to_user((void __user *) (unsigned long) cmd.response, 2724 &resp, sizeof resp)) { 2725 err = -EFAULT; 2726 goto err_destroy; 2727 } 2728 2729 atomic_inc(&xrcd->usecnt); 2730 put_xrcd_read(uobj); 2731 obj->qp_num = qp_num; 2732 obj->domain_handle = cmd.xrc_domain_handle; 2733 xrcd_uobj = container_of(uobj, struct ib_uxrcd_object, uobject); 2734 mutex_lock(&file->device->ib_dev->xrcd_table_mutex); 2735 list_add_tail(&obj->list, &xrcd_uobj->xrc_reg_qp_list); 2736 mutex_unlock(&file->device->ib_dev->xrcd_table_mutex); 2737 2738 return in_len; 2739 2740err_destroy: 2741 xrcd->device->unreg_xrc_rcv_qp(xrcd, file, qp_num); 2742err_put: 2743 put_xrcd_read(uobj); 2744err_out: 2745 kfree(obj); 2746 return err; 2747} 2748 2749ssize_t ib_uverbs_modify_xrc_rcv_qp(struct ib_uverbs_file *file, 2750 const char __user *buf, int in_len, 2751 int out_len) 2752{ 2753 struct ib_uverbs_modify_xrc_rcv_qp cmd; 2754 struct ib_qp_attr *attr; 2755 struct ib_xrcd *xrcd; 2756 struct ib_uobject *uobj; 2757 int err; 2758 2759 if (copy_from_user(&cmd, buf, sizeof cmd)) 2760 return -EFAULT; 2761 2762 attr = kzalloc(sizeof *attr, GFP_KERNEL); 2763 if (!attr) 2764 return -ENOMEM; 2765 2766 xrcd = idr_read_xrcd(cmd.xrc_domain_handle, file->ucontext, &uobj); 2767 if (!xrcd) { 2768 kfree(attr); 2769 return -EINVAL; 2770 } 2771 2772 attr->qp_state = cmd.qp_state; 2773 attr->cur_qp_state = cmd.cur_qp_state; 2774 attr->qp_access_flags = cmd.qp_access_flags; 2775 attr->pkey_index = cmd.pkey_index; 2776 attr->port_num = cmd.port_num; 2777 attr->path_mtu = cmd.path_mtu; 2778 attr->path_mig_state = cmd.path_mig_state; 2779 attr->qkey = cmd.qkey; 2780 attr->rq_psn = cmd.rq_psn; 2781 attr->sq_psn = cmd.sq_psn; 2782 attr->dest_qp_num = cmd.dest_qp_num; 2783 attr->alt_pkey_index = cmd.alt_pkey_index; 2784 attr->en_sqd_async_notify = cmd.en_sqd_async_notify; 2785 attr->max_rd_atomic = cmd.max_rd_atomic; 2786 attr->max_dest_rd_atomic = cmd.max_dest_rd_atomic; 2787 attr->min_rnr_timer = cmd.min_rnr_timer; 2788 attr->port_num = cmd.port_num; 2789 attr->timeout = cmd.timeout; 2790 attr->retry_cnt = cmd.retry_cnt; 2791 attr->rnr_retry = cmd.rnr_retry; 2792 attr->alt_port_num = cmd.alt_port_num; 2793 attr->alt_timeout = cmd.alt_timeout; 2794 2795 memcpy(attr->ah_attr.grh.dgid.raw, cmd.dest.dgid, 16); 2796 attr->ah_attr.grh.flow_label = cmd.dest.flow_label; 2797 attr->ah_attr.grh.sgid_index = cmd.dest.sgid_index; 2798 attr->ah_attr.grh.hop_limit = cmd.dest.hop_limit; 2799 attr->ah_attr.grh.traffic_class = cmd.dest.traffic_class; 2800 attr->ah_attr.dlid = cmd.dest.dlid; 2801 attr->ah_attr.sl = cmd.dest.sl; 2802 attr->ah_attr.src_path_bits = cmd.dest.src_path_bits; 2803 attr->ah_attr.static_rate = cmd.dest.static_rate; 2804 attr->ah_attr.ah_flags = cmd.dest.is_global ? IB_AH_GRH : 0; 2805 attr->ah_attr.port_num = cmd.dest.port_num; 2806 2807 memcpy(attr->alt_ah_attr.grh.dgid.raw, cmd.alt_dest.dgid, 16); 2808 attr->alt_ah_attr.grh.flow_label = cmd.alt_dest.flow_label; 2809 attr->alt_ah_attr.grh.sgid_index = cmd.alt_dest.sgid_index; 2810 attr->alt_ah_attr.grh.hop_limit = cmd.alt_dest.hop_limit; 2811 attr->alt_ah_attr.grh.traffic_class = cmd.alt_dest.traffic_class; 2812 attr->alt_ah_attr.dlid = cmd.alt_dest.dlid; 2813 attr->alt_ah_attr.sl = cmd.alt_dest.sl; 2814 attr->alt_ah_attr.src_path_bits = cmd.alt_dest.src_path_bits; 2815 attr->alt_ah_attr.static_rate = cmd.alt_dest.static_rate; 2816 attr->alt_ah_attr.ah_flags = cmd.alt_dest.is_global ? IB_AH_GRH : 0; 2817 attr->alt_ah_attr.port_num = cmd.alt_dest.port_num; 2818 2819 err = xrcd->device->modify_xrc_rcv_qp(xrcd, cmd.qp_num, attr, cmd.attr_mask); 2820 put_xrcd_read(uobj); 2821 kfree(attr); 2822 return err ? err : in_len; 2823} 2824 2825ssize_t ib_uverbs_query_xrc_rcv_qp(struct ib_uverbs_file *file, 2826 const char __user *buf, int in_len, 2827 int out_len) 2828{ 2829 struct ib_uverbs_query_xrc_rcv_qp cmd; 2830 struct ib_uverbs_query_qp_resp resp; 2831 struct ib_qp_attr *attr; 2832 struct ib_qp_init_attr *init_attr; 2833 struct ib_xrcd *xrcd; 2834 struct ib_uobject *uobj; 2835 int ret; 2836 2837 if (copy_from_user(&cmd, buf, sizeof cmd)) 2838 return -EFAULT; 2839 2840 attr = kmalloc(sizeof *attr, GFP_KERNEL); 2841 init_attr = kmalloc(sizeof *init_attr, GFP_KERNEL); 2842 if (!attr || !init_attr) { 2843 ret = -ENOMEM; 2844 goto out; 2845 } 2846 2847 xrcd = idr_read_xrcd(cmd.xrc_domain_handle, file->ucontext, &uobj); 2848 if (!xrcd) { 2849 ret = -EINVAL; 2850 goto out; 2851 } 2852 2853 ret = xrcd->device->query_xrc_rcv_qp(xrcd, cmd.qp_num, attr, 2854 cmd.attr_mask, init_attr); 2855 2856 put_xrcd_read(uobj); 2857 2858 if (ret) 2859 goto out; 2860 2861 memset(&resp, 0, sizeof resp); 2862 resp.qp_state = attr->qp_state; 2863 resp.cur_qp_state = attr->cur_qp_state; 2864 resp.path_mtu = attr->path_mtu; 2865 resp.path_mig_state = attr->path_mig_state; 2866 resp.qkey = attr->qkey; 2867 resp.rq_psn = attr->rq_psn; 2868 resp.sq_psn = attr->sq_psn; 2869 resp.dest_qp_num = attr->dest_qp_num; 2870 resp.qp_access_flags = attr->qp_access_flags; 2871 resp.pkey_index = attr->pkey_index; 2872 resp.alt_pkey_index = attr->alt_pkey_index; 2873 resp.sq_draining = attr->sq_draining; 2874 resp.max_rd_atomic = attr->max_rd_atomic; 2875 resp.max_dest_rd_atomic = attr->max_dest_rd_atomic; 2876 resp.min_rnr_timer = attr->min_rnr_timer; 2877 resp.port_num = attr->port_num; 2878 resp.timeout = attr->timeout; 2879 resp.retry_cnt = attr->retry_cnt; 2880 resp.rnr_retry = attr->rnr_retry; 2881 resp.alt_port_num = attr->alt_port_num; 2882 resp.alt_timeout = attr->alt_timeout; 2883 2884 memcpy(resp.dest.dgid, attr->ah_attr.grh.dgid.raw, 16); 2885 resp.dest.flow_label = attr->ah_attr.grh.flow_label; 2886 resp.dest.sgid_index = attr->ah_attr.grh.sgid_index; 2887 resp.dest.hop_limit = attr->ah_attr.grh.hop_limit; 2888 resp.dest.traffic_class = attr->ah_attr.grh.traffic_class; 2889 resp.dest.dlid = attr->ah_attr.dlid; 2890 resp.dest.sl = attr->ah_attr.sl; 2891 resp.dest.src_path_bits = attr->ah_attr.src_path_bits; 2892 resp.dest.static_rate = attr->ah_attr.static_rate; 2893 resp.dest.is_global = !!(attr->ah_attr.ah_flags & IB_AH_GRH); 2894 resp.dest.port_num = attr->ah_attr.port_num; 2895 2896 memcpy(resp.alt_dest.dgid, attr->alt_ah_attr.grh.dgid.raw, 16); 2897 resp.alt_dest.flow_label = attr->alt_ah_attr.grh.flow_label; 2898 resp.alt_dest.sgid_index = attr->alt_ah_attr.grh.sgid_index; 2899 resp.alt_dest.hop_limit = attr->alt_ah_attr.grh.hop_limit; 2900 resp.alt_dest.traffic_class = attr->alt_ah_attr.grh.traffic_class; 2901 resp.alt_dest.dlid = attr->alt_ah_attr.dlid; 2902 resp.alt_dest.sl = attr->alt_ah_attr.sl; 2903 resp.alt_dest.src_path_bits = attr->alt_ah_attr.src_path_bits; 2904 resp.alt_dest.static_rate = attr->alt_ah_attr.static_rate; 2905 resp.alt_dest.is_global = !!(attr->alt_ah_attr.ah_flags & IB_AH_GRH); 2906 resp.alt_dest.port_num = attr->alt_ah_attr.port_num; 2907 2908 resp.max_send_wr = init_attr->cap.max_send_wr; 2909 resp.max_recv_wr = init_attr->cap.max_recv_wr; 2910 resp.max_send_sge = init_attr->cap.max_send_sge; 2911 resp.max_recv_sge = init_attr->cap.max_recv_sge; 2912 resp.max_inline_data = init_attr->cap.max_inline_data; 2913 resp.sq_sig_all = init_attr->sq_sig_type == IB_SIGNAL_ALL_WR; 2914 2915 if (copy_to_user((void __user *) (unsigned long) cmd.response, 2916 &resp, sizeof resp)) 2917 ret = -EFAULT; 2918 2919out: 2920 kfree(attr); 2921 kfree(init_attr); 2922 2923 return ret ? ret : in_len; 2924} 2925 2926ssize_t ib_uverbs_reg_xrc_rcv_qp(struct ib_uverbs_file *file, 2927 const char __user *buf, int in_len, 2928 int out_len) 2929{ 2930 struct ib_uverbs_reg_xrc_rcv_qp cmd; 2931 struct ib_uxrc_rcv_object *qp_obj, *tmp; 2932 struct ib_xrcd *xrcd; 2933 struct ib_uobject *uobj; 2934 struct ib_uxrcd_object *xrcd_uobj; 2935 int ret; 2936 2937 if (copy_from_user(&cmd, buf, sizeof cmd)) 2938 return -EFAULT; 2939 2940 qp_obj = kmalloc(sizeof *qp_obj, GFP_KERNEL); 2941 if (!qp_obj) 2942 return -ENOMEM; 2943 2944 xrcd = idr_read_xrcd(cmd.xrc_domain_handle, file->ucontext, &uobj); 2945 if (!xrcd) { 2946 ret = -EINVAL; 2947 goto err_out; 2948 } 2949 2950 ret = xrcd->device->reg_xrc_rcv_qp(xrcd, file, cmd.qp_num); 2951 if (ret) 2952 goto err_put; 2953 2954 xrcd_uobj = container_of(uobj, struct ib_uxrcd_object, uobject); 2955 mutex_lock(&file->device->ib_dev->xrcd_table_mutex); 2956 list_for_each_entry(tmp, &xrcd_uobj->xrc_reg_qp_list, list) 2957 if (cmd.qp_num == tmp->qp_num) { 2958 kfree(qp_obj); 2959 mutex_unlock(&file->device->ib_dev->xrcd_table_mutex); 2960 put_xrcd_read(uobj); 2961 return in_len; 2962 } 2963 qp_obj->qp_num = cmd.qp_num; 2964 qp_obj->domain_handle = cmd.xrc_domain_handle; 2965 list_add_tail(&qp_obj->list, &xrcd_uobj->xrc_reg_qp_list); 2966 mutex_unlock(&file->device->ib_dev->xrcd_table_mutex); 2967 atomic_inc(&xrcd->usecnt); 2968 put_xrcd_read(uobj); 2969 return in_len; 2970 2971err_put: 2972 put_xrcd_read(uobj); 2973err_out: 2974 2975 kfree(qp_obj); 2976 return ret; 2977} 2978 2979int ib_uverbs_cleanup_xrc_rcv_qp(struct ib_uverbs_file *file, 2980 struct ib_xrcd *xrcd, u32 qp_num) 2981{ 2982 int err; 2983 err = xrcd->device->unreg_xrc_rcv_qp(xrcd, file, qp_num); 2984 if (!err) 2985 atomic_dec(&xrcd->usecnt); 2986 return err; 2987} 2988 2989ssize_t ib_uverbs_unreg_xrc_rcv_qp(struct ib_uverbs_file *file, 2990 const char __user *buf, int in_len, 2991 int out_len) 2992{ 2993 struct ib_uverbs_unreg_xrc_rcv_qp cmd; 2994 struct ib_uxrc_rcv_object *qp_obj, *tmp; 2995 struct ib_xrcd *xrcd; 2996 struct ib_uobject *uobj; 2997 struct ib_uxrcd_object *xrcd_uobj; 2998 int ret; 2999 3000 if (copy_from_user(&cmd, buf, sizeof cmd)) 3001 return -EFAULT; 3002 3003 xrcd = idr_read_xrcd(cmd.xrc_domain_handle, file->ucontext, &uobj); 3004 if (!xrcd) 3005 return -EINVAL; 3006 3007 ret = xrcd->device->unreg_xrc_rcv_qp(xrcd, file, cmd.qp_num); 3008 if (ret) { 3009 put_xrcd_read(uobj); 3010 return -EINVAL; 3011 } 3012 atomic_dec(&xrcd->usecnt); 3013 3014 xrcd_uobj = container_of(uobj, struct ib_uxrcd_object, uobject); 3015 mutex_lock(&file->device->ib_dev->xrcd_table_mutex); 3016 list_for_each_entry_safe(qp_obj, tmp, &xrcd_uobj->xrc_reg_qp_list, list) 3017 if (cmd.qp_num == qp_obj->qp_num) { 3018 list_del(&qp_obj->list); 3019 kfree(qp_obj); 3020 break; 3021 } 3022 mutex_unlock(&file->device->ib_dev->xrcd_table_mutex); 3023 put_xrcd_read(uobj); 3024 return in_len; 3025} 3026