1/* 2 * Copyright (c) 2005-2006 Intel Corporation. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33#include <linux/completion.h> 34#include <linux/file.h> 35#include <linux/mutex.h> 36#include <linux/poll.h> 37#include <linux/sched.h> 38#include <linux/idr.h> 39#include <linux/in.h> 40#include <linux/in6.h> 41#include <linux/miscdevice.h> 42#include <linux/slab.h> 43#include <linux/sysctl.h> 44#include <linux/module.h> 45#include <linux/nsproxy.h> 46 47#include <linux/nospec.h> 48 49#include <rdma/rdma_user_cm.h> 50#include <rdma/ib_marshall.h> 51#include <rdma/rdma_cm.h> 52#include <rdma/rdma_cm_ib.h> 53#include <rdma/ib_addr.h> 54#include <rdma/ib.h> 55#include <rdma/ib_cm.h> 56#include <rdma/rdma_netlink.h> 57#include "core_priv.h" 58 59MODULE_AUTHOR("Sean Hefty"); 60MODULE_DESCRIPTION("RDMA Userspace Connection Manager Access"); 61MODULE_LICENSE("Dual BSD/GPL"); 62 63static unsigned int max_backlog = 1024; 64 65static struct ctl_table_header *ucma_ctl_table_hdr; 66static struct ctl_table ucma_ctl_table[] = { 67 { 68 .procname = "max_backlog", 69 .data = &max_backlog, 70 .maxlen = sizeof max_backlog, 71 .mode = 0644, 72 .proc_handler = proc_dointvec, 73 }, 74}; 75 76struct ucma_file { 77 struct mutex mut; 78 struct file *filp; 79 struct list_head ctx_list; 80 struct list_head event_list; 81 wait_queue_head_t poll_wait; 82}; 83 84struct ucma_context { 85 u32 id; 86 struct completion comp; 87 refcount_t ref; 88 int events_reported; 89 atomic_t backlog; 90 91 struct ucma_file *file; 92 struct rdma_cm_id *cm_id; 93 struct mutex mutex; 94 u64 uid; 95 96 struct list_head list; 97 struct list_head mc_list; 98 struct work_struct close_work; 99}; 100 101struct ucma_multicast { 102 struct ucma_context *ctx; 103 u32 id; 104 int events_reported; 105 106 u64 uid; 107 u8 join_state; 108 struct list_head list; 109 struct sockaddr_storage addr; 110}; 111 112struct ucma_event { 113 struct ucma_context *ctx; 114 struct ucma_context *conn_req_ctx; 115 struct ucma_multicast *mc; 116 struct list_head list; 117 struct rdma_ucm_event_resp resp; 118}; 119 120static DEFINE_XARRAY_ALLOC(ctx_table); 121static DEFINE_XARRAY_ALLOC(multicast_table); 122 123static const struct file_operations ucma_fops; 124static int ucma_destroy_private_ctx(struct ucma_context *ctx); 125 126static inline struct ucma_context *_ucma_find_context(int id, 127 struct ucma_file *file) 128{ 129 struct ucma_context *ctx; 130 131 ctx = xa_load(&ctx_table, id); 132 if (!ctx) 133 ctx = ERR_PTR(-ENOENT); 134 else if (ctx->file != file) 135 ctx = ERR_PTR(-EINVAL); 136 return ctx; 137} 138 139static struct ucma_context *ucma_get_ctx(struct ucma_file *file, int id) 140{ 141 struct ucma_context *ctx; 142 143 xa_lock(&ctx_table); 144 ctx = _ucma_find_context(id, file); 145 if (!IS_ERR(ctx)) 146 if (!refcount_inc_not_zero(&ctx->ref)) 147 ctx = ERR_PTR(-ENXIO); 148 xa_unlock(&ctx_table); 149 return ctx; 150} 151 152static void ucma_put_ctx(struct ucma_context *ctx) 153{ 154 if (refcount_dec_and_test(&ctx->ref)) 155 complete(&ctx->comp); 156} 157 158/* 159 * Same as ucm_get_ctx but requires that ->cm_id->device is valid, eg that the 160 * CM_ID is bound. 161 */ 162static struct ucma_context *ucma_get_ctx_dev(struct ucma_file *file, int id) 163{ 164 struct ucma_context *ctx = ucma_get_ctx(file, id); 165 166 if (IS_ERR(ctx)) 167 return ctx; 168 if (!ctx->cm_id->device) { 169 ucma_put_ctx(ctx); 170 return ERR_PTR(-EINVAL); 171 } 172 return ctx; 173} 174 175static void ucma_close_id(struct work_struct *work) 176{ 177 struct ucma_context *ctx = container_of(work, struct ucma_context, close_work); 178 179 /* once all inflight tasks are finished, we close all underlying 180 * resources. The context is still alive till its explicit destryoing 181 * by its creator. This puts back the xarray's reference. 182 */ 183 ucma_put_ctx(ctx); 184 wait_for_completion(&ctx->comp); 185 /* No new events will be generated after destroying the id. */ 186 rdma_destroy_id(ctx->cm_id); 187 188 /* Reading the cm_id without holding a positive ref is not allowed */ 189 ctx->cm_id = NULL; 190} 191 192static struct ucma_context *ucma_alloc_ctx(struct ucma_file *file) 193{ 194 struct ucma_context *ctx; 195 196 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); 197 if (!ctx) 198 return NULL; 199 200 INIT_WORK(&ctx->close_work, ucma_close_id); 201 init_completion(&ctx->comp); 202 INIT_LIST_HEAD(&ctx->mc_list); 203 /* So list_del() will work if we don't do ucma_finish_ctx() */ 204 INIT_LIST_HEAD(&ctx->list); 205 ctx->file = file; 206 mutex_init(&ctx->mutex); 207 208 if (xa_alloc(&ctx_table, &ctx->id, NULL, xa_limit_32b, GFP_KERNEL)) { 209 kfree(ctx); 210 return NULL; 211 } 212 return ctx; 213} 214 215static void ucma_set_ctx_cm_id(struct ucma_context *ctx, 216 struct rdma_cm_id *cm_id) 217{ 218 refcount_set(&ctx->ref, 1); 219 ctx->cm_id = cm_id; 220} 221 222static void ucma_finish_ctx(struct ucma_context *ctx) 223{ 224 lockdep_assert_held(&ctx->file->mut); 225 list_add_tail(&ctx->list, &ctx->file->ctx_list); 226 xa_store(&ctx_table, ctx->id, ctx, GFP_KERNEL); 227} 228 229static void ucma_copy_conn_event(struct rdma_ucm_conn_param *dst, 230 struct rdma_conn_param *src) 231{ 232 if (src->private_data_len) 233 memcpy(dst->private_data, src->private_data, 234 src->private_data_len); 235 dst->private_data_len = src->private_data_len; 236 dst->responder_resources = src->responder_resources; 237 dst->initiator_depth = src->initiator_depth; 238 dst->flow_control = src->flow_control; 239 dst->retry_count = src->retry_count; 240 dst->rnr_retry_count = src->rnr_retry_count; 241 dst->srq = src->srq; 242 dst->qp_num = src->qp_num; 243} 244 245static void ucma_copy_ud_event(struct ib_device *device, 246 struct rdma_ucm_ud_param *dst, 247 struct rdma_ud_param *src) 248{ 249 if (src->private_data_len) 250 memcpy(dst->private_data, src->private_data, 251 src->private_data_len); 252 dst->private_data_len = src->private_data_len; 253 ib_copy_ah_attr_to_user(device, &dst->ah_attr, &src->ah_attr); 254 dst->qp_num = src->qp_num; 255 dst->qkey = src->qkey; 256} 257 258static struct ucma_event *ucma_create_uevent(struct ucma_context *ctx, 259 struct rdma_cm_event *event) 260{ 261 struct ucma_event *uevent; 262 263 uevent = kzalloc(sizeof(*uevent), GFP_KERNEL); 264 if (!uevent) 265 return NULL; 266 267 uevent->ctx = ctx; 268 switch (event->event) { 269 case RDMA_CM_EVENT_MULTICAST_JOIN: 270 case RDMA_CM_EVENT_MULTICAST_ERROR: 271 uevent->mc = (struct ucma_multicast *) 272 event->param.ud.private_data; 273 uevent->resp.uid = uevent->mc->uid; 274 uevent->resp.id = uevent->mc->id; 275 break; 276 default: 277 uevent->resp.uid = ctx->uid; 278 uevent->resp.id = ctx->id; 279 break; 280 } 281 uevent->resp.event = event->event; 282 uevent->resp.status = event->status; 283 if (ctx->cm_id->qp_type == IB_QPT_UD) 284 ucma_copy_ud_event(ctx->cm_id->device, &uevent->resp.param.ud, 285 &event->param.ud); 286 else 287 ucma_copy_conn_event(&uevent->resp.param.conn, 288 &event->param.conn); 289 290 uevent->resp.ece.vendor_id = event->ece.vendor_id; 291 uevent->resp.ece.attr_mod = event->ece.attr_mod; 292 return uevent; 293} 294 295static int ucma_connect_event_handler(struct rdma_cm_id *cm_id, 296 struct rdma_cm_event *event) 297{ 298 struct ucma_context *listen_ctx = cm_id->context; 299 struct ucma_context *ctx; 300 struct ucma_event *uevent; 301 302 if (!atomic_add_unless(&listen_ctx->backlog, -1, 0)) 303 return -ENOMEM; 304 ctx = ucma_alloc_ctx(listen_ctx->file); 305 if (!ctx) 306 goto err_backlog; 307 ucma_set_ctx_cm_id(ctx, cm_id); 308 309 uevent = ucma_create_uevent(listen_ctx, event); 310 if (!uevent) 311 goto err_alloc; 312 uevent->conn_req_ctx = ctx; 313 uevent->resp.id = ctx->id; 314 315 ctx->cm_id->context = ctx; 316 317 mutex_lock(&ctx->file->mut); 318 ucma_finish_ctx(ctx); 319 list_add_tail(&uevent->list, &ctx->file->event_list); 320 mutex_unlock(&ctx->file->mut); 321 wake_up_interruptible(&ctx->file->poll_wait); 322 return 0; 323 324err_alloc: 325 ucma_destroy_private_ctx(ctx); 326err_backlog: 327 atomic_inc(&listen_ctx->backlog); 328 /* Returning error causes the new ID to be destroyed */ 329 return -ENOMEM; 330} 331 332static int ucma_event_handler(struct rdma_cm_id *cm_id, 333 struct rdma_cm_event *event) 334{ 335 struct ucma_event *uevent; 336 struct ucma_context *ctx = cm_id->context; 337 338 if (event->event == RDMA_CM_EVENT_CONNECT_REQUEST) 339 return ucma_connect_event_handler(cm_id, event); 340 341 /* 342 * We ignore events for new connections until userspace has set their 343 * context. This can only happen if an error occurs on a new connection 344 * before the user accepts it. This is okay, since the accept will just 345 * fail later. However, we do need to release the underlying HW 346 * resources in case of a device removal event. 347 */ 348 if (ctx->uid) { 349 uevent = ucma_create_uevent(ctx, event); 350 if (!uevent) 351 return 0; 352 353 mutex_lock(&ctx->file->mut); 354 list_add_tail(&uevent->list, &ctx->file->event_list); 355 mutex_unlock(&ctx->file->mut); 356 wake_up_interruptible(&ctx->file->poll_wait); 357 } 358 359 if (event->event == RDMA_CM_EVENT_DEVICE_REMOVAL) { 360 xa_lock(&ctx_table); 361 if (xa_load(&ctx_table, ctx->id) == ctx) 362 queue_work(system_unbound_wq, &ctx->close_work); 363 xa_unlock(&ctx_table); 364 } 365 return 0; 366} 367 368static ssize_t ucma_get_event(struct ucma_file *file, const char __user *inbuf, 369 int in_len, int out_len) 370{ 371 struct rdma_ucm_get_event cmd; 372 struct ucma_event *uevent; 373 374 /* 375 * Old 32 bit user space does not send the 4 byte padding in the 376 * reserved field. We don't care, allow it to keep working. 377 */ 378 if (out_len < sizeof(uevent->resp) - sizeof(uevent->resp.reserved) - 379 sizeof(uevent->resp.ece)) 380 return -ENOSPC; 381 382 if (copy_from_user(&cmd, inbuf, sizeof(cmd))) 383 return -EFAULT; 384 385 mutex_lock(&file->mut); 386 while (list_empty(&file->event_list)) { 387 mutex_unlock(&file->mut); 388 389 if (file->filp->f_flags & O_NONBLOCK) 390 return -EAGAIN; 391 392 if (wait_event_interruptible(file->poll_wait, 393 !list_empty(&file->event_list))) 394 return -ERESTARTSYS; 395 396 mutex_lock(&file->mut); 397 } 398 399 uevent = list_first_entry(&file->event_list, struct ucma_event, list); 400 401 if (copy_to_user(u64_to_user_ptr(cmd.response), 402 &uevent->resp, 403 min_t(size_t, out_len, sizeof(uevent->resp)))) { 404 mutex_unlock(&file->mut); 405 return -EFAULT; 406 } 407 408 list_del(&uevent->list); 409 uevent->ctx->events_reported++; 410 if (uevent->mc) 411 uevent->mc->events_reported++; 412 if (uevent->resp.event == RDMA_CM_EVENT_CONNECT_REQUEST) 413 atomic_inc(&uevent->ctx->backlog); 414 mutex_unlock(&file->mut); 415 416 kfree(uevent); 417 return 0; 418} 419 420static int ucma_get_qp_type(struct rdma_ucm_create_id *cmd, enum ib_qp_type *qp_type) 421{ 422 switch (cmd->ps) { 423 case RDMA_PS_TCP: 424 *qp_type = IB_QPT_RC; 425 return 0; 426 case RDMA_PS_UDP: 427 case RDMA_PS_IPOIB: 428 *qp_type = IB_QPT_UD; 429 return 0; 430 case RDMA_PS_IB: 431 *qp_type = cmd->qp_type; 432 return 0; 433 default: 434 return -EINVAL; 435 } 436} 437 438static ssize_t ucma_create_id(struct ucma_file *file, const char __user *inbuf, 439 int in_len, int out_len) 440{ 441 struct rdma_ucm_create_id cmd; 442 struct rdma_ucm_create_id_resp resp; 443 struct ucma_context *ctx; 444 struct rdma_cm_id *cm_id; 445 enum ib_qp_type qp_type; 446 int ret; 447 448 if (out_len < sizeof(resp)) 449 return -ENOSPC; 450 451 if (copy_from_user(&cmd, inbuf, sizeof(cmd))) 452 return -EFAULT; 453 454 ret = ucma_get_qp_type(&cmd, &qp_type); 455 if (ret) 456 return ret; 457 458 ctx = ucma_alloc_ctx(file); 459 if (!ctx) 460 return -ENOMEM; 461 462 ctx->uid = cmd.uid; 463 cm_id = rdma_create_user_id(ucma_event_handler, ctx, cmd.ps, qp_type); 464 if (IS_ERR(cm_id)) { 465 ret = PTR_ERR(cm_id); 466 goto err1; 467 } 468 ucma_set_ctx_cm_id(ctx, cm_id); 469 470 resp.id = ctx->id; 471 if (copy_to_user(u64_to_user_ptr(cmd.response), 472 &resp, sizeof(resp))) { 473 ret = -EFAULT; 474 goto err1; 475 } 476 477 mutex_lock(&file->mut); 478 ucma_finish_ctx(ctx); 479 mutex_unlock(&file->mut); 480 return 0; 481 482err1: 483 ucma_destroy_private_ctx(ctx); 484 return ret; 485} 486 487static void ucma_cleanup_multicast(struct ucma_context *ctx) 488{ 489 struct ucma_multicast *mc, *tmp; 490 491 xa_lock(&multicast_table); 492 list_for_each_entry_safe(mc, tmp, &ctx->mc_list, list) { 493 list_del(&mc->list); 494 /* 495 * At this point mc->ctx->ref is 0 so the mc cannot leave the 496 * lock on the reader and this is enough serialization 497 */ 498 __xa_erase(&multicast_table, mc->id); 499 kfree(mc); 500 } 501 xa_unlock(&multicast_table); 502} 503 504static void ucma_cleanup_mc_events(struct ucma_multicast *mc) 505{ 506 struct ucma_event *uevent, *tmp; 507 508 rdma_lock_handler(mc->ctx->cm_id); 509 mutex_lock(&mc->ctx->file->mut); 510 list_for_each_entry_safe(uevent, tmp, &mc->ctx->file->event_list, list) { 511 if (uevent->mc != mc) 512 continue; 513 514 list_del(&uevent->list); 515 kfree(uevent); 516 } 517 mutex_unlock(&mc->ctx->file->mut); 518 rdma_unlock_handler(mc->ctx->cm_id); 519} 520 521static int ucma_cleanup_ctx_events(struct ucma_context *ctx) 522{ 523 int events_reported; 524 struct ucma_event *uevent, *tmp; 525 LIST_HEAD(list); 526 527 /* Cleanup events not yet reported to the user.*/ 528 mutex_lock(&ctx->file->mut); 529 list_for_each_entry_safe(uevent, tmp, &ctx->file->event_list, list) { 530 if (uevent->ctx != ctx) 531 continue; 532 533 if (uevent->resp.event == RDMA_CM_EVENT_CONNECT_REQUEST && 534 xa_cmpxchg(&ctx_table, uevent->conn_req_ctx->id, 535 uevent->conn_req_ctx, XA_ZERO_ENTRY, 536 GFP_KERNEL) == uevent->conn_req_ctx) { 537 list_move_tail(&uevent->list, &list); 538 continue; 539 } 540 list_del(&uevent->list); 541 kfree(uevent); 542 } 543 list_del(&ctx->list); 544 events_reported = ctx->events_reported; 545 mutex_unlock(&ctx->file->mut); 546 547 /* 548 * If this was a listening ID then any connections spawned from it that 549 * have not been delivered to userspace are cleaned up too. Must be done 550 * outside any locks. 551 */ 552 list_for_each_entry_safe(uevent, tmp, &list, list) { 553 ucma_destroy_private_ctx(uevent->conn_req_ctx); 554 kfree(uevent); 555 } 556 return events_reported; 557} 558 559/* 560 * When this is called the xarray must have a XA_ZERO_ENTRY in the ctx->id (ie 561 * the ctx is not public to the user). This either because: 562 * - ucma_finish_ctx() hasn't been called 563 * - xa_cmpxchg() succeed to remove the entry (only one thread can succeed) 564 */ 565static int ucma_destroy_private_ctx(struct ucma_context *ctx) 566{ 567 int events_reported; 568 569 /* 570 * Destroy the underlying cm_id. New work queuing is prevented now by 571 * the removal from the xarray. Once the work is cancled ref will either 572 * be 0 because the work ran to completion and consumed the ref from the 573 * xarray, or it will be positive because we still have the ref from the 574 * xarray. This can also be 0 in cases where cm_id was never set 575 */ 576 cancel_work_sync(&ctx->close_work); 577 if (refcount_read(&ctx->ref)) 578 ucma_close_id(&ctx->close_work); 579 580 events_reported = ucma_cleanup_ctx_events(ctx); 581 ucma_cleanup_multicast(ctx); 582 583 WARN_ON(xa_cmpxchg(&ctx_table, ctx->id, XA_ZERO_ENTRY, NULL, 584 GFP_KERNEL) != NULL); 585 mutex_destroy(&ctx->mutex); 586 kfree(ctx); 587 return events_reported; 588} 589 590static ssize_t ucma_destroy_id(struct ucma_file *file, const char __user *inbuf, 591 int in_len, int out_len) 592{ 593 struct rdma_ucm_destroy_id cmd; 594 struct rdma_ucm_destroy_id_resp resp; 595 struct ucma_context *ctx; 596 int ret = 0; 597 598 if (out_len < sizeof(resp)) 599 return -ENOSPC; 600 601 if (copy_from_user(&cmd, inbuf, sizeof(cmd))) 602 return -EFAULT; 603 604 xa_lock(&ctx_table); 605 ctx = _ucma_find_context(cmd.id, file); 606 if (!IS_ERR(ctx)) { 607 if (__xa_cmpxchg(&ctx_table, ctx->id, ctx, XA_ZERO_ENTRY, 608 GFP_KERNEL) != ctx) 609 ctx = ERR_PTR(-ENOENT); 610 } 611 xa_unlock(&ctx_table); 612 613 if (IS_ERR(ctx)) 614 return PTR_ERR(ctx); 615 616 resp.events_reported = ucma_destroy_private_ctx(ctx); 617 if (copy_to_user(u64_to_user_ptr(cmd.response), 618 &resp, sizeof(resp))) 619 ret = -EFAULT; 620 621 return ret; 622} 623 624static ssize_t ucma_bind_ip(struct ucma_file *file, const char __user *inbuf, 625 int in_len, int out_len) 626{ 627 struct rdma_ucm_bind_ip cmd; 628 struct ucma_context *ctx; 629 int ret; 630 631 if (copy_from_user(&cmd, inbuf, sizeof(cmd))) 632 return -EFAULT; 633 634 if (!rdma_addr_size_in6(&cmd.addr)) 635 return -EINVAL; 636 637 ctx = ucma_get_ctx(file, cmd.id); 638 if (IS_ERR(ctx)) 639 return PTR_ERR(ctx); 640 641 mutex_lock(&ctx->mutex); 642 ret = rdma_bind_addr(ctx->cm_id, (struct sockaddr *) &cmd.addr); 643 mutex_unlock(&ctx->mutex); 644 645 ucma_put_ctx(ctx); 646 return ret; 647} 648 649static ssize_t ucma_bind(struct ucma_file *file, const char __user *inbuf, 650 int in_len, int out_len) 651{ 652 struct rdma_ucm_bind cmd; 653 struct ucma_context *ctx; 654 int ret; 655 656 if (copy_from_user(&cmd, inbuf, sizeof(cmd))) 657 return -EFAULT; 658 659 if (cmd.reserved || !cmd.addr_size || 660 cmd.addr_size != rdma_addr_size_kss(&cmd.addr)) 661 return -EINVAL; 662 663 ctx = ucma_get_ctx(file, cmd.id); 664 if (IS_ERR(ctx)) 665 return PTR_ERR(ctx); 666 667 mutex_lock(&ctx->mutex); 668 ret = rdma_bind_addr(ctx->cm_id, (struct sockaddr *) &cmd.addr); 669 mutex_unlock(&ctx->mutex); 670 ucma_put_ctx(ctx); 671 return ret; 672} 673 674static ssize_t ucma_resolve_ip(struct ucma_file *file, 675 const char __user *inbuf, 676 int in_len, int out_len) 677{ 678 struct rdma_ucm_resolve_ip cmd; 679 struct ucma_context *ctx; 680 int ret; 681 682 if (copy_from_user(&cmd, inbuf, sizeof(cmd))) 683 return -EFAULT; 684 685 if ((cmd.src_addr.sin6_family && !rdma_addr_size_in6(&cmd.src_addr)) || 686 !rdma_addr_size_in6(&cmd.dst_addr)) 687 return -EINVAL; 688 689 ctx = ucma_get_ctx(file, cmd.id); 690 if (IS_ERR(ctx)) 691 return PTR_ERR(ctx); 692 693 mutex_lock(&ctx->mutex); 694 ret = rdma_resolve_addr(ctx->cm_id, (struct sockaddr *) &cmd.src_addr, 695 (struct sockaddr *) &cmd.dst_addr, cmd.timeout_ms); 696 mutex_unlock(&ctx->mutex); 697 ucma_put_ctx(ctx); 698 return ret; 699} 700 701static ssize_t ucma_resolve_addr(struct ucma_file *file, 702 const char __user *inbuf, 703 int in_len, int out_len) 704{ 705 struct rdma_ucm_resolve_addr cmd; 706 struct ucma_context *ctx; 707 int ret; 708 709 if (copy_from_user(&cmd, inbuf, sizeof(cmd))) 710 return -EFAULT; 711 712 if (cmd.reserved || 713 (cmd.src_size && (cmd.src_size != rdma_addr_size_kss(&cmd.src_addr))) || 714 !cmd.dst_size || (cmd.dst_size != rdma_addr_size_kss(&cmd.dst_addr))) 715 return -EINVAL; 716 717 ctx = ucma_get_ctx(file, cmd.id); 718 if (IS_ERR(ctx)) 719 return PTR_ERR(ctx); 720 721 mutex_lock(&ctx->mutex); 722 ret = rdma_resolve_addr(ctx->cm_id, (struct sockaddr *) &cmd.src_addr, 723 (struct sockaddr *) &cmd.dst_addr, cmd.timeout_ms); 724 mutex_unlock(&ctx->mutex); 725 ucma_put_ctx(ctx); 726 return ret; 727} 728 729static ssize_t ucma_resolve_route(struct ucma_file *file, 730 const char __user *inbuf, 731 int in_len, int out_len) 732{ 733 struct rdma_ucm_resolve_route cmd; 734 struct ucma_context *ctx; 735 int ret; 736 737 if (copy_from_user(&cmd, inbuf, sizeof(cmd))) 738 return -EFAULT; 739 740 ctx = ucma_get_ctx_dev(file, cmd.id); 741 if (IS_ERR(ctx)) 742 return PTR_ERR(ctx); 743 744 mutex_lock(&ctx->mutex); 745 ret = rdma_resolve_route(ctx->cm_id, cmd.timeout_ms); 746 mutex_unlock(&ctx->mutex); 747 ucma_put_ctx(ctx); 748 return ret; 749} 750 751static void ucma_copy_ib_route(struct rdma_ucm_query_route_resp *resp, 752 struct rdma_route *route) 753{ 754 struct rdma_dev_addr *dev_addr; 755 756 resp->num_paths = route->num_pri_alt_paths; 757 switch (route->num_pri_alt_paths) { 758 case 0: 759 dev_addr = &route->addr.dev_addr; 760 rdma_addr_get_dgid(dev_addr, 761 (union ib_gid *) &resp->ib_route[0].dgid); 762 rdma_addr_get_sgid(dev_addr, 763 (union ib_gid *) &resp->ib_route[0].sgid); 764 resp->ib_route[0].pkey = cpu_to_be16(ib_addr_get_pkey(dev_addr)); 765 break; 766 case 2: 767 ib_copy_path_rec_to_user(&resp->ib_route[1], 768 &route->path_rec[1]); 769 fallthrough; 770 case 1: 771 ib_copy_path_rec_to_user(&resp->ib_route[0], 772 &route->path_rec[0]); 773 break; 774 default: 775 break; 776 } 777} 778 779static void ucma_copy_iboe_route(struct rdma_ucm_query_route_resp *resp, 780 struct rdma_route *route) 781{ 782 783 resp->num_paths = route->num_pri_alt_paths; 784 switch (route->num_pri_alt_paths) { 785 case 0: 786 rdma_ip2gid((struct sockaddr *)&route->addr.dst_addr, 787 (union ib_gid *)&resp->ib_route[0].dgid); 788 rdma_ip2gid((struct sockaddr *)&route->addr.src_addr, 789 (union ib_gid *)&resp->ib_route[0].sgid); 790 resp->ib_route[0].pkey = cpu_to_be16(0xffff); 791 break; 792 case 2: 793 ib_copy_path_rec_to_user(&resp->ib_route[1], 794 &route->path_rec[1]); 795 fallthrough; 796 case 1: 797 ib_copy_path_rec_to_user(&resp->ib_route[0], 798 &route->path_rec[0]); 799 break; 800 default: 801 break; 802 } 803} 804 805static void ucma_copy_iw_route(struct rdma_ucm_query_route_resp *resp, 806 struct rdma_route *route) 807{ 808 struct rdma_dev_addr *dev_addr; 809 810 dev_addr = &route->addr.dev_addr; 811 rdma_addr_get_dgid(dev_addr, (union ib_gid *) &resp->ib_route[0].dgid); 812 rdma_addr_get_sgid(dev_addr, (union ib_gid *) &resp->ib_route[0].sgid); 813} 814 815static ssize_t ucma_query_route(struct ucma_file *file, 816 const char __user *inbuf, 817 int in_len, int out_len) 818{ 819 struct rdma_ucm_query cmd; 820 struct rdma_ucm_query_route_resp resp; 821 struct ucma_context *ctx; 822 struct sockaddr *addr; 823 int ret = 0; 824 825 if (out_len < offsetof(struct rdma_ucm_query_route_resp, ibdev_index)) 826 return -ENOSPC; 827 828 if (copy_from_user(&cmd, inbuf, sizeof(cmd))) 829 return -EFAULT; 830 831 ctx = ucma_get_ctx(file, cmd.id); 832 if (IS_ERR(ctx)) 833 return PTR_ERR(ctx); 834 835 mutex_lock(&ctx->mutex); 836 memset(&resp, 0, sizeof resp); 837 addr = (struct sockaddr *) &ctx->cm_id->route.addr.src_addr; 838 memcpy(&resp.src_addr, addr, addr->sa_family == AF_INET ? 839 sizeof(struct sockaddr_in) : 840 sizeof(struct sockaddr_in6)); 841 addr = (struct sockaddr *) &ctx->cm_id->route.addr.dst_addr; 842 memcpy(&resp.dst_addr, addr, addr->sa_family == AF_INET ? 843 sizeof(struct sockaddr_in) : 844 sizeof(struct sockaddr_in6)); 845 if (!ctx->cm_id->device) 846 goto out; 847 848 resp.node_guid = (__force __u64) ctx->cm_id->device->node_guid; 849 resp.ibdev_index = ctx->cm_id->device->index; 850 resp.port_num = ctx->cm_id->port_num; 851 852 if (rdma_cap_ib_sa(ctx->cm_id->device, ctx->cm_id->port_num)) 853 ucma_copy_ib_route(&resp, &ctx->cm_id->route); 854 else if (rdma_protocol_roce(ctx->cm_id->device, ctx->cm_id->port_num)) 855 ucma_copy_iboe_route(&resp, &ctx->cm_id->route); 856 else if (rdma_protocol_iwarp(ctx->cm_id->device, ctx->cm_id->port_num)) 857 ucma_copy_iw_route(&resp, &ctx->cm_id->route); 858 859out: 860 mutex_unlock(&ctx->mutex); 861 if (copy_to_user(u64_to_user_ptr(cmd.response), &resp, 862 min_t(size_t, out_len, sizeof(resp)))) 863 ret = -EFAULT; 864 865 ucma_put_ctx(ctx); 866 return ret; 867} 868 869static void ucma_query_device_addr(struct rdma_cm_id *cm_id, 870 struct rdma_ucm_query_addr_resp *resp) 871{ 872 if (!cm_id->device) 873 return; 874 875 resp->node_guid = (__force __u64) cm_id->device->node_guid; 876 resp->ibdev_index = cm_id->device->index; 877 resp->port_num = cm_id->port_num; 878 resp->pkey = (__force __u16) cpu_to_be16( 879 ib_addr_get_pkey(&cm_id->route.addr.dev_addr)); 880} 881 882static ssize_t ucma_query_addr(struct ucma_context *ctx, 883 void __user *response, int out_len) 884{ 885 struct rdma_ucm_query_addr_resp resp; 886 struct sockaddr *addr; 887 int ret = 0; 888 889 if (out_len < offsetof(struct rdma_ucm_query_addr_resp, ibdev_index)) 890 return -ENOSPC; 891 892 memset(&resp, 0, sizeof resp); 893 894 addr = (struct sockaddr *) &ctx->cm_id->route.addr.src_addr; 895 resp.src_size = rdma_addr_size(addr); 896 memcpy(&resp.src_addr, addr, resp.src_size); 897 898 addr = (struct sockaddr *) &ctx->cm_id->route.addr.dst_addr; 899 resp.dst_size = rdma_addr_size(addr); 900 memcpy(&resp.dst_addr, addr, resp.dst_size); 901 902 ucma_query_device_addr(ctx->cm_id, &resp); 903 904 if (copy_to_user(response, &resp, min_t(size_t, out_len, sizeof(resp)))) 905 ret = -EFAULT; 906 907 return ret; 908} 909 910static ssize_t ucma_query_path(struct ucma_context *ctx, 911 void __user *response, int out_len) 912{ 913 struct rdma_ucm_query_path_resp *resp; 914 int i, ret = 0; 915 916 if (out_len < sizeof(*resp)) 917 return -ENOSPC; 918 919 resp = kzalloc(out_len, GFP_KERNEL); 920 if (!resp) 921 return -ENOMEM; 922 923 resp->num_paths = ctx->cm_id->route.num_pri_alt_paths; 924 for (i = 0, out_len -= sizeof(*resp); 925 i < resp->num_paths && out_len > sizeof(struct ib_path_rec_data); 926 i++, out_len -= sizeof(struct ib_path_rec_data)) { 927 struct sa_path_rec *rec = &ctx->cm_id->route.path_rec[i]; 928 929 resp->path_data[i].flags = IB_PATH_GMP | IB_PATH_PRIMARY | 930 IB_PATH_BIDIRECTIONAL; 931 if (rec->rec_type == SA_PATH_REC_TYPE_OPA) { 932 struct sa_path_rec ib; 933 934 sa_convert_path_opa_to_ib(&ib, rec); 935 ib_sa_pack_path(&ib, &resp->path_data[i].path_rec); 936 937 } else { 938 ib_sa_pack_path(rec, &resp->path_data[i].path_rec); 939 } 940 } 941 942 if (copy_to_user(response, resp, struct_size(resp, path_data, i))) 943 ret = -EFAULT; 944 945 kfree(resp); 946 return ret; 947} 948 949static ssize_t ucma_query_gid(struct ucma_context *ctx, 950 void __user *response, int out_len) 951{ 952 struct rdma_ucm_query_addr_resp resp; 953 struct sockaddr_ib *addr; 954 int ret = 0; 955 956 if (out_len < offsetof(struct rdma_ucm_query_addr_resp, ibdev_index)) 957 return -ENOSPC; 958 959 memset(&resp, 0, sizeof resp); 960 961 ucma_query_device_addr(ctx->cm_id, &resp); 962 963 addr = (struct sockaddr_ib *) &resp.src_addr; 964 resp.src_size = sizeof(*addr); 965 if (ctx->cm_id->route.addr.src_addr.ss_family == AF_IB) { 966 memcpy(addr, &ctx->cm_id->route.addr.src_addr, resp.src_size); 967 } else { 968 addr->sib_family = AF_IB; 969 addr->sib_pkey = (__force __be16) resp.pkey; 970 rdma_read_gids(ctx->cm_id, (union ib_gid *)&addr->sib_addr, 971 NULL); 972 addr->sib_sid = rdma_get_service_id(ctx->cm_id, (struct sockaddr *) 973 &ctx->cm_id->route.addr.src_addr); 974 } 975 976 addr = (struct sockaddr_ib *) &resp.dst_addr; 977 resp.dst_size = sizeof(*addr); 978 if (ctx->cm_id->route.addr.dst_addr.ss_family == AF_IB) { 979 memcpy(addr, &ctx->cm_id->route.addr.dst_addr, resp.dst_size); 980 } else { 981 addr->sib_family = AF_IB; 982 addr->sib_pkey = (__force __be16) resp.pkey; 983 rdma_read_gids(ctx->cm_id, NULL, 984 (union ib_gid *)&addr->sib_addr); 985 addr->sib_sid = rdma_get_service_id(ctx->cm_id, (struct sockaddr *) 986 &ctx->cm_id->route.addr.dst_addr); 987 } 988 989 if (copy_to_user(response, &resp, min_t(size_t, out_len, sizeof(resp)))) 990 ret = -EFAULT; 991 992 return ret; 993} 994 995static ssize_t ucma_query(struct ucma_file *file, 996 const char __user *inbuf, 997 int in_len, int out_len) 998{ 999 struct rdma_ucm_query cmd; 1000 struct ucma_context *ctx; 1001 void __user *response; 1002 int ret; 1003 1004 if (copy_from_user(&cmd, inbuf, sizeof(cmd))) 1005 return -EFAULT; 1006 1007 response = u64_to_user_ptr(cmd.response); 1008 ctx = ucma_get_ctx(file, cmd.id); 1009 if (IS_ERR(ctx)) 1010 return PTR_ERR(ctx); 1011 1012 mutex_lock(&ctx->mutex); 1013 switch (cmd.option) { 1014 case RDMA_USER_CM_QUERY_ADDR: 1015 ret = ucma_query_addr(ctx, response, out_len); 1016 break; 1017 case RDMA_USER_CM_QUERY_PATH: 1018 ret = ucma_query_path(ctx, response, out_len); 1019 break; 1020 case RDMA_USER_CM_QUERY_GID: 1021 ret = ucma_query_gid(ctx, response, out_len); 1022 break; 1023 default: 1024 ret = -ENOSYS; 1025 break; 1026 } 1027 mutex_unlock(&ctx->mutex); 1028 1029 ucma_put_ctx(ctx); 1030 return ret; 1031} 1032 1033static void ucma_copy_conn_param(struct rdma_cm_id *id, 1034 struct rdma_conn_param *dst, 1035 struct rdma_ucm_conn_param *src) 1036{ 1037 dst->private_data = src->private_data; 1038 dst->private_data_len = src->private_data_len; 1039 dst->responder_resources = src->responder_resources; 1040 dst->initiator_depth = src->initiator_depth; 1041 dst->flow_control = src->flow_control; 1042 dst->retry_count = src->retry_count; 1043 dst->rnr_retry_count = src->rnr_retry_count; 1044 dst->srq = src->srq; 1045 dst->qp_num = src->qp_num & 0xFFFFFF; 1046 dst->qkey = (id->route.addr.src_addr.ss_family == AF_IB) ? src->qkey : 0; 1047} 1048 1049static ssize_t ucma_connect(struct ucma_file *file, const char __user *inbuf, 1050 int in_len, int out_len) 1051{ 1052 struct rdma_conn_param conn_param; 1053 struct rdma_ucm_ece ece = {}; 1054 struct rdma_ucm_connect cmd; 1055 struct ucma_context *ctx; 1056 size_t in_size; 1057 int ret; 1058 1059 if (in_len < offsetofend(typeof(cmd), reserved)) 1060 return -EINVAL; 1061 in_size = min_t(size_t, in_len, sizeof(cmd)); 1062 if (copy_from_user(&cmd, inbuf, in_size)) 1063 return -EFAULT; 1064 1065 if (!cmd.conn_param.valid) 1066 return -EINVAL; 1067 1068 ctx = ucma_get_ctx_dev(file, cmd.id); 1069 if (IS_ERR(ctx)) 1070 return PTR_ERR(ctx); 1071 1072 ucma_copy_conn_param(ctx->cm_id, &conn_param, &cmd.conn_param); 1073 if (offsetofend(typeof(cmd), ece) <= in_size) { 1074 ece.vendor_id = cmd.ece.vendor_id; 1075 ece.attr_mod = cmd.ece.attr_mod; 1076 } 1077 1078 mutex_lock(&ctx->mutex); 1079 ret = rdma_connect_ece(ctx->cm_id, &conn_param, &ece); 1080 mutex_unlock(&ctx->mutex); 1081 ucma_put_ctx(ctx); 1082 return ret; 1083} 1084 1085static ssize_t ucma_listen(struct ucma_file *file, const char __user *inbuf, 1086 int in_len, int out_len) 1087{ 1088 struct rdma_ucm_listen cmd; 1089 struct ucma_context *ctx; 1090 int ret; 1091 1092 if (copy_from_user(&cmd, inbuf, sizeof(cmd))) 1093 return -EFAULT; 1094 1095 ctx = ucma_get_ctx(file, cmd.id); 1096 if (IS_ERR(ctx)) 1097 return PTR_ERR(ctx); 1098 1099 if (cmd.backlog <= 0 || cmd.backlog > max_backlog) 1100 cmd.backlog = max_backlog; 1101 atomic_set(&ctx->backlog, cmd.backlog); 1102 1103 mutex_lock(&ctx->mutex); 1104 ret = rdma_listen(ctx->cm_id, cmd.backlog); 1105 mutex_unlock(&ctx->mutex); 1106 ucma_put_ctx(ctx); 1107 return ret; 1108} 1109 1110static ssize_t ucma_accept(struct ucma_file *file, const char __user *inbuf, 1111 int in_len, int out_len) 1112{ 1113 struct rdma_ucm_accept cmd; 1114 struct rdma_conn_param conn_param; 1115 struct rdma_ucm_ece ece = {}; 1116 struct ucma_context *ctx; 1117 size_t in_size; 1118 int ret; 1119 1120 if (in_len < offsetofend(typeof(cmd), reserved)) 1121 return -EINVAL; 1122 in_size = min_t(size_t, in_len, sizeof(cmd)); 1123 if (copy_from_user(&cmd, inbuf, in_size)) 1124 return -EFAULT; 1125 1126 ctx = ucma_get_ctx_dev(file, cmd.id); 1127 if (IS_ERR(ctx)) 1128 return PTR_ERR(ctx); 1129 1130 if (offsetofend(typeof(cmd), ece) <= in_size) { 1131 ece.vendor_id = cmd.ece.vendor_id; 1132 ece.attr_mod = cmd.ece.attr_mod; 1133 } 1134 1135 if (cmd.conn_param.valid) { 1136 ucma_copy_conn_param(ctx->cm_id, &conn_param, &cmd.conn_param); 1137 mutex_lock(&ctx->mutex); 1138 rdma_lock_handler(ctx->cm_id); 1139 ret = rdma_accept_ece(ctx->cm_id, &conn_param, &ece); 1140 if (!ret) { 1141 /* The uid must be set atomically with the handler */ 1142 ctx->uid = cmd.uid; 1143 } 1144 rdma_unlock_handler(ctx->cm_id); 1145 mutex_unlock(&ctx->mutex); 1146 } else { 1147 mutex_lock(&ctx->mutex); 1148 rdma_lock_handler(ctx->cm_id); 1149 ret = rdma_accept_ece(ctx->cm_id, NULL, &ece); 1150 rdma_unlock_handler(ctx->cm_id); 1151 mutex_unlock(&ctx->mutex); 1152 } 1153 ucma_put_ctx(ctx); 1154 return ret; 1155} 1156 1157static ssize_t ucma_reject(struct ucma_file *file, const char __user *inbuf, 1158 int in_len, int out_len) 1159{ 1160 struct rdma_ucm_reject cmd; 1161 struct ucma_context *ctx; 1162 int ret; 1163 1164 if (copy_from_user(&cmd, inbuf, sizeof(cmd))) 1165 return -EFAULT; 1166 1167 if (!cmd.reason) 1168 cmd.reason = IB_CM_REJ_CONSUMER_DEFINED; 1169 1170 switch (cmd.reason) { 1171 case IB_CM_REJ_CONSUMER_DEFINED: 1172 case IB_CM_REJ_VENDOR_OPTION_NOT_SUPPORTED: 1173 break; 1174 default: 1175 return -EINVAL; 1176 } 1177 1178 ctx = ucma_get_ctx_dev(file, cmd.id); 1179 if (IS_ERR(ctx)) 1180 return PTR_ERR(ctx); 1181 1182 mutex_lock(&ctx->mutex); 1183 ret = rdma_reject(ctx->cm_id, cmd.private_data, cmd.private_data_len, 1184 cmd.reason); 1185 mutex_unlock(&ctx->mutex); 1186 ucma_put_ctx(ctx); 1187 return ret; 1188} 1189 1190static ssize_t ucma_disconnect(struct ucma_file *file, const char __user *inbuf, 1191 int in_len, int out_len) 1192{ 1193 struct rdma_ucm_disconnect cmd; 1194 struct ucma_context *ctx; 1195 int ret; 1196 1197 if (copy_from_user(&cmd, inbuf, sizeof(cmd))) 1198 return -EFAULT; 1199 1200 ctx = ucma_get_ctx_dev(file, cmd.id); 1201 if (IS_ERR(ctx)) 1202 return PTR_ERR(ctx); 1203 1204 mutex_lock(&ctx->mutex); 1205 ret = rdma_disconnect(ctx->cm_id); 1206 mutex_unlock(&ctx->mutex); 1207 ucma_put_ctx(ctx); 1208 return ret; 1209} 1210 1211static ssize_t ucma_init_qp_attr(struct ucma_file *file, 1212 const char __user *inbuf, 1213 int in_len, int out_len) 1214{ 1215 struct rdma_ucm_init_qp_attr cmd; 1216 struct ib_uverbs_qp_attr resp; 1217 struct ucma_context *ctx; 1218 struct ib_qp_attr qp_attr; 1219 int ret; 1220 1221 if (out_len < sizeof(resp)) 1222 return -ENOSPC; 1223 1224 if (copy_from_user(&cmd, inbuf, sizeof(cmd))) 1225 return -EFAULT; 1226 1227 if (cmd.qp_state > IB_QPS_ERR) 1228 return -EINVAL; 1229 1230 ctx = ucma_get_ctx_dev(file, cmd.id); 1231 if (IS_ERR(ctx)) 1232 return PTR_ERR(ctx); 1233 1234 resp.qp_attr_mask = 0; 1235 memset(&qp_attr, 0, sizeof qp_attr); 1236 qp_attr.qp_state = cmd.qp_state; 1237 mutex_lock(&ctx->mutex); 1238 ret = rdma_init_qp_attr(ctx->cm_id, &qp_attr, &resp.qp_attr_mask); 1239 mutex_unlock(&ctx->mutex); 1240 if (ret) 1241 goto out; 1242 1243 ib_copy_qp_attr_to_user(ctx->cm_id->device, &resp, &qp_attr); 1244 if (copy_to_user(u64_to_user_ptr(cmd.response), 1245 &resp, sizeof(resp))) 1246 ret = -EFAULT; 1247 1248out: 1249 ucma_put_ctx(ctx); 1250 return ret; 1251} 1252 1253static int ucma_set_option_id(struct ucma_context *ctx, int optname, 1254 void *optval, size_t optlen) 1255{ 1256 int ret = 0; 1257 1258 switch (optname) { 1259 case RDMA_OPTION_ID_TOS: 1260 if (optlen != sizeof(u8)) { 1261 ret = -EINVAL; 1262 break; 1263 } 1264 rdma_set_service_type(ctx->cm_id, *((u8 *) optval)); 1265 break; 1266 case RDMA_OPTION_ID_REUSEADDR: 1267 if (optlen != sizeof(int)) { 1268 ret = -EINVAL; 1269 break; 1270 } 1271 ret = rdma_set_reuseaddr(ctx->cm_id, *((int *) optval) ? 1 : 0); 1272 break; 1273 case RDMA_OPTION_ID_AFONLY: 1274 if (optlen != sizeof(int)) { 1275 ret = -EINVAL; 1276 break; 1277 } 1278 ret = rdma_set_afonly(ctx->cm_id, *((int *) optval) ? 1 : 0); 1279 break; 1280 case RDMA_OPTION_ID_ACK_TIMEOUT: 1281 if (optlen != sizeof(u8)) { 1282 ret = -EINVAL; 1283 break; 1284 } 1285 ret = rdma_set_ack_timeout(ctx->cm_id, *((u8 *)optval)); 1286 break; 1287 default: 1288 ret = -ENOSYS; 1289 } 1290 1291 return ret; 1292} 1293 1294static int ucma_set_ib_path(struct ucma_context *ctx, 1295 struct ib_path_rec_data *path_data, size_t optlen) 1296{ 1297 struct sa_path_rec sa_path; 1298 struct rdma_cm_event event; 1299 int ret; 1300 1301 if (optlen % sizeof(*path_data)) 1302 return -EINVAL; 1303 1304 for (; optlen; optlen -= sizeof(*path_data), path_data++) { 1305 if (path_data->flags == (IB_PATH_GMP | IB_PATH_PRIMARY | 1306 IB_PATH_BIDIRECTIONAL)) 1307 break; 1308 } 1309 1310 if (!optlen) 1311 return -EINVAL; 1312 1313 if (!ctx->cm_id->device) 1314 return -EINVAL; 1315 1316 memset(&sa_path, 0, sizeof(sa_path)); 1317 1318 sa_path.rec_type = SA_PATH_REC_TYPE_IB; 1319 ib_sa_unpack_path(path_data->path_rec, &sa_path); 1320 1321 if (rdma_cap_opa_ah(ctx->cm_id->device, ctx->cm_id->port_num)) { 1322 struct sa_path_rec opa; 1323 1324 sa_convert_path_ib_to_opa(&opa, &sa_path); 1325 mutex_lock(&ctx->mutex); 1326 ret = rdma_set_ib_path(ctx->cm_id, &opa); 1327 mutex_unlock(&ctx->mutex); 1328 } else { 1329 mutex_lock(&ctx->mutex); 1330 ret = rdma_set_ib_path(ctx->cm_id, &sa_path); 1331 mutex_unlock(&ctx->mutex); 1332 } 1333 if (ret) 1334 return ret; 1335 1336 memset(&event, 0, sizeof event); 1337 event.event = RDMA_CM_EVENT_ROUTE_RESOLVED; 1338 return ucma_event_handler(ctx->cm_id, &event); 1339} 1340 1341static int ucma_set_option_ib(struct ucma_context *ctx, int optname, 1342 void *optval, size_t optlen) 1343{ 1344 int ret; 1345 1346 switch (optname) { 1347 case RDMA_OPTION_IB_PATH: 1348 ret = ucma_set_ib_path(ctx, optval, optlen); 1349 break; 1350 default: 1351 ret = -ENOSYS; 1352 } 1353 1354 return ret; 1355} 1356 1357static int ucma_set_option_level(struct ucma_context *ctx, int level, 1358 int optname, void *optval, size_t optlen) 1359{ 1360 int ret; 1361 1362 switch (level) { 1363 case RDMA_OPTION_ID: 1364 mutex_lock(&ctx->mutex); 1365 ret = ucma_set_option_id(ctx, optname, optval, optlen); 1366 mutex_unlock(&ctx->mutex); 1367 break; 1368 case RDMA_OPTION_IB: 1369 ret = ucma_set_option_ib(ctx, optname, optval, optlen); 1370 break; 1371 default: 1372 ret = -ENOSYS; 1373 } 1374 1375 return ret; 1376} 1377 1378static ssize_t ucma_set_option(struct ucma_file *file, const char __user *inbuf, 1379 int in_len, int out_len) 1380{ 1381 struct rdma_ucm_set_option cmd; 1382 struct ucma_context *ctx; 1383 void *optval; 1384 int ret; 1385 1386 if (copy_from_user(&cmd, inbuf, sizeof(cmd))) 1387 return -EFAULT; 1388 1389 if (unlikely(cmd.optlen > KMALLOC_MAX_SIZE)) 1390 return -EINVAL; 1391 1392 ctx = ucma_get_ctx(file, cmd.id); 1393 if (IS_ERR(ctx)) 1394 return PTR_ERR(ctx); 1395 1396 optval = memdup_user(u64_to_user_ptr(cmd.optval), 1397 cmd.optlen); 1398 if (IS_ERR(optval)) { 1399 ret = PTR_ERR(optval); 1400 goto out; 1401 } 1402 1403 ret = ucma_set_option_level(ctx, cmd.level, cmd.optname, optval, 1404 cmd.optlen); 1405 kfree(optval); 1406 1407out: 1408 ucma_put_ctx(ctx); 1409 return ret; 1410} 1411 1412static ssize_t ucma_notify(struct ucma_file *file, const char __user *inbuf, 1413 int in_len, int out_len) 1414{ 1415 struct rdma_ucm_notify cmd; 1416 struct ucma_context *ctx; 1417 int ret = -EINVAL; 1418 1419 if (copy_from_user(&cmd, inbuf, sizeof(cmd))) 1420 return -EFAULT; 1421 1422 ctx = ucma_get_ctx(file, cmd.id); 1423 if (IS_ERR(ctx)) 1424 return PTR_ERR(ctx); 1425 1426 mutex_lock(&ctx->mutex); 1427 if (ctx->cm_id->device) 1428 ret = rdma_notify(ctx->cm_id, (enum ib_event_type)cmd.event); 1429 mutex_unlock(&ctx->mutex); 1430 1431 ucma_put_ctx(ctx); 1432 return ret; 1433} 1434 1435static ssize_t ucma_process_join(struct ucma_file *file, 1436 struct rdma_ucm_join_mcast *cmd, int out_len) 1437{ 1438 struct rdma_ucm_create_id_resp resp; 1439 struct ucma_context *ctx; 1440 struct ucma_multicast *mc; 1441 struct sockaddr *addr; 1442 int ret; 1443 u8 join_state; 1444 1445 if (out_len < sizeof(resp)) 1446 return -ENOSPC; 1447 1448 addr = (struct sockaddr *) &cmd->addr; 1449 if (cmd->addr_size != rdma_addr_size(addr)) 1450 return -EINVAL; 1451 1452 if (cmd->join_flags == RDMA_MC_JOIN_FLAG_FULLMEMBER) 1453 join_state = BIT(FULLMEMBER_JOIN); 1454 else if (cmd->join_flags == RDMA_MC_JOIN_FLAG_SENDONLY_FULLMEMBER) 1455 join_state = BIT(SENDONLY_FULLMEMBER_JOIN); 1456 else 1457 return -EINVAL; 1458 1459 ctx = ucma_get_ctx_dev(file, cmd->id); 1460 if (IS_ERR(ctx)) 1461 return PTR_ERR(ctx); 1462 1463 mc = kzalloc(sizeof(*mc), GFP_KERNEL); 1464 if (!mc) { 1465 ret = -ENOMEM; 1466 goto err_put_ctx; 1467 } 1468 1469 mc->ctx = ctx; 1470 mc->join_state = join_state; 1471 mc->uid = cmd->uid; 1472 memcpy(&mc->addr, addr, cmd->addr_size); 1473 1474 xa_lock(&multicast_table); 1475 if (__xa_alloc(&multicast_table, &mc->id, NULL, xa_limit_32b, 1476 GFP_KERNEL)) { 1477 ret = -ENOMEM; 1478 goto err_free_mc; 1479 } 1480 1481 list_add_tail(&mc->list, &ctx->mc_list); 1482 xa_unlock(&multicast_table); 1483 1484 mutex_lock(&ctx->mutex); 1485 ret = rdma_join_multicast(ctx->cm_id, (struct sockaddr *)&mc->addr, 1486 join_state, mc); 1487 mutex_unlock(&ctx->mutex); 1488 if (ret) 1489 goto err_xa_erase; 1490 1491 resp.id = mc->id; 1492 if (copy_to_user(u64_to_user_ptr(cmd->response), 1493 &resp, sizeof(resp))) { 1494 ret = -EFAULT; 1495 goto err_leave_multicast; 1496 } 1497 1498 xa_store(&multicast_table, mc->id, mc, 0); 1499 1500 ucma_put_ctx(ctx); 1501 return 0; 1502 1503err_leave_multicast: 1504 mutex_lock(&ctx->mutex); 1505 rdma_leave_multicast(ctx->cm_id, (struct sockaddr *) &mc->addr); 1506 mutex_unlock(&ctx->mutex); 1507 ucma_cleanup_mc_events(mc); 1508err_xa_erase: 1509 xa_lock(&multicast_table); 1510 list_del(&mc->list); 1511 __xa_erase(&multicast_table, mc->id); 1512err_free_mc: 1513 xa_unlock(&multicast_table); 1514 kfree(mc); 1515err_put_ctx: 1516 ucma_put_ctx(ctx); 1517 return ret; 1518} 1519 1520static ssize_t ucma_join_ip_multicast(struct ucma_file *file, 1521 const char __user *inbuf, 1522 int in_len, int out_len) 1523{ 1524 struct rdma_ucm_join_ip_mcast cmd; 1525 struct rdma_ucm_join_mcast join_cmd; 1526 1527 if (copy_from_user(&cmd, inbuf, sizeof(cmd))) 1528 return -EFAULT; 1529 1530 join_cmd.response = cmd.response; 1531 join_cmd.uid = cmd.uid; 1532 join_cmd.id = cmd.id; 1533 join_cmd.addr_size = rdma_addr_size_in6(&cmd.addr); 1534 if (!join_cmd.addr_size) 1535 return -EINVAL; 1536 1537 join_cmd.join_flags = RDMA_MC_JOIN_FLAG_FULLMEMBER; 1538 memcpy(&join_cmd.addr, &cmd.addr, join_cmd.addr_size); 1539 1540 return ucma_process_join(file, &join_cmd, out_len); 1541} 1542 1543static ssize_t ucma_join_multicast(struct ucma_file *file, 1544 const char __user *inbuf, 1545 int in_len, int out_len) 1546{ 1547 struct rdma_ucm_join_mcast cmd; 1548 1549 if (copy_from_user(&cmd, inbuf, sizeof(cmd))) 1550 return -EFAULT; 1551 1552 if (!rdma_addr_size_kss(&cmd.addr)) 1553 return -EINVAL; 1554 1555 return ucma_process_join(file, &cmd, out_len); 1556} 1557 1558static ssize_t ucma_leave_multicast(struct ucma_file *file, 1559 const char __user *inbuf, 1560 int in_len, int out_len) 1561{ 1562 struct rdma_ucm_destroy_id cmd; 1563 struct rdma_ucm_destroy_id_resp resp; 1564 struct ucma_multicast *mc; 1565 int ret = 0; 1566 1567 if (out_len < sizeof(resp)) 1568 return -ENOSPC; 1569 1570 if (copy_from_user(&cmd, inbuf, sizeof(cmd))) 1571 return -EFAULT; 1572 1573 xa_lock(&multicast_table); 1574 mc = xa_load(&multicast_table, cmd.id); 1575 if (!mc) 1576 mc = ERR_PTR(-ENOENT); 1577 else if (READ_ONCE(mc->ctx->file) != file) 1578 mc = ERR_PTR(-EINVAL); 1579 else if (!refcount_inc_not_zero(&mc->ctx->ref)) 1580 mc = ERR_PTR(-ENXIO); 1581 1582 if (IS_ERR(mc)) { 1583 xa_unlock(&multicast_table); 1584 ret = PTR_ERR(mc); 1585 goto out; 1586 } 1587 1588 list_del(&mc->list); 1589 __xa_erase(&multicast_table, mc->id); 1590 xa_unlock(&multicast_table); 1591 1592 mutex_lock(&mc->ctx->mutex); 1593 rdma_leave_multicast(mc->ctx->cm_id, (struct sockaddr *) &mc->addr); 1594 mutex_unlock(&mc->ctx->mutex); 1595 1596 ucma_cleanup_mc_events(mc); 1597 1598 ucma_put_ctx(mc->ctx); 1599 resp.events_reported = mc->events_reported; 1600 kfree(mc); 1601 1602 if (copy_to_user(u64_to_user_ptr(cmd.response), 1603 &resp, sizeof(resp))) 1604 ret = -EFAULT; 1605out: 1606 return ret; 1607} 1608 1609static ssize_t ucma_migrate_id(struct ucma_file *new_file, 1610 const char __user *inbuf, 1611 int in_len, int out_len) 1612{ 1613 struct rdma_ucm_migrate_id cmd; 1614 struct rdma_ucm_migrate_resp resp; 1615 struct ucma_event *uevent, *tmp; 1616 struct ucma_context *ctx; 1617 LIST_HEAD(event_list); 1618 struct fd f; 1619 struct ucma_file *cur_file; 1620 int ret = 0; 1621 1622 if (copy_from_user(&cmd, inbuf, sizeof(cmd))) 1623 return -EFAULT; 1624 1625 /* Get current fd to protect against it being closed */ 1626 f = fdget(cmd.fd); 1627 if (!f.file) 1628 return -ENOENT; 1629 if (f.file->f_op != &ucma_fops) { 1630 ret = -EINVAL; 1631 goto file_put; 1632 } 1633 cur_file = f.file->private_data; 1634 1635 /* Validate current fd and prevent destruction of id. */ 1636 ctx = ucma_get_ctx(cur_file, cmd.id); 1637 if (IS_ERR(ctx)) { 1638 ret = PTR_ERR(ctx); 1639 goto file_put; 1640 } 1641 1642 rdma_lock_handler(ctx->cm_id); 1643 /* 1644 * ctx->file can only be changed under the handler & xa_lock. xa_load() 1645 * must be checked again to ensure the ctx hasn't begun destruction 1646 * since the ucma_get_ctx(). 1647 */ 1648 xa_lock(&ctx_table); 1649 if (_ucma_find_context(cmd.id, cur_file) != ctx) { 1650 xa_unlock(&ctx_table); 1651 ret = -ENOENT; 1652 goto err_unlock; 1653 } 1654 ctx->file = new_file; 1655 xa_unlock(&ctx_table); 1656 1657 mutex_lock(&cur_file->mut); 1658 list_del(&ctx->list); 1659 /* 1660 * At this point lock_handler() prevents addition of new uevents for 1661 * this ctx. 1662 */ 1663 list_for_each_entry_safe(uevent, tmp, &cur_file->event_list, list) 1664 if (uevent->ctx == ctx) 1665 list_move_tail(&uevent->list, &event_list); 1666 resp.events_reported = ctx->events_reported; 1667 mutex_unlock(&cur_file->mut); 1668 1669 mutex_lock(&new_file->mut); 1670 list_add_tail(&ctx->list, &new_file->ctx_list); 1671 list_splice_tail(&event_list, &new_file->event_list); 1672 mutex_unlock(&new_file->mut); 1673 1674 if (copy_to_user(u64_to_user_ptr(cmd.response), 1675 &resp, sizeof(resp))) 1676 ret = -EFAULT; 1677 1678err_unlock: 1679 rdma_unlock_handler(ctx->cm_id); 1680 ucma_put_ctx(ctx); 1681file_put: 1682 fdput(f); 1683 return ret; 1684} 1685 1686static ssize_t (*ucma_cmd_table[])(struct ucma_file *file, 1687 const char __user *inbuf, 1688 int in_len, int out_len) = { 1689 [RDMA_USER_CM_CMD_CREATE_ID] = ucma_create_id, 1690 [RDMA_USER_CM_CMD_DESTROY_ID] = ucma_destroy_id, 1691 [RDMA_USER_CM_CMD_BIND_IP] = ucma_bind_ip, 1692 [RDMA_USER_CM_CMD_RESOLVE_IP] = ucma_resolve_ip, 1693 [RDMA_USER_CM_CMD_RESOLVE_ROUTE] = ucma_resolve_route, 1694 [RDMA_USER_CM_CMD_QUERY_ROUTE] = ucma_query_route, 1695 [RDMA_USER_CM_CMD_CONNECT] = ucma_connect, 1696 [RDMA_USER_CM_CMD_LISTEN] = ucma_listen, 1697 [RDMA_USER_CM_CMD_ACCEPT] = ucma_accept, 1698 [RDMA_USER_CM_CMD_REJECT] = ucma_reject, 1699 [RDMA_USER_CM_CMD_DISCONNECT] = ucma_disconnect, 1700 [RDMA_USER_CM_CMD_INIT_QP_ATTR] = ucma_init_qp_attr, 1701 [RDMA_USER_CM_CMD_GET_EVENT] = ucma_get_event, 1702 [RDMA_USER_CM_CMD_GET_OPTION] = NULL, 1703 [RDMA_USER_CM_CMD_SET_OPTION] = ucma_set_option, 1704 [RDMA_USER_CM_CMD_NOTIFY] = ucma_notify, 1705 [RDMA_USER_CM_CMD_JOIN_IP_MCAST] = ucma_join_ip_multicast, 1706 [RDMA_USER_CM_CMD_LEAVE_MCAST] = ucma_leave_multicast, 1707 [RDMA_USER_CM_CMD_MIGRATE_ID] = ucma_migrate_id, 1708 [RDMA_USER_CM_CMD_QUERY] = ucma_query, 1709 [RDMA_USER_CM_CMD_BIND] = ucma_bind, 1710 [RDMA_USER_CM_CMD_RESOLVE_ADDR] = ucma_resolve_addr, 1711 [RDMA_USER_CM_CMD_JOIN_MCAST] = ucma_join_multicast 1712}; 1713 1714static ssize_t ucma_write(struct file *filp, const char __user *buf, 1715 size_t len, loff_t *pos) 1716{ 1717 struct ucma_file *file = filp->private_data; 1718 struct rdma_ucm_cmd_hdr hdr; 1719 ssize_t ret; 1720 1721 if (!ib_safe_file_access(filp)) { 1722 pr_err_once("%s: process %d (%s) changed security contexts after opening file descriptor, this is not allowed.\n", 1723 __func__, task_tgid_vnr(current), current->comm); 1724 return -EACCES; 1725 } 1726 1727 if (len < sizeof(hdr)) 1728 return -EINVAL; 1729 1730 if (copy_from_user(&hdr, buf, sizeof(hdr))) 1731 return -EFAULT; 1732 1733 if (hdr.cmd >= ARRAY_SIZE(ucma_cmd_table)) 1734 return -EINVAL; 1735 hdr.cmd = array_index_nospec(hdr.cmd, ARRAY_SIZE(ucma_cmd_table)); 1736 1737 if (hdr.in + sizeof(hdr) > len) 1738 return -EINVAL; 1739 1740 if (!ucma_cmd_table[hdr.cmd]) 1741 return -ENOSYS; 1742 1743 ret = ucma_cmd_table[hdr.cmd](file, buf + sizeof(hdr), hdr.in, hdr.out); 1744 if (!ret) 1745 ret = len; 1746 1747 return ret; 1748} 1749 1750static __poll_t ucma_poll(struct file *filp, struct poll_table_struct *wait) 1751{ 1752 struct ucma_file *file = filp->private_data; 1753 __poll_t mask = 0; 1754 1755 poll_wait(filp, &file->poll_wait, wait); 1756 1757 if (!list_empty(&file->event_list)) 1758 mask = EPOLLIN | EPOLLRDNORM; 1759 1760 return mask; 1761} 1762 1763/* 1764 * ucma_open() does not need the BKL: 1765 * 1766 * - no global state is referred to; 1767 * - there is no ioctl method to race against; 1768 * - no further module initialization is required for open to work 1769 * after the device is registered. 1770 */ 1771static int ucma_open(struct inode *inode, struct file *filp) 1772{ 1773 struct ucma_file *file; 1774 1775 file = kmalloc(sizeof *file, GFP_KERNEL); 1776 if (!file) 1777 return -ENOMEM; 1778 1779 INIT_LIST_HEAD(&file->event_list); 1780 INIT_LIST_HEAD(&file->ctx_list); 1781 init_waitqueue_head(&file->poll_wait); 1782 mutex_init(&file->mut); 1783 1784 filp->private_data = file; 1785 file->filp = filp; 1786 1787 return stream_open(inode, filp); 1788} 1789 1790static int ucma_close(struct inode *inode, struct file *filp) 1791{ 1792 struct ucma_file *file = filp->private_data; 1793 1794 /* 1795 * All paths that touch ctx_list or ctx_list starting from write() are 1796 * prevented by this being a FD release function. The list_add_tail() in 1797 * ucma_connect_event_handler() can run concurrently, however it only 1798 * adds to the list *after* a listening ID. By only reading the first of 1799 * the list, and relying on ucma_destroy_private_ctx() to block 1800 * ucma_connect_event_handler(), no additional locking is needed. 1801 */ 1802 while (!list_empty(&file->ctx_list)) { 1803 struct ucma_context *ctx = list_first_entry( 1804 &file->ctx_list, struct ucma_context, list); 1805 1806 WARN_ON(xa_cmpxchg(&ctx_table, ctx->id, ctx, XA_ZERO_ENTRY, 1807 GFP_KERNEL) != ctx); 1808 ucma_destroy_private_ctx(ctx); 1809 } 1810 kfree(file); 1811 return 0; 1812} 1813 1814static const struct file_operations ucma_fops = { 1815 .owner = THIS_MODULE, 1816 .open = ucma_open, 1817 .release = ucma_close, 1818 .write = ucma_write, 1819 .poll = ucma_poll, 1820 .llseek = no_llseek, 1821}; 1822 1823static struct miscdevice ucma_misc = { 1824 .minor = MISC_DYNAMIC_MINOR, 1825 .name = "rdma_cm", 1826 .nodename = "infiniband/rdma_cm", 1827 .mode = 0666, 1828 .fops = &ucma_fops, 1829}; 1830 1831static int ucma_get_global_nl_info(struct ib_client_nl_info *res) 1832{ 1833 res->abi = RDMA_USER_CM_ABI_VERSION; 1834 res->cdev = ucma_misc.this_device; 1835 return 0; 1836} 1837 1838static struct ib_client rdma_cma_client = { 1839 .name = "rdma_cm", 1840 .get_global_nl_info = ucma_get_global_nl_info, 1841}; 1842MODULE_ALIAS_RDMA_CLIENT("rdma_cm"); 1843 1844static ssize_t abi_version_show(struct device *dev, 1845 struct device_attribute *attr, char *buf) 1846{ 1847 return sysfs_emit(buf, "%d\n", RDMA_USER_CM_ABI_VERSION); 1848} 1849static DEVICE_ATTR_RO(abi_version); 1850 1851static int __init ucma_init(void) 1852{ 1853 int ret; 1854 1855 ret = misc_register(&ucma_misc); 1856 if (ret) 1857 return ret; 1858 1859 ret = device_create_file(ucma_misc.this_device, &dev_attr_abi_version); 1860 if (ret) { 1861 pr_err("rdma_ucm: couldn't create abi_version attr\n"); 1862 goto err1; 1863 } 1864 1865 ucma_ctl_table_hdr = register_net_sysctl(&init_net, "net/rdma_ucm", ucma_ctl_table); 1866 if (!ucma_ctl_table_hdr) { 1867 pr_err("rdma_ucm: couldn't register sysctl paths\n"); 1868 ret = -ENOMEM; 1869 goto err2; 1870 } 1871 1872 ret = ib_register_client(&rdma_cma_client); 1873 if (ret) 1874 goto err3; 1875 1876 return 0; 1877err3: 1878 unregister_net_sysctl_table(ucma_ctl_table_hdr); 1879err2: 1880 device_remove_file(ucma_misc.this_device, &dev_attr_abi_version); 1881err1: 1882 misc_deregister(&ucma_misc); 1883 return ret; 1884} 1885 1886static void __exit ucma_cleanup(void) 1887{ 1888 ib_unregister_client(&rdma_cma_client); 1889 unregister_net_sysctl_table(ucma_ctl_table_hdr); 1890 device_remove_file(ucma_misc.this_device, &dev_attr_abi_version); 1891 misc_deregister(&ucma_misc); 1892} 1893 1894module_init(ucma_init); 1895module_exit(ucma_cleanup); 1896