1/* 2 * Copyright (c) 2017, Mellanox Technologies inc. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33#include <rdma/rdma_user_ioctl.h> 34#include <rdma/uverbs_ioctl.h> 35#include "rdma_core.h" 36#include "uverbs.h" 37 38struct bundle_alloc_head { 39 struct_group_tagged(bundle_alloc_head_hdr, hdr, 40 struct bundle_alloc_head *next; 41 ); 42 u8 data[]; 43}; 44 45struct bundle_priv { 46 /* Must be first */ 47 struct bundle_alloc_head_hdr alloc_head; 48 struct bundle_alloc_head *allocated_mem; 49 size_t internal_avail; 50 size_t internal_used; 51 52 struct radix_tree_root *radix; 53 const struct uverbs_api_ioctl_method *method_elm; 54 void __rcu **radix_slots; 55 unsigned long radix_slots_len; 56 u32 method_key; 57 58 struct ib_uverbs_attr __user *user_attrs; 59 struct ib_uverbs_attr *uattrs; 60 61 DECLARE_BITMAP(uobj_finalize, UVERBS_API_ATTR_BKEY_LEN); 62 DECLARE_BITMAP(spec_finalize, UVERBS_API_ATTR_BKEY_LEN); 63 DECLARE_BITMAP(uobj_hw_obj_valid, UVERBS_API_ATTR_BKEY_LEN); 64 65 /* 66 * Must be last. bundle ends in a flex array which overlaps 67 * internal_buffer. 68 */ 69 struct uverbs_attr_bundle_hdr bundle; 70 u64 internal_buffer[32]; 71}; 72 73/* 74 * Each method has an absolute minimum amount of memory it needs to allocate, 75 * precompute that amount and determine if the onstack memory can be used or 76 * if allocation is need. 77 */ 78void uapi_compute_bundle_size(struct uverbs_api_ioctl_method *method_elm, 79 unsigned int num_attrs) 80{ 81 struct bundle_priv *pbundle; 82 struct uverbs_attr_bundle *bundle; 83 size_t bundle_size = 84 offsetof(struct bundle_priv, internal_buffer) + 85 sizeof(*bundle->attrs) * method_elm->key_bitmap_len + 86 sizeof(*pbundle->uattrs) * num_attrs; 87 88 method_elm->use_stack = bundle_size <= sizeof(*pbundle); 89 method_elm->bundle_size = 90 ALIGN(bundle_size + 256, sizeof(*pbundle->internal_buffer)); 91 92 /* Do not want order-2 allocations for this. */ 93 WARN_ON_ONCE(method_elm->bundle_size > PAGE_SIZE); 94} 95 96/** 97 * _uverbs_alloc() - Quickly allocate memory for use with a bundle 98 * @bundle: The bundle 99 * @size: Number of bytes to allocate 100 * @flags: Allocator flags 101 * 102 * The bundle allocator is intended for allocations that are connected with 103 * processing the system call related to the bundle. The allocated memory is 104 * always freed once the system call completes, and cannot be freed any other 105 * way. 106 * 107 * This tries to use a small pool of pre-allocated memory for performance. 108 */ 109__malloc void *_uverbs_alloc(struct uverbs_attr_bundle *bundle, size_t size, 110 gfp_t flags) 111{ 112 struct bundle_priv *pbundle = 113 container_of(&bundle->hdr, struct bundle_priv, bundle); 114 size_t new_used; 115 void *res; 116 117 if (check_add_overflow(size, pbundle->internal_used, &new_used)) 118 return ERR_PTR(-EOVERFLOW); 119 120 if (new_used > pbundle->internal_avail) { 121 struct bundle_alloc_head *buf; 122 123 buf = kvmalloc(struct_size(buf, data, size), flags); 124 if (!buf) 125 return ERR_PTR(-ENOMEM); 126 buf->next = pbundle->allocated_mem; 127 pbundle->allocated_mem = buf; 128 return buf->data; 129 } 130 131 res = (void *)pbundle->internal_buffer + pbundle->internal_used; 132 pbundle->internal_used = 133 ALIGN(new_used, sizeof(*pbundle->internal_buffer)); 134 if (want_init_on_alloc(flags)) 135 memset(res, 0, size); 136 return res; 137} 138EXPORT_SYMBOL(_uverbs_alloc); 139 140static bool uverbs_is_attr_cleared(const struct ib_uverbs_attr *uattr, 141 u16 len) 142{ 143 if (uattr->len > sizeof_field(struct ib_uverbs_attr, data)) 144 return ib_is_buffer_cleared(u64_to_user_ptr(uattr->data) + len, 145 uattr->len - len); 146 147 return !memchr_inv((const void *)&uattr->data + len, 148 0, uattr->len - len); 149} 150 151static int uverbs_set_output(const struct uverbs_attr_bundle *bundle, 152 const struct uverbs_attr *attr) 153{ 154 struct bundle_priv *pbundle = 155 container_of(&bundle->hdr, struct bundle_priv, bundle); 156 u16 flags; 157 158 flags = pbundle->uattrs[attr->ptr_attr.uattr_idx].flags | 159 UVERBS_ATTR_F_VALID_OUTPUT; 160 if (put_user(flags, 161 &pbundle->user_attrs[attr->ptr_attr.uattr_idx].flags)) 162 return -EFAULT; 163 return 0; 164} 165 166static int uverbs_process_idrs_array(struct bundle_priv *pbundle, 167 const struct uverbs_api_attr *attr_uapi, 168 struct uverbs_objs_arr_attr *attr, 169 struct ib_uverbs_attr *uattr, 170 u32 attr_bkey) 171{ 172 struct uverbs_attr_bundle *bundle = 173 container_of(&pbundle->bundle, struct uverbs_attr_bundle, hdr); 174 const struct uverbs_attr_spec *spec = &attr_uapi->spec; 175 size_t array_len; 176 u32 *idr_vals; 177 int ret = 0; 178 size_t i; 179 180 if (uattr->attr_data.reserved) 181 return -EINVAL; 182 183 if (uattr->len % sizeof(u32)) 184 return -EINVAL; 185 186 array_len = uattr->len / sizeof(u32); 187 if (array_len < spec->u2.objs_arr.min_len || 188 array_len > spec->u2.objs_arr.max_len) 189 return -EINVAL; 190 191 attr->uobjects = 192 uverbs_alloc(bundle, 193 array_size(array_len, sizeof(*attr->uobjects))); 194 if (IS_ERR(attr->uobjects)) 195 return PTR_ERR(attr->uobjects); 196 197 /* 198 * Since idr is 4B and *uobjects is >= 4B, we can use attr->uobjects 199 * to store idrs array and avoid additional memory allocation. The 200 * idrs array is offset to the end of the uobjects array so we will be 201 * able to read idr and replace with a pointer. 202 */ 203 idr_vals = (u32 *)(attr->uobjects + array_len) - array_len; 204 205 if (uattr->len > sizeof(uattr->data)) { 206 ret = copy_from_user(idr_vals, u64_to_user_ptr(uattr->data), 207 uattr->len); 208 if (ret) 209 return -EFAULT; 210 } else { 211 memcpy(idr_vals, &uattr->data, uattr->len); 212 } 213 214 for (i = 0; i != array_len; i++) { 215 attr->uobjects[i] = uverbs_get_uobject_from_file( 216 spec->u2.objs_arr.obj_type, spec->u2.objs_arr.access, 217 idr_vals[i], bundle); 218 if (IS_ERR(attr->uobjects[i])) { 219 ret = PTR_ERR(attr->uobjects[i]); 220 break; 221 } 222 } 223 224 attr->len = i; 225 __set_bit(attr_bkey, pbundle->spec_finalize); 226 return ret; 227} 228 229static void uverbs_free_idrs_array(const struct uverbs_api_attr *attr_uapi, 230 struct uverbs_objs_arr_attr *attr, 231 bool commit, 232 struct uverbs_attr_bundle *attrs) 233{ 234 const struct uverbs_attr_spec *spec = &attr_uapi->spec; 235 size_t i; 236 237 for (i = 0; i != attr->len; i++) 238 uverbs_finalize_object(attr->uobjects[i], 239 spec->u2.objs_arr.access, false, commit, 240 attrs); 241} 242 243static int uverbs_process_attr(struct bundle_priv *pbundle, 244 const struct uverbs_api_attr *attr_uapi, 245 struct ib_uverbs_attr *uattr, u32 attr_bkey) 246{ 247 const struct uverbs_attr_spec *spec = &attr_uapi->spec; 248 struct uverbs_attr_bundle *bundle = 249 container_of(&pbundle->bundle, struct uverbs_attr_bundle, hdr); 250 struct uverbs_attr *e = &bundle->attrs[attr_bkey]; 251 const struct uverbs_attr_spec *val_spec = spec; 252 struct uverbs_obj_attr *o_attr; 253 254 switch (spec->type) { 255 case UVERBS_ATTR_TYPE_ENUM_IN: 256 if (uattr->attr_data.enum_data.elem_id >= spec->u.enum_def.num_elems) 257 return -EOPNOTSUPP; 258 259 if (uattr->attr_data.enum_data.reserved) 260 return -EINVAL; 261 262 val_spec = &spec->u2.enum_def.ids[uattr->attr_data.enum_data.elem_id]; 263 264 /* Currently we only support PTR_IN based enums */ 265 if (val_spec->type != UVERBS_ATTR_TYPE_PTR_IN) 266 return -EOPNOTSUPP; 267 268 e->ptr_attr.enum_id = uattr->attr_data.enum_data.elem_id; 269 fallthrough; 270 case UVERBS_ATTR_TYPE_PTR_IN: 271 /* Ensure that any data provided by userspace beyond the known 272 * struct is zero. Userspace that knows how to use some future 273 * longer struct will fail here if used with an old kernel and 274 * non-zero content, making ABI compat/discovery simpler. 275 */ 276 if (uattr->len > val_spec->u.ptr.len && 277 val_spec->zero_trailing && 278 !uverbs_is_attr_cleared(uattr, val_spec->u.ptr.len)) 279 return -EOPNOTSUPP; 280 281 fallthrough; 282 case UVERBS_ATTR_TYPE_PTR_OUT: 283 if (uattr->len < val_spec->u.ptr.min_len || 284 (!val_spec->zero_trailing && 285 uattr->len > val_spec->u.ptr.len)) 286 return -EINVAL; 287 288 if (spec->type != UVERBS_ATTR_TYPE_ENUM_IN && 289 uattr->attr_data.reserved) 290 return -EINVAL; 291 292 e->ptr_attr.uattr_idx = uattr - pbundle->uattrs; 293 e->ptr_attr.len = uattr->len; 294 295 if (val_spec->alloc_and_copy && !uverbs_attr_ptr_is_inline(e)) { 296 void *p; 297 298 p = uverbs_alloc(bundle, uattr->len); 299 if (IS_ERR(p)) 300 return PTR_ERR(p); 301 302 e->ptr_attr.ptr = p; 303 304 if (copy_from_user(p, u64_to_user_ptr(uattr->data), 305 uattr->len)) 306 return -EFAULT; 307 } else { 308 e->ptr_attr.data = uattr->data; 309 } 310 break; 311 312 case UVERBS_ATTR_TYPE_IDR: 313 case UVERBS_ATTR_TYPE_FD: 314 if (uattr->attr_data.reserved) 315 return -EINVAL; 316 317 if (uattr->len != 0) 318 return -EINVAL; 319 320 o_attr = &e->obj_attr; 321 o_attr->attr_elm = attr_uapi; 322 323 /* 324 * The type of uattr->data is u64 for UVERBS_ATTR_TYPE_IDR and 325 * s64 for UVERBS_ATTR_TYPE_FD. We can cast the u64 to s64 326 * here without caring about truncation as we know that the 327 * IDR implementation today rejects negative IDs 328 */ 329 o_attr->uobject = uverbs_get_uobject_from_file( 330 spec->u.obj.obj_type, spec->u.obj.access, 331 uattr->data_s64, bundle); 332 if (IS_ERR(o_attr->uobject)) 333 return PTR_ERR(o_attr->uobject); 334 __set_bit(attr_bkey, pbundle->uobj_finalize); 335 336 if (spec->u.obj.access == UVERBS_ACCESS_NEW) { 337 unsigned int uattr_idx = uattr - pbundle->uattrs; 338 s64 id = o_attr->uobject->id; 339 340 /* Copy the allocated id to the user-space */ 341 if (put_user(id, &pbundle->user_attrs[uattr_idx].data)) 342 return -EFAULT; 343 } 344 345 break; 346 347 case UVERBS_ATTR_TYPE_RAW_FD: 348 if (uattr->attr_data.reserved || uattr->len != 0 || 349 uattr->data_s64 < INT_MIN || uattr->data_s64 > INT_MAX) 350 return -EINVAL; 351 /* _uverbs_get_const_signed() is the accessor */ 352 e->ptr_attr.data = uattr->data_s64; 353 break; 354 355 case UVERBS_ATTR_TYPE_IDRS_ARRAY: 356 return uverbs_process_idrs_array(pbundle, attr_uapi, 357 &e->objs_arr_attr, uattr, 358 attr_bkey); 359 default: 360 return -EOPNOTSUPP; 361 } 362 363 return 0; 364} 365 366/* 367 * We search the radix tree with the method prefix and now we want to fast 368 * search the suffix bits to get a particular attribute pointer. It is not 369 * totally clear to me if this breaks the radix tree encasulation or not, but 370 * it uses the iter data to determine if the method iter points at the same 371 * chunk that will store the attribute, if so it just derefs it directly. By 372 * construction in most kernel configs the method and attrs will all fit in a 373 * single radix chunk, so in most cases this will have no search. Other cases 374 * this falls back to a full search. 375 */ 376static void __rcu **uapi_get_attr_for_method(struct bundle_priv *pbundle, 377 u32 attr_key) 378{ 379 void __rcu **slot; 380 381 if (likely(attr_key < pbundle->radix_slots_len)) { 382 void *entry; 383 384 slot = pbundle->radix_slots + attr_key; 385 entry = rcu_dereference_raw(*slot); 386 if (likely(!radix_tree_is_internal_node(entry) && entry)) 387 return slot; 388 } 389 390 return radix_tree_lookup_slot(pbundle->radix, 391 pbundle->method_key | attr_key); 392} 393 394static int uverbs_set_attr(struct bundle_priv *pbundle, 395 struct ib_uverbs_attr *uattr) 396{ 397 u32 attr_key = uapi_key_attr(uattr->attr_id); 398 u32 attr_bkey = uapi_bkey_attr(attr_key); 399 const struct uverbs_api_attr *attr; 400 void __rcu **slot; 401 int ret; 402 403 slot = uapi_get_attr_for_method(pbundle, attr_key); 404 if (!slot) { 405 /* 406 * Kernel does not support the attribute but user-space says it 407 * is mandatory 408 */ 409 if (uattr->flags & UVERBS_ATTR_F_MANDATORY) 410 return -EPROTONOSUPPORT; 411 return 0; 412 } 413 attr = rcu_dereference_protected(*slot, true); 414 415 /* Reject duplicate attributes from user-space */ 416 if (test_bit(attr_bkey, pbundle->bundle.attr_present)) 417 return -EINVAL; 418 419 ret = uverbs_process_attr(pbundle, attr, uattr, attr_bkey); 420 if (ret) 421 return ret; 422 423 __set_bit(attr_bkey, pbundle->bundle.attr_present); 424 425 return 0; 426} 427 428static int ib_uverbs_run_method(struct bundle_priv *pbundle, 429 unsigned int num_attrs) 430{ 431 int (*handler)(struct uverbs_attr_bundle *attrs); 432 struct uverbs_attr_bundle *bundle = 433 container_of(&pbundle->bundle, struct uverbs_attr_bundle, hdr); 434 size_t uattrs_size = array_size(sizeof(*pbundle->uattrs), num_attrs); 435 unsigned int destroy_bkey = pbundle->method_elm->destroy_bkey; 436 unsigned int i; 437 int ret; 438 439 /* See uverbs_disassociate_api() */ 440 handler = srcu_dereference( 441 pbundle->method_elm->handler, 442 &pbundle->bundle.ufile->device->disassociate_srcu); 443 if (!handler) 444 return -EIO; 445 446 pbundle->uattrs = uverbs_alloc(bundle, uattrs_size); 447 if (IS_ERR(pbundle->uattrs)) 448 return PTR_ERR(pbundle->uattrs); 449 if (copy_from_user(pbundle->uattrs, pbundle->user_attrs, uattrs_size)) 450 return -EFAULT; 451 452 for (i = 0; i != num_attrs; i++) { 453 ret = uverbs_set_attr(pbundle, &pbundle->uattrs[i]); 454 if (unlikely(ret)) 455 return ret; 456 } 457 458 /* User space did not provide all the mandatory attributes */ 459 if (unlikely(!bitmap_subset(pbundle->method_elm->attr_mandatory, 460 pbundle->bundle.attr_present, 461 pbundle->method_elm->key_bitmap_len))) 462 return -EINVAL; 463 464 if (pbundle->method_elm->has_udata) 465 uverbs_fill_udata(bundle, &pbundle->bundle.driver_udata, 466 UVERBS_ATTR_UHW_IN, UVERBS_ATTR_UHW_OUT); 467 else 468 pbundle->bundle.driver_udata = (struct ib_udata){}; 469 470 if (destroy_bkey != UVERBS_API_ATTR_BKEY_LEN) { 471 struct uverbs_obj_attr *destroy_attr = &bundle->attrs[destroy_bkey].obj_attr; 472 473 ret = uobj_destroy(destroy_attr->uobject, bundle); 474 if (ret) 475 return ret; 476 __clear_bit(destroy_bkey, pbundle->uobj_finalize); 477 478 ret = handler(bundle); 479 uobj_put_destroy(destroy_attr->uobject); 480 } else { 481 ret = handler(bundle); 482 } 483 484 /* 485 * Until the drivers are revised to use the bundle directly we have to 486 * assume that the driver wrote to its UHW_OUT and flag userspace 487 * appropriately. 488 */ 489 if (!ret && pbundle->method_elm->has_udata) { 490 const struct uverbs_attr *attr = 491 uverbs_attr_get(bundle, UVERBS_ATTR_UHW_OUT); 492 493 if (!IS_ERR(attr)) 494 ret = uverbs_set_output(bundle, attr); 495 } 496 497 /* 498 * EPROTONOSUPPORT is ONLY to be returned if the ioctl framework can 499 * not invoke the method because the request is not supported. No 500 * other cases should return this code. 501 */ 502 if (WARN_ON_ONCE(ret == -EPROTONOSUPPORT)) 503 return -EINVAL; 504 505 return ret; 506} 507 508static void bundle_destroy(struct bundle_priv *pbundle, bool commit) 509{ 510 unsigned int key_bitmap_len = pbundle->method_elm->key_bitmap_len; 511 struct uverbs_attr_bundle *bundle = 512 container_of(&pbundle->bundle, struct uverbs_attr_bundle, hdr); 513 struct bundle_alloc_head *memblock; 514 unsigned int i; 515 516 /* fast path for simple uobjects */ 517 i = -1; 518 while ((i = find_next_bit(pbundle->uobj_finalize, key_bitmap_len, 519 i + 1)) < key_bitmap_len) { 520 struct uverbs_attr *attr = &bundle->attrs[i]; 521 522 uverbs_finalize_object( 523 attr->obj_attr.uobject, 524 attr->obj_attr.attr_elm->spec.u.obj.access, 525 test_bit(i, pbundle->uobj_hw_obj_valid), 526 commit, bundle); 527 } 528 529 i = -1; 530 while ((i = find_next_bit(pbundle->spec_finalize, key_bitmap_len, 531 i + 1)) < key_bitmap_len) { 532 struct uverbs_attr *attr = &bundle->attrs[i]; 533 const struct uverbs_api_attr *attr_uapi; 534 void __rcu **slot; 535 536 slot = uapi_get_attr_for_method( 537 pbundle, 538 pbundle->method_key | uapi_bkey_to_key_attr(i)); 539 if (WARN_ON(!slot)) 540 continue; 541 542 attr_uapi = rcu_dereference_protected(*slot, true); 543 544 if (attr_uapi->spec.type == UVERBS_ATTR_TYPE_IDRS_ARRAY) { 545 uverbs_free_idrs_array(attr_uapi, &attr->objs_arr_attr, 546 commit, bundle); 547 } 548 } 549 550 for (memblock = pbundle->allocated_mem; memblock;) { 551 struct bundle_alloc_head *tmp = memblock; 552 553 memblock = memblock->next; 554 kvfree(tmp); 555 } 556} 557 558static int ib_uverbs_cmd_verbs(struct ib_uverbs_file *ufile, 559 struct ib_uverbs_ioctl_hdr *hdr, 560 struct ib_uverbs_attr __user *user_attrs) 561{ 562 const struct uverbs_api_ioctl_method *method_elm; 563 struct uverbs_api *uapi = ufile->device->uapi; 564 struct radix_tree_iter attrs_iter; 565 struct bundle_priv *pbundle; 566 struct bundle_priv onstack; 567 void __rcu **slot; 568 int ret; 569 570 if (unlikely(hdr->driver_id != uapi->driver_id)) 571 return -EINVAL; 572 573 slot = radix_tree_iter_lookup( 574 &uapi->radix, &attrs_iter, 575 uapi_key_obj(hdr->object_id) | 576 uapi_key_ioctl_method(hdr->method_id)); 577 if (unlikely(!slot)) 578 return -EPROTONOSUPPORT; 579 method_elm = rcu_dereference_protected(*slot, true); 580 581 if (!method_elm->use_stack) { 582 pbundle = kmalloc(method_elm->bundle_size, GFP_KERNEL); 583 if (!pbundle) 584 return -ENOMEM; 585 pbundle->internal_avail = 586 method_elm->bundle_size - 587 offsetof(struct bundle_priv, internal_buffer); 588 pbundle->alloc_head.next = NULL; 589 pbundle->allocated_mem = container_of(&pbundle->alloc_head, 590 struct bundle_alloc_head, hdr); 591 } else { 592 pbundle = &onstack; 593 pbundle->internal_avail = sizeof(pbundle->internal_buffer); 594 pbundle->allocated_mem = NULL; 595 } 596 597 /* Space for the pbundle->bundle.attrs flex array */ 598 pbundle->method_elm = method_elm; 599 pbundle->method_key = attrs_iter.index; 600 pbundle->bundle.ufile = ufile; 601 pbundle->bundle.context = NULL; /* only valid if bundle has uobject */ 602 pbundle->radix = &uapi->radix; 603 pbundle->radix_slots = slot; 604 pbundle->radix_slots_len = radix_tree_chunk_size(&attrs_iter); 605 pbundle->user_attrs = user_attrs; 606 607 pbundle->internal_used = ALIGN(pbundle->method_elm->key_bitmap_len * 608 sizeof(*container_of(&pbundle->bundle, 609 struct uverbs_attr_bundle, hdr)->attrs), 610 sizeof(*pbundle->internal_buffer)); 611 memset(pbundle->bundle.attr_present, 0, 612 sizeof(pbundle->bundle.attr_present)); 613 memset(pbundle->uobj_finalize, 0, sizeof(pbundle->uobj_finalize)); 614 memset(pbundle->spec_finalize, 0, sizeof(pbundle->spec_finalize)); 615 memset(pbundle->uobj_hw_obj_valid, 0, 616 sizeof(pbundle->uobj_hw_obj_valid)); 617 618 ret = ib_uverbs_run_method(pbundle, hdr->num_attrs); 619 bundle_destroy(pbundle, ret == 0); 620 return ret; 621} 622 623long ib_uverbs_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) 624{ 625 struct ib_uverbs_file *file = filp->private_data; 626 struct ib_uverbs_ioctl_hdr __user *user_hdr = 627 (struct ib_uverbs_ioctl_hdr __user *)arg; 628 struct ib_uverbs_ioctl_hdr hdr; 629 int srcu_key; 630 int err; 631 632 if (unlikely(cmd != RDMA_VERBS_IOCTL)) 633 return -ENOIOCTLCMD; 634 635 err = copy_from_user(&hdr, user_hdr, sizeof(hdr)); 636 if (err) 637 return -EFAULT; 638 639 if (hdr.length > PAGE_SIZE || 640 hdr.length != struct_size(&hdr, attrs, hdr.num_attrs)) 641 return -EINVAL; 642 643 if (hdr.reserved1 || hdr.reserved2) 644 return -EPROTONOSUPPORT; 645 646 srcu_key = srcu_read_lock(&file->device->disassociate_srcu); 647 err = ib_uverbs_cmd_verbs(file, &hdr, user_hdr->attrs); 648 srcu_read_unlock(&file->device->disassociate_srcu, srcu_key); 649 return err; 650} 651 652int uverbs_get_flags64(u64 *to, const struct uverbs_attr_bundle *attrs_bundle, 653 size_t idx, u64 allowed_bits) 654{ 655 const struct uverbs_attr *attr; 656 u64 flags; 657 658 attr = uverbs_attr_get(attrs_bundle, idx); 659 /* Missing attribute means 0 flags */ 660 if (IS_ERR(attr)) { 661 *to = 0; 662 return 0; 663 } 664 665 /* 666 * New userspace code should use 8 bytes to pass flags, but we 667 * transparently support old userspaces that were using 4 bytes as 668 * well. 669 */ 670 if (attr->ptr_attr.len == 8) 671 flags = attr->ptr_attr.data; 672 else if (attr->ptr_attr.len == 4) 673 flags = *(u32 *)&attr->ptr_attr.data; 674 else 675 return -EINVAL; 676 677 if (flags & ~allowed_bits) 678 return -EINVAL; 679 680 *to = flags; 681 return 0; 682} 683EXPORT_SYMBOL(uverbs_get_flags64); 684 685int uverbs_get_flags32(u32 *to, const struct uverbs_attr_bundle *attrs_bundle, 686 size_t idx, u64 allowed_bits) 687{ 688 u64 flags; 689 int ret; 690 691 ret = uverbs_get_flags64(&flags, attrs_bundle, idx, allowed_bits); 692 if (ret) 693 return ret; 694 695 if (flags > U32_MAX) 696 return -EINVAL; 697 *to = flags; 698 699 return 0; 700} 701EXPORT_SYMBOL(uverbs_get_flags32); 702 703/* 704 * Fill a ib_udata struct (core or uhw) using the given attribute IDs. 705 * This is primarily used to convert the UVERBS_ATTR_UHW() into the 706 * ib_udata format used by the drivers. 707 */ 708void uverbs_fill_udata(struct uverbs_attr_bundle *bundle, 709 struct ib_udata *udata, unsigned int attr_in, 710 unsigned int attr_out) 711{ 712 struct bundle_priv *pbundle = 713 container_of(&bundle->hdr, struct bundle_priv, bundle); 714 struct uverbs_attr_bundle *bundle_aux = 715 container_of(&pbundle->bundle, struct uverbs_attr_bundle, hdr); 716 const struct uverbs_attr *in = 717 uverbs_attr_get(bundle_aux, attr_in); 718 const struct uverbs_attr *out = 719 uverbs_attr_get(bundle_aux, attr_out); 720 721 if (!IS_ERR(in)) { 722 udata->inlen = in->ptr_attr.len; 723 if (uverbs_attr_ptr_is_inline(in)) 724 udata->inbuf = 725 &pbundle->user_attrs[in->ptr_attr.uattr_idx] 726 .data; 727 else 728 udata->inbuf = u64_to_user_ptr(in->ptr_attr.data); 729 } else { 730 udata->inbuf = NULL; 731 udata->inlen = 0; 732 } 733 734 if (!IS_ERR(out)) { 735 udata->outbuf = u64_to_user_ptr(out->ptr_attr.data); 736 udata->outlen = out->ptr_attr.len; 737 } else { 738 udata->outbuf = NULL; 739 udata->outlen = 0; 740 } 741} 742 743int uverbs_copy_to(const struct uverbs_attr_bundle *bundle, size_t idx, 744 const void *from, size_t size) 745{ 746 const struct uverbs_attr *attr = uverbs_attr_get(bundle, idx); 747 size_t min_size; 748 749 if (IS_ERR(attr)) 750 return PTR_ERR(attr); 751 752 min_size = min_t(size_t, attr->ptr_attr.len, size); 753 if (copy_to_user(u64_to_user_ptr(attr->ptr_attr.data), from, min_size)) 754 return -EFAULT; 755 756 return uverbs_set_output(bundle, attr); 757} 758EXPORT_SYMBOL(uverbs_copy_to); 759 760 761/* 762 * This is only used if the caller has directly used copy_to_use to write the 763 * data. It signals to user space that the buffer is filled in. 764 */ 765int uverbs_output_written(const struct uverbs_attr_bundle *bundle, size_t idx) 766{ 767 const struct uverbs_attr *attr = uverbs_attr_get(bundle, idx); 768 769 if (IS_ERR(attr)) 770 return PTR_ERR(attr); 771 772 return uverbs_set_output(bundle, attr); 773} 774 775int _uverbs_get_const_signed(s64 *to, 776 const struct uverbs_attr_bundle *attrs_bundle, 777 size_t idx, s64 lower_bound, u64 upper_bound, 778 s64 *def_val) 779{ 780 const struct uverbs_attr *attr; 781 782 attr = uverbs_attr_get(attrs_bundle, idx); 783 if (IS_ERR(attr)) { 784 if ((PTR_ERR(attr) != -ENOENT) || !def_val) 785 return PTR_ERR(attr); 786 787 *to = *def_val; 788 } else { 789 *to = attr->ptr_attr.data; 790 } 791 792 if (*to < lower_bound || (*to > 0 && (u64)*to > upper_bound)) 793 return -EINVAL; 794 795 return 0; 796} 797EXPORT_SYMBOL(_uverbs_get_const_signed); 798 799int _uverbs_get_const_unsigned(u64 *to, 800 const struct uverbs_attr_bundle *attrs_bundle, 801 size_t idx, u64 upper_bound, u64 *def_val) 802{ 803 const struct uverbs_attr *attr; 804 805 attr = uverbs_attr_get(attrs_bundle, idx); 806 if (IS_ERR(attr)) { 807 if ((PTR_ERR(attr) != -ENOENT) || !def_val) 808 return PTR_ERR(attr); 809 810 *to = *def_val; 811 } else { 812 *to = attr->ptr_attr.data; 813 } 814 815 if (*to > upper_bound) 816 return -EINVAL; 817 818 return 0; 819} 820EXPORT_SYMBOL(_uverbs_get_const_unsigned); 821 822int uverbs_copy_to_struct_or_zero(const struct uverbs_attr_bundle *bundle, 823 size_t idx, const void *from, size_t size) 824{ 825 const struct uverbs_attr *attr = uverbs_attr_get(bundle, idx); 826 827 if (IS_ERR(attr)) 828 return PTR_ERR(attr); 829 830 if (size < attr->ptr_attr.len) { 831 if (clear_user(u64_to_user_ptr(attr->ptr_attr.data) + size, 832 attr->ptr_attr.len - size)) 833 return -EFAULT; 834 } 835 return uverbs_copy_to(bundle, idx, from, size); 836} 837EXPORT_SYMBOL(uverbs_copy_to_struct_or_zero); 838 839/* Once called an abort will call through to the type's destroy_hw() */ 840void uverbs_finalize_uobj_create(const struct uverbs_attr_bundle *bundle, 841 u16 idx) 842{ 843 struct bundle_priv *pbundle = 844 container_of(&bundle->hdr, struct bundle_priv, bundle); 845 846 __set_bit(uapi_bkey_attr(uapi_key_attr(idx)), 847 pbundle->uobj_hw_obj_valid); 848} 849EXPORT_SYMBOL(uverbs_finalize_uobj_create); 850