1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2/* 3 * Copyright (c) 2021, Mellanox Technologies inc. All rights reserved. 4 */ 5 6#include <rdma/uverbs_std_types.h> 7#include "dm.h" 8 9#define UVERBS_MODULE_NAME mlx5_ib 10#include <rdma/uverbs_named_ioctl.h> 11 12static int mlx5_cmd_alloc_memic(struct mlx5_dm *dm, phys_addr_t *addr, 13 u64 length, u32 alignment) 14{ 15 struct mlx5_core_dev *dev = dm->dev; 16 u64 num_memic_hw_pages = MLX5_CAP_DEV_MEM(dev, memic_bar_size) 17 >> PAGE_SHIFT; 18 u64 hw_start_addr = MLX5_CAP64_DEV_MEM(dev, memic_bar_start_addr); 19 u32 max_alignment = MLX5_CAP_DEV_MEM(dev, log_max_memic_addr_alignment); 20 u32 num_pages = DIV_ROUND_UP(length, PAGE_SIZE); 21 u32 out[MLX5_ST_SZ_DW(alloc_memic_out)] = {}; 22 u32 in[MLX5_ST_SZ_DW(alloc_memic_in)] = {}; 23 u32 mlx5_alignment; 24 u64 page_idx = 0; 25 int ret = 0; 26 27 if (!length || (length & MLX5_MEMIC_ALLOC_SIZE_MASK)) 28 return -EINVAL; 29 30 /* mlx5 device sets alignment as 64*2^driver_value 31 * so normalizing is needed. 32 */ 33 mlx5_alignment = (alignment < MLX5_MEMIC_BASE_ALIGN) ? 0 : 34 alignment - MLX5_MEMIC_BASE_ALIGN; 35 if (mlx5_alignment > max_alignment) 36 return -EINVAL; 37 38 MLX5_SET(alloc_memic_in, in, opcode, MLX5_CMD_OP_ALLOC_MEMIC); 39 MLX5_SET(alloc_memic_in, in, range_size, num_pages * PAGE_SIZE); 40 MLX5_SET(alloc_memic_in, in, memic_size, length); 41 MLX5_SET(alloc_memic_in, in, log_memic_addr_alignment, 42 mlx5_alignment); 43 44 while (page_idx < num_memic_hw_pages) { 45 spin_lock(&dm->lock); 46 page_idx = bitmap_find_next_zero_area(dm->memic_alloc_pages, 47 num_memic_hw_pages, 48 page_idx, 49 num_pages, 0); 50 51 if (page_idx < num_memic_hw_pages) 52 bitmap_set(dm->memic_alloc_pages, 53 page_idx, num_pages); 54 55 spin_unlock(&dm->lock); 56 57 if (page_idx >= num_memic_hw_pages) 58 break; 59 60 MLX5_SET64(alloc_memic_in, in, range_start_addr, 61 hw_start_addr + (page_idx * PAGE_SIZE)); 62 63 ret = mlx5_cmd_exec_inout(dev, alloc_memic, in, out); 64 if (ret) { 65 spin_lock(&dm->lock); 66 bitmap_clear(dm->memic_alloc_pages, 67 page_idx, num_pages); 68 spin_unlock(&dm->lock); 69 70 if (ret == -EAGAIN) { 71 page_idx++; 72 continue; 73 } 74 75 return ret; 76 } 77 78 *addr = dev->bar_addr + 79 MLX5_GET64(alloc_memic_out, out, memic_start_addr); 80 81 return 0; 82 } 83 84 return -ENOMEM; 85} 86 87void mlx5_cmd_dealloc_memic(struct mlx5_dm *dm, phys_addr_t addr, 88 u64 length) 89{ 90 struct mlx5_core_dev *dev = dm->dev; 91 u64 hw_start_addr = MLX5_CAP64_DEV_MEM(dev, memic_bar_start_addr); 92 u32 num_pages = DIV_ROUND_UP(length, PAGE_SIZE); 93 u32 in[MLX5_ST_SZ_DW(dealloc_memic_in)] = {}; 94 u64 start_page_idx; 95 int err; 96 97 addr -= dev->bar_addr; 98 start_page_idx = (addr - hw_start_addr) >> PAGE_SHIFT; 99 100 MLX5_SET(dealloc_memic_in, in, opcode, MLX5_CMD_OP_DEALLOC_MEMIC); 101 MLX5_SET64(dealloc_memic_in, in, memic_start_addr, addr); 102 MLX5_SET(dealloc_memic_in, in, memic_size, length); 103 104 err = mlx5_cmd_exec_in(dev, dealloc_memic, in); 105 if (err) 106 return; 107 108 spin_lock(&dm->lock); 109 bitmap_clear(dm->memic_alloc_pages, 110 start_page_idx, num_pages); 111 spin_unlock(&dm->lock); 112} 113 114void mlx5_cmd_dealloc_memic_op(struct mlx5_dm *dm, phys_addr_t addr, 115 u8 operation) 116{ 117 u32 in[MLX5_ST_SZ_DW(modify_memic_in)] = {}; 118 struct mlx5_core_dev *dev = dm->dev; 119 120 MLX5_SET(modify_memic_in, in, opcode, MLX5_CMD_OP_MODIFY_MEMIC); 121 MLX5_SET(modify_memic_in, in, op_mod, MLX5_MODIFY_MEMIC_OP_MOD_DEALLOC); 122 MLX5_SET(modify_memic_in, in, memic_operation_type, operation); 123 MLX5_SET64(modify_memic_in, in, memic_start_addr, addr - dev->bar_addr); 124 125 mlx5_cmd_exec_in(dev, modify_memic, in); 126} 127 128static int mlx5_cmd_alloc_memic_op(struct mlx5_dm *dm, phys_addr_t addr, 129 u8 operation, phys_addr_t *op_addr) 130{ 131 u32 out[MLX5_ST_SZ_DW(modify_memic_out)] = {}; 132 u32 in[MLX5_ST_SZ_DW(modify_memic_in)] = {}; 133 struct mlx5_core_dev *dev = dm->dev; 134 int err; 135 136 MLX5_SET(modify_memic_in, in, opcode, MLX5_CMD_OP_MODIFY_MEMIC); 137 MLX5_SET(modify_memic_in, in, op_mod, MLX5_MODIFY_MEMIC_OP_MOD_ALLOC); 138 MLX5_SET(modify_memic_in, in, memic_operation_type, operation); 139 MLX5_SET64(modify_memic_in, in, memic_start_addr, addr - dev->bar_addr); 140 141 err = mlx5_cmd_exec_inout(dev, modify_memic, in, out); 142 if (err) 143 return err; 144 145 *op_addr = dev->bar_addr + 146 MLX5_GET64(modify_memic_out, out, memic_operation_addr); 147 return 0; 148} 149 150static int add_dm_mmap_entry(struct ib_ucontext *context, 151 struct mlx5_user_mmap_entry *mentry, u8 mmap_flag, 152 size_t size, u64 address) 153{ 154 mentry->mmap_flag = mmap_flag; 155 mentry->address = address; 156 157 return rdma_user_mmap_entry_insert_range( 158 context, &mentry->rdma_entry, size, 159 MLX5_IB_MMAP_DEVICE_MEM << 16, 160 (MLX5_IB_MMAP_DEVICE_MEM << 16) + (1UL << 16) - 1); 161} 162 163static void mlx5_ib_dm_memic_free(struct kref *kref) 164{ 165 struct mlx5_ib_dm_memic *dm = 166 container_of(kref, struct mlx5_ib_dm_memic, ref); 167 struct mlx5_ib_dev *dev = to_mdev(dm->base.ibdm.device); 168 169 mlx5_cmd_dealloc_memic(&dev->dm, dm->base.dev_addr, dm->base.size); 170 kfree(dm); 171} 172 173static int copy_op_to_user(struct mlx5_ib_dm_op_entry *op_entry, 174 struct uverbs_attr_bundle *attrs) 175{ 176 u64 start_offset; 177 u16 page_idx; 178 int err; 179 180 page_idx = op_entry->mentry.rdma_entry.start_pgoff & 0xFFFF; 181 start_offset = op_entry->op_addr & ~PAGE_MASK; 182 err = uverbs_copy_to(attrs, MLX5_IB_ATTR_DM_MAP_OP_ADDR_RESP_PAGE_INDEX, 183 &page_idx, sizeof(page_idx)); 184 if (err) 185 return err; 186 187 return uverbs_copy_to(attrs, 188 MLX5_IB_ATTR_DM_MAP_OP_ADDR_RESP_START_OFFSET, 189 &start_offset, sizeof(start_offset)); 190} 191 192static int map_existing_op(struct mlx5_ib_dm_memic *dm, u8 op, 193 struct uverbs_attr_bundle *attrs) 194{ 195 struct mlx5_ib_dm_op_entry *op_entry; 196 197 op_entry = xa_load(&dm->ops, op); 198 if (!op_entry) 199 return -ENOENT; 200 201 return copy_op_to_user(op_entry, attrs); 202} 203 204static int UVERBS_HANDLER(MLX5_IB_METHOD_DM_MAP_OP_ADDR)( 205 struct uverbs_attr_bundle *attrs) 206{ 207 struct ib_uobject *uobj = uverbs_attr_get_uobject( 208 attrs, MLX5_IB_ATTR_DM_MAP_OP_ADDR_REQ_HANDLE); 209 struct mlx5_ib_dev *dev = to_mdev(uobj->context->device); 210 struct ib_dm *ibdm = uobj->object; 211 struct mlx5_ib_dm_memic *dm = to_memic(ibdm); 212 struct mlx5_ib_dm_op_entry *op_entry; 213 int err; 214 u8 op; 215 216 err = uverbs_copy_from(&op, attrs, MLX5_IB_ATTR_DM_MAP_OP_ADDR_REQ_OP); 217 if (err) 218 return err; 219 220 if (op >= BITS_PER_TYPE(u32)) 221 return -EOPNOTSUPP; 222 223 if (!(MLX5_CAP_DEV_MEM(dev->mdev, memic_operations) & BIT(op))) 224 return -EOPNOTSUPP; 225 226 mutex_lock(&dm->ops_xa_lock); 227 err = map_existing_op(dm, op, attrs); 228 if (!err || err != -ENOENT) 229 goto err_unlock; 230 231 op_entry = kzalloc(sizeof(*op_entry), GFP_KERNEL); 232 if (!op_entry) 233 goto err_unlock; 234 235 err = mlx5_cmd_alloc_memic_op(&dev->dm, dm->base.dev_addr, op, 236 &op_entry->op_addr); 237 if (err) { 238 kfree(op_entry); 239 goto err_unlock; 240 } 241 op_entry->op = op; 242 op_entry->dm = dm; 243 244 err = add_dm_mmap_entry(uobj->context, &op_entry->mentry, 245 MLX5_IB_MMAP_TYPE_MEMIC_OP, dm->base.size, 246 op_entry->op_addr & PAGE_MASK); 247 if (err) { 248 mlx5_cmd_dealloc_memic_op(&dev->dm, dm->base.dev_addr, op); 249 kfree(op_entry); 250 goto err_unlock; 251 } 252 /* From this point, entry will be freed by mmap_free */ 253 kref_get(&dm->ref); 254 255 err = copy_op_to_user(op_entry, attrs); 256 if (err) 257 goto err_remove; 258 259 err = xa_insert(&dm->ops, op, op_entry, GFP_KERNEL); 260 if (err) 261 goto err_remove; 262 mutex_unlock(&dm->ops_xa_lock); 263 264 return 0; 265 266err_remove: 267 rdma_user_mmap_entry_remove(&op_entry->mentry.rdma_entry); 268err_unlock: 269 mutex_unlock(&dm->ops_xa_lock); 270 271 return err; 272} 273 274static struct ib_dm *handle_alloc_dm_memic(struct ib_ucontext *ctx, 275 struct ib_dm_alloc_attr *attr, 276 struct uverbs_attr_bundle *attrs) 277{ 278 struct mlx5_dm *dm_db = &to_mdev(ctx->device)->dm; 279 struct mlx5_ib_dm_memic *dm; 280 u64 start_offset; 281 u16 page_idx; 282 int err; 283 u64 address; 284 285 if (!MLX5_CAP_DEV_MEM(dm_db->dev, memic)) 286 return ERR_PTR(-EOPNOTSUPP); 287 288 dm = kzalloc(sizeof(*dm), GFP_KERNEL); 289 if (!dm) 290 return ERR_PTR(-ENOMEM); 291 292 dm->base.type = MLX5_IB_UAPI_DM_TYPE_MEMIC; 293 dm->base.size = roundup(attr->length, MLX5_MEMIC_BASE_SIZE); 294 dm->base.ibdm.device = ctx->device; 295 296 kref_init(&dm->ref); 297 xa_init(&dm->ops); 298 mutex_init(&dm->ops_xa_lock); 299 dm->req_length = attr->length; 300 301 err = mlx5_cmd_alloc_memic(dm_db, &dm->base.dev_addr, 302 dm->base.size, attr->alignment); 303 if (err) { 304 kfree(dm); 305 return ERR_PTR(err); 306 } 307 308 address = dm->base.dev_addr & PAGE_MASK; 309 err = add_dm_mmap_entry(ctx, &dm->mentry, MLX5_IB_MMAP_TYPE_MEMIC, 310 dm->base.size, address); 311 if (err) { 312 mlx5_cmd_dealloc_memic(dm_db, dm->base.dev_addr, dm->base.size); 313 kfree(dm); 314 return ERR_PTR(err); 315 } 316 317 page_idx = dm->mentry.rdma_entry.start_pgoff & 0xFFFF; 318 err = uverbs_copy_to(attrs, MLX5_IB_ATTR_ALLOC_DM_RESP_PAGE_INDEX, 319 &page_idx, sizeof(page_idx)); 320 if (err) 321 goto err_copy; 322 323 start_offset = dm->base.dev_addr & ~PAGE_MASK; 324 err = uverbs_copy_to(attrs, 325 MLX5_IB_ATTR_ALLOC_DM_RESP_START_OFFSET, 326 &start_offset, sizeof(start_offset)); 327 if (err) 328 goto err_copy; 329 330 return &dm->base.ibdm; 331 332err_copy: 333 rdma_user_mmap_entry_remove(&dm->mentry.rdma_entry); 334 return ERR_PTR(err); 335} 336 337static enum mlx5_sw_icm_type get_icm_type(int uapi_type) 338{ 339 switch (uapi_type) { 340 case MLX5_IB_UAPI_DM_TYPE_HEADER_MODIFY_SW_ICM: 341 return MLX5_SW_ICM_TYPE_HEADER_MODIFY; 342 case MLX5_IB_UAPI_DM_TYPE_HEADER_MODIFY_PATTERN_SW_ICM: 343 return MLX5_SW_ICM_TYPE_HEADER_MODIFY_PATTERN; 344 case MLX5_IB_UAPI_DM_TYPE_ENCAP_SW_ICM: 345 return MLX5_SW_ICM_TYPE_SW_ENCAP; 346 case MLX5_IB_UAPI_DM_TYPE_STEERING_SW_ICM: 347 default: 348 return MLX5_SW_ICM_TYPE_STEERING; 349 } 350} 351 352static struct ib_dm *handle_alloc_dm_sw_icm(struct ib_ucontext *ctx, 353 struct ib_dm_alloc_attr *attr, 354 struct uverbs_attr_bundle *attrs, 355 int type) 356{ 357 struct mlx5_core_dev *dev = to_mdev(ctx->device)->mdev; 358 enum mlx5_sw_icm_type icm_type; 359 struct mlx5_ib_dm_icm *dm; 360 u64 act_size; 361 int err; 362 363 if (!capable(CAP_SYS_RAWIO) || !capable(CAP_NET_RAW)) 364 return ERR_PTR(-EPERM); 365 366 switch (type) { 367 case MLX5_IB_UAPI_DM_TYPE_STEERING_SW_ICM: 368 case MLX5_IB_UAPI_DM_TYPE_HEADER_MODIFY_SW_ICM: 369 case MLX5_IB_UAPI_DM_TYPE_ENCAP_SW_ICM: 370 if (!(MLX5_CAP_FLOWTABLE_NIC_RX(dev, sw_owner) || 371 MLX5_CAP_FLOWTABLE_NIC_TX(dev, sw_owner) || 372 MLX5_CAP_FLOWTABLE_NIC_RX(dev, sw_owner_v2) || 373 MLX5_CAP_FLOWTABLE_NIC_TX(dev, sw_owner_v2))) 374 return ERR_PTR(-EOPNOTSUPP); 375 break; 376 case MLX5_IB_UAPI_DM_TYPE_HEADER_MODIFY_PATTERN_SW_ICM: 377 if (!MLX5_CAP_FLOWTABLE_NIC_RX(dev, sw_owner_v2) || 378 !MLX5_CAP_FLOWTABLE_NIC_TX(dev, sw_owner_v2)) 379 return ERR_PTR(-EOPNOTSUPP); 380 break; 381 default: 382 return ERR_PTR(-EOPNOTSUPP); 383 } 384 385 dm = kzalloc(sizeof(*dm), GFP_KERNEL); 386 if (!dm) 387 return ERR_PTR(-ENOMEM); 388 389 dm->base.type = type; 390 dm->base.ibdm.device = ctx->device; 391 392 /* Allocation size must a multiple of the basic block size 393 * and a power of 2. 394 */ 395 act_size = round_up(attr->length, MLX5_SW_ICM_BLOCK_SIZE(dev)); 396 act_size = roundup_pow_of_two(act_size); 397 398 dm->base.size = act_size; 399 icm_type = get_icm_type(type); 400 401 err = mlx5_dm_sw_icm_alloc(dev, icm_type, act_size, attr->alignment, 402 to_mucontext(ctx)->devx_uid, 403 &dm->base.dev_addr, &dm->obj_id); 404 if (err) 405 goto free; 406 407 err = uverbs_copy_to(attrs, MLX5_IB_ATTR_ALLOC_DM_RESP_START_OFFSET, 408 &dm->base.dev_addr, sizeof(dm->base.dev_addr)); 409 if (err) { 410 mlx5_dm_sw_icm_dealloc(dev, icm_type, dm->base.size, 411 to_mucontext(ctx)->devx_uid, 412 dm->base.dev_addr, dm->obj_id); 413 goto free; 414 } 415 return &dm->base.ibdm; 416free: 417 kfree(dm); 418 return ERR_PTR(err); 419} 420 421struct ib_dm *mlx5_ib_alloc_dm(struct ib_device *ibdev, 422 struct ib_ucontext *context, 423 struct ib_dm_alloc_attr *attr, 424 struct uverbs_attr_bundle *attrs) 425{ 426 enum mlx5_ib_uapi_dm_type type; 427 int err; 428 429 err = uverbs_get_const_default(&type, attrs, 430 MLX5_IB_ATTR_ALLOC_DM_REQ_TYPE, 431 MLX5_IB_UAPI_DM_TYPE_MEMIC); 432 if (err) 433 return ERR_PTR(err); 434 435 mlx5_ib_dbg(to_mdev(ibdev), "alloc_dm req: dm_type=%d user_length=0x%llx log_alignment=%d\n", 436 type, attr->length, attr->alignment); 437 438 switch (type) { 439 case MLX5_IB_UAPI_DM_TYPE_MEMIC: 440 return handle_alloc_dm_memic(context, attr, attrs); 441 case MLX5_IB_UAPI_DM_TYPE_STEERING_SW_ICM: 442 case MLX5_IB_UAPI_DM_TYPE_HEADER_MODIFY_SW_ICM: 443 case MLX5_IB_UAPI_DM_TYPE_HEADER_MODIFY_PATTERN_SW_ICM: 444 case MLX5_IB_UAPI_DM_TYPE_ENCAP_SW_ICM: 445 return handle_alloc_dm_sw_icm(context, attr, attrs, type); 446 default: 447 return ERR_PTR(-EOPNOTSUPP); 448 } 449} 450 451static void dm_memic_remove_ops(struct mlx5_ib_dm_memic *dm) 452{ 453 struct mlx5_ib_dm_op_entry *entry; 454 unsigned long idx; 455 456 mutex_lock(&dm->ops_xa_lock); 457 xa_for_each(&dm->ops, idx, entry) { 458 xa_erase(&dm->ops, idx); 459 rdma_user_mmap_entry_remove(&entry->mentry.rdma_entry); 460 } 461 mutex_unlock(&dm->ops_xa_lock); 462} 463 464static void mlx5_dm_memic_dealloc(struct mlx5_ib_dm_memic *dm) 465{ 466 dm_memic_remove_ops(dm); 467 rdma_user_mmap_entry_remove(&dm->mentry.rdma_entry); 468} 469 470static int mlx5_dm_icm_dealloc(struct mlx5_ib_ucontext *ctx, 471 struct mlx5_ib_dm_icm *dm) 472{ 473 enum mlx5_sw_icm_type type = get_icm_type(dm->base.type); 474 struct mlx5_core_dev *dev = to_mdev(dm->base.ibdm.device)->mdev; 475 int err; 476 477 err = mlx5_dm_sw_icm_dealloc(dev, type, dm->base.size, ctx->devx_uid, 478 dm->base.dev_addr, dm->obj_id); 479 if (!err) 480 kfree(dm); 481 return 0; 482} 483 484static int mlx5_ib_dealloc_dm(struct ib_dm *ibdm, 485 struct uverbs_attr_bundle *attrs) 486{ 487 struct mlx5_ib_ucontext *ctx = rdma_udata_to_drv_context( 488 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext); 489 struct mlx5_ib_dm *dm = to_mdm(ibdm); 490 491 switch (dm->type) { 492 case MLX5_IB_UAPI_DM_TYPE_MEMIC: 493 mlx5_dm_memic_dealloc(to_memic(ibdm)); 494 return 0; 495 case MLX5_IB_UAPI_DM_TYPE_STEERING_SW_ICM: 496 case MLX5_IB_UAPI_DM_TYPE_HEADER_MODIFY_SW_ICM: 497 case MLX5_IB_UAPI_DM_TYPE_HEADER_MODIFY_PATTERN_SW_ICM: 498 case MLX5_IB_UAPI_DM_TYPE_ENCAP_SW_ICM: 499 return mlx5_dm_icm_dealloc(ctx, to_icm(ibdm)); 500 default: 501 return -EOPNOTSUPP; 502 } 503} 504 505static int UVERBS_HANDLER(MLX5_IB_METHOD_DM_QUERY)( 506 struct uverbs_attr_bundle *attrs) 507{ 508 struct ib_dm *ibdm = 509 uverbs_attr_get_obj(attrs, MLX5_IB_ATTR_QUERY_DM_REQ_HANDLE); 510 struct mlx5_ib_dm *dm = to_mdm(ibdm); 511 struct mlx5_ib_dm_memic *memic; 512 u64 start_offset; 513 u16 page_idx; 514 int err; 515 516 if (dm->type != MLX5_IB_UAPI_DM_TYPE_MEMIC) 517 return -EOPNOTSUPP; 518 519 memic = to_memic(ibdm); 520 page_idx = memic->mentry.rdma_entry.start_pgoff & 0xFFFF; 521 err = uverbs_copy_to(attrs, MLX5_IB_ATTR_QUERY_DM_RESP_PAGE_INDEX, 522 &page_idx, sizeof(page_idx)); 523 if (err) 524 return err; 525 526 start_offset = memic->base.dev_addr & ~PAGE_MASK; 527 err = uverbs_copy_to(attrs, MLX5_IB_ATTR_QUERY_DM_RESP_START_OFFSET, 528 &start_offset, sizeof(start_offset)); 529 if (err) 530 return err; 531 532 return uverbs_copy_to(attrs, MLX5_IB_ATTR_QUERY_DM_RESP_LENGTH, 533 &memic->req_length, 534 sizeof(memic->req_length)); 535} 536 537void mlx5_ib_dm_mmap_free(struct mlx5_ib_dev *dev, 538 struct mlx5_user_mmap_entry *mentry) 539{ 540 struct mlx5_ib_dm_op_entry *op_entry; 541 struct mlx5_ib_dm_memic *mdm; 542 543 switch (mentry->mmap_flag) { 544 case MLX5_IB_MMAP_TYPE_MEMIC: 545 mdm = container_of(mentry, struct mlx5_ib_dm_memic, mentry); 546 kref_put(&mdm->ref, mlx5_ib_dm_memic_free); 547 break; 548 case MLX5_IB_MMAP_TYPE_MEMIC_OP: 549 op_entry = container_of(mentry, struct mlx5_ib_dm_op_entry, 550 mentry); 551 mdm = op_entry->dm; 552 mlx5_cmd_dealloc_memic_op(&dev->dm, mdm->base.dev_addr, 553 op_entry->op); 554 kfree(op_entry); 555 kref_put(&mdm->ref, mlx5_ib_dm_memic_free); 556 break; 557 default: 558 WARN_ON(true); 559 } 560} 561 562DECLARE_UVERBS_NAMED_METHOD( 563 MLX5_IB_METHOD_DM_QUERY, 564 UVERBS_ATTR_IDR(MLX5_IB_ATTR_QUERY_DM_REQ_HANDLE, UVERBS_OBJECT_DM, 565 UVERBS_ACCESS_READ, UA_MANDATORY), 566 UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_QUERY_DM_RESP_START_OFFSET, 567 UVERBS_ATTR_TYPE(u64), UA_MANDATORY), 568 UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_QUERY_DM_RESP_PAGE_INDEX, 569 UVERBS_ATTR_TYPE(u16), UA_MANDATORY), 570 UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_QUERY_DM_RESP_LENGTH, 571 UVERBS_ATTR_TYPE(u64), UA_MANDATORY)); 572 573ADD_UVERBS_ATTRIBUTES_SIMPLE( 574 mlx5_ib_dm, UVERBS_OBJECT_DM, UVERBS_METHOD_DM_ALLOC, 575 UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_ALLOC_DM_RESP_START_OFFSET, 576 UVERBS_ATTR_TYPE(u64), UA_MANDATORY), 577 UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_ALLOC_DM_RESP_PAGE_INDEX, 578 UVERBS_ATTR_TYPE(u16), UA_OPTIONAL), 579 UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_ALLOC_DM_REQ_TYPE, 580 enum mlx5_ib_uapi_dm_type, UA_OPTIONAL)); 581 582DECLARE_UVERBS_NAMED_METHOD( 583 MLX5_IB_METHOD_DM_MAP_OP_ADDR, 584 UVERBS_ATTR_IDR(MLX5_IB_ATTR_DM_MAP_OP_ADDR_REQ_HANDLE, 585 UVERBS_OBJECT_DM, 586 UVERBS_ACCESS_READ, 587 UA_MANDATORY), 588 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DM_MAP_OP_ADDR_REQ_OP, 589 UVERBS_ATTR_TYPE(u8), 590 UA_MANDATORY), 591 UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_DM_MAP_OP_ADDR_RESP_START_OFFSET, 592 UVERBS_ATTR_TYPE(u64), 593 UA_MANDATORY), 594 UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_DM_MAP_OP_ADDR_RESP_PAGE_INDEX, 595 UVERBS_ATTR_TYPE(u16), 596 UA_OPTIONAL)); 597 598DECLARE_UVERBS_GLOBAL_METHODS(UVERBS_OBJECT_DM, 599 &UVERBS_METHOD(MLX5_IB_METHOD_DM_MAP_OP_ADDR), 600 &UVERBS_METHOD(MLX5_IB_METHOD_DM_QUERY)); 601 602const struct uapi_definition mlx5_ib_dm_defs[] = { 603 UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_DM, &mlx5_ib_dm), 604 UAPI_DEF_CHAIN_OBJ_TREE_NAMED(UVERBS_OBJECT_DM), 605 {}, 606}; 607 608const struct ib_device_ops mlx5_ib_dev_dm_ops = { 609 .alloc_dm = mlx5_ib_alloc_dm, 610 .dealloc_dm = mlx5_ib_dealloc_dm, 611 .reg_dm_mr = mlx5_ib_reg_dm_mr, 612}; 613