1// SPDX-License-Identifier: GPL-2.0-only OR MIT 2/* Copyright (c) 2023 Imagination Technologies Ltd. */ 3 4#include "pvr_free_list.h" 5#include "pvr_gem.h" 6#include "pvr_hwrt.h" 7#include "pvr_rogue_fwif.h" 8#include "pvr_vm.h" 9 10#include <drm/drm_gem.h> 11#include <linux/slab.h> 12#include <linux/xarray.h> 13#include <uapi/drm/pvr_drm.h> 14 15#define FREE_LIST_ENTRY_SIZE sizeof(u32) 16 17#define FREE_LIST_ALIGNMENT \ 18 ((ROGUE_BIF_PM_FREELIST_BASE_ADDR_ALIGNSIZE / FREE_LIST_ENTRY_SIZE) - 1) 19 20#define FREE_LIST_MIN_PAGES 50 21#define FREE_LIST_MIN_PAGES_BRN66011 40 22#define FREE_LIST_MIN_PAGES_ROGUEXE 25 23 24/** 25 * pvr_get_free_list_min_pages() - Get minimum free list size for this device 26 * @pvr_dev: Device pointer. 27 * 28 * Returns: 29 * * Minimum free list size, in PM physical pages. 30 */ 31u32 32pvr_get_free_list_min_pages(struct pvr_device *pvr_dev) 33{ 34 u32 value; 35 36 if (PVR_HAS_FEATURE(pvr_dev, roguexe)) { 37 if (PVR_HAS_QUIRK(pvr_dev, 66011)) 38 value = FREE_LIST_MIN_PAGES_BRN66011; 39 else 40 value = FREE_LIST_MIN_PAGES_ROGUEXE; 41 } else { 42 value = FREE_LIST_MIN_PAGES; 43 } 44 45 return value; 46} 47 48static int 49free_list_create_kernel_structure(struct pvr_file *pvr_file, 50 struct drm_pvr_ioctl_create_free_list_args *args, 51 struct pvr_free_list *free_list) 52{ 53 struct pvr_gem_object *free_list_obj; 54 struct pvr_vm_context *vm_ctx; 55 u64 free_list_size; 56 int err; 57 58 if (args->grow_threshold > 100 || 59 args->initial_num_pages > args->max_num_pages || 60 args->grow_num_pages > args->max_num_pages || 61 args->max_num_pages == 0 || 62 (args->initial_num_pages < args->max_num_pages && !args->grow_num_pages) || 63 (args->initial_num_pages == args->max_num_pages && args->grow_num_pages)) 64 return -EINVAL; 65 66 if ((args->initial_num_pages & FREE_LIST_ALIGNMENT) || 67 (args->max_num_pages & FREE_LIST_ALIGNMENT) || 68 (args->grow_num_pages & FREE_LIST_ALIGNMENT)) 69 return -EINVAL; 70 71 vm_ctx = pvr_vm_context_lookup(pvr_file, args->vm_context_handle); 72 if (!vm_ctx) 73 return -EINVAL; 74 75 free_list_obj = pvr_vm_find_gem_object(vm_ctx, args->free_list_gpu_addr, 76 NULL, &free_list_size); 77 if (!free_list_obj) { 78 err = -EINVAL; 79 goto err_put_vm_context; 80 } 81 82 if ((free_list_obj->flags & DRM_PVR_BO_ALLOW_CPU_USERSPACE_ACCESS) || 83 !(free_list_obj->flags & DRM_PVR_BO_PM_FW_PROTECT) || 84 free_list_size < (args->max_num_pages * FREE_LIST_ENTRY_SIZE)) { 85 err = -EINVAL; 86 goto err_put_free_list_obj; 87 } 88 89 free_list->pvr_dev = pvr_file->pvr_dev; 90 free_list->current_pages = 0; 91 free_list->max_pages = args->max_num_pages; 92 free_list->grow_pages = args->grow_num_pages; 93 free_list->grow_threshold = args->grow_threshold; 94 free_list->obj = free_list_obj; 95 free_list->free_list_gpu_addr = args->free_list_gpu_addr; 96 free_list->initial_num_pages = args->initial_num_pages; 97 98 pvr_vm_context_put(vm_ctx); 99 100 return 0; 101 102err_put_free_list_obj: 103 pvr_gem_object_put(free_list_obj); 104 105err_put_vm_context: 106 pvr_vm_context_put(vm_ctx); 107 108 return err; 109} 110 111static void 112free_list_destroy_kernel_structure(struct pvr_free_list *free_list) 113{ 114 WARN_ON(!list_empty(&free_list->hwrt_list)); 115 116 pvr_gem_object_put(free_list->obj); 117} 118 119/** 120 * calculate_free_list_ready_pages_locked() - Function to work out the number of free 121 * list pages to reserve for growing within 122 * the FW without having to wait for the 123 * host to progress a grow request 124 * @free_list: Pointer to free list. 125 * @pages: Total pages currently in free list. 126 * 127 * If the threshold or grow size means less than the alignment size (4 pages on 128 * Rogue), then the feature is not used. 129 * 130 * Caller must hold &free_list->lock. 131 * 132 * Return: number of pages to reserve. 133 */ 134static u32 135calculate_free_list_ready_pages_locked(struct pvr_free_list *free_list, u32 pages) 136{ 137 u32 ready_pages; 138 139 lockdep_assert_held(&free_list->lock); 140 141 ready_pages = ((pages * free_list->grow_threshold) / 100); 142 143 /* The number of pages must be less than the grow size. */ 144 ready_pages = min(ready_pages, free_list->grow_pages); 145 146 /* 147 * The number of pages must be a multiple of the free list align size. 148 */ 149 ready_pages &= ~FREE_LIST_ALIGNMENT; 150 151 return ready_pages; 152} 153 154static u32 155calculate_free_list_ready_pages(struct pvr_free_list *free_list, u32 pages) 156{ 157 u32 ret; 158 159 mutex_lock(&free_list->lock); 160 161 ret = calculate_free_list_ready_pages_locked(free_list, pages); 162 163 mutex_unlock(&free_list->lock); 164 165 return ret; 166} 167 168static void 169free_list_fw_init(void *cpu_ptr, void *priv) 170{ 171 struct rogue_fwif_freelist *fw_data = cpu_ptr; 172 struct pvr_free_list *free_list = priv; 173 u32 ready_pages; 174 175 /* Fill out FW structure */ 176 ready_pages = calculate_free_list_ready_pages(free_list, 177 free_list->initial_num_pages); 178 179 fw_data->max_pages = free_list->max_pages; 180 fw_data->current_pages = free_list->initial_num_pages - ready_pages; 181 fw_data->grow_pages = free_list->grow_pages; 182 fw_data->ready_pages = ready_pages; 183 fw_data->freelist_id = free_list->fw_id; 184 fw_data->grow_pending = false; 185 fw_data->current_stack_top = fw_data->current_pages - 1; 186 fw_data->freelist_dev_addr = free_list->free_list_gpu_addr; 187 fw_data->current_dev_addr = (fw_data->freelist_dev_addr + 188 ((fw_data->max_pages - fw_data->current_pages) * 189 FREE_LIST_ENTRY_SIZE)) & 190 ~((u64)ROGUE_BIF_PM_FREELIST_BASE_ADDR_ALIGNSIZE - 1); 191} 192 193static int 194free_list_create_fw_structure(struct pvr_file *pvr_file, 195 struct drm_pvr_ioctl_create_free_list_args *args, 196 struct pvr_free_list *free_list) 197{ 198 struct pvr_device *pvr_dev = pvr_file->pvr_dev; 199 200 /* 201 * Create and map the FW structure so we can initialise it. This is not 202 * accessed on the CPU side post-initialisation so the mapping lifetime 203 * is only for this function. 204 */ 205 free_list->fw_data = pvr_fw_object_create_and_map(pvr_dev, sizeof(*free_list->fw_data), 206 PVR_BO_FW_FLAGS_DEVICE_UNCACHED, 207 free_list_fw_init, free_list, 208 &free_list->fw_obj); 209 if (IS_ERR(free_list->fw_data)) 210 return PTR_ERR(free_list->fw_data); 211 212 return 0; 213} 214 215static void 216free_list_destroy_fw_structure(struct pvr_free_list *free_list) 217{ 218 pvr_fw_object_unmap_and_destroy(free_list->fw_obj); 219} 220 221static int 222pvr_free_list_insert_pages_locked(struct pvr_free_list *free_list, 223 struct sg_table *sgt, u32 offset, u32 num_pages) 224{ 225 struct sg_dma_page_iter dma_iter; 226 u32 *page_list; 227 228 lockdep_assert_held(&free_list->lock); 229 230 page_list = pvr_gem_object_vmap(free_list->obj); 231 if (IS_ERR(page_list)) 232 return PTR_ERR(page_list); 233 234 offset /= FREE_LIST_ENTRY_SIZE; 235 /* clang-format off */ 236 for_each_sgtable_dma_page(sgt, &dma_iter, 0) { 237 dma_addr_t dma_addr = sg_page_iter_dma_address(&dma_iter); 238 u64 dma_pfn = dma_addr >> 239 ROGUE_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT; 240 u32 dma_addr_offset; 241 242 BUILD_BUG_ON(ROGUE_BIF_PM_PHYSICAL_PAGE_SIZE > PAGE_SIZE); 243 244 for (dma_addr_offset = 0; dma_addr_offset < PAGE_SIZE; 245 dma_addr_offset += ROGUE_BIF_PM_PHYSICAL_PAGE_SIZE) { 246 WARN_ON_ONCE(dma_pfn >> 32); 247 248 page_list[offset++] = (u32)dma_pfn; 249 dma_pfn++; 250 251 num_pages--; 252 if (!num_pages) 253 break; 254 } 255 256 if (!num_pages) 257 break; 258 } 259 /* clang-format on */ 260 261 /* Make sure our free_list update is flushed. */ 262 wmb(); 263 264 pvr_gem_object_vunmap(free_list->obj); 265 266 return 0; 267} 268 269static int 270pvr_free_list_insert_node_locked(struct pvr_free_list_node *free_list_node) 271{ 272 struct pvr_free_list *free_list = free_list_node->free_list; 273 struct sg_table *sgt; 274 u32 start_page; 275 u32 offset; 276 int err; 277 278 lockdep_assert_held(&free_list->lock); 279 280 start_page = free_list->max_pages - free_list->current_pages - 281 free_list_node->num_pages; 282 offset = (start_page * FREE_LIST_ENTRY_SIZE) & 283 ~((u64)ROGUE_BIF_PM_FREELIST_BASE_ADDR_ALIGNSIZE - 1); 284 285 sgt = drm_gem_shmem_get_pages_sgt(&free_list_node->mem_obj->base); 286 if (WARN_ON(IS_ERR(sgt))) 287 return PTR_ERR(sgt); 288 289 err = pvr_free_list_insert_pages_locked(free_list, sgt, 290 offset, free_list_node->num_pages); 291 if (!err) 292 free_list->current_pages += free_list_node->num_pages; 293 294 return err; 295} 296 297static int 298pvr_free_list_grow(struct pvr_free_list *free_list, u32 num_pages) 299{ 300 struct pvr_device *pvr_dev = free_list->pvr_dev; 301 struct pvr_free_list_node *free_list_node; 302 int err; 303 304 mutex_lock(&free_list->lock); 305 306 if (num_pages & FREE_LIST_ALIGNMENT) { 307 err = -EINVAL; 308 goto err_unlock; 309 } 310 311 free_list_node = kzalloc(sizeof(*free_list_node), GFP_KERNEL); 312 if (!free_list_node) { 313 err = -ENOMEM; 314 goto err_unlock; 315 } 316 317 free_list_node->num_pages = num_pages; 318 free_list_node->free_list = free_list; 319 320 free_list_node->mem_obj = pvr_gem_object_create(pvr_dev, 321 num_pages << 322 ROGUE_BIF_PM_PHYSICAL_PAGE_ALIGNSHIFT, 323 PVR_BO_FW_FLAGS_DEVICE_CACHED); 324 if (IS_ERR(free_list_node->mem_obj)) { 325 err = PTR_ERR(free_list_node->mem_obj); 326 goto err_free; 327 } 328 329 err = pvr_free_list_insert_node_locked(free_list_node); 330 if (err) 331 goto err_destroy_gem_object; 332 333 list_add_tail(&free_list_node->node, &free_list->mem_block_list); 334 335 /* 336 * Reserve a number ready pages to allow the FW to process OOM quickly 337 * and asynchronously request a grow. 338 */ 339 free_list->ready_pages = 340 calculate_free_list_ready_pages_locked(free_list, 341 free_list->current_pages); 342 free_list->current_pages -= free_list->ready_pages; 343 344 mutex_unlock(&free_list->lock); 345 346 return 0; 347 348err_destroy_gem_object: 349 pvr_gem_object_put(free_list_node->mem_obj); 350 351err_free: 352 kfree(free_list_node); 353 354err_unlock: 355 mutex_unlock(&free_list->lock); 356 357 return err; 358} 359 360void pvr_free_list_process_grow_req(struct pvr_device *pvr_dev, 361 struct rogue_fwif_fwccb_cmd_freelist_gs_data *req) 362{ 363 struct pvr_free_list *free_list = pvr_free_list_lookup_id(pvr_dev, req->freelist_id); 364 struct rogue_fwif_kccb_cmd resp_cmd = { 365 .cmd_type = ROGUE_FWIF_KCCB_CMD_FREELIST_GROW_UPDATE, 366 }; 367 struct rogue_fwif_freelist_gs_data *resp = &resp_cmd.cmd_data.free_list_gs_data; 368 u32 grow_pages = 0; 369 370 /* If we don't have a freelist registered for this ID, we can't do much. */ 371 if (WARN_ON(!free_list)) 372 return; 373 374 /* Since the FW made the request, it has already consumed the ready pages, 375 * update the host struct. 376 */ 377 free_list->current_pages += free_list->ready_pages; 378 free_list->ready_pages = 0; 379 380 /* If the grow succeeds, update the grow_pages argument. */ 381 if (!pvr_free_list_grow(free_list, free_list->grow_pages)) 382 grow_pages = free_list->grow_pages; 383 384 /* Now prepare the response and send it back to the FW. */ 385 pvr_fw_object_get_fw_addr(free_list->fw_obj, &resp->freelist_fw_addr); 386 resp->delta_pages = grow_pages; 387 resp->new_pages = free_list->current_pages + free_list->ready_pages; 388 resp->ready_pages = free_list->ready_pages; 389 pvr_free_list_put(free_list); 390 391 WARN_ON(pvr_kccb_send_cmd(pvr_dev, &resp_cmd, NULL)); 392} 393 394static void 395pvr_free_list_free_node(struct pvr_free_list_node *free_list_node) 396{ 397 pvr_gem_object_put(free_list_node->mem_obj); 398 399 kfree(free_list_node); 400} 401 402/** 403 * pvr_free_list_create() - Create a new free list and return an object pointer 404 * @pvr_file: Pointer to pvr_file structure. 405 * @args: Creation arguments from userspace. 406 * 407 * Return: 408 * * Pointer to new free_list, or 409 * * ERR_PTR(-%ENOMEM) on out of memory. 410 */ 411struct pvr_free_list * 412pvr_free_list_create(struct pvr_file *pvr_file, 413 struct drm_pvr_ioctl_create_free_list_args *args) 414{ 415 struct pvr_free_list *free_list; 416 int err; 417 418 /* Create and fill out the kernel structure */ 419 free_list = kzalloc(sizeof(*free_list), GFP_KERNEL); 420 421 if (!free_list) 422 return ERR_PTR(-ENOMEM); 423 424 kref_init(&free_list->ref_count); 425 INIT_LIST_HEAD(&free_list->mem_block_list); 426 INIT_LIST_HEAD(&free_list->hwrt_list); 427 mutex_init(&free_list->lock); 428 429 err = free_list_create_kernel_structure(pvr_file, args, free_list); 430 if (err < 0) 431 goto err_free; 432 433 /* Allocate global object ID for firmware. */ 434 err = xa_alloc(&pvr_file->pvr_dev->free_list_ids, 435 &free_list->fw_id, 436 free_list, 437 xa_limit_32b, 438 GFP_KERNEL); 439 if (err) 440 goto err_destroy_kernel_structure; 441 442 err = free_list_create_fw_structure(pvr_file, args, free_list); 443 if (err < 0) 444 goto err_free_fw_id; 445 446 err = pvr_free_list_grow(free_list, args->initial_num_pages); 447 if (err < 0) 448 goto err_fw_struct_cleanup; 449 450 return free_list; 451 452err_fw_struct_cleanup: 453 WARN_ON(pvr_fw_structure_cleanup(free_list->pvr_dev, 454 ROGUE_FWIF_CLEANUP_FREELIST, 455 free_list->fw_obj, 0)); 456 457err_free_fw_id: 458 xa_erase(&free_list->pvr_dev->free_list_ids, free_list->fw_id); 459 460err_destroy_kernel_structure: 461 free_list_destroy_kernel_structure(free_list); 462 463err_free: 464 mutex_destroy(&free_list->lock); 465 kfree(free_list); 466 467 return ERR_PTR(err); 468} 469 470static void 471pvr_free_list_release(struct kref *ref_count) 472{ 473 struct pvr_free_list *free_list = 474 container_of(ref_count, struct pvr_free_list, ref_count); 475 struct list_head *pos, *n; 476 int err; 477 478 xa_erase(&free_list->pvr_dev->free_list_ids, free_list->fw_id); 479 480 err = pvr_fw_structure_cleanup(free_list->pvr_dev, 481 ROGUE_FWIF_CLEANUP_FREELIST, 482 free_list->fw_obj, 0); 483 if (err == -EBUSY) { 484 /* Flush the FWCCB to process any HWR or freelist reconstruction 485 * request that might keep the freelist busy, and try again. 486 */ 487 pvr_fwccb_process(free_list->pvr_dev); 488 err = pvr_fw_structure_cleanup(free_list->pvr_dev, 489 ROGUE_FWIF_CLEANUP_FREELIST, 490 free_list->fw_obj, 0); 491 } 492 493 WARN_ON(err); 494 495 /* clang-format off */ 496 list_for_each_safe(pos, n, &free_list->mem_block_list) { 497 struct pvr_free_list_node *free_list_node = 498 container_of(pos, struct pvr_free_list_node, node); 499 500 list_del(pos); 501 pvr_free_list_free_node(free_list_node); 502 } 503 /* clang-format on */ 504 505 free_list_destroy_kernel_structure(free_list); 506 free_list_destroy_fw_structure(free_list); 507 mutex_destroy(&free_list->lock); 508 kfree(free_list); 509} 510 511/** 512 * pvr_destroy_free_lists_for_file: Destroy any free lists associated with the 513 * given file. 514 * @pvr_file: Pointer to pvr_file structure. 515 * 516 * Removes all free lists associated with @pvr_file from the device free_list 517 * list and drops initial references. Free lists will then be destroyed once 518 * all outstanding references are dropped. 519 */ 520void pvr_destroy_free_lists_for_file(struct pvr_file *pvr_file) 521{ 522 struct pvr_free_list *free_list; 523 unsigned long handle; 524 525 xa_for_each(&pvr_file->free_list_handles, handle, free_list) { 526 (void)free_list; 527 pvr_free_list_put(xa_erase(&pvr_file->free_list_handles, handle)); 528 } 529} 530 531/** 532 * pvr_free_list_put() - Release reference on free list 533 * @free_list: Pointer to list to release reference on 534 */ 535void 536pvr_free_list_put(struct pvr_free_list *free_list) 537{ 538 if (free_list) 539 kref_put(&free_list->ref_count, pvr_free_list_release); 540} 541 542void pvr_free_list_add_hwrt(struct pvr_free_list *free_list, struct pvr_hwrt_data *hwrt_data) 543{ 544 mutex_lock(&free_list->lock); 545 546 list_add_tail(&hwrt_data->freelist_node, &free_list->hwrt_list); 547 548 mutex_unlock(&free_list->lock); 549} 550 551void pvr_free_list_remove_hwrt(struct pvr_free_list *free_list, struct pvr_hwrt_data *hwrt_data) 552{ 553 mutex_lock(&free_list->lock); 554 555 list_del(&hwrt_data->freelist_node); 556 557 mutex_unlock(&free_list->lock); 558} 559 560static void 561pvr_free_list_reconstruct(struct pvr_device *pvr_dev, u32 freelist_id) 562{ 563 struct pvr_free_list *free_list = pvr_free_list_lookup_id(pvr_dev, freelist_id); 564 struct pvr_free_list_node *free_list_node; 565 struct rogue_fwif_freelist *fw_data; 566 struct pvr_hwrt_data *hwrt_data; 567 568 if (!free_list) 569 return; 570 571 mutex_lock(&free_list->lock); 572 573 /* Rebuild the free list based on the memory block list. */ 574 free_list->current_pages = 0; 575 576 list_for_each_entry(free_list_node, &free_list->mem_block_list, node) 577 WARN_ON(pvr_free_list_insert_node_locked(free_list_node)); 578 579 /* 580 * Remove the ready pages, which are reserved to allow the FW to process OOM quickly and 581 * asynchronously request a grow. 582 */ 583 free_list->current_pages -= free_list->ready_pages; 584 585 fw_data = free_list->fw_data; 586 fw_data->current_stack_top = fw_data->current_pages - 1; 587 fw_data->allocated_page_count = 0; 588 fw_data->allocated_mmu_page_count = 0; 589 590 /* Reset the state of any associated HWRTs. */ 591 list_for_each_entry(hwrt_data, &free_list->hwrt_list, freelist_node) { 592 struct rogue_fwif_hwrtdata *hwrt_fw_data = pvr_fw_object_vmap(hwrt_data->fw_obj); 593 594 if (!WARN_ON(IS_ERR(hwrt_fw_data))) { 595 hwrt_fw_data->state = ROGUE_FWIF_RTDATA_STATE_HWR; 596 hwrt_fw_data->hwrt_data_flags &= ~HWRTDATA_HAS_LAST_GEOM; 597 } 598 599 pvr_fw_object_vunmap(hwrt_data->fw_obj); 600 } 601 602 mutex_unlock(&free_list->lock); 603 604 pvr_free_list_put(free_list); 605} 606 607void 608pvr_free_list_process_reconstruct_req(struct pvr_device *pvr_dev, 609 struct rogue_fwif_fwccb_cmd_freelists_reconstruction_data *req) 610{ 611 struct rogue_fwif_kccb_cmd resp_cmd = { 612 .cmd_type = ROGUE_FWIF_KCCB_CMD_FREELISTS_RECONSTRUCTION_UPDATE, 613 }; 614 struct rogue_fwif_freelists_reconstruction_data *resp = 615 &resp_cmd.cmd_data.free_lists_reconstruction_data; 616 617 for (u32 i = 0; i < req->freelist_count; i++) 618 pvr_free_list_reconstruct(pvr_dev, req->freelist_ids[i]); 619 620 resp->freelist_count = req->freelist_count; 621 memcpy(resp->freelist_ids, req->freelist_ids, 622 req->freelist_count * sizeof(resp->freelist_ids[0])); 623 624 WARN_ON(pvr_kccb_send_cmd(pvr_dev, &resp_cmd, NULL)); 625} 626