1// SPDX-License-Identifier: GPL-2.0-only OR MIT 2/* Copyright (c) 2023 Imagination Technologies Ltd. */ 3 4#include "pvr_free_list.h" 5#include "pvr_hwrt.h" 6#include "pvr_gem.h" 7#include "pvr_rogue_cr_defs_client.h" 8#include "pvr_rogue_fwif.h" 9 10#include <drm/drm_gem.h> 11#include <linux/bitops.h> 12#include <linux/math.h> 13#include <linux/slab.h> 14#include <linux/xarray.h> 15#include <uapi/drm/pvr_drm.h> 16 17static_assert(ROGUE_FWIF_NUM_RTDATAS == 2); 18static_assert(ROGUE_FWIF_NUM_GEOMDATAS == 1); 19static_assert(ROGUE_FWIF_NUM_RTDATA_FREELISTS == 2); 20 21/* 22 * struct pvr_rt_mtile_info - Render target macrotile information 23 */ 24struct pvr_rt_mtile_info { 25 u32 mtile_x[3]; 26 u32 mtile_y[3]; 27 u32 tile_max_x; 28 u32 tile_max_y; 29 u32 tile_size_x; 30 u32 tile_size_y; 31 u32 num_tiles_x; 32 u32 num_tiles_y; 33}; 34 35/* Size of Shadow Render Target Cache entry */ 36#define SRTC_ENTRY_SIZE sizeof(u32) 37/* Size of Renders Accumulation Array entry */ 38#define RAA_ENTRY_SIZE sizeof(u32) 39 40static int 41hwrt_init_kernel_structure(struct pvr_file *pvr_file, 42 struct drm_pvr_ioctl_create_hwrt_dataset_args *args, 43 struct pvr_hwrt_dataset *hwrt) 44{ 45 struct pvr_device *pvr_dev = pvr_file->pvr_dev; 46 int err; 47 int i; 48 49 hwrt->pvr_dev = pvr_dev; 50 hwrt->max_rts = args->layers; 51 52 /* Get pointers to the free lists */ 53 for (i = 0; i < ARRAY_SIZE(hwrt->free_lists); i++) { 54 hwrt->free_lists[i] = pvr_free_list_lookup(pvr_file, args->free_list_handles[i]); 55 if (!hwrt->free_lists[i]) { 56 err = -EINVAL; 57 goto err_put_free_lists; 58 } 59 } 60 61 if (hwrt->free_lists[ROGUE_FW_LOCAL_FREELIST]->current_pages < 62 pvr_get_free_list_min_pages(pvr_dev)) { 63 err = -EINVAL; 64 goto err_put_free_lists; 65 } 66 67 return 0; 68 69err_put_free_lists: 70 for (i = 0; i < ARRAY_SIZE(hwrt->free_lists); i++) { 71 pvr_free_list_put(hwrt->free_lists[i]); 72 hwrt->free_lists[i] = NULL; 73 } 74 75 return err; 76} 77 78static void 79hwrt_fini_kernel_structure(struct pvr_hwrt_dataset *hwrt) 80{ 81 int i; 82 83 for (i = 0; i < ARRAY_SIZE(hwrt->free_lists); i++) { 84 pvr_free_list_put(hwrt->free_lists[i]); 85 hwrt->free_lists[i] = NULL; 86 } 87} 88 89static void 90hwrt_fini_common_fw_structure(struct pvr_hwrt_dataset *hwrt) 91{ 92 pvr_fw_object_destroy(hwrt->common_fw_obj); 93} 94 95static int 96get_cr_isp_mtile_size_val(struct pvr_device *pvr_dev, u32 samples, 97 struct pvr_rt_mtile_info *info, u32 *value_out) 98{ 99 u32 x = info->mtile_x[0]; 100 u32 y = info->mtile_y[0]; 101 u32 samples_per_pixel; 102 int err; 103 104 err = PVR_FEATURE_VALUE(pvr_dev, isp_samples_per_pixel, &samples_per_pixel); 105 if (err) 106 return err; 107 108 if (samples_per_pixel == 1) { 109 if (samples >= 4) 110 x <<= 1; 111 if (samples >= 2) 112 y <<= 1; 113 } else if (samples_per_pixel == 2) { 114 if (samples >= 8) 115 x <<= 1; 116 if (samples >= 4) 117 y <<= 1; 118 } else if (samples_per_pixel == 4) { 119 if (samples >= 8) 120 y <<= 1; 121 } else { 122 WARN(true, "Unsupported ISP samples per pixel value"); 123 return -EINVAL; 124 } 125 126 *value_out = ((x << ROGUE_CR_ISP_MTILE_SIZE_X_SHIFT) & ~ROGUE_CR_ISP_MTILE_SIZE_X_CLRMSK) | 127 ((y << ROGUE_CR_ISP_MTILE_SIZE_Y_SHIFT) & ~ROGUE_CR_ISP_MTILE_SIZE_Y_CLRMSK); 128 129 return 0; 130} 131 132static int 133get_cr_multisamplectl_val(u32 samples, bool y_flip, u64 *value_out) 134{ 135 static const struct { 136 u8 x[8]; 137 u8 y[8]; 138 } sample_positions[4] = { 139 /* 1 sample */ 140 { 141 .x = { 8 }, 142 .y = { 8 }, 143 }, 144 /* 2 samples */ 145 { 146 .x = { 12, 4 }, 147 .y = { 12, 4 }, 148 }, 149 /* 4 samples */ 150 { 151 .x = { 6, 14, 2, 10 }, 152 .y = { 2, 6, 10, 14 }, 153 }, 154 /* 8 samples */ 155 { 156 .x = { 9, 7, 13, 5, 3, 1, 11, 15 }, 157 .y = { 5, 11, 9, 3, 13, 7, 15, 1 }, 158 }, 159 }; 160 const int idx = fls(samples) - 1; 161 u64 value = 0; 162 163 if (idx < 0 || idx > 3) 164 return -EINVAL; 165 166 for (u32 i = 0; i < 8; i++) { 167 value |= ((u64)sample_positions[idx].x[i]) << (i * 8); 168 if (y_flip) 169 value |= (((u64)(16 - sample_positions[idx].y[i]) & 0xf)) << (i * 8 + 4); 170 else 171 value |= ((u64)sample_positions[idx].y[i]) << (i * 8 + 4); 172 } 173 174 *value_out = value; 175 176 return 0; 177} 178 179static int 180get_cr_te_aa_val(struct pvr_device *pvr_dev, u32 samples, u32 *value_out) 181{ 182 u32 samples_per_pixel; 183 u32 value = 0; 184 int err = 0; 185 186 err = PVR_FEATURE_VALUE(pvr_dev, isp_samples_per_pixel, &samples_per_pixel); 187 if (err) 188 return err; 189 190 switch (samples_per_pixel) { 191 case 1: 192 if (samples >= 2) 193 value |= ROGUE_CR_TE_AA_Y_EN; 194 if (samples >= 4) 195 value |= ROGUE_CR_TE_AA_X_EN; 196 break; 197 case 2: 198 if (samples >= 2) 199 value |= ROGUE_CR_TE_AA_X2_EN; 200 if (samples >= 4) 201 value |= ROGUE_CR_TE_AA_Y_EN; 202 if (samples >= 8) 203 value |= ROGUE_CR_TE_AA_X_EN; 204 break; 205 case 4: 206 if (samples >= 2) 207 value |= ROGUE_CR_TE_AA_X2_EN; 208 if (samples >= 4) 209 value |= ROGUE_CR_TE_AA_Y2_EN; 210 if (samples >= 8) 211 value |= ROGUE_CR_TE_AA_Y_EN; 212 break; 213 default: 214 WARN(true, "Unsupported ISP samples per pixel value"); 215 return -EINVAL; 216 } 217 218 *value_out = value; 219 220 return 0; 221} 222 223static void 224hwrtdata_common_init(void *cpu_ptr, void *priv) 225{ 226 struct pvr_hwrt_dataset *hwrt = priv; 227 228 memcpy(cpu_ptr, &hwrt->common, sizeof(hwrt->common)); 229} 230 231static int 232hwrt_init_common_fw_structure(struct pvr_file *pvr_file, 233 struct drm_pvr_ioctl_create_hwrt_dataset_args *args, 234 struct pvr_hwrt_dataset *hwrt) 235{ 236 struct drm_pvr_create_hwrt_geom_data_args *geom_data_args = &args->geom_data_args; 237 struct pvr_device *pvr_dev = pvr_file->pvr_dev; 238 struct pvr_rt_mtile_info info; 239 int err; 240 241 err = PVR_FEATURE_VALUE(pvr_dev, tile_size_x, &info.tile_size_x); 242 if (WARN_ON(err)) 243 return err; 244 245 err = PVR_FEATURE_VALUE(pvr_dev, tile_size_y, &info.tile_size_y); 246 if (WARN_ON(err)) 247 return err; 248 249 info.num_tiles_x = DIV_ROUND_UP(args->width, info.tile_size_x); 250 info.num_tiles_y = DIV_ROUND_UP(args->height, info.tile_size_y); 251 252 if (PVR_HAS_FEATURE(pvr_dev, simple_parameter_format_version)) { 253 u32 parameter_format; 254 255 err = PVR_FEATURE_VALUE(pvr_dev, simple_parameter_format_version, 256 ¶meter_format); 257 if (WARN_ON(err)) 258 return err; 259 260 WARN_ON(parameter_format != 2); 261 262 /* 263 * Set up 16 macrotiles with a multiple of 2x2 tiles per macrotile, which is 264 * aligned to a tile group. 265 */ 266 info.mtile_x[0] = DIV_ROUND_UP(info.num_tiles_x, 8) * 2; 267 info.mtile_y[0] = DIV_ROUND_UP(info.num_tiles_y, 8) * 2; 268 info.mtile_x[1] = 0; 269 info.mtile_y[1] = 0; 270 info.mtile_x[2] = 0; 271 info.mtile_y[2] = 0; 272 info.tile_max_x = round_up(info.num_tiles_x, 2) - 1; 273 info.tile_max_y = round_up(info.num_tiles_y, 2) - 1; 274 } else { 275 /* Set up 16 macrotiles with a multiple of 4x4 tiles per macrotile. */ 276 info.mtile_x[0] = round_up(DIV_ROUND_UP(info.num_tiles_x, 4), 4); 277 info.mtile_y[0] = round_up(DIV_ROUND_UP(info.num_tiles_y, 4), 4); 278 info.mtile_x[1] = info.mtile_x[0] * 2; 279 info.mtile_y[1] = info.mtile_y[0] * 2; 280 info.mtile_x[2] = info.mtile_x[0] * 3; 281 info.mtile_y[2] = info.mtile_y[0] * 3; 282 info.tile_max_x = info.num_tiles_x - 1; 283 info.tile_max_y = info.num_tiles_y - 1; 284 } 285 286 hwrt->common.geom_caches_need_zeroing = false; 287 288 hwrt->common.isp_merge_lower_x = args->isp_merge_lower_x; 289 hwrt->common.isp_merge_lower_y = args->isp_merge_lower_y; 290 hwrt->common.isp_merge_upper_x = args->isp_merge_upper_x; 291 hwrt->common.isp_merge_upper_y = args->isp_merge_upper_y; 292 hwrt->common.isp_merge_scale_x = args->isp_merge_scale_x; 293 hwrt->common.isp_merge_scale_y = args->isp_merge_scale_y; 294 295 err = get_cr_multisamplectl_val(args->samples, false, 296 &hwrt->common.multi_sample_ctl); 297 if (err) 298 return err; 299 300 err = get_cr_multisamplectl_val(args->samples, true, 301 &hwrt->common.flipped_multi_sample_ctl); 302 if (err) 303 return err; 304 305 hwrt->common.mtile_stride = info.mtile_x[0] * info.mtile_y[0]; 306 307 err = get_cr_te_aa_val(pvr_dev, args->samples, &hwrt->common.teaa); 308 if (err) 309 return err; 310 311 hwrt->common.screen_pixel_max = 312 (((args->width - 1) << ROGUE_CR_PPP_SCREEN_PIXXMAX_SHIFT) & 313 ~ROGUE_CR_PPP_SCREEN_PIXXMAX_CLRMSK) | 314 (((args->height - 1) << ROGUE_CR_PPP_SCREEN_PIXYMAX_SHIFT) & 315 ~ROGUE_CR_PPP_SCREEN_PIXYMAX_CLRMSK); 316 317 hwrt->common.te_screen = 318 ((info.tile_max_x << ROGUE_CR_TE_SCREEN_XMAX_SHIFT) & 319 ~ROGUE_CR_TE_SCREEN_XMAX_CLRMSK) | 320 ((info.tile_max_y << ROGUE_CR_TE_SCREEN_YMAX_SHIFT) & 321 ~ROGUE_CR_TE_SCREEN_YMAX_CLRMSK); 322 hwrt->common.te_mtile1 = 323 ((info.mtile_x[0] << ROGUE_CR_TE_MTILE1_X1_SHIFT) & ~ROGUE_CR_TE_MTILE1_X1_CLRMSK) | 324 ((info.mtile_x[1] << ROGUE_CR_TE_MTILE1_X2_SHIFT) & ~ROGUE_CR_TE_MTILE1_X2_CLRMSK) | 325 ((info.mtile_x[2] << ROGUE_CR_TE_MTILE1_X3_SHIFT) & ~ROGUE_CR_TE_MTILE1_X3_CLRMSK); 326 hwrt->common.te_mtile2 = 327 ((info.mtile_y[0] << ROGUE_CR_TE_MTILE2_Y1_SHIFT) & ~ROGUE_CR_TE_MTILE2_Y1_CLRMSK) | 328 ((info.mtile_y[1] << ROGUE_CR_TE_MTILE2_Y2_SHIFT) & ~ROGUE_CR_TE_MTILE2_Y2_CLRMSK) | 329 ((info.mtile_y[2] << ROGUE_CR_TE_MTILE2_Y3_SHIFT) & ~ROGUE_CR_TE_MTILE2_Y3_CLRMSK); 330 331 err = get_cr_isp_mtile_size_val(pvr_dev, args->samples, &info, 332 &hwrt->common.isp_mtile_size); 333 if (err) 334 return err; 335 336 hwrt->common.tpc_stride = geom_data_args->tpc_stride; 337 hwrt->common.tpc_size = geom_data_args->tpc_size; 338 339 hwrt->common.rgn_header_size = args->region_header_size; 340 341 err = pvr_fw_object_create(pvr_dev, sizeof(struct rogue_fwif_hwrtdata_common), 342 PVR_BO_FW_FLAGS_DEVICE_UNCACHED, hwrtdata_common_init, hwrt, 343 &hwrt->common_fw_obj); 344 345 return err; 346} 347 348static void 349hwrt_fw_data_init(void *cpu_ptr, void *priv) 350{ 351 struct pvr_hwrt_data *hwrt_data = priv; 352 353 memcpy(cpu_ptr, &hwrt_data->data, sizeof(hwrt_data->data)); 354} 355 356static int 357hwrt_data_init_fw_structure(struct pvr_file *pvr_file, 358 struct pvr_hwrt_dataset *hwrt, 359 struct drm_pvr_ioctl_create_hwrt_dataset_args *args, 360 struct drm_pvr_create_hwrt_rt_data_args *rt_data_args, 361 struct pvr_hwrt_data *hwrt_data) 362{ 363 struct drm_pvr_create_hwrt_geom_data_args *geom_data_args = &args->geom_data_args; 364 struct pvr_device *pvr_dev = pvr_file->pvr_dev; 365 struct rogue_fwif_rta_ctl *rta_ctl; 366 int free_list_i; 367 int err; 368 369 pvr_fw_object_get_fw_addr(hwrt->common_fw_obj, 370 &hwrt_data->data.hwrt_data_common_fw_addr); 371 372 for (free_list_i = 0; free_list_i < ARRAY_SIZE(hwrt->free_lists); free_list_i++) { 373 pvr_fw_object_get_fw_addr(hwrt->free_lists[free_list_i]->fw_obj, 374 &hwrt_data->data.freelists_fw_addr[free_list_i]); 375 } 376 377 hwrt_data->data.tail_ptrs_dev_addr = geom_data_args->tpc_dev_addr; 378 hwrt_data->data.vheap_table_dev_addr = geom_data_args->vheap_table_dev_addr; 379 hwrt_data->data.rtc_dev_addr = geom_data_args->rtc_dev_addr; 380 381 hwrt_data->data.pm_mlist_dev_addr = rt_data_args->pm_mlist_dev_addr; 382 hwrt_data->data.macrotile_array_dev_addr = rt_data_args->macrotile_array_dev_addr; 383 hwrt_data->data.rgn_header_dev_addr = rt_data_args->region_header_dev_addr; 384 385 rta_ctl = &hwrt_data->data.rta_ctl; 386 387 rta_ctl->render_target_index = 0; 388 rta_ctl->active_render_targets = 0; 389 rta_ctl->valid_render_targets_fw_addr = 0; 390 rta_ctl->rta_num_partial_renders_fw_addr = 0; 391 rta_ctl->max_rts = args->layers; 392 393 if (args->layers > 1) { 394 err = pvr_fw_object_create(pvr_dev, args->layers * SRTC_ENTRY_SIZE, 395 PVR_BO_FW_FLAGS_DEVICE_UNCACHED, 396 NULL, NULL, &hwrt_data->srtc_obj); 397 if (err) 398 return err; 399 pvr_fw_object_get_fw_addr(hwrt_data->srtc_obj, 400 &rta_ctl->valid_render_targets_fw_addr); 401 402 err = pvr_fw_object_create(pvr_dev, args->layers * RAA_ENTRY_SIZE, 403 PVR_BO_FW_FLAGS_DEVICE_UNCACHED, 404 NULL, NULL, &hwrt_data->raa_obj); 405 if (err) 406 goto err_put_shadow_rt_cache; 407 pvr_fw_object_get_fw_addr(hwrt_data->raa_obj, 408 &rta_ctl->rta_num_partial_renders_fw_addr); 409 } 410 411 err = pvr_fw_object_create(pvr_dev, sizeof(struct rogue_fwif_hwrtdata), 412 PVR_BO_FW_FLAGS_DEVICE_UNCACHED, 413 hwrt_fw_data_init, hwrt_data, &hwrt_data->fw_obj); 414 if (err) 415 goto err_put_raa_obj; 416 417 pvr_free_list_add_hwrt(hwrt->free_lists[0], hwrt_data); 418 419 return 0; 420 421err_put_raa_obj: 422 if (args->layers > 1) 423 pvr_fw_object_destroy(hwrt_data->raa_obj); 424 425err_put_shadow_rt_cache: 426 if (args->layers > 1) 427 pvr_fw_object_destroy(hwrt_data->srtc_obj); 428 429 return err; 430} 431 432static void 433hwrt_data_fini_fw_structure(struct pvr_hwrt_dataset *hwrt, int hwrt_nr) 434{ 435 struct pvr_hwrt_data *hwrt_data = &hwrt->data[hwrt_nr]; 436 437 pvr_free_list_remove_hwrt(hwrt->free_lists[0], hwrt_data); 438 439 if (hwrt->max_rts > 1) { 440 pvr_fw_object_destroy(hwrt_data->raa_obj); 441 pvr_fw_object_destroy(hwrt_data->srtc_obj); 442 } 443 444 pvr_fw_object_destroy(hwrt_data->fw_obj); 445} 446 447/** 448 * pvr_hwrt_dataset_create() - Create a new HWRT dataset 449 * @pvr_file: Pointer to pvr_file structure. 450 * @args: Creation arguments from userspace. 451 * 452 * Return: 453 * * Pointer to new HWRT, or 454 * * ERR_PTR(-%ENOMEM) on out of memory. 455 */ 456struct pvr_hwrt_dataset * 457pvr_hwrt_dataset_create(struct pvr_file *pvr_file, 458 struct drm_pvr_ioctl_create_hwrt_dataset_args *args) 459{ 460 struct pvr_hwrt_dataset *hwrt; 461 int err, i = 0; 462 463 /* Create and fill out the kernel structure */ 464 hwrt = kzalloc(sizeof(*hwrt), GFP_KERNEL); 465 466 if (!hwrt) 467 return ERR_PTR(-ENOMEM); 468 469 err = hwrt_init_kernel_structure(pvr_file, args, hwrt); 470 if (err < 0) 471 goto err_free; 472 473 err = hwrt_init_common_fw_structure(pvr_file, args, hwrt); 474 if (err < 0) 475 goto err_fini_kernel_structure; 476 477 for (; i < ARRAY_SIZE(hwrt->data); i++) { 478 err = hwrt_data_init_fw_structure(pvr_file, hwrt, args, 479 &args->rt_data_args[i], 480 &hwrt->data[i]); 481 if (err < 0) 482 goto err_fini_data_structures; 483 484 hwrt->data[i].hwrt_dataset = hwrt; 485 } 486 487 kref_init(&hwrt->ref_count); 488 return hwrt; 489 490err_fini_data_structures: 491 while (--i >= 0) 492 hwrt_data_fini_fw_structure(hwrt, i); 493 494err_fini_kernel_structure: 495 hwrt_fini_kernel_structure(hwrt); 496 497err_free: 498 kfree(hwrt); 499 500 return ERR_PTR(err); 501} 502 503static void 504pvr_hwrt_dataset_release(struct kref *ref_count) 505{ 506 struct pvr_hwrt_dataset *hwrt = 507 container_of(ref_count, struct pvr_hwrt_dataset, ref_count); 508 509 for (int i = ARRAY_SIZE(hwrt->data) - 1; i >= 0; i--) { 510 WARN_ON(pvr_fw_structure_cleanup(hwrt->pvr_dev, ROGUE_FWIF_CLEANUP_HWRTDATA, 511 hwrt->data[i].fw_obj, 0)); 512 hwrt_data_fini_fw_structure(hwrt, i); 513 } 514 515 hwrt_fini_common_fw_structure(hwrt); 516 hwrt_fini_kernel_structure(hwrt); 517 518 kfree(hwrt); 519} 520 521/** 522 * pvr_destroy_hwrt_datasets_for_file: Destroy any HWRT datasets associated 523 * with the given file. 524 * @pvr_file: Pointer to pvr_file structure. 525 * 526 * Removes all HWRT datasets associated with @pvr_file from the device 527 * hwrt_dataset list and drops initial references. HWRT datasets will then be 528 * destroyed once all outstanding references are dropped. 529 */ 530void pvr_destroy_hwrt_datasets_for_file(struct pvr_file *pvr_file) 531{ 532 struct pvr_hwrt_dataset *hwrt; 533 unsigned long handle; 534 535 xa_for_each(&pvr_file->hwrt_handles, handle, hwrt) { 536 (void)hwrt; 537 pvr_hwrt_dataset_put(xa_erase(&pvr_file->hwrt_handles, handle)); 538 } 539} 540 541/** 542 * pvr_hwrt_dataset_put() - Release reference on HWRT dataset 543 * @hwrt: Pointer to HWRT dataset to release reference on 544 */ 545void 546pvr_hwrt_dataset_put(struct pvr_hwrt_dataset *hwrt) 547{ 548 if (hwrt) 549 kref_put(&hwrt->ref_count, pvr_hwrt_dataset_release); 550} 551