1// SPDX-License-Identifier: GPL-2.0-only OR MIT 2/* Copyright (c) 2023 Imagination Technologies Ltd. */ 3 4#include "pvr_device.h" 5#include "pvr_gem.h" 6#include "pvr_vm.h" 7 8#include <drm/drm_gem.h> 9#include <drm/drm_prime.h> 10 11#include <linux/compiler.h> 12#include <linux/compiler_attributes.h> 13#include <linux/dma-buf.h> 14#include <linux/dma-direction.h> 15#include <linux/dma-mapping.h> 16#include <linux/err.h> 17#include <linux/gfp.h> 18#include <linux/iosys-map.h> 19#include <linux/log2.h> 20#include <linux/mutex.h> 21#include <linux/pagemap.h> 22#include <linux/refcount.h> 23#include <linux/scatterlist.h> 24 25static void pvr_gem_object_free(struct drm_gem_object *obj) 26{ 27 drm_gem_shmem_object_free(obj); 28} 29 30static int pvr_gem_mmap(struct drm_gem_object *gem_obj, struct vm_area_struct *vma) 31{ 32 struct pvr_gem_object *pvr_obj = gem_to_pvr_gem(gem_obj); 33 struct drm_gem_shmem_object *shmem_obj = shmem_gem_from_pvr_gem(pvr_obj); 34 35 if (!(pvr_obj->flags & DRM_PVR_BO_ALLOW_CPU_USERSPACE_ACCESS)) 36 return -EINVAL; 37 38 return drm_gem_shmem_mmap(shmem_obj, vma); 39} 40 41static const struct drm_gem_object_funcs pvr_gem_object_funcs = { 42 .free = pvr_gem_object_free, 43 .print_info = drm_gem_shmem_object_print_info, 44 .pin = drm_gem_shmem_object_pin, 45 .unpin = drm_gem_shmem_object_unpin, 46 .get_sg_table = drm_gem_shmem_object_get_sg_table, 47 .vmap = drm_gem_shmem_object_vmap, 48 .vunmap = drm_gem_shmem_object_vunmap, 49 .mmap = pvr_gem_mmap, 50 .vm_ops = &drm_gem_shmem_vm_ops, 51}; 52 53/** 54 * pvr_gem_object_flags_validate() - Verify that a collection of PowerVR GEM 55 * mapping and/or creation flags form a valid combination. 56 * @flags: PowerVR GEM mapping/creation flags to validate. 57 * 58 * This function explicitly allows kernel-only flags. All ioctl entrypoints 59 * should do their own validation as well as relying on this function. 60 * 61 * Return: 62 * * %true if @flags contains valid mapping and/or creation flags, or 63 * * %false otherwise. 64 */ 65static bool 66pvr_gem_object_flags_validate(u64 flags) 67{ 68 static const u64 invalid_combinations[] = { 69 /* 70 * Memory flagged as PM/FW-protected cannot be mapped to 71 * userspace. To make this explicit, we require that the two 72 * flags allowing each of these respective features are never 73 * specified together. 74 */ 75 (DRM_PVR_BO_PM_FW_PROTECT | 76 DRM_PVR_BO_ALLOW_CPU_USERSPACE_ACCESS), 77 }; 78 79 int i; 80 81 /* 82 * Check for bits set in undefined regions. Reserved regions refer to 83 * options that can only be set by the kernel. These are explicitly 84 * allowed in most cases, and must be checked specifically in IOCTL 85 * callback code. 86 */ 87 if ((flags & PVR_BO_UNDEFINED_MASK) != 0) 88 return false; 89 90 /* 91 * Check for all combinations of flags marked as invalid in the array 92 * above. 93 */ 94 for (i = 0; i < ARRAY_SIZE(invalid_combinations); ++i) { 95 u64 combo = invalid_combinations[i]; 96 97 if ((flags & combo) == combo) 98 return false; 99 } 100 101 return true; 102} 103 104/** 105 * pvr_gem_object_into_handle() - Convert a reference to an object into a 106 * userspace-accessible handle. 107 * @pvr_obj: [IN] Target PowerVR-specific object. 108 * @pvr_file: [IN] File to associate the handle with. 109 * @handle: [OUT] Pointer to store the created handle in. Remains unmodified if 110 * an error is encountered. 111 * 112 * If an error is encountered, ownership of @pvr_obj will not have been 113 * transferred. If this function succeeds, however, further use of @pvr_obj is 114 * considered undefined behaviour unless another reference to it is explicitly 115 * held. 116 * 117 * Return: 118 * * 0 on success, or 119 * * Any error encountered while attempting to allocate a handle on @pvr_file. 120 */ 121int 122pvr_gem_object_into_handle(struct pvr_gem_object *pvr_obj, 123 struct pvr_file *pvr_file, u32 *handle) 124{ 125 struct drm_gem_object *gem_obj = gem_from_pvr_gem(pvr_obj); 126 struct drm_file *file = from_pvr_file(pvr_file); 127 128 u32 new_handle; 129 int err; 130 131 err = drm_gem_handle_create(file, gem_obj, &new_handle); 132 if (err) 133 return err; 134 135 /* 136 * Release our reference to @pvr_obj, effectively transferring 137 * ownership to the handle. 138 */ 139 pvr_gem_object_put(pvr_obj); 140 141 /* 142 * Do not store the new handle in @handle until no more errors can 143 * occur. 144 */ 145 *handle = new_handle; 146 147 return 0; 148} 149 150/** 151 * pvr_gem_object_from_handle() - Obtain a reference to an object from a 152 * userspace handle. 153 * @pvr_file: PowerVR-specific file to which @handle is associated. 154 * @handle: Userspace handle referencing the target object. 155 * 156 * On return, @handle always maintains its reference to the requested object 157 * (if it had one in the first place). If this function succeeds, the returned 158 * object will hold an additional reference. When the caller is finished with 159 * the returned object, they should call pvr_gem_object_put() on it to release 160 * this reference. 161 * 162 * Return: 163 * * A pointer to the requested PowerVR-specific object on success, or 164 * * %NULL otherwise. 165 */ 166struct pvr_gem_object * 167pvr_gem_object_from_handle(struct pvr_file *pvr_file, u32 handle) 168{ 169 struct drm_file *file = from_pvr_file(pvr_file); 170 struct drm_gem_object *gem_obj; 171 172 gem_obj = drm_gem_object_lookup(file, handle); 173 if (!gem_obj) 174 return NULL; 175 176 return gem_to_pvr_gem(gem_obj); 177} 178 179/** 180 * pvr_gem_object_vmap() - Map a PowerVR GEM object into CPU virtual address 181 * space. 182 * @pvr_obj: Target PowerVR GEM object. 183 * 184 * Once the caller is finished with the CPU mapping, they must call 185 * pvr_gem_object_vunmap() on @pvr_obj. 186 * 187 * If @pvr_obj is CPU-cached, dma_sync_sgtable_for_cpu() is called to make 188 * sure the CPU mapping is consistent. 189 * 190 * Return: 191 * * A pointer to the CPU mapping on success, 192 * * -%ENOMEM if the mapping fails, or 193 * * Any error encountered while attempting to acquire a reference to the 194 * backing pages for @pvr_obj. 195 */ 196void * 197pvr_gem_object_vmap(struct pvr_gem_object *pvr_obj) 198{ 199 struct drm_gem_shmem_object *shmem_obj = shmem_gem_from_pvr_gem(pvr_obj); 200 struct drm_gem_object *obj = gem_from_pvr_gem(pvr_obj); 201 struct iosys_map map; 202 int err; 203 204 dma_resv_lock(obj->resv, NULL); 205 206 err = drm_gem_shmem_vmap(shmem_obj, &map); 207 if (err) 208 goto err_unlock; 209 210 if (pvr_obj->flags & PVR_BO_CPU_CACHED) { 211 struct device *dev = shmem_obj->base.dev->dev; 212 213 /* If shmem_obj->sgt is NULL, that means the buffer hasn't been mapped 214 * in GPU space yet. 215 */ 216 if (shmem_obj->sgt) 217 dma_sync_sgtable_for_cpu(dev, shmem_obj->sgt, DMA_BIDIRECTIONAL); 218 } 219 220 dma_resv_unlock(obj->resv); 221 222 return map.vaddr; 223 224err_unlock: 225 dma_resv_unlock(obj->resv); 226 227 return ERR_PTR(err); 228} 229 230/** 231 * pvr_gem_object_vunmap() - Unmap a PowerVR memory object from CPU virtual 232 * address space. 233 * @pvr_obj: Target PowerVR GEM object. 234 * 235 * If @pvr_obj is CPU-cached, dma_sync_sgtable_for_device() is called to make 236 * sure the GPU mapping is consistent. 237 */ 238void 239pvr_gem_object_vunmap(struct pvr_gem_object *pvr_obj) 240{ 241 struct drm_gem_shmem_object *shmem_obj = shmem_gem_from_pvr_gem(pvr_obj); 242 struct iosys_map map = IOSYS_MAP_INIT_VADDR(shmem_obj->vaddr); 243 struct drm_gem_object *obj = gem_from_pvr_gem(pvr_obj); 244 245 if (WARN_ON(!map.vaddr)) 246 return; 247 248 dma_resv_lock(obj->resv, NULL); 249 250 if (pvr_obj->flags & PVR_BO_CPU_CACHED) { 251 struct device *dev = shmem_obj->base.dev->dev; 252 253 /* If shmem_obj->sgt is NULL, that means the buffer hasn't been mapped 254 * in GPU space yet. 255 */ 256 if (shmem_obj->sgt) 257 dma_sync_sgtable_for_device(dev, shmem_obj->sgt, DMA_BIDIRECTIONAL); 258 } 259 260 drm_gem_shmem_vunmap(shmem_obj, &map); 261 262 dma_resv_unlock(obj->resv); 263} 264 265/** 266 * pvr_gem_object_zero() - Zeroes the physical memory behind an object. 267 * @pvr_obj: Target PowerVR GEM object. 268 * 269 * Return: 270 * * 0 on success, or 271 * * Any error encountered while attempting to map @pvr_obj to the CPU (see 272 * pvr_gem_object_vmap()). 273 */ 274static int 275pvr_gem_object_zero(struct pvr_gem_object *pvr_obj) 276{ 277 void *cpu_ptr; 278 279 cpu_ptr = pvr_gem_object_vmap(pvr_obj); 280 if (IS_ERR(cpu_ptr)) 281 return PTR_ERR(cpu_ptr); 282 283 memset(cpu_ptr, 0, pvr_gem_object_size(pvr_obj)); 284 285 /* Make sure the zero-ing is done before vumap-ing the object. */ 286 wmb(); 287 288 pvr_gem_object_vunmap(pvr_obj); 289 290 return 0; 291} 292 293/** 294 * pvr_gem_create_object() - Allocate and pre-initializes a pvr_gem_object 295 * @drm_dev: DRM device creating this object. 296 * @size: Size of the object to allocate in bytes. 297 * 298 * Return: 299 * * The new pre-initialized GEM object on success, 300 * * -ENOMEM if the allocation failed. 301 */ 302struct drm_gem_object *pvr_gem_create_object(struct drm_device *drm_dev, size_t size) 303{ 304 struct drm_gem_object *gem_obj; 305 struct pvr_gem_object *pvr_obj; 306 307 pvr_obj = kzalloc(sizeof(*pvr_obj), GFP_KERNEL); 308 if (!pvr_obj) 309 return ERR_PTR(-ENOMEM); 310 311 gem_obj = gem_from_pvr_gem(pvr_obj); 312 gem_obj->funcs = &pvr_gem_object_funcs; 313 314 return gem_obj; 315} 316 317/** 318 * pvr_gem_object_create() - Creates a PowerVR-specific buffer object. 319 * @pvr_dev: Target PowerVR device. 320 * @size: Size of the object to allocate in bytes. Must be greater than zero. 321 * Any value which is not an exact multiple of the system page size will be 322 * rounded up to satisfy this condition. 323 * @flags: Options which affect both this operation and future mapping 324 * operations performed on the returned object. Must be a combination of 325 * DRM_PVR_BO_* and/or PVR_BO_* flags. 326 * 327 * The created object may be larger than @size, but can never be smaller. To 328 * get the exact size, call pvr_gem_object_size() on the returned pointer. 329 * 330 * Return: 331 * * The newly-minted PowerVR-specific buffer object on success, 332 * * -%EINVAL if @size is zero or @flags is not valid, 333 * * -%ENOMEM if sufficient physical memory cannot be allocated, or 334 * * Any other error returned by drm_gem_create_mmap_offset(). 335 */ 336struct pvr_gem_object * 337pvr_gem_object_create(struct pvr_device *pvr_dev, size_t size, u64 flags) 338{ 339 struct drm_gem_shmem_object *shmem_obj; 340 struct pvr_gem_object *pvr_obj; 341 struct sg_table *sgt; 342 int err; 343 344 /* Verify @size and @flags before continuing. */ 345 if (size == 0 || !pvr_gem_object_flags_validate(flags)) 346 return ERR_PTR(-EINVAL); 347 348 shmem_obj = drm_gem_shmem_create(from_pvr_device(pvr_dev), size); 349 if (IS_ERR(shmem_obj)) 350 return ERR_CAST(shmem_obj); 351 352 shmem_obj->pages_mark_dirty_on_put = true; 353 shmem_obj->map_wc = !(flags & PVR_BO_CPU_CACHED); 354 pvr_obj = shmem_gem_to_pvr_gem(shmem_obj); 355 pvr_obj->flags = flags; 356 357 sgt = drm_gem_shmem_get_pages_sgt(shmem_obj); 358 if (IS_ERR(sgt)) { 359 err = PTR_ERR(sgt); 360 goto err_shmem_object_free; 361 } 362 363 dma_sync_sgtable_for_device(shmem_obj->base.dev->dev, sgt, 364 DMA_BIDIRECTIONAL); 365 366 /* 367 * Do this last because pvr_gem_object_zero() requires a fully 368 * configured instance of struct pvr_gem_object. 369 */ 370 pvr_gem_object_zero(pvr_obj); 371 372 return pvr_obj; 373 374err_shmem_object_free: 375 drm_gem_shmem_free(shmem_obj); 376 377 return ERR_PTR(err); 378} 379 380/** 381 * pvr_gem_get_dma_addr() - Get DMA address for given offset in object 382 * @pvr_obj: Pointer to object to lookup address in. 383 * @offset: Offset within object to lookup address at. 384 * @dma_addr_out: Pointer to location to store DMA address. 385 * 386 * Returns: 387 * * 0 on success, or 388 * * -%EINVAL if object is not currently backed, or if @offset is out of valid 389 * range for this object. 390 */ 391int 392pvr_gem_get_dma_addr(struct pvr_gem_object *pvr_obj, u32 offset, 393 dma_addr_t *dma_addr_out) 394{ 395 struct drm_gem_shmem_object *shmem_obj = shmem_gem_from_pvr_gem(pvr_obj); 396 u32 accumulated_offset = 0; 397 struct scatterlist *sgl; 398 unsigned int sgt_idx; 399 400 WARN_ON(!shmem_obj->sgt); 401 for_each_sgtable_dma_sg(shmem_obj->sgt, sgl, sgt_idx) { 402 u32 new_offset = accumulated_offset + sg_dma_len(sgl); 403 404 if (offset >= accumulated_offset && offset < new_offset) { 405 *dma_addr_out = sg_dma_address(sgl) + 406 (offset - accumulated_offset); 407 return 0; 408 } 409 410 accumulated_offset = new_offset; 411 } 412 413 return -EINVAL; 414} 415