1235783Skib/*- 2235783Skib * Copyright (c) 2011 The FreeBSD Foundation 3235783Skib * All rights reserved. 4235783Skib * 5235783Skib * This software was developed by Konstantin Belousov under sponsorship from 6235783Skib * the FreeBSD Foundation. 7235783Skib * 8235783Skib * Redistribution and use in source and binary forms, with or without 9235783Skib * modification, are permitted provided that the following conditions 10235783Skib * are met: 11235783Skib * 1. Redistributions of source code must retain the above copyright 12235783Skib * notice, this list of conditions and the following disclaimer. 13235783Skib * 2. Redistributions in binary form must reproduce the above copyright 14235783Skib * notice, this list of conditions and the following disclaimer in the 15235783Skib * documentation and/or other materials provided with the distribution. 16235783Skib * 17235783Skib * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18235783Skib * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19235783Skib * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20235783Skib * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21235783Skib * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22235783Skib * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23235783Skib * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24235783Skib * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25235783Skib * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26235783Skib * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27235783Skib * SUCH DAMAGE. 28235783Skib */ 29235783Skib 30235783Skib#include <sys/cdefs.h> 31235783Skib__FBSDID("$FreeBSD$"); 32235783Skib 33235783Skib#include "opt_vm.h" 34235783Skib 35235783Skib#include <sys/param.h> 36235783Skib#include <sys/systm.h> 37235783Skib#include <sys/limits.h> 38235783Skib#include <sys/lock.h> 39235783Skib#include <sys/mutex.h> 40235783Skib 41235783Skib#include <vm/vm.h> 42235783Skib#include <vm/vm_page.h> 43235783Skib 44235783Skib#include <dev/drm2/drmP.h> 45235783Skib#include <dev/drm2/drm.h> 46235783Skib#include <dev/drm2/drm_sarea.h> 47235783Skib 48235783Skib/* 49235783Skib * We make up offsets for buffer objects so we can recognize them at 50235783Skib * mmap time. 51235783Skib */ 52235783Skib 53235783Skib/* pgoff in mmap is an unsigned long, so we need to make sure that 54235783Skib * the faked up offset will fit 55235783Skib */ 56235783Skib 57282199Sdumbbell#if BITS_PER_LONG == 64 58235783Skib#define DRM_FILE_PAGE_OFFSET_START ((0xFFFFFFFFUL >> PAGE_SHIFT) + 1) 59235783Skib#define DRM_FILE_PAGE_OFFSET_SIZE ((0xFFFFFFFFUL >> PAGE_SHIFT) * 16) 60235783Skib#else 61235783Skib#define DRM_FILE_PAGE_OFFSET_START ((0xFFFFFFFUL >> PAGE_SHIFT) + 1) 62235783Skib#define DRM_FILE_PAGE_OFFSET_SIZE ((0xFFFFFFFUL >> PAGE_SHIFT) * 16) 63235783Skib#endif 64235783Skib 65282199Sdumbbell/** 66282199Sdumbbell * Initialize the GEM device fields 67282199Sdumbbell */ 68282199Sdumbbell 69235783Skibint 70235783Skibdrm_gem_init(struct drm_device *dev) 71235783Skib{ 72235783Skib struct drm_gem_mm *mm; 73235783Skib 74235783Skib drm_gem_names_init(&dev->object_names); 75282199Sdumbbell 76282199Sdumbbell mm = malloc(sizeof(*mm), DRM_MEM_DRIVER, M_NOWAIT); 77282199Sdumbbell if (!mm) { 78282199Sdumbbell DRM_ERROR("out of memory\n"); 79282199Sdumbbell return -ENOMEM; 80282199Sdumbbell } 81282199Sdumbbell 82235783Skib dev->mm_private = mm; 83282199Sdumbbell 84282199Sdumbbell if (drm_ht_create(&mm->offset_hash, 19)) { 85235783Skib free(mm, DRM_MEM_DRIVER); 86282199Sdumbbell return -ENOMEM; 87235783Skib } 88282199Sdumbbell 89235783Skib mm->idxunr = new_unrhdr(0, DRM_GEM_MAX_IDX, NULL); 90282199Sdumbbell 91282199Sdumbbell return 0; 92235783Skib} 93235783Skib 94235783Skibvoid 95235783Skibdrm_gem_destroy(struct drm_device *dev) 96235783Skib{ 97282199Sdumbbell struct drm_gem_mm *mm = dev->mm_private; 98235783Skib 99235783Skib dev->mm_private = NULL; 100235783Skib drm_ht_remove(&mm->offset_hash); 101235783Skib delete_unrhdr(mm->idxunr); 102235783Skib free(mm, DRM_MEM_DRIVER); 103235783Skib drm_gem_names_fini(&dev->object_names); 104235783Skib} 105235783Skib 106282199Sdumbbellint drm_gem_object_init(struct drm_device *dev, 107282199Sdumbbell struct drm_gem_object *obj, size_t size) 108235783Skib{ 109235783Skib KASSERT((size & (PAGE_SIZE - 1)) == 0, 110235783Skib ("Bad size %ju", (uintmax_t)size)); 111235783Skib 112235783Skib obj->dev = dev; 113235783Skib obj->vm_obj = vm_pager_allocate(OBJT_DEFAULT, NULL, size, 114235783Skib VM_PROT_READ | VM_PROT_WRITE, 0, curthread->td_ucred); 115235783Skib 116235783Skib obj->refcount = 1; 117235783Skib obj->handle_count = 0; 118235783Skib obj->size = size; 119235783Skib 120282199Sdumbbell return 0; 121235783Skib} 122282199SdumbbellEXPORT_SYMBOL(drm_gem_object_init); 123235783Skib 124282199Sdumbbell/** 125282199Sdumbbell * Initialize an already allocated GEM object of the specified size with 126282199Sdumbbell * no GEM provided backing store. Instead the caller is responsible for 127282199Sdumbbell * backing the object and handling it. 128282199Sdumbbell */ 129282199Sdumbbellint drm_gem_private_object_init(struct drm_device *dev, 130282199Sdumbbell struct drm_gem_object *obj, size_t size) 131235783Skib{ 132235783Skib MPASS((size & (PAGE_SIZE - 1)) == 0); 133235783Skib 134235783Skib obj->dev = dev; 135235783Skib obj->vm_obj = NULL; 136235783Skib 137235783Skib obj->refcount = 1; 138254880Sdumbbell atomic_store_rel_int(&obj->handle_count, 0); 139235783Skib obj->size = size; 140235783Skib 141282199Sdumbbell return 0; 142235783Skib} 143282199SdumbbellEXPORT_SYMBOL(drm_gem_private_object_init); 144235783Skib 145235783Skibstruct drm_gem_object * 146235783Skibdrm_gem_object_alloc(struct drm_device *dev, size_t size) 147235783Skib{ 148235783Skib struct drm_gem_object *obj; 149235783Skib 150282199Sdumbbell obj = malloc(sizeof(*obj), DRM_MEM_DRIVER, M_NOWAIT | M_ZERO); 151282199Sdumbbell if (!obj) 152282199Sdumbbell goto free; 153282199Sdumbbell 154235783Skib if (drm_gem_object_init(dev, obj, size) != 0) 155235783Skib goto free; 156235783Skib 157235783Skib if (dev->driver->gem_init_object != NULL && 158282199Sdumbbell dev->driver->gem_init_object(obj) != 0) { 159235783Skib goto dealloc; 160282199Sdumbbell } 161282199Sdumbbell return obj; 162235783Skibdealloc: 163235783Skib vm_object_deallocate(obj->vm_obj); 164235783Skibfree: 165235783Skib free(obj, DRM_MEM_DRIVER); 166282199Sdumbbell return NULL; 167235783Skib} 168282199SdumbbellEXPORT_SYMBOL(drm_gem_object_alloc); 169235783Skib 170282199Sdumbbell#if defined(FREEBSD_NOTYET) 171282199Sdumbbellstatic void 172282199Sdumbbelldrm_gem_remove_prime_handles(struct drm_gem_object *obj, struct drm_file *filp) 173235783Skib{ 174282199Sdumbbell if (obj->import_attach) { 175282199Sdumbbell drm_prime_remove_buf_handle(&filp->prime, 176282199Sdumbbell obj->import_attach->dmabuf); 177282199Sdumbbell } 178282199Sdumbbell if (obj->export_dma_buf) { 179282199Sdumbbell drm_prime_remove_buf_handle(&filp->prime, 180282199Sdumbbell obj->export_dma_buf); 181282199Sdumbbell } 182282199Sdumbbell} 183282199Sdumbbell#endif 184282199Sdumbbell 185282199Sdumbbell/** 186282199Sdumbbell * Removes the mapping from handle to filp for this object. 187282199Sdumbbell */ 188282199Sdumbbellint 189282199Sdumbbelldrm_gem_handle_delete(struct drm_file *filp, u32 handle) 190282199Sdumbbell{ 191235783Skib struct drm_device *dev; 192282199Sdumbbell struct drm_gem_object *obj; 193235783Skib 194282199Sdumbbell obj = drm_gem_names_remove(&filp->object_names, handle); 195282199Sdumbbell if (obj == NULL) { 196282199Sdumbbell return -EINVAL; 197282199Sdumbbell } 198235783Skib dev = obj->dev; 199235783Skib 200282199Sdumbbell#if defined(FREEBSD_NOTYET) 201282199Sdumbbell drm_gem_remove_prime_handles(obj, filp); 202282199Sdumbbell#endif 203235783Skib 204282199Sdumbbell if (dev->driver->gem_close_object) 205282199Sdumbbell dev->driver->gem_close_object(obj, filp); 206282199Sdumbbell drm_gem_object_handle_unreference_unlocked(obj); 207282199Sdumbbell 208282199Sdumbbell return 0; 209235783Skib} 210282199SdumbbellEXPORT_SYMBOL(drm_gem_handle_delete); 211235783Skib 212282199Sdumbbell/** 213282199Sdumbbell * Create a handle for this object. This adds a handle reference 214282199Sdumbbell * to the object, which includes a regular reference count. Callers 215282199Sdumbbell * will likely want to dereference the object afterwards. 216282199Sdumbbell */ 217282199Sdumbbellint 218282199Sdumbbelldrm_gem_handle_create(struct drm_file *file_priv, 219282199Sdumbbell struct drm_gem_object *obj, 220282199Sdumbbell u32 *handlep) 221235783Skib{ 222282199Sdumbbell struct drm_device *dev = obj->dev; 223282199Sdumbbell int ret; 224235783Skib 225282199Sdumbbell *handlep = 0; 226282199Sdumbbell ret = drm_gem_name_create(&file_priv->object_names, obj, handlep); 227282199Sdumbbell if (ret != 0) 228282199Sdumbbell return ret; 229282199Sdumbbell 230282199Sdumbbell drm_gem_object_handle_reference(obj); 231282199Sdumbbell 232282199Sdumbbell if (dev->driver->gem_open_object) { 233282199Sdumbbell ret = dev->driver->gem_open_object(obj, file_priv); 234282199Sdumbbell if (ret) { 235282199Sdumbbell drm_gem_handle_delete(file_priv, *handlep); 236282199Sdumbbell return ret; 237282199Sdumbbell } 238282199Sdumbbell } 239282199Sdumbbell 240282199Sdumbbell return 0; 241235783Skib} 242282199SdumbbellEXPORT_SYMBOL(drm_gem_handle_create); 243235783Skib 244235783Skibvoid 245282199Sdumbbelldrm_gem_free_mmap_offset(struct drm_gem_object *obj) 246235783Skib{ 247282199Sdumbbell struct drm_device *dev = obj->dev; 248282199Sdumbbell struct drm_gem_mm *mm = dev->mm_private; 249282199Sdumbbell struct drm_hash_item *list = &obj->map_list; 250235783Skib 251282199Sdumbbell if (!obj->on_map) 252235783Skib return; 253282199Sdumbbell 254282199Sdumbbell drm_ht_remove_item(&mm->offset_hash, list); 255282199Sdumbbell free_unr(mm->idxunr, list->key); 256282199Sdumbbell obj->on_map = false; 257235783Skib} 258282199SdumbbellEXPORT_SYMBOL(drm_gem_free_mmap_offset); 259235783Skib 260282199Sdumbbellint 261282199Sdumbbelldrm_gem_create_mmap_offset(struct drm_gem_object *obj) 262235783Skib{ 263282199Sdumbbell struct drm_device *dev = obj->dev; 264282199Sdumbbell struct drm_gem_mm *mm = dev->mm_private; 265282199Sdumbbell int ret; 266235783Skib 267282199Sdumbbell if (obj->on_map) 268282199Sdumbbell return 0; 269235783Skib 270282199Sdumbbell obj->map_list.key = alloc_unr(mm->idxunr); 271282199Sdumbbell ret = drm_ht_insert_item(&mm->offset_hash, &obj->map_list); 272282199Sdumbbell if (ret) { 273282199Sdumbbell DRM_ERROR("failed to add to map hash\n"); 274282199Sdumbbell free_unr(mm->idxunr, obj->map_list.key); 275282199Sdumbbell return ret; 276235783Skib } 277282199Sdumbbell obj->on_map = true; 278235783Skib 279282199Sdumbbell return 0; 280235783Skib} 281282199SdumbbellEXPORT_SYMBOL(drm_gem_create_mmap_offset); 282235783Skib 283282199Sdumbbell/** Returns a reference to the object named by the handle. */ 284282199Sdumbbellstruct drm_gem_object * 285282199Sdumbbelldrm_gem_object_lookup(struct drm_device *dev, struct drm_file *filp, 286282199Sdumbbell u32 handle) 287235783Skib{ 288282199Sdumbbell struct drm_gem_object *obj; 289235783Skib 290282199Sdumbbell obj = drm_gem_name_ref(&filp->object_names, handle, 291282199Sdumbbell (void (*)(void *))drm_gem_object_reference); 292235783Skib 293282199Sdumbbell return obj; 294235783Skib} 295282199SdumbbellEXPORT_SYMBOL(drm_gem_object_lookup); 296235783Skib 297235783Skibint 298282199Sdumbbelldrm_gem_close_ioctl(struct drm_device *dev, void *data, 299282199Sdumbbell struct drm_file *file_priv) 300235783Skib{ 301282199Sdumbbell struct drm_gem_close *args = data; 302254836Sdumbbell int ret; 303235783Skib 304282199Sdumbbell if (!(dev->driver->driver_features & DRIVER_GEM)) 305282199Sdumbbell return -ENODEV; 306254836Sdumbbell 307282199Sdumbbell ret = drm_gem_handle_delete(file_priv, args->handle); 308254836Sdumbbell 309282199Sdumbbell return ret; 310235783Skib} 311235783Skib 312235783Skibint 313282199Sdumbbelldrm_gem_flink_ioctl(struct drm_device *dev, void *data, 314282199Sdumbbell struct drm_file *file_priv) 315235783Skib{ 316282199Sdumbbell struct drm_gem_flink *args = data; 317235783Skib struct drm_gem_object *obj; 318282199Sdumbbell int ret; 319235783Skib 320282199Sdumbbell if (!(dev->driver->driver_features & DRIVER_GEM)) 321282199Sdumbbell return -ENODEV; 322282199Sdumbbell 323282199Sdumbbell obj = drm_gem_object_lookup(dev, file_priv, args->handle); 324235783Skib if (obj == NULL) 325282199Sdumbbell return -ENOENT; 326254836Sdumbbell 327282199Sdumbbell ret = drm_gem_name_create(&dev->object_names, obj, &obj->name); 328282199Sdumbbell if (ret != 0) { 329282199Sdumbbell if (ret == -EALREADY) 330282199Sdumbbell ret = 0; 331282199Sdumbbell drm_gem_object_unreference_unlocked(obj); 332282199Sdumbbell } 333282199Sdumbbell if (ret == 0) 334282199Sdumbbell args->name = obj->name; 335282199Sdumbbell return ret; 336235783Skib} 337235783Skib 338235783Skibint 339235783Skibdrm_gem_open_ioctl(struct drm_device *dev, void *data, 340282199Sdumbbell struct drm_file *file_priv) 341235783Skib{ 342282199Sdumbbell struct drm_gem_open *args = data; 343235783Skib struct drm_gem_object *obj; 344235783Skib int ret; 345282199Sdumbbell u32 handle; 346235783Skib 347282199Sdumbbell if (!(dev->driver->driver_features & DRIVER_GEM)) 348282199Sdumbbell return -ENODEV; 349235783Skib 350235783Skib obj = drm_gem_name_ref(&dev->object_names, args->name, 351235783Skib (void (*)(void *))drm_gem_object_reference); 352282199Sdumbbell if (!obj) 353282199Sdumbbell return -ENOENT; 354282199Sdumbbell 355235783Skib ret = drm_gem_handle_create(file_priv, obj, &handle); 356235783Skib drm_gem_object_unreference_unlocked(obj); 357282199Sdumbbell if (ret) 358282199Sdumbbell return ret; 359282199Sdumbbell 360235783Skib args->handle = handle; 361235783Skib args->size = obj->size; 362235783Skib 363282199Sdumbbell return 0; 364235783Skib} 365235783Skib 366235783Skibvoid 367282199Sdumbbelldrm_gem_open(struct drm_device *dev, struct drm_file *file_private) 368235783Skib{ 369235783Skib 370282199Sdumbbell drm_gem_names_init(&file_private->object_names); 371235783Skib} 372235783Skib 373235783Skibstatic int 374282199Sdumbbelldrm_gem_object_release_handle(uint32_t name, void *ptr, void *data) 375235783Skib{ 376282199Sdumbbell struct drm_file *file_priv = data; 377282199Sdumbbell struct drm_gem_object *obj = ptr; 378282199Sdumbbell struct drm_device *dev = obj->dev; 379235783Skib 380282199Sdumbbell#if defined(FREEBSD_NOTYET) 381282199Sdumbbell drm_gem_remove_prime_handles(obj, file_priv); 382282199Sdumbbell#endif 383254837Sdumbbell 384254837Sdumbbell if (dev->driver->gem_close_object) 385254837Sdumbbell dev->driver->gem_close_object(obj, file_priv); 386254837Sdumbbell 387282199Sdumbbell drm_gem_object_handle_unreference_unlocked(obj); 388282199Sdumbbell 389282199Sdumbbell return 0; 390235783Skib} 391235783Skib 392235783Skibvoid 393282199Sdumbbelldrm_gem_release(struct drm_device *dev, struct drm_file *file_private) 394235783Skib{ 395282199Sdumbbell drm_gem_names_foreach(&file_private->object_names, 396282199Sdumbbell drm_gem_object_release_handle, file_private); 397235783Skib 398282199Sdumbbell drm_gem_names_fini(&file_private->object_names); 399235783Skib} 400235783Skib 401282199Sdumbbellvoid 402282199Sdumbbelldrm_gem_object_release(struct drm_gem_object *obj) 403235783Skib{ 404235783Skib 405282199Sdumbbell /* 406282199Sdumbbell * obj->vm_obj can be NULL for private gem objects. 407282199Sdumbbell */ 408282199Sdumbbell vm_object_deallocate(obj->vm_obj); 409235783Skib} 410282199SdumbbellEXPORT_SYMBOL(drm_gem_object_release); 411235783Skib 412282199Sdumbbellvoid 413282199Sdumbbelldrm_gem_object_free(struct drm_gem_object *obj) 414235783Skib{ 415282199Sdumbbell struct drm_device *dev = obj->dev; 416235783Skib 417282199Sdumbbell DRM_LOCK_ASSERT(dev); 418282199Sdumbbell if (dev->driver->gem_free_object != NULL) 419282199Sdumbbell dev->driver->gem_free_object(obj); 420235783Skib} 421282199SdumbbellEXPORT_SYMBOL(drm_gem_object_free); 422235783Skib 423282199Sdumbbellvoid drm_gem_object_handle_free(struct drm_gem_object *obj) 424235783Skib{ 425282199Sdumbbell struct drm_device *dev = obj->dev; 426282199Sdumbbell struct drm_gem_object *obj1; 427235783Skib 428282199Sdumbbell if (obj->name) { 429282199Sdumbbell obj1 = drm_gem_names_remove(&dev->object_names, obj->name); 430282199Sdumbbell obj->name = 0; 431282199Sdumbbell drm_gem_object_unreference(obj1); 432282199Sdumbbell } 433235783Skib} 434235783Skib 435235783Skibstatic struct drm_gem_object * 436235783Skibdrm_gem_object_from_offset(struct drm_device *dev, vm_ooffset_t offset) 437235783Skib{ 438235783Skib struct drm_gem_object *obj; 439235783Skib struct drm_gem_mm *mm; 440235783Skib struct drm_hash_item *map_list; 441235783Skib 442235783Skib if ((offset & DRM_GEM_MAPPING_MASK) != DRM_GEM_MAPPING_KEY) 443235783Skib return (NULL); 444235783Skib offset &= ~DRM_GEM_MAPPING_KEY; 445235783Skib mm = dev->mm_private; 446235783Skib if (drm_ht_find_item(&mm->offset_hash, DRM_GEM_MAPPING_IDX(offset), 447235783Skib &map_list) != 0) { 448235783Skib DRM_DEBUG("drm_gem_object_from_offset: offset 0x%jx obj not found\n", 449235783Skib (uintmax_t)offset); 450235783Skib return (NULL); 451235783Skib } 452240539Sed obj = __containerof(map_list, struct drm_gem_object, map_list); 453235783Skib return (obj); 454235783Skib} 455235783Skib 456235783Skibint 457247835Skibdrm_gem_mmap_single(struct drm_device *dev, vm_ooffset_t *offset, vm_size_t size, 458235783Skib struct vm_object **obj_res, int nprot) 459235783Skib{ 460235783Skib struct drm_gem_object *gem_obj; 461235783Skib struct vm_object *vm_obj; 462235783Skib 463235783Skib DRM_LOCK(dev); 464235783Skib gem_obj = drm_gem_object_from_offset(dev, *offset); 465235783Skib if (gem_obj == NULL) { 466235783Skib DRM_UNLOCK(dev); 467282199Sdumbbell return (-ENODEV); 468235783Skib } 469235783Skib drm_gem_object_reference(gem_obj); 470235783Skib DRM_UNLOCK(dev); 471235783Skib vm_obj = cdev_pager_allocate(gem_obj, OBJT_MGTDEVICE, 472235783Skib dev->driver->gem_pager_ops, size, nprot, 473235783Skib DRM_GEM_MAPPING_MAPOFF(*offset), curthread->td_ucred); 474235783Skib if (vm_obj == NULL) { 475235783Skib drm_gem_object_unreference_unlocked(gem_obj); 476282199Sdumbbell return (-EINVAL); 477235783Skib } 478235783Skib *offset = DRM_GEM_MAPPING_MAPOFF(*offset); 479235783Skib *obj_res = vm_obj; 480235783Skib return (0); 481235783Skib} 482235783Skib 483235783Skibvoid 484235783Skibdrm_gem_pager_dtr(void *handle) 485235783Skib{ 486235783Skib struct drm_gem_object *obj; 487235783Skib struct drm_device *dev; 488235783Skib 489235783Skib obj = handle; 490235783Skib dev = obj->dev; 491235783Skib 492235783Skib DRM_LOCK(dev); 493235783Skib drm_gem_free_mmap_offset(obj); 494235783Skib drm_gem_object_unreference(obj); 495235783Skib DRM_UNLOCK(dev); 496235783Skib} 497