1247835Skib/************************************************************************** 2247835Skib * 3247835Skib * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA 4247835Skib * All Rights Reserved. 5247835Skib * 6247835Skib * Permission is hereby granted, free of charge, to any person obtaining a 7247835Skib * copy of this software and associated documentation files (the 8247835Skib * "Software"), to deal in the Software without restriction, including 9247835Skib * without limitation the rights to use, copy, modify, merge, publish, 10247835Skib * distribute, sub license, and/or sell copies of the Software, and to 11247835Skib * permit persons to whom the Software is furnished to do so, subject to 12247835Skib * the following conditions: 13247835Skib * 14247835Skib * The above copyright notice and this permission notice (including the 15247835Skib * next paragraph) shall be included in all copies or substantial portions 16247835Skib * of the Software. 17247835Skib * 18247835Skib * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19247835Skib * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20247835Skib * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 21247835Skib * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 22247835Skib * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 23247835Skib * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 24247835Skib * USE OR OTHER DEALINGS IN THE SOFTWARE. 25247835Skib * 26247835Skib **************************************************************************/ 27247835Skib 28247835Skib#include <sys/cdefs.h> 29247835Skib__FBSDID("$FreeBSD$"); 30247835Skib 31247835Skib#include <dev/drm2/drmP.h> 32247835Skib#include <dev/drm2/ttm/ttm_execbuf_util.h> 33247835Skib#include <dev/drm2/ttm/ttm_bo_driver.h> 34247835Skib#include <dev/drm2/ttm/ttm_placement.h> 35247835Skib 36247835Skibstatic void ttm_eu_backoff_reservation_locked(struct list_head *list) 37247835Skib{ 38247835Skib struct ttm_validate_buffer *entry; 39247835Skib 40247835Skib list_for_each_entry(entry, list, head) { 41247835Skib struct ttm_buffer_object *bo = entry->bo; 42247835Skib if (!entry->reserved) 43247835Skib continue; 44247835Skib 45247835Skib if (entry->removed) { 46247835Skib ttm_bo_add_to_lru(bo); 47247835Skib entry->removed = false; 48247835Skib 49247835Skib } 50247835Skib entry->reserved = false; 51247835Skib atomic_set(&bo->reserved, 0); 52247835Skib wakeup(bo); 53247835Skib } 54247835Skib} 55247835Skib 56247835Skibstatic void ttm_eu_del_from_lru_locked(struct list_head *list) 57247835Skib{ 58247835Skib struct ttm_validate_buffer *entry; 59247835Skib 60247835Skib list_for_each_entry(entry, list, head) { 61247835Skib struct ttm_buffer_object *bo = entry->bo; 62247835Skib if (!entry->reserved) 63247835Skib continue; 64247835Skib 65247835Skib if (!entry->removed) { 66247835Skib entry->put_count = ttm_bo_del_from_lru(bo); 67247835Skib entry->removed = true; 68247835Skib } 69247835Skib } 70247835Skib} 71247835Skib 72247835Skibstatic void ttm_eu_list_ref_sub(struct list_head *list) 73247835Skib{ 74247835Skib struct ttm_validate_buffer *entry; 75247835Skib 76247835Skib list_for_each_entry(entry, list, head) { 77247835Skib struct ttm_buffer_object *bo = entry->bo; 78247835Skib 79247835Skib if (entry->put_count) { 80247835Skib ttm_bo_list_ref_sub(bo, entry->put_count, true); 81247835Skib entry->put_count = 0; 82247835Skib } 83247835Skib } 84247835Skib} 85247835Skib 86247835Skibvoid ttm_eu_backoff_reservation(struct list_head *list) 87247835Skib{ 88247835Skib struct ttm_validate_buffer *entry; 89247835Skib struct ttm_bo_global *glob; 90247835Skib 91247835Skib if (list_empty(list)) 92247835Skib return; 93247835Skib 94247835Skib entry = list_first_entry(list, struct ttm_validate_buffer, head); 95247835Skib glob = entry->bo->glob; 96247835Skib mtx_lock(&glob->lru_lock); 97247835Skib ttm_eu_backoff_reservation_locked(list); 98247835Skib mtx_unlock(&glob->lru_lock); 99247835Skib} 100247835Skib 101247835Skib/* 102247835Skib * Reserve buffers for validation. 103247835Skib * 104247835Skib * If a buffer in the list is marked for CPU access, we back off and 105247835Skib * wait for that buffer to become free for GPU access. 106247835Skib * 107247835Skib * If a buffer is reserved for another validation, the validator with 108247835Skib * the highest validation sequence backs off and waits for that buffer 109247835Skib * to become unreserved. This prevents deadlocks when validating multiple 110247835Skib * buffers in different orders. 111247835Skib */ 112247835Skib 113247835Skibint ttm_eu_reserve_buffers(struct list_head *list) 114247835Skib{ 115247835Skib struct ttm_bo_global *glob; 116247835Skib struct ttm_validate_buffer *entry; 117247835Skib int ret; 118247835Skib uint32_t val_seq; 119247835Skib 120247835Skib if (list_empty(list)) 121247835Skib return 0; 122247835Skib 123247835Skib list_for_each_entry(entry, list, head) { 124247835Skib entry->reserved = false; 125247835Skib entry->put_count = 0; 126247835Skib entry->removed = false; 127247835Skib } 128247835Skib 129247835Skib entry = list_first_entry(list, struct ttm_validate_buffer, head); 130247835Skib glob = entry->bo->glob; 131247835Skib 132247835Skib mtx_lock(&glob->lru_lock); 133247835Skib val_seq = entry->bo->bdev->val_seq++; 134247835Skib 135254864Sdumbbellretry_locked: 136247835Skib list_for_each_entry(entry, list, head) { 137247835Skib struct ttm_buffer_object *bo = entry->bo; 138247835Skib 139254864Sdumbbell /* already slowpath reserved? */ 140254864Sdumbbell if (entry->reserved) 141254864Sdumbbell continue; 142254864Sdumbbell 143254861Sdumbbell ret = ttm_bo_reserve_nolru(bo, true, true, true, val_seq); 144247835Skib switch (ret) { 145247835Skib case 0: 146247835Skib break; 147247835Skib case -EBUSY: 148254862Sdumbbell ttm_eu_del_from_lru_locked(list); 149254862Sdumbbell ret = ttm_bo_reserve_nolru(bo, true, false, 150254862Sdumbbell true, val_seq); 151254862Sdumbbell if (!ret) 152254862Sdumbbell break; 153254862Sdumbbell 154254862Sdumbbell if (unlikely(ret != -EAGAIN)) 155254862Sdumbbell goto err; 156254862Sdumbbell 157254862Sdumbbell /* fallthrough */ 158247835Skib case -EAGAIN: 159247835Skib ttm_eu_backoff_reservation_locked(list); 160254864Sdumbbell 161254864Sdumbbell /* 162254864Sdumbbell * temporarily increase sequence number every retry, 163254864Sdumbbell * to prevent us from seeing our old reservation 164254864Sdumbbell * sequence when someone else reserved the buffer, 165254864Sdumbbell * but hasn't updated the seq_valid/seqno members yet. 166254864Sdumbbell */ 167254864Sdumbbell val_seq = entry->bo->bdev->val_seq++; 168254864Sdumbbell 169247835Skib ttm_eu_list_ref_sub(list); 170254864Sdumbbell ret = ttm_bo_reserve_slowpath_nolru(bo, true, val_seq); 171247835Skib if (unlikely(ret != 0)) { 172247835Skib mtx_unlock(&glob->lru_lock); 173247835Skib return ret; 174247835Skib } 175254864Sdumbbell entry->reserved = true; 176254864Sdumbbell if (unlikely(atomic_read(&bo->cpu_writers) > 0)) { 177254864Sdumbbell ret = -EBUSY; 178254864Sdumbbell goto err; 179254864Sdumbbell } 180247835Skib goto retry_locked; 181247835Skib default: 182254862Sdumbbell goto err; 183247835Skib } 184247835Skib 185247835Skib entry->reserved = true; 186247835Skib if (unlikely(atomic_read(&bo->cpu_writers) > 0)) { 187254862Sdumbbell ret = -EBUSY; 188254862Sdumbbell goto err; 189247835Skib } 190247835Skib } 191247835Skib 192247835Skib ttm_eu_del_from_lru_locked(list); 193247835Skib mtx_unlock(&glob->lru_lock); 194247835Skib ttm_eu_list_ref_sub(list); 195247835Skib 196247835Skib return 0; 197254862Sdumbbell 198254862Sdumbbellerr: 199254862Sdumbbell ttm_eu_backoff_reservation_locked(list); 200254862Sdumbbell mtx_unlock(&glob->lru_lock); 201254862Sdumbbell ttm_eu_list_ref_sub(list); 202254862Sdumbbell return ret; 203247835Skib} 204247835Skib 205247835Skibvoid ttm_eu_fence_buffer_objects(struct list_head *list, void *sync_obj) 206247835Skib{ 207247835Skib struct ttm_validate_buffer *entry; 208247835Skib struct ttm_buffer_object *bo; 209247835Skib struct ttm_bo_global *glob; 210247835Skib struct ttm_bo_device *bdev; 211247835Skib struct ttm_bo_driver *driver; 212247835Skib 213247835Skib if (list_empty(list)) 214247835Skib return; 215247835Skib 216247835Skib bo = list_first_entry(list, struct ttm_validate_buffer, head)->bo; 217247835Skib bdev = bo->bdev; 218247835Skib driver = bdev->driver; 219247835Skib glob = bo->glob; 220247835Skib 221247835Skib mtx_lock(&glob->lru_lock); 222247835Skib mtx_lock(&bdev->fence_lock); 223247835Skib 224247835Skib list_for_each_entry(entry, list, head) { 225247835Skib bo = entry->bo; 226247835Skib entry->old_sync_obj = bo->sync_obj; 227247835Skib bo->sync_obj = driver->sync_obj_ref(sync_obj); 228247835Skib ttm_bo_unreserve_locked(bo); 229247835Skib entry->reserved = false; 230247835Skib } 231247835Skib mtx_unlock(&bdev->fence_lock); 232247835Skib mtx_unlock(&glob->lru_lock); 233247835Skib 234247835Skib list_for_each_entry(entry, list, head) { 235247835Skib if (entry->old_sync_obj) 236247835Skib driver->sync_obj_unref(&entry->old_sync_obj); 237247835Skib } 238247835Skib} 239