1254885Sdumbbell/* 2254885Sdumbbell * Copyright 2008 Jerome Glisse. 3254885Sdumbbell * All Rights Reserved. 4254885Sdumbbell * 5254885Sdumbbell * Permission is hereby granted, free of charge, to any person obtaining a 6254885Sdumbbell * copy of this software and associated documentation files (the "Software"), 7254885Sdumbbell * to deal in the Software without restriction, including without limitation 8254885Sdumbbell * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9254885Sdumbbell * and/or sell copies of the Software, and to permit persons to whom the 10254885Sdumbbell * Software is furnished to do so, subject to the following conditions: 11254885Sdumbbell * 12254885Sdumbbell * The above copyright notice and this permission notice (including the next 13254885Sdumbbell * paragraph) shall be included in all copies or substantial portions of the 14254885Sdumbbell * Software. 15254885Sdumbbell * 16254885Sdumbbell * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17254885Sdumbbell * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18254885Sdumbbell * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19254885Sdumbbell * PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR 20254885Sdumbbell * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 21254885Sdumbbell * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 22254885Sdumbbell * DEALINGS IN THE SOFTWARE. 23254885Sdumbbell * 24254885Sdumbbell * Authors: 25254885Sdumbbell * Jerome Glisse <glisse@freedesktop.org> 26254885Sdumbbell */ 27254885Sdumbbell 28254885Sdumbbell#include <sys/cdefs.h> 29254885Sdumbbell__FBSDID("$FreeBSD$"); 30254885Sdumbbell 31254885Sdumbbell#include <dev/drm2/drmP.h> 32254885Sdumbbell#include <dev/drm2/radeon/radeon_drm.h> 33254885Sdumbbell#include "radeon_reg.h" 34254885Sdumbbell#include "radeon.h" 35254885Sdumbbell 36254885Sdumbbellvoid r100_cs_dump_packet(struct radeon_cs_parser *p, 37254885Sdumbbell struct radeon_cs_packet *pkt); 38254885Sdumbbell 39254885Sdumbbellstatic int radeon_cs_parser_relocs(struct radeon_cs_parser *p) 40254885Sdumbbell{ 41254885Sdumbbell struct drm_device *ddev = p->rdev->ddev; 42254885Sdumbbell struct radeon_cs_chunk *chunk; 43254885Sdumbbell unsigned i, j; 44254885Sdumbbell bool duplicate; 45254885Sdumbbell 46254885Sdumbbell if (p->chunk_relocs_idx == -1) { 47254885Sdumbbell return 0; 48254885Sdumbbell } 49254885Sdumbbell chunk = &p->chunks[p->chunk_relocs_idx]; 50254885Sdumbbell p->dma_reloc_idx = 0; 51254885Sdumbbell /* FIXME: we assume that each relocs use 4 dwords */ 52254885Sdumbbell p->nrelocs = chunk->length_dw / 4; 53254885Sdumbbell p->relocs_ptr = malloc(p->nrelocs * sizeof(void *), 54282199Sdumbbell DRM_MEM_DRIVER, M_NOWAIT | M_ZERO); 55254885Sdumbbell if (p->relocs_ptr == NULL) { 56254885Sdumbbell return -ENOMEM; 57254885Sdumbbell } 58254885Sdumbbell p->relocs = malloc(p->nrelocs * sizeof(struct radeon_cs_reloc), 59282199Sdumbbell DRM_MEM_DRIVER, M_NOWAIT | M_ZERO); 60254885Sdumbbell if (p->relocs == NULL) { 61254885Sdumbbell return -ENOMEM; 62254885Sdumbbell } 63254885Sdumbbell for (i = 0; i < p->nrelocs; i++) { 64254885Sdumbbell struct drm_radeon_cs_reloc *r; 65254885Sdumbbell 66254885Sdumbbell duplicate = false; 67254885Sdumbbell r = (struct drm_radeon_cs_reloc *)&chunk->kdata[i*4]; 68254885Sdumbbell for (j = 0; j < i; j++) { 69254885Sdumbbell if (r->handle == p->relocs[j].handle) { 70254885Sdumbbell p->relocs_ptr[i] = &p->relocs[j]; 71254885Sdumbbell duplicate = true; 72254885Sdumbbell break; 73254885Sdumbbell } 74254885Sdumbbell } 75254885Sdumbbell if (!duplicate) { 76254885Sdumbbell p->relocs[i].gobj = drm_gem_object_lookup(ddev, 77254885Sdumbbell p->filp, 78254885Sdumbbell r->handle); 79254885Sdumbbell if (p->relocs[i].gobj == NULL) { 80254885Sdumbbell DRM_ERROR("gem object lookup failed 0x%x\n", 81254885Sdumbbell r->handle); 82254885Sdumbbell return -ENOENT; 83254885Sdumbbell } 84254885Sdumbbell p->relocs_ptr[i] = &p->relocs[i]; 85254885Sdumbbell p->relocs[i].robj = gem_to_radeon_bo(p->relocs[i].gobj); 86254885Sdumbbell p->relocs[i].lobj.bo = p->relocs[i].robj; 87254885Sdumbbell p->relocs[i].lobj.wdomain = r->write_domain; 88254885Sdumbbell p->relocs[i].lobj.rdomain = r->read_domains; 89254885Sdumbbell p->relocs[i].lobj.tv.bo = &p->relocs[i].robj->tbo; 90254885Sdumbbell p->relocs[i].handle = r->handle; 91254885Sdumbbell p->relocs[i].flags = r->flags; 92254885Sdumbbell radeon_bo_list_add_object(&p->relocs[i].lobj, 93254885Sdumbbell &p->validated); 94254885Sdumbbell 95254885Sdumbbell } else 96254885Sdumbbell p->relocs[i].handle = 0; 97254885Sdumbbell } 98254885Sdumbbell return radeon_bo_list_validate(&p->validated); 99254885Sdumbbell} 100254885Sdumbbell 101254885Sdumbbellstatic int radeon_cs_get_ring(struct radeon_cs_parser *p, u32 ring, s32 priority) 102254885Sdumbbell{ 103254885Sdumbbell p->priority = priority; 104254885Sdumbbell 105254885Sdumbbell switch (ring) { 106254885Sdumbbell default: 107254885Sdumbbell DRM_ERROR("unknown ring id: %d\n", ring); 108254885Sdumbbell return -EINVAL; 109254885Sdumbbell case RADEON_CS_RING_GFX: 110254885Sdumbbell p->ring = RADEON_RING_TYPE_GFX_INDEX; 111254885Sdumbbell break; 112254885Sdumbbell case RADEON_CS_RING_COMPUTE: 113254885Sdumbbell if (p->rdev->family >= CHIP_TAHITI) { 114254885Sdumbbell if (p->priority > 0) 115254885Sdumbbell p->ring = CAYMAN_RING_TYPE_CP1_INDEX; 116254885Sdumbbell else 117254885Sdumbbell p->ring = CAYMAN_RING_TYPE_CP2_INDEX; 118254885Sdumbbell } else 119254885Sdumbbell p->ring = RADEON_RING_TYPE_GFX_INDEX; 120254885Sdumbbell break; 121254885Sdumbbell case RADEON_CS_RING_DMA: 122254885Sdumbbell if (p->rdev->family >= CHIP_CAYMAN) { 123254885Sdumbbell if (p->priority > 0) 124254885Sdumbbell p->ring = R600_RING_TYPE_DMA_INDEX; 125254885Sdumbbell else 126254885Sdumbbell p->ring = CAYMAN_RING_TYPE_DMA1_INDEX; 127254885Sdumbbell } else if (p->rdev->family >= CHIP_R600) { 128254885Sdumbbell p->ring = R600_RING_TYPE_DMA_INDEX; 129254885Sdumbbell } else { 130254885Sdumbbell return -EINVAL; 131254885Sdumbbell } 132254885Sdumbbell break; 133254885Sdumbbell } 134254885Sdumbbell return 0; 135254885Sdumbbell} 136254885Sdumbbell 137254885Sdumbbellstatic void radeon_cs_sync_to(struct radeon_cs_parser *p, 138254885Sdumbbell struct radeon_fence *fence) 139254885Sdumbbell{ 140254885Sdumbbell struct radeon_fence *other; 141254885Sdumbbell 142254885Sdumbbell if (!fence) 143254885Sdumbbell return; 144254885Sdumbbell 145254885Sdumbbell other = p->ib.sync_to[fence->ring]; 146254885Sdumbbell p->ib.sync_to[fence->ring] = radeon_fence_later(fence, other); 147254885Sdumbbell} 148254885Sdumbbell 149254885Sdumbbellstatic void radeon_cs_sync_rings(struct radeon_cs_parser *p) 150254885Sdumbbell{ 151254885Sdumbbell int i; 152254885Sdumbbell 153254885Sdumbbell for (i = 0; i < p->nrelocs; i++) { 154254885Sdumbbell if (!p->relocs[i].robj) 155254885Sdumbbell continue; 156254885Sdumbbell 157254885Sdumbbell radeon_cs_sync_to(p, p->relocs[i].robj->tbo.sync_obj); 158254885Sdumbbell } 159254885Sdumbbell} 160254885Sdumbbell 161254885Sdumbbell/* XXX: note that this is called from the legacy UMS CS ioctl as well */ 162254885Sdumbbellint radeon_cs_parser_init(struct radeon_cs_parser *p, void *data) 163254885Sdumbbell{ 164254885Sdumbbell struct drm_radeon_cs *cs = data; 165254885Sdumbbell uint64_t *chunk_array_ptr; 166254885Sdumbbell unsigned size, i; 167254885Sdumbbell u32 ring = RADEON_CS_RING_GFX; 168254885Sdumbbell s32 priority = 0; 169254885Sdumbbell 170254885Sdumbbell if (!cs->num_chunks) { 171254885Sdumbbell return 0; 172254885Sdumbbell } 173254885Sdumbbell /* get chunks */ 174254885Sdumbbell INIT_LIST_HEAD(&p->validated); 175254885Sdumbbell p->idx = 0; 176254885Sdumbbell p->ib.sa_bo = NULL; 177254885Sdumbbell p->ib.semaphore = NULL; 178254885Sdumbbell p->const_ib.sa_bo = NULL; 179254885Sdumbbell p->const_ib.semaphore = NULL; 180254885Sdumbbell p->chunk_ib_idx = -1; 181254885Sdumbbell p->chunk_relocs_idx = -1; 182254885Sdumbbell p->chunk_flags_idx = -1; 183254885Sdumbbell p->chunk_const_ib_idx = -1; 184254885Sdumbbell p->chunks_array = malloc(cs->num_chunks * sizeof(uint64_t), 185282199Sdumbbell DRM_MEM_DRIVER, M_NOWAIT | M_ZERO); 186254885Sdumbbell if (p->chunks_array == NULL) { 187254885Sdumbbell return -ENOMEM; 188254885Sdumbbell } 189254885Sdumbbell chunk_array_ptr = (uint64_t *)(unsigned long)(cs->chunks); 190254885Sdumbbell if (DRM_COPY_FROM_USER(p->chunks_array, chunk_array_ptr, 191254885Sdumbbell sizeof(uint64_t)*cs->num_chunks)) { 192254885Sdumbbell return -EFAULT; 193254885Sdumbbell } 194254885Sdumbbell p->cs_flags = 0; 195254885Sdumbbell p->nchunks = cs->num_chunks; 196254885Sdumbbell p->chunks = malloc(p->nchunks * sizeof(struct radeon_cs_chunk), 197282199Sdumbbell DRM_MEM_DRIVER, M_NOWAIT | M_ZERO); 198254885Sdumbbell if (p->chunks == NULL) { 199254885Sdumbbell return -ENOMEM; 200254885Sdumbbell } 201254885Sdumbbell for (i = 0; i < p->nchunks; i++) { 202254885Sdumbbell struct drm_radeon_cs_chunk __user **chunk_ptr = NULL; 203254885Sdumbbell struct drm_radeon_cs_chunk user_chunk; 204254885Sdumbbell uint32_t __user *cdata; 205254885Sdumbbell 206254885Sdumbbell chunk_ptr = (void __user*)(unsigned long)p->chunks_array[i]; 207254885Sdumbbell if (DRM_COPY_FROM_USER(&user_chunk, chunk_ptr, 208254885Sdumbbell sizeof(struct drm_radeon_cs_chunk))) { 209254885Sdumbbell return -EFAULT; 210254885Sdumbbell } 211254885Sdumbbell p->chunks[i].length_dw = user_chunk.length_dw; 212254885Sdumbbell p->chunks[i].kdata = NULL; 213254885Sdumbbell p->chunks[i].chunk_id = user_chunk.chunk_id; 214254885Sdumbbell 215254885Sdumbbell if (p->chunks[i].chunk_id == RADEON_CHUNK_ID_RELOCS) { 216254885Sdumbbell p->chunk_relocs_idx = i; 217254885Sdumbbell } 218254885Sdumbbell if (p->chunks[i].chunk_id == RADEON_CHUNK_ID_IB) { 219254885Sdumbbell p->chunk_ib_idx = i; 220254885Sdumbbell /* zero length IB isn't useful */ 221254885Sdumbbell if (p->chunks[i].length_dw == 0) 222254885Sdumbbell return -EINVAL; 223254885Sdumbbell } 224254885Sdumbbell if (p->chunks[i].chunk_id == RADEON_CHUNK_ID_CONST_IB) { 225254885Sdumbbell p->chunk_const_ib_idx = i; 226254885Sdumbbell /* zero length CONST IB isn't useful */ 227254885Sdumbbell if (p->chunks[i].length_dw == 0) 228254885Sdumbbell return -EINVAL; 229254885Sdumbbell } 230254885Sdumbbell if (p->chunks[i].chunk_id == RADEON_CHUNK_ID_FLAGS) { 231254885Sdumbbell p->chunk_flags_idx = i; 232254885Sdumbbell /* zero length flags aren't useful */ 233254885Sdumbbell if (p->chunks[i].length_dw == 0) 234254885Sdumbbell return -EINVAL; 235254885Sdumbbell } 236254885Sdumbbell 237254885Sdumbbell p->chunks[i].length_dw = user_chunk.length_dw; 238254885Sdumbbell p->chunks[i].user_ptr = (void __user *)(unsigned long)user_chunk.chunk_data; 239254885Sdumbbell 240254885Sdumbbell cdata = (uint32_t *)(unsigned long)user_chunk.chunk_data; 241254885Sdumbbell if ((p->chunks[i].chunk_id == RADEON_CHUNK_ID_RELOCS) || 242254885Sdumbbell (p->chunks[i].chunk_id == RADEON_CHUNK_ID_FLAGS)) { 243254885Sdumbbell size = p->chunks[i].length_dw * sizeof(uint32_t); 244282199Sdumbbell p->chunks[i].kdata = malloc(size, DRM_MEM_DRIVER, M_NOWAIT); 245254885Sdumbbell if (p->chunks[i].kdata == NULL) { 246254885Sdumbbell return -ENOMEM; 247254885Sdumbbell } 248254885Sdumbbell if (DRM_COPY_FROM_USER(p->chunks[i].kdata, 249254885Sdumbbell p->chunks[i].user_ptr, size)) { 250254885Sdumbbell return -EFAULT; 251254885Sdumbbell } 252254885Sdumbbell if (p->chunks[i].chunk_id == RADEON_CHUNK_ID_FLAGS) { 253254885Sdumbbell p->cs_flags = p->chunks[i].kdata[0]; 254254885Sdumbbell if (p->chunks[i].length_dw > 1) 255254885Sdumbbell ring = p->chunks[i].kdata[1]; 256254885Sdumbbell if (p->chunks[i].length_dw > 2) 257254885Sdumbbell priority = (s32)p->chunks[i].kdata[2]; 258254885Sdumbbell } 259254885Sdumbbell } 260254885Sdumbbell } 261254885Sdumbbell 262254885Sdumbbell /* these are KMS only */ 263254885Sdumbbell if (p->rdev) { 264254885Sdumbbell if ((p->cs_flags & RADEON_CS_USE_VM) && 265254885Sdumbbell !p->rdev->vm_manager.enabled) { 266254885Sdumbbell DRM_ERROR("VM not active on asic!\n"); 267254885Sdumbbell return -EINVAL; 268254885Sdumbbell } 269254885Sdumbbell 270254885Sdumbbell /* we only support VM on SI+ */ 271254885Sdumbbell if ((p->rdev->family >= CHIP_TAHITI) && 272254885Sdumbbell ((p->cs_flags & RADEON_CS_USE_VM) == 0)) { 273254885Sdumbbell DRM_ERROR("VM required on SI+!\n"); 274254885Sdumbbell return -EINVAL; 275254885Sdumbbell } 276254885Sdumbbell 277254885Sdumbbell if (radeon_cs_get_ring(p, ring, priority)) 278254885Sdumbbell return -EINVAL; 279254885Sdumbbell } 280254885Sdumbbell 281254885Sdumbbell /* deal with non-vm */ 282254885Sdumbbell if ((p->chunk_ib_idx != -1) && 283254885Sdumbbell ((p->cs_flags & RADEON_CS_USE_VM) == 0) && 284254885Sdumbbell (p->chunks[p->chunk_ib_idx].chunk_id == RADEON_CHUNK_ID_IB)) { 285254885Sdumbbell if (p->chunks[p->chunk_ib_idx].length_dw > (16 * 1024)) { 286254885Sdumbbell DRM_ERROR("cs IB too big: %d\n", 287254885Sdumbbell p->chunks[p->chunk_ib_idx].length_dw); 288254885Sdumbbell return -EINVAL; 289254885Sdumbbell } 290254885Sdumbbell if (p->rdev && (p->rdev->flags & RADEON_IS_AGP)) { 291282199Sdumbbell p->chunks[p->chunk_ib_idx].kpage[0] = malloc(PAGE_SIZE, DRM_MEM_DRIVER, M_NOWAIT); 292282199Sdumbbell p->chunks[p->chunk_ib_idx].kpage[1] = malloc(PAGE_SIZE, DRM_MEM_DRIVER, M_NOWAIT); 293254885Sdumbbell if (p->chunks[p->chunk_ib_idx].kpage[0] == NULL || 294254885Sdumbbell p->chunks[p->chunk_ib_idx].kpage[1] == NULL) { 295254885Sdumbbell free(p->chunks[p->chunk_ib_idx].kpage[0], DRM_MEM_DRIVER); 296254885Sdumbbell free(p->chunks[p->chunk_ib_idx].kpage[1], DRM_MEM_DRIVER); 297254885Sdumbbell p->chunks[p->chunk_ib_idx].kpage[0] = NULL; 298254885Sdumbbell p->chunks[p->chunk_ib_idx].kpage[1] = NULL; 299254885Sdumbbell return -ENOMEM; 300254885Sdumbbell } 301254885Sdumbbell } 302254885Sdumbbell p->chunks[p->chunk_ib_idx].kpage_idx[0] = -1; 303254885Sdumbbell p->chunks[p->chunk_ib_idx].kpage_idx[1] = -1; 304254885Sdumbbell p->chunks[p->chunk_ib_idx].last_copied_page = -1; 305254885Sdumbbell p->chunks[p->chunk_ib_idx].last_page_index = 306254885Sdumbbell ((p->chunks[p->chunk_ib_idx].length_dw * 4) - 1) / PAGE_SIZE; 307254885Sdumbbell } 308254885Sdumbbell 309254885Sdumbbell return 0; 310254885Sdumbbell} 311254885Sdumbbell 312254885Sdumbbell/** 313254885Sdumbbell * cs_parser_fini() - clean parser states 314254885Sdumbbell * @parser: parser structure holding parsing context. 315254885Sdumbbell * @error: error number 316254885Sdumbbell * 317254885Sdumbbell * If error is set than unvalidate buffer, otherwise just free memory 318254885Sdumbbell * used by parsing context. 319254885Sdumbbell **/ 320254885Sdumbbellstatic void radeon_cs_parser_fini(struct radeon_cs_parser *parser, int error) 321254885Sdumbbell{ 322254885Sdumbbell unsigned i; 323254885Sdumbbell 324254885Sdumbbell if (!error) { 325254885Sdumbbell ttm_eu_fence_buffer_objects(&parser->validated, 326254885Sdumbbell parser->ib.fence); 327254885Sdumbbell } else { 328254885Sdumbbell ttm_eu_backoff_reservation(&parser->validated); 329254885Sdumbbell } 330254885Sdumbbell 331254885Sdumbbell if (parser->relocs != NULL) { 332254885Sdumbbell for (i = 0; i < parser->nrelocs; i++) { 333254885Sdumbbell if (parser->relocs[i].gobj) 334254885Sdumbbell drm_gem_object_unreference_unlocked(parser->relocs[i].gobj); 335254885Sdumbbell } 336254885Sdumbbell } 337254885Sdumbbell free(parser->track, DRM_MEM_DRIVER); 338254885Sdumbbell free(parser->relocs, DRM_MEM_DRIVER); 339254885Sdumbbell free(parser->relocs_ptr, DRM_MEM_DRIVER); 340254885Sdumbbell for (i = 0; i < parser->nchunks; i++) { 341254885Sdumbbell free(parser->chunks[i].kdata, DRM_MEM_DRIVER); 342254885Sdumbbell if ((parser->rdev->flags & RADEON_IS_AGP)) { 343254885Sdumbbell free(parser->chunks[i].kpage[0], DRM_MEM_DRIVER); 344254885Sdumbbell free(parser->chunks[i].kpage[1], DRM_MEM_DRIVER); 345254885Sdumbbell } 346254885Sdumbbell } 347254885Sdumbbell free(parser->chunks, DRM_MEM_DRIVER); 348254885Sdumbbell free(parser->chunks_array, DRM_MEM_DRIVER); 349254885Sdumbbell radeon_ib_free(parser->rdev, &parser->ib); 350254885Sdumbbell radeon_ib_free(parser->rdev, &parser->const_ib); 351254885Sdumbbell} 352254885Sdumbbell 353254885Sdumbbellstatic int radeon_cs_ib_chunk(struct radeon_device *rdev, 354254885Sdumbbell struct radeon_cs_parser *parser) 355254885Sdumbbell{ 356254885Sdumbbell struct radeon_cs_chunk *ib_chunk; 357254885Sdumbbell int r; 358254885Sdumbbell 359254885Sdumbbell if (parser->chunk_ib_idx == -1) 360254885Sdumbbell return 0; 361254885Sdumbbell 362254885Sdumbbell if (parser->cs_flags & RADEON_CS_USE_VM) 363254885Sdumbbell return 0; 364254885Sdumbbell 365254885Sdumbbell ib_chunk = &parser->chunks[parser->chunk_ib_idx]; 366254885Sdumbbell /* Copy the packet into the IB, the parser will read from the 367254885Sdumbbell * input memory (cached) and write to the IB (which can be 368254885Sdumbbell * uncached). 369254885Sdumbbell */ 370254885Sdumbbell r = radeon_ib_get(rdev, parser->ring, &parser->ib, 371254885Sdumbbell NULL, ib_chunk->length_dw * 4); 372254885Sdumbbell if (r) { 373254885Sdumbbell DRM_ERROR("Failed to get ib !\n"); 374254885Sdumbbell return r; 375254885Sdumbbell } 376254885Sdumbbell parser->ib.length_dw = ib_chunk->length_dw; 377254885Sdumbbell r = radeon_cs_parse(rdev, parser->ring, parser); 378254885Sdumbbell if (r || parser->parser_error) { 379254885Sdumbbell DRM_ERROR("Invalid command stream !\n"); 380254885Sdumbbell return r; 381254885Sdumbbell } 382254885Sdumbbell r = radeon_cs_finish_pages(parser); 383254885Sdumbbell if (r) { 384254885Sdumbbell DRM_ERROR("Invalid command stream !\n"); 385254885Sdumbbell return r; 386254885Sdumbbell } 387254885Sdumbbell radeon_cs_sync_rings(parser); 388254885Sdumbbell r = radeon_ib_schedule(rdev, &parser->ib, NULL); 389254885Sdumbbell if (r) { 390254885Sdumbbell DRM_ERROR("Failed to schedule IB !\n"); 391254885Sdumbbell } 392254885Sdumbbell return r; 393254885Sdumbbell} 394254885Sdumbbell 395254885Sdumbbellstatic int radeon_bo_vm_update_pte(struct radeon_cs_parser *parser, 396254885Sdumbbell struct radeon_vm *vm) 397254885Sdumbbell{ 398254885Sdumbbell struct radeon_device *rdev = parser->rdev; 399254885Sdumbbell struct radeon_bo_list *lobj; 400254885Sdumbbell struct radeon_bo *bo; 401254885Sdumbbell int r; 402254885Sdumbbell 403254885Sdumbbell r = radeon_vm_bo_update_pte(rdev, vm, rdev->ring_tmp_bo.bo, &rdev->ring_tmp_bo.bo->tbo.mem); 404254885Sdumbbell if (r) { 405254885Sdumbbell return r; 406254885Sdumbbell } 407254885Sdumbbell list_for_each_entry(lobj, &parser->validated, tv.head) { 408254885Sdumbbell bo = lobj->bo; 409254885Sdumbbell r = radeon_vm_bo_update_pte(parser->rdev, vm, bo, &bo->tbo.mem); 410254885Sdumbbell if (r) { 411254885Sdumbbell return r; 412254885Sdumbbell } 413254885Sdumbbell } 414254885Sdumbbell return 0; 415254885Sdumbbell} 416254885Sdumbbell 417254885Sdumbbellstatic int radeon_cs_ib_vm_chunk(struct radeon_device *rdev, 418254885Sdumbbell struct radeon_cs_parser *parser) 419254885Sdumbbell{ 420254885Sdumbbell struct radeon_cs_chunk *ib_chunk; 421254885Sdumbbell struct radeon_fpriv *fpriv = parser->filp->driver_priv; 422254885Sdumbbell struct radeon_vm *vm = &fpriv->vm; 423254885Sdumbbell int r; 424254885Sdumbbell 425254885Sdumbbell if (parser->chunk_ib_idx == -1) 426254885Sdumbbell return 0; 427254885Sdumbbell if ((parser->cs_flags & RADEON_CS_USE_VM) == 0) 428254885Sdumbbell return 0; 429254885Sdumbbell 430254885Sdumbbell if ((rdev->family >= CHIP_TAHITI) && 431254885Sdumbbell (parser->chunk_const_ib_idx != -1)) { 432254885Sdumbbell ib_chunk = &parser->chunks[parser->chunk_const_ib_idx]; 433254885Sdumbbell if (ib_chunk->length_dw > RADEON_IB_VM_MAX_SIZE) { 434254885Sdumbbell DRM_ERROR("cs IB CONST too big: %d\n", ib_chunk->length_dw); 435254885Sdumbbell return -EINVAL; 436254885Sdumbbell } 437254885Sdumbbell r = radeon_ib_get(rdev, parser->ring, &parser->const_ib, 438254885Sdumbbell vm, ib_chunk->length_dw * 4); 439254885Sdumbbell if (r) { 440254885Sdumbbell DRM_ERROR("Failed to get const ib !\n"); 441254885Sdumbbell return r; 442254885Sdumbbell } 443254885Sdumbbell parser->const_ib.is_const_ib = true; 444254885Sdumbbell parser->const_ib.length_dw = ib_chunk->length_dw; 445254885Sdumbbell /* Copy the packet into the IB */ 446254885Sdumbbell if (DRM_COPY_FROM_USER(parser->const_ib.ptr, ib_chunk->user_ptr, 447254885Sdumbbell ib_chunk->length_dw * 4)) { 448254885Sdumbbell return -EFAULT; 449254885Sdumbbell } 450254885Sdumbbell r = radeon_ring_ib_parse(rdev, parser->ring, &parser->const_ib); 451254885Sdumbbell if (r) { 452254885Sdumbbell return r; 453254885Sdumbbell } 454254885Sdumbbell } 455254885Sdumbbell 456254885Sdumbbell ib_chunk = &parser->chunks[parser->chunk_ib_idx]; 457254885Sdumbbell if (ib_chunk->length_dw > RADEON_IB_VM_MAX_SIZE) { 458254885Sdumbbell DRM_ERROR("cs IB too big: %d\n", ib_chunk->length_dw); 459254885Sdumbbell return -EINVAL; 460254885Sdumbbell } 461254885Sdumbbell r = radeon_ib_get(rdev, parser->ring, &parser->ib, 462254885Sdumbbell vm, ib_chunk->length_dw * 4); 463254885Sdumbbell if (r) { 464254885Sdumbbell DRM_ERROR("Failed to get ib !\n"); 465254885Sdumbbell return r; 466254885Sdumbbell } 467254885Sdumbbell parser->ib.length_dw = ib_chunk->length_dw; 468254885Sdumbbell /* Copy the packet into the IB */ 469254885Sdumbbell if (DRM_COPY_FROM_USER(parser->ib.ptr, ib_chunk->user_ptr, 470254885Sdumbbell ib_chunk->length_dw * 4)) { 471254885Sdumbbell return -EFAULT; 472254885Sdumbbell } 473254885Sdumbbell r = radeon_ring_ib_parse(rdev, parser->ring, &parser->ib); 474254885Sdumbbell if (r) { 475254885Sdumbbell return r; 476254885Sdumbbell } 477254885Sdumbbell 478254885Sdumbbell sx_xlock(&rdev->vm_manager.lock); 479254885Sdumbbell sx_xlock(&vm->mutex); 480254885Sdumbbell r = radeon_vm_alloc_pt(rdev, vm); 481254885Sdumbbell if (r) { 482254885Sdumbbell goto out; 483254885Sdumbbell } 484254885Sdumbbell r = radeon_bo_vm_update_pte(parser, vm); 485254885Sdumbbell if (r) { 486254885Sdumbbell goto out; 487254885Sdumbbell } 488254885Sdumbbell radeon_cs_sync_rings(parser); 489254885Sdumbbell radeon_cs_sync_to(parser, vm->fence); 490254885Sdumbbell radeon_cs_sync_to(parser, radeon_vm_grab_id(rdev, vm, parser->ring)); 491254885Sdumbbell 492254885Sdumbbell if ((rdev->family >= CHIP_TAHITI) && 493254885Sdumbbell (parser->chunk_const_ib_idx != -1)) { 494254885Sdumbbell r = radeon_ib_schedule(rdev, &parser->ib, &parser->const_ib); 495254885Sdumbbell } else { 496254885Sdumbbell r = radeon_ib_schedule(rdev, &parser->ib, NULL); 497254885Sdumbbell } 498254885Sdumbbell 499254885Sdumbbell if (!r) { 500254885Sdumbbell radeon_vm_fence(rdev, vm, parser->ib.fence); 501254885Sdumbbell } 502254885Sdumbbell 503254885Sdumbbellout: 504254885Sdumbbell radeon_vm_add_to_lru(rdev, vm); 505254885Sdumbbell sx_xunlock(&vm->mutex); 506254885Sdumbbell sx_xunlock(&rdev->vm_manager.lock); 507254885Sdumbbell return r; 508254885Sdumbbell} 509254885Sdumbbell 510254885Sdumbbellstatic int radeon_cs_handle_lockup(struct radeon_device *rdev, int r) 511254885Sdumbbell{ 512254885Sdumbbell if (r == -EDEADLK) { 513254885Sdumbbell r = radeon_gpu_reset(rdev); 514254885Sdumbbell if (!r) 515254885Sdumbbell r = -EAGAIN; 516254885Sdumbbell } 517254885Sdumbbell return r; 518254885Sdumbbell} 519254885Sdumbbell 520254885Sdumbbellint radeon_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) 521254885Sdumbbell{ 522254885Sdumbbell struct radeon_device *rdev = dev->dev_private; 523254885Sdumbbell struct radeon_cs_parser parser; 524254885Sdumbbell int r; 525254885Sdumbbell 526254885Sdumbbell sx_slock(&rdev->exclusive_lock); 527254885Sdumbbell if (!rdev->accel_working) { 528254885Sdumbbell sx_sunlock(&rdev->exclusive_lock); 529254885Sdumbbell return -EBUSY; 530254885Sdumbbell } 531254885Sdumbbell /* initialize parser */ 532254885Sdumbbell memset(&parser, 0, sizeof(struct radeon_cs_parser)); 533254885Sdumbbell parser.filp = filp; 534254885Sdumbbell parser.rdev = rdev; 535254885Sdumbbell parser.dev = rdev->dev; 536254885Sdumbbell parser.family = rdev->family; 537254885Sdumbbell r = radeon_cs_parser_init(&parser, data); 538254885Sdumbbell if (r) { 539254885Sdumbbell DRM_ERROR("Failed to initialize parser !\n"); 540254885Sdumbbell radeon_cs_parser_fini(&parser, r); 541254885Sdumbbell sx_sunlock(&rdev->exclusive_lock); 542254885Sdumbbell r = radeon_cs_handle_lockup(rdev, r); 543254885Sdumbbell return r; 544254885Sdumbbell } 545254885Sdumbbell r = radeon_cs_parser_relocs(&parser); 546254885Sdumbbell if (r) { 547254885Sdumbbell if (r != -ERESTARTSYS) 548254885Sdumbbell DRM_ERROR("Failed to parse relocation %d!\n", r); 549254885Sdumbbell radeon_cs_parser_fini(&parser, r); 550254885Sdumbbell sx_sunlock(&rdev->exclusive_lock); 551254885Sdumbbell r = radeon_cs_handle_lockup(rdev, r); 552254885Sdumbbell return r; 553254885Sdumbbell } 554254885Sdumbbell r = radeon_cs_ib_chunk(rdev, &parser); 555254885Sdumbbell if (r) { 556254885Sdumbbell goto out; 557254885Sdumbbell } 558254885Sdumbbell r = radeon_cs_ib_vm_chunk(rdev, &parser); 559254885Sdumbbell if (r) { 560254885Sdumbbell goto out; 561254885Sdumbbell } 562254885Sdumbbellout: 563254885Sdumbbell radeon_cs_parser_fini(&parser, r); 564254885Sdumbbell sx_sunlock(&rdev->exclusive_lock); 565254885Sdumbbell r = radeon_cs_handle_lockup(rdev, r); 566254885Sdumbbell return r; 567254885Sdumbbell} 568254885Sdumbbell 569254885Sdumbbellint radeon_cs_finish_pages(struct radeon_cs_parser *p) 570254885Sdumbbell{ 571254885Sdumbbell struct radeon_cs_chunk *ibc = &p->chunks[p->chunk_ib_idx]; 572254885Sdumbbell int i; 573254885Sdumbbell int size = PAGE_SIZE; 574254885Sdumbbell 575254885Sdumbbell for (i = ibc->last_copied_page + 1; i <= ibc->last_page_index; i++) { 576254885Sdumbbell if (i == ibc->last_page_index) { 577254885Sdumbbell size = (ibc->length_dw * 4) % PAGE_SIZE; 578254885Sdumbbell if (size == 0) 579254885Sdumbbell size = PAGE_SIZE; 580254885Sdumbbell } 581254885Sdumbbell 582254885Sdumbbell if (DRM_COPY_FROM_USER(p->ib.ptr + (i * (PAGE_SIZE/4)), 583254885Sdumbbell (char *)ibc->user_ptr + (i * PAGE_SIZE), 584254885Sdumbbell size)) 585254885Sdumbbell return -EFAULT; 586254885Sdumbbell } 587254885Sdumbbell return 0; 588254885Sdumbbell} 589254885Sdumbbell 590254885Sdumbbellstatic int radeon_cs_update_pages(struct radeon_cs_parser *p, int pg_idx) 591254885Sdumbbell{ 592254885Sdumbbell int new_page; 593254885Sdumbbell struct radeon_cs_chunk *ibc = &p->chunks[p->chunk_ib_idx]; 594254885Sdumbbell int i; 595254885Sdumbbell int size = PAGE_SIZE; 596254885Sdumbbell bool copy1 = (p->rdev && (p->rdev->flags & RADEON_IS_AGP)) ? 597254885Sdumbbell false : true; 598254885Sdumbbell 599254885Sdumbbell for (i = ibc->last_copied_page + 1; i < pg_idx; i++) { 600254885Sdumbbell if (DRM_COPY_FROM_USER(p->ib.ptr + (i * (PAGE_SIZE/4)), 601254885Sdumbbell (char *)ibc->user_ptr + (i * PAGE_SIZE), 602254885Sdumbbell PAGE_SIZE)) { 603254885Sdumbbell p->parser_error = -EFAULT; 604254885Sdumbbell return 0; 605254885Sdumbbell } 606254885Sdumbbell } 607254885Sdumbbell 608254885Sdumbbell if (pg_idx == ibc->last_page_index) { 609254885Sdumbbell size = (ibc->length_dw * 4) % PAGE_SIZE; 610254885Sdumbbell if (size == 0) 611254885Sdumbbell size = PAGE_SIZE; 612254885Sdumbbell } 613254885Sdumbbell 614254885Sdumbbell new_page = ibc->kpage_idx[0] < ibc->kpage_idx[1] ? 0 : 1; 615254885Sdumbbell if (copy1) 616254885Sdumbbell ibc->kpage[new_page] = p->ib.ptr + (pg_idx * (PAGE_SIZE / 4)); 617254885Sdumbbell 618254885Sdumbbell if (DRM_COPY_FROM_USER(ibc->kpage[new_page], 619254885Sdumbbell (char *)ibc->user_ptr + (pg_idx * PAGE_SIZE), 620254885Sdumbbell size)) { 621254885Sdumbbell p->parser_error = -EFAULT; 622254885Sdumbbell return 0; 623254885Sdumbbell } 624254885Sdumbbell 625254885Sdumbbell /* copy to IB for non single case */ 626254885Sdumbbell if (!copy1) 627254885Sdumbbell memcpy((void *)(p->ib.ptr+(pg_idx*(PAGE_SIZE/4))), ibc->kpage[new_page], size); 628254885Sdumbbell 629254885Sdumbbell ibc->last_copied_page = pg_idx; 630254885Sdumbbell ibc->kpage_idx[new_page] = pg_idx; 631254885Sdumbbell 632254885Sdumbbell return new_page; 633254885Sdumbbell} 634254885Sdumbbell 635254885Sdumbbellu32 radeon_get_ib_value(struct radeon_cs_parser *p, int idx) 636254885Sdumbbell{ 637254885Sdumbbell struct radeon_cs_chunk *ibc = &p->chunks[p->chunk_ib_idx]; 638254885Sdumbbell u32 pg_idx, pg_offset; 639254885Sdumbbell u32 idx_value = 0; 640254885Sdumbbell int new_page; 641254885Sdumbbell 642254885Sdumbbell pg_idx = (idx * 4) / PAGE_SIZE; 643254885Sdumbbell pg_offset = (idx * 4) % PAGE_SIZE; 644254885Sdumbbell 645254885Sdumbbell if (ibc->kpage_idx[0] == pg_idx) 646254885Sdumbbell return ibc->kpage[0][pg_offset/4]; 647254885Sdumbbell if (ibc->kpage_idx[1] == pg_idx) 648254885Sdumbbell return ibc->kpage[1][pg_offset/4]; 649254885Sdumbbell 650254885Sdumbbell new_page = radeon_cs_update_pages(p, pg_idx); 651254885Sdumbbell if (new_page < 0) { 652254885Sdumbbell p->parser_error = new_page; 653254885Sdumbbell return 0; 654254885Sdumbbell } 655254885Sdumbbell 656254885Sdumbbell idx_value = ibc->kpage[new_page][pg_offset/4]; 657254885Sdumbbell return idx_value; 658254885Sdumbbell} 659