1/* 2 * Copyright �� 2010 Daniel Vetter 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 */ 24 25#include <sys/cdefs.h> 26__FBSDID("$FreeBSD: stable/11/sys/dev/drm2/i915/i915_gem_gtt.c 324129 2017-09-30 18:52:59Z alc $"); 27 28#include <dev/drm2/drmP.h> 29#include <dev/drm2/i915/i915_drm.h> 30#include <dev/drm2/i915/i915_drv.h> 31#include <dev/drm2/i915/intel_drv.h> 32#include <sys/sched.h> 33#include <sys/sf_buf.h> 34#include <vm/vm_pageout.h> 35 36typedef uint32_t gtt_pte_t; 37 38/* PPGTT stuff */ 39#define GEN6_GTT_ADDR_ENCODE(addr) ((addr) | (((addr) >> 28) & 0xff0)) 40 41#define GEN6_PDE_VALID (1 << 0) 42/* gen6+ has bit 11-4 for physical addr bit 39-32 */ 43#define GEN6_PDE_ADDR_ENCODE(addr) GEN6_GTT_ADDR_ENCODE(addr) 44 45#define GEN6_PTE_VALID (1 << 0) 46#define GEN6_PTE_UNCACHED (1 << 1) 47#define HSW_PTE_UNCACHED (0) 48#define GEN6_PTE_CACHE_LLC (2 << 1) 49#define GEN6_PTE_CACHE_LLC_MLC (3 << 1) 50#define GEN6_PTE_ADDR_ENCODE(addr) GEN6_GTT_ADDR_ENCODE(addr) 51 52static inline gtt_pte_t pte_encode(struct drm_device *dev, 53 dma_addr_t addr, 54 enum i915_cache_level level) 55{ 56 gtt_pte_t pte = GEN6_PTE_VALID; 57 pte |= GEN6_PTE_ADDR_ENCODE(addr); 58 59 switch (level) { 60 case I915_CACHE_LLC_MLC: 61 /* Haswell doesn't set L3 this way */ 62 if (IS_HASWELL(dev)) 63 pte |= GEN6_PTE_CACHE_LLC; 64 else 65 pte |= GEN6_PTE_CACHE_LLC_MLC; 66 break; 67 case I915_CACHE_LLC: 68 pte |= GEN6_PTE_CACHE_LLC; 69 break; 70 case I915_CACHE_NONE: 71 if (IS_HASWELL(dev)) 72 pte |= HSW_PTE_UNCACHED; 73 else 74 pte |= GEN6_PTE_UNCACHED; 75 break; 76 default: 77 BUG(); 78 } 79 80 81 return pte; 82} 83 84/* PPGTT support for Sandybdrige/Gen6 and later */ 85static void i915_ppgtt_clear_range(struct i915_hw_ppgtt *ppgtt, 86 unsigned first_entry, 87 unsigned num_entries) 88{ 89 gtt_pte_t *pt_vaddr; 90 gtt_pte_t scratch_pte; 91 unsigned act_pd = first_entry / I915_PPGTT_PT_ENTRIES; 92 unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES; 93 unsigned last_pte, i; 94 struct sf_buf *sf; 95 96 scratch_pte = pte_encode(ppgtt->dev, ppgtt->scratch_page_dma_addr, 97 I915_CACHE_LLC); 98 99 while (num_entries) { 100 last_pte = first_pte + num_entries; 101 if (last_pte > I915_PPGTT_PT_ENTRIES) 102 last_pte = I915_PPGTT_PT_ENTRIES; 103 104 sched_pin(); 105 sf = sf_buf_alloc(ppgtt->pt_pages[act_pd], SFB_CPUPRIVATE); 106 pt_vaddr = (uint32_t *)(uintptr_t)sf_buf_kva(sf); 107 108 for (i = first_pte; i < last_pte; i++) 109 pt_vaddr[i] = scratch_pte; 110 111 sf_buf_free(sf); 112 sched_unpin(); 113 114 num_entries -= last_pte - first_pte; 115 first_pte = 0; 116 act_pd++; 117 } 118} 119 120int i915_gem_init_aliasing_ppgtt(struct drm_device *dev) 121{ 122 struct drm_i915_private *dev_priv = dev->dev_private; 123 struct i915_hw_ppgtt *ppgtt; 124 unsigned first_pd_entry_in_global_pt; 125 int i; 126 int ret = -ENOMEM; 127 128 /* ppgtt PDEs reside in the global gtt pagetable, which has 512*1024 129 * entries. For aliasing ppgtt support we just steal them at the end for 130 * now. */ 131 first_pd_entry_in_global_pt = dev_priv->mm.gtt->gtt_total_entries - I915_PPGTT_PD_ENTRIES; 132 133 ppgtt = malloc(sizeof(*ppgtt), DRM_I915_GEM, M_WAITOK | M_ZERO); 134 if (!ppgtt) 135 return ret; 136 137 ppgtt->dev = dev; 138 ppgtt->num_pd_entries = I915_PPGTT_PD_ENTRIES; 139 ppgtt->pt_pages = malloc(sizeof(struct page *)*ppgtt->num_pd_entries, 140 DRM_I915_GEM, M_WAITOK | M_ZERO); 141 if (!ppgtt->pt_pages) 142 goto err_ppgtt; 143 144 for (i = 0; i < ppgtt->num_pd_entries; i++) { 145 ppgtt->pt_pages[i] = vm_page_alloc(NULL, 0, 146 VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | 147 VM_ALLOC_ZERO); 148 if (!ppgtt->pt_pages[i]) 149 goto err_pt_alloc; 150 } 151 152 if (dev_priv->mm.gtt->needs_dmar) { 153 ppgtt->pt_dma_addr = malloc(sizeof(dma_addr_t) 154 *ppgtt->num_pd_entries, 155 DRM_I915_GEM, M_WAITOK | M_ZERO); 156 if (!ppgtt->pt_dma_addr) 157 goto err_pt_alloc; 158 159#ifdef CONFIG_INTEL_IOMMU /* <- Added as a marker on FreeBSD. */ 160 for (i = 0; i < ppgtt->num_pd_entries; i++) { 161 dma_addr_t pt_addr; 162 163 pt_addr = pci_map_page(dev->pdev, ppgtt->pt_pages[i], 164 0, 4096, 165 PCI_DMA_BIDIRECTIONAL); 166 167 if (pci_dma_mapping_error(dev->pdev, 168 pt_addr)) { 169 ret = -EIO; 170 goto err_pd_pin; 171 172 } 173 ppgtt->pt_dma_addr[i] = pt_addr; 174 } 175#endif 176 } 177 178 ppgtt->scratch_page_dma_addr = dev_priv->mm.gtt->scratch_page_dma; 179 180 i915_ppgtt_clear_range(ppgtt, 0, 181 ppgtt->num_pd_entries*I915_PPGTT_PT_ENTRIES); 182 183 ppgtt->pd_offset = (first_pd_entry_in_global_pt)*sizeof(gtt_pte_t); 184 185 dev_priv->mm.aliasing_ppgtt = ppgtt; 186 187 return 0; 188 189#ifdef CONFIG_INTEL_IOMMU /* <- Added as a marker on FreeBSD. */ 190err_pd_pin: 191 if (ppgtt->pt_dma_addr) { 192 for (i--; i >= 0; i--) 193 pci_unmap_page(dev->pdev, ppgtt->pt_dma_addr[i], 194 4096, PCI_DMA_BIDIRECTIONAL); 195 } 196#endif 197err_pt_alloc: 198 free(ppgtt->pt_dma_addr, DRM_I915_GEM); 199 for (i = 0; i < ppgtt->num_pd_entries; i++) { 200 if (ppgtt->pt_pages[i]) { 201 vm_page_unwire(ppgtt->pt_pages[i], PQ_NONE); 202 vm_page_free(ppgtt->pt_pages[i]); 203 } 204 } 205 free(ppgtt->pt_pages, DRM_I915_GEM); 206err_ppgtt: 207 free(ppgtt, DRM_I915_GEM); 208 209 return ret; 210} 211 212void i915_gem_cleanup_aliasing_ppgtt(struct drm_device *dev) 213{ 214 struct drm_i915_private *dev_priv = dev->dev_private; 215 struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt; 216 int i; 217 218 if (!ppgtt) 219 return; 220 221#ifdef CONFIG_INTEL_IOMMU /* <- Added as a marker on FreeBSD. */ 222 if (ppgtt->pt_dma_addr) { 223 for (i = 0; i < ppgtt->num_pd_entries; i++) 224 pci_unmap_page(dev->pdev, ppgtt->pt_dma_addr[i], 225 4096, PCI_DMA_BIDIRECTIONAL); 226 } 227#endif 228 229 free(ppgtt->pt_dma_addr, DRM_I915_GEM); 230 for (i = 0; i < ppgtt->num_pd_entries; i++) { 231 vm_page_unwire(ppgtt->pt_pages[i], PQ_NONE); 232 vm_page_free(ppgtt->pt_pages[i]); 233 } 234 free(ppgtt->pt_pages, DRM_I915_GEM); 235 free(ppgtt, DRM_I915_GEM); 236} 237 238static void i915_ppgtt_insert_pages(struct i915_hw_ppgtt *ppgtt, 239 vm_page_t *pages, 240 unsigned first_entry, 241 unsigned num_entries, 242 enum i915_cache_level cache_level) 243{ 244 uint32_t *pt_vaddr; 245 unsigned act_pd = first_entry / I915_PPGTT_PT_ENTRIES; 246 unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES; 247 unsigned j, last_pte; 248 vm_paddr_t page_addr; 249 struct sf_buf *sf; 250 251 while (num_entries) { 252 last_pte = first_pte + num_entries; 253 if (last_pte > I915_PPGTT_PT_ENTRIES) 254 last_pte = I915_PPGTT_PT_ENTRIES; 255 256 sched_pin(); 257 sf = sf_buf_alloc(ppgtt->pt_pages[act_pd], SFB_CPUPRIVATE); 258 pt_vaddr = (uint32_t *)(uintptr_t)sf_buf_kva(sf); 259 260 for (j = first_pte; j < last_pte; j++) { 261 page_addr = VM_PAGE_TO_PHYS(*pages); 262 pt_vaddr[j] = pte_encode(ppgtt->dev, page_addr, 263 cache_level); 264 265 pages++; 266 } 267 268 sf_buf_free(sf); 269 sched_unpin(); 270 271 num_entries -= last_pte - first_pte; 272 first_pte = 0; 273 act_pd++; 274 } 275} 276 277void i915_ppgtt_bind_object(struct i915_hw_ppgtt *ppgtt, 278 struct drm_i915_gem_object *obj, 279 enum i915_cache_level cache_level) 280{ 281 i915_ppgtt_insert_pages(ppgtt, 282 obj->pages, 283 obj->gtt_space->start >> PAGE_SHIFT, 284 obj->base.size >> PAGE_SHIFT, 285 cache_level); 286} 287 288void i915_ppgtt_unbind_object(struct i915_hw_ppgtt *ppgtt, 289 struct drm_i915_gem_object *obj) 290{ 291 i915_ppgtt_clear_range(ppgtt, 292 obj->gtt_space->start >> PAGE_SHIFT, 293 obj->base.size >> PAGE_SHIFT); 294} 295 296void i915_gem_init_ppgtt(struct drm_device *dev) 297{ 298 drm_i915_private_t *dev_priv = dev->dev_private; 299 uint32_t pd_offset; 300 struct intel_ring_buffer *ring; 301 struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt; 302 uint32_t __iomem *pd_addr; 303 uint32_t pd_entry; 304 int i; 305 306 if (!dev_priv->mm.aliasing_ppgtt) 307 return; 308 309 310 pd_addr = dev_priv->mm.gtt->gtt + ppgtt->pd_offset/sizeof(uint32_t); 311 for (i = 0; i < ppgtt->num_pd_entries; i++) { 312 vm_paddr_t pt_addr; 313 314 if (dev_priv->mm.gtt->needs_dmar) 315 pt_addr = ppgtt->pt_dma_addr[i]; 316 else 317 pt_addr = VM_PAGE_TO_PHYS(ppgtt->pt_pages[i]); 318 319 pd_entry = GEN6_PDE_ADDR_ENCODE(pt_addr); 320 pd_entry |= GEN6_PDE_VALID; 321 322 /* NOTE Linux<->FreeBSD: Arguments of writel() are reversed. */ 323 writel(pd_addr + i, pd_entry); 324 } 325 readl(pd_addr); 326 327 pd_offset = ppgtt->pd_offset; 328 pd_offset /= 64; /* in cachelines, */ 329 pd_offset <<= 16; 330 331 if (INTEL_INFO(dev)->gen == 6) { 332 uint32_t ecochk, gab_ctl, ecobits; 333 334 ecobits = I915_READ(GAC_ECO_BITS); 335 I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_PPGTT_CACHE64B); 336 337 gab_ctl = I915_READ(GAB_CTL); 338 I915_WRITE(GAB_CTL, gab_ctl | GAB_CTL_CONT_AFTER_PAGEFAULT); 339 340 ecochk = I915_READ(GAM_ECOCHK); 341 I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT | 342 ECOCHK_PPGTT_CACHE64B); 343 I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE)); 344 } else if (INTEL_INFO(dev)->gen >= 7) { 345 I915_WRITE(GAM_ECOCHK, ECOCHK_PPGTT_CACHE64B); 346 /* GFX_MODE is per-ring on gen7+ */ 347 } 348 349 for_each_ring(ring, dev_priv, i) { 350 if (INTEL_INFO(dev)->gen >= 7) 351 I915_WRITE(RING_MODE_GEN7(ring), 352 _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE)); 353 354 I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G); 355 I915_WRITE(RING_PP_DIR_BASE(ring), pd_offset); 356 } 357} 358 359static bool do_idling(struct drm_i915_private *dev_priv) 360{ 361 bool ret = dev_priv->mm.interruptible; 362 363 if (unlikely(dev_priv->mm.gtt->do_idle_maps)) { 364 dev_priv->mm.interruptible = false; 365 if (i915_gpu_idle(dev_priv->dev)) { 366 DRM_ERROR("Couldn't idle GPU\n"); 367 /* Wait a bit, in hopes it avoids the hang */ 368 udelay(10); 369 } 370 } 371 372 return ret; 373} 374 375static void undo_idling(struct drm_i915_private *dev_priv, bool interruptible) 376{ 377 if (unlikely(dev_priv->mm.gtt->do_idle_maps)) 378 dev_priv->mm.interruptible = interruptible; 379} 380 381 382static void i915_ggtt_clear_range(struct drm_device *dev, 383 unsigned first_entry, 384 unsigned num_entries) 385{ 386 struct drm_i915_private *dev_priv = dev->dev_private; 387 gtt_pte_t scratch_pte; 388 gtt_pte_t __iomem *gtt_base = dev_priv->mm.gtt->gtt + first_entry; 389 const int max_entries = dev_priv->mm.gtt->gtt_total_entries - first_entry; 390 int i; 391 392 if (INTEL_INFO(dev)->gen < 6) { 393 intel_gtt_clear_range(first_entry, num_entries); 394 return; 395 } 396 397 if (WARN(num_entries > max_entries, 398 "First entry = %d; Num entries = %d (max=%d)\n", 399 first_entry, num_entries, max_entries)) 400 num_entries = max_entries; 401 402 scratch_pte = pte_encode(dev, dev_priv->mm.gtt->scratch_page_dma, I915_CACHE_LLC); 403 for (i = 0; i < num_entries; i++) 404 iowrite32(scratch_pte, >t_base[i]); 405 readl(gtt_base); 406} 407 408void i915_gem_restore_gtt_mappings(struct drm_device *dev) 409{ 410 struct drm_i915_private *dev_priv = dev->dev_private; 411 struct drm_i915_gem_object *obj; 412 413 /* First fill our portion of the GTT with scratch pages */ 414 i915_ggtt_clear_range(dev, dev_priv->mm.gtt_start / PAGE_SIZE, 415 (dev_priv->mm.gtt_end - dev_priv->mm.gtt_start) / PAGE_SIZE); 416 417 list_for_each_entry(obj, &dev_priv->mm.bound_list, gtt_list) { 418 i915_gem_clflush_object(obj); 419 i915_gem_gtt_bind_object(obj, obj->cache_level); 420 } 421 422 i915_gem_chipset_flush(dev); 423} 424 425int i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj) 426{ 427 if (obj->has_dma_mapping) 428 return 0; 429 430#ifdef FREEBSD_WIP 431 if (!dma_map_sg(&obj->base.dev->pdev->dev, 432 obj->pages->sgl, obj->pages->nents, 433 PCI_DMA_BIDIRECTIONAL)) 434 return -ENOSPC; 435#endif /* FREEBSD_WIP */ 436 437 return 0; 438} 439 440/* 441 * Binds an object into the global gtt with the specified cache level. The object 442 * will be accessible to the GPU via commands whose operands reference offsets 443 * within the global GTT as well as accessible by the GPU through the GMADR 444 * mapped BAR (dev_priv->mm.gtt->gtt). 445 */ 446static void gen6_ggtt_bind_object(struct drm_i915_gem_object *obj, 447 enum i915_cache_level level) 448{ 449 struct drm_device *dev = obj->base.dev; 450 struct drm_i915_private *dev_priv = dev->dev_private; 451 const int first_entry = obj->gtt_space->start >> PAGE_SHIFT; 452#if defined(INVARIANTS) 453 const int max_entries = dev_priv->mm.gtt->gtt_total_entries - first_entry; 454#endif 455 gtt_pte_t __iomem *gtt_entries = dev_priv->mm.gtt->gtt + first_entry; 456 int i = 0; 457 vm_paddr_t addr; 458 459 for (i = 0; i < obj->base.size >> PAGE_SHIFT; ++i) { 460 addr = VM_PAGE_TO_PHYS(obj->pages[i]); 461 iowrite32(pte_encode(dev, addr, level), >t_entries[i]); 462 } 463 464 BUG_ON(i > max_entries); 465 BUG_ON(i != obj->base.size / PAGE_SIZE); 466 467 /* XXX: This serves as a posting read to make sure that the PTE has 468 * actually been updated. There is some concern that even though 469 * registers and PTEs are within the same BAR that they are potentially 470 * of NUMA access patterns. Therefore, even with the way we assume 471 * hardware should work, we must keep this posting read for paranoia. 472 */ 473 if (i != 0) 474 WARN_ON(readl(>t_entries[i-1]) != pte_encode(dev, addr, level)); 475 476 /* This next bit makes the above posting read even more important. We 477 * want to flush the TLBs only after we're certain all the PTE updates 478 * have finished. 479 */ 480 I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN); 481 POSTING_READ(GFX_FLSH_CNTL_GEN6); 482} 483 484void i915_gem_gtt_bind_object(struct drm_i915_gem_object *obj, 485 enum i915_cache_level cache_level) 486{ 487 struct drm_device *dev = obj->base.dev; 488 if (INTEL_INFO(dev)->gen < 6) { 489 unsigned int flags = (cache_level == I915_CACHE_NONE) ? 490 AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY; 491 intel_gtt_insert_pages(obj->gtt_space->start >> PAGE_SHIFT, 492 obj->base.size >> PAGE_SHIFT, 493 obj->pages, 494 flags); 495 } else { 496 gen6_ggtt_bind_object(obj, cache_level); 497 } 498 499 obj->has_global_gtt_mapping = 1; 500} 501 502void i915_gem_gtt_unbind_object(struct drm_i915_gem_object *obj) 503{ 504 i915_ggtt_clear_range(obj->base.dev, 505 obj->gtt_space->start >> PAGE_SHIFT, 506 obj->base.size >> PAGE_SHIFT); 507 508 obj->has_global_gtt_mapping = 0; 509} 510 511void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj) 512{ 513 struct drm_device *dev = obj->base.dev; 514 struct drm_i915_private *dev_priv = dev->dev_private; 515 bool interruptible; 516 517 interruptible = do_idling(dev_priv); 518 519#ifdef FREEBSD_WIP 520 if (!obj->has_dma_mapping) 521 dma_unmap_sg(&dev->pdev->dev, 522 obj->pages->sgl, obj->pages->nents, 523 PCI_DMA_BIDIRECTIONAL); 524#endif /* FREEBSD_WIP */ 525 526 undo_idling(dev_priv, interruptible); 527} 528 529static void i915_gtt_color_adjust(struct drm_mm_node *node, 530 unsigned long color, 531 unsigned long *start, 532 unsigned long *end) 533{ 534 if (node->color != color) 535 *start += 4096; 536 537 if (!list_empty(&node->node_list)) { 538 node = list_entry(node->node_list.next, 539 struct drm_mm_node, 540 node_list); 541 if (node->allocated && node->color != color) 542 *end -= 4096; 543 } 544} 545 546void i915_gem_init_global_gtt(struct drm_device *dev, 547 unsigned long start, 548 unsigned long mappable_end, 549 unsigned long end) 550{ 551 drm_i915_private_t *dev_priv = dev->dev_private; 552 553 /* Subtract the guard page ... */ 554 drm_mm_init(&dev_priv->mm.gtt_space, start, end - start - PAGE_SIZE); 555 if (!HAS_LLC(dev)) 556 dev_priv->mm.gtt_space.color_adjust = i915_gtt_color_adjust; 557 558 dev_priv->mm.gtt_start = start; 559 dev_priv->mm.gtt_mappable_end = mappable_end; 560 dev_priv->mm.gtt_end = end; 561 dev_priv->mm.gtt_total = end - start; 562 dev_priv->mm.mappable_gtt_total = min(end, mappable_end) - start; 563 564 /* ... but ensure that we clear the entire range. */ 565 i915_ggtt_clear_range(dev, start / PAGE_SIZE, (end-start) / PAGE_SIZE); 566 567 device_printf(dev->dev, 568 "taking over the fictitious range 0x%jx-0x%jx\n", 569 (uintmax_t)(dev_priv->mm.gtt_base_addr + start), 570 (uintmax_t)(dev_priv->mm.gtt_base_addr + start + 571 dev_priv->mm.mappable_gtt_total)); 572 vm_phys_fictitious_reg_range(dev_priv->mm.gtt_base_addr + start, 573 dev_priv->mm.gtt_base_addr + start + dev_priv->mm.mappable_gtt_total, 574 VM_MEMATTR_WRITE_COMBINING); 575} 576 577static int setup_scratch_page(struct drm_device *dev) 578{ 579 struct drm_i915_private *dev_priv = dev->dev_private; 580 vm_page_t page; 581 dma_addr_t dma_addr; 582 int tries = 0; 583 int req = VM_ALLOC_ZERO | VM_ALLOC_NOOBJ; 584 585retry: 586 page = vm_page_alloc_contig(NULL, 0, req, 1, 0, 0xffffffff, 587 PAGE_SIZE, 0, VM_MEMATTR_UNCACHEABLE); 588 if (page == NULL) { 589 if (tries < 1) { 590 if (!vm_page_reclaim_contig(req, 1, 0, 0xffffffff, 591 PAGE_SIZE, 0)) 592 VM_WAIT; 593 tries++; 594 goto retry; 595 } 596 return -ENOMEM; 597 } 598 if ((page->flags & PG_ZERO) == 0) 599 pmap_zero_page(page); 600 601#ifdef CONFIG_INTEL_IOMMU 602 dma_addr = pci_map_page(dev->pdev, page, 0, PAGE_SIZE, 603 PCI_DMA_BIDIRECTIONAL); 604 if (pci_dma_mapping_error(dev->pdev, dma_addr)) 605 return -EINVAL; 606#else 607 dma_addr = VM_PAGE_TO_PHYS(page); 608#endif 609 dev_priv->mm.gtt->scratch_page = page; 610 dev_priv->mm.gtt->scratch_page_dma = dma_addr; 611 612 return 0; 613} 614 615static void teardown_scratch_page(struct drm_device *dev) 616{ 617#ifdef CONFIG_INTEL_IOMMU /* <- Added as a marker on FreeBSD. */ 618 struct drm_i915_private *dev_priv = dev->dev_private; 619 pci_unmap_page(dev->pdev, dev_priv->mm.gtt->scratch_page_dma, 620 PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); 621#endif 622} 623 624static inline unsigned int gen6_get_total_gtt_size(u16 snb_gmch_ctl) 625{ 626 snb_gmch_ctl >>= SNB_GMCH_GGMS_SHIFT; 627 snb_gmch_ctl &= SNB_GMCH_GGMS_MASK; 628 return snb_gmch_ctl << 20; 629} 630 631static inline unsigned int gen6_get_stolen_size(u16 snb_gmch_ctl) 632{ 633 snb_gmch_ctl >>= SNB_GMCH_GMS_SHIFT; 634 snb_gmch_ctl &= SNB_GMCH_GMS_MASK; 635 return snb_gmch_ctl << 25; /* 32 MB units */ 636} 637 638static inline unsigned int gen7_get_stolen_size(u16 snb_gmch_ctl) 639{ 640 static const int stolen_decoder[] = { 641 0, 0, 0, 0, 0, 32, 48, 64, 128, 256, 96, 160, 224, 352}; 642 snb_gmch_ctl >>= IVB_GMCH_GMS_SHIFT; 643 snb_gmch_ctl &= IVB_GMCH_GMS_MASK; 644 return stolen_decoder[snb_gmch_ctl] << 20; 645} 646 647int i915_gem_gtt_init(struct drm_device *dev) 648{ 649 struct drm_i915_private *dev_priv = dev->dev_private; 650 vm_paddr_t gtt_bus_addr; 651 u16 snb_gmch_ctl; 652 int ret; 653 654 /* On modern platforms we need not worry ourself with the legacy 655 * hostbridge query stuff. Skip it entirely 656 */ 657 if (INTEL_INFO(dev)->gen < 6) { 658#ifdef FREEBSD_WIP 659 ret = intel_gmch_probe(dev_priv->bridge_dev, dev->pdev, NULL); 660 if (!ret) { 661 DRM_ERROR("failed to set up gmch\n"); 662 return -EIO; 663 } 664#endif /* FREEBSD_WIP */ 665 666 dev_priv->mm.gtt = intel_gtt_get(); 667 if (!dev_priv->mm.gtt) { 668 DRM_ERROR("Failed to initialize GTT\n"); 669#ifdef FREEBSD_WIP 670 intel_gmch_remove(); 671#endif /* FREEBSD_WIP */ 672 return -ENODEV; 673 } 674 return 0; 675 } 676 677 dev_priv->mm.gtt = malloc(sizeof(*dev_priv->mm.gtt), DRM_I915_GEM, M_WAITOK | M_ZERO); 678 if (!dev_priv->mm.gtt) 679 return -ENOMEM; 680 681#ifdef FREEBSD_WIP 682 if (!pci_set_dma_mask(dev->pdev, DMA_BIT_MASK(40))) 683 pci_set_consistent_dma_mask(dev->pdev, DMA_BIT_MASK(40)); 684#endif /* FREEBSD_WIP */ 685 686#ifdef CONFIG_INTEL_IOMMU 687 dev_priv->mm.gtt->needs_dmar = 1; 688#endif 689 690 /* For GEN6+ the PTEs for the ggtt live at 2MB + BAR0 */ 691 gtt_bus_addr = drm_get_resource_start(dev, 0) + (2<<20); 692 dev_priv->mm.gtt->gma_bus_addr = drm_get_resource_start(dev, 2); 693 694 /* i9xx_setup */ 695 pci_read_config_word(dev->dev, SNB_GMCH_CTRL, &snb_gmch_ctl); 696 dev_priv->mm.gtt->gtt_total_entries = 697 gen6_get_total_gtt_size(snb_gmch_ctl) / sizeof(gtt_pte_t); 698 if (INTEL_INFO(dev)->gen < 7) 699 dev_priv->mm.gtt->stolen_size = gen6_get_stolen_size(snb_gmch_ctl); 700 else 701 dev_priv->mm.gtt->stolen_size = gen7_get_stolen_size(snb_gmch_ctl); 702 703 dev_priv->mm.gtt->gtt_mappable_entries = drm_get_resource_len(dev, 2) >> PAGE_SHIFT; 704 /* 64/512MB is the current min/max we actually know of, but this is just a 705 * coarse sanity check. 706 */ 707 if ((dev_priv->mm.gtt->gtt_mappable_entries >> 8) < 64 || 708 dev_priv->mm.gtt->gtt_mappable_entries > dev_priv->mm.gtt->gtt_total_entries) { 709 DRM_ERROR("Unknown GMADR entries (%d)\n", 710 dev_priv->mm.gtt->gtt_mappable_entries); 711 ret = -ENXIO; 712 goto err_out; 713 } 714 715 ret = setup_scratch_page(dev); 716 if (ret) { 717 DRM_ERROR("Scratch setup failed\n"); 718 goto err_out; 719 } 720 721 dev_priv->mm.gtt->gtt = pmap_mapdev_attr(gtt_bus_addr, 722 /* The size is used later by pmap_unmapdev. */ 723 dev_priv->mm.gtt->gtt_total_entries * sizeof(gtt_pte_t), 724 VM_MEMATTR_WRITE_COMBINING); 725 if (!dev_priv->mm.gtt->gtt) { 726 DRM_ERROR("Failed to map the gtt page table\n"); 727 teardown_scratch_page(dev); 728 ret = -ENOMEM; 729 goto err_out; 730 } 731 732 /* GMADR is the PCI aperture used by SW to access tiled GFX surfaces in a linear fashion. */ 733 DRM_INFO("Memory usable by graphics device = %dM\n", dev_priv->mm.gtt->gtt_total_entries >> 8); 734 DRM_DEBUG_DRIVER("GMADR size = %dM\n", dev_priv->mm.gtt->gtt_mappable_entries >> 8); 735 DRM_DEBUG_DRIVER("GTT stolen size = %dM\n", dev_priv->mm.gtt->stolen_size >> 20); 736 737 return 0; 738 739err_out: 740 free(dev_priv->mm.gtt, DRM_I915_GEM); 741#ifdef FREEBSD_WIP 742 if (INTEL_INFO(dev)->gen < 6) 743 intel_gmch_remove(); 744#endif /* FREEBSD_WIP */ 745 return ret; 746} 747 748void i915_gem_gtt_fini(struct drm_device *dev) 749{ 750 struct drm_i915_private *dev_priv = dev->dev_private; 751 pmap_unmapdev((vm_offset_t)dev_priv->mm.gtt->gtt, 752 dev_priv->mm.gtt->gtt_total_entries * sizeof(gtt_pte_t)); 753 teardown_scratch_page(dev); 754#ifdef FREEBSD_WIP 755 if (INTEL_INFO(dev)->gen < 6) 756 intel_gmch_remove(); 757#endif /* FREEBSD_WIP */ 758 if (INTEL_INFO(dev)->gen >= 6) 759 free(dev_priv->mm.gtt, DRM_I915_GEM); 760} 761