1235783Skib/* 2235783Skib * Copyright �� 2010 Daniel Vetter 3235783Skib * 4235783Skib * Permission is hereby granted, free of charge, to any person obtaining a 5235783Skib * copy of this software and associated documentation files (the "Software"), 6235783Skib * to deal in the Software without restriction, including without limitation 7235783Skib * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8235783Skib * and/or sell copies of the Software, and to permit persons to whom the 9235783Skib * Software is furnished to do so, subject to the following conditions: 10235783Skib * 11235783Skib * The above copyright notice and this permission notice (including the next 12235783Skib * paragraph) shall be included in all copies or substantial portions of the 13235783Skib * Software. 14235783Skib * 15235783Skib * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16235783Skib * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17235783Skib * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18235783Skib * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19235783Skib * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20235783Skib * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21235783Skib * IN THE SOFTWARE. 22235783Skib * 23235783Skib */ 24235783Skib 25235783Skib#include <sys/cdefs.h> 26235783Skib__FBSDID("$FreeBSD: stable/11/sys/dev/drm2/i915/i915_gem_gtt.c 324129 2017-09-30 18:52:59Z alc $"); 27235783Skib 28235783Skib#include <dev/drm2/drmP.h> 29235783Skib#include <dev/drm2/i915/i915_drm.h> 30235783Skib#include <dev/drm2/i915/i915_drv.h> 31235783Skib#include <dev/drm2/i915/intel_drv.h> 32235783Skib#include <sys/sched.h> 33235783Skib#include <sys/sf_buf.h> 34296548Sdumbbell#include <vm/vm_pageout.h> 35235783Skib 36296548Sdumbbelltypedef uint32_t gtt_pte_t; 37296548Sdumbbell 38296548Sdumbbell/* PPGTT stuff */ 39296548Sdumbbell#define GEN6_GTT_ADDR_ENCODE(addr) ((addr) | (((addr) >> 28) & 0xff0)) 40296548Sdumbbell 41296548Sdumbbell#define GEN6_PDE_VALID (1 << 0) 42296548Sdumbbell/* gen6+ has bit 11-4 for physical addr bit 39-32 */ 43296548Sdumbbell#define GEN6_PDE_ADDR_ENCODE(addr) GEN6_GTT_ADDR_ENCODE(addr) 44296548Sdumbbell 45296548Sdumbbell#define GEN6_PTE_VALID (1 << 0) 46296548Sdumbbell#define GEN6_PTE_UNCACHED (1 << 1) 47296548Sdumbbell#define HSW_PTE_UNCACHED (0) 48296548Sdumbbell#define GEN6_PTE_CACHE_LLC (2 << 1) 49296548Sdumbbell#define GEN6_PTE_CACHE_LLC_MLC (3 << 1) 50296548Sdumbbell#define GEN6_PTE_ADDR_ENCODE(addr) GEN6_GTT_ADDR_ENCODE(addr) 51296548Sdumbbell 52296548Sdumbbellstatic inline gtt_pte_t pte_encode(struct drm_device *dev, 53296548Sdumbbell dma_addr_t addr, 54296548Sdumbbell enum i915_cache_level level) 55296548Sdumbbell{ 56296548Sdumbbell gtt_pte_t pte = GEN6_PTE_VALID; 57296548Sdumbbell pte |= GEN6_PTE_ADDR_ENCODE(addr); 58296548Sdumbbell 59296548Sdumbbell switch (level) { 60296548Sdumbbell case I915_CACHE_LLC_MLC: 61296548Sdumbbell /* Haswell doesn't set L3 this way */ 62296548Sdumbbell if (IS_HASWELL(dev)) 63296548Sdumbbell pte |= GEN6_PTE_CACHE_LLC; 64296548Sdumbbell else 65296548Sdumbbell pte |= GEN6_PTE_CACHE_LLC_MLC; 66296548Sdumbbell break; 67296548Sdumbbell case I915_CACHE_LLC: 68296548Sdumbbell pte |= GEN6_PTE_CACHE_LLC; 69296548Sdumbbell break; 70296548Sdumbbell case I915_CACHE_NONE: 71296548Sdumbbell if (IS_HASWELL(dev)) 72296548Sdumbbell pte |= HSW_PTE_UNCACHED; 73296548Sdumbbell else 74296548Sdumbbell pte |= GEN6_PTE_UNCACHED; 75296548Sdumbbell break; 76296548Sdumbbell default: 77296548Sdumbbell BUG(); 78296548Sdumbbell } 79296548Sdumbbell 80296548Sdumbbell 81296548Sdumbbell return pte; 82296548Sdumbbell} 83296548Sdumbbell 84235783Skib/* PPGTT support for Sandybdrige/Gen6 and later */ 85287174Sbaptstatic void i915_ppgtt_clear_range(struct i915_hw_ppgtt *ppgtt, 86287174Sbapt unsigned first_entry, 87287174Sbapt unsigned num_entries) 88235783Skib{ 89296548Sdumbbell gtt_pte_t *pt_vaddr; 90296548Sdumbbell gtt_pte_t scratch_pte; 91296548Sdumbbell unsigned act_pd = first_entry / I915_PPGTT_PT_ENTRIES; 92296548Sdumbbell unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES; 93296548Sdumbbell unsigned last_pte, i; 94235783Skib struct sf_buf *sf; 95235783Skib 96296548Sdumbbell scratch_pte = pte_encode(ppgtt->dev, ppgtt->scratch_page_dma_addr, 97296548Sdumbbell I915_CACHE_LLC); 98235783Skib 99235783Skib while (num_entries) { 100235783Skib last_pte = first_pte + num_entries; 101235783Skib if (last_pte > I915_PPGTT_PT_ENTRIES) 102235783Skib last_pte = I915_PPGTT_PT_ENTRIES; 103235783Skib 104235783Skib sched_pin(); 105235783Skib sf = sf_buf_alloc(ppgtt->pt_pages[act_pd], SFB_CPUPRIVATE); 106235783Skib pt_vaddr = (uint32_t *)(uintptr_t)sf_buf_kva(sf); 107235783Skib 108235783Skib for (i = first_pte; i < last_pte; i++) 109235783Skib pt_vaddr[i] = scratch_pte; 110235783Skib 111235783Skib sf_buf_free(sf); 112235783Skib sched_unpin(); 113235783Skib 114235783Skib num_entries -= last_pte - first_pte; 115235783Skib first_pte = 0; 116235783Skib act_pd++; 117235783Skib } 118235783Skib} 119235783Skib 120287174Sbaptint i915_gem_init_aliasing_ppgtt(struct drm_device *dev) 121235783Skib{ 122287174Sbapt struct drm_i915_private *dev_priv = dev->dev_private; 123235783Skib struct i915_hw_ppgtt *ppgtt; 124287174Sbapt unsigned first_pd_entry_in_global_pt; 125287174Sbapt int i; 126296548Sdumbbell int ret = -ENOMEM; 127235783Skib 128287174Sbapt /* ppgtt PDEs reside in the global gtt pagetable, which has 512*1024 129235783Skib * entries. For aliasing ppgtt support we just steal them at the end for 130296548Sdumbbell * now. */ 131296548Sdumbbell first_pd_entry_in_global_pt = dev_priv->mm.gtt->gtt_total_entries - I915_PPGTT_PD_ENTRIES; 132235783Skib 133235783Skib ppgtt = malloc(sizeof(*ppgtt), DRM_I915_GEM, M_WAITOK | M_ZERO); 134296548Sdumbbell if (!ppgtt) 135296548Sdumbbell return ret; 136235783Skib 137296548Sdumbbell ppgtt->dev = dev; 138235783Skib ppgtt->num_pd_entries = I915_PPGTT_PD_ENTRIES; 139296548Sdumbbell ppgtt->pt_pages = malloc(sizeof(struct page *)*ppgtt->num_pd_entries, 140296548Sdumbbell DRM_I915_GEM, M_WAITOK | M_ZERO); 141296548Sdumbbell if (!ppgtt->pt_pages) 142296548Sdumbbell goto err_ppgtt; 143235783Skib 144235783Skib for (i = 0; i < ppgtt->num_pd_entries; i++) { 145235783Skib ppgtt->pt_pages[i] = vm_page_alloc(NULL, 0, 146235783Skib VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | 147235783Skib VM_ALLOC_ZERO); 148296548Sdumbbell if (!ppgtt->pt_pages[i]) 149296548Sdumbbell goto err_pt_alloc; 150296548Sdumbbell } 151296548Sdumbbell 152296548Sdumbbell if (dev_priv->mm.gtt->needs_dmar) { 153296548Sdumbbell ppgtt->pt_dma_addr = malloc(sizeof(dma_addr_t) 154296548Sdumbbell *ppgtt->num_pd_entries, 155296548Sdumbbell DRM_I915_GEM, M_WAITOK | M_ZERO); 156296548Sdumbbell if (!ppgtt->pt_dma_addr) 157296548Sdumbbell goto err_pt_alloc; 158296548Sdumbbell 159296548Sdumbbell#ifdef CONFIG_INTEL_IOMMU /* <- Added as a marker on FreeBSD. */ 160296548Sdumbbell for (i = 0; i < ppgtt->num_pd_entries; i++) { 161296548Sdumbbell dma_addr_t pt_addr; 162296548Sdumbbell 163296548Sdumbbell pt_addr = pci_map_page(dev->pdev, ppgtt->pt_pages[i], 164296548Sdumbbell 0, 4096, 165296548Sdumbbell PCI_DMA_BIDIRECTIONAL); 166296548Sdumbbell 167296548Sdumbbell if (pci_dma_mapping_error(dev->pdev, 168296548Sdumbbell pt_addr)) { 169296548Sdumbbell ret = -EIO; 170296548Sdumbbell goto err_pd_pin; 171296548Sdumbbell 172296548Sdumbbell } 173296548Sdumbbell ppgtt->pt_dma_addr[i] = pt_addr; 174235783Skib } 175296548Sdumbbell#endif 176235783Skib } 177235783Skib 178296548Sdumbbell ppgtt->scratch_page_dma_addr = dev_priv->mm.gtt->scratch_page_dma; 179235783Skib 180296548Sdumbbell i915_ppgtt_clear_range(ppgtt, 0, 181296548Sdumbbell ppgtt->num_pd_entries*I915_PPGTT_PT_ENTRIES); 182296548Sdumbbell 183296548Sdumbbell ppgtt->pd_offset = (first_pd_entry_in_global_pt)*sizeof(gtt_pte_t); 184296548Sdumbbell 185235783Skib dev_priv->mm.aliasing_ppgtt = ppgtt; 186293837Sdumbbell 187293837Sdumbbell return 0; 188296548Sdumbbell 189296548Sdumbbell#ifdef CONFIG_INTEL_IOMMU /* <- Added as a marker on FreeBSD. */ 190296548Sdumbbellerr_pd_pin: 191296548Sdumbbell if (ppgtt->pt_dma_addr) { 192296548Sdumbbell for (i--; i >= 0; i--) 193296548Sdumbbell pci_unmap_page(dev->pdev, ppgtt->pt_dma_addr[i], 194296548Sdumbbell 4096, PCI_DMA_BIDIRECTIONAL); 195296548Sdumbbell } 196296548Sdumbbell#endif 197296548Sdumbbellerr_pt_alloc: 198296548Sdumbbell free(ppgtt->pt_dma_addr, DRM_I915_GEM); 199296548Sdumbbell for (i = 0; i < ppgtt->num_pd_entries; i++) { 200296548Sdumbbell if (ppgtt->pt_pages[i]) { 201324129Salc vm_page_unwire(ppgtt->pt_pages[i], PQ_NONE); 202296548Sdumbbell vm_page_free(ppgtt->pt_pages[i]); 203296548Sdumbbell } 204296548Sdumbbell } 205296548Sdumbbell free(ppgtt->pt_pages, DRM_I915_GEM); 206296548Sdumbbellerr_ppgtt: 207296548Sdumbbell free(ppgtt, DRM_I915_GEM); 208296548Sdumbbell 209296548Sdumbbell return ret; 210235783Skib} 211235783Skib 212296548Sdumbbellvoid i915_gem_cleanup_aliasing_ppgtt(struct drm_device *dev) 213296548Sdumbbell{ 214296548Sdumbbell struct drm_i915_private *dev_priv = dev->dev_private; 215296548Sdumbbell struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt; 216296548Sdumbbell int i; 217296548Sdumbbell 218296548Sdumbbell if (!ppgtt) 219296548Sdumbbell return; 220296548Sdumbbell 221296548Sdumbbell#ifdef CONFIG_INTEL_IOMMU /* <- Added as a marker on FreeBSD. */ 222296548Sdumbbell if (ppgtt->pt_dma_addr) { 223296548Sdumbbell for (i = 0; i < ppgtt->num_pd_entries; i++) 224296548Sdumbbell pci_unmap_page(dev->pdev, ppgtt->pt_dma_addr[i], 225296548Sdumbbell 4096, PCI_DMA_BIDIRECTIONAL); 226296548Sdumbbell } 227296548Sdumbbell#endif 228296548Sdumbbell 229296548Sdumbbell free(ppgtt->pt_dma_addr, DRM_I915_GEM); 230296548Sdumbbell for (i = 0; i < ppgtt->num_pd_entries; i++) { 231324129Salc vm_page_unwire(ppgtt->pt_pages[i], PQ_NONE); 232296548Sdumbbell vm_page_free(ppgtt->pt_pages[i]); 233296548Sdumbbell } 234296548Sdumbbell free(ppgtt->pt_pages, DRM_I915_GEM); 235296548Sdumbbell free(ppgtt, DRM_I915_GEM); 236296548Sdumbbell} 237296548Sdumbbell 238293837Sdumbbellstatic void i915_ppgtt_insert_pages(struct i915_hw_ppgtt *ppgtt, 239296548Sdumbbell vm_page_t *pages, 240293837Sdumbbell unsigned first_entry, 241293837Sdumbbell unsigned num_entries, 242296548Sdumbbell enum i915_cache_level cache_level) 243235783Skib{ 244296548Sdumbbell uint32_t *pt_vaddr; 245293837Sdumbbell unsigned act_pd = first_entry / I915_PPGTT_PT_ENTRIES; 246293837Sdumbbell unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES; 247293837Sdumbbell unsigned j, last_pte; 248293837Sdumbbell vm_paddr_t page_addr; 249235783Skib struct sf_buf *sf; 250235783Skib 251235783Skib while (num_entries) { 252235783Skib last_pte = first_pte + num_entries; 253235783Skib if (last_pte > I915_PPGTT_PT_ENTRIES) 254235783Skib last_pte = I915_PPGTT_PT_ENTRIES; 255235783Skib 256235783Skib sched_pin(); 257235783Skib sf = sf_buf_alloc(ppgtt->pt_pages[act_pd], SFB_CPUPRIVATE); 258235783Skib pt_vaddr = (uint32_t *)(uintptr_t)sf_buf_kva(sf); 259235783Skib 260293837Sdumbbell for (j = first_pte; j < last_pte; j++) { 261235783Skib page_addr = VM_PAGE_TO_PHYS(*pages); 262296548Sdumbbell pt_vaddr[j] = pte_encode(ppgtt->dev, page_addr, 263296548Sdumbbell cache_level); 264235783Skib 265235783Skib pages++; 266235783Skib } 267235783Skib 268235783Skib sf_buf_free(sf); 269235783Skib sched_unpin(); 270235783Skib 271235783Skib num_entries -= last_pte - first_pte; 272235783Skib first_pte = 0; 273235783Skib act_pd++; 274235783Skib } 275235783Skib} 276235783Skib 277287174Sbaptvoid i915_ppgtt_bind_object(struct i915_hw_ppgtt *ppgtt, 278287174Sbapt struct drm_i915_gem_object *obj, 279287174Sbapt enum i915_cache_level cache_level) 280235783Skib{ 281296548Sdumbbell i915_ppgtt_insert_pages(ppgtt, 282296548Sdumbbell obj->pages, 283296548Sdumbbell obj->gtt_space->start >> PAGE_SHIFT, 284296548Sdumbbell obj->base.size >> PAGE_SHIFT, 285296548Sdumbbell cache_level); 286235783Skib} 287235783Skib 288235783Skibvoid i915_ppgtt_unbind_object(struct i915_hw_ppgtt *ppgtt, 289235783Skib struct drm_i915_gem_object *obj) 290235783Skib{ 291287174Sbapt i915_ppgtt_clear_range(ppgtt, 292287174Sbapt obj->gtt_space->start >> PAGE_SHIFT, 293287174Sbapt obj->base.size >> PAGE_SHIFT); 294235783Skib} 295235783Skib 296285988Sdumbbellvoid i915_gem_init_ppgtt(struct drm_device *dev) 297285988Sdumbbell{ 298287174Sbapt drm_i915_private_t *dev_priv = dev->dev_private; 299287174Sbapt uint32_t pd_offset; 300287174Sbapt struct intel_ring_buffer *ring; 301287174Sbapt struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt; 302296548Sdumbbell uint32_t __iomem *pd_addr; 303287174Sbapt uint32_t pd_entry; 304287174Sbapt int i; 305285988Sdumbbell 306287174Sbapt if (!dev_priv->mm.aliasing_ppgtt) 307285988Sdumbbell return; 308285988Sdumbbell 309293837Sdumbbell 310296548Sdumbbell pd_addr = dev_priv->mm.gtt->gtt + ppgtt->pd_offset/sizeof(uint32_t); 311285988Sdumbbell for (i = 0; i < ppgtt->num_pd_entries; i++) { 312293837Sdumbbell vm_paddr_t pt_addr; 313293837Sdumbbell 314296548Sdumbbell if (dev_priv->mm.gtt->needs_dmar) 315296548Sdumbbell pt_addr = ppgtt->pt_dma_addr[i]; 316296548Sdumbbell else 317296548Sdumbbell pt_addr = VM_PAGE_TO_PHYS(ppgtt->pt_pages[i]); 318296548Sdumbbell 319285988Sdumbbell pd_entry = GEN6_PDE_ADDR_ENCODE(pt_addr); 320285988Sdumbbell pd_entry |= GEN6_PDE_VALID; 321293837Sdumbbell 322296548Sdumbbell /* NOTE Linux<->FreeBSD: Arguments of writel() are reversed. */ 323296548Sdumbbell writel(pd_addr + i, pd_entry); 324285988Sdumbbell } 325296548Sdumbbell readl(pd_addr); 326285988Sdumbbell 327285988Sdumbbell pd_offset = ppgtt->pd_offset; 328285988Sdumbbell pd_offset /= 64; /* in cachelines, */ 329285988Sdumbbell pd_offset <<= 16; 330285988Sdumbbell 331285988Sdumbbell if (INTEL_INFO(dev)->gen == 6) { 332285988Sdumbbell uint32_t ecochk, gab_ctl, ecobits; 333285988Sdumbbell 334293837Sdumbbell ecobits = I915_READ(GAC_ECO_BITS); 335285988Sdumbbell I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_PPGTT_CACHE64B); 336285988Sdumbbell 337285988Sdumbbell gab_ctl = I915_READ(GAB_CTL); 338285988Sdumbbell I915_WRITE(GAB_CTL, gab_ctl | GAB_CTL_CONT_AFTER_PAGEFAULT); 339285988Sdumbbell 340285988Sdumbbell ecochk = I915_READ(GAM_ECOCHK); 341285988Sdumbbell I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT | 342285988Sdumbbell ECOCHK_PPGTT_CACHE64B); 343285988Sdumbbell I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE)); 344285988Sdumbbell } else if (INTEL_INFO(dev)->gen >= 7) { 345285988Sdumbbell I915_WRITE(GAM_ECOCHK, ECOCHK_PPGTT_CACHE64B); 346285988Sdumbbell /* GFX_MODE is per-ring on gen7+ */ 347285988Sdumbbell } 348285988Sdumbbell 349285988Sdumbbell for_each_ring(ring, dev_priv, i) { 350285988Sdumbbell if (INTEL_INFO(dev)->gen >= 7) 351285988Sdumbbell I915_WRITE(RING_MODE_GEN7(ring), 352285988Sdumbbell _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE)); 353285988Sdumbbell 354285988Sdumbbell I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G); 355285988Sdumbbell I915_WRITE(RING_PP_DIR_BASE(ring), pd_offset); 356285988Sdumbbell } 357285988Sdumbbell} 358285988Sdumbbell 359287174Sbaptstatic bool do_idling(struct drm_i915_private *dev_priv) 360287174Sbapt{ 361287174Sbapt bool ret = dev_priv->mm.interruptible; 362287174Sbapt 363296548Sdumbbell if (unlikely(dev_priv->mm.gtt->do_idle_maps)) { 364287174Sbapt dev_priv->mm.interruptible = false; 365287174Sbapt if (i915_gpu_idle(dev_priv->dev)) { 366287174Sbapt DRM_ERROR("Couldn't idle GPU\n"); 367287174Sbapt /* Wait a bit, in hopes it avoids the hang */ 368296548Sdumbbell udelay(10); 369287174Sbapt } 370287174Sbapt } 371287174Sbapt 372287174Sbapt return ret; 373287174Sbapt} 374287174Sbapt 375287174Sbaptstatic void undo_idling(struct drm_i915_private *dev_priv, bool interruptible) 376287174Sbapt{ 377296548Sdumbbell if (unlikely(dev_priv->mm.gtt->do_idle_maps)) 378287174Sbapt dev_priv->mm.interruptible = interruptible; 379287174Sbapt} 380287174Sbapt 381296548Sdumbbell 382296548Sdumbbellstatic void i915_ggtt_clear_range(struct drm_device *dev, 383296548Sdumbbell unsigned first_entry, 384296548Sdumbbell unsigned num_entries) 385235783Skib{ 386296548Sdumbbell struct drm_i915_private *dev_priv = dev->dev_private; 387296548Sdumbbell gtt_pte_t scratch_pte; 388296548Sdumbbell gtt_pte_t __iomem *gtt_base = dev_priv->mm.gtt->gtt + first_entry; 389296548Sdumbbell const int max_entries = dev_priv->mm.gtt->gtt_total_entries - first_entry; 390235783Skib int i; 391235783Skib 392296548Sdumbbell if (INTEL_INFO(dev)->gen < 6) { 393296548Sdumbbell intel_gtt_clear_range(first_entry, num_entries); 394235783Skib return; 395235783Skib } 396235783Skib 397296548Sdumbbell if (WARN(num_entries > max_entries, 398296548Sdumbbell "First entry = %d; Num entries = %d (max=%d)\n", 399296548Sdumbbell first_entry, num_entries, max_entries)) 400296548Sdumbbell num_entries = max_entries; 401235783Skib 402296548Sdumbbell scratch_pte = pte_encode(dev, dev_priv->mm.gtt->scratch_page_dma, I915_CACHE_LLC); 403296548Sdumbbell for (i = 0; i < num_entries; i++) 404296548Sdumbbell iowrite32(scratch_pte, >t_base[i]); 405296548Sdumbbell readl(gtt_base); 406235783Skib} 407235783Skib 408287174Sbaptvoid i915_gem_restore_gtt_mappings(struct drm_device *dev) 409235783Skib{ 410287174Sbapt struct drm_i915_private *dev_priv = dev->dev_private; 411235783Skib struct drm_i915_gem_object *obj; 412235783Skib 413235783Skib /* First fill our portion of the GTT with scratch pages */ 414296548Sdumbbell i915_ggtt_clear_range(dev, dev_priv->mm.gtt_start / PAGE_SIZE, 415287174Sbapt (dev_priv->mm.gtt_end - dev_priv->mm.gtt_start) / PAGE_SIZE); 416235783Skib 417296548Sdumbbell list_for_each_entry(obj, &dev_priv->mm.bound_list, gtt_list) { 418235783Skib i915_gem_clflush_object(obj); 419277487Skib i915_gem_gtt_bind_object(obj, obj->cache_level); 420235783Skib } 421235783Skib 422296548Sdumbbell i915_gem_chipset_flush(dev); 423235783Skib} 424235783Skib 425287174Sbaptint i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj) 426235783Skib{ 427296548Sdumbbell if (obj->has_dma_mapping) 428296548Sdumbbell return 0; 429235783Skib 430296548Sdumbbell#ifdef FREEBSD_WIP 431296548Sdumbbell if (!dma_map_sg(&obj->base.dev->pdev->dev, 432296548Sdumbbell obj->pages->sgl, obj->pages->nents, 433296548Sdumbbell PCI_DMA_BIDIRECTIONAL)) 434296548Sdumbbell return -ENOSPC; 435296548Sdumbbell#endif /* FREEBSD_WIP */ 436296548Sdumbbell 437287174Sbapt return 0; 438235783Skib} 439235783Skib 440296548Sdumbbell/* 441296548Sdumbbell * Binds an object into the global gtt with the specified cache level. The object 442296548Sdumbbell * will be accessible to the GPU via commands whose operands reference offsets 443296548Sdumbbell * within the global GTT as well as accessible by the GPU through the GMADR 444296548Sdumbbell * mapped BAR (dev_priv->mm.gtt->gtt). 445296548Sdumbbell */ 446296548Sdumbbellstatic void gen6_ggtt_bind_object(struct drm_i915_gem_object *obj, 447296548Sdumbbell enum i915_cache_level level) 448296548Sdumbbell{ 449296548Sdumbbell struct drm_device *dev = obj->base.dev; 450296548Sdumbbell struct drm_i915_private *dev_priv = dev->dev_private; 451296548Sdumbbell const int first_entry = obj->gtt_space->start >> PAGE_SHIFT; 452296548Sdumbbell#if defined(INVARIANTS) 453296548Sdumbbell const int max_entries = dev_priv->mm.gtt->gtt_total_entries - first_entry; 454296548Sdumbbell#endif 455296548Sdumbbell gtt_pte_t __iomem *gtt_entries = dev_priv->mm.gtt->gtt + first_entry; 456296548Sdumbbell int i = 0; 457296548Sdumbbell vm_paddr_t addr; 458296548Sdumbbell 459296548Sdumbbell for (i = 0; i < obj->base.size >> PAGE_SHIFT; ++i) { 460296548Sdumbbell addr = VM_PAGE_TO_PHYS(obj->pages[i]); 461296548Sdumbbell iowrite32(pte_encode(dev, addr, level), >t_entries[i]); 462296548Sdumbbell } 463296548Sdumbbell 464296548Sdumbbell BUG_ON(i > max_entries); 465296548Sdumbbell BUG_ON(i != obj->base.size / PAGE_SIZE); 466296548Sdumbbell 467296548Sdumbbell /* XXX: This serves as a posting read to make sure that the PTE has 468296548Sdumbbell * actually been updated. There is some concern that even though 469296548Sdumbbell * registers and PTEs are within the same BAR that they are potentially 470296548Sdumbbell * of NUMA access patterns. Therefore, even with the way we assume 471296548Sdumbbell * hardware should work, we must keep this posting read for paranoia. 472296548Sdumbbell */ 473296548Sdumbbell if (i != 0) 474296548Sdumbbell WARN_ON(readl(>t_entries[i-1]) != pte_encode(dev, addr, level)); 475296548Sdumbbell 476296548Sdumbbell /* This next bit makes the above posting read even more important. We 477296548Sdumbbell * want to flush the TLBs only after we're certain all the PTE updates 478296548Sdumbbell * have finished. 479296548Sdumbbell */ 480296548Sdumbbell I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN); 481296548Sdumbbell POSTING_READ(GFX_FLSH_CNTL_GEN6); 482296548Sdumbbell} 483296548Sdumbbell 484293837Sdumbbellvoid i915_gem_gtt_bind_object(struct drm_i915_gem_object *obj, 485293837Sdumbbell enum i915_cache_level cache_level) 486235783Skib{ 487296548Sdumbbell struct drm_device *dev = obj->base.dev; 488296548Sdumbbell if (INTEL_INFO(dev)->gen < 6) { 489296548Sdumbbell unsigned int flags = (cache_level == I915_CACHE_NONE) ? 490296548Sdumbbell AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY; 491296548Sdumbbell intel_gtt_insert_pages(obj->gtt_space->start >> PAGE_SHIFT, 492296548Sdumbbell obj->base.size >> PAGE_SHIFT, 493296548Sdumbbell obj->pages, 494296548Sdumbbell flags); 495296548Sdumbbell } else { 496296548Sdumbbell gen6_ggtt_bind_object(obj, cache_level); 497296548Sdumbbell } 498235783Skib 499277487Skib obj->has_global_gtt_mapping = 1; 500235783Skib} 501235783Skib 502287174Sbaptvoid i915_gem_gtt_unbind_object(struct drm_i915_gem_object *obj) 503235783Skib{ 504296548Sdumbbell i915_ggtt_clear_range(obj->base.dev, 505296548Sdumbbell obj->gtt_space->start >> PAGE_SHIFT, 506296548Sdumbbell obj->base.size >> PAGE_SHIFT); 507277487Skib 508277487Skib obj->has_global_gtt_mapping = 0; 509277487Skib} 510277487Skib 511287174Sbaptvoid i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj) 512277487Skib{ 513235783Skib struct drm_device *dev = obj->base.dev; 514235783Skib struct drm_i915_private *dev_priv = dev->dev_private; 515235783Skib bool interruptible; 516235783Skib 517235783Skib interruptible = do_idling(dev_priv); 518235783Skib 519296548Sdumbbell#ifdef FREEBSD_WIP 520296548Sdumbbell if (!obj->has_dma_mapping) 521296548Sdumbbell dma_unmap_sg(&dev->pdev->dev, 522296548Sdumbbell obj->pages->sgl, obj->pages->nents, 523296548Sdumbbell PCI_DMA_BIDIRECTIONAL); 524296548Sdumbbell#endif /* FREEBSD_WIP */ 525296548Sdumbbell 526235783Skib undo_idling(dev_priv, interruptible); 527235783Skib} 528277487Skib 529296548Sdumbbellstatic void i915_gtt_color_adjust(struct drm_mm_node *node, 530296548Sdumbbell unsigned long color, 531296548Sdumbbell unsigned long *start, 532296548Sdumbbell unsigned long *end) 533296548Sdumbbell{ 534296548Sdumbbell if (node->color != color) 535296548Sdumbbell *start += 4096; 536296548Sdumbbell 537296548Sdumbbell if (!list_empty(&node->node_list)) { 538296548Sdumbbell node = list_entry(node->node_list.next, 539296548Sdumbbell struct drm_mm_node, 540296548Sdumbbell node_list); 541296548Sdumbbell if (node->allocated && node->color != color) 542296548Sdumbbell *end -= 4096; 543296548Sdumbbell } 544296548Sdumbbell} 545296548Sdumbbell 546296548Sdumbbellvoid i915_gem_init_global_gtt(struct drm_device *dev, 547293837Sdumbbell unsigned long start, 548293837Sdumbbell unsigned long mappable_end, 549293837Sdumbbell unsigned long end) 550277487Skib{ 551293837Sdumbbell drm_i915_private_t *dev_priv = dev->dev_private; 552277487Skib 553298955Spfg /* Subtract the guard page ... */ 554277487Skib drm_mm_init(&dev_priv->mm.gtt_space, start, end - start - PAGE_SIZE); 555296548Sdumbbell if (!HAS_LLC(dev)) 556296548Sdumbbell dev_priv->mm.gtt_space.color_adjust = i915_gtt_color_adjust; 557277487Skib 558277487Skib dev_priv->mm.gtt_start = start; 559277487Skib dev_priv->mm.gtt_mappable_end = mappable_end; 560277487Skib dev_priv->mm.gtt_end = end; 561277487Skib dev_priv->mm.gtt_total = end - start; 562296548Sdumbbell dev_priv->mm.mappable_gtt_total = min(end, mappable_end) - start; 563277487Skib 564277487Skib /* ... but ensure that we clear the entire range. */ 565296548Sdumbbell i915_ggtt_clear_range(dev, start / PAGE_SIZE, (end-start) / PAGE_SIZE); 566296548Sdumbbell 567280183Sdumbbell device_printf(dev->dev, 568296862Sbz "taking over the fictitious range 0x%jx-0x%jx\n", 569296862Sbz (uintmax_t)(dev_priv->mm.gtt_base_addr + start), 570296862Sbz (uintmax_t)(dev_priv->mm.gtt_base_addr + start + 571296862Sbz dev_priv->mm.mappable_gtt_total)); 572296548Sdumbbell vm_phys_fictitious_reg_range(dev_priv->mm.gtt_base_addr + start, 573296548Sdumbbell dev_priv->mm.gtt_base_addr + start + dev_priv->mm.mappable_gtt_total, 574296548Sdumbbell VM_MEMATTR_WRITE_COMBINING); 575277487Skib} 576296548Sdumbbell 577296548Sdumbbellstatic int setup_scratch_page(struct drm_device *dev) 578296548Sdumbbell{ 579296548Sdumbbell struct drm_i915_private *dev_priv = dev->dev_private; 580296548Sdumbbell vm_page_t page; 581296548Sdumbbell dma_addr_t dma_addr; 582296548Sdumbbell int tries = 0; 583296548Sdumbbell int req = VM_ALLOC_ZERO | VM_ALLOC_NOOBJ; 584296548Sdumbbell 585296548Sdumbbellretry: 586296548Sdumbbell page = vm_page_alloc_contig(NULL, 0, req, 1, 0, 0xffffffff, 587296548Sdumbbell PAGE_SIZE, 0, VM_MEMATTR_UNCACHEABLE); 588296548Sdumbbell if (page == NULL) { 589296548Sdumbbell if (tries < 1) { 590296548Sdumbbell if (!vm_page_reclaim_contig(req, 1, 0, 0xffffffff, 591296548Sdumbbell PAGE_SIZE, 0)) 592296548Sdumbbell VM_WAIT; 593296548Sdumbbell tries++; 594296548Sdumbbell goto retry; 595296548Sdumbbell } 596296548Sdumbbell return -ENOMEM; 597296548Sdumbbell } 598296548Sdumbbell if ((page->flags & PG_ZERO) == 0) 599296548Sdumbbell pmap_zero_page(page); 600296548Sdumbbell 601296548Sdumbbell#ifdef CONFIG_INTEL_IOMMU 602296548Sdumbbell dma_addr = pci_map_page(dev->pdev, page, 0, PAGE_SIZE, 603296548Sdumbbell PCI_DMA_BIDIRECTIONAL); 604296548Sdumbbell if (pci_dma_mapping_error(dev->pdev, dma_addr)) 605296548Sdumbbell return -EINVAL; 606296548Sdumbbell#else 607296548Sdumbbell dma_addr = VM_PAGE_TO_PHYS(page); 608296548Sdumbbell#endif 609296548Sdumbbell dev_priv->mm.gtt->scratch_page = page; 610296548Sdumbbell dev_priv->mm.gtt->scratch_page_dma = dma_addr; 611296548Sdumbbell 612296548Sdumbbell return 0; 613296548Sdumbbell} 614296548Sdumbbell 615296548Sdumbbellstatic void teardown_scratch_page(struct drm_device *dev) 616296548Sdumbbell{ 617296548Sdumbbell#ifdef CONFIG_INTEL_IOMMU /* <- Added as a marker on FreeBSD. */ 618296548Sdumbbell struct drm_i915_private *dev_priv = dev->dev_private; 619296548Sdumbbell pci_unmap_page(dev->pdev, dev_priv->mm.gtt->scratch_page_dma, 620296548Sdumbbell PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); 621296548Sdumbbell#endif 622296548Sdumbbell} 623296548Sdumbbell 624296548Sdumbbellstatic inline unsigned int gen6_get_total_gtt_size(u16 snb_gmch_ctl) 625296548Sdumbbell{ 626296548Sdumbbell snb_gmch_ctl >>= SNB_GMCH_GGMS_SHIFT; 627296548Sdumbbell snb_gmch_ctl &= SNB_GMCH_GGMS_MASK; 628296548Sdumbbell return snb_gmch_ctl << 20; 629296548Sdumbbell} 630296548Sdumbbell 631296548Sdumbbellstatic inline unsigned int gen6_get_stolen_size(u16 snb_gmch_ctl) 632296548Sdumbbell{ 633296548Sdumbbell snb_gmch_ctl >>= SNB_GMCH_GMS_SHIFT; 634296548Sdumbbell snb_gmch_ctl &= SNB_GMCH_GMS_MASK; 635296548Sdumbbell return snb_gmch_ctl << 25; /* 32 MB units */ 636296548Sdumbbell} 637296548Sdumbbell 638296548Sdumbbellstatic inline unsigned int gen7_get_stolen_size(u16 snb_gmch_ctl) 639296548Sdumbbell{ 640296548Sdumbbell static const int stolen_decoder[] = { 641296548Sdumbbell 0, 0, 0, 0, 0, 32, 48, 64, 128, 256, 96, 160, 224, 352}; 642296548Sdumbbell snb_gmch_ctl >>= IVB_GMCH_GMS_SHIFT; 643296548Sdumbbell snb_gmch_ctl &= IVB_GMCH_GMS_MASK; 644296548Sdumbbell return stolen_decoder[snb_gmch_ctl] << 20; 645296548Sdumbbell} 646296548Sdumbbell 647296548Sdumbbellint i915_gem_gtt_init(struct drm_device *dev) 648296548Sdumbbell{ 649296548Sdumbbell struct drm_i915_private *dev_priv = dev->dev_private; 650296548Sdumbbell vm_paddr_t gtt_bus_addr; 651296548Sdumbbell u16 snb_gmch_ctl; 652296548Sdumbbell int ret; 653296548Sdumbbell 654296548Sdumbbell /* On modern platforms we need not worry ourself with the legacy 655296548Sdumbbell * hostbridge query stuff. Skip it entirely 656296548Sdumbbell */ 657296548Sdumbbell if (INTEL_INFO(dev)->gen < 6) { 658296548Sdumbbell#ifdef FREEBSD_WIP 659296548Sdumbbell ret = intel_gmch_probe(dev_priv->bridge_dev, dev->pdev, NULL); 660296548Sdumbbell if (!ret) { 661296548Sdumbbell DRM_ERROR("failed to set up gmch\n"); 662296548Sdumbbell return -EIO; 663296548Sdumbbell } 664296548Sdumbbell#endif /* FREEBSD_WIP */ 665296548Sdumbbell 666296548Sdumbbell dev_priv->mm.gtt = intel_gtt_get(); 667296548Sdumbbell if (!dev_priv->mm.gtt) { 668296548Sdumbbell DRM_ERROR("Failed to initialize GTT\n"); 669296548Sdumbbell#ifdef FREEBSD_WIP 670296548Sdumbbell intel_gmch_remove(); 671296548Sdumbbell#endif /* FREEBSD_WIP */ 672296548Sdumbbell return -ENODEV; 673296548Sdumbbell } 674296548Sdumbbell return 0; 675296548Sdumbbell } 676296548Sdumbbell 677296548Sdumbbell dev_priv->mm.gtt = malloc(sizeof(*dev_priv->mm.gtt), DRM_I915_GEM, M_WAITOK | M_ZERO); 678296548Sdumbbell if (!dev_priv->mm.gtt) 679296548Sdumbbell return -ENOMEM; 680296548Sdumbbell 681296548Sdumbbell#ifdef FREEBSD_WIP 682296548Sdumbbell if (!pci_set_dma_mask(dev->pdev, DMA_BIT_MASK(40))) 683296548Sdumbbell pci_set_consistent_dma_mask(dev->pdev, DMA_BIT_MASK(40)); 684296548Sdumbbell#endif /* FREEBSD_WIP */ 685296548Sdumbbell 686296548Sdumbbell#ifdef CONFIG_INTEL_IOMMU 687296548Sdumbbell dev_priv->mm.gtt->needs_dmar = 1; 688296548Sdumbbell#endif 689296548Sdumbbell 690296548Sdumbbell /* For GEN6+ the PTEs for the ggtt live at 2MB + BAR0 */ 691296548Sdumbbell gtt_bus_addr = drm_get_resource_start(dev, 0) + (2<<20); 692296548Sdumbbell dev_priv->mm.gtt->gma_bus_addr = drm_get_resource_start(dev, 2); 693296548Sdumbbell 694296548Sdumbbell /* i9xx_setup */ 695296548Sdumbbell pci_read_config_word(dev->dev, SNB_GMCH_CTRL, &snb_gmch_ctl); 696296548Sdumbbell dev_priv->mm.gtt->gtt_total_entries = 697296548Sdumbbell gen6_get_total_gtt_size(snb_gmch_ctl) / sizeof(gtt_pte_t); 698296548Sdumbbell if (INTEL_INFO(dev)->gen < 7) 699296548Sdumbbell dev_priv->mm.gtt->stolen_size = gen6_get_stolen_size(snb_gmch_ctl); 700296548Sdumbbell else 701296548Sdumbbell dev_priv->mm.gtt->stolen_size = gen7_get_stolen_size(snb_gmch_ctl); 702296548Sdumbbell 703296548Sdumbbell dev_priv->mm.gtt->gtt_mappable_entries = drm_get_resource_len(dev, 2) >> PAGE_SHIFT; 704296548Sdumbbell /* 64/512MB is the current min/max we actually know of, but this is just a 705296548Sdumbbell * coarse sanity check. 706296548Sdumbbell */ 707296548Sdumbbell if ((dev_priv->mm.gtt->gtt_mappable_entries >> 8) < 64 || 708296548Sdumbbell dev_priv->mm.gtt->gtt_mappable_entries > dev_priv->mm.gtt->gtt_total_entries) { 709296548Sdumbbell DRM_ERROR("Unknown GMADR entries (%d)\n", 710296548Sdumbbell dev_priv->mm.gtt->gtt_mappable_entries); 711296548Sdumbbell ret = -ENXIO; 712296548Sdumbbell goto err_out; 713296548Sdumbbell } 714296548Sdumbbell 715296548Sdumbbell ret = setup_scratch_page(dev); 716296548Sdumbbell if (ret) { 717296548Sdumbbell DRM_ERROR("Scratch setup failed\n"); 718296548Sdumbbell goto err_out; 719296548Sdumbbell } 720296548Sdumbbell 721296548Sdumbbell dev_priv->mm.gtt->gtt = pmap_mapdev_attr(gtt_bus_addr, 722296548Sdumbbell /* The size is used later by pmap_unmapdev. */ 723296548Sdumbbell dev_priv->mm.gtt->gtt_total_entries * sizeof(gtt_pte_t), 724296548Sdumbbell VM_MEMATTR_WRITE_COMBINING); 725296548Sdumbbell if (!dev_priv->mm.gtt->gtt) { 726296548Sdumbbell DRM_ERROR("Failed to map the gtt page table\n"); 727296548Sdumbbell teardown_scratch_page(dev); 728296548Sdumbbell ret = -ENOMEM; 729296548Sdumbbell goto err_out; 730296548Sdumbbell } 731296548Sdumbbell 732296548Sdumbbell /* GMADR is the PCI aperture used by SW to access tiled GFX surfaces in a linear fashion. */ 733296548Sdumbbell DRM_INFO("Memory usable by graphics device = %dM\n", dev_priv->mm.gtt->gtt_total_entries >> 8); 734296548Sdumbbell DRM_DEBUG_DRIVER("GMADR size = %dM\n", dev_priv->mm.gtt->gtt_mappable_entries >> 8); 735296548Sdumbbell DRM_DEBUG_DRIVER("GTT stolen size = %dM\n", dev_priv->mm.gtt->stolen_size >> 20); 736296548Sdumbbell 737296548Sdumbbell return 0; 738296548Sdumbbell 739296548Sdumbbellerr_out: 740296548Sdumbbell free(dev_priv->mm.gtt, DRM_I915_GEM); 741296548Sdumbbell#ifdef FREEBSD_WIP 742296548Sdumbbell if (INTEL_INFO(dev)->gen < 6) 743296548Sdumbbell intel_gmch_remove(); 744296548Sdumbbell#endif /* FREEBSD_WIP */ 745296548Sdumbbell return ret; 746296548Sdumbbell} 747296548Sdumbbell 748296548Sdumbbellvoid i915_gem_gtt_fini(struct drm_device *dev) 749296548Sdumbbell{ 750296548Sdumbbell struct drm_i915_private *dev_priv = dev->dev_private; 751296548Sdumbbell pmap_unmapdev((vm_offset_t)dev_priv->mm.gtt->gtt, 752296548Sdumbbell dev_priv->mm.gtt->gtt_total_entries * sizeof(gtt_pte_t)); 753296548Sdumbbell teardown_scratch_page(dev); 754296548Sdumbbell#ifdef FREEBSD_WIP 755296548Sdumbbell if (INTEL_INFO(dev)->gen < 6) 756296548Sdumbbell intel_gmch_remove(); 757296548Sdumbbell#endif /* FREEBSD_WIP */ 758296548Sdumbbell if (INTEL_INFO(dev)->gen >= 6) 759296548Sdumbbell free(dev_priv->mm.gtt, DRM_I915_GEM); 760296548Sdumbbell} 761