1235783Skib/* 2235783Skib * Copyright �� 2008 Intel Corporation 3235783Skib * 4235783Skib * Permission is hereby granted, free of charge, to any person obtaining a 5235783Skib * copy of this software and associated documentation files (the "Software"), 6235783Skib * to deal in the Software without restriction, including without limitation 7235783Skib * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8235783Skib * and/or sell copies of the Software, and to permit persons to whom the 9235783Skib * Software is furnished to do so, subject to the following conditions: 10235783Skib * 11235783Skib * The above copyright notice and this permission notice (including the next 12235783Skib * paragraph) shall be included in all copies or substantial portions of the 13235783Skib * Software. 14235783Skib * 15235783Skib * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16235783Skib * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17235783Skib * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18235783Skib * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19235783Skib * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20235783Skib * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21235783Skib * IN THE SOFTWARE. 22235783Skib * 23235783Skib * Authors: 24235783Skib * Eric Anholt <eric@anholt.net> 25235783Skib * 26235783Skib */ 27235783Skib 28235783Skib#include <sys/cdefs.h> 29235783Skib__FBSDID("$FreeBSD$"); 30235783Skib 31235783Skib#include <dev/drm2/drmP.h> 32235783Skib#include <dev/drm2/i915/i915_drm.h> 33235783Skib#include <dev/drm2/i915/i915_drv.h> 34235783Skib 35235783Skib#include <sys/sf_buf.h> 36235783Skib 37235783Skib/** @file i915_gem_tiling.c 38235783Skib * 39235783Skib * Support for managing tiling state of buffer objects. 40235783Skib * 41235783Skib * The idea behind tiling is to increase cache hit rates by rearranging 42235783Skib * pixel data so that a group of pixel accesses are in the same cacheline. 43235783Skib * Performance improvement from doing this on the back/depth buffer are on 44235783Skib * the order of 30%. 45235783Skib * 46235783Skib * Intel architectures make this somewhat more complicated, though, by 47235783Skib * adjustments made to addressing of data when the memory is in interleaved 48235783Skib * mode (matched pairs of DIMMS) to improve memory bandwidth. 49235783Skib * For interleaved memory, the CPU sends every sequential 64 bytes 50235783Skib * to an alternate memory channel so it can get the bandwidth from both. 51235783Skib * 52235783Skib * The GPU also rearranges its accesses for increased bandwidth to interleaved 53235783Skib * memory, and it matches what the CPU does for non-tiled. However, when tiled 54235783Skib * it does it a little differently, since one walks addresses not just in the 55235783Skib * X direction but also Y. So, along with alternating channels when bit 56235783Skib * 6 of the address flips, it also alternates when other bits flip -- Bits 9 57235783Skib * (every 512 bytes, an X tile scanline) and 10 (every two X tile scanlines) 58235783Skib * are common to both the 915 and 965-class hardware. 59235783Skib * 60235783Skib * The CPU also sometimes XORs in higher bits as well, to improve 61235783Skib * bandwidth doing strided access like we do so frequently in graphics. This 62235783Skib * is called "Channel XOR Randomization" in the MCH documentation. The result 63235783Skib * is that the CPU is XORing in either bit 11 or bit 17 to bit 6 of its address 64235783Skib * decode. 65235783Skib * 66235783Skib * All of this bit 6 XORing has an effect on our memory management, 67235783Skib * as we need to make sure that the 3d driver can correctly address object 68235783Skib * contents. 69235783Skib * 70235783Skib * If we don't have interleaved memory, all tiling is safe and no swizzling is 71235783Skib * required. 72235783Skib * 73235783Skib * When bit 17 is XORed in, we simply refuse to tile at all. Bit 74235783Skib * 17 is not just a page offset, so as we page an objet out and back in, 75235783Skib * individual pages in it will have different bit 17 addresses, resulting in 76235783Skib * each 64 bytes being swapped with its neighbor! 77235783Skib * 78235783Skib * Otherwise, if interleaved, we have to tell the 3d driver what the address 79235783Skib * swizzling it needs to do is, since it's writing with the CPU to the pages 80235783Skib * (bit 6 and potentially bit 11 XORed in), and the GPU is reading from the 81235783Skib * pages (bit 6, 9, and 10 XORed in), resulting in a cumulative bit swizzling 82235783Skib * required by the CPU of XORing in bit 6, 9, 10, and potentially 11, in order 83235783Skib * to match what the GPU expects. 84235783Skib */ 85235783Skib 86235783Skib/** 87235783Skib * Detects bit 6 swizzling of address lookup between IGD access and CPU 88235783Skib * access through main memory. 89235783Skib */ 90235783Skibvoid 91235783Skibi915_gem_detect_bit_6_swizzle(struct drm_device *dev) 92235783Skib{ 93235783Skib drm_i915_private_t *dev_priv = dev->dev_private; 94235783Skib uint32_t swizzle_x = I915_BIT_6_SWIZZLE_UNKNOWN; 95235783Skib uint32_t swizzle_y = I915_BIT_6_SWIZZLE_UNKNOWN; 96235783Skib 97296548Sdumbbell if (IS_VALLEYVIEW(dev)) { 98296548Sdumbbell swizzle_x = I915_BIT_6_SWIZZLE_NONE; 99296548Sdumbbell swizzle_y = I915_BIT_6_SWIZZLE_NONE; 100296548Sdumbbell } else if (INTEL_INFO(dev)->gen >= 6) { 101235783Skib uint32_t dimm_c0, dimm_c1; 102235783Skib dimm_c0 = I915_READ(MAD_DIMM_C0); 103235783Skib dimm_c1 = I915_READ(MAD_DIMM_C1); 104235783Skib dimm_c0 &= MAD_DIMM_A_SIZE_MASK | MAD_DIMM_B_SIZE_MASK; 105235783Skib dimm_c1 &= MAD_DIMM_A_SIZE_MASK | MAD_DIMM_B_SIZE_MASK; 106235783Skib /* Enable swizzling when the channels are populated with 107235783Skib * identically sized dimms. We don't need to check the 3rd 108235783Skib * channel because no cpu with gpu attached ships in that 109235783Skib * configuration. Also, swizzling only makes sense for 2 110235783Skib * channels anyway. */ 111235783Skib if (dimm_c0 == dimm_c1) { 112235783Skib swizzle_x = I915_BIT_6_SWIZZLE_9_10; 113235783Skib swizzle_y = I915_BIT_6_SWIZZLE_9; 114235783Skib } else { 115235783Skib swizzle_x = I915_BIT_6_SWIZZLE_NONE; 116235783Skib swizzle_y = I915_BIT_6_SWIZZLE_NONE; 117235783Skib } 118235783Skib } else if (IS_GEN5(dev)) { 119235783Skib /* On Ironlake whatever DRAM config, GPU always do 120235783Skib * same swizzling setup. 121235783Skib */ 122235783Skib swizzle_x = I915_BIT_6_SWIZZLE_9_10; 123235783Skib swizzle_y = I915_BIT_6_SWIZZLE_9; 124235783Skib } else if (IS_GEN2(dev)) { 125235783Skib /* As far as we know, the 865 doesn't have these bit 6 126235783Skib * swizzling issues. 127235783Skib */ 128235783Skib swizzle_x = I915_BIT_6_SWIZZLE_NONE; 129235783Skib swizzle_y = I915_BIT_6_SWIZZLE_NONE; 130235783Skib } else if (IS_MOBILE(dev) || (IS_GEN3(dev) && !IS_G33(dev))) { 131235783Skib uint32_t dcc; 132235783Skib 133235783Skib /* On 9xx chipsets, channel interleave by the CPU is 134235783Skib * determined by DCC. For single-channel, neither the CPU 135235783Skib * nor the GPU do swizzling. For dual channel interleaved, 136235783Skib * the GPU's interleave is bit 9 and 10 for X tiled, and bit 137235783Skib * 9 for Y tiled. The CPU's interleave is independent, and 138235783Skib * can be based on either bit 11 (haven't seen this yet) or 139235783Skib * bit 17 (common). 140235783Skib */ 141235783Skib dcc = I915_READ(DCC); 142235783Skib switch (dcc & DCC_ADDRESSING_MODE_MASK) { 143235783Skib case DCC_ADDRESSING_MODE_SINGLE_CHANNEL: 144235783Skib case DCC_ADDRESSING_MODE_DUAL_CHANNEL_ASYMMETRIC: 145235783Skib swizzle_x = I915_BIT_6_SWIZZLE_NONE; 146235783Skib swizzle_y = I915_BIT_6_SWIZZLE_NONE; 147235783Skib break; 148235783Skib case DCC_ADDRESSING_MODE_DUAL_CHANNEL_INTERLEAVED: 149235783Skib if (dcc & DCC_CHANNEL_XOR_DISABLE) { 150235783Skib /* This is the base swizzling by the GPU for 151235783Skib * tiled buffers. 152235783Skib */ 153235783Skib swizzle_x = I915_BIT_6_SWIZZLE_9_10; 154235783Skib swizzle_y = I915_BIT_6_SWIZZLE_9; 155235783Skib } else if ((dcc & DCC_CHANNEL_XOR_BIT_17) == 0) { 156235783Skib /* Bit 11 swizzling by the CPU in addition. */ 157235783Skib swizzle_x = I915_BIT_6_SWIZZLE_9_10_11; 158235783Skib swizzle_y = I915_BIT_6_SWIZZLE_9_11; 159235783Skib } else { 160235783Skib /* Bit 17 swizzling by the CPU in addition. */ 161235783Skib swizzle_x = I915_BIT_6_SWIZZLE_9_10_17; 162235783Skib swizzle_y = I915_BIT_6_SWIZZLE_9_17; 163235783Skib } 164235783Skib break; 165235783Skib } 166235783Skib if (dcc == 0xffffffff) { 167235783Skib DRM_ERROR("Couldn't read from MCHBAR. " 168235783Skib "Disabling tiling.\n"); 169235783Skib swizzle_x = I915_BIT_6_SWIZZLE_UNKNOWN; 170235783Skib swizzle_y = I915_BIT_6_SWIZZLE_UNKNOWN; 171235783Skib } 172235783Skib } else { 173235783Skib /* The 965, G33, and newer, have a very flexible memory 174235783Skib * configuration. It will enable dual-channel mode 175235783Skib * (interleaving) on as much memory as it can, and the GPU 176235783Skib * will additionally sometimes enable different bit 6 177235783Skib * swizzling for tiled objects from the CPU. 178235783Skib * 179235783Skib * Here's what I found on the G965: 180235783Skib * slot fill memory size swizzling 181235783Skib * 0A 0B 1A 1B 1-ch 2-ch 182235783Skib * 512 0 0 0 512 0 O 183235783Skib * 512 0 512 0 16 1008 X 184235783Skib * 512 0 0 512 16 1008 X 185235783Skib * 0 512 0 512 16 1008 X 186235783Skib * 1024 1024 1024 0 2048 1024 O 187235783Skib * 188235783Skib * We could probably detect this based on either the DRB 189235783Skib * matching, which was the case for the swizzling required in 190235783Skib * the table above, or from the 1-ch value being less than 191235783Skib * the minimum size of a rank. 192235783Skib */ 193235783Skib if (I915_READ16(C0DRB3) != I915_READ16(C1DRB3)) { 194235783Skib swizzle_x = I915_BIT_6_SWIZZLE_NONE; 195235783Skib swizzle_y = I915_BIT_6_SWIZZLE_NONE; 196235783Skib } else { 197235783Skib swizzle_x = I915_BIT_6_SWIZZLE_9_10; 198235783Skib swizzle_y = I915_BIT_6_SWIZZLE_9; 199235783Skib } 200235783Skib } 201235783Skib 202235783Skib dev_priv->mm.bit_6_swizzle_x = swizzle_x; 203235783Skib dev_priv->mm.bit_6_swizzle_y = swizzle_y; 204235783Skib} 205235783Skib 206235783Skib/* Check pitch constriants for all chips & tiling formats */ 207235783Skibstatic bool 208235783Skibi915_tiling_ok(struct drm_device *dev, int stride, int size, int tiling_mode) 209235783Skib{ 210235783Skib int tile_width; 211235783Skib 212235783Skib /* Linear is always fine */ 213235783Skib if (tiling_mode == I915_TILING_NONE) 214287174Sbapt return true; 215235783Skib 216235783Skib if (IS_GEN2(dev) || 217235783Skib (tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))) 218235783Skib tile_width = 128; 219235783Skib else 220235783Skib tile_width = 512; 221235783Skib 222235783Skib /* check maximum stride & object size */ 223235783Skib if (INTEL_INFO(dev)->gen >= 4) { 224235783Skib /* i965 stores the end address of the gtt mapping in the fence 225235783Skib * reg, so dont bother to check the size */ 226235783Skib if (stride / 128 > I965_FENCE_MAX_PITCH_VAL) 227287174Sbapt return false; 228235783Skib } else { 229235783Skib if (stride > 8192) 230287174Sbapt return false; 231235783Skib 232235783Skib if (IS_GEN3(dev)) { 233235783Skib if (size > I830_FENCE_MAX_SIZE_VAL << 20) 234287174Sbapt return false; 235235783Skib } else { 236235783Skib if (size > I830_FENCE_MAX_SIZE_VAL << 19) 237287174Sbapt return false; 238235783Skib } 239235783Skib } 240235783Skib 241235783Skib /* 965+ just needs multiples of tile width */ 242235783Skib if (INTEL_INFO(dev)->gen >= 4) { 243235783Skib if (stride & (tile_width - 1)) 244287174Sbapt return false; 245287174Sbapt return true; 246235783Skib } 247235783Skib 248235783Skib /* Pre-965 needs power of two tile widths */ 249235783Skib if (stride < tile_width) 250287174Sbapt return false; 251235783Skib 252235783Skib if (stride & (stride - 1)) 253287174Sbapt return false; 254235783Skib 255287174Sbapt return true; 256235783Skib} 257235783Skib 258235783Skib/* Is the current GTT allocation valid for the change in tiling? */ 259235783Skibstatic bool 260235783Skibi915_gem_object_fence_ok(struct drm_i915_gem_object *obj, int tiling_mode) 261235783Skib{ 262235783Skib u32 size; 263235783Skib 264235783Skib if (tiling_mode == I915_TILING_NONE) 265287174Sbapt return true; 266235783Skib 267235783Skib if (INTEL_INFO(obj->base.dev)->gen >= 4) 268287174Sbapt return true; 269235783Skib 270235783Skib if (INTEL_INFO(obj->base.dev)->gen == 3) { 271235783Skib if (obj->gtt_offset & ~I915_FENCE_START_MASK) 272287174Sbapt return false; 273235783Skib } else { 274235783Skib if (obj->gtt_offset & ~I830_FENCE_START_MASK) 275287174Sbapt return false; 276235783Skib } 277235783Skib 278235783Skib /* 279235783Skib * Previous chips need to be aligned to the size of the smallest 280235783Skib * fence register that can contain the object. 281235783Skib */ 282235783Skib if (INTEL_INFO(obj->base.dev)->gen == 3) 283235783Skib size = 1024*1024; 284235783Skib else 285235783Skib size = 512*1024; 286235783Skib 287235783Skib while (size < obj->base.size) 288235783Skib size <<= 1; 289235783Skib 290235783Skib if (obj->gtt_space->size != size) 291287174Sbapt return false; 292235783Skib 293235783Skib if (obj->gtt_offset & (size - 1)) 294287174Sbapt return false; 295235783Skib 296287174Sbapt return true; 297235783Skib} 298235783Skib 299235783Skib/** 300235783Skib * Sets the tiling mode of an object, returning the required swizzling of 301235783Skib * bit 6 of addresses in the object. 302235783Skib */ 303235783Skibint 304235783Skibi915_gem_set_tiling(struct drm_device *dev, void *data, 305235783Skib struct drm_file *file) 306235783Skib{ 307235783Skib struct drm_i915_gem_set_tiling *args = data; 308235783Skib drm_i915_private_t *dev_priv = dev->dev_private; 309235783Skib struct drm_i915_gem_object *obj; 310287174Sbapt int ret = 0; 311235783Skib 312235783Skib obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 313235783Skib if (&obj->base == NULL) 314235783Skib return -ENOENT; 315235783Skib 316235783Skib if (!i915_tiling_ok(dev, 317235783Skib args->stride, obj->base.size, args->tiling_mode)) { 318296548Sdumbbell drm_gem_object_unreference_unlocked(&obj->base); 319235783Skib return -EINVAL; 320235783Skib } 321235783Skib 322235783Skib if (obj->pin_count) { 323296548Sdumbbell drm_gem_object_unreference_unlocked(&obj->base); 324235783Skib return -EBUSY; 325235783Skib } 326235783Skib 327235783Skib if (args->tiling_mode == I915_TILING_NONE) { 328235783Skib args->swizzle_mode = I915_BIT_6_SWIZZLE_NONE; 329235783Skib args->stride = 0; 330235783Skib } else { 331235783Skib if (args->tiling_mode == I915_TILING_X) 332235783Skib args->swizzle_mode = dev_priv->mm.bit_6_swizzle_x; 333235783Skib else 334235783Skib args->swizzle_mode = dev_priv->mm.bit_6_swizzle_y; 335235783Skib 336235783Skib /* Hide bit 17 swizzling from the user. This prevents old Mesa 337235783Skib * from aborting the application on sw fallbacks to bit 17, 338235783Skib * and we use the pread/pwrite bit17 paths to swizzle for it. 339235783Skib * If there was a user that was relying on the swizzle 340235783Skib * information for drm_intel_bo_map()ed reads/writes this would 341235783Skib * break it, but we don't have any of those. 342235783Skib */ 343235783Skib if (args->swizzle_mode == I915_BIT_6_SWIZZLE_9_17) 344235783Skib args->swizzle_mode = I915_BIT_6_SWIZZLE_9; 345235783Skib if (args->swizzle_mode == I915_BIT_6_SWIZZLE_9_10_17) 346235783Skib args->swizzle_mode = I915_BIT_6_SWIZZLE_9_10; 347235783Skib 348235783Skib /* If we can't handle the swizzling, make it untiled. */ 349235783Skib if (args->swizzle_mode == I915_BIT_6_SWIZZLE_UNKNOWN) { 350235783Skib args->tiling_mode = I915_TILING_NONE; 351235783Skib args->swizzle_mode = I915_BIT_6_SWIZZLE_NONE; 352235783Skib args->stride = 0; 353235783Skib } 354235783Skib } 355235783Skib 356280183Sdumbbell DRM_LOCK(dev); 357235783Skib if (args->tiling_mode != obj->tiling_mode || 358235783Skib args->stride != obj->stride) { 359235783Skib /* We need to rebind the object if its current allocation 360235783Skib * no longer meets the alignment restrictions for its new 361235783Skib * tiling mode. Otherwise we can just leave it alone, but 362277487Skib * need to ensure that any fence register is updated before 363277487Skib * the next fenced (either through the GTT or by the BLT unit 364277487Skib * on older GPUs) access. 365277487Skib * 366277487Skib * After updating the tiling parameters, we then flag whether 367277487Skib * we need to update an associated fence register. Note this 368277487Skib * has to also include the unfenced register the GPU uses 369277487Skib * whilst executing a fenced command for an untiled object. 370235783Skib */ 371235783Skib 372277487Skib obj->map_and_fenceable = 373277487Skib obj->gtt_space == NULL || 374287174Sbapt (obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end && 375287174Sbapt i915_gem_object_fence_ok(obj, args->tiling_mode)); 376235783Skib 377235783Skib /* Rebind if we need a change of alignment */ 378235783Skib if (!obj->map_and_fenceable) { 379287174Sbapt u32 unfenced_alignment = 380287174Sbapt i915_gem_get_unfenced_gtt_alignment(dev, 381287174Sbapt obj->base.size, 382287174Sbapt args->tiling_mode); 383235783Skib if (obj->gtt_offset & (unfenced_alignment - 1)) 384235783Skib ret = i915_gem_object_unbind(obj); 385235783Skib } 386277487Skib 387235783Skib if (ret == 0) { 388277487Skib obj->fence_dirty = 389277487Skib obj->fenced_gpu_access || 390277487Skib obj->fence_reg != I915_FENCE_REG_NONE; 391277487Skib 392235783Skib obj->tiling_mode = args->tiling_mode; 393235783Skib obj->stride = args->stride; 394277487Skib 395277487Skib /* Force the fence to be reacquired for GTT access */ 396277487Skib i915_gem_release_mmap(obj); 397235783Skib } 398277487Skib } 399235783Skib /* we have to maintain this existing ABI... */ 400235783Skib args->stride = obj->stride; 401235783Skib args->tiling_mode = obj->tiling_mode; 402235783Skib drm_gem_object_unreference(&obj->base); 403280183Sdumbbell DRM_UNLOCK(dev); 404235783Skib 405287174Sbapt return ret; 406235783Skib} 407235783Skib 408235783Skib/** 409235783Skib * Returns the current tiling mode and required bit 6 swizzling for the object. 410235783Skib */ 411235783Skibint 412235783Skibi915_gem_get_tiling(struct drm_device *dev, void *data, 413235783Skib struct drm_file *file) 414235783Skib{ 415235783Skib struct drm_i915_gem_get_tiling *args = data; 416235783Skib drm_i915_private_t *dev_priv = dev->dev_private; 417235783Skib struct drm_i915_gem_object *obj; 418235783Skib 419235783Skib obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 420235783Skib if (&obj->base == NULL) 421235783Skib return -ENOENT; 422235783Skib 423280183Sdumbbell DRM_LOCK(dev); 424280183Sdumbbell 425235783Skib args->tiling_mode = obj->tiling_mode; 426235783Skib switch (obj->tiling_mode) { 427235783Skib case I915_TILING_X: 428235783Skib args->swizzle_mode = dev_priv->mm.bit_6_swizzle_x; 429235783Skib break; 430235783Skib case I915_TILING_Y: 431235783Skib args->swizzle_mode = dev_priv->mm.bit_6_swizzle_y; 432235783Skib break; 433235783Skib case I915_TILING_NONE: 434235783Skib args->swizzle_mode = I915_BIT_6_SWIZZLE_NONE; 435235783Skib break; 436235783Skib default: 437235783Skib DRM_ERROR("unknown tiling mode\n"); 438235783Skib } 439235783Skib 440235783Skib /* Hide bit 17 from the user -- see comment in i915_gem_set_tiling */ 441235783Skib if (args->swizzle_mode == I915_BIT_6_SWIZZLE_9_17) 442235783Skib args->swizzle_mode = I915_BIT_6_SWIZZLE_9; 443235783Skib if (args->swizzle_mode == I915_BIT_6_SWIZZLE_9_10_17) 444235783Skib args->swizzle_mode = I915_BIT_6_SWIZZLE_9_10; 445235783Skib 446235783Skib drm_gem_object_unreference(&obj->base); 447280183Sdumbbell DRM_UNLOCK(dev); 448235783Skib 449235783Skib return 0; 450235783Skib} 451235783Skib 452235783Skib/** 453235783Skib * Swap every 64 bytes of this page around, to account for it having a new 454235783Skib * bit 17 of its physical address and therefore being interpreted differently 455235783Skib * by the GPU. 456235783Skib */ 457235783Skibstatic void 458293837Sdumbbelli915_gem_swizzle_page(vm_page_t page) 459235783Skib{ 460235783Skib char temp[64]; 461293837Sdumbbell struct sf_buf *sf; 462235783Skib char *vaddr; 463235783Skib int i; 464235783Skib 465235783Skib /* XXXKIB sleep */ 466293837Sdumbbell sf = sf_buf_alloc(page, SFB_DEFAULT); 467235783Skib vaddr = (char *)sf_buf_kva(sf); 468235783Skib 469235783Skib for (i = 0; i < PAGE_SIZE; i += 128) { 470235783Skib memcpy(temp, &vaddr[i], 64); 471235783Skib memcpy(&vaddr[i], &vaddr[i + 64], 64); 472235783Skib memcpy(&vaddr[i + 64], temp, 64); 473235783Skib } 474235783Skib 475235783Skib sf_buf_free(sf); 476235783Skib} 477235783Skib 478235783Skibvoid 479277487Skibi915_gem_object_do_bit_17_swizzle_page(struct drm_i915_gem_object *obj, 480277487Skib vm_page_t m) 481277487Skib{ 482277487Skib char new_bit_17; 483277487Skib 484277487Skib if (obj->bit_17 == NULL) 485277487Skib return; 486277487Skib 487277487Skib new_bit_17 = VM_PAGE_TO_PHYS(m) >> 17; 488296548Sdumbbell if ((new_bit_17 & 0x1) != 489296548Sdumbbell (test_bit(m->pindex, obj->bit_17) != 0)) { 490277487Skib i915_gem_swizzle_page(m); 491277487Skib vm_page_dirty(m); 492277487Skib } 493277487Skib} 494277487Skib 495277487Skibvoid 496235783Skibi915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj) 497235783Skib{ 498235783Skib int page_count = obj->base.size >> PAGE_SHIFT; 499235783Skib int i; 500235783Skib 501235783Skib if (obj->bit_17 == NULL) 502235783Skib return; 503235783Skib 504235783Skib for (i = 0; i < page_count; i++) { 505296548Sdumbbell vm_page_t page = obj->pages[i]; 506296548Sdumbbell char new_bit_17 = VM_PAGE_TO_PHYS(page) >> 17; 507235783Skib if ((new_bit_17 & 0x1) != 508235783Skib (test_bit(i, obj->bit_17) != 0)) { 509296548Sdumbbell i915_gem_swizzle_page(page); 510296548Sdumbbell vm_page_dirty(page); 511235783Skib } 512235783Skib } 513235783Skib} 514235783Skib 515235783Skibvoid 516235783Skibi915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object *obj) 517235783Skib{ 518235783Skib int page_count = obj->base.size >> PAGE_SHIFT; 519235783Skib int i; 520235783Skib 521235783Skib if (obj->bit_17 == NULL) { 522235783Skib obj->bit_17 = malloc(BITS_TO_LONGS(page_count) * 523296548Sdumbbell sizeof(long), DRM_I915_GEM, M_WAITOK); 524296548Sdumbbell if (obj->bit_17 == NULL) { 525296548Sdumbbell DRM_ERROR("Failed to allocate memory for bit 17 " 526296548Sdumbbell "record\n"); 527296548Sdumbbell return; 528296548Sdumbbell } 529235783Skib } 530235783Skib 531235783Skib /* XXXKIB: review locking, atomics might be not needed there */ 532235783Skib for (i = 0; i < page_count; i++) { 533296548Sdumbbell vm_page_t page = obj->pages[i]; 534296548Sdumbbell if (VM_PAGE_TO_PHYS(page) & (1 << 17)) 535296548Sdumbbell __set_bit(i, obj->bit_17); 536235783Skib else 537296548Sdumbbell __clear_bit(i, obj->bit_17); 538235783Skib } 539235783Skib} 540