Deleted Added
sdiff udiff text old ( 271705 ) new ( 277487 )
full compact
i915_gem.c (271705) i915_gem.c (277487)
1/*-
2 * Copyright �� 2008 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the

--- 38 unchanged lines hidden (view full) ---

47 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
48 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
49 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
50 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
51 * SUCH DAMAGE.
52 */
53
54#include <sys/cdefs.h>
1/*-
2 * Copyright �� 2008 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the

--- 38 unchanged lines hidden (view full) ---

47 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
48 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
49 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
50 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
51 * SUCH DAMAGE.
52 */
53
54#include <sys/cdefs.h>
55__FBSDID("$FreeBSD: head/sys/dev/drm2/i915/i915_gem.c 271705 2014-09-17 08:28:50Z dumbbell $");
55__FBSDID("$FreeBSD: head/sys/dev/drm2/i915/i915_gem.c 277487 2015-01-21 16:10:37Z kib $");
56
57#include <dev/drm2/drmP.h>
58#include <dev/drm2/drm.h>
59#include <dev/drm2/i915/i915_drm.h>
60#include <dev/drm2/i915/i915_drv.h>
61#include <dev/drm2/i915/intel_drv.h>
62#include <dev/drm2/i915/intel_ringbuffer.h>
63#include <sys/resourcevar.h>
64#include <sys/sched.h>
65#include <sys/sf_buf.h>
66
67#include <vm/vm.h>
68#include <vm/vm_pageout.h>
69
56
57#include <dev/drm2/drmP.h>
58#include <dev/drm2/drm.h>
59#include <dev/drm2/i915/i915_drm.h>
60#include <dev/drm2/i915/i915_drv.h>
61#include <dev/drm2/i915/intel_drv.h>
62#include <dev/drm2/i915/intel_ringbuffer.h>
63#include <sys/resourcevar.h>
64#include <sys/sched.h>
65#include <sys/sf_buf.h>
66
67#include <vm/vm.h>
68#include <vm/vm_pageout.h>
69
70#include <machine/md_var.h>
71
70static void i915_gem_object_flush_cpu_write_domain(
71 struct drm_i915_gem_object *obj);
72static uint32_t i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size,
73 int tiling_mode);
74static uint32_t i915_gem_get_gtt_alignment(struct drm_device *dev,
75 uint32_t size, int tiling_mode);
76static int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
77 unsigned alignment, bool map_and_fenceable);
78static int i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj,
79 int flags);
80static void i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj);
72static void i915_gem_object_flush_cpu_write_domain(
73 struct drm_i915_gem_object *obj);
74static uint32_t i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size,
75 int tiling_mode);
76static uint32_t i915_gem_get_gtt_alignment(struct drm_device *dev,
77 uint32_t size, int tiling_mode);
78static int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
79 unsigned alignment, bool map_and_fenceable);
80static int i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj,
81 int flags);
82static void i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj);
81static int i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj,
82 bool write);
83static void i915_gem_object_set_to_full_cpu_read_domain(
84 struct drm_i915_gem_object *obj);
85static int i915_gem_object_set_cpu_read_domain_range(
86 struct drm_i915_gem_object *obj, uint64_t offset, uint64_t size);
83static void i915_gem_object_put_pages_range(struct drm_i915_gem_object *obj,
84 off_t start, off_t end);
85static int i915_gem_object_get_pages_range(struct drm_i915_gem_object *obj,
86 off_t start, off_t end);
87static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj);
88static void i915_gem_object_truncate(struct drm_i915_gem_object *obj);
89static int i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj);
90static bool i915_gem_object_is_inactive(struct drm_i915_gem_object *obj);
91static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj);
87static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj);
88static void i915_gem_object_truncate(struct drm_i915_gem_object *obj);
89static int i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj);
90static bool i915_gem_object_is_inactive(struct drm_i915_gem_object *obj);
91static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj);
92static vm_page_t i915_gem_wire_page(vm_object_t object, vm_pindex_t pindex);
92static vm_page_t i915_gem_wire_page(vm_object_t object, vm_pindex_t pindex,
93 bool *fresh);
93static void i915_gem_process_flushing_list(struct intel_ring_buffer *ring,
94 uint32_t flush_domains);
94static void i915_gem_process_flushing_list(struct intel_ring_buffer *ring,
95 uint32_t flush_domains);
95static void i915_gem_clear_fence_reg(struct drm_device *dev,
96 struct drm_i915_fence_reg *reg);
97static void i915_gem_reset_fences(struct drm_device *dev);
98static void i915_gem_retire_task_handler(void *arg, int pending);
96static void i915_gem_reset_fences(struct drm_device *dev);
97static void i915_gem_retire_task_handler(void *arg, int pending);
99static int i915_gem_phys_pwrite(struct drm_device *dev,
100 struct drm_i915_gem_object *obj, uint64_t data_ptr, uint64_t offset,
101 uint64_t size, struct drm_file *file_priv);
102static void i915_gem_lowmem(void *arg);
98static void i915_gem_lowmem(void *arg);
99static void i915_gem_write_fence(struct drm_device *dev, int reg,
100 struct drm_i915_gem_object *obj);
101static int __wait_seqno(struct intel_ring_buffer *ring, u32 seqno,
102 bool interruptible);
103static int i915_gem_check_olr(struct intel_ring_buffer *ring, u32 seqno);
103
104MALLOC_DEFINE(DRM_I915_GEM, "i915gem", "Allocations from i915 gem");
105long i915_gem_wired_pages_cnt;
106
104
105MALLOC_DEFINE(DRM_I915_GEM, "i915gem", "Allocations from i915 gem");
106long i915_gem_wired_pages_cnt;
107
108static bool cpu_cache_is_coherent(struct drm_device *dev,
109 enum i915_cache_level level)
110{
111 return HAS_LLC(dev) || level != I915_CACHE_NONE;
112}
113
114static bool cpu_write_needs_clflush(struct drm_i915_gem_object *obj)
115{
116 if (!cpu_cache_is_coherent(obj->base.dev, obj->cache_level))
117 return true;
118
119 return obj->pin_display;
120}
121
122static inline void i915_gem_object_fence_lost(struct drm_i915_gem_object *obj)
123{
124 if (obj->tiling_mode)
125 i915_gem_release_mmap(obj);
126
127 /* As we do not have an associated fence register, we will force
128 * a tiling change if we ever need to acquire one.
129 */
130 obj->fence_dirty = false;
131 obj->fence_reg = I915_FENCE_REG_NONE;
132}
133
107static void
108i915_gem_info_add_obj(struct drm_i915_private *dev_priv, size_t size)
109{
110
111 dev_priv->mm.object_count++;
112 dev_priv->mm.object_memory += size;
113}
114

--- 52 unchanged lines hidden (view full) ---

167 ret = sx_xlock_sig(&dev->dev_struct_lock);
168 if (ret != 0)
169 return (-ret);
170
171 return (0);
172}
173
174
134static void
135i915_gem_info_add_obj(struct drm_i915_private *dev_priv, size_t size)
136{
137
138 dev_priv->mm.object_count++;
139 dev_priv->mm.object_memory += size;
140}
141

--- 52 unchanged lines hidden (view full) ---

194 ret = sx_xlock_sig(&dev->dev_struct_lock);
195 if (ret != 0)
196 return (-ret);
197
198 return (0);
199}
200
201
175static void
176i915_gem_free_object_tail(struct drm_i915_gem_object *obj)
202void
203i915_gem_free_object(struct drm_gem_object *gem_obj)
177{
204{
205 struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
178 struct drm_device *dev;
179 drm_i915_private_t *dev_priv;
206 struct drm_device *dev;
207 drm_i915_private_t *dev_priv;
180 int ret;
181
182 dev = obj->base.dev;
183 dev_priv = dev->dev_private;
184
208
209 dev = obj->base.dev;
210 dev_priv = dev->dev_private;
211
185 ret = i915_gem_object_unbind(obj);
186 if (ret == -ERESTART) {
187 list_move(&obj->mm_list, &dev_priv->mm.deferred_free_list);
188 return;
212 CTR1(KTR_DRM, "object_destroy_tail %p", obj);
213
214 if (obj->phys_obj)
215 i915_gem_detach_phys_object(dev, obj);
216
217 obj->pin_count = 0;
218 if (i915_gem_object_unbind(obj) == -ERESTARTSYS) {
219 bool was_interruptible;
220
221 was_interruptible = dev_priv->mm.interruptible;
222 dev_priv->mm.interruptible = false;
223
224 if (i915_gem_object_unbind(obj))
225 printf("i915_gem_free_object: unbind\n");
226
227 dev_priv->mm.interruptible = was_interruptible;
189 }
190
228 }
229
191 CTR1(KTR_DRM, "object_destroy_tail %p", obj);
192 drm_gem_free_mmap_offset(&obj->base);
193 drm_gem_object_release(&obj->base);
194 i915_gem_info_remove_obj(dev_priv, obj->base.size);
195
230 drm_gem_free_mmap_offset(&obj->base);
231 drm_gem_object_release(&obj->base);
232 i915_gem_info_remove_obj(dev_priv, obj->base.size);
233
196 free(obj->page_cpu_valid, DRM_I915_GEM);
197 free(obj->bit_17, DRM_I915_GEM);
198 free(obj, DRM_I915_GEM);
199}
200
234 free(obj->bit_17, DRM_I915_GEM);
235 free(obj, DRM_I915_GEM);
236}
237
201void
202i915_gem_free_object(struct drm_gem_object *gem_obj)
203{
204 struct drm_i915_gem_object *obj;
205 struct drm_device *dev;
206
207 obj = to_intel_bo(gem_obj);
208 dev = obj->base.dev;
209
210 while (obj->pin_count > 0)
211 i915_gem_object_unpin(obj);
212
213 if (obj->phys_obj != NULL)
214 i915_gem_detach_phys_object(dev, obj);
215
216 i915_gem_free_object_tail(obj);
217}
218
219static void
220init_ring_lists(struct intel_ring_buffer *ring)
221{
222
223 INIT_LIST_HEAD(&ring->active_list);
224 INIT_LIST_HEAD(&ring->request_list);
225 INIT_LIST_HEAD(&ring->gpu_write_list);
226}

--- 4 unchanged lines hidden (view full) ---

231 drm_i915_private_t *dev_priv;
232 int i;
233
234 dev_priv = dev->dev_private;
235
236 INIT_LIST_HEAD(&dev_priv->mm.active_list);
237 INIT_LIST_HEAD(&dev_priv->mm.flushing_list);
238 INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
238static void
239init_ring_lists(struct intel_ring_buffer *ring)
240{
241
242 INIT_LIST_HEAD(&ring->active_list);
243 INIT_LIST_HEAD(&ring->request_list);
244 INIT_LIST_HEAD(&ring->gpu_write_list);
245}

--- 4 unchanged lines hidden (view full) ---

250 drm_i915_private_t *dev_priv;
251 int i;
252
253 dev_priv = dev->dev_private;
254
255 INIT_LIST_HEAD(&dev_priv->mm.active_list);
256 INIT_LIST_HEAD(&dev_priv->mm.flushing_list);
257 INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
239 INIT_LIST_HEAD(&dev_priv->mm.pinned_list);
240 INIT_LIST_HEAD(&dev_priv->mm.fence_list);
258 INIT_LIST_HEAD(&dev_priv->mm.fence_list);
241 INIT_LIST_HEAD(&dev_priv->mm.deferred_free_list);
242 INIT_LIST_HEAD(&dev_priv->mm.gtt_list);
243 for (i = 0; i < I915_NUM_RINGS; i++)
244 init_ring_lists(&dev_priv->rings[i]);
245 for (i = 0; i < I915_MAX_NUM_FENCES; i++)
246 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
247 TIMEOUT_TASK_INIT(dev_priv->tq, &dev_priv->mm.retire_task, 0,
248 i915_gem_retire_task_handler, dev_priv);
249 dev_priv->error_completion = 0;
250
251 /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
252 if (IS_GEN3(dev)) {
259 INIT_LIST_HEAD(&dev_priv->mm.gtt_list);
260 for (i = 0; i < I915_NUM_RINGS; i++)
261 init_ring_lists(&dev_priv->rings[i]);
262 for (i = 0; i < I915_MAX_NUM_FENCES; i++)
263 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
264 TIMEOUT_TASK_INIT(dev_priv->tq, &dev_priv->mm.retire_task, 0,
265 i915_gem_retire_task_handler, dev_priv);
266 dev_priv->error_completion = 0;
267
268 /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
269 if (IS_GEN3(dev)) {
253 u32 tmp = I915_READ(MI_ARB_STATE);
254 if (!(tmp & MI_ARB_C3_LP_WRITE_ENABLE)) {
255 /*
256 * arb state is a masked write, so set bit +
257 * bit in mask.
258 */
259 tmp = MI_ARB_C3_LP_WRITE_ENABLE |
260 (MI_ARB_C3_LP_WRITE_ENABLE << MI_ARB_MASK_SHIFT);
261 I915_WRITE(MI_ARB_STATE, tmp);
262 }
270 I915_WRITE(MI_ARB_STATE,
271 _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
263 }
264
265 dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
266
267 /* Old X drivers will take 0-2 for front, back, depth buffers */
268 if (!drm_core_check_feature(dev, DRIVER_MODESET))
269 dev_priv->fence_reg_start = 3;
270
271 if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) ||
272 IS_G33(dev))
273 dev_priv->num_fence_regs = 16;
274 else
275 dev_priv->num_fence_regs = 8;
276
277 /* Initialize fence registers to zero */
272 }
273
274 dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
275
276 /* Old X drivers will take 0-2 for front, back, depth buffers */
277 if (!drm_core_check_feature(dev, DRIVER_MODESET))
278 dev_priv->fence_reg_start = 3;
279
280 if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) ||
281 IS_G33(dev))
282 dev_priv->num_fence_regs = 16;
283 else
284 dev_priv->num_fence_regs = 8;
285
286 /* Initialize fence registers to zero */
278 for (i = 0; i < dev_priv->num_fence_regs; i++) {
279 i915_gem_clear_fence_reg(dev, &dev_priv->fence_regs[i]);
280 }
287 i915_gem_reset_fences(dev);
288
281 i915_gem_detect_bit_6_swizzle(dev);
282 dev_priv->mm.interruptible = true;
283
284 dev_priv->mm.i915_lowmem = EVENTHANDLER_REGISTER(vm_lowmem,
285 i915_gem_lowmem, dev, EVENTHANDLER_PRI_ANY);
286}
287
288int
289 i915_gem_detect_bit_6_swizzle(dev);
290 dev_priv->mm.interruptible = true;
291
292 dev_priv->mm.i915_lowmem = EVENTHANDLER_REGISTER(vm_lowmem,
293 i915_gem_lowmem, dev, EVENTHANDLER_PRI_ANY);
294}
295
296int
289i915_gem_do_init(struct drm_device *dev, unsigned long start,
290 unsigned long mappable_end, unsigned long end)
291{
292 drm_i915_private_t *dev_priv;
293 unsigned long mappable;
294 int error;
295
296 dev_priv = dev->dev_private;
297 mappable = min(end, mappable_end) - start;
298
299 drm_mm_init(&dev_priv->mm.gtt_space, start, end - start);
300
301 dev_priv->mm.gtt_start = start;
302 dev_priv->mm.gtt_mappable_end = mappable_end;
303 dev_priv->mm.gtt_end = end;
304 dev_priv->mm.gtt_total = end - start;
305 dev_priv->mm.mappable_gtt_total = mappable;
306
307 /* Take over this portion of the GTT */
308 intel_gtt_clear_range(start / PAGE_SIZE, (end-start) / PAGE_SIZE);
309 device_printf(dev->device,
310 "taking over the fictitious range 0x%lx-0x%lx\n",
311 dev->agp->base + start, dev->agp->base + start + mappable);
312 error = -vm_phys_fictitious_reg_range(dev->agp->base + start,
313 dev->agp->base + start + mappable, VM_MEMATTR_WRITE_COMBINING);
314 return (error);
315}
316
317int
318i915_gem_init_ioctl(struct drm_device *dev, void *data,
319 struct drm_file *file)
320{
321 struct drm_i915_gem_init *args;
322 drm_i915_private_t *dev_priv;
323
297i915_gem_init_ioctl(struct drm_device *dev, void *data,
298 struct drm_file *file)
299{
300 struct drm_i915_gem_init *args;
301 drm_i915_private_t *dev_priv;
302
303 if (drm_core_check_feature(dev, DRIVER_MODESET))
304 return -ENODEV;
305
324 dev_priv = dev->dev_private;
325 args = data;
326
327 if (args->gtt_start >= args->gtt_end ||
328 (args->gtt_end | args->gtt_start) & (PAGE_SIZE - 1))
329 return (-EINVAL);
330
331 if (mtx_initialized(&dev_priv->mm.gtt_space.unused_lock))
332 return (-EBUSY);
306 dev_priv = dev->dev_private;
307 args = data;
308
309 if (args->gtt_start >= args->gtt_end ||
310 (args->gtt_end | args->gtt_start) & (PAGE_SIZE - 1))
311 return (-EINVAL);
312
313 if (mtx_initialized(&dev_priv->mm.gtt_space.unused_lock))
314 return (-EBUSY);
315
316 /* GEM with user mode setting was never supported on ilk and later. */
317 if (INTEL_INFO(dev)->gen >= 5)
318 return -ENODEV;
319
333 /*
334 * XXXKIB. The second-time initialization should be guarded
335 * against.
336 */
320 /*
321 * XXXKIB. The second-time initialization should be guarded
322 * against.
323 */
337 return (i915_gem_do_init(dev, args->gtt_start, args->gtt_end,
324 return (i915_gem_init_global_gtt(dev, args->gtt_start, args->gtt_end,
338 args->gtt_end));
339}
340
341int
342i915_gem_idle(struct drm_device *dev)
343{
344 drm_i915_private_t *dev_priv;
345 int ret;
346
347 dev_priv = dev->dev_private;
348 if (dev_priv->mm.suspended)
349 return (0);
350
325 args->gtt_end));
326}
327
328int
329i915_gem_idle(struct drm_device *dev)
330{
331 drm_i915_private_t *dev_priv;
332 int ret;
333
334 dev_priv = dev->dev_private;
335 if (dev_priv->mm.suspended)
336 return (0);
337
351 ret = i915_gpu_idle(dev, true);
338 ret = i915_gpu_idle(dev);
352 if (ret != 0)
353 return (ret);
339 if (ret != 0)
340 return (ret);
341 i915_gem_retire_requests(dev);
354
355 /* Under UMS, be paranoid and evict. */
356 if (!drm_core_check_feature(dev, DRIVER_MODESET)) {
342
343 /* Under UMS, be paranoid and evict. */
344 if (!drm_core_check_feature(dev, DRIVER_MODESET)) {
357 ret = i915_gem_evict_inactive(dev, false);
345 ret = i915_gem_evict_everything(dev, false);
358 if (ret != 0)
359 return ret;
360 }
361
362 i915_gem_reset_fences(dev);
363
364 /* Hack! Don't let anybody do execbuf while we don't control the chip.
365 * We need to replace this with a semaphore, or something.

--- 22 unchanged lines hidden (view full) ---

388 return;
389
390 I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
391 DISP_TILE_SURFACE_SWIZZLING);
392
393 if (IS_GEN5(dev))
394 return;
395
346 if (ret != 0)
347 return ret;
348 }
349
350 i915_gem_reset_fences(dev);
351
352 /* Hack! Don't let anybody do execbuf while we don't control the chip.
353 * We need to replace this with a semaphore, or something.

--- 22 unchanged lines hidden (view full) ---

376 return;
377
378 I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
379 DISP_TILE_SURFACE_SWIZZLING);
380
381 if (IS_GEN5(dev))
382 return;
383
384
396 I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL);
397 if (IS_GEN6(dev))
385 I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL);
386 if (IS_GEN6(dev))
398 I915_WRITE(ARB_MODE, ARB_MODE_ENABLE(ARB_MODE_SWIZZLE_SNB));
387 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
399 else
388 else
400 I915_WRITE(ARB_MODE, ARB_MODE_ENABLE(ARB_MODE_SWIZZLE_IVB));
389 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
401}
402
390}
391
403void
404i915_gem_init_ppgtt(struct drm_device *dev)
392void i915_gem_init_ppgtt(struct drm_device *dev)
405{
406 drm_i915_private_t *dev_priv;
407 struct i915_hw_ppgtt *ppgtt;
408 uint32_t pd_offset, pd_entry;
409 vm_paddr_t pt_addr;
410 struct intel_ring_buffer *ring;
411 u_int first_pd_entry_in_global_pt, i;
412

--- 11 unchanged lines hidden (view full) ---

424 }
425 intel_gtt_read_pte(first_pd_entry_in_global_pt);
426
427 pd_offset = ppgtt->pd_offset;
428 pd_offset /= 64; /* in cachelines, */
429 pd_offset <<= 16;
430
431 if (INTEL_INFO(dev)->gen == 6) {
393{
394 drm_i915_private_t *dev_priv;
395 struct i915_hw_ppgtt *ppgtt;
396 uint32_t pd_offset, pd_entry;
397 vm_paddr_t pt_addr;
398 struct intel_ring_buffer *ring;
399 u_int first_pd_entry_in_global_pt, i;
400

--- 11 unchanged lines hidden (view full) ---

412 }
413 intel_gtt_read_pte(first_pd_entry_in_global_pt);
414
415 pd_offset = ppgtt->pd_offset;
416 pd_offset /= 64; /* in cachelines, */
417 pd_offset <<= 16;
418
419 if (INTEL_INFO(dev)->gen == 6) {
432 uint32_t ecochk = I915_READ(GAM_ECOCHK);
420 uint32_t ecochk, gab_ctl, ecobits;
421
422 ecobits = I915_READ(GAC_ECO_BITS);
423 I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_PPGTT_CACHE64B);
424
425 gab_ctl = I915_READ(GAB_CTL);
426 I915_WRITE(GAB_CTL, gab_ctl | GAB_CTL_CONT_AFTER_PAGEFAULT);
427
428 ecochk = I915_READ(GAM_ECOCHK);
433 I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT |
434 ECOCHK_PPGTT_CACHE64B);
429 I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT |
430 ECOCHK_PPGTT_CACHE64B);
435 I915_WRITE(GFX_MODE, GFX_MODE_ENABLE(GFX_PPGTT_ENABLE));
431 I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
436 } else if (INTEL_INFO(dev)->gen >= 7) {
437 I915_WRITE(GAM_ECOCHK, ECOCHK_PPGTT_CACHE64B);
438 /* GFX_MODE is per-ring on gen7+ */
439 }
440
432 } else if (INTEL_INFO(dev)->gen >= 7) {
433 I915_WRITE(GAM_ECOCHK, ECOCHK_PPGTT_CACHE64B);
434 /* GFX_MODE is per-ring on gen7+ */
435 }
436
441 for (i = 0; i < I915_NUM_RINGS; i++) {
442 ring = &dev_priv->rings[i];
443
437 for_each_ring(ring, dev_priv, i) {
444 if (INTEL_INFO(dev)->gen >= 7)
445 I915_WRITE(RING_MODE_GEN7(ring),
438 if (INTEL_INFO(dev)->gen >= 7)
439 I915_WRITE(RING_MODE_GEN7(ring),
446 GFX_MODE_ENABLE(GFX_PPGTT_ENABLE));
440 _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
447
448 I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G);
449 I915_WRITE(RING_PP_DIR_BASE(ring), pd_offset);
450 }
451}
452
453int
454i915_gem_init_hw(struct drm_device *dev)

--- 28 unchanged lines hidden (view full) ---

483
484cleanup_bsd_ring:
485 intel_cleanup_ring_buffer(&dev_priv->rings[VCS]);
486cleanup_render_ring:
487 intel_cleanup_ring_buffer(&dev_priv->rings[RCS]);
488 return (ret);
489}
490
441
442 I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G);
443 I915_WRITE(RING_PP_DIR_BASE(ring), pd_offset);
444 }
445}
446
447int
448i915_gem_init_hw(struct drm_device *dev)

--- 28 unchanged lines hidden (view full) ---

477
478cleanup_bsd_ring:
479 intel_cleanup_ring_buffer(&dev_priv->rings[VCS]);
480cleanup_render_ring:
481 intel_cleanup_ring_buffer(&dev_priv->rings[RCS]);
482 return (ret);
483}
484
485static bool
486intel_enable_ppgtt(struct drm_device *dev)
487{
488 if (i915_enable_ppgtt >= 0)
489 return i915_enable_ppgtt;
490
491 /* Disable ppgtt on SNB if VT-d is on. */
492 if (INTEL_INFO(dev)->gen == 6 && intel_iommu_enabled)
493 return false;
494
495 return true;
496}
497
498int i915_gem_init(struct drm_device *dev)
499{
500 struct drm_i915_private *dev_priv = dev->dev_private;
501 unsigned long gtt_size, mappable_size;
502 int ret;
503
504 gtt_size = dev_priv->mm.gtt.gtt_total_entries << PAGE_SHIFT;
505 mappable_size = dev_priv->mm.gtt.gtt_mappable_entries << PAGE_SHIFT;
506
507 DRM_LOCK(dev);
508 if (intel_enable_ppgtt(dev) && HAS_ALIASING_PPGTT(dev)) {
509 /* PPGTT pdes are stolen from global gtt ptes, so shrink the
510 * aperture accordingly when using aliasing ppgtt. */
511 gtt_size -= I915_PPGTT_PD_ENTRIES*PAGE_SIZE;
512
513 i915_gem_init_global_gtt(dev, 0, mappable_size, gtt_size);
514
515 ret = i915_gem_init_aliasing_ppgtt(dev);
516 if (ret) {
517 DRM_UNLOCK(dev);
518 return ret;
519 }
520 } else {
521 /* Let GEM Manage all of the aperture.
522 *
523 * However, leave one page at the end still bound to the scratch
524 * page. There are a number of places where the hardware
525 * apparently prefetches past the end of the object, and we've
526 * seen multiple hangs with the GPU head pointer stuck in a
527 * batchbuffer bound at the last page of the aperture. One page
528 * should be enough to keep any prefetching inside of the
529 * aperture.
530 */
531 i915_gem_init_global_gtt(dev, 0, mappable_size,
532 gtt_size);
533 }
534
535 ret = i915_gem_init_hw(dev);
536 DRM_UNLOCK(dev);
537 if (ret != 0) {
538 i915_gem_cleanup_aliasing_ppgtt(dev);
539 return (ret);
540 }
541
542 /* Allow hardware batchbuffers unless told otherwise, but not for KMS. */
543 if (!drm_core_check_feature(dev, DRIVER_MODESET))
544 dev_priv->dri1.allow_batchbuffer = 1;
545 return 0;
546}
547
491int
492i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
493 struct drm_file *file)
494{
495 struct drm_i915_private *dev_priv;
496 struct drm_i915_gem_get_aperture *args;
497 struct drm_i915_gem_object *obj;
498 size_t pinned;
499
500 dev_priv = dev->dev_private;
501 args = data;
502
548int
549i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
550 struct drm_file *file)
551{
552 struct drm_i915_private *dev_priv;
553 struct drm_i915_gem_get_aperture *args;
554 struct drm_i915_gem_object *obj;
555 size_t pinned;
556
557 dev_priv = dev->dev_private;
558 args = data;
559
503 if (!(dev->driver->driver_features & DRIVER_GEM))
504 return (-ENODEV);
505
506 pinned = 0;
507 DRM_LOCK(dev);
560 pinned = 0;
561 DRM_LOCK(dev);
508 list_for_each_entry(obj, &dev_priv->mm.pinned_list, mm_list)
509 pinned += obj->gtt_space->size;
562 list_for_each_entry(obj, &dev_priv->mm.gtt_list, gtt_list)
563 if (obj->pin_count)
564 pinned += obj->gtt_space->size;
510 DRM_UNLOCK(dev);
511
512 args->aper_size = dev_priv->mm.gtt_total;
513 args->aper_available_size = args->aper_size - pinned;
514
515 return (0);
516}
517
518int
519i915_gem_object_pin(struct drm_i915_gem_object *obj, uint32_t alignment,
520 bool map_and_fenceable)
521{
565 DRM_UNLOCK(dev);
566
567 args->aper_size = dev_priv->mm.gtt_total;
568 args->aper_available_size = args->aper_size - pinned;
569
570 return (0);
571}
572
573int
574i915_gem_object_pin(struct drm_i915_gem_object *obj, uint32_t alignment,
575 bool map_and_fenceable)
576{
522 struct drm_device *dev;
523 struct drm_i915_private *dev_priv;
524 int ret;
525
577 int ret;
578
526 dev = obj->base.dev;
527 dev_priv = dev->dev_private;
579 if (obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT)
580 return (-EBUSY);
528
581
529 KASSERT(obj->pin_count != DRM_I915_GEM_OBJECT_MAX_PIN_COUNT,
530 ("Max pin count"));
531
532 if (obj->gtt_space != NULL) {
533 if ((alignment && obj->gtt_offset & (alignment - 1)) ||
534 (map_and_fenceable && !obj->map_and_fenceable)) {
535 DRM_DEBUG("bo is already pinned with incorrect alignment:"
536 " offset=%x, req.alignment=%x, req.map_and_fenceable=%d,"
537 " obj->map_and_fenceable=%d\n",
538 obj->gtt_offset, alignment,
539 map_and_fenceable,

--- 6 unchanged lines hidden (view full) ---

546
547 if (obj->gtt_space == NULL) {
548 ret = i915_gem_object_bind_to_gtt(obj, alignment,
549 map_and_fenceable);
550 if (ret)
551 return (ret);
552 }
553
582 if (obj->gtt_space != NULL) {
583 if ((alignment && obj->gtt_offset & (alignment - 1)) ||
584 (map_and_fenceable && !obj->map_and_fenceable)) {
585 DRM_DEBUG("bo is already pinned with incorrect alignment:"
586 " offset=%x, req.alignment=%x, req.map_and_fenceable=%d,"
587 " obj->map_and_fenceable=%d\n",
588 obj->gtt_offset, alignment,
589 map_and_fenceable,

--- 6 unchanged lines hidden (view full) ---

596
597 if (obj->gtt_space == NULL) {
598 ret = i915_gem_object_bind_to_gtt(obj, alignment,
599 map_and_fenceable);
600 if (ret)
601 return (ret);
602 }
603
554 if (obj->pin_count++ == 0 && !obj->active)
555 list_move_tail(&obj->mm_list, &dev_priv->mm.pinned_list);
604 if (!obj->has_global_gtt_mapping && map_and_fenceable)
605 i915_gem_gtt_bind_object(obj, obj->cache_level);
606
607 obj->pin_count++;
556 obj->pin_mappable |= map_and_fenceable;
557
608 obj->pin_mappable |= map_and_fenceable;
609
558#if 1
559 KIB_NOTYET();
560#else
561 WARN_ON(i915_verify_lists(dev));
562#endif
563 return (0);
610 return 0;
564}
565
566void
567i915_gem_object_unpin(struct drm_i915_gem_object *obj)
568{
611}
612
613void
614i915_gem_object_unpin(struct drm_i915_gem_object *obj)
615{
569 struct drm_device *dev;
570 drm_i915_private_t *dev_priv;
571
572 dev = obj->base.dev;
573 dev_priv = dev->dev_private;
574
575#if 1
576 KIB_NOTYET();
577#else
578 WARN_ON(i915_verify_lists(dev));
579#endif
580
581 KASSERT(obj->pin_count != 0, ("zero pin count"));
582 KASSERT(obj->gtt_space != NULL, ("No gtt mapping"));
583
616
617 KASSERT(obj->pin_count != 0, ("zero pin count"));
618 KASSERT(obj->gtt_space != NULL, ("No gtt mapping"));
619
584 if (--obj->pin_count == 0) {
585 if (!obj->active)
586 list_move_tail(&obj->mm_list,
587 &dev_priv->mm.inactive_list);
620 if (--obj->pin_count == 0)
588 obj->pin_mappable = false;
621 obj->pin_mappable = false;
589 }
590#if 1
591 KIB_NOTYET();
592#else
593 WARN_ON(i915_verify_lists(dev));
594#endif
595}
596
597int
598i915_gem_pin_ioctl(struct drm_device *dev, void *data,
599 struct drm_file *file)
600{
601 struct drm_i915_gem_pin *args;
602 struct drm_i915_gem_object *obj;

--- 85 unchanged lines hidden (view full) ---

688}
689
690int
691i915_gem_busy_ioctl(struct drm_device *dev, void *data,
692 struct drm_file *file)
693{
694 struct drm_i915_gem_busy *args;
695 struct drm_i915_gem_object *obj;
622}
623
624int
625i915_gem_pin_ioctl(struct drm_device *dev, void *data,
626 struct drm_file *file)
627{
628 struct drm_i915_gem_pin *args;
629 struct drm_i915_gem_object *obj;

--- 85 unchanged lines hidden (view full) ---

715}
716
717int
718i915_gem_busy_ioctl(struct drm_device *dev, void *data,
719 struct drm_file *file)
720{
721 struct drm_i915_gem_busy *args;
722 struct drm_i915_gem_object *obj;
696 struct drm_i915_gem_request *request;
697 int ret;
698
699 args = data;
700
701 ret = i915_mutex_lock_interruptible(dev);
702 if (ret != 0)
703 return ret;
704
705 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
706 if (&obj->base == NULL) {
707 ret = -ENOENT;
708 goto unlock;
709 }
710
711 args->busy = obj->active;
712 if (args->busy) {
713 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
714 ret = i915_gem_flush_ring(obj->ring,
715 0, obj->base.write_domain);
723 int ret;
724
725 args = data;
726
727 ret = i915_mutex_lock_interruptible(dev);
728 if (ret != 0)
729 return ret;
730
731 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
732 if (&obj->base == NULL) {
733 ret = -ENOENT;
734 goto unlock;
735 }
736
737 args->busy = obj->active;
738 if (args->busy) {
739 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
740 ret = i915_gem_flush_ring(obj->ring,
741 0, obj->base.write_domain);
716 } else if (obj->ring->outstanding_lazy_request ==
717 obj->last_rendering_seqno) {
718 request = malloc(sizeof(*request), DRM_I915_GEM,
719 M_WAITOK | M_ZERO);
720 ret = i915_add_request(obj->ring, NULL, request);
721 if (ret != 0)
722 free(request, DRM_I915_GEM);
742 } else {
743 ret = i915_gem_check_olr(obj->ring,
744 obj->last_rendering_seqno);
723 }
724
725 i915_gem_retire_requests_ring(obj->ring);
726 args->busy = obj->active;
727 }
728
729 drm_gem_object_unreference(&obj->base);
730unlock:

--- 27 unchanged lines hidden (view full) ---

758 break;
759 ring = request->ring;
760 seqno = request->seqno;
761 }
762 mtx_unlock(&file_priv->mm.lck);
763 if (seqno == 0)
764 return (0);
765
745 }
746
747 i915_gem_retire_requests_ring(obj->ring);
748 args->busy = obj->active;
749 }
750
751 drm_gem_object_unreference(&obj->base);
752unlock:

--- 27 unchanged lines hidden (view full) ---

780 break;
781 ring = request->ring;
782 seqno = request->seqno;
783 }
784 mtx_unlock(&file_priv->mm.lck);
785 if (seqno == 0)
786 return (0);
787
766 ret = 0;
767 mtx_lock(&ring->irq_lock);
768 if (!i915_seqno_passed(ring->get_seqno(ring), seqno)) {
769 if (ring->irq_get(ring)) {
770 while (ret == 0 &&
771 !(i915_seqno_passed(ring->get_seqno(ring), seqno) ||
772 atomic_load_acq_int(&dev_priv->mm.wedged)))
773 ret = -msleep(ring, &ring->irq_lock, PCATCH,
774 "915thr", 0);
775 ring->irq_put(ring);
776 if (ret == 0 && atomic_load_acq_int(&dev_priv->mm.wedged))
777 ret = -EIO;
778 } else if (_intel_wait_for(dev,
779 i915_seqno_passed(ring->get_seqno(ring), seqno) ||
780 atomic_load_acq_int(&dev_priv->mm.wedged), 3000, 0, "915rtr")) {
781 ret = -EBUSY;
782 }
783 }
784 mtx_unlock(&ring->irq_lock);
785
788 ret = __wait_seqno(ring, seqno, true);
786 if (ret == 0)
787 taskqueue_enqueue_timeout(dev_priv->tq,
788 &dev_priv->mm.retire_task, 0);
789
790 return (ret);
791}
792
793int

--- 48 unchanged lines hidden (view full) ---

842 DRM_UNLOCK(dev);
843 return (ret);
844}
845
846void
847i915_gem_cleanup_ringbuffer(struct drm_device *dev)
848{
849 drm_i915_private_t *dev_priv;
789 if (ret == 0)
790 taskqueue_enqueue_timeout(dev_priv->tq,
791 &dev_priv->mm.retire_task, 0);
792
793 return (ret);
794}
795
796int

--- 48 unchanged lines hidden (view full) ---

845 DRM_UNLOCK(dev);
846 return (ret);
847}
848
849void
850i915_gem_cleanup_ringbuffer(struct drm_device *dev)
851{
852 drm_i915_private_t *dev_priv;
853 struct intel_ring_buffer *ring;
850 int i;
851
852 dev_priv = dev->dev_private;
854 int i;
855
856 dev_priv = dev->dev_private;
853 for (i = 0; i < I915_NUM_RINGS; i++)
854 intel_cleanup_ring_buffer(&dev_priv->rings[i]);
857 for_each_ring(ring, dev_priv, i)
858 intel_cleanup_ring_buffer(ring);
855}
856
857int
858i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
859 struct drm_file *file_priv)
860{
861 drm_i915_private_t *dev_priv;
859}
860
861int
862i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
863 struct drm_file *file_priv)
864{
865 drm_i915_private_t *dev_priv;
862 int ret, i;
866 int ret;
863
864 if (drm_core_check_feature(dev, DRIVER_MODESET))
865 return (0);
866 dev_priv = dev->dev_private;
867 if (atomic_load_acq_int(&dev_priv->mm.wedged) != 0) {
868 DRM_ERROR("Reenabling wedged hardware, good luck\n");
869 atomic_store_rel_int(&dev_priv->mm.wedged, 0);
870 }
871
872 dev_priv->mm.suspended = 0;
873
874 ret = i915_gem_init_hw(dev);
875 if (ret != 0) {
876 return (ret);
877 }
878
879 KASSERT(list_empty(&dev_priv->mm.active_list), ("active list"));
880 KASSERT(list_empty(&dev_priv->mm.flushing_list), ("flushing list"));
881 KASSERT(list_empty(&dev_priv->mm.inactive_list), ("inactive list"));
867
868 if (drm_core_check_feature(dev, DRIVER_MODESET))
869 return (0);
870 dev_priv = dev->dev_private;
871 if (atomic_load_acq_int(&dev_priv->mm.wedged) != 0) {
872 DRM_ERROR("Reenabling wedged hardware, good luck\n");
873 atomic_store_rel_int(&dev_priv->mm.wedged, 0);
874 }
875
876 dev_priv->mm.suspended = 0;
877
878 ret = i915_gem_init_hw(dev);
879 if (ret != 0) {
880 return (ret);
881 }
882
883 KASSERT(list_empty(&dev_priv->mm.active_list), ("active list"));
884 KASSERT(list_empty(&dev_priv->mm.flushing_list), ("flushing list"));
885 KASSERT(list_empty(&dev_priv->mm.inactive_list), ("inactive list"));
882 for (i = 0; i < I915_NUM_RINGS; i++) {
883 KASSERT(list_empty(&dev_priv->rings[i].active_list),
884 ("ring %d active list", i));
885 KASSERT(list_empty(&dev_priv->rings[i].request_list),
886 ("ring %d request list", i));
887 }
888
889 DRM_UNLOCK(dev);
890 ret = drm_irq_install(dev);
891 DRM_LOCK(dev);
892 if (ret)
893 goto cleanup_ringbuffer;
894
895 return (0);
896

--- 71 unchanged lines hidden (view full) ---

968i915_gem_create_ioctl(struct drm_device *dev, void *data,
969 struct drm_file *file)
970{
971 struct drm_i915_gem_create *args = data;
972
973 return (i915_gem_create(file, dev, args->size, &args->handle));
974}
975
886 DRM_UNLOCK(dev);
887 ret = drm_irq_install(dev);
888 DRM_LOCK(dev);
889 if (ret)
890 goto cleanup_ringbuffer;
891
892 return (0);
893

--- 71 unchanged lines hidden (view full) ---

965i915_gem_create_ioctl(struct drm_device *dev, void *data,
966 struct drm_file *file)
967{
968 struct drm_i915_gem_create *args = data;
969
970 return (i915_gem_create(file, dev, args->size, &args->handle));
971}
972
973#define __user
974#define __force
975#define __iomem
976#define to_user_ptr(x) ((void *)(uintptr_t)(x))
977#define offset_in_page(x) ((x) & PAGE_MASK)
978#define page_to_phys(x) VM_PAGE_TO_PHYS(x)
979static inline long
980__copy_to_user(void __user *to, const void *from, unsigned long n)
981{
982 return (copyout(from, to, n) != 0 ? n : 0);
983}
984static inline int
985__copy_to_user_inatomic(void __user *to, const void *from, unsigned n)
986{
987 return (copyout_nofault(from, to, n) != 0 ? n : 0);
988}
989static inline unsigned long
990__copy_from_user(void *to, const void __user *from, unsigned long n)
991{
992 return ((copyin(__DECONST(void *, from), to, n) != 0 ? n : 0));
993}
994#define copy_from_user(to, from, n) __copy_from_user((to), (from), (n))
995static inline unsigned long
996__copy_from_user_inatomic_nocache(void *to, const void __user *from,
997 unsigned long n)
998{
999
1000 /*
1001 * XXXKIB. Equivalent Linux function is implemented using
1002 * MOVNTI for aligned moves. For unaligned head and tail,
1003 * normal move is performed. As such, it is not incorrect, if
1004 * only somewhat slower, to use normal copyin. All uses
1005 * except shmem_pwrite_fast() have the destination mapped WC.
1006 */
1007 return ((copyin_nofault(__DECONST(void *, from), to, n) != 0 ? n : 0));
1008}
1009static inline int
1010fault_in_multipages_readable(const char __user *uaddr, int size)
1011{
1012 char c;
1013 int ret = 0;
1014 const char __user *end = uaddr + size - 1;
1015
1016 if (unlikely(size == 0))
1017 return ret;
1018
1019 while (uaddr <= end) {
1020 ret = copyin(uaddr, &c, 1);
1021 if (ret != 0)
1022 return -EFAULT;
1023 uaddr += PAGE_SIZE;
1024 }
1025
1026 /* Check whether the range spilled into the next page. */
1027 if (((unsigned long)uaddr & ~PAGE_MASK) ==
1028 ((unsigned long)end & ~PAGE_MASK)) {
1029 ret = copyin(end, &c, 1);
1030 }
1031
1032 return -ret;
1033}
1034
1035static inline int
1036fault_in_multipages_writeable(char __user *uaddr, int size)
1037{
1038 int ret = 0;
1039 char __user *end = uaddr + size - 1;
1040
1041 if (unlikely(size == 0))
1042 return ret;
1043
1044 /*
1045 * Writing zeroes into userspace here is OK, because we know that if
1046 * the zero gets there, we'll be overwriting it.
1047 */
1048 while (uaddr <= end) {
1049 ret = subyte(uaddr, 0);
1050 if (ret != 0)
1051 return -EFAULT;
1052 uaddr += PAGE_SIZE;
1053 }
1054
1055 /* Check whether the range spilled into the next page. */
1056 if (((unsigned long)uaddr & ~PAGE_MASK) ==
1057 ((unsigned long)end & ~PAGE_MASK))
1058 ret = subyte(end, 0);
1059
1060 return ret;
1061}
1062
1063static inline int
1064__copy_to_user_swizzled(char __user *cpu_vaddr,
1065 const char *gpu_vaddr, int gpu_offset,
1066 int length)
1067{
1068 int ret, cpu_offset = 0;
1069
1070 while (length > 0) {
1071 int cacheline_end = roundup2(gpu_offset + 1, 64);
1072 int this_length = min(cacheline_end - gpu_offset, length);
1073 int swizzled_gpu_offset = gpu_offset ^ 64;
1074
1075 ret = __copy_to_user(cpu_vaddr + cpu_offset,
1076 gpu_vaddr + swizzled_gpu_offset,
1077 this_length);
1078 if (ret)
1079 return ret + length;
1080
1081 cpu_offset += this_length;
1082 gpu_offset += this_length;
1083 length -= this_length;
1084 }
1085
1086 return 0;
1087}
1088
1089static inline int
1090__copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset,
1091 const char __user *cpu_vaddr,
1092 int length)
1093{
1094 int ret, cpu_offset = 0;
1095
1096 while (length > 0) {
1097 int cacheline_end = roundup2(gpu_offset + 1, 64);
1098 int this_length = min(cacheline_end - gpu_offset, length);
1099 int swizzled_gpu_offset = gpu_offset ^ 64;
1100
1101 ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset,
1102 cpu_vaddr + cpu_offset,
1103 this_length);
1104 if (ret)
1105 return ret + length;
1106
1107 cpu_offset += this_length;
1108 gpu_offset += this_length;
1109 length -= this_length;
1110 }
1111
1112 return 0;
1113}
1114
976static int
1115static int
977i915_gem_swap_io(struct drm_device *dev, struct drm_i915_gem_object *obj,
978 uint64_t data_ptr, uint64_t size, uint64_t offset, enum uio_rw rw,
979 struct drm_file *file)
1116i915_gem_phys_pwrite(struct drm_device *dev,
1117 struct drm_i915_gem_object *obj,
1118 struct drm_i915_gem_pwrite *args,
1119 struct drm_file *file_priv)
980{
1120{
981 vm_object_t vm_obj;
982 vm_page_t m;
983 struct sf_buf *sf;
984 vm_offset_t mkva;
985 vm_pindex_t obj_pi;
986 int cnt, do_bit17_swizzling, length, obj_po, ret, swizzled_po;
1121 void *vaddr = (char *)obj->phys_obj->handle->vaddr + args->offset;
1122 char __user *user_data = to_user_ptr(args->data_ptr);
987
1123
988 if (obj->gtt_offset != 0 && rw == UIO_READ)
989 do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
990 else
991 do_bit17_swizzling = 0;
1124 if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) {
1125 unsigned long unwritten;
992
1126
993 obj->dirty = 1;
994 vm_obj = obj->base.vm_obj;
995 ret = 0;
1127 /* The physical object once assigned is fixed for the lifetime
1128 * of the obj, so we can safely drop the lock and continue
1129 * to access vaddr.
1130 */
1131 DRM_UNLOCK(dev);
1132 unwritten = copy_from_user(vaddr, user_data, args->size);
1133 DRM_LOCK(dev);
1134 if (unwritten)
1135 return -EFAULT;
1136 }
996
1137
997 VM_OBJECT_WLOCK(vm_obj);
998 vm_object_pip_add(vm_obj, 1);
999 while (size > 0) {
1000 obj_pi = OFF_TO_IDX(offset);
1001 obj_po = offset & PAGE_MASK;
1138 i915_gem_chipset_flush(dev);
1139 return 0;
1140}
1002
1141
1003 m = i915_gem_wire_page(vm_obj, obj_pi);
1004 VM_OBJECT_WUNLOCK(vm_obj);
1142/* Per-page copy function for the shmem pread fastpath.
1143 * Flushes invalid cachelines before reading the target if
1144 * needs_clflush is set. */
1145static int
1146shmem_pread_fast(vm_page_t page, int shmem_page_offset, int page_length,
1147 char __user *user_data,
1148 bool page_do_bit17_swizzling, bool needs_clflush)
1149{
1150 char *vaddr;
1151 struct sf_buf *sf;
1152 int ret;
1005
1153
1006 sched_pin();
1007 sf = sf_buf_alloc(m, SFB_CPUPRIVATE);
1008 mkva = sf_buf_kva(sf);
1009 length = min(size, PAGE_SIZE - obj_po);
1010 while (length > 0) {
1011 if (do_bit17_swizzling &&
1012 (VM_PAGE_TO_PHYS(m) & (1 << 17)) != 0) {
1013 cnt = roundup2(obj_po + 1, 64);
1014 cnt = min(cnt - obj_po, length);
1015 swizzled_po = obj_po ^ 64;
1016 } else {
1017 cnt = length;
1018 swizzled_po = obj_po;
1019 }
1020 if (rw == UIO_READ)
1021 ret = -copyout_nofault(
1022 (char *)mkva + swizzled_po,
1023 (void *)(uintptr_t)data_ptr, cnt);
1024 else
1025 ret = -copyin_nofault(
1026 (void *)(uintptr_t)data_ptr,
1027 (char *)mkva + swizzled_po, cnt);
1028 if (ret != 0)
1029 break;
1030 data_ptr += cnt;
1031 size -= cnt;
1032 length -= cnt;
1033 offset += cnt;
1034 obj_po += cnt;
1035 }
1036 sf_buf_free(sf);
1154 if (unlikely(page_do_bit17_swizzling))
1155 return -EINVAL;
1156
1157 sched_pin();
1158 sf = sf_buf_alloc(page, SFB_NOWAIT | SFB_CPUPRIVATE);
1159 if (sf == NULL) {
1037 sched_unpin();
1160 sched_unpin();
1038 VM_OBJECT_WLOCK(vm_obj);
1039 if (rw == UIO_WRITE)
1040 vm_page_dirty(m);
1041 vm_page_reference(m);
1042 vm_page_lock(m);
1043 vm_page_unwire(m, PQ_ACTIVE);
1044 vm_page_unlock(m);
1045 atomic_add_long(&i915_gem_wired_pages_cnt, -1);
1161 return (-EFAULT);
1162 }
1163 vaddr = (char *)sf_buf_kva(sf);
1164 if (needs_clflush)
1165 drm_clflush_virt_range(vaddr + shmem_page_offset,
1166 page_length);
1167 ret = __copy_to_user_inatomic(user_data,
1168 vaddr + shmem_page_offset,
1169 page_length);
1170 sf_buf_free(sf);
1171 sched_unpin();
1046
1172
1047 if (ret != 0)
1048 break;
1173 return ret ? -EFAULT : 0;
1174}
1175
1176static void
1177shmem_clflush_swizzled_range(char *addr, unsigned long length,
1178 bool swizzled)
1179{
1180 if (unlikely(swizzled)) {
1181 unsigned long start = (unsigned long) addr;
1182 unsigned long end = (unsigned long) addr + length;
1183
1184 /* For swizzling simply ensure that we always flush both
1185 * channels. Lame, but simple and it works. Swizzled
1186 * pwrite/pread is far from a hotpath - current userspace
1187 * doesn't use it at all. */
1188 start = rounddown2(start, 128);
1189 end = roundup2(end, 128);
1190
1191 drm_clflush_virt_range((void *)start, end - start);
1192 } else {
1193 drm_clflush_virt_range(addr, length);
1049 }
1194 }
1050 vm_object_pip_wakeup(vm_obj);
1051 VM_OBJECT_WUNLOCK(vm_obj);
1052
1195
1053 return (ret);
1054}
1055
1196}
1197
1198/* Only difference to the fast-path function is that this can handle bit17
1199 * and uses non-atomic copy and kmap functions. */
1056static int
1200static int
1057i915_gem_gtt_write(struct drm_device *dev, struct drm_i915_gem_object *obj,
1058 uint64_t data_ptr, uint64_t size, uint64_t offset, struct drm_file *file)
1201shmem_pread_slow(vm_page_t page, int shmem_page_offset, int page_length,
1202 char __user *user_data,
1203 bool page_do_bit17_swizzling, bool needs_clflush)
1059{
1204{
1060 vm_offset_t mkva;
1061 vm_pindex_t obj_pi;
1062 int obj_po, ret;
1205 char *vaddr;
1206 struct sf_buf *sf;
1207 int ret;
1063
1208
1064 obj_pi = OFF_TO_IDX(offset);
1065 obj_po = offset & PAGE_MASK;
1209 sf = sf_buf_alloc(page, 0);
1210 vaddr = (char *)sf_buf_kva(sf);
1211 if (needs_clflush)
1212 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
1213 page_length,
1214 page_do_bit17_swizzling);
1066
1215
1067 mkva = (vm_offset_t)pmap_mapdev_attr(dev->agp->base + obj->gtt_offset +
1068 IDX_TO_OFF(obj_pi), size, PAT_WRITE_COMBINING);
1069 ret = -copyin_nofault((void *)(uintptr_t)data_ptr, (char *)mkva +
1070 obj_po, size);
1071 pmap_unmapdev(mkva, size);
1072 return (ret);
1216 if (page_do_bit17_swizzling)
1217 ret = __copy_to_user_swizzled(user_data,
1218 vaddr, shmem_page_offset,
1219 page_length);
1220 else
1221 ret = __copy_to_user(user_data,
1222 vaddr + shmem_page_offset,
1223 page_length);
1224 sf_buf_free(sf);
1225
1226 return ret ? - EFAULT : 0;
1073}
1074
1075static int
1227}
1228
1229static int
1076i915_gem_obj_io(struct drm_device *dev, uint32_t handle, uint64_t data_ptr,
1077 uint64_t size, uint64_t offset, enum uio_rw rw, struct drm_file *file)
1230i915_gem_shmem_pread(struct drm_device *dev,
1231 struct drm_i915_gem_object *obj,
1232 struct drm_i915_gem_pread *args,
1233 struct drm_file *file)
1078{
1234{
1079 struct drm_i915_gem_object *obj;
1080 vm_page_t *ma;
1081 vm_offset_t start, end;
1082 int npages, ret;
1235 char __user *user_data;
1236 ssize_t remain, sremain;
1237 off_t offset, soffset;
1238 int shmem_page_offset, page_length, ret = 0;
1239 int obj_do_bit17_swizzling, page_do_bit17_swizzling;
1240 int prefaulted = 0;
1241 int needs_clflush = 0;
1083
1242
1084 if (size == 0)
1085 return (0);
1086 start = trunc_page(data_ptr);
1087 end = round_page(data_ptr + size);
1088 npages = howmany(end - start, PAGE_SIZE);
1089 ma = malloc(npages * sizeof(vm_page_t), DRM_I915_GEM, M_WAITOK |
1090 M_ZERO);
1091 npages = vm_fault_quick_hold_pages(&curproc->p_vmspace->vm_map,
1092 (vm_offset_t)data_ptr, size,
1093 (rw == UIO_READ ? VM_PROT_WRITE : 0 ) | VM_PROT_READ, ma, npages);
1094 if (npages == -1) {
1095 ret = -EFAULT;
1096 goto free_ma;
1243 user_data = to_user_ptr(args->data_ptr);
1244 sremain = remain = args->size;
1245
1246 obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
1247
1248 if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) {
1249 /* If we're not in the cpu read domain, set ourself into the gtt
1250 * read domain and manually flush cachelines (if required). This
1251 * optimizes for the case when the gpu will dirty the data
1252 * anyway again before the next pread happens. */
1253 needs_clflush = !cpu_cache_is_coherent(dev, obj->cache_level);
1254 ret = i915_gem_object_set_to_gtt_domain(obj, false);
1255 if (ret)
1256 return ret;
1097 }
1098
1257 }
1258
1259 soffset = offset = args->offset;
1260 ret = i915_gem_object_get_pages_range(obj, soffset, soffset + sremain);
1261 if (ret)
1262 return ret;
1263
1264 i915_gem_object_pin_pages(obj);
1265
1266 VM_OBJECT_WLOCK(obj->base.vm_obj);
1267 for (vm_page_t page = vm_page_find_least(obj->base.vm_obj,
1268 OFF_TO_IDX(offset));; page = vm_page_next(page)) {
1269 VM_OBJECT_WUNLOCK(obj->base.vm_obj);
1270
1271 if (remain <= 0)
1272 break;
1273
1274 /* Operation in this page
1275 *
1276 * shmem_page_offset = offset within page in shmem file
1277 * page_length = bytes to copy for this page
1278 */
1279 shmem_page_offset = offset_in_page(offset);
1280 page_length = remain;
1281 if ((shmem_page_offset + page_length) > PAGE_SIZE)
1282 page_length = PAGE_SIZE - shmem_page_offset;
1283
1284 page_do_bit17_swizzling = obj_do_bit17_swizzling &&
1285 (page_to_phys(page) & (1 << 17)) != 0;
1286
1287 ret = shmem_pread_fast(page, shmem_page_offset, page_length,
1288 user_data, page_do_bit17_swizzling,
1289 needs_clflush);
1290 if (ret == 0)
1291 goto next_page;
1292
1293 DRM_UNLOCK(dev);
1294
1295 if (likely(!i915_prefault_disable) && !prefaulted) {
1296 ret = fault_in_multipages_writeable(user_data, remain);
1297 /* Userspace is tricking us, but we've already clobbered
1298 * its pages with the prefault and promised to write the
1299 * data up to the first fault. Hence ignore any errors
1300 * and just continue. */
1301 (void)ret;
1302 prefaulted = 1;
1303 }
1304
1305 ret = shmem_pread_slow(page, shmem_page_offset, page_length,
1306 user_data, page_do_bit17_swizzling,
1307 needs_clflush);
1308
1309 DRM_LOCK(dev);
1310
1311next_page:
1312 vm_page_reference(page);
1313
1314 if (ret)
1315 goto out;
1316
1317 remain -= page_length;
1318 user_data += page_length;
1319 offset += page_length;
1320 VM_OBJECT_WLOCK(obj->base.vm_obj);
1321 }
1322
1323out:
1324 i915_gem_object_unpin_pages(obj);
1325 i915_gem_object_put_pages_range(obj, soffset, soffset + sremain);
1326
1327 return ret;
1328}
1329
1330/**
1331 * Reads data from the object referenced by handle.
1332 *
1333 * On error, the contents of *data are undefined.
1334 */
1335int
1336i915_gem_pread_ioctl(struct drm_device *dev, void *data,
1337 struct drm_file *file)
1338{
1339 struct drm_i915_gem_pread *args = data;
1340 struct drm_i915_gem_object *obj;
1341 int ret = 0;
1342
1343 if (args->size == 0)
1344 return 0;
1345
1346 if (!useracc(to_user_ptr(args->data_ptr), args->size, VM_PROT_WRITE))
1347 return -EFAULT;
1348
1099 ret = i915_mutex_lock_interruptible(dev);
1349 ret = i915_mutex_lock_interruptible(dev);
1100 if (ret != 0)
1101 goto unlocked;
1350 if (ret)
1351 return ret;
1102
1352
1103 obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
1353 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1104 if (&obj->base == NULL) {
1105 ret = -ENOENT;
1106 goto unlock;
1107 }
1354 if (&obj->base == NULL) {
1355 ret = -ENOENT;
1356 goto unlock;
1357 }
1108 if (offset > obj->base.size || size > obj->base.size - offset) {
1358
1359 /* Bounds check source. */
1360 if (args->offset > obj->base.size ||
1361 args->size > obj->base.size - args->offset) {
1109 ret = -EINVAL;
1110 goto out;
1111 }
1112
1362 ret = -EINVAL;
1363 goto out;
1364 }
1365
1113 if (rw == UIO_READ) {
1114 CTR3(KTR_DRM, "object_pread %p %jx %jx", obj, offset, size);
1115 ret = i915_gem_object_set_cpu_read_domain_range(obj,
1116 offset, size);
1117 if (ret != 0)
1118 goto out;
1119 ret = i915_gem_swap_io(dev, obj, data_ptr, size, offset,
1120 UIO_READ, file);
1121 } else {
1122 if (obj->phys_obj) {
1123 CTR3(KTR_DRM, "object_phys_write %p %jx %jx", obj,
1124 offset, size);
1125 ret = i915_gem_phys_pwrite(dev, obj, data_ptr, offset,
1126 size, file);
1127 } else if (obj->gtt_space &&
1128 obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
1129 CTR3(KTR_DRM, "object_gtt_write %p %jx %jx", obj,
1130 offset, size);
1131 ret = i915_gem_object_pin(obj, 0, true);
1132 if (ret != 0)
1133 goto out;
1134 ret = i915_gem_object_set_to_gtt_domain(obj, true);
1135 if (ret != 0)
1136 goto out_unpin;
1137 ret = i915_gem_object_put_fence(obj);
1138 if (ret != 0)
1139 goto out_unpin;
1140 ret = i915_gem_gtt_write(dev, obj, data_ptr, size,
1141 offset, file);
1142out_unpin:
1143 i915_gem_object_unpin(obj);
1144 } else {
1145 CTR3(KTR_DRM, "object_pwrite %p %jx %jx", obj,
1146 offset, size);
1147 ret = i915_gem_object_set_to_cpu_domain(obj, true);
1148 if (ret != 0)
1149 goto out;
1150 ret = i915_gem_swap_io(dev, obj, data_ptr, size, offset,
1151 UIO_WRITE, file);
1152 }
1366#if 1
1367 KIB_NOTYET();
1368#else
1369 /* prime objects have no backing filp to GEM pread/pwrite
1370 * pages from.
1371 */
1372 if (!obj->base.filp) {
1373 ret = -EINVAL;
1374 goto out;
1153 }
1375 }
1376#endif
1377
1378 CTR3(KTR_DRM, "pread %p %jx %jx", obj, args->offset, args->size);
1379
1380 ret = i915_gem_shmem_pread(dev, obj, args, file);
1381
1154out:
1155 drm_gem_object_unreference(&obj->base);
1156unlock:
1157 DRM_UNLOCK(dev);
1382out:
1383 drm_gem_object_unreference(&obj->base);
1384unlock:
1385 DRM_UNLOCK(dev);
1158unlocked:
1159 vm_page_unhold_pages(ma, npages);
1160free_ma:
1161 free(ma, DRM_I915_GEM);
1162 return (ret);
1386 return ret;
1163}
1164
1387}
1388
1165int
1166i915_gem_pread_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
1389/* This is the fast write path which cannot handle
1390 * page faults in the source data
1391 */
1392
1393static inline int
1394fast_user_write(struct drm_device *dev,
1395 off_t page_base, int page_offset,
1396 char __user *user_data,
1397 int length)
1167{
1398{
1168 struct drm_i915_gem_pread *args;
1399 void __iomem *vaddr_atomic;
1400 void *vaddr;
1401 unsigned long unwritten;
1169
1402
1170 args = data;
1171 return (i915_gem_obj_io(dev, args->handle, args->data_ptr, args->size,
1172 args->offset, UIO_READ, file));
1403 vaddr_atomic = pmap_mapdev_attr(dev->agp->base + page_base,
1404 length, PAT_WRITE_COMBINING);
1405 /* We can use the cpu mem copy function because this is X86. */
1406 vaddr = (char *)vaddr_atomic + page_offset;
1407 unwritten = __copy_from_user_inatomic_nocache(vaddr,
1408 user_data, length);
1409 pmap_unmapdev((vm_offset_t)vaddr_atomic, length);
1410 return unwritten;
1173}
1174
1411}
1412
1413/**
1414 * This is the fast pwrite path, where we copy the data directly from the
1415 * user into the GTT, uncached.
1416 */
1417static int
1418i915_gem_gtt_pwrite_fast(struct drm_device *dev,
1419 struct drm_i915_gem_object *obj,
1420 struct drm_i915_gem_pwrite *args,
1421 struct drm_file *file)
1422{
1423 ssize_t remain;
1424 off_t offset, page_base;
1425 char __user *user_data;
1426 int page_offset, page_length, ret;
1427
1428 ret = i915_gem_object_pin(obj, 0, true);
1429 /* XXXKIB ret = i915_gem_obj_ggtt_pin(obj, 0, true, true); */
1430 if (ret != 0)
1431 goto out;
1432
1433 ret = i915_gem_object_set_to_gtt_domain(obj, true);
1434 if (ret)
1435 goto out_unpin;
1436
1437 ret = i915_gem_object_put_fence(obj);
1438 if (ret)
1439 goto out_unpin;
1440
1441 user_data = to_user_ptr(args->data_ptr);
1442 remain = args->size;
1443
1444 offset = obj->gtt_offset + args->offset;
1445
1446 while (remain > 0) {
1447 /* Operation in this page
1448 *
1449 * page_base = page offset within aperture
1450 * page_offset = offset within page
1451 * page_length = bytes to copy for this page
1452 */
1453 page_base = offset & ~PAGE_MASK;
1454 page_offset = offset_in_page(offset);
1455 page_length = remain;
1456 if ((page_offset + remain) > PAGE_SIZE)
1457 page_length = PAGE_SIZE - page_offset;
1458
1459 /* If we get a fault while copying data, then (presumably) our
1460 * source page isn't available. Return the error and we'll
1461 * retry in the slow path.
1462 */
1463 if (fast_user_write(dev, page_base,
1464 page_offset, user_data, page_length)) {
1465 ret = -EFAULT;
1466 goto out_unpin;
1467 }
1468
1469 remain -= page_length;
1470 user_data += page_length;
1471 offset += page_length;
1472 }
1473
1474out_unpin:
1475 i915_gem_object_unpin(obj);
1476out:
1477 return ret;
1478}
1479
1480/* Per-page copy function for the shmem pwrite fastpath.
1481 * Flushes invalid cachelines before writing to the target if
1482 * needs_clflush_before is set and flushes out any written cachelines after
1483 * writing if needs_clflush is set. */
1484static int
1485shmem_pwrite_fast(vm_page_t page, int shmem_page_offset, int page_length,
1486 char __user *user_data,
1487 bool page_do_bit17_swizzling,
1488 bool needs_clflush_before,
1489 bool needs_clflush_after)
1490{
1491 char *vaddr;
1492 struct sf_buf *sf;
1493 int ret;
1494
1495 if (unlikely(page_do_bit17_swizzling))
1496 return -EINVAL;
1497
1498 sched_pin();
1499 sf = sf_buf_alloc(page, SFB_NOWAIT | SFB_CPUPRIVATE);
1500 if (sf == NULL) {
1501 sched_unpin();
1502 return (-EFAULT);
1503 }
1504 vaddr = (char *)sf_buf_kva(sf);
1505 if (needs_clflush_before)
1506 drm_clflush_virt_range(vaddr + shmem_page_offset,
1507 page_length);
1508 ret = __copy_from_user_inatomic_nocache(vaddr + shmem_page_offset,
1509 user_data,
1510 page_length);
1511 if (needs_clflush_after)
1512 drm_clflush_virt_range(vaddr + shmem_page_offset,
1513 page_length);
1514 sf_buf_free(sf);
1515 sched_unpin();
1516
1517 return ret ? -EFAULT : 0;
1518}
1519
1520/* Only difference to the fast-path function is that this can handle bit17
1521 * and uses non-atomic copy and kmap functions. */
1522static int
1523shmem_pwrite_slow(vm_page_t page, int shmem_page_offset, int page_length,
1524 char __user *user_data,
1525 bool page_do_bit17_swizzling,
1526 bool needs_clflush_before,
1527 bool needs_clflush_after)
1528{
1529 char *vaddr;
1530 struct sf_buf *sf;
1531 int ret;
1532
1533 sf = sf_buf_alloc(page, 0);
1534 vaddr = (char *)sf_buf_kva(sf);
1535 if (unlikely(needs_clflush_before || page_do_bit17_swizzling))
1536 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
1537 page_length,
1538 page_do_bit17_swizzling);
1539 if (page_do_bit17_swizzling)
1540 ret = __copy_from_user_swizzled(vaddr, shmem_page_offset,
1541 user_data,
1542 page_length);
1543 else
1544 ret = __copy_from_user(vaddr + shmem_page_offset,
1545 user_data,
1546 page_length);
1547 if (needs_clflush_after)
1548 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
1549 page_length,
1550 page_do_bit17_swizzling);
1551 sf_buf_free(sf);
1552
1553 return ret ? -EFAULT : 0;
1554}
1555
1556static int
1557i915_gem_shmem_pwrite(struct drm_device *dev,
1558 struct drm_i915_gem_object *obj,
1559 struct drm_i915_gem_pwrite *args,
1560 struct drm_file *file)
1561{
1562 ssize_t remain, sremain;
1563 off_t offset, soffset;
1564 char __user *user_data;
1565 int shmem_page_offset, page_length, ret = 0;
1566 int obj_do_bit17_swizzling, page_do_bit17_swizzling;
1567 int hit_slowpath = 0;
1568 int needs_clflush_after = 0;
1569 int needs_clflush_before = 0;
1570
1571 user_data = to_user_ptr(args->data_ptr);
1572 sremain = remain = args->size;
1573
1574 obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
1575
1576 if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
1577 /* If we're not in the cpu write domain, set ourself into the gtt
1578 * write domain and manually flush cachelines (if required). This
1579 * optimizes for the case when the gpu will use the data
1580 * right away and we therefore have to clflush anyway. */
1581 needs_clflush_after = cpu_write_needs_clflush(obj);
1582 ret = i915_gem_object_set_to_gtt_domain(obj, true);
1583 if (ret)
1584 return ret;
1585 }
1586 /* Same trick applies to invalidate partially written cachelines read
1587 * before writing. */
1588 if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0)
1589 needs_clflush_before =
1590 !cpu_cache_is_coherent(dev, obj->cache_level);
1591
1592 soffset = offset = args->offset;
1593 ret = i915_gem_object_get_pages_range(obj, soffset, soffset + sremain);
1594 if (ret)
1595 return ret;
1596
1597 i915_gem_object_pin_pages(obj);
1598
1599 obj->dirty = 1;
1600
1601 VM_OBJECT_WLOCK(obj->base.vm_obj);
1602 for (vm_page_t page = vm_page_find_least(obj->base.vm_obj,
1603 OFF_TO_IDX(offset));; page = vm_page_next(page)) {
1604 VM_OBJECT_WUNLOCK(obj->base.vm_obj);
1605 int partial_cacheline_write;
1606
1607 if (remain <= 0)
1608 break;
1609
1610 /* Operation in this page
1611 *
1612 * shmem_page_offset = offset within page in shmem file
1613 * page_length = bytes to copy for this page
1614 */
1615 shmem_page_offset = offset_in_page(offset);
1616
1617 page_length = remain;
1618 if ((shmem_page_offset + page_length) > PAGE_SIZE)
1619 page_length = PAGE_SIZE - shmem_page_offset;
1620
1621 /* If we don't overwrite a cacheline completely we need to be
1622 * careful to have up-to-date data by first clflushing. Don't
1623 * overcomplicate things and flush the entire patch. */
1624 partial_cacheline_write = needs_clflush_before &&
1625 ((shmem_page_offset | page_length)
1626 & (cpu_clflush_line_size - 1));
1627
1628 page_do_bit17_swizzling = obj_do_bit17_swizzling &&
1629 (page_to_phys(page) & (1 << 17)) != 0;
1630
1631 ret = shmem_pwrite_fast(page, shmem_page_offset, page_length,
1632 user_data, page_do_bit17_swizzling,
1633 partial_cacheline_write,
1634 needs_clflush_after);
1635 if (ret == 0)
1636 goto next_page;
1637
1638 hit_slowpath = 1;
1639 DRM_UNLOCK(dev);
1640 ret = shmem_pwrite_slow(page, shmem_page_offset, page_length,
1641 user_data, page_do_bit17_swizzling,
1642 partial_cacheline_write,
1643 needs_clflush_after);
1644
1645 DRM_LOCK(dev);
1646
1647next_page:
1648 vm_page_dirty(page);
1649 vm_page_reference(page);
1650
1651 if (ret)
1652 goto out;
1653
1654 remain -= page_length;
1655 user_data += page_length;
1656 offset += page_length;
1657 VM_OBJECT_WLOCK(obj->base.vm_obj);
1658 }
1659
1660out:
1661 i915_gem_object_unpin_pages(obj);
1662 i915_gem_object_put_pages_range(obj, soffset, soffset + sremain);
1663
1664 if (hit_slowpath) {
1665 /*
1666 * Fixup: Flush cpu caches in case we didn't flush the dirty
1667 * cachelines in-line while writing and the object moved
1668 * out of the cpu write domain while we've dropped the lock.
1669 */
1670 if (!needs_clflush_after &&
1671 obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
1672 i915_gem_clflush_object(obj);
1673 i915_gem_chipset_flush(dev);
1674 }
1675 }
1676
1677 if (needs_clflush_after)
1678 i915_gem_chipset_flush(dev);
1679
1680 return ret;
1681}
1682
1683/**
1684 * Writes data to the object referenced by handle.
1685 *
1686 * On error, the contents of the buffer that were to be modified are undefined.
1687 */
1175int
1688int
1176i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
1689i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
1690 struct drm_file *file)
1177{
1691{
1178 struct drm_i915_gem_pwrite *args;
1692 struct drm_i915_gem_pwrite *args = data;
1693 struct drm_i915_gem_object *obj;
1694 int ret;
1179
1695
1180 args = data;
1181 return (i915_gem_obj_io(dev, args->handle, args->data_ptr, args->size,
1182 args->offset, UIO_WRITE, file));
1696 if (args->size == 0)
1697 return 0;
1698
1699 if (!useracc(to_user_ptr(args->data_ptr), args->size, VM_PROT_READ))
1700 return -EFAULT;
1701
1702 if (likely(!i915_prefault_disable)) {
1703 ret = fault_in_multipages_readable(to_user_ptr(args->data_ptr),
1704 args->size);
1705 if (ret)
1706 return -EFAULT;
1707 }
1708
1709 ret = i915_mutex_lock_interruptible(dev);
1710 if (ret)
1711 return ret;
1712
1713 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1714 if (&obj->base == NULL) {
1715 ret = -ENOENT;
1716 goto unlock;
1717 }
1718
1719 /* Bounds check destination. */
1720 if (args->offset > obj->base.size ||
1721 args->size > obj->base.size - args->offset) {
1722 ret = -EINVAL;
1723 goto out;
1724 }
1725
1726#if 1
1727 KIB_NOTYET();
1728#else
1729 /* prime objects have no backing filp to GEM pread/pwrite
1730 * pages from.
1731 */
1732 if (!obj->base.filp) {
1733 ret = -EINVAL;
1734 goto out;
1735 }
1736#endif
1737
1738 CTR3(KTR_DRM, "pwrite %p %jx %jx", obj, args->offset, args->size);
1739
1740 ret = -EFAULT;
1741 /* We can only do the GTT pwrite on untiled buffers, as otherwise
1742 * it would end up going through the fenced access, and we'll get
1743 * different detiling behavior between reading and writing.
1744 * pread/pwrite currently are reading and writing from the CPU
1745 * perspective, requiring manual detiling by the client.
1746 */
1747 if (obj->phys_obj) {
1748 ret = i915_gem_phys_pwrite(dev, obj, args, file);
1749 goto out;
1750 }
1751
1752 if (obj->tiling_mode == I915_TILING_NONE &&
1753 obj->base.write_domain != I915_GEM_DOMAIN_CPU &&
1754 cpu_write_needs_clflush(obj)) {
1755 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
1756 /* Note that the gtt paths might fail with non-page-backed user
1757 * pointers (e.g. gtt mappings when moving data between
1758 * textures). Fallback to the shmem path in that case. */
1759 }
1760
1761 if (ret == -EFAULT || ret == -ENOSPC)
1762 ret = i915_gem_shmem_pwrite(dev, obj, args, file);
1763
1764out:
1765 drm_gem_object_unreference(&obj->base);
1766unlock:
1767 DRM_UNLOCK(dev);
1768 return ret;
1183}
1769}
1770#undef __user
1771#undef __force
1772#undef __iomem
1773#undef to_user_ptr
1774#undef offset_in_page
1775#undef page_to_phys
1184
1185int
1186i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
1187 struct drm_file *file)
1188{
1189 struct drm_i915_gem_set_domain *args;
1190 struct drm_i915_gem_object *obj;
1191 uint32_t read_domains;
1192 uint32_t write_domain;
1193 int ret;
1194
1776
1777int
1778i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
1779 struct drm_file *file)
1780{
1781 struct drm_i915_gem_set_domain *args;
1782 struct drm_i915_gem_object *obj;
1783 uint32_t read_domains;
1784 uint32_t write_domain;
1785 int ret;
1786
1195 if ((dev->driver->driver_features & DRIVER_GEM) == 0)
1196 return (-ENODEV);
1197
1198 args = data;
1199 read_domains = args->read_domains;
1200 write_domain = args->write_domain;
1201
1202 if ((write_domain & I915_GEM_GPU_DOMAINS) != 0 ||
1203 (read_domains & I915_GEM_GPU_DOMAINS) != 0 ||
1204 (write_domain != 0 && read_domains != write_domain))
1205 return (-EINVAL);

--- 25 unchanged lines hidden (view full) ---

1231i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1232 struct drm_file *file)
1233{
1234 struct drm_i915_gem_sw_finish *args;
1235 struct drm_i915_gem_object *obj;
1236 int ret;
1237
1238 args = data;
1787 args = data;
1788 read_domains = args->read_domains;
1789 write_domain = args->write_domain;
1790
1791 if ((write_domain & I915_GEM_GPU_DOMAINS) != 0 ||
1792 (read_domains & I915_GEM_GPU_DOMAINS) != 0 ||
1793 (write_domain != 0 && read_domains != write_domain))
1794 return (-EINVAL);

--- 25 unchanged lines hidden (view full) ---

1820i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1821 struct drm_file *file)
1822{
1823 struct drm_i915_gem_sw_finish *args;
1824 struct drm_i915_gem_object *obj;
1825 int ret;
1826
1827 args = data;
1239 ret = 0;
1240 if ((dev->driver->driver_features & DRIVER_GEM) == 0)
1241 return (ENODEV);
1828
1242 ret = i915_mutex_lock_interruptible(dev);
1243 if (ret != 0)
1244 return (ret);
1245 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1246 if (&obj->base == NULL) {
1247 ret = -ENOENT;
1248 goto unlock;
1249 }

--- 14 unchanged lines hidden (view full) ---

1264 struct proc *p;
1265 vm_map_t map;
1266 vm_offset_t addr;
1267 vm_size_t size;
1268 int error, rv;
1269
1270 args = data;
1271
1829 ret = i915_mutex_lock_interruptible(dev);
1830 if (ret != 0)
1831 return (ret);
1832 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1833 if (&obj->base == NULL) {
1834 ret = -ENOENT;
1835 goto unlock;
1836 }

--- 14 unchanged lines hidden (view full) ---

1851 struct proc *p;
1852 vm_map_t map;
1853 vm_offset_t addr;
1854 vm_size_t size;
1855 int error, rv;
1856
1857 args = data;
1858
1272 if ((dev->driver->driver_features & DRIVER_GEM) == 0)
1273 return (-ENODEV);
1274
1275 obj = drm_gem_object_lookup(dev, file, args->handle);
1276 if (obj == NULL)
1277 return (-ENOENT);
1278 error = 0;
1279 if (args->size == 0)
1280 goto out;
1281 p = curproc;
1282 map = &p->p_vmspace->vm_map;

--- 127 unchanged lines hidden (view full) ---

1410
1411 ret = i915_gem_object_set_to_gtt_domain(obj, write);
1412 if (ret != 0) {
1413 cause = 40;
1414 goto unlock;
1415 }
1416 }
1417
1859 obj = drm_gem_object_lookup(dev, file, args->handle);
1860 if (obj == NULL)
1861 return (-ENOENT);
1862 error = 0;
1863 if (args->size == 0)
1864 goto out;
1865 p = curproc;
1866 map = &p->p_vmspace->vm_map;

--- 127 unchanged lines hidden (view full) ---

1994
1995 ret = i915_gem_object_set_to_gtt_domain(obj, write);
1996 if (ret != 0) {
1997 cause = 40;
1998 goto unlock;
1999 }
2000 }
2001
1418 if (obj->tiling_mode == I915_TILING_NONE)
1419 ret = i915_gem_object_put_fence(obj);
1420 else
1421 ret = i915_gem_object_get_fence(obj, NULL);
2002 if (!obj->has_global_gtt_mapping)
2003 i915_gem_gtt_bind_object(obj, obj->cache_level);
2004
2005 ret = i915_gem_object_get_fence(obj);
1422 if (ret != 0) {
1423 cause = 50;
1424 goto unlock;
1425 }
1426
1427 if (i915_gem_object_is_inactive(obj))
1428 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
1429

--- 82 unchanged lines hidden (view full) ---

1512int
1513i915_gem_mmap_gtt(struct drm_file *file, struct drm_device *dev,
1514 uint32_t handle, uint64_t *offset)
1515{
1516 struct drm_i915_private *dev_priv;
1517 struct drm_i915_gem_object *obj;
1518 int ret;
1519
2006 if (ret != 0) {
2007 cause = 50;
2008 goto unlock;
2009 }
2010
2011 if (i915_gem_object_is_inactive(obj))
2012 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
2013

--- 82 unchanged lines hidden (view full) ---

2096int
2097i915_gem_mmap_gtt(struct drm_file *file, struct drm_device *dev,
2098 uint32_t handle, uint64_t *offset)
2099{
2100 struct drm_i915_private *dev_priv;
2101 struct drm_i915_gem_object *obj;
2102 int ret;
2103
1520 if (!(dev->driver->driver_features & DRIVER_GEM))
1521 return (-ENODEV);
1522
1523 dev_priv = dev->dev_private;
1524
1525 ret = i915_mutex_lock_interruptible(dev);
1526 if (ret != 0)
1527 return (ret);
1528
1529 obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
1530 if (&obj->base == NULL) {

--- 143 unchanged lines hidden (view full) ---

1674
1675 CTR3(KTR_DRM, "object_change_domain flush gtt_write %p %x %x", obj,
1676 obj->base.read_domains, old_write_domain);
1677}
1678
1679int
1680i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
1681{
2104 dev_priv = dev->dev_private;
2105
2106 ret = i915_mutex_lock_interruptible(dev);
2107 if (ret != 0)
2108 return (ret);
2109
2110 obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
2111 if (&obj->base == NULL) {

--- 143 unchanged lines hidden (view full) ---

2255
2256 CTR3(KTR_DRM, "object_change_domain flush gtt_write %p %x %x", obj,
2257 obj->base.read_domains, old_write_domain);
2258}
2259
2260int
2261i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
2262{
2263 drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
1682 uint32_t old_write_domain, old_read_domains;
1683 int ret;
1684
1685 if (obj->gtt_space == NULL)
1686 return (-EINVAL);
1687
1688 if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
1689 return 0;

--- 17 unchanged lines hidden (view full) ---

1707 ("In GTT write domain"));
1708 obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
1709 if (write) {
1710 obj->base.read_domains = I915_GEM_DOMAIN_GTT;
1711 obj->base.write_domain = I915_GEM_DOMAIN_GTT;
1712 obj->dirty = 1;
1713 }
1714
2264 uint32_t old_write_domain, old_read_domains;
2265 int ret;
2266
2267 if (obj->gtt_space == NULL)
2268 return (-EINVAL);
2269
2270 if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
2271 return 0;

--- 17 unchanged lines hidden (view full) ---

2289 ("In GTT write domain"));
2290 obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
2291 if (write) {
2292 obj->base.read_domains = I915_GEM_DOMAIN_GTT;
2293 obj->base.write_domain = I915_GEM_DOMAIN_GTT;
2294 obj->dirty = 1;
2295 }
2296
2297 /* And bump the LRU for this access */
2298 if (i915_gem_object_is_inactive(obj))
2299 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
2300
1715 CTR3(KTR_DRM, "object_change_domain set_to_gtt %p %x %x", obj,
1716 old_read_domains, old_write_domain);
1717 return (0);
1718}
1719
1720int
1721i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
1722 enum i915_cache_level cache_level)

--- 24 unchanged lines hidden (view full) ---

1747 * currently pointing to our region in the aperture.
1748 */
1749 if (INTEL_INFO(obj->base.dev)->gen < 6) {
1750 ret = i915_gem_object_put_fence(obj);
1751 if (ret != 0)
1752 return (ret);
1753 }
1754
2301 CTR3(KTR_DRM, "object_change_domain set_to_gtt %p %x %x", obj,
2302 old_read_domains, old_write_domain);
2303 return (0);
2304}
2305
2306int
2307i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
2308 enum i915_cache_level cache_level)

--- 24 unchanged lines hidden (view full) ---

2333 * currently pointing to our region in the aperture.
2334 */
2335 if (INTEL_INFO(obj->base.dev)->gen < 6) {
2336 ret = i915_gem_object_put_fence(obj);
2337 if (ret != 0)
2338 return (ret);
2339 }
2340
1755 i915_gem_gtt_rebind_object(obj, cache_level);
2341 if (obj->has_global_gtt_mapping)
2342 i915_gem_gtt_bind_object(obj, cache_level);
1756 if (obj->has_aliasing_ppgtt_mapping)
1757 i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt,
1758 obj, cache_level);
1759 }
1760
1761 if (cache_level == I915_CACHE_NONE) {
1762 u32 old_read_domains, old_write_domain;
1763

--- 17 unchanged lines hidden (view full) ---

1781 CTR3(KTR_DRM, "object_change_domain set_cache_level %p %x %x",
1782 obj, old_read_domains, old_write_domain);
1783 }
1784
1785 obj->cache_level = cache_level;
1786 return (0);
1787}
1788
2343 if (obj->has_aliasing_ppgtt_mapping)
2344 i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt,
2345 obj, cache_level);
2346 }
2347
2348 if (cache_level == I915_CACHE_NONE) {
2349 u32 old_read_domains, old_write_domain;
2350

--- 17 unchanged lines hidden (view full) ---

2368 CTR3(KTR_DRM, "object_change_domain set_cache_level %p %x %x",
2369 obj, old_read_domains, old_write_domain);
2370 }
2371
2372 obj->cache_level = cache_level;
2373 return (0);
2374}
2375
2376static bool is_pin_display(struct drm_i915_gem_object *obj)
2377{
2378 /* There are 3 sources that pin objects:
2379 * 1. The display engine (scanouts, sprites, cursors);
2380 * 2. Reservations for execbuffer;
2381 * 3. The user.
2382 *
2383 * We can ignore reservations as we hold the struct_mutex and
2384 * are only called outside of the reservation path. The user
2385 * can only increment pin_count once, and so if after
2386 * subtracting the potential reference by the user, any pin_count
2387 * remains, it must be due to another use by the display engine.
2388 */
2389 return obj->pin_count - !!obj->user_pin_count;
2390}
2391
1789int
1790i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
1791 u32 alignment, struct intel_ring_buffer *pipelined)
1792{
1793 u32 old_read_domains, old_write_domain;
1794 int ret;
1795
1796 ret = i915_gem_object_flush_gpu_write_domain(obj);
1797 if (ret != 0)
1798 return (ret);
1799
1800 if (pipelined != obj->ring) {
2392int
2393i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
2394 u32 alignment, struct intel_ring_buffer *pipelined)
2395{
2396 u32 old_read_domains, old_write_domain;
2397 int ret;
2398
2399 ret = i915_gem_object_flush_gpu_write_domain(obj);
2400 if (ret != 0)
2401 return (ret);
2402
2403 if (pipelined != obj->ring) {
1801 ret = i915_gem_object_wait_rendering(obj);
1802 if (ret == -ERESTART || ret == -EINTR)
2404 ret = i915_gem_object_sync(obj, pipelined);
2405 if (ret)
1803 return (ret);
1804 }
1805
2406 return (ret);
2407 }
2408
2409 obj->pin_display = true;
1806 ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE);
1807 if (ret != 0)
2410 ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE);
2411 if (ret != 0)
1808 return (ret);
2412 goto err_unpin_display;
1809
1810 ret = i915_gem_object_pin(obj, alignment, true);
1811 if (ret != 0)
2413
2414 ret = i915_gem_object_pin(obj, alignment, true);
2415 if (ret != 0)
1812 return (ret);
2416 goto err_unpin_display;
1813
1814 i915_gem_object_flush_cpu_write_domain(obj);
1815
1816 old_write_domain = obj->base.write_domain;
1817 old_read_domains = obj->base.read_domains;
1818
1819 KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) == 0,
1820 ("obj %p in GTT write domain", obj));
1821 obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
1822
1823 CTR3(KTR_DRM, "object_change_domain pin_to_display_plan %p %x %x",
1824 obj, old_read_domains, obj->base.write_domain);
1825 return (0);
2417
2418 i915_gem_object_flush_cpu_write_domain(obj);
2419
2420 old_write_domain = obj->base.write_domain;
2421 old_read_domains = obj->base.read_domains;
2422
2423 KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) == 0,
2424 ("obj %p in GTT write domain", obj));
2425 obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
2426
2427 CTR3(KTR_DRM, "object_change_domain pin_to_display_plan %p %x %x",
2428 obj, old_read_domains, obj->base.write_domain);
2429 return (0);
2430
2431err_unpin_display:
2432 obj->pin_display = is_pin_display(obj);
2433 return ret;
1826}
1827
2434}
2435
2436void
2437i915_gem_object_unpin_from_display_plane(struct drm_i915_gem_object *obj)
2438{
2439 i915_gem_object_unpin(obj);
2440 obj->pin_display = is_pin_display(obj);
2441}
2442
1828int
1829i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj)
1830{
1831 int ret;
1832
1833 if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0)
1834 return (0);
1835

--- 7 unchanged lines hidden (view full) ---

1843 if (ret != 0)
1844 return (ret);
1845
1846 obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
1847
1848 return (0);
1849}
1850
2443int
2444i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj)
2445{
2446 int ret;
2447
2448 if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0)
2449 return (0);
2450

--- 7 unchanged lines hidden (view full) ---

2458 if (ret != 0)
2459 return (ret);
2460
2461 obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
2462
2463 return (0);
2464}
2465
1851static int
2466int
1852i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
1853{
1854 uint32_t old_write_domain, old_read_domains;
1855 int ret;
1856
1857 if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
1858 return 0;
1859
1860 ret = i915_gem_object_flush_gpu_write_domain(obj);
1861 if (ret != 0)
1862 return (ret);
1863
2467i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
2468{
2469 uint32_t old_write_domain, old_read_domains;
2470 int ret;
2471
2472 if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
2473 return 0;
2474
2475 ret = i915_gem_object_flush_gpu_write_domain(obj);
2476 if (ret != 0)
2477 return (ret);
2478
1864 ret = i915_gem_object_wait_rendering(obj);
1865 if (ret != 0)
1866 return (ret);
2479 if (write || obj->pending_gpu_write) {
2480 ret = i915_gem_object_wait_rendering(obj);
2481 if (ret != 0)
2482 return (ret);
2483 }
1867
1868 i915_gem_object_flush_gtt_write_domain(obj);
2484
2485 i915_gem_object_flush_gtt_write_domain(obj);
1869 i915_gem_object_set_to_full_cpu_read_domain(obj);
1870
1871 old_write_domain = obj->base.write_domain;
1872 old_read_domains = obj->base.read_domains;
1873
1874 if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
1875 i915_gem_clflush_object(obj);
1876 obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
1877 }

--- 6 unchanged lines hidden (view full) ---

1884 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
1885 }
1886
1887 CTR3(KTR_DRM, "object_change_domain set_to_cpu %p %x %x", obj,
1888 old_read_domains, old_write_domain);
1889 return (0);
1890}
1891
2486
2487 old_write_domain = obj->base.write_domain;
2488 old_read_domains = obj->base.read_domains;
2489
2490 if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
2491 i915_gem_clflush_object(obj);
2492 obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
2493 }

--- 6 unchanged lines hidden (view full) ---

2500 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
2501 }
2502
2503 CTR3(KTR_DRM, "object_change_domain set_to_cpu %p %x %x", obj,
2504 old_read_domains, old_write_domain);
2505 return (0);
2506}
2507
1892static void
1893i915_gem_object_set_to_full_cpu_read_domain(struct drm_i915_gem_object *obj)
1894{
1895 int i;
1896
1897 if (obj->page_cpu_valid == NULL)
1898 return;
1899
1900 if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) != 0) {
1901 for (i = 0; i <= (obj->base.size - 1) / PAGE_SIZE; i++) {
1902 if (obj->page_cpu_valid[i] != 0)
1903 continue;
1904 drm_clflush_pages(obj->pages + i, 1);
1905 }
1906 }
1907
1908 free(obj->page_cpu_valid, DRM_I915_GEM);
1909 obj->page_cpu_valid = NULL;
1910}
1911
1912static int
1913i915_gem_object_set_cpu_read_domain_range(struct drm_i915_gem_object *obj,
1914 uint64_t offset, uint64_t size)
1915{
1916 uint32_t old_read_domains;
1917 int i, ret;
1918
1919 if (offset == 0 && size == obj->base.size)
1920 return (i915_gem_object_set_to_cpu_domain(obj, 0));
1921
1922 ret = i915_gem_object_flush_gpu_write_domain(obj);
1923 if (ret != 0)
1924 return (ret);
1925 ret = i915_gem_object_wait_rendering(obj);
1926 if (ret != 0)
1927 return (ret);
1928
1929 i915_gem_object_flush_gtt_write_domain(obj);
1930
1931 if (obj->page_cpu_valid == NULL &&
1932 (obj->base.read_domains & I915_GEM_DOMAIN_CPU) != 0)
1933 return (0);
1934
1935 if (obj->page_cpu_valid == NULL) {
1936 obj->page_cpu_valid = malloc(obj->base.size / PAGE_SIZE,
1937 DRM_I915_GEM, M_WAITOK | M_ZERO);
1938 } else if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0)
1939 memset(obj->page_cpu_valid, 0, obj->base.size / PAGE_SIZE);
1940
1941 for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE;
1942 i++) {
1943 if (obj->page_cpu_valid[i])
1944 continue;
1945 drm_clflush_pages(obj->pages + i, 1);
1946 obj->page_cpu_valid[i] = 1;
1947 }
1948
1949 KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) == 0,
1950 ("In gpu write domain"));
1951
1952 old_read_domains = obj->base.read_domains;
1953 obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
1954
1955 CTR3(KTR_DRM, "object_change_domain set_cpu_read %p %x %x", obj,
1956 old_read_domains, obj->base.write_domain);
1957 return (0);
1958}
1959
1960static uint32_t
1961i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
1962{
1963 uint32_t gtt_size;
1964
1965 if (INTEL_INFO(dev)->gen >= 4 ||
1966 tiling_mode == I915_TILING_NONE)
1967 return (size);

--- 134 unchanged lines hidden (view full) ---

2102 obj->gtt_space = NULL;
2103 /*
2104 * i915_gem_object_get_pages_gtt() cannot return
2105 * ENOMEM, since we use vm_page_grab().
2106 */
2107 return (ret);
2108 }
2109
2508static uint32_t
2509i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
2510{
2511 uint32_t gtt_size;
2512
2513 if (INTEL_INFO(dev)->gen >= 4 ||
2514 tiling_mode == I915_TILING_NONE)
2515 return (size);

--- 134 unchanged lines hidden (view full) ---

2650 obj->gtt_space = NULL;
2651 /*
2652 * i915_gem_object_get_pages_gtt() cannot return
2653 * ENOMEM, since we use vm_page_grab().
2654 */
2655 return (ret);
2656 }
2657
2110 ret = i915_gem_gtt_bind_object(obj);
2658 ret = i915_gem_gtt_prepare_object(obj);
2111 if (ret != 0) {
2112 i915_gem_object_put_pages_gtt(obj);
2113 drm_mm_put_block(obj->gtt_space);
2114 obj->gtt_space = NULL;
2115 if (i915_gem_evict_everything(dev, false))
2116 return (ret);
2117 goto search_free;
2118 }
2119
2659 if (ret != 0) {
2660 i915_gem_object_put_pages_gtt(obj);
2661 drm_mm_put_block(obj->gtt_space);
2662 obj->gtt_space = NULL;
2663 if (i915_gem_evict_everything(dev, false))
2664 return (ret);
2665 goto search_free;
2666 }
2667
2668 if (!dev_priv->mm.aliasing_ppgtt)
2669 i915_gem_gtt_bind_object(obj, obj->cache_level);
2670
2120 list_add_tail(&obj->gtt_list, &dev_priv->mm.gtt_list);
2121 list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
2122
2123 KASSERT((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0,
2124 ("Object in gpu read domain"));
2125 KASSERT((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0,
2126 ("Object in gpu write domain"));
2127

--- 7 unchanged lines hidden (view full) ---

2135 obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end;
2136 obj->map_and_fenceable = mappable && fenceable;
2137
2138 CTR4(KTR_DRM, "object_bind %p %x %x %d", obj, obj->gtt_offset,
2139 obj->base.size, map_and_fenceable);
2140 return (0);
2141}
2142
2671 list_add_tail(&obj->gtt_list, &dev_priv->mm.gtt_list);
2672 list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
2673
2674 KASSERT((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0,
2675 ("Object in gpu read domain"));
2676 KASSERT((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0,
2677 ("Object in gpu write domain"));
2678

--- 7 unchanged lines hidden (view full) ---

2686 obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end;
2687 obj->map_and_fenceable = mappable && fenceable;
2688
2689 CTR4(KTR_DRM, "object_bind %p %x %x %d", obj, obj->gtt_offset,
2690 obj->base.size, map_and_fenceable);
2691 return (0);
2692}
2693
2143static void
2144i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
2694int
2695i915_gem_object_sync(struct drm_i915_gem_object *obj,
2696 struct intel_ring_buffer *to)
2145{
2697{
2698 struct intel_ring_buffer *from = obj->ring;
2699 u32 seqno;
2700 int ret, idx;
2701
2702 if (from == NULL || to == from)
2703 return 0;
2704
2705 if (to == NULL || !i915_semaphore_is_enabled(obj->base.dev))
2706 return i915_gem_object_wait_rendering(obj);
2707
2708 idx = intel_ring_sync_index(from, to);
2709
2710 seqno = obj->last_rendering_seqno;
2711 if (seqno <= from->sync_seqno[idx])
2712 return 0;
2713
2714 if (seqno == from->outstanding_lazy_request) {
2715 struct drm_i915_gem_request *request;
2716
2717 request = malloc(sizeof(*request), DRM_I915_GEM,
2718 M_WAITOK | M_ZERO);
2719 ret = i915_add_request(from, NULL, request);
2720 if (ret) {
2721 free(request, DRM_I915_GEM);
2722 return ret;
2723 }
2724 seqno = request->seqno;
2725 }
2726
2727
2728 ret = to->sync_to(to, from, seqno);
2729 if (!ret)
2730 from->sync_seqno[idx] = seqno;
2731
2732 return ret;
2733}
2734
2735static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
2736{
2146 u32 old_write_domain, old_read_domains;
2147
2148 /* Act a barrier for all accesses through the GTT */
2149 mb();
2150
2151 /* Force a pagefault for domain tracking on next user access */
2152 i915_gem_release_mmap(obj);
2153

--- 37 unchanged lines hidden (view full) ---

2191 return (ret);
2192 if (ret != 0) {
2193 i915_gem_clflush_object(obj);
2194 obj->base.read_domains = obj->base.write_domain =
2195 I915_GEM_DOMAIN_CPU;
2196 }
2197
2198 ret = i915_gem_object_put_fence(obj);
2737 u32 old_write_domain, old_read_domains;
2738
2739 /* Act a barrier for all accesses through the GTT */
2740 mb();
2741
2742 /* Force a pagefault for domain tracking on next user access */
2743 i915_gem_release_mmap(obj);
2744

--- 37 unchanged lines hidden (view full) ---

2782 return (ret);
2783 if (ret != 0) {
2784 i915_gem_clflush_object(obj);
2785 obj->base.read_domains = obj->base.write_domain =
2786 I915_GEM_DOMAIN_CPU;
2787 }
2788
2789 ret = i915_gem_object_put_fence(obj);
2199 if (ret == -ERESTART)
2790 if (ret)
2200 return (ret);
2201
2791 return (ret);
2792
2202 i915_gem_gtt_unbind_object(obj);
2793 if (obj->has_global_gtt_mapping)
2794 i915_gem_gtt_unbind_object(obj);
2203 if (obj->has_aliasing_ppgtt_mapping) {
2204 i915_ppgtt_unbind_object(dev_priv->mm.aliasing_ppgtt, obj);
2205 obj->has_aliasing_ppgtt_mapping = 0;
2206 }
2795 if (obj->has_aliasing_ppgtt_mapping) {
2796 i915_ppgtt_unbind_object(dev_priv->mm.aliasing_ppgtt, obj);
2797 obj->has_aliasing_ppgtt_mapping = 0;
2798 }
2799 i915_gem_gtt_finish_object(obj);
2800
2207 i915_gem_object_put_pages_gtt(obj);
2208
2209 list_del_init(&obj->gtt_list);
2210 list_del_init(&obj->mm_list);
2211 obj->map_and_fenceable = true;
2212
2213 drm_mm_put_block(obj->gtt_space);
2214 obj->gtt_space = NULL;
2215 obj->gtt_offset = 0;
2216
2217 if (i915_gem_object_is_purgeable(obj))
2218 i915_gem_object_truncate(obj);
2219 CTR1(KTR_DRM, "object_unbind %p", obj);
2220
2221 return (ret);
2222}
2223
2801 i915_gem_object_put_pages_gtt(obj);
2802
2803 list_del_init(&obj->gtt_list);
2804 list_del_init(&obj->mm_list);
2805 obj->map_and_fenceable = true;
2806
2807 drm_mm_put_block(obj->gtt_space);
2808 obj->gtt_space = NULL;
2809 obj->gtt_offset = 0;
2810
2811 if (i915_gem_object_is_purgeable(obj))
2812 i915_gem_object_truncate(obj);
2813 CTR1(KTR_DRM, "object_unbind %p", obj);
2814
2815 return (ret);
2816}
2817
2818static void
2819i915_gem_object_put_pages_range_locked(struct drm_i915_gem_object *obj,
2820 vm_pindex_t si, vm_pindex_t ei)
2821{
2822 vm_object_t vm_obj;
2823 vm_page_t m;
2824 vm_pindex_t i;
2825
2826 vm_obj = obj->base.vm_obj;
2827 VM_OBJECT_ASSERT_LOCKED(vm_obj);
2828 for (i = si, m = vm_page_lookup(vm_obj, i); i < ei;
2829 m = vm_page_next(m), i++) {
2830 KASSERT(m->pindex == i, ("pindex %jx %jx",
2831 (uintmax_t)m->pindex, (uintmax_t)i));
2832 vm_page_lock(m);
2833 vm_page_unwire(m, PQ_INACTIVE);
2834 if (m->wire_count == 0)
2835 atomic_add_long(&i915_gem_wired_pages_cnt, -1);
2836 vm_page_unlock(m);
2837 }
2838}
2839
2840static void
2841i915_gem_object_put_pages_range(struct drm_i915_gem_object *obj,
2842 off_t start, off_t end)
2843{
2844 vm_object_t vm_obj;
2845
2846 vm_obj = obj->base.vm_obj;
2847 VM_OBJECT_WLOCK(vm_obj);
2848 i915_gem_object_put_pages_range_locked(obj,
2849 OFF_TO_IDX(trunc_page(start)), OFF_TO_IDX(round_page(end)));
2850 VM_OBJECT_WUNLOCK(vm_obj);
2851}
2852
2224static int
2853static int
2854i915_gem_object_get_pages_range(struct drm_i915_gem_object *obj,
2855 off_t start, off_t end)
2856{
2857 vm_object_t vm_obj;
2858 vm_page_t m;
2859 vm_pindex_t si, ei, i;
2860 bool need_swizzle, fresh;
2861
2862 need_swizzle = i915_gem_object_needs_bit17_swizzle(obj) != 0;
2863 vm_obj = obj->base.vm_obj;
2864 si = OFF_TO_IDX(trunc_page(start));
2865 ei = OFF_TO_IDX(round_page(end));
2866 VM_OBJECT_WLOCK(vm_obj);
2867 for (i = si; i < ei; i++) {
2868 m = i915_gem_wire_page(vm_obj, i, &fresh);
2869 if (m == NULL)
2870 goto failed;
2871 if (need_swizzle && fresh)
2872 i915_gem_object_do_bit_17_swizzle_page(obj, m);
2873 }
2874 VM_OBJECT_WUNLOCK(vm_obj);
2875 return (0);
2876failed:
2877 i915_gem_object_put_pages_range_locked(obj, si, i);
2878 VM_OBJECT_WUNLOCK(vm_obj);
2879 return (-EIO);
2880}
2881
2882static int
2225i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj,
2226 int flags)
2227{
2228 struct drm_device *dev;
2229 vm_object_t vm_obj;
2230 vm_page_t m;
2883i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj,
2884 int flags)
2885{
2886 struct drm_device *dev;
2887 vm_object_t vm_obj;
2888 vm_page_t m;
2231 int page_count, i, j;
2889 vm_pindex_t i, page_count;
2890 int res;
2232
2233 dev = obj->base.dev;
2234 KASSERT(obj->pages == NULL, ("Obj already has pages"));
2891
2892 dev = obj->base.dev;
2893 KASSERT(obj->pages == NULL, ("Obj already has pages"));
2235 page_count = obj->base.size / PAGE_SIZE;
2894 page_count = OFF_TO_IDX(obj->base.size);
2236 obj->pages = malloc(page_count * sizeof(vm_page_t), DRM_I915_GEM,
2237 M_WAITOK);
2895 obj->pages = malloc(page_count * sizeof(vm_page_t), DRM_I915_GEM,
2896 M_WAITOK);
2897 res = i915_gem_object_get_pages_range(obj, 0, obj->base.size);
2898 if (res != 0) {
2899 free(obj->pages, DRM_I915_GEM);
2900 obj->pages = NULL;
2901 return (res);
2902 }
2238 vm_obj = obj->base.vm_obj;
2239 VM_OBJECT_WLOCK(vm_obj);
2903 vm_obj = obj->base.vm_obj;
2904 VM_OBJECT_WLOCK(vm_obj);
2240 for (i = 0; i < page_count; i++) {
2241 if ((obj->pages[i] = i915_gem_wire_page(vm_obj, i)) == NULL)
2242 goto failed;
2905 for (i = 0, m = vm_page_lookup(vm_obj, 0); i < page_count;
2906 i++, m = vm_page_next(m)) {
2907 KASSERT(m->pindex == i, ("pindex %jx %jx",
2908 (uintmax_t)m->pindex, (uintmax_t)i));
2909 obj->pages[i] = m;
2243 }
2244 VM_OBJECT_WUNLOCK(vm_obj);
2910 }
2911 VM_OBJECT_WUNLOCK(vm_obj);
2245 if (i915_gem_object_needs_bit17_swizzle(obj))
2246 i915_gem_object_do_bit_17_swizzle(obj);
2247 return (0);
2912 return (0);
2248
2249failed:
2250 for (j = 0; j < i; j++) {
2251 m = obj->pages[j];
2252 vm_page_lock(m);
2253 vm_page_unwire(m, PQ_INACTIVE);
2254 vm_page_unlock(m);
2255 atomic_add_long(&i915_gem_wired_pages_cnt, -1);
2256 }
2257 VM_OBJECT_WUNLOCK(vm_obj);
2258 free(obj->pages, DRM_I915_GEM);
2259 obj->pages = NULL;
2260 return (-EIO);
2261}
2262
2263#define GEM_PARANOID_CHECK_GTT 0
2264#if GEM_PARANOID_CHECK_GTT
2265static void
2266i915_gem_assert_pages_not_mapped(struct drm_device *dev, vm_page_t *ma,
2267 int page_count)
2268{

--- 92 unchanged lines hidden (view full) ---

2361
2362 KASSERT((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0,
2363 ("In GPU write domain"));
2364
2365 CTR5(KTR_DRM, "object_wait_rendering %p %s %x %d %d", obj,
2366 obj->ring != NULL ? obj->ring->name : "none", obj->gtt_offset,
2367 obj->active, obj->last_rendering_seqno);
2368 if (obj->active) {
2913}
2914
2915#define GEM_PARANOID_CHECK_GTT 0
2916#if GEM_PARANOID_CHECK_GTT
2917static void
2918i915_gem_assert_pages_not_mapped(struct drm_device *dev, vm_page_t *ma,
2919 int page_count)
2920{

--- 92 unchanged lines hidden (view full) ---

3013
3014 KASSERT((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0,
3015 ("In GPU write domain"));
3016
3017 CTR5(KTR_DRM, "object_wait_rendering %p %s %x %d %d", obj,
3018 obj->ring != NULL ? obj->ring->name : "none", obj->gtt_offset,
3019 obj->active, obj->last_rendering_seqno);
3020 if (obj->active) {
2369 ret = i915_wait_request(obj->ring, obj->last_rendering_seqno,
2370 true);
3021 ret = i915_wait_request(obj->ring, obj->last_rendering_seqno);
2371 if (ret != 0)
2372 return (ret);
3022 if (ret != 0)
3023 return (ret);
3024 i915_gem_retire_requests_ring(obj->ring);
2373 }
2374 return (0);
2375}
2376
2377void
2378i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
2379 struct intel_ring_buffer *ring, uint32_t seqno)
2380{

--- 12 unchanged lines hidden (view full) ---

2393
2394 /* Move from whatever list we were on to the tail of execution. */
2395 list_move_tail(&obj->mm_list, &dev_priv->mm.active_list);
2396 list_move_tail(&obj->ring_list, &ring->active_list);
2397
2398 obj->last_rendering_seqno = seqno;
2399 if (obj->fenced_gpu_access) {
2400 obj->last_fenced_seqno = seqno;
3025 }
3026 return (0);
3027}
3028
3029void
3030i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
3031 struct intel_ring_buffer *ring, uint32_t seqno)
3032{

--- 12 unchanged lines hidden (view full) ---

3045
3046 /* Move from whatever list we were on to the tail of execution. */
3047 list_move_tail(&obj->mm_list, &dev_priv->mm.active_list);
3048 list_move_tail(&obj->ring_list, &ring->active_list);
3049
3050 obj->last_rendering_seqno = seqno;
3051 if (obj->fenced_gpu_access) {
3052 obj->last_fenced_seqno = seqno;
2401 obj->last_fenced_ring = ring;
2402
2403 /* Bump MRU to take account of the delayed flush */
2404 if (obj->fence_reg != I915_FENCE_REG_NONE) {
2405 reg = &dev_priv->fence_regs[obj->fence_reg];
2406 list_move_tail(&reg->lru_list,
2407 &dev_priv->mm.fence_list);
2408 }
2409 }

--- 20 unchanged lines hidden (view full) ---

2430}
2431
2432static void
2433i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
2434{
2435 struct drm_device *dev = obj->base.dev;
2436 struct drm_i915_private *dev_priv = dev->dev_private;
2437
3053
3054 /* Bump MRU to take account of the delayed flush */
3055 if (obj->fence_reg != I915_FENCE_REG_NONE) {
3056 reg = &dev_priv->fence_regs[obj->fence_reg];
3057 list_move_tail(&reg->lru_list,
3058 &dev_priv->mm.fence_list);
3059 }
3060 }

--- 20 unchanged lines hidden (view full) ---

3081}
3082
3083static void
3084i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
3085{
3086 struct drm_device *dev = obj->base.dev;
3087 struct drm_i915_private *dev_priv = dev->dev_private;
3088
2438 if (obj->pin_count != 0)
2439 list_move_tail(&obj->mm_list, &dev_priv->mm.pinned_list);
2440 else
2441 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
3089 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
2442
2443 KASSERT(list_empty(&obj->gpu_write_list), ("On gpu_write_list"));
2444 KASSERT(obj->active, ("Object not active"));
2445 obj->ring = NULL;
3090
3091 KASSERT(list_empty(&obj->gpu_write_list), ("On gpu_write_list"));
3092 KASSERT(obj->active, ("Object not active"));
3093 obj->ring = NULL;
2446 obj->last_fenced_ring = NULL;
2447
2448 i915_gem_object_move_off_active(obj);
2449 obj->fenced_gpu_access = false;
2450
2451 obj->active = 0;
2452 obj->pending_gpu_write = false;
2453 drm_gem_object_unreference(&obj->base);
2454

--- 8 unchanged lines hidden (view full) ---

2463i915_gem_object_truncate(struct drm_i915_gem_object *obj)
2464{
2465 vm_object_t vm_obj;
2466
2467 vm_obj = obj->base.vm_obj;
2468 VM_OBJECT_WLOCK(vm_obj);
2469 vm_object_page_remove(vm_obj, 0, 0, false);
2470 VM_OBJECT_WUNLOCK(vm_obj);
3094
3095 i915_gem_object_move_off_active(obj);
3096 obj->fenced_gpu_access = false;
3097
3098 obj->active = 0;
3099 obj->pending_gpu_write = false;
3100 drm_gem_object_unreference(&obj->base);
3101

--- 8 unchanged lines hidden (view full) ---

3110i915_gem_object_truncate(struct drm_i915_gem_object *obj)
3111{
3112 vm_object_t vm_obj;
3113
3114 vm_obj = obj->base.vm_obj;
3115 VM_OBJECT_WLOCK(vm_obj);
3116 vm_object_page_remove(vm_obj, 0, 0, false);
3117 VM_OBJECT_WUNLOCK(vm_obj);
3118 drm_gem_free_mmap_offset(&obj->base);
2471 obj->madv = I915_MADV_PURGED_INTERNAL;
2472}
2473
2474static inline int
2475i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
2476{
2477
2478 return (obj->madv == I915_MADV_DONTNEED);

--- 27 unchanged lines hidden (view full) ---

2506 drm_i915_private_t *dev_priv;
2507
2508 dev_priv = obj->base.dev->dev_private;
2509 return (dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
2510 obj->tiling_mode != I915_TILING_NONE);
2511}
2512
2513static vm_page_t
3119 obj->madv = I915_MADV_PURGED_INTERNAL;
3120}
3121
3122static inline int
3123i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
3124{
3125
3126 return (obj->madv == I915_MADV_DONTNEED);

--- 27 unchanged lines hidden (view full) ---

3154 drm_i915_private_t *dev_priv;
3155
3156 dev_priv = obj->base.dev->dev_private;
3157 return (dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
3158 obj->tiling_mode != I915_TILING_NONE);
3159}
3160
3161static vm_page_t
2514i915_gem_wire_page(vm_object_t object, vm_pindex_t pindex)
3162i915_gem_wire_page(vm_object_t object, vm_pindex_t pindex, bool *fresh)
2515{
2516 vm_page_t m;
2517 int rv;
2518
2519 VM_OBJECT_ASSERT_WLOCKED(object);
2520 m = vm_page_grab(object, pindex, VM_ALLOC_NORMAL);
2521 if (m->valid != VM_PAGE_BITS_ALL) {
2522 if (vm_pager_has_page(object, pindex, NULL, NULL)) {
2523 rv = vm_pager_get_pages(object, &m, 1, 0);
2524 m = vm_page_lookup(object, pindex);
2525 if (m == NULL)
2526 return (NULL);
2527 if (rv != VM_PAGER_OK) {
2528 vm_page_lock(m);
2529 vm_page_free(m);
2530 vm_page_unlock(m);
2531 return (NULL);
2532 }
3163{
3164 vm_page_t m;
3165 int rv;
3166
3167 VM_OBJECT_ASSERT_WLOCKED(object);
3168 m = vm_page_grab(object, pindex, VM_ALLOC_NORMAL);
3169 if (m->valid != VM_PAGE_BITS_ALL) {
3170 if (vm_pager_has_page(object, pindex, NULL, NULL)) {
3171 rv = vm_pager_get_pages(object, &m, 1, 0);
3172 m = vm_page_lookup(object, pindex);
3173 if (m == NULL)
3174 return (NULL);
3175 if (rv != VM_PAGER_OK) {
3176 vm_page_lock(m);
3177 vm_page_free(m);
3178 vm_page_unlock(m);
3179 return (NULL);
3180 }
3181 if (fresh != NULL)
3182 *fresh = true;
2533 } else {
2534 pmap_zero_page(m);
2535 m->valid = VM_PAGE_BITS_ALL;
2536 m->dirty = 0;
3183 } else {
3184 pmap_zero_page(m);
3185 m->valid = VM_PAGE_BITS_ALL;
3186 m->dirty = 0;
3187 if (fresh != NULL)
3188 *fresh = false;
2537 }
3189 }
3190 } else if (fresh != NULL) {
3191 *fresh = false;
2538 }
2539 vm_page_lock(m);
2540 vm_page_wire(m);
2541 vm_page_unlock(m);
2542 vm_page_xunbusy(m);
2543 atomic_add_long(&i915_gem_wired_pages_cnt, 1);
2544 return (m);
2545}

--- 14 unchanged lines hidden (view full) ---

2560 return ret;
2561
2562 if (flush_domains & I915_GEM_GPU_DOMAINS)
2563 i915_gem_process_flushing_list(ring, flush_domains);
2564 return 0;
2565}
2566
2567static int
3192 }
3193 vm_page_lock(m);
3194 vm_page_wire(m);
3195 vm_page_unlock(m);
3196 vm_page_xunbusy(m);
3197 atomic_add_long(&i915_gem_wired_pages_cnt, 1);
3198 return (m);
3199}

--- 14 unchanged lines hidden (view full) ---

3214 return ret;
3215
3216 if (flush_domains & I915_GEM_GPU_DOMAINS)
3217 i915_gem_process_flushing_list(ring, flush_domains);
3218 return 0;
3219}
3220
3221static int
2568i915_ring_idle(struct intel_ring_buffer *ring, bool do_retire)
3222i915_ring_idle(struct intel_ring_buffer *ring)
2569{
2570 int ret;
2571
2572 if (list_empty(&ring->gpu_write_list) && list_empty(&ring->active_list))
2573 return 0;
2574
2575 if (!list_empty(&ring->gpu_write_list)) {
2576 ret = i915_gem_flush_ring(ring, I915_GEM_GPU_DOMAINS,
2577 I915_GEM_GPU_DOMAINS);
2578 if (ret != 0)
2579 return ret;
2580 }
2581
3223{
3224 int ret;
3225
3226 if (list_empty(&ring->gpu_write_list) && list_empty(&ring->active_list))
3227 return 0;
3228
3229 if (!list_empty(&ring->gpu_write_list)) {
3230 ret = i915_gem_flush_ring(ring, I915_GEM_GPU_DOMAINS,
3231 I915_GEM_GPU_DOMAINS);
3232 if (ret != 0)
3233 return ret;
3234 }
3235
2582 return (i915_wait_request(ring, i915_gem_next_request_seqno(ring),
2583 do_retire));
3236 return (i915_wait_request(ring, i915_gem_next_request_seqno(ring)));
2584}
2585
2586int
3237}
3238
3239int
2587i915_gpu_idle(struct drm_device *dev, bool do_retire)
3240i915_gpu_idle(struct drm_device *dev)
2588{
2589 drm_i915_private_t *dev_priv = dev->dev_private;
2590 struct intel_ring_buffer *ring;
2591 int ret, i;
2592
2593 /* Flush everything onto the inactive list. */
2594 for_each_ring(ring, dev_priv, i) {
2595 ret = i915_switch_context(ring, NULL, DEFAULT_CONTEXT_ID);
2596 if (ret)
2597 return ret;
2598
3241{
3242 drm_i915_private_t *dev_priv = dev->dev_private;
3243 struct intel_ring_buffer *ring;
3244 int ret, i;
3245
3246 /* Flush everything onto the inactive list. */
3247 for_each_ring(ring, dev_priv, i) {
3248 ret = i915_switch_context(ring, NULL, DEFAULT_CONTEXT_ID);
3249 if (ret)
3250 return ret;
3251
2599 ret = i915_ring_idle(ring, do_retire);
3252 ret = i915_ring_idle(ring);
2600 if (ret)
2601 return ret;
3253 if (ret)
3254 return ret;
3255
3256 /* Is the device fubar? */
3257 if (!list_empty(&ring->gpu_write_list))
3258 return -EBUSY;
2602 }
2603
2604 return 0;
2605}
2606
3259 }
3260
3261 return 0;
3262}
3263
2607int
2608i915_wait_request(struct intel_ring_buffer *ring, uint32_t seqno, bool do_retire)
3264static int
3265i915_gem_check_wedge(struct drm_i915_private *dev_priv)
2609{
3266{
2610 drm_i915_private_t *dev_priv;
2611 struct drm_i915_gem_request *request;
2612 uint32_t ier;
2613 int flags, ret;
2614 bool recovery_complete;
3267 DRM_LOCK_ASSERT(dev_priv->dev);
2615
3268
2616 KASSERT(seqno != 0, ("Zero seqno"));
2617
2618 dev_priv = ring->dev->dev_private;
2619 ret = 0;
2620
2621 if (atomic_load_acq_int(&dev_priv->mm.wedged) != 0) {
3269 if (atomic_load_acq_int(&dev_priv->mm.wedged) != 0) {
3270 bool recovery_complete;
2622 /* Give the error handler a chance to run. */
2623 mtx_lock(&dev_priv->error_completion_lock);
2624 recovery_complete = (&dev_priv->error_completion) > 0;
2625 mtx_unlock(&dev_priv->error_completion_lock);
2626 return (recovery_complete ? -EIO : -EAGAIN);
2627 }
2628
3271 /* Give the error handler a chance to run. */
3272 mtx_lock(&dev_priv->error_completion_lock);
3273 recovery_complete = (&dev_priv->error_completion) > 0;
3274 mtx_unlock(&dev_priv->error_completion_lock);
3275 return (recovery_complete ? -EIO : -EAGAIN);
3276 }
3277
3278 return 0;
3279}
3280
3281/*
3282 * Compare seqno against outstanding lazy request. Emit a request if they are
3283 * equal.
3284 */
3285static int
3286i915_gem_check_olr(struct intel_ring_buffer *ring, u32 seqno)
3287{
3288 int ret = 0;
3289
3290 DRM_LOCK_ASSERT(ring->dev);
3291
2629 if (seqno == ring->outstanding_lazy_request) {
3292 if (seqno == ring->outstanding_lazy_request) {
3293 struct drm_i915_gem_request *request;
3294
2630 request = malloc(sizeof(*request), DRM_I915_GEM,
2631 M_WAITOK | M_ZERO);
3295 request = malloc(sizeof(*request), DRM_I915_GEM,
3296 M_WAITOK | M_ZERO);
2632 if (request == NULL)
2633 return (-ENOMEM);
2634
2635 ret = i915_add_request(ring, NULL, request);
2636 if (ret != 0) {
2637 free(request, DRM_I915_GEM);
2638 return (ret);
2639 }
2640
3297
3298 ret = i915_add_request(ring, NULL, request);
3299 if (ret != 0) {
3300 free(request, DRM_I915_GEM);
3301 return (ret);
3302 }
3303
2641 seqno = request->seqno;
3304 MPASS(seqno == request->seqno);
2642 }
3305 }
3306 return ret;
3307}
2643
3308
2644 if (!i915_seqno_passed(ring->get_seqno(ring), seqno)) {
2645 if (HAS_PCH_SPLIT(ring->dev))
2646 ier = I915_READ(DEIER) | I915_READ(GTIER);
2647 else
2648 ier = I915_READ(IER);
2649 if (!ier) {
2650 DRM_ERROR("something (likely vbetool) disabled "
2651 "interrupts, re-enabling\n");
2652 ring->dev->driver->irq_preinstall(ring->dev);
2653 ring->dev->driver->irq_postinstall(ring->dev);
2654 }
3309static int __wait_seqno(struct intel_ring_buffer *ring, u32 seqno,
3310 bool interruptible)
3311{
3312 drm_i915_private_t *dev_priv = ring->dev->dev_private;
3313 int ret = 0, flags;
2655
3314
2656 CTR2(KTR_DRM, "request_wait_begin %s %d", ring->name, seqno);
3315 if (i915_seqno_passed(ring->get_seqno(ring), seqno))
3316 return 0;
2657
3317
2658 ring->waiting_seqno = seqno;
2659 mtx_lock(&ring->irq_lock);
2660 if (ring->irq_get(ring)) {
2661 flags = dev_priv->mm.interruptible ? PCATCH : 0;
2662 while (!i915_seqno_passed(ring->get_seqno(ring), seqno)
2663 && !atomic_load_acq_int(&dev_priv->mm.wedged) &&
2664 ret == 0) {
2665 ret = -msleep(ring, &ring->irq_lock, flags,
2666 "915gwr", 0);
2667 }
2668 ring->irq_put(ring);
2669 mtx_unlock(&ring->irq_lock);
2670 } else {
2671 mtx_unlock(&ring->irq_lock);
2672 if (_intel_wait_for(ring->dev,
2673 i915_seqno_passed(ring->get_seqno(ring), seqno) ||
2674 atomic_load_acq_int(&dev_priv->mm.wedged), 3000,
2675 0, "i915wrq") != 0)
2676 ret = -EBUSY;
2677 }
2678 ring->waiting_seqno = 0;
3318 CTR2(KTR_DRM, "request_wait_begin %s %d", ring->name, seqno);
2679
3319
2680 CTR3(KTR_DRM, "request_wait_end %s %d %d", ring->name, seqno,
2681 ret);
3320 mtx_lock(&dev_priv->irq_lock);
3321 if (!ring->irq_get(ring)) {
3322 mtx_unlock(&dev_priv->irq_lock);
3323 return (-ENODEV);
2682 }
3324 }
3325
3326 flags = interruptible ? PCATCH : 0;
3327 while (!i915_seqno_passed(ring->get_seqno(ring), seqno)
3328 && !atomic_load_acq_int(&dev_priv->mm.wedged) &&
3329 ret == 0)
3330 ret = -msleep(ring, &dev_priv->irq_lock, flags, "915gwr", 0);
3331 ring->irq_put(ring);
3332 mtx_unlock(&dev_priv->irq_lock);
3333
3334 CTR3(KTR_DRM, "request_wait_end %s %d %d", ring->name, seqno, ret);
3335
3336 return ret;
3337}
3338
3339int
3340i915_wait_request(struct intel_ring_buffer *ring, uint32_t seqno)
3341{
3342 drm_i915_private_t *dev_priv;
3343 int ret;
3344
3345 KASSERT(seqno != 0, ("Zero seqno"));
3346
3347 dev_priv = ring->dev->dev_private;
3348 ret = 0;
3349
3350 ret = i915_gem_check_wedge(dev_priv);
3351 if (ret)
3352 return ret;
3353
3354 ret = i915_gem_check_olr(ring, seqno);
3355 if (ret)
3356 return ret;
3357
3358 ret = __wait_seqno(ring, seqno, dev_priv->mm.interruptible);
2683 if (atomic_load_acq_int(&dev_priv->mm.wedged))
2684 ret = -EAGAIN;
2685
3359 if (atomic_load_acq_int(&dev_priv->mm.wedged))
3360 ret = -EAGAIN;
3361
2686 /* Directly dispatch request retiring. While we have the work queue
2687 * to handle this, the waiter on a request often wants an associated
2688 * buffer to have made it to the inactive list, and we would need
2689 * a separate wait queue to handle that.
2690 */
2691 if (ret == 0 && do_retire)
2692 i915_gem_retire_requests_ring(ring);
2693
2694 return (ret);
2695}
2696
2697static u32
2698i915_gem_get_seqno(struct drm_device *dev)
2699{
2700 drm_i915_private_t *dev_priv = dev->dev_private;
2701 u32 seqno = dev_priv->next_seqno;

--- 144 unchanged lines hidden (view full) ---

2846static void
2847i915_gem_reset_fences(struct drm_device *dev)
2848{
2849 struct drm_i915_private *dev_priv = dev->dev_private;
2850 int i;
2851
2852 for (i = 0; i < dev_priv->num_fence_regs; i++) {
2853 struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
3362 return (ret);
3363}
3364
3365static u32
3366i915_gem_get_seqno(struct drm_device *dev)
3367{
3368 drm_i915_private_t *dev_priv = dev->dev_private;
3369 u32 seqno = dev_priv->next_seqno;

--- 144 unchanged lines hidden (view full) ---

3514static void
3515i915_gem_reset_fences(struct drm_device *dev)
3516{
3517 struct drm_i915_private *dev_priv = dev->dev_private;
3518 int i;
3519
3520 for (i = 0; i < dev_priv->num_fence_regs; i++) {
3521 struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
2854 struct drm_i915_gem_object *obj = reg->obj;
2855
3522
2856 if (!obj)
2857 continue;
3523 i915_gem_write_fence(dev, i, NULL);
2858
3524
2859 if (obj->tiling_mode)
2860 i915_gem_release_mmap(obj);
3525 if (reg->obj)
3526 i915_gem_object_fence_lost(reg->obj);
2861
3527
2862 reg->obj->fence_reg = I915_FENCE_REG_NONE;
2863 reg->obj->fenced_gpu_access = false;
2864 reg->obj->last_fenced_seqno = 0;
2865 reg->obj->last_fenced_ring = NULL;
2866 i915_gem_clear_fence_reg(dev, reg);
3528 reg->pin_count = 0;
3529 reg->obj = NULL;
3530 INIT_LIST_HEAD(&reg->lru_list);
2867 }
3531 }
3532
3533 INIT_LIST_HEAD(&dev_priv->mm.fence_list);
2868}
2869
2870void
2871i915_gem_reset(struct drm_device *dev)
2872{
2873 struct drm_i915_private *dev_priv = dev->dev_private;
2874 struct drm_i915_gem_object *obj;
3534}
3535
3536void
3537i915_gem_reset(struct drm_device *dev)
3538{
3539 struct drm_i915_private *dev_priv = dev->dev_private;
3540 struct drm_i915_gem_object *obj;
3541 struct intel_ring_buffer *ring;
2875 int i;
2876
3542 int i;
3543
2877 for (i = 0; i < I915_NUM_RINGS; i++)
2878 i915_gem_reset_ring_lists(dev_priv, &dev_priv->rings[i]);
3544 for_each_ring(ring, dev_priv, i)
3545 i915_gem_reset_ring_lists(dev_priv, ring);
2879
2880 /* Remove anything from the flushing lists. The GPU cache is likely
2881 * to be lost on reset along with the data, so simply move the
2882 * lost bo to the inactive list.
2883 */
2884 while (!list_empty(&dev_priv->mm.flushing_list)) {
2885 obj = list_first_entry(&dev_priv->mm.flushing_list,
2886 struct drm_i915_gem_object,

--- 69 unchanged lines hidden (view full) ---

2956 if (obj->base.write_domain != 0)
2957 i915_gem_object_move_to_flushing(obj);
2958 else
2959 i915_gem_object_move_to_inactive(obj);
2960 }
2961
2962 if (ring->trace_irq_seqno &&
2963 i915_seqno_passed(seqno, ring->trace_irq_seqno)) {
3546
3547 /* Remove anything from the flushing lists. The GPU cache is likely
3548 * to be lost on reset along with the data, so simply move the
3549 * lost bo to the inactive list.
3550 */
3551 while (!list_empty(&dev_priv->mm.flushing_list)) {
3552 obj = list_first_entry(&dev_priv->mm.flushing_list,
3553 struct drm_i915_gem_object,

--- 69 unchanged lines hidden (view full) ---

3623 if (obj->base.write_domain != 0)
3624 i915_gem_object_move_to_flushing(obj);
3625 else
3626 i915_gem_object_move_to_inactive(obj);
3627 }
3628
3629 if (ring->trace_irq_seqno &&
3630 i915_seqno_passed(seqno, ring->trace_irq_seqno)) {
2964 mtx_lock(&ring->irq_lock);
3631 struct drm_i915_private *dev_priv = ring->dev->dev_private;
3632 mtx_lock(&dev_priv->irq_lock);
2965 ring->irq_put(ring);
3633 ring->irq_put(ring);
2966 mtx_unlock(&ring->irq_lock);
3634 mtx_unlock(&dev_priv->irq_lock);
2967 ring->trace_irq_seqno = 0;
2968 }
2969}
2970
2971void
2972i915_gem_retire_requests(struct drm_device *dev)
2973{
2974 drm_i915_private_t *dev_priv = dev->dev_private;
3635 ring->trace_irq_seqno = 0;
3636 }
3637}
3638
3639void
3640i915_gem_retire_requests(struct drm_device *dev)
3641{
3642 drm_i915_private_t *dev_priv = dev->dev_private;
2975 struct drm_i915_gem_object *obj, *next;
3643 struct intel_ring_buffer *ring;
2976 int i;
2977
3644 int i;
3645
2978 if (!list_empty(&dev_priv->mm.deferred_free_list)) {
2979 list_for_each_entry_safe(obj, next,
2980 &dev_priv->mm.deferred_free_list, mm_list)
2981 i915_gem_free_object_tail(obj);
2982 }
2983
2984 for (i = 0; i < I915_NUM_RINGS; i++)
2985 i915_gem_retire_requests_ring(&dev_priv->rings[i]);
3646 for_each_ring(ring, dev_priv, i)
3647 i915_gem_retire_requests_ring(ring);
2986}
2987
3648}
3649
2988static int
2989sandybridge_write_fence_reg(struct drm_i915_gem_object *obj,
2990 struct intel_ring_buffer *pipelined)
3650static void sandybridge_write_fence_reg(struct drm_device *dev, int reg,
3651 struct drm_i915_gem_object *obj)
2991{
3652{
2992 struct drm_device *dev = obj->base.dev;
2993 drm_i915_private_t *dev_priv = dev->dev_private;
3653 drm_i915_private_t *dev_priv = dev->dev_private;
2994 u32 size = obj->gtt_space->size;
2995 int regnum = obj->fence_reg;
2996 uint64_t val;
2997
3654 uint64_t val;
3655
2998 val = (uint64_t)((obj->gtt_offset + size - 4096) &
2999 0xfffff000) << 32;
3000 val |= obj->gtt_offset & 0xfffff000;
3001 val |= (uint64_t)((obj->stride / 128) - 1) <<
3002 SANDYBRIDGE_FENCE_PITCH_SHIFT;
3656 if (obj) {
3657 u32 size = obj->gtt_space->size;
3003
3658
3004 if (obj->tiling_mode == I915_TILING_Y)
3005 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
3006 val |= I965_FENCE_REG_VALID;
3659 val = (uint64_t)((obj->gtt_offset + size - 4096) &
3660 0xfffff000) << 32;
3661 val |= obj->gtt_offset & 0xfffff000;
3662 val |= (uint64_t)((obj->stride / 128) - 1) <<
3663 SANDYBRIDGE_FENCE_PITCH_SHIFT;
3007
3664
3008 if (pipelined) {
3009 int ret = intel_ring_begin(pipelined, 6);
3010 if (ret)
3011 return ret;
3012
3013 intel_ring_emit(pipelined, MI_NOOP);
3014 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(2));
3015 intel_ring_emit(pipelined, FENCE_REG_SANDYBRIDGE_0 + regnum*8);
3016 intel_ring_emit(pipelined, (u32)val);
3017 intel_ring_emit(pipelined, FENCE_REG_SANDYBRIDGE_0 + regnum*8 + 4);
3018 intel_ring_emit(pipelined, (u32)(val >> 32));
3019 intel_ring_advance(pipelined);
3665 if (obj->tiling_mode == I915_TILING_Y)
3666 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
3667 val |= I965_FENCE_REG_VALID;
3020 } else
3668 } else
3021 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + regnum * 8, val);
3669 val = 0;
3022
3670
3023 return 0;
3671 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + reg * 8, val);
3672 POSTING_READ(FENCE_REG_SANDYBRIDGE_0 + reg * 8);
3024}
3025
3673}
3674
3026static int
3027i965_write_fence_reg(struct drm_i915_gem_object *obj,
3028 struct intel_ring_buffer *pipelined)
3675static void i965_write_fence_reg(struct drm_device *dev, int reg,
3676 struct drm_i915_gem_object *obj)
3029{
3677{
3030 struct drm_device *dev = obj->base.dev;
3031 drm_i915_private_t *dev_priv = dev->dev_private;
3678 drm_i915_private_t *dev_priv = dev->dev_private;
3032 u32 size = obj->gtt_space->size;
3033 int regnum = obj->fence_reg;
3034 uint64_t val;
3035
3679 uint64_t val;
3680
3036 val = (uint64_t)((obj->gtt_offset + size - 4096) &
3037 0xfffff000) << 32;
3038 val |= obj->gtt_offset & 0xfffff000;
3039 val |= ((obj->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
3040 if (obj->tiling_mode == I915_TILING_Y)
3041 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
3042 val |= I965_FENCE_REG_VALID;
3681 if (obj) {
3682 u32 size = obj->gtt_space->size;
3043
3683
3044 if (pipelined) {
3045 int ret = intel_ring_begin(pipelined, 6);
3046 if (ret)
3047 return ret;
3048
3049 intel_ring_emit(pipelined, MI_NOOP);
3050 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(2));
3051 intel_ring_emit(pipelined, FENCE_REG_965_0 + regnum*8);
3052 intel_ring_emit(pipelined, (u32)val);
3053 intel_ring_emit(pipelined, FENCE_REG_965_0 + regnum*8 + 4);
3054 intel_ring_emit(pipelined, (u32)(val >> 32));
3055 intel_ring_advance(pipelined);
3684 val = (uint64_t)((obj->gtt_offset + size - 4096) &
3685 0xfffff000) << 32;
3686 val |= obj->gtt_offset & 0xfffff000;
3687 val |= ((obj->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
3688 if (obj->tiling_mode == I915_TILING_Y)
3689 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
3690 val |= I965_FENCE_REG_VALID;
3056 } else
3691 } else
3057 I915_WRITE64(FENCE_REG_965_0 + regnum * 8, val);
3692 val = 0;
3058
3693
3059 return 0;
3694 I915_WRITE64(FENCE_REG_965_0 + reg * 8, val);
3695 POSTING_READ(FENCE_REG_965_0 + reg * 8);
3060}
3061
3696}
3697
3062static int
3063i915_write_fence_reg(struct drm_i915_gem_object *obj,
3064 struct intel_ring_buffer *pipelined)
3698static void i915_write_fence_reg(struct drm_device *dev, int reg,
3699 struct drm_i915_gem_object *obj)
3065{
3700{
3066 struct drm_device *dev = obj->base.dev;
3067 drm_i915_private_t *dev_priv = dev->dev_private;
3701 drm_i915_private_t *dev_priv = dev->dev_private;
3068 u32 size = obj->gtt_space->size;
3069 u32 fence_reg, val, pitch_val;
3070 int tile_width;
3702 u32 val;
3071
3703
3072 if ((obj->gtt_offset & ~I915_FENCE_START_MASK) ||
3073 (size & -size) != size || (obj->gtt_offset & (size - 1))) {
3074 printf(
3075"object 0x%08x [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
3076 obj->gtt_offset, obj->map_and_fenceable, size);
3077 return -EINVAL;
3078 }
3704 if (obj) {
3705 u32 size = obj->gtt_space->size;
3706 int pitch_val;
3707 int tile_width;
3079
3708
3080 if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
3081 tile_width = 128;
3082 else
3083 tile_width = 512;
3709 if ((obj->gtt_offset & ~I915_FENCE_START_MASK) ||
3710 (size & -size) != size ||
3711 (obj->gtt_offset & (size - 1)))
3712 printf(
3713 "object 0x%08x [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
3714 obj->gtt_offset, obj->map_and_fenceable, size);
3084
3715
3085 /* Note: pitch better be a power of two tile widths */
3086 pitch_val = obj->stride / tile_width;
3087 pitch_val = ffs(pitch_val) - 1;
3716 if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
3717 tile_width = 128;
3718 else
3719 tile_width = 512;
3088
3720
3089 val = obj->gtt_offset;
3090 if (obj->tiling_mode == I915_TILING_Y)
3091 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
3092 val |= I915_FENCE_SIZE_BITS(size);
3093 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
3094 val |= I830_FENCE_REG_VALID;
3721 /* Note: pitch better be a power of two tile widths */
3722 pitch_val = obj->stride / tile_width;
3723 pitch_val = ffs(pitch_val) - 1;
3095
3724
3096 fence_reg = obj->fence_reg;
3097 if (fence_reg < 8)
3098 fence_reg = FENCE_REG_830_0 + fence_reg * 4;
3099 else
3100 fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4;
3101
3102 if (pipelined) {
3103 int ret = intel_ring_begin(pipelined, 4);
3104 if (ret)
3105 return ret;
3106
3107 intel_ring_emit(pipelined, MI_NOOP);
3108 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(1));
3109 intel_ring_emit(pipelined, fence_reg);
3110 intel_ring_emit(pipelined, val);
3111 intel_ring_advance(pipelined);
3725 val = obj->gtt_offset;
3726 if (obj->tiling_mode == I915_TILING_Y)
3727 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
3728 val |= I915_FENCE_SIZE_BITS(size);
3729 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
3730 val |= I830_FENCE_REG_VALID;
3112 } else
3731 } else
3113 I915_WRITE(fence_reg, val);
3732 val = 0;
3114
3733
3115 return 0;
3734 if (reg < 8)
3735 reg = FENCE_REG_830_0 + reg * 4;
3736 else
3737 reg = FENCE_REG_945_8 + (reg - 8) * 4;
3738
3739 I915_WRITE(reg, val);
3740 POSTING_READ(reg);
3116}
3117
3741}
3742
3118static int
3119i830_write_fence_reg(struct drm_i915_gem_object *obj,
3120 struct intel_ring_buffer *pipelined)
3743static void i830_write_fence_reg(struct drm_device *dev, int reg,
3744 struct drm_i915_gem_object *obj)
3121{
3745{
3122 struct drm_device *dev = obj->base.dev;
3123 drm_i915_private_t *dev_priv = dev->dev_private;
3746 drm_i915_private_t *dev_priv = dev->dev_private;
3124 u32 size = obj->gtt_space->size;
3125 int regnum = obj->fence_reg;
3126 uint32_t val;
3747 uint32_t val;
3127 uint32_t pitch_val;
3128
3748
3129 if ((obj->gtt_offset & ~I830_FENCE_START_MASK) ||
3130 (size & -size) != size || (obj->gtt_offset & (size - 1))) {
3131 printf(
3132"object 0x%08x not 512K or pot-size 0x%08x aligned\n",
3133 obj->gtt_offset, size);
3134 return -EINVAL;
3135 }
3749 if (obj) {
3750 u32 size = obj->gtt_space->size;
3751 uint32_t pitch_val;
3136
3752
3137 pitch_val = obj->stride / 128;
3138 pitch_val = ffs(pitch_val) - 1;
3753 if ((obj->gtt_offset & ~I830_FENCE_START_MASK) ||
3754 (size & -size) != size ||
3755 (obj->gtt_offset & (size - 1)))
3756 printf(
3757 "object 0x%08x not 512K or pot-size 0x%08x aligned\n",
3758 obj->gtt_offset, size);
3139
3759
3140 val = obj->gtt_offset;
3141 if (obj->tiling_mode == I915_TILING_Y)
3142 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
3143 val |= I830_FENCE_SIZE_BITS(size);
3144 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
3145 val |= I830_FENCE_REG_VALID;
3760 pitch_val = obj->stride / 128;
3761 pitch_val = ffs(pitch_val) - 1;
3146
3762
3147 if (pipelined) {
3148 int ret = intel_ring_begin(pipelined, 4);
3149 if (ret)
3150 return ret;
3151
3152 intel_ring_emit(pipelined, MI_NOOP);
3153 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(1));
3154 intel_ring_emit(pipelined, FENCE_REG_830_0 + regnum*4);
3155 intel_ring_emit(pipelined, val);
3156 intel_ring_advance(pipelined);
3763 val = obj->gtt_offset;
3764 if (obj->tiling_mode == I915_TILING_Y)
3765 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
3766 val |= I830_FENCE_SIZE_BITS(size);
3767 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
3768 val |= I830_FENCE_REG_VALID;
3157 } else
3769 } else
3158 I915_WRITE(FENCE_REG_830_0 + regnum * 4, val);
3770 val = 0;
3159
3771
3160 return 0;
3772 I915_WRITE(FENCE_REG_830_0 + reg * 4, val);
3773 POSTING_READ(FENCE_REG_830_0 + reg * 4);
3161}
3162
3774}
3775
3163static bool ring_passed_seqno(struct intel_ring_buffer *ring, u32 seqno)
3776static void i915_gem_write_fence(struct drm_device *dev, int reg,
3777 struct drm_i915_gem_object *obj)
3164{
3778{
3165 return i915_seqno_passed(ring->get_seqno(ring), seqno);
3779 switch (INTEL_INFO(dev)->gen) {
3780 case 7:
3781 case 6: sandybridge_write_fence_reg(dev, reg, obj); break;
3782 case 5:
3783 case 4: i965_write_fence_reg(dev, reg, obj); break;
3784 case 3: i915_write_fence_reg(dev, reg, obj); break;
3785 case 2: i830_write_fence_reg(dev, reg, obj); break;
3786 default: break;
3787 }
3166}
3167
3788}
3789
3790static inline int fence_number(struct drm_i915_private *dev_priv,
3791 struct drm_i915_fence_reg *fence)
3792{
3793 return fence - dev_priv->fence_regs;
3794}
3795
3796static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
3797 struct drm_i915_fence_reg *fence,
3798 bool enable)
3799{
3800 struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
3801 int reg = fence_number(dev_priv, fence);
3802
3803 i915_gem_write_fence(obj->base.dev, reg, enable ? obj : NULL);
3804
3805 if (enable) {
3806 obj->fence_reg = reg;
3807 fence->obj = obj;
3808 list_move_tail(&fence->lru_list, &dev_priv->mm.fence_list);
3809 } else {
3810 obj->fence_reg = I915_FENCE_REG_NONE;
3811 fence->obj = NULL;
3812 list_del_init(&fence->lru_list);
3813 }
3814}
3815
3168static int
3816static int
3169i915_gem_object_flush_fence(struct drm_i915_gem_object *obj,
3170 struct intel_ring_buffer *pipelined)
3817i915_gem_object_flush_fence(struct drm_i915_gem_object *obj)
3171{
3172 int ret;
3173
3174 if (obj->fenced_gpu_access) {
3175 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
3818{
3819 int ret;
3820
3821 if (obj->fenced_gpu_access) {
3822 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
3176 ret = i915_gem_flush_ring(obj->last_fenced_ring, 0,
3177 obj->base.write_domain);
3823 ret = i915_gem_flush_ring(obj->ring,
3824 0, obj->base.write_domain);
3178 if (ret)
3179 return ret;
3180 }
3181
3182 obj->fenced_gpu_access = false;
3183 }
3184
3825 if (ret)
3826 return ret;
3827 }
3828
3829 obj->fenced_gpu_access = false;
3830 }
3831
3185 if (obj->last_fenced_seqno && pipelined != obj->last_fenced_ring) {
3186 if (!ring_passed_seqno(obj->last_fenced_ring,
3187 obj->last_fenced_seqno)) {
3188 ret = i915_wait_request(obj->last_fenced_ring,
3189 obj->last_fenced_seqno,
3190 true);
3191 if (ret)
3192 return ret;
3193 }
3832 if (obj->last_fenced_seqno) {
3833 ret = i915_wait_request(obj->ring,
3834 obj->last_fenced_seqno);
3835 if (ret)
3836 return ret;
3194
3195 obj->last_fenced_seqno = 0;
3837
3838 obj->last_fenced_seqno = 0;
3196 obj->last_fenced_ring = NULL;
3197 }
3198
3199 /* Ensure that all CPU reads are completed before installing a fence
3200 * and all writes before removing the fence.
3201 */
3202 if (obj->base.read_domains & I915_GEM_DOMAIN_GTT)
3203 mb();
3204
3205 return 0;
3206}
3207
3208int
3209i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
3210{
3839 }
3840
3841 /* Ensure that all CPU reads are completed before installing a fence
3842 * and all writes before removing the fence.
3843 */
3844 if (obj->base.read_domains & I915_GEM_DOMAIN_GTT)
3845 mb();
3846
3847 return 0;
3848}
3849
3850int
3851i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
3852{
3853 struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
3211 int ret;
3212
3854 int ret;
3855
3213 if (obj->tiling_mode)
3214 i915_gem_release_mmap(obj);
3215
3216 ret = i915_gem_object_flush_fence(obj, NULL);
3856 ret = i915_gem_object_flush_fence(obj);
3217 if (ret)
3218 return ret;
3219
3857 if (ret)
3858 return ret;
3859
3220 if (obj->fence_reg != I915_FENCE_REG_NONE) {
3221 struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
3860 if (obj->fence_reg == I915_FENCE_REG_NONE)
3861 return 0;
3222
3862
3223 if (dev_priv->fence_regs[obj->fence_reg].pin_count != 0)
3224 printf("%s: pin_count %d\n", __func__,
3225 dev_priv->fence_regs[obj->fence_reg].pin_count);
3226 i915_gem_clear_fence_reg(obj->base.dev,
3227 &dev_priv->fence_regs[obj->fence_reg]);
3863 i915_gem_object_update_fence(obj,
3864 &dev_priv->fence_regs[obj->fence_reg],
3865 false);
3866 i915_gem_object_fence_lost(obj);
3228
3867
3229 obj->fence_reg = I915_FENCE_REG_NONE;
3230 }
3231
3232 return 0;
3233}
3234
3235static struct drm_i915_fence_reg *
3868 return 0;
3869}
3870
3871static struct drm_i915_fence_reg *
3236i915_find_fence_reg(struct drm_device *dev, struct intel_ring_buffer *pipelined)
3872i915_find_fence_reg(struct drm_device *dev)
3237{
3238 struct drm_i915_private *dev_priv = dev->dev_private;
3873{
3874 struct drm_i915_private *dev_priv = dev->dev_private;
3239 struct drm_i915_fence_reg *reg, *first, *avail;
3875 struct drm_i915_fence_reg *reg, *avail;
3240 int i;
3241
3242 /* First try to find a free reg */
3243 avail = NULL;
3244 for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
3245 reg = &dev_priv->fence_regs[i];
3246 if (!reg->obj)
3247 return reg;
3248
3249 if (!reg->pin_count)
3250 avail = reg;
3251 }
3252
3253 if (avail == NULL)
3254 return NULL;
3255
3256 /* None available, try to steal one or wait for a user to finish */
3876 int i;
3877
3878 /* First try to find a free reg */
3879 avail = NULL;
3880 for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
3881 reg = &dev_priv->fence_regs[i];
3882 if (!reg->obj)
3883 return reg;
3884
3885 if (!reg->pin_count)
3886 avail = reg;
3887 }
3888
3889 if (avail == NULL)
3890 return NULL;
3891
3892 /* None available, try to steal one or wait for a user to finish */
3257 avail = first = NULL;
3258 list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) {
3259 if (reg->pin_count)
3260 continue;
3261
3893 list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) {
3894 if (reg->pin_count)
3895 continue;
3896
3262 if (first == NULL)
3263 first = reg;
3264
3265 if (!pipelined ||
3266 !reg->obj->last_fenced_ring ||
3267 reg->obj->last_fenced_ring == pipelined) {
3268 avail = reg;
3269 break;
3270 }
3897 return reg;
3271 }
3272
3898 }
3899
3273 if (avail == NULL)
3274 avail = first;
3275
3276 return avail;
3900 return NULL;
3277}
3278
3279int
3901}
3902
3903int
3280i915_gem_object_get_fence(struct drm_i915_gem_object *obj,
3281 struct intel_ring_buffer *pipelined)
3904i915_gem_object_get_fence(struct drm_i915_gem_object *obj)
3282{
3283 struct drm_device *dev = obj->base.dev;
3284 struct drm_i915_private *dev_priv = dev->dev_private;
3905{
3906 struct drm_device *dev = obj->base.dev;
3907 struct drm_i915_private *dev_priv = dev->dev_private;
3908 bool enable = obj->tiling_mode != I915_TILING_NONE;
3285 struct drm_i915_fence_reg *reg;
3286 int ret;
3287
3909 struct drm_i915_fence_reg *reg;
3910 int ret;
3911
3288 pipelined = NULL;
3912 /* Have we updated the tiling parameters upon the object and so
3913 * will need to serialise the write to the associated fence register?
3914 */
3915 if (obj->fence_dirty) {
3916 ret = i915_gem_object_flush_fence(obj);
3917 if (ret)
3918 return ret;
3919 }
3920
3289 ret = 0;
3290
3291 if (obj->fence_reg != I915_FENCE_REG_NONE) {
3292 reg = &dev_priv->fence_regs[obj->fence_reg];
3921 ret = 0;
3922
3923 if (obj->fence_reg != I915_FENCE_REG_NONE) {
3924 reg = &dev_priv->fence_regs[obj->fence_reg];
3293 list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
3294
3295 if (obj->tiling_changed) {
3296 ret = i915_gem_object_flush_fence(obj, pipelined);
3297 if (ret)
3298 return ret;
3299
3300 if (!obj->fenced_gpu_access && !obj->last_fenced_seqno)
3301 pipelined = NULL;
3302
3303 if (pipelined) {
3304 reg->setup_seqno =
3305 i915_gem_next_request_seqno(pipelined);
3306 obj->last_fenced_seqno = reg->setup_seqno;
3307 obj->last_fenced_ring = pipelined;
3308 }
3309
3310 goto update;
3925 if (!obj->fence_dirty) {
3926 list_move_tail(&reg->lru_list,
3927 &dev_priv->mm.fence_list);
3928 return 0;
3311 }
3929 }
3930 } else if (enable) {
3931 reg = i915_find_fence_reg(dev);
3932 if (reg == NULL)
3933 return -EDEADLK;
3312
3934
3313 if (!pipelined) {
3314 if (reg->setup_seqno) {
3315 if (!ring_passed_seqno(obj->last_fenced_ring,
3316 reg->setup_seqno)) {
3317 ret = i915_wait_request(
3318 obj->last_fenced_ring,
3319 reg->setup_seqno,
3320 true);
3321 if (ret)
3322 return ret;
3323 }
3935 if (reg->obj) {
3936 struct drm_i915_gem_object *old = reg->obj;
3324
3937
3325 reg->setup_seqno = 0;
3326 }
3327 } else if (obj->last_fenced_ring &&
3328 obj->last_fenced_ring != pipelined) {
3329 ret = i915_gem_object_flush_fence(obj, pipelined);
3938 ret = i915_gem_object_flush_fence(old);
3330 if (ret)
3331 return ret;
3939 if (ret)
3940 return ret;
3332 }
3333
3941
3334 if (!obj->fenced_gpu_access && !obj->last_fenced_seqno)
3335 pipelined = NULL;
3336 KASSERT(pipelined || reg->setup_seqno == 0, ("!pipelined"));
3337
3338 if (obj->tiling_changed) {
3339 if (pipelined) {
3340 reg->setup_seqno =
3341 i915_gem_next_request_seqno(pipelined);
3342 obj->last_fenced_seqno = reg->setup_seqno;
3343 obj->last_fenced_ring = pipelined;
3344 }
3345 goto update;
3942 i915_gem_object_fence_lost(old);
3346 }
3943 }
3347
3944 } else
3348 return 0;
3945 return 0;
3349 }
3350
3946
3351 reg = i915_find_fence_reg(dev, pipelined);
3352 if (reg == NULL)
3353 return -EDEADLK;
3947 i915_gem_object_update_fence(obj, reg, enable);
3948 obj->fence_dirty = false;
3354
3949
3355 ret = i915_gem_object_flush_fence(obj, pipelined);
3356 if (ret)
3357 return ret;
3358
3359 if (reg->obj) {
3360 struct drm_i915_gem_object *old = reg->obj;
3361
3362 drm_gem_object_reference(&old->base);
3363
3364 if (old->tiling_mode)
3365 i915_gem_release_mmap(old);
3366
3367 ret = i915_gem_object_flush_fence(old, pipelined);
3368 if (ret) {
3369 drm_gem_object_unreference(&old->base);
3370 return ret;
3371 }
3372
3373 if (old->last_fenced_seqno == 0 && obj->last_fenced_seqno == 0)
3374 pipelined = NULL;
3375
3376 old->fence_reg = I915_FENCE_REG_NONE;
3377 old->last_fenced_ring = pipelined;
3378 old->last_fenced_seqno =
3379 pipelined ? i915_gem_next_request_seqno(pipelined) : 0;
3380
3381 drm_gem_object_unreference(&old->base);
3382 } else if (obj->last_fenced_seqno == 0)
3383 pipelined = NULL;
3384
3385 reg->obj = obj;
3386 list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
3387 obj->fence_reg = reg - dev_priv->fence_regs;
3388 obj->last_fenced_ring = pipelined;
3389
3390 reg->setup_seqno =
3391 pipelined ? i915_gem_next_request_seqno(pipelined) : 0;
3392 obj->last_fenced_seqno = reg->setup_seqno;
3393
3394update:
3395 obj->tiling_changed = false;
3396 switch (INTEL_INFO(dev)->gen) {
3397 case 7:
3398 case 6:
3399 ret = sandybridge_write_fence_reg(obj, pipelined);
3400 break;
3401 case 5:
3402 case 4:
3403 ret = i965_write_fence_reg(obj, pipelined);
3404 break;
3405 case 3:
3406 ret = i915_write_fence_reg(obj, pipelined);
3407 break;
3408 case 2:
3409 ret = i830_write_fence_reg(obj, pipelined);
3410 break;
3411 }
3412
3413 return ret;
3950 return 0;
3414}
3415
3951}
3952
3416static void
3417i915_gem_clear_fence_reg(struct drm_device *dev, struct drm_i915_fence_reg *reg)
3418{
3419 drm_i915_private_t *dev_priv = dev->dev_private;
3420 uint32_t fence_reg = reg - dev_priv->fence_regs;
3421
3422 switch (INTEL_INFO(dev)->gen) {
3423 case 7:
3424 case 6:
3425 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + fence_reg*8, 0);
3426 break;
3427 case 5:
3428 case 4:
3429 I915_WRITE64(FENCE_REG_965_0 + fence_reg*8, 0);
3430 break;
3431 case 3:
3432 if (fence_reg >= 8)
3433 fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4;
3434 else
3435 case 2:
3436 fence_reg = FENCE_REG_830_0 + fence_reg * 4;
3437
3438 I915_WRITE(fence_reg, 0);
3439 break;
3440 }
3441
3442 list_del_init(&reg->lru_list);
3443 reg->obj = NULL;
3444 reg->setup_seqno = 0;
3445 reg->pin_count = 0;
3446}
3447
3448int
3449i915_gem_init_object(struct drm_gem_object *obj)
3450{
3451
3452 printf("i915_gem_init_object called\n");
3453 return (0);
3454}
3455
3456static bool
3457i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
3458{
3459
3953int
3954i915_gem_init_object(struct drm_gem_object *obj)
3955{
3956
3957 printf("i915_gem_init_object called\n");
3958 return (0);
3959}
3960
3961static bool
3962i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
3963{
3964
3460 return (obj->gtt_space && !obj->active && obj->pin_count == 0);
3965 return !obj->active;
3461}
3462
3463static void
3464i915_gem_retire_task_handler(void *arg, int pending)
3465{
3466 drm_i915_private_t *dev_priv;
3467 struct drm_device *dev;
3966}
3967
3968static void
3969i915_gem_retire_task_handler(void *arg, int pending)
3970{
3971 drm_i915_private_t *dev_priv;
3972 struct drm_device *dev;
3973 struct intel_ring_buffer *ring;
3468 bool idle;
3469 int i;
3470
3471 dev_priv = arg;
3472 dev = dev_priv->dev;
3473
3474 /* Come back later if the device is busy... */
3475 if (!sx_try_xlock(&dev->dev_struct_lock)) {

--- 5 unchanged lines hidden (view full) ---

3481 CTR0(KTR_DRM, "retire_task");
3482
3483 i915_gem_retire_requests(dev);
3484
3485 /* Send a periodic flush down the ring so we don't hold onto GEM
3486 * objects indefinitely.
3487 */
3488 idle = true;
3974 bool idle;
3975 int i;
3976
3977 dev_priv = arg;
3978 dev = dev_priv->dev;
3979
3980 /* Come back later if the device is busy... */
3981 if (!sx_try_xlock(&dev->dev_struct_lock)) {

--- 5 unchanged lines hidden (view full) ---

3987 CTR0(KTR_DRM, "retire_task");
3988
3989 i915_gem_retire_requests(dev);
3990
3991 /* Send a periodic flush down the ring so we don't hold onto GEM
3992 * objects indefinitely.
3993 */
3994 idle = true;
3489 for (i = 0; i < I915_NUM_RINGS; i++) {
3995 for_each_ring(ring, dev_priv, i) {
3490 struct intel_ring_buffer *ring = &dev_priv->rings[i];
3491
3492 if (!list_empty(&ring->gpu_write_list)) {
3493 struct drm_i915_gem_request *request;
3494 int ret;
3495
3496 ret = i915_gem_flush_ring(ring,
3497 0, I915_GEM_GPU_DOMAINS);

--- 99 unchanged lines hidden (view full) ---

3597
3598 if (obj->phys_obj == NULL)
3599 return;
3600 vaddr = obj->phys_obj->handle->vaddr;
3601
3602 page_count = obj->base.size / PAGE_SIZE;
3603 VM_OBJECT_WLOCK(obj->base.vm_obj);
3604 for (i = 0; i < page_count; i++) {
3996 struct intel_ring_buffer *ring = &dev_priv->rings[i];
3997
3998 if (!list_empty(&ring->gpu_write_list)) {
3999 struct drm_i915_gem_request *request;
4000 int ret;
4001
4002 ret = i915_gem_flush_ring(ring,
4003 0, I915_GEM_GPU_DOMAINS);

--- 99 unchanged lines hidden (view full) ---

4103
4104 if (obj->phys_obj == NULL)
4105 return;
4106 vaddr = obj->phys_obj->handle->vaddr;
4107
4108 page_count = obj->base.size / PAGE_SIZE;
4109 VM_OBJECT_WLOCK(obj->base.vm_obj);
4110 for (i = 0; i < page_count; i++) {
3605 m = i915_gem_wire_page(obj->base.vm_obj, i);
4111 m = i915_gem_wire_page(obj->base.vm_obj, i, NULL);
3606 if (m == NULL)
3607 continue; /* XXX */
3608
3609 VM_OBJECT_WUNLOCK(obj->base.vm_obj);
3610 sf = sf_buf_alloc(m, 0);
3611 if (sf != NULL) {
3612 dst = (char *)sf_buf_kva(sf);
3613 memcpy(dst, vaddr + IDX_TO_OFF(i), PAGE_SIZE);

--- 49 unchanged lines hidden (view full) ---

3663 obj->phys_obj = dev_priv->mm.phys_objs[id - 1];
3664 obj->phys_obj->cur_obj = obj;
3665
3666 page_count = obj->base.size / PAGE_SIZE;
3667
3668 VM_OBJECT_WLOCK(obj->base.vm_obj);
3669 ret = 0;
3670 for (i = 0; i < page_count; i++) {
4112 if (m == NULL)
4113 continue; /* XXX */
4114
4115 VM_OBJECT_WUNLOCK(obj->base.vm_obj);
4116 sf = sf_buf_alloc(m, 0);
4117 if (sf != NULL) {
4118 dst = (char *)sf_buf_kva(sf);
4119 memcpy(dst, vaddr + IDX_TO_OFF(i), PAGE_SIZE);

--- 49 unchanged lines hidden (view full) ---

4169 obj->phys_obj = dev_priv->mm.phys_objs[id - 1];
4170 obj->phys_obj->cur_obj = obj;
4171
4172 page_count = obj->base.size / PAGE_SIZE;
4173
4174 VM_OBJECT_WLOCK(obj->base.vm_obj);
4175 ret = 0;
4176 for (i = 0; i < page_count; i++) {
3671 m = i915_gem_wire_page(obj->base.vm_obj, i);
4177 m = i915_gem_wire_page(obj->base.vm_obj, i, NULL);
3672 if (m == NULL) {
3673 ret = -EIO;
3674 break;
3675 }
3676 VM_OBJECT_WUNLOCK(obj->base.vm_obj);
3677 sf = sf_buf_alloc(m, 0);
3678 src = (char *)sf_buf_kva(sf);
3679 dst = (char *)obj->phys_obj->handle->vaddr + IDX_TO_OFF(i);

--- 9 unchanged lines hidden (view full) ---

3689 atomic_add_long(&i915_gem_wired_pages_cnt, -1);
3690 }
3691 VM_OBJECT_WUNLOCK(obj->base.vm_obj);
3692
3693 return (0);
3694}
3695
3696static int
4178 if (m == NULL) {
4179 ret = -EIO;
4180 break;
4181 }
4182 VM_OBJECT_WUNLOCK(obj->base.vm_obj);
4183 sf = sf_buf_alloc(m, 0);
4184 src = (char *)sf_buf_kva(sf);
4185 dst = (char *)obj->phys_obj->handle->vaddr + IDX_TO_OFF(i);

--- 9 unchanged lines hidden (view full) ---

4195 atomic_add_long(&i915_gem_wired_pages_cnt, -1);
4196 }
4197 VM_OBJECT_WUNLOCK(obj->base.vm_obj);
4198
4199 return (0);
4200}
4201
4202static int
3697i915_gem_phys_pwrite(struct drm_device *dev, struct drm_i915_gem_object *obj,
3698 uint64_t data_ptr, uint64_t offset, uint64_t size,
3699 struct drm_file *file_priv)
3700{
3701 char *user_data, *vaddr;
3702 int ret;
3703
3704 vaddr = (char *)obj->phys_obj->handle->vaddr + offset;
3705 user_data = (char *)(uintptr_t)data_ptr;
3706
3707 if (copyin_nofault(user_data, vaddr, size) != 0) {
3708 /* The physical object once assigned is fixed for the lifetime
3709 * of the obj, so we can safely drop the lock and continue
3710 * to access vaddr.
3711 */
3712 DRM_UNLOCK(dev);
3713 ret = -copyin(user_data, vaddr, size);
3714 DRM_LOCK(dev);
3715 if (ret != 0)
3716 return (ret);
3717 }
3718
3719 intel_gtt_chipset_flush();
3720 return (0);
3721}
3722
3723static int
3724i915_gpu_is_active(struct drm_device *dev)
3725{
3726 drm_i915_private_t *dev_priv;
3727
3728 dev_priv = dev->dev_private;
3729 return (!list_empty(&dev_priv->mm.flushing_list) ||
3730 !list_empty(&dev_priv->mm.active_list));
3731}

--- 40 unchanged lines hidden (view full) ---

3772
3773 if (cnt_fail > cnt_total / 100 && i915_gpu_is_active(dev)) {
3774 /*
3775 * We are desperate for pages, so as a last resort, wait
3776 * for the GPU to finish and discard whatever we can.
3777 * This has a dramatic impact to reduce the number of
3778 * OOM-killer events whilst running the GPU aggressively.
3779 */
4203i915_gpu_is_active(struct drm_device *dev)
4204{
4205 drm_i915_private_t *dev_priv;
4206
4207 dev_priv = dev->dev_private;
4208 return (!list_empty(&dev_priv->mm.flushing_list) ||
4209 !list_empty(&dev_priv->mm.active_list));
4210}

--- 40 unchanged lines hidden (view full) ---

4251
4252 if (cnt_fail > cnt_total / 100 && i915_gpu_is_active(dev)) {
4253 /*
4254 * We are desperate for pages, so as a last resort, wait
4255 * for the GPU to finish and discard whatever we can.
4256 * This has a dramatic impact to reduce the number of
4257 * OOM-killer events whilst running the GPU aggressively.
4258 */
3780 if (i915_gpu_idle(dev, true) == 0)
4259 if (i915_gpu_idle(dev) == 0)
3781 goto rescan;
3782 }
3783 DRM_UNLOCK(dev);
3784}
3785
3786void
3787i915_gem_unload(struct drm_device *dev)
3788{
3789 struct drm_i915_private *dev_priv;
3790
3791 dev_priv = dev->dev_private;
3792 EVENTHANDLER_DEREGISTER(vm_lowmem, dev_priv->mm.i915_lowmem);
3793}
4260 goto rescan;
4261 }
4262 DRM_UNLOCK(dev);
4263}
4264
4265void
4266i915_gem_unload(struct drm_device *dev)
4267{
4268 struct drm_i915_private *dev_priv;
4269
4270 dev_priv = dev->dev_private;
4271 EVENTHANDLER_DEREGISTER(vm_lowmem, dev_priv->mm.i915_lowmem);
4272}