i915/gem/i915_gem_userptr.c

72 i915_gem_userptr_init__mmu_notifier(struct drm_i915_gem_object *obj)
74 	return mmu_interval_notifier_insert(&obj->userptr.notifier, current->mm,
75 					    obj->userptr.ptr, obj->base.size,
79 static void i915_gem_object_userptr_drop_ref(struct drm_i915_gem_object *obj)
83 	assert_object_held_shared(obj);
85 	if (!--obj->userptr.page_ref) {
86 		pvec = obj->userptr.pvec;
87 		obj->userptr.pvec = NULL;
89 	GEM_BUG_ON(obj->userptr.page_ref < 0);
92 		const unsigned long num_pages = obj->base.size >> PAGE_SHIFT;
99 static int i915_gem_userptr_get_pages(struct drm_i915_gem_object *obj)
101 	unsigned int max_segment = i915_sg_segment_size(obj->base.dev->dev);
107 	if (overflows_type(obj->base.size >> PAGE_SHIFT, num_pages))
110 	num_pages = obj->base.size >> PAGE_SHIFT;
115 	if (!obj->userptr.page_ref) {
120 	obj->userptr.page_ref++;
121 	pvec = obj->userptr.pvec;
130 	ret = i915_gem_gtt_prepare_pages(obj, st);
142 	WARN_ON_ONCE(!(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE));
143 	if (i915_gem_object_can_bypass_llc(obj))
144 		obj->cache_dirty = true;
146 	__i915_gem_object_set_pages(obj, st);
151 	i915_gem_object_userptr_drop_ref(obj);
158 i915_gem_userptr_put_pages(struct drm_i915_gem_object *obj,
167 	__i915_gem_object_release_shmem(obj, pages, true);
168 	i915_gem_gtt_finish_pages(obj, pages);
175 	if (i915_gem_object_is_readonly(obj))
176 		obj->mm.dirty = false;
179 		if (obj->mm.dirty && trylock_page(page)) {
204 	obj->mm.dirty = false;
209 	i915_gem_object_userptr_drop_ref(obj);
212 static int i915_gem_object_userptr_unbind(struct drm_i915_gem_object *obj)
217 	err = i915_gem_object_unbind(obj, I915_GEM_OBJECT_UNBIND_ACTIVE);
221 	if (GEM_WARN_ON(i915_gem_object_has_pinned_pages(obj)))
224 	assert_object_held(obj);
226 	pages = __i915_gem_object_unset_pages(obj);
228 		i915_gem_userptr_put_pages(obj, pages);
233 int i915_gem_object_userptr_submit_init(struct drm_i915_gem_object *obj)
235 	const unsigned long num_pages = obj->base.size >> PAGE_SHIFT;
241 	if (obj->userptr.notifier.mm != current->mm)
244 	notifier_seq = mmu_interval_read_begin(&obj->userptr.notifier);
246 	ret = i915_gem_object_lock_interruptible(obj, NULL);
250 	if (notifier_seq == obj->userptr.notifier_seq && obj->userptr.pvec) {
251 		i915_gem_object_unlock(obj);
255 	ret = i915_gem_object_userptr_unbind(obj);
256 	i915_gem_object_unlock(obj);
264 	if (!i915_gem_object_is_readonly(obj))
269 		ret = pin_user_pages_fast(obj->userptr.ptr + pinned * PAGE_SIZE,
278 	ret = i915_gem_object_lock_interruptible(obj, NULL);
282 	if (mmu_interval_read_retry(&obj->userptr.notifier,
283 		!obj->userptr.page_ref ? notifier_seq :
284 		obj->userptr.notifier_seq)) {
289 	if (!obj->userptr.page_ref++) {
290 		obj->userptr.pvec = pvec;
291 		obj->userptr.notifier_seq = notifier_seq;
293 		ret = ____i915_gem_object_get_pages(obj);
296 	obj->userptr.page_ref--;
299 	i915_gem_object_unlock(obj);
310 int i915_gem_object_userptr_submit_done(struct drm_i915_gem_object *obj)
312 	if (mmu_interval_read_retry(&obj->userptr.notifier,
313 				    obj->userptr.notifier_seq)) {
322 int i915_gem_object_userptr_validate(struct drm_i915_gem_object *obj)
326 	err = i915_gem_object_userptr_submit_init(obj);
330 	err = i915_gem_object_lock_interruptible(obj, NULL);
337 		err = i915_gem_object_pin_pages(obj);
339 			i915_gem_object_unpin_pages(obj);
341 		i915_gem_object_unlock(obj);
348 i915_gem_userptr_release(struct drm_i915_gem_object *obj)
350 	GEM_WARN_ON(obj->userptr.page_ref);
352 	if (!obj->userptr.notifier.mm)
355 	mmu_interval_notifier_remove(&obj->userptr.notifier);
356 	obj->userptr.notifier.mm = NULL;
360 i915_gem_userptr_dmabuf_export(struct drm_i915_gem_object *obj)
362 	drm_dbg(obj->base.dev, "Exporting userptr no longer allowed\n");
368 i915_gem_userptr_pwrite(struct drm_i915_gem_object *obj,
371 	drm_dbg(obj->base.dev, "pwrite to userptr no longer allowed\n");
377 i915_gem_userptr_pread(struct drm_i915_gem_object *obj,
380 	drm_dbg(obj->base.dev, "pread from userptr no longer allowed\n");
468 	struct drm_i915_gem_object __maybe_unused *obj;
519 	obj = i915_gem_object_alloc();
520 	if (obj == NULL)
523 	drm_gem_private_object_init(dev, &obj->base, args->user_size);
524 	i915_gem_object_init(obj, &i915_gem_userptr_ops, &lock_class,
526 	obj->mem_flags = I915_BO_FLAG_STRUCT_PAGE;
527 	obj->read_domains = I915_GEM_DOMAIN_CPU;
528 	obj->write_domain = I915_GEM_DOMAIN_CPU;
529 	i915_gem_object_set_cache_coherency(obj, I915_CACHE_LLC);
531 	obj->userptr.ptr = args->user_ptr;
532 	obj->userptr.notifier_seq = ULONG_MAX;
534 		i915_gem_object_set_readonly(obj);
540 	ret = i915_gem_userptr_init__mmu_notifier(obj);
542 		ret = drm_gem_handle_create(file, &obj->base, &handle);
545 	i915_gem_object_put(obj);