1/* 2 * Copyright �� 2016 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 */ 24 25#include "gem/i915_gem_internal.h" 26#include "gem/i915_gem_pm.h" 27#include "gem/selftests/igt_gem_utils.h" 28#include "gem/selftests/mock_context.h" 29#include "gt/intel_gt.h" 30#include "gt/intel_gt_print.h" 31 32#include "i915_selftest.h" 33 34#include "igt_flush_test.h" 35#include "lib_sw_fence.h" 36#include "mock_drm.h" 37#include "mock_gem_device.h" 38 39static void quirk_add(struct drm_i915_gem_object *obj, 40 struct list_head *objects) 41{ 42 /* quirk is only for live tiled objects, use it to declare ownership */ 43 GEM_BUG_ON(i915_gem_object_has_tiling_quirk(obj)); 44 i915_gem_object_set_tiling_quirk(obj); 45 list_add(&obj->st_link, objects); 46} 47 48static int populate_ggtt(struct i915_ggtt *ggtt, struct list_head *objects) 49{ 50 struct drm_i915_gem_object *obj; 51 unsigned long count; 52 53 count = 0; 54 do { 55 struct i915_vma *vma; 56 57 obj = i915_gem_object_create_internal(ggtt->vm.i915, 58 I915_GTT_PAGE_SIZE); 59 if (IS_ERR(obj)) 60 return PTR_ERR(obj); 61 62 vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, 0); 63 if (IS_ERR(vma)) { 64 i915_gem_object_put(obj); 65 if (vma == ERR_PTR(-ENOSPC)) 66 break; 67 68 return PTR_ERR(vma); 69 } 70 71 quirk_add(obj, objects); 72 count++; 73 } while (1); 74 pr_debug("Filled GGTT with %lu pages [%llu total]\n", 75 count, ggtt->vm.total / PAGE_SIZE); 76 77 if (list_empty(&ggtt->vm.bound_list)) { 78 pr_err("No objects on the GGTT inactive list!\n"); 79 return -EINVAL; 80 } 81 82 return 0; 83} 84 85static void unpin_ggtt(struct i915_ggtt *ggtt) 86{ 87 struct i915_vma *vma; 88 89 list_for_each_entry(vma, &ggtt->vm.bound_list, vm_link) 90 if (i915_gem_object_has_tiling_quirk(vma->obj)) 91 i915_vma_unpin(vma); 92} 93 94static void cleanup_objects(struct i915_ggtt *ggtt, struct list_head *list) 95{ 96 struct drm_i915_gem_object *obj, *on; 97 98 list_for_each_entry_safe(obj, on, list, st_link) { 99 GEM_BUG_ON(!i915_gem_object_has_tiling_quirk(obj)); 100 i915_gem_object_set_tiling_quirk(obj); 101 i915_gem_object_put(obj); 102 } 103 104 i915_gem_drain_freed_objects(ggtt->vm.i915); 105} 106 107static int igt_evict_something(void *arg) 108{ 109 struct intel_gt *gt = arg; 110 struct i915_ggtt *ggtt = gt->ggtt; 111 LIST_HEAD(objects); 112 int err; 113 114 /* Fill the GGTT with pinned objects and try to evict one. */ 115 116 err = populate_ggtt(ggtt, &objects); 117 if (err) 118 goto cleanup; 119 120 /* Everything is pinned, nothing should happen */ 121 mutex_lock(&ggtt->vm.mutex); 122 err = i915_gem_evict_something(&ggtt->vm, NULL, 123 I915_GTT_PAGE_SIZE, 0, 0, 124 0, U64_MAX, 125 0); 126 mutex_unlock(&ggtt->vm.mutex); 127 if (err != -ENOSPC) { 128 pr_err("i915_gem_evict_something failed on a full GGTT with err=%d\n", 129 err); 130 goto cleanup; 131 } 132 133 unpin_ggtt(ggtt); 134 135 /* Everything is unpinned, we should be able to evict something */ 136 mutex_lock(&ggtt->vm.mutex); 137 err = i915_gem_evict_something(&ggtt->vm, NULL, 138 I915_GTT_PAGE_SIZE, 0, 0, 139 0, U64_MAX, 140 0); 141 mutex_unlock(&ggtt->vm.mutex); 142 if (err) { 143 pr_err("i915_gem_evict_something failed on a full GGTT with err=%d\n", 144 err); 145 goto cleanup; 146 } 147 148cleanup: 149 cleanup_objects(ggtt, &objects); 150 return err; 151} 152 153static int igt_overcommit(void *arg) 154{ 155 struct intel_gt *gt = arg; 156 struct i915_ggtt *ggtt = gt->ggtt; 157 struct drm_i915_gem_object *obj; 158 struct i915_vma *vma; 159 LIST_HEAD(objects); 160 int err; 161 162 /* Fill the GGTT with pinned objects and then try to pin one more. 163 * We expect it to fail. 164 */ 165 166 err = populate_ggtt(ggtt, &objects); 167 if (err) 168 goto cleanup; 169 170 obj = i915_gem_object_create_internal(gt->i915, I915_GTT_PAGE_SIZE); 171 if (IS_ERR(obj)) { 172 err = PTR_ERR(obj); 173 goto cleanup; 174 } 175 176 quirk_add(obj, &objects); 177 178 vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, 0); 179 if (vma != ERR_PTR(-ENOSPC)) { 180 pr_err("Failed to evict+insert, i915_gem_object_ggtt_pin returned err=%d\n", (int)PTR_ERR_OR_ZERO(vma)); 181 err = -EINVAL; 182 goto cleanup; 183 } 184 185cleanup: 186 cleanup_objects(ggtt, &objects); 187 return err; 188} 189 190static int igt_evict_for_vma(void *arg) 191{ 192 struct intel_gt *gt = arg; 193 struct i915_ggtt *ggtt = gt->ggtt; 194 struct drm_mm_node target = { 195 .start = 0, 196 .size = 4096, 197 }; 198 LIST_HEAD(objects); 199 int err; 200 201 /* Fill the GGTT with pinned objects and try to evict a range. */ 202 203 err = populate_ggtt(ggtt, &objects); 204 if (err) 205 goto cleanup; 206 207 /* Everything is pinned, nothing should happen */ 208 mutex_lock(&ggtt->vm.mutex); 209 err = i915_gem_evict_for_node(&ggtt->vm, NULL, &target, 0); 210 mutex_unlock(&ggtt->vm.mutex); 211 if (err != -ENOSPC) { 212 pr_err("i915_gem_evict_for_node on a full GGTT returned err=%d\n", 213 err); 214 goto cleanup; 215 } 216 217 unpin_ggtt(ggtt); 218 219 /* Everything is unpinned, we should be able to evict the node */ 220 mutex_lock(&ggtt->vm.mutex); 221 err = i915_gem_evict_for_node(&ggtt->vm, NULL, &target, 0); 222 mutex_unlock(&ggtt->vm.mutex); 223 if (err) { 224 pr_err("i915_gem_evict_for_node returned err=%d\n", 225 err); 226 goto cleanup; 227 } 228 229cleanup: 230 cleanup_objects(ggtt, &objects); 231 return err; 232} 233 234static void mock_color_adjust(const struct drm_mm_node *node, 235 unsigned long color, 236 u64 *start, 237 u64 *end) 238{ 239} 240 241static int igt_evict_for_cache_color(void *arg) 242{ 243 struct intel_gt *gt = arg; 244 struct i915_ggtt *ggtt = gt->ggtt; 245 const unsigned long flags = PIN_OFFSET_FIXED; 246 struct drm_mm_node target = { 247 .start = I915_GTT_PAGE_SIZE * 2, 248 .size = I915_GTT_PAGE_SIZE, 249 .color = i915_gem_get_pat_index(gt->i915, I915_CACHE_LLC), 250 }; 251 struct drm_i915_gem_object *obj; 252 struct i915_vma *vma; 253 LIST_HEAD(objects); 254 int err; 255 256 /* 257 * Currently the use of color_adjust for the GGTT is limited to cache 258 * coloring and guard pages, and so the presence of mm.color_adjust for 259 * the GGTT is assumed to be i915_ggtt_color_adjust, hence using a mock 260 * color adjust will work just fine for our purposes. 261 */ 262 ggtt->vm.mm.color_adjust = mock_color_adjust; 263 GEM_BUG_ON(!i915_vm_has_cache_coloring(&ggtt->vm)); 264 265 obj = i915_gem_object_create_internal(gt->i915, I915_GTT_PAGE_SIZE); 266 if (IS_ERR(obj)) { 267 err = PTR_ERR(obj); 268 goto cleanup; 269 } 270 i915_gem_object_set_cache_coherency(obj, I915_CACHE_LLC); 271 quirk_add(obj, &objects); 272 273 vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, 274 I915_GTT_PAGE_SIZE | flags); 275 if (IS_ERR(vma)) { 276 pr_err("[0]i915_gem_object_ggtt_pin failed\n"); 277 err = PTR_ERR(vma); 278 goto cleanup; 279 } 280 281 obj = i915_gem_object_create_internal(gt->i915, I915_GTT_PAGE_SIZE); 282 if (IS_ERR(obj)) { 283 err = PTR_ERR(obj); 284 goto cleanup; 285 } 286 i915_gem_object_set_cache_coherency(obj, I915_CACHE_LLC); 287 quirk_add(obj, &objects); 288 289 /* Neighbouring; same colour - should fit */ 290 vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, 291 (I915_GTT_PAGE_SIZE * 2) | flags); 292 if (IS_ERR(vma)) { 293 pr_err("[1]i915_gem_object_ggtt_pin failed\n"); 294 err = PTR_ERR(vma); 295 goto cleanup; 296 } 297 298 i915_vma_unpin(vma); 299 300 /* Remove just the second vma */ 301 mutex_lock(&ggtt->vm.mutex); 302 err = i915_gem_evict_for_node(&ggtt->vm, NULL, &target, 0); 303 mutex_unlock(&ggtt->vm.mutex); 304 if (err) { 305 pr_err("[0]i915_gem_evict_for_node returned err=%d\n", err); 306 goto cleanup; 307 } 308 309 /* Attempt to remove the first *pinned* vma, by removing the (empty) 310 * neighbour -- this should fail. 311 */ 312 target.color = i915_gem_get_pat_index(gt->i915, I915_CACHE_L3_LLC); 313 314 mutex_lock(&ggtt->vm.mutex); 315 err = i915_gem_evict_for_node(&ggtt->vm, NULL, &target, 0); 316 mutex_unlock(&ggtt->vm.mutex); 317 if (!err) { 318 pr_err("[1]i915_gem_evict_for_node returned err=%d\n", err); 319 err = -EINVAL; 320 goto cleanup; 321 } 322 323 err = 0; 324 325cleanup: 326 unpin_ggtt(ggtt); 327 cleanup_objects(ggtt, &objects); 328 ggtt->vm.mm.color_adjust = NULL; 329 return err; 330} 331 332static int igt_evict_vm(void *arg) 333{ 334 struct intel_gt *gt = arg; 335 struct i915_ggtt *ggtt = gt->ggtt; 336 struct i915_gem_ww_ctx ww; 337 LIST_HEAD(objects); 338 int err; 339 340 /* Fill the GGTT with pinned objects and try to evict everything. */ 341 342 err = populate_ggtt(ggtt, &objects); 343 if (err) 344 goto cleanup; 345 346 /* Everything is pinned, nothing should happen */ 347 mutex_lock(&ggtt->vm.mutex); 348 err = i915_gem_evict_vm(&ggtt->vm, NULL, NULL); 349 mutex_unlock(&ggtt->vm.mutex); 350 if (err) { 351 pr_err("i915_gem_evict_vm on a full GGTT returned err=%d]\n", 352 err); 353 goto cleanup; 354 } 355 356 unpin_ggtt(ggtt); 357 358 for_i915_gem_ww(&ww, err, false) { 359 mutex_lock(&ggtt->vm.mutex); 360 err = i915_gem_evict_vm(&ggtt->vm, &ww, NULL); 361 mutex_unlock(&ggtt->vm.mutex); 362 } 363 364 if (err) { 365 pr_err("i915_gem_evict_vm on a full GGTT returned err=%d]\n", 366 err); 367 goto cleanup; 368 } 369 370cleanup: 371 cleanup_objects(ggtt, &objects); 372 return err; 373} 374 375static int igt_evict_contexts(void *arg) 376{ 377 const u64 PRETEND_GGTT_SIZE = 16ull << 20; 378 struct intel_gt *gt = arg; 379 struct i915_ggtt *ggtt = gt->ggtt; 380 struct drm_i915_private *i915 = gt->i915; 381 struct intel_engine_cs *engine; 382 enum intel_engine_id id; 383 struct reserved { 384 struct drm_mm_node node; 385 struct reserved *next; 386 } *reserved = NULL; 387 intel_wakeref_t wakeref; 388 struct drm_mm_node hole; 389 unsigned long count; 390 int err; 391 392 /* 393 * The purpose of this test is to verify that we will trigger an 394 * eviction in the GGTT when constructing a request that requires 395 * additional space in the GGTT for pinning the context. This space 396 * is not directly tied to the request so reclaiming it requires 397 * extra work. 398 * 399 * As such this test is only meaningful for full-ppgtt environments 400 * where the GTT space of the request is separate from the GGTT 401 * allocation required to build the request. 402 */ 403 if (!HAS_FULL_PPGTT(i915)) 404 return 0; 405 406 wakeref = intel_runtime_pm_get(&i915->runtime_pm); 407 408 /* Reserve a block so that we know we have enough to fit a few rq */ 409 memset(&hole, 0, sizeof(hole)); 410 mutex_lock(&ggtt->vm.mutex); 411 err = i915_gem_gtt_insert(&ggtt->vm, NULL, &hole, 412 PRETEND_GGTT_SIZE, 0, I915_COLOR_UNEVICTABLE, 413 0, ggtt->vm.total, 414 PIN_NOEVICT); 415 if (err) 416 goto out_locked; 417 418 /* Make the GGTT appear small by filling it with unevictable nodes */ 419 count = 0; 420 do { 421 struct reserved *r; 422 423 mutex_unlock(&ggtt->vm.mutex); 424 r = kcalloc(1, sizeof(*r), GFP_KERNEL); 425 mutex_lock(&ggtt->vm.mutex); 426 if (!r) { 427 err = -ENOMEM; 428 goto out_locked; 429 } 430 431 if (i915_gem_gtt_insert(&ggtt->vm, NULL, &r->node, 432 1ul << 20, 0, I915_COLOR_UNEVICTABLE, 433 0, ggtt->vm.total, 434 PIN_NOEVICT)) { 435 kfree(r); 436 break; 437 } 438 439 r->next = reserved; 440 reserved = r; 441 442 count++; 443 } while (1); 444 drm_mm_remove_node(&hole); 445 mutex_unlock(&ggtt->vm.mutex); 446 pr_info("Filled GGTT with %lu 1MiB nodes\n", count); 447 448 /* Overfill the GGTT with context objects and so try to evict one. */ 449 for_each_engine(engine, gt, id) { 450 struct i915_sw_fence fence; 451 struct i915_request *last = NULL; 452 453 count = 0; 454 onstack_fence_init(&fence); 455 do { 456 struct intel_context *ce; 457 struct i915_request *rq; 458 459 ce = intel_context_create(engine); 460 if (IS_ERR(ce)) 461 break; 462 463 /* We will need some GGTT space for the rq's context */ 464 igt_evict_ctl.fail_if_busy = true; 465 rq = intel_context_create_request(ce); 466 igt_evict_ctl.fail_if_busy = false; 467 intel_context_put(ce); 468 469 if (IS_ERR(rq)) { 470 /* When full, fail_if_busy will trigger EBUSY */ 471 if (PTR_ERR(rq) != -EBUSY) { 472 pr_err("Unexpected error from request alloc (on %s): %d\n", 473 engine->name, 474 (int)PTR_ERR(rq)); 475 err = PTR_ERR(rq); 476 } 477 break; 478 } 479 480 /* Keep every request/ctx pinned until we are full */ 481 err = i915_sw_fence_await_sw_fence_gfp(&rq->submit, 482 &fence, 483 GFP_KERNEL); 484 if (err < 0) 485 break; 486 487 i915_request_add(rq); 488 count++; 489 if (last) 490 i915_request_put(last); 491 last = i915_request_get(rq); 492 err = 0; 493 } while(1); 494 onstack_fence_fini(&fence); 495 pr_info("Submitted %lu contexts/requests on %s\n", 496 count, engine->name); 497 if (err) 498 break; 499 if (last) { 500 if (i915_request_wait(last, 0, HZ) < 0) { 501 err = -EIO; 502 i915_request_put(last); 503 pr_err("Failed waiting for last request (on %s)", 504 engine->name); 505 break; 506 } 507 i915_request_put(last); 508 } 509 err = intel_gt_wait_for_idle(engine->gt, HZ * 3); 510 if (err) { 511 gt_err(engine->gt, "Failed to idle GT (on %s)", 512 engine->name); 513 break; 514 } 515 } 516 517 mutex_lock(&ggtt->vm.mutex); 518out_locked: 519 if (igt_flush_test(i915)) 520 err = -EIO; 521 while (reserved) { 522 struct reserved *next = reserved->next; 523 524 drm_mm_remove_node(&reserved->node); 525 kfree(reserved); 526 527 reserved = next; 528 } 529 if (drm_mm_node_allocated(&hole)) 530 drm_mm_remove_node(&hole); 531 mutex_unlock(&ggtt->vm.mutex); 532 intel_runtime_pm_put(&i915->runtime_pm, wakeref); 533 534 return err; 535} 536 537int i915_gem_evict_mock_selftests(void) 538{ 539 static const struct i915_subtest tests[] = { 540 SUBTEST(igt_evict_something), 541 SUBTEST(igt_evict_for_vma), 542 SUBTEST(igt_evict_for_cache_color), 543 SUBTEST(igt_evict_vm), 544 SUBTEST(igt_overcommit), 545 }; 546 struct drm_i915_private *i915; 547 intel_wakeref_t wakeref; 548 int err = 0; 549 550 i915 = mock_gem_device(); 551 if (!i915) 552 return -ENOMEM; 553 554 with_intel_runtime_pm(&i915->runtime_pm, wakeref) 555 err = i915_subtests(tests, to_gt(i915)); 556 557 mock_destroy_device(i915); 558 return err; 559} 560 561int i915_gem_evict_live_selftests(struct drm_i915_private *i915) 562{ 563 static const struct i915_subtest tests[] = { 564 SUBTEST(igt_evict_contexts), 565 }; 566 567 if (intel_gt_is_wedged(to_gt(i915))) 568 return 0; 569 570 return intel_gt_live_subtests(tests, to_gt(i915)); 571} 572