Lines Matching refs:guc
108 * guc->submission_state.lock
122 * guc->submission_state.lock -> ce->guc_state.lock
161 #define NUMBER_MULTI_LRC_GUC_ID(guc) \
162 ((guc)->submission_state.num_guc_ids / 16)
451 GEM_BUG_ON(!ce->parallel.guc.parent_page);
453 return ce->parallel.guc.parent_page * PAGE_SIZE;
471 * parallel.guc.parent_page is the offset into ce->state while
504 CIRC_SPACE(ce->parallel.guc.wqi_tail, ce->parallel.guc.wqi_head, WQ_SIZE)
506 ce->parallel.guc.wqi_head = READ_ONCE(*ce->parallel.guc.wq_head);
513 return &__get_parent_scratch(ce)->wq[ce->parallel.guc.wqi_tail / sizeof(u32)];
516 static inline struct intel_context *__get_context(struct intel_guc *guc, u32 id)
518 struct intel_context *ce = xa_load(&guc->context_lookup, id);
525 static struct guc_lrc_desc_v69 *__get_lrc_desc_v69(struct intel_guc *guc, u32 index)
527 struct guc_lrc_desc_v69 *base = guc->lrc_desc_pool_vaddr_v69;
537 static int guc_lrc_desc_pool_create_v69(struct intel_guc *guc)
544 ret = intel_guc_allocate_and_map_vma(guc, size, &guc->lrc_desc_pool_v69,
545 (void **)&guc->lrc_desc_pool_vaddr_v69);
552 static void guc_lrc_desc_pool_destroy_v69(struct intel_guc *guc)
554 if (!guc->lrc_desc_pool_vaddr_v69)
557 guc->lrc_desc_pool_vaddr_v69 = NULL;
558 i915_vma_unpin_and_release(&guc->lrc_desc_pool_v69, I915_VMA_RELEASE_MAP);
561 static inline bool guc_submission_initialized(struct intel_guc *guc)
563 return guc->submission_initialized;
566 static inline void _reset_lrc_desc_v69(struct intel_guc *guc, u32 id)
568 struct guc_lrc_desc_v69 *desc = __get_lrc_desc_v69(guc, id);
574 static inline bool ctx_id_mapped(struct intel_guc *guc, u32 id)
576 return __get_context(guc, id);
579 static inline void set_ctx_id_mapping(struct intel_guc *guc, u32 id,
588 xa_lock_irqsave(&guc->context_lookup, flags);
589 __xa_store(&guc->context_lookup, id, ce, GFP_ATOMIC);
590 xa_unlock_irqrestore(&guc->context_lookup, flags);
593 static inline void clr_ctx_id_mapping(struct intel_guc *guc, u32 id)
597 if (unlikely(!guc_submission_initialized(guc)))
600 _reset_lrc_desc_v69(guc, id);
606 xa_lock_irqsave(&guc->context_lookup, flags);
607 __xa_erase(&guc->context_lookup, id);
608 xa_unlock_irqrestore(&guc->context_lookup, flags);
611 static void decr_outstanding_submission_g2h(struct intel_guc *guc)
613 if (atomic_dec_and_test(&guc->outstanding_submission_g2h))
614 wake_up_all(&guc->ct.wq);
617 static int guc_submission_send_busy_loop(struct intel_guc *guc,
633 atomic_inc(&guc->outstanding_submission_g2h);
635 ret = intel_guc_send_busy_loop(guc, action, len, g2h_len_dw, loop);
637 atomic_dec(&guc->outstanding_submission_g2h);
642 int intel_guc_wait_for_pending_msg(struct intel_guc *guc,
661 prepare_to_wait(&guc->ct.wq, &wait, state);
678 finish_wait(&guc->ct.wq, &wait);
683 int intel_guc_wait_for_idle(struct intel_guc *guc, long timeout)
685 if (!intel_uc_uses_guc_submission(&guc_to_gt(guc)->uc))
688 return intel_guc_wait_for_pending_msg(guc,
689 &guc->outstanding_submission_g2h,
696 static int __guc_add_request(struct intel_guc *guc, struct i915_request *rq)
750 err = intel_guc_send_nb(guc, action, len, g2h_len_dw);
753 atomic_inc(&guc->outstanding_submission_g2h);
765 err = intel_guc_send_nb(guc, action, len - 1, 0);
779 static int guc_add_request(struct intel_guc *guc, struct i915_request *rq)
781 int ret = __guc_add_request(guc, rq);
784 guc->stalled_request = rq;
785 guc->submission_stall_reason = STALL_ADD_REQUEST;
816 return (WQ_SIZE - ce->parallel.guc.wqi_tail);
828 ce->parallel.guc.wqi_tail = (ce->parallel.guc.wqi_tail + wqi_size) &
830 WRITE_ONCE(*ce->parallel.guc.wq_tail, ce->parallel.guc.wqi_tail);
845 ce->parallel.guc.wqi_tail = 0;
893 static int guc_wq_item_append(struct intel_guc *guc,
904 guc->stalled_request = rq;
905 guc->submission_stall_reason = STALL_MOVE_LRC_TAIL;
927 static int guc_dequeue_one_context(struct intel_guc *guc)
929 struct i915_sched_engine * const sched_engine = guc->sched_engine;
937 if (guc->stalled_request) {
939 last = guc->stalled_request;
941 switch (guc->submission_stall_reason) {
949 MISSING_CASE(guc->submission_stall_reason);
992 if (unlikely(!ctx_id_mapped(guc, ce->guc_id.id) &&
998 guc->stalled_request = last;
999 guc->submission_stall_reason =
1010 ret = guc_wq_item_append(guc, last);
1022 ret = guc_add_request(guc, last);
1033 guc->stalled_request = NULL;
1034 guc->submission_stall_reason = STALL_NONE;
1072 static void release_guc_id(struct intel_guc *guc, struct intel_context *ce);
1077 static void scrub_guc_desc_for_outstanding_g2h(struct intel_guc *guc)
1083 xa_lock_irqsave(&guc->context_lookup, flags);
1084 xa_for_each(&guc->context_lookup, index, ce) {
1092 xa_unlock(&guc->context_lookup);
1119 decr_outstanding_submission_g2h(guc);
1123 intel_gt_pm_put_async_untracked(guc_to_gt(guc));
1124 release_guc_id(guc, ce);
1139 decr_outstanding_submission_g2h(guc);
1150 xa_lock(&guc->context_lookup);
1152 xa_unlock_irqrestore(&guc->context_lookup, flags);
1182 __extend_last_switch(struct intel_guc *guc, u64 *prev_start, u32 new_start)
1184 u32 gt_stamp_hi = upper_32_bits(guc->timestamp.gt_stamp);
1185 u32 gt_stamp_last = lower_32_bits(guc->timestamp.gt_stamp);
1248 struct intel_engine_guc_stats *stats = &engine->stats.guc;
1249 struct intel_guc *guc = gt_to_guc(engine->gt);
1252 lockdep_assert_held(&guc->timestamp.lock);
1258 __extend_last_switch(guc, &stats->start_gt_clk, last_switch);
1284 static void guc_update_pm_timestamp(struct intel_guc *guc, ktime_t *now)
1286 struct intel_gt *gt = guc_to_gt(guc);
1290 lockdep_assert_held(&guc->timestamp.lock);
1292 gt_stamp_hi = upper_32_bits(guc->timestamp.gt_stamp);
1294 MISC_STATUS1) >> guc->timestamp.shift;
1298 if (gt_stamp_lo < lower_32_bits(guc->timestamp.gt_stamp))
1301 guc->timestamp.gt_stamp = ((u64)gt_stamp_hi << 32) | gt_stamp_lo;
1311 struct intel_engine_guc_stats stats_saved, *stats = &engine->stats.guc;
1314 struct intel_guc *guc = gt_to_guc(gt);
1321 spin_lock_irqsave(&guc->timestamp.lock, flags);
1345 gt_stamp_saved = guc->timestamp.gt_stamp;
1351 guc_update_pm_timestamp(guc, now);
1355 guc->timestamp.gt_stamp = gt_stamp_saved;
1361 u64 clk = guc->timestamp.gt_stamp - stats->start_gt_clk;
1366 spin_unlock_irqrestore(&guc->timestamp.lock, flags);
1371 static void guc_enable_busyness_worker(struct intel_guc *guc)
1373 mod_delayed_work(system_highpri_wq, &guc->timestamp.work, guc->timestamp.ping_delay);
1376 static void guc_cancel_busyness_worker(struct intel_guc *guc)
1415 if (mutex_is_locked(&guc_to_gt(guc)->reset.mutex) ||
1416 test_bit(I915_RESET_BACKOFF, &guc_to_gt(guc)->reset.flags))
1417 cancel_delayed_work(&guc->timestamp.work);
1419 cancel_delayed_work_sync(&guc->timestamp.work);
1422 static void __reset_guc_busyness_stats(struct intel_guc *guc)
1424 struct intel_gt *gt = guc_to_gt(guc);
1430 spin_lock_irqsave(&guc->timestamp.lock, flags);
1432 guc_update_pm_timestamp(guc, &unused);
1435 engine->stats.guc.prev_total = 0;
1438 spin_unlock_irqrestore(&guc->timestamp.lock, flags);
1441 static void __update_guc_busyness_stats(struct intel_guc *guc)
1443 struct intel_gt *gt = guc_to_gt(guc);
1449 guc->timestamp.last_stat_jiffies = jiffies;
1451 spin_lock_irqsave(&guc->timestamp.lock, flags);
1453 guc_update_pm_timestamp(guc, &unused);
1457 spin_unlock_irqrestore(&guc->timestamp.lock, flags);
1462 struct intel_guc *guc = ce_to_guc(ce);
1465 spin_lock_irqsave(&guc->timestamp.lock, flags);
1467 spin_unlock_irqrestore(&guc->timestamp.lock, flags);
1481 struct intel_guc *guc = container_of(wrk, typeof(*guc),
1483 struct intel_uc *uc = container_of(guc, typeof(*uc), guc);
1484 struct intel_gt *gt = guc_to_gt(guc);
1522 * corrupt the engine/guc stats. NB: can't actually block waiting
1530 __update_guc_busyness_stats(guc);
1533 xa_for_each(&guc->context_lookup, index, ce)
1538 guc_enable_busyness_worker(guc);
1544 static int guc_action_enable_usage_stats(struct intel_guc *guc)
1546 u32 offset = intel_guc_engine_usage_offset(guc);
1553 return intel_guc_send(guc, action, ARRAY_SIZE(action));
1556 static int guc_init_engine_stats(struct intel_guc *guc)
1558 struct intel_gt *gt = guc_to_gt(guc);
1563 ret = guc_action_enable_usage_stats(guc);
1566 guc_err(guc, "Failed to enable usage stats: %pe\n", ERR_PTR(ret));
1568 guc_enable_busyness_worker(guc);
1573 static void guc_fini_engine_stats(struct intel_guc *guc)
1575 guc_cancel_busyness_worker(guc);
1580 struct intel_guc *guc = gt_to_guc(gt);
1582 if (!guc_submission_initialized(guc))
1590 guc_cancel_busyness_worker(guc);
1597 if (guc->timestamp.last_stat_jiffies &&
1598 !time_after(jiffies, guc->timestamp.last_stat_jiffies +
1599 (guc->timestamp.ping_delay / 2)))
1602 __update_guc_busyness_stats(guc);
1607 struct intel_guc *guc = gt_to_guc(gt);
1611 if (!guc_submission_initialized(guc))
1614 spin_lock_irqsave(&guc->timestamp.lock, flags);
1615 guc_update_pm_timestamp(guc, &unused);
1616 spin_unlock_irqrestore(&guc->timestamp.lock, flags);
1617 guc_enable_busyness_worker(guc);
1621 submission_disabled(struct intel_guc *guc)
1623 struct i915_sched_engine * const sched_engine = guc->sched_engine;
1627 intel_gt_is_wedged(guc_to_gt(guc)));
1630 static void disable_submission(struct intel_guc *guc)
1632 struct i915_sched_engine * const sched_engine = guc->sched_engine;
1635 GEM_BUG_ON(!guc->ct.enabled);
1641 static void enable_submission(struct intel_guc *guc)
1643 struct i915_sched_engine * const sched_engine = guc->sched_engine;
1646 spin_lock_irqsave(&guc->sched_engine->lock, flags);
1651 GEM_BUG_ON(!guc->ct.enabled);
1656 spin_unlock_irqrestore(&guc->sched_engine->lock, flags);
1659 static void guc_flush_submissions(struct intel_guc *guc)
1661 struct i915_sched_engine * const sched_engine = guc->sched_engine;
1668 void intel_guc_submission_flush_work(struct intel_guc *guc)
1670 flush_work(&guc->submission_state.destroyed_worker);
1673 static void guc_flush_destroyed_contexts(struct intel_guc *guc);
1675 void intel_guc_submission_reset_prepare(struct intel_guc *guc)
1677 if (unlikely(!guc_submission_initialized(guc))) {
1682 intel_gt_park_heartbeats(guc_to_gt(guc));
1683 disable_submission(guc);
1684 guc->interrupts.disable(guc);
1685 __reset_guc_busyness_stats(guc);
1688 spin_lock_irq(guc_to_gt(guc)->irq_lock);
1689 spin_unlock_irq(guc_to_gt(guc)->irq_lock);
1691 guc_flush_submissions(guc);
1692 guc_flush_destroyed_contexts(guc);
1693 flush_work(&guc->ct.requests.worker);
1695 scrub_guc_desc_for_outstanding_g2h(guc);
1859 void wake_up_all_tlb_invalidate(struct intel_guc *guc)
1864 if (!intel_guc_tlb_invalidation_is_available(guc))
1867 xa_lock_irq(&guc->tlb_lookup);
1868 xa_for_each(&guc->tlb_lookup, i, wait)
1870 xa_unlock_irq(&guc->tlb_lookup);
1873 void intel_guc_submission_reset(struct intel_guc *guc, intel_engine_mask_t stalled)
1879 if (unlikely(!guc_submission_initialized(guc))) {
1884 xa_lock_irqsave(&guc->context_lookup, flags);
1885 xa_for_each(&guc->context_lookup, index, ce) {
1889 xa_unlock(&guc->context_lookup);
1897 xa_lock(&guc->context_lookup);
1899 xa_unlock_irqrestore(&guc->context_lookup, flags);
1902 xa_destroy(&guc->context_lookup);
1971 void intel_guc_submission_cancel_requests(struct intel_guc *guc)
1977 xa_lock_irqsave(&guc->context_lookup, flags);
1978 xa_for_each(&guc->context_lookup, index, ce) {
1982 xa_unlock(&guc->context_lookup);
1990 xa_lock(&guc->context_lookup);
1992 xa_unlock_irqrestore(&guc->context_lookup, flags);
1994 guc_cancel_sched_engine_requests(guc->sched_engine);
1997 xa_destroy(&guc->context_lookup);
2003 wake_up_all_tlb_invalidate(guc);
2006 void intel_guc_submission_reset_finish(struct intel_guc *guc)
2009 if (unlikely(!guc_submission_initialized(guc) ||
2010 !intel_guc_is_fw_running(guc) ||
2011 intel_gt_is_wedged(guc_to_gt(guc)))) {
2021 GEM_WARN_ON(atomic_read(&guc->outstanding_submission_g2h));
2022 atomic_set(&guc->outstanding_submission_g2h, 0);
2024 intel_guc_global_policies_update(guc);
2025 enable_submission(guc);
2026 intel_gt_unpark_heartbeats(guc_to_gt(guc));
2032 wake_up_all_tlb_invalidate(guc);
2038 bool intel_guc_tlb_invalidation_is_available(struct intel_guc *guc)
2040 return HAS_GUC_TLB_INVALIDATION(guc_to_gt(guc)->i915) &&
2041 intel_guc_is_ready(guc);
2044 static int init_tlb_lookup(struct intel_guc *guc)
2049 if (!HAS_GUC_TLB_INVALIDATION(guc_to_gt(guc)->i915))
2052 xa_init_flags(&guc->tlb_lookup, XA_FLAGS_ALLOC);
2061 err = xa_alloc_cyclic_irq(&guc->tlb_lookup, &guc->serial_slot, wait,
2062 xa_limit_32b, &guc->next_seqno, GFP_KERNEL);
2071 static void fini_tlb_lookup(struct intel_guc *guc)
2075 if (!HAS_GUC_TLB_INVALIDATION(guc_to_gt(guc)->i915))
2078 wait = xa_load(&guc->tlb_lookup, guc->serial_slot);
2080 guc_err(guc, "Unexpected busy item in tlb_lookup on fini\n");
2083 xa_destroy(&guc->tlb_lookup);
2090 int intel_guc_submission_init(struct intel_guc *guc)
2092 struct intel_gt *gt = guc_to_gt(guc);
2095 if (guc->submission_initialized)
2098 if (GUC_SUBMIT_VER(guc) < MAKE_GUC_VER(1, 0, 0)) {
2099 ret = guc_lrc_desc_pool_create_v69(guc);
2104 ret = init_tlb_lookup(guc);
2108 guc->submission_state.guc_ids_bitmap =
2109 bitmap_zalloc(NUMBER_MULTI_LRC_GUC_ID(guc), GFP_KERNEL);
2110 if (!guc->submission_state.guc_ids_bitmap) {
2115 guc->timestamp.ping_delay = (POLL_TIME_CLKS / gt->clock_frequency + 1) * HZ;
2116 guc->timestamp.shift = gpm_timestamp_shift(gt);
2117 guc->submission_initialized = true;
2122 fini_tlb_lookup(guc);
2124 guc_lrc_desc_pool_destroy_v69(guc);
2128 void intel_guc_submission_fini(struct intel_guc *guc)
2130 if (!guc->submission_initialized)
2133 guc_fini_engine_stats(guc);
2134 guc_flush_destroyed_contexts(guc);
2135 guc_lrc_desc_pool_destroy_v69(guc);
2136 i915_sched_engine_put(guc->sched_engine);
2137 bitmap_free(guc->submission_state.guc_ids_bitmap);
2138 fini_tlb_lookup(guc);
2139 guc->submission_initialized = false;
2153 static int guc_bypass_tasklet_submit(struct intel_guc *guc,
2164 ret = guc_wq_item_append(guc, rq);
2166 ret = guc_add_request(guc, rq);
2170 ret = guc_add_request(guc, rq);
2174 disable_submission(guc);
2179 static bool need_tasklet(struct intel_guc *guc, struct i915_request *rq)
2184 return submission_disabled(guc) || guc->stalled_request ||
2186 !ctx_id_mapped(guc, ce->guc_id.id);
2192 struct intel_guc *guc = gt_to_guc(rq->engine->gt);
2198 if (need_tasklet(guc, rq))
2200 else if (guc_bypass_tasklet_submit(guc, rq) == -EBUSY)
2206 static int new_guc_id(struct intel_guc *guc, struct intel_context *ce)
2213 ret = bitmap_find_free_region(guc->submission_state.guc_ids_bitmap,
2214 NUMBER_MULTI_LRC_GUC_ID(guc),
2218 ret = ida_alloc_range(&guc->submission_state.guc_ids,
2219 NUMBER_MULTI_LRC_GUC_ID(guc),
2220 guc->submission_state.num_guc_ids - 1,
2226 ++guc->submission_state.guc_ids_in_use;
2232 static void __release_guc_id(struct intel_guc *guc, struct intel_context *ce)
2238 bitmap_release_region(guc->submission_state.guc_ids_bitmap,
2243 --guc->submission_state.guc_ids_in_use;
2244 ida_free(&guc->submission_state.guc_ids,
2247 clr_ctx_id_mapping(guc, ce->guc_id.id);
2254 static void release_guc_id(struct intel_guc *guc, struct intel_context *ce)
2258 spin_lock_irqsave(&guc->submission_state.lock, flags);
2259 __release_guc_id(guc, ce);
2260 spin_unlock_irqrestore(&guc->submission_state.lock, flags);
2263 static int steal_guc_id(struct intel_guc *guc, struct intel_context *ce)
2267 lockdep_assert_held(&guc->submission_state.lock);
2271 if (!list_empty(&guc->submission_state.guc_id_list)) {
2272 cn = list_first_entry(&guc->submission_state.guc_id_list,
2291 guc->number_guc_id_stolen++;
2300 static int assign_guc_id(struct intel_guc *guc, struct intel_context *ce)
2304 lockdep_assert_held(&guc->submission_state.lock);
2307 ret = new_guc_id(guc, ce);
2312 ret = steal_guc_id(guc, ce);
2329 static int pin_guc_id(struct intel_guc *guc, struct intel_context *ce)
2337 spin_lock_irqsave(&guc->submission_state.lock, flags);
2342 ret = assign_guc_id(guc, ce);
2352 spin_unlock_irqrestore(&guc->submission_state.lock, flags);
2372 intel_gt_retire_requests(guc_to_gt(guc));
2379 static void unpin_guc_id(struct intel_guc *guc, struct intel_context *ce)
2390 spin_lock_irqsave(&guc->submission_state.lock, flags);
2394 &guc->submission_state.guc_id_list);
2395 spin_unlock_irqrestore(&guc->submission_state.lock, flags);
2398 static int __guc_action_register_multi_lrc_v69(struct intel_guc *guc,
2419 return guc_submission_send_busy_loop(guc, action, len, 0, loop);
2422 static int __guc_action_register_multi_lrc_v70(struct intel_guc *guc,
2462 return guc_submission_send_busy_loop(guc, action, len, 0, loop);
2465 static int __guc_action_register_context_v69(struct intel_guc *guc,
2476 return guc_submission_send_busy_loop(guc, action, ARRAY_SIZE(action),
2480 static int __guc_action_register_context_v70(struct intel_guc *guc,
2499 return guc_submission_send_busy_loop(guc, action, ARRAY_SIZE(action),
2508 register_context_v69(struct intel_guc *guc, struct intel_context *ce, bool loop)
2510 u32 offset = intel_guc_ggtt_offset(guc, guc->lrc_desc_pool_v69) +
2516 return __guc_action_register_multi_lrc_v69(guc, ce, ce->guc_id.id,
2519 return __guc_action_register_context_v69(guc, ce->guc_id.id,
2524 register_context_v70(struct intel_guc *guc, struct intel_context *ce, bool loop)
2531 return __guc_action_register_multi_lrc_v70(guc, ce, &info, loop);
2533 return __guc_action_register_context_v70(guc, &info, loop);
2538 struct intel_guc *guc = ce_to_guc(ce);
2544 if (GUC_SUBMIT_VER(guc) >= MAKE_GUC_VER(1, 0, 0))
2545 ret = register_context_v70(guc, ce, loop);
2547 ret = register_context_v69(guc, ce, loop);
2556 if (GUC_SUBMIT_VER(guc) >= MAKE_GUC_VER(1, 0, 0))
2563 static int __guc_action_deregister_context(struct intel_guc *guc,
2571 return guc_submission_send_busy_loop(guc, action, ARRAY_SIZE(action),
2578 struct intel_guc *guc = ce_to_guc(ce);
2583 return __guc_action_deregister_context(guc, guc_id);
2646 static int __guc_context_set_context_policies(struct intel_guc *guc,
2650 return guc_submission_send_busy_loop(guc, (u32 *)&policy->h2g,
2658 struct intel_guc *guc = gt_to_guc(engine->gt);
2687 ret = __guc_context_set_context_policies(guc, &policy, loop);
2739 struct intel_guc *guc = gt_to_guc(engine->gt);
2750 GEM_BUG_ON(i915_gem_object_is_lmem(guc->ct.vma->obj) !=
2753 desc = __get_lrc_desc_v69(guc, ctx_id);
2769 ce->parallel.guc.wqi_tail = 0;
2770 ce->parallel.guc.wqi_head = 0;
2785 ce->parallel.guc.wq_head = &pdesc->head;
2786 ce->parallel.guc.wq_tail = &pdesc->tail;
2787 ce->parallel.guc.wq_status = &pdesc->wq_status;
2790 desc = __get_lrc_desc_v69(guc, child->guc_id.id);
2808 struct intel_guc *guc = gt_to_guc(engine->gt);
2817 GEM_BUG_ON(i915_gem_object_is_lmem(guc->ct.vma->obj) !=
2842 ce->parallel.guc.wqi_tail = 0;
2843 ce->parallel.guc.wqi_head = 0;
2859 ce->parallel.guc.wq_head = &wq_desc->head;
2860 ce->parallel.guc.wq_tail = &wq_desc->tail;
2861 ce->parallel.guc.wq_status = &wq_desc->wq_status;
2871 struct intel_guc *guc = gt_to_guc(engine->gt);
2879 context_registered = ctx_id_mapped(guc, ctx_id);
2881 clr_ctx_id_mapping(guc, ctx_id);
2882 set_ctx_id_mapping(guc, ctx_id, ce);
2901 disabled = submission_disabled(guc);
2908 clr_ctx_id_mapping(guc, ctx_id);
2924 clr_ctx_id_mapping(guc, ctx_id);
2926 clr_ctx_id_mapping(guc, ctx_id);
2977 struct intel_guc *guc = ce_to_guc(ce);
2980 unpin_guc_id(guc, ce);
2992 static void __guc_context_sched_enable(struct intel_guc *guc,
3003 guc_submission_send_busy_loop(guc, action, ARRAY_SIZE(action),
3007 static void __guc_context_sched_disable(struct intel_guc *guc,
3022 guc_submission_send_busy_loop(guc, action, ARRAY_SIZE(action),
3064 struct intel_guc *guc = ce_to_guc(ce);
3078 if (unlikely(!enabled || submission_disabled(guc))) {
3096 __guc_context_sched_disable(guc, ce, guc_id);
3120 struct intel_guc *guc = ce_to_guc(ce);
3131 if (unlikely(submission_disabled(guc) ||
3147 __guc_context_sched_enable(guc, ce);
3174 static void __guc_context_set_preemption_timeout(struct intel_guc *guc,
3178 if (GUC_SUBMIT_VER(guc) >= MAKE_GUC_VER(1, 0, 0)) {
3183 __guc_context_set_context_policies(guc, &policy, true);
3191 intel_guc_send_busy_loop(guc, action, ARRAY_SIZE(action), 0, true);
3199 struct intel_guc *guc = ce_to_guc(ce);
3207 guc_flush_submissions(guc);
3212 if (submission_disabled(guc) ||
3236 __guc_context_set_preemption_timeout(guc, guc_id,
3238 __guc_context_sched_disable(guc, ce, guc_id);
3243 __guc_context_set_preemption_timeout(guc,
3250 static void do_sched_disable(struct intel_guc *guc, struct intel_context *ce,
3264 __guc_context_sched_disable(guc, ce, guc_id);
3267 static bool bypass_sched_disable(struct intel_guc *guc,
3273 if (submission_disabled(guc) || context_guc_id_invalid(ce) ||
3274 !ctx_id_mapped(guc, ce->guc_id.id)) {
3286 struct intel_guc *guc = ce_to_guc(ce);
3291 if (bypass_sched_disable(guc, ce)) {
3295 do_sched_disable(guc, ce, flags);
3299 static bool guc_id_pressure(struct intel_guc *guc, struct intel_context *ce)
3311 return guc->submission_state.guc_ids_in_use >
3312 guc->submission_state.sched_disable_gucid_threshold;
3317 struct intel_guc *guc = ce_to_guc(ce);
3318 u64 delay = guc->submission_state.sched_disable_delay_ms;
3323 if (bypass_sched_disable(guc, ce)) {
3326 } else if (!intel_context_is_closed(ce) && !guc_id_pressure(guc, ce) &&
3333 do_sched_disable(guc, ce, flags);
3352 struct intel_guc *guc = ce_to_guc(ce);
3353 struct intel_gt *gt = guc_to_gt(guc);
3359 GEM_BUG_ON(!ctx_id_mapped(guc, ce->guc_id.id));
3360 GEM_BUG_ON(ce != __get_context(guc, ce->guc_id.id));
3365 disabled = submission_disabled(guc);
3378 release_guc_id(guc, ce);
3427 static void guc_flush_destroyed_contexts(struct intel_guc *guc)
3432 GEM_BUG_ON(!submission_disabled(guc) &&
3433 guc_submission_initialized(guc));
3435 while (!list_empty(&guc->submission_state.destroyed_contexts)) {
3436 spin_lock_irqsave(&guc->submission_state.lock, flags);
3437 ce = list_first_entry_or_null(&guc->submission_state.destroyed_contexts,
3442 spin_unlock_irqrestore(&guc->submission_state.lock, flags);
3447 release_guc_id(guc, ce);
3452 static void deregister_destroyed_contexts(struct intel_guc *guc)
3457 while (!list_empty(&guc->submission_state.destroyed_contexts)) {
3458 spin_lock_irqsave(&guc->submission_state.lock, flags);
3459 ce = list_first_entry_or_null(&guc->submission_state.destroyed_contexts,
3464 spin_unlock_irqrestore(&guc->submission_state.lock, flags);
3477 spin_lock_irqsave(&guc->submission_state.lock, flags);
3479 &guc->submission_state.destroyed_contexts);
3480 spin_unlock_irqrestore(&guc->submission_state.lock, flags);
3490 struct intel_guc *guc = container_of(w, struct intel_guc,
3492 struct intel_gt *gt = guc_to_gt(guc);
3503 if (!intel_guc_is_ready(guc))
3507 deregister_destroyed_contexts(guc);
3513 struct intel_guc *guc = ce_to_guc(ce);
3522 spin_lock_irqsave(&guc->submission_state.lock, flags);
3523 destroy = submission_disabled(guc) || context_guc_id_invalid(ce) ||
3524 !ctx_id_mapped(guc, ce->guc_id.id);
3529 &guc->submission_state.destroyed_contexts);
3531 __release_guc_id(guc, ce);
3533 spin_unlock_irqrestore(&guc->submission_state.lock, flags);
3544 queue_work(system_unbound_wq, &guc->submission_state.destroyed_worker);
3552 static void __guc_context_set_prio(struct intel_guc *guc,
3555 if (GUC_SUBMIT_VER(guc) >= MAKE_GUC_VER(1, 0, 0)) {
3560 __guc_context_set_context_policies(guc, &policy, true);
3568 guc_submission_send_busy_loop(guc, action, ARRAY_SIZE(action), 0, true);
3572 static void guc_context_set_prio(struct intel_guc *guc,
3580 if (ce->guc_state.prio == prio || submission_disabled(guc) ||
3587 __guc_context_set_prio(guc, ce);
3630 struct intel_guc *guc = &ce->engine->gt->uc.guc;
3640 guc_context_set_prio(guc, ce, i);
3810 struct intel_guc *guc = ce_to_guc(ce);
3859 guc_warn(guc, "timed out waiting on context sched close before realloc\n");
3880 ret = pin_guc_id(guc, ce); /* returns 1 if new guc_id assigned */
3887 disable_submission(guc);
3891 unpin_guc_id(guc, ce);
3945 struct intel_guc *guc = ce_to_guc(ce);
3950 unpin_guc_id(guc, ce);
4015 struct intel_guc *guc = ce_to_guc(ce);
4021 ret = pin_guc_id(guc, ce);
4041 struct intel_guc *guc = ce_to_guc(ce);
4048 unpin_guc_id(guc, ce);
4377 static inline int guc_kernel_context_pin(struct intel_guc *guc,
4390 ret = pin_guc_id(guc, ce);
4401 unpin_guc_id(guc, ce);
4406 static inline int guc_init_submission(struct intel_guc *guc)
4408 struct intel_gt *gt = guc_to_gt(guc);
4413 xa_destroy(&guc->context_lookup);
4419 guc->stalled_request = NULL;
4420 guc->submission_stall_reason = STALL_NONE;
4434 int ret = guc_kernel_context_pin(guc, ce);
4548 struct intel_guc *guc = sched_engine->private_data;
4550 guc->sched_engine = NULL;
4558 struct intel_guc *guc = gt_to_guc(engine->gt);
4566 if (!guc->sched_engine) {
4567 guc->sched_engine = i915_sched_engine_create(ENGINE_VIRTUAL);
4568 if (!guc->sched_engine)
4571 guc->sched_engine->schedule = i915_schedule;
4572 guc->sched_engine->disabled = guc_sched_engine_disabled;
4573 guc->sched_engine->private_data = guc;
4574 guc->sched_engine->destroy = guc_sched_engine_destroy;
4575 guc->sched_engine->bump_inflight_request_prio =
4577 guc->sched_engine->retire_inflight_request_prio =
4579 tasklet_setup(&guc->sched_engine->tasklet,
4583 engine->sched_engine = i915_sched_engine_get(guc->sched_engine);
4641 static int __guc_action_set_scheduling_policies(struct intel_guc *guc,
4646 ret = intel_guc_send(guc, (u32 *)&policy->h2g,
4649 guc_probe_error(guc, "Failed to configure global scheduling policies: %pe!\n",
4655 guc_warn(guc, "global scheduler policy processed %d of %d KLVs!",
4664 static int guc_init_global_schedule_policy(struct intel_guc *guc)
4667 struct intel_gt *gt = guc_to_gt(guc);
4671 if (GUC_SUBMIT_VER(guc) < MAKE_GUC_VER(1, 1, 0))
4686 ret = __guc_action_set_scheduling_policies(guc, &policy);
4692 static void guc_route_semaphores(struct intel_guc *guc, bool to_guc)
4694 struct intel_gt *gt = guc_to_gt(guc);
4708 int intel_guc_submission_enable(struct intel_guc *guc)
4713 guc_route_semaphores(guc, true);
4715 ret = guc_init_submission(guc);
4719 ret = guc_init_engine_stats(guc);
4723 ret = guc_init_global_schedule_policy(guc);
4730 guc_fini_engine_stats(guc);
4732 guc_route_semaphores(guc, false);
4737 void intel_guc_submission_disable(struct intel_guc *guc)
4739 guc_cancel_busyness_worker(guc);
4742 guc_route_semaphores(guc, false);
4745 static bool __guc_submission_supported(struct intel_guc *guc)
4748 return intel_guc_is_supported(guc) &&
4749 GRAPHICS_VER(guc_to_i915(guc)) >= 11;
4752 static bool __guc_submission_selected(struct intel_guc *guc)
4754 struct drm_i915_private *i915 = guc_to_i915(guc);
4756 if (!intel_guc_submission_is_supported(guc))
4762 int intel_guc_sched_disable_gucid_threshold_max(struct intel_guc *guc)
4764 return guc->submission_state.num_guc_ids - NUMBER_MULTI_LRC_GUC_ID(guc);
4780 (((intel_guc_sched_disable_gucid_threshold_max(guc)) * 3) / 4)
4782 void intel_guc_submission_init_early(struct intel_guc *guc)
4784 xa_init_flags(&guc->context_lookup, XA_FLAGS_LOCK_IRQ);
4786 spin_lock_init(&guc->submission_state.lock);
4787 INIT_LIST_HEAD(&guc->submission_state.guc_id_list);
4788 ida_init(&guc->submission_state.guc_ids);
4789 INIT_LIST_HEAD(&guc->submission_state.destroyed_contexts);
4790 INIT_WORK(&guc->submission_state.destroyed_worker,
4792 INIT_WORK(&guc->submission_state.reset_fail_worker,
4795 spin_lock_init(&guc->timestamp.lock);
4796 INIT_DELAYED_WORK(&guc->timestamp.work, guc_timestamp_ping);
4798 guc->submission_state.sched_disable_delay_ms = SCHED_DISABLE_DELAY_MS;
4799 guc->submission_state.num_guc_ids = GUC_MAX_CONTEXT_ID;
4800 guc->submission_state.sched_disable_gucid_threshold =
4801 NUM_SCHED_DISABLE_GUCIDS_DEFAULT_THRESHOLD(guc);
4802 guc->submission_supported = __guc_submission_supported(guc);
4803 guc->submission_selected = __guc_submission_selected(guc);
4807 g2h_context_lookup(struct intel_guc *guc, u32 ctx_id)
4812 guc_err(guc, "Invalid ctx_id %u\n", ctx_id);
4816 ce = __get_context(guc, ctx_id);
4818 guc_err(guc, "Context is NULL, ctx_id %u\n", ctx_id);
4823 guc_err(guc, "Context is child, ctx_id %u\n", ctx_id);
4830 static void wait_wake_outstanding_tlb_g2h(struct intel_guc *guc, u32 seqno)
4835 xa_lock_irqsave(&guc->tlb_lookup, flags);
4836 wait = xa_load(&guc->tlb_lookup, seqno);
4841 guc_dbg(guc,
4844 xa_unlock_irqrestore(&guc->tlb_lookup, flags);
4847 int intel_guc_tlb_invalidation_done(struct intel_guc *guc,
4853 wait_wake_outstanding_tlb_g2h(guc, payload[0]);
4889 static int guc_send_invalidate_tlb(struct intel_guc *guc,
4893 struct intel_gt *gt = guc_to_gt(guc);
4916 if (xa_alloc_cyclic_irq(&guc->tlb_lookup, &seqno, wq,
4917 xa_limit_32b, &guc->next_seqno,
4920 xa_lock_irq(&guc->tlb_lookup);
4921 wq = xa_load(&guc->tlb_lookup, guc->serial_slot);
4924 guc->tlb_lookup.xa_lock);
4932 xa_unlock_irq(&guc->tlb_lookup);
4934 seqno = guc->serial_slot;
4942 err = intel_guc_send_busy_loop(guc, action, size, G2H_LEN_DW_INVALIDATE_TLB, true);
4954 guc_err(guc,
4960 if (seqno != guc->serial_slot)
4961 xa_erase_irq(&guc->tlb_lookup, seqno);
4967 int intel_guc_invalidate_tlb_engines(struct intel_guc *guc)
4969 return guc_send_invalidate_tlb(guc, INTEL_GUC_TLB_INVAL_ENGINES);
4973 int intel_guc_invalidate_tlb_guc(struct intel_guc *guc)
4975 return guc_send_invalidate_tlb(guc, INTEL_GUC_TLB_INVAL_GUC);
4978 int intel_guc_deregister_done_process_msg(struct intel_guc *guc,
4986 guc_err(guc, "Invalid length %u\n", len);
4991 ce = g2h_context_lookup(guc, ctx_id);
5019 intel_gt_pm_put_async_untracked(guc_to_gt(guc));
5020 release_guc_id(guc, ce);
5024 decr_outstanding_submission_g2h(guc);
5029 int intel_guc_sched_done_process_msg(struct intel_guc *guc,
5038 guc_err(guc, "Invalid length %u\n", len);
5043 ce = g2h_context_lookup(guc, ctx_id);
5050 guc_err(guc, "Bad context sched_state 0x%x, ctx_id %u\n",
5101 decr_outstanding_submission_g2h(guc);
5107 static void capture_error_state(struct intel_guc *guc,
5110 struct intel_gt *gt = guc_to_gt(guc);
5132 guc_warn(guc, "No matching physical engine capture for virtual engine context 0x%04X / %s",
5154 static void guc_handle_context_reset(struct intel_guc *guc,
5161 guc_dbg(guc, "%s context reset notification: 0x%04X on %s, exiting = %s, banned = %s\n",
5168 capture_error_state(guc, ce);
5173 int intel_guc_context_reset_process_msg(struct intel_guc *guc,
5181 guc_err(guc, "Invalid length %u", len);
5193 xa_lock_irqsave(&guc->context_lookup, flags);
5194 ce = g2h_context_lookup(guc, ctx_id);
5197 xa_unlock_irqrestore(&guc->context_lookup, flags);
5202 guc_handle_context_reset(guc, ce);
5208 int intel_guc_error_capture_process_msg(struct intel_guc *guc,
5214 guc_dbg(guc, "Invalid length %u", len);
5220 guc_warn(guc, "No space for error capture");
5222 intel_guc_capture_process(guc);
5228 intel_guc_lookup_engine(struct intel_guc *guc, u8 guc_class, u8 instance)
5230 struct intel_gt *gt = guc_to_gt(guc);
5241 struct intel_guc *guc = container_of(w, struct intel_guc,
5243 struct intel_gt *gt = guc_to_gt(guc);
5247 spin_lock_irqsave(&guc->submission_state.lock, flags);
5248 reset_fail_mask = guc->submission_state.reset_fail_mask;
5249 guc->submission_state.reset_fail_mask = 0;
5250 spin_unlock_irqrestore(&guc->submission_state.lock, flags);
5272 int intel_guc_engine_failure_process_msg(struct intel_guc *guc,
5281 guc_err(guc, "Invalid length %u", len);
5289 engine = intel_guc_lookup_engine(guc, guc_class, instance);
5291 guc_err(guc, "Invalid engine %d:%d", guc_class, instance);
5299 guc_err(guc, "Engine reset failed on %d:%d (%s) because 0x%08X",
5302 spin_lock_irqsave(&guc->submission_state.lock, flags);
5303 guc->submission_state.reset_fail_mask |= engine->mask;
5304 spin_unlock_irqrestore(&guc->submission_state.lock, flags);
5310 queue_work(system_unbound_wq, &guc->submission_state.reset_fail_worker);
5317 struct intel_guc *guc = gt_to_guc(engine->gt);
5324 if (unlikely(!guc_submission_initialized(guc)))
5327 xa_lock_irqsave(&guc->context_lookup, flags);
5328 xa_for_each(&guc->context_lookup, index, ce) {
5334 xa_unlock(&guc->context_lookup);
5363 xa_lock(&guc->context_lookup);
5369 xa_lock(&guc->context_lookup);
5372 xa_unlock_irqrestore(&guc->context_lookup, flags);
5379 struct intel_guc *guc = gt_to_guc(engine->gt);
5385 if (unlikely(!guc_submission_initialized(guc)))
5388 xa_lock_irqsave(&guc->context_lookup, flags);
5389 xa_for_each(&guc->context_lookup, index, ce) {
5393 xa_unlock(&guc->context_lookup);
5413 xa_lock(&guc->context_lookup);
5415 xa_unlock_irqrestore(&guc->context_lookup, flags);
5418 void intel_guc_submission_print_info(struct intel_guc *guc,
5421 struct i915_sched_engine *sched_engine = guc->sched_engine;
5429 guc->submission_version.major, guc->submission_version.minor,
5430 guc->submission_version.patch);
5432 atomic_read(&guc->outstanding_submission_g2h));
5485 void intel_guc_submission_print_context_info(struct intel_guc *guc,
5492 xa_lock_irqsave(&guc->context_lookup, flags);
5493 xa_for_each(&guc->context_lookup, index, ce) {
5505 if (ce->parallel.guc.wq_status) {
5507 READ_ONCE(*ce->parallel.guc.wq_head));
5509 READ_ONCE(*ce->parallel.guc.wq_tail));
5511 READ_ONCE(*ce->parallel.guc.wq_status));
5529 xa_unlock_irqrestore(&guc->context_lookup, flags);
5823 struct intel_guc *guc;
5831 guc = gt_to_guc(siblings[0]->gt);
5845 ve->base.sched_engine = i915_sched_engine_get(guc->sched_engine);
5865 guc_dbg(guc, "duplicate %s entry in load balancer\n",
5875 guc_dbg(guc, "invalid mixing of engine class, sibling %d, already %d\n",