Lines Matching defs:guc
46 return &q->gt->uc.guc;
65 return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_REGISTERED;
70 atomic_or(EXEC_QUEUE_STATE_REGISTERED, &q->guc->state);
75 atomic_and(~EXEC_QUEUE_STATE_REGISTERED, &q->guc->state);
80 return atomic_read(&q->guc->state) & ENGINE_STATE_ENABLED;
85 atomic_or(ENGINE_STATE_ENABLED, &q->guc->state);
90 atomic_and(~ENGINE_STATE_ENABLED, &q->guc->state);
95 return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_PENDING_ENABLE;
100 atomic_or(EXEC_QUEUE_STATE_PENDING_ENABLE, &q->guc->state);
105 atomic_and(~EXEC_QUEUE_STATE_PENDING_ENABLE, &q->guc->state);
110 return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_PENDING_DISABLE;
115 atomic_or(EXEC_QUEUE_STATE_PENDING_DISABLE, &q->guc->state);
120 atomic_and(~EXEC_QUEUE_STATE_PENDING_DISABLE, &q->guc->state);
125 return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_DESTROYED;
130 atomic_or(EXEC_QUEUE_STATE_DESTROYED, &q->guc->state);
145 return atomic_read(&q->guc->state) & ENGINE_STATE_SUSPENDED;
150 atomic_or(ENGINE_STATE_SUSPENDED, &q->guc->state);
155 atomic_and(~ENGINE_STATE_SUSPENDED, &q->guc->state);
160 return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_RESET;
165 atomic_or(EXEC_QUEUE_STATE_RESET, &q->guc->state);
170 return atomic_read(&q->guc->state) & ENGINE_STATE_KILLED;
175 atomic_or(ENGINE_STATE_KILLED, &q->guc->state);
184 static int alloc_submit_wq(struct xe_guc *guc)
189 guc->submission_state.submit_wq_pool[i] =
191 if (!guc->submission_state.submit_wq_pool[i])
199 destroy_workqueue(guc->submission_state.submit_wq_pool[--i]);
204 static void free_submit_wq(struct xe_guc *guc)
209 destroy_workqueue(guc->submission_state.submit_wq_pool[i]);
212 static struct workqueue_struct *get_submit_wq(struct xe_guc *guc)
214 int idx = guc->submission_state.submit_wq_idx++ % NUM_SUBMIT_WQ;
216 return guc->submission_state.submit_wq_pool[idx];
219 static int alloc_submit_wq(struct xe_guc *guc)
224 static void free_submit_wq(struct xe_guc *guc)
229 static struct workqueue_struct *get_submit_wq(struct xe_guc *guc)
237 struct xe_guc *guc = arg;
239 xa_destroy(&guc->submission_state.exec_queue_lookup);
240 free_submit_wq(guc);
245 static void primelockdep(struct xe_guc *guc)
252 mutex_lock(&guc->submission_state.lock);
253 might_lock(&guc->submission_state.suspend.lock);
254 mutex_unlock(&guc->submission_state.lock);
259 int xe_guc_submit_init(struct xe_guc *guc)
261 struct xe_device *xe = guc_to_xe(guc);
262 struct xe_gt *gt = guc_to_gt(guc);
265 err = drmm_mutex_init(&xe->drm, &guc->submission_state.lock);
269 err = xe_guc_id_mgr_init(&guc->submission_state.idm, ~0);
273 err = alloc_submit_wq(guc);
279 xa_init(&guc->submission_state.exec_queue_lookup);
281 spin_lock_init(&guc->submission_state.suspend.lock);
282 guc->submission_state.suspend.context = dma_fence_context_alloc(1);
284 primelockdep(guc);
286 return drmm_add_action_or_reset(&xe->drm, guc_submit_fini, guc);
289 static void __release_guc_id(struct xe_guc *guc, struct xe_exec_queue *q, u32 xa_count)
293 lockdep_assert_held(&guc->submission_state.lock);
296 xa_erase(&guc->submission_state.exec_queue_lookup, q->guc->id + i);
298 xe_guc_id_mgr_release_locked(&guc->submission_state.idm,
299 q->guc->id, q->width);
302 static int alloc_guc_id(struct xe_guc *guc, struct xe_exec_queue *q)
315 lockdep_assert_held(&guc->submission_state.lock);
317 ret = xe_guc_id_mgr_reserve_locked(&guc->submission_state.idm,
322 q->guc->id = ret;
325 ptr = xa_store(&guc->submission_state.exec_queue_lookup,
326 q->guc->id + i, q, GFP_NOWAIT);
336 __release_guc_id(guc, q, i);
341 static void release_guc_id(struct xe_guc *guc, struct xe_exec_queue *q)
343 mutex_lock(&guc->submission_state.lock);
344 __release_guc_id(guc, q, q->width);
345 mutex_unlock(&guc->submission_state.lock);
396 static void init_policies(struct xe_guc *guc, struct xe_exec_queue *q)
399 struct xe_device *xe = guc_to_xe(guc);
406 __guc_exec_queue_policy_start_klv(&policy, q->guc->id);
411 xe_guc_ct_send(&guc->ct, (u32 *)&policy.h2g,
415 static void set_min_preemption_timeout(struct xe_guc *guc, struct xe_exec_queue *q)
419 __guc_exec_queue_policy_start_klv(&policy, q->guc->id);
422 xe_guc_ct_send(&guc->ct, (u32 *)&policy.h2g,
433 static void __register_mlrc_engine(struct xe_guc *guc,
438 struct xe_device *xe = guc_to_xe(guc);
469 xe_guc_ct_send(&guc->ct, action, len, 0, 0);
472 static void __register_engine(struct xe_guc *guc,
490 xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), 0, 0);
495 struct xe_guc *guc = exec_queue_to_guc(q);
496 struct xe_device *xe = guc_to_xe(guc);
503 info.context_idx = q->guc->id;
524 q->guc->wqi_head = 0;
525 q->guc->wqi_tail = 0;
541 __register_mlrc_engine(guc, q, &info);
543 __register_engine(guc, &info);
544 init_policies(guc, q);
549 return (WQ_SIZE - q->guc->wqi_tail);
554 struct xe_guc *guc = exec_queue_to_guc(q);
555 struct xe_device *xe = guc_to_xe(guc);
560 CIRC_SPACE(q->guc->wqi_tail, q->guc->wqi_head, WQ_SIZE)
563 q->guc->wqi_head = parallel_read(xe, map, wq_desc.head);
582 struct xe_guc *guc = exec_queue_to_guc(q);
583 struct xe_device *xe = guc_to_xe(guc);
592 parallel_write(xe, map, wq[q->guc->wqi_tail / sizeof(u32)],
595 q->guc->wqi_tail = 0;
602 struct xe_guc *guc = exec_queue_to_guc(q);
603 struct xe_device *xe = guc_to_xe(guc);
621 wqi[i++] = FIELD_PREP(WQ_GUC_ID_MASK, q->guc->id) |
633 wq[q->guc->wqi_tail / sizeof(u32)]));
635 q->guc->wqi_tail += wqi_size;
636 xe_assert(xe, q->guc->wqi_tail <= WQ_SIZE);
641 parallel_write(xe, map, wq_desc.tail, q->guc->wqi_tail);
647 struct xe_guc *guc = exec_queue_to_guc(q);
648 struct xe_device *xe = guc_to_xe(guc);
668 action[len++] = q->guc->id;
675 q->guc->resume_time = RESUME_PENDING;
681 action[len++] = q->guc->id;
685 xe_guc_ct_send(&guc->ct, action, len, g2h_len, num_g2h);
690 action[len++] = q->guc->id;
693 xe_guc_ct_send(&guc->ct, action, len, 0, 0);
702 struct xe_guc *guc = exec_queue_to_guc(q);
703 struct xe_device *xe = guc_to_xe(guc);
737 static int guc_read_stopped(struct xe_guc *guc)
739 return atomic_read(&guc->submission_state.stopped);
745 q->guc->id, \
749 static void disable_scheduling_deregister(struct xe_guc *guc,
753 struct xe_device *xe = guc_to_xe(guc);
756 set_min_preemption_timeout(guc, q);
758 ret = wait_event_timeout(guc->ct.wq, !exec_queue_pending_enable(q) ||
759 guc_read_stopped(guc), HZ * 5);
761 struct xe_gpu_scheduler *sched = &q->guc->sched;
779 xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action),
789 struct xe_guc *guc = exec_queue_to_guc(q);
790 struct xe_device *xe = guc_to_xe(guc);
811 if (xe_force_wake_get(gt_to_fw(guc_to_gt(guc)), XE_FORCEWAKE_ALL))
812 xe_gt_info(guc_to_gt(guc),
814 xe_guc_ct_print(&guc->ct, &p, true);
816 for_each_hw_engine(hwe, guc_to_gt(guc), id) {
823 xe_force_wake_put(gt_to_fw(guc_to_gt(guc)), XE_FORCEWAKE_ALL);
835 struct xe_guc *guc = exec_queue_to_guc(q);
836 struct xe_device *xe = guc_to_xe(guc);
842 queue_work(guc_to_gt(guc)->ordered_wq, &q->guc->lr_tdr);
844 xe_sched_tdr_queue_imm(&q->guc->sched);
852 struct xe_guc *guc = exec_queue_to_guc(q);
853 struct xe_device *xe = guc_to_xe(guc);
874 struct xe_guc *guc = exec_queue_to_guc(q);
878 disable_scheduling_deregister(guc, q);
884 ret = wait_event_timeout(guc->ct.wq,
886 guc_read_stopped(guc), HZ * 5);
904 struct xe_gpu_scheduler *sched = &q->guc->sched;
920 xe_sched_job_seqno(job), q->guc->id, q->flags);
950 struct xe_guc *guc = exec_queue_to_guc(q);
958 disable_scheduling_deregister(guc, q);
970 ret = wait_event_timeout(guc->ct.wq,
972 guc_read_stopped(guc), HZ * 5);
973 if (!ret || guc_read_stopped(guc)) {
1012 struct xe_guc *guc = exec_queue_to_guc(q);
1018 release_guc_id(guc, q);
1028 INIT_WORK(&q->guc->fini_async, __guc_exec_queue_fini_async);
1032 __guc_exec_queue_fini_async(&q->guc->fini_async);
1034 queue_work(system_wq, &q->guc->fini_async);
1037 static void __guc_exec_queue_fini(struct xe_guc *guc, struct xe_exec_queue *q)
1052 struct xe_guc *guc = exec_queue_to_guc(q);
1053 struct xe_device *xe = guc_to_xe(guc);
1059 disable_scheduling_deregister(guc, q);
1061 __guc_exec_queue_fini(guc, q);
1072 struct xe_guc *guc = exec_queue_to_guc(q);
1075 init_policies(guc, q);
1081 struct xe_guc *guc = exec_queue_to_guc(q);
1082 struct xe_device *xe = guc_to_xe(guc);
1085 guc_read_stopped(guc));
1086 xe_assert(xe, q->guc->suspend_pending);
1088 q->guc->suspend_pending = false;
1090 wake_up(&q->guc->suspend_wait);
1096 struct xe_guc *guc = exec_queue_to_guc(q);
1100 wait_event(guc->ct.wq, q->guc->resume_time != RESUME_PENDING ||
1101 guc_read_stopped(guc));
1103 if (!guc_read_stopped(guc)) {
1107 q->guc->resume_time);
1111 if (wait_ms > 0 && q->guc->resume_time)
1119 xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action),
1122 } else if (q->guc->suspend_pending) {
1131 struct xe_guc *guc = exec_queue_to_guc(q);
1136 q->guc->resume_time = RESUME_PENDING;
1142 xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action),
1189 struct xe_guc *guc = exec_queue_to_guc(q);
1190 struct xe_device *xe = guc_to_xe(guc);
1195 xe_assert(xe, xe_device_uc_enabled(guc_to_xe(guc)));
1201 q->guc = ge;
1208 get_submit_wq(guc),
1210 timeout, guc_to_gt(guc)->ordered_wq, NULL,
1221 INIT_WORK(&q->guc->lr_tdr, xe_guc_exec_queue_lr_cleanup);
1223 mutex_lock(&guc->submission_state.lock);
1225 err = alloc_guc_id(guc, q);
1231 if (guc_read_stopped(guc))
1234 mutex_unlock(&guc->submission_state.lock);
1236 xe_exec_queue_assign_name(q, q->guc->id);
1267 xe_sched_add_msg(&q->guc->sched, msg);
1275 struct xe_sched_msg *msg = q->guc->static_msgs + STATIC_MSG_CLEANUP;
1340 struct xe_sched_msg *msg = q->guc->static_msgs + STATIC_MSG_SUSPEND;
1342 if (exec_queue_killed_or_banned(q) || q->guc->suspend_pending)
1345 q->guc->suspend_pending = true;
1353 struct xe_guc *guc = exec_queue_to_guc(q);
1355 wait_event(q->guc->suspend_wait, !q->guc->suspend_pending ||
1356 guc_read_stopped(guc));
1361 struct xe_sched_msg *msg = q->guc->static_msgs + STATIC_MSG_RESUME;
1362 struct xe_guc *guc = exec_queue_to_guc(q);
1363 struct xe_device *xe = guc_to_xe(guc);
1365 xe_assert(xe, !q->guc->suspend_pending);
1394 static void guc_exec_queue_stop(struct xe_guc *guc, struct xe_exec_queue *q)
1396 struct xe_gpu_scheduler *sched = &q->guc->sched;
1407 __guc_exec_queue_fini(guc, q);
1409 if (q->guc->suspend_pending) {
1414 &q->guc->state);
1415 q->guc->resume_time = 0;
1431 xe_sched_tdr_queue_imm(&q->guc->sched);
1438 int xe_guc_submit_reset_prepare(struct xe_guc *guc)
1447 * and releasing any TDRs waiting on guc->submission_state.stopped.
1449 ret = atomic_fetch_or(1, &guc->submission_state.stopped);
1451 wake_up_all(&guc->ct.wq);
1456 void xe_guc_submit_reset_wait(struct xe_guc *guc)
1458 wait_event(guc->ct.wq, !guc_read_stopped(guc));
1461 int xe_guc_submit_stop(struct xe_guc *guc)
1465 struct xe_device *xe = guc_to_xe(guc);
1467 xe_assert(xe, guc_read_stopped(guc) == 1);
1469 mutex_lock(&guc->submission_state.lock);
1471 xa_for_each(&guc->submission_state.exec_queue_lookup, index, q)
1472 guc_exec_queue_stop(guc, q);
1474 mutex_unlock(&guc->submission_state.lock);
1478 * creation which is protected by guc->submission_state.lock.
1486 struct xe_gpu_scheduler *sched = &q->guc->sched;
1500 int xe_guc_submit_start(struct xe_guc *guc)
1504 struct xe_device *xe = guc_to_xe(guc);
1506 xe_assert(xe, guc_read_stopped(guc) == 1);
1508 mutex_lock(&guc->submission_state.lock);
1509 atomic_dec(&guc->submission_state.stopped);
1510 xa_for_each(&guc->submission_state.exec_queue_lookup, index, q)
1512 mutex_unlock(&guc->submission_state.lock);
1514 wake_up_all(&guc->ct.wq);
1520 g2h_exec_queue_lookup(struct xe_guc *guc, u32 guc_id)
1522 struct xe_device *xe = guc_to_xe(guc);
1530 q = xa_load(&guc->submission_state.exec_queue_lookup, guc_id);
1536 xe_assert(xe, guc_id >= q->guc->id);
1537 xe_assert(xe, guc_id < (q->guc->id + q->width));
1542 static void deregister_exec_queue(struct xe_guc *guc, struct xe_exec_queue *q)
1546 q->guc->id,
1551 xe_guc_ct_send_g2h_handler(&guc->ct, action, ARRAY_SIZE(action));
1554 static void handle_sched_done(struct xe_guc *guc, struct xe_exec_queue *q)
1559 q->guc->resume_time = ktime_get();
1562 wake_up_all(&guc->ct.wq);
1565 if (q->guc->suspend_pending) {
1570 wake_up_all(&guc->ct.wq);
1572 deregister_exec_queue(guc, q);
1577 int xe_guc_sched_done_handler(struct xe_guc *guc, u32 *msg, u32 len)
1579 struct xe_device *xe = guc_to_xe(guc);
1588 q = g2h_exec_queue_lookup(guc, guc_id);
1595 atomic_read(&q->guc->state));
1599 handle_sched_done(guc, q);
1604 static void handle_deregister_done(struct xe_guc *guc, struct xe_exec_queue *q)
1613 __guc_exec_queue_fini(guc, q);
1616 int xe_guc_deregister_done_handler(struct xe_guc *guc, u32 *msg, u32 len)
1618 struct xe_device *xe = guc_to_xe(guc);
1627 q = g2h_exec_queue_lookup(guc, guc_id);
1634 atomic_read(&q->guc->state));
1638 handle_deregister_done(guc, q);
1643 int xe_guc_exec_queue_reset_handler(struct xe_guc *guc, u32 *msg, u32 len)
1645 struct xe_device *xe = guc_to_xe(guc);
1654 q = g2h_exec_queue_lookup(guc, guc_id);
1677 int xe_guc_exec_queue_memory_cat_error_handler(struct xe_guc *guc, u32 *msg,
1680 struct xe_device *xe = guc_to_xe(guc);
1689 q = g2h_exec_queue_lookup(guc, guc_id);
1704 int xe_guc_exec_queue_reset_failure_handler(struct xe_guc *guc, u32 *msg, u32 len)
1706 struct xe_device *xe = guc_to_xe(guc);
1723 xe_gt_reset_async(guc_to_gt(guc));
1732 struct xe_guc *guc = exec_queue_to_guc(q);
1733 struct xe_device *xe = guc_to_xe(guc);
1737 snapshot->guc.wqi_head = q->guc->wqi_head;
1738 snapshot->guc.wqi_tail = q->guc->wqi_tail;
1761 snapshot->guc.wqi_head, snapshot->parallel.wq_desc.head);
1763 snapshot->guc.wqi_tail, snapshot->parallel.wq_desc.tail);
1789 struct xe_gpu_scheduler *sched = &q->guc->sched;
1798 snapshot->guc.id = q->guc->id;
1820 snapshot->schedule_state = atomic_read(&q->guc->state);
1888 drm_printf(p, "\nGuC ID: %d\n", snapshot->guc.id);
1952 * @guc: GuC.
1957 void xe_guc_submit_print(struct xe_guc *guc, struct drm_printer *p)
1962 if (!xe_device_uc_enabled(guc_to_xe(guc)))
1965 mutex_lock(&guc->submission_state.lock);
1966 xa_for_each(&guc->submission_state.exec_queue_lookup, index, q)
1968 mutex_unlock(&guc->submission_state.lock);