Lines Matching defs:cs
23 * enum hl_cs_wait_status - cs wait status
24 * @CS_WAIT_STATUS_BUSY: cs was not completed yet
25 * @CS_WAIT_STATUS_COMPLETED: cs completed
26 * @CS_WAIT_STATUS_GONE: cs completed but fence is already gone
250 void cs_get(struct hl_cs *cs)
252 kref_get(&cs->refcount);
255 static int cs_get_unless_zero(struct hl_cs *cs)
257 return kref_get_unless_zero(&cs->refcount);
260 static void cs_put(struct hl_cs *cs)
262 kref_put(&cs->refcount, cs_do_release);
277 bool cs_needs_completion(struct hl_cs *cs)
282 if (cs->staged_cs && !cs->staged_last)
288 bool cs_needs_timeout(struct hl_cs *cs)
293 if (cs->staged_cs && !cs->staged_first)
322 parser.ctx_id = job->cs->ctx->asid;
323 parser.cs_sequence = job->cs->sequence;
334 parser.completion = cs_needs_completion(job->cs);
363 struct hl_cs *cs = job->cs;
393 spin_lock(&cs->job_lock);
395 spin_unlock(&cs->job_lock);
411 if (cs_needs_completion(cs) &&
418 cs->completion_timestamp = job->timestamp;
420 cs_put(cs);
438 struct hl_cs *cs;
440 list_for_each_entry_reverse(cs, &hdev->cs_mirror_list, mirror_node)
441 if (cs->staged_cs && cs->staged_first &&
442 cs->sequence == cs_seq)
443 return cs;
452 * @cs: staged submission member
455 bool is_staged_cs_last_exists(struct hl_device *hdev, struct hl_cs *cs)
459 last_entry = list_last_entry(&cs->staged_cs_node, struct hl_cs,
472 * @cs: current CS
478 static void staged_cs_get(struct hl_device *hdev, struct hl_cs *cs)
485 if (!cs->staged_last)
486 cs_get(cs);
493 * @cs: CS to put
497 static void staged_cs_put(struct hl_device *hdev, struct hl_cs *cs)
502 if (!cs_needs_completion(cs))
503 cs_put(cs);
506 static void cs_handle_tdr(struct hl_device *hdev, struct hl_cs *cs)
510 if (!cs_needs_timeout(cs))
518 * In case single staged cs was submitted which has both first and last
520 * removed the cs node from the list before getting here,
521 * in such cases just continue with the cs to cancel it's TDR work.
523 if (cs->staged_cs && cs->staged_last) {
524 first_cs = hl_staged_cs_find_first(hdev, cs->staged_sequence);
526 cs = first_cs;
534 if (cs->timedout || hdev->timeout_jiffies == MAX_SCHEDULE_TIMEOUT)
537 if (cs->tdr_active)
538 cancel_delayed_work_sync(&cs->work_tdr);
595 * @cs: CS structure
606 static void complete_multi_cs(struct hl_device *hdev, struct hl_cs *cs)
608 struct hl_fence *fence = cs->fence;
612 if (cs->staged_cs && !cs->staged_first)
656 struct hl_cs *cs,
659 /* Skip this handler if the cs wasn't submitted, to avoid putting
663 if (!hl_cs_cmpl->hw_sob || !cs->submitted)
669 * we get refcount upon reservation of signals or signal/wait cs for the
670 * hw_sob object, and need to put it when the first staged cs
671 * (which contains the encaps signals) or cs signal/wait is completed.
696 struct hl_cs *cs = container_of(ref, struct hl_cs, refcount);
697 struct hl_device *hdev = cs->ctx->hdev;
700 container_of(cs->fence, struct hl_cs_compl, base_fence);
702 cs->completed = true;
712 list_for_each_entry_safe(job, tmp, &cs->job_list, cs_node)
715 if (!cs->submitted) {
721 if (cs->type == CS_TYPE_WAIT ||
722 cs->type == CS_TYPE_COLLECTIVE_WAIT)
723 hl_fence_put(cs->signal_fence);
729 hl_hw_queue_update_ci(cs);
733 list_del_init(&cs->mirror_node);
736 cs_handle_tdr(hdev, cs);
738 if (cs->staged_cs) {
742 if (cs->staged_last) {
746 &cs->staged_cs_node, staged_cs_node)
754 if (cs->submitted) {
756 list_del(&cs->staged_cs_node);
760 /* decrement refcount to handle when first staged cs
768 if ((cs->type == CS_TYPE_WAIT || cs->type == CS_TYPE_COLLECTIVE_WAIT) && cs->encaps_signals)
769 kref_put(&cs->encaps_sig_hdl->refcount, hl_encaps_release_handle_and_put_ctx);
775 hl_debugfs_remove_cs(cs);
777 hdev->shadow_cs_queue[cs->sequence & (hdev->asic_prop.max_pending_cs - 1)] = NULL;
783 if (cs->timedout)
784 cs->fence->error = -ETIMEDOUT;
785 else if (cs->aborted)
786 cs->fence->error = -EIO;
787 else if (!cs->submitted)
788 cs->fence->error = -EBUSY;
790 if (unlikely(cs->skip_reset_on_timeout)) {
793 cs->sequence,
794 div_u64(jiffies - cs->submission_time_jiffies, HZ));
797 if (cs->timestamp) {
798 cs->fence->timestamp = cs->completion_timestamp;
799 hl_push_cs_outcome(hdev, &cs->ctx->outcome_store, cs->sequence,
800 cs->fence->timestamp, cs->fence->error);
803 hl_ctx_put(cs->ctx);
805 complete_all(&cs->fence->completion);
806 complete_multi_cs(hdev, cs);
808 cs_release_sob_reset_handler(hdev, cs, hl_cs_cmpl);
810 hl_fence_put(cs->fence);
812 kfree(cs->jobs_in_queue_cnt);
813 kfree(cs);
818 struct hl_cs *cs = container_of(work, struct hl_cs, work_tdr.work);
825 skip_reset_on_timeout = cs->skip_reset_on_timeout;
827 rc = cs_get_unless_zero(cs);
831 if ((!cs->submitted) || (cs->completed)) {
832 cs_put(cs);
836 hdev = cs->ctx->hdev;
845 cs->timedout = true;
852 hdev->captured_err_info.cs_timeout.seq = cs->sequence;
858 switch (cs->type) {
862 cs->sequence, timeout_sec);
868 cs->sequence, timeout_sec);
874 cs->sequence, timeout_sec);
880 cs->sequence, timeout_sec);
888 cs_put(cs);
905 struct hl_cs *cs;
910 cs = kzalloc(sizeof(*cs), GFP_ATOMIC);
911 if (!cs)
912 cs = kzalloc(sizeof(*cs), GFP_KERNEL);
914 if (!cs) {
923 cs->ctx = ctx;
924 cs->submitted = false;
925 cs->completed = false;
926 cs->type = cs_type;
927 cs->timestamp = !!(flags & HL_CS_FLAGS_TIMESTAMP);
928 cs->encaps_signals = !!(flags & HL_CS_FLAGS_ENCAP_SIGNALS);
929 cs->timeout_jiffies = timeout;
930 cs->skip_reset_on_timeout =
933 cs->submission_time_jiffies = jiffies;
934 INIT_LIST_HEAD(&cs->job_list);
935 INIT_DELAYED_WORK(&cs->work_tdr, cs_timedout);
936 kref_init(&cs->refcount);
937 spin_lock_init(&cs->job_lock);
950 cs->jobs_in_queue_cnt = kcalloc(hdev->asic_prop.max_queues,
951 sizeof(*cs->jobs_in_queue_cnt), GFP_ATOMIC);
952 if (!cs->jobs_in_queue_cnt)
953 cs->jobs_in_queue_cnt = kcalloc(hdev->asic_prop.max_queues,
954 sizeof(*cs->jobs_in_queue_cnt), GFP_KERNEL);
956 if (!cs->jobs_in_queue_cnt) {
964 cs_cmpl->type = cs->type;
966 cs->fence = &cs_cmpl->base_fence;
998 cs->sequence = cs_cmpl->cs_seq;
1011 *cs_new = cs;
1017 kfree(cs->jobs_in_queue_cnt);
1021 kfree(cs);
1026 static void cs_rollback(struct hl_device *hdev, struct hl_cs *cs)
1030 staged_cs_put(hdev, cs);
1032 list_for_each_entry_safe(job, tmp, &cs->job_list, cs_node)
1067 struct hl_cs *cs, *tmp;
1082 list_for_each_entry_safe(cs, tmp, &hdev->cs_mirror_list, mirror_node) {
1083 cs_get(cs);
1084 cs->aborted = true;
1086 cs->ctx->asid, cs->sequence);
1087 cs_rollback(hdev, cs);
1088 cs_put(cs);
1148 struct hl_cs *cs;
1152 list_for_each_entry(cs, &hdev->cs_mirror_list, mirror_node) {
1153 cs->fence->error = -EIO;
1154 complete_all(&cs->fence->completion);
1170 struct hl_cs *cs = job->cs;
1171 struct hl_device *hdev = cs->ctx->hdev;
1179 struct hl_cs *cs = container_of(work, struct hl_cs, finish_work);
1180 struct hl_device *hdev = cs->ctx->hdev;
1183 list_for_each_entry_safe(job, tmp, &cs->job_list, cs_node)
1190 struct hl_cs *cs;
1194 list_for_each_entry(cs, &hdev->cs_mirror_list, mirror_node)
1195 if (!cs->completed)
1438 dev_err(hdev->dev, "Failed to copy cs chunk array from user\n");
1446 static int cs_staged_submission(struct hl_device *hdev, struct hl_cs *cs,
1453 cs->staged_last = !!(flags & HL_CS_FLAGS_STAGED_SUBMISSION_LAST);
1454 cs->staged_first = !!(flags & HL_CS_FLAGS_STAGED_SUBMISSION_FIRST);
1456 if (cs->staged_first) {
1458 INIT_LIST_HEAD(&cs->staged_cs_node);
1459 cs->staged_sequence = cs->sequence;
1461 if (cs->encaps_signals)
1462 cs->encaps_sig_hdl_id = encaps_signal_handle;
1467 cs->staged_sequence = sequence;
1471 staged_cs_get(hdev, cs);
1473 cs->staged_cs = true;
1500 struct hl_cs *cs;
1522 staged_mid ? user_sequence : ULLONG_MAX, &cs, flags,
1527 *cs_seq = cs->sequence;
1529 hl_debugfs_add_cs(cs);
1531 rc = cs_staged_submission(hdev, cs, user_sequence, flags,
1539 if (cs->staged_cs)
1540 *cs_seq = cs->staged_sequence;
1600 job->cs = cs;
1605 cs->jobs_in_queue_cnt[job->hw_queue_id]++;
1606 cs->jobs_cnt++;
1608 list_add_tail(&job->cs_node, &cs->job_list);
1616 if (cs_needs_completion(cs) &&
1619 cs_get(cs);
1629 cs->ctx->asid, cs->sequence, job->id, rc);
1637 if (int_queues_only && cs_needs_completion(cs)) {
1642 cs->ctx->asid, cs->sequence);
1648 INIT_WORK(&cs->finish_work, cs_completion);
1655 cs->fence->stream_master_qid_map = stream_master_qid_map;
1657 rc = hl_hw_queue_schedule_cs(cs);
1662 cs->ctx->asid, cs->sequence, rc);
1666 *signal_initial_sob_count = cs->initial_sob_count;
1675 cs_rollback(hdev, cs);
1680 cs_put(cs);
1851 * for the reservation or the next signal cs.
1852 * we do it here, and for both encaps and regular signal cs
1856 * in addition, if we have combination of cs signal and
1858 * no more reservations and only signal cs keep coming,
1939 struct hl_ctx *ctx, struct hl_cs *cs,
1957 if (cs->type == CS_TYPE_WAIT)
1971 job->cs = cs;
1977 if ((cs->type == CS_TYPE_WAIT || cs->type == CS_TYPE_COLLECTIVE_WAIT)
1978 && cs->encaps_signals)
1991 cs_get(cs);
1993 cs->jobs_in_queue_cnt[job->hw_queue_id]++;
1994 cs->jobs_cnt++;
1996 list_add_tail(&job->cs_node, &cs->job_list);
2217 struct hl_cs *cs;
2289 /* check if cs sequence has encapsulated
2300 * needed when multiple wait cs are used with offset
2377 rc = allocate_cs(hdev, ctx, cs_type, ULLONG_MAX, &cs, flags, timeout);
2388 * for encaps signals case, we save the cs sequence and handle pointer
2392 cs->signal_fence = sig_fence;
2397 if (cs->encaps_signals)
2398 cs->encaps_sig_hdl = encaps_sig_hdl;
2401 hl_debugfs_add_cs(cs);
2403 *cs_seq = cs->sequence;
2406 rc = cs_ioctl_signal_wait_create_jobs(hdev, ctx, cs, q_type,
2410 cs, q_idx, collective_engine_id,
2422 INIT_WORK(&cs->finish_work, cs_completion);
2424 rc = hl_hw_queue_schedule_cs(cs);
2426 /* In case wait cs failed here, it means the signal cs
2435 ctx->asid, cs->sequence, rc);
2439 *signal_sob_addr_offset = cs->sob_addr_offset;
2440 *signal_initial_sob_count = cs->initial_sob_count;
2448 cs_rollback(hdev, cs);
2453 cs_put(cs);
3050 dev_err(hdev->dev, "Failed to copy multi-cs sequence array from user\n");