Lines Matching refs:hdev
36 * @hdev: pointer to habanalabs device structure.
49 static u64 hl_set_dram_bar(struct hl_device *hdev, u64 addr, struct pci_mem_region *region,
52 struct asic_fixed_properties *prop = &hdev->asic_prop;
62 old_base = hdev->asic_funcs->set_dram_bar_base(hdev, bar_base_addr);
71 int hl_access_sram_dram_region(struct hl_device *hdev, u64 addr, u64 *val,
74 struct pci_mem_region *region = &hdev->pci_mem_region[region_type];
79 old_base = hl_set_dram_bar(hdev, addr, region, &bar_region_base);
84 acc_addr = hdev->pcie_bar[region->bar_id] + region->offset_in_bar +
109 rc = hl_set_dram_bar(hdev, old_base, region, NULL);
117 static void *hl_dma_alloc_common(struct hl_device *hdev, size_t size, dma_addr_t *dma_handle,
125 ptr = hdev->asic_funcs->asic_dma_alloc_coherent(hdev, size, dma_handle, flag);
128 ptr = hdev->asic_funcs->asic_dma_pool_zalloc(hdev, size, flag, dma_handle);
133 trace_habanalabs_dma_alloc(hdev->dev, (u64) (uintptr_t) ptr, *dma_handle, size,
139 static void hl_asic_dma_free_common(struct hl_device *hdev, size_t size, void *cpu_addr,
148 hdev->asic_funcs->asic_dma_free_coherent(hdev, size, cpu_addr, dma_handle);
151 hdev->asic_funcs->asic_dma_pool_free(hdev, cpu_addr, dma_handle);
155 trace_habanalabs_dma_free(hdev->dev, store_cpu_addr, dma_handle, size, caller);
158 void *hl_asic_dma_alloc_coherent_caller(struct hl_device *hdev, size_t size, dma_addr_t *dma_handle,
161 return hl_dma_alloc_common(hdev, size, dma_handle, flag, DMA_ALLOC_COHERENT, caller);
164 void hl_asic_dma_free_coherent_caller(struct hl_device *hdev, size_t size, void *cpu_addr,
167 hl_asic_dma_free_common(hdev, size, cpu_addr, dma_handle, DMA_ALLOC_COHERENT, caller);
170 void *hl_asic_dma_pool_zalloc_caller(struct hl_device *hdev, size_t size, gfp_t mem_flags,
173 return hl_dma_alloc_common(hdev, size, dma_handle, mem_flags, DMA_ALLOC_POOL, caller);
176 void hl_asic_dma_pool_free_caller(struct hl_device *hdev, void *vaddr, dma_addr_t dma_addr,
179 hl_asic_dma_free_common(hdev, 0, vaddr, dma_addr, DMA_ALLOC_POOL, caller);
182 void *hl_cpu_accessible_dma_pool_alloc(struct hl_device *hdev, size_t size, dma_addr_t *dma_handle)
184 return hdev->asic_funcs->cpu_accessible_dma_pool_alloc(hdev, size, dma_handle);
187 void hl_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size, void *vaddr)
189 hdev->asic_funcs->cpu_accessible_dma_pool_free(hdev, size, vaddr);
192 int hl_dma_map_sgtable_caller(struct hl_device *hdev, struct sg_table *sgt,
195 struct asic_fixed_properties *prop = &hdev->asic_prop;
199 rc = hdev->asic_funcs->dma_map_sgtable(hdev, sgt, dir);
207 trace_habanalabs_dma_map_page(hdev->dev,
220 int hl_asic_dma_map_sgtable(struct hl_device *hdev, struct sg_table *sgt,
223 struct asic_fixed_properties *prop = &hdev->asic_prop;
227 rc = dma_map_sgtable(&hdev->pdev->dev, sgt, dir, 0);
239 void hl_dma_unmap_sgtable_caller(struct hl_device *hdev, struct sg_table *sgt,
242 struct asic_fixed_properties *prop = &hdev->asic_prop;
246 hdev->asic_funcs->dma_unmap_sgtable(hdev, sgt, dir);
250 trace_habanalabs_dma_unmap_page(hdev->dev, page_to_phys(sg_page(sg)),
261 void hl_asic_dma_unmap_sgtable(struct hl_device *hdev, struct sg_table *sgt,
264 struct asic_fixed_properties *prop = &hdev->asic_prop;
273 dma_unmap_sgtable(&hdev->pdev->dev, sgt, dir, 0);
279 * @hdev: pointer to habanalabs device structure
284 int hl_access_cfg_region(struct hl_device *hdev, u64 addr, u64 *val,
287 struct pci_mem_region *cfg_region = &hdev->pci_mem_region[PCI_REGION_CFG];
291 dev_err(hdev->dev, "address %#llx not a multiple of %zu\n", addr, sizeof(u32));
313 dev_err(hdev->dev, "access type %d is not supported\n", acc_type);
323 * @hdev: pointer to habanalabs device structure
329 int hl_access_dev_mem(struct hl_device *hdev, enum pci_region region_type,
334 return hl_access_cfg_region(hdev, addr, val, acc_type);
337 return hl_access_sram_dram_region(hdev, addr, val, acc_type,
370 enum hl_device_status hl_device_status(struct hl_device *hdev)
374 if (hdev->device_fini_pending) {
376 } else if (hdev->reset_info.in_reset) {
377 if (hdev->reset_info.in_compute_reset)
381 } else if (hdev->reset_info.needs_reset) {
383 } else if (hdev->disabled) {
385 } else if (!hdev->init_done) {
394 bool hl_device_operational(struct hl_device *hdev,
399 current_status = hl_device_status(hdev);
416 bool hl_ctrl_device_operational(struct hl_device *hdev,
421 current_status = hl_device_status(hdev);
438 static void print_idle_status_mask(struct hl_device *hdev, const char *message,
442 dev_err(hdev->dev, "%s (mask %#llx_%016llx_%016llx_%016llx)\n",
445 dev_err(hdev->dev, "%s (mask %#llx_%016llx_%016llx)\n",
448 dev_err(hdev->dev, "%s (mask %#llx_%016llx)\n",
451 dev_err(hdev->dev, "%s (mask %#llx)\n", message, idle_mask[0]);
459 struct hl_device *hdev;
463 hdev = hpriv->hdev;
465 hdev->asic_funcs->send_device_activity(hdev, false);
478 reset_device = hdev->reset_upon_device_release || hdev->reset_info.watchdog_active;
483 if (!hdev->reset_info.in_reset && !reset_device && !hdev->pldm)
484 device_is_idle = hdev->asic_funcs->is_device_idle(hdev, idle_mask,
487 print_idle_status_mask(hdev, "device is not idle after user context is closed",
501 mutex_lock(&hdev->fpriv_list_lock);
503 mutex_unlock(&hdev->fpriv_list_lock);
508 hl_device_reset(hdev, HL_DRV_RESET_DEV_RELEASE);
511 int rc = hdev->asic_funcs->scrub_device_mem(hdev);
514 dev_err(hdev->dev, "failed to scrub memory from hpriv release (%d)\n", rc);
515 hl_device_reset(hdev, HL_DRV_RESET_HARD);
523 mutex_lock(&hdev->fpriv_list_lock);
524 hdev->is_compute_ctx_active = false;
525 mutex_unlock(&hdev->fpriv_list_lock);
527 hdev->compute_ctx_in_release = 0;
548 static void print_device_in_use_info(struct hl_device *hdev, const char *message)
559 active_cs_num = hl_get_active_cs_num(hdev);
565 dmabuf_export_cnt = atomic_read(&hdev->dmabuf_export_cnt);
575 dev_notice(hdev->dev, "%s%s\n", message, buf);
588 struct hl_device *hdev = to_hl_device(ddev);
590 if (!hdev) {
595 hl_ctx_mgr_fini(hdev, &hpriv->ctx_mgr);
602 hdev->compute_ctx_in_release = 1;
605 print_device_in_use_info(hdev, "User process closed FD but device still in use");
606 hl_device_reset(hdev, HL_DRV_RESET_HARD);
609 hdev->last_open_session_duration_jif = jiffies - hdev->last_successful_open_jif;
615 struct hl_device *hdev = hpriv->hdev;
619 if (!hdev) {
624 mutex_lock(&hdev->fpriv_ctrl_list_lock);
626 mutex_unlock(&hdev->fpriv_ctrl_list_lock);
637 struct hl_device *hdev = hpriv->hdev;
640 if (!hdev) {
692 * @hdev: pointer to habanalabs device structure
702 static int device_init_cdev(struct hl_device *hdev, const struct class *class,
715 (*dev)->devt = MKDEV(hdev->major, minor);
718 dev_set_drvdata(*dev, hdev);
724 static int cdev_sysfs_debugfs_add(struct hl_device *hdev)
726 const struct class *accel_class = hdev->drm.accel->kdev->class;
730 hdev->cdev_idx = hdev->drm.accel->index;
733 snprintf(name, sizeof(name), "accel_controlD%d", hdev->cdev_idx);
734 rc = device_init_cdev(hdev, accel_class, hdev->cdev_idx, &hl_ctrl_ops, name,
735 &hdev->cdev_ctrl, &hdev->dev_ctrl);
739 rc = cdev_device_add(&hdev->cdev_ctrl, hdev->dev_ctrl);
741 dev_err(hdev->dev_ctrl,
746 rc = hl_sysfs_init(hdev);
748 dev_err(hdev->dev, "failed to initialize sysfs\n");
752 hl_debugfs_add_device(hdev);
754 hdev->cdev_sysfs_debugfs_created = true;
759 cdev_device_del(&hdev->cdev_ctrl, hdev->dev_ctrl);
761 put_device(hdev->dev_ctrl);
765 static void cdev_sysfs_debugfs_remove(struct hl_device *hdev)
767 if (!hdev->cdev_sysfs_debugfs_created)
770 hl_sysfs_fini(hdev);
772 cdev_device_del(&hdev->cdev_ctrl, hdev->dev_ctrl);
773 put_device(hdev->dev_ctrl);
780 struct hl_device *hdev = device_reset_work->hdev;
786 rc = hl_device_reset(hdev, flags);
788 if ((rc == -EBUSY) && !hdev->device_fini_pending) {
789 struct hl_ctx *ctx = hl_get_compute_ctx(hdev);
795 dev_info(hdev->dev,
800 dev_info(hdev->dev, "Could not reset device. will try again in %u seconds",
804 queue_delayed_work(hdev->reset_wq, &device_reset_work->reset_work,
813 struct hl_device *hdev = watchdog_work->hdev;
816 dev_dbg(hdev->dev, "Device wasn't released in time. Initiate hard-reset.\n");
820 hl_device_reset(hdev, flags);
826 * @hdev: pointer to habanalabs device structure
831 static int device_early_init(struct hl_device *hdev)
836 switch (hdev->asic_type) {
838 goya_set_asic_funcs(hdev);
839 strscpy(hdev->asic_name, "GOYA", sizeof(hdev->asic_name));
842 gaudi_set_asic_funcs(hdev);
843 strscpy(hdev->asic_name, "GAUDI", sizeof(hdev->asic_name));
846 gaudi_set_asic_funcs(hdev);
847 strscpy(hdev->asic_name, "GAUDI SEC", sizeof(hdev->asic_name));
850 gaudi2_set_asic_funcs(hdev);
851 strscpy(hdev->asic_name, "GAUDI2", sizeof(hdev->asic_name));
854 gaudi2_set_asic_funcs(hdev);
855 strscpy(hdev->asic_name, "GAUDI2B", sizeof(hdev->asic_name));
858 gaudi2_set_asic_funcs(hdev);
859 strscpy(hdev->asic_name, "GAUDI2C", sizeof(hdev->asic_name));
862 dev_err(hdev->dev, "Unrecognized ASIC type %d\n",
863 hdev->asic_type);
867 rc = hdev->asic_funcs->early_init(hdev);
871 rc = hl_asid_init(hdev);
875 if (hdev->asic_prop.completion_queues_count) {
876 hdev->cq_wq = kcalloc(hdev->asic_prop.completion_queues_count,
879 if (!hdev->cq_wq) {
885 for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++) {
886 snprintf(workq_name, 32, "hl%u-free-jobs-%u", hdev->cdev_idx, (u32) i);
887 hdev->cq_wq[i] = create_singlethread_workqueue(workq_name);
888 if (hdev->cq_wq[i] == NULL) {
889 dev_err(hdev->dev, "Failed to allocate CQ workqueue\n");
895 snprintf(workq_name, 32, "hl%u-events", hdev->cdev_idx);
896 hdev->eq_wq = create_singlethread_workqueue(workq_name);
897 if (hdev->eq_wq == NULL) {
898 dev_err(hdev->dev, "Failed to allocate EQ workqueue\n");
903 snprintf(workq_name, 32, "hl%u-cs-completions", hdev->cdev_idx);
904 hdev->cs_cmplt_wq = alloc_workqueue(workq_name, WQ_UNBOUND, 0);
905 if (!hdev->cs_cmplt_wq) {
906 dev_err(hdev->dev,
912 snprintf(workq_name, 32, "hl%u-ts-free-obj", hdev->cdev_idx);
913 hdev->ts_free_obj_wq = alloc_workqueue(workq_name, WQ_UNBOUND, 0);
914 if (!hdev->ts_free_obj_wq) {
915 dev_err(hdev->dev,
921 snprintf(workq_name, 32, "hl%u-prefetch", hdev->cdev_idx);
922 hdev->prefetch_wq = alloc_workqueue(workq_name, WQ_UNBOUND, 0);
923 if (!hdev->prefetch_wq) {
924 dev_err(hdev->dev, "Failed to allocate MMU prefetch workqueue\n");
929 hdev->hl_chip_info = kzalloc(sizeof(struct hwmon_chip_info), GFP_KERNEL);
930 if (!hdev->hl_chip_info) {
935 rc = hl_mmu_if_set_funcs(hdev);
939 hl_mem_mgr_init(hdev->dev, &hdev->kernel_mem_mgr);
941 snprintf(workq_name, 32, "hl%u_device_reset", hdev->cdev_idx);
942 hdev->reset_wq = create_singlethread_workqueue(workq_name);
943 if (!hdev->reset_wq) {
945 dev_err(hdev->dev, "Failed to create device reset WQ\n");
949 INIT_DELAYED_WORK(&hdev->device_reset_work.reset_work, device_hard_reset_pending);
950 hdev->device_reset_work.hdev = hdev;
951 hdev->device_fini_pending = 0;
953 INIT_DELAYED_WORK(&hdev->device_release_watchdog_work.reset_work,
955 hdev->device_release_watchdog_work.hdev = hdev;
957 mutex_init(&hdev->send_cpu_message_lock);
958 mutex_init(&hdev->debug_lock);
959 INIT_LIST_HEAD(&hdev->cs_mirror_list);
960 spin_lock_init(&hdev->cs_mirror_lock);
961 spin_lock_init(&hdev->reset_info.lock);
962 INIT_LIST_HEAD(&hdev->fpriv_list);
963 INIT_LIST_HEAD(&hdev->fpriv_ctrl_list);
964 mutex_init(&hdev->fpriv_list_lock);
965 mutex_init(&hdev->fpriv_ctrl_list_lock);
966 mutex_init(&hdev->clk_throttling.lock);
971 hl_mem_mgr_fini(&hdev->kernel_mem_mgr);
972 hl_mem_mgr_idr_destroy(&hdev->kernel_mem_mgr);
974 kfree(hdev->hl_chip_info);
976 destroy_workqueue(hdev->prefetch_wq);
978 destroy_workqueue(hdev->ts_free_obj_wq);
980 destroy_workqueue(hdev->cs_cmplt_wq);
982 destroy_workqueue(hdev->eq_wq);
984 for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++)
985 if (hdev->cq_wq[i])
986 destroy_workqueue(hdev->cq_wq[i]);
987 kfree(hdev->cq_wq);
989 hl_asid_fini(hdev);
991 if (hdev->asic_funcs->early_fini)
992 hdev->asic_funcs->early_fini(hdev);
1000 * @hdev: pointer to habanalabs device structure
1003 static void device_early_fini(struct hl_device *hdev)
1007 mutex_destroy(&hdev->debug_lock);
1008 mutex_destroy(&hdev->send_cpu_message_lock);
1010 mutex_destroy(&hdev->fpriv_list_lock);
1011 mutex_destroy(&hdev->fpriv_ctrl_list_lock);
1013 mutex_destroy(&hdev->clk_throttling.lock);
1015 hl_mem_mgr_fini(&hdev->kernel_mem_mgr);
1016 hl_mem_mgr_idr_destroy(&hdev->kernel_mem_mgr);
1018 kfree(hdev->hl_chip_info);
1020 destroy_workqueue(hdev->prefetch_wq);
1021 destroy_workqueue(hdev->ts_free_obj_wq);
1022 destroy_workqueue(hdev->cs_cmplt_wq);
1023 destroy_workqueue(hdev->eq_wq);
1024 destroy_workqueue(hdev->reset_wq);
1026 for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++)
1027 destroy_workqueue(hdev->cq_wq[i]);
1028 kfree(hdev->cq_wq);
1030 hl_asid_fini(hdev);
1032 if (hdev->asic_funcs->early_fini)
1033 hdev->asic_funcs->early_fini(hdev);
1036 static bool is_pci_link_healthy(struct hl_device *hdev)
1040 if (!hdev->pdev)
1043 pci_read_config_word(hdev->pdev, PCI_DEVICE_ID, &device_id);
1045 return (device_id == hdev->pdev->device);
1048 static int hl_device_eq_heartbeat_check(struct hl_device *hdev)
1050 struct asic_fixed_properties *prop = &hdev->asic_prop;
1055 if (hdev->eq_heartbeat_received) {
1056 hdev->eq_heartbeat_received = false;
1058 dev_err(hdev->dev, "EQ heartbeat event was not received!\n");
1067 struct hl_device *hdev = container_of(work, struct hl_device,
1073 if (!hl_device_operational(hdev, NULL) || !hdev->init_done)
1081 if ((!hl_device_eq_heartbeat_check(hdev)) && (!hdev->asic_funcs->send_heartbeat(hdev)))
1084 if (hl_device_operational(hdev, NULL))
1085 dev_err(hdev->dev, "Device heartbeat failed! PCI link is %s\n",
1086 is_pci_link_healthy(hdev) ? "healthy" : "broken");
1090 hl_handle_fw_err(hdev, &info);
1091 hl_device_cond_reset(hdev, HL_DRV_RESET_HARD | HL_DRV_RESET_HEARTBEAT, event_mask);
1106 if (!hdev->reset_info.in_reset)
1107 hdev->reset_info.prev_reset_trigger = HL_RESET_TRIGGER_DEFAULT;
1109 schedule_delayed_work(&hdev->work_heartbeat,
1116 * @hdev: pointer to habanalabs device structure
1121 static int device_late_init(struct hl_device *hdev)
1125 if (hdev->asic_funcs->late_init) {
1126 rc = hdev->asic_funcs->late_init(hdev);
1128 dev_err(hdev->dev,
1134 hdev->high_pll = hdev->asic_prop.high_pll;
1136 if (hdev->heartbeat) {
1142 hdev->eq_heartbeat_received = true;
1144 INIT_DELAYED_WORK(&hdev->work_heartbeat, hl_device_heartbeat);
1146 schedule_delayed_work(&hdev->work_heartbeat,
1150 hdev->late_init_done = true;
1158 * @hdev: pointer to habanalabs device structure
1161 static void device_late_fini(struct hl_device *hdev)
1163 if (!hdev->late_init_done)
1166 if (hdev->heartbeat)
1167 cancel_delayed_work_sync(&hdev->work_heartbeat);
1169 if (hdev->asic_funcs->late_fini)
1170 hdev->asic_funcs->late_fini(hdev);
1172 hdev->late_init_done = false;
1175 int hl_device_utilization(struct hl_device *hdev, u32 *utilization)
1180 max_power = hdev->max_power;
1181 dc_power = hdev->asic_prop.dc_power_default;
1184 dev_warn(hdev->dev, "device utilization is not supported\n");
1187 rc = hl_fw_cpucp_power_get(hdev, &curr_power);
1200 int hl_device_set_debug_mode(struct hl_device *hdev, struct hl_ctx *ctx, bool enable)
1204 mutex_lock(&hdev->debug_lock);
1207 if (!hdev->in_debug) {
1208 dev_err(hdev->dev,
1214 if (!hdev->reset_info.hard_reset_pending)
1215 hdev->asic_funcs->halt_coresight(hdev, ctx);
1217 hdev->in_debug = 0;
1222 if (hdev->in_debug) {
1223 dev_err(hdev->dev,
1229 hdev->in_debug = 1;
1232 mutex_unlock(&hdev->debug_lock);
1237 static void take_release_locks(struct hl_device *hdev)
1242 hdev->asic_funcs->hw_queues_lock(hdev);
1243 hdev->asic_funcs->hw_queues_unlock(hdev);
1246 mutex_lock(&hdev->send_cpu_message_lock);
1247 mutex_unlock(&hdev->send_cpu_message_lock);
1250 mutex_lock(&hdev->fpriv_list_lock);
1251 mutex_unlock(&hdev->fpriv_list_lock);
1252 mutex_lock(&hdev->fpriv_ctrl_list_lock);
1253 mutex_unlock(&hdev->fpriv_ctrl_list_lock);
1256 static void hl_abort_waiting_for_completions(struct hl_device *hdev)
1258 hl_abort_waiting_for_cs_completions(hdev);
1263 hl_release_pending_user_interrupts(hdev);
1266 static void cleanup_resources(struct hl_device *hdev, bool hard_reset, bool fw_reset,
1270 device_late_fini(hdev);
1277 hdev->asic_funcs->halt_engines(hdev, hard_reset, fw_reset);
1280 hl_cs_rollback_all(hdev, skip_wq_flush);
1283 flush_workqueue(hdev->prefetch_wq);
1285 hl_abort_waiting_for_completions(hdev);
1291 * @hdev: pointer to habanalabs device structure
1297 int hl_device_suspend(struct hl_device *hdev)
1301 pci_save_state(hdev->pdev);
1304 spin_lock(&hdev->reset_info.lock);
1305 if (hdev->reset_info.in_reset) {
1306 spin_unlock(&hdev->reset_info.lock);
1307 dev_err(hdev->dev, "Can't suspend while in reset\n");
1310 hdev->reset_info.in_reset = 1;
1311 spin_unlock(&hdev->reset_info.lock);
1314 hdev->disabled = true;
1316 take_release_locks(hdev);
1318 rc = hdev->asic_funcs->suspend(hdev);
1320 dev_err(hdev->dev,
1324 pci_disable_device(hdev->pdev);
1325 pci_set_power_state(hdev->pdev, PCI_D3hot);
1333 * @hdev: pointer to habanalabs device structure
1339 int hl_device_resume(struct hl_device *hdev)
1343 pci_set_power_state(hdev->pdev, PCI_D0);
1344 pci_restore_state(hdev->pdev);
1345 rc = pci_enable_device_mem(hdev->pdev);
1347 dev_err(hdev->dev,
1352 pci_set_master(hdev->pdev);
1354 rc = hdev->asic_funcs->resume(hdev);
1356 dev_err(hdev->dev, "Failed to resume device after suspend\n");
1364 spin_lock(&hdev->reset_info.lock);
1365 hdev->reset_info.in_reset = 0;
1366 spin_unlock(&hdev->reset_info.lock);
1368 rc = hl_device_reset(hdev, HL_DRV_RESET_HARD);
1370 dev_err(hdev->dev, "Failed to reset device during resume\n");
1377 pci_disable_device(hdev->pdev);
1382 static int device_kill_open_processes(struct hl_device *hdev, u32 timeout, bool control_dev)
1390 hpriv_lock = control_dev ? &hdev->fpriv_ctrl_list_lock : &hdev->fpriv_list_lock;
1391 hpriv_list = control_dev ? &hdev->fpriv_ctrl_list : &hdev->fpriv_list;
1402 if (hdev->process_kill_trial_cnt) {
1420 dev_info(hdev->dev, "Killing user process pid=%d\n",
1427 dev_dbg(hdev->dev,
1446 dev_dbg(hdev->dev,
1459 if (hdev->process_kill_trial_cnt == HL_PENDING_RESET_MAX_TRIALS)
1462 hdev->process_kill_trial_cnt++;
1467 static void device_disable_open_processes(struct hl_device *hdev, bool control_dev)
1473 hpriv_lock = control_dev ? &hdev->fpriv_ctrl_list_lock : &hdev->fpriv_list_lock;
1474 hpriv_list = control_dev ? &hdev->fpriv_ctrl_list : &hdev->fpriv_list;
1478 hpriv->hdev = NULL;
1482 static void send_disable_pci_access(struct hl_device *hdev, u32 flags)
1498 if (hl_fw_send_pci_access_msg(hdev, CPUCP_PACKET_DISABLE_PCI_ACCESS, 0x0)) {
1499 dev_warn(hdev->dev, "Failed to disable FW's PCI access\n");
1504 if (hdev->cpu_queues_enable)
1505 synchronize_irq(pci_irq_vector(hdev->pdev,
1506 hdev->asic_prop.eq_interrupt_id));
1510 static void handle_reset_trigger(struct hl_device *hdev, u32 flags)
1515 if (hdev->is_compute_ctx_active)
1525 hdev->reset_info.curr_reset_cause = HL_RESET_CAUSE_HEARTBEAT;
1528 hdev->reset_info.curr_reset_cause = HL_RESET_CAUSE_TDR;
1531 hdev->reset_info.curr_reset_cause = HL_RESET_CAUSE_UNKNOWN;
1534 hdev->reset_info.curr_reset_cause = HL_RESET_CAUSE_UNKNOWN;
1542 if (hdev->reset_info.prev_reset_trigger != cur_reset_trigger) {
1543 hdev->reset_info.prev_reset_trigger = cur_reset_trigger;
1544 hdev->reset_info.reset_trigger_repeated = 0;
1546 hdev->reset_info.reset_trigger_repeated = 1;
1553 * @hdev: pointer to habanalabs device structure
1566 int hl_device_reset(struct hl_device *hdev, u32 flags)
1574 if (!hdev->init_done) {
1575 dev_err(hdev->dev, "Can't reset before initialization is done\n");
1585 reset_upon_device_release = hdev->reset_upon_device_release && from_dev_release;
1587 if (!hard_reset && (hl_device_status(hdev) == HL_DEVICE_STATUS_MALFUNCTION)) {
1588 dev_dbg(hdev->dev, "soft-reset isn't supported on a malfunctioning device\n");
1592 if (!hard_reset && !hdev->asic_prop.supports_compute_reset) {
1593 dev_dbg(hdev->dev, "asic doesn't support compute reset - do hard-reset instead\n");
1599 dev_crit(hdev->dev,
1607 if (!hard_reset && !hdev->asic_prop.allow_inference_soft_reset) {
1608 dev_dbg(hdev->dev,
1615 if (from_hard_reset_thread && hdev->process_kill_trial_cnt)
1625 spin_lock(&hdev->reset_info.lock);
1626 if (hdev->reset_info.in_reset) {
1628 if (hard_reset && hdev->reset_info.in_compute_reset)
1629 hdev->reset_info.hard_reset_schedule_flags = flags;
1630 spin_unlock(&hdev->reset_info.lock);
1637 hdev->reset_info.in_compute_reset = !hard_reset;
1639 hdev->reset_info.in_reset = 1;
1641 spin_unlock(&hdev->reset_info.lock);
1647 if ((hard_reset || from_dev_release) && hdev->reset_info.watchdog_active) {
1649 &hdev->device_release_watchdog_work;
1651 hdev->reset_info.watchdog_active = 0;
1656 hdev->reset_info.in_compute_reset = 0;
1667 handle_reset_trigger(hdev, flags);
1668 send_disable_pci_access(hdev, flags);
1671 hdev->disabled = true;
1673 take_release_locks(hdev);
1676 dev_info(hdev->dev, "Going to reset device\n");
1678 dev_dbg(hdev->dev, "Going to reset device after release by user\n");
1680 dev_dbg(hdev->dev, "Going to reset engines of inference device\n");
1684 hdev->reset_info.hard_reset_pending = true;
1686 hdev->process_kill_trial_cnt = 0;
1688 hdev->device_reset_work.flags = flags;
1694 queue_delayed_work(hdev->reset_wq, &hdev->device_reset_work.reset_work, 0);
1699 cleanup_resources(hdev, hard_reset, fw_reset, from_dev_release);
1707 rc = device_kill_open_processes(hdev, 0, false);
1710 if (hdev->device_fini_pending) {
1711 dev_crit(hdev->dev,
1713 dev_name(&(hdev)->pdev->dev));
1722 dev_crit(hdev->dev,
1724 dev_name(&(hdev)->pdev->dev));
1731 flush_workqueue(hdev->eq_wq);
1735 hw_fini_rc = hdev->asic_funcs->hw_fini(hdev, hard_reset, fw_reset);
1738 hdev->fw_loader.fw_comp_loaded = FW_TYPE_NONE;
1741 if (hdev->kernel_ctx && hl_ctx_put(hdev->kernel_ctx) == 1)
1742 hdev->kernel_ctx = NULL;
1744 hl_vm_fini(hdev);
1745 hl_mmu_fini(hdev);
1746 hl_eq_reset(hdev, &hdev->event_queue);
1750 hl_hw_queue_reset(hdev, hard_reset);
1751 for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++)
1752 hl_cq_reset(hdev, &hdev->completion_queue[i]);
1755 ctx = hl_get_compute_ctx(hdev);
1769 hdev->device_cpu_disabled = false;
1770 hdev->reset_info.hard_reset_pending = false;
1776 if (hdev->reset_info.reset_trigger_repeated &&
1777 (hdev->reset_info.prev_reset_trigger == HL_DRV_RESET_FW_FATAL_ERR ||
1778 hdev->reset_info.prev_reset_trigger ==
1780 dev_crit(hdev->dev,
1782 dev_name(&(hdev)->pdev->dev));
1787 if (hdev->kernel_ctx) {
1788 dev_crit(hdev->dev,
1790 dev_name(&(hdev)->pdev->dev));
1795 rc = hl_mmu_init(hdev);
1797 dev_err(hdev->dev,
1803 hdev->kernel_ctx = kzalloc(sizeof(*hdev->kernel_ctx),
1805 if (!hdev->kernel_ctx) {
1807 hl_mmu_fini(hdev);
1811 hdev->is_compute_ctx_active = false;
1813 rc = hl_ctx_init(hdev, hdev->kernel_ctx, true);
1815 dev_err(hdev->dev,
1817 kfree(hdev->kernel_ctx);
1818 hdev->kernel_ctx = NULL;
1819 hl_mmu_fini(hdev);
1828 hdev->disabled = false;
1832 rc = hl_fw_read_preboot_status(hdev);
1837 rc = hdev->asic_funcs->hw_init(hdev);
1839 dev_err(hdev->dev, "failed to initialize the H/W after reset\n");
1844 if (!hdev->asic_funcs->is_device_idle(hdev, idle_mask,
1846 print_idle_status_mask(hdev, "device is not idle after reset", idle_mask);
1852 rc = hdev->asic_funcs->test_queues(hdev);
1854 dev_err(hdev->dev, "Failed to detect if device is alive after reset\n");
1859 rc = device_late_init(hdev);
1861 dev_err(hdev->dev, "Failed late init after hard reset\n");
1865 rc = hl_vm_init(hdev);
1867 dev_err(hdev->dev, "Failed to init memory module after hard reset\n");
1871 if (!hdev->asic_prop.fw_security_enabled)
1872 hl_fw_set_max_power(hdev);
1874 rc = hdev->asic_funcs->compute_reset_late_init(hdev);
1877 dev_err(hdev->dev,
1880 dev_err(hdev->dev, "Failed late init after compute reset\n");
1885 rc = hdev->asic_funcs->scrub_device_mem(hdev);
1887 dev_err(hdev->dev, "scrub mem failed from device reset (%d)\n", rc);
1891 spin_lock(&hdev->reset_info.lock);
1892 hdev->reset_info.in_compute_reset = 0;
1898 if (!hard_reset && hdev->reset_info.hard_reset_schedule_flags)
1901 hdev->reset_info.in_reset = 0;
1903 spin_unlock(&hdev->reset_info.lock);
1905 hdev->reset_info.needs_reset = false;
1908 dev_info(hdev->dev,
1910 dev_name(&(hdev)->pdev->dev));
1912 dev_dbg(hdev->dev,
1914 dev_name(&(hdev)->pdev->dev));
1917 hdev->reset_info.hard_reset_cnt++;
1924 hdev->asic_funcs->enable_events_from_fw(hdev);
1927 hdev->reset_info.compute_reset_cnt++;
1930 dev_info(hdev->dev, "Performing hard reset scheduled during compute reset\n");
1931 flags = hdev->reset_info.hard_reset_schedule_flags;
1932 hdev->reset_info.hard_reset_schedule_flags = 0;
1941 hdev->disabled = true;
1943 spin_lock(&hdev->reset_info.lock);
1944 hdev->reset_info.in_compute_reset = 0;
1947 dev_err(hdev->dev,
1949 dev_name(&(hdev)->pdev->dev));
1950 hdev->reset_info.hard_reset_cnt++;
1953 dev_err(hdev->dev, "Failed to reset device after user release\n");
1956 dev_err(hdev->dev, "Failed to do compute reset\n");
1957 hdev->reset_info.compute_reset_cnt++;
1960 spin_unlock(&hdev->reset_info.lock);
1966 hdev->reset_info.in_reset = 0;
1968 spin_unlock(&hdev->reset_info.lock);
1975 * @hdev: pointer to habanalabs device structure.
1982 int hl_device_cond_reset(struct hl_device *hdev, u32 flags, u64 event_mask)
1992 dev_err(hdev->dev, "Resetting device without a reset indication to user\n");
1996 ctx = hl_get_compute_ctx(hdev);
2006 if (!ctx->hpriv->notifier_event.eventfd && !hdev->reset_info.watchdog_active)
2012 spin_lock(&hdev->reset_info.lock);
2013 if (hdev->reset_info.in_reset) {
2014 spin_unlock(&hdev->reset_info.lock);
2018 if (hdev->reset_info.watchdog_active) {
2019 hdev->device_release_watchdog_work.flags |= flags;
2023 hdev->device_release_watchdog_work.flags = flags;
2024 dev_dbg(hdev->dev, "Device is going to be hard-reset in %u sec unless being released\n",
2025 hdev->device_release_watchdog_timeout_sec);
2026 schedule_delayed_work(&hdev->device_release_watchdog_work.reset_work,
2027 msecs_to_jiffies(hdev->device_release_watchdog_timeout_sec * 1000));
2028 hdev->reset_info.watchdog_active = 1;
2030 spin_unlock(&hdev->reset_info.lock);
2032 hl_notifier_event_send_all(hdev, event_mask);
2036 hl_abort_waiting_for_completions(hdev);
2042 hl_notifier_event_send_all(hdev, event_mask);
2046 return hl_device_reset(hdev, flags | HL_DRV_RESET_HARD);
2063 * @hdev: pointer to habanalabs device structure
2067 void hl_notifier_event_send_all(struct hl_device *hdev, u64 event_mask)
2072 dev_warn(hdev->dev, "Skip sending zero event");
2076 mutex_lock(&hdev->fpriv_list_lock);
2078 list_for_each_entry(hpriv, &hdev->fpriv_list, dev_node)
2081 mutex_unlock(&hdev->fpriv_list_lock);
2087 * @hdev: pointer to habanalabs device structure
2093 int hl_device_init(struct hl_device *hdev)
2101 rc = device_early_init(hdev);
2105 user_interrupt_cnt = hdev->asic_prop.user_dec_intr_count +
2106 hdev->asic_prop.user_interrupt_count;
2109 hdev->user_interrupt = kcalloc(user_interrupt_cnt, sizeof(*hdev->user_interrupt),
2111 if (!hdev->user_interrupt) {
2117 if (hdev->asic_prop.first_available_cq[0] != USHRT_MAX) {
2125 free_jobs_data = &hdev->user_interrupt[i].ts_free_jobs_data;
2133 free_jobs_data = &hdev->common_user_cq_interrupt.ts_free_jobs_data;
2149 rc = hdev->asic_funcs->sw_init(hdev);
2155 hl_multi_cs_completion_init(hdev);
2162 rc = hl_hw_queues_create(hdev);
2164 dev_err(hdev->dev, "failed to initialize kernel queues\n");
2168 cq_cnt = hdev->asic_prop.completion_queues_count;
2176 hdev->completion_queue = kcalloc(cq_cnt,
2177 sizeof(*hdev->completion_queue),
2180 if (!hdev->completion_queue) {
2181 dev_err(hdev->dev,
2189 rc = hl_cq_init(hdev, &hdev->completion_queue[i],
2190 hdev->asic_funcs->get_queue_id_for_cq(hdev, i));
2192 dev_err(hdev->dev,
2196 hdev->completion_queue[i].cq_idx = i;
2199 hdev->shadow_cs_queue = kcalloc(hdev->asic_prop.max_pending_cs,
2201 if (!hdev->shadow_cs_queue) {
2211 rc = hl_eq_init(hdev, &hdev->event_queue);
2213 dev_err(hdev->dev, "failed to initialize event queue\n");
2218 rc = hl_mmu_init(hdev);
2220 dev_err(hdev->dev, "Failed to initialize MMU S/W structures\n");
2225 hdev->kernel_ctx = kzalloc(sizeof(*hdev->kernel_ctx), GFP_KERNEL);
2226 if (!hdev->kernel_ctx) {
2231 hdev->is_compute_ctx_active = false;
2233 hdev->asic_funcs->state_dump_init(hdev);
2235 hdev->device_release_watchdog_timeout_sec = HL_DEVICE_RELEASE_WATCHDOG_TIMEOUT_SEC;
2237 hdev->memory_scrub_val = MEM_SCRUB_DEFAULT_VAL;
2239 rc = hl_debugfs_device_init(hdev);
2241 dev_err(hdev->dev, "failed to initialize debugfs entry structure\n");
2242 kfree(hdev->kernel_ctx);
2249 rc = hl_ctx_init(hdev, hdev->kernel_ctx, true);
2251 dev_err(hdev->dev, "failed to initialize kernel context\n");
2252 kfree(hdev->kernel_ctx);
2256 rc = hl_cb_pool_init(hdev);
2258 dev_err(hdev->dev, "failed to initialize CB pool\n");
2262 rc = hl_dec_init(hdev);
2264 dev_err(hdev->dev, "Failed to initialize the decoder module\n");
2278 hdev->disabled = false;
2280 rc = hdev->asic_funcs->hw_init(hdev);
2282 dev_err(hdev->dev, "failed to initialize the H/W\n");
2288 rc = hdev->asic_funcs->test_queues(hdev);
2290 dev_err(hdev->dev, "Failed to detect if device is alive\n");
2295 rc = device_late_init(hdev);
2297 dev_err(hdev->dev, "Failed late initialization\n");
2302 dev_info(hdev->dev, "Found %s device with %lluGB DRAM\n",
2303 hdev->asic_name,
2304 hdev->asic_prop.dram_size / SZ_1G);
2306 rc = hl_vm_init(hdev);
2308 dev_err(hdev->dev, "Failed to initialize memory module\n");
2319 rc = drm_dev_register(&hdev->drm, 0);
2321 dev_err(hdev->dev, "Failed to register DRM device, rc %d\n", rc);
2326 rc = cdev_sysfs_debugfs_add(hdev);
2328 dev_err(hdev->dev, "Failed to add char devices and sysfs/debugfs files\n");
2336 if (hdev->asic_prop.set_max_power_on_device_init &&
2337 !hdev->asic_prop.fw_security_enabled)
2338 hl_fw_set_max_power(hdev);
2346 rc = hl_hwmon_init(hdev);
2348 dev_err(hdev->dev, "Failed to initialize hwmon\n");
2353 dev_notice(hdev->dev,
2355 dev_name(&(hdev)->pdev->dev));
2362 hdev->asic_funcs->enable_events_from_fw(hdev);
2364 hdev->init_done = true;
2369 hl_cb_pool_fini(hdev);
2371 if (hl_ctx_put(hdev->kernel_ctx) != 1)
2372 dev_err(hdev->dev,
2375 hl_debugfs_device_fini(hdev);
2377 hl_mmu_fini(hdev);
2379 hl_eq_fini(hdev, &hdev->event_queue);
2381 kfree(hdev->shadow_cs_queue);
2384 hl_cq_fini(hdev, &hdev->completion_queue[i]);
2385 kfree(hdev->completion_queue);
2387 hl_hw_queues_destroy(hdev);
2389 hdev->asic_funcs->sw_fini(hdev);
2391 vfree(hdev->common_user_cq_interrupt.ts_free_jobs_data.free_nodes_pool);
2395 if (!hdev->user_interrupt[i].ts_free_jobs_data.free_nodes_pool)
2397 vfree(hdev->user_interrupt[i].ts_free_jobs_data.free_nodes_pool);
2399 kfree(hdev->user_interrupt);
2402 device_early_fini(hdev);
2404 hdev->disabled = true;
2406 drm_dev_register(&hdev->drm, 0);
2407 cdev_sysfs_debugfs_add(hdev);
2411 hdev->cdev_idx, dev_name(&hdev->pdev->dev));
2419 * @hdev: pointer to habanalabs device structure
2423 void hl_device_fini(struct hl_device *hdev)
2431 dev_info(hdev->dev, "Removing device %s\n", dev_name(&(hdev)->pdev->dev));
2433 hdev->device_fini_pending = 1;
2434 flush_delayed_work(&hdev->device_reset_work.reset_work);
2436 if (hdev->pldm)
2451 spin_lock(&hdev->reset_info.lock);
2452 device_in_reset = !!hdev->reset_info.in_reset;
2454 hdev->reset_info.in_reset = 1;
2455 spin_unlock(&hdev->reset_info.lock);
2460 spin_lock(&hdev->reset_info.lock);
2461 device_in_reset = !!hdev->reset_info.in_reset;
2463 hdev->reset_info.in_reset = 1;
2464 spin_unlock(&hdev->reset_info.lock);
2467 dev_crit(hdev->dev,
2469 dev_name(&(hdev)->pdev->dev));
2474 cancel_delayed_work_sync(&hdev->device_release_watchdog_work.reset_work);
2483 hl_fw_send_pci_access_msg(hdev, CPUCP_PACKET_DISABLE_PCI_ACCESS, 0x0);
2486 hdev->disabled = true;
2488 take_release_locks(hdev);
2490 hdev->reset_info.hard_reset_pending = true;
2492 hl_hwmon_fini(hdev);
2494 cleanup_resources(hdev, true, false, false);
2500 dev_info(hdev->dev,
2504 hdev->process_kill_trial_cnt = 0;
2505 rc = device_kill_open_processes(hdev, HL_WAIT_PROCESS_KILL_ON_DEVICE_FINI, false);
2507 dev_crit(hdev->dev, "Failed to kill all open processes (%d)\n", rc);
2508 device_disable_open_processes(hdev, false);
2511 hdev->process_kill_trial_cnt = 0;
2512 rc = device_kill_open_processes(hdev, 0, true);
2514 dev_crit(hdev->dev, "Failed to kill all control device open processes (%d)\n", rc);
2515 device_disable_open_processes(hdev, true);
2518 hl_cb_pool_fini(hdev);
2521 rc = hdev->asic_funcs->hw_fini(hdev, true, false);
2523 dev_err(hdev->dev, "hw_fini failed in device fini while removing device %d\n", rc);
2525 hdev->fw_loader.fw_comp_loaded = FW_TYPE_NONE;
2528 if ((hdev->kernel_ctx) && (hl_ctx_put(hdev->kernel_ctx) != 1))
2529 dev_err(hdev->dev, "kernel ctx is still alive\n");
2531 hl_dec_fini(hdev);
2533 hl_vm_fini(hdev);
2535 hl_mmu_fini(hdev);
2537 vfree(hdev->captured_err_info.page_fault_info.user_mappings);
2539 hl_eq_fini(hdev, &hdev->event_queue);
2541 kfree(hdev->shadow_cs_queue);
2543 for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++)
2544 hl_cq_fini(hdev, &hdev->completion_queue[i]);
2545 kfree(hdev->completion_queue);
2547 user_interrupt_cnt = hdev->asic_prop.user_dec_intr_count +
2548 hdev->asic_prop.user_interrupt_count;
2551 if (hdev->asic_prop.first_available_cq[0] != USHRT_MAX) {
2553 vfree(hdev->user_interrupt[i].ts_free_jobs_data.free_nodes_pool);
2556 kfree(hdev->user_interrupt);
2559 vfree(hdev->common_user_cq_interrupt.ts_free_jobs_data.free_nodes_pool);
2561 hl_hw_queues_destroy(hdev);
2564 hdev->asic_funcs->sw_fini(hdev);
2566 device_early_fini(hdev);
2569 cdev_sysfs_debugfs_remove(hdev);
2570 drm_dev_unregister(&hdev->drm);
2572 hl_debugfs_device_fini(hdev);
2584 * @hdev: pointer to habanalabs device structure
2590 inline u32 hl_rreg(struct hl_device *hdev, u32 reg)
2592 u32 val = readl(hdev->rmmio + reg);
2595 trace_habanalabs_rreg32(hdev->dev, reg, val);
2603 * @hdev: pointer to habanalabs device structure
2610 inline void hl_wreg(struct hl_device *hdev, u32 reg, u32 val)
2613 trace_habanalabs_wreg32(hdev->dev, reg, val);
2615 writel(val, hdev->rmmio + reg);
2618 void hl_capture_razwi(struct hl_device *hdev, u64 addr, u16 *engine_id, u16 num_of_engines,
2621 struct razwi_info *razwi_info = &hdev->captured_err_info.razwi_info;
2624 dev_err(hdev->dev,
2631 if (atomic_cmpxchg(&hdev->captured_err_info.razwi_info.razwi_detected, 0, 1))
2644 void hl_handle_razwi(struct hl_device *hdev, u64 addr, u16 *engine_id, u16 num_of_engines,
2647 hl_capture_razwi(hdev, addr, engine_id, num_of_engines, flags);
2653 static void hl_capture_user_mappings(struct hl_device *hdev, bool is_pmmu)
2655 struct page_fault_info *pgf_info = &hdev->captured_err_info.page_fault_info;
2667 ctx = hl_get_compute_ctx(hdev);
2669 dev_err(hdev->dev, "Can't get user context for user mappings\n");
2715 void hl_capture_page_fault(struct hl_device *hdev, u64 addr, u16 eng_id, bool is_pmmu)
2717 struct page_fault_info *pgf_info = &hdev->captured_err_info.page_fault_info;
2726 hl_capture_user_mappings(hdev, is_pmmu);
2731 void hl_handle_page_fault(struct hl_device *hdev, u64 addr, u16 eng_id, bool is_pmmu,
2734 hl_capture_page_fault(hdev, addr, eng_id, is_pmmu);
2740 static void hl_capture_hw_err(struct hl_device *hdev, u16 event_id)
2742 struct hw_err_info *info = &hdev->captured_err_info.hw_err;
2754 void hl_handle_critical_hw_err(struct hl_device *hdev, u16 event_id, u64 *event_mask)
2756 hl_capture_hw_err(hdev, event_id);
2762 static void hl_capture_fw_err(struct hl_device *hdev, struct hl_info_fw_err_info *fw_info)
2764 struct fw_err_info *info = &hdev->captured_err_info.fw_err;
2778 void hl_handle_fw_err(struct hl_device *hdev, struct hl_info_fw_err_info *info)
2780 hl_capture_fw_err(hdev, info);
2786 void hl_capture_engine_err(struct hl_device *hdev, u16 engine_id, u16 error_count)
2788 struct engine_err_info *info = &hdev->captured_err_info.engine_err;
2808 void hl_init_cpu_for_irq(struct hl_device *hdev)
2811 struct cpumask *available_mask = &hdev->irq_affinity_mask;
2812 int numa_node = hdev->pdev->dev.numa_node, i;
2819 dev_err(hdev->dev, "No available affinities in current numa node\n");
2829 void hl_set_irq_affinity(struct hl_device *hdev, int irq)
2831 if (cpumask_empty(&hdev->irq_affinity_mask)) {
2832 dev_dbg(hdev->dev, "affinity mask is empty\n");
2836 if (irq_set_affinity_and_hint(irq, &hdev->irq_affinity_mask))
2837 dev_err(hdev->dev, "Failed setting irq %d affinity\n", irq);