Lines Matching defs:ipc_imem
25 static int ipc_imem_msg_send_device_sleep(struct iosm_imem *ipc_imem, u32 state)
32 ipc_imem->device_sleep = state;
34 return ipc_protocol_tq_msg_send(ipc_imem->ipc_protocol,
38 static bool ipc_imem_dl_skb_alloc(struct iosm_imem *ipc_imem,
45 return ipc_protocol_dl_td_prepare(ipc_imem->ipc_protocol, pipe);
51 static int ipc_imem_tq_td_alloc_timer(struct iosm_imem *ipc_imem, int arg,
59 struct ipc_pipe *pipe = &ipc_imem->channels[i].dl_pipe;
64 while (ipc_imem_dl_skb_alloc(ipc_imem, pipe))
72 ipc_protocol_doorbell_trigger(ipc_imem->ipc_protocol,
76 ipc_imem->hrtimer_period =
78 if (!hrtimer_active(&ipc_imem->td_alloc_timer))
79 hrtimer_start(&ipc_imem->td_alloc_timer,
80 ipc_imem->hrtimer_period,
88 struct iosm_imem *ipc_imem =
91 ipc_task_queue_send_task(ipc_imem, ipc_imem_tq_td_alloc_timer, 0, NULL,
97 static int ipc_imem_tq_fast_update_timer_cb(struct iosm_imem *ipc_imem, int arg,
100 ipc_protocol_doorbell_trigger(ipc_imem->ipc_protocol,
109 struct iosm_imem *ipc_imem =
112 ipc_task_queue_send_task(ipc_imem, ipc_imem_tq_fast_update_timer_cb, 0,
117 static int ipc_imem_tq_adb_timer_cb(struct iosm_imem *ipc_imem, int arg,
120 ipc_mux_ul_adb_finish(ipc_imem->mux);
127 struct iosm_imem *ipc_imem =
130 ipc_task_queue_send_task(ipc_imem, ipc_imem_tq_adb_timer_cb, 0,
135 static int ipc_imem_setup_cp_mux_cap_init(struct iosm_imem *ipc_imem,
138 ipc_mmio_update_cp_capability(ipc_imem->mmio);
140 if (ipc_imem->mmio->mux_protocol == MUX_UNKNOWN) {
141 dev_err(ipc_imem->dev, "Failed to get Mux capability.");
145 cfg->protocol = ipc_imem->mmio->mux_protocol;
147 cfg->ul_flow = (ipc_imem->mmio->has_ul_flow_credit == 1) ?
159 void ipc_imem_msg_send_feature_set(struct iosm_imem *ipc_imem,
166 ipc_protocol_tq_msg_send(ipc_imem->ipc_protocol,
170 ipc_protocol_msg_send(ipc_imem->ipc_protocol,
176 * @ipc_imem: Pointer to imem data-struct
178 void ipc_imem_td_update_timer_start(struct iosm_imem *ipc_imem)
181 if (!ipc_imem->enter_runtime || ipc_imem->td_update_timer_suspended) {
183 ipc_protocol_doorbell_trigger(ipc_imem->ipc_protocol,
188 if (!hrtimer_active(&ipc_imem->tdupdate_timer)) {
189 ipc_imem->hrtimer_period =
191 if (!hrtimer_active(&ipc_imem->tdupdate_timer))
192 hrtimer_start(&ipc_imem->tdupdate_timer,
193 ipc_imem->hrtimer_period,
206 * @ipc_imem: Pointer to imem data-struct
208 void ipc_imem_adb_timer_start(struct iosm_imem *ipc_imem)
210 if (!hrtimer_active(&ipc_imem->adb_timer)) {
211 ipc_imem->hrtimer_period =
213 hrtimer_start(&ipc_imem->adb_timer,
214 ipc_imem->hrtimer_period,
219 bool ipc_imem_ul_write_td(struct iosm_imem *ipc_imem)
229 for (i = 0; i < ipc_imem->nr_of_channels; i++) {
230 channel = &ipc_imem->channels[i];
243 ipc_protocol_ul_td_send(ipc_imem->ipc_protocol,
247 ipc_protocol_ul_td_send(ipc_imem->ipc_protocol,
255 ipc_protocol_doorbell_trigger(ipc_imem->ipc_protocol,
262 void ipc_imem_ipc_init_check(struct iosm_imem *ipc_imem)
266 ipc_imem->ipc_requested_state = IPC_MEM_DEVICE_IPC_INIT;
269 ipc_doorbell_fire(ipc_imem->pcie, IPC_DOORBELL_IRQ_IPC,
273 if (ipc_mmio_get_ipc_state(ipc_imem->mmio) ==
274 ipc_imem->ipc_requested_state) {
276 ipc_mmio_config(ipc_imem->mmio);
279 ipc_imem->ipc_requested_state =
281 ipc_doorbell_fire(ipc_imem->pcie, IPC_DOORBELL_IRQ_IPC,
290 dev_err(ipc_imem->dev, "%s: ipc_status(%d) ne. IPC_MEM_DEVICE_IPC_INIT",
291 ipc_imem_phase_get_string(ipc_imem->phase),
292 ipc_mmio_get_ipc_state(ipc_imem->mmio));
294 ipc_uevent_send(ipc_imem->dev, UEVENT_MDM_TIMEOUT);
298 static void ipc_imem_dl_skb_process(struct iosm_imem *ipc_imem,
310 ipc_pcie_addr_unmap(ipc_imem->pcie, IPC_CB(skb)->len,
314 ipc_imem_sys_devlink_notify_rx(ipc_imem->ipc_devlink,
316 else if (ipc_is_trace_channel(ipc_imem, port_id))
317 ipc_trace_port_rx(ipc_imem, skb);
319 wwan_port_rx(ipc_imem->ipc_port[port_id]->iosm_port,
325 ipc_mux_dl_decode(ipc_imem->mux, skb);
328 dev_err(ipc_imem->dev, "Invalid channel type");
334 static void ipc_imem_dl_pipe_process(struct iosm_imem *ipc_imem,
345 ipc_protocol_get_head_tail_index(ipc_imem->ipc_protocol, pipe, &head,
358 skb = ipc_protocol_dl_td_process(ipc_imem->ipc_protocol, pipe);
361 ipc_imem_dl_skb_process(ipc_imem, pipe, skb);
365 while (ipc_imem_dl_skb_alloc(ipc_imem, pipe))
370 ipc_protocol_doorbell_trigger(ipc_imem->ipc_protocol,
375 ipc_imem_hrtimer_stop(&ipc_imem->fast_update_timer);
383 ipc_imem->hrtimer_period =
385 hrtimer_start(&ipc_imem->fast_update_timer,
386 ipc_imem->hrtimer_period, HRTIMER_MODE_REL);
389 if (ipc_imem->app_notify_dl_pend)
390 complete(&ipc_imem->dl_pend_sem);
394 static void ipc_imem_ul_pipe_process(struct iosm_imem *ipc_imem,
405 ipc_protocol_get_head_tail_index(ipc_imem->ipc_protocol, pipe, &head,
417 skb = ipc_protocol_ul_td_process(ipc_imem->ipc_protocol, pipe);
431 ipc_mux_ul_encoded_process(ipc_imem->mux, skb);
433 dev_err(ipc_imem->dev,
437 ipc_pcie_kfree_skb(ipc_imem->pcie, skb);
443 ipc_mux_check_n_restart_tx(ipc_imem->mux);
445 if (ipc_imem->app_notify_ul_pend)
446 complete(&ipc_imem->ul_pend_sem);
450 static void ipc_imem_rom_irq_exec(struct iosm_imem *ipc_imem)
454 channel = ipc_imem->ipc_devlink->devlink_sio.channel;
455 ipc_imem->rom_exit_code = ipc_mmio_get_rom_exit_code(ipc_imem->mmio);
460 static int ipc_imem_tq_td_update_timer_cb(struct iosm_imem *ipc_imem, int arg,
463 ipc_protocol_doorbell_trigger(ipc_imem->ipc_protocol,
469 static void ipc_imem_slp_control_exec(struct iosm_imem *ipc_imem)
472 if (ipc_protocol_pm_dev_sleep_handle(ipc_imem->ipc_protocol) &&
473 hrtimer_active(&ipc_imem->tdupdate_timer)) {
475 ipc_imem_tq_td_update_timer_cb(ipc_imem, 0, NULL, 0);
477 ipc_imem_hrtimer_stop(&ipc_imem->tdupdate_timer);
479 ipc_imem_hrtimer_stop(&ipc_imem->fast_update_timer);
484 static int ipc_imem_tq_startup_timer_cb(struct iosm_imem *ipc_imem, int arg,
488 if (ipc_imem_phase_update(ipc_imem) != IPC_P_RUN)
491 if (ipc_mmio_get_ipc_state(ipc_imem->mmio) ==
493 ipc_imem->ipc_requested_state = IPC_MEM_DEVICE_IPC_INIT;
495 ipc_doorbell_fire(ipc_imem->pcie, IPC_DOORBELL_IRQ_IPC,
498 ipc_imem->hrtimer_period = ktime_set(0, 100 * 1000UL * 1000ULL);
500 if (!hrtimer_active(&ipc_imem->startup_timer))
501 hrtimer_start(&ipc_imem->startup_timer,
502 ipc_imem->hrtimer_period,
504 } else if (ipc_mmio_get_ipc_state(ipc_imem->mmio) ==
507 ipc_imem_hrtimer_stop(&ipc_imem->startup_timer);
510 ipc_mmio_config(ipc_imem->mmio);
511 ipc_imem->ipc_requested_state = IPC_MEM_DEVICE_IPC_RUNNING;
512 ipc_doorbell_fire(ipc_imem->pcie, IPC_DOORBELL_IRQ_IPC,
522 struct iosm_imem *ipc_imem =
525 if (ktime_to_ns(ipc_imem->hrtimer_period)) {
526 hrtimer_forward_now(&ipc_imem->startup_timer,
527 ipc_imem->hrtimer_period);
531 ipc_task_queue_send_task(ipc_imem, ipc_imem_tq_startup_timer_cb, 0,
538 ipc_imem_get_exec_stage_buffered(struct iosm_imem *ipc_imem)
540 return (ipc_imem->phase == IPC_P_RUN &&
541 ipc_imem->ipc_status == IPC_MEM_DEVICE_IPC_RUNNING) ?
542 ipc_protocol_get_ap_exec_stage(ipc_imem->ipc_protocol) :
543 ipc_mmio_get_exec_stage(ipc_imem->mmio);
547 static int ipc_imem_send_mdm_rdy_cb(struct iosm_imem *ipc_imem, int arg,
551 ipc_imem_get_exec_stage_buffered(ipc_imem);
554 ipc_uevent_send(ipc_imem->dev, UEVENT_MDM_READY);
566 struct iosm_imem *ipc_imem;
570 ipc_imem = container_of(instance, struct iosm_imem, run_state_worker);
572 if (ipc_imem->phase != IPC_P_RUN) {
573 dev_err(ipc_imem->dev,
578 if (test_and_clear_bit(IOSM_DEVLINK_INIT, &ipc_imem->flag))
579 ipc_devlink_deinit(ipc_imem->ipc_devlink);
581 ret = ipc_imem_setup_cp_mux_cap_init(ipc_imem, &mux_cfg);
585 ipc_imem->mux = ipc_mux_init(&mux_cfg, ipc_imem);
586 if (!ipc_imem->mux)
589 ret = ipc_imem_wwan_channel_init(ipc_imem, mux_cfg.protocol);
593 ipc_imem->mux->wwan = ipc_imem->wwan;
597 ipc_imem->ipc_port[ctrl_chl_idx] = NULL;
599 if (ipc_imem->pcie->pci->device == INTEL_CP_DEVICE_7560_ID &&
605 if (ipc_imem->pcie->pci->device == INTEL_CP_DEVICE_7360_ID &&
611 ipc_imem_channel_init(ipc_imem, IPC_CTYPE_CTRL,
614 ipc_imem->ipc_port[ctrl_chl_idx] =
615 ipc_port_init(ipc_imem, chnl_cfg_port);
621 ipc_debugfs_init(ipc_imem);
623 ipc_task_queue_send_task(ipc_imem, ipc_imem_send_mdm_rdy_cb, 0, NULL, 0,
629 set_bit(FULLY_FUNCTIONAL, &ipc_imem->flag);
637 ipc_mux_deinit(ipc_imem->mux);
639 ipc_uevent_send(ipc_imem->dev, UEVENT_CD_READY_LINK_DOWN);
642 static void ipc_imem_handle_irq(struct iosm_imem *ipc_imem, int irq)
651 ipc_imem->ev_irq_pending[irq] = false;
654 old_phase = ipc_imem->phase;
657 dev_dbg(ipc_imem->dev,
664 phase = ipc_imem_phase_update(ipc_imem);
668 if (!ipc_imem->enter_runtime) {
670 ipc_imem->enter_runtime = 1;
675 ipc_imem_msg_send_device_sleep(ipc_imem,
676 ipc_imem->device_sleep);
678 ipc_imem_msg_send_feature_set(ipc_imem,
684 ipc_protocol_get_ipc_status(ipc_imem->ipc_protocol);
687 if (ipc_imem->ipc_status != curr_ipc_status) {
688 ipc_imem->ipc_status = curr_ipc_status;
690 if (ipc_imem->ipc_status ==
692 schedule_work(&ipc_imem->run_state_worker);
697 ipc_imem_slp_control_exec(ipc_imem);
703 dev_err(ipc_imem->dev, "confused phase %s",
719 ipc_imem_rom_irq_exec(ipc_imem);
727 ipc_protocol_msg_process(ipc_imem, irq);
731 struct ipc_pipe *ul_pipe = &ipc_imem->channels[i].ul_pipe;
732 struct ipc_pipe *dl_pipe = &ipc_imem->channels[i].dl_pipe;
736 ipc_imem_dl_pipe_process(ipc_imem, dl_pipe);
743 ipc_imem_ul_pipe_process(ipc_imem, ul_pipe);
747 if (ipc_mux_ul_data_encode(ipc_imem->mux)) {
748 ipc_imem_td_update_timer_start(ipc_imem);
749 if (ipc_imem->mux->protocol == MUX_AGGREGATION)
750 ipc_imem_adb_timer_start(ipc_imem);
756 ul_pending |= ipc_imem_ul_write_td(ipc_imem);
760 ipc_imem->hrtimer_period =
762 if (!hrtimer_active(&ipc_imem->tdupdate_timer))
763 hrtimer_start(&ipc_imem->tdupdate_timer,
764 ipc_imem->hrtimer_period,
773 ipc_imem->ipc_requested_state == IPC_MEM_DEVICE_IPC_RUNNING &&
774 ipc_mmio_get_ipc_state(ipc_imem->mmio) ==
776 complete(&ipc_imem->ipc_devlink->devlink_sio.channel->ul_sem);
780 ipc_imem->ipc_requested_state = IPC_MEM_DEVICE_IPC_DONT_CARE;
783 ipc_imem->hrtimer_period =
785 if (!hrtimer_active(&ipc_imem->td_alloc_timer))
786 hrtimer_start(&ipc_imem->td_alloc_timer,
787 ipc_imem->hrtimer_period,
793 static int ipc_imem_tq_irq_cb(struct iosm_imem *ipc_imem, int arg, void *msg,
796 ipc_imem_handle_irq(ipc_imem, arg);
801 void ipc_imem_ul_send(struct iosm_imem *ipc_imem)
804 if (ipc_imem_ul_write_td(ipc_imem))
805 ipc_imem_td_update_timer_start(ipc_imem);
809 static enum ipc_phase ipc_imem_phase_update_check(struct iosm_imem *ipc_imem,
814 if (ipc_imem->phase != IPC_P_ROM) {
816 ipc_uevent_send(ipc_imem->dev, UEVENT_ROM_READY);
819 ipc_imem->phase = IPC_P_ROM;
823 ipc_imem->phase = IPC_P_PSI;
827 ipc_imem->phase = IPC_P_EBL;
831 if (ipc_imem->phase != IPC_P_RUN &&
832 ipc_imem->ipc_status == IPC_MEM_DEVICE_IPC_RUNNING) {
833 ipc_uevent_send(ipc_imem->dev, UEVENT_MDM_READY);
835 ipc_imem->phase = IPC_P_RUN;
839 if (ipc_imem->phase != IPC_P_CRASH)
840 ipc_uevent_send(ipc_imem->dev, UEVENT_CRASH);
842 ipc_imem->phase = IPC_P_CRASH;
846 if (ipc_imem->phase != IPC_P_CD_READY)
847 ipc_uevent_send(ipc_imem->dev, UEVENT_CD_READY);
848 ipc_imem->phase = IPC_P_CD_READY;
855 ipc_uevent_send(ipc_imem->dev, UEVENT_CD_READY_LINK_DOWN);
859 return ipc_imem->phase;
863 static bool ipc_imem_pipe_open(struct iosm_imem *ipc_imem,
870 if (ipc_protocol_msg_send(ipc_imem->ipc_protocol,
878 static int ipc_imem_tq_pipe_td_alloc(struct iosm_imem *ipc_imem, int arg,
886 processed |= ipc_imem_dl_skb_alloc(ipc_imem, dl_pipe);
892 ipc_protocol_doorbell_trigger(ipc_imem->ipc_protocol, arg);
900 struct iosm_imem *ipc_imem =
903 ipc_task_queue_send_task(ipc_imem, ipc_imem_tq_td_update_timer_cb, 0,
909 enum ipc_phase ipc_imem_phase_update(struct iosm_imem *ipc_imem)
912 ipc_imem_get_exec_stage_buffered(ipc_imem);
914 return ipc_imem->phase == IPC_P_OFF_REQ ?
915 ipc_imem->phase :
916 ipc_imem_phase_update_check(ipc_imem, exec_stage);
951 void ipc_imem_pipe_close(struct iosm_imem *ipc_imem, struct ipc_pipe *pipe)
956 ipc_protocol_msg_send(ipc_imem->ipc_protocol, IPC_MSG_PREP_PIPE_CLOSE,
959 ipc_imem_pipe_cleanup(ipc_imem, pipe);
962 void ipc_imem_channel_close(struct iosm_imem *ipc_imem, int channel_id)
966 if (channel_id < 0 || channel_id >= ipc_imem->nr_of_channels) {
967 dev_err(ipc_imem->dev, "invalid channel id %d", channel_id);
971 channel = &ipc_imem->channels[channel_id];
974 dev_err(ipc_imem->dev, "ch[%d]: invalid channel state %d",
984 if (ipc_imem->phase == IPC_P_RUN) {
985 ipc_imem_pipe_close(ipc_imem, &channel->ul_pipe);
986 ipc_imem_pipe_close(ipc_imem, &channel->dl_pipe);
989 ipc_imem_pipe_cleanup(ipc_imem, &channel->ul_pipe);
990 ipc_imem_pipe_cleanup(ipc_imem, &channel->dl_pipe);
996 struct ipc_mem_channel *ipc_imem_channel_open(struct iosm_imem *ipc_imem,
1002 dev_err(ipc_imem->dev, "invalid channel ID: %d", channel_id);
1006 channel = &ipc_imem->channels[channel_id];
1010 if (!ipc_imem_pipe_open(ipc_imem, &channel->ul_pipe))
1013 if (!ipc_imem_pipe_open(ipc_imem, &channel->dl_pipe))
1017 if (ipc_task_queue_send_task(ipc_imem, ipc_imem_tq_pipe_td_alloc, db_id,
1019 dev_err(ipc_imem->dev, "td allocation failed : %d", channel_id);
1026 ipc_imem_pipe_close(ipc_imem, &channel->dl_pipe);
1028 ipc_imem_pipe_close(ipc_imem, &channel->ul_pipe);
1034 void ipc_imem_pm_suspend(struct iosm_imem *ipc_imem)
1036 ipc_protocol_suspend(ipc_imem->ipc_protocol);
1039 void ipc_imem_pm_s2idle_sleep(struct iosm_imem *ipc_imem, bool sleep)
1041 ipc_protocol_s2idle_sleep(ipc_imem->ipc_protocol, sleep);
1044 void ipc_imem_pm_resume(struct iosm_imem *ipc_imem)
1048 if (ipc_protocol_resume(ipc_imem->ipc_protocol)) {
1049 stage = ipc_mmio_get_exec_stage(ipc_imem->mmio);
1050 ipc_imem_phase_update_check(ipc_imem, stage);
1060 int ipc_imem_channel_alloc(struct iosm_imem *ipc_imem, int index,
1067 for (i = 0; i < ipc_imem->nr_of_channels; i++) {
1068 channel = &ipc_imem->channels[i];
1073 if (i >= ipc_imem->nr_of_channels) {
1074 dev_dbg(ipc_imem->dev,
1080 if (ipc_imem->channels[i].state != IMEM_CHANNEL_FREE) {
1081 dev_dbg(ipc_imem->dev, "channel is in use");
1095 void ipc_imem_channel_init(struct iosm_imem *ipc_imem, enum ipc_ctype ctype,
1102 dev_err(ipc_imem->dev, "invalid pipe: ul_pipe=%d, dl_pipe=%d",
1107 if (ipc_imem->nr_of_channels >= IPC_MEM_MAX_CHANNELS) {
1108 dev_err(ipc_imem->dev, "too many channels");
1112 channel = &ipc_imem->channels[ipc_imem->nr_of_channels];
1113 channel->channel_id = ipc_imem->nr_of_channels;
1118 ipc_imem->nr_of_channels++;
1120 ipc_imem_channel_update(ipc_imem, channel->channel_id, chnl_cfg,
1128 void ipc_imem_channel_update(struct iosm_imem *ipc_imem, int id,
1133 if (id < 0 || id >= ipc_imem->nr_of_channels) {
1134 dev_err(ipc_imem->dev, "invalid channel id %d", id);
1138 channel = &ipc_imem->channels[id];
1142 dev_err(ipc_imem->dev, "invalid channel state %d",
1168 static void ipc_imem_channel_reset(struct iosm_imem *ipc_imem)
1172 for (i = 0; i < ipc_imem->nr_of_channels; i++) {
1175 channel = &ipc_imem->channels[i];
1177 ipc_imem_pipe_cleanup(ipc_imem, &channel->dl_pipe);
1178 ipc_imem_pipe_cleanup(ipc_imem, &channel->ul_pipe);
1184 void ipc_imem_pipe_cleanup(struct iosm_imem *ipc_imem, struct ipc_pipe *pipe)
1195 ipc_pcie_kfree_skb(ipc_imem->pcie, skb);
1197 ipc_protocol_pipe_cleanup(ipc_imem->ipc_protocol, pipe);
1201 static void ipc_imem_device_ipc_uninit(struct iosm_imem *ipc_imem)
1209 if (ipc_pcie_check_data_link_active(ipc_imem->pcie)) {
1214 ipc_doorbell_fire(ipc_imem->pcie, IPC_DOORBELL_IRQ_IPC,
1216 ipc_state = ipc_mmio_get_ipc_state(ipc_imem->mmio);
1226 ipc_state = ipc_mmio_get_ipc_state(ipc_imem->mmio);
1231 void ipc_imem_cleanup(struct iosm_imem *ipc_imem)
1233 ipc_imem->phase = IPC_P_OFF_REQ;
1236 ipc_uevent_send(ipc_imem->dev, UEVENT_MDM_NOT_READY);
1238 hrtimer_cancel(&ipc_imem->td_alloc_timer);
1239 hrtimer_cancel(&ipc_imem->tdupdate_timer);
1240 hrtimer_cancel(&ipc_imem->fast_update_timer);
1241 hrtimer_cancel(&ipc_imem->startup_timer);
1244 cancel_work_sync(&ipc_imem->run_state_worker);
1246 if (test_and_clear_bit(FULLY_FUNCTIONAL, &ipc_imem->flag)) {
1247 ipc_mux_deinit(ipc_imem->mux);
1248 ipc_debugfs_deinit(ipc_imem);
1249 ipc_wwan_deinit(ipc_imem->wwan);
1250 ipc_port_deinit(ipc_imem->ipc_port);
1253 if (test_and_clear_bit(IOSM_DEVLINK_INIT, &ipc_imem->flag))
1254 ipc_devlink_deinit(ipc_imem->ipc_devlink);
1256 ipc_imem_device_ipc_uninit(ipc_imem);
1257 ipc_imem_channel_reset(ipc_imem);
1259 ipc_protocol_deinit(ipc_imem->ipc_protocol);
1260 ipc_task_deinit(ipc_imem->ipc_task);
1262 kfree(ipc_imem->ipc_task);
1263 kfree(ipc_imem->mmio);
1265 ipc_imem->phase = IPC_P_OFF;
1273 static int ipc_imem_config(struct iosm_imem *ipc_imem)
1278 init_completion(&ipc_imem->ul_pend_sem);
1280 init_completion(&ipc_imem->dl_pend_sem);
1283 ipc_imem->ipc_status = IPC_MEM_DEVICE_IPC_UNINIT;
1284 ipc_imem->enter_runtime = 0;
1286 phase = ipc_imem_phase_update(ipc_imem);
1291 ipc_imem->hrtimer_period = ktime_set(0, 1000 * 1000 * 1000ULL);
1293 if (!hrtimer_active(&ipc_imem->startup_timer))
1294 hrtimer_start(&ipc_imem->startup_timer,
1295 ipc_imem->hrtimer_period,
1303 ipc_imem->ipc_requested_state = IPC_MEM_DEVICE_IPC_UNINIT;
1306 if (ipc_imem->ipc_requested_state ==
1307 ipc_mmio_get_ipc_state(ipc_imem->mmio)) {
1308 ipc_imem_ipc_init_check(ipc_imem);
1312 dev_err(ipc_imem->dev,
1314 ipc_mmio_get_ipc_state(ipc_imem->mmio));
1318 dev_dbg(ipc_imem->dev,
1323 dev_err(ipc_imem->dev, "unexpected operation phase %d", phase);
1327 complete(&ipc_imem->dl_pend_sem);
1328 complete(&ipc_imem->ul_pend_sem);
1329 ipc_imem->phase = IPC_P_OFF;
1337 struct iosm_imem *ipc_imem = kzalloc(sizeof(*pcie->imem), GFP_KERNEL);
1340 if (!ipc_imem)
1344 ipc_imem->pcie = pcie;
1345 ipc_imem->dev = dev;
1347 ipc_imem->pci_device_id = device_id;
1349 ipc_imem->cp_version = 0;
1350 ipc_imem->device_sleep = IPC_HOST_SLEEP_ENTER_SLEEP;
1353 ipc_imem->nr_of_channels = 0;
1356 ipc_imem->mmio = ipc_mmio_init(mmio, ipc_imem->dev);
1357 if (!ipc_imem->mmio) {
1358 dev_err(ipc_imem->dev, "failed to initialize mmio region");
1362 ipc_imem->ipc_task = kzalloc(sizeof(*ipc_imem->ipc_task),
1366 if (!ipc_imem->ipc_task)
1369 if (ipc_task_init(ipc_imem->ipc_task))
1372 ipc_imem->ipc_task->dev = ipc_imem->dev;
1374 INIT_WORK(&ipc_imem->run_state_worker, ipc_imem_run_state_worker);
1376 ipc_imem->ipc_protocol = ipc_protocol_init(ipc_imem);
1378 if (!ipc_imem->ipc_protocol)
1382 ipc_imem->phase = IPC_P_OFF;
1384 hrtimer_init(&ipc_imem->startup_timer, CLOCK_MONOTONIC,
1386 ipc_imem->startup_timer.function = ipc_imem_startup_timer_cb;
1388 hrtimer_init(&ipc_imem->tdupdate_timer, CLOCK_MONOTONIC,
1390 ipc_imem->tdupdate_timer.function = ipc_imem_td_update_timer_cb;
1392 hrtimer_init(&ipc_imem->fast_update_timer, CLOCK_MONOTONIC,
1394 ipc_imem->fast_update_timer.function = ipc_imem_fast_update_timer_cb;
1396 hrtimer_init(&ipc_imem->td_alloc_timer, CLOCK_MONOTONIC,
1398 ipc_imem->td_alloc_timer.function = ipc_imem_td_alloc_timer_cb;
1400 hrtimer_init(&ipc_imem->adb_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1401 ipc_imem->adb_timer.function = ipc_imem_adb_timer_cb;
1403 if (ipc_imem_config(ipc_imem)) {
1404 dev_err(ipc_imem->dev, "failed to initialize the imem");
1408 stage = ipc_mmio_get_exec_stage(ipc_imem->mmio);
1411 ipc_imem->ipc_devlink = ipc_devlink_init(ipc_imem);
1412 if (!ipc_imem->ipc_devlink) {
1413 dev_err(ipc_imem->dev, "Devlink register failed");
1417 if (ipc_flash_link_establish(ipc_imem))
1420 set_bit(IOSM_DEVLINK_INIT, &ipc_imem->flag);
1422 return ipc_imem;
1424 ipc_devlink_deinit(ipc_imem->ipc_devlink);
1426 hrtimer_cancel(&ipc_imem->td_alloc_timer);
1427 hrtimer_cancel(&ipc_imem->fast_update_timer);
1428 hrtimer_cancel(&ipc_imem->tdupdate_timer);
1429 hrtimer_cancel(&ipc_imem->startup_timer);
1431 cancel_work_sync(&ipc_imem->run_state_worker);
1432 ipc_task_deinit(ipc_imem->ipc_task);
1434 kfree(ipc_imem->ipc_task);
1436 kfree(ipc_imem->mmio);
1438 kfree(ipc_imem);
1442 void ipc_imem_irq_process(struct iosm_imem *ipc_imem, int irq)
1445 if (ipc_imem && !ipc_imem->ev_irq_pending[irq]) {
1446 ipc_imem->ev_irq_pending[irq] = true;
1447 ipc_task_queue_send_task(ipc_imem, ipc_imem_tq_irq_cb, irq,
1452 void ipc_imem_td_update_timer_suspend(struct iosm_imem *ipc_imem, bool suspend)
1454 ipc_imem->td_update_timer_suspended = suspend;
1460 static int ipc_imem_devlink_trigger_chip_info_cb(struct iosm_imem *ipc_imem,
1470 stage = ipc_mmio_get_exec_stage(ipc_imem->mmio);
1472 dev_err(ipc_imem->dev,
1477 size = ipc_imem->mmio->chip_info_size;
1481 skb = ipc_pcie_alloc_local_skb(ipc_imem->pcie, GFP_ATOMIC, size);
1483 dev_err(ipc_imem->dev, "exhausted skbuf kernel DL memory");
1488 ipc_mmio_copy_chip_info(ipc_imem->mmio, skb_put(skb, size), size);
1490 dev_dbg(ipc_imem->dev, "execution_stage[%X] eq. BOOT", stage);
1491 ipc_imem->phase = ipc_imem_phase_update(ipc_imem);
1492 ipc_imem_sys_devlink_notify_rx(ipc_imem->ipc_devlink, skb);
1498 int ipc_imem_devlink_trigger_chip_info(struct iosm_imem *ipc_imem)
1500 return ipc_task_queue_send_task(ipc_imem,