mlx5/core/health.c

87 	struct mlx5_core_health *health = &dev->priv.health;
88 	struct health_buffer __iomem *h = health->health;
101 	struct mlx5_core_health *health = &dev->priv.health;
102 	struct health_buffer __iomem *h = health->health;
171 	/* The reset only needs to be issued by one PF. The health buffer is
296 		if (dev->priv.health.fatal_error != MLX5_SENSOR_PCI_COMM_ERR)
330 		mlx5_core_err(dev, "health recovery flow aborted, PCI reads still not working\n");
333 	mlx5_core_err(dev, "starting health recovery flow\n");
335 		mlx5_core_err(dev, "health recovery failed\n");
339 	mlx5_core_info(dev, "health recovery succeeded\n");
406 	struct mlx5_core_health *health = &dev->priv.health;
407 	struct health_buffer __iomem *h = health->health;
451 	struct mlx5_core_health *health = &dev->priv.health;
452 	struct health_buffer __iomem *h = health->health;
481 	struct mlx5_core_health *health = &dev->priv.health;
482 	struct health_buffer __iomem *h = health->health;
489 	devlink_fmsg_pair_nest_start(fmsg, "health buffer");
538 	struct mlx5_core_health *health;
540 	health = container_of(work, struct mlx5_core_health, report_work);
542 	if (IS_ERR_OR_NULL(health->fw_reporter))
545 	fw_reporter_ctx.err_synd = health->synd;
546 	fw_reporter_ctx.miss_counter = health->miss_counter;
548 		devlink_health_report(health->fw_reporter,
553 		devlink_health_report(health->fw_reporter,
585 	u32 crdump_size = dev->priv.health.crdump_size;
615 	struct mlx5_core_health *health;
620 	health = container_of(work, struct mlx5_core_health, fatal_report_work);
621 	priv = container_of(health, struct mlx5_priv, health);
626 	if (test_bit(MLX5_DROP_HEALTH_WORK, &health->flags)) {
627 		mlx5_core_err(dev, "health works are not permitted at this stage\n");
633 	if (IS_ERR_OR_NULL(health->fw_fatal_reporter)) {
636 			mlx5_core_err(dev, "health recovery failed\n");
640 	fw_reporter_ctx.err_synd = health->synd;
641 	fw_reporter_ctx.miss_counter = health->miss_counter;
642 	if (devlink_health_report(health->fw_fatal_reporter,
673 	struct mlx5_core_health *health = &dev->priv.health;
691 	health->fw_reporter =
693 	if (IS_ERR(health->fw_reporter))
695 			       PTR_ERR(health->fw_reporter));
697 	health->fw_fatal_reporter =
702 	if (IS_ERR(health->fw_fatal_reporter))
704 			       PTR_ERR(health->fw_fatal_reporter));
709 	struct mlx5_core_health *health = &dev->priv.health;
711 	if (!IS_ERR_OR_NULL(health->fw_reporter))
712 		devlink_health_reporter_destroy(health->fw_reporter);
714 	if (!IS_ERR_OR_NULL(health->fw_fatal_reporter))
715 		devlink_health_reporter_destroy(health->fw_fatal_reporter);
731 	struct mlx5_core_health *health = &dev->priv.health;
734 		queue_work(health->wq, &health->fatal_report_work);
743 	struct mlx5_core_health *health;
748 	health = container_of(dwork, struct mlx5_core_health, update_fw_log_ts_work);
749 	priv = container_of(health, struct mlx5_priv, health);
758 	queue_delayed_work(health->wq, &health->update_fw_log_ts_work,
764 	struct mlx5_core_dev *dev = from_timer(dev, t, priv.health.timer);
765 	struct mlx5_core_health *health = &dev->priv.health;
766 	struct health_buffer __iomem *h = health->health;
776 	if (fatal_error && !health->fatal_error) {
778 		dev->priv.health.fatal_error = fatal_error;
785 	count = ioread32be(health->health_counter);
786 	if (count == health->prev)
787 		++health->miss_counter;
789 		health->miss_counter = 0;
791 	health->prev = count;
792 	if (health->miss_counter == MAX_MISSES) {
793 		mlx5_core_err(dev, "device's health compromised - reached miss count\n");
795 		queue_work(health->wq, &health->report_work);
798 	prev_synd = health->synd;
799 	health->synd = ioread8(&h->synd);
800 	if (health->synd && health->synd != prev_synd)
801 		queue_work(health->wq, &health->report_work);
804 	mod_timer(&health->timer, get_next_poll_jiffies(dev));
810 	struct mlx5_core_health *health = &dev->priv.health;
812 	timer_setup(&health->timer, poll_health, 0);
813 	health->fatal_error = MLX5_SENSOR_NO_ERR;
814 	clear_bit(MLX5_DROP_HEALTH_WORK, &health->flags);
815 	health->health = &dev->iseg->health;
816 	health->health_counter = &dev->iseg->health_counter;
818 	health->timer.expires = jiffies + msecs_to_jiffies(poll_interval_ms);
819 	add_timer(&health->timer);
824 	struct mlx5_core_health *health = &dev->priv.health;
827 		set_bit(MLX5_DROP_HEALTH_WORK, &health->flags);
829 	del_timer_sync(&health->timer);
834 	struct mlx5_core_health *health = &dev->priv.health;
837 		queue_delayed_work(health->wq, &health->update_fw_log_ts_work, 0);
842 	struct mlx5_core_health *health = &dev->priv.health;
844 	set_bit(MLX5_DROP_HEALTH_WORK, &health->flags);
845 	cancel_delayed_work_sync(&health->update_fw_log_ts_work);
846 	cancel_work_sync(&health->report_work);
847 	cancel_work_sync(&health->fatal_report_work);
852 	struct mlx5_core_health *health = &dev->priv.health;
854 	cancel_delayed_work_sync(&health->update_fw_log_ts_work);
855 	destroy_workqueue(health->wq);
863 	struct mlx5_core_health *health;
873 	health = &dev->priv.health;
880 	health->wq = create_singlethread_workqueue(name);
882 	if (!health->wq)
884 	INIT_WORK(&health->fatal_report_work, mlx5_fw_fatal_reporter_err_work);
885 	INIT_WORK(&health->report_work, mlx5_fw_reporter_err_work);
886 	INIT_DELAYED_WORK(&health->update_fw_log_ts_work, mlx5_health_log_ts_update);