Deleted Added
sdiff udiff text old ( 331574 ) new ( 331580 )
full compact
mlx5_main.c (331574) mlx5_main.c (331580)
1/*-
2 * Copyright (c) 2013-2017, Mellanox Technologies, Ltd. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.

--- 8 unchanged lines hidden (view full) ---

17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
23 * SUCH DAMAGE.
24 *
1/*-
2 * Copyright (c) 2013-2017, Mellanox Technologies, Ltd. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.

--- 8 unchanged lines hidden (view full) ---

17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
23 * SUCH DAMAGE.
24 *
25 * $FreeBSD: stable/11/sys/dev/mlx5/mlx5_core/mlx5_main.c 331574 2018-03-26 20:09:36Z hselasky $
25 * $FreeBSD: stable/11/sys/dev/mlx5/mlx5_core/mlx5_main.c 331580 2018-03-26 20:33:31Z hselasky $
26 */
27
28#define LINUXKPI_PARAM_PREFIX mlx5_
29
30#include <linux/kmod.h>
31#include <linux/module.h>
32#include <linux/errno.h>
33#include <linux/pci.h>

--- 24 unchanged lines hidden (view full) ---

58
59#define MLX5_DEFAULT_PROF 2
60static int prof_sel = MLX5_DEFAULT_PROF;
61module_param_named(prof_sel, prof_sel, int, 0444);
62MODULE_PARM_DESC(prof_sel, "profile selector. Valid range 0 - 2");
63
64#define NUMA_NO_NODE -1
65
26 */
27
28#define LINUXKPI_PARAM_PREFIX mlx5_
29
30#include <linux/kmod.h>
31#include <linux/module.h>
32#include <linux/errno.h>
33#include <linux/pci.h>

--- 24 unchanged lines hidden (view full) ---

58
59#define MLX5_DEFAULT_PROF 2
60static int prof_sel = MLX5_DEFAULT_PROF;
61module_param_named(prof_sel, prof_sel, int, 0444);
62MODULE_PARM_DESC(prof_sel, "profile selector. Valid range 0 - 2");
63
64#define NUMA_NO_NODE -1
65
66struct workqueue_struct *mlx5_core_wq;
67static LIST_HEAD(intf_list);
68static LIST_HEAD(dev_list);
69static DEFINE_MUTEX(intf_mutex);
70
71struct mlx5_device_context {
72 struct list_head list;
73 struct mlx5_interface *intf;
74 void *context;

--- 106 unchanged lines hidden (view full) ---

181 return err;
182 }
183 }
184
185 dma_set_max_seg_size(&pdev->dev, 2u * 1024 * 1024 * 1024);
186 return err;
187}
188
66static LIST_HEAD(intf_list);
67static LIST_HEAD(dev_list);
68static DEFINE_MUTEX(intf_mutex);
69
70struct mlx5_device_context {
71 struct list_head list;
72 struct mlx5_interface *intf;
73 void *context;

--- 106 unchanged lines hidden (view full) ---

180 return err;
181 }
182 }
183
184 dma_set_max_seg_size(&pdev->dev, 2u * 1024 * 1024 * 1024);
185 return err;
186}
187
188static int mlx5_pci_enable_device(struct mlx5_core_dev *dev)
189{
190 struct pci_dev *pdev = dev->pdev;
191 int err = 0;
192
193 mutex_lock(&dev->pci_status_mutex);
194 if (dev->pci_status == MLX5_PCI_STATUS_DISABLED) {
195 err = pci_enable_device(pdev);
196 if (!err)
197 dev->pci_status = MLX5_PCI_STATUS_ENABLED;
198 }
199 mutex_unlock(&dev->pci_status_mutex);
200
201 return err;
202}
203
204static void mlx5_pci_disable_device(struct mlx5_core_dev *dev)
205{
206 struct pci_dev *pdev = dev->pdev;
207
208 mutex_lock(&dev->pci_status_mutex);
209 if (dev->pci_status == MLX5_PCI_STATUS_ENABLED) {
210 pci_disable_device(pdev);
211 dev->pci_status = MLX5_PCI_STATUS_DISABLED;
212 }
213 mutex_unlock(&dev->pci_status_mutex);
214}
215
189static int request_bar(struct pci_dev *pdev)
190{
191 int err = 0;
192
193 if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) {
194 device_printf((&pdev->dev)->bsddev, "ERR: ""Missing registers BAR, aborting\n");
195 return -ENODEV;
196 }

--- 478 unchanged lines hidden (view full) ---

675 break;
676 }
677 msleep(FW_INIT_WAIT_MS);
678 }
679
680 return err;
681}
682
216static int request_bar(struct pci_dev *pdev)
217{
218 int err = 0;
219
220 if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) {
221 device_printf((&pdev->dev)->bsddev, "ERR: ""Missing registers BAR, aborting\n");
222 return -ENODEV;
223 }

--- 478 unchanged lines hidden (view full) ---

702 break;
703 }
704 msleep(FW_INIT_WAIT_MS);
705 }
706
707 return err;
708}
709
683static int mlx5_dev_init(struct mlx5_core_dev *dev, struct pci_dev *pdev)
710static void mlx5_add_device(struct mlx5_interface *intf, struct mlx5_priv *priv)
684{
711{
712 struct mlx5_device_context *dev_ctx;
713 struct mlx5_core_dev *dev = container_of(priv, struct mlx5_core_dev, priv);
714
715 dev_ctx = kzalloc(sizeof(*dev_ctx), GFP_KERNEL);
716 if (!dev_ctx)
717 return;
718
719 dev_ctx->intf = intf;
720 CURVNET_SET_QUIET(vnet0);
721 dev_ctx->context = intf->add(dev);
722 CURVNET_RESTORE();
723
724 if (dev_ctx->context) {
725 spin_lock_irq(&priv->ctx_lock);
726 list_add_tail(&dev_ctx->list, &priv->ctx_list);
727 spin_unlock_irq(&priv->ctx_lock);
728 } else {
729 kfree(dev_ctx);
730 }
731}
732
733static void mlx5_remove_device(struct mlx5_interface *intf, struct mlx5_priv *priv)
734{
735 struct mlx5_device_context *dev_ctx;
736 struct mlx5_core_dev *dev = container_of(priv, struct mlx5_core_dev, priv);
737
738 list_for_each_entry(dev_ctx, &priv->ctx_list, list)
739 if (dev_ctx->intf == intf) {
740 spin_lock_irq(&priv->ctx_lock);
741 list_del(&dev_ctx->list);
742 spin_unlock_irq(&priv->ctx_lock);
743
744 intf->remove(dev, dev_ctx->context);
745 kfree(dev_ctx);
746 return;
747 }
748}
749
750static int mlx5_register_device(struct mlx5_core_dev *dev)
751{
685 struct mlx5_priv *priv = &dev->priv;
752 struct mlx5_priv *priv = &dev->priv;
686 int err;
753 struct mlx5_interface *intf;
687
754
688 dev->pdev = pdev;
755 mutex_lock(&intf_mutex);
756 list_add_tail(&priv->dev_list, &dev_list);
757 list_for_each_entry(intf, &intf_list, list)
758 mlx5_add_device(intf, priv);
759 mutex_unlock(&intf_mutex);
760
761 return 0;
762}
763
764static void mlx5_unregister_device(struct mlx5_core_dev *dev)
765{
766 struct mlx5_priv *priv = &dev->priv;
767 struct mlx5_interface *intf;
768
769 mutex_lock(&intf_mutex);
770 list_for_each_entry(intf, &intf_list, list)
771 mlx5_remove_device(intf, priv);
772 list_del(&priv->dev_list);
773 mutex_unlock(&intf_mutex);
774}
775
776int mlx5_register_interface(struct mlx5_interface *intf)
777{
778 struct mlx5_priv *priv;
779
780 if (!intf->add || !intf->remove)
781 return -EINVAL;
782
783 mutex_lock(&intf_mutex);
784 list_add_tail(&intf->list, &intf_list);
785 list_for_each_entry(priv, &dev_list, dev_list)
786 mlx5_add_device(intf, priv);
787 mutex_unlock(&intf_mutex);
788
789 return 0;
790}
791EXPORT_SYMBOL(mlx5_register_interface);
792
793void mlx5_unregister_interface(struct mlx5_interface *intf)
794{
795 struct mlx5_priv *priv;
796
797 mutex_lock(&intf_mutex);
798 list_for_each_entry(priv, &dev_list, dev_list)
799 mlx5_remove_device(intf, priv);
800 list_del(&intf->list);
801 mutex_unlock(&intf_mutex);
802}
803EXPORT_SYMBOL(mlx5_unregister_interface);
804
805void *mlx5_get_protocol_dev(struct mlx5_core_dev *mdev, int protocol)
806{
807 struct mlx5_priv *priv = &mdev->priv;
808 struct mlx5_device_context *dev_ctx;
809 unsigned long flags;
810 void *result = NULL;
811
812 spin_lock_irqsave(&priv->ctx_lock, flags);
813
814 list_for_each_entry(dev_ctx, &mdev->priv.ctx_list, list)
815 if ((dev_ctx->intf->protocol == protocol) &&
816 dev_ctx->intf->get_dev) {
817 result = dev_ctx->intf->get_dev(dev_ctx->context);
818 break;
819 }
820
821 spin_unlock_irqrestore(&priv->ctx_lock, flags);
822
823 return result;
824}
825EXPORT_SYMBOL(mlx5_get_protocol_dev);
826
827static int mlx5_pci_init(struct mlx5_core_dev *dev, struct mlx5_priv *priv)
828{
829 struct pci_dev *pdev = dev->pdev;
830 int err = 0;
831
689 pci_set_drvdata(dev->pdev, dev);
690 strncpy(priv->name, dev_name(&pdev->dev), MLX5_MAX_NAME_LEN);
691 priv->name[MLX5_MAX_NAME_LEN - 1] = 0;
692
693 mutex_init(&priv->pgdir_mutex);
694 INIT_LIST_HEAD(&priv->pgdir_list);
695 spin_lock_init(&priv->mkey_lock);
696
697 priv->numa_node = NUMA_NO_NODE;
698
832 pci_set_drvdata(dev->pdev, dev);
833 strncpy(priv->name, dev_name(&pdev->dev), MLX5_MAX_NAME_LEN);
834 priv->name[MLX5_MAX_NAME_LEN - 1] = 0;
835
836 mutex_init(&priv->pgdir_mutex);
837 INIT_LIST_HEAD(&priv->pgdir_list);
838 spin_lock_init(&priv->mkey_lock);
839
840 priv->numa_node = NUMA_NO_NODE;
841
699 err = pci_enable_device(pdev);
842 err = mlx5_pci_enable_device(dev);
700 if (err) {
701 device_printf((&pdev->dev)->bsddev, "ERR: ""Cannot enable PCI device, aborting\n");
702 goto err_dbg;
703 }
704
705 err = request_bar(pdev);
706 if (err) {
707 device_printf((&pdev->dev)->bsddev, "ERR: ""error requesting BARs, aborting\n");

--- 10 unchanged lines hidden (view full) ---

718
719 dev->iseg_base = pci_resource_start(dev->pdev, 0);
720 dev->iseg = ioremap(dev->iseg_base, sizeof(*dev->iseg));
721 if (!dev->iseg) {
722 err = -ENOMEM;
723 device_printf((&pdev->dev)->bsddev, "ERR: ""Failed mapping initialization segment, aborting\n");
724 goto err_clr_master;
725 }
843 if (err) {
844 device_printf((&pdev->dev)->bsddev, "ERR: ""Cannot enable PCI device, aborting\n");
845 goto err_dbg;
846 }
847
848 err = request_bar(pdev);
849 if (err) {
850 device_printf((&pdev->dev)->bsddev, "ERR: ""error requesting BARs, aborting\n");

--- 10 unchanged lines hidden (view full) ---

861
862 dev->iseg_base = pci_resource_start(dev->pdev, 0);
863 dev->iseg = ioremap(dev->iseg_base, sizeof(*dev->iseg));
864 if (!dev->iseg) {
865 err = -ENOMEM;
866 device_printf((&pdev->dev)->bsddev, "ERR: ""Failed mapping initialization segment, aborting\n");
867 goto err_clr_master;
868 }
869
870 return 0;
871
872err_clr_master:
873 pci_clear_master(dev->pdev);
874 release_bar(dev->pdev);
875err_disable:
876 mlx5_pci_disable_device(dev);
877err_dbg:
878 return err;
879}
880
881static void mlx5_pci_close(struct mlx5_core_dev *dev, struct mlx5_priv *priv)
882{
883 iounmap(dev->iseg);
884 pci_clear_master(dev->pdev);
885 release_bar(dev->pdev);
886 mlx5_pci_disable_device(dev);
887}
888
889static int mlx5_load_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv)
890{
891 struct pci_dev *pdev = dev->pdev;
892 int err;
893
894 mutex_lock(&dev->intf_state_mutex);
895 if (test_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state)) {
896 dev_warn(&dev->pdev->dev, "%s: interface is up, NOP\n",
897 __func__);
898 goto out;
899 }
900
726 device_printf((&pdev->dev)->bsddev, "INFO: ""firmware version: %d.%d.%d\n", fw_rev_maj(dev), fw_rev_min(dev), fw_rev_sub(dev));
727
728 /*
729 * On load removing any previous indication of internal error,
730 * device is up
731 */
732 dev->state = MLX5_DEVICE_STATE_UP;
733
734 err = mlx5_cmd_init(dev);
735 if (err) {
736 device_printf((&pdev->dev)->bsddev, "ERR: ""Failed initializing command interface, aborting\n");
901 device_printf((&pdev->dev)->bsddev, "INFO: ""firmware version: %d.%d.%d\n", fw_rev_maj(dev), fw_rev_min(dev), fw_rev_sub(dev));
902
903 /*
904 * On load removing any previous indication of internal error,
905 * device is up
906 */
907 dev->state = MLX5_DEVICE_STATE_UP;
908
909 err = mlx5_cmd_init(dev);
910 if (err) {
911 device_printf((&pdev->dev)->bsddev, "ERR: ""Failed initializing command interface, aborting\n");
737 goto err_unmap;
912 goto out_err;
738 }
739
740 err = wait_fw_init(dev, FW_INIT_TIMEOUT_MILI);
741 if (err) {
742 device_printf((&dev->pdev->dev)->bsddev, "ERR: ""Firmware over %d MS in initializing state, aborting\n", FW_INIT_TIMEOUT_MILI);
743 goto err_cmd_cleanup;
744 }
745

--- 108 unchanged lines hidden (view full) ---

854 mlx5_init_mr_table(dev);
855
856 err = mlx5_init_fs(dev);
857 if (err) {
858 mlx5_core_err(dev, "flow steering init %d\n", err);
859 goto err_init_tables;
860 }
861
913 }
914
915 err = wait_fw_init(dev, FW_INIT_TIMEOUT_MILI);
916 if (err) {
917 device_printf((&dev->pdev->dev)->bsddev, "ERR: ""Firmware over %d MS in initializing state, aborting\n", FW_INIT_TIMEOUT_MILI);
918 goto err_cmd_cleanup;
919 }
920

--- 108 unchanged lines hidden (view full) ---

1029 mlx5_init_mr_table(dev);
1030
1031 err = mlx5_init_fs(dev);
1032 if (err) {
1033 mlx5_core_err(dev, "flow steering init %d\n", err);
1034 goto err_init_tables;
1035 }
1036
1037 err = mlx5_register_device(dev);
1038 if (err) {
1039 dev_err(&pdev->dev, "mlx5_register_device failed %d\n", err);
1040 goto err_reg_dev;
1041 }
1042
1043 clear_bit(MLX5_INTERFACE_STATE_DOWN, &dev->intf_state);
1044 set_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state);
1045
1046out:
1047 mutex_unlock(&dev->intf_state_mutex);
862 return 0;
863
1048 return 0;
1049
1050err_reg_dev:
1051 mlx5_cleanup_fs(dev);
864err_init_tables:
865 mlx5_cleanup_mr_table(dev);
866 mlx5_cleanup_srq_table(dev);
867 mlx5_cleanup_qp_table(dev);
868 mlx5_cleanup_cq_table(dev);
869 unmap_bf_area(dev);
870
871err_stop_eqs:

--- 7 unchanged lines hidden (view full) ---

879
880disable_msix:
881 mlx5_disable_msix(dev);
882
883err_stop_poll:
884 mlx5_stop_health_poll(dev);
885 if (mlx5_cmd_teardown_hca(dev)) {
886 device_printf((&dev->pdev->dev)->bsddev, "ERR: ""tear_down_hca failed, skip cleanup\n");
1052err_init_tables:
1053 mlx5_cleanup_mr_table(dev);
1054 mlx5_cleanup_srq_table(dev);
1055 mlx5_cleanup_qp_table(dev);
1056 mlx5_cleanup_cq_table(dev);
1057 unmap_bf_area(dev);
1058
1059err_stop_eqs:

--- 7 unchanged lines hidden (view full) ---

1067
1068disable_msix:
1069 mlx5_disable_msix(dev);
1070
1071err_stop_poll:
1072 mlx5_stop_health_poll(dev);
1073 if (mlx5_cmd_teardown_hca(dev)) {
1074 device_printf((&dev->pdev->dev)->bsddev, "ERR: ""tear_down_hca failed, skip cleanup\n");
887 return err;
1075 goto out_err;
888 }
889
890reclaim_boot_pages:
891 mlx5_reclaim_startup_pages(dev);
892
893err_pagealloc_stop:
894 mlx5_pagealloc_stop(dev);
895
896err_disable_hca:
897 mlx5_core_disable_hca(dev);
898
899err_pagealloc_cleanup:
900 mlx5_pagealloc_cleanup(dev);
1076 }
1077
1078reclaim_boot_pages:
1079 mlx5_reclaim_startup_pages(dev);
1080
1081err_pagealloc_stop:
1082 mlx5_pagealloc_stop(dev);
1083
1084err_disable_hca:
1085 mlx5_core_disable_hca(dev);
1086
1087err_pagealloc_cleanup:
1088 mlx5_pagealloc_cleanup(dev);
1089
901err_cmd_cleanup:
902 mlx5_cmd_cleanup(dev);
903
1090err_cmd_cleanup:
1091 mlx5_cmd_cleanup(dev);
1092
904err_unmap:
905 iounmap(dev->iseg);
906
907err_clr_master:
908 pci_clear_master(dev->pdev);
909 release_bar(dev->pdev);
910
911err_disable:
912 pci_disable_device(dev->pdev);
913
914err_dbg:
1093out_err:
915 dev->state = MLX5_DEVICE_STATE_INTERNAL_ERROR;
1094 dev->state = MLX5_DEVICE_STATE_INTERNAL_ERROR;
1095 mutex_unlock(&dev->intf_state_mutex);
1096
916 return err;
917}
918
1097 return err;
1098}
1099
919static void mlx5_dev_cleanup(struct mlx5_core_dev *dev)
1100static int mlx5_unload_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv)
920{
1101{
921 struct mlx5_priv *priv = &dev->priv;
1102 int err = 0;
922
1103
1104 mutex_lock(&dev->intf_state_mutex);
1105 if (test_bit(MLX5_INTERFACE_STATE_DOWN, &dev->intf_state)) {
1106 dev_warn(&dev->pdev->dev, "%s: interface is down, NOP\n", __func__);
1107 goto out;
1108 }
1109
1110 mlx5_unregister_device(dev);
1111
923 mlx5_cleanup_fs(dev);
924 mlx5_cleanup_mr_table(dev);
925 mlx5_cleanup_srq_table(dev);
926 mlx5_cleanup_qp_table(dev);
927 mlx5_cleanup_cq_table(dev);
928 unmap_bf_area(dev);
929 mlx5_wait_for_reclaim_vfs_pages(dev);
930 free_comp_eqs(dev);
931 mlx5_stop_eqs(dev);
932 mlx5_free_uuars(dev, &priv->uuari);
933 mlx5_eq_cleanup(dev);
934 mlx5_disable_msix(dev);
935 mlx5_stop_health_poll(dev);
1112 mlx5_cleanup_fs(dev);
1113 mlx5_cleanup_mr_table(dev);
1114 mlx5_cleanup_srq_table(dev);
1115 mlx5_cleanup_qp_table(dev);
1116 mlx5_cleanup_cq_table(dev);
1117 unmap_bf_area(dev);
1118 mlx5_wait_for_reclaim_vfs_pages(dev);
1119 free_comp_eqs(dev);
1120 mlx5_stop_eqs(dev);
1121 mlx5_free_uuars(dev, &priv->uuari);
1122 mlx5_eq_cleanup(dev);
1123 mlx5_disable_msix(dev);
1124 mlx5_stop_health_poll(dev);
936 if (mlx5_cmd_teardown_hca(dev)) {
1125 err = mlx5_cmd_teardown_hca(dev);
1126 if (err) {
937 device_printf((&dev->pdev->dev)->bsddev, "ERR: ""tear_down_hca failed, skip cleanup\n");
1127 device_printf((&dev->pdev->dev)->bsddev, "ERR: ""tear_down_hca failed, skip cleanup\n");
938 return;
1128 goto out;
939 }
940 mlx5_pagealloc_stop(dev);
941 mlx5_reclaim_startup_pages(dev);
942 mlx5_core_disable_hca(dev);
943 mlx5_pagealloc_cleanup(dev);
944 mlx5_cmd_cleanup(dev);
1129 }
1130 mlx5_pagealloc_stop(dev);
1131 mlx5_reclaim_startup_pages(dev);
1132 mlx5_core_disable_hca(dev);
1133 mlx5_pagealloc_cleanup(dev);
1134 mlx5_cmd_cleanup(dev);
945 iounmap(dev->iseg);
946 pci_clear_master(dev->pdev);
947 release_bar(dev->pdev);
948 pci_disable_device(dev->pdev);
949}
950
1135
951static void mlx5_add_device(struct mlx5_interface *intf, struct mlx5_priv *priv)
952{
953 struct mlx5_device_context *dev_ctx;
954 struct mlx5_core_dev *dev = container_of(priv, struct mlx5_core_dev, priv);
955
956 dev_ctx = kmalloc(sizeof(*dev_ctx), GFP_KERNEL);
957
958 dev_ctx->intf = intf;
959 CURVNET_SET_QUIET(vnet0);
960 dev_ctx->context = intf->add(dev);
961 CURVNET_RESTORE();
962
963 if (dev_ctx->context) {
964 spin_lock_irq(&priv->ctx_lock);
965 list_add_tail(&dev_ctx->list, &priv->ctx_list);
966 spin_unlock_irq(&priv->ctx_lock);
967 } else {
968 kfree(dev_ctx);
969 }
1136out:
1137 clear_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state);
1138 set_bit(MLX5_INTERFACE_STATE_DOWN, &dev->intf_state);
1139 mutex_unlock(&dev->intf_state_mutex);
1140 return err;
970}
971
1141}
1142
972static void mlx5_remove_device(struct mlx5_interface *intf, struct mlx5_priv *priv)
1143void mlx5_core_event(struct mlx5_core_dev *dev, enum mlx5_dev_event event,
1144 unsigned long param)
973{
1145{
974 struct mlx5_device_context *dev_ctx;
975 struct mlx5_core_dev *dev = container_of(priv, struct mlx5_core_dev, priv);
976
977 list_for_each_entry(dev_ctx, &priv->ctx_list, list)
978 if (dev_ctx->intf == intf) {
979 spin_lock_irq(&priv->ctx_lock);
980 list_del(&dev_ctx->list);
981 spin_unlock_irq(&priv->ctx_lock);
982
983 intf->remove(dev, dev_ctx->context);
984 kfree(dev_ctx);
985 return;
986 }
987}
988static int mlx5_register_device(struct mlx5_core_dev *dev)
989{
990 struct mlx5_priv *priv = &dev->priv;
1146 struct mlx5_priv *priv = &dev->priv;
991 struct mlx5_interface *intf;
992
993 mutex_lock(&intf_mutex);
994 list_add_tail(&priv->dev_list, &dev_list);
995 list_for_each_entry(intf, &intf_list, list)
996 mlx5_add_device(intf, priv);
997 mutex_unlock(&intf_mutex);
998
999 return 0;
1000}
1001static void mlx5_unregister_device(struct mlx5_core_dev *dev)
1002{
1003 struct mlx5_priv *priv = &dev->priv;
1004 struct mlx5_interface *intf;
1005
1006 mutex_lock(&intf_mutex);
1007 list_for_each_entry(intf, &intf_list, list)
1008 mlx5_remove_device(intf, priv);
1009 list_del(&priv->dev_list);
1010 mutex_unlock(&intf_mutex);
1011}
1012
1013int mlx5_register_interface(struct mlx5_interface *intf)
1014{
1015 struct mlx5_priv *priv;
1016
1017 if (!intf->add || !intf->remove)
1018 return -EINVAL;
1019
1020 mutex_lock(&intf_mutex);
1021 list_add_tail(&intf->list, &intf_list);
1022 list_for_each_entry(priv, &dev_list, dev_list)
1023 mlx5_add_device(intf, priv);
1024 mutex_unlock(&intf_mutex);
1025
1026 return 0;
1027}
1028EXPORT_SYMBOL(mlx5_register_interface);
1029
1030void mlx5_unregister_interface(struct mlx5_interface *intf)
1031{
1032 struct mlx5_priv *priv;
1033
1034 mutex_lock(&intf_mutex);
1035 list_for_each_entry(priv, &dev_list, dev_list)
1036 mlx5_remove_device(intf, priv);
1037 list_del(&intf->list);
1038 mutex_unlock(&intf_mutex);
1039}
1040EXPORT_SYMBOL(mlx5_unregister_interface);
1041
1042void *mlx5_get_protocol_dev(struct mlx5_core_dev *mdev, int protocol)
1043{
1044 struct mlx5_priv *priv = &mdev->priv;
1045 struct mlx5_device_context *dev_ctx;
1046 unsigned long flags;
1147 struct mlx5_device_context *dev_ctx;
1148 unsigned long flags;
1047 void *result = NULL;
1048
1049 spin_lock_irqsave(&priv->ctx_lock, flags);
1050
1149
1150 spin_lock_irqsave(&priv->ctx_lock, flags);
1151
1051 list_for_each_entry(dev_ctx, &mdev->priv.ctx_list, list)
1052 if ((dev_ctx->intf->protocol == protocol) &&
1053 dev_ctx->intf->get_dev) {
1054 result = dev_ctx->intf->get_dev(dev_ctx->context);
1055 break;
1056 }
1057
1058 spin_unlock_irqrestore(&priv->ctx_lock, flags);
1059
1060 return result;
1061}
1062EXPORT_SYMBOL(mlx5_get_protocol_dev);
1063
1064static void mlx5_core_event(struct mlx5_core_dev *dev, enum mlx5_dev_event event,
1065 unsigned long param)
1066{
1067 struct mlx5_priv *priv = &dev->priv;
1068 struct mlx5_device_context *dev_ctx;
1069 unsigned long flags;
1070
1071 spin_lock_irqsave(&priv->ctx_lock, flags);
1072
1073 list_for_each_entry(dev_ctx, &priv->ctx_list, list)
1074 if (dev_ctx->intf->event)
1075 dev_ctx->intf->event(dev, dev_ctx->context, event, param);
1076
1077 spin_unlock_irqrestore(&priv->ctx_lock, flags);
1078}
1079
1080struct mlx5_core_event_handler {

--- 15 unchanged lines hidden (view full) ---

1096 if (id)
1097 priv->pci_dev_data = id->driver_data;
1098
1099 if (prof_sel < 0 || prof_sel >= ARRAY_SIZE(profiles)) {
1100 printf("mlx5_core: WARN: ""selected profile out of range, selecting default (%d)\n", MLX5_DEFAULT_PROF);
1101 prof_sel = MLX5_DEFAULT_PROF;
1102 }
1103 dev->profile = &profiles[prof_sel];
1152 list_for_each_entry(dev_ctx, &priv->ctx_list, list)
1153 if (dev_ctx->intf->event)
1154 dev_ctx->intf->event(dev, dev_ctx->context, event, param);
1155
1156 spin_unlock_irqrestore(&priv->ctx_lock, flags);
1157}
1158
1159struct mlx5_core_event_handler {

--- 15 unchanged lines hidden (view full) ---

1175 if (id)
1176 priv->pci_dev_data = id->driver_data;
1177
1178 if (prof_sel < 0 || prof_sel >= ARRAY_SIZE(profiles)) {
1179 printf("mlx5_core: WARN: ""selected profile out of range, selecting default (%d)\n", MLX5_DEFAULT_PROF);
1180 prof_sel = MLX5_DEFAULT_PROF;
1181 }
1182 dev->profile = &profiles[prof_sel];
1183 dev->pdev = pdev;
1104 dev->event = mlx5_core_event;
1105
1106 INIT_LIST_HEAD(&priv->ctx_list);
1107 spin_lock_init(&priv->ctx_lock);
1184 dev->event = mlx5_core_event;
1185
1186 INIT_LIST_HEAD(&priv->ctx_list);
1187 spin_lock_init(&priv->ctx_lock);
1108 err = mlx5_dev_init(dev, pdev);
1188 mutex_init(&dev->pci_status_mutex);
1189 mutex_init(&dev->intf_state_mutex);
1190 err = mlx5_pci_init(dev, priv);
1109 if (err) {
1191 if (err) {
1110 device_printf((&pdev->dev)->bsddev, "ERR: ""mlx5_dev_init failed %d\n", err);
1111 goto out;
1192 device_printf((&pdev->dev)->bsddev, "ERR: ""mlx5_pci_init failed %d\n", err);
1193 goto clean_dev;
1112 }
1113
1194 }
1195
1114 err = mlx5_register_device(dev);
1196 err = mlx5_health_init(dev);
1197 if (err) {
1198 device_printf((&pdev->dev)->bsddev, "ERR: ""mlx5_health_init failed %d\n", err);
1199 goto close_pci;
1200 }
1201
1202 err = mlx5_load_one(dev, priv);
1115 if (err) {
1116 device_printf((&pdev->dev)->bsddev, "ERR: ""mlx5_register_device failed %d\n", err);
1203 if (err) {
1204 device_printf((&pdev->dev)->bsddev, "ERR: ""mlx5_register_device failed %d\n", err);
1117 goto out_init;
1205 goto clean_health;
1118 }
1119
1120
1121 return 0;
1122
1206 }
1207
1208
1209 return 0;
1210
1123out_init:
1124 mlx5_dev_cleanup(dev);
1125out:
1211clean_health:
1212 mlx5_health_cleanup(dev);
1213close_pci:
1214 mlx5_pci_close(dev, priv);
1215clean_dev:
1126 kfree(dev);
1127 return err;
1128}
1129
1130static void remove_one(struct pci_dev *pdev)
1131{
1132 struct mlx5_core_dev *dev = pci_get_drvdata(pdev);
1216 kfree(dev);
1217 return err;
1218}
1219
1220static void remove_one(struct pci_dev *pdev)
1221{
1222 struct mlx5_core_dev *dev = pci_get_drvdata(pdev);
1223 struct mlx5_priv *priv = &dev->priv;
1133
1224
1134 mlx5_unregister_device(dev);
1135 mlx5_dev_cleanup(dev);
1225 if (mlx5_unload_one(dev, priv)) {
1226 dev_err(&dev->pdev->dev, "mlx5_unload_one failed\n");
1227 mlx5_health_cleanup(dev);
1228 return;
1229 }
1230
1231 mlx5_health_cleanup(dev);
1232 mlx5_pci_close(dev, priv);
1233 pci_set_drvdata(pdev, NULL);
1136 kfree(dev);
1137}
1138
1234 kfree(dev);
1235}
1236
1237static pci_ers_result_t mlx5_pci_err_detected(struct pci_dev *pdev,
1238 pci_channel_state_t state)
1239{
1240 struct mlx5_core_dev *dev = pci_get_drvdata(pdev);
1241 struct mlx5_priv *priv = &dev->priv;
1242
1243 dev_info(&pdev->dev, "%s was called\n", __func__);
1244 mlx5_enter_error_state(dev);
1245 mlx5_unload_one(dev, priv);
1246 mlx5_pci_disable_device(dev);
1247 return state == pci_channel_io_perm_failure ?
1248 PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_NEED_RESET;
1249}
1250
1251static pci_ers_result_t mlx5_pci_slot_reset(struct pci_dev *pdev)
1252{
1253 struct mlx5_core_dev *dev = pci_get_drvdata(pdev);
1254 int err = 0;
1255
1256 dev_info(&pdev->dev, "%s was called\n", __func__);
1257
1258 err = mlx5_pci_enable_device(dev);
1259 if (err) {
1260 dev_err(&pdev->dev, "%s: mlx5_pci_enable_device failed with error code: %d\n"
1261 , __func__, err);
1262 return PCI_ERS_RESULT_DISCONNECT;
1263 }
1264 pci_set_master(pdev);
1265 pci_set_powerstate(pdev->dev.bsddev, PCI_POWERSTATE_D0);
1266 pci_restore_state(pdev->dev.bsddev);
1267
1268 return err ? PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_RECOVERED;
1269}
1270
1271void mlx5_disable_device(struct mlx5_core_dev *dev)
1272{
1273 mlx5_pci_err_detected(dev->pdev, 0);
1274}
1275
1276/* wait for the device to show vital signs. For now we check
1277 * that we can read the device ID and that the health buffer
1278 * shows a non zero value which is different than 0xffffffff
1279 */
1280static void wait_vital(struct pci_dev *pdev)
1281{
1282 struct mlx5_core_dev *dev = pci_get_drvdata(pdev);
1283 struct mlx5_core_health *health = &dev->priv.health;
1284 const int niter = 100;
1285 u32 count;
1286 u16 did;
1287 int i;
1288
1289 /* Wait for firmware to be ready after reset */
1290 msleep(1000);
1291 for (i = 0; i < niter; i++) {
1292 if (pci_read_config_word(pdev, 2, &did)) {
1293 dev_warn(&pdev->dev, "failed reading config word\n");
1294 break;
1295 }
1296 if (did == pdev->device) {
1297 dev_info(&pdev->dev, "device ID correctly read after %d iterations\n", i);
1298 break;
1299 }
1300 msleep(50);
1301 }
1302 if (i == niter)
1303 dev_warn(&pdev->dev, "%s-%d: could not read device ID\n", __func__, __LINE__);
1304
1305 for (i = 0; i < niter; i++) {
1306 count = ioread32be(health->health_counter);
1307 if (count && count != 0xffffffff) {
1308 dev_info(&pdev->dev, "Counter value 0x%x after %d iterations\n", count, i);
1309 break;
1310 }
1311 msleep(50);
1312 }
1313
1314 if (i == niter)
1315 dev_warn(&pdev->dev, "%s-%d: could not read device ID\n", __func__, __LINE__);
1316}
1317
1318static void mlx5_pci_resume(struct pci_dev *pdev)
1319{
1320 struct mlx5_core_dev *dev = pci_get_drvdata(pdev);
1321 struct mlx5_priv *priv = &dev->priv;
1322 int err;
1323
1324 dev_info(&pdev->dev, "%s was called\n", __func__);
1325
1326 pci_save_state(pdev->dev.bsddev);
1327 wait_vital(pdev);
1328
1329 err = mlx5_load_one(dev, priv);
1330 if (err)
1331 dev_err(&pdev->dev, "%s: mlx5_load_one failed with error code: %d\n"
1332 , __func__, err);
1333 else
1334 dev_info(&pdev->dev, "%s: device recovered\n", __func__);
1335}
1336
1337static const struct pci_error_handlers mlx5_err_handler = {
1338 .error_detected = mlx5_pci_err_detected,
1339 .slot_reset = mlx5_pci_slot_reset,
1340 .resume = mlx5_pci_resume
1341};
1342
1139static void shutdown_one(struct pci_dev *pdev)
1140{
1343static void shutdown_one(struct pci_dev *pdev)
1344{
1141 /* prevent device from accessing host memory after shutdown */
1142 pci_clear_master(pdev);
1345 struct mlx5_core_dev *dev = pci_get_drvdata(pdev);
1346 struct mlx5_priv *priv = &dev->priv;
1347
1348 set_bit(MLX5_INTERFACE_STATE_SHUTDOWN, &dev->intf_state);
1349 mlx5_unload_one(dev, priv);
1350 mlx5_pci_disable_device(dev);
1143}
1144
1145static const struct pci_device_id mlx5_core_pci_table[] = {
1146 { PCI_VDEVICE(MELLANOX, 4113) }, /* Connect-IB */
1147 { PCI_VDEVICE(MELLANOX, 4114) }, /* Connect-IB VF */
1148 { PCI_VDEVICE(MELLANOX, 4115) }, /* ConnectX-4 */
1149 { PCI_VDEVICE(MELLANOX, 4116) }, /* ConnectX-4 VF */
1150 { PCI_VDEVICE(MELLANOX, 4117) }, /* ConnectX-4LX */

--- 29 unchanged lines hidden (view full) ---

1180
1181MODULE_DEVICE_TABLE(pci, mlx5_core_pci_table);
1182
1183static struct pci_driver mlx5_core_driver = {
1184 .name = DRIVER_NAME,
1185 .id_table = mlx5_core_pci_table,
1186 .shutdown = shutdown_one,
1187 .probe = init_one,
1351}
1352
1353static const struct pci_device_id mlx5_core_pci_table[] = {
1354 { PCI_VDEVICE(MELLANOX, 4113) }, /* Connect-IB */
1355 { PCI_VDEVICE(MELLANOX, 4114) }, /* Connect-IB VF */
1356 { PCI_VDEVICE(MELLANOX, 4115) }, /* ConnectX-4 */
1357 { PCI_VDEVICE(MELLANOX, 4116) }, /* ConnectX-4 VF */
1358 { PCI_VDEVICE(MELLANOX, 4117) }, /* ConnectX-4LX */

--- 29 unchanged lines hidden (view full) ---

1388
1389MODULE_DEVICE_TABLE(pci, mlx5_core_pci_table);
1390
1391static struct pci_driver mlx5_core_driver = {
1392 .name = DRIVER_NAME,
1393 .id_table = mlx5_core_pci_table,
1394 .shutdown = shutdown_one,
1395 .probe = init_one,
1188 .remove = remove_one
1396 .remove = remove_one,
1397 .err_handler = &mlx5_err_handler
1189};
1190
1191static int __init init(void)
1192{
1193 int err;
1194
1398};
1399
1400static int __init init(void)
1401{
1402 int err;
1403
1195 mlx5_core_wq = create_singlethread_workqueue("mlx5_core_wq");
1196 if (!mlx5_core_wq) {
1197 err = -ENOMEM;
1198 goto err_debug;
1199 }
1200 mlx5_health_init();
1201
1202 err = pci_register_driver(&mlx5_core_driver);
1203 if (err)
1404 err = pci_register_driver(&mlx5_core_driver);
1405 if (err)
1204 goto err_health;
1406 goto err_debug;
1205
1206
1207 return 0;
1208
1407
1408
1409 return 0;
1410
1209err_health:
1210 mlx5_health_cleanup();
1211 destroy_workqueue(mlx5_core_wq);
1212err_debug:
1213 return err;
1214}
1215
1216static void __exit cleanup(void)
1217{
1218 pci_unregister_driver(&mlx5_core_driver);
1411err_debug:
1412 return err;
1413}
1414
1415static void __exit cleanup(void)
1416{
1417 pci_unregister_driver(&mlx5_core_driver);
1219 mlx5_health_cleanup();
1220 destroy_workqueue(mlx5_core_wq);
1221}
1222
1223module_init(init);
1224module_exit(cleanup);
1418}
1419
1420module_init(init);
1421module_exit(cleanup);
1225
1226void mlx5_enter_error_state(struct mlx5_core_dev *dev)
1227{
1228 if (dev->state != MLX5_DEVICE_STATE_UP)
1229 return;
1230
1231 dev->state = MLX5_DEVICE_STATE_INTERNAL_ERROR;
1232 mlx5_trigger_cmd_completions(dev);
1233}
1234EXPORT_SYMBOL(mlx5_enter_error_state);