187189Spdeuskar// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 287189Spdeuskar/* Copyright (c) 2015 - 2021 Intel Corporation */ 3146662Stackerman#include "main.h" 487189Spdeuskar#include "../../../net/ethernet/intel/ice/ice.h" 587189Spdeuskar 6103895SpdeuskarMODULE_ALIAS("i40iw"); 7103895SpdeuskarMODULE_AUTHOR("Intel Corporation, <e1000-rdma@lists.sourceforge.net>"); 887189SpdeuskarMODULE_DESCRIPTION("Intel(R) Ethernet Protocol Driver for RDMA"); 9103895SpdeuskarMODULE_LICENSE("Dual BSD/GPL"); 10103895Spdeuskar 1187189Spdeuskarstatic struct notifier_block irdma_inetaddr_notifier = { 12103895Spdeuskar .notifier_call = irdma_inetaddr_event 13103895Spdeuskar}; 14103895Spdeuskar 1587189Spdeuskarstatic struct notifier_block irdma_inetaddr6_notifier = { 1687189Spdeuskar .notifier_call = irdma_inet6addr_event 17103895Spdeuskar}; 18103895Spdeuskar 1987189Spdeuskarstatic struct notifier_block irdma_net_notifier = { 2087189Spdeuskar .notifier_call = irdma_net_event 2187189Spdeuskar}; 2287189Spdeuskar 23103895Spdeuskarstatic struct notifier_block irdma_netdevice_notifier = { 24103895Spdeuskar .notifier_call = irdma_netdevice_event 25103895Spdeuskar}; 26103895Spdeuskar 27103895Spdeuskarstatic void irdma_register_notifiers(void) 28103895Spdeuskar{ 29103895Spdeuskar register_inetaddr_notifier(&irdma_inetaddr_notifier); 30103895Spdeuskar register_inet6addr_notifier(&irdma_inetaddr6_notifier); 3187189Spdeuskar register_netevent_notifier(&irdma_net_notifier); 3287189Spdeuskar register_netdevice_notifier(&irdma_netdevice_notifier); 3387189Spdeuskar} 3490628Spdeuskar 3590628Spdeuskarstatic void irdma_unregister_notifiers(void) 36150968Sglebius{ 37150968Sglebius unregister_netevent_notifier(&irdma_net_notifier); 38150968Sglebius unregister_inetaddr_notifier(&irdma_inetaddr_notifier); 39150968Sglebius unregister_inet6addr_notifier(&irdma_inetaddr6_notifier); 4087189Spdeuskar unregister_netdevice_notifier(&irdma_netdevice_notifier); 4187189Spdeuskar} 4287189Spdeuskar 4387189Spdeuskarstatic void irdma_prep_tc_change(struct irdma_device *iwdev) 4487189Spdeuskar{ 4587189Spdeuskar iwdev->vsi.tc_change_pending = true; 4687189Spdeuskar irdma_sc_suspend_resume_qps(&iwdev->vsi, IRDMA_OP_SUSPEND); 4787189Spdeuskar 4887189Spdeuskar /* Wait for all qp's to suspend */ 4987189Spdeuskar wait_event_timeout(iwdev->suspend_wq, 5087189Spdeuskar !atomic_read(&iwdev->vsi.qp_suspend_reqs), 51152740Sglebius msecs_to_jiffies(IRDMA_EVENT_TIMEOUT_MS)); 5287189Spdeuskar irdma_ws_reset(&iwdev->vsi); 5387189Spdeuskar} 5487189Spdeuskar 5587189Spdeuskarstatic void irdma_log_invalid_mtu(u16 mtu, struct irdma_sc_dev *dev) 5687189Spdeuskar{ 5787189Spdeuskar if (mtu < IRDMA_MIN_MTU_IPV4) 5887189Spdeuskar ibdev_warn(to_ibdev(dev), "MTU setting [%d] too low for RDMA traffic. Minimum MTU is 576 for IPv4\n", mtu); 5987189Spdeuskar else if (mtu < IRDMA_MIN_MTU_IPV6) 6087189Spdeuskar ibdev_warn(to_ibdev(dev), "MTU setting [%d] too low for RDMA traffic. Minimum MTU is 1280 for IPv6\\n", mtu); 6187189Spdeuskar} 6287189Spdeuskar 63112472Spdeuskarstatic void irdma_fill_qos_info(struct irdma_l2params *l2params, 6487189Spdeuskar struct iidc_qos_params *qos_info) 6587189Spdeuskar{ 66108229Spdeuskar int i; 67146662Stackerman 68146662Stackerman l2params->num_tc = qos_info->num_tc; 69146662Stackerman l2params->vsi_prio_type = qos_info->vport_priority_type; 70146662Stackerman l2params->vsi_rel_bw = qos_info->vport_relative_bw; 71146662Stackerman for (i = 0; i < l2params->num_tc; i++) { 72146662Stackerman l2params->tc_info[i].egress_virt_up = 73146662Stackerman qos_info->tc_info[i].egress_virt_up; 74146662Stackerman l2params->tc_info[i].ingress_virt_up = 75146662Stackerman qos_info->tc_info[i].ingress_virt_up; 76146662Stackerman l2params->tc_info[i].prio_type = qos_info->tc_info[i].prio_type; 77146662Stackerman l2params->tc_info[i].rel_bw = qos_info->tc_info[i].rel_bw; 78146662Stackerman l2params->tc_info[i].tc_ctx = qos_info->tc_info[i].tc_ctx; 79146662Stackerman } 80146662Stackerman for (i = 0; i < IIDC_MAX_USER_PRIORITY; i++) 81146662Stackerman l2params->up2tc[i] = qos_info->up2tc[i]; 82146662Stackerman if (qos_info->pfc_mode == IIDC_DSCP_PFC_MODE) { 83146662Stackerman l2params->dscp_mode = true; 84146662Stackerman memcpy(l2params->dscp_map, qos_info->dscp_map, sizeof(l2params->dscp_map)); 85146662Stackerman } 86146662Stackerman} 87146662Stackerman 88146662Stackermanstatic void irdma_iidc_event_handler(struct ice_pf *pf, struct iidc_event *event) 89146662Stackerman{ 90146662Stackerman struct irdma_device *iwdev = dev_get_drvdata(&pf->adev->dev); 91146662Stackerman struct irdma_l2params l2params = {}; 92146662Stackerman 93146662Stackerman if (*event->type & BIT(IIDC_EVENT_AFTER_MTU_CHANGE)) { 94146662Stackerman ibdev_dbg(&iwdev->ibdev, "CLNT: new MTU = %d\n", iwdev->netdev->mtu); 95146662Stackerman if (iwdev->vsi.mtu != iwdev->netdev->mtu) { 96146662Stackerman l2params.mtu = iwdev->netdev->mtu; 97146662Stackerman l2params.mtu_changed = true; 98146662Stackerman irdma_log_invalid_mtu(l2params.mtu, &iwdev->rf->sc_dev); 99146662Stackerman irdma_change_l2params(&iwdev->vsi, &l2params); 100146662Stackerman } 101146662Stackerman } else if (*event->type & BIT(IIDC_EVENT_BEFORE_TC_CHANGE)) { 102146662Stackerman if (iwdev->vsi.tc_change_pending) 103146662Stackerman return; 104146662Stackerman 105146662Stackerman irdma_prep_tc_change(iwdev); 106146662Stackerman } else if (*event->type & BIT(IIDC_EVENT_AFTER_TC_CHANGE)) { 107146662Stackerman struct iidc_qos_params qos_info = {}; 108146662Stackerman 109146662Stackerman if (!iwdev->vsi.tc_change_pending) 110152740Sglebius return; 111152740Sglebius 112152740Sglebius l2params.tc_changed = true; 113152740Sglebius ibdev_dbg(&iwdev->ibdev, "CLNT: TC Change\n"); 114152740Sglebius ice_get_qos_params(pf, &qos_info); 115152740Sglebius irdma_fill_qos_info(&l2params, &qos_info); 116152740Sglebius if (iwdev->rf->protocol_used != IRDMA_IWARP_PROTOCOL_ONLY) 117152740Sglebius iwdev->dcb_vlan_mode = qos_info.num_tc > 1 && !l2params.dscp_mode; 118146662Stackerman irdma_change_l2params(&iwdev->vsi, &l2params); 119146662Stackerman } else if (*event->type & BIT(IIDC_EVENT_CRIT_ERR)) { 120152740Sglebius ibdev_warn(&iwdev->ibdev, "ICE OICR event notification: oicr = 0x%08x\n", 121146662Stackerman event->reg); 122108229Spdeuskar if (event->reg & IRDMAPFINT_OICR_PE_CRITERR_M) { 123108229Spdeuskar u32 pe_criterr; 12487189Spdeuskar 12587189Spdeuskar pe_criterr = readl(iwdev->rf->sc_dev.hw_regs[IRDMA_GLPE_CRITERR]); 12687189Spdeuskar#define IRDMA_Q1_RESOURCE_ERR 0x0001024d 12787189Spdeuskar if (pe_criterr != IRDMA_Q1_RESOURCE_ERR) { 12887189Spdeuskar ibdev_err(&iwdev->ibdev, "critical PE Error, GLPE_CRITERR=0x%08x\n", 12987189Spdeuskar pe_criterr); 13087189Spdeuskar iwdev->rf->reset = true; 13197785Spdeuskar } else { 13287189Spdeuskar ibdev_warn(&iwdev->ibdev, "Q1 Resource Check\n"); 13387189Spdeuskar } 13487189Spdeuskar } 13587189Spdeuskar if (event->reg & IRDMAPFINT_OICR_HMC_ERR_M) { 13687189Spdeuskar ibdev_err(&iwdev->ibdev, "HMC Error\n"); 13793914Spdeuskar iwdev->rf->reset = true; 13893914Spdeuskar } 13993914Spdeuskar if (event->reg & IRDMAPFINT_OICR_PE_PUSH_M) { 14093914Spdeuskar ibdev_err(&iwdev->ibdev, "PE Push Error\n"); 141153729Sglebius iwdev->rf->reset = true; 142153729Sglebius } 14392739Salfred if (iwdev->rf->reset) 14492739Salfred iwdev->rf->gen_ops.request_reset(iwdev->rf); 145153729Sglebius } 146112472Spdeuskar} 14792739Salfred 14892739Salfred/** 149120364Ssam * irdma_request_reset - Request a reset 15092739Salfred * @rf: RDMA PCI function 15192739Salfred */ 15293914Spdeuskarstatic void irdma_request_reset(struct irdma_pci_f *rf) 15392739Salfred{ 15493914Spdeuskar struct ice_pf *pf = rf->cdev; 15592739Salfred 15692739Salfred ibdev_warn(&rf->iwdev->ibdev, "Requesting a reset\n"); 15793914Spdeuskar ice_rdma_request_reset(pf, IIDC_PFR); 15892739Salfred} 15993914Spdeuskar 16092739Salfred/** 16193914Spdeuskar * irdma_lan_register_qset - Register qset with LAN driver 16292739Salfred * @vsi: vsi structure 16393914Spdeuskar * @tc_node: Traffic class node 16493914Spdeuskar */ 16592739Salfredstatic int irdma_lan_register_qset(struct irdma_sc_vsi *vsi, 16692739Salfred struct irdma_ws_node *tc_node) 16792739Salfred{ 16892739Salfred struct irdma_device *iwdev = vsi->back_vsi; 16993914Spdeuskar struct ice_pf *pf = iwdev->rf->cdev; 17093914Spdeuskar struct iidc_rdma_qset_params qset = {}; 171107242Sluigi int ret; 172153474Syongari 173153474Syongari qset.qs_handle = tc_node->qs_handle; 174153474Syongari qset.tc = tc_node->traffic_class; 17592739Salfred qset.vport_id = vsi->vsi_idx; 176112472Spdeuskar ret = ice_add_rdma_qset(pf, &qset); 177108229Spdeuskar if (ret) { 178102452Spdeuskar ibdev_dbg(&iwdev->ibdev, "WS: LAN alloc_res for rdma qset failed.\n"); 179108229Spdeuskar return ret; 180108229Spdeuskar } 181108229Spdeuskar 18292739Salfred tc_node->l2_sched_node_id = qset.teid; 18392739Salfred vsi->qos[tc_node->user_pri].l2_sched_node_id = qset.teid; 18492739Salfred 18592739Salfred return 0; 18692739Salfred} 187108229Spdeuskar 188106649Spdeuskar/** 189112472Spdeuskar * irdma_lan_unregister_qset - Unregister qset with LAN driver 190140859Syar * @vsi: vsi structure 191137609Srwatson * @tc_node: Traffic class node 192112472Spdeuskar */ 193112472Spdeuskarstatic void irdma_lan_unregister_qset(struct irdma_sc_vsi *vsi, 194112472Spdeuskar struct irdma_ws_node *tc_node) 195112472Spdeuskar{ 196112472Spdeuskar struct irdma_device *iwdev = vsi->back_vsi; 197121106Sdeischen struct ice_pf *pf = iwdev->rf->cdev; 198114554Spdeuskar struct iidc_rdma_qset_params qset = {}; 199114554Spdeuskar 200114554Spdeuskar qset.qs_handle = tc_node->qs_handle; 201115878Spdeuskar qset.tc = tc_node->traffic_class; 202115878Spdeuskar qset.vport_id = vsi->vsi_idx; 203115878Spdeuskar qset.teid = tc_node->l2_sched_node_id; 204115878Spdeuskar 205152225Syongari if (ice_del_rdma_qset(pf, &qset)) 206119509Spdeuskar ibdev_dbg(&iwdev->ibdev, "WS: LAN free_res for rdma qset failed.\n"); 207119509Spdeuskar} 208118314Sjdp 209118314Sjdpstatic void irdma_remove(struct auxiliary_device *aux_dev) 210118314Sjdp{ 211118314Sjdp struct iidc_auxiliary_dev *iidc_adev = container_of(aux_dev, 212150789Sglebius struct iidc_auxiliary_dev, 213150789Sglebius adev); 214150789Sglebius struct ice_pf *pf = iidc_adev->pf; 21597785Spdeuskar struct irdma_device *iwdev = auxiliary_get_drvdata(aux_dev); 21687189Spdeuskar 21787189Spdeuskar irdma_ib_unregister_device(iwdev); 21887189Spdeuskar ice_rdma_update_vsi_filter(pf, iwdev->vsi_num, false); 21987189Spdeuskar 22087189Spdeuskar pr_debug("INIT: Gen2 PF[%d] device remove success\n", PCI_FUNC(pf->pdev->devfn)); 22197785Spdeuskar} 22297785Spdeuskar 22397785Spdeuskarstatic void irdma_fill_device_info(struct irdma_device *iwdev, struct ice_pf *pf, 22497785Spdeuskar struct ice_vsi *vsi) 22597785Spdeuskar{ 226153729Sglebius struct irdma_pci_f *rf = iwdev->rf; 227153729Sglebius 22897785Spdeuskar rf->cdev = pf; 22987189Spdeuskar rf->gen_ops.register_qset = irdma_lan_register_qset; 23087189Spdeuskar rf->gen_ops.unregister_qset = irdma_lan_unregister_qset; 23187189Spdeuskar rf->hw.hw_addr = pf->hw.hw_addr; 23297785Spdeuskar rf->pcidev = pf->pdev; 23387189Spdeuskar rf->msix_count = pf->num_rdma_msix; 23487189Spdeuskar rf->pf_id = pf->hw.pf_id; 23587189Spdeuskar rf->msix_entries = &pf->msix_entries[pf->rdma_base_vector]; 236113506Smdodd rf->default_vsi.vsi_idx = vsi->vsi_num; 237113506Smdodd rf->protocol_used = pf->rdma_mode & IIDC_RDMA_PROTOCOL_ROCEV2 ? 238113506Smdodd IRDMA_ROCE_PROTOCOL_ONLY : IRDMA_IWARP_PROTOCOL_ONLY; 23987189Spdeuskar rf->rdma_ver = IRDMA_GEN_2; 24087189Spdeuskar rf->rsrc_profile = IRDMA_HMC_PROFILE_DEFAULT; 241118314Sjdp rf->rst_to = IRDMA_RST_TIMEOUT_HZ; 242118314Sjdp rf->gen_ops.request_reset = irdma_request_reset; 243118314Sjdp rf->limits_sel = 7; 244118314Sjdp rf->iwdev = iwdev; 245118314Sjdp mutex_init(&iwdev->ah_tbl_lock); 246118314Sjdp iwdev->netdev = vsi->netdev; 247118314Sjdp iwdev->vsi_num = vsi->vsi_num; 248118314Sjdp iwdev->init_state = INITIAL_STATE; 249118314Sjdp iwdev->roce_cwnd = IRDMA_ROCE_CWND_DEFAULT; 250118314Sjdp iwdev->roce_ackcreds = IRDMA_ROCE_ACKCREDS_DEFAULT; 251152545Sglebius iwdev->rcv_wnd = IRDMA_CM_DEFAULT_RCV_WND_SCALED; 252152545Sglebius iwdev->rcv_wscale = IRDMA_CM_DEFAULT_RCV_WND_SCALE; 253118314Sjdp if (rf->protocol_used == IRDMA_ROCE_PROTOCOL_ONLY) 254118314Sjdp iwdev->roce_mode = true; 255118314Sjdp} 256118314Sjdp 257118314Sjdpstatic int irdma_probe(struct auxiliary_device *aux_dev, const struct auxiliary_device_id *id) 258152545Sglebius{ 259152545Sglebius struct iidc_auxiliary_dev *iidc_adev = container_of(aux_dev, 260118314Sjdp struct iidc_auxiliary_dev, 261118314Sjdp adev); 26287189Spdeuskar struct ice_pf *pf = iidc_adev->pf; 26387189Spdeuskar struct ice_vsi *vsi = ice_get_main_vsi(pf); 26487189Spdeuskar struct iidc_qos_params qos_info = {}; 26587189Spdeuskar struct irdma_device *iwdev; 26687189Spdeuskar struct irdma_pci_f *rf; 267143161Simp struct irdma_l2params l2params = {}; 26887189Spdeuskar int err; 26987189Spdeuskar 27087189Spdeuskar if (!vsi) 27187189Spdeuskar return -EIO; 27287189Spdeuskar iwdev = ib_alloc_device(irdma_device, ibdev); 27397785Spdeuskar if (!iwdev) 27487189Spdeuskar return -ENOMEM; 27597785Spdeuskar iwdev->rf = kzalloc(sizeof(*rf), GFP_KERNEL); 27697785Spdeuskar if (!iwdev->rf) { 27797785Spdeuskar ib_dealloc_device(&iwdev->ibdev); 27897785Spdeuskar return -ENOMEM; 27997785Spdeuskar } 28087189Spdeuskar 28197785Spdeuskar irdma_fill_device_info(iwdev, pf, vsi); 28287189Spdeuskar rf = iwdev->rf; 28397785Spdeuskar 28497785Spdeuskar err = irdma_ctrl_init_hw(rf); 28597785Spdeuskar if (err) 28687189Spdeuskar goto err_ctrl_init; 28797785Spdeuskar 28897785Spdeuskar l2params.mtu = iwdev->netdev->mtu; 28997785Spdeuskar ice_get_qos_params(pf, &qos_info); 29087189Spdeuskar irdma_fill_qos_info(&l2params, &qos_info); 29197785Spdeuskar if (iwdev->rf->protocol_used != IRDMA_IWARP_PROTOCOL_ONLY) 29297785Spdeuskar iwdev->dcb_vlan_mode = l2params.num_tc > 1 && !l2params.dscp_mode; 29397785Spdeuskar 29497785Spdeuskar err = irdma_rt_init_hw(iwdev, &l2params); 29587189Spdeuskar if (err) 29697785Spdeuskar goto err_rt_init; 29797785Spdeuskar 29887189Spdeuskar err = irdma_ib_register_device(iwdev); 29997785Spdeuskar if (err) 30097785Spdeuskar goto err_ibreg; 301152740Sglebius 30297785Spdeuskar ice_rdma_update_vsi_filter(pf, iwdev->vsi_num, true); 30397785Spdeuskar 30497785Spdeuskar ibdev_dbg(&iwdev->ibdev, "INIT: Gen2 PF[%d] device probe success\n", PCI_FUNC(rf->pcidev->devfn)); 305143161Simp auxiliary_set_drvdata(aux_dev, iwdev); 30697785Spdeuskar 30797785Spdeuskar return 0; 30897785Spdeuskar 30997785Spdeuskarerr_ibreg: 31097785Spdeuskar irdma_rt_deinit_hw(iwdev); 31187189Spdeuskarerr_rt_init: 31287189Spdeuskar irdma_ctrl_deinit_hw(rf); 31387189Spdeuskarerr_ctrl_init: 31487189Spdeuskar kfree(iwdev->rf); 31587189Spdeuskar ib_dealloc_device(&iwdev->ibdev); 31687189Spdeuskar 31787189Spdeuskar return err; 31887189Spdeuskar} 31987189Spdeuskar 32087189Spdeuskarstatic const struct auxiliary_device_id irdma_auxiliary_id_table[] = { 32187189Spdeuskar {.name = "ice.iwarp", }, 32287189Spdeuskar {.name = "ice.roce", }, 32387189Spdeuskar {}, 32487189Spdeuskar}; 32587189Spdeuskar 32697785SpdeuskarMODULE_DEVICE_TABLE(auxiliary, irdma_auxiliary_id_table); 32797785Spdeuskar 328115878Spdeuskarstatic struct iidc_auxiliary_drv irdma_auxiliary_drv = { 32987189Spdeuskar .adrv = { 33097785Spdeuskar .id_table = irdma_auxiliary_id_table, 33187189Spdeuskar .probe = irdma_probe, 33297785Spdeuskar .remove = irdma_remove, 33397785Spdeuskar }, 33497785Spdeuskar .event_handler = irdma_iidc_event_handler, 33597785Spdeuskar}; 33697785Spdeuskar 33797785Spdeuskarstatic int __init irdma_init_module(void) 33897785Spdeuskar{ 33997785Spdeuskar int ret; 34097785Spdeuskar 341120989Ssam ret = auxiliary_driver_register(&i40iw_auxiliary_drv); 34287189Spdeuskar if (ret) { 343115878Spdeuskar pr_err("Failed i40iw(gen_1) auxiliary_driver_register() ret=%d\n", 344137575Sbms ret); 345137575Sbms return ret; 346115878Spdeuskar } 347115878Spdeuskar 348115878Spdeuskar ret = auxiliary_driver_register(&irdma_auxiliary_drv.adrv); 349115878Spdeuskar if (ret) { 350137575Sbms auxiliary_driver_unregister(&i40iw_auxiliary_drv); 351137575Sbms pr_err("Failed irdma auxiliary_driver_register() ret=%d\n", 352115878Spdeuskar ret); 353115878Spdeuskar return ret; 354115878Spdeuskar } 355115878Spdeuskar 356120364Ssam irdma_register_notifiers(); 357120364Ssam 35887189Spdeuskar return 0; 35997785Spdeuskar} 36097785Spdeuskar 361118314Sjdpstatic void __exit irdma_exit_module(void) 362118314Sjdp{ 363118314Sjdp irdma_unregister_notifiers(); 364118314Sjdp auxiliary_driver_unregister(&irdma_auxiliary_drv.adrv); 365118314Sjdp auxiliary_driver_unregister(&i40iw_auxiliary_drv); 366118314Sjdp} 367118314Sjdp 368118314Sjdpmodule_init(irdma_init_module); 369118314Sjdpmodule_exit(irdma_exit_module); 370118314Sjdp