1// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) 2/* Copyright(c) 2014 - 2020 Intel Corporation */ 3#include <linux/kernel.h> 4#include <linux/init.h> 5#include <linux/types.h> 6#include <linux/pci.h> 7#include <linux/slab.h> 8#include <linux/errno.h> 9#include <linux/interrupt.h> 10#include <linux/workqueue.h> 11#include "adf_accel_devices.h" 12#include "adf_common_drv.h" 13#include "adf_cfg.h" 14#include "adf_cfg_strings.h" 15#include "adf_cfg_common.h" 16#include "adf_transport_access_macros.h" 17#include "adf_transport_internal.h" 18 19#define ADF_VINTSOU_OFFSET 0x204 20#define ADF_VINTMSK_OFFSET 0x208 21#define ADF_VINTSOU_BUN BIT(0) 22#define ADF_VINTSOU_PF2VF BIT(1) 23 24static struct workqueue_struct *adf_vf_stop_wq; 25 26struct adf_vf_stop_data { 27 struct adf_accel_dev *accel_dev; 28 struct work_struct work; 29}; 30 31void adf_enable_pf2vf_interrupts(struct adf_accel_dev *accel_dev) 32{ 33 void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev); 34 35 ADF_CSR_WR(pmisc_addr, ADF_VINTMSK_OFFSET, 0x0); 36} 37 38void adf_disable_pf2vf_interrupts(struct adf_accel_dev *accel_dev) 39{ 40 void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev); 41 42 ADF_CSR_WR(pmisc_addr, ADF_VINTMSK_OFFSET, 0x2); 43} 44EXPORT_SYMBOL_GPL(adf_disable_pf2vf_interrupts); 45 46static int adf_enable_msi(struct adf_accel_dev *accel_dev) 47{ 48 struct adf_accel_pci *pci_dev_info = &accel_dev->accel_pci_dev; 49 int stat = pci_alloc_irq_vectors(pci_dev_info->pci_dev, 1, 1, 50 PCI_IRQ_MSI); 51 if (unlikely(stat < 0)) { 52 dev_err(&GET_DEV(accel_dev), 53 "Failed to enable MSI interrupt: %d\n", stat); 54 return stat; 55 } 56 57 return 0; 58} 59 60static void adf_disable_msi(struct adf_accel_dev *accel_dev) 61{ 62 struct pci_dev *pdev = accel_to_pci_dev(accel_dev); 63 64 pci_free_irq_vectors(pdev); 65} 66 67static void adf_dev_stop_async(struct work_struct *work) 68{ 69 struct adf_vf_stop_data *stop_data = 70 container_of(work, struct adf_vf_stop_data, work); 71 struct adf_accel_dev *accel_dev = stop_data->accel_dev; 72 73 adf_dev_restarting_notify(accel_dev); 74 adf_dev_down(accel_dev, false); 75 76 /* Re-enable PF2VF interrupts */ 77 adf_enable_pf2vf_interrupts(accel_dev); 78 kfree(stop_data); 79} 80 81int adf_pf2vf_handle_pf_restarting(struct adf_accel_dev *accel_dev) 82{ 83 struct adf_vf_stop_data *stop_data; 84 85 clear_bit(ADF_STATUS_PF_RUNNING, &accel_dev->status); 86 stop_data = kzalloc(sizeof(*stop_data), GFP_ATOMIC); 87 if (!stop_data) { 88 dev_err(&GET_DEV(accel_dev), 89 "Couldn't schedule stop for vf_%d\n", 90 accel_dev->accel_id); 91 return -ENOMEM; 92 } 93 stop_data->accel_dev = accel_dev; 94 INIT_WORK(&stop_data->work, adf_dev_stop_async); 95 queue_work(adf_vf_stop_wq, &stop_data->work); 96 97 return 0; 98} 99 100static void adf_pf2vf_bh_handler(void *data) 101{ 102 struct adf_accel_dev *accel_dev = data; 103 bool ret; 104 105 ret = adf_recv_and_handle_pf2vf_msg(accel_dev); 106 if (ret) 107 /* Re-enable PF2VF interrupts */ 108 adf_enable_pf2vf_interrupts(accel_dev); 109 110 return; 111 112} 113 114static int adf_setup_pf2vf_bh(struct adf_accel_dev *accel_dev) 115{ 116 tasklet_init(&accel_dev->vf.pf2vf_bh_tasklet, 117 (void *)adf_pf2vf_bh_handler, (unsigned long)accel_dev); 118 119 mutex_init(&accel_dev->vf.vf2pf_lock); 120 return 0; 121} 122 123static void adf_cleanup_pf2vf_bh(struct adf_accel_dev *accel_dev) 124{ 125 tasklet_disable(&accel_dev->vf.pf2vf_bh_tasklet); 126 tasklet_kill(&accel_dev->vf.pf2vf_bh_tasklet); 127 mutex_destroy(&accel_dev->vf.vf2pf_lock); 128} 129 130static irqreturn_t adf_isr(int irq, void *privdata) 131{ 132 struct adf_accel_dev *accel_dev = privdata; 133 struct adf_hw_device_data *hw_data = accel_dev->hw_device; 134 struct adf_hw_csr_ops *csr_ops = &hw_data->csr_ops; 135 struct adf_bar *pmisc = 136 &GET_BARS(accel_dev)[hw_data->get_misc_bar_id(hw_data)]; 137 void __iomem *pmisc_bar_addr = pmisc->virt_addr; 138 bool handled = false; 139 u32 v_int, v_mask; 140 141 /* Read VF INT source CSR to determine the source of VF interrupt */ 142 v_int = ADF_CSR_RD(pmisc_bar_addr, ADF_VINTSOU_OFFSET); 143 144 /* Read VF INT mask CSR to determine which sources are masked */ 145 v_mask = ADF_CSR_RD(pmisc_bar_addr, ADF_VINTMSK_OFFSET); 146 147 /* 148 * Recompute v_int ignoring sources that are masked. This is to 149 * avoid rescheduling the tasklet for interrupts already handled 150 */ 151 v_int &= ~v_mask; 152 153 /* Check for PF2VF interrupt */ 154 if (v_int & ADF_VINTSOU_PF2VF) { 155 /* Disable PF to VF interrupt */ 156 adf_disable_pf2vf_interrupts(accel_dev); 157 158 /* Schedule tasklet to handle interrupt BH */ 159 tasklet_hi_schedule(&accel_dev->vf.pf2vf_bh_tasklet); 160 handled = true; 161 } 162 163 /* Check bundle interrupt */ 164 if (v_int & ADF_VINTSOU_BUN) { 165 struct adf_etr_data *etr_data = accel_dev->transport; 166 struct adf_etr_bank_data *bank = &etr_data->banks[0]; 167 168 /* Disable Flag and Coalesce Ring Interrupts */ 169 csr_ops->write_csr_int_flag_and_col(bank->csr_addr, 170 bank->bank_number, 0); 171 tasklet_hi_schedule(&bank->resp_handler); 172 handled = true; 173 } 174 175 return handled ? IRQ_HANDLED : IRQ_NONE; 176} 177 178static int adf_request_msi_irq(struct adf_accel_dev *accel_dev) 179{ 180 struct pci_dev *pdev = accel_to_pci_dev(accel_dev); 181 unsigned int cpu; 182 int ret; 183 184 snprintf(accel_dev->vf.irq_name, ADF_MAX_MSIX_VECTOR_NAME, 185 "qat_%02x:%02d.%02d", pdev->bus->number, PCI_SLOT(pdev->devfn), 186 PCI_FUNC(pdev->devfn)); 187 ret = request_irq(pdev->irq, adf_isr, 0, accel_dev->vf.irq_name, 188 (void *)accel_dev); 189 if (ret) { 190 dev_err(&GET_DEV(accel_dev), "failed to enable irq for %s\n", 191 accel_dev->vf.irq_name); 192 return ret; 193 } 194 cpu = accel_dev->accel_id % num_online_cpus(); 195 irq_set_affinity_hint(pdev->irq, get_cpu_mask(cpu)); 196 accel_dev->vf.irq_enabled = true; 197 198 return ret; 199} 200 201static int adf_setup_bh(struct adf_accel_dev *accel_dev) 202{ 203 struct adf_etr_data *priv_data = accel_dev->transport; 204 205 tasklet_init(&priv_data->banks[0].resp_handler, adf_response_handler, 206 (unsigned long)priv_data->banks); 207 return 0; 208} 209 210static void adf_cleanup_bh(struct adf_accel_dev *accel_dev) 211{ 212 struct adf_etr_data *priv_data = accel_dev->transport; 213 214 tasklet_disable(&priv_data->banks[0].resp_handler); 215 tasklet_kill(&priv_data->banks[0].resp_handler); 216} 217 218/** 219 * adf_vf_isr_resource_free() - Free IRQ for acceleration device 220 * @accel_dev: Pointer to acceleration device. 221 * 222 * Function frees interrupts for acceleration device virtual function. 223 */ 224void adf_vf_isr_resource_free(struct adf_accel_dev *accel_dev) 225{ 226 struct pci_dev *pdev = accel_to_pci_dev(accel_dev); 227 228 if (accel_dev->vf.irq_enabled) { 229 irq_set_affinity_hint(pdev->irq, NULL); 230 free_irq(pdev->irq, accel_dev); 231 } 232 adf_cleanup_bh(accel_dev); 233 adf_cleanup_pf2vf_bh(accel_dev); 234 adf_disable_msi(accel_dev); 235} 236EXPORT_SYMBOL_GPL(adf_vf_isr_resource_free); 237 238/** 239 * adf_vf_isr_resource_alloc() - Allocate IRQ for acceleration device 240 * @accel_dev: Pointer to acceleration device. 241 * 242 * Function allocates interrupts for acceleration device virtual function. 243 * 244 * Return: 0 on success, error code otherwise. 245 */ 246int adf_vf_isr_resource_alloc(struct adf_accel_dev *accel_dev) 247{ 248 if (adf_enable_msi(accel_dev)) 249 goto err_out; 250 251 if (adf_setup_pf2vf_bh(accel_dev)) 252 goto err_disable_msi; 253 254 if (adf_setup_bh(accel_dev)) 255 goto err_cleanup_pf2vf_bh; 256 257 if (adf_request_msi_irq(accel_dev)) 258 goto err_cleanup_bh; 259 260 return 0; 261 262err_cleanup_bh: 263 adf_cleanup_bh(accel_dev); 264 265err_cleanup_pf2vf_bh: 266 adf_cleanup_pf2vf_bh(accel_dev); 267 268err_disable_msi: 269 adf_disable_msi(accel_dev); 270 271err_out: 272 return -EFAULT; 273} 274EXPORT_SYMBOL_GPL(adf_vf_isr_resource_alloc); 275 276/** 277 * adf_flush_vf_wq() - Flush workqueue for VF 278 * @accel_dev: Pointer to acceleration device. 279 * 280 * Function disables the PF/VF interrupts on the VF so that no new messages 281 * are received and flushes the workqueue 'adf_vf_stop_wq'. 282 * 283 * Return: void. 284 */ 285void adf_flush_vf_wq(struct adf_accel_dev *accel_dev) 286{ 287 adf_disable_pf2vf_interrupts(accel_dev); 288 289 flush_workqueue(adf_vf_stop_wq); 290} 291EXPORT_SYMBOL_GPL(adf_flush_vf_wq); 292 293/** 294 * adf_init_vf_wq() - Init workqueue for VF 295 * 296 * Return: 0 on success, error code otherwise. 297 */ 298int __init adf_init_vf_wq(void) 299{ 300 adf_vf_stop_wq = alloc_workqueue("adf_vf_stop_wq", WQ_MEM_RECLAIM, 0); 301 302 return !adf_vf_stop_wq ? -EFAULT : 0; 303} 304 305void adf_exit_vf_wq(void) 306{ 307 if (adf_vf_stop_wq) 308 destroy_workqueue(adf_vf_stop_wq); 309 310 adf_vf_stop_wq = NULL; 311} 312