1/* SPDX-License-Identifier: BSD-3-Clause */
2/* Copyright(c) 2007-2022 Intel Corporation */
3#include "qat_freebsd.h"
4#include <sys/kernel.h>
5#include <sys/systm.h>
6
7#include <sys/types.h>
8#include <sys/interrupt.h>
9#include <dev/pci/pcivar.h>
10#include <sys/param.h>
11#include <linux/workqueue.h>
12#include "adf_accel_devices.h"
13#include "adf_common_drv.h"
14#include "adf_cfg.h"
15#include "adf_cfg_strings.h"
16#include "adf_cfg_common.h"
17#include "adf_transport_access_macros.h"
18#include "adf_transport_internal.h"
19#include "adf_pfvf_utils.h"
20
21static TASKQUEUE_DEFINE_THREAD(qat_vf);
22static TASKQUEUE_DEFINE_THREAD(qat_bank_handler);
23
24static struct workqueue_struct *adf_vf_stop_wq;
25static DEFINE_MUTEX(vf_stop_wq_lock);
26
27struct adf_vf_stop_data {
28	struct adf_accel_dev *accel_dev;
29	struct work_struct work;
30};
31
32static int
33adf_enable_msi(struct adf_accel_dev *accel_dev)
34{
35	int stat;
36	int count = 1;
37	stat = pci_alloc_msi(accel_to_pci_dev(accel_dev), &count);
38	if (stat) {
39		device_printf(GET_DEV(accel_dev),
40			      "Failed to enable MSI interrupts\n");
41		return stat;
42	}
43
44	return stat;
45}
46
47static void
48adf_disable_msi(struct adf_accel_dev *accel_dev)
49{
50	device_t pdev = accel_to_pci_dev(accel_dev);
51	pci_release_msi(pdev);
52}
53
54static void
55adf_dev_stop_async(struct work_struct *work)
56{
57	struct adf_vf_stop_data *stop_data =
58	    container_of(work, struct adf_vf_stop_data, work);
59	struct adf_accel_dev *accel_dev = stop_data->accel_dev;
60	struct adf_hw_device_data *hw_data = accel_dev->hw_device;
61
62	adf_dev_restarting_notify(accel_dev);
63	adf_dev_stop(accel_dev);
64	adf_dev_shutdown(accel_dev);
65
66	/* Re-enable PF2VF interrupts */
67	hw_data->enable_pf2vf_interrupt(accel_dev);
68	kfree(stop_data);
69}
70
71int
72adf_pf2vf_handle_pf_restarting(struct adf_accel_dev *accel_dev)
73{
74	struct adf_vf_stop_data *stop_data;
75
76	clear_bit(ADF_STATUS_PF_RUNNING, &accel_dev->status);
77	stop_data = kzalloc(sizeof(*stop_data), GFP_ATOMIC);
78	if (!stop_data) {
79		device_printf(GET_DEV(accel_dev),
80			      "Couldn't schedule stop for vf_%d\n",
81			      accel_dev->accel_id);
82		return -ENOMEM;
83	}
84	stop_data->accel_dev = accel_dev;
85	INIT_WORK(&stop_data->work, adf_dev_stop_async);
86	queue_work(adf_vf_stop_wq, &stop_data->work);
87
88	return 0;
89}
90
91int
92adf_pf2vf_handle_pf_rp_reset(struct adf_accel_dev *accel_dev,
93			     struct pfvf_message msg)
94{
95	accel_dev->u1.vf.rpreset_sts = msg.data;
96	if (accel_dev->u1.vf.rpreset_sts == RPRESET_SUCCESS)
97		device_printf(
98		    GET_DEV(accel_dev),
99		    "rpreset resp(success) from PF type:0x%x data:0x%x\n",
100		    msg.type,
101		    msg.data);
102	else if (accel_dev->u1.vf.rpreset_sts == RPRESET_NOT_SUPPORTED)
103		device_printf(
104		    GET_DEV(accel_dev),
105		    "rpreset resp(not supported) from PF type:0x%x data:0x%x\n",
106		    msg.type,
107		    msg.data);
108	else if (accel_dev->u1.vf.rpreset_sts == RPRESET_INVAL_BANK)
109		device_printf(
110		    GET_DEV(accel_dev),
111		    "rpreset resp(invalid bank) from PF type:0x%x data:0x%x\n",
112		    msg.type,
113		    msg.data);
114	else
115		device_printf(
116		    GET_DEV(accel_dev),
117		    "rpreset resp(timeout) from PF type:0x%x data:0x%x\nn",
118		    msg.type,
119		    msg.data);
120
121	complete(&accel_dev->u1.vf.msg_received);
122
123	return 0;
124}
125
126static void
127adf_pf2vf_bh_handler(void *data, int pending)
128{
129	struct adf_accel_dev *accel_dev = data;
130	struct adf_hw_device_data *hw_data = accel_dev->hw_device;
131
132	if (adf_recv_and_handle_pf2vf_msg(accel_dev))
133		/* Re-enable PF2VF interrupts */
134		hw_data->enable_pf2vf_interrupt(accel_dev);
135
136	return;
137}
138
139static int
140adf_setup_pf2vf_bh(struct adf_accel_dev *accel_dev)
141{
142	TASK_INIT(&accel_dev->u1.vf.pf2vf_bh_tasklet,
143		  0,
144		  adf_pf2vf_bh_handler,
145		  accel_dev);
146	mutex_init(&accel_dev->u1.vf.vf2pf_lock);
147
148	return 0;
149}
150
151static void
152adf_cleanup_pf2vf_bh(struct adf_accel_dev *accel_dev)
153{
154	taskqueue_cancel(taskqueue_qat_vf,
155			 &accel_dev->u1.vf.pf2vf_bh_tasklet,
156			 NULL);
157	taskqueue_drain(taskqueue_qat_vf, &accel_dev->u1.vf.pf2vf_bh_tasklet);
158	mutex_destroy(&accel_dev->u1.vf.vf2pf_lock);
159}
160
161static void
162adf_bh_handler(void *data, int pending)
163{
164	struct adf_etr_bank_data *bank = (void *)data;
165
166	adf_response_handler((uintptr_t)bank);
167
168	return;
169}
170
171static int
172adf_setup_bh(struct adf_accel_dev *accel_dev)
173{
174	int i = 0;
175	struct adf_etr_data *priv_data = accel_dev->transport;
176
177	for (i = 0; i < GET_MAX_BANKS(accel_dev); i++) {
178		TASK_INIT(&priv_data->banks[i].resp_handler,
179			  0,
180			  adf_bh_handler,
181			  &priv_data->banks[i]);
182	}
183
184	return 0;
185}
186
187static void
188adf_cleanup_bh(struct adf_accel_dev *accel_dev)
189{
190	int i = 0;
191	struct adf_etr_data *transport;
192
193	if (!accel_dev || !accel_dev->transport)
194		return;
195
196	transport = accel_dev->transport;
197	for (i = 0; i < GET_MAX_BANKS(accel_dev); i++) {
198		taskqueue_cancel(taskqueue_qat_bank_handler,
199				 &transport->banks[i].resp_handler,
200				 NULL);
201		taskqueue_drain(taskqueue_qat_bank_handler,
202				&transport->banks[i].resp_handler);
203	}
204}
205
206static void
207adf_isr(void *privdata)
208{
209	struct adf_accel_dev *accel_dev = privdata;
210	struct adf_hw_device_data *hw_data = accel_dev->hw_device;
211	struct adf_hw_csr_ops *csr_ops = &hw_data->csr_info.csr_ops;
212	int int_active_bundles = 0;
213	int i = 0;
214
215	/* Check for PF2VF interrupt */
216	if (hw_data->interrupt_active_pf2vf(accel_dev)) {
217		/* Disable PF to VF interrupt */
218		hw_data->disable_pf2vf_interrupt(accel_dev);
219		/* Schedule tasklet to handle interrupt BH */
220		taskqueue_enqueue(taskqueue_qat_vf,
221				  &accel_dev->u1.vf.pf2vf_bh_tasklet);
222	}
223
224	if (hw_data->get_int_active_bundles)
225		int_active_bundles = hw_data->get_int_active_bundles(accel_dev);
226
227	for (i = 0; i < GET_MAX_BANKS(accel_dev); i++) {
228		if (int_active_bundles & BIT(i)) {
229			struct adf_etr_data *etr_data = accel_dev->transport;
230			struct adf_etr_bank_data *bank = &etr_data->banks[i];
231
232			/* Disable Flag and Coalesce Ring Interrupts */
233			csr_ops->write_csr_int_flag_and_col(bank->csr_addr,
234							    bank->bank_number,
235							    0);
236			/* Schedule tasklet to handle interrupt BH */
237			taskqueue_enqueue(taskqueue_qat_bank_handler,
238					  &bank->resp_handler);
239		}
240	}
241}
242
243static int
244adf_request_msi_irq(struct adf_accel_dev *accel_dev)
245{
246	device_t pdev = accel_to_pci_dev(accel_dev);
247	int ret;
248	int rid = 1;
249	int cpu;
250
251	accel_dev->u1.vf.irq =
252	    bus_alloc_resource_any(pdev, SYS_RES_IRQ, &rid, RF_ACTIVE);
253	if (accel_dev->u1.vf.irq == NULL) {
254		device_printf(GET_DEV(accel_dev), "failed to allocate IRQ\n");
255		return ENXIO;
256	}
257	ret = bus_setup_intr(pdev,
258			     accel_dev->u1.vf.irq,
259			     INTR_TYPE_MISC | INTR_MPSAFE,
260			     NULL,
261			     adf_isr,
262			     accel_dev,
263			     &accel_dev->u1.vf.cookie);
264	if (ret) {
265		device_printf(GET_DEV(accel_dev), "failed to enable irq\n");
266		goto errout;
267	}
268
269	cpu = accel_dev->accel_id % num_online_cpus();
270	ret = bus_bind_intr(pdev, accel_dev->u1.vf.irq, cpu);
271	if (ret) {
272		device_printf(GET_DEV(accel_dev),
273			      "failed to bind IRQ handler to cpu core\n");
274		goto errout;
275	}
276	accel_dev->u1.vf.irq_enabled = true;
277
278	return ret;
279errout:
280	bus_free_resource(pdev, SYS_RES_IRQ, accel_dev->u1.vf.irq);
281
282	return ret;
283}
284
285/**
286 * adf_vf_isr_resource_free() - Free IRQ for acceleration device
287 * @accel_dev:  Pointer to acceleration device.
288 *
289 * Function frees interrupts for acceleration device virtual function.
290 */
291void
292adf_vf_isr_resource_free(struct adf_accel_dev *accel_dev)
293{
294	device_t pdev = accel_to_pci_dev(accel_dev);
295
296	if (accel_dev->u1.vf.irq_enabled) {
297		bus_teardown_intr(pdev,
298				  accel_dev->u1.vf.irq,
299				  accel_dev->u1.vf.cookie);
300		bus_free_resource(pdev, SYS_RES_IRQ, accel_dev->u1.vf.irq);
301	}
302	adf_cleanup_bh(accel_dev);
303	adf_cleanup_pf2vf_bh(accel_dev);
304	adf_disable_msi(accel_dev);
305}
306
307/**
308 * adf_vf_isr_resource_alloc() - Allocate IRQ for acceleration device
309 * @accel_dev:  Pointer to acceleration device.
310 *
311 * Function allocates interrupts for acceleration device virtual function.
312 *
313 * Return: 0 on success, error code otherwise.
314 */
315int
316adf_vf_isr_resource_alloc(struct adf_accel_dev *accel_dev)
317{
318	if (adf_enable_msi(accel_dev))
319		goto err_out;
320
321	if (adf_setup_pf2vf_bh(accel_dev))
322		goto err_disable_msi;
323
324	if (adf_setup_bh(accel_dev))
325		goto err_out;
326
327	if (adf_request_msi_irq(accel_dev))
328		goto err_disable_msi;
329
330	return 0;
331
332err_disable_msi:
333	adf_disable_msi(accel_dev);
334
335err_out:
336	return -EFAULT;
337}
338
339/**
340 * adf_flush_vf_wq() - Flush workqueue for VF
341 * @accel_dev:  Pointer to acceleration device.
342 *
343 * Function disables the PF/VF interrupts on the VF so that no new messages
344 * are received and flushes the workqueue 'adf_vf_stop_wq'.
345 *
346 * Return: void.
347 */
348void
349adf_flush_vf_wq(struct adf_accel_dev *accel_dev)
350{
351	struct adf_hw_device_data *hw_data = accel_dev->hw_device;
352
353	hw_data->disable_pf2vf_interrupt(accel_dev);
354
355	if (adf_vf_stop_wq)
356		flush_workqueue(adf_vf_stop_wq);
357}
358
359/**
360 * adf_init_vf_wq() - Init workqueue for VF
361 *
362 * Function init workqueue 'adf_vf_stop_wq' for VF.
363 *
364 * Return: 0 on success, error code otherwise.
365 */
366int
367adf_init_vf_wq(void)
368{
369	int ret = 0;
370
371	mutex_lock(&vf_stop_wq_lock);
372	if (!adf_vf_stop_wq)
373		adf_vf_stop_wq =
374		    alloc_workqueue("adf_vf_stop_wq", WQ_MEM_RECLAIM, 0);
375
376	if (!adf_vf_stop_wq)
377		ret = ENOMEM;
378
379	mutex_unlock(&vf_stop_wq_lock);
380	return ret;
381}
382
383void
384adf_exit_vf_wq(void)
385{
386	if (adf_vf_stop_wq)
387		destroy_workqueue(adf_vf_stop_wq);
388
389	adf_vf_stop_wq = NULL;
390}
391