1// SPDX-License-Identifier: GPL-2.0-only
2#include <linux/delay.h>
3#include <linux/firmware.h>
4#include <linux/list.h>
5#include <linux/module.h>
6#include <linux/mutex.h>
7#include <linux/pci.h>
8#include <linux/pci_ids.h>
9
10#include "nitrox_dev.h"
11#include "nitrox_common.h"
12#include "nitrox_csr.h"
13#include "nitrox_hal.h"
14#include "nitrox_isr.h"
15#include "nitrox_debugfs.h"
16
17#define CNN55XX_DEV_ID	0x12
18#define UCODE_HLEN 48
19#define DEFAULT_SE_GROUP 0
20#define DEFAULT_AE_GROUP 0
21
22#define DRIVER_VERSION "1.2"
23#define CNN55XX_UCD_BLOCK_SIZE 32768
24#define CNN55XX_MAX_UCODE_SIZE (CNN55XX_UCD_BLOCK_SIZE * 2)
25#define FW_DIR "cavium/"
26/* SE microcode */
27#define SE_FW	FW_DIR "cnn55xx_se.fw"
28/* AE microcode */
29#define AE_FW	FW_DIR "cnn55xx_ae.fw"
30
31static const char nitrox_driver_name[] = "CNN55XX";
32
33static LIST_HEAD(ndevlist);
34static DEFINE_MUTEX(devlist_lock);
35static unsigned int num_devices;
36
37/*
38 * nitrox_pci_tbl - PCI Device ID Table
39 */
40static const struct pci_device_id nitrox_pci_tbl[] = {
41	{PCI_VDEVICE(CAVIUM, CNN55XX_DEV_ID), 0},
42	/* required last entry */
43	{0, }
44};
45MODULE_DEVICE_TABLE(pci, nitrox_pci_tbl);
46
47static unsigned int qlen = DEFAULT_CMD_QLEN;
48module_param(qlen, uint, 0644);
49MODULE_PARM_DESC(qlen, "Command queue length - default 2048");
50
51/**
52 * struct ucode - Firmware Header
53 * @id: microcode ID
54 * @version: firmware version
55 * @code_size: code section size
56 * @raz: alignment
57 * @code: code section
58 */
59struct ucode {
60	u8 id;
61	char version[VERSION_LEN - 1];
62	__be32 code_size;
63	u8 raz[12];
64	u64 code[];
65};
66
67/*
68 * write_to_ucd_unit - Write Firmware to NITROX UCD unit
69 */
70static void write_to_ucd_unit(struct nitrox_device *ndev, u32 ucode_size,
71			      u64 *ucode_data, int block_num)
72{
73	u32 code_size;
74	u64 offset, data;
75	int i = 0;
76
77	/*
78	 * UCD structure
79	 *
80	 *  -------------
81	 *  |    BLK 7  |
82	 *  -------------
83	 *  |    BLK 6  |
84	 *  -------------
85	 *  |    ...    |
86	 *  -------------
87	 *  |    BLK 0  |
88	 *  -------------
89	 *  Total of 8 blocks, each size 32KB
90	 */
91
92	/* set the block number */
93	offset = UCD_UCODE_LOAD_BLOCK_NUM;
94	nitrox_write_csr(ndev, offset, block_num);
95
96	code_size = roundup(ucode_size, 16);
97	while (code_size) {
98		data = ucode_data[i];
99		/* write 8 bytes at a time */
100		offset = UCD_UCODE_LOAD_IDX_DATAX(i);
101		nitrox_write_csr(ndev, offset, data);
102		code_size -= 8;
103		i++;
104	}
105
106	usleep_range(300, 400);
107}
108
109static int nitrox_load_fw(struct nitrox_device *ndev)
110{
111	const struct firmware *fw;
112	const char *fw_name;
113	struct ucode *ucode;
114	u64 *ucode_data;
115	u64 offset;
116	union ucd_core_eid_ucode_block_num core_2_eid_val;
117	union aqm_grp_execmsk_lo aqm_grp_execmask_lo;
118	union aqm_grp_execmsk_hi aqm_grp_execmask_hi;
119	u32 ucode_size;
120	int ret, i = 0;
121
122	fw_name = SE_FW;
123	dev_info(DEV(ndev), "Loading firmware \"%s\"\n", fw_name);
124
125	ret = request_firmware(&fw, fw_name, DEV(ndev));
126	if (ret < 0) {
127		dev_err(DEV(ndev), "failed to get firmware %s\n", fw_name);
128		return ret;
129	}
130
131	ucode = (struct ucode *)fw->data;
132
133	ucode_size = be32_to_cpu(ucode->code_size) * 2;
134	if (!ucode_size || ucode_size > CNN55XX_MAX_UCODE_SIZE) {
135		dev_err(DEV(ndev), "Invalid ucode size: %u for firmware %s\n",
136			ucode_size, fw_name);
137		release_firmware(fw);
138		return -EINVAL;
139	}
140	ucode_data = ucode->code;
141
142	/* copy the firmware version */
143	memcpy(&ndev->hw.fw_name[0][0], ucode->version, (VERSION_LEN - 2));
144	ndev->hw.fw_name[0][VERSION_LEN - 1] = '\0';
145
146	/* Load SE Firmware on UCD Block 0 */
147	write_to_ucd_unit(ndev, ucode_size, ucode_data, 0);
148
149	release_firmware(fw);
150
151	/* put all SE cores in DEFAULT_SE_GROUP */
152	offset = POM_GRP_EXECMASKX(DEFAULT_SE_GROUP);
153	nitrox_write_csr(ndev, offset, (~0ULL));
154
155	/* write block number and firmware length
156	 * bit:<2:0> block number
157	 * bit:3 is set SE uses 32KB microcode
158	 * bit:3 is clear SE uses 64KB microcode
159	 */
160	core_2_eid_val.value = 0ULL;
161	core_2_eid_val.ucode_blk = 0;
162	if (ucode_size <= CNN55XX_UCD_BLOCK_SIZE)
163		core_2_eid_val.ucode_len = 1;
164	else
165		core_2_eid_val.ucode_len = 0;
166
167	for (i = 0; i < ndev->hw.se_cores; i++) {
168		offset = UCD_SE_EID_UCODE_BLOCK_NUMX(i);
169		nitrox_write_csr(ndev, offset, core_2_eid_val.value);
170	}
171
172
173	fw_name = AE_FW;
174	dev_info(DEV(ndev), "Loading firmware \"%s\"\n", fw_name);
175
176	ret = request_firmware(&fw, fw_name, DEV(ndev));
177	if (ret < 0) {
178		dev_err(DEV(ndev), "failed to get firmware %s\n", fw_name);
179		return ret;
180	}
181
182	ucode = (struct ucode *)fw->data;
183
184	ucode_size = be32_to_cpu(ucode->code_size) * 2;
185	if (!ucode_size || ucode_size > CNN55XX_MAX_UCODE_SIZE) {
186		dev_err(DEV(ndev), "Invalid ucode size: %u for firmware %s\n",
187			ucode_size, fw_name);
188		release_firmware(fw);
189		return -EINVAL;
190	}
191	ucode_data = ucode->code;
192
193	/* copy the firmware version */
194	memcpy(&ndev->hw.fw_name[1][0], ucode->version, (VERSION_LEN - 2));
195	ndev->hw.fw_name[1][VERSION_LEN - 1] = '\0';
196
197	/* Load AE Firmware on UCD Block 2 */
198	write_to_ucd_unit(ndev, ucode_size, ucode_data, 2);
199
200	release_firmware(fw);
201
202	/* put all AE cores in DEFAULT_AE_GROUP */
203	offset = AQM_GRP_EXECMSK_LOX(DEFAULT_AE_GROUP);
204	aqm_grp_execmask_lo.exec_0_to_39 = 0xFFFFFFFFFFULL;
205	nitrox_write_csr(ndev, offset, aqm_grp_execmask_lo.value);
206	offset = AQM_GRP_EXECMSK_HIX(DEFAULT_AE_GROUP);
207	aqm_grp_execmask_hi.exec_40_to_79 = 0xFFFFFFFFFFULL;
208	nitrox_write_csr(ndev, offset, aqm_grp_execmask_hi.value);
209
210	/* write block number and firmware length
211	 * bit:<2:0> block number
212	 * bit:3 is set AE uses 32KB microcode
213	 * bit:3 is clear AE uses 64KB microcode
214	 */
215	core_2_eid_val.value = 0ULL;
216	core_2_eid_val.ucode_blk = 2;
217	if (ucode_size <= CNN55XX_UCD_BLOCK_SIZE)
218		core_2_eid_val.ucode_len = 1;
219	else
220		core_2_eid_val.ucode_len = 0;
221
222	for (i = 0; i < ndev->hw.ae_cores; i++) {
223		offset = UCD_AE_EID_UCODE_BLOCK_NUMX(i);
224		nitrox_write_csr(ndev, offset, core_2_eid_val.value);
225	}
226
227	return 0;
228}
229
230/**
231 * nitrox_add_to_devlist - add NITROX device to global device list
232 * @ndev: NITROX device
233 */
234static int nitrox_add_to_devlist(struct nitrox_device *ndev)
235{
236	struct nitrox_device *dev;
237	int ret = 0;
238
239	INIT_LIST_HEAD(&ndev->list);
240	refcount_set(&ndev->refcnt, 1);
241
242	mutex_lock(&devlist_lock);
243	list_for_each_entry(dev, &ndevlist, list) {
244		if (dev == ndev) {
245			ret = -EEXIST;
246			goto unlock;
247		}
248	}
249	ndev->idx = num_devices++;
250	list_add_tail(&ndev->list, &ndevlist);
251unlock:
252	mutex_unlock(&devlist_lock);
253	return ret;
254}
255
256/**
257 * nitrox_remove_from_devlist - remove NITROX device from
258 *   global device list
259 * @ndev: NITROX device
260 */
261static void nitrox_remove_from_devlist(struct nitrox_device *ndev)
262{
263	mutex_lock(&devlist_lock);
264	list_del(&ndev->list);
265	num_devices--;
266	mutex_unlock(&devlist_lock);
267}
268
269struct nitrox_device *nitrox_get_first_device(void)
270{
271	struct nitrox_device *ndev = NULL, *iter;
272
273	mutex_lock(&devlist_lock);
274	list_for_each_entry(iter, &ndevlist, list) {
275		if (nitrox_ready(iter)) {
276			ndev = iter;
277			break;
278		}
279	}
280	mutex_unlock(&devlist_lock);
281	if (!ndev)
282		return NULL;
283
284	refcount_inc(&ndev->refcnt);
285	/* barrier to sync with other cpus */
286	smp_mb__after_atomic();
287	return ndev;
288}
289
290void nitrox_put_device(struct nitrox_device *ndev)
291{
292	if (!ndev)
293		return;
294
295	refcount_dec(&ndev->refcnt);
296	/* barrier to sync with other cpus */
297	smp_mb__after_atomic();
298}
299
300static int nitrox_device_flr(struct pci_dev *pdev)
301{
302	int pos = 0;
303
304	pos = pci_save_state(pdev);
305	if (pos) {
306		dev_err(&pdev->dev, "Failed to save pci state\n");
307		return -ENOMEM;
308	}
309
310	pcie_reset_flr(pdev, PCI_RESET_DO_RESET);
311
312	pci_restore_state(pdev);
313
314	return 0;
315}
316
317static int nitrox_pf_sw_init(struct nitrox_device *ndev)
318{
319	int err;
320
321	err = nitrox_common_sw_init(ndev);
322	if (err)
323		return err;
324
325	err = nitrox_register_interrupts(ndev);
326	if (err)
327		nitrox_common_sw_cleanup(ndev);
328
329	return err;
330}
331
332static void nitrox_pf_sw_cleanup(struct nitrox_device *ndev)
333{
334	nitrox_unregister_interrupts(ndev);
335	nitrox_common_sw_cleanup(ndev);
336}
337
338/**
339 * nitrox_bist_check - Check NITROX BIST registers status
340 * @ndev: NITROX device
341 */
342static int nitrox_bist_check(struct nitrox_device *ndev)
343{
344	u64 value = 0;
345	int i;
346
347	for (i = 0; i < NR_CLUSTERS; i++) {
348		value += nitrox_read_csr(ndev, EMU_BIST_STATUSX(i));
349		value += nitrox_read_csr(ndev, EFL_CORE_BIST_REGX(i));
350	}
351	value += nitrox_read_csr(ndev, UCD_BIST_STATUS);
352	value += nitrox_read_csr(ndev, NPS_CORE_BIST_REG);
353	value += nitrox_read_csr(ndev, NPS_CORE_NPC_BIST_REG);
354	value += nitrox_read_csr(ndev, NPS_PKT_SLC_BIST_REG);
355	value += nitrox_read_csr(ndev, NPS_PKT_IN_BIST_REG);
356	value += nitrox_read_csr(ndev, POM_BIST_REG);
357	value += nitrox_read_csr(ndev, BMI_BIST_REG);
358	value += nitrox_read_csr(ndev, EFL_TOP_BIST_STAT);
359	value += nitrox_read_csr(ndev, BMO_BIST_REG);
360	value += nitrox_read_csr(ndev, LBC_BIST_STATUS);
361	value += nitrox_read_csr(ndev, PEM_BIST_STATUSX(0));
362	if (value)
363		return -EIO;
364	return 0;
365}
366
367static int nitrox_pf_hw_init(struct nitrox_device *ndev)
368{
369	int err;
370
371	err = nitrox_bist_check(ndev);
372	if (err) {
373		dev_err(&ndev->pdev->dev, "BIST check failed\n");
374		return err;
375	}
376	/* get cores information */
377	nitrox_get_hwinfo(ndev);
378
379	nitrox_config_nps_core_unit(ndev);
380	nitrox_config_aqm_unit(ndev);
381	nitrox_config_nps_pkt_unit(ndev);
382	nitrox_config_pom_unit(ndev);
383	nitrox_config_efl_unit(ndev);
384	/* configure IO units */
385	nitrox_config_bmi_unit(ndev);
386	nitrox_config_bmo_unit(ndev);
387	/* configure Local Buffer Cache */
388	nitrox_config_lbc_unit(ndev);
389	nitrox_config_rand_unit(ndev);
390
391	/* load firmware on cores */
392	err = nitrox_load_fw(ndev);
393	if (err)
394		return err;
395
396	nitrox_config_emu_unit(ndev);
397
398	return 0;
399}
400
401/**
402 * nitrox_probe - NITROX Initialization function.
403 * @pdev: PCI device information struct
404 * @id: entry in nitrox_pci_tbl
405 *
406 * Return: 0, if the driver is bound to the device, or
407 *         a negative error if there is failure.
408 */
409static int nitrox_probe(struct pci_dev *pdev,
410			const struct pci_device_id *id)
411{
412	struct nitrox_device *ndev;
413	int err;
414
415	dev_info_once(&pdev->dev, "%s driver version %s\n",
416		      nitrox_driver_name, DRIVER_VERSION);
417
418	err = pci_enable_device_mem(pdev);
419	if (err)
420		return err;
421
422	/* do FLR */
423	err = nitrox_device_flr(pdev);
424	if (err) {
425		dev_err(&pdev->dev, "FLR failed\n");
426		goto flr_fail;
427	}
428
429	if (!dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64))) {
430		dev_dbg(&pdev->dev, "DMA to 64-BIT address\n");
431	} else {
432		err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
433		if (err) {
434			dev_err(&pdev->dev, "DMA configuration failed\n");
435			goto flr_fail;
436		}
437	}
438
439	err = pci_request_mem_regions(pdev, nitrox_driver_name);
440	if (err)
441		goto flr_fail;
442	pci_set_master(pdev);
443
444	ndev = kzalloc(sizeof(*ndev), GFP_KERNEL);
445	if (!ndev) {
446		err = -ENOMEM;
447		goto ndev_fail;
448	}
449
450	pci_set_drvdata(pdev, ndev);
451	ndev->pdev = pdev;
452
453	/* add to device list */
454	nitrox_add_to_devlist(ndev);
455
456	ndev->hw.vendor_id = pdev->vendor;
457	ndev->hw.device_id = pdev->device;
458	ndev->hw.revision_id = pdev->revision;
459	/* command timeout in jiffies */
460	ndev->timeout = msecs_to_jiffies(CMD_TIMEOUT);
461	ndev->node = dev_to_node(&pdev->dev);
462	if (ndev->node == NUMA_NO_NODE)
463		ndev->node = 0;
464
465	ndev->bar_addr = ioremap(pci_resource_start(pdev, 0),
466				 pci_resource_len(pdev, 0));
467	if (!ndev->bar_addr) {
468		err = -EIO;
469		goto ioremap_err;
470	}
471	/* allocate command queus based on cpus, max queues are 64 */
472	ndev->nr_queues = min_t(u32, MAX_PF_QUEUES, num_online_cpus());
473	ndev->qlen = qlen;
474
475	err = nitrox_pf_sw_init(ndev);
476	if (err)
477		goto pf_sw_fail;
478
479	err = nitrox_pf_hw_init(ndev);
480	if (err)
481		goto pf_hw_fail;
482
483	nitrox_debugfs_init(ndev);
484
485	/* clear the statistics */
486	atomic64_set(&ndev->stats.posted, 0);
487	atomic64_set(&ndev->stats.completed, 0);
488	atomic64_set(&ndev->stats.dropped, 0);
489
490	atomic_set(&ndev->state, __NDEV_READY);
491	/* barrier to sync with other cpus */
492	smp_mb__after_atomic();
493
494	err = nitrox_crypto_register();
495	if (err)
496		goto crypto_fail;
497
498	return 0;
499
500crypto_fail:
501	nitrox_debugfs_exit(ndev);
502	atomic_set(&ndev->state, __NDEV_NOT_READY);
503	/* barrier to sync with other cpus */
504	smp_mb__after_atomic();
505pf_hw_fail:
506	nitrox_pf_sw_cleanup(ndev);
507pf_sw_fail:
508	iounmap(ndev->bar_addr);
509ioremap_err:
510	nitrox_remove_from_devlist(ndev);
511	kfree(ndev);
512	pci_set_drvdata(pdev, NULL);
513ndev_fail:
514	pci_release_mem_regions(pdev);
515flr_fail:
516	pci_disable_device(pdev);
517	return err;
518}
519
520/**
521 * nitrox_remove - Unbind the driver from the device.
522 * @pdev: PCI device information struct
523 */
524static void nitrox_remove(struct pci_dev *pdev)
525{
526	struct nitrox_device *ndev = pci_get_drvdata(pdev);
527
528	if (!ndev)
529		return;
530
531	if (!refcount_dec_and_test(&ndev->refcnt)) {
532		dev_err(DEV(ndev), "Device refcnt not zero (%d)\n",
533			refcount_read(&ndev->refcnt));
534		return;
535	}
536
537	dev_info(DEV(ndev), "Removing Device %x:%x\n",
538		 ndev->hw.vendor_id, ndev->hw.device_id);
539
540	atomic_set(&ndev->state, __NDEV_NOT_READY);
541	/* barrier to sync with other cpus */
542	smp_mb__after_atomic();
543
544	nitrox_remove_from_devlist(ndev);
545
546	/* disable SR-IOV */
547	nitrox_sriov_configure(pdev, 0);
548	nitrox_crypto_unregister();
549	nitrox_debugfs_exit(ndev);
550	nitrox_pf_sw_cleanup(ndev);
551
552	iounmap(ndev->bar_addr);
553	kfree(ndev);
554
555	pci_set_drvdata(pdev, NULL);
556	pci_release_mem_regions(pdev);
557	pci_disable_device(pdev);
558}
559
560static void nitrox_shutdown(struct pci_dev *pdev)
561{
562	pci_set_drvdata(pdev, NULL);
563	pci_release_mem_regions(pdev);
564	pci_disable_device(pdev);
565}
566
567static struct pci_driver nitrox_driver = {
568	.name = nitrox_driver_name,
569	.id_table = nitrox_pci_tbl,
570	.probe = nitrox_probe,
571	.remove	= nitrox_remove,
572	.shutdown = nitrox_shutdown,
573	.sriov_configure = nitrox_sriov_configure,
574};
575
576module_pci_driver(nitrox_driver);
577
578MODULE_AUTHOR("Srikanth Jampala <Jampala.Srikanth@cavium.com>");
579MODULE_DESCRIPTION("Cavium CNN55XX PF Driver" DRIVER_VERSION " ");
580MODULE_LICENSE("GPL");
581MODULE_VERSION(DRIVER_VERSION);
582MODULE_FIRMWARE(SE_FW);
583