1/*
2 *   BSD LICENSE
3 *
4 *   Copyright(c) 2017 Cavium, Inc.. All rights reserved.
5 *   All rights reserved.
6 *
7 *   Redistribution and use in source and binary forms, with or without
8 *   modification, are permitted provided that the following conditions
9 *   are met:
10 *
11 *     * Redistributions of source code must retain the above copyright
12 *       notice, this list of conditions and the following disclaimer.
13 *     * Redistributions in binary form must reproduce the above copyright
14 *       notice, this list of conditions and the following disclaimer in
15 *       the documentation and/or other materials provided with the
16 *       distribution.
17 *     * Neither the name of Cavium, Inc. nor the names of its
18 *       contributors may be used to endorse or promote products derived
19 *       from this software without specific prior written permission.
20 *
21 *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 *   OWNER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 */
33
34#include "lio_bsd.h"
35#include "lio_common.h"
36#include "lio_droq.h"
37#include "lio_iq.h"
38#include "lio_response_manager.h"
39#include "lio_device.h"
40#include "cn23xx_pf_device.h"
41#include "lio_main.h"
42#include "lio_rss.h"
43
44static int
45lio_cn23xx_pf_soft_reset(struct octeon_device *oct)
46{
47
48	lio_write_csr64(oct, LIO_CN23XX_SLI_WIN_WR_MASK_REG, 0xFF);
49
50	lio_dev_dbg(oct, "BIST enabled for CN23XX soft reset\n");
51
52	lio_write_csr64(oct, LIO_CN23XX_SLI_SCRATCH1, 0x1234ULL);
53
54	/* Initiate chip-wide soft reset */
55	lio_pci_readq(oct, LIO_CN23XX_RST_SOFT_RST);
56	lio_pci_writeq(oct, 1, LIO_CN23XX_RST_SOFT_RST);
57
58	/* Wait for 100ms as Octeon resets. */
59	lio_mdelay(100);
60
61	if (lio_read_csr64(oct, LIO_CN23XX_SLI_SCRATCH1)) {
62		lio_dev_err(oct, "Soft reset failed\n");
63		return (1);
64	}
65
66	lio_dev_dbg(oct, "Reset completed\n");
67
68	/* restore the  reset value */
69	lio_write_csr64(oct, LIO_CN23XX_SLI_WIN_WR_MASK_REG, 0xFF);
70
71	return (0);
72}
73
74static void
75lio_cn23xx_pf_enable_error_reporting(struct octeon_device *oct)
76{
77	uint32_t	corrtable_err_status, uncorrectable_err_mask, regval;
78
79	regval = lio_read_pci_cfg(oct, LIO_CN23XX_CFG_PCIE_DEVCTL);
80	if (regval & LIO_CN23XX_CFG_PCIE_DEVCTL_MASK) {
81		uncorrectable_err_mask = 0;
82		corrtable_err_status = 0;
83		uncorrectable_err_mask =
84		    lio_read_pci_cfg(oct,
85				     LIO_CN23XX_CFG_PCIE_UNCORRECT_ERR_MASK);
86		corrtable_err_status =
87		    lio_read_pci_cfg(oct,
88				     LIO_CN23XX_CFG_PCIE_CORRECT_ERR_STATUS);
89		lio_dev_err(oct, "PCI-E Fatal error detected;\n"
90			    "\tdev_ctl_status_reg = 0x%08x\n"
91			    "\tuncorrectable_error_mask_reg = 0x%08x\n"
92			    "\tcorrectable_error_status_reg = 0x%08x\n",
93			    regval, uncorrectable_err_mask,
94			    corrtable_err_status);
95	}
96
97	regval |= 0xf;	/* Enable Link error reporting */
98
99	lio_dev_dbg(oct, "Enabling PCI-E error reporting..\n");
100	lio_write_pci_cfg(oct, LIO_CN23XX_CFG_PCIE_DEVCTL, regval);
101}
102
103static uint32_t
104lio_cn23xx_pf_coprocessor_clock(struct octeon_device *oct)
105{
106	/*
107	 * Bits 29:24 of RST_BOOT[PNR_MUL] holds the ref.clock MULTIPLIER
108	 * for SLI.
109	 */
110
111	/* TBD: get the info in Hand-shake */
112	return (((lio_pci_readq(oct, LIO_CN23XX_RST_BOOT) >> 24) & 0x3f) * 50);
113}
114
115uint32_t
116lio_cn23xx_pf_get_oq_ticks(struct octeon_device *oct, uint32_t time_intr_in_us)
117{
118	/* This gives the SLI clock per microsec */
119	uint32_t	oqticks_per_us = lio_cn23xx_pf_coprocessor_clock(oct);
120
121	oct->pfvf_hsword.coproc_tics_per_us = oqticks_per_us;
122
123	/* This gives the clock cycles per millisecond */
124	oqticks_per_us *= 1000;
125
126	/* This gives the oq ticks (1024 core clock cycles) per millisecond */
127	oqticks_per_us /= 1024;
128
129	/*
130	 * time_intr is in microseconds. The next 2 steps gives the oq ticks
131	 * corresponding to time_intr.
132	 */
133	oqticks_per_us *= time_intr_in_us;
134	oqticks_per_us /= 1000;
135
136	return (oqticks_per_us);
137}
138
139static void
140lio_cn23xx_pf_setup_global_mac_regs(struct octeon_device *oct)
141{
142	uint64_t	reg_val;
143	uint16_t	mac_no = oct->pcie_port;
144	uint16_t	pf_num = oct->pf_num;
145	/* programming SRN and TRS for each MAC(0..3)  */
146
147	lio_dev_dbg(oct, "%s: Using pcie port %d\n", __func__, mac_no);
148	/* By default, mapping all 64 IOQs to  a single MACs */
149
150	reg_val =
151	    lio_read_csr64(oct, LIO_CN23XX_SLI_PKT_MAC_RINFO64(mac_no, pf_num));
152
153	/* setting SRN <6:0>  */
154	reg_val = pf_num * LIO_CN23XX_PF_MAX_RINGS;
155
156	/* setting TRS <23:16> */
157	reg_val = reg_val |
158	    (oct->sriov_info.trs << LIO_CN23XX_PKT_MAC_CTL_RINFO_TRS_BIT_POS);
159
160	/* write these settings to MAC register */
161	lio_write_csr64(oct, LIO_CN23XX_SLI_PKT_MAC_RINFO64(mac_no, pf_num),
162			reg_val);
163
164	lio_dev_dbg(oct, "SLI_PKT_MAC(%d)_PF(%d)_RINFO : 0x%016llx\n", mac_no,
165		    pf_num,
166		    LIO_CAST64(lio_read_csr64(oct,
167				   LIO_CN23XX_SLI_PKT_MAC_RINFO64(mac_no,
168								  pf_num))));
169}
170
171static int
172lio_cn23xx_pf_reset_io_queues(struct octeon_device *oct)
173{
174	uint64_t	d64;
175	uint32_t	ern, loop = BUSY_READING_REG_PF_LOOP_COUNT;
176	uint32_t	q_no, srn;
177	int		ret_val = 0;
178
179	srn = oct->sriov_info.pf_srn;
180	ern = srn + oct->sriov_info.num_pf_rings;
181
182	/* As per HRM reg description, s/w cant write 0 to ENB. */
183	/* to make the queue off, need to set the RST bit. */
184
185	/* Reset the Enable bit for all the 64 IQs.  */
186	for (q_no = srn; q_no < ern; q_no++) {
187		/* set RST bit to 1. This bit applies to both IQ and OQ */
188		d64 = lio_read_csr64(oct,
189				     LIO_CN23XX_SLI_IQ_PKT_CONTROL64(q_no));
190		d64 = d64 | LIO_CN23XX_PKT_INPUT_CTL_RST;
191		lio_write_csr64(oct,
192				LIO_CN23XX_SLI_IQ_PKT_CONTROL64(q_no), d64);
193	}
194
195	/* wait until the RST bit is clear or the RST and quiet bits are set */
196	for (q_no = srn; q_no < ern; q_no++) {
197		volatile uint64_t reg_val =
198			lio_read_csr64(oct,
199				       LIO_CN23XX_SLI_IQ_PKT_CONTROL64(q_no));
200		while ((reg_val & LIO_CN23XX_PKT_INPUT_CTL_RST) &&
201		       !(reg_val & LIO_CN23XX_PKT_INPUT_CTL_QUIET) &&
202		       loop) {
203			reg_val = lio_read_csr64(oct,
204				       LIO_CN23XX_SLI_IQ_PKT_CONTROL64(q_no));
205			loop--;
206		}
207
208		if (!loop) {
209			lio_dev_err(oct,
210				    "clearing the reset reg failed or setting the quiet reg failed for qno: %u\n",
211				    q_no);
212			return (-1);
213		}
214
215		reg_val &= ~LIO_CN23XX_PKT_INPUT_CTL_RST;
216		lio_write_csr64(oct, LIO_CN23XX_SLI_IQ_PKT_CONTROL64(q_no),
217				reg_val);
218
219		reg_val = lio_read_csr64(oct,
220					 LIO_CN23XX_SLI_IQ_PKT_CONTROL64(q_no));
221		if (reg_val & LIO_CN23XX_PKT_INPUT_CTL_RST) {
222			lio_dev_err(oct, "clearing the reset failed for qno: %u\n",
223				    q_no);
224			ret_val = -1;
225		}
226	}
227
228	return (ret_val);
229}
230
231static int
232lio_cn23xx_pf_setup_global_input_regs(struct octeon_device *oct)
233{
234	struct lio_cn23xx_pf	*cn23xx = (struct lio_cn23xx_pf *)oct->chip;
235	struct lio_instr_queue	*iq;
236	uint64_t		intr_threshold;
237	uint64_t		pf_num, reg_val;
238	uint32_t		q_no, ern, srn;
239
240	pf_num = oct->pf_num;
241
242	srn = oct->sriov_info.pf_srn;
243	ern = srn + oct->sriov_info.num_pf_rings;
244
245	if (lio_cn23xx_pf_reset_io_queues(oct))
246		return (-1);
247
248	/*
249	 * Set the MAC_NUM and PVF_NUM in IQ_PKT_CONTROL reg
250	 * for all queues.Only PF can set these bits.
251	 * bits 29:30 indicate the MAC num.
252	 * bits 32:47 indicate the PVF num.
253	 */
254	for (q_no = 0; q_no < ern; q_no++) {
255		reg_val = oct->pcie_port <<
256			LIO_CN23XX_PKT_INPUT_CTL_MAC_NUM_POS;
257
258		reg_val |= pf_num << LIO_CN23XX_PKT_INPUT_CTL_PF_NUM_POS;
259
260		lio_write_csr64(oct, LIO_CN23XX_SLI_IQ_PKT_CONTROL64(q_no),
261				reg_val);
262	}
263
264	/*
265	 * Select ES, RO, NS, RDSIZE,DPTR Fomat#0 for
266	 * pf queues
267	 */
268	for (q_no = srn; q_no < ern; q_no++) {
269		uint32_t	inst_cnt_reg;
270
271		iq = oct->instr_queue[q_no];
272		if (iq != NULL)
273			inst_cnt_reg = iq->inst_cnt_reg;
274		else
275			inst_cnt_reg = LIO_CN23XX_SLI_IQ_INSTR_COUNT64(q_no);
276
277		reg_val =
278		    lio_read_csr64(oct, LIO_CN23XX_SLI_IQ_PKT_CONTROL64(q_no));
279
280		reg_val |= LIO_CN23XX_PKT_INPUT_CTL_MASK;
281
282		lio_write_csr64(oct, LIO_CN23XX_SLI_IQ_PKT_CONTROL64(q_no),
283				reg_val);
284
285		/* Set WMARK level for triggering PI_INT */
286		/* intr_threshold = LIO_CN23XX_DEF_IQ_INTR_THRESHOLD & */
287		intr_threshold = LIO_GET_IQ_INTR_PKT_CFG(cn23xx->conf) &
288		    LIO_CN23XX_PKT_IN_DONE_WMARK_MASK;
289
290		lio_write_csr64(oct, inst_cnt_reg,
291				(lio_read_csr64(oct, inst_cnt_reg) &
292				 ~(LIO_CN23XX_PKT_IN_DONE_WMARK_MASK <<
293				   LIO_CN23XX_PKT_IN_DONE_WMARK_BIT_POS)) |
294				(intr_threshold <<
295				 LIO_CN23XX_PKT_IN_DONE_WMARK_BIT_POS));
296	}
297	return (0);
298}
299
300static void
301lio_cn23xx_pf_setup_global_output_regs(struct octeon_device *oct)
302{
303	struct lio_cn23xx_pf *cn23xx = (struct lio_cn23xx_pf *)oct->chip;
304	uint64_t	time_threshold;
305	uint32_t	ern, q_no, reg_val, srn;
306
307	srn = oct->sriov_info.pf_srn;
308	ern = srn + oct->sriov_info.num_pf_rings;
309
310	if (LIO_GET_IS_SLI_BP_ON_CFG(cn23xx->conf)) {
311		lio_write_csr64(oct, LIO_CN23XX_SLI_OQ_WMARK, 32);
312	} else {
313		/* Set Output queue watermark to 0 to disable backpressure */
314		lio_write_csr64(oct, LIO_CN23XX_SLI_OQ_WMARK, 0);
315	}
316
317	for (q_no = srn; q_no < ern; q_no++) {
318		reg_val = lio_read_csr32(oct,
319					 LIO_CN23XX_SLI_OQ_PKT_CONTROL(q_no));
320
321		/* set IPTR & DPTR */
322		reg_val |= LIO_CN23XX_PKT_OUTPUT_CTL_DPTR;
323
324		/* reset BMODE */
325		reg_val &= ~(LIO_CN23XX_PKT_OUTPUT_CTL_BMODE);
326
327		/*
328		 * No Relaxed Ordering, No Snoop, 64-bit Byte swap for
329		 * Output Queue ScatterList reset ROR_P, NSR_P
330		 */
331		reg_val &= ~(LIO_CN23XX_PKT_OUTPUT_CTL_ROR_P);
332		reg_val &= ~(LIO_CN23XX_PKT_OUTPUT_CTL_NSR_P);
333
334#if BYTE_ORDER == LITTLE_ENDIAN
335		reg_val &= ~(LIO_CN23XX_PKT_OUTPUT_CTL_ES_P);
336#else	/* BYTE_ORDER != LITTLE_ENDIAN  */
337		reg_val |= (LIO_CN23XX_PKT_OUTPUT_CTL_ES_P);
338#endif	/* BYTE_ORDER == LITTLE_ENDIAN */
339
340		/*
341		 * No Relaxed Ordering, No Snoop, 64-bit Byte swap for
342		 * Output Queue Data reset ROR, NSR
343		 */
344		reg_val &= ~(LIO_CN23XX_PKT_OUTPUT_CTL_ROR);
345		reg_val &= ~(LIO_CN23XX_PKT_OUTPUT_CTL_NSR);
346		/* set the ES bit */
347		reg_val |= (LIO_CN23XX_PKT_OUTPUT_CTL_ES);
348
349		/* write all the selected settings */
350		lio_write_csr32(oct, LIO_CN23XX_SLI_OQ_PKT_CONTROL(q_no),
351				reg_val);
352
353		/*
354		 * Enabling these interrupt in oct->fn_list.enable_interrupt()
355		 * routine which called after IOQ init.
356		 * Set up interrupt packet and time thresholds
357		 * for all the OQs
358		 */
359		time_threshold =lio_cn23xx_pf_get_oq_ticks(
360		       oct, (uint32_t)LIO_GET_OQ_INTR_TIME_CFG(cn23xx->conf));
361
362		lio_write_csr64(oct, LIO_CN23XX_SLI_OQ_PKT_INT_LEVELS(q_no),
363				(LIO_GET_OQ_INTR_PKT_CFG(cn23xx->conf) |
364				 (time_threshold << 32)));
365	}
366
367	/* Setting the water mark level for pko back pressure * */
368	lio_write_csr64(oct, LIO_CN23XX_SLI_OQ_WMARK, 0x40);
369
370	/* Enable channel-level backpressure */
371	if (oct->pf_num)
372		lio_write_csr64(oct, LIO_CN23XX_SLI_OUT_BP_EN2_W1S,
373				0xffffffffffffffffULL);
374	else
375		lio_write_csr64(oct, LIO_CN23XX_SLI_OUT_BP_EN_W1S,
376				0xffffffffffffffffULL);
377}
378
379static int
380lio_cn23xx_pf_setup_device_regs(struct octeon_device *oct)
381{
382
383	lio_cn23xx_pf_enable_error_reporting(oct);
384
385	/* program the MAC(0..3)_RINFO before setting up input/output regs */
386	lio_cn23xx_pf_setup_global_mac_regs(oct);
387
388	if (lio_cn23xx_pf_setup_global_input_regs(oct))
389		return (-1);
390
391	lio_cn23xx_pf_setup_global_output_regs(oct);
392
393	/*
394	 * Default error timeout value should be 0x200000 to avoid host hang
395	 * when reads invalid register
396	 */
397	lio_write_csr64(oct, LIO_CN23XX_SLI_WINDOW_CTL,
398			LIO_CN23XX_SLI_WINDOW_CTL_DEFAULT);
399
400	/* set SLI_PKT_IN_JABBER to handle large VXLAN packets */
401	lio_write_csr64(oct, LIO_CN23XX_SLI_PKT_IN_JABBER,
402			LIO_CN23XX_MAX_INPUT_JABBER);
403	return (0);
404}
405
406static void
407lio_cn23xx_pf_setup_iq_regs(struct octeon_device *oct, uint32_t iq_no)
408{
409	struct lio_instr_queue	*iq = oct->instr_queue[iq_no];
410	uint64_t		pkt_in_done;
411
412	iq_no += oct->sriov_info.pf_srn;
413
414	/* Write the start of the input queue's ring and its size  */
415	lio_write_csr64(oct, LIO_CN23XX_SLI_IQ_BASE_ADDR64(iq_no),
416			iq->base_addr_dma);
417	lio_write_csr32(oct, LIO_CN23XX_SLI_IQ_SIZE(iq_no), iq->max_count);
418
419	/*
420	 * Remember the doorbell & instruction count register addr
421	 * for this queue
422	 */
423	iq->doorbell_reg = LIO_CN23XX_SLI_IQ_DOORBELL(iq_no);
424	iq->inst_cnt_reg = LIO_CN23XX_SLI_IQ_INSTR_COUNT64(iq_no);
425	lio_dev_dbg(oct, "InstQ[%d]:dbell reg @ 0x%x instcnt_reg @ 0x%x\n",
426		    iq_no, iq->doorbell_reg, iq->inst_cnt_reg);
427
428	/*
429	 * Store the current instruction counter (used in flush_iq
430	 * calculation)
431	 */
432	pkt_in_done = lio_read_csr64(oct, iq->inst_cnt_reg);
433
434	if (oct->msix_on) {
435		/* Set CINT_ENB to enable IQ interrupt   */
436		lio_write_csr64(oct, iq->inst_cnt_reg,
437				(pkt_in_done | LIO_CN23XX_INTR_CINT_ENB));
438	} else {
439		/*
440		 * Clear the count by writing back what we read, but don't
441		 * enable interrupts
442		 */
443		lio_write_csr64(oct, iq->inst_cnt_reg, pkt_in_done);
444	}
445
446	iq->reset_instr_cnt = 0;
447}
448
449static void
450lio_cn23xx_pf_setup_oq_regs(struct octeon_device *oct, uint32_t oq_no)
451{
452	struct lio_droq		*droq = oct->droq[oq_no];
453	struct lio_cn23xx_pf	*cn23xx = (struct lio_cn23xx_pf *)oct->chip;
454	uint64_t		cnt_threshold;
455	uint64_t		time_threshold;
456	uint32_t		reg_val;
457
458	oq_no += oct->sriov_info.pf_srn;
459
460	lio_write_csr64(oct, LIO_CN23XX_SLI_OQ_BASE_ADDR64(oq_no),
461			droq->desc_ring_dma);
462	lio_write_csr32(oct, LIO_CN23XX_SLI_OQ_SIZE(oq_no), droq->max_count);
463
464	lio_write_csr32(oct, LIO_CN23XX_SLI_OQ_BUFF_INFO_SIZE(oq_no),
465			droq->buffer_size);
466
467	/* pkt_sent and pkts_credit regs */
468	droq->pkts_sent_reg = LIO_CN23XX_SLI_OQ_PKTS_SENT(oq_no);
469	droq->pkts_credit_reg = LIO_CN23XX_SLI_OQ_PKTS_CREDIT(oq_no);
470
471	if (!oct->msix_on) {
472		/*
473		 * Enable this output queue to generate Packet Timer
474		 * Interrupt
475		 */
476		reg_val =
477		    lio_read_csr32(oct, LIO_CN23XX_SLI_OQ_PKT_CONTROL(oq_no));
478		reg_val |= LIO_CN23XX_PKT_OUTPUT_CTL_TENB;
479		lio_write_csr32(oct, LIO_CN23XX_SLI_OQ_PKT_CONTROL(oq_no),
480				reg_val);
481
482		/*
483		 * Enable this output queue to generate Packet Count
484		 * Interrupt
485		 */
486		reg_val =
487		    lio_read_csr32(oct, LIO_CN23XX_SLI_OQ_PKT_CONTROL(oq_no));
488		reg_val |= LIO_CN23XX_PKT_OUTPUT_CTL_CENB;
489		lio_write_csr32(oct, LIO_CN23XX_SLI_OQ_PKT_CONTROL(oq_no),
490				reg_val);
491	} else {
492		time_threshold = lio_cn23xx_pf_get_oq_ticks(oct,
493			(uint32_t)LIO_GET_OQ_INTR_TIME_CFG(cn23xx->conf));
494		cnt_threshold = (uint32_t)LIO_GET_OQ_INTR_PKT_CFG(cn23xx->conf);
495
496		lio_write_csr64(oct, LIO_CN23XX_SLI_OQ_PKT_INT_LEVELS(oq_no),
497				((time_threshold << 32 | cnt_threshold)));
498	}
499}
500
501
502static int
503lio_cn23xx_pf_enable_io_queues(struct octeon_device *oct)
504{
505	uint64_t	reg_val;
506	uint32_t	ern, loop = BUSY_READING_REG_PF_LOOP_COUNT;
507	uint32_t	q_no, srn;
508
509	srn = oct->sriov_info.pf_srn;
510	ern = srn + oct->num_iqs;
511
512	for (q_no = srn; q_no < ern; q_no++) {
513		/* set the corresponding IQ IS_64B bit */
514		if (oct->io_qmask.iq64B & BIT_ULL(q_no - srn)) {
515			reg_val = lio_read_csr64(oct,
516					LIO_CN23XX_SLI_IQ_PKT_CONTROL64(q_no));
517			reg_val = reg_val | LIO_CN23XX_PKT_INPUT_CTL_IS_64B;
518			lio_write_csr64(oct,
519					LIO_CN23XX_SLI_IQ_PKT_CONTROL64(q_no),
520					reg_val);
521		}
522		/* set the corresponding IQ ENB bit */
523		if (oct->io_qmask.iq & BIT_ULL(q_no - srn)) {
524			/*
525			 * IOQs are in reset by default in PEM2 mode,
526			 * clearing reset bit
527			 */
528			reg_val = lio_read_csr64(oct,
529					LIO_CN23XX_SLI_IQ_PKT_CONTROL64(q_no));
530
531			if (reg_val & LIO_CN23XX_PKT_INPUT_CTL_RST) {
532				while ((reg_val &
533					LIO_CN23XX_PKT_INPUT_CTL_RST) &&
534				       !(reg_val &
535					 LIO_CN23XX_PKT_INPUT_CTL_QUIET) &&
536				       loop) {
537					reg_val = lio_read_csr64(oct,
538					LIO_CN23XX_SLI_IQ_PKT_CONTROL64(q_no));
539					loop--;
540				}
541				if (!loop) {
542					lio_dev_err(oct, "clearing the reset reg failed or setting the quiet reg failed for qno: %u\n",
543						    q_no);
544					return (-1);
545				}
546				reg_val = reg_val &
547					~LIO_CN23XX_PKT_INPUT_CTL_RST;
548				lio_write_csr64(oct,
549					LIO_CN23XX_SLI_IQ_PKT_CONTROL64(q_no),
550					reg_val);
551
552				reg_val = lio_read_csr64(oct,
553					LIO_CN23XX_SLI_IQ_PKT_CONTROL64(q_no));
554				if (reg_val & LIO_CN23XX_PKT_INPUT_CTL_RST) {
555					lio_dev_err(oct, "clearing the reset failed for qno: %u\n",
556						    q_no);
557					return (-1);
558				}
559			}
560			reg_val = lio_read_csr64(oct,
561					LIO_CN23XX_SLI_IQ_PKT_CONTROL64(q_no));
562			reg_val = reg_val | LIO_CN23XX_PKT_INPUT_CTL_RING_ENB;
563			lio_write_csr64(oct,
564					LIO_CN23XX_SLI_IQ_PKT_CONTROL64(q_no),
565					reg_val);
566		}
567	}
568	for (q_no = srn; q_no < ern; q_no++) {
569		uint32_t	reg_val;
570		/* set the corresponding OQ ENB bit */
571		if (oct->io_qmask.oq & BIT_ULL(q_no - srn)) {
572			reg_val = lio_read_csr32(oct,
573					LIO_CN23XX_SLI_OQ_PKT_CONTROL(q_no));
574			reg_val = reg_val | LIO_CN23XX_PKT_OUTPUT_CTL_RING_ENB;
575			lio_write_csr32(oct,
576					LIO_CN23XX_SLI_OQ_PKT_CONTROL(q_no),
577					reg_val);
578		}
579	}
580	return (0);
581}
582
583static void
584lio_cn23xx_pf_disable_io_queues(struct octeon_device *oct)
585{
586	volatile uint64_t	d64;
587	volatile uint32_t	d32;
588	int			loop;
589	unsigned int		q_no;
590	uint32_t		ern, srn;
591
592	srn = oct->sriov_info.pf_srn;
593	ern = srn + oct->num_iqs;
594
595	/* Disable Input Queues. */
596	for (q_no = srn; q_no < ern; q_no++) {
597		loop = lio_ms_to_ticks(1000);
598
599		/* start the Reset for a particular ring */
600		d64 = lio_read_csr64(oct,
601				     LIO_CN23XX_SLI_IQ_PKT_CONTROL64(q_no));
602		d64 &= ~LIO_CN23XX_PKT_INPUT_CTL_RING_ENB;
603		d64 |= LIO_CN23XX_PKT_INPUT_CTL_RST;
604		lio_write_csr64(oct, LIO_CN23XX_SLI_IQ_PKT_CONTROL64(q_no),
605				d64);
606
607		/*
608		 * Wait until hardware indicates that the particular IQ
609		 * is out of reset.
610		 */
611		d64 = lio_read_csr64(oct, LIO_CN23XX_SLI_PKT_IOQ_RING_RST);
612		while (!(d64 & BIT_ULL(q_no)) && loop--) {
613			d64 = lio_read_csr64(oct,
614					     LIO_CN23XX_SLI_PKT_IOQ_RING_RST);
615			lio_sleep_timeout(1);
616			loop--;
617		}
618
619		/* Reset the doorbell register for this Input Queue. */
620		lio_write_csr32(oct, LIO_CN23XX_SLI_IQ_DOORBELL(q_no),
621				0xFFFFFFFF);
622		while (((lio_read_csr64(oct,
623					LIO_CN23XX_SLI_IQ_DOORBELL(q_no))) !=
624			0ULL) && loop--) {
625			lio_sleep_timeout(1);
626		}
627	}
628
629	/* Disable Output Queues. */
630	for (q_no = srn; q_no < ern; q_no++) {
631		loop = lio_ms_to_ticks(1000);
632
633		/*
634		 * Wait until hardware indicates that the particular IQ
635		 * is out of reset.It given that SLI_PKT_RING_RST is
636		 * common for both IQs and OQs
637		 */
638		d64 = lio_read_csr64(oct, LIO_CN23XX_SLI_PKT_IOQ_RING_RST);
639		while (!(d64 & BIT_ULL(q_no)) && loop--) {
640			d64 = lio_read_csr64(oct,
641					     LIO_CN23XX_SLI_PKT_IOQ_RING_RST);
642			lio_sleep_timeout(1);
643			loop--;
644		}
645
646		/* Reset the doorbell register for this Output Queue. */
647		lio_write_csr32(oct, LIO_CN23XX_SLI_OQ_PKTS_CREDIT(q_no),
648				0xFFFFFFFF);
649		while ((lio_read_csr64(oct,
650				       LIO_CN23XX_SLI_OQ_PKTS_CREDIT(q_no)) !=
651			0ULL) && loop--) {
652			lio_sleep_timeout(1);
653		}
654
655		/* clear the SLI_PKT(0..63)_CNTS[CNT] reg value */
656		d32 = lio_read_csr32(oct, LIO_CN23XX_SLI_OQ_PKTS_SENT(q_no));
657		lio_write_csr32(oct, LIO_CN23XX_SLI_OQ_PKTS_SENT(q_no),	d32);
658	}
659}
660
661static uint64_t
662lio_cn23xx_pf_msix_interrupt_handler(void *dev)
663{
664	struct lio_ioq_vector	*ioq_vector = (struct lio_ioq_vector *)dev;
665	struct octeon_device	*oct = ioq_vector->oct_dev;
666	struct lio_droq		*droq = oct->droq[ioq_vector->droq_index];
667	uint64_t		pkts_sent;
668	uint64_t		ret = 0;
669
670	if (droq == NULL) {
671		lio_dev_err(oct, "23XX bringup FIXME: oct pfnum:%d ioq_vector->ioq_num :%d droq is NULL\n",
672			    oct->pf_num, ioq_vector->ioq_num);
673		return (0);
674	}
675	pkts_sent = lio_read_csr64(oct, droq->pkts_sent_reg);
676
677	/*
678	 * If our device has interrupted, then proceed. Also check
679	 * for all f's if interrupt was triggered on an error
680	 * and the PCI read fails.
681	 */
682	if (!pkts_sent || (pkts_sent == 0xFFFFFFFFFFFFFFFFULL))
683		return (ret);
684
685	/* Write count reg in sli_pkt_cnts to clear these int. */
686	if (pkts_sent & LIO_CN23XX_INTR_PO_INT)
687		ret |= LIO_MSIX_PO_INT;
688
689	if (pkts_sent & LIO_CN23XX_INTR_PI_INT)
690		/* We will clear the count when we update the read_index. */
691		ret |= LIO_MSIX_PI_INT;
692
693	/*
694	 * Never need to handle msix mbox intr for pf. They arrive on the last
695	 * msix
696	 */
697	return (ret);
698}
699
700static void
701lio_cn23xx_pf_interrupt_handler(void *dev)
702{
703	struct octeon_device	*oct = (struct octeon_device *)dev;
704	struct lio_cn23xx_pf	*cn23xx = (struct lio_cn23xx_pf *)oct->chip;
705	uint64_t		intr64;
706
707	lio_dev_dbg(oct, "In %s octeon_dev @ %p\n", __func__, oct);
708	intr64 = lio_read_csr64(oct, cn23xx->intr_sum_reg64);
709
710	oct->int_status = 0;
711
712	if (intr64 & LIO_CN23XX_INTR_ERR)
713		lio_dev_err(oct, "Error Intr: 0x%016llx\n",
714			    LIO_CAST64(intr64));
715
716	if (oct->msix_on != LIO_FLAG_MSIX_ENABLED) {
717		if (intr64 & LIO_CN23XX_INTR_PKT_DATA)
718			oct->int_status |= LIO_DEV_INTR_PKT_DATA;
719	}
720
721	if (intr64 & (LIO_CN23XX_INTR_DMA0_FORCE))
722		oct->int_status |= LIO_DEV_INTR_DMA0_FORCE;
723
724	if (intr64 & (LIO_CN23XX_INTR_DMA1_FORCE))
725		oct->int_status |= LIO_DEV_INTR_DMA1_FORCE;
726
727	/* Clear the current interrupts */
728	lio_write_csr64(oct, cn23xx->intr_sum_reg64, intr64);
729}
730
731static void
732lio_cn23xx_pf_bar1_idx_setup(struct octeon_device *oct, uint64_t core_addr,
733			     uint32_t idx, int valid)
734{
735	volatile uint64_t	bar1;
736	uint64_t		reg_adr;
737
738	if (!valid) {
739		reg_adr = lio_pci_readq(oct,
740				LIO_CN23XX_PEM_BAR1_INDEX_REG(oct->pcie_port,
741							      idx));
742		bar1 = reg_adr;
743		lio_pci_writeq(oct, (bar1 & 0xFFFFFFFEULL),
744			       LIO_CN23XX_PEM_BAR1_INDEX_REG(oct->pcie_port,
745							     idx));
746		reg_adr = lio_pci_readq(oct,
747				LIO_CN23XX_PEM_BAR1_INDEX_REG(oct->pcie_port,
748							      idx));
749		bar1 = reg_adr;
750		return;
751	}
752	/*
753	 *  The PEM(0..3)_BAR1_INDEX(0..15)[ADDR_IDX]<23:4> stores
754	 *  bits <41:22> of the Core Addr
755	 */
756	lio_pci_writeq(oct, (((core_addr >> 22) << 4) | LIO_PCI_BAR1_MASK),
757		       LIO_CN23XX_PEM_BAR1_INDEX_REG(oct->pcie_port, idx));
758
759	bar1 = lio_pci_readq(oct, LIO_CN23XX_PEM_BAR1_INDEX_REG(oct->pcie_port,
760								idx));
761}
762
763static void
764lio_cn23xx_pf_bar1_idx_write(struct octeon_device *oct, uint32_t idx,
765			     uint32_t mask)
766{
767
768	lio_pci_writeq(oct, mask,
769		       LIO_CN23XX_PEM_BAR1_INDEX_REG(oct->pcie_port, idx));
770}
771
772static uint32_t
773lio_cn23xx_pf_bar1_idx_read(struct octeon_device *oct, uint32_t idx)
774{
775
776	return ((uint32_t)lio_pci_readq(oct,
777				LIO_CN23XX_PEM_BAR1_INDEX_REG(oct->pcie_port,
778							      idx)));
779}
780
781/* always call with lock held */
782static uint32_t
783lio_cn23xx_pf_update_read_index(struct lio_instr_queue *iq)
784{
785	struct octeon_device	*oct = iq->oct_dev;
786	uint32_t	new_idx;
787	uint32_t	last_done;
788	uint32_t	pkt_in_done = lio_read_csr32(oct, iq->inst_cnt_reg);
789
790	last_done = pkt_in_done - iq->pkt_in_done;
791	iq->pkt_in_done = pkt_in_done;
792
793	/*
794	 * Modulo of the new index with the IQ size will give us
795	 * the new index.  The iq->reset_instr_cnt is always zero for
796	 * cn23xx, so no extra adjustments are needed.
797	 */
798	new_idx = (iq->octeon_read_index +
799		   ((uint32_t)(last_done & LIO_CN23XX_PKT_IN_DONE_CNT_MASK))) %
800	    iq->max_count;
801
802	return (new_idx);
803}
804
805static void
806lio_cn23xx_pf_enable_interrupt(struct octeon_device *oct, uint8_t intr_flag)
807{
808	struct lio_cn23xx_pf	*cn23xx = (struct lio_cn23xx_pf *)oct->chip;
809	uint64_t		intr_val = 0;
810
811	/* Divide the single write to multiple writes based on the flag. */
812	/* Enable Interrupt */
813	if (intr_flag == OCTEON_ALL_INTR) {
814		lio_write_csr64(oct, cn23xx->intr_enb_reg64,
815				cn23xx->intr_mask64);
816	} else if (intr_flag & OCTEON_OUTPUT_INTR) {
817		intr_val = lio_read_csr64(oct, cn23xx->intr_enb_reg64);
818		intr_val |= LIO_CN23XX_INTR_PKT_DATA;
819		lio_write_csr64(oct, cn23xx->intr_enb_reg64, intr_val);
820	}
821}
822
823static void
824lio_cn23xx_pf_disable_interrupt(struct octeon_device *oct, uint8_t intr_flag)
825{
826	struct lio_cn23xx_pf	*cn23xx = (struct lio_cn23xx_pf *)oct->chip;
827	uint64_t		intr_val = 0;
828
829	/* Disable Interrupts */
830	if (intr_flag == OCTEON_ALL_INTR) {
831		lio_write_csr64(oct, cn23xx->intr_enb_reg64, 0);
832	} else if (intr_flag & OCTEON_OUTPUT_INTR) {
833		intr_val = lio_read_csr64(oct, cn23xx->intr_enb_reg64);
834		intr_val &= ~LIO_CN23XX_INTR_PKT_DATA;
835		lio_write_csr64(oct, cn23xx->intr_enb_reg64, intr_val);
836	}
837}
838
839static void
840lio_cn23xx_pf_get_pcie_qlmport(struct octeon_device *oct)
841{
842	oct->pcie_port = (lio_read_csr32(oct,
843					 LIO_CN23XX_SLI_MAC_NUMBER)) & 0xff;
844
845	lio_dev_dbg(oct, "CN23xx uses PCIE Port %d\n",
846		    oct->pcie_port);
847}
848
849static void
850lio_cn23xx_pf_get_pf_num(struct octeon_device *oct)
851{
852	uint32_t	fdl_bit;
853
854	/* Read Function Dependency Link reg to get the function number */
855	fdl_bit = lio_read_pci_cfg(oct, LIO_CN23XX_PCIE_SRIOV_FDL);
856	oct->pf_num = ((fdl_bit >> LIO_CN23XX_PCIE_SRIOV_FDL_BIT_POS) &
857		       LIO_CN23XX_PCIE_SRIOV_FDL_MASK);
858}
859
860static void
861lio_cn23xx_pf_setup_reg_address(struct octeon_device *oct)
862{
863	struct lio_cn23xx_pf	*cn23xx = (struct lio_cn23xx_pf *)oct->chip;
864
865	oct->reg_list.pci_win_wr_addr = LIO_CN23XX_SLI_WIN_WR_ADDR64;
866
867	oct->reg_list.pci_win_rd_addr_hi = LIO_CN23XX_SLI_WIN_RD_ADDR_HI;
868	oct->reg_list.pci_win_rd_addr_lo = LIO_CN23XX_SLI_WIN_RD_ADDR64;
869	oct->reg_list.pci_win_rd_addr = LIO_CN23XX_SLI_WIN_RD_ADDR64;
870
871	oct->reg_list.pci_win_wr_data_hi = LIO_CN23XX_SLI_WIN_WR_DATA_HI;
872	oct->reg_list.pci_win_wr_data_lo = LIO_CN23XX_SLI_WIN_WR_DATA_LO;
873	oct->reg_list.pci_win_wr_data = LIO_CN23XX_SLI_WIN_WR_DATA64;
874
875	oct->reg_list.pci_win_rd_data = LIO_CN23XX_SLI_WIN_RD_DATA64;
876
877	lio_cn23xx_pf_get_pcie_qlmport(oct);
878
879	cn23xx->intr_mask64 = LIO_CN23XX_INTR_MASK;
880	if (!oct->msix_on)
881		cn23xx->intr_mask64 |= LIO_CN23XX_INTR_PKT_TIME;
882
883	cn23xx->intr_sum_reg64 =
884	    LIO_CN23XX_SLI_MAC_PF_INT_SUM64(oct->pcie_port, oct->pf_num);
885	cn23xx->intr_enb_reg64 =
886	    LIO_CN23XX_SLI_MAC_PF_INT_ENB64(oct->pcie_port, oct->pf_num);
887}
888
889static int
890lio_cn23xx_pf_sriov_config(struct octeon_device *oct)
891{
892	struct lio_cn23xx_pf	*cn23xx = (struct lio_cn23xx_pf *)oct->chip;
893	uint32_t		num_pf_rings, total_rings, max_rings;
894	cn23xx->conf = (struct lio_config *)lio_get_config_info(oct, LIO_23XX);
895
896	max_rings = LIO_CN23XX_PF_MAX_RINGS;
897
898	if (oct->sriov_info.num_pf_rings) {
899		num_pf_rings = oct->sriov_info.num_pf_rings;
900		if (num_pf_rings > max_rings) {
901			num_pf_rings = min(mp_ncpus, max_rings);
902			lio_dev_warn(oct, "num_queues_per_pf requested %u is more than available rings (%u). Reducing to %u\n",
903				     oct->sriov_info.num_pf_rings,
904				     max_rings, num_pf_rings);
905		}
906	} else {
907#ifdef RSS
908		num_pf_rings = min(rss_getnumbuckets(), mp_ncpus);
909#else
910		num_pf_rings = min(mp_ncpus, max_rings);
911#endif
912
913	}
914
915	total_rings = num_pf_rings;
916	oct->sriov_info.trs = total_rings;
917	oct->sriov_info.pf_srn = total_rings - num_pf_rings;
918	oct->sriov_info.num_pf_rings = num_pf_rings;
919
920	lio_dev_dbg(oct, "trs:%d pf_srn:%d num_pf_rings:%d\n",
921		    oct->sriov_info.trs, oct->sriov_info.pf_srn,
922		    oct->sriov_info.num_pf_rings);
923
924	return (0);
925}
926
927int
928lio_cn23xx_pf_setup_device(struct octeon_device *oct)
929{
930	uint64_t	BAR0, BAR1;
931	uint32_t	data32;
932
933	data32 = lio_read_pci_cfg(oct, 0x10);
934	BAR0 = (uint64_t)(data32 & ~0xf);
935	data32 = lio_read_pci_cfg(oct, 0x14);
936	BAR0 |= ((uint64_t)data32 << 32);
937	data32 = lio_read_pci_cfg(oct, 0x18);
938	BAR1 = (uint64_t)(data32 & ~0xf);
939	data32 = lio_read_pci_cfg(oct, 0x1c);
940	BAR1 |= ((uint64_t)data32 << 32);
941
942	if (!BAR0 || !BAR1) {
943		if (!BAR0)
944			lio_dev_err(oct, "Device BAR0 unassigned\n");
945
946		if (!BAR1)
947			lio_dev_err(oct, "Device BAR1 unassigned\n");
948
949		return (1);
950	}
951
952	if (lio_map_pci_barx(oct, 0))
953		return (1);
954
955	if (lio_map_pci_barx(oct, 1)) {
956		lio_dev_err(oct, "%s CN23XX BAR1 map failed\n", __func__);
957		lio_unmap_pci_barx(oct, 0);
958		return (1);
959	}
960
961	lio_cn23xx_pf_get_pf_num(oct);
962
963	if (lio_cn23xx_pf_sriov_config(oct)) {
964		lio_unmap_pci_barx(oct, 0);
965		lio_unmap_pci_barx(oct, 1);
966		return (1);
967	}
968	lio_write_csr64(oct, LIO_CN23XX_SLI_MAC_CREDIT_CNT,
969			0x3F802080802080ULL);
970
971	oct->fn_list.setup_iq_regs = lio_cn23xx_pf_setup_iq_regs;
972	oct->fn_list.setup_oq_regs = lio_cn23xx_pf_setup_oq_regs;
973	oct->fn_list.process_interrupt_regs = lio_cn23xx_pf_interrupt_handler;
974	oct->fn_list.msix_interrupt_handler =
975		lio_cn23xx_pf_msix_interrupt_handler;
976
977	oct->fn_list.soft_reset = lio_cn23xx_pf_soft_reset;
978	oct->fn_list.setup_device_regs = lio_cn23xx_pf_setup_device_regs;
979	oct->fn_list.update_iq_read_idx = lio_cn23xx_pf_update_read_index;
980
981	oct->fn_list.bar1_idx_setup = lio_cn23xx_pf_bar1_idx_setup;
982	oct->fn_list.bar1_idx_write = lio_cn23xx_pf_bar1_idx_write;
983	oct->fn_list.bar1_idx_read = lio_cn23xx_pf_bar1_idx_read;
984
985	oct->fn_list.enable_interrupt = lio_cn23xx_pf_enable_interrupt;
986	oct->fn_list.disable_interrupt = lio_cn23xx_pf_disable_interrupt;
987
988	oct->fn_list.enable_io_queues = lio_cn23xx_pf_enable_io_queues;
989	oct->fn_list.disable_io_queues = lio_cn23xx_pf_disable_io_queues;
990
991	lio_cn23xx_pf_setup_reg_address(oct);
992
993	oct->coproc_clock_rate = 1000000ULL *
994		lio_cn23xx_pf_coprocessor_clock(oct);
995
996	return (0);
997}
998
999int
1000lio_cn23xx_pf_fw_loaded(struct octeon_device *oct)
1001{
1002	uint64_t	val;
1003
1004	val = lio_read_csr64(oct, LIO_CN23XX_SLI_SCRATCH2);
1005	return ((val >> SCR2_BIT_FW_LOADED) & 1ULL);
1006}
1007
1008