1// SPDX-License-Identifier: GPL-2.0
2/* psycho_common.c: Code common to PSYCHO and derivative PCI controllers.
3 *
4 * Copyright (C) 2008 David S. Miller <davem@davemloft.net>
5 */
6#include <linux/kernel.h>
7#include <linux/interrupt.h>
8#include <linux/numa.h>
9#include <linux/platform_device.h>
10
11#include <asm/upa.h>
12
13#include "pci_impl.h"
14#include "iommu_common.h"
15#include "psycho_common.h"
16
17#define  PSYCHO_STRBUF_CTRL_DENAB	0x0000000000000002ULL
18#define  PSYCHO_STCERR_WRITE		0x0000000000000002ULL
19#define  PSYCHO_STCERR_READ		0x0000000000000001ULL
20#define  PSYCHO_STCTAG_PPN		0x0fffffff00000000ULL
21#define  PSYCHO_STCTAG_VPN		0x00000000ffffe000ULL
22#define  PSYCHO_STCTAG_VALID		0x0000000000000002ULL
23#define  PSYCHO_STCTAG_WRITE		0x0000000000000001ULL
24#define  PSYCHO_STCLINE_LINDX		0x0000000001e00000ULL
25#define  PSYCHO_STCLINE_SPTR		0x00000000001f8000ULL
26#define  PSYCHO_STCLINE_LADDR		0x0000000000007f00ULL
27#define  PSYCHO_STCLINE_EPTR		0x00000000000000fcULL
28#define  PSYCHO_STCLINE_VALID		0x0000000000000002ULL
29#define  PSYCHO_STCLINE_FOFN		0x0000000000000001ULL
30
31static DEFINE_SPINLOCK(stc_buf_lock);
32static unsigned long stc_error_buf[128];
33static unsigned long stc_tag_buf[16];
34static unsigned long stc_line_buf[16];
35
36static void psycho_check_stc_error(struct pci_pbm_info *pbm)
37{
38	unsigned long err_base, tag_base, line_base;
39	struct strbuf *strbuf = &pbm->stc;
40	u64 control;
41	int i;
42
43	if (!strbuf->strbuf_control)
44		return;
45
46	err_base = strbuf->strbuf_err_stat;
47	tag_base = strbuf->strbuf_tag_diag;
48	line_base = strbuf->strbuf_line_diag;
49
50	spin_lock(&stc_buf_lock);
51
52	/* This is __REALLY__ dangerous.  When we put the streaming
53	 * buffer into diagnostic mode to probe its tags and error
54	 * status, we _must_ clear all of the line tag valid bits
55	 * before re-enabling the streaming buffer.  If any dirty data
56	 * lives in the STC when we do this, we will end up
57	 * invalidating it before it has a chance to reach main
58	 * memory.
59	 */
60	control = upa_readq(strbuf->strbuf_control);
61	upa_writeq(control | PSYCHO_STRBUF_CTRL_DENAB, strbuf->strbuf_control);
62	for (i = 0; i < 128; i++) {
63		u64 val;
64
65		val = upa_readq(err_base + (i * 8UL));
66		upa_writeq(0UL, err_base + (i * 8UL));
67		stc_error_buf[i] = val;
68	}
69	for (i = 0; i < 16; i++) {
70		stc_tag_buf[i] = upa_readq(tag_base + (i * 8UL));
71		stc_line_buf[i] = upa_readq(line_base + (i * 8UL));
72		upa_writeq(0UL, tag_base + (i * 8UL));
73		upa_writeq(0UL, line_base + (i * 8UL));
74	}
75
76	/* OK, state is logged, exit diagnostic mode. */
77	upa_writeq(control, strbuf->strbuf_control);
78
79	for (i = 0; i < 16; i++) {
80		int j, saw_error, first, last;
81
82		saw_error = 0;
83		first = i * 8;
84		last = first + 8;
85		for (j = first; j < last; j++) {
86			u64 errval = stc_error_buf[j];
87			if (errval != 0) {
88				saw_error++;
89				printk(KERN_ERR "%s: STC_ERR(%d)[wr(%d)"
90				       "rd(%d)]\n",
91				       pbm->name,
92				       j,
93				       (errval & PSYCHO_STCERR_WRITE) ? 1 : 0,
94				       (errval & PSYCHO_STCERR_READ) ? 1 : 0);
95			}
96		}
97		if (saw_error != 0) {
98			u64 tagval = stc_tag_buf[i];
99			u64 lineval = stc_line_buf[i];
100			printk(KERN_ERR "%s: STC_TAG(%d)[PA(%016llx)VA(%08llx)"
101			       "V(%d)W(%d)]\n",
102			       pbm->name,
103			       i,
104			       ((tagval & PSYCHO_STCTAG_PPN) >> 19UL),
105			       (tagval & PSYCHO_STCTAG_VPN),
106			       ((tagval & PSYCHO_STCTAG_VALID) ? 1 : 0),
107			       ((tagval & PSYCHO_STCTAG_WRITE) ? 1 : 0));
108			printk(KERN_ERR "%s: STC_LINE(%d)[LIDX(%llx)SP(%llx)"
109			       "LADDR(%llx)EP(%llx)V(%d)FOFN(%d)]\n",
110			       pbm->name,
111			       i,
112			       ((lineval & PSYCHO_STCLINE_LINDX) >> 21UL),
113			       ((lineval & PSYCHO_STCLINE_SPTR) >> 15UL),
114			       ((lineval & PSYCHO_STCLINE_LADDR) >> 8UL),
115			       ((lineval & PSYCHO_STCLINE_EPTR) >> 2UL),
116			       ((lineval & PSYCHO_STCLINE_VALID) ? 1 : 0),
117			       ((lineval & PSYCHO_STCLINE_FOFN) ? 1 : 0));
118		}
119	}
120
121	spin_unlock(&stc_buf_lock);
122}
123
124#define PSYCHO_IOMMU_TAG		0xa580UL
125#define PSYCHO_IOMMU_DATA		0xa600UL
126
127static void psycho_record_iommu_tags_and_data(struct pci_pbm_info *pbm,
128					      u64 *tag, u64 *data)
129{
130	int i;
131
132	for (i = 0; i < 16; i++) {
133		unsigned long base = pbm->controller_regs;
134		unsigned long off = i * 8UL;
135
136		tag[i] = upa_readq(base + PSYCHO_IOMMU_TAG+off);
137		data[i] = upa_readq(base + PSYCHO_IOMMU_DATA+off);
138
139		/* Now clear out the entry. */
140		upa_writeq(0, base + PSYCHO_IOMMU_TAG + off);
141		upa_writeq(0, base + PSYCHO_IOMMU_DATA + off);
142	}
143}
144
145#define  PSYCHO_IOMMU_TAG_ERRSTS (0x3UL << 23UL)
146#define  PSYCHO_IOMMU_TAG_ERR	 (0x1UL << 22UL)
147#define  PSYCHO_IOMMU_TAG_WRITE	 (0x1UL << 21UL)
148#define  PSYCHO_IOMMU_TAG_STREAM (0x1UL << 20UL)
149#define  PSYCHO_IOMMU_TAG_SIZE	 (0x1UL << 19UL)
150#define  PSYCHO_IOMMU_TAG_VPAGE	 0x7ffffULL
151#define  PSYCHO_IOMMU_DATA_VALID (1UL << 30UL)
152#define  PSYCHO_IOMMU_DATA_CACHE (1UL << 28UL)
153#define  PSYCHO_IOMMU_DATA_PPAGE 0xfffffffULL
154
155static void psycho_dump_iommu_tags_and_data(struct pci_pbm_info *pbm,
156					    u64 *tag, u64 *data)
157{
158	int i;
159
160	for (i = 0; i < 16; i++) {
161		u64 tag_val, data_val;
162		const char *type_str;
163		tag_val = tag[i];
164		if (!(tag_val & PSYCHO_IOMMU_TAG_ERR))
165			continue;
166
167		data_val = data[i];
168		switch((tag_val & PSYCHO_IOMMU_TAG_ERRSTS) >> 23UL) {
169		case 0:
170			type_str = "Protection Error";
171			break;
172		case 1:
173			type_str = "Invalid Error";
174			break;
175		case 2:
176			type_str = "TimeOut Error";
177			break;
178		case 3:
179		default:
180			type_str = "ECC Error";
181			break;
182		}
183
184		printk(KERN_ERR "%s: IOMMU TAG(%d)[error(%s) wr(%d) "
185		       "str(%d) sz(%dK) vpg(%08llx)]\n",
186		       pbm->name, i, type_str,
187		       ((tag_val & PSYCHO_IOMMU_TAG_WRITE) ? 1 : 0),
188		       ((tag_val & PSYCHO_IOMMU_TAG_STREAM) ? 1 : 0),
189		       ((tag_val & PSYCHO_IOMMU_TAG_SIZE) ? 64 : 8),
190		       (tag_val & PSYCHO_IOMMU_TAG_VPAGE) << IOMMU_PAGE_SHIFT);
191		printk(KERN_ERR "%s: IOMMU DATA(%d)[valid(%d) cache(%d) "
192		       "ppg(%016llx)]\n",
193		       pbm->name, i,
194		       ((data_val & PSYCHO_IOMMU_DATA_VALID) ? 1 : 0),
195		       ((data_val & PSYCHO_IOMMU_DATA_CACHE) ? 1 : 0),
196		       (data_val & PSYCHO_IOMMU_DATA_PPAGE) << IOMMU_PAGE_SHIFT);
197	}
198}
199
200#define  PSYCHO_IOMMU_CTRL_XLTESTAT	0x0000000006000000UL
201#define  PSYCHO_IOMMU_CTRL_XLTEERR	0x0000000001000000UL
202
203void psycho_check_iommu_error(struct pci_pbm_info *pbm,
204			      unsigned long afsr,
205			      unsigned long afar,
206			      enum psycho_error_type type)
207{
208	u64 control, iommu_tag[16], iommu_data[16];
209	struct iommu *iommu = pbm->iommu;
210	unsigned long flags;
211
212	spin_lock_irqsave(&iommu->lock, flags);
213	control = upa_readq(iommu->iommu_control);
214	if (control & PSYCHO_IOMMU_CTRL_XLTEERR) {
215		const char *type_str;
216
217		control &= ~PSYCHO_IOMMU_CTRL_XLTEERR;
218		upa_writeq(control, iommu->iommu_control);
219
220		switch ((control & PSYCHO_IOMMU_CTRL_XLTESTAT) >> 25UL) {
221		case 0:
222			type_str = "Protection Error";
223			break;
224		case 1:
225			type_str = "Invalid Error";
226			break;
227		case 2:
228			type_str = "TimeOut Error";
229			break;
230		case 3:
231		default:
232			type_str = "ECC Error";
233			break;
234		}
235		printk(KERN_ERR "%s: IOMMU Error, type[%s]\n",
236		       pbm->name, type_str);
237
238		/* It is very possible for another DVMA to occur while
239		 * we do this probe, and corrupt the system further.
240		 * But we are so screwed at this point that we are
241		 * likely to crash hard anyways, so get as much
242		 * diagnostic information to the console as we can.
243		 */
244		psycho_record_iommu_tags_and_data(pbm, iommu_tag, iommu_data);
245		psycho_dump_iommu_tags_and_data(pbm, iommu_tag, iommu_data);
246	}
247	psycho_check_stc_error(pbm);
248	spin_unlock_irqrestore(&iommu->lock, flags);
249}
250
251#define  PSYCHO_PCICTRL_SBH_ERR	 0x0000000800000000UL
252#define  PSYCHO_PCICTRL_SERR	 0x0000000400000000UL
253
254static irqreturn_t psycho_pcierr_intr_other(struct pci_pbm_info *pbm)
255{
256	irqreturn_t ret = IRQ_NONE;
257	u64 csr, csr_error_bits;
258	u16 stat, *addr;
259
260	csr = upa_readq(pbm->pci_csr);
261	csr_error_bits = csr & (PSYCHO_PCICTRL_SBH_ERR | PSYCHO_PCICTRL_SERR);
262	if (csr_error_bits) {
263		/* Clear the errors.  */
264		upa_writeq(csr, pbm->pci_csr);
265
266		/* Log 'em.  */
267		if (csr_error_bits & PSYCHO_PCICTRL_SBH_ERR)
268			printk(KERN_ERR "%s: PCI streaming byte hole "
269			       "error asserted.\n", pbm->name);
270		if (csr_error_bits & PSYCHO_PCICTRL_SERR)
271			printk(KERN_ERR "%s: PCI SERR signal asserted.\n",
272			       pbm->name);
273		ret = IRQ_HANDLED;
274	}
275	addr = psycho_pci_config_mkaddr(pbm, pbm->pci_first_busno,
276					0, PCI_STATUS);
277	pci_config_read16(addr, &stat);
278	if (stat & (PCI_STATUS_PARITY |
279		    PCI_STATUS_SIG_TARGET_ABORT |
280		    PCI_STATUS_REC_TARGET_ABORT |
281		    PCI_STATUS_REC_MASTER_ABORT |
282		    PCI_STATUS_SIG_SYSTEM_ERROR)) {
283		printk(KERN_ERR "%s: PCI bus error, PCI_STATUS[%04x]\n",
284		       pbm->name, stat);
285		pci_config_write16(addr, 0xffff);
286		ret = IRQ_HANDLED;
287	}
288	return ret;
289}
290
291#define  PSYCHO_PCIAFSR_PMA	0x8000000000000000ULL
292#define  PSYCHO_PCIAFSR_PTA	0x4000000000000000ULL
293#define  PSYCHO_PCIAFSR_PRTRY	0x2000000000000000ULL
294#define  PSYCHO_PCIAFSR_PPERR	0x1000000000000000ULL
295#define  PSYCHO_PCIAFSR_SMA	0x0800000000000000ULL
296#define  PSYCHO_PCIAFSR_STA	0x0400000000000000ULL
297#define  PSYCHO_PCIAFSR_SRTRY	0x0200000000000000ULL
298#define  PSYCHO_PCIAFSR_SPERR	0x0100000000000000ULL
299#define  PSYCHO_PCIAFSR_RESV1	0x00ff000000000000ULL
300#define  PSYCHO_PCIAFSR_BMSK	0x0000ffff00000000ULL
301#define  PSYCHO_PCIAFSR_BLK	0x0000000080000000ULL
302#define  PSYCHO_PCIAFSR_RESV2	0x0000000040000000ULL
303#define  PSYCHO_PCIAFSR_MID	0x000000003e000000ULL
304#define  PSYCHO_PCIAFSR_RESV3	0x0000000001ffffffULL
305
306irqreturn_t psycho_pcierr_intr(int irq, void *dev_id)
307{
308	struct pci_pbm_info *pbm = dev_id;
309	u64 afsr, afar, error_bits;
310	int reported;
311
312	afsr = upa_readq(pbm->pci_afsr);
313	afar = upa_readq(pbm->pci_afar);
314	error_bits = afsr &
315		(PSYCHO_PCIAFSR_PMA | PSYCHO_PCIAFSR_PTA |
316		 PSYCHO_PCIAFSR_PRTRY | PSYCHO_PCIAFSR_PPERR |
317		 PSYCHO_PCIAFSR_SMA | PSYCHO_PCIAFSR_STA |
318		 PSYCHO_PCIAFSR_SRTRY | PSYCHO_PCIAFSR_SPERR);
319	if (!error_bits)
320		return psycho_pcierr_intr_other(pbm);
321	upa_writeq(error_bits, pbm->pci_afsr);
322	printk(KERN_ERR "%s: PCI Error, primary error type[%s]\n",
323	       pbm->name,
324	       (((error_bits & PSYCHO_PCIAFSR_PMA) ?
325		 "Master Abort" :
326		 ((error_bits & PSYCHO_PCIAFSR_PTA) ?
327		  "Target Abort" :
328		  ((error_bits & PSYCHO_PCIAFSR_PRTRY) ?
329		   "Excessive Retries" :
330		   ((error_bits & PSYCHO_PCIAFSR_PPERR) ?
331		    "Parity Error" : "???"))))));
332	printk(KERN_ERR "%s: bytemask[%04llx] UPA_MID[%02llx] was_block(%d)\n",
333	       pbm->name,
334	       (afsr & PSYCHO_PCIAFSR_BMSK) >> 32UL,
335	       (afsr & PSYCHO_PCIAFSR_MID) >> 25UL,
336	       (afsr & PSYCHO_PCIAFSR_BLK) ? 1 : 0);
337	printk(KERN_ERR "%s: PCI AFAR [%016llx]\n", pbm->name, afar);
338	printk(KERN_ERR "%s: PCI Secondary errors [", pbm->name);
339	reported = 0;
340	if (afsr & PSYCHO_PCIAFSR_SMA) {
341		reported++;
342		printk("(Master Abort)");
343	}
344	if (afsr & PSYCHO_PCIAFSR_STA) {
345		reported++;
346		printk("(Target Abort)");
347	}
348	if (afsr & PSYCHO_PCIAFSR_SRTRY) {
349		reported++;
350		printk("(Excessive Retries)");
351	}
352	if (afsr & PSYCHO_PCIAFSR_SPERR) {
353		reported++;
354		printk("(Parity Error)");
355	}
356	if (!reported)
357		printk("(none)");
358	printk("]\n");
359
360	if (error_bits & (PSYCHO_PCIAFSR_PTA | PSYCHO_PCIAFSR_STA)) {
361		psycho_check_iommu_error(pbm, afsr, afar, PCI_ERR);
362		pci_scan_for_target_abort(pbm, pbm->pci_bus);
363	}
364	if (error_bits & (PSYCHO_PCIAFSR_PMA | PSYCHO_PCIAFSR_SMA))
365		pci_scan_for_master_abort(pbm, pbm->pci_bus);
366
367	if (error_bits & (PSYCHO_PCIAFSR_PPERR | PSYCHO_PCIAFSR_SPERR))
368		pci_scan_for_parity_error(pbm, pbm->pci_bus);
369
370	return IRQ_HANDLED;
371}
372
373static void psycho_iommu_flush(struct pci_pbm_info *pbm)
374{
375	int i;
376
377	for (i = 0; i < 16; i++) {
378		unsigned long off = i * 8;
379
380		upa_writeq(0, pbm->controller_regs + PSYCHO_IOMMU_TAG + off);
381		upa_writeq(0, pbm->controller_regs + PSYCHO_IOMMU_DATA + off);
382	}
383}
384
385#define PSYCHO_IOMMU_CONTROL		0x0200UL
386#define  PSYCHO_IOMMU_CTRL_TSBSZ	0x0000000000070000UL
387#define  PSYCHO_IOMMU_TSBSZ_1K      	0x0000000000000000UL
388#define  PSYCHO_IOMMU_TSBSZ_2K      	0x0000000000010000UL
389#define  PSYCHO_IOMMU_TSBSZ_4K      	0x0000000000020000UL
390#define  PSYCHO_IOMMU_TSBSZ_8K      	0x0000000000030000UL
391#define  PSYCHO_IOMMU_TSBSZ_16K     	0x0000000000040000UL
392#define  PSYCHO_IOMMU_TSBSZ_32K     	0x0000000000050000UL
393#define  PSYCHO_IOMMU_TSBSZ_64K     	0x0000000000060000UL
394#define  PSYCHO_IOMMU_TSBSZ_128K    	0x0000000000070000UL
395#define  PSYCHO_IOMMU_CTRL_TBWSZ    	0x0000000000000004UL
396#define  PSYCHO_IOMMU_CTRL_DENAB    	0x0000000000000002UL
397#define  PSYCHO_IOMMU_CTRL_ENAB     	0x0000000000000001UL
398#define PSYCHO_IOMMU_FLUSH		0x0210UL
399#define PSYCHO_IOMMU_TSBBASE		0x0208UL
400
401int psycho_iommu_init(struct pci_pbm_info *pbm, int tsbsize,
402		      u32 dvma_offset, u32 dma_mask,
403		      unsigned long write_complete_offset)
404{
405	struct iommu *iommu = pbm->iommu;
406	u64 control;
407	int err;
408
409	iommu->iommu_control  = pbm->controller_regs + PSYCHO_IOMMU_CONTROL;
410	iommu->iommu_tsbbase  = pbm->controller_regs + PSYCHO_IOMMU_TSBBASE;
411	iommu->iommu_flush    = pbm->controller_regs + PSYCHO_IOMMU_FLUSH;
412	iommu->iommu_tags     = pbm->controller_regs + PSYCHO_IOMMU_TAG;
413	iommu->write_complete_reg = (pbm->controller_regs +
414				     write_complete_offset);
415
416	iommu->iommu_ctxflush = 0;
417
418	control = upa_readq(iommu->iommu_control);
419	control |= PSYCHO_IOMMU_CTRL_DENAB;
420	upa_writeq(control, iommu->iommu_control);
421
422	psycho_iommu_flush(pbm);
423
424	/* Leave diag mode enabled for full-flushing done in pci_iommu.c */
425	err = iommu_table_init(iommu, tsbsize * 1024 * 8,
426			       dvma_offset, dma_mask, pbm->numa_node);
427	if (err)
428		return err;
429
430	upa_writeq(__pa(iommu->page_table), iommu->iommu_tsbbase);
431
432	control = upa_readq(iommu->iommu_control);
433	control &= ~(PSYCHO_IOMMU_CTRL_TSBSZ | PSYCHO_IOMMU_CTRL_TBWSZ);
434	control |= PSYCHO_IOMMU_CTRL_ENAB;
435
436	switch (tsbsize) {
437	case 64:
438		control |= PSYCHO_IOMMU_TSBSZ_64K;
439		break;
440	case 128:
441		control |= PSYCHO_IOMMU_TSBSZ_128K;
442		break;
443	default:
444		return -EINVAL;
445	}
446
447	upa_writeq(control, iommu->iommu_control);
448
449	return 0;
450
451}
452
453void psycho_pbm_init_common(struct pci_pbm_info *pbm, struct platform_device *op,
454			    const char *chip_name, int chip_type)
455{
456	struct device_node *dp = op->dev.of_node;
457
458	pbm->name = dp->full_name;
459	pbm->numa_node = NUMA_NO_NODE;
460	pbm->chip_type = chip_type;
461	pbm->chip_version = of_getintprop_default(dp, "version#", 0);
462	pbm->chip_revision = of_getintprop_default(dp, "module-revision#", 0);
463	pbm->op = op;
464	pbm->pci_ops = &sun4u_pci_ops;
465	pbm->config_space_reg_bits = 8;
466	pbm->index = pci_num_pbms++;
467	pci_get_pbm_props(pbm);
468	pci_determine_mem_io_space(pbm);
469
470	printk(KERN_INFO "%s: %s PCI Bus Module ver[%x:%x]\n",
471	       pbm->name, chip_name,
472	       pbm->chip_version, pbm->chip_revision);
473}
474