1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2012 NVIDIA CORPORATION.  All rights reserved.
4 */
5
6#include <linux/bitfield.h>
7#include <linux/delay.h>
8#include <linux/device.h>
9#include <linux/mutex.h>
10#include <linux/of.h>
11#include <linux/slab.h>
12#include <linux/string.h>
13
14#include <dt-bindings/memory/tegra20-mc.h>
15
16#include "mc.h"
17
18#define MC_STAT_CONTROL				0x90
19#define MC_STAT_EMC_CLOCK_LIMIT			0xa0
20#define MC_STAT_EMC_CLOCKS			0xa4
21#define MC_STAT_EMC_CONTROL_0			0xa8
22#define MC_STAT_EMC_CONTROL_1			0xac
23#define MC_STAT_EMC_COUNT_0			0xb8
24#define MC_STAT_EMC_COUNT_1			0xbc
25
26#define MC_STAT_CONTROL_CLIENT_ID		GENMASK(13,  8)
27#define MC_STAT_CONTROL_EVENT			GENMASK(23, 16)
28#define MC_STAT_CONTROL_PRI_EVENT		GENMASK(25, 24)
29#define MC_STAT_CONTROL_FILTER_CLIENT_ENABLE	GENMASK(26, 26)
30#define MC_STAT_CONTROL_FILTER_PRI		GENMASK(29, 28)
31
32#define MC_STAT_CONTROL_PRI_EVENT_HP		0
33#define MC_STAT_CONTROL_PRI_EVENT_TM		1
34#define MC_STAT_CONTROL_PRI_EVENT_BW		2
35
36#define MC_STAT_CONTROL_FILTER_PRI_DISABLE	0
37#define MC_STAT_CONTROL_FILTER_PRI_NO		1
38#define MC_STAT_CONTROL_FILTER_PRI_YES		2
39
40#define MC_STAT_CONTROL_EVENT_QUALIFIED		0
41#define MC_STAT_CONTROL_EVENT_ANY_READ		1
42#define MC_STAT_CONTROL_EVENT_ANY_WRITE		2
43#define MC_STAT_CONTROL_EVENT_RD_WR_CHANGE	3
44#define MC_STAT_CONTROL_EVENT_SUCCESSIVE	4
45#define MC_STAT_CONTROL_EVENT_ARB_BANK_AA	5
46#define MC_STAT_CONTROL_EVENT_ARB_BANK_BB	6
47#define MC_STAT_CONTROL_EVENT_PAGE_MISS		7
48#define MC_STAT_CONTROL_EVENT_AUTO_PRECHARGE	8
49
50#define EMC_GATHER_RST				(0 << 8)
51#define EMC_GATHER_CLEAR			(1 << 8)
52#define EMC_GATHER_DISABLE			(2 << 8)
53#define EMC_GATHER_ENABLE			(3 << 8)
54
55#define MC_STAT_SAMPLE_TIME_USEC		16000
56
57/* we store collected statistics as a fixed point values */
58#define MC_FX_FRAC_SCALE			100
59
60static DEFINE_MUTEX(tegra20_mc_stat_lock);
61
62struct tegra20_mc_stat_gather {
63	unsigned int pri_filter;
64	unsigned int pri_event;
65	unsigned int result;
66	unsigned int client;
67	unsigned int event;
68	bool client_enb;
69};
70
71struct tegra20_mc_stat {
72	struct tegra20_mc_stat_gather gather0;
73	struct tegra20_mc_stat_gather gather1;
74	unsigned int sample_time_usec;
75	const struct tegra_mc *mc;
76};
77
78struct tegra20_mc_client_stat {
79	unsigned int events;
80	unsigned int arb_high_prio;
81	unsigned int arb_timeout;
82	unsigned int arb_bandwidth;
83	unsigned int rd_wr_change;
84	unsigned int successive;
85	unsigned int page_miss;
86	unsigned int auto_precharge;
87	unsigned int arb_bank_aa;
88	unsigned int arb_bank_bb;
89};
90
91static const struct tegra_mc_client tegra20_mc_clients[] = {
92	{
93		.id = 0x00,
94		.name = "display0a",
95	}, {
96		.id = 0x01,
97		.name = "display0ab",
98	}, {
99		.id = 0x02,
100		.name = "display0b",
101	}, {
102		.id = 0x03,
103		.name = "display0bb",
104	}, {
105		.id = 0x04,
106		.name = "display0c",
107	}, {
108		.id = 0x05,
109		.name = "display0cb",
110	}, {
111		.id = 0x06,
112		.name = "display1b",
113	}, {
114		.id = 0x07,
115		.name = "display1bb",
116	}, {
117		.id = 0x08,
118		.name = "eppup",
119	}, {
120		.id = 0x09,
121		.name = "g2pr",
122	}, {
123		.id = 0x0a,
124		.name = "g2sr",
125	}, {
126		.id = 0x0b,
127		.name = "mpeunifbr",
128	}, {
129		.id = 0x0c,
130		.name = "viruv",
131	}, {
132		.id = 0x0d,
133		.name = "avpcarm7r",
134	}, {
135		.id = 0x0e,
136		.name = "displayhc",
137	}, {
138		.id = 0x0f,
139		.name = "displayhcb",
140	}, {
141		.id = 0x10,
142		.name = "fdcdrd",
143	}, {
144		.id = 0x11,
145		.name = "g2dr",
146	}, {
147		.id = 0x12,
148		.name = "host1xdmar",
149	}, {
150		.id = 0x13,
151		.name = "host1xr",
152	}, {
153		.id = 0x14,
154		.name = "idxsrd",
155	}, {
156		.id = 0x15,
157		.name = "mpcorer",
158	}, {
159		.id = 0x16,
160		.name = "mpe_ipred",
161	}, {
162		.id = 0x17,
163		.name = "mpeamemrd",
164	}, {
165		.id = 0x18,
166		.name = "mpecsrd",
167	}, {
168		.id = 0x19,
169		.name = "ppcsahbdmar",
170	}, {
171		.id = 0x1a,
172		.name = "ppcsahbslvr",
173	}, {
174		.id = 0x1b,
175		.name = "texsrd",
176	}, {
177		.id = 0x1c,
178		.name = "vdebsevr",
179	}, {
180		.id = 0x1d,
181		.name = "vdember",
182	}, {
183		.id = 0x1e,
184		.name = "vdemcer",
185	}, {
186		.id = 0x1f,
187		.name = "vdetper",
188	}, {
189		.id = 0x20,
190		.name = "eppu",
191	}, {
192		.id = 0x21,
193		.name = "eppv",
194	}, {
195		.id = 0x22,
196		.name = "eppy",
197	}, {
198		.id = 0x23,
199		.name = "mpeunifbw",
200	}, {
201		.id = 0x24,
202		.name = "viwsb",
203	}, {
204		.id = 0x25,
205		.name = "viwu",
206	}, {
207		.id = 0x26,
208		.name = "viwv",
209	}, {
210		.id = 0x27,
211		.name = "viwy",
212	}, {
213		.id = 0x28,
214		.name = "g2dw",
215	}, {
216		.id = 0x29,
217		.name = "avpcarm7w",
218	}, {
219		.id = 0x2a,
220		.name = "fdcdwr",
221	}, {
222		.id = 0x2b,
223		.name = "host1xw",
224	}, {
225		.id = 0x2c,
226		.name = "ispw",
227	}, {
228		.id = 0x2d,
229		.name = "mpcorew",
230	}, {
231		.id = 0x2e,
232		.name = "mpecswr",
233	}, {
234		.id = 0x2f,
235		.name = "ppcsahbdmaw",
236	}, {
237		.id = 0x30,
238		.name = "ppcsahbslvw",
239	}, {
240		.id = 0x31,
241		.name = "vdebsevw",
242	}, {
243		.id = 0x32,
244		.name = "vdembew",
245	}, {
246		.id = 0x33,
247		.name = "vdetpmw",
248	},
249};
250
251#define TEGRA20_MC_RESET(_name, _control, _status, _reset, _bit)	\
252	{								\
253		.name = #_name,						\
254		.id = TEGRA20_MC_RESET_##_name,				\
255		.control = _control,					\
256		.status = _status,					\
257		.reset = _reset,					\
258		.bit = _bit,						\
259	}
260
261static const struct tegra_mc_reset tegra20_mc_resets[] = {
262	TEGRA20_MC_RESET(AVPC,   0x100, 0x140, 0x104,  0),
263	TEGRA20_MC_RESET(DC,     0x100, 0x144, 0x104,  1),
264	TEGRA20_MC_RESET(DCB,    0x100, 0x148, 0x104,  2),
265	TEGRA20_MC_RESET(EPP,    0x100, 0x14c, 0x104,  3),
266	TEGRA20_MC_RESET(2D,     0x100, 0x150, 0x104,  4),
267	TEGRA20_MC_RESET(HC,     0x100, 0x154, 0x104,  5),
268	TEGRA20_MC_RESET(ISP,    0x100, 0x158, 0x104,  6),
269	TEGRA20_MC_RESET(MPCORE, 0x100, 0x15c, 0x104,  7),
270	TEGRA20_MC_RESET(MPEA,   0x100, 0x160, 0x104,  8),
271	TEGRA20_MC_RESET(MPEB,   0x100, 0x164, 0x104,  9),
272	TEGRA20_MC_RESET(MPEC,   0x100, 0x168, 0x104, 10),
273	TEGRA20_MC_RESET(3D,     0x100, 0x16c, 0x104, 11),
274	TEGRA20_MC_RESET(PPCS,   0x100, 0x170, 0x104, 12),
275	TEGRA20_MC_RESET(VDE,    0x100, 0x174, 0x104, 13),
276	TEGRA20_MC_RESET(VI,     0x100, 0x178, 0x104, 14),
277};
278
279static int tegra20_mc_hotreset_assert(struct tegra_mc *mc,
280				      const struct tegra_mc_reset *rst)
281{
282	unsigned long flags;
283	u32 value;
284
285	spin_lock_irqsave(&mc->lock, flags);
286
287	value = mc_readl(mc, rst->reset);
288	mc_writel(mc, value & ~BIT(rst->bit), rst->reset);
289
290	spin_unlock_irqrestore(&mc->lock, flags);
291
292	return 0;
293}
294
295static int tegra20_mc_hotreset_deassert(struct tegra_mc *mc,
296					const struct tegra_mc_reset *rst)
297{
298	unsigned long flags;
299	u32 value;
300
301	spin_lock_irqsave(&mc->lock, flags);
302
303	value = mc_readl(mc, rst->reset);
304	mc_writel(mc, value | BIT(rst->bit), rst->reset);
305
306	spin_unlock_irqrestore(&mc->lock, flags);
307
308	return 0;
309}
310
311static int tegra20_mc_block_dma(struct tegra_mc *mc,
312				const struct tegra_mc_reset *rst)
313{
314	unsigned long flags;
315	u32 value;
316
317	spin_lock_irqsave(&mc->lock, flags);
318
319	value = mc_readl(mc, rst->control) & ~BIT(rst->bit);
320	mc_writel(mc, value, rst->control);
321
322	spin_unlock_irqrestore(&mc->lock, flags);
323
324	return 0;
325}
326
327static bool tegra20_mc_dma_idling(struct tegra_mc *mc,
328				  const struct tegra_mc_reset *rst)
329{
330	return mc_readl(mc, rst->status) == 0;
331}
332
333static int tegra20_mc_reset_status(struct tegra_mc *mc,
334				   const struct tegra_mc_reset *rst)
335{
336	return (mc_readl(mc, rst->reset) & BIT(rst->bit)) == 0;
337}
338
339static int tegra20_mc_unblock_dma(struct tegra_mc *mc,
340				  const struct tegra_mc_reset *rst)
341{
342	unsigned long flags;
343	u32 value;
344
345	spin_lock_irqsave(&mc->lock, flags);
346
347	value = mc_readl(mc, rst->control) | BIT(rst->bit);
348	mc_writel(mc, value, rst->control);
349
350	spin_unlock_irqrestore(&mc->lock, flags);
351
352	return 0;
353}
354
355static const struct tegra_mc_reset_ops tegra20_mc_reset_ops = {
356	.hotreset_assert = tegra20_mc_hotreset_assert,
357	.hotreset_deassert = tegra20_mc_hotreset_deassert,
358	.block_dma = tegra20_mc_block_dma,
359	.dma_idling = tegra20_mc_dma_idling,
360	.unblock_dma = tegra20_mc_unblock_dma,
361	.reset_status = tegra20_mc_reset_status,
362};
363
364static int tegra20_mc_icc_set(struct icc_node *src, struct icc_node *dst)
365{
366	/*
367	 * It should be possible to tune arbitration knobs here, but the
368	 * default values are known to work well on all devices. Hence
369	 * nothing to do here so far.
370	 */
371	return 0;
372}
373
374static int tegra20_mc_icc_aggreate(struct icc_node *node, u32 tag, u32 avg_bw,
375				   u32 peak_bw, u32 *agg_avg, u32 *agg_peak)
376{
377	/*
378	 * ISO clients need to reserve extra bandwidth up-front because
379	 * there could be high bandwidth pressure during initial filling
380	 * of the client's FIFO buffers.  Secondly, we need to take into
381	 * account impurities of the memory subsystem.
382	 */
383	if (tag & TEGRA_MC_ICC_TAG_ISO)
384		peak_bw = tegra_mc_scale_percents(peak_bw, 300);
385
386	*agg_avg += avg_bw;
387	*agg_peak = max(*agg_peak, peak_bw);
388
389	return 0;
390}
391
392static struct icc_node_data *
393tegra20_mc_of_icc_xlate_extended(const struct of_phandle_args *spec, void *data)
394{
395	struct tegra_mc *mc = icc_provider_to_tegra_mc(data);
396	unsigned int i, idx = spec->args[0];
397	struct icc_node_data *ndata;
398	struct icc_node *node;
399
400	list_for_each_entry(node, &mc->provider.nodes, node_list) {
401		if (node->id != idx)
402			continue;
403
404		ndata = kzalloc(sizeof(*ndata), GFP_KERNEL);
405		if (!ndata)
406			return ERR_PTR(-ENOMEM);
407
408		ndata->node = node;
409
410		/* these clients are isochronous by default */
411		if (strstarts(node->name, "display") ||
412		    strstarts(node->name, "vi"))
413			ndata->tag = TEGRA_MC_ICC_TAG_ISO;
414		else
415			ndata->tag = TEGRA_MC_ICC_TAG_DEFAULT;
416
417		return ndata;
418	}
419
420	for (i = 0; i < mc->soc->num_clients; i++) {
421		if (mc->soc->clients[i].id == idx)
422			return ERR_PTR(-EPROBE_DEFER);
423	}
424
425	dev_err(mc->dev, "invalid ICC client ID %u\n", idx);
426
427	return ERR_PTR(-EINVAL);
428}
429
430static const struct tegra_mc_icc_ops tegra20_mc_icc_ops = {
431	.xlate_extended = tegra20_mc_of_icc_xlate_extended,
432	.aggregate = tegra20_mc_icc_aggreate,
433	.set = tegra20_mc_icc_set,
434};
435
436static u32 tegra20_mc_stat_gather_control(const struct tegra20_mc_stat_gather *g)
437{
438	u32 control;
439
440	control  = FIELD_PREP(MC_STAT_CONTROL_EVENT, g->event);
441	control |= FIELD_PREP(MC_STAT_CONTROL_CLIENT_ID, g->client);
442	control |= FIELD_PREP(MC_STAT_CONTROL_PRI_EVENT, g->pri_event);
443	control |= FIELD_PREP(MC_STAT_CONTROL_FILTER_PRI, g->pri_filter);
444	control |= FIELD_PREP(MC_STAT_CONTROL_FILTER_CLIENT_ENABLE, g->client_enb);
445
446	return control;
447}
448
449static void tegra20_mc_stat_gather(struct tegra20_mc_stat *stat)
450{
451	u32 clocks, count0, count1, control_0, control_1;
452	const struct tegra_mc *mc = stat->mc;
453
454	control_0 = tegra20_mc_stat_gather_control(&stat->gather0);
455	control_1 = tegra20_mc_stat_gather_control(&stat->gather1);
456
457	/*
458	 * Reset statistic gathers state, select statistics collection mode
459	 * and set clocks counter saturation limit to maximum.
460	 */
461	mc_writel(mc, 0x00000000, MC_STAT_CONTROL);
462	mc_writel(mc,  control_0, MC_STAT_EMC_CONTROL_0);
463	mc_writel(mc,  control_1, MC_STAT_EMC_CONTROL_1);
464	mc_writel(mc, 0xffffffff, MC_STAT_EMC_CLOCK_LIMIT);
465
466	mc_writel(mc, EMC_GATHER_ENABLE, MC_STAT_CONTROL);
467	fsleep(stat->sample_time_usec);
468	mc_writel(mc, EMC_GATHER_DISABLE, MC_STAT_CONTROL);
469
470	count0 = mc_readl(mc, MC_STAT_EMC_COUNT_0);
471	count1 = mc_readl(mc, MC_STAT_EMC_COUNT_1);
472	clocks = mc_readl(mc, MC_STAT_EMC_CLOCKS);
473	clocks = max(clocks / 100 / MC_FX_FRAC_SCALE, 1u);
474
475	stat->gather0.result = DIV_ROUND_UP(count0, clocks);
476	stat->gather1.result = DIV_ROUND_UP(count1, clocks);
477}
478
479static void tegra20_mc_stat_events(const struct tegra_mc *mc,
480				   const struct tegra_mc_client *client0,
481				   const struct tegra_mc_client *client1,
482				   unsigned int pri_filter,
483				   unsigned int pri_event,
484				   unsigned int event,
485				   unsigned int *result0,
486				   unsigned int *result1)
487{
488	struct tegra20_mc_stat stat = {};
489
490	stat.gather0.client = client0 ? client0->id : 0;
491	stat.gather0.pri_filter = pri_filter;
492	stat.gather0.client_enb = !!client0;
493	stat.gather0.pri_event = pri_event;
494	stat.gather0.event = event;
495
496	stat.gather1.client = client1 ? client1->id : 0;
497	stat.gather1.pri_filter = pri_filter;
498	stat.gather1.client_enb = !!client1;
499	stat.gather1.pri_event = pri_event;
500	stat.gather1.event = event;
501
502	stat.sample_time_usec = MC_STAT_SAMPLE_TIME_USEC;
503	stat.mc = mc;
504
505	tegra20_mc_stat_gather(&stat);
506
507	*result0 = stat.gather0.result;
508	*result1 = stat.gather1.result;
509}
510
511static void tegra20_mc_collect_stats(const struct tegra_mc *mc,
512				     struct tegra20_mc_client_stat *stats)
513{
514	const struct tegra_mc_client *client0, *client1;
515	unsigned int i;
516
517	/* collect memory controller utilization percent for each client */
518	for (i = 0; i < mc->soc->num_clients; i += 2) {
519		client0 = &mc->soc->clients[i];
520		client1 = &mc->soc->clients[i + 1];
521
522		if (i + 1 == mc->soc->num_clients)
523			client1 = NULL;
524
525		tegra20_mc_stat_events(mc, client0, client1,
526				       MC_STAT_CONTROL_FILTER_PRI_DISABLE,
527				       MC_STAT_CONTROL_PRI_EVENT_HP,
528				       MC_STAT_CONTROL_EVENT_QUALIFIED,
529				       &stats[i + 0].events,
530				       &stats[i + 1].events);
531	}
532
533	/* collect more info from active clients */
534	for (i = 0; i < mc->soc->num_clients; i++) {
535		unsigned int clienta, clientb = mc->soc->num_clients;
536
537		for (client0 = NULL; i < mc->soc->num_clients; i++) {
538			if (stats[i].events) {
539				client0 = &mc->soc->clients[i];
540				clienta = i++;
541				break;
542			}
543		}
544
545		for (client1 = NULL; i < mc->soc->num_clients; i++) {
546			if (stats[i].events) {
547				client1 = &mc->soc->clients[i];
548				clientb = i;
549				break;
550			}
551		}
552
553		if (!client0 && !client1)
554			break;
555
556		tegra20_mc_stat_events(mc, client0, client1,
557				       MC_STAT_CONTROL_FILTER_PRI_YES,
558				       MC_STAT_CONTROL_PRI_EVENT_HP,
559				       MC_STAT_CONTROL_EVENT_QUALIFIED,
560				       &stats[clienta].arb_high_prio,
561				       &stats[clientb].arb_high_prio);
562
563		tegra20_mc_stat_events(mc, client0, client1,
564				       MC_STAT_CONTROL_FILTER_PRI_YES,
565				       MC_STAT_CONTROL_PRI_EVENT_TM,
566				       MC_STAT_CONTROL_EVENT_QUALIFIED,
567				       &stats[clienta].arb_timeout,
568				       &stats[clientb].arb_timeout);
569
570		tegra20_mc_stat_events(mc, client0, client1,
571				       MC_STAT_CONTROL_FILTER_PRI_YES,
572				       MC_STAT_CONTROL_PRI_EVENT_BW,
573				       MC_STAT_CONTROL_EVENT_QUALIFIED,
574				       &stats[clienta].arb_bandwidth,
575				       &stats[clientb].arb_bandwidth);
576
577		tegra20_mc_stat_events(mc, client0, client1,
578				       MC_STAT_CONTROL_FILTER_PRI_DISABLE,
579				       MC_STAT_CONTROL_PRI_EVENT_HP,
580				       MC_STAT_CONTROL_EVENT_RD_WR_CHANGE,
581				       &stats[clienta].rd_wr_change,
582				       &stats[clientb].rd_wr_change);
583
584		tegra20_mc_stat_events(mc, client0, client1,
585				       MC_STAT_CONTROL_FILTER_PRI_DISABLE,
586				       MC_STAT_CONTROL_PRI_EVENT_HP,
587				       MC_STAT_CONTROL_EVENT_SUCCESSIVE,
588				       &stats[clienta].successive,
589				       &stats[clientb].successive);
590
591		tegra20_mc_stat_events(mc, client0, client1,
592				       MC_STAT_CONTROL_FILTER_PRI_DISABLE,
593				       MC_STAT_CONTROL_PRI_EVENT_HP,
594				       MC_STAT_CONTROL_EVENT_PAGE_MISS,
595				       &stats[clienta].page_miss,
596				       &stats[clientb].page_miss);
597	}
598}
599
600static void tegra20_mc_printf_percents(struct seq_file *s,
601				       const char *fmt,
602				       unsigned int percents_fx)
603{
604	char percents_str[8];
605
606	snprintf(percents_str, ARRAY_SIZE(percents_str), "%3u.%02u%%",
607		 percents_fx / MC_FX_FRAC_SCALE, percents_fx % MC_FX_FRAC_SCALE);
608
609	seq_printf(s, fmt, percents_str);
610}
611
612static int tegra20_mc_stats_show(struct seq_file *s, void *unused)
613{
614	const struct tegra_mc *mc = dev_get_drvdata(s->private);
615	struct tegra20_mc_client_stat *stats;
616	unsigned int i;
617
618	stats = kcalloc(mc->soc->num_clients + 1, sizeof(*stats), GFP_KERNEL);
619	if (!stats)
620		return -ENOMEM;
621
622	mutex_lock(&tegra20_mc_stat_lock);
623
624	tegra20_mc_collect_stats(mc, stats);
625
626	mutex_unlock(&tegra20_mc_stat_lock);
627
628	seq_puts(s, "Memory client   Events   Timeout   High priority   Bandwidth ARB   RW change   Successive   Page miss\n");
629	seq_puts(s, "-----------------------------------------------------------------------------------------------------\n");
630
631	for (i = 0; i < mc->soc->num_clients; i++) {
632		seq_printf(s, "%-14s  ", mc->soc->clients[i].name);
633
634		/* An event is generated when client performs R/W request. */
635		tegra20_mc_printf_percents(s,  "%-9s", stats[i].events);
636
637		/*
638		 * An event is generated based on the timeout (TM) signal
639		 * accompanying a request for arbitration.
640		 */
641		tegra20_mc_printf_percents(s, "%-10s", stats[i].arb_timeout);
642
643		/*
644		 * An event is generated based on the high-priority (HP) signal
645		 * accompanying a request for arbitration.
646		 */
647		tegra20_mc_printf_percents(s, "%-16s", stats[i].arb_high_prio);
648
649		/*
650		 * An event is generated based on the bandwidth (BW) signal
651		 * accompanying a request for arbitration.
652		 */
653		tegra20_mc_printf_percents(s, "%-16s", stats[i].arb_bandwidth);
654
655		/*
656		 * An event is generated when the memory controller switches
657		 * between making a read request to making a write request.
658		 */
659		tegra20_mc_printf_percents(s, "%-12s", stats[i].rd_wr_change);
660
661		/*
662		 * An even generated when the chosen client has wins arbitration
663		 * when it was also the winner at the previous request.  If a
664		 * client makes N requests in a row that are honored, SUCCESSIVE
665		 * will be counted (N-1) times.  Large values for this event
666		 * imply that if we were patient enough, all of those requests
667		 * could have been coalesced.
668		 */
669		tegra20_mc_printf_percents(s, "%-13s", stats[i].successive);
670
671		/*
672		 * An event is generated when the memory controller detects a
673		 * page miss for the current request.
674		 */
675		tegra20_mc_printf_percents(s, "%-12s\n", stats[i].page_miss);
676	}
677
678	kfree(stats);
679
680	return 0;
681}
682
683static int tegra20_mc_probe(struct tegra_mc *mc)
684{
685	debugfs_create_devm_seqfile(mc->dev, "stats", mc->debugfs.root,
686				    tegra20_mc_stats_show);
687
688	return 0;
689}
690
691static irqreturn_t tegra20_mc_handle_irq(int irq, void *data)
692{
693	struct tegra_mc *mc = data;
694	unsigned long status;
695	unsigned int bit;
696
697	/* mask all interrupts to avoid flooding */
698	status = mc_readl(mc, MC_INTSTATUS) & mc->soc->intmask;
699	if (!status)
700		return IRQ_NONE;
701
702	for_each_set_bit(bit, &status, 32) {
703		const char *error = tegra_mc_status_names[bit];
704		const char *direction = "read", *secure = "";
705		const char *client, *desc;
706		phys_addr_t addr;
707		u32 value, reg;
708		u8 id, type;
709
710		switch (BIT(bit)) {
711		case MC_INT_DECERR_EMEM:
712			reg = MC_DECERR_EMEM_OTHERS_STATUS;
713			value = mc_readl(mc, reg);
714
715			id = value & mc->soc->client_id_mask;
716			desc = tegra_mc_error_names[2];
717
718			if (value & BIT(31))
719				direction = "write";
720			break;
721
722		case MC_INT_INVALID_GART_PAGE:
723			reg = MC_GART_ERROR_REQ;
724			value = mc_readl(mc, reg);
725
726			id = (value >> 1) & mc->soc->client_id_mask;
727			desc = tegra_mc_error_names[2];
728
729			if (value & BIT(0))
730				direction = "write";
731			break;
732
733		case MC_INT_SECURITY_VIOLATION:
734			reg = MC_SECURITY_VIOLATION_STATUS;
735			value = mc_readl(mc, reg);
736
737			id = value & mc->soc->client_id_mask;
738			type = (value & BIT(30)) ? 4 : 3;
739			desc = tegra_mc_error_names[type];
740			secure = "secure ";
741
742			if (value & BIT(31))
743				direction = "write";
744			break;
745
746		default:
747			continue;
748		}
749
750		client = mc->soc->clients[id].name;
751		addr = mc_readl(mc, reg + sizeof(u32));
752
753		dev_err_ratelimited(mc->dev, "%s: %s%s @%pa: %s (%s)\n",
754				    client, secure, direction, &addr, error,
755				    desc);
756	}
757
758	/* clear interrupts */
759	mc_writel(mc, status, MC_INTSTATUS);
760
761	return IRQ_HANDLED;
762}
763
764static const struct tegra_mc_ops tegra20_mc_ops = {
765	.probe = tegra20_mc_probe,
766	.handle_irq = tegra20_mc_handle_irq,
767};
768
769const struct tegra_mc_soc tegra20_mc_soc = {
770	.clients = tegra20_mc_clients,
771	.num_clients = ARRAY_SIZE(tegra20_mc_clients),
772	.num_address_bits = 32,
773	.client_id_mask = 0x3f,
774	.intmask = MC_INT_SECURITY_VIOLATION | MC_INT_INVALID_GART_PAGE |
775		   MC_INT_DECERR_EMEM,
776	.reset_ops = &tegra20_mc_reset_ops,
777	.resets = tegra20_mc_resets,
778	.num_resets = ARRAY_SIZE(tegra20_mc_resets),
779	.icc_ops = &tegra20_mc_icc_ops,
780	.ops = &tegra20_mc_ops,
781};
782