1// SPDX-License-Identifier: GPL-2.0
2/*
3 * ARM Generic Interrupt Controller (GIC) v3 support
4 */
5
6#include <linux/sizes.h>
7
8#include "kvm_util.h"
9#include "processor.h"
10#include "delay.h"
11
12#include "gic_v3.h"
13#include "gic_private.h"
14
15struct gicv3_data {
16	void *dist_base;
17	void *redist_base[GICV3_MAX_CPUS];
18	unsigned int nr_cpus;
19	unsigned int nr_spis;
20};
21
22#define sgi_base_from_redist(redist_base)	(redist_base + SZ_64K)
23#define DIST_BIT				(1U << 31)
24
25enum gicv3_intid_range {
26	SGI_RANGE,
27	PPI_RANGE,
28	SPI_RANGE,
29	INVALID_RANGE,
30};
31
32static struct gicv3_data gicv3_data;
33
34static void gicv3_gicd_wait_for_rwp(void)
35{
36	unsigned int count = 100000; /* 1s */
37
38	while (readl(gicv3_data.dist_base + GICD_CTLR) & GICD_CTLR_RWP) {
39		GUEST_ASSERT(count--);
40		udelay(10);
41	}
42}
43
44static void gicv3_gicr_wait_for_rwp(void *redist_base)
45{
46	unsigned int count = 100000; /* 1s */
47
48	while (readl(redist_base + GICR_CTLR) & GICR_CTLR_RWP) {
49		GUEST_ASSERT(count--);
50		udelay(10);
51	}
52}
53
54static void gicv3_wait_for_rwp(uint32_t cpu_or_dist)
55{
56	if (cpu_or_dist & DIST_BIT)
57		gicv3_gicd_wait_for_rwp();
58	else
59		gicv3_gicr_wait_for_rwp(gicv3_data.redist_base[cpu_or_dist]);
60}
61
62static enum gicv3_intid_range get_intid_range(unsigned int intid)
63{
64	switch (intid) {
65	case 0 ... 15:
66		return SGI_RANGE;
67	case 16 ... 31:
68		return PPI_RANGE;
69	case 32 ... 1019:
70		return SPI_RANGE;
71	}
72
73	/* We should not be reaching here */
74	GUEST_ASSERT(0);
75
76	return INVALID_RANGE;
77}
78
79static uint64_t gicv3_read_iar(void)
80{
81	uint64_t irqstat = read_sysreg_s(SYS_ICC_IAR1_EL1);
82
83	dsb(sy);
84	return irqstat;
85}
86
87static void gicv3_write_eoir(uint32_t irq)
88{
89	write_sysreg_s(irq, SYS_ICC_EOIR1_EL1);
90	isb();
91}
92
93static void gicv3_write_dir(uint32_t irq)
94{
95	write_sysreg_s(irq, SYS_ICC_DIR_EL1);
96	isb();
97}
98
99static void gicv3_set_priority_mask(uint64_t mask)
100{
101	write_sysreg_s(mask, SYS_ICC_PMR_EL1);
102}
103
104static void gicv3_set_eoi_split(bool split)
105{
106	uint32_t val;
107
108	/*
109	 * All other fields are read-only, so no need to read CTLR first. In
110	 * fact, the kernel does the same.
111	 */
112	val = split ? (1U << 1) : 0;
113	write_sysreg_s(val, SYS_ICC_CTLR_EL1);
114	isb();
115}
116
117uint32_t gicv3_reg_readl(uint32_t cpu_or_dist, uint64_t offset)
118{
119	void *base = cpu_or_dist & DIST_BIT ? gicv3_data.dist_base
120		: sgi_base_from_redist(gicv3_data.redist_base[cpu_or_dist]);
121	return readl(base + offset);
122}
123
124void gicv3_reg_writel(uint32_t cpu_or_dist, uint64_t offset, uint32_t reg_val)
125{
126	void *base = cpu_or_dist & DIST_BIT ? gicv3_data.dist_base
127		: sgi_base_from_redist(gicv3_data.redist_base[cpu_or_dist]);
128	writel(reg_val, base + offset);
129}
130
131uint32_t gicv3_getl_fields(uint32_t cpu_or_dist, uint64_t offset, uint32_t mask)
132{
133	return gicv3_reg_readl(cpu_or_dist, offset) & mask;
134}
135
136void gicv3_setl_fields(uint32_t cpu_or_dist, uint64_t offset,
137		uint32_t mask, uint32_t reg_val)
138{
139	uint32_t tmp = gicv3_reg_readl(cpu_or_dist, offset) & ~mask;
140
141	tmp |= (reg_val & mask);
142	gicv3_reg_writel(cpu_or_dist, offset, tmp);
143}
144
145/*
146 * We use a single offset for the distributor and redistributor maps as they
147 * have the same value in both. The only exceptions are registers that only
148 * exist in one and not the other, like GICR_WAKER that doesn't exist in the
149 * distributor map. Such registers are conveniently marked as reserved in the
150 * map that doesn't implement it; like GICR_WAKER's offset of 0x0014 being
151 * marked as "Reserved" in the Distributor map.
152 */
153static void gicv3_access_reg(uint32_t intid, uint64_t offset,
154		uint32_t reg_bits, uint32_t bits_per_field,
155		bool write, uint32_t *val)
156{
157	uint32_t cpu = guest_get_vcpuid();
158	enum gicv3_intid_range intid_range = get_intid_range(intid);
159	uint32_t fields_per_reg, index, mask, shift;
160	uint32_t cpu_or_dist;
161
162	GUEST_ASSERT(bits_per_field <= reg_bits);
163	GUEST_ASSERT(!write || *val < (1U << bits_per_field));
164	/*
165	 * This function does not support 64 bit accesses. Just asserting here
166	 * until we implement readq/writeq.
167	 */
168	GUEST_ASSERT(reg_bits == 32);
169
170	fields_per_reg = reg_bits / bits_per_field;
171	index = intid % fields_per_reg;
172	shift = index * bits_per_field;
173	mask = ((1U << bits_per_field) - 1) << shift;
174
175	/* Set offset to the actual register holding intid's config. */
176	offset += (intid / fields_per_reg) * (reg_bits / 8);
177
178	cpu_or_dist = (intid_range == SPI_RANGE) ? DIST_BIT : cpu;
179
180	if (write)
181		gicv3_setl_fields(cpu_or_dist, offset, mask, *val << shift);
182	*val = gicv3_getl_fields(cpu_or_dist, offset, mask) >> shift;
183}
184
185static void gicv3_write_reg(uint32_t intid, uint64_t offset,
186		uint32_t reg_bits, uint32_t bits_per_field, uint32_t val)
187{
188	gicv3_access_reg(intid, offset, reg_bits,
189			bits_per_field, true, &val);
190}
191
192static uint32_t gicv3_read_reg(uint32_t intid, uint64_t offset,
193		uint32_t reg_bits, uint32_t bits_per_field)
194{
195	uint32_t val;
196
197	gicv3_access_reg(intid, offset, reg_bits,
198			bits_per_field, false, &val);
199	return val;
200}
201
202static void gicv3_set_priority(uint32_t intid, uint32_t prio)
203{
204	gicv3_write_reg(intid, GICD_IPRIORITYR, 32, 8, prio);
205}
206
207/* Sets the intid to be level-sensitive or edge-triggered. */
208static void gicv3_irq_set_config(uint32_t intid, bool is_edge)
209{
210	uint32_t val;
211
212	/* N/A for private interrupts. */
213	GUEST_ASSERT(get_intid_range(intid) == SPI_RANGE);
214	val = is_edge ? 2 : 0;
215	gicv3_write_reg(intid, GICD_ICFGR, 32, 2, val);
216}
217
218static void gicv3_irq_enable(uint32_t intid)
219{
220	bool is_spi = get_intid_range(intid) == SPI_RANGE;
221	uint32_t cpu = guest_get_vcpuid();
222
223	gicv3_write_reg(intid, GICD_ISENABLER, 32, 1, 1);
224	gicv3_wait_for_rwp(is_spi ? DIST_BIT : cpu);
225}
226
227static void gicv3_irq_disable(uint32_t intid)
228{
229	bool is_spi = get_intid_range(intid) == SPI_RANGE;
230	uint32_t cpu = guest_get_vcpuid();
231
232	gicv3_write_reg(intid, GICD_ICENABLER, 32, 1, 1);
233	gicv3_wait_for_rwp(is_spi ? DIST_BIT : cpu);
234}
235
236static void gicv3_irq_set_active(uint32_t intid)
237{
238	gicv3_write_reg(intid, GICD_ISACTIVER, 32, 1, 1);
239}
240
241static void gicv3_irq_clear_active(uint32_t intid)
242{
243	gicv3_write_reg(intid, GICD_ICACTIVER, 32, 1, 1);
244}
245
246static bool gicv3_irq_get_active(uint32_t intid)
247{
248	return gicv3_read_reg(intid, GICD_ISACTIVER, 32, 1);
249}
250
251static void gicv3_irq_set_pending(uint32_t intid)
252{
253	gicv3_write_reg(intid, GICD_ISPENDR, 32, 1, 1);
254}
255
256static void gicv3_irq_clear_pending(uint32_t intid)
257{
258	gicv3_write_reg(intid, GICD_ICPENDR, 32, 1, 1);
259}
260
261static bool gicv3_irq_get_pending(uint32_t intid)
262{
263	return gicv3_read_reg(intid, GICD_ISPENDR, 32, 1);
264}
265
266static void gicv3_enable_redist(void *redist_base)
267{
268	uint32_t val = readl(redist_base + GICR_WAKER);
269	unsigned int count = 100000; /* 1s */
270
271	val &= ~GICR_WAKER_ProcessorSleep;
272	writel(val, redist_base + GICR_WAKER);
273
274	/* Wait until the processor is 'active' */
275	while (readl(redist_base + GICR_WAKER) & GICR_WAKER_ChildrenAsleep) {
276		GUEST_ASSERT(count--);
277		udelay(10);
278	}
279}
280
281static inline void *gicr_base_cpu(void *redist_base, uint32_t cpu)
282{
283	/* Align all the redistributors sequentially */
284	return redist_base + cpu * SZ_64K * 2;
285}
286
287static void gicv3_cpu_init(unsigned int cpu, void *redist_base)
288{
289	void *sgi_base;
290	unsigned int i;
291	void *redist_base_cpu;
292
293	GUEST_ASSERT(cpu < gicv3_data.nr_cpus);
294
295	redist_base_cpu = gicr_base_cpu(redist_base, cpu);
296	sgi_base = sgi_base_from_redist(redist_base_cpu);
297
298	gicv3_enable_redist(redist_base_cpu);
299
300	/*
301	 * Mark all the SGI and PPI interrupts as non-secure Group-1.
302	 * Also, deactivate and disable them.
303	 */
304	writel(~0, sgi_base + GICR_IGROUPR0);
305	writel(~0, sgi_base + GICR_ICACTIVER0);
306	writel(~0, sgi_base + GICR_ICENABLER0);
307
308	/* Set a default priority for all the SGIs and PPIs */
309	for (i = 0; i < 32; i += 4)
310		writel(GICD_INT_DEF_PRI_X4,
311				sgi_base + GICR_IPRIORITYR0 + i);
312
313	gicv3_gicr_wait_for_rwp(redist_base_cpu);
314
315	/* Enable the GIC system register (ICC_*) access */
316	write_sysreg_s(read_sysreg_s(SYS_ICC_SRE_EL1) | ICC_SRE_EL1_SRE,
317			SYS_ICC_SRE_EL1);
318
319	/* Set a default priority threshold */
320	write_sysreg_s(ICC_PMR_DEF_PRIO, SYS_ICC_PMR_EL1);
321
322	/* Enable non-secure Group-1 interrupts */
323	write_sysreg_s(ICC_IGRPEN1_EL1_ENABLE, SYS_ICC_GRPEN1_EL1);
324
325	gicv3_data.redist_base[cpu] = redist_base_cpu;
326}
327
328static void gicv3_dist_init(void)
329{
330	void *dist_base = gicv3_data.dist_base;
331	unsigned int i;
332
333	/* Disable the distributor until we set things up */
334	writel(0, dist_base + GICD_CTLR);
335	gicv3_gicd_wait_for_rwp();
336
337	/*
338	 * Mark all the SPI interrupts as non-secure Group-1.
339	 * Also, deactivate and disable them.
340	 */
341	for (i = 32; i < gicv3_data.nr_spis; i += 32) {
342		writel(~0, dist_base + GICD_IGROUPR + i / 8);
343		writel(~0, dist_base + GICD_ICACTIVER + i / 8);
344		writel(~0, dist_base + GICD_ICENABLER + i / 8);
345	}
346
347	/* Set a default priority for all the SPIs */
348	for (i = 32; i < gicv3_data.nr_spis; i += 4)
349		writel(GICD_INT_DEF_PRI_X4,
350				dist_base + GICD_IPRIORITYR + i);
351
352	/* Wait for the settings to sync-in */
353	gicv3_gicd_wait_for_rwp();
354
355	/* Finally, enable the distributor globally with ARE */
356	writel(GICD_CTLR_ARE_NS | GICD_CTLR_ENABLE_G1A |
357			GICD_CTLR_ENABLE_G1, dist_base + GICD_CTLR);
358	gicv3_gicd_wait_for_rwp();
359}
360
361static void gicv3_init(unsigned int nr_cpus, void *dist_base)
362{
363	GUEST_ASSERT(nr_cpus <= GICV3_MAX_CPUS);
364
365	gicv3_data.nr_cpus = nr_cpus;
366	gicv3_data.dist_base = dist_base;
367	gicv3_data.nr_spis = GICD_TYPER_SPIS(
368				readl(gicv3_data.dist_base + GICD_TYPER));
369	if (gicv3_data.nr_spis > 1020)
370		gicv3_data.nr_spis = 1020;
371
372	/*
373	 * Initialize only the distributor for now.
374	 * The redistributor and CPU interfaces are initialized
375	 * later for every PE.
376	 */
377	gicv3_dist_init();
378}
379
380const struct gic_common_ops gicv3_ops = {
381	.gic_init = gicv3_init,
382	.gic_cpu_init = gicv3_cpu_init,
383	.gic_irq_enable = gicv3_irq_enable,
384	.gic_irq_disable = gicv3_irq_disable,
385	.gic_read_iar = gicv3_read_iar,
386	.gic_write_eoir = gicv3_write_eoir,
387	.gic_write_dir = gicv3_write_dir,
388	.gic_set_priority_mask = gicv3_set_priority_mask,
389	.gic_set_eoi_split = gicv3_set_eoi_split,
390	.gic_set_priority = gicv3_set_priority,
391	.gic_irq_set_active = gicv3_irq_set_active,
392	.gic_irq_clear_active = gicv3_irq_clear_active,
393	.gic_irq_get_active = gicv3_irq_get_active,
394	.gic_irq_set_pending = gicv3_irq_set_pending,
395	.gic_irq_clear_pending = gicv3_irq_clear_pending,
396	.gic_irq_get_pending = gicv3_irq_get_pending,
397	.gic_irq_set_config = gicv3_irq_set_config,
398};
399