1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (C) 2021 Western Digital Corporation or its affiliates.
4 * Copyright (C) 2022 Ventana Micro Systems Inc.
5 */
6
7#define pr_fmt(fmt) "riscv-imsic: " fmt
8#include <linux/bitmap.h>
9#include <linux/cpu.h>
10#include <linux/interrupt.h>
11#include <linux/io.h>
12#include <linux/irq.h>
13#include <linux/irqchip.h>
14#include <linux/irqdomain.h>
15#include <linux/module.h>
16#include <linux/msi.h>
17#include <linux/pci.h>
18#include <linux/platform_device.h>
19#include <linux/spinlock.h>
20#include <linux/smp.h>
21
22#include "irq-riscv-imsic-state.h"
23
24static bool imsic_cpu_page_phys(unsigned int cpu, unsigned int guest_index,
25				phys_addr_t *out_msi_pa)
26{
27	struct imsic_global_config *global;
28	struct imsic_local_config *local;
29
30	global = &imsic->global;
31	local = per_cpu_ptr(global->local, cpu);
32
33	if (BIT(global->guest_index_bits) <= guest_index)
34		return false;
35
36	if (out_msi_pa)
37		*out_msi_pa = local->msi_pa + (guest_index * IMSIC_MMIO_PAGE_SZ);
38
39	return true;
40}
41
42static void imsic_irq_mask(struct irq_data *d)
43{
44	imsic_vector_mask(irq_data_get_irq_chip_data(d));
45}
46
47static void imsic_irq_unmask(struct irq_data *d)
48{
49	imsic_vector_unmask(irq_data_get_irq_chip_data(d));
50}
51
52static int imsic_irq_retrigger(struct irq_data *d)
53{
54	struct imsic_vector *vec = irq_data_get_irq_chip_data(d);
55	struct imsic_local_config *local;
56
57	if (WARN_ON(!vec))
58		return -ENOENT;
59
60	local = per_cpu_ptr(imsic->global.local, vec->cpu);
61	writel_relaxed(vec->local_id, local->msi_va);
62	return 0;
63}
64
65static void imsic_irq_compose_vector_msg(struct imsic_vector *vec, struct msi_msg *msg)
66{
67	phys_addr_t msi_addr;
68
69	if (WARN_ON(!vec))
70		return;
71
72	if (WARN_ON(!imsic_cpu_page_phys(vec->cpu, 0, &msi_addr)))
73		return;
74
75	msg->address_hi = upper_32_bits(msi_addr);
76	msg->address_lo = lower_32_bits(msi_addr);
77	msg->data = vec->local_id;
78}
79
80static void imsic_irq_compose_msg(struct irq_data *d, struct msi_msg *msg)
81{
82	imsic_irq_compose_vector_msg(irq_data_get_irq_chip_data(d), msg);
83}
84
85#ifdef CONFIG_SMP
86static void imsic_msi_update_msg(struct irq_data *d, struct imsic_vector *vec)
87{
88	struct msi_msg msg = { };
89
90	imsic_irq_compose_vector_msg(vec, &msg);
91	irq_data_get_irq_chip(d)->irq_write_msi_msg(d, &msg);
92}
93
94static int imsic_irq_set_affinity(struct irq_data *d, const struct cpumask *mask_val,
95				  bool force)
96{
97	struct imsic_vector *old_vec, *new_vec;
98	struct irq_data *pd = d->parent_data;
99
100	old_vec = irq_data_get_irq_chip_data(pd);
101	if (WARN_ON(!old_vec))
102		return -ENOENT;
103
104	/* If old vector cpu belongs to the target cpumask then do nothing */
105	if (cpumask_test_cpu(old_vec->cpu, mask_val))
106		return IRQ_SET_MASK_OK_DONE;
107
108	/* If move is already in-flight then return failure */
109	if (imsic_vector_get_move(old_vec))
110		return -EBUSY;
111
112	/* Get a new vector on the desired set of CPUs */
113	new_vec = imsic_vector_alloc(old_vec->hwirq, mask_val);
114	if (!new_vec)
115		return -ENOSPC;
116
117	/* Point device to the new vector */
118	imsic_msi_update_msg(d, new_vec);
119
120	/* Update irq descriptors with the new vector */
121	pd->chip_data = new_vec;
122
123	/* Update effective affinity of parent irq data */
124	irq_data_update_effective_affinity(pd, cpumask_of(new_vec->cpu));
125
126	/* Move state of the old vector to the new vector */
127	imsic_vector_move(old_vec, new_vec);
128
129	return IRQ_SET_MASK_OK_DONE;
130}
131#endif
132
133static struct irq_chip imsic_irq_base_chip = {
134	.name			= "IMSIC",
135	.irq_mask		= imsic_irq_mask,
136	.irq_unmask		= imsic_irq_unmask,
137	.irq_retrigger		= imsic_irq_retrigger,
138	.irq_compose_msi_msg	= imsic_irq_compose_msg,
139	.flags			= IRQCHIP_SKIP_SET_WAKE |
140				  IRQCHIP_MASK_ON_SUSPEND,
141};
142
143static int imsic_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
144				  unsigned int nr_irqs, void *args)
145{
146	struct imsic_vector *vec;
147
148	/* Multi-MSI is not supported yet. */
149	if (nr_irqs > 1)
150		return -EOPNOTSUPP;
151
152	vec = imsic_vector_alloc(virq, cpu_online_mask);
153	if (!vec)
154		return -ENOSPC;
155
156	irq_domain_set_info(domain, virq, virq, &imsic_irq_base_chip, vec,
157			    handle_simple_irq, NULL, NULL);
158	irq_set_noprobe(virq);
159	irq_set_affinity(virq, cpu_online_mask);
160	irq_data_update_effective_affinity(irq_get_irq_data(virq), cpumask_of(vec->cpu));
161
162	return 0;
163}
164
165static void imsic_irq_domain_free(struct irq_domain *domain, unsigned int virq,
166				  unsigned int nr_irqs)
167{
168	struct irq_data *d = irq_domain_get_irq_data(domain, virq);
169
170	imsic_vector_free(irq_data_get_irq_chip_data(d));
171	irq_domain_free_irqs_parent(domain, virq, nr_irqs);
172}
173
174static int imsic_irq_domain_select(struct irq_domain *domain, struct irq_fwspec *fwspec,
175				   enum irq_domain_bus_token bus_token)
176{
177	const struct msi_parent_ops *ops = domain->msi_parent_ops;
178	u32 busmask = BIT(bus_token);
179
180	if (fwspec->fwnode != domain->fwnode || fwspec->param_count != 0)
181		return 0;
182
183	/* Handle pure domain searches */
184	if (bus_token == ops->bus_select_token)
185		return 1;
186
187	return !!(ops->bus_select_mask & busmask);
188}
189
190#ifdef CONFIG_GENERIC_IRQ_DEBUGFS
191static void imsic_irq_debug_show(struct seq_file *m, struct irq_domain *d,
192				 struct irq_data *irqd, int ind)
193{
194	if (!irqd) {
195		imsic_vector_debug_show_summary(m, ind);
196		return;
197	}
198
199	imsic_vector_debug_show(m, irq_data_get_irq_chip_data(irqd), ind);
200}
201#endif
202
203static const struct irq_domain_ops imsic_base_domain_ops = {
204	.alloc		= imsic_irq_domain_alloc,
205	.free		= imsic_irq_domain_free,
206	.select		= imsic_irq_domain_select,
207#ifdef CONFIG_GENERIC_IRQ_DEBUGFS
208	.debug_show	= imsic_irq_debug_show,
209#endif
210};
211
212#ifdef CONFIG_RISCV_IMSIC_PCI
213
214static void imsic_pci_mask_irq(struct irq_data *d)
215{
216	pci_msi_mask_irq(d);
217	irq_chip_mask_parent(d);
218}
219
220static void imsic_pci_unmask_irq(struct irq_data *d)
221{
222	irq_chip_unmask_parent(d);
223	pci_msi_unmask_irq(d);
224}
225
226#define MATCH_PCI_MSI		BIT(DOMAIN_BUS_PCI_MSI)
227
228#else
229
230#define MATCH_PCI_MSI		0
231
232#endif
233
234static bool imsic_init_dev_msi_info(struct device *dev,
235				    struct irq_domain *domain,
236				    struct irq_domain *real_parent,
237				    struct msi_domain_info *info)
238{
239	const struct msi_parent_ops *pops = real_parent->msi_parent_ops;
240
241	/* MSI parent domain specific settings */
242	switch (real_parent->bus_token) {
243	case DOMAIN_BUS_NEXUS:
244		if (WARN_ON_ONCE(domain != real_parent))
245			return false;
246#ifdef CONFIG_SMP
247		info->chip->irq_set_affinity = imsic_irq_set_affinity;
248#endif
249		break;
250	default:
251		WARN_ON_ONCE(1);
252		return false;
253	}
254
255	/* Is the target supported? */
256	switch (info->bus_token) {
257#ifdef CONFIG_RISCV_IMSIC_PCI
258	case DOMAIN_BUS_PCI_DEVICE_MSI:
259	case DOMAIN_BUS_PCI_DEVICE_MSIX:
260		info->chip->irq_mask = imsic_pci_mask_irq;
261		info->chip->irq_unmask = imsic_pci_unmask_irq;
262		break;
263#endif
264	case DOMAIN_BUS_DEVICE_MSI:
265		/*
266		 * Per-device MSI should never have any MSI feature bits
267		 * set. It's sole purpose is to create a dumb interrupt
268		 * chip which has a device specific irq_write_msi_msg()
269		 * callback.
270		 */
271		if (WARN_ON_ONCE(info->flags))
272			return false;
273
274		/* Core managed MSI descriptors */
275		info->flags |= MSI_FLAG_ALLOC_SIMPLE_MSI_DESCS |
276			       MSI_FLAG_FREE_MSI_DESCS;
277		break;
278	case DOMAIN_BUS_WIRED_TO_MSI:
279		break;
280	default:
281		WARN_ON_ONCE(1);
282		return false;
283	}
284
285	/* Use hierarchial chip operations re-trigger */
286	info->chip->irq_retrigger = irq_chip_retrigger_hierarchy;
287
288	/*
289	 * Mask out the domain specific MSI feature flags which are not
290	 * supported by the real parent.
291	 */
292	info->flags &= pops->supported_flags;
293
294	/* Enforce the required flags */
295	info->flags |= pops->required_flags;
296
297	return true;
298}
299
300#define MATCH_PLATFORM_MSI		BIT(DOMAIN_BUS_PLATFORM_MSI)
301
302static const struct msi_parent_ops imsic_msi_parent_ops = {
303	.supported_flags	= MSI_GENERIC_FLAGS_MASK |
304				  MSI_FLAG_PCI_MSIX,
305	.required_flags		= MSI_FLAG_USE_DEF_DOM_OPS |
306				  MSI_FLAG_USE_DEF_CHIP_OPS,
307	.bus_select_token	= DOMAIN_BUS_NEXUS,
308	.bus_select_mask	= MATCH_PCI_MSI | MATCH_PLATFORM_MSI,
309	.init_dev_msi_info	= imsic_init_dev_msi_info,
310};
311
312int imsic_irqdomain_init(void)
313{
314	struct imsic_global_config *global;
315
316	if (!imsic || !imsic->fwnode) {
317		pr_err("early driver not probed\n");
318		return -ENODEV;
319	}
320
321	if (imsic->base_domain) {
322		pr_err("%pfwP: irq domain already created\n", imsic->fwnode);
323		return -ENODEV;
324	}
325
326	/* Create Base IRQ domain */
327	imsic->base_domain = irq_domain_create_tree(imsic->fwnode,
328						    &imsic_base_domain_ops, imsic);
329	if (!imsic->base_domain) {
330		pr_err("%pfwP: failed to create IMSIC base domain\n", imsic->fwnode);
331		return -ENOMEM;
332	}
333	imsic->base_domain->flags |= IRQ_DOMAIN_FLAG_MSI_PARENT;
334	imsic->base_domain->msi_parent_ops = &imsic_msi_parent_ops;
335
336	irq_domain_update_bus_token(imsic->base_domain, DOMAIN_BUS_NEXUS);
337
338	global = &imsic->global;
339	pr_info("%pfwP:  hart-index-bits: %d,  guest-index-bits: %d\n",
340		imsic->fwnode, global->hart_index_bits, global->guest_index_bits);
341	pr_info("%pfwP: group-index-bits: %d, group-index-shift: %d\n",
342		imsic->fwnode, global->group_index_bits, global->group_index_shift);
343	pr_info("%pfwP: per-CPU IDs %d at base PPN %pa\n",
344		imsic->fwnode, global->nr_ids, &global->base_addr);
345	pr_info("%pfwP: total %d interrupts available\n",
346		imsic->fwnode, num_possible_cpus() * (global->nr_ids - 1));
347
348	return 0;
349}
350
351static int imsic_platform_probe(struct platform_device *pdev)
352{
353	struct device *dev = &pdev->dev;
354
355	if (imsic && imsic->fwnode != dev->fwnode) {
356		dev_err(dev, "fwnode mismatch\n");
357		return -ENODEV;
358	}
359
360	return imsic_irqdomain_init();
361}
362
363static const struct of_device_id imsic_platform_match[] = {
364	{ .compatible = "riscv,imsics" },
365	{}
366};
367
368static struct platform_driver imsic_platform_driver = {
369	.driver = {
370		.name		= "riscv-imsic",
371		.of_match_table	= imsic_platform_match,
372	},
373	.probe = imsic_platform_probe,
374};
375builtin_platform_driver(imsic_platform_driver);
376