1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Guest ITS library, generously donated by drivers/irqchip/irq-gic-v3-its.c
4 * over in the kernel tree.
5 */
6
7#include <linux/kvm.h>
8#include <linux/sizes.h>
9#include <asm/kvm_para.h>
10#include <asm/kvm.h>
11
12#include "kvm_util.h"
13#include "vgic.h"
14#include "gic.h"
15#include "gic_v3.h"
16#include "processor.h"
17
18static u64 its_read_u64(unsigned long offset)
19{
20	return readq_relaxed(GITS_BASE_GVA + offset);
21}
22
23static void its_write_u64(unsigned long offset, u64 val)
24{
25	writeq_relaxed(val, GITS_BASE_GVA + offset);
26}
27
28static u32 its_read_u32(unsigned long offset)
29{
30	return readl_relaxed(GITS_BASE_GVA + offset);
31}
32
33static void its_write_u32(unsigned long offset, u32 val)
34{
35	writel_relaxed(val, GITS_BASE_GVA + offset);
36}
37
38static unsigned long its_find_baser(unsigned int type)
39{
40	int i;
41
42	for (i = 0; i < GITS_BASER_NR_REGS; i++) {
43		u64 baser;
44		unsigned long offset = GITS_BASER + (i * sizeof(baser));
45
46		baser = its_read_u64(offset);
47		if (GITS_BASER_TYPE(baser) == type)
48			return offset;
49	}
50
51	GUEST_FAIL("Couldn't find an ITS BASER of type %u", type);
52	return -1;
53}
54
55static void its_install_table(unsigned int type, vm_paddr_t base, size_t size)
56{
57	unsigned long offset = its_find_baser(type);
58	u64 baser;
59
60	baser = ((size / SZ_64K) - 1) |
61		GITS_BASER_PAGE_SIZE_64K |
62		GITS_BASER_InnerShareable |
63		base |
64		GITS_BASER_RaWaWb |
65		GITS_BASER_VALID;
66
67	its_write_u64(offset, baser);
68}
69
70static void its_install_cmdq(vm_paddr_t base, size_t size)
71{
72	u64 cbaser;
73
74	cbaser = ((size / SZ_4K) - 1) |
75		 GITS_CBASER_InnerShareable |
76		 base |
77		 GITS_CBASER_RaWaWb |
78		 GITS_CBASER_VALID;
79
80	its_write_u64(GITS_CBASER, cbaser);
81}
82
83void its_init(vm_paddr_t coll_tbl, size_t coll_tbl_sz,
84	      vm_paddr_t device_tbl, size_t device_tbl_sz,
85	      vm_paddr_t cmdq, size_t cmdq_size)
86{
87	u32 ctlr;
88
89	its_install_table(GITS_BASER_TYPE_COLLECTION, coll_tbl, coll_tbl_sz);
90	its_install_table(GITS_BASER_TYPE_DEVICE, device_tbl, device_tbl_sz);
91	its_install_cmdq(cmdq, cmdq_size);
92
93	ctlr = its_read_u32(GITS_CTLR);
94	ctlr |= GITS_CTLR_ENABLE;
95	its_write_u32(GITS_CTLR, ctlr);
96}
97
98struct its_cmd_block {
99	union {
100		u64	raw_cmd[4];
101		__le64	raw_cmd_le[4];
102	};
103};
104
105static inline void its_fixup_cmd(struct its_cmd_block *cmd)
106{
107	/* Let's fixup BE commands */
108	cmd->raw_cmd_le[0] = cpu_to_le64(cmd->raw_cmd[0]);
109	cmd->raw_cmd_le[1] = cpu_to_le64(cmd->raw_cmd[1]);
110	cmd->raw_cmd_le[2] = cpu_to_le64(cmd->raw_cmd[2]);
111	cmd->raw_cmd_le[3] = cpu_to_le64(cmd->raw_cmd[3]);
112}
113
114static void its_mask_encode(u64 *raw_cmd, u64 val, int h, int l)
115{
116	u64 mask = GENMASK_ULL(h, l);
117	*raw_cmd &= ~mask;
118	*raw_cmd |= (val << l) & mask;
119}
120
121static void its_encode_cmd(struct its_cmd_block *cmd, u8 cmd_nr)
122{
123	its_mask_encode(&cmd->raw_cmd[0], cmd_nr, 7, 0);
124}
125
126static void its_encode_devid(struct its_cmd_block *cmd, u32 devid)
127{
128	its_mask_encode(&cmd->raw_cmd[0], devid, 63, 32);
129}
130
131static void its_encode_event_id(struct its_cmd_block *cmd, u32 id)
132{
133	its_mask_encode(&cmd->raw_cmd[1], id, 31, 0);
134}
135
136static void its_encode_phys_id(struct its_cmd_block *cmd, u32 phys_id)
137{
138	its_mask_encode(&cmd->raw_cmd[1], phys_id, 63, 32);
139}
140
141static void its_encode_size(struct its_cmd_block *cmd, u8 size)
142{
143	its_mask_encode(&cmd->raw_cmd[1], size, 4, 0);
144}
145
146static void its_encode_itt(struct its_cmd_block *cmd, u64 itt_addr)
147{
148	its_mask_encode(&cmd->raw_cmd[2], itt_addr >> 8, 51, 8);
149}
150
151static void its_encode_valid(struct its_cmd_block *cmd, int valid)
152{
153	its_mask_encode(&cmd->raw_cmd[2], !!valid, 63, 63);
154}
155
156static void its_encode_target(struct its_cmd_block *cmd, u64 target_addr)
157{
158	its_mask_encode(&cmd->raw_cmd[2], target_addr >> 16, 51, 16);
159}
160
161static void its_encode_collection(struct its_cmd_block *cmd, u16 col)
162{
163	its_mask_encode(&cmd->raw_cmd[2], col, 15, 0);
164}
165
166#define GITS_CMDQ_POLL_ITERATIONS	0
167
168static void its_send_cmd(void *cmdq_base, struct its_cmd_block *cmd)
169{
170	u64 cwriter = its_read_u64(GITS_CWRITER);
171	struct its_cmd_block *dst = cmdq_base + cwriter;
172	u64 cbaser = its_read_u64(GITS_CBASER);
173	size_t cmdq_size;
174	u64 next;
175	int i;
176
177	cmdq_size = ((cbaser & 0xFF) + 1) * SZ_4K;
178
179	its_fixup_cmd(cmd);
180
181	WRITE_ONCE(*dst, *cmd);
182	dsb(ishst);
183	next = (cwriter + sizeof(*cmd)) % cmdq_size;
184	its_write_u64(GITS_CWRITER, next);
185
186	/*
187	 * Polling isn't necessary considering KVM's ITS emulation at the time
188	 * of writing this, as the CMDQ is processed synchronously after a write
189	 * to CWRITER.
190	 */
191	for (i = 0; its_read_u64(GITS_CREADR) != next; i++) {
192		__GUEST_ASSERT(i < GITS_CMDQ_POLL_ITERATIONS,
193			       "ITS didn't process command at offset %lu after %d iterations\n",
194			       cwriter, i);
195
196		cpu_relax();
197	}
198}
199
200void its_send_mapd_cmd(void *cmdq_base, u32 device_id, vm_paddr_t itt_base,
201		       size_t itt_size, bool valid)
202{
203	struct its_cmd_block cmd = {};
204
205	its_encode_cmd(&cmd, GITS_CMD_MAPD);
206	its_encode_devid(&cmd, device_id);
207	its_encode_size(&cmd, ilog2(itt_size) - 1);
208	its_encode_itt(&cmd, itt_base);
209	its_encode_valid(&cmd, valid);
210
211	its_send_cmd(cmdq_base, &cmd);
212}
213
214void its_send_mapc_cmd(void *cmdq_base, u32 vcpu_id, u32 collection_id, bool valid)
215{
216	struct its_cmd_block cmd = {};
217
218	its_encode_cmd(&cmd, GITS_CMD_MAPC);
219	its_encode_collection(&cmd, collection_id);
220	its_encode_target(&cmd, vcpu_id);
221	its_encode_valid(&cmd, valid);
222
223	its_send_cmd(cmdq_base, &cmd);
224}
225
226void its_send_mapti_cmd(void *cmdq_base, u32 device_id, u32 event_id,
227			u32 collection_id, u32 intid)
228{
229	struct its_cmd_block cmd = {};
230
231	its_encode_cmd(&cmd, GITS_CMD_MAPTI);
232	its_encode_devid(&cmd, device_id);
233	its_encode_event_id(&cmd, event_id);
234	its_encode_phys_id(&cmd, intid);
235	its_encode_collection(&cmd, collection_id);
236
237	its_send_cmd(cmdq_base, &cmd);
238}
239
240void its_send_invall_cmd(void *cmdq_base, u32 collection_id)
241{
242	struct its_cmd_block cmd = {};
243
244	its_encode_cmd(&cmd, GITS_CMD_INVALL);
245	its_encode_collection(&cmd, collection_id);
246
247	its_send_cmd(cmdq_base, &cmd);
248}
249