1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * Copyright (C) 2016,2017 ARM Limited, All Rights Reserved. 4 * Author: Marc Zyngier <marc.zyngier@arm.com> 5 */ 6 7#include <linux/interrupt.h> 8#include <linux/irq.h> 9#include <linux/irqdomain.h> 10#include <linux/msi.h> 11#include <linux/pid.h> 12#include <linux/sched.h> 13 14#include <linux/irqchip/arm-gic-v4.h> 15 16/* 17 * WARNING: The blurb below assumes that you understand the 18 * intricacies of GICv3, GICv4, and how a guest's view of a GICv3 gets 19 * translated into GICv4 commands. So it effectively targets at most 20 * two individuals. You know who you are. 21 * 22 * The core GICv4 code is designed to *avoid* exposing too much of the 23 * core GIC code (that would in turn leak into the hypervisor code), 24 * and instead provide a hypervisor agnostic interface to the HW (of 25 * course, the astute reader will quickly realize that hypervisor 26 * agnostic actually means KVM-specific - what were you thinking?). 27 * 28 * In order to achieve a modicum of isolation, we try to hide most of 29 * the GICv4 "stuff" behind normal irqchip operations: 30 * 31 * - Any guest-visible VLPI is backed by a Linux interrupt (and a 32 * physical LPI which gets unmapped when the guest maps the 33 * VLPI). This allows the same DevID/EventID pair to be either 34 * mapped to the LPI (host) or the VLPI (guest). Note that this is 35 * exclusive, and you cannot have both. 36 * 37 * - Enabling/disabling a VLPI is done by issuing mask/unmask calls. 38 * 39 * - Guest INT/CLEAR commands are implemented through 40 * irq_set_irqchip_state(). 41 * 42 * - The *bizarre* stuff (mapping/unmapping an interrupt to a VLPI, or 43 * issuing an INV after changing a priority) gets shoved into the 44 * irq_set_vcpu_affinity() method. While this is quite horrible 45 * (let's face it, this is the irqchip version of an ioctl), it 46 * confines the crap to a single location. And map/unmap really is 47 * about setting the affinity of a VLPI to a vcpu, so only INV is 48 * majorly out of place. So there. 49 * 50 * A number of commands are simply not provided by this interface, as 51 * they do not make direct sense. For example, MAPD is purely local to 52 * the virtual ITS (because it references a virtual device, and the 53 * physical ITS is still very much in charge of the physical 54 * device). Same goes for things like MAPC (the physical ITS deals 55 * with the actual vPE affinity, and not the braindead concept of 56 * collection). SYNC is not provided either, as each and every command 57 * is followed by a VSYNC. This could be relaxed in the future, should 58 * this be seen as a bottleneck (yes, this means *never*). 59 * 60 * But handling VLPIs is only one side of the job of the GICv4 61 * code. The other (darker) side is to take care of the doorbell 62 * interrupts which are delivered when a VLPI targeting a non-running 63 * vcpu is being made pending. 64 * 65 * The choice made here is that each vcpu (VPE in old northern GICv4 66 * dialect) gets a single doorbell LPI, no matter how many interrupts 67 * are targeting it. This has a nice property, which is that the 68 * interrupt becomes a handle for the VPE, and that the hypervisor 69 * code can manipulate it through the normal interrupt API: 70 * 71 * - VMs (or rather the VM abstraction that matters to the GIC) 72 * contain an irq domain where each interrupt maps to a VPE. In 73 * turn, this domain sits on top of the normal LPI allocator, and a 74 * specially crafted irq_chip implementation. 75 * 76 * - mask/unmask do what is expected on the doorbell interrupt. 77 * 78 * - irq_set_affinity is used to move a VPE from one redistributor to 79 * another. 80 * 81 * - irq_set_vcpu_affinity once again gets hijacked for the purpose of 82 * creating a new sub-API, namely scheduling/descheduling a VPE 83 * (which involves programming GICR_V{PROP,PEND}BASER) and 84 * performing INVALL operations. 85 */ 86 87static struct irq_domain *gic_domain; 88static const struct irq_domain_ops *vpe_domain_ops; 89static const struct irq_domain_ops *sgi_domain_ops; 90 91#ifdef CONFIG_ARM64 92#include <asm/cpufeature.h> 93 94bool gic_cpuif_has_vsgi(void) 95{ 96 unsigned long fld, reg = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1); 97 98 fld = cpuid_feature_extract_unsigned_field(reg, ID_AA64PFR0_EL1_GIC_SHIFT); 99 100 return fld >= 0x3; 101} 102#else 103bool gic_cpuif_has_vsgi(void) 104{ 105 return false; 106} 107#endif 108 109static bool has_v4_1(void) 110{ 111 return !!sgi_domain_ops; 112} 113 114static bool has_v4_1_sgi(void) 115{ 116 return has_v4_1() && gic_cpuif_has_vsgi(); 117} 118 119static int its_alloc_vcpu_sgis(struct its_vpe *vpe, int idx) 120{ 121 char *name; 122 int sgi_base; 123 124 if (!has_v4_1_sgi()) 125 return 0; 126 127 name = kasprintf(GFP_KERNEL, "GICv4-sgi-%d", task_pid_nr(current)); 128 if (!name) 129 goto err; 130 131 vpe->fwnode = irq_domain_alloc_named_id_fwnode(name, idx); 132 if (!vpe->fwnode) 133 goto err; 134 135 kfree(name); 136 name = NULL; 137 138 vpe->sgi_domain = irq_domain_create_linear(vpe->fwnode, 16, 139 sgi_domain_ops, vpe); 140 if (!vpe->sgi_domain) 141 goto err; 142 143 sgi_base = irq_domain_alloc_irqs(vpe->sgi_domain, 16, NUMA_NO_NODE, vpe); 144 if (sgi_base <= 0) 145 goto err; 146 147 return 0; 148 149err: 150 if (vpe->sgi_domain) 151 irq_domain_remove(vpe->sgi_domain); 152 if (vpe->fwnode) 153 irq_domain_free_fwnode(vpe->fwnode); 154 kfree(name); 155 return -ENOMEM; 156} 157 158int its_alloc_vcpu_irqs(struct its_vm *vm) 159{ 160 int vpe_base_irq, i; 161 162 vm->fwnode = irq_domain_alloc_named_id_fwnode("GICv4-vpe", 163 task_pid_nr(current)); 164 if (!vm->fwnode) 165 goto err; 166 167 vm->domain = irq_domain_create_hierarchy(gic_domain, 0, vm->nr_vpes, 168 vm->fwnode, vpe_domain_ops, 169 vm); 170 if (!vm->domain) 171 goto err; 172 173 for (i = 0; i < vm->nr_vpes; i++) { 174 vm->vpes[i]->its_vm = vm; 175 vm->vpes[i]->idai = true; 176 } 177 178 vpe_base_irq = irq_domain_alloc_irqs(vm->domain, vm->nr_vpes, 179 NUMA_NO_NODE, vm); 180 if (vpe_base_irq <= 0) 181 goto err; 182 183 for (i = 0; i < vm->nr_vpes; i++) { 184 int ret; 185 vm->vpes[i]->irq = vpe_base_irq + i; 186 ret = its_alloc_vcpu_sgis(vm->vpes[i], i); 187 if (ret) 188 goto err; 189 } 190 191 return 0; 192 193err: 194 if (vm->domain) 195 irq_domain_remove(vm->domain); 196 if (vm->fwnode) 197 irq_domain_free_fwnode(vm->fwnode); 198 199 return -ENOMEM; 200} 201 202static void its_free_sgi_irqs(struct its_vm *vm) 203{ 204 int i; 205 206 if (!has_v4_1_sgi()) 207 return; 208 209 for (i = 0; i < vm->nr_vpes; i++) { 210 unsigned int irq = irq_find_mapping(vm->vpes[i]->sgi_domain, 0); 211 212 if (WARN_ON(!irq)) 213 continue; 214 215 irq_domain_free_irqs(irq, 16); 216 irq_domain_remove(vm->vpes[i]->sgi_domain); 217 irq_domain_free_fwnode(vm->vpes[i]->fwnode); 218 } 219} 220 221void its_free_vcpu_irqs(struct its_vm *vm) 222{ 223 its_free_sgi_irqs(vm); 224 irq_domain_free_irqs(vm->vpes[0]->irq, vm->nr_vpes); 225 irq_domain_remove(vm->domain); 226 irq_domain_free_fwnode(vm->fwnode); 227} 228 229static int its_send_vpe_cmd(struct its_vpe *vpe, struct its_cmd_info *info) 230{ 231 return irq_set_vcpu_affinity(vpe->irq, info); 232} 233 234int its_make_vpe_non_resident(struct its_vpe *vpe, bool db) 235{ 236 struct irq_desc *desc = irq_to_desc(vpe->irq); 237 struct its_cmd_info info = { }; 238 int ret; 239 240 WARN_ON(preemptible()); 241 242 info.cmd_type = DESCHEDULE_VPE; 243 if (has_v4_1()) { 244 /* GICv4.1 can directly deal with doorbells */ 245 info.req_db = db; 246 } else { 247 /* Undo the nested disable_irq() calls... */ 248 while (db && irqd_irq_disabled(&desc->irq_data)) 249 enable_irq(vpe->irq); 250 } 251 252 ret = its_send_vpe_cmd(vpe, &info); 253 if (!ret) 254 vpe->resident = false; 255 256 vpe->ready = false; 257 258 return ret; 259} 260 261int its_make_vpe_resident(struct its_vpe *vpe, bool g0en, bool g1en) 262{ 263 struct its_cmd_info info = { }; 264 int ret; 265 266 WARN_ON(preemptible()); 267 268 info.cmd_type = SCHEDULE_VPE; 269 if (has_v4_1()) { 270 info.g0en = g0en; 271 info.g1en = g1en; 272 } else { 273 /* Disabled the doorbell, as we're about to enter the guest */ 274 disable_irq_nosync(vpe->irq); 275 } 276 277 ret = its_send_vpe_cmd(vpe, &info); 278 if (!ret) 279 vpe->resident = true; 280 281 return ret; 282} 283 284int its_commit_vpe(struct its_vpe *vpe) 285{ 286 struct its_cmd_info info = { 287 .cmd_type = COMMIT_VPE, 288 }; 289 int ret; 290 291 WARN_ON(preemptible()); 292 293 ret = its_send_vpe_cmd(vpe, &info); 294 if (!ret) 295 vpe->ready = true; 296 297 return ret; 298} 299 300 301int its_invall_vpe(struct its_vpe *vpe) 302{ 303 struct its_cmd_info info = { 304 .cmd_type = INVALL_VPE, 305 }; 306 307 return its_send_vpe_cmd(vpe, &info); 308} 309 310int its_map_vlpi(int irq, struct its_vlpi_map *map) 311{ 312 struct its_cmd_info info = { 313 .cmd_type = MAP_VLPI, 314 { 315 .map = map, 316 }, 317 }; 318 int ret; 319 320 /* 321 * The host will never see that interrupt firing again, so it 322 * is vital that we don't do any lazy masking. 323 */ 324 irq_set_status_flags(irq, IRQ_DISABLE_UNLAZY); 325 326 ret = irq_set_vcpu_affinity(irq, &info); 327 if (ret) 328 irq_clear_status_flags(irq, IRQ_DISABLE_UNLAZY); 329 330 return ret; 331} 332 333int its_get_vlpi(int irq, struct its_vlpi_map *map) 334{ 335 struct its_cmd_info info = { 336 .cmd_type = GET_VLPI, 337 { 338 .map = map, 339 }, 340 }; 341 342 return irq_set_vcpu_affinity(irq, &info); 343} 344 345int its_unmap_vlpi(int irq) 346{ 347 irq_clear_status_flags(irq, IRQ_DISABLE_UNLAZY); 348 return irq_set_vcpu_affinity(irq, NULL); 349} 350 351int its_prop_update_vlpi(int irq, u8 config, bool inv) 352{ 353 struct its_cmd_info info = { 354 .cmd_type = inv ? PROP_UPDATE_AND_INV_VLPI : PROP_UPDATE_VLPI, 355 { 356 .config = config, 357 }, 358 }; 359 360 return irq_set_vcpu_affinity(irq, &info); 361} 362 363int its_prop_update_vsgi(int irq, u8 priority, bool group) 364{ 365 struct its_cmd_info info = { 366 .cmd_type = PROP_UPDATE_VSGI, 367 { 368 .priority = priority, 369 .group = group, 370 }, 371 }; 372 373 return irq_set_vcpu_affinity(irq, &info); 374} 375 376int its_init_v4(struct irq_domain *domain, 377 const struct irq_domain_ops *vpe_ops, 378 const struct irq_domain_ops *sgi_ops) 379{ 380 if (domain) { 381 pr_info("ITS: Enabling GICv4 support\n"); 382 gic_domain = domain; 383 vpe_domain_ops = vpe_ops; 384 sgi_domain_ops = sgi_ops; 385 return 0; 386 } 387 388 pr_err("ITS: No GICv4 VPE domain allocated\n"); 389 return -ENODEV; 390} 391