1/* 2 * Copyright (c) 2014 Roger Pau Monn�� <roger.pau@citrix.com> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27#include <sys/cdefs.h> 28__FBSDID("$FreeBSD: stable/11/sys/x86/xen/xen_apic.c 361795 2020-06-04 17:08:07Z mav $"); 29 30#include <sys/param.h> 31#include <sys/bus.h> 32#include <sys/kernel.h> 33#include <sys/malloc.h> 34#include <sys/proc.h> 35#include <sys/smp.h> 36#include <sys/systm.h> 37 38#include <vm/vm.h> 39#include <vm/pmap.h> 40 41#include <machine/cpufunc.h> 42#include <machine/cpu.h> 43#include <machine/intr_machdep.h> 44#include <machine/md_var.h> 45#include <machine/smp.h> 46 47#include <x86/apicreg.h> 48#include <x86/apicvar.h> 49 50#include <xen/xen-os.h> 51#include <xen/features.h> 52#include <xen/gnttab.h> 53#include <xen/hypervisor.h> 54#include <xen/hvm.h> 55#include <xen/xen_intr.h> 56 57#include <xen/interface/vcpu.h> 58 59/*--------------------------------- Macros -----------------------------------*/ 60 61#define XEN_APIC_UNSUPPORTED \ 62 panic("%s: not available in Xen PV port.", __func__) 63 64 65/*--------------------------- Forward Declarations ---------------------------*/ 66#ifdef SMP 67static driver_filter_t xen_smp_rendezvous_action; 68static driver_filter_t xen_invltlb; 69static driver_filter_t xen_invlpg; 70static driver_filter_t xen_invlrng; 71static driver_filter_t xen_invlcache; 72static driver_filter_t xen_ipi_bitmap_handler; 73static driver_filter_t xen_cpustop_handler; 74static driver_filter_t xen_cpususpend_handler; 75#endif 76 77/*---------------------------------- Macros ----------------------------------*/ 78#define IPI_TO_IDX(ipi) ((ipi) - APIC_IPI_INTS) 79 80/*--------------------------------- Xen IPIs ---------------------------------*/ 81#ifdef SMP 82struct xen_ipi_handler 83{ 84 driver_filter_t *filter; 85 const char *description; 86}; 87 88static struct xen_ipi_handler xen_ipis[] = 89{ 90 [IPI_TO_IDX(IPI_RENDEZVOUS)] = { xen_smp_rendezvous_action, "r" }, 91 [IPI_TO_IDX(IPI_INVLTLB)] = { xen_invltlb, "itlb"}, 92 [IPI_TO_IDX(IPI_INVLPG)] = { xen_invlpg, "ipg" }, 93 [IPI_TO_IDX(IPI_INVLRNG)] = { xen_invlrng, "irg" }, 94 [IPI_TO_IDX(IPI_INVLCACHE)] = { xen_invlcache, "ic" }, 95 [IPI_TO_IDX(IPI_BITMAP_VECTOR)] = { xen_ipi_bitmap_handler, "b" }, 96 [IPI_TO_IDX(IPI_STOP)] = { xen_cpustop_handler, "st" }, 97 [IPI_TO_IDX(IPI_SUSPEND)] = { xen_cpususpend_handler, "sp" }, 98}; 99#endif 100 101/*------------------------------- Per-CPU Data -------------------------------*/ 102#ifdef SMP 103DPCPU_DEFINE(xen_intr_handle_t, ipi_handle[nitems(xen_ipis)]); 104#endif 105 106/*------------------------------- Xen PV APIC --------------------------------*/ 107 108static void 109xen_pv_lapic_create(u_int apic_id, int boot_cpu) 110{ 111#ifdef SMP 112 cpu_add(apic_id, boot_cpu); 113#endif 114} 115 116static void 117xen_pv_lapic_init(vm_paddr_t addr) 118{ 119 120} 121 122static void 123xen_pv_lapic_setup(int boot) 124{ 125 126} 127 128static void 129xen_pv_lapic_dump(const char *str) 130{ 131 132 printf("cpu%d %s XEN PV LAPIC\n", PCPU_GET(cpuid), str); 133} 134 135static void 136xen_pv_lapic_disable(void) 137{ 138 139} 140 141static bool 142xen_pv_lapic_is_x2apic(void) 143{ 144 145 return (false); 146} 147 148static void 149xen_pv_lapic_eoi(void) 150{ 151 152 XEN_APIC_UNSUPPORTED; 153} 154 155static int 156xen_pv_lapic_id(void) 157{ 158 159 return (PCPU_GET(apic_id)); 160} 161 162static int 163xen_pv_lapic_intr_pending(u_int vector) 164{ 165 166 XEN_APIC_UNSUPPORTED; 167 return (0); 168} 169 170static u_int 171xen_pv_apic_cpuid(u_int apic_id) 172{ 173#ifdef SMP 174 return (apic_cpuids[apic_id]); 175#else 176 return (0); 177#endif 178} 179 180static u_int 181xen_pv_apic_alloc_vector(u_int apic_id, u_int irq) 182{ 183 184 XEN_APIC_UNSUPPORTED; 185 return (0); 186} 187 188static u_int 189xen_pv_apic_alloc_vectors(u_int apic_id, u_int *irqs, u_int count, u_int align) 190{ 191 192 XEN_APIC_UNSUPPORTED; 193 return (0); 194} 195 196static void 197xen_pv_apic_disable_vector(u_int apic_id, u_int vector) 198{ 199 200 XEN_APIC_UNSUPPORTED; 201} 202 203static void 204xen_pv_apic_enable_vector(u_int apic_id, u_int vector) 205{ 206 207 XEN_APIC_UNSUPPORTED; 208} 209 210static void 211xen_pv_apic_free_vector(u_int apic_id, u_int vector, u_int irq) 212{ 213 214 XEN_APIC_UNSUPPORTED; 215} 216 217static void 218xen_pv_lapic_set_logical_id(u_int apic_id, u_int cluster, u_int cluster_id) 219{ 220 221 XEN_APIC_UNSUPPORTED; 222} 223 224static int 225xen_pv_lapic_enable_pmc(void) 226{ 227 228 XEN_APIC_UNSUPPORTED; 229 return (0); 230} 231 232static void 233xen_pv_lapic_disable_pmc(void) 234{ 235 236 XEN_APIC_UNSUPPORTED; 237} 238 239static void 240xen_pv_lapic_reenable_pmc(void) 241{ 242 243 XEN_APIC_UNSUPPORTED; 244} 245 246static void 247xen_pv_lapic_enable_cmc(void) 248{ 249 250} 251 252#ifdef SMP 253static void 254xen_pv_lapic_ipi_raw(register_t icrlo, u_int dest) 255{ 256 257 XEN_APIC_UNSUPPORTED; 258} 259 260#define PCPU_ID_GET(id, field) (pcpu_find(id)->pc_##field) 261static void 262send_nmi(int dest) 263{ 264 unsigned int cpu; 265 266 /* 267 * NMIs are not routed over event channels, and instead delivered as on 268 * native using the exception vector (#2). Triggering them can be done 269 * using the local APIC, or an hypercall as a shortcut like it's done 270 * below. 271 */ 272 switch(dest) { 273 case APIC_IPI_DEST_SELF: 274 HYPERVISOR_vcpu_op(VCPUOP_send_nmi, PCPU_GET(vcpu_id), NULL); 275 break; 276 case APIC_IPI_DEST_ALL: 277 CPU_FOREACH(cpu) 278 HYPERVISOR_vcpu_op(VCPUOP_send_nmi, 279 PCPU_ID_GET(cpu, vcpu_id), NULL); 280 break; 281 case APIC_IPI_DEST_OTHERS: 282 CPU_FOREACH(cpu) 283 if (cpu != PCPU_GET(cpuid)) 284 HYPERVISOR_vcpu_op(VCPUOP_send_nmi, 285 PCPU_ID_GET(cpu, vcpu_id), NULL); 286 break; 287 default: 288 HYPERVISOR_vcpu_op(VCPUOP_send_nmi, 289 PCPU_ID_GET(apic_cpuid(dest), vcpu_id), NULL); 290 break; 291 } 292} 293#undef PCPU_ID_GET 294 295static void 296xen_pv_lapic_ipi_vectored(u_int vector, int dest) 297{ 298 xen_intr_handle_t *ipi_handle; 299 int ipi_idx, to_cpu, self; 300 301 if (vector >= IPI_NMI_FIRST) { 302 send_nmi(dest); 303 return; 304 } 305 306 ipi_idx = IPI_TO_IDX(vector); 307 if (ipi_idx >= nitems(xen_ipis)) 308 panic("IPI out of range"); 309 310 switch(dest) { 311 case APIC_IPI_DEST_SELF: 312 ipi_handle = DPCPU_GET(ipi_handle); 313 xen_intr_signal(ipi_handle[ipi_idx]); 314 break; 315 case APIC_IPI_DEST_ALL: 316 CPU_FOREACH(to_cpu) { 317 ipi_handle = DPCPU_ID_GET(to_cpu, ipi_handle); 318 xen_intr_signal(ipi_handle[ipi_idx]); 319 } 320 break; 321 case APIC_IPI_DEST_OTHERS: 322 self = PCPU_GET(cpuid); 323 CPU_FOREACH(to_cpu) { 324 if (to_cpu != self) { 325 ipi_handle = DPCPU_ID_GET(to_cpu, ipi_handle); 326 xen_intr_signal(ipi_handle[ipi_idx]); 327 } 328 } 329 break; 330 default: 331 to_cpu = apic_cpuid(dest); 332 ipi_handle = DPCPU_ID_GET(to_cpu, ipi_handle); 333 xen_intr_signal(ipi_handle[ipi_idx]); 334 break; 335 } 336} 337 338static int 339xen_pv_lapic_ipi_wait(int delay) 340{ 341 342 XEN_APIC_UNSUPPORTED; 343 return (0); 344} 345#endif /* SMP */ 346 347static int 348xen_pv_lapic_ipi_alloc(inthand_t *ipifunc) 349{ 350 351 XEN_APIC_UNSUPPORTED; 352 return (-1); 353} 354 355static void 356xen_pv_lapic_ipi_free(int vector) 357{ 358 359 XEN_APIC_UNSUPPORTED; 360} 361 362static int 363xen_pv_lapic_set_lvt_mask(u_int apic_id, u_int lvt, u_char masked) 364{ 365 366 XEN_APIC_UNSUPPORTED; 367 return (0); 368} 369 370static int 371xen_pv_lapic_set_lvt_mode(u_int apic_id, u_int lvt, uint32_t mode) 372{ 373 374 XEN_APIC_UNSUPPORTED; 375 return (0); 376} 377 378static int 379xen_pv_lapic_set_lvt_polarity(u_int apic_id, u_int lvt, enum intr_polarity pol) 380{ 381 382 XEN_APIC_UNSUPPORTED; 383 return (0); 384} 385 386static int 387xen_pv_lapic_set_lvt_triggermode(u_int apic_id, u_int lvt, 388 enum intr_trigger trigger) 389{ 390 391 XEN_APIC_UNSUPPORTED; 392 return (0); 393} 394 395/* Xen apic_ops implementation */ 396struct apic_ops xen_apic_ops = { 397 .create = xen_pv_lapic_create, 398 .init = xen_pv_lapic_init, 399 .xapic_mode = xen_pv_lapic_disable, 400 .is_x2apic = xen_pv_lapic_is_x2apic, 401 .setup = xen_pv_lapic_setup, 402 .dump = xen_pv_lapic_dump, 403 .disable = xen_pv_lapic_disable, 404 .eoi = xen_pv_lapic_eoi, 405 .id = xen_pv_lapic_id, 406 .intr_pending = xen_pv_lapic_intr_pending, 407 .set_logical_id = xen_pv_lapic_set_logical_id, 408 .cpuid = xen_pv_apic_cpuid, 409 .alloc_vector = xen_pv_apic_alloc_vector, 410 .alloc_vectors = xen_pv_apic_alloc_vectors, 411 .enable_vector = xen_pv_apic_enable_vector, 412 .disable_vector = xen_pv_apic_disable_vector, 413 .free_vector = xen_pv_apic_free_vector, 414 .enable_pmc = xen_pv_lapic_enable_pmc, 415 .disable_pmc = xen_pv_lapic_disable_pmc, 416 .reenable_pmc = xen_pv_lapic_reenable_pmc, 417 .enable_cmc = xen_pv_lapic_enable_cmc, 418#ifdef SMP 419 .ipi_raw = xen_pv_lapic_ipi_raw, 420 .ipi_vectored = xen_pv_lapic_ipi_vectored, 421 .ipi_wait = xen_pv_lapic_ipi_wait, 422#endif 423 .ipi_alloc = xen_pv_lapic_ipi_alloc, 424 .ipi_free = xen_pv_lapic_ipi_free, 425 .set_lvt_mask = xen_pv_lapic_set_lvt_mask, 426 .set_lvt_mode = xen_pv_lapic_set_lvt_mode, 427 .set_lvt_polarity = xen_pv_lapic_set_lvt_polarity, 428 .set_lvt_triggermode = xen_pv_lapic_set_lvt_triggermode, 429}; 430 431#ifdef SMP 432/*---------------------------- XEN PV IPI Handlers ---------------------------*/ 433/* 434 * These are C clones of the ASM functions found in apic_vector. 435 */ 436static int 437xen_ipi_bitmap_handler(void *arg) 438{ 439 struct trapframe *frame; 440 441 frame = arg; 442 ipi_bitmap_handler(*frame); 443 return (FILTER_HANDLED); 444} 445 446static int 447xen_smp_rendezvous_action(void *arg) 448{ 449#ifdef COUNT_IPIS 450 (*ipi_rendezvous_counts[PCPU_GET(cpuid)])++; 451#endif /* COUNT_IPIS */ 452 453 smp_rendezvous_action(); 454 return (FILTER_HANDLED); 455} 456 457static int 458xen_invltlb(void *arg) 459{ 460 461 invltlb_handler(); 462 return (FILTER_HANDLED); 463} 464 465#ifdef __amd64__ 466static int 467xen_invltlb_invpcid(void *arg) 468{ 469 470 invltlb_invpcid_handler(); 471 return (FILTER_HANDLED); 472} 473 474static int 475xen_invltlb_pcid(void *arg) 476{ 477 478 invltlb_pcid_handler(); 479 return (FILTER_HANDLED); 480} 481 482static int 483xen_invltlb_invpcid_pti(void *arg) 484{ 485 486 invltlb_invpcid_pti_handler(); 487 return (FILTER_HANDLED); 488} 489 490static int 491xen_invlpg_invpcid_handler(void *arg) 492{ 493 494 invlpg_invpcid_handler(); 495 return (FILTER_HANDLED); 496} 497 498static int 499xen_invlpg_pcid_handler(void *arg) 500{ 501 502 invlpg_pcid_handler(); 503 return (FILTER_HANDLED); 504} 505 506static int 507xen_invlrng_invpcid_handler(void *arg) 508{ 509 510 invlrng_invpcid_handler(); 511 return (FILTER_HANDLED); 512} 513 514static int 515xen_invlrng_pcid_handler(void *arg) 516{ 517 518 invlrng_pcid_handler(); 519 return (FILTER_HANDLED); 520} 521#endif 522 523static int 524xen_invlpg(void *arg) 525{ 526 527 invlpg_handler(); 528 return (FILTER_HANDLED); 529} 530 531static int 532xen_invlrng(void *arg) 533{ 534 535 invlrng_handler(); 536 return (FILTER_HANDLED); 537} 538 539static int 540xen_invlcache(void *arg) 541{ 542 543 invlcache_handler(); 544 return (FILTER_HANDLED); 545} 546 547static int 548xen_cpustop_handler(void *arg) 549{ 550 551 cpustop_handler(); 552 return (FILTER_HANDLED); 553} 554 555static int 556xen_cpususpend_handler(void *arg) 557{ 558 559 cpususpend_handler(); 560 return (FILTER_HANDLED); 561} 562 563/*----------------------------- XEN PV IPI setup -----------------------------*/ 564/* 565 * Those functions are provided outside of the Xen PV APIC implementation 566 * so PVHVM guests can also use PV IPIs without having an actual Xen PV APIC, 567 * because on PVHVM there's an emulated LAPIC provided by Xen. 568 */ 569static void 570xen_cpu_ipi_init(int cpu) 571{ 572 xen_intr_handle_t *ipi_handle; 573 const struct xen_ipi_handler *ipi; 574 int idx, rc; 575 576 ipi_handle = DPCPU_ID_GET(cpu, ipi_handle); 577 578 for (ipi = xen_ipis, idx = 0; idx < nitems(xen_ipis); ipi++, idx++) { 579 580 if (ipi->filter == NULL) { 581 ipi_handle[idx] = NULL; 582 continue; 583 } 584 585 rc = xen_intr_alloc_and_bind_ipi(cpu, ipi->filter, 586 INTR_TYPE_TTY, &ipi_handle[idx]); 587 if (rc != 0) 588 panic("Unable to allocate a XEN IPI port"); 589 xen_intr_describe(ipi_handle[idx], "%s", ipi->description); 590 } 591} 592 593static void 594xen_setup_cpus(void) 595{ 596 int i; 597 598 if (!xen_vector_callback_enabled) 599 return; 600 601#ifdef __amd64__ 602 if (pmap_pcid_enabled) { 603 if (pti) 604 xen_ipis[IPI_TO_IDX(IPI_INVLTLB)].filter = 605 invpcid_works ? xen_invltlb_invpcid_pti : 606 xen_invltlb_pcid; 607 else 608 xen_ipis[IPI_TO_IDX(IPI_INVLTLB)].filter = 609 invpcid_works ? xen_invltlb_invpcid : 610 xen_invltlb_pcid; 611 xen_ipis[IPI_TO_IDX(IPI_INVLPG)].filter = invpcid_works ? 612 xen_invlpg_invpcid_handler : xen_invlpg_pcid_handler; 613 xen_ipis[IPI_TO_IDX(IPI_INVLRNG)].filter = invpcid_works ? 614 xen_invlrng_invpcid_handler : xen_invlrng_pcid_handler; 615 } 616#endif 617 CPU_FOREACH(i) 618 xen_cpu_ipi_init(i); 619 620 /* Set the xen pv ipi ops to replace the native ones */ 621 if (xen_hvm_domain()) 622 apic_ops.ipi_vectored = xen_pv_lapic_ipi_vectored; 623} 624 625/* We need to setup IPIs before APs are started */ 626SYSINIT(xen_setup_cpus, SI_SUB_SMP-1, SI_ORDER_FIRST, xen_setup_cpus, NULL); 627#endif /* SMP */ 628