1// Copyright 2016 The Fuchsia Authors 2// 3// Use of this source code is governed by a MIT-style 4// license that can be found in the LICENSE file or at 5// https://opensource.org/licenses/MIT 6 7#include <assert.h> 8 9#include <arch/x86/apic.h> 10#include <arch/x86/interrupts.h> 11#include <err.h> 12#include <fbl/array.h> 13#include <kernel/auto_lock.h> 14#include <kernel/spinlock.h> 15#include <trace.h> 16#include <vm/physmap.h> 17#include <vm/pmm.h> 18#include <vm/vm_aspace.h> 19#include <zircon/types.h> 20#include <zxcpp/new.h> 21 22#define IO_APIC_IND(base) ((volatile uint32_t*)(((uint8_t*)(base)) + IO_APIC_IOREGSEL)) 23#define IO_APIC_DAT(base) ((volatile uint32_t*)(((uint8_t*)(base)) + IO_APIC_IOWIN)) 24#define IO_APIC_EOIR(base) ((volatile uint32_t*)(((uint8_t*)(base)) + 0x40)) 25// The minimum address space required past the base address 26#define IO_APIC_WINDOW_SIZE 0x44 27// The minimum version that supported the EOIR 28#define IO_APIC_EOIR_MIN_VERSION 0x20 29 30// IO APIC register offsets 31#define IO_APIC_REG_RTE(idx) (0x10 + 2 * (idx)) 32 33// Macros for extracting data from REG_ID 34#define IO_APIC_ID_ID(v) (((v) >> 24) & 0xf) 35// Macros for extracting data from REG_VER 36#define IO_APIC_VER_MAX_REDIR_ENTRY(v) (((v) >> 16) & 0xff) 37#define IO_APIC_VER_VERSION(v) ((v)&0xff) 38// Macros for writing REG_RTE entries 39#define IO_APIC_RTE_DST(v) (((uint64_t)(v)) << 56) 40#define IO_APIC_RTE_EXTENDED_DST_ID(v) (((uint64_t)((v)&0xf)) << 48) 41#define IO_APIC_RTE_MASKED (1ULL << 16) 42#define IO_APIC_RTE_TRIGGER_MODE(tm) (((uint64_t)(tm)) << 15) 43#define IO_APIC_RTE_POLARITY(p) (((uint64_t)(p)) << 13) 44#define IO_APIC_RTE_DST_MODE(dm) (((uint64_t)(dm)) << 11) 45#define IO_APIC_RTE_DELIVERY_MODE(dm) ((((uint64_t)(dm)) & 0x7) << 8) 46#define IO_APIC_RTE_VECTOR(x) (((uint64_t)(x)) & 0xff) 47#define IO_APIC_RTE_MASK IO_APIC_RTE_VECTOR(0xff) 48// Macros for reading REG_RTE entries 49#define IO_APIC_RTE_REMOTE_IRR (1ULL << 14) 50#define IO_APIC_RTE_DELIVERY_STATUS (1ULL << 12) 51#define IO_APIC_RTE_GET_POLARITY(r) \ 52 ((enum interrupt_polarity)(((r) >> 13) & 0x1)) 53#define IO_APIC_RTE_GET_TRIGGER_MODE(r) \ 54 ((enum interrupt_trigger_mode)(((r) >> 15) & 0x1)) 55#define IO_APIC_RTE_GET_VECTOR(r) \ 56 ((uint8_t)((r)&0xFF)) 57 58// Technically this can be larger, but the spec as of the 100-Series doesn't 59// guarantee where the additional redirections will be. 60#define IO_APIC_NUM_REDIRECTIONS 120 61 62#define LOCAL_TRACE 0 63 64// Struct for tracking all we need to know about each IO APIC 65struct io_apic { 66 struct io_apic_descriptor desc; 67 68 // Virtual address of the base of this IOAPIC's MMIO 69 void* vaddr; 70 71 uint8_t version; 72 // The index of the last redirection entry 73 uint8_t max_redirection_entry; 74 75 // Pre-allocated space for suspend/resume bookkeeping 76 uint64_t saved_rtes[IO_APIC_NUM_REDIRECTIONS]; 77}; 78 79// This lock guards all access to IO APIC registers 80static SpinLock lock; 81 82// General register accessors 83static inline uint32_t apic_io_read_reg(struct io_apic* io_apic, uint8_t reg) TA_REQ(lock); 84static inline void apic_io_write_reg(struct io_apic* io_apic, uint8_t reg, 85 uint32_t val) TA_REQ(lock); 86 87// Register-specific accessors 88static uint64_t apic_io_read_redirection_entry(struct io_apic* io_apic, 89 uint32_t global_irq) TA_REQ(lock); 90static void apic_io_write_redirection_entry(struct io_apic* io_apic, uint32_t global_irq, 91 uint64_t value) TA_REQ(lock); 92 93// Utility for finding the right IO APIC for a specific global IRQ, cannot fail 94static struct io_apic* apic_io_resolve_global_irq(uint32_t irq); 95// Utility for finding the right IO APIC for a specific global IRQ, can fail 96static struct io_apic* apic_io_resolve_global_irq_no_panic(uint32_t irq); 97 98// Track all IO APICs in the system 99static fbl::Array<io_apic> io_apics; 100static uint32_t num_io_apics; 101 102// The first 16 global IRQs are identity mapped to the legacy ISA IRQs unless 103// we are told otherwise. This tracks the actual mapping. 104// Read-only after initialization in apic_io_init() 105static struct io_apic_isa_override isa_overrides[NUM_ISA_IRQS]; 106 107void apic_io_init( 108 struct io_apic_descriptor* io_apic_descs, 109 unsigned int num_io_apic_descs, 110 struct io_apic_isa_override* overrides, 111 unsigned int num_overrides) { 112 ASSERT(!io_apics); 113 114 num_io_apics = num_io_apic_descs; 115 { 116 fbl::AllocChecker ac; 117 io_apics.reset(new (&ac) io_apic[num_io_apics], num_io_apics); 118 ASSERT(ac.check()); 119 } 120 for (unsigned int i = 0; i < num_io_apics; ++i) { 121 io_apics[i].desc = io_apic_descs[i]; 122 } 123 124 // Allocate windows to their control pages 125 for (uint32_t i = 0; i < num_io_apics; ++i) { 126 struct io_apic* apic = &io_apics[i]; 127 paddr_t paddr = apic->desc.paddr; 128 void* vaddr = paddr_to_physmap(paddr); 129 // If the window isn't mapped yet (multiple IO APICs can be in the 130 // same page), map it in. 131 if (vaddr == nullptr) { 132 paddr_t paddr_page_base = ROUNDDOWN(paddr, PAGE_SIZE); 133 ASSERT(paddr + IO_APIC_WINDOW_SIZE <= paddr_page_base + PAGE_SIZE); 134 zx_status_t res = VmAspace::kernel_aspace()->AllocPhysical( 135 "ioapic", 136 PAGE_SIZE, // size 137 &vaddr, // requested virtual vaddress 138 PAGE_SIZE_SHIFT, // alignment log2 139 paddr_page_base, // physical vaddress 140 0, // vmm flags 141 ARCH_MMU_FLAG_PERM_READ | ARCH_MMU_FLAG_PERM_WRITE | 142 ARCH_MMU_FLAG_UNCACHED_DEVICE); // arch mmu flags 143 ASSERT(res == ZX_OK); 144 vaddr = (void*)((uintptr_t)vaddr + paddr - paddr_page_base); 145 } 146 147 // Populate the rest of the descriptor 148 apic->vaddr = vaddr; 149 150 AutoSpinLock guard(&lock); 151 152 uint32_t ver = apic_io_read_reg(apic, IO_APIC_REG_VER); 153 apic->version = IO_APIC_VER_VERSION(ver); 154 apic->max_redirection_entry = IO_APIC_VER_MAX_REDIR_ENTRY(ver); 155 LTRACEF("Found an IO APIC at phys %p, virt %p: ver %08x\n", (void*)paddr, vaddr, ver); 156 if (apic->max_redirection_entry > IO_APIC_NUM_REDIRECTIONS - 1) { 157 TRACEF("IO APIC supports more redirections than kernel: %08x\n", 158 ver); 159 apic->max_redirection_entry = IO_APIC_NUM_REDIRECTIONS - 1; 160 } 161 162 // Cleanout the redirection entries 163 for (unsigned int j = 0; j <= apic->max_redirection_entry; ++j) { 164 apic_io_write_redirection_entry( 165 apic, j + apic->desc.global_irq_base, IO_APIC_RTE_MASKED); 166 } 167 } 168 169 // Process ISA IRQ overrides 170 for (unsigned int i = 0; i < num_overrides; ++i) { 171 uint8_t isa_irq = overrides[i].isa_irq; 172 ASSERT(isa_irq < NUM_ISA_IRQS); 173 isa_overrides[isa_irq] = overrides[i]; 174 LTRACEF("ISA IRQ override for ISA IRQ %u, mapping to %u\n", 175 isa_irq, overrides[i].global_irq); 176 } 177} 178 179static struct io_apic* apic_io_resolve_global_irq_no_panic(uint32_t irq) { 180 for (uint32_t i = 0; i < num_io_apics; ++i) { 181 uint32_t start = io_apics[i].desc.global_irq_base; 182 uint32_t end = start + io_apics[i].max_redirection_entry; 183 if (start <= irq && irq <= end) { 184 return &io_apics[i]; 185 } 186 } 187 return nullptr; 188} 189 190static struct io_apic* apic_io_resolve_global_irq(uint32_t irq) { 191 struct io_apic* res = apic_io_resolve_global_irq_no_panic(irq); 192 if (res) { 193 return res; 194 } 195 // Treat this as fatal, since dealing with an unmapped IRQ is a bug. 196 panic("Could not resolve global IRQ: %u\n", irq); 197} 198 199static inline uint32_t apic_io_read_reg( 200 struct io_apic* io_apic, 201 uint8_t reg) { 202 ASSERT(io_apic != nullptr); 203 DEBUG_ASSERT(lock.IsHeld()); 204 *IO_APIC_IND(io_apic->vaddr) = reg; 205 uint32_t val = *IO_APIC_DAT(io_apic->vaddr); 206 return val; 207} 208 209static inline void apic_io_write_reg( 210 struct io_apic* io_apic, 211 uint8_t reg, 212 uint32_t val) { 213 ASSERT(io_apic != nullptr); 214 DEBUG_ASSERT(lock.IsHeld()); 215 *IO_APIC_IND(io_apic->vaddr) = reg; 216 *IO_APIC_DAT(io_apic->vaddr) = val; 217} 218 219static uint64_t apic_io_read_redirection_entry( 220 struct io_apic* io_apic, 221 uint32_t global_irq) { 222 DEBUG_ASSERT(lock.IsHeld()); 223 224 ASSERT(global_irq >= io_apic->desc.global_irq_base); 225 uint32_t offset = global_irq - io_apic->desc.global_irq_base; 226 ASSERT(offset <= io_apic->max_redirection_entry); 227 228 uint8_t reg_id = (uint8_t)IO_APIC_REG_RTE(offset); 229 uint64_t result = 0; 230 result |= apic_io_read_reg(io_apic, reg_id); 231 result |= ((uint64_t)apic_io_read_reg(io_apic, (uint8_t)(reg_id + 1))) << 32; 232 return result; 233} 234 235static void apic_io_write_redirection_entry( 236 struct io_apic* io_apic, 237 uint32_t global_irq, 238 uint64_t value) { 239 DEBUG_ASSERT(lock.IsHeld()); 240 241 ASSERT(global_irq >= io_apic->desc.global_irq_base); 242 uint32_t offset = global_irq - io_apic->desc.global_irq_base; 243 ASSERT(offset <= io_apic->max_redirection_entry); 244 245 uint8_t reg_id = (uint8_t)IO_APIC_REG_RTE(offset); 246 apic_io_write_reg(io_apic, reg_id, (uint32_t)value); 247 apic_io_write_reg(io_apic, (uint8_t)(reg_id + 1), (uint32_t)(value >> 32)); 248} 249 250bool apic_io_is_valid_irq(uint32_t global_irq) { 251 return apic_io_resolve_global_irq_no_panic(global_irq) != nullptr; 252} 253 254/* 255 * To correctly use this function, we need to do some work first. 256 * 1) We need to check for EOI-broadcast suppression support in the local APIC 257 * version register. 258 * 2) We need to check that the IOAPIC is new enough to support the EOI 259 * 3) We need to enable suppression in the spurious interrupt register. 260 * 4) Call this function after calling apic_issue_eoi() (or maybe modify 261 * apic_issue_eoi() to call this automatically). 262 * 263 * In the mean time, IO APIC EOIs are automatically issued via broadcast to 264 * all IO APICs whenever the local APIC receives an EOI for a level-triggered 265 * interrupt. 266 */ 267void apic_io_issue_eoi(uint32_t global_irq, uint8_t vec) { 268 struct io_apic* io_apic = apic_io_resolve_global_irq(global_irq); 269 270 AutoSpinLock guard(&lock); 271 272 ASSERT(io_apic->version >= IO_APIC_EOIR_MIN_VERSION); 273 *IO_APIC_EOIR(io_apic->vaddr) = vec; 274} 275 276void apic_io_mask_irq(uint32_t global_irq, bool mask) { 277 struct io_apic* io_apic = apic_io_resolve_global_irq(global_irq); 278 279 AutoSpinLock guard(&lock); 280 281 uint64_t reg = apic_io_read_redirection_entry(io_apic, global_irq); 282 if (mask) { 283 reg |= IO_APIC_RTE_MASKED; 284 } else { 285 /* If we are unmasking, we had better have been assigned a valid vector */ 286 DEBUG_ASSERT((IO_APIC_RTE_GET_VECTOR(reg) >= X86_INT_PLATFORM_BASE) && 287 (IO_APIC_RTE_GET_VECTOR(reg) <= X86_INT_PLATFORM_MAX)); 288 reg &= ~IO_APIC_RTE_MASKED; 289 } 290 apic_io_write_redirection_entry(io_apic, global_irq, reg); 291} 292 293void apic_io_configure_irq( 294 uint32_t global_irq, 295 enum interrupt_trigger_mode trig_mode, 296 enum interrupt_polarity polarity, 297 enum apic_interrupt_delivery_mode del_mode, 298 bool mask, 299 enum apic_interrupt_dst_mode dst_mode, 300 uint8_t dst, 301 uint8_t vector) { 302 struct io_apic* io_apic = apic_io_resolve_global_irq(global_irq); 303 304 AutoSpinLock guard(&lock); 305 306 /* If we are configuring an invalid vector, for the IRQ to be masked. */ 307 if ((del_mode == DELIVERY_MODE_FIXED || del_mode == DELIVERY_MODE_LOWEST_PRI) && 308 ((vector < X86_INT_PLATFORM_BASE) || (vector > X86_INT_PLATFORM_MAX))) { 309 310 mask = true; 311 } 312 313 uint64_t reg = 0; 314 reg |= IO_APIC_RTE_TRIGGER_MODE(trig_mode); 315 reg |= IO_APIC_RTE_POLARITY(polarity); 316 reg |= IO_APIC_RTE_DELIVERY_MODE(del_mode); 317 reg |= IO_APIC_RTE_DST_MODE(dst_mode); 318 reg |= IO_APIC_RTE_DST(dst); 319 reg |= IO_APIC_RTE_VECTOR(vector); 320 if (mask) { 321 reg |= IO_APIC_RTE_MASKED; 322 } 323 apic_io_write_redirection_entry(io_apic, global_irq, reg); 324} 325 326zx_status_t apic_io_fetch_irq_config( 327 uint32_t global_irq, 328 enum interrupt_trigger_mode* trig_mode, 329 enum interrupt_polarity* polarity) { 330 struct io_apic* io_apic = apic_io_resolve_global_irq(global_irq); 331 332 if (!io_apic) 333 return ZX_ERR_INVALID_ARGS; 334 335 AutoSpinLock guard(&lock); 336 337 uint64_t reg = apic_io_read_redirection_entry(io_apic, global_irq); 338 if (trig_mode) 339 *trig_mode = IO_APIC_RTE_GET_TRIGGER_MODE(reg); 340 if (polarity) 341 *polarity = IO_APIC_RTE_GET_POLARITY(reg); 342 343 return ZX_OK; 344} 345 346void apic_io_configure_irq_vector( 347 uint32_t global_irq, 348 uint8_t vector) { 349 struct io_apic* io_apic = apic_io_resolve_global_irq(global_irq); 350 351 AutoSpinLock guard(&lock); 352 353 uint64_t reg = apic_io_read_redirection_entry(io_apic, global_irq); 354 355 /* If we are configuring an invalid vector, automatically mask the IRQ. */ 356 if ((IO_APIC_RTE_GET_VECTOR(reg) < X86_INT_PLATFORM_BASE) || 357 (IO_APIC_RTE_GET_VECTOR(reg) > X86_INT_PLATFORM_MAX)) { 358 reg |= IO_APIC_RTE_MASKED; 359 } 360 361 reg &= ~IO_APIC_RTE_MASK; 362 reg |= IO_APIC_RTE_VECTOR(vector); 363 apic_io_write_redirection_entry(io_apic, global_irq, reg); 364} 365 366uint8_t apic_io_fetch_irq_vector(uint32_t global_irq) { 367 struct io_apic* io_apic = apic_io_resolve_global_irq(global_irq); 368 369 AutoSpinLock guard(&lock); 370 371 uint64_t reg = apic_io_read_redirection_entry(io_apic, global_irq); 372 uint8_t vector = IO_APIC_RTE_GET_VECTOR(reg); 373 374 return vector; 375} 376 377void apic_io_mask_isa_irq(uint8_t isa_irq, bool mask) { 378 ASSERT(isa_irq < NUM_ISA_IRQS); 379 uint32_t global_irq = isa_irq; 380 if (isa_overrides[isa_irq].remapped) { 381 global_irq = isa_overrides[isa_irq].global_irq; 382 } 383 apic_io_mask_irq(global_irq, mask); 384} 385 386void apic_io_configure_isa_irq( 387 uint8_t isa_irq, 388 enum apic_interrupt_delivery_mode del_mode, 389 bool mask, 390 enum apic_interrupt_dst_mode dst_mode, 391 uint8_t dst, 392 uint8_t vector) { 393 ASSERT(isa_irq < NUM_ISA_IRQS); 394 uint32_t global_irq = isa_irq; 395 enum interrupt_trigger_mode trig_mode = IRQ_TRIGGER_MODE_EDGE; 396 enum interrupt_polarity polarity = IRQ_POLARITY_ACTIVE_HIGH; 397 if (isa_overrides[isa_irq].remapped) { 398 global_irq = isa_overrides[isa_irq].global_irq; 399 trig_mode = isa_overrides[isa_irq].tm; 400 polarity = isa_overrides[isa_irq].pol; 401 } 402 403 apic_io_configure_irq( 404 global_irq, 405 trig_mode, 406 polarity, 407 del_mode, 408 mask, 409 dst_mode, 410 dst, 411 vector); 412} 413 414// Convert a legacy ISA IRQ number into a global IRQ number 415uint32_t apic_io_isa_to_global(uint8_t isa_irq) { 416 // It is a programming bug for this to be invoked with an invalid value. 417 ASSERT(isa_irq < NUM_ISA_IRQS); 418 if (isa_overrides[isa_irq].remapped) { 419 return isa_overrides[isa_irq].global_irq; 420 } 421 return isa_irq; 422} 423 424void apic_io_save(void) { 425 DEBUG_ASSERT(arch_ints_disabled()); 426 AutoSpinLockNoIrqSave guard(&lock); 427 for (uint32_t i = 0; i < num_io_apics; ++i) { 428 struct io_apic* apic = &io_apics[i]; 429 for (uint8_t j = 0; j <= apic->max_redirection_entry; ++j) { 430 uint32_t global_irq = apic->desc.global_irq_base + j; 431 uint64_t reg = apic_io_read_redirection_entry(apic, global_irq); 432 apic->saved_rtes[j] = reg; 433 } 434 } 435} 436 437void apic_io_restore(void) { 438 DEBUG_ASSERT(arch_ints_disabled()); 439 AutoSpinLockNoIrqSave guard(&lock); 440 for (uint32_t i = 0; i < num_io_apics; ++i) { 441 struct io_apic* apic = &io_apics[i]; 442 for (uint8_t j = 0; j <= apic->max_redirection_entry; ++j) { 443 uint32_t global_irq = apic->desc.global_irq_base + j; 444 apic_io_write_redirection_entry(apic, global_irq, apic->saved_rtes[j]); 445 } 446 } 447} 448 449void apic_io_debug(void) { 450 AutoSpinLock guard(&lock); 451 for (uint32_t i = 0; i < num_io_apics; ++i) { 452 struct io_apic* apic = &io_apics[i]; 453 printf("IO APIC idx %u:\n", i); 454 printf(" id: %08x\n", apic->desc.apic_id); 455 printf(" version: %08hhx\n", apic->version); 456 printf(" entries: %08x\n", apic->max_redirection_entry + 1U); 457 for (uint8_t j = 0; j <= apic->max_redirection_entry; ++j) { 458 uint32_t global_irq = apic->desc.global_irq_base + j; 459 uint64_t reg = apic_io_read_redirection_entry(apic, global_irq); 460 printf(" %4u: dst: %s %02hhx, %s, %s, %s, dm %hhx, vec %2hhx, %s %s\n", 461 global_irq, 462 (reg & (1 << 11)) ? "l" : "p", 463 (uint8_t)(reg >> 56), 464 (reg & IO_APIC_RTE_MASKED) ? "masked" : "unmasked", 465 IO_APIC_RTE_GET_TRIGGER_MODE(reg) ? "level" : "edge", 466 IO_APIC_RTE_GET_POLARITY(reg) ? "low" : "high", 467 (uint8_t)((reg >> 8) & 0x7), 468 (uint8_t)reg, 469 (reg & (1 << 12)) ? "pending" : "", 470 (reg & (1 << 14)) ? "RIRR" : ""); 471 } 472 } 473} 474