1/* 2 * Copyright 2014, General Dynamics C4 Systems 3 * 4 * This software may be distributed and modified according to the terms of 5 * the GNU General Public License version 2. Note that NO WARRANTY is provided. 6 * See "LICENSE_GPLv2.txt" for details. 7 * 8 * @TAG(GD_GPL) 9 */ 10 11#include <config.h> 12 13#ifdef CONFIG_IOMMU 14 15#include <kernel/boot.h> 16#include <machine.h> 17#include <machine/io.h> 18#include <arch/kernel/apic.h> 19#include <arch/model/statedata.h> 20#include <linker.h> 21#include <plat/machine/acpi.h> 22#include <plat/machine/intel-vtd.h> 23#include <util.h> 24 25#define RTADDR_REG 0x20 26#define GCMD_REG 0x18 27#define GSTS_REG 0x1C 28#define CCMD_REG 0x28 29#define ECAP_REG 0x10 30#define IOTLB_REG 0x08 31#define FSTS_REG 0x34 32#define FECTL_REG 0x38 33#define FEDATA_REG 0x3C 34#define FEADDR_REG 0x40 35#define FEUADDR_REG 0x44 36#define CAP_REG 0x08 37 38/* Bit Positions within Registers */ 39#define SRTP 30 /* Set Root Table Pointer */ 40#define RTPS 30 /* Root Table Pointer Status */ 41#define TE 31 /* Translation Enable */ 42#define TES 31 /* Translation Enable Status */ 43 44/* ICC is 63rd bit in CCMD_REG, but since we will be 45 * accessing this register as 4 byte word, ICC becomes 46 * 31st bit in the upper 32bit word. 47 */ 48#define ICC (31 + 32) /* Invalidate Context Cache */ 49#define CIRG (29 + 32) /* Context Invalidation Request Granularity */ 50#define CAIG 27 /* Context Actual Invalidation Granularity */ 51#define CAIG_MASK 0x3 52#define IVO_MASK 0x3FF 53#define IVT 31 /* Invalidate IOTLB */ 54#define IIRG 28 /* IOTLB Invalidation Request Granularity */ 55#define IAIG 25 /* IOTLB Actual Invalidation Granularity */ 56#define IAIG_MASK 0x7 57#define IP 30 /* Interrupt Pending */ 58#define FRI 0x8 /* Fault Recording Index */ 59#define FRI_MASK 0xFF 60#define FRO 24 61#define FRO_MASK 0xFF 62#define FI 12 63#define SID_MASK 0xFFFF 64#define SID_BUS(a) (MASK(8) & (a >> 8)) 65#define SID_DEV(a) (MASK(5) & (a >> 3)) 66#define SID_FUNC(a) (MASK(3) & a) 67#define FR_MASK 0xFF 68#define FAULT_TYPE 30 69#define FAULT 31 70#define NFR 8 /* high word of CAP_REG */ 71#define NFR_MASK 0xff 72#define PPF 1 73#define PPF_MASK 1 74#define PRESENT 1 75#define WBF 27 76#define WBFS 27 77#define DID 8 78#define RW 0x3 79 80#define SAGAW 8 81#define SAGAW_2_LEVEL 0x01 82#define SAGAW_3_LEVEL 0x02 83#define SAGAW_4_LEVEL 0x04 84#define SAGAW_5_LEVEL 0x08 85#define SAGAW_6_LEVEL 0x10 86 87#define CONTEXT_GLOBAL_INVALIDATE 0x1 88#define IOTLB_GLOBAL_INVALIDATE 0x1 89 90#define DMA_TLB_READ_DRAIN BIT(17) 91#define DMA_TLB_WRITE_DRAIN BIT(16) 92 93typedef uint32_t drhu_id_t; 94 95static inline uint32_t vtd_read32(drhu_id_t drhu_id, uint32_t offset) 96{ 97 return *(volatile uint32_t*)(PPTR_DRHU_START + (drhu_id << PAGE_BITS) + offset); 98} 99 100static inline void vtd_write32(drhu_id_t drhu_id, uint32_t offset, uint32_t value) 101{ 102 *(volatile uint32_t*)(PPTR_DRHU_START + (drhu_id << PAGE_BITS) + offset) = value; 103} 104 105 106static inline uint64_t vtd_read64(drhu_id_t drhu_id, uint32_t offset) 107{ 108 return *(volatile uint64_t *)(PPTR_DRHU_START + (drhu_id << PAGE_BITS) + offset); 109} 110 111static inline void vtd_write64(drhu_id_t drhu_id, uint32_t offset, uint64_t value) 112{ 113 *(volatile uint64_t *)(PPTR_DRHU_START + (drhu_id << PAGE_BITS) + offset) = value; 114} 115 116static inline uint32_t get_ivo(drhu_id_t drhu_id) 117{ 118 return ((vtd_read32(drhu_id, ECAP_REG) >> 8) & IVO_MASK) * 16; 119} 120 121static uint32_t get_fro_offset(drhu_id_t drhu_id) 122{ 123 uint32_t fro_offset; 124 125 /* Get bits 31 to 24 from lower Capability Register */ 126 fro_offset = (vtd_read32(drhu_id, CAP_REG) >> FRO) & FRO_MASK; 127 128 /* Get bits 33 to 32 from higher Capability Register */ 129 fro_offset |= (vtd_read32(drhu_id, CAP_REG + 4) & 0x3) << 8; 130 131 return fro_offset << 4; 132} 133 134void invalidate_context_cache(void) 135{ 136 /* FIXME - bugzilla bug 172 137 * 1. Instead of assuming global invalidation, this function should 138 * accept a parameter to control the granularity of invalidation 139 * request. 140 * 2. Instead of doing invalidation for all the IOMMUs, it should 141 * only do it for the IOMMU responsible for the requesting PCI 142 * device. 143 */ 144 145 drhu_id_t i; 146 147 for (i = 0; i < x86KSnumDrhu; i++) { 148 /* Wait till ICC bit is clear */ 149 uint64_t ccmd = 0; 150 while ((vtd_read64(i, CCMD_REG) >> ICC) & 1); 151 152 /* Program CIRG for Global Invalidation by setting bit 61 which 153 * will be bit 29 in upper 32 bits of CCMD_REG 154 */ 155 ccmd = ((uint64_t)CONTEXT_GLOBAL_INVALIDATE << CIRG) | (1ull << ICC); 156 157 /* Invalidate Context Cache */ 158 vtd_write64(i, CCMD_REG, ccmd); 159 160 /* Wait for the invalidation to complete */ 161 while ((vtd_read64(i, CCMD_REG) >> ICC) & 1); 162 } 163} 164 165void invalidate_iotlb(void) 166{ 167 /* FIXME - bugzilla bug 172 168 * 1. Instead of assuming global invalidation, this function should 169 * accept a parameter to control the granularity of invalidation 170 * request. 171 * 2. Instead of doing invalidation for all the IOMMUs, it should 172 * only do it for the IOMMU responsible for the requesting PCI 173 * device. 174 */ 175 176 uint8_t invalidate_command = IOTLB_GLOBAL_INVALIDATE; 177 uint32_t iotlb_reg_upper; 178 uint32_t ivo_offset; 179 drhu_id_t i; 180 181 for (i = 0; i < x86KSnumDrhu; i++) { 182 ivo_offset = get_ivo(i); 183 184 /* Wait till IVT bit is clear */ 185 while ((vtd_read32(i, ivo_offset + IOTLB_REG + 4) >> IVT) & 1); 186 187 /* Program IIRG for Global Invalidation by setting bit 60 which 188 * will be bit 28 in upper 32 bits of IOTLB_REG 189 */ 190 iotlb_reg_upper = invalidate_command << IIRG; 191 192 /* Invalidate IOTLB */ 193 iotlb_reg_upper |= BIT(IVT); 194 iotlb_reg_upper |= DMA_TLB_READ_DRAIN | DMA_TLB_WRITE_DRAIN; 195 196 vtd_write32(i, ivo_offset + IOTLB_REG, 0); 197 vtd_write32(i, ivo_offset + IOTLB_REG + 4, iotlb_reg_upper); 198 199 /* Wait for the invalidation to complete */ 200 while ((vtd_read32(i, ivo_offset + IOTLB_REG + 4) >> IVT) & 1); 201 } 202} 203 204static void vtd_clear_fault(drhu_id_t i, word_t fr_reg) 205{ 206 /* Clear the 'F' (Fault) bit to indicate that this fault is processed */ 207 vtd_write32(i, fr_reg + 12, BIT(FAULT)); 208} 209 210static void vtd_process_faults(drhu_id_t i) 211{ 212 /* Fault Recording register offset relative to the base register */ 213 uint32_t fro_offset; 214 uint32_t source_id UNUSED; 215 uint32_t fault_type UNUSED; 216 uint32_t address[2] UNUSED; 217 uint32_t reason UNUSED; 218 uint32_t num_fault_regs; 219 uint32_t fr_reg; 220 uint32_t fault_status; 221 uint32_t fault_record_index; 222 223 /* Retrieves FRO by looking into Capability register bits 33 to 24 */ 224 fro_offset = get_fro_offset(i); 225 fault_status = (vtd_read32(i, FSTS_REG) >> PPF) & PPF_MASK; 226 227 if (fault_status) { 228 num_fault_regs = ((vtd_read32(i, CAP_REG + 4) >> NFR) & NFR_MASK) + 1; 229 fault_record_index = (vtd_read32(i, FSTS_REG) >> FRI) & FRI_MASK; 230 fr_reg = fro_offset + 16 * fault_record_index; 231 232 /* Traverse the fault register ring buffer */ 233 do { 234 source_id = vtd_read32(i, fr_reg + 8) & SID_MASK; 235 236 fault_type = (vtd_read32(i, fr_reg + 12) >> FAULT_TYPE) & 1; 237 address[1] = vtd_read32(i, fr_reg + 4); 238 address[0] = vtd_read32(i, fr_reg); 239 reason = vtd_read32(i, fr_reg + 12) & FR_MASK; 240 241 printf("IOMMU: DMA %s page fault ", fault_type ? "read" : "write"); 242 printf("from 0x%x (bus: 0x%lx/dev: 0x%lx/fun: 0x%lx) ", source_id, 243 SID_BUS(source_id), SID_DEV(source_id), SID_FUNC(source_id)); 244 printf("on address 0x%x:%x ", address[1], address[0]); 245 printf("with reason code 0x%x\n", reason); 246 247 vtd_clear_fault(i, fr_reg); 248 249 fault_record_index = (fault_record_index + 1) % num_fault_regs; 250 fr_reg = fro_offset + 16 * fault_record_index; 251 } while ((vtd_read32(i, fr_reg + 12) >> FAULT) & 1); 252 253 /* Check for Primary Fault Overflow */ 254 if (vtd_read32(i, FSTS_REG) & 1) { 255 /* Clear PFO bit, so new faults will be generated again ! */ 256 vtd_write32(i, FSTS_REG, 1); 257 } 258 } 259} 260 261void vtd_handle_fault(void) 262{ 263 drhu_id_t i; 264 265 for (i = 0; i < x86KSnumDrhu; i++) { 266 vtd_process_faults(i); 267 } 268} 269 270BOOT_CODE static void 271vtd_create_root_table(void) 272{ 273 x86KSvtdRootTable = (void*)alloc_region(VTD_RT_SIZE_BITS); 274 memzero((void*)x86KSvtdRootTable, BIT(VTD_RT_SIZE_BITS)); 275} 276 277/* This function is a simplistic duplication of some of the logic 278 * in iospace.c 279 */ 280BOOT_CODE static void 281vtd_map_reserved_page(vtd_cte_t *vtd_context_table, int context_index, paddr_t addr) 282{ 283 int i; 284 vtd_pte_t *iopt; 285 vtd_pte_t *vtd_pte_slot; 286 /* first check for the first page table */ 287 vtd_cte_t *vtd_context_slot = vtd_context_table + context_index; 288 if (!vtd_cte_ptr_get_present(vtd_context_slot)) { 289 iopt = (vtd_pte_t*)alloc_region(seL4_IOPageTableBits); 290 if (!iopt) { 291 fail("Failed to allocate IO page table"); 292 } 293 memzero(iopt, BIT(seL4_IOPageTableBits)); 294 flushCacheRange(iopt, seL4_IOPageTableBits); 295 296 *vtd_context_slot = vtd_cte_new( 297 x86KSFirstValidIODomain, /* Domain ID */ 298 true, /* RMRR Mapping */ 299 x86KSnumIOPTLevels - 2, /* Address Width */ 300 pptr_to_paddr(iopt), /* Address Space Root */ 301 0, /* Translation Type */ 302 true); /* Present */ 303 x86KSFirstValidIODomain++; 304 flushCacheRange(vtd_context_slot, VTD_CTE_SIZE_BITS); 305 } else { 306 iopt = (vtd_pte_t*)paddr_to_pptr(vtd_cte_ptr_get_asr(vtd_context_slot)); 307 } 308 /* now recursively find and map page tables */ 309 for (i = x86KSnumIOPTLevels - 1; i >= 0; i--) { 310 uint32_t iopt_index; 311 /* If we are still looking up bits beyond the 32bit of physical 312 * that we support then we select entry 0 in the current PT */ 313 if (VTD_PT_INDEX_BITS * i >= 32) { 314 iopt_index = 0; 315 } else { 316 iopt_index = ( (addr >> seL4_PageBits) >> (VTD_PT_INDEX_BITS * i)) & MASK(VTD_PT_INDEX_BITS); 317 } 318 vtd_pte_slot = iopt + iopt_index; 319 if (i == 0) { 320 /* Now put the mapping in */ 321 *vtd_pte_slot = vtd_pte_new(addr, 1, 1); 322 flushCacheRange(vtd_pte_slot, VTD_PTE_SIZE_BITS); 323 } else { 324 if (!vtd_pte_ptr_get_write(vtd_pte_slot)) { 325 iopt = (vtd_pte_t*)alloc_region(seL4_IOPageTableBits); 326 if (!iopt) { 327 fail("Failed to allocate IO page table"); 328 } 329 memzero(iopt, BIT(seL4_IOPageTableBits)); 330 flushCacheRange(iopt, seL4_IOPageTableBits); 331 332 *vtd_pte_slot = vtd_pte_new(pptr_to_paddr(iopt), 1, 1); 333 flushCacheRange(vtd_pte_slot, VTD_PTE_SIZE_BITS); 334 } else { 335 iopt = (vtd_pte_t*)paddr_to_pptr(vtd_pte_ptr_get_addr(vtd_pte_slot)); 336 } 337 } 338 } 339} 340 341BOOT_CODE static void 342vtd_create_context_table( 343 uint8_t bus, 344 uint32_t max_num_iopt_levels, 345 acpi_rmrr_list_t *rmrr_list 346) 347{ 348 word_t i; 349 vtd_cte_t* vtd_context_table = (vtd_cte_t*)alloc_region(VTD_CT_SIZE_BITS); 350 if (!vtd_context_table) { 351 fail("Failed to allocate context table"); 352 } 353 354 printf("IOMMU: Create VTD context table for PCI bus 0x%x (pptr=%p)\n", bus, vtd_context_table); 355 memzero(vtd_context_table, BIT(VTD_CT_SIZE_BITS)); 356 flushCacheRange(vtd_context_table, VTD_CT_SIZE_BITS); 357 358 x86KSvtdRootTable[bus] = 359 vtd_rte_new( 360 pptr_to_paddr(vtd_context_table), /* Context Table Pointer */ 361 true /* Present */ 362 ); 363 /* map in any RMRR regions */ 364 for (i = 0; i < rmrr_list->num; i++) { 365 if (vtd_get_root_index(rmrr_list->entries[i].device) == bus) { 366 uint32_t addr; 367 for (addr = rmrr_list->entries[i].base; addr < rmrr_list->entries[i].limit; addr += BIT(seL4_PageBits)) { 368 (void)vtd_map_reserved_page; 369 vtd_map_reserved_page(vtd_context_table, vtd_get_context_index(rmrr_list->entries[i].device), addr); 370 } 371 } 372 } 373} 374 375BOOT_CODE static bool_t 376vtd_enable(cpu_id_t cpu_id) 377{ 378 drhu_id_t i; 379 uint32_t status = 0; 380 381 for (i = 0; i < x86KSnumDrhu; i++) { 382 pptr_t pa = (pptr_t)pptr_to_paddr((void *)x86KSvtdRootTable); 383 384 /* Set the Root Table Register */ 385 vtd_write64(i, RTADDR_REG, pa); 386 status = vtd_read32(i, GSTS_REG); 387 status |= BIT(SRTP); 388 /* Set SRTP bit in GCMD_REG */ 389 vtd_write32(i, GCMD_REG, status); 390 391 /* Wait for SRTP operation to complete by polling 392 * RTPS bit from GSTS_REG 393 */ 394 while (!((vtd_read32(i, GSTS_REG) >> RTPS) & 1)); 395 } 396 397 /* Globally invalidate context cache of all IOMMUs */ 398 invalidate_context_cache(); 399 400 /* Globally invalidate IOTLB of all IOMMUs */ 401 invalidate_iotlb(); 402 403 for (i = 0; i < x86KSnumDrhu; i++) { 404 uint32_t data, addr; 405 406 data = int_iommu; 407 addr = apic_get_base_paddr(); 408 if (!addr) { 409 return false; 410 } 411 addr |= (cpu_id << 12); 412 413 vtd_process_faults(i); 414 vtd_write32(i, FECTL_REG, 0); 415 vtd_write32(i, FEDATA_REG, data); 416 vtd_write32(i, FEADDR_REG, addr); 417 vtd_write32(i, FEUADDR_REG, 0); 418 status = vtd_read32(i, GSTS_REG); 419 status |= BIT(WBF); 420 /*flush IOMMU write buffer */ 421 vtd_write32(i, GCMD_REG, status); 422 while (((vtd_read32(i, GSTS_REG) >> WBFS) & 1)); 423 424 printf("IOMMU 0x%x: enabling...", i); 425 426 status = vtd_read32(i, GSTS_REG); 427 status |= BIT(TE); 428 /* Enable the DMA translation by setting TE bit in GCMD_REG */ 429 vtd_write32(i, GCMD_REG, status); 430 431 /* Wait for Translation Enable operation to complete by polling 432 * TES bit from GSTS_REG 433 */ 434 while (!((vtd_read32(i, GSTS_REG) >> TES) & 1)); 435 436 printf(" enabled\n"); 437 } 438 return true; 439} 440 441BOOT_CODE bool_t 442vtd_init( 443 cpu_id_t cpu_id, 444 uint32_t num_drhu, 445 acpi_rmrr_list_t *rmrr_list 446) 447{ 448 drhu_id_t i; 449 uint32_t bus; 450 uint32_t aw_bitmask = 0xffffffff; 451 uint32_t max_num_iopt_levels; 452 /* Start the number of domains at 16 bits */ 453 uint32_t num_domain_id_bits = 16; 454 455 x86KSnumDrhu = num_drhu; 456 x86KSFirstValidIODomain = 0; 457 458 if (x86KSnumDrhu == 0) { 459 return true; 460 } 461 462 for (i = 0; i < x86KSnumDrhu; i++) { 463 uint32_t bits_supported = 4 + 2 * (vtd_read32(i, CAP_REG) & 7); 464 aw_bitmask &= vtd_read32(i, CAP_REG) >> SAGAW; 465 printf("IOMMU 0x%x: %d-bit domain IDs supported\n", i, bits_supported); 466 if (bits_supported < num_domain_id_bits) { 467 num_domain_id_bits = bits_supported; 468 } 469 } 470 471 x86KSnumIODomainIDBits = num_domain_id_bits; 472 473 if (aw_bitmask & SAGAW_6_LEVEL) { 474 max_num_iopt_levels = 6; 475 } else if (aw_bitmask & SAGAW_5_LEVEL) { 476 max_num_iopt_levels = 5; 477 } else if (aw_bitmask & SAGAW_4_LEVEL) { 478 max_num_iopt_levels = 4; 479 } else if (aw_bitmask & SAGAW_3_LEVEL) { 480 max_num_iopt_levels = 3; 481 } else if (aw_bitmask & SAGAW_2_LEVEL) { 482 max_num_iopt_levels = 2; 483 } else { 484 printf("IOMMU: mismatch of supported number of PT levels between IOMMUs\n"); 485 return false; 486 } 487 488 if (aw_bitmask & SAGAW_3_LEVEL) { 489 x86KSnumIOPTLevels = 3; 490 } else if (aw_bitmask & SAGAW_4_LEVEL) { 491 x86KSnumIOPTLevels = 4; 492 } else if (aw_bitmask & SAGAW_5_LEVEL) { 493 x86KSnumIOPTLevels = 5; 494 } else if (aw_bitmask & SAGAW_6_LEVEL) { 495 x86KSnumIOPTLevels = 6; 496 } else if (aw_bitmask & SAGAW_2_LEVEL) { 497 x86KSnumIOPTLevels = 2; 498 } else { 499 printf("IOMMU: mismatch of supported number of PT levels between IOMMUs\n"); 500 return false; 501 } 502 503 printf("IOMMU: Using %d page-table levels (max. supported: %d)\n", x86KSnumIOPTLevels, max_num_iopt_levels); 504 505 vtd_create_root_table(); 506 507 for (bus = 0; bus < 256; bus++) { 508 vtd_create_context_table( 509 bus, 510 max_num_iopt_levels, 511 rmrr_list 512 ); 513 } 514 515 flushCacheRange(x86KSvtdRootTable, VTD_RT_SIZE_BITS); 516 517 if (!vtd_enable(cpu_id)) { 518 return false; 519 } 520 return true; 521} 522 523#endif /* CONFIG_IOMMU */ 524