1// Copyright 2018 The Fuchsia Authors 2// 3// Use of this source code is governed by a MIT-style 4// license that can be found in the LICENSE file or at 5// https://opensource.org/licenses/MIT 6 7#include "iommu_impl.h" 8 9#include <dev/interrupt.h> 10#include <err.h> 11#include <fbl/algorithm.h> 12#include <fbl/auto_lock.h> 13#include <fbl/limits.h> 14#include <fbl/ref_ptr.h> 15#include <fbl/unique_ptr.h> 16#include <platform.h> 17#include <trace.h> 18#include <vm/vm_aspace.h> 19#include <vm/vm_object_paged.h> 20#include <vm/vm_object_physical.h> 21#include <zircon/time.h> 22#include <zxcpp/new.h> 23 24#include "context_table_state.h" 25#include "device_context.h" 26#include "hw.h" 27 28#define LOCAL_TRACE 0 29 30namespace intel_iommu { 31 32IommuImpl::IommuImpl(volatile void* register_base, 33 fbl::unique_ptr<const uint8_t[]> desc, size_t desc_len) 34 : desc_(fbl::move(desc)), desc_len_(desc_len), mmio_(register_base) { 35 memset(&irq_block_, 0, sizeof(irq_block_)); 36 // desc_len_ is currently unused, but we stash it so we can use the length 37 // of it later in case we need it. This silences a warning in Clang. 38 desc_len_ = desc_len; 39} 40 41zx_status_t IommuImpl::Create(fbl::unique_ptr<const uint8_t[]> desc_bytes, size_t desc_len, 42 fbl::RefPtr<Iommu>* out) { 43 zx_status_t status = ValidateIommuDesc(desc_bytes, desc_len); 44 if (status != ZX_OK) { 45 return status; 46 } 47 48 auto desc = reinterpret_cast<const zx_iommu_desc_intel_t*>(desc_bytes.get()); 49 const uint64_t register_base = desc->register_base; 50 51 auto kernel_aspace = VmAspace::kernel_aspace(); 52 void *vaddr; 53 status = kernel_aspace->AllocPhysical( 54 "iommu", 55 PAGE_SIZE, 56 &vaddr, 57 PAGE_SIZE_SHIFT, 58 register_base, 59 0, 60 ARCH_MMU_FLAG_PERM_READ | ARCH_MMU_FLAG_PERM_WRITE | ARCH_MMU_FLAG_UNCACHED); 61 if (status != ZX_OK) { 62 return status; 63 } 64 65 fbl::AllocChecker ac; 66 auto instance = fbl::AdoptRef<IommuImpl>(new (&ac) IommuImpl(vaddr, fbl::move(desc_bytes), 67 desc_len)); 68 if (!ac.check()) { 69 kernel_aspace->FreeRegion(reinterpret_cast<vaddr_t>(vaddr)); 70 return ZX_ERR_NO_MEMORY; 71 } 72 73 status = instance->Initialize(); 74 if (status != ZX_OK) { 75 return status; 76 } 77 78 *out = fbl::move(instance); 79 return ZX_OK; 80} 81 82IommuImpl::~IommuImpl() { 83 fbl::AutoLock guard(&lock_); 84 85 // We cannot unpin memory until translation is disabled 86 zx_status_t status = SetTranslationEnableLocked(false, ZX_TIME_INFINITE); 87 ASSERT(status == ZX_OK); 88 89 DisableFaultsLocked(); 90 msi_free_block(&irq_block_); 91 92 VmAspace::kernel_aspace()->FreeRegion(mmio_.base()); 93} 94 95// Validate the IOMMU descriptor from userspace. 96// 97// The IOMMU descriptor identifies either a whitelist (if whole_segment is false) 98// or a blacklist (if whole_segment is true) of devices that are decoded by this 99// IOMMU. An entry in the list is described by a "scope" below. A scope 100// identifies a single PCIe device. If the device is behind a bridge, it will be 101// described using multiple "hops", one for each bridge in the way and one for 102// the device itself. A hop identifies the address of a bridge on the path to 103// the device, or (in the final entry) the address of the device itself. 104// 105// The descriptor also contains a list of "Reserved Memory Regions", which 106// describes regions of physical address space that must be identity-mapped for 107// specific devices to function correctly. There is typically one region for 108// the i915 gpu (initial framebuffer) and one for the XHCI controller 109// (scratch space for the BIOS before the OS takes ownership of the controller). 110zx_status_t IommuImpl::ValidateIommuDesc(const fbl::unique_ptr<const uint8_t[]>& desc_bytes, 111 size_t desc_len) { 112 auto desc = reinterpret_cast<const zx_iommu_desc_intel_t*>(desc_bytes.get()); 113 114 // Validate the size 115 if (desc_len < sizeof(*desc)) { 116 LTRACEF("desc too short: %zu < %zu\n", desc_len, sizeof(*desc)); 117 return ZX_ERR_INVALID_ARGS; 118 } 119 static_assert(sizeof(desc->scope_bytes) < sizeof(size_t), 120 "if this changes, need to check for overflow"); 121 size_t actual_size = sizeof(*desc); 122 if (add_overflow(actual_size, desc->scope_bytes, &actual_size) || 123 add_overflow(actual_size, desc->reserved_memory_bytes, &actual_size) || 124 actual_size != desc_len) { 125 126 LTRACEF("desc size mismatch: %zu != %zu\n", desc_len, actual_size); 127 return ZX_ERR_INVALID_ARGS; 128 } 129 130 // Validate scopes 131 if (desc->scope_bytes == 0 && !desc->whole_segment) { 132 LTRACEF("desc has no scopes\n"); 133 return ZX_ERR_INVALID_ARGS; 134 } 135 const size_t num_scopes = desc->scope_bytes / sizeof(zx_iommu_desc_intel_scope_t); 136 size_t scope_bytes = num_scopes; 137 if (mul_overflow(scope_bytes, sizeof(zx_iommu_desc_intel_scope_t), &scope_bytes) || 138 scope_bytes != desc->scope_bytes) { 139 140 LTRACEF("desc has invalid scope_bytes field\n"); 141 return ZX_ERR_INVALID_ARGS; 142 } 143 144 auto scopes = reinterpret_cast<zx_iommu_desc_intel_scope_t*>( 145 reinterpret_cast<uintptr_t>(desc) + sizeof(*desc)); 146 for (size_t i = 0; i < num_scopes; ++i) { 147 if (scopes[i].num_hops == 0) { 148 LTRACEF("desc scope %zu has no hops\n", i); 149 return ZX_ERR_INVALID_ARGS; 150 } 151 if (scopes[i].num_hops > fbl::count_of(scopes[0].dev_func)) { 152 LTRACEF("desc scope %zu has too many hops\n", i); 153 return ZX_ERR_INVALID_ARGS; 154 } 155 } 156 157 // Validate reserved memory regions 158 size_t cursor_bytes = sizeof(*desc) + desc->scope_bytes; 159 while (cursor_bytes + sizeof(zx_iommu_desc_intel_reserved_memory_t) < desc_len) { 160 auto mem = reinterpret_cast<zx_iommu_desc_intel_reserved_memory_t*>( 161 reinterpret_cast<uintptr_t>(desc) + cursor_bytes); 162 163 size_t next_entry = cursor_bytes; 164 if (add_overflow(next_entry, sizeof(zx_iommu_desc_intel_reserved_memory_t), &next_entry) || 165 add_overflow(next_entry, mem->scope_bytes, &next_entry) || 166 next_entry > desc_len) { 167 168 LTRACEF("desc reserved memory entry has invalid scope_bytes\n"); 169 return ZX_ERR_INVALID_ARGS; 170 } 171 172 // TODO(teisenbe): Make sure that the reserved memory regions are not in our 173 // allocatable RAM pools 174 175 // Validate scopes 176 if (mem->scope_bytes == 0) { 177 LTRACEF("desc reserved memory entry has no scopes\n"); 178 return ZX_ERR_INVALID_ARGS; 179 } 180 const size_t num_scopes = mem->scope_bytes / sizeof(zx_iommu_desc_intel_scope_t); 181 size_t scope_bytes = num_scopes; 182 if (mul_overflow(scope_bytes, sizeof(zx_iommu_desc_intel_scope_t), &scope_bytes) || 183 scope_bytes != desc->scope_bytes) { 184 185 LTRACEF("desc reserved memory entry has invalid scope_bytes field\n"); 186 return ZX_ERR_INVALID_ARGS; 187 } 188 189 auto scopes = reinterpret_cast<zx_iommu_desc_intel_scope_t*>( 190 reinterpret_cast<uintptr_t>(mem) + sizeof(*mem)); 191 for (size_t i = 0; i < num_scopes; ++i) { 192 if (scopes[i].num_hops == 0) { 193 LTRACEF("desc reserved memory entry scope %zu has no hops\n", i); 194 return ZX_ERR_INVALID_ARGS; 195 } 196 if (scopes[i].num_hops > fbl::count_of(scopes[0].dev_func)) { 197 LTRACEF("desc reserved memory entry scope %zu has too many hops\n", i); 198 return ZX_ERR_INVALID_ARGS; 199 } 200 } 201 202 cursor_bytes = next_entry; 203 } 204 if (cursor_bytes != desc_len) { 205 LTRACEF("desc has invalid reserved_memory_bytes field\n"); 206 return ZX_ERR_INVALID_ARGS; 207 } 208 209 LTRACEF("validated desc\n"); 210 return ZX_OK; 211} 212 213bool IommuImpl::IsValidBusTxnId(uint64_t bus_txn_id) const { 214 if (bus_txn_id > UINT16_MAX) { 215 return false; 216 } 217 218 ds::Bdf bdf = decode_bus_txn_id(bus_txn_id); 219 220 auto desc = reinterpret_cast<const zx_iommu_desc_intel_t*>(desc_.get()); 221 const size_t num_scopes = desc->scope_bytes / sizeof(zx_iommu_desc_intel_scope_t); 222 auto scopes = reinterpret_cast<zx_iommu_desc_intel_scope_t*>( 223 reinterpret_cast<uintptr_t>(desc) + sizeof(*desc)); 224 225 // Search for this BDF in the scopes we have 226 for (size_t i = 0; i < num_scopes; ++i) { 227 if (scopes[i].num_hops != 1) { 228 // TODO(teisenbe): Implement 229 continue; 230 } 231 232 if (scopes[i].start_bus == bdf.bus() && 233 scopes[i].dev_func[0] == bdf.packed_dev_and_func()) { 234 return !desc->whole_segment; 235 } 236 } 237 238 if (desc->whole_segment) { 239 // Since we only support single segment currently, just return true 240 // here. To support more segments, we need to make sure the segment 241 // matches, too. 242 return true; 243 } 244 245 return false; 246} 247 248zx_status_t IommuImpl::Map(uint64_t bus_txn_id, const fbl::RefPtr<VmObject>& vmo, 249 uint64_t offset, size_t size, uint32_t perms, 250 dev_vaddr_t* vaddr, size_t* mapped_len) { 251 DEBUG_ASSERT(vaddr); 252 if (!IS_PAGE_ALIGNED(offset) || size == 0) { 253 return ZX_ERR_INVALID_ARGS; 254 } 255 if (perms & ~(IOMMU_FLAG_PERM_READ | IOMMU_FLAG_PERM_WRITE | IOMMU_FLAG_PERM_EXECUTE)) { 256 return ZX_ERR_INVALID_ARGS; 257 } 258 if (perms == 0) { 259 return ZX_ERR_INVALID_ARGS; 260 } 261 if (!IsValidBusTxnId(bus_txn_id)) { 262 return ZX_ERR_NOT_FOUND; 263 } 264 265 ds::Bdf bdf = decode_bus_txn_id(bus_txn_id); 266 267 fbl::AutoLock guard(&lock_); 268 DeviceContext* dev; 269 zx_status_t status = GetOrCreateDeviceContextLocked(bdf, &dev); 270 if (status != ZX_OK) { 271 return status; 272 } 273 return dev->SecondLevelMap(vmo, offset, size, perms, false /* map_contiguous */, 274 vaddr, mapped_len); 275} 276 277zx_status_t IommuImpl::MapContiguous(uint64_t bus_txn_id, const fbl::RefPtr<VmObject>& vmo, 278 uint64_t offset, size_t size, uint32_t perms, 279 dev_vaddr_t* vaddr, size_t* mapped_len) { 280 DEBUG_ASSERT(vaddr); 281 if (!IS_PAGE_ALIGNED(offset) || size == 0) { 282 return ZX_ERR_INVALID_ARGS; 283 } 284 if (perms & ~(IOMMU_FLAG_PERM_READ | IOMMU_FLAG_PERM_WRITE | IOMMU_FLAG_PERM_EXECUTE)) { 285 return ZX_ERR_INVALID_ARGS; 286 } 287 if (perms == 0) { 288 return ZX_ERR_INVALID_ARGS; 289 } 290 if (!IsValidBusTxnId(bus_txn_id)) { 291 return ZX_ERR_NOT_FOUND; 292 } 293 294 ds::Bdf bdf = decode_bus_txn_id(bus_txn_id); 295 296 fbl::AutoLock guard(&lock_); 297 DeviceContext* dev; 298 zx_status_t status = GetOrCreateDeviceContextLocked(bdf, &dev); 299 if (status != ZX_OK) { 300 return status; 301 } 302 return dev->SecondLevelMap(vmo, offset, size, perms, true /* map_contiguous */, 303 vaddr, mapped_len); 304} 305 306zx_status_t IommuImpl::Unmap(uint64_t bus_txn_id, dev_vaddr_t vaddr, size_t size) { 307 if (!IS_PAGE_ALIGNED(vaddr) || !IS_PAGE_ALIGNED(size)) { 308 return ZX_ERR_INVALID_ARGS; 309 } 310 if (!IsValidBusTxnId(bus_txn_id)) { 311 return ZX_ERR_NOT_FOUND; 312 } 313 314 ds::Bdf bdf = decode_bus_txn_id(bus_txn_id); 315 316 fbl::AutoLock guard(&lock_); 317 DeviceContext* dev; 318 zx_status_t status = GetOrCreateDeviceContextLocked(bdf, &dev); 319 if (status != ZX_OK) { 320 return status; 321 } 322 status = dev->SecondLevelUnmap(vaddr, size); 323 if (status != ZX_OK) { 324 return status; 325 } 326 327 return ZX_OK; 328} 329 330zx_status_t IommuImpl::ClearMappingsForBusTxnId(uint64_t bus_txn_id) { 331 PANIC_UNIMPLEMENTED; 332 return ZX_ERR_NOT_SUPPORTED; 333} 334 335zx_status_t IommuImpl::Initialize() { 336 fbl::AutoLock guard(&lock_); 337 338 // Ensure we support this device version 339 auto version = reg::Version::Get().ReadFrom(&mmio_); 340 if (version.major() != 1 && version.minor() != 0) { 341 LTRACEF("Unsupported IOMMU version: %u.%u\n", version.major(), version.minor()); 342 return ZX_ERR_NOT_SUPPORTED; 343 } 344 345 // Cache useful capability info 346 caps_ = reg::Capability::Get().ReadFrom(&mmio_); 347 extended_caps_ = reg::ExtendedCapability::Get().ReadFrom(&mmio_); 348 349 max_guest_addr_mask_ = (1ULL << (caps_.max_guest_addr_width() + 1)) - 1; 350 fault_recording_reg_offset_ = static_cast<uint32_t>( 351 caps_.fault_recording_register_offset() * 16); 352 num_fault_recording_reg_ = static_cast<uint32_t>(caps_.num_fault_recording_reg() + 1); 353 iotlb_reg_offset_ = static_cast<uint32_t>(extended_caps_.iotlb_register_offset() * 16); 354 355 constexpr size_t kIoTlbRegisterBankSize = 16; 356 if (iotlb_reg_offset_ > PAGE_SIZE - kIoTlbRegisterBankSize) { 357 LTRACEF("Unsupported IOMMU: IOTLB offset runs past the register page\n"); 358 return ZX_ERR_NOT_SUPPORTED; 359 } 360 supports_extended_context_ = extended_caps_.supports_extended_context(); 361 if (extended_caps_.supports_pasid()) { 362 valid_pasid_mask_ = static_cast<uint32_t>((1ULL << (extended_caps_.pasid_size() + 1)) - 1); 363 } 364 365 const uint64_t num_domains_raw = caps_.num_domains(); 366 if (num_domains_raw > 0x6) { 367 LTRACEF("Unknown num_domains value\n"); 368 return ZX_ERR_NOT_SUPPORTED; 369 } 370 const uint32_t num_supported_domains = static_cast<uint32_t>(1ul << (4 + 2 * num_domains_raw)); 371 domain_allocator_.set_num_domains(num_supported_domains); 372 373 // Sanity check initial configuration 374 auto global_ctl = reg::GlobalControl::Get().ReadFrom(&mmio_); 375 if (global_ctl.translation_enable()) { 376 LTRACEF("DMA remapping already enabled?!\n"); 377 return ZX_ERR_BAD_STATE; 378 } 379 if (global_ctl.interrupt_remap_enable()) { 380 LTRACEF("IRQ remapping already enabled?!\n"); 381 return ZX_ERR_BAD_STATE; 382 } 383 384 // Allocate and setup the root table 385 zx_status_t status = IommuPage::AllocatePage(&root_table_page_); 386 if (status != ZX_OK) { 387 LTRACEF("alloc root table failed\n"); 388 return status; 389 } 390 status = SetRootTablePointerLocked(root_table_page_.paddr()); 391 if (status != ZX_OK) { 392 LTRACEF("set root table failed\n"); 393 return status; 394 } 395 396 // Enable interrupts before we enable translation 397 status = ConfigureFaultEventInterruptLocked(); 398 if (status != ZX_OK) { 399 LTRACEF("configuring fault event irq failed\n"); 400 return status; 401 } 402 403 status = EnableBiosReservedMappingsLocked(); 404 if (status != ZX_OK) { 405 LTRACEF("enable bios reserved mappings failed\n"); 406 return status; 407 } 408 409 status = SetTranslationEnableLocked(true, zx_time_add_duration(current_time(), ZX_SEC(1))); 410 if (status != ZX_OK) { 411 LTRACEF("set translation enable failed\n"); 412 return status; 413 } 414 415 return ZX_OK; 416} 417 418zx_status_t IommuImpl::EnableBiosReservedMappingsLocked() { 419 auto desc = reinterpret_cast<const zx_iommu_desc_intel_t*>(desc_.get()); 420 421 size_t cursor_bytes = 0; 422 while (cursor_bytes + sizeof(zx_iommu_desc_intel_reserved_memory_t) < desc->reserved_memory_bytes) { 423 // The descriptor has already been validated, so no need to check again. 424 auto mem = reinterpret_cast<zx_iommu_desc_intel_reserved_memory_t*>( 425 reinterpret_cast<uintptr_t>(desc) + sizeof(*desc) + desc->scope_bytes + 426 cursor_bytes); 427 428 const size_t num_scopes = mem->scope_bytes / sizeof(zx_iommu_desc_intel_scope_t); 429 auto scopes = reinterpret_cast<zx_iommu_desc_intel_scope_t*>( 430 reinterpret_cast<uintptr_t>(mem) + sizeof(*mem)); 431 for (size_t i = 0; i < num_scopes; ++i) { 432 if (scopes[i].num_hops != 1) { 433 // TODO(teisenbe): Implement 434 return ZX_ERR_NOT_SUPPORTED; 435 } 436 437 ds::Bdf bdf; 438 bdf.set_bus(scopes[i].start_bus); 439 bdf.set_dev(static_cast<uint8_t>(scopes[i].dev_func[0] >> 3)); 440 bdf.set_func(static_cast<uint8_t>(scopes[i].dev_func[0] & 0x7)); 441 442 DeviceContext* dev; 443 zx_status_t status = GetOrCreateDeviceContextLocked(bdf, &dev); 444 if (status != ZX_OK) { 445 return status; 446 } 447 448 LTRACEF("Enabling region [%lx, %lx) for %02x:%02x.%02x\n", mem->base_addr, 449 mem->base_addr + mem->len, bdf.bus(), bdf.dev(), bdf.func()); 450 size_t size = ROUNDUP(mem->len, PAGE_SIZE); 451 const uint32_t perms = IOMMU_FLAG_PERM_READ | IOMMU_FLAG_PERM_WRITE; 452 status = dev->SecondLevelMapIdentity(mem->base_addr, size, perms); 453 if (status != ZX_OK) { 454 return status; 455 } 456 } 457 458 cursor_bytes += sizeof(*mem) + mem->scope_bytes; 459 } 460 461 return ZX_OK; 462} 463 464// Sets the root table pointer and invalidates the context-cache and IOTLB. 465zx_status_t IommuImpl::SetRootTablePointerLocked(paddr_t pa) { 466 DEBUG_ASSERT(IS_PAGE_ALIGNED(pa)); 467 468 auto root_table_addr = reg::RootTableAddress::Get().FromValue(0); 469 // If we support extended contexts, use it. 470 root_table_addr.set_root_table_type(supports_extended_context_); 471 root_table_addr.set_root_table_address(pa >> PAGE_SIZE_SHIFT); 472 root_table_addr.WriteTo(&mmio_); 473 474 auto global_ctl = reg::GlobalControl::Get().ReadFrom(&mmio_); 475 DEBUG_ASSERT(!global_ctl.translation_enable()); 476 global_ctl.set_root_table_ptr(1); 477 global_ctl.WriteTo(&mmio_); 478 zx_status_t status = WaitForValueLocked(&global_ctl, &decltype(global_ctl)::root_table_ptr, 479 1, zx_time_add_duration(current_time(), ZX_SEC(1))); 480 if (status != ZX_OK) { 481 LTRACEF("Timed out waiting for root_table_ptr bit to take\n"); 482 return status; 483 } 484 485 InvalidateContextCacheGlobalLocked(); 486 InvalidateIotlbGlobalLocked(); 487 488 return ZX_OK; 489} 490 491zx_status_t IommuImpl::SetTranslationEnableLocked(bool enabled, zx_time_t deadline) { 492 auto global_ctl = reg::GlobalControl::Get().ReadFrom(&mmio_); 493 global_ctl.set_translation_enable(enabled); 494 global_ctl.WriteTo(&mmio_); 495 496 return WaitForValueLocked(&global_ctl, &decltype(global_ctl)::translation_enable, 497 enabled, deadline); 498} 499 500void IommuImpl::InvalidateContextCacheGlobalLocked() { 501 DEBUG_ASSERT(lock_.IsHeld()); 502 503 auto context_cmd = reg::ContextCommand::Get().FromValue(0); 504 context_cmd.set_invld_context_cache(1); 505 context_cmd.set_invld_request_granularity(reg::ContextCommand::kGlobalInvld); 506 context_cmd.WriteTo(&mmio_); 507 508 WaitForValueLocked(&context_cmd, &decltype(context_cmd)::invld_context_cache, 0, 509 ZX_TIME_INFINITE); 510} 511 512void IommuImpl::InvalidateContextCacheDomainLocked(uint32_t domain_id) { 513 DEBUG_ASSERT(lock_.IsHeld()); 514 515 auto context_cmd = reg::ContextCommand::Get().FromValue(0); 516 context_cmd.set_invld_context_cache(1); 517 context_cmd.set_invld_request_granularity(reg::ContextCommand::kDomainInvld); 518 context_cmd.set_domain_id(domain_id); 519 context_cmd.WriteTo(&mmio_); 520 521 WaitForValueLocked(&context_cmd, &decltype(context_cmd)::invld_context_cache, 0, 522 ZX_TIME_INFINITE); 523} 524 525void IommuImpl::InvalidateContextCacheGlobal() { 526 fbl::AutoLock guard(&lock_); 527 InvalidateContextCacheGlobalLocked(); 528} 529 530void IommuImpl::InvalidateContextCacheDomain(uint32_t domain_id) { 531 fbl::AutoLock guard(&lock_); 532 InvalidateContextCacheDomainLocked(domain_id); 533} 534 535void IommuImpl::InvalidateIotlbGlobalLocked() { 536 DEBUG_ASSERT(lock_.IsHeld()); 537 ASSERT(!caps_.required_write_buf_flushing()); 538 539 // TODO(teisenbe): Read/write draining? 540 auto iotlb_invld = reg::IotlbInvalidate::Get(iotlb_reg_offset_).ReadFrom(&mmio_); 541 iotlb_invld.set_invld_iotlb(1); 542 iotlb_invld.set_invld_request_granularity(reg::IotlbInvalidate::kGlobalInvld); 543 iotlb_invld.WriteTo(&mmio_); 544 545 WaitForValueLocked(&iotlb_invld, &decltype(iotlb_invld)::invld_iotlb, 0, 546 ZX_TIME_INFINITE); 547} 548 549void IommuImpl::InvalidateIotlbDomainAllLocked(uint32_t domain_id) { 550 DEBUG_ASSERT(lock_.IsHeld()); 551 ASSERT(!caps_.required_write_buf_flushing()); 552 553 // TODO(teisenbe): Read/write draining? 554 auto iotlb_invld = reg::IotlbInvalidate::Get(iotlb_reg_offset_).ReadFrom(&mmio_); 555 iotlb_invld.set_invld_iotlb(1); 556 iotlb_invld.set_invld_request_granularity(reg::IotlbInvalidate::kDomainAllInvld); 557 iotlb_invld.set_domain_id(domain_id); 558 iotlb_invld.WriteTo(&mmio_); 559 560 WaitForValueLocked(&iotlb_invld, &decltype(iotlb_invld)::invld_iotlb, 0, 561 ZX_TIME_INFINITE); 562} 563 564void IommuImpl::InvalidateIotlbPageLocked(uint32_t domain_id, dev_vaddr_t vaddr, uint pages_pow2) { 565 DEBUG_ASSERT(lock_.IsHeld()); 566 DEBUG_ASSERT(IS_PAGE_ALIGNED(vaddr)); 567 DEBUG_ASSERT(pages_pow2 < 64); 568 DEBUG_ASSERT(pages_pow2 <= caps_.max_addr_mask_value()); 569 ASSERT(!caps_.required_write_buf_flushing()); 570 571 auto invld_addr = reg::InvalidateAddress::Get(iotlb_reg_offset_).FromValue(0); 572 invld_addr.set_address(vaddr >> 12); 573 invld_addr.set_invld_hint(0); 574 invld_addr.set_address_mask(pages_pow2); 575 invld_addr.WriteTo(&mmio_); 576 577 // TODO(teisenbe): Read/write draining? 578 auto iotlb_invld = reg::IotlbInvalidate::Get(iotlb_reg_offset_).ReadFrom(&mmio_); 579 iotlb_invld.set_invld_iotlb(1); 580 iotlb_invld.set_invld_request_granularity(reg::IotlbInvalidate::kDomainPageInvld); 581 iotlb_invld.set_domain_id(domain_id); 582 iotlb_invld.WriteTo(&mmio_); 583 584 WaitForValueLocked(&iotlb_invld, &decltype(iotlb_invld)::invld_iotlb, 0, 585 ZX_TIME_INFINITE); 586} 587 588void IommuImpl::InvalidateIotlbGlobal() { 589 fbl::AutoLock guard(&lock_); 590 InvalidateIotlbGlobalLocked(); 591} 592 593void IommuImpl::InvalidateIotlbDomainAll(uint32_t domain_id) { 594 fbl::AutoLock guard(&lock_); 595 InvalidateIotlbDomainAllLocked(domain_id); 596} 597 598template <class RegType> 599zx_status_t IommuImpl::WaitForValueLocked(RegType* reg, 600 typename RegType::ValueType (RegType::*getter)() const, 601 typename RegType::ValueType value, 602 zx_time_t deadline) { 603 DEBUG_ASSERT(lock_.IsHeld()); 604 605 const zx_time_t kMaxSleepDuration = ZX_USEC(10); 606 607 while (true) { 608 // Read the register and check if it matches the expected value. If 609 // not, sleep for a bit and try again. 610 reg->ReadFrom(&mmio_); 611 if ((reg->*getter)() == value) { 612 return ZX_OK; 613 } 614 615 const zx_time_t now = current_time(); 616 if (now > deadline) { 617 break; 618 } 619 620 zx_time_t sleep_deadline = fbl::min(zx_time_add_duration(now, kMaxSleepDuration), deadline); 621 thread_sleep(sleep_deadline); 622 } 623 return ZX_ERR_TIMED_OUT; 624} 625 626void IommuImpl::FaultHandler(void* ctx) { 627 auto self = static_cast<IommuImpl*>(ctx); 628 auto status = reg::FaultStatus::Get().ReadFrom(&self->mmio_); 629 630 if (!status.primary_pending_fault()) { 631 TRACEF("Non primary fault\n"); 632 return; 633 } 634 635 auto caps = reg::Capability::Get().ReadFrom(&self->mmio_); 636 const uint32_t num_regs = static_cast<uint32_t>(caps.num_fault_recording_reg() + 1); 637 const uint32_t reg_offset = static_cast<uint32_t>(caps.fault_recording_register_offset() * 16); 638 639 uint32_t index = status.fault_record_index(); 640 while (1) { 641 auto rec_high = reg::FaultRecordHigh::Get(reg_offset, index).ReadFrom(&self->mmio_); 642 if (!rec_high.fault()) { 643 break; 644 } 645 auto rec_low = reg::FaultRecordLow::Get(reg_offset, index).ReadFrom(&self->mmio_); 646 uint64_t source = rec_high.source_id(); 647 TRACEF("IOMMU Fault: access %c, PASID (%c) %#04lx, reason %#02lx, source %02lx:%02lx.%lx, info: %lx\n", 648 rec_high.request_type() ? 'R' : 'W', 649 rec_high.pasid_present() ? 'V' : '-', 650 rec_high.pasid_value(), 651 rec_high.fault_reason(), 652 source >> 8, (source >> 3) & 0x1f, source & 0x7, 653 rec_low.fault_info() << 12); 654 655 // Clear this fault (RW1CS) 656 rec_high.WriteTo(&self->mmio_); 657 658 ++index; 659 if (index >= num_regs) { 660 index -= num_regs; 661 } 662 } 663 664 status.set_reg_value(0); 665 // Clear the primary fault overflow condition (RW1CS) 666 // TODO(teisenbe): How do we guarantee we get an interrupt on the next fault/if we left a fault unprocessed? 667 status.set_primary_fault_overflow(1); 668 status.WriteTo(&self->mmio_); 669} 670 671zx_status_t IommuImpl::ConfigureFaultEventInterruptLocked() { 672 DEBUG_ASSERT(lock_.IsHeld()); 673 674 if (!msi_is_supported()) { 675 return ZX_ERR_NOT_SUPPORTED; 676 } 677 zx_status_t status = msi_alloc_block(1, false/* can_target_64bit */, 678 false /* msi x */, &irq_block_); 679 if (status != ZX_OK) { 680 return status; 681 } 682 683 auto event_data = reg::FaultEventData::Get().FromValue(irq_block_.tgt_data); 684 auto event_addr = reg::FaultEventAddress::Get().FromValue( 685 static_cast<uint32_t>(irq_block_.tgt_addr)); 686 auto event_upper_addr = reg::FaultEventUpperAddress::Get().FromValue( 687 static_cast<uint32_t>(irq_block_.tgt_addr >> 32)); 688 689 event_data.WriteTo(&mmio_); 690 event_addr.WriteTo(&mmio_); 691 event_upper_addr.WriteTo(&mmio_); 692 693 // Clear all primary fault records 694 for (uint32_t i = 0; i < num_fault_recording_reg_; ++i) { 695 const uint32_t offset = fault_recording_reg_offset_; 696 auto record_high = reg::FaultRecordHigh::Get(offset, i).ReadFrom(&mmio_); 697 record_high.WriteTo(&mmio_); 698 } 699 700 // Clear all pending faults 701 auto fault_status_ctl = reg::FaultStatus::Get().ReadFrom(&mmio_); 702 fault_status_ctl.WriteTo(&mmio_); 703 704 msi_register_handler(&irq_block_, 0, FaultHandler, this); 705 706 // Unmask interrupts 707 auto fault_event_ctl = reg::FaultEventControl::Get().ReadFrom(&mmio_); 708 fault_event_ctl.set_interrupt_mask(0); 709 fault_event_ctl.WriteTo(&mmio_); 710 711 return ZX_OK; 712} 713 714void IommuImpl::DisableFaultsLocked() { 715 auto fault_event_ctl = reg::FaultEventControl::Get().ReadFrom(&mmio_); 716 fault_event_ctl.set_interrupt_mask(1); 717 fault_event_ctl.WriteTo(&mmio_); 718} 719 720zx_status_t IommuImpl::GetOrCreateContextTableLocked(ds::Bdf bdf, ContextTableState** tbl) { 721 DEBUG_ASSERT(lock_.IsHeld()); 722 723 volatile ds::RootTable* root_table = this->root_table(); 724 DEBUG_ASSERT(root_table); 725 726 volatile ds::RootEntrySubentry* target_entry = &root_table->entry[bdf.bus()].lower; 727 if (supports_extended_context_ && bdf.dev() >= 16) { 728 // If this is an extended root table and the device is in the upper half 729 // of the bus address space, use the upper pointer. 730 target_entry = &root_table->entry[bdf.bus()].upper; 731 } 732 733 ds::RootEntrySubentry entry; 734 entry.ReadFrom(target_entry); 735 if (entry.present()) { 736 // We know the entry exists, so search our list of tables for it. 737 for (ContextTableState& context_table : context_tables_) { 738 if (context_table.includes_bdf(bdf)) { 739 *tbl = &context_table; 740 return ZX_OK; 741 } 742 } 743 } 744 745 // Couldn't find the ContextTable, so create it. 746 fbl::unique_ptr<ContextTableState> table; 747 zx_status_t status = ContextTableState::Create(static_cast<uint8_t>(bdf.bus()), 748 supports_extended_context_, 749 bdf.dev() >= 16 /* upper */, 750 this, target_entry, &table); 751 if (status != ZX_OK) { 752 return status; 753 } 754 755 *tbl = table.get(); 756 context_tables_.push_back(fbl::move(table)); 757 758 return ZX_OK; 759} 760 761zx_status_t IommuImpl::GetOrCreateDeviceContextLocked(ds::Bdf bdf, DeviceContext** context) { 762 DEBUG_ASSERT(lock_.IsHeld()); 763 764 ContextTableState* ctx_table_state; 765 zx_status_t status = GetOrCreateContextTableLocked(bdf, &ctx_table_state); 766 if (status != ZX_OK) { 767 return status; 768 } 769 770 status = ctx_table_state->GetDeviceContext(bdf, context); 771 if (status != ZX_ERR_NOT_FOUND) { 772 // Either status was ZX_OK and we're done, or some error occurred. 773 return status; 774 } 775 776 uint32_t domain_id; 777 status = domain_allocator_.Allocate(&domain_id); 778 if (status != ZX_OK) { 779 return status; 780 } 781 return ctx_table_state->CreateDeviceContext(bdf, domain_id, context); 782} 783 784uint64_t IommuImpl::minimum_contiguity(uint64_t bus_txn_id) { 785 if (!IsValidBusTxnId(bus_txn_id)) { 786 return 0; 787 } 788 789 ds::Bdf bdf = decode_bus_txn_id(bus_txn_id); 790 791 fbl::AutoLock guard(&lock_); 792 DeviceContext* dev; 793 zx_status_t status = GetOrCreateDeviceContextLocked(bdf, &dev); 794 if (status != ZX_OK) { 795 return status; 796 } 797 798 return dev->minimum_contiguity(); 799} 800 801uint64_t IommuImpl::aspace_size(uint64_t bus_txn_id) { 802 if (!IsValidBusTxnId(bus_txn_id)) { 803 return 0; 804 } 805 806 ds::Bdf bdf = decode_bus_txn_id(bus_txn_id); 807 808 fbl::AutoLock guard(&lock_); 809 DeviceContext* dev; 810 zx_status_t status = GetOrCreateDeviceContextLocked(bdf, &dev); 811 if (status != ZX_OK) { 812 return status; 813 } 814 815 return dev->aspace_size(); 816} 817 818} // namespace intel_iommu 819