1/* 2 * Copyright 2014, General Dynamics C4 Systems 3 * 4 * SPDX-License-Identifier: GPL-2.0-only 5 */ 6 7#include <config.h> 8 9#ifdef CONFIG_IOMMU 10 11#include <api/syscall.h> 12#include <machine/io.h> 13#include <kernel/thread.h> 14#include <arch/api/invocation.h> 15#include <arch/object/iospace.h> 16#include <arch/model/statedata.h> 17#include <linker.h> 18#include <plat/machine/intel-vtd.h> 19 20 21typedef struct lookupVTDContextSlot_ret { 22 vtd_cte_t *cte; 23 word_t index; 24} lookupVTDContextSlot_ret_t; 25 26 27BOOT_CODE cap_t master_iospace_cap(void) 28{ 29 if (x86KSnumDrhu == 0) { 30 return cap_null_cap_new(); 31 } 32 33 return 34 cap_io_space_cap_new( 35 0, /* capDomainID */ 36 0 /* capPCIDevice */ 37 ); 38} 39 40static vtd_cte_t *lookup_vtd_context_slot(cap_t cap) 41{ 42 uint32_t vtd_root_index; 43 uint32_t vtd_context_index; 44 uint32_t pci_request_id; 45 vtd_rte_t *vtd_root_slot; 46 vtd_cte_t *vtd_context; 47 vtd_cte_t *vtd_context_slot; 48 49 switch (cap_get_capType(cap)) { 50 case cap_io_space_cap: 51 pci_request_id = cap_io_space_cap_get_capPCIDevice(cap); 52 break; 53 54 case cap_io_page_table_cap: 55 pci_request_id = cap_io_page_table_cap_get_capIOPTIOASID(cap); 56 break; 57 58 case cap_frame_cap: 59 pci_request_id = cap_frame_cap_get_capFMappedASID(cap); 60 break; 61 62 default: 63 fail("Invalid cap type"); 64 } 65 66 vtd_root_index = get_pci_bus(pci_request_id); 67 vtd_root_slot = x86KSvtdRootTable + vtd_root_index; 68 69 vtd_context = (vtd_cte_t *)paddr_to_pptr(vtd_rte_ptr_get_ctp(vtd_root_slot)); 70 vtd_context_index = (get_pci_dev(pci_request_id) << 3) | get_pci_fun(pci_request_id); 71 vtd_context_slot = &vtd_context[vtd_context_index]; 72 73 return vtd_context_slot; 74} 75 76static lookupIOPTSlot_ret_t lookupIOPTSlot_resolve_levels(vtd_pte_t *iopt, word_t translation, 77 word_t levels_to_resolve, word_t levels_remaining) 78{ 79 lookupIOPTSlot_ret_t ret; 80 81 word_t iopt_index = 0; 82 vtd_pte_t *iopt_slot = 0; 83 vtd_pte_t *next_iopt_slot = 0; 84 85 if (iopt == 0) { 86 ret.ioptSlot = 0; 87 ret.level = levels_remaining; 88 ret.status = EXCEPTION_LOOKUP_FAULT; 89 return ret; 90 } 91 92 iopt_index = (translation >> (VTD_PT_INDEX_BITS * (x86KSnumIOPTLevels - 1 - (levels_to_resolve - levels_remaining)))) & 93 MASK(VTD_PT_INDEX_BITS); 94 iopt_slot = iopt + iopt_index; 95 96 if (!vtd_pte_ptr_get_write(iopt_slot) || levels_remaining == 0) { 97 ret.ioptSlot = iopt_slot; 98 ret.level = levels_remaining; 99 ret.status = EXCEPTION_NONE; 100 return ret; 101 } 102 next_iopt_slot = (vtd_pte_t *)paddr_to_pptr(vtd_pte_ptr_get_addr(iopt_slot)); 103 return lookupIOPTSlot_resolve_levels(next_iopt_slot, translation, levels_to_resolve, levels_remaining - 1); 104} 105 106 107static inline lookupIOPTSlot_ret_t lookupIOPTSlot(vtd_pte_t *iopt, word_t io_address) 108{ 109 lookupIOPTSlot_ret_t ret; 110 111 if (iopt == 0) { 112 ret.ioptSlot = 0; 113 ret.level = 0; 114 ret.status = EXCEPTION_LOOKUP_FAULT; 115 return ret; 116 } else { 117 return lookupIOPTSlot_resolve_levels(iopt, io_address >> PAGE_BITS, 118 x86KSnumIOPTLevels - 1, x86KSnumIOPTLevels - 1); 119 } 120} 121 122void unmapVTDContextEntry(cap_t cap) 123{ 124 vtd_cte_t *cte = lookup_vtd_context_slot(cap); 125 assert(cte != 0); 126 *cte = vtd_cte_new( 127 0, 128 false, 129 0, 130 0, 131 0, 132 false 133 ); 134 135 flushCacheRange(cte, VTD_CTE_SIZE_BITS); 136 invalidate_iotlb(); 137 setThreadState(NODE_STATE(ksCurThread), ThreadState_Restart); 138 return; 139} 140 141static exception_t performX86IOPTInvocationUnmap(cap_t cap, cte_t *ctSlot) 142{ 143 deleteIOPageTable(cap); 144 cap = cap_io_page_table_cap_set_capIOPTIsMapped(cap, 0); 145 ctSlot->cap = cap; 146 147 return EXCEPTION_NONE; 148} 149 150static exception_t performX86IOPTInvocationMapContextRoot(cap_t cap, cte_t *ctSlot, vtd_cte_t vtd_cte, 151 vtd_cte_t *vtd_context_slot) 152{ 153 *vtd_context_slot = vtd_cte; 154 flushCacheRange(vtd_context_slot, VTD_CTE_SIZE_BITS); 155 ctSlot->cap = cap; 156 157 return EXCEPTION_NONE; 158} 159 160static exception_t performX86IOPTInvocationMapPT(cap_t cap, cte_t *ctSlot, vtd_pte_t iopte, vtd_pte_t *ioptSlot) 161{ 162 *ioptSlot = iopte; 163 flushCacheRange(ioptSlot, VTD_PTE_SIZE_BITS); 164 ctSlot->cap = cap; 165 166 return EXCEPTION_NONE; 167} 168 169exception_t decodeX86IOPTInvocation( 170 word_t invLabel, 171 word_t length, 172 cte_t *slot, 173 cap_t cap, 174 extra_caps_t excaps, 175 word_t *buffer 176) 177{ 178 cap_t io_space; 179 paddr_t paddr; 180 uint32_t pci_request_id; 181 word_t io_address; 182 uint16_t domain_id; 183 vtd_cte_t *vtd_context_slot; 184 vtd_pte_t *vtd_pte; 185 186 if (invLabel == X86IOPageTableUnmap) { 187 188 setThreadState(NODE_STATE(ksCurThread), ThreadState_Restart); 189 return performX86IOPTInvocationUnmap(cap, slot); 190 } 191 192 if (invLabel != X86IOPageTableMap) { 193 userError("X86IOPageTable: Illegal operation."); 194 current_syscall_error.type = seL4_IllegalOperation; 195 return EXCEPTION_SYSCALL_ERROR; 196 } 197 198 if (excaps.excaprefs[0] == NULL || length < 1) { 199 userError("X86IOPageTableMap: Truncated message."); 200 current_syscall_error.type = seL4_TruncatedMessage; 201 return EXCEPTION_SYSCALL_ERROR; 202 } 203 204 io_space = excaps.excaprefs[0]->cap; 205 io_address = getSyscallArg(0, buffer) & ~MASK(VTD_PT_INDEX_BITS + seL4_PageBits); 206 207 if (cap_io_page_table_cap_get_capIOPTIsMapped(cap)) { 208 userError("X86IOPageTableMap: IO page table is already mapped."); 209 current_syscall_error.type = seL4_InvalidCapability; 210 current_syscall_error.invalidCapNumber = 0; 211 return EXCEPTION_SYSCALL_ERROR; 212 } 213 214 if (cap_get_capType(io_space) != cap_io_space_cap) { 215 userError("X86IOPageTableMap: Invalid IO space capability."); 216 current_syscall_error.type = seL4_InvalidCapability; 217 current_syscall_error.invalidCapNumber = 0; 218 return EXCEPTION_SYSCALL_ERROR; 219 } 220 221 pci_request_id = cap_io_space_cap_get_capPCIDevice(io_space); 222 domain_id = cap_io_space_cap_get_capDomainID(io_space); 223 if (pci_request_id == asidInvalid) { 224 current_syscall_error.type = seL4_InvalidCapability; 225 current_syscall_error.invalidCapNumber = 0; 226 227 return EXCEPTION_SYSCALL_ERROR; 228 } 229 230 paddr = pptr_to_paddr(VTD_PTE_PTR(cap_io_page_table_cap_get_capIOPTBasePtr(cap))); 231 vtd_context_slot = lookup_vtd_context_slot(io_space); 232 233 if (!vtd_cte_ptr_get_present(vtd_context_slot)) { 234 235 /* 1st Level Page Table */ 236 vtd_cte_t vtd_cte = vtd_cte_new( 237 domain_id, /* domain ID */ 238 false, /* RMRR */ 239 x86KSnumIOPTLevels - 2, /* addr width (x = levels - 2) */ 240 paddr, /* address space root */ 241 0, /* translation type */ 242 true /* present */ 243 ); 244 245 cap = cap_io_page_table_cap_set_capIOPTIsMapped(cap, 1); 246 cap = cap_io_page_table_cap_set_capIOPTLevel(cap, 0); 247 cap = cap_io_page_table_cap_set_capIOPTIOASID(cap, pci_request_id); 248 249 setThreadState(NODE_STATE(ksCurThread), ThreadState_Restart); 250 return performX86IOPTInvocationMapContextRoot(cap, slot, vtd_cte, vtd_context_slot); 251 } else { 252 lookupIOPTSlot_ret_t lu_ret; 253 vtd_pte_t iopte; 254 255 vtd_pte = (vtd_pte_t *)paddr_to_pptr(vtd_cte_ptr_get_asr(vtd_context_slot)); 256 lu_ret = lookupIOPTSlot(vtd_pte, io_address); 257 258 if (lu_ret.status != EXCEPTION_NONE) { 259 current_syscall_error.type = seL4_FailedLookup; 260 current_syscall_error.failedLookupWasSource = false; 261 return EXCEPTION_SYSCALL_ERROR; 262 } 263 264 lu_ret.level = x86KSnumIOPTLevels - lu_ret.level; 265 if (vtd_pte_ptr_get_addr(lu_ret.ioptSlot) != 0) { 266 current_syscall_error.type = seL4_DeleteFirst; 267 268 return EXCEPTION_SYSCALL_ERROR; 269 } 270 271 iopte = vtd_pte_new( 272 paddr, /* physical addr */ 273 1, /* write permission flag */ 274 1 /* read permission flag */ 275 ); 276 277 cap = cap_io_page_table_cap_set_capIOPTIsMapped(cap, 1); 278 cap = cap_io_page_table_cap_set_capIOPTLevel(cap, lu_ret.level); 279 cap = cap_io_page_table_cap_set_capIOPTIOASID(cap, pci_request_id); 280 cap = cap_io_page_table_cap_set_capIOPTMappedAddress(cap, io_address); 281 282 setThreadState(NODE_STATE(ksCurThread), ThreadState_Restart); 283 return performX86IOPTInvocationMapPT(cap, slot, iopte, lu_ret.ioptSlot); 284 } 285} 286 287static exception_t performX86IOInvocationMap(cap_t cap, cte_t *ctSlot, vtd_pte_t iopte, vtd_pte_t *ioptSlot) 288{ 289 ctSlot->cap = cap; 290 *ioptSlot = iopte; 291 flushCacheRange(ioptSlot, VTD_PTE_SIZE_BITS); 292 293 return EXCEPTION_NONE; 294} 295 296 297exception_t decodeX86IOMapInvocation( 298 word_t length, 299 cte_t *slot, 300 cap_t cap, 301 extra_caps_t excaps, 302 word_t *buffer 303) 304{ 305 cap_t io_space; 306 word_t io_address; 307 uint32_t pci_request_id; 308 vtd_cte_t *vtd_context_slot; 309 vtd_pte_t *vtd_pte; 310 vtd_pte_t iopte; 311 paddr_t paddr; 312 lookupIOPTSlot_ret_t lu_ret; 313 vm_rights_t frame_cap_rights; 314 seL4_CapRights_t dma_cap_rights_mask; 315 316 if (excaps.excaprefs[0] == NULL || length < 2) { 317 userError("X86PageMapIO: Truncated message."); 318 current_syscall_error.type = seL4_TruncatedMessage; 319 return EXCEPTION_SYSCALL_ERROR; 320 } 321 322 if (cap_frame_cap_get_capFSize(cap) != X86_SmallPage) { 323 userError("X86PageMapIO: Invalid page size."); 324 current_syscall_error.type = seL4_InvalidCapability; 325 current_syscall_error.invalidCapNumber = 0; 326 return EXCEPTION_SYSCALL_ERROR; 327 } 328 329 if (cap_frame_cap_get_capFMappedASID(cap) != asidInvalid) { 330 userError("X86PageMapIO: Page already mapped."); 331 current_syscall_error.type = seL4_InvalidCapability; 332 current_syscall_error.invalidCapNumber = 0; 333 return EXCEPTION_SYSCALL_ERROR; 334 } 335 336 io_space = excaps.excaprefs[0]->cap; 337 io_address = getSyscallArg(1, buffer) & ~MASK(PAGE_BITS); 338 paddr = pptr_to_paddr((void *)cap_frame_cap_get_capFBasePtr(cap)); 339 340 if (cap_get_capType(io_space) != cap_io_space_cap) { 341 userError("X86PageMapIO: Invalid IO space capability."); 342 current_syscall_error.type = seL4_InvalidCapability; 343 current_syscall_error.invalidCapNumber = 0; 344 return EXCEPTION_SYSCALL_ERROR; 345 } 346 347 pci_request_id = cap_io_space_cap_get_capPCIDevice(io_space); 348 349 if (pci_request_id == asidInvalid) { 350 userError("X86PageMapIO: Invalid PCI device."); 351 current_syscall_error.type = seL4_InvalidCapability; 352 current_syscall_error.invalidCapNumber = 0; 353 return EXCEPTION_SYSCALL_ERROR; 354 } 355 356 vtd_context_slot = lookup_vtd_context_slot(io_space); 357 358 if (!vtd_cte_ptr_get_present(vtd_context_slot)) { 359 /* 1st Level Page Table is not installed */ 360 current_syscall_error.type = seL4_FailedLookup; 361 current_syscall_error.failedLookupWasSource = false; 362 return EXCEPTION_SYSCALL_ERROR; 363 } 364 365 vtd_pte = (vtd_pte_t *)paddr_to_pptr(vtd_cte_ptr_get_asr(vtd_context_slot)); 366 lu_ret = lookupIOPTSlot(vtd_pte, io_address); 367 if (lu_ret.status != EXCEPTION_NONE || lu_ret.level != 0) { 368 current_syscall_error.type = seL4_FailedLookup; 369 current_syscall_error.failedLookupWasSource = false; 370 return EXCEPTION_SYSCALL_ERROR; 371 } 372 373 if (vtd_pte_ptr_get_addr(lu_ret.ioptSlot) != 0) { 374 current_syscall_error.type = seL4_DeleteFirst; 375 return EXCEPTION_SYSCALL_ERROR; 376 } 377 378 dma_cap_rights_mask = rightsFromWord(getSyscallArg(0, buffer)); 379 frame_cap_rights = cap_frame_cap_get_capFVMRights(cap); 380 381 bool_t write = seL4_CapRights_get_capAllowWrite(dma_cap_rights_mask) && (frame_cap_rights == VMReadWrite); 382 bool_t read = seL4_CapRights_get_capAllowRead(dma_cap_rights_mask) && (frame_cap_rights != VMKernelOnly); 383 if (write || read) { 384 iopte = vtd_pte_new(paddr, !!write, !!read); 385 } else { 386 current_syscall_error.type = seL4_InvalidArgument; 387 current_syscall_error.invalidArgumentNumber = 0; 388 return EXCEPTION_SYSCALL_ERROR; 389 } 390 391 cap = cap_frame_cap_set_capFMapType(cap, X86_MappingIOSpace); 392 cap = cap_frame_cap_set_capFMappedASID(cap, pci_request_id); 393 cap = cap_frame_cap_set_capFMappedAddress(cap, io_address); 394 395 setThreadState(NODE_STATE(ksCurThread), ThreadState_Restart); 396 return performX86IOInvocationMap(cap, slot, iopte, lu_ret.ioptSlot); 397} 398 399void deleteIOPageTable(cap_t io_pt_cap) 400{ 401 lookupIOPTSlot_ret_t lu_ret; 402 uint32_t level; 403 word_t io_address; 404 vtd_cte_t *vtd_context_slot; 405 vtd_pte_t *vtd_pte; 406 407 if (cap_io_page_table_cap_get_capIOPTIsMapped(io_pt_cap)) { 408 io_pt_cap = cap_io_page_table_cap_set_capIOPTIsMapped(io_pt_cap, 0); 409 level = cap_io_page_table_cap_get_capIOPTLevel(io_pt_cap); 410 vtd_context_slot = lookup_vtd_context_slot(io_pt_cap); 411 412 if (!vtd_cte_ptr_get_present(vtd_context_slot)) { 413 return; 414 } 415 416 vtd_pte = (vtd_pte_t *)paddr_to_pptr(vtd_cte_ptr_get_asr(vtd_context_slot)); 417 418 if (level == 0) { 419 /* if we have been overmapped or something */ 420 if (pptr_to_paddr(vtd_pte) != pptr_to_paddr((void *)cap_io_page_table_cap_get_capIOPTBasePtr(io_pt_cap))) { 421 return; 422 } 423 *vtd_context_slot = vtd_cte_new( 424 0, /* Domain ID */ 425 false, /* RMRR */ 426 0, /* Address Width */ 427 0, /* Address Space Root */ 428 0, /* Translation Type */ 429 0 /* Present */ 430 ); 431 flushCacheRange(vtd_context_slot, VTD_CTE_SIZE_BITS); 432 } else { 433 io_address = cap_io_page_table_cap_get_capIOPTMappedAddress(io_pt_cap); 434 lu_ret = lookupIOPTSlot_resolve_levels(vtd_pte, io_address >> PAGE_BITS, level - 1, level - 1); 435 436 /* if we have been overmapped or something */ 437 if (lu_ret.status != EXCEPTION_NONE || lu_ret.level != 0) { 438 return; 439 } 440 if (vtd_pte_ptr_get_addr(lu_ret.ioptSlot) != pptr_to_paddr((void *)cap_io_page_table_cap_get_capIOPTBasePtr( 441 io_pt_cap))) { 442 return; 443 } 444 *lu_ret.ioptSlot = vtd_pte_new( 445 0, /* Physical Address */ 446 0, /* Read Permission */ 447 0 /* Write Permission */ 448 ); 449 flushCacheRange(lu_ret.ioptSlot, VTD_PTE_SIZE_BITS); 450 } 451 invalidate_iotlb(); 452 } 453} 454 455void unmapIOPage(cap_t cap) 456{ 457 lookupIOPTSlot_ret_t lu_ret; 458 word_t io_address; 459 vtd_cte_t *vtd_context_slot; 460 vtd_pte_t *vtd_pte; 461 462 io_address = cap_frame_cap_get_capFMappedAddress(cap); 463 vtd_context_slot = lookup_vtd_context_slot(cap); 464 465 466 if (!vtd_cte_ptr_get_present(vtd_context_slot)) { 467 return; 468 } 469 470 vtd_pte = (vtd_pte_t *)paddr_to_pptr(vtd_cte_ptr_get_asr(vtd_context_slot)); 471 472 lu_ret = lookupIOPTSlot(vtd_pte, io_address); 473 if (lu_ret.status != EXCEPTION_NONE || lu_ret.level != 0) { 474 return; 475 } 476 477 if (vtd_pte_ptr_get_addr(lu_ret.ioptSlot) != pptr_to_paddr((void *)cap_frame_cap_get_capFBasePtr(cap))) { 478 return; 479 } 480 481 *lu_ret.ioptSlot = vtd_pte_new( 482 0, /* Physical Address */ 483 0, /* Read Permission */ 484 0 /* Write Permission */ 485 ); 486 487 flushCacheRange(lu_ret.ioptSlot, VTD_PTE_SIZE_BITS); 488 invalidate_iotlb(); 489} 490 491exception_t performX86IOUnMapInvocation(cap_t cap, cte_t *ctSlot) 492{ 493 unmapIOPage(ctSlot->cap); 494 495 ctSlot->cap = cap_frame_cap_set_capFMappedAddress(ctSlot->cap, 0); 496 ctSlot->cap = cap_frame_cap_set_capFMapType(ctSlot->cap, X86_MappingNone); 497 ctSlot->cap = cap_frame_cap_set_capFMappedASID(ctSlot->cap, asidInvalid); 498 499 return EXCEPTION_NONE; 500} 501 502exception_t decodeX86IOSpaceInvocation(word_t invLabel, cap_t cap) 503{ 504 userError("IOSpace capability has no invocations"); 505 current_syscall_error.type = seL4_IllegalOperation; 506 return EXCEPTION_SYSCALL_ERROR; 507} 508 509#endif /* CONFIG_IOMMU */ 510