1/* 2 * Copyright 2020, Data61, CSIRO (ABN 41 687 119 230) 3 * 4 * SPDX-License-Identifier: GPL-2.0-only 5 */ 6 7#include <config.h> 8 9#ifdef CONFIG_VTX 10 11#include <model/statedata.h> 12#include <arch/kernel/ept.h> 13#include <arch/api/invocation.h> 14 15struct lookupEPTPDPTSlot_ret { 16 exception_t status; 17 ept_pdpte_t *pdptSlot; 18}; 19typedef struct lookupEPTPDPTSlot_ret lookupEPTPDPTSlot_ret_t; 20 21struct lookupEPTPDSlot_ret { 22 exception_t status; 23 ept_pde_t *pdSlot; 24}; 25typedef struct lookupEPTPDSlot_ret lookupEPTPDSlot_ret_t; 26 27struct lookupEPTPTSlot_ret { 28 exception_t status; 29 ept_pte_t *ptSlot; 30}; 31typedef struct lookupEPTPTSlot_ret lookupEPTPTSlot_ret_t; 32 33enum ept_cache_options { 34 EPTUncacheable = 0, 35 EPTWriteCombining = 1, 36 EPTWriteThrough = 4, 37 EPTWriteProtected = 5, 38 EPTWriteBack = 6 39}; 40typedef enum ept_cache_options ept_cache_options_t; 41 42void deleteEPTASID(asid_t asid, ept_pml4e_t *ept) 43{ 44 asid_pool_t *poolPtr; 45 46 poolPtr = x86KSASIDTable[asid >> asidLowBits]; 47 if (poolPtr != NULL) { 48 asid_map_t asid_map = poolPtr->array[asid & MASK(asidLowBits)]; 49 if (asid_map_get_type(asid_map) == asid_map_asid_map_ept && 50 (ept_pml4e_t *)asid_map_asid_map_ept_get_ept_root(asid_map) == ept) { 51 poolPtr->array[asid & MASK(asidLowBits)] = asid_map_asid_map_none_new(); 52 } 53 } 54} 55 56exception_t performX86EPTPageInvocationUnmap(cap_t cap, cte_t *ctSlot) 57{ 58 unmapEPTPage( 59 cap_frame_cap_get_capFSize(cap), 60 cap_frame_cap_get_capFMappedASID(cap), 61 cap_frame_cap_get_capFMappedAddress(cap), 62 (void *)cap_frame_cap_get_capFBasePtr(cap) 63 ); 64 65 cap_frame_cap_ptr_set_capFMappedAddress(&ctSlot->cap, 0); 66 cap_frame_cap_ptr_set_capFMappedASID(&ctSlot->cap, asidInvalid); 67 cap_frame_cap_ptr_set_capFMapType(&ctSlot->cap, X86_MappingNone); 68 69 return EXCEPTION_NONE; 70} 71 72findEPTForASID_ret_t findEPTForASID(asid_t asid) 73{ 74 findEPTForASID_ret_t ret; 75 asid_map_t asid_map; 76 77 asid_map = findMapForASID(asid); 78 if (asid_map_get_type(asid_map) != asid_map_asid_map_ept) { 79 current_lookup_fault = lookup_fault_invalid_root_new(); 80 81 ret.ept = NULL; 82 ret.status = EXCEPTION_LOOKUP_FAULT; 83 return ret; 84 } 85 86 ret.ept = (ept_pml4e_t *)asid_map_asid_map_ept_get_ept_root(asid_map); 87 ret.status = EXCEPTION_NONE; 88 return ret; 89} 90 91static ept_pml4e_t *CONST lookupEPTPML4Slot(ept_pml4e_t *pml4, vptr_t vptr) 92{ 93 return pml4 + GET_EPT_PML4_INDEX(vptr); 94} 95 96static lookupEPTPDPTSlot_ret_t CONST lookupEPTPDPTSlot(ept_pml4e_t *pml4, vptr_t vptr) 97{ 98 lookupEPTPDPTSlot_ret_t ret; 99 ept_pml4e_t *pml4Slot; 100 101 pml4Slot = lookupEPTPML4Slot(pml4, vptr); 102 103 if (!ept_pml4e_ptr_get_read(pml4Slot)) { 104 current_lookup_fault = lookup_fault_missing_capability_new(EPT_PML4_INDEX_OFFSET); 105 106 ret.pdptSlot = NULL; 107 ret.status = EXCEPTION_LOOKUP_FAULT; 108 return ret; 109 } 110 111 ept_pdpte_t *pdpt = paddr_to_pptr(ept_pml4e_ptr_get_pdpt_base_address(pml4Slot)); 112 uint32_t index = GET_EPT_PDPT_INDEX(vptr); 113 ret.pdptSlot = pdpt + index; 114 ret.status = EXCEPTION_NONE; 115 return ret; 116} 117 118static lookupEPTPDSlot_ret_t lookupEPTPDSlot(ept_pml4e_t *pml4, vptr_t vptr) 119{ 120 lookupEPTPDSlot_ret_t ret; 121 lookupEPTPDPTSlot_ret_t lu_ret; 122 123 lu_ret = lookupEPTPDPTSlot(pml4, vptr); 124 if (lu_ret.status != EXCEPTION_NONE) { 125 current_syscall_error.type = seL4_FailedLookup; 126 current_syscall_error.failedLookupWasSource = false; 127 /* current_lookup_fault will have been set by lookupEPTPDPTSlot */ 128 ret.pdSlot = NULL; 129 ret.status = EXCEPTION_LOOKUP_FAULT; 130 return ret; 131 } 132 133 if (!ept_pdpte_ptr_get_read(lu_ret.pdptSlot)) { 134 current_lookup_fault = lookup_fault_missing_capability_new(EPT_PDPT_INDEX_OFFSET); 135 136 ret.pdSlot = NULL; 137 ret.status = EXCEPTION_LOOKUP_FAULT; 138 return ret; 139 } 140 141 ept_pde_t *pd = paddr_to_pptr(ept_pdpte_ptr_get_pd_base_address(lu_ret.pdptSlot)); 142 uint32_t index = GET_EPT_PD_INDEX(vptr); 143 ret.pdSlot = pd + index; 144 ret.status = EXCEPTION_NONE; 145 return ret; 146} 147 148static lookupEPTPTSlot_ret_t lookupEPTPTSlot(ept_pml4e_t *pml4, vptr_t vptr) 149{ 150 lookupEPTPTSlot_ret_t ret; 151 lookupEPTPDSlot_ret_t lu_ret; 152 153 lu_ret = lookupEPTPDSlot(pml4, vptr); 154 if (lu_ret.status != EXCEPTION_NONE) { 155 current_syscall_error.type = seL4_FailedLookup; 156 current_syscall_error.failedLookupWasSource = false; 157 /* current_lookup_fault will have been set by lookupEPTPDSlot */ 158 ret.ptSlot = NULL; 159 ret.status = EXCEPTION_LOOKUP_FAULT; 160 return ret; 161 } 162 163 if ((ept_pde_ptr_get_page_size(lu_ret.pdSlot) != ept_pde_ept_pde_pt) || 164 !ept_pde_ept_pde_pt_ptr_get_read(lu_ret.pdSlot)) { 165 current_lookup_fault = lookup_fault_missing_capability_new(EPT_PD_INDEX_OFFSET); 166 167 ret.ptSlot = NULL; 168 ret.status = EXCEPTION_LOOKUP_FAULT; 169 return ret; 170 } 171 172 ept_pte_t *pt = paddr_to_pptr(ept_pde_ept_pde_pt_ptr_get_pt_base_address(lu_ret.pdSlot)); 173 uint32_t index = GET_EPT_PT_INDEX(vptr); 174 175 ret.ptSlot = pt + index; 176 ret.status = EXCEPTION_NONE; 177 return ret; 178} 179 180static ept_cache_options_t eptCacheFromVmAttr(vm_attributes_t vmAttr) 181{ 182 /* PAT cache options are 1-1 with ept_cache_options. But need to 183 verify user has specified a sensible option */ 184 ept_cache_options_t option = vmAttr.words[0]; 185 if (option != EPTUncacheable || 186 option != EPTWriteCombining || 187 option != EPTWriteThrough || 188 option != EPTWriteBack) { 189 /* No failure mode is supported here, vmAttr settings should be verified earlier */ 190 option = EPTWriteBack; 191 } 192 return option; 193} 194 195EPTPDPTMapped_ret_t EPTPDPTMapped(asid_t asid, vptr_t vptr, ept_pdpte_t *pdpt) 196{ 197 EPTPDPTMapped_ret_t ret; 198 findEPTForASID_ret_t asid_ret; 199 ept_pml4e_t *pml4Slot; 200 201 asid_ret = findEPTForASID(asid); 202 if (asid_ret.status != EXCEPTION_NONE) { 203 ret.pml4 = NULL; 204 ret.pml4Slot = NULL; 205 ret.status = asid_ret.status; 206 return ret; 207 } 208 209 pml4Slot = lookupEPTPML4Slot(asid_ret.ept, vptr); 210 211 if (ept_pml4e_ptr_get_read(pml4Slot) 212 && ptrFromPAddr(ept_pml4e_ptr_get_pdpt_base_address(pml4Slot)) == pdpt) { 213 ret.pml4 = asid_ret.ept; 214 ret.pml4Slot = pml4Slot; 215 ret.status = EXCEPTION_NONE; 216 return ret; 217 } else { 218 ret.pml4 = NULL; 219 ret.pml4Slot = NULL; 220 ret.status = EXCEPTION_LOOKUP_FAULT; 221 return ret; 222 } 223} 224 225void unmapEPTPDPT(asid_t asid, vptr_t vaddr, ept_pdpte_t *pdpt) 226{ 227 EPTPDPTMapped_ret_t lu_ret; 228 229 lu_ret = EPTPDPTMapped(asid, vaddr, pdpt); 230 231 if (lu_ret.status == EXCEPTION_NONE) { 232 *lu_ret.pml4Slot = ept_pml4e_new(0, 0, 0, 0); 233 invept(lu_ret.pml4); 234 } 235} 236 237static exception_t performEPTPDPTInvocationUnmap(cap_t cap, cte_t *cte) 238{ 239 if (cap_ept_pdpt_cap_get_capPDPTIsMapped(cap)) { 240 ept_pdpte_t *pdpt = (ept_pdpte_t *)cap_ept_pdpt_cap_get_capPDPTBasePtr(cap); 241 unmapEPTPDPT( 242 cap_ept_pdpt_cap_get_capPDPTMappedASID(cap), 243 cap_ept_pdpt_cap_get_capPDPTMappedAddress(cap), 244 pdpt); 245 clearMemory((void *)pdpt, cap_get_capSizeBits(cap)); 246 } 247 cap_ept_pdpt_cap_ptr_set_capPDPTIsMapped(&(cte->cap), 0); 248 249 return EXCEPTION_NONE; 250} 251 252static exception_t performEPTPDPTInvocationMap(cap_t cap, cte_t *cte, ept_pml4e_t pml4e, ept_pml4e_t *pml4Slot, 253 ept_pml4e_t *pml4) 254{ 255 cte->cap = cap; 256 *pml4Slot = pml4e; 257 invept(pml4); 258 259 return EXCEPTION_NONE; 260} 261 262static exception_t decodeX86EPTPDPTInvocation( 263 word_t invLabel, 264 word_t length, 265 cte_t *cte, 266 cap_t cap, 267 extra_caps_t excaps, 268 word_t *buffer 269) 270{ 271 word_t vaddr; 272 cap_t pml4Cap; 273 ept_pml4e_t *pml4; 274 ept_pml4e_t pml4e; 275 paddr_t paddr; 276 asid_t asid; 277 findEPTForASID_ret_t find_ret; 278 ept_pml4e_t *pml4Slot; 279 280 if (invLabel == X86EPTPDPTUnmap) { 281 if (!isFinalCapability(cte)) { 282 current_syscall_error.type = seL4_RevokeFirst; 283 return EXCEPTION_SYSCALL_ERROR; 284 } 285 setThreadState(NODE_STATE(ksCurThread), ThreadState_Restart); 286 return performEPTPDPTInvocationUnmap(cap, cte); 287 } 288 289 if (invLabel != X86EPTPDPTMap) { 290 userError("X86EPTPDPT Illegal operation."); 291 current_syscall_error.type = seL4_IllegalOperation; 292 return EXCEPTION_SYSCALL_ERROR; 293 } 294 295 if (length < 2 || excaps.excaprefs[0] == NULL) { 296 userError("X86EPTPDPTMap: Truncated message."); 297 current_syscall_error.type = seL4_TruncatedMessage; 298 return EXCEPTION_SYSCALL_ERROR; 299 } 300 301 if (cap_ept_pdpt_cap_get_capPDPTIsMapped(cap)) { 302 userError("X86EPTPDPTMap: EPT PDPT is already mapped to a PML4."); 303 current_syscall_error.type = seL4_InvalidCapability; 304 current_syscall_error.invalidCapNumber = 0; 305 306 return EXCEPTION_SYSCALL_ERROR; 307 } 308 309 vaddr = getSyscallArg(0, buffer); 310 /* cannot use ~MASK(EPT_PML4_INDEX_OFFSET) because on 32-bit compilations 311 * this results in an error shifting by greater than 31 bits, so we manually 312 * force a 64-bit variable to do the shifting with */ 313 vaddr = vaddr & ~(((uint64_t)1 << EPT_PML4_INDEX_OFFSET) - 1); 314 pml4Cap = excaps.excaprefs[0]->cap; 315 316 if (cap_get_capType(pml4Cap) != cap_ept_pml4_cap) { 317 userError("X86EPTPDPTMap: Not a valid EPT PML4."); 318 current_syscall_error.type = seL4_InvalidCapability; 319 current_syscall_error.invalidCapNumber = 1; 320 321 return EXCEPTION_SYSCALL_ERROR; 322 } 323 324 pml4 = (ept_pml4e_t *)cap_ept_pml4_cap_get_capPML4BasePtr(pml4Cap); 325 asid = cap_ept_pml4_cap_get_capPML4MappedASID(pml4Cap); 326 327 find_ret = findEPTForASID(asid); 328 if (find_ret.status != EXCEPTION_NONE) { 329 current_syscall_error.type = seL4_FailedLookup; 330 current_syscall_error.failedLookupWasSource = false; 331 332 return EXCEPTION_SYSCALL_ERROR; 333 } 334 335 if (find_ret.ept != pml4) { 336 current_syscall_error.type = seL4_InvalidCapability; 337 current_syscall_error.invalidCapNumber = 1; 338 339 return EXCEPTION_SYSCALL_ERROR; 340 } 341 342 pml4Slot = lookupEPTPML4Slot(pml4, vaddr); 343 344 if (ept_pml4e_ptr_get_read(pml4Slot)) { 345 userError("X86EPTPDPTMap: PDPT already mapped here."); 346 current_syscall_error.type = seL4_DeleteFirst; 347 return EXCEPTION_SYSCALL_ERROR; 348 } 349 350 paddr = pptr_to_paddr((void *)cap_ept_pdpt_cap_get_capPDPTBasePtr(cap)); 351 pml4e = ept_pml4e_new( 352 paddr, 353 1, 354 1, 355 1 356 ); 357 358 cap = cap_ept_pdpt_cap_set_capPDPTIsMapped(cap, 1); 359 cap = cap_ept_pdpt_cap_set_capPDPTMappedASID(cap, asid); 360 cap = cap_ept_pdpt_cap_set_capPDPTMappedAddress(cap, vaddr); 361 362 setThreadState(NODE_STATE(ksCurThread), ThreadState_Restart); 363 return performEPTPDPTInvocationMap(cap, cte, pml4e, pml4Slot, pml4); 364} 365 366exception_t decodeX86EPTInvocation( 367 word_t invLabel, 368 word_t length, 369 cptr_t cptr, 370 cte_t *cte, 371 cap_t cap, 372 extra_caps_t excaps, 373 word_t *buffer 374) 375{ 376 switch (cap_get_capType(cap)) { 377 case cap_ept_pdpt_cap: 378 return decodeX86EPTPDPTInvocation(invLabel, length, cte, cap, excaps, buffer); 379 case cap_ept_pd_cap: 380 return decodeX86EPTPDInvocation(invLabel, length, cte, cap, excaps, buffer); 381 case cap_ept_pt_cap: 382 return decodeX86EPTPTInvocation(invLabel, length, cte, cap, excaps, buffer); 383 default: 384 fail("Invalid cap type"); 385 } 386} 387 388EPTPageDirectoryMapped_ret_t EPTPageDirectoryMapped(asid_t asid, vptr_t vaddr, ept_pde_t *pd) 389{ 390 EPTPageDirectoryMapped_ret_t ret; 391 lookupEPTPDPTSlot_ret_t find_ret; 392 findEPTForASID_ret_t asid_ret; 393 394 asid_ret = findEPTForASID(asid); 395 if (asid_ret.status != EXCEPTION_NONE) { 396 ret.pml4 = NULL; 397 ret.pdptSlot = NULL; 398 ret.status = asid_ret.status; 399 return ret; 400 } 401 402 find_ret = lookupEPTPDPTSlot(asid_ret.ept, vaddr); 403 if (find_ret.status != EXCEPTION_NONE) { 404 ret.pml4 = NULL; 405 ret.pdptSlot = NULL; 406 ret.status = find_ret.status; 407 return ret; 408 } 409 410 if (ept_pdpte_ptr_get_read(find_ret.pdptSlot) 411 && ptrFromPAddr(ept_pdpte_ptr_get_pd_base_address(find_ret.pdptSlot)) == pd) { 412 ret.pml4 = asid_ret.ept; 413 ret.pdptSlot = find_ret.pdptSlot; 414 ret.status = EXCEPTION_NONE; 415 return ret; 416 } else { 417 ret.pml4 = NULL; 418 ret.pdptSlot = NULL; 419 ret.status = EXCEPTION_LOOKUP_FAULT; 420 return ret; 421 } 422} 423 424void unmapEPTPageDirectory(asid_t asid, vptr_t vaddr, ept_pde_t *pd) 425{ 426 EPTPageDirectoryMapped_ret_t lu_ret; 427 428 lu_ret = EPTPageDirectoryMapped(asid, vaddr, pd); 429 430 if (lu_ret.status == EXCEPTION_NONE) { 431 *lu_ret.pdptSlot = ept_pdpte_new( 432 0, /* pd_base_address */ 433 0, /* avl_cte_depth */ 434 0, /* execute */ 435 0, /* write */ 436 0 /* read */ 437 ); 438 invept(lu_ret.pml4); 439 } 440} 441 442static exception_t performEPTPDInvocationUnmap(cap_t cap, cte_t *cte) 443{ 444 if (cap_ept_pd_cap_get_capPDIsMapped(cap)) { 445 ept_pde_t *pd = (ept_pde_t *)cap_ept_pd_cap_get_capPDBasePtr(cap); 446 unmapEPTPageDirectory( 447 cap_ept_pd_cap_get_capPDMappedASID(cap), 448 cap_ept_pd_cap_get_capPDMappedAddress(cap), 449 pd); 450 clearMemory((void *)pd, cap_get_capSizeBits(cap)); 451 } 452 cap_ept_pd_cap_ptr_set_capPDIsMapped(&(cte->cap), 0); 453 454 return EXCEPTION_NONE; 455} 456 457static exception_t performEPTPDInvocationMap(cap_t cap, cte_t *cte, ept_pdpte_t pdpte, ept_pdpte_t *pdptSlot, 458 ept_pml4e_t *pml4) 459{ 460 cte->cap = cap; 461 *pdptSlot = pdpte; 462 invept(pml4); 463 464 return EXCEPTION_NONE; 465} 466 467exception_t decodeX86EPTPDInvocation( 468 word_t invLabel, 469 word_t length, 470 cte_t *cte, 471 cap_t cap, 472 extra_caps_t excaps, 473 word_t *buffer 474) 475{ 476 word_t vaddr; 477 cap_t pml4Cap; 478 ept_pml4e_t *pml4; 479 ept_pdpte_t pdpte; 480 paddr_t paddr; 481 asid_t asid; 482 findEPTForASID_ret_t find_ret; 483 lookupEPTPDPTSlot_ret_t lu_ret; 484 485 if (invLabel == X86EPTPDUnmap) { 486 if (!isFinalCapability(cte)) { 487 current_syscall_error.type = seL4_RevokeFirst; 488 return EXCEPTION_SYSCALL_ERROR; 489 } 490 setThreadState(NODE_STATE(ksCurThread), ThreadState_Restart); 491 return performEPTPDInvocationUnmap(cap, cte); 492 } 493 494 if (invLabel != X86EPTPDMap) { 495 userError("X86EPTPD Illegal operation."); 496 current_syscall_error.type = seL4_IllegalOperation; 497 return EXCEPTION_SYSCALL_ERROR; 498 } 499 500 if (length < 2 || excaps.excaprefs[0] == NULL) { 501 userError("X86EPTPDMap: Truncated message."); 502 current_syscall_error.type = seL4_TruncatedMessage; 503 return EXCEPTION_SYSCALL_ERROR; 504 } 505 506 if (cap_ept_pd_cap_get_capPDIsMapped(cap)) { 507 userError("X86EPTPDMap: EPT Page directory is already mapped to a PDPT."); 508 current_syscall_error.type = 509 seL4_InvalidCapability; 510 current_syscall_error.invalidCapNumber = 0; 511 512 return EXCEPTION_SYSCALL_ERROR; 513 } 514 515 vaddr = getSyscallArg(0, buffer); 516 vaddr = vaddr & ~MASK(EPT_PDPT_INDEX_OFFSET); 517 pml4Cap = excaps.excaprefs[0]->cap; 518 519 if (cap_get_capType(pml4Cap) != cap_ept_pml4_cap) { 520 userError("X86EPTPDMap: Not a valid EPT pml4."); 521 current_syscall_error.type = seL4_InvalidCapability; 522 current_syscall_error.invalidCapNumber = 1; 523 524 return EXCEPTION_SYSCALL_ERROR; 525 } 526 527 pml4 = (ept_pml4e_t *)cap_ept_pml4_cap_get_capPML4BasePtr(pml4Cap); 528 asid = cap_ept_pml4_cap_get_capPML4MappedASID(pml4Cap); 529 530 find_ret = findEPTForASID(asid); 531 if (find_ret.status != EXCEPTION_NONE) { 532 userError("X86EPTPDMap: EPT PML4 is not mapped."); 533 current_syscall_error.type = seL4_FailedLookup; 534 current_syscall_error.failedLookupWasSource = false; 535 536 return EXCEPTION_SYSCALL_ERROR; 537 } 538 539 if (find_ret.ept != pml4) { 540 userError("X86EPTPDMap: EPT PML4 asid is invalid."); 541 current_syscall_error.type = seL4_InvalidCapability; 542 current_syscall_error.invalidCapNumber = 1; 543 544 return EXCEPTION_SYSCALL_ERROR; 545 } 546 547 lu_ret = lookupEPTPDPTSlot(pml4, vaddr); 548 if (lu_ret.status != EXCEPTION_NONE) { 549 current_syscall_error.type = seL4_FailedLookup; 550 current_syscall_error.failedLookupWasSource = false; 551 return EXCEPTION_SYSCALL_ERROR; 552 } 553 554 if (ept_pdpte_ptr_get_read(lu_ret.pdptSlot)) { 555 userError("X86EPTPDMap: Page directory already mapped here."); 556 current_syscall_error.type = seL4_DeleteFirst; 557 return EXCEPTION_SYSCALL_ERROR; 558 } 559 560 paddr = pptr_to_paddr((void *)(cap_ept_pd_cap_get_capPDBasePtr(cap))); 561 pdpte = ept_pdpte_new( 562 paddr, /* pd_base_address */ 563 0, /* avl_cte_depth */ 564 1, /* execute */ 565 1, /* write */ 566 1 /* read */ 567 ); 568 569 cap = cap_ept_pd_cap_set_capPDIsMapped(cap, 1); 570 cap = cap_ept_pd_cap_set_capPDMappedASID(cap, asid); 571 cap = cap_ept_pd_cap_set_capPDMappedAddress(cap, vaddr); 572 573 setThreadState(NODE_STATE(ksCurThread), ThreadState_Restart); 574 return performEPTPDInvocationMap(cap, cte, pdpte, lu_ret.pdptSlot, pml4); 575} 576 577EPTPageTableMapped_ret_t EPTPageTableMapped(asid_t asid, vptr_t vaddr, ept_pte_t *pt) 578{ 579 EPTPageTableMapped_ret_t ret; 580 lookupEPTPDSlot_ret_t find_ret; 581 findEPTForASID_ret_t asid_ret; 582 583 asid_ret = findEPTForASID(asid); 584 if (asid_ret.status != EXCEPTION_NONE) { 585 ret.pml4 = NULL; 586 ret.pdSlot = NULL; 587 ret.status = asid_ret.status; 588 return ret; 589 } 590 591 find_ret = lookupEPTPDSlot(asid_ret.ept, vaddr); 592 if (find_ret.status != EXCEPTION_NONE) { 593 ret.pml4 = NULL; 594 ret.pdSlot = NULL; 595 ret.status = find_ret.status; 596 return ret; 597 } 598 599 if (ept_pde_ptr_get_page_size(find_ret.pdSlot) == ept_pde_ept_pde_pt 600 && ptrFromPAddr(ept_pde_ept_pde_pt_ptr_get_pt_base_address(find_ret.pdSlot)) == pt) { 601 ret.pml4 = asid_ret.ept; 602 ret.pdSlot = find_ret.pdSlot; 603 ret.status = EXCEPTION_NONE; 604 return ret; 605 } else { 606 ret.pml4 = NULL; 607 ret.pdSlot = NULL; 608 ret.status = EXCEPTION_LOOKUP_FAULT; 609 return ret; 610 } 611} 612 613void unmapEPTPageTable(asid_t asid, vptr_t vaddr, ept_pte_t *pt) 614{ 615 EPTPageTableMapped_ret_t lu_ret; 616 617 lu_ret = EPTPageTableMapped(asid, vaddr, pt); 618 619 if (lu_ret.status == EXCEPTION_NONE) { 620 *lu_ret.pdSlot = ept_pde_ept_pde_pt_new( 621 0, /* pt_base_address */ 622 0, /* avl_cte_depth */ 623 0, /* execute */ 624 0, /* write */ 625 0 /* read */ 626 ); 627 invept(lu_ret.pml4); 628 } 629} 630 631static exception_t performEPTPTInvocationUnmap(cap_t cap, cte_t *cte) 632{ 633 if (cap_ept_pt_cap_get_capPTIsMapped(cap)) { 634 ept_pte_t *pt = (ept_pte_t *)cap_ept_pt_cap_get_capPTBasePtr(cap); 635 unmapEPTPageTable( 636 cap_ept_pt_cap_get_capPTMappedASID(cap), 637 cap_ept_pt_cap_get_capPTMappedAddress(cap), 638 pt); 639 clearMemory((void *)pt, cap_get_capSizeBits(cap)); 640 } 641 cap_ept_pt_cap_ptr_set_capPTIsMapped(&(cte->cap), 0); 642 643 return EXCEPTION_NONE; 644} 645 646static exception_t performEPTPTInvocationMap(cap_t cap, cte_t *cte, ept_pde_t pde, ept_pde_t *pdSlot, ept_pml4e_t *pml4) 647{ 648 cte->cap = cap; 649 *pdSlot = pde; 650 invept(pml4); 651 652 return EXCEPTION_NONE; 653} 654 655exception_t decodeX86EPTPTInvocation( 656 word_t invLabel, 657 word_t length, 658 cte_t *cte, 659 cap_t cap, 660 extra_caps_t excaps, 661 word_t *buffer 662) 663{ 664 word_t vaddr; 665 cap_t pml4Cap; 666 ept_pml4e_t *pml4; 667 ept_pde_t pde; 668 paddr_t paddr; 669 asid_t asid; 670 findEPTForASID_ret_t find_ret; 671 lookupEPTPDSlot_ret_t lu_ret; 672 673 if (invLabel == X86EPTPTUnmap) { 674 if (!isFinalCapability(cte)) { 675 current_syscall_error.type = seL4_RevokeFirst; 676 return EXCEPTION_SYSCALL_ERROR; 677 } 678 setThreadState(NODE_STATE(ksCurThread), ThreadState_Restart); 679 return performEPTPTInvocationUnmap(cap, cte); 680 } 681 682 if (invLabel != X86EPTPTMap) { 683 userError("X86EPTPT Illegal operation."); 684 current_syscall_error.type = seL4_IllegalOperation; 685 return EXCEPTION_SYSCALL_ERROR; 686 } 687 688 if (length < 2 || excaps.excaprefs[0] == NULL) { 689 userError("X86EPTPT: Truncated message."); 690 current_syscall_error.type = seL4_TruncatedMessage; 691 return EXCEPTION_SYSCALL_ERROR; 692 } 693 694 if (cap_ept_pt_cap_get_capPTIsMapped(cap)) { 695 userError("X86EPTPT EPT Page table is already mapped to an EPT page directory."); 696 current_syscall_error.type = 697 seL4_InvalidCapability; 698 current_syscall_error.invalidCapNumber = 0; 699 700 return EXCEPTION_SYSCALL_ERROR; 701 } 702 703 vaddr = getSyscallArg(0, buffer); 704 vaddr = vaddr & ~MASK(EPT_PD_INDEX_OFFSET); 705 pml4Cap = excaps.excaprefs[0]->cap; 706 707 if (cap_get_capType(pml4Cap) != cap_ept_pml4_cap || 708 !cap_ept_pml4_cap_get_capPML4IsMapped(pml4Cap)) { 709 userError("X86EPTPTMap: Not a valid EPT pml4."); 710 current_syscall_error.type = seL4_InvalidCapability; 711 current_syscall_error.invalidCapNumber = 1; 712 713 return EXCEPTION_SYSCALL_ERROR; 714 } 715 716 pml4 = (ept_pml4e_t *)(cap_ept_pml4_cap_get_capPML4BasePtr(pml4Cap)); 717 asid = cap_ept_pml4_cap_get_capPML4MappedASID(pml4Cap); 718 719 find_ret = findEPTForASID(asid); 720 if (find_ret.status != EXCEPTION_NONE) { 721 current_syscall_error.type = seL4_FailedLookup; 722 current_syscall_error.failedLookupWasSource = false; 723 724 return EXCEPTION_SYSCALL_ERROR; 725 } 726 727 if (find_ret.ept != pml4) { 728 current_syscall_error.type = seL4_InvalidCapability; 729 current_syscall_error.invalidCapNumber = 1; 730 731 return EXCEPTION_SYSCALL_ERROR; 732 } 733 734 lu_ret = lookupEPTPDSlot(pml4, vaddr); 735 if (lu_ret.status != EXCEPTION_NONE) { 736 current_syscall_error.type = seL4_FailedLookup; 737 current_syscall_error.failedLookupWasSource = false; 738 /* current_lookup_fault will have been set by lookupPTSlot */ 739 return EXCEPTION_SYSCALL_ERROR; 740 } 741 742 if (((ept_pde_ptr_get_page_size(lu_ret.pdSlot) == ept_pde_ept_pde_pt) && 743 ept_pde_ept_pde_pt_ptr_get_read(lu_ret.pdSlot)) || 744 ((ept_pde_ptr_get_page_size(lu_ret.pdSlot) == ept_pde_ept_pde_2m) && 745 ept_pde_ept_pde_2m_ptr_get_read(lu_ret.pdSlot))) { 746 userError("X86EPTPTMap: Page table already mapped here"); 747 current_syscall_error.type = seL4_DeleteFirst; 748 return EXCEPTION_SYSCALL_ERROR; 749 } 750 751 paddr = pptr_to_paddr((void *)(cap_ept_pt_cap_get_capPTBasePtr(cap))); 752 pde = ept_pde_ept_pde_pt_new( 753 paddr,/* pt_base_address */ 754 0, /* avl_cte_depth */ 755 1, /* execute */ 756 1, /* write */ 757 1 /* read */ 758 ); 759 760 cap = cap_ept_pt_cap_set_capPTIsMapped(cap, 1); 761 cap = cap_ept_pt_cap_set_capPTMappedASID(cap, asid); 762 cap = cap_ept_pt_cap_set_capPTMappedAddress(cap, vaddr); 763 764 setThreadState(NODE_STATE(ksCurThread), ThreadState_Restart); 765 return performEPTPTInvocationMap(cap, cte, pde, lu_ret.pdSlot, pml4); 766} 767 768static exception_t performEPTPageMapPTE(cap_t cap, cte_t *cte, ept_pte_t *ptSlot, ept_pte_t pte, ept_pml4e_t *pml4) 769{ 770 *ptSlot = pte; 771 cte->cap = cap; 772 invept(pml4); 773 774 return EXCEPTION_NONE; 775} 776 777static exception_t performEPTPageMapPDE(cap_t cap, cte_t *cte, ept_pde_t *pdSlot, ept_pde_t pde1, ept_pde_t pde2, 778 ept_pml4e_t *pml4) 779{ 780 pdSlot[0] = pde1; 781 if (LARGE_PAGE_BITS == 22) { 782 pdSlot[1] = pde2; 783 } 784 cte->cap = cap; 785 invept(pml4); 786 787 return EXCEPTION_NONE; 788} 789 790exception_t decodeX86EPTPageMap( 791 word_t invLabel, 792 word_t length, 793 cte_t *cte, 794 cap_t cap, 795 extra_caps_t excaps, 796 word_t *buffer) 797{ 798 word_t vaddr; 799 word_t w_rightsMask; 800 paddr_t paddr; 801 cap_t pml4Cap; 802 ept_pml4e_t *pml4; 803 vm_rights_t capVMRights; 804 vm_rights_t vmRights; 805 vm_attributes_t vmAttr; 806 vm_page_size_t frameSize; 807 asid_t asid; 808 809 frameSize = cap_frame_cap_get_capFSize(cap); 810 vaddr = getSyscallArg(0, buffer); 811 vaddr = vaddr & ~MASK(EPT_PT_INDEX_OFFSET); 812 w_rightsMask = getSyscallArg(1, buffer); 813 vmAttr = vmAttributesFromWord(getSyscallArg(2, buffer)); 814 pml4Cap = excaps.excaprefs[0]->cap; 815 816 capVMRights = cap_frame_cap_get_capFVMRights(cap); 817 818 if (cap_frame_cap_get_capFMappedASID(cap) != asidInvalid) { 819 userError("X86EPTPageMap: Frame already mapped."); 820 current_syscall_error.type = seL4_InvalidCapability; 821 current_syscall_error.invalidCapNumber = 0; 822 823 return EXCEPTION_SYSCALL_ERROR; 824 } 825 826 assert(cap_frame_cap_get_capFMapType(cap) == X86_MappingNone); 827 828 if (cap_get_capType(pml4Cap) != cap_ept_pml4_cap || 829 !cap_ept_pml4_cap_get_capPML4IsMapped(pml4Cap)) { 830 userError("X86EPTPageMap: Attempting to map frame into invalid ept pml4."); 831 current_syscall_error.type = seL4_InvalidCapability; 832 current_syscall_error.invalidCapNumber = 1; 833 834 return EXCEPTION_SYSCALL_ERROR; 835 } 836 837 pml4 = (ept_pml4e_t *)(cap_ept_pml4_cap_get_capPML4BasePtr(pml4Cap)); 838 asid = cap_ept_pml4_cap_get_capPML4MappedASID(pml4Cap); 839 840 findEPTForASID_ret_t find_ret = findEPTForASID(asid); 841 if (find_ret.status != EXCEPTION_NONE) { 842 current_syscall_error.type = seL4_FailedLookup; 843 current_syscall_error.failedLookupWasSource = false; 844 845 return EXCEPTION_SYSCALL_ERROR; 846 } 847 848 if (find_ret.ept != pml4) { 849 current_syscall_error.type = seL4_InvalidCapability; 850 current_syscall_error.invalidCapNumber = 1; 851 852 return EXCEPTION_SYSCALL_ERROR; 853 } 854 855 856 vmRights = maskVMRights(capVMRights, rightsFromWord(w_rightsMask)); 857 858 if (!checkVPAlignment(frameSize, vaddr)) { 859 current_syscall_error.type = seL4_AlignmentError; 860 861 return EXCEPTION_SYSCALL_ERROR; 862 } 863 864 paddr = pptr_to_paddr((void *)cap_frame_cap_get_capFBasePtr(cap)); 865 866 cap = cap_frame_cap_set_capFMappedASID(cap, asid); 867 cap = cap_frame_cap_set_capFMappedAddress(cap, vaddr); 868 cap = cap_frame_cap_set_capFMapType(cap, X86_MappingEPT); 869 870 switch (frameSize) { 871 /* PTE mappings */ 872 case X86_SmallPage: { 873 lookupEPTPTSlot_ret_t lu_ret; 874 ept_pte_t pte; 875 876 lu_ret = lookupEPTPTSlot(pml4, vaddr); 877 if (lu_ret.status != EXCEPTION_NONE) { 878 current_syscall_error.type = seL4_FailedLookup; 879 current_syscall_error.failedLookupWasSource = false; 880 /* current_lookup_fault will have been set by lookupEPTPTSlot */ 881 return EXCEPTION_SYSCALL_ERROR; 882 } 883 884 if (ept_pte_ptr_get_read(lu_ret.ptSlot)) { 885 userError("X86EPTPageMap: Mapping already present."); 886 current_syscall_error.type = seL4_DeleteFirst; 887 return EXCEPTION_SYSCALL_ERROR; 888 } 889 890 pte = ept_pte_new( 891 paddr, 892 0, 893 0, 894 eptCacheFromVmAttr(vmAttr), 895 1, 896 WritableFromVMRights(vmRights), 897 1); 898 899 setThreadState(NODE_STATE(ksCurThread), ThreadState_Restart); 900 return performEPTPageMapPTE(cap, cte, lu_ret.ptSlot, pte, pml4); 901 } 902 903 /* PDE mappings */ 904 case X86_LargePage: { 905 lookupEPTPDSlot_ret_t lu_ret; 906 907 lu_ret = lookupEPTPDSlot(pml4, vaddr); 908 if (lu_ret.status != EXCEPTION_NONE) { 909 userError("X86EPTPageMap: Need a page directory first."); 910 current_syscall_error.type = seL4_FailedLookup; 911 current_syscall_error.failedLookupWasSource = false; 912 /* current_lookup_fault will have been set by lookupEPTPDSlot */ 913 return EXCEPTION_SYSCALL_ERROR; 914 } 915 916 917 if ((ept_pde_ptr_get_page_size(lu_ret.pdSlot) == ept_pde_ept_pde_pt) && 918 ept_pde_ept_pde_pt_ptr_get_read(lu_ret.pdSlot)) { 919 userError("X86EPTPageMap: Page table already present."); 920 current_syscall_error.type = seL4_DeleteFirst; 921 return EXCEPTION_SYSCALL_ERROR; 922 } 923 if (LARGE_PAGE_BITS != EPT_PD_INDEX_OFFSET && 924 (ept_pde_ptr_get_page_size(lu_ret.pdSlot + 1) == ept_pde_ept_pde_pt) && 925 ept_pde_ept_pde_pt_ptr_get_read(lu_ret.pdSlot + 1)) { 926 userError("X86EPTPageMap: Page table already present."); 927 current_syscall_error.type = seL4_DeleteFirst; 928 return EXCEPTION_SYSCALL_ERROR; 929 } 930 if ((ept_pde_ptr_get_page_size(lu_ret.pdSlot) == ept_pde_ept_pde_2m) && 931 ept_pde_ept_pde_2m_ptr_get_read(lu_ret.pdSlot)) { 932 userError("X86EPTPageMap: Mapping already present."); 933 current_syscall_error.type = seL4_DeleteFirst; 934 return EXCEPTION_SYSCALL_ERROR; 935 } 936 937 ept_pde_t pde1 = ept_pde_ept_pde_2m_new( 938 paddr, 939 0, 940 0, 941 eptCacheFromVmAttr(vmAttr), 942 1, 943 WritableFromVMRights(vmRights), 944 1); 945 946 ept_pde_t pde2 = ept_pde_ept_pde_2m_new( 947 paddr + BIT(EPT_PD_INDEX_OFFSET), 948 0, 949 0, 950 eptCacheFromVmAttr(vmAttr), 951 1, 952 WritableFromVMRights(vmRights), 953 1); 954 955 setThreadState(NODE_STATE(ksCurThread), ThreadState_Restart); 956 return performEPTPageMapPDE(cap, cte, lu_ret.pdSlot, pde1, pde2, pml4); 957 } 958 959 default: 960 /* When initializing EPT we only checked for support for 4K and 2M 961 * pages, so we must disallow attempting to use any other */ 962 userError("X86EPTPageMap: Attempted to map unsupported page size."); 963 current_syscall_error.type = seL4_InvalidCapability; 964 current_syscall_error.invalidCapNumber = 0; 965 return EXCEPTION_SYSCALL_ERROR; 966 } 967} 968 969void unmapEPTPage(vm_page_size_t page_size, asid_t asid, vptr_t vptr, void *pptr) 970{ 971 findEPTForASID_ret_t find_ret; 972 paddr_t addr = addrFromPPtr(pptr); 973 974 find_ret = findEPTForASID(asid); 975 if (find_ret.status != EXCEPTION_NONE) { 976 return; 977 } 978 979 switch (page_size) { 980 case X86_SmallPage: { 981 lookupEPTPTSlot_ret_t lu_ret; 982 983 lu_ret = lookupEPTPTSlot(find_ret.ept, vptr); 984 if (lu_ret.status != EXCEPTION_NONE) { 985 return; 986 } 987 if (!ept_pte_ptr_get_read(lu_ret.ptSlot)) { 988 return; 989 } 990 if (ept_pte_ptr_get_page_base_address(lu_ret.ptSlot) != addr) { 991 return; 992 } 993 994 *lu_ret.ptSlot = ept_pte_new(0, 0, 0, 0, 0, 0, 0); 995 break; 996 } 997 case X86_LargePage: { 998 lookupEPTPDSlot_ret_t lu_ret; 999 1000 lu_ret = lookupEPTPDSlot(find_ret.ept, vptr); 1001 if (lu_ret.status != EXCEPTION_NONE) { 1002 return; 1003 } 1004 if (ept_pde_ptr_get_page_size(lu_ret.pdSlot) != ept_pde_ept_pde_2m) { 1005 return; 1006 } 1007 if (!ept_pde_ept_pde_2m_ptr_get_read(lu_ret.pdSlot)) { 1008 return; 1009 } 1010 if (ept_pde_ept_pde_2m_ptr_get_page_base_address(lu_ret.pdSlot) != addr) { 1011 return; 1012 } 1013 1014 lu_ret.pdSlot[0] = ept_pde_ept_pde_2m_new(0, 0, 0, 0, 0, 0, 0); 1015 1016 if (LARGE_PAGE_BITS != EPT_PD_INDEX_OFFSET) { 1017 assert(ept_pde_ptr_get_page_size(lu_ret.pdSlot + 1) == ept_pde_ept_pde_2m); 1018 assert(ept_pde_ept_pde_2m_ptr_get_read(lu_ret.pdSlot + 1)); 1019 assert(ept_pde_ept_pde_2m_ptr_get_page_base_address(lu_ret.pdSlot + 1) == addr + BIT(21)); 1020 1021 lu_ret.pdSlot[1] = ept_pde_ept_pde_2m_new(0, 0, 0, 0, 0, 0, 0); 1022 } 1023 break; 1024 } 1025 default: 1026 /* we did not allow mapping additional page sizes into EPT objects, 1027 * so this should not happen. As we have no way to return an error 1028 * all we can do is assert */ 1029 assert(!"Invalid page size for unmap"); 1030 } 1031} 1032 1033#endif /* CONFIG_VTX */ 1034