1// Copyright 2016 The Fuchsia Authors 2// 3// Use of this source code is governed by a MIT-style 4// license that can be found in the LICENSE file or at 5// https://opensource.org/licenses/MIT 6 7#include <vm/vm_address_region.h> 8 9#include "vm_priv.h" 10#include <assert.h> 11#include <err.h> 12#include <fbl/alloc_checker.h> 13#include <fbl/auto_call.h> 14#include <inttypes.h> 15#include <trace.h> 16#include <vm/fault.h> 17#include <vm/vm.h> 18#include <vm/vm_aspace.h> 19#include <vm/vm_object.h> 20#include <zircon/types.h> 21 22#define LOCAL_TRACE MAX(VM_GLOBAL_TRACE, 0) 23 24VmMapping::VmMapping(VmAddressRegion& parent, vaddr_t base, size_t size, uint32_t vmar_flags, 25 fbl::RefPtr<VmObject> vmo, uint64_t vmo_offset, uint arch_mmu_flags) 26 : VmAddressRegionOrMapping(base, size, vmar_flags, 27 parent.aspace_.get(), &parent), 28 object_(fbl::move(vmo)), object_offset_(vmo_offset), arch_mmu_flags_(arch_mmu_flags) { 29 30 LTRACEF("%p aspace %p base %#" PRIxPTR " size %#zx offset %#" PRIx64 "\n", 31 this, aspace_.get(), base_, size_, vmo_offset); 32} 33 34VmMapping::~VmMapping() { 35 canary_.Assert(); 36 LTRACEF("%p aspace %p base %#" PRIxPTR " size %#zx\n", 37 this, aspace_.get(), base_, size_); 38} 39 40size_t VmMapping::AllocatedPagesLocked() const { 41 canary_.Assert(); 42 DEBUG_ASSERT(aspace_->lock()->lock().IsHeld()); 43 44 if (state_ != LifeCycleState::ALIVE) { 45 return 0; 46 } 47 return object_->AllocatedPagesInRange(object_offset_, size_); 48} 49 50void VmMapping::Dump(uint depth, bool verbose) const { 51 canary_.Assert(); 52 for (uint i = 0; i < depth; ++i) { 53 printf(" "); 54 } 55 char vmo_name[32]; 56 object_->get_name(vmo_name, sizeof(vmo_name)); 57 printf("map %p [%#" PRIxPTR " %#" PRIxPTR "] sz %#zx mmufl %#x\n", 58 this, base_, base_ + size_ - 1, size_, arch_mmu_flags_); 59 for (uint i = 0; i < depth + 1; ++i) { 60 printf(" "); 61 } 62 printf("vmo %p/k%" PRIu64 " off %#" PRIx64 63 " pages %zu ref %d '%s'\n", 64 object_.get(), object_->user_id(), object_offset_, 65 // TODO(dbort): Use AllocatePagesLocked() once Dump() is locked 66 // consistently. Currently, Dump() may be called without the aspace 67 // lock. 68 object_->AllocatedPagesInRange(object_offset_, size_), 69 ref_count_debug(), vmo_name); 70 if (verbose) { 71 object_->Dump(depth + 1, false); 72 } 73} 74 75zx_status_t VmMapping::Protect(vaddr_t base, size_t size, uint new_arch_mmu_flags) { 76 canary_.Assert(); 77 LTRACEF("%p %#" PRIxPTR " %#x %#x\n", this, base_, flags_, new_arch_mmu_flags); 78 79 if (!IS_PAGE_ALIGNED(base)) { 80 return ZX_ERR_INVALID_ARGS; 81 } 82 83 size = ROUNDUP(size, PAGE_SIZE); 84 85 Guard<fbl::Mutex> guard{aspace_->lock()}; 86 if (state_ != LifeCycleState::ALIVE) { 87 return ZX_ERR_BAD_STATE; 88 } 89 90 if (size == 0 || !is_in_range(base, size)) { 91 return ZX_ERR_INVALID_ARGS; 92 } 93 94 return ProtectLocked(base, size, new_arch_mmu_flags); 95} 96 97namespace { 98 99// Implementation helper for ProtectLocked 100zx_status_t ProtectOrUnmap(const fbl::RefPtr<VmAspace>& aspace, vaddr_t base, size_t size, 101 uint new_arch_mmu_flags) { 102 if (new_arch_mmu_flags & ARCH_MMU_FLAG_PERM_RWX_MASK) { 103 return aspace->arch_aspace().Protect(base, size / PAGE_SIZE, new_arch_mmu_flags); 104 } else { 105 return aspace->arch_aspace().Unmap(base, size / PAGE_SIZE, nullptr); 106 } 107} 108 109} // namespace 110 111zx_status_t VmMapping::ProtectLocked(vaddr_t base, size_t size, uint new_arch_mmu_flags) { 112 DEBUG_ASSERT(aspace_->lock()->lock().IsHeld()); 113 DEBUG_ASSERT(size != 0 && IS_PAGE_ALIGNED(base) && IS_PAGE_ALIGNED(size)); 114 115 // Do not allow changing caching 116 if (new_arch_mmu_flags & ARCH_MMU_FLAG_CACHE_MASK) { 117 return ZX_ERR_INVALID_ARGS; 118 } 119 120 if (!is_valid_mapping_flags(new_arch_mmu_flags)) { 121 return ZX_ERR_ACCESS_DENIED; 122 } 123 124 DEBUG_ASSERT(object_); 125 // grab the lock for the vmo 126 Guard<fbl::Mutex> guard{object_->lock()}; 127 128 // Persist our current caching mode 129 new_arch_mmu_flags |= (arch_mmu_flags_ & ARCH_MMU_FLAG_CACHE_MASK); 130 131 // If we're not actually changing permissions, return fast. 132 if (new_arch_mmu_flags == arch_mmu_flags_) { 133 return ZX_OK; 134 } 135 136 // TODO(teisenbe): deal with error mapping on arch_mmu_protect fail 137 138 // If we're changing the whole mapping, just make the change. 139 if (base_ == base && size_ == size) { 140 zx_status_t status = ProtectOrUnmap(aspace_, base, size, new_arch_mmu_flags); 141 LTRACEF("arch_mmu_protect returns %d\n", status); 142 arch_mmu_flags_ = new_arch_mmu_flags; 143 return ZX_OK; 144 } 145 146 // Handle changing from the left 147 if (base_ == base) { 148 // Create a new mapping for the right half (has old perms) 149 fbl::AllocChecker ac; 150 fbl::RefPtr<VmMapping> mapping(fbl::AdoptRef( 151 new (&ac) VmMapping(*parent_, base + size, size_ - size, flags_, 152 object_, object_offset_ + size, arch_mmu_flags_))); 153 if (!ac.check()) { 154 return ZX_ERR_NO_MEMORY; 155 } 156 157 zx_status_t status = ProtectOrUnmap(aspace_, base, size, new_arch_mmu_flags); 158 LTRACEF("arch_mmu_protect returns %d\n", status); 159 arch_mmu_flags_ = new_arch_mmu_flags; 160 161 size_ = size; 162 mapping->ActivateLocked(); 163 return ZX_OK; 164 } 165 166 // Handle changing from the right 167 if (base_ + size_ == base + size) { 168 // Create a new mapping for the right half (has new perms) 169 fbl::AllocChecker ac; 170 171 fbl::RefPtr<VmMapping> mapping(fbl::AdoptRef( 172 new (&ac) VmMapping(*parent_, base, size, flags_, 173 object_, object_offset_ + base - base_, 174 new_arch_mmu_flags))); 175 if (!ac.check()) { 176 return ZX_ERR_NO_MEMORY; 177 } 178 179 zx_status_t status = ProtectOrUnmap(aspace_, base, size, new_arch_mmu_flags); 180 LTRACEF("arch_mmu_protect returns %d\n", status); 181 182 size_ -= size; 183 mapping->ActivateLocked(); 184 return ZX_OK; 185 } 186 187 // We're unmapping from the center, so we need to create two new mappings 188 const size_t left_size = base - base_; 189 const size_t right_size = (base_ + size_) - (base + size); 190 const uint64_t center_vmo_offset = object_offset_ + base - base_; 191 const uint64_t right_vmo_offset = center_vmo_offset + size; 192 193 fbl::AllocChecker ac; 194 fbl::RefPtr<VmMapping> center_mapping(fbl::AdoptRef( 195 new (&ac) VmMapping(*parent_, base, size, flags_, 196 object_, center_vmo_offset, new_arch_mmu_flags))); 197 if (!ac.check()) { 198 return ZX_ERR_NO_MEMORY; 199 } 200 fbl::RefPtr<VmMapping> right_mapping(fbl::AdoptRef( 201 new (&ac) VmMapping(*parent_, base + size, right_size, flags_, 202 object_, right_vmo_offset, arch_mmu_flags_))); 203 if (!ac.check()) { 204 return ZX_ERR_NO_MEMORY; 205 } 206 207 zx_status_t status = ProtectOrUnmap(aspace_, base, size, new_arch_mmu_flags); 208 LTRACEF("arch_mmu_protect returns %d\n", status); 209 210 // Turn us into the left half 211 size_ = left_size; 212 213 center_mapping->ActivateLocked(); 214 right_mapping->ActivateLocked(); 215 return ZX_OK; 216} 217 218zx_status_t VmMapping::Unmap(vaddr_t base, size_t size) { 219 LTRACEF("%p %#" PRIxPTR " %zu\n", this, base, size); 220 221 if (!IS_PAGE_ALIGNED(base)) { 222 return ZX_ERR_INVALID_ARGS; 223 } 224 225 size = ROUNDUP(size, PAGE_SIZE); 226 227 fbl::RefPtr<VmAspace> aspace(aspace_); 228 if (!aspace) { 229 return ZX_ERR_BAD_STATE; 230 } 231 232 Guard<fbl::Mutex> guard{aspace_->lock()}; 233 if (state_ != LifeCycleState::ALIVE) { 234 return ZX_ERR_BAD_STATE; 235 } 236 237 if (size == 0 || !is_in_range(base, size)) { 238 return ZX_ERR_INVALID_ARGS; 239 } 240 241 // If we're unmapping everything, destroy this mapping 242 if (base == base_ && size == size_) { 243 return DestroyLocked(); 244 } 245 246 return UnmapLocked(base, size); 247} 248 249zx_status_t VmMapping::UnmapLocked(vaddr_t base, size_t size) { 250 canary_.Assert(); 251 DEBUG_ASSERT(aspace_->lock()->lock().IsHeld()); 252 DEBUG_ASSERT(size != 0 && IS_PAGE_ALIGNED(size) && IS_PAGE_ALIGNED(base)); 253 DEBUG_ASSERT(base >= base_ && base - base_ < size_); 254 DEBUG_ASSERT(size_ - (base - base_) >= size); 255 DEBUG_ASSERT(parent_); 256 257 if (state_ != LifeCycleState::ALIVE) { 258 return ZX_ERR_BAD_STATE; 259 } 260 261 // If our parent VMAR is DEAD, then we can only unmap everything. 262 DEBUG_ASSERT(parent_->state_ != LifeCycleState::DEAD || (base == base_ && size == size_)); 263 264 LTRACEF("%p\n", this); 265 266 // grab the lock for the vmo 267 DEBUG_ASSERT(object_); 268 Guard<fbl::Mutex> guard{object_->lock()}; 269 270 // Check if unmapping from one of the ends 271 if (base_ == base || base + size == base_ + size_) { 272 LTRACEF("unmapping base %#lx size %#zx\n", base, size); 273 zx_status_t status = aspace_->arch_aspace().Unmap(base, size / PAGE_SIZE, nullptr); 274 if (status != ZX_OK) { 275 return status; 276 } 277 278 if (base_ == base && size_ != size) { 279 // We need to remove ourselves from tree before updating base_, 280 // since base_ is the tree key. 281 fbl::RefPtr<VmAddressRegionOrMapping> ref(parent_->subregions_.erase(*this)); 282 base_ += size; 283 object_offset_ += size; 284 parent_->subregions_.insert(fbl::move(ref)); 285 } 286 size_ -= size; 287 288 return ZX_OK; 289 } 290 291 // We're unmapping from the center, so we need to split the mapping 292 DEBUG_ASSERT(parent_->state_ == LifeCycleState::ALIVE); 293 294 const uint64_t vmo_offset = object_offset_ + (base + size) - base_; 295 const vaddr_t new_base = base + size; 296 const size_t new_size = (base_ + size_) - new_base; 297 298 fbl::AllocChecker ac; 299 fbl::RefPtr<VmMapping> mapping(fbl::AdoptRef( 300 new (&ac) VmMapping(*parent_, new_base, new_size, flags_, object_, vmo_offset, 301 arch_mmu_flags_))); 302 if (!ac.check()) { 303 return ZX_ERR_NO_MEMORY; 304 } 305 306 // Unmap the middle segment 307 LTRACEF("unmapping base %#lx size %#zx\n", base, size); 308 zx_status_t status = aspace_->arch_aspace().Unmap(base, size / PAGE_SIZE, nullptr); 309 if (status != ZX_OK) { 310 return status; 311 } 312 313 // Turn us into the left half 314 size_ = base - base_; 315 mapping->ActivateLocked(); 316 return ZX_OK; 317} 318 319zx_status_t VmMapping::UnmapVmoRangeLocked(uint64_t offset, uint64_t len) const { 320 canary_.Assert(); 321 322 LTRACEF("region %p obj_offset %#" PRIx64 " size %zu, offset %#" PRIx64 " len %#" PRIx64 "\n", 323 this, object_offset_, size_, offset, len); 324 325 // NOTE: must be acquired with the vmo lock held, but doesn't need to take 326 // the address space lock, since it will not manipulate its location in the 327 // vmar tree. However, it must be held in the ALIVE state across this call. 328 // 329 // Avoids a race with DestroyLocked() since it removes ourself from the VMO's 330 // mapping list with the VMO lock held before dropping this state to DEAD. The 331 // VMO cant call back to us once we're out of their list. 332 DEBUG_ASSERT(state_ == LifeCycleState::ALIVE); 333 334 DEBUG_ASSERT(object_); 335 DEBUG_ASSERT(object_->lock()->lock().IsHeld()); 336 337 DEBUG_ASSERT(IS_PAGE_ALIGNED(offset)); 338 DEBUG_ASSERT(IS_PAGE_ALIGNED(len)); 339 DEBUG_ASSERT(len > 0); 340 341 // If we're currently faulting and are responsible for the vmo code to be calling 342 // back to us, detect the recursion and abort here. 343 // The specific path we're avoiding is if the VMO calls back into us during vmo->GetPageLocked() 344 // via UnmapVmoRangeLocked(). If we set this flag we're short circuiting the unmap operation 345 // so that we don't do extra work. 346 if (likely(currently_faulting_)) { 347 LTRACEF("recursing to ourself, abort\n"); 348 return ZX_OK; 349 } 350 351 if (len == 0) { 352 return ZX_OK; 353 } 354 355 // compute the intersection of the passed in vmo range and our mapping 356 uint64_t offset_new; 357 uint64_t len_new; 358 if (!GetIntersect(object_offset_, static_cast<uint64_t>(size_), offset, len, 359 &offset_new, &len_new)) { 360 return ZX_OK; 361 } 362 363 DEBUG_ASSERT(len_new > 0 && len_new <= SIZE_MAX); 364 DEBUG_ASSERT(offset_new >= object_offset_); 365 366 LTRACEF("intersection offset %#" PRIx64 ", len %#" PRIx64 "\n", offset_new, len_new); 367 368 // make sure the base + offset is within our address space 369 // should be, according to the range stored in base_ + size_ 370 vaddr_t unmap_base; 371 bool overflowed = add_overflow(base_, offset_new - object_offset_, &unmap_base); 372 ASSERT(!overflowed); 373 374 // make sure we're only unmapping within our window 375 ASSERT(unmap_base >= base_); 376 ASSERT((unmap_base + len_new - 1) <= (base_ + size_ - 1)); 377 378 LTRACEF("going to unmap %#" PRIxPTR ", len %#" PRIx64 " aspace %p\n", 379 unmap_base, len_new, aspace_.get()); 380 381 zx_status_t status = aspace_->arch_aspace().Unmap(unmap_base, 382 static_cast<size_t>(len_new) / PAGE_SIZE, nullptr); 383 if (status != ZX_OK) { 384 return status; 385 } 386 387 return ZX_OK; 388} 389 390namespace { 391 392class VmMappingCoalescer { 393public: 394 VmMappingCoalescer(VmMapping* mapping, vaddr_t base); 395 ~VmMappingCoalescer(); 396 397 // Add a page to the mapping run. If this fails, the VmMappingCoalescer is 398 // no longer valid. 399 zx_status_t Append(vaddr_t vaddr, paddr_t paddr) { 400 DEBUG_ASSERT(!aborted_); 401 // If this isn't the expected vaddr, flush the run we have first. 402 if (count_ >= fbl::count_of(phys_) || vaddr != base_ + count_ * PAGE_SIZE) { 403 zx_status_t status = Flush(); 404 if (status != ZX_OK) { 405 return status; 406 } 407 base_ = vaddr; 408 } 409 phys_[count_] = paddr; 410 ++count_; 411 return ZX_OK; 412 } 413 414 // Submit any outstanding mappings to the MMU. If this fails, the 415 // VmMappingCoalescer is no longer valid. 416 zx_status_t Flush(); 417 418 // Drop the current outstanding mappings without sending them to the MMU. 419 // After this call, the VmMappingCoalescer is no longer valid. 420 void Abort() { 421 aborted_ = true; 422 } 423 424private: 425 DISALLOW_COPY_ASSIGN_AND_MOVE(VmMappingCoalescer); 426 427 VmMapping* mapping_; 428 vaddr_t base_; 429 paddr_t phys_[16]; 430 size_t count_; 431 bool aborted_; 432}; 433 434VmMappingCoalescer::VmMappingCoalescer(VmMapping* mapping, vaddr_t base) 435 : mapping_(mapping), base_(base), count_(0), aborted_(false) {} 436 437VmMappingCoalescer::~VmMappingCoalescer() { 438 // Make sure we've flushed or aborted 439 DEBUG_ASSERT(count_ == 0 || aborted_); 440} 441 442zx_status_t VmMappingCoalescer::Flush() { 443 if (count_ == 0) { 444 return ZX_OK; 445 } 446 447 uint flags = mapping_->arch_mmu_flags(); 448 if (flags & ARCH_MMU_FLAG_PERM_RWX_MASK) { 449 size_t mapped; 450 zx_status_t ret = mapping_->aspace()->arch_aspace().Map(base_, phys_, count_, flags, 451 &mapped); 452 if (ret != ZX_OK) { 453 TRACEF("error %d mapping %zu pages starting at va %#" PRIxPTR "\n", ret, count_, base_); 454 aborted_ = true; 455 return ret; 456 } 457 DEBUG_ASSERT(mapped == count_); 458 } 459 base_ += count_ * PAGE_SIZE; 460 count_ = 0; 461 return ZX_OK; 462} 463 464} // namespace 465 466zx_status_t VmMapping::MapRange(size_t offset, size_t len, bool commit) { 467 canary_.Assert(); 468 469 len = ROUNDUP(len, PAGE_SIZE); 470 if (len == 0) { 471 return ZX_ERR_INVALID_ARGS; 472 } 473 474 Guard<fbl::Mutex> aspace_guard{aspace_->lock()}; 475 if (state_ != LifeCycleState::ALIVE) { 476 return ZX_ERR_BAD_STATE; 477 } 478 479 LTRACEF("region %p, offset %#zx, size %#zx, commit %d\n", this, offset, len, commit); 480 481 DEBUG_ASSERT(object_); 482 if (!IS_PAGE_ALIGNED(offset) || !is_in_range(base_ + offset, len)) { 483 return ZX_ERR_INVALID_ARGS; 484 } 485 486 // precompute the flags we'll pass GetPageLocked 487 // if committing, then tell it to soft fault in a page 488 uint pf_flags = VMM_PF_FLAG_WRITE; 489 if (commit) { 490 pf_flags |= VMM_PF_FLAG_SW_FAULT; 491 } 492 493 // grab the lock for the vmo 494 Guard<fbl::Mutex> object_guard{object_->lock()}; 495 496 // set the currently faulting flag for any recursive calls the vmo may make back into us. 497 DEBUG_ASSERT(!currently_faulting_); 498 currently_faulting_ = true; 499 auto ac = fbl::MakeAutoCall([&]() { currently_faulting_ = false; }); 500 501 // iterate through the range, grabbing a page from the underlying object and 502 // mapping it in 503 size_t o; 504 VmMappingCoalescer coalescer(this, base_ + offset); 505 for (o = offset; o < offset + len; o += PAGE_SIZE) { 506 uint64_t vmo_offset = object_offset_ + o; 507 508 zx_status_t status; 509 paddr_t pa; 510 status = object_->GetPageLocked(vmo_offset, pf_flags, nullptr, nullptr, &pa); 511 if (status != ZX_OK) { 512 // no page to map 513 if (commit) { 514 // fail when we can't commit every requested page 515 coalescer.Abort(); 516 return status; 517 } 518 519 // skip ahead 520 continue; 521 } 522 523 vaddr_t va = base_ + o; 524 LTRACEF_LEVEL(2, "mapping pa %#" PRIxPTR " to va %#" PRIxPTR "\n", pa, va); 525 status = coalescer.Append(va, pa); 526 if (status != ZX_OK) { 527 return status; 528 } 529 } 530 return coalescer.Flush(); 531} 532 533zx_status_t VmMapping::DecommitRange(size_t offset, size_t len, 534 size_t* decommitted) { 535 canary_.Assert(); 536 LTRACEF("%p [%#zx+%#zx], offset %#zx, len %#zx\n", 537 this, base_, size_, offset, len); 538 539 Guard<fbl::Mutex> guard{aspace_->lock()}; 540 if (state_ != LifeCycleState::ALIVE) { 541 return ZX_ERR_BAD_STATE; 542 } 543 if (offset + len < offset || offset + len > size_) { 544 return ZX_ERR_OUT_OF_RANGE; 545 } 546 // VmObject::DecommitRange will typically call back into our instance's 547 // VmMapping::UnmapVmoRangeLocked. 548 return object_->DecommitRange(object_offset_ + offset, len, decommitted); 549} 550 551zx_status_t VmMapping::DestroyLocked() { 552 canary_.Assert(); 553 DEBUG_ASSERT(aspace_->lock()->lock().IsHeld()); 554 LTRACEF("%p\n", this); 555 556 // Take a reference to ourself, so that we do not get destructed after 557 // dropping our last reference in this method (e.g. when calling 558 // subregions_.erase below). 559 fbl::RefPtr<VmMapping> self(this); 560 561 // The vDSO code mapping can never be unmapped, not even 562 // by VMAR destruction (except for process exit, of course). 563 // TODO(mcgrathr): Turn this into a policy-driven process-fatal case 564 // at some point. teisenbe@ wants to eventually make zx_vmar_destroy 565 // never fail. 566 if (aspace_->vdso_code_mapping_ == self) { 567 return ZX_ERR_ACCESS_DENIED; 568 } 569 570 // unmap our entire range 571 zx_status_t status = UnmapLocked(base_, size_); 572 if (status != ZX_OK) { 573 return status; 574 } 575 576 // Unmap should have reset our size to 0 577 DEBUG_ASSERT(size_ == 0); 578 579 // grab the object lock and remove ourself from its list 580 { 581 Guard<fbl::Mutex> guard{object_->lock()}; 582 object_->RemoveMappingLocked(this); 583 } 584 585 // detach from any object we have mapped 586 object_.reset(); 587 588 // Detach the now dead region from the parent 589 if (parent_) { 590 DEBUG_ASSERT(subregion_list_node_.InContainer()); 591 parent_->RemoveSubregion(this); 592 } 593 594 // mark ourself as dead 595 parent_ = nullptr; 596 state_ = LifeCycleState::DEAD; 597 return ZX_OK; 598} 599 600zx_status_t VmMapping::PageFault(vaddr_t va, const uint pf_flags) { 601 canary_.Assert(); 602 DEBUG_ASSERT(aspace_->lock()->lock().IsHeld()); 603 604 DEBUG_ASSERT(va >= base_ && va <= base_ + size_ - 1); 605 606 va = ROUNDDOWN(va, PAGE_SIZE); 607 uint64_t vmo_offset = va - base_ + object_offset_; 608 609 __UNUSED char pf_string[5]; 610 LTRACEF("%p va %#" PRIxPTR " vmo_offset %#" PRIx64 ", pf_flags %#x (%s)\n", 611 this, va, vmo_offset, pf_flags, 612 vmm_pf_flags_to_string(pf_flags, pf_string)); 613 614 // make sure we have permission to continue 615 if ((pf_flags & VMM_PF_FLAG_USER) && !(arch_mmu_flags_ & ARCH_MMU_FLAG_PERM_USER)) { 616 // user page fault on non user mapped region 617 LTRACEF("permission failure: user fault on non user region\n"); 618 return ZX_ERR_ACCESS_DENIED; 619 } 620 if ((pf_flags & VMM_PF_FLAG_WRITE) && !(arch_mmu_flags_ & ARCH_MMU_FLAG_PERM_WRITE)) { 621 // write to a non-writeable region 622 LTRACEF("permission failure: write fault on non-writable region\n"); 623 return ZX_ERR_ACCESS_DENIED; 624 } 625 if (!(pf_flags & VMM_PF_FLAG_WRITE) && !(arch_mmu_flags_ & ARCH_MMU_FLAG_PERM_READ)) { 626 // read to a non-readable region 627 LTRACEF("permission failure: read fault on non-readable region\n"); 628 return ZX_ERR_ACCESS_DENIED; 629 } 630 if ((pf_flags & VMM_PF_FLAG_INSTRUCTION) && !(arch_mmu_flags_ & ARCH_MMU_FLAG_PERM_EXECUTE)) { 631 // instruction fetch from a no execute region 632 LTRACEF("permission failure: execute fault on no execute region\n"); 633 return ZX_ERR_ACCESS_DENIED; 634 } 635 636 // grab the lock for the vmo 637 Guard<fbl::Mutex> guard{object_->lock()}; 638 639 // set the currently faulting flag for any recursive calls the vmo may make back into us 640 // The specific path we're avoiding is if the VMO calls back into us during vmo->GetPageLocked() 641 // via UnmapVmoRangeLocked(). Since we're responsible for that page, signal to ourself to skip 642 // the unmap operation. 643 DEBUG_ASSERT(!currently_faulting_); 644 currently_faulting_ = true; 645 auto ac = fbl::MakeAutoCall([&]() { currently_faulting_ = false; }); 646 647 // fault in or grab an existing page 648 paddr_t new_pa; 649 vm_page_t* page; 650 zx_status_t status = object_->GetPageLocked(vmo_offset, pf_flags, nullptr, &page, &new_pa); 651 if (status != ZX_OK) { 652 // TODO(cpu): This trace was originally TRACEF() always on, but it fires if the 653 // VMO was resized, rather than just when the system is running out of memory. 654 LTRACEF("ERROR: failed to fault in or grab existing page\n"); 655 LTRACEF("%p vmo_offset %#" PRIx64 ", pf_flags %#x\n", this, vmo_offset, pf_flags); 656 return status; 657 } 658 659 // if we read faulted, make sure we map or modify the page without any write permissions 660 // this ensures we will fault again if a write is attempted so we can potentially 661 // replace this page with a copy or a new one 662 uint mmu_flags = arch_mmu_flags_; 663 if (!(pf_flags & VMM_PF_FLAG_WRITE)) { 664 // we read faulted, so only map with read permissions 665 mmu_flags &= ~ARCH_MMU_FLAG_PERM_WRITE; 666 } 667 668 // see if something is mapped here now 669 // this may happen if we are one of multiple threads racing on a single address 670 uint page_flags; 671 paddr_t pa; 672 zx_status_t err = aspace_->arch_aspace().Query(va, &pa, &page_flags); 673 if (err >= 0) { 674 LTRACEF("queried va, page at pa %#" PRIxPTR ", flags %#x is already there\n", pa, 675 page_flags); 676 if (pa == new_pa) { 677 // page was already mapped, are the permissions compatible? 678 // test that the page is already mapped with either the region's mmu flags 679 // or the flags that we're about to try to switch it to, which may be read-only 680 if (page_flags == arch_mmu_flags_ || page_flags == mmu_flags) { 681 return ZX_OK; 682 } 683 684 // assert that we're not accidentally marking the zero page writable 685 DEBUG_ASSERT((pa != vm_get_zero_page_paddr()) || !(mmu_flags & ARCH_MMU_FLAG_PERM_WRITE)); 686 687 // same page, different permission 688 status = aspace_->arch_aspace().Protect(va, 1, mmu_flags); 689 if (status != ZX_OK) { 690 TRACEF("failed to modify permissions on existing mapping\n"); 691 return ZX_ERR_NO_MEMORY; 692 } 693 } else { 694 // some other page is mapped there already 695 LTRACEF("thread %s faulted on va %#" PRIxPTR ", different page was present\n", 696 get_current_thread()->name, va); 697 LTRACEF("old pa %#" PRIxPTR " new pa %#" PRIxPTR "\n", pa, new_pa); 698 699 // assert that we're not accidentally mapping the zero page writable 700 DEBUG_ASSERT((new_pa != vm_get_zero_page_paddr()) || !(mmu_flags & ARCH_MMU_FLAG_PERM_WRITE)); 701 702 // unmap the old one and put the new one in place 703 status = aspace_->arch_aspace().Unmap(va, 1, nullptr); 704 if (status != ZX_OK) { 705 TRACEF("failed to remove old mapping before replacing\n"); 706 return ZX_ERR_NO_MEMORY; 707 } 708 709 size_t mapped; 710 status = aspace_->arch_aspace().MapContiguous(va, new_pa, 1, mmu_flags, &mapped); 711 if (status != ZX_OK) { 712 TRACEF("failed to map replacement page\n"); 713 return ZX_ERR_NO_MEMORY; 714 } 715 DEBUG_ASSERT(mapped == 1); 716 717 return ZX_OK; 718 } 719 } else { 720 // nothing was mapped there before, map it now 721 LTRACEF("mapping pa %#" PRIxPTR " to va %#" PRIxPTR " is zero page %d\n", 722 new_pa, va, (new_pa == vm_get_zero_page_paddr())); 723 724 // assert that we're not accidentally mapping the zero page writable 725 DEBUG_ASSERT((new_pa != vm_get_zero_page_paddr()) || !(mmu_flags & ARCH_MMU_FLAG_PERM_WRITE)); 726 727 size_t mapped; 728 status = aspace_->arch_aspace().MapContiguous(va, new_pa, 1, mmu_flags, &mapped); 729 if (status != ZX_OK) { 730 TRACEF("failed to map page\n"); 731 return ZX_ERR_NO_MEMORY; 732 } 733 DEBUG_ASSERT(mapped == 1); 734 } 735 736// TODO: figure out what to do with this 737#if ARCH_ARM64 738 if (pf_flags & VMM_PF_FLAG_GUEST) { 739 // TODO(abdulla): Correctly handle page fault for guest. 740 } else if (arch_mmu_flags_ & ARCH_MMU_FLAG_PERM_EXECUTE) { 741 arch_sync_cache_range(va, PAGE_SIZE); 742 } 743#endif 744 return ZX_OK; 745} 746 747// We disable thread safety analysis here because one of the common uses of this 748// function is for splitting one mapping object into several that will be backed 749// by the same VmObject. In that case, object_->lock() gets aliased across all 750// of the VmMappings involved, but we have no way of informing the analyzer of 751// this, resulting in spurious warnings. We could disable analysis on the 752// splitting functions instead, but they are much more involved, and we'd rather 753// have the analysis mostly functioning on those than on this much simpler 754// function. 755void VmMapping::ActivateLocked() TA_NO_THREAD_SAFETY_ANALYSIS { 756 DEBUG_ASSERT(state_ == LifeCycleState::NOT_READY); 757 DEBUG_ASSERT(aspace_->lock()->lock().IsHeld()); 758 DEBUG_ASSERT(object_->lock()->lock().IsHeld()); 759 DEBUG_ASSERT(parent_); 760 761 state_ = LifeCycleState::ALIVE; 762 object_->AddMappingLocked(this); 763 parent_->subregions_.insert(fbl::RefPtr<VmAddressRegionOrMapping>(this)); 764} 765 766void VmMapping::Activate() { 767 Guard<fbl::Mutex> guard{object_->lock()}; 768 ActivateLocked(); 769} 770