1/* 2 * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 29#define _IOMEMORYDESCRIPTOR_INTERNAL_ 30 31#include <IOKit/assert.h> 32#include <IOKit/system.h> 33 34#include <IOKit/IOLib.h> 35#include <IOKit/IOMapper.h> 36#include <IOKit/IOBufferMemoryDescriptor.h> 37#include <libkern/OSDebug.h> 38#include <mach/mach_vm.h> 39 40#include "IOKitKernelInternal.h" 41 42#ifdef IOALLOCDEBUG 43#include <libkern/c++/OSCPPDebug.h> 44#endif 45#include <IOKit/IOStatisticsPrivate.h> 46 47#if IOKITSTATS 48#define IOStatisticsAlloc(type, size) \ 49do { \ 50 IOStatistics::countAlloc(type, size); \ 51} while (0) 52#else 53#define IOStatisticsAlloc(type, size) 54#endif /* IOKITSTATS */ 55 56 57__BEGIN_DECLS 58void ipc_port_release_send(ipc_port_t port); 59#include <vm/pmap.h> 60 61__END_DECLS 62 63/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 64 65enum 66{ 67 kInternalFlagPhysical = 0x00000001, 68 kInternalFlagPageSized = 0x00000002, 69 kInternalFlagPageAllocated = 0x00000004 70}; 71 72/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 73 74#define super IOGeneralMemoryDescriptor 75OSDefineMetaClassAndStructors(IOBufferMemoryDescriptor, 76 IOGeneralMemoryDescriptor); 77 78/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 79 80static uintptr_t IOBMDPageProc(iopa_t * a) 81{ 82 kern_return_t kr; 83 vm_address_t vmaddr = 0; 84 int options = 0; // KMA_LOMEM; 85 86 kr = kernel_memory_allocate(kernel_map, &vmaddr, 87 page_size, 0, options); 88 89 if (KERN_SUCCESS != kr) vmaddr = 0; 90 else bzero((void *) vmaddr, page_size); 91 92 return ((uintptr_t) vmaddr); 93} 94 95/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ 96 97#ifndef __LP64__ 98bool IOBufferMemoryDescriptor::initWithOptions( 99 IOOptionBits options, 100 vm_size_t capacity, 101 vm_offset_t alignment, 102 task_t inTask) 103{ 104 mach_vm_address_t physicalMask = 0; 105 return (initWithPhysicalMask(inTask, options, capacity, alignment, physicalMask)); 106} 107#endif /* !__LP64__ */ 108 109bool IOBufferMemoryDescriptor::initWithPhysicalMask( 110 task_t inTask, 111 IOOptionBits options, 112 mach_vm_size_t capacity, 113 mach_vm_address_t alignment, 114 mach_vm_address_t physicalMask) 115{ 116 task_t mapTask = NULL; 117 vm_map_t vmmap = NULL; 118 mach_vm_address_t highestMask = 0; 119 IOOptionBits iomdOptions = kIOMemoryTypeVirtual64 | kIOMemoryAsReference; 120 IODMAMapSpecification mapSpec; 121 bool mapped = false; 122 bool needZero; 123 124 if (!capacity) return false; 125 126 _options = options; 127 _capacity = capacity; 128 _internalFlags = 0; 129 _internalReserved = 0; 130 _buffer = 0; 131 132 _ranges.v64 = IONew(IOAddressRange, 1); 133 if (!_ranges.v64) 134 return (false); 135 _ranges.v64->address = 0; 136 _ranges.v64->length = 0; 137 // make sure super::free doesn't dealloc _ranges before super::init 138 _flags = kIOMemoryAsReference; 139 140 // Grab IOMD bits from the Buffer MD options 141 iomdOptions |= (options & kIOBufferDescriptorMemoryFlags); 142 143 if (!(kIOMemoryMapperNone & options)) 144 { 145 IOMapper::checkForSystemMapper(); 146 mapped = (0 != IOMapper::gSystem); 147 } 148 needZero = (mapped || (0 != (kIOMemorySharingTypeMask & options))); 149 150 if (physicalMask && (alignment <= 1)) 151 { 152 alignment = ((physicalMask ^ (-1ULL)) & (physicalMask - 1)); 153 highestMask = (physicalMask | alignment); 154 alignment++; 155 if (alignment < page_size) 156 alignment = page_size; 157 } 158 159 if ((options & (kIOMemorySharingTypeMask | kIOMapCacheMask | kIOMemoryClearEncrypt)) && (alignment < page_size)) 160 alignment = page_size; 161 162 if (alignment >= page_size) 163 capacity = round_page(capacity); 164 165 if (alignment > page_size) 166 options |= kIOMemoryPhysicallyContiguous; 167 168 _alignment = alignment; 169 170 if ((inTask != kernel_task) && !(options & kIOMemoryPageable)) 171 return false; 172 173 bzero(&mapSpec, sizeof(mapSpec)); 174 mapSpec.alignment = _alignment; 175 mapSpec.numAddressBits = 64; 176 if (highestMask && mapped) 177 { 178 if (highestMask <= 0xFFFFFFFF) 179 mapSpec.numAddressBits = (32 - __builtin_clz((unsigned int) highestMask)); 180 else 181 mapSpec.numAddressBits = (64 - __builtin_clz((unsigned int) (highestMask >> 32))); 182 highestMask = 0; 183 } 184 185 // set memory entry cache mode, pageable, purgeable 186 iomdOptions |= ((options & kIOMapCacheMask) >> kIOMapCacheShift) << kIOMemoryBufferCacheShift; 187 if (options & kIOMemoryPageable) 188 { 189 iomdOptions |= kIOMemoryBufferPageable; 190 if (options & kIOMemoryPurgeable) iomdOptions |= kIOMemoryBufferPurgeable; 191 } 192 else 193 { 194 vmmap = kernel_map; 195 196 // Buffer shouldn't auto prepare they should be prepared explicitly 197 // But it never was enforced so what are you going to do? 198 iomdOptions |= kIOMemoryAutoPrepare; 199 200 /* Allocate a wired-down buffer inside kernel space. */ 201 202 bool contig = (0 != (options & kIOMemoryHostPhysicallyContiguous)); 203 204 if (!contig && (0 != (options & kIOMemoryPhysicallyContiguous))) 205 { 206 contig |= (!mapped); 207 contig |= (0 != (kIOMemoryMapperNone & options)); 208#if 0 209 // treat kIOMemoryPhysicallyContiguous as kIOMemoryHostPhysicallyContiguous for now 210 contig |= true; 211#endif 212 } 213 214 if (contig || highestMask || (alignment > page_size)) 215 { 216 _internalFlags |= kInternalFlagPhysical; 217 if (highestMask) 218 { 219 _internalFlags |= kInternalFlagPageSized; 220 capacity = round_page(capacity); 221 } 222 _buffer = (void *) IOKernelAllocateWithPhysicalRestrict( 223 capacity, highestMask, alignment, contig); 224 } 225 else if (needZero 226 && ((capacity + alignment) <= (page_size - gIOPageAllocChunkBytes))) 227 { 228 _internalFlags |= kInternalFlagPageAllocated; 229 needZero = false; 230 _buffer = (void *) iopa_alloc(&gIOBMDPageAllocator, &IOBMDPageProc, capacity, alignment); 231 if (_buffer) 232 { 233 IOStatisticsAlloc(kIOStatisticsMallocAligned, capacity); 234#if IOALLOCDEBUG 235 debug_iomalloc_size += capacity; 236#endif 237 } 238 } 239 else if (alignment > 1) 240 { 241 _buffer = IOMallocAligned(capacity, alignment); 242 } 243 else 244 { 245 _buffer = IOMalloc(capacity); 246 } 247 if (!_buffer) 248 { 249 return false; 250 } 251 if (needZero) bzero(_buffer, capacity); 252 } 253 254 if( (options & (kIOMemoryPageable | kIOMapCacheMask))) { 255 vm_size_t size = round_page(capacity); 256 257 // initWithOptions will create memory entry 258 iomdOptions |= kIOMemoryPersistent; 259 260 if( options & kIOMemoryPageable) { 261#if IOALLOCDEBUG 262 debug_iomallocpageable_size += size; 263#endif 264 mapTask = inTask; 265 if (NULL == inTask) 266 inTask = kernel_task; 267 } 268 else if (options & kIOMapCacheMask) 269 { 270 // Prefetch each page to put entries into the pmap 271 volatile UInt8 * startAddr = (UInt8 *)_buffer; 272 volatile UInt8 * endAddr = (UInt8 *)_buffer + capacity; 273 274 while (startAddr < endAddr) 275 { 276 UInt8 dummyVar = *startAddr; 277 (void) dummyVar; 278 startAddr += page_size; 279 } 280 } 281 } 282 283 _ranges.v64->address = (mach_vm_address_t) _buffer;; 284 _ranges.v64->length = _capacity; 285 286 if (!super::initWithOptions(_ranges.v64, 1, 0, 287 inTask, iomdOptions, /* System mapper */ 0)) 288 return false; 289 290 // give any system mapper the allocation params 291 if (kIOReturnSuccess != dmaCommandOperation(kIOMDAddDMAMapSpec, 292 &mapSpec, sizeof(mapSpec))) 293 return false; 294 295 if (mapTask) 296 { 297 if (!reserved) { 298 reserved = IONew( ExpansionData, 1 ); 299 if( !reserved) 300 return( false ); 301 } 302 reserved->map = createMappingInTask(mapTask, 0, 303 kIOMapAnywhere | (options & kIOMapPrefault) | (options & kIOMapCacheMask), 0, 0); 304 if (!reserved->map) 305 { 306 _buffer = 0; 307 return( false ); 308 } 309 release(); // map took a retain on this 310 reserved->map->retain(); 311 removeMapping(reserved->map); 312 mach_vm_address_t buffer = reserved->map->getAddress(); 313 _buffer = (void *) buffer; 314 if (kIOMemoryTypeVirtual64 == (kIOMemoryTypeMask & iomdOptions)) 315 _ranges.v64->address = buffer; 316 } 317 318 setLength(_capacity); 319 320 return true; 321} 322 323IOBufferMemoryDescriptor * IOBufferMemoryDescriptor::inTaskWithOptions( 324 task_t inTask, 325 IOOptionBits options, 326 vm_size_t capacity, 327 vm_offset_t alignment) 328{ 329 IOBufferMemoryDescriptor *me = new IOBufferMemoryDescriptor; 330 331 if (me && !me->initWithPhysicalMask(inTask, options, capacity, alignment, 0)) { 332 me->release(); 333 me = 0; 334 } 335 return me; 336} 337 338IOBufferMemoryDescriptor * IOBufferMemoryDescriptor::inTaskWithPhysicalMask( 339 task_t inTask, 340 IOOptionBits options, 341 mach_vm_size_t capacity, 342 mach_vm_address_t physicalMask) 343{ 344 IOBufferMemoryDescriptor *me = new IOBufferMemoryDescriptor; 345 346 if (me && !me->initWithPhysicalMask(inTask, options, capacity, 1, physicalMask)) 347 { 348 me->release(); 349 me = 0; 350 } 351 return me; 352} 353 354#ifndef __LP64__ 355bool IOBufferMemoryDescriptor::initWithOptions( 356 IOOptionBits options, 357 vm_size_t capacity, 358 vm_offset_t alignment) 359{ 360 return (initWithPhysicalMask(kernel_task, options, capacity, alignment, (mach_vm_address_t)0)); 361} 362#endif /* !__LP64__ */ 363 364IOBufferMemoryDescriptor * IOBufferMemoryDescriptor::withOptions( 365 IOOptionBits options, 366 vm_size_t capacity, 367 vm_offset_t alignment) 368{ 369 IOBufferMemoryDescriptor *me = new IOBufferMemoryDescriptor; 370 371 if (me && !me->initWithPhysicalMask(kernel_task, options, capacity, alignment, 0)) { 372 me->release(); 373 me = 0; 374 } 375 return me; 376} 377 378 379/* 380 * withCapacity: 381 * 382 * Returns a new IOBufferMemoryDescriptor with a buffer large enough to 383 * hold capacity bytes. The descriptor's length is initially set to the capacity. 384 */ 385IOBufferMemoryDescriptor * 386IOBufferMemoryDescriptor::withCapacity(vm_size_t inCapacity, 387 IODirection inDirection, 388 bool inContiguous) 389{ 390 return( IOBufferMemoryDescriptor::withOptions( 391 inDirection | kIOMemoryUnshared 392 | (inContiguous ? kIOMemoryPhysicallyContiguous : 0), 393 inCapacity, inContiguous ? inCapacity : 1 )); 394} 395 396#ifndef __LP64__ 397/* 398 * initWithBytes: 399 * 400 * Initialize a new IOBufferMemoryDescriptor preloaded with bytes (copied). 401 * The descriptor's length and capacity are set to the input buffer's size. 402 */ 403bool IOBufferMemoryDescriptor::initWithBytes(const void * inBytes, 404 vm_size_t inLength, 405 IODirection inDirection, 406 bool inContiguous) 407{ 408 if (!initWithPhysicalMask(kernel_task, inDirection | kIOMemoryUnshared 409 | (inContiguous ? kIOMemoryPhysicallyContiguous : 0), 410 inLength, inLength, (mach_vm_address_t)0)) 411 return false; 412 413 // start out with no data 414 setLength(0); 415 416 if (!appendBytes(inBytes, inLength)) 417 return false; 418 419 return true; 420} 421#endif /* !__LP64__ */ 422 423/* 424 * withBytes: 425 * 426 * Returns a new IOBufferMemoryDescriptor preloaded with bytes (copied). 427 * The descriptor's length and capacity are set to the input buffer's size. 428 */ 429IOBufferMemoryDescriptor * 430IOBufferMemoryDescriptor::withBytes(const void * inBytes, 431 vm_size_t inLength, 432 IODirection inDirection, 433 bool inContiguous) 434{ 435 IOBufferMemoryDescriptor *me = new IOBufferMemoryDescriptor; 436 437 if (me && !me->initWithPhysicalMask( 438 kernel_task, inDirection | kIOMemoryUnshared 439 | (inContiguous ? kIOMemoryPhysicallyContiguous : 0), 440 inLength, inLength, 0 )) 441 { 442 me->release(); 443 me = 0; 444 } 445 446 if (me) 447 { 448 // start out with no data 449 me->setLength(0); 450 451 if (!me->appendBytes(inBytes, inLength)) 452 { 453 me->release(); 454 me = 0; 455 } 456 } 457 return me; 458} 459 460/* 461 * free: 462 * 463 * Free resources 464 */ 465void IOBufferMemoryDescriptor::free() 466{ 467 // Cache all of the relevant information on the stack for use 468 // after we call super::free()! 469 IOOptionBits flags = _flags; 470 IOOptionBits internalFlags = _internalFlags; 471 IOOptionBits options = _options; 472 vm_size_t size = _capacity; 473 void * buffer = _buffer; 474 IOMemoryMap * map = 0; 475 IOAddressRange * range = _ranges.v64; 476 vm_offset_t alignment = _alignment; 477 478 if (alignment >= page_size) 479 size = round_page(size); 480 481 if (reserved) 482 { 483 map = reserved->map; 484 IODelete( reserved, ExpansionData, 1 ); 485 if (map) 486 map->release(); 487 } 488 489 /* super::free may unwire - deallocate buffer afterwards */ 490 super::free(); 491 492 if (options & kIOMemoryPageable) 493 { 494#if IOALLOCDEBUG 495 debug_iomallocpageable_size -= round_page(size); 496#endif 497 } 498 else if (buffer) 499 { 500 if (kInternalFlagPageSized & internalFlags) size = round_page(size); 501 502 if (kInternalFlagPhysical & internalFlags) 503 { 504 IOKernelFreePhysical((mach_vm_address_t) buffer, size); 505 } 506 else if (kInternalFlagPageAllocated & internalFlags) 507 { 508 uintptr_t page; 509 page = iopa_free(&gIOBMDPageAllocator, (uintptr_t) buffer, size); 510 if (page) 511 { 512 kmem_free(kernel_map, page, page_size); 513 } 514#if IOALLOCDEBUG 515 debug_iomalloc_size -= size; 516#endif 517 IOStatisticsAlloc(kIOStatisticsFreeAligned, size); 518 } 519 else if (alignment > 1) 520 { 521 IOFreeAligned(buffer, size); 522 } 523 else 524 { 525 IOFree(buffer, size); 526 } 527 } 528 if (range && (kIOMemoryAsReference & flags)) 529 IODelete(range, IOAddressRange, 1); 530} 531 532/* 533 * getCapacity: 534 * 535 * Get the buffer capacity 536 */ 537vm_size_t IOBufferMemoryDescriptor::getCapacity() const 538{ 539 return _capacity; 540} 541 542/* 543 * setLength: 544 * 545 * Change the buffer length of the memory descriptor. When a new buffer 546 * is created, the initial length of the buffer is set to be the same as 547 * the capacity. The length can be adjusted via setLength for a shorter 548 * transfer (there is no need to create more buffer descriptors when you 549 * can reuse an existing one, even for different transfer sizes). Note 550 * that the specified length must not exceed the capacity of the buffer. 551 */ 552void IOBufferMemoryDescriptor::setLength(vm_size_t length) 553{ 554 assert(length <= _capacity); 555 556 _length = length; 557 _ranges.v64->length = length; 558} 559 560/* 561 * setDirection: 562 * 563 * Change the direction of the transfer. This method allows one to redirect 564 * the descriptor's transfer direction. This eliminates the need to destroy 565 * and create new buffers when different transfer directions are needed. 566 */ 567void IOBufferMemoryDescriptor::setDirection(IODirection direction) 568{ 569 _flags = (_flags & ~kIOMemoryDirectionMask) | direction; 570#ifndef __LP64__ 571 _direction = (IODirection) (_flags & kIOMemoryDirectionMask); 572#endif /* !__LP64__ */ 573} 574 575/* 576 * appendBytes: 577 * 578 * Add some data to the end of the buffer. This method automatically 579 * maintains the memory descriptor buffer length. Note that appendBytes 580 * will not copy past the end of the memory descriptor's current capacity. 581 */ 582bool 583IOBufferMemoryDescriptor::appendBytes(const void * bytes, vm_size_t withLength) 584{ 585 vm_size_t actualBytesToCopy = min(withLength, _capacity - _length); 586 IOByteCount offset; 587 588 assert(_length <= _capacity); 589 590 offset = _length; 591 _length += actualBytesToCopy; 592 _ranges.v64->length += actualBytesToCopy; 593 594 if (_task == kernel_task) 595 bcopy(/* from */ bytes, (void *)(_ranges.v64->address + offset), 596 actualBytesToCopy); 597 else 598 writeBytes(offset, bytes, actualBytesToCopy); 599 600 return true; 601} 602 603/* 604 * getBytesNoCopy: 605 * 606 * Return the virtual address of the beginning of the buffer 607 */ 608void * IOBufferMemoryDescriptor::getBytesNoCopy() 609{ 610 if (kIOMemoryTypePhysical64 == (_flags & kIOMemoryTypeMask)) 611 return _buffer; 612 else 613 return (void *)_ranges.v64->address; 614} 615 616 617/* 618 * getBytesNoCopy: 619 * 620 * Return the virtual address of an offset from the beginning of the buffer 621 */ 622void * 623IOBufferMemoryDescriptor::getBytesNoCopy(vm_size_t start, vm_size_t withLength) 624{ 625 IOVirtualAddress address; 626 if (kIOMemoryTypePhysical64 == (_flags & kIOMemoryTypeMask)) 627 address = (IOVirtualAddress) _buffer; 628 else 629 address = _ranges.v64->address; 630 631 if (start < _length && (start + withLength) <= _length) 632 return (void *)(address + start); 633 return 0; 634} 635 636#ifndef __LP64__ 637void * IOBufferMemoryDescriptor::getVirtualSegment(IOByteCount offset, 638 IOByteCount * lengthOfSegment) 639{ 640 void * bytes = getBytesNoCopy(offset, 0); 641 642 if (bytes && lengthOfSegment) 643 *lengthOfSegment = _length - offset; 644 645 return bytes; 646} 647#endif /* !__LP64__ */ 648 649#ifdef __LP64__ 650OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 0); 651OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 1); 652#else /* !__LP64__ */ 653OSMetaClassDefineReservedUsed(IOBufferMemoryDescriptor, 0); 654OSMetaClassDefineReservedUsed(IOBufferMemoryDescriptor, 1); 655#endif /* !__LP64__ */ 656OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 2); 657OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 3); 658OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 4); 659OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 5); 660OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 6); 661OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 7); 662OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 8); 663OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 9); 664OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 10); 665OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 11); 666OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 12); 667OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 13); 668OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 14); 669OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 15); 670