1/* 2 * Copyright (c) 2003-2007 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 29/* 30 * Here's what to do if you want to add a new routine to the comm page: 31 * 32 * 1. Add a definition for it's address in osfmk/i386/cpu_capabilities.h, 33 * being careful to reserve room for future expansion. 34 * 35 * 2. Write one or more versions of the routine, each with it's own 36 * commpage_descriptor. The tricky part is getting the "special", 37 * "musthave", and "canthave" fields right, so that exactly one 38 * version of the routine is selected for every machine. 39 * The source files should be in osfmk/i386/commpage/. 40 * 41 * 3. Add a ptr to your new commpage_descriptor(s) in the "routines" 42 * array in osfmk/i386/commpage/commpage_asm.s. There are two 43 * arrays, one for the 32-bit and one for the 64-bit commpage. 44 * 45 * 4. Write the code in Libc to use the new routine. 46 */ 47 48#include <mach/mach_types.h> 49#include <mach/machine.h> 50#include <mach/vm_map.h> 51#include <i386/machine_routines.h> 52#include <i386/misc_protos.h> 53#include <i386/tsc.h> 54#include <i386/cpu_data.h> 55#include <machine/cpu_capabilities.h> 56#include <machine/commpage.h> 57#include <machine/pmap.h> 58#include <vm/vm_kern.h> 59#include <vm/vm_map.h> 60#include <ipc/ipc_port.h> 61 62#include <kern/page_decrypt.h> 63 64/* the lists of commpage routines are in commpage_asm.s */ 65extern commpage_descriptor* commpage_32_routines[]; 66extern commpage_descriptor* commpage_64_routines[]; 67 68/* translated commpage descriptors from commpage_sigs.c */ 69extern commpage_descriptor sigdata_descriptor; 70extern commpage_descriptor *ba_descriptors[]; 71 72extern vm_map_t commpage32_map; // the shared submap, set up in vm init 73extern vm_map_t commpage64_map; // the shared submap, set up in vm init 74 75char *commPagePtr32 = NULL; // virtual addr in kernel map of 32-bit commpage 76char *commPagePtr64 = NULL; // ...and of 64-bit commpage 77int _cpu_capabilities = 0; // define the capability vector 78 79int noVMX = 0; /* if true, do not set kHasAltivec in ppc _cpu_capabilities */ 80 81static uintptr_t next; // next available byte in comm page 82static int cur_routine; // comm page address of "current" routine 83static int matched; // true if we've found a match for "current" routine 84 85static char *commPagePtr; // virtual addr in kernel map of commpage we are working on 86static size_t commPageBaseOffset; // add to 32-bit runtime address to get offset in commpage 87 88static commpage_time_data *time_data32 = NULL; 89static commpage_time_data *time_data64 = NULL; 90 91/* Allocate the commpage and add to the shared submap created by vm: 92 * 1. allocate a page in the kernel map (RW) 93 * 2. wire it down 94 * 3. make a memory entry out of it 95 * 4. map that entry into the shared comm region map (R-only) 96 */ 97 98static void* 99commpage_allocate( 100 vm_map_t submap, // commpage32_map or commpage_map64 101 size_t area_used ) // _COMM_PAGE32_AREA_USED or _COMM_PAGE64_AREA_USED 102{ 103 vm_offset_t kernel_addr = 0; // address of commpage in kernel map 104 vm_offset_t zero = 0; 105 vm_size_t size = area_used; // size actually populated 106 vm_map_entry_t entry; 107 ipc_port_t handle; 108 109 if (submap == NULL) 110 panic("commpage submap is null"); 111 112 if (vm_map(kernel_map,&kernel_addr,area_used,0,VM_FLAGS_ANYWHERE,NULL,0,FALSE,VM_PROT_ALL,VM_PROT_ALL,VM_INHERIT_NONE)) 113 panic("cannot allocate commpage"); 114 115 if (vm_map_wire(kernel_map,kernel_addr,kernel_addr+area_used,VM_PROT_DEFAULT,FALSE)) 116 panic("cannot wire commpage"); 117 118 /* 119 * Now that the object is created and wired into the kernel map, mark it so that no delay 120 * copy-on-write will ever be performed on it as a result of mapping it into user-space. 121 * If such a delayed copy ever occurred, we could remove the kernel's wired mapping - and 122 * that would be a real disaster. 123 * 124 * JMM - What we really need is a way to create it like this in the first place. 125 */ 126 if (!vm_map_lookup_entry( kernel_map, vm_map_trunc_page(kernel_addr), &entry) || entry->is_sub_map) 127 panic("cannot find commpage entry"); 128 entry->object.vm_object->copy_strategy = MEMORY_OBJECT_COPY_NONE; 129 130 if (mach_make_memory_entry( kernel_map, // target map 131 &size, // size 132 kernel_addr, // offset (address in kernel map) 133 VM_PROT_ALL, // map it RWX 134 &handle, // this is the object handle we get 135 NULL )) // parent_entry (what is this?) 136 panic("cannot make entry for commpage"); 137 138 if (vm_map_64( submap, // target map (shared submap) 139 &zero, // address (map into 1st page in submap) 140 area_used, // size 141 0, // mask 142 VM_FLAGS_FIXED, // flags (it must be 1st page in submap) 143 handle, // port is the memory entry we just made 144 0, // offset (map 1st page in memory entry) 145 FALSE, // copy 146 VM_PROT_READ|VM_PROT_EXECUTE, // cur_protection (R-only in user map) 147 VM_PROT_READ|VM_PROT_EXECUTE, // max_protection 148 VM_INHERIT_SHARE )) // inheritance 149 panic("cannot map commpage"); 150 151 ipc_port_release(handle); 152 153 return (void*) kernel_addr; // return address in kernel map 154} 155 156/* Get address (in kernel map) of a commpage field. */ 157 158static void* 159commpage_addr_of( 160 int addr_at_runtime ) 161{ 162 return (void*) ((uintptr_t)commPagePtr + addr_at_runtime - commPageBaseOffset); 163} 164 165/* Determine number of CPUs on this system. We cannot rely on 166 * machine_info.max_cpus this early in the boot. 167 */ 168static int 169commpage_cpus( void ) 170{ 171 int cpus; 172 173 cpus = ml_get_max_cpus(); // NB: this call can block 174 175 if (cpus == 0) 176 panic("commpage cpus==0"); 177 if (cpus > 0xFF) 178 cpus = 0xFF; 179 180 return cpus; 181} 182 183/* Initialize kernel version of _cpu_capabilities vector (used by KEXTs.) */ 184 185static void 186commpage_init_cpu_capabilities( void ) 187{ 188 int bits; 189 int cpus; 190 ml_cpu_info_t cpu_info; 191 192 bits = 0; 193 ml_cpu_get_info(&cpu_info); 194 195 switch (cpu_info.vector_unit) { 196 case 8: 197 bits |= kHasSSE4_2; 198 /* fall thru */ 199 case 7: 200 bits |= kHasSSE4_1; 201 /* fall thru */ 202 case 6: 203 bits |= kHasSupplementalSSE3; 204 /* fall thru */ 205 case 5: 206 bits |= kHasSSE3; 207 /* fall thru */ 208 case 4: 209 bits |= kHasSSE2; 210 /* fall thru */ 211 case 3: 212 bits |= kHasSSE; 213 /* fall thru */ 214 case 2: 215 bits |= kHasMMX; 216 default: 217 break; 218 } 219 switch (cpu_info.cache_line_size) { 220 case 128: 221 bits |= kCache128; 222 break; 223 case 64: 224 bits |= kCache64; 225 break; 226 case 32: 227 bits |= kCache32; 228 break; 229 default: 230 break; 231 } 232 cpus = commpage_cpus(); // how many CPUs do we have 233 234 if (cpus == 1) 235 bits |= kUP; 236 237 bits |= (cpus << kNumCPUsShift); 238 239 bits |= kFastThreadLocalStorage; // we use %gs for TLS 240 241 if (cpu_mode_is64bit()) // k64Bit means processor is 64-bit capable 242 bits |= k64Bit; 243 244 if (tscFreq <= SLOW_TSC_THRESHOLD) /* is TSC too slow for _commpage_nanotime? */ 245 bits |= kSlow; 246 247 _cpu_capabilities = bits; // set kernel version for use by drivers etc 248} 249 250int 251_get_cpu_capabilities(void) 252{ 253 return _cpu_capabilities; 254} 255 256/* Copy data into commpage. */ 257 258static void 259commpage_stuff( 260 int address, 261 const void *source, 262 int length ) 263{ 264 void *dest = commpage_addr_of(address); 265 266 if ((uintptr_t)dest < next) 267 panic("commpage overlap at address 0x%x, %p < 0x%lx", address, dest, next); 268 269 bcopy(source,dest,length); 270 271 next = ((uintptr_t)dest + length); 272} 273 274static void 275commpage_stuff_swap( 276 int address, 277 void *source, 278 int length, 279 int legacy ) 280{ 281 if ( legacy ) { 282 void *dest = commpage_addr_of(address); 283 dest = (void *)((uintptr_t) dest + _COMM_PAGE_SIGS_OFFSET); 284 switch (length) { 285 case 2: 286 OSWriteSwapInt16(dest, 0, *(uint16_t *)source); 287 break; 288 case 4: 289 OSWriteSwapInt32(dest, 0, *(uint32_t *)source); 290 break; 291 case 8: 292 OSWriteSwapInt64(dest, 0, *(uint64_t *)source); 293 break; 294 } 295 } 296} 297 298static void 299commpage_stuff2( 300 int address, 301 void *source, 302 int length, 303 int legacy ) 304{ 305 commpage_stuff_swap(address, source, length, legacy); 306 commpage_stuff(address, source, length); 307} 308 309/* Copy a routine into comm page if it matches running machine. 310 */ 311static void 312commpage_stuff_routine( 313 commpage_descriptor *rd ) 314{ 315 int must,cant; 316 317 if (rd->commpage_address != cur_routine) { 318 if ((cur_routine!=0) && (matched==0)) 319 panic("commpage no match for last, next address %08lx", rd->commpage_address); 320 cur_routine = rd->commpage_address; 321 matched = 0; 322 } 323 324 must = _cpu_capabilities & rd->musthave; 325 cant = _cpu_capabilities & rd->canthave; 326 327 if ((must == rd->musthave) && (cant == 0)) { 328 if (matched) 329 panic("commpage multiple matches for address %08lx", rd->commpage_address); 330 matched = 1; 331 332 commpage_stuff(rd->commpage_address,rd->code_address,rd->code_length); 333 } 334} 335 336/* Fill in the 32- or 64-bit commpage. Called once for each. 337 * The 32-bit ("legacy") commpage has a bunch of stuff added to it 338 * for translated processes, some of which is byte-swapped. 339 */ 340 341static void 342commpage_populate_one( 343 vm_map_t submap, // commpage32_map or compage64_map 344 char ** kernAddressPtr, // &commPagePtr32 or &commPagePtr64 345 size_t area_used, // _COMM_PAGE32_AREA_USED or _COMM_PAGE64_AREA_USED 346 size_t base_offset, // will become commPageBaseOffset 347 commpage_descriptor** commpage_routines, // list of routine ptrs for this commpage 348 boolean_t legacy, // true if 32-bit commpage 349 commpage_time_data** time_data, // &time_data32 or &time_data64 350 const char* signature ) // "commpage 32-bit" or "commpage 64-bit" 351{ 352 short c2; 353 static double two52 = 1048576.0 * 1048576.0 * 4096.0; // 2**52 354 static double ten6 = 1000000.0; // 10**6 355 commpage_descriptor **rd; 356 short version = _COMM_PAGE_THIS_VERSION; 357 int swapcaps; 358 359 next = (uintptr_t) NULL; 360 cur_routine = 0; 361 commPagePtr = (char *)commpage_allocate( submap, (vm_size_t) area_used ); 362 *kernAddressPtr = commPagePtr; // save address either in commPagePtr32 or 64 363 commPageBaseOffset = base_offset; 364 365 *time_data = commpage_addr_of( _COMM_PAGE_TIME_DATA_START ); 366 367 /* Stuff in the constants. We move things into the comm page in strictly 368 * ascending order, so we can check for overlap and panic if so. 369 */ 370 commpage_stuff(_COMM_PAGE_SIGNATURE,signature,strlen(signature)); 371 commpage_stuff2(_COMM_PAGE_VERSION,&version,sizeof(short),legacy); 372 commpage_stuff(_COMM_PAGE_CPU_CAPABILITIES,&_cpu_capabilities,sizeof(int)); 373 374 /* excuse our magic constants, we cannot include ppc/cpu_capabilities.h */ 375 /* always set kCache32 and kDcbaAvailable */ 376 swapcaps = 0x44; 377 if ( _cpu_capabilities & kUP ) 378 swapcaps |= (kUP + (1 << kNumCPUsShift)); 379 else 380 swapcaps |= 2 << kNumCPUsShift; /* limit #cpus to 2 */ 381 if ( ! noVMX ) /* if rosetta will be emulating altivec... */ 382 swapcaps |= 0x101; /* ...then set kHasAltivec and kDataStreamsAvailable too */ 383 commpage_stuff_swap(_COMM_PAGE_CPU_CAPABILITIES, &swapcaps, sizeof(int), legacy); 384 c2 = 32; 385 commpage_stuff_swap(_COMM_PAGE_CACHE_LINESIZE,&c2,2,legacy); 386 387 if (_cpu_capabilities & kCache32) 388 c2 = 32; 389 else if (_cpu_capabilities & kCache64) 390 c2 = 64; 391 else if (_cpu_capabilities & kCache128) 392 c2 = 128; 393 commpage_stuff(_COMM_PAGE_CACHE_LINESIZE,&c2,2); 394 395 if ( legacy ) { 396 commpage_stuff2(_COMM_PAGE_2_TO_52,&two52,8,legacy); 397 commpage_stuff2(_COMM_PAGE_10_TO_6,&ten6,8,legacy); 398 } 399 400 for( rd = commpage_routines; *rd != NULL ; rd++ ) 401 commpage_stuff_routine(*rd); 402 403 if (!matched) 404 panic("commpage no match on last routine"); 405 406 if (next > (uintptr_t)_COMM_PAGE_END) 407 panic("commpage overflow: next = 0x%08lx, commPagePtr = 0x%08lx", next, (uintptr_t)commPagePtr); 408 409 if ( legacy ) { 410 next = (uintptr_t) NULL; 411 for( rd = ba_descriptors; *rd != NULL ; rd++ ) 412 commpage_stuff_routine(*rd); 413 414 next = (uintptr_t) NULL; 415 commpage_stuff_routine(&sigdata_descriptor); 416 } 417} 418 419 420/* Fill in commpages: called once, during kernel initialization, from the 421 * startup thread before user-mode code is running. 422 * 423 * See the top of this file for a list of what you have to do to add 424 * a new routine to the commpage. 425 */ 426 427void 428commpage_populate( void ) 429{ 430 commpage_init_cpu_capabilities(); 431 432 commpage_populate_one( commpage32_map, 433 &commPagePtr32, 434 _COMM_PAGE32_AREA_USED, 435 _COMM_PAGE32_BASE_ADDRESS, 436 commpage_32_routines, 437 TRUE, /* legacy (32-bit) commpage */ 438 &time_data32, 439 "commpage 32-bit"); 440 pmap_commpage32_init((vm_offset_t) commPagePtr32, _COMM_PAGE32_BASE_ADDRESS, 441 _COMM_PAGE32_AREA_USED/INTEL_PGBYTES); 442 443 time_data64 = time_data32; /* if no 64-bit commpage, point to 32-bit */ 444 445 if (_cpu_capabilities & k64Bit) { 446 commpage_populate_one( commpage64_map, 447 &commPagePtr64, 448 _COMM_PAGE64_AREA_USED, 449 _COMM_PAGE32_START_ADDRESS, /* because kernel is built 32-bit */ 450 commpage_64_routines, 451 FALSE, /* not a legacy commpage */ 452 &time_data64, 453 "commpage 64-bit"); 454 pmap_commpage64_init((vm_offset_t) commPagePtr64, _COMM_PAGE64_BASE_ADDRESS, 455 _COMM_PAGE64_AREA_USED/INTEL_PGBYTES); 456 } 457 458 rtc_nanotime_init_commpage(); 459} 460 461 462/* Update commpage nanotime information. Note that we interleave 463 * setting the 32- and 64-bit commpages, in order to keep nanotime more 464 * nearly in sync between the two environments. 465 * 466 * This routine must be serialized by some external means, ie a lock. 467 */ 468 469void 470commpage_set_nanotime( 471 uint64_t tsc_base, 472 uint64_t ns_base, 473 uint32_t scale, 474 uint32_t shift ) 475{ 476 commpage_time_data *p32 = time_data32; 477 commpage_time_data *p64 = time_data64; 478 static uint32_t generation = 0; 479 uint32_t next_gen; 480 481 if (p32 == NULL) /* have commpages been allocated yet? */ 482 return; 483 484 if ( generation != p32->nt_generation ) 485 panic("nanotime trouble 1"); /* possibly not serialized */ 486 if ( ns_base < p32->nt_ns_base ) 487 panic("nanotime trouble 2"); 488 if ((shift != 32) && ((_cpu_capabilities & kSlow)==0) ) 489 panic("nanotime trouble 3"); 490 491 next_gen = ++generation; 492 if (next_gen == 0) 493 next_gen = ++generation; 494 495 p32->nt_generation = 0; /* mark invalid, so commpage won't try to use it */ 496 p64->nt_generation = 0; 497 498 p32->nt_tsc_base = tsc_base; 499 p64->nt_tsc_base = tsc_base; 500 501 p32->nt_ns_base = ns_base; 502 p64->nt_ns_base = ns_base; 503 504 p32->nt_scale = scale; 505 p64->nt_scale = scale; 506 507 p32->nt_shift = shift; 508 p64->nt_shift = shift; 509 510 p32->nt_generation = next_gen; /* mark data as valid */ 511 p64->nt_generation = next_gen; 512} 513 514 515/* Disable commpage gettimeofday(), forcing commpage to call through to the kernel. */ 516 517void 518commpage_disable_timestamp( void ) 519{ 520 time_data32->gtod_generation = 0; 521 time_data64->gtod_generation = 0; 522} 523 524 525/* Update commpage gettimeofday() information. As with nanotime(), we interleave 526 * updates to the 32- and 64-bit commpage, in order to keep time more nearly in sync 527 * between the two environments. 528 * 529 * This routine must be serializeed by some external means, ie a lock. 530 */ 531 532 void 533 commpage_set_timestamp( 534 uint64_t abstime, 535 uint64_t secs ) 536{ 537 commpage_time_data *p32 = time_data32; 538 commpage_time_data *p64 = time_data64; 539 static uint32_t generation = 0; 540 uint32_t next_gen; 541 542 next_gen = ++generation; 543 if (next_gen == 0) 544 next_gen = ++generation; 545 546 p32->gtod_generation = 0; /* mark invalid, so commpage won't try to use it */ 547 p64->gtod_generation = 0; 548 549 p32->gtod_ns_base = abstime; 550 p64->gtod_ns_base = abstime; 551 552 p32->gtod_sec_base = secs; 553 p64->gtod_sec_base = secs; 554 555 p32->gtod_generation = next_gen; /* mark data as valid */ 556 p64->gtod_generation = next_gen; 557} 558