1/* 2 * Copyright (c) 2000-2009 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28/* 29 * @OSF_COPYRIGHT@ 30 */ 31/* 32 * Mach Operating System 33 * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University 34 * All Rights Reserved. 35 * 36 * Permission to use, copy, modify and distribute this software and its 37 * documentation is hereby granted, provided that both the copyright 38 * notice and this permission notice appear in all copies of the 39 * software, derivative works or modified versions, and any portions 40 * thereof, and that both notices appear in supporting documentation. 41 * 42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR 44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 45 * 46 * Carnegie Mellon requests users of this software to return to 47 * 48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU 49 * School of Computer Science 50 * Carnegie Mellon University 51 * Pittsburgh PA 15213-3890 52 * 53 * any improvements or extensions that they make and grant Carnegie Mellon 54 * the rights to redistribute these changes. 55 */ 56/* 57 */ 58 59#include <mach/boolean.h> 60#include <mach/thread_switch.h> 61#include <ipc/ipc_port.h> 62#include <ipc/ipc_space.h> 63#include <kern/counters.h> 64#include <kern/ipc_kobject.h> 65#include <kern/processor.h> 66#include <kern/sched.h> 67#include <kern/sched_prim.h> 68#include <kern/spl.h> 69#include <kern/task.h> 70#include <kern/thread.h> 71#include <mach/policy.h> 72 73#include <kern/syscall_subr.h> 74#include <mach/mach_host_server.h> 75#include <mach/mach_syscalls.h> 76 77 78#ifdef MACH_BSD 79extern void workqueue_thread_yielded(void); 80#endif /* MACH_BSD */ 81 82 83/* Called from commpage to take a delayed preemption when exiting 84 * the "Preemption Free Zone" (PFZ). 85 */ 86kern_return_t 87pfz_exit( 88__unused struct pfz_exit_args *args) 89{ 90 /* For now, nothing special to do. We'll pick up the ASTs on kernel exit. */ 91 92 return (KERN_SUCCESS); 93} 94 95 96/* 97 * swtch and swtch_pri both attempt to context switch (logic in 98 * thread_block no-ops the context switch if nothing would happen). 99 * A boolean is returned that indicates whether there is anything 100 * else runnable. 101 * 102 * This boolean can be used by a thread waiting on a 103 * lock or condition: If FALSE is returned, the thread is justified 104 * in becoming a resource hog by continuing to spin because there's 105 * nothing else useful that the processor could do. If TRUE is 106 * returned, the thread should make one more check on the 107 * lock and then be a good citizen and really suspend. 108 */ 109 110static void 111swtch_continue(void) 112{ 113 register processor_t myprocessor; 114 boolean_t result; 115 116 disable_preemption(); 117 myprocessor = current_processor(); 118 result = !SCHED(processor_queue_empty)(myprocessor) || rt_runq.count > 0; 119 enable_preemption(); 120 121 thread_syscall_return(result); 122 /*NOTREACHED*/ 123} 124 125boolean_t 126swtch( 127 __unused struct swtch_args *args) 128{ 129 register processor_t myprocessor; 130 boolean_t result; 131 132 disable_preemption(); 133 myprocessor = current_processor(); 134 if (SCHED(processor_queue_empty)(myprocessor) && rt_runq.count == 0) { 135 mp_enable_preemption(); 136 137 return (FALSE); 138 } 139 enable_preemption(); 140 141 counter(c_swtch_block++); 142 143 thread_block_reason((thread_continue_t)swtch_continue, NULL, AST_YIELD); 144 145 disable_preemption(); 146 myprocessor = current_processor(); 147 result = !SCHED(processor_queue_empty)(myprocessor) || rt_runq.count > 0; 148 enable_preemption(); 149 150 return (result); 151} 152 153static void 154swtch_pri_continue(void) 155{ 156 register processor_t myprocessor; 157 boolean_t result; 158 159 thread_depress_abort_internal(current_thread()); 160 161 disable_preemption(); 162 myprocessor = current_processor(); 163 result = !SCHED(processor_queue_empty)(myprocessor) || rt_runq.count > 0; 164 mp_enable_preemption(); 165 166 thread_syscall_return(result); 167 /*NOTREACHED*/ 168} 169 170boolean_t 171swtch_pri( 172__unused struct swtch_pri_args *args) 173{ 174 register processor_t myprocessor; 175 boolean_t result; 176 177 disable_preemption(); 178 myprocessor = current_processor(); 179 if (SCHED(processor_queue_empty)(myprocessor) && rt_runq.count == 0) { 180 mp_enable_preemption(); 181 182 return (FALSE); 183 } 184 enable_preemption(); 185 186 counter(c_swtch_pri_block++); 187 188 thread_depress_abstime(thread_depress_time); 189 190 thread_block_reason((thread_continue_t)swtch_pri_continue, NULL, AST_YIELD); 191 192 thread_depress_abort_internal(current_thread()); 193 194 disable_preemption(); 195 myprocessor = current_processor(); 196 result = !SCHED(processor_queue_empty)(myprocessor) || rt_runq.count > 0; 197 enable_preemption(); 198 199 return (result); 200} 201 202static void 203thread_switch_continue(void) 204{ 205 register thread_t self = current_thread(); 206 int option = self->saved.swtch.option; 207 208 if (option == SWITCH_OPTION_DEPRESS) 209 thread_depress_abort_internal(self); 210 211 thread_syscall_return(KERN_SUCCESS); 212 /*NOTREACHED*/ 213} 214 215/* 216 * thread_switch: 217 * 218 * Context switch. User may supply thread hint. 219 */ 220kern_return_t 221thread_switch( 222 struct thread_switch_args *args) 223{ 224 register thread_t thread, self = current_thread(); 225 mach_port_name_t thread_name = args->thread_name; 226 int option = args->option; 227 mach_msg_timeout_t option_time = args->option_time; 228 229 /* 230 * Process option. 231 */ 232 switch (option) { 233 234 case SWITCH_OPTION_NONE: 235 case SWITCH_OPTION_DEPRESS: 236 case SWITCH_OPTION_WAIT: 237 break; 238 239 default: 240 return (KERN_INVALID_ARGUMENT); 241 } 242 243 workqueue_thread_yielded(); 244 245 /* 246 * Translate the port name if supplied. 247 */ 248 if (thread_name != MACH_PORT_NULL) { 249 ipc_port_t port; 250 251 if (ipc_port_translate_send(self->task->itk_space, 252 thread_name, &port) == KERN_SUCCESS) { 253 ip_reference(port); 254 ip_unlock(port); 255 256 thread = convert_port_to_thread(port); 257 ip_release(port); 258 259 if (thread == self) { 260 (void)thread_deallocate_internal(thread); 261 thread = THREAD_NULL; 262 } 263 } 264 else 265 thread = THREAD_NULL; 266 } 267 else 268 thread = THREAD_NULL; 269 270 /* 271 * Try to handoff if supplied. 272 */ 273 if (thread != THREAD_NULL) { 274 processor_t processor; 275 spl_t s; 276 277 s = splsched(); 278 thread_lock(thread); 279 280 /* 281 * Check that the thread is not bound 282 * to a different processor, and that realtime 283 * is not involved. 284 * 285 * Next, pull it off its run queue. If it 286 * doesn't come, it's not eligible. 287 */ 288 processor = current_processor(); 289 if (processor->current_pri < BASEPRI_RTQUEUES && 290 thread->sched_pri < BASEPRI_RTQUEUES && 291 (thread->bound_processor == PROCESSOR_NULL || 292 thread->bound_processor == processor) && 293 thread_run_queue_remove(thread) ) { 294 /* 295 * Hah, got it!! 296 */ 297 thread_unlock(thread); 298 299 (void)thread_deallocate_internal(thread); 300 301 if (option == SWITCH_OPTION_WAIT) 302 assert_wait_timeout((event_t)assert_wait_timeout, THREAD_ABORTSAFE, 303 option_time, 1000*NSEC_PER_USEC); 304 else 305 if (option == SWITCH_OPTION_DEPRESS) 306 thread_depress_ms(option_time); 307 308 self->saved.swtch.option = option; 309 310 thread_run(self, (thread_continue_t)thread_switch_continue, NULL, thread); 311 /* NOTREACHED */ 312 } 313 314 thread_unlock(thread); 315 splx(s); 316 317 thread_deallocate(thread); 318 } 319 320 if (option == SWITCH_OPTION_WAIT) 321 assert_wait_timeout((event_t)assert_wait_timeout, THREAD_ABORTSAFE, option_time, 1000*NSEC_PER_USEC); 322 else 323 if (option == SWITCH_OPTION_DEPRESS) 324 thread_depress_ms(option_time); 325 326 self->saved.swtch.option = option; 327 328 thread_block_reason((thread_continue_t)thread_switch_continue, NULL, AST_YIELD); 329 330 if (option == SWITCH_OPTION_DEPRESS) 331 thread_depress_abort_internal(self); 332 333 return (KERN_SUCCESS); 334} 335 336/* 337 * Depress thread's priority to lowest possible for the specified interval, 338 * with a value of zero resulting in no timeout being scheduled. 339 */ 340void 341thread_depress_abstime( 342 uint64_t interval) 343{ 344 register thread_t self = current_thread(); 345 uint64_t deadline; 346 spl_t s; 347 348 s = splsched(); 349 thread_lock(self); 350 if (!(self->sched_flags & TH_SFLAG_DEPRESSED_MASK)) { 351 processor_t myprocessor = self->last_processor; 352 353 self->sched_pri = DEPRESSPRI; 354 myprocessor->current_pri = self->sched_pri; 355 self->sched_flags |= TH_SFLAG_DEPRESS; 356 357 if (interval != 0) { 358 clock_absolutetime_interval_to_deadline(interval, &deadline); 359 if (!timer_call_enter(&self->depress_timer, deadline, TIMER_CALL_CRITICAL)) 360 self->depress_timer_active++; 361 } 362 } 363 thread_unlock(self); 364 splx(s); 365} 366 367void 368thread_depress_ms( 369 mach_msg_timeout_t interval) 370{ 371 uint64_t abstime; 372 373 clock_interval_to_absolutetime_interval( 374 interval, 1000*NSEC_PER_USEC, &abstime); 375 thread_depress_abstime(abstime); 376} 377 378/* 379 * Priority depression expiration. 380 */ 381void 382thread_depress_expire( 383 void *p0, 384 __unused void *p1) 385{ 386 thread_t thread = p0; 387 spl_t s; 388 389 s = splsched(); 390 thread_lock(thread); 391 if (--thread->depress_timer_active == 0) { 392 thread->sched_flags &= ~TH_SFLAG_DEPRESSED_MASK; 393 SCHED(compute_priority)(thread, FALSE); 394 } 395 thread_unlock(thread); 396 splx(s); 397} 398 399/* 400 * Prematurely abort priority depression if there is one. 401 */ 402kern_return_t 403thread_depress_abort_internal( 404 thread_t thread) 405{ 406 kern_return_t result = KERN_NOT_DEPRESSED; 407 spl_t s; 408 409 s = splsched(); 410 thread_lock(thread); 411 if (!(thread->sched_flags & TH_SFLAG_POLLDEPRESS)) { 412 if (thread->sched_flags & TH_SFLAG_DEPRESSED_MASK) { 413 thread->sched_flags &= ~TH_SFLAG_DEPRESSED_MASK; 414 SCHED(compute_priority)(thread, FALSE); 415 result = KERN_SUCCESS; 416 } 417 418 if (timer_call_cancel(&thread->depress_timer)) 419 thread->depress_timer_active--; 420 } 421 thread_unlock(thread); 422 splx(s); 423 424 return (result); 425} 426 427void 428thread_poll_yield( 429 thread_t self) 430{ 431 spl_t s; 432 433 assert(self == current_thread()); 434 435 s = splsched(); 436 if (self->sched_mode == TH_MODE_FIXED) { 437 uint64_t total_computation, abstime; 438 439 abstime = mach_absolute_time(); 440 total_computation = abstime - self->computation_epoch; 441 total_computation += self->computation_metered; 442 if (total_computation >= max_poll_computation) { 443 processor_t myprocessor = current_processor(); 444 ast_t preempt; 445 446 thread_lock(self); 447 if (!(self->sched_flags & TH_SFLAG_DEPRESSED_MASK)) { 448 self->sched_pri = DEPRESSPRI; 449 myprocessor->current_pri = self->sched_pri; 450 } 451 self->computation_epoch = abstime; 452 self->computation_metered = 0; 453 self->sched_flags |= TH_SFLAG_POLLDEPRESS; 454 455 abstime += (total_computation >> sched_poll_yield_shift); 456 if (!timer_call_enter(&self->depress_timer, abstime, TIMER_CALL_CRITICAL)) 457 self->depress_timer_active++; 458 thread_unlock(self); 459 460 if ((preempt = csw_check(myprocessor)) != AST_NONE) 461 ast_on(preempt); 462 } 463 } 464 splx(s); 465} 466 467 468void 469thread_yield_internal( 470 mach_msg_timeout_t ms) 471{ 472 processor_t myprocessor; 473 474 disable_preemption(); 475 myprocessor = current_processor(); 476 if (SCHED(processor_queue_empty)(myprocessor) && rt_runq.count == 0) { 477 mp_enable_preemption(); 478 479 return; 480 } 481 enable_preemption(); 482 483 thread_depress_ms(ms); 484 485 thread_block_reason(THREAD_CONTINUE_NULL, NULL, AST_YIELD); 486 487 thread_depress_abort_internal(current_thread()); 488} 489 490